[Gluster-users] GlusterFS disk full.

Fujii Yasuhiro eyes.works at gmail.com
Fri Sep 11 05:44:40 UTC 2015


Hi.

I have a question.

GlusterFS hdd space can't be recovered automatically after glusterfs
is disk full and I delete files.
It will be recovered restarting glusterfsd.
I can find deleted file and glusterfsd do not relese the file I don't know why.
Is this OK?

[version]
CentOS release 6.7 (Final)
Linux web2 2.6.32-573.3.1.el6.x86_64 #1 SMP Thu Aug 13 22:55:16 UTC
2015 x86_64 x86_64 x86_64 GNU/Linux
glusterfs-3.6.5-1.el6.x86_64
glusterfs-api-3.6.5-1.el6.x86_64
glusterfs-fuse-3.6.5-1.el6.x86_64
glusterfs-server-3.6.5-1.el6.x86_64
glusterfs-libs-3.6.5-1.el6.x86_64
glusterfs-cli-3.6.5-1.el6.x86_6

[test]
dd if=/dev/zero of=./test.dat bs=1M count=100000
dd: writing `./test.dat': Input/output error
dd: closing output file `./test.dat': No space left on device

[root at web2 www_virtual]# df
Filesystem           1K-blocks      Used Available Use% Mounted on
/dev/xvda1             8124856   2853600   4851880  38% /
tmpfs                   509256         0    509256   0% /dev/shm
/dev/xvdf1           101441468 101419972         0 100% /glusterfs/vol01
web2:/vol_replica_01 101441408 101420032         0 100% /mnt/glusterfs

[root at web2 www_virtual]# rm test.dat
rm: remove regular file `test.dat'? y

[root at web2 www_virtual]# sync
[root at web2 www_virtual]# df
Filesystem           1K-blocks      Used Available Use% Mounted on
/dev/xvda1             8124856   2856744   4848736  38% /
tmpfs                   509256         0    509256   0% /dev/shm
/dev/xvdf1           101441468 101419972         0 100% /glusterfs/vol01
web2:/vol_replica_01 101441408 101420032         0 100% /mnt/glusterfs

Glusterfs is still disk full.
The other glusterfs server is same.

[find the deleted file]
(server web2)
[root at web2 www_virtual]# ls -l /proc/*/fd/* | grep deleted
lr-x------ 1 root root 64 Sep 11 14:15 /proc/1753/fd/14 ->
/var/lib/glusterd/snaps/missed_snaps_list (deleted)
lrwx------ 1 root root 64 Sep 11 14:17 /proc/1775/fd/21 ->
/glusterfs/vol01/brick/.glusterfs/1f/c2/1fc2b7b4-ecd6-4eff-a874-962c2283823f
(deleted)

[root at web2 .glusterfs]# ps ax | grep 1775 | grep -v grep
 1775 ?        Ssl    5:43 /usr/sbin/glusterfsd -s web2 --volfile-id
vol_replica_01.web2.glusterfs-vol01-brick -p
/var/lib/glusterd/vols/vol_replica_01/run/web2-glusterfs-vol01-brick.pid
-S /var/run/1cf98ee59b5dff8cfd793b8ec39851db.socket --brick-name
/glusterfs/vol01/brick -l
/var/log/glusterfs/bricks/glusterfs-vol01-brick.log --xlator-option
*-posix.glusterd-uuid=029cf626-935f-4546-a8df-f9d79a6959da
--brick-port 49152 --xlator-option
vol_replica_01-server.listen-port=49152

[root at web2 www_virtual]# lsof -p 1775
COMMAND    PID USER   FD   TYPE             DEVICE    SIZE/OFF    NODE NAME
glusterfs 1775 root  cwd    DIR              202,1        4096       2 /
glusterfs 1775 root  rtd    DIR              202,1        4096       2 /
glusterfs 1775 root  txt    REG              202,1       78056  266837
/usr/sbin/glusterfsd
glusterfs 1775 root  mem    REG              202,1        8560     854
/usr/lib64/glusterfs/3.6.5/auth/login.so
glusterfs 1775 root  mem    REG              202,1       13248     853
/usr/lib64/glusterfs/3.6.5/auth/addr.so
glusterfs 1775 root  mem    REG              202,1      224920    4334
/usr/lib64/glusterfs/3.6.5/xlator/protocol/server.so
glusterfs 1775 root  mem    REG              202,1      118760    2421
/usr/lib64/glusterfs/3.6.5/xlator/debug/io-stats.so
glusterfs 1775 root  mem    REG              202,1      119688    2507
/usr/lib64/glusterfs/3.6.5/xlator/features/quota.so
glusterfs 1775 root  mem    REG              202,1      139784    2491
/usr/lib64/glusterfs/3.6.5/xlator/features/marker.so
glusterfs 1775 root  mem    REG              202,1       42304    2475
/usr/lib64/glusterfs/3.6.5/xlator/features/index.so
glusterfs 1775 root  mem    REG              202,1       34360    2447
/usr/lib64/glusterfs/3.6.5/xlator/features/barrier.so
glusterfs 1775 root  mem    REG              202,1       46936    2561
/usr/lib64/glusterfs/3.6.5/xlator/performance/io-threads.so
glusterfs 1775 root  mem    REG              202,1      107984    2476
/usr/lib64/glusterfs/3.6.5/xlator/features/locks.so
glusterfs 1775 root  mem    REG              202,1       58440    3940
/usr/lib64/glusterfs/3.6.5/xlator/system/posix-acl.so
glusterfs 1775 root  mem    REG              202,1       90880  266322
/lib64/libgcc_s-4.4.7-20120601.so.1
glusterfs 1775 root  mem    REG              202,1       96200    2460
/usr/lib64/glusterfs/3.6.5/xlator/features/changelog.so
glusterfs 1775 root  mem    REG              202,1        3944  272350
/lib64/libaio.so.1.0.1
glusterfs 1775 root  mem    REG              202,1      188376    4337
/usr/lib64/glusterfs/3.6.5/xlator/storage/posix.so
glusterfs 1775 root  mem    REG              202,1       65928  273938
/lib64/libnss_files-2.12.so
glusterfs 1775 root  mem    REG              202,1      122040  264857
/lib64/libselinux.so.1
glusterfs 1775 root  mem    REG              202,1      110960  273940
/lib64/libresolv-2.12.so
glusterfs 1775 root  mem    REG              202,1       10192  267134
/lib64/libkeyutils.so.1.3
glusterfs 1775 root  mem    REG              202,1       43728  267143
/lib64/libkrb5support.so.0.1
glusterfs 1775 root  mem    REG              202,1      174840  267139
/lib64/libk5crypto.so.3.1
glusterfs 1775 root  mem    REG              202,1       14664  265208
/lib64/libcom_err.so.2.1
glusterfs 1775 root  mem    REG              202,1      946048  267141
/lib64/libkrb5.so.3.3
glusterfs 1775 root  mem    REG              202,1      277704  267135
/lib64/libgssapi_krb5.so.2.2
glusterfs 1775 root  mem    REG              202,1      441256  280919
/usr/lib64/libssl.so.1.0.1e
glusterfs 1775 root  mem    REG              202,1       83384    2328
/usr/lib64/glusterfs/3.6.5/rpc-transport/socket.so
glusterfs 1775 root  mem    REG              202,1    99158576  281041
/usr/lib/locale/locale-archive
glusterfs 1775 root  mem    REG              202,1       88600  265291
/lib64/libz.so.1.2.3
glusterfs 1775 root  mem    REG              202,1       19536  273934
/lib64/libdl-2.12.so
glusterfs 1775 root  mem    REG              202,1     1921216  264860
/lib64/libc-2.12.so
glusterfs 1775 root  mem    REG              202,1     1963296  267216
/usr/lib64/libcrypto.so.1.0.1e
glusterfs 1775 root  mem    REG              202,1      142640  264887
/lib64/libpthread-2.12.so
glusterfs 1775 root  mem    REG              202,1       43880  273941
/lib64/librt-2.12.so
glusterfs 1775 root  mem    REG              202,1       95968  265199
/usr/lib64/libgfxdr.so.0.0.1
glusterfs 1775 root  mem    REG              202,1      107536  264893
/usr/lib64/libgfrpc.so.0.0.1
glusterfs 1775 root  mem    REG              202,1      633304  265202
/usr/lib64/libglusterfs.so.0.0.1
glusterfs 1775 root  mem    REG              202,1      154664  264855
/lib64/ld-2.12.so
glusterfs 1775 root  mem    REG              202,1       26060  265172
/usr/lib64/gconv/gconv-modules.cache
glusterfs 1775 root    0r   CHR                1,3         0t0    3923 /dev/null
glusterfs 1775 root    1w   CHR                1,3         0t0    3923 /dev/null
glusterfs 1775 root    2w   CHR                1,3         0t0    3923 /dev/null
glusterfs 1775 root    3u   REG                0,9           0    3919
[eventpoll]
glusterfs 1775 root    4u  unix 0xffff88003d027440         0t0  146494 socket
glusterfs 1775 root    5w   REG              202,1       28428     778
/var/log/glusterfs/bricks/glusterfs-vol01-brick.log
glusterfs 1775 root    6uW  REG              202,1           5     687
/var/lib/glusterd/vols/vol_replica_01/run/web2-glusterfs-vol01-brick.pid
glusterfs 1775 root    7u  unix 0xffff880037b470c0         0t0  146497
/var/run/1cf98ee59b5dff8cfd793b8ec39851db.socket
glusterfs 1775 root    8u  IPv4             146668         0t0     TCP
web2:49152->web2:1018 (ESTABLISHED)
glusterfs 1775 root    9u  IPv4             146505         0t0     TCP
web2:1020->web2:24007 (ESTABLISHED)
glusterfs 1775 root   10u  IPv4             146729         0t0     TCP
web2:49152->web2:1009 (ESTABLISHED)
glusterfs 1775 root   11u  IPv4             146608         0t0     TCP
*:49152 (LISTEN)
glusterfs 1775 root   12r   CHR                1,9         0t0    3928
/dev/urandom
glusterfs 1775 root   13r   DIR             202,81        4096 5505025
/glusterfs/vol01/brick
glusterfs 1775 root   14u  unix 0xffff88003d027780         0t0  146737
/var/run/1cf98ee59b5dff8cfd793b8ec39851db.socket
glusterfs 1775 root   15u  IPv4             146749         0t0     TCP
web2:49152->web1:1020 (ESTABLISHED)
glusterfs 1775 root   16u  IPv4             146751         0t0     TCP
web2:49152->web1:1017 (ESTABLISHED)
glusterfs 1775 root   17u  IPv4             146753         0t0     TCP
web2:49152->web1:1014 (ESTABLISHED)
glusterfs 1775 root   18u  IPv4             146759         0t0     TCP
web2:49152->web2:1006 (ESTABLISHED)
glusterfs 1775 root   20r   DIR             202,81        4096 5505028
/glusterfs/vol01/brick/.glusterfs/indices/xattrop
glusterfs 1775 root   21u   REG             202,81 16349855744 5520418
/glusterfs/vol01/brick/.glusterfs/1f/c2/1fc2b7b4-ecd6-4eff-a874-962c2283823f
(deleted)

(server web1)
[root at web1 www_virtual]# df
Filesystem           1K-blocks      Used Available Use% Mounted on
/dev/xvda1             8124856   5608396   2097084  73% /
tmpfs                   960324         0    960324   0% /dev/shm
/dev/xvdf1           101441468 101420068         0 100% /glusterfs/vol01
web1:/vol_replica_01 101441408 101420032         0 100% /mnt/glusterfs

[root at web1 www_virtual]#  ls -l /proc/*/fd/* | grep deleted
ls: cannot access /proc/2348/fd/255: No such file or directory
ls: cannot access /proc/2348/fd/3: No such file or directory
ls: cannot access /proc/self/fd/255: No such file or directory
ls: cannot access /proc/self/fd/3: No such file or directory
lr-x------ 1 root root 64 Sep 11 14:21 /proc/1061/fd/14 ->
/var/lib/glusterd/snaps/missed_snaps_list (deleted)
lrwx------ 1 root root 64 Sep 11 14:21 /proc/1436/fd/16 ->
/glusterfs/vol01/brick/.glusterfs/1f/c2/1fc2b7b4-ecd6-4eff-a874-962c2283823f
(deleted)

[root at web1 www_virtual]# ps ax | grep 1436
 1436 ?        Ssl   15:24 /usr/sbin/glusterfsd -s web1 --volfile-id
vol_replica_01.web1.glusterfs-vol01-brick -p
/var/lib/glusterd/vols/vol_replica_01/run/web1-glusterfs-vol01-brick.pid
-S /var/run/623572fd107c86f79cd247c24572d645.socket --brick-name
/glusterfs/vol01/brick -l
/var/log/glusterfs/bricks/glusterfs-vol01-brick.log --xlator-option
*-posix.glusterd-uuid=cd648272-ac40-47c0-bbda-f2bb4894480a
--brick-port 49152 --xlator-option
vol_replica_01-server.listen-port=49152

[root at web1 www_virtual]# lsof -p 1436
COMMAND    PID USER   FD   TYPE             DEVICE    SIZE/OFF    NODE NAME
glusterfs 1436 root  cwd    DIR              202,1        4096       2 /
glusterfs 1436 root  rtd    DIR              202,1        4096       2 /
glusterfs 1436 root  txt    REG              202,1       78056  272348
/usr/sbin/glusterfsd
glusterfs 1436 root  mem    REG              202,1        8560  405980
/usr/lib64/glusterfs/3.6.5/auth/login.so
glusterfs 1436 root  mem    REG              202,1       13248  405978
/usr/lib64/glusterfs/3.6.5/auth/addr.so
glusterfs 1436 root  mem    REG              202,1      224920  406066
/usr/lib64/glusterfs/3.6.5/xlator/protocol/server.so
glusterfs 1436 root  mem    REG              202,1      118760  406003
/usr/lib64/glusterfs/3.6.5/xlator/debug/io-stats.so
glusterfs 1436 root  mem    REG              202,1      119688  406028
/usr/lib64/glusterfs/3.6.5/xlator/features/quota.so
glusterfs 1436 root  mem    REG              202,1      139784  406022
/usr/lib64/glusterfs/3.6.5/xlator/features/marker.so
glusterfs 1436 root  mem    REG              202,1       42304  406020
/usr/lib64/glusterfs/3.6.5/xlator/features/index.so
glusterfs 1436 root  mem    REG              202,1       34360  406016
/usr/lib64/glusterfs/3.6.5/xlator/features/barrier.so
glusterfs 1436 root  mem    REG              202,1       46936  406037
/usr/lib64/glusterfs/3.6.5/xlator/performance/io-threads.so
glusterfs 1436 root  mem    REG              202,1      107984  406021
/usr/lib64/glusterfs/3.6.5/xlator/features/locks.so
glusterfs 1436 root  mem    REG              202,1       58440  406048
/usr/lib64/glusterfs/3.6.5/xlator/system/posix-acl.so
glusterfs 1436 root  mem    REG              202,1       90880  267449
/lib64/libgcc_s-4.4.7-20120601.so.1
glusterfs 1436 root  mem    REG              202,1       96200  406018
/usr/lib64/glusterfs/3.6.5/xlator/features/changelog.so
glusterfs 1436 root  mem    REG              202,1        3944  272159
/lib64/libaio.so.1.0.1
glusterfs 1436 root  mem    REG              202,1      188376  406069
/usr/lib64/glusterfs/3.6.5/xlator/storage/posix.so
glusterfs 1436 root  mem    REG              202,1       65928  275388
/lib64/libnss_files-2.12.so
glusterfs 1436 root  mem    REG              202,1      122040  264857
/lib64/libselinux.so.1
glusterfs 1436 root  mem    REG              202,1      110960  275392
/lib64/libresolv-2.12.so
glusterfs 1436 root  mem    REG              202,1       10192  267134
/lib64/libkeyutils.so.1.3
glusterfs 1436 root  mem    REG              202,1       43728  267143
/lib64/libkrb5support.so.0.1
glusterfs 1436 root  mem    REG              202,1      174840  267139
/lib64/libk5crypto.so.3.1
glusterfs 1436 root  mem    REG              202,1       14664  265208
/lib64/libcom_err.so.2.1
glusterfs 1436 root  mem    REG              202,1      946048  267141
/lib64/libkrb5.so.3.3
glusterfs 1436 root  mem    REG              202,1      277704  267135
/lib64/libgssapi_krb5.so.2.2
glusterfs 1436 root  mem    REG              202,1      441256  281908
/usr/lib64/libssl.so.1.0.1e
glusterfs 1436 root  mem    REG              202,1       83384  405982
/usr/lib64/glusterfs/3.6.5/rpc-transport/socket.so
glusterfs 1436 root  mem    REG              202,1    99158576  272725
/usr/lib/locale/locale-archive
glusterfs 1436 root  mem    REG              202,1       88600  265291
/lib64/libz.so.1.2.3
glusterfs 1436 root  mem    REG              202,1       19536  274985
/lib64/libdl-2.12.so
glusterfs 1436 root  mem    REG              202,1     1921216  264860
/lib64/libc-2.12.so
glusterfs 1436 root  mem    REG              202,1     1963296  267216
/usr/lib64/libcrypto.so.1.0.1e
glusterfs 1436 root  mem    REG              202,1      142640  264887
/lib64/libpthread-2.12.so
glusterfs 1436 root  mem    REG              202,1       43880  275393
/lib64/librt-2.12.so
glusterfs 1436 root  mem    REG              202,1       95968  272324
/usr/lib64/libgfxdr.so.0.0.1
glusterfs 1436 root  mem    REG              202,1      107536  272323
/usr/lib64/libgfrpc.so.0.0.1
glusterfs 1436 root  mem    REG              202,1      633304  272325
/usr/lib64/libglusterfs.so.0.0.1
glusterfs 1436 root  mem    REG              202,1      154664  264855
/lib64/ld-2.12.so
glusterfs 1436 root  mem    REG              202,1       26060  265172
/usr/lib64/gconv/gconv-modules.cache
glusterfs 1436 root    0r   CHR                1,3         0t0    3923 /dev/null
glusterfs 1436 root    1w   CHR                1,3         0t0    3923 /dev/null
glusterfs 1436 root    2w   CHR                1,3         0t0    3923 /dev/null
glusterfs 1436 root    3u   REG                0,9           0    3919
[eventpoll]
glusterfs 1436 root    4u  unix 0xffff88007b2b23c0         0t0   75161 socket
glusterfs 1436 root    5w   REG              202,1       13674     838
/var/log/glusterfs/bricks/glusterfs-vol01-brick.log
glusterfs 1436 root    6uW  REG              202,1           5  401217
/var/lib/glusterd/vols/vol_replica_01/run/web1-glusterfs-vol01-brick.pid
glusterfs 1436 root    7u  unix 0xffff88003799f0c0         0t0   75221
/var/run/623572fd107c86f79cd247c24572d645.socket
glusterfs 1436 root    8u  IPv4              75365         0t0     TCP
web1:49152->web1:1016 (ESTABLISHED)
glusterfs 1436 root    9u  IPv4              75231         0t0     TCP
web1:1018->web1:24007 (ESTABLISHED)
glusterfs 1436 root   10u  IPv4              75397         0t0     TCP
web1:49152->web1:1009 (ESTABLISHED)
glusterfs 1436 root   11u  IPv4              75275         0t0     TCP
*:49152 (LISTEN)
glusterfs 1436 root   12r   CHR                1,9         0t0    3928
/dev/urandom
glusterfs 1436 root   13r   DIR             202,81        4096 2621441
/glusterfs/vol01/brick
glusterfs 1436 root   14u  IPv4              75401         0t0     TCP
web1:49152->web2:1015 (ESTABLISHED)
glusterfs 1436 root   16u   REG             202,81 16349941760 2639508
/glusterfs/vol01/brick/.glusterfs/1f/c2/1fc2b7b4-ecd6-4eff-a874-962c2283823f
(deleted)
glusterfs 1436 root   17u  unix 0xffff880079b85100         0t0   75422
/var/run/623572fd107c86f79cd247c24572d645.socket
glusterfs 1436 root   18u  IPv4              75428         0t0     TCP
web1:49152->web1:1007 (ESTABLISHED)
glusterfs 1436 root   19u  IPv4              92958         0t0     TCP
web1:49152->web2:1008 (ESTABLISHED)
glusterfs 1436 root   20u  IPv4              92955         0t0     TCP
web1:49152->web2:1017 (ESTABLISHED)

[gluster status]

[root at web2 www_virtual]# gluster volume info

Volume Name: vol_replica_01
Type: Replicate
Volume ID: c6f5b52c-865f-4dc1-9377-ea81d933b859
Status: Started
Number of Bricks: 1 x 2 = 2
Transport-type: tcp
Bricks:
Brick1: web1:/glusterfs/vol01/brick
Brick2: web2:/glusterfs/vol01/brick
Options Reconfigured:
network.ping-timeout: 15
performance.cache-max-file-size: 100MB

[root at web1 www_virtual]# gluster volume info

Volume Name: vol_replica_01
Type: Replicate
Volume ID: c6f5b52c-865f-4dc1-9377-ea81d933b859
Status: Started
Number of Bricks: 1 x 2 = 2
Transport-type: tcp
Bricks:
Brick1: web1:/glusterfs/vol01/brick
Brick2: web2:/glusterfs/vol01/brick
Options Reconfigured:
network.ping-timeout: 15
performance.cache-max-file-size: 100MB

[gluster config]
/etc/glusterfs/glusterd.vol
volume management
    type mgmt/glusterd
    option working-directory /var/lib/glusterd
    option transport-type socket
    option transport.socket.keepalive-time 10
    option transport.socket.keepalive-interval 2
    option transport.socket.read-fail-log off
    option ping-timeout 30
end-volume

Best,
Yasu


More information about the Gluster-users mailing list