[Gluster-users] Problem no. 2

[artur.k]

I've got 4 glusterfs servers (unify + afr).Everythinh works fine. I do the following: 


#!/bin/bash


for ((i = 0; i < 100000000; i++)) {

    KAT=$RANDOM
    mkdir /mnt/glusterfs/$KAT
    for ((j=0; j< $RANDOM; j++)) {
        PLIK=$RANDOM.$RANDOM
        echo "PLIK: $KAT/$PLIK "
        dd if=/dev/urandom of=/mnt/glusterfs/$KAT/$PLIK bs=$RANDOM count=1
        dd if=/mnt/glusterfs/$KAT/$PLIK of=/dev/null
        rm -f /mnt/glusterfs/$KAT/$PLIK
    }
    rm -rf /mnt/glusterfs/$KAT

I stop gluster server number 1. I wait 1 minute, start the server again and everything works fine.
I stop gluster server number 2. I wait 1 minute, start the server again and everything works fine.
I stop gluster server number 3 and I get: 


11613 bytes (12 kB) copied, 0.001298 seconds, 8.9 MB/s
PLIK: 25687/3290.6708
1+0 records in
1+0 records out
18442 bytes (18 kB) copied, 0.005242 seconds, 3.5 MB/s
36+1 records in
36+1 records out
18442 bytes (18 kB) copied, 0.001331 seconds, 13.9 MB/s
PLIK: 25687/25858.17300
dd: opening `/mnt/glusterfs/25687/25858.17300': Transport endpoint is not connected
dd: opening `/mnt/glusterfs/25687/25858.17300': No such file or directory
PLIK: 25687/3626.14066
dd: opening `/mnt/glusterfs/25687/3626.14066': Transport endpoint is not connected
dd: opening `/mnt/glusterfs/25687/3626.14066': No such file or directory
PLIK: 25687/29220.20962
dd: opening `/mnt/glusterfs/25687/29220.20962': Transport endpoint is not connected
dd: opening `/mnt/glusterfs/25687/29220.20962': No such file or directory
PLIK: 25687/3106.1086
dd: opening `/mnt/glusterfs/25687/3106.1086': Transport endpoint is not connected
dd: opening `/mnt/glusterfs/25687/3106.1086': No such file or directory
PLIK: 25687/32498.4016
dd: opening `/mnt/glusterfs/25687/32498.4016': Transport endpoint is not connected
dd: opening `/mnt/glusterfs/25687/32498.4016': No such file or directory
PLIK: 25687/21239.13312
dd: opening `/mnt/glusterfs/25687/21239.13312': Transport endpoint is not connected
dd: opening `/mnt/glusterfs/25687/21239.13312': No such file or directory
PLIK: 25687/6123.21534

noc-test-2:~# cat /mnt/glusterfs/25687/6123.21534
cat: /mnt/glusterfs/25687/6123.21534: No such file or directory
noc-test-2:~# echo 1 > /mnt/glusterfs/25687/6123.21534
bash: /mnt/glusterfs/25687/6123.21534: Transport endpoint is not connected

In the logs I have:
2009-01-14 14:01:48 E [fuse-bridge.c:1423:fuse_create_cbk] glusterfs-fuse: 31336: /25687/29931.14091 => -1 (Transport endpoint is not connected)
2009-01-14 14:01:48 E [unify.c:1614:unify_ns_create_cbk] unify: no node online to schedule create:(file /25687/19473.19860) sending unlink to namespace
2009-01-14 14:01:48 E [fuse-bridge.c:1423:fuse_create_cbk] glusterfs-fuse: 31340: /25687/19473.19860 => -1 (Transport endpoint is not connected)
2009-01-14 14:01:49 E [unify.c:1614:unify_ns_create_cbk] unify: no node online to schedule create:(file /25687/15325.23330) sending unlink to namespace
2009-01-14 14:01:49 E [fuse-bridge.c:1423:fuse_create_cbk] glusterfs-fuse: 31344: /25687/15325.23330 => -1 (Transport endpoint is not connected)
2009-01-14 14:01:49 E [unify.c:1614:unify_ns_create_cbk] unify: no node online to schedule create:(file /25687/7787.29078) sending unlink to namespace
2009-01-14 14:01:49 E [fuse-bridge.c:1423:fuse_create_cbk] glusterfs-fuse: 31348: /25687/7787.29078 => -1 (Transport endpoint is not connected)


client:
volume client1
  type protocol/client
  option transport-type tcp/client
  # noc-test-1
  option remote-host xx
  option remote-port 6996
  option remote-subvolume brick
end-volume

volume client1-ns
  type protocol/client
  option transport-type tcp/client
  # noc-test-1
  option remote-host xx
  option remote-port 6996
  option remote-subvolume brick-ns
end-volume

volume client2
 type protocol/client
 option transport-type tcp/client
 # www-storage-2
 option remote-host xx
 option remote-port 6996
 option remote-subvolume brick
end-volume

volume client2-ns
 type protocol/client
 option transport-type tcp/client
 # www-storage-2
 option remote-host xx
 option remote-port 6996
 option remote-subvolume brick-ns
end-volume


volume client3
 type protocol/client
 option transport-type tcp/client
 # trac-storage-1
 option remote-host xx
 option remote-port 6996
 option remote-subvolume brick
end-volume

volume client3-ns
 type protocol/client
 option transport-type tcp/client
 # trac-storage-1
 option remote-host xx
 option remote-port 6996
 option remote-subvolume brick-ns
end-volume


volume client4
 type protocol/client
 option transport-type tcp/client
 # trac-storage-2
 option remote-host xx
 option remote-port 6996
 option remote-subvolume brick
end-volume

volume client4-ns
 type protocol/client
 option transport-type tcp/client
 # trac-storage-2
 option remote-host xx
 option remote-port 6996
 option remote-subvolume brick-ns
end-volume


volume afr-1
  type cluster/afr
  subvolumes client1 client3
#  option entry-self-heal on
#  option data-self-heal on
#  option metadata-self-heal off
  option favorite-child client1
end-volume

volume afr-2
  type cluster/afr
  subvolumes client2 client4
#  option entry-self-heal on
#  option data-self-heal on
#  option metadata-self-heal off
  option favorite-child client4
end-volume


volume afr-ns
  type cluster/afr
  subvolumes client1-ns client2-ns client3-ns client4-ns
#  option entry-self-heal on
#  option data-self-heal on
#  option metadata-self-heal off
  option favorite-child client1-ns
end-volume

volume unify
    type cluster/unify
    option namespace afr-ns
    option scheduler rr
    subvolumes afr-1 afr-2
end-volume


volume wh
  type performance/write-behind
  option flush-behind on
  subvolumes unify
end-volume

volume io-cache
  type performance/io-cache
  option cache-size 64MB
  option page-size 1MB
#  option cache-timeout 2
  subvolumes wh
end-volume

volume iot
  type performance/io-threads
  subvolumes io-cache
  option thread-count 4
end-volume



glusterfs 2.0.0qa1 built on Jan  9 2009 14:14:17
Repository revision: glusterfs--mainline--3.0--patch-840