[Gluster-devel] Distribute problem on rc2 release

Thu Feb 26 11:55:08 UTC 2009

Dear All,
after upgrade to glusterfs rc2 I have this scenario:

[root at xen1 glusterfs]# time dd if=/dev/zero of=/mnt/glusterfs/debian_40.img
bs=1M count=10
10+0 records in
10+0 records out
10485760 bytes (10 MB) copied, 0,413117 seconds, 25,4 MB/s

real    0m0.417s
user    0m0.000s
sys     0m0.004s
[root at xen1 glusterfs]# time dd if=/dev/zero of=/mnt/glusterfs/debian_40.img
bs=1M count=100
100+0 records in
100+0 records out
104857600 bytes (105 MB) copied, 3,48037 seconds, 30,1 MB/s

real    0m3.489s
user    0m0.000s
sys     0m0.056s
[root at xen1 glusterfs]# time dd if=/dev/zero of=/mnt/glusterfs/debian_50.img
bs=1M count=150
150+0 records in
150+0 records out
157286400 bytes (157 MB) copied, 1,75172 seconds, 89,8 MB/s

real    0m1.755s
user    0m0.000s
sys     0m0.300s
[root at xen1 glusterfs]# time dd if=/dev/zero of=/mnt/glusterfs/debian_50.img
bs=1M count=200
dd: scrittura di `/mnt/glusterfs/debian_50.img': Transport endpoint is not
connected
dd: chiusura del file di output `/mnt/glusterfs/debian_50.img': Transport
endpoint is not connected

real    0m20.878s
user    0m0.000s
sys     0m0.080s

Now, into CLIENT log I retry this error:
2009-02-26 12:45:21 N [glusterfsd.c:1121:main] glusterfs: Successfully
started
2009-02-26 12:45:21 N [client-protocol.c:5853:client_setvolume_cbk]
remote2: connection and handshake succeeded
2009-02-26 12:45:21 N [client-protocol.c:5853:client_setvolume_cbk]
remote2: connection and handshake succeeded
2009-02-26 12:45:21 N [client-protocol.c:5853:client_setvolume_cbk]
remote1: connection and handshake succeeded
2009-02-26 12:45:21 N [client-protocol.c:5853:client_setvolume_cbk]
remote1: connection and handshake succeeded
2009-02-26 12:46:30 E [client-protocol.c:416:client_ping_timer_expired]
remote2: ping timer expired! bailing transport
2009-02-26 12:46:30 E [ib-verbs.c:1991:ib_verbs_event_handler]
transport/ib-verbs: remote2: pollin received on tcp socket (peer:
10.0.0.11:6996) after handshake is complete
2009-02-26 12:46:30 E [saved-frames.c:149:saved_frames_unwind] remote2:
forced unwinding frame type(1) op(WRITE)
2009-02-26 12:46:30 E [fuse-bridge.c:1602:fuse_writev_cbk] glusterfs-fuse:
109467: WRITE => -1 (Transport endpoint is not connected)
2009-02-26 12:46:30 E [ib-verbs.c:392:ib_verbs_writev] remote2: ib-verbs is
not connected to post a send request
2009-02-26 12:46:30 E [fuse-bridge.c:1602:fuse_writev_cbk] glusterfs-fuse:
109468: WRITE => -1 (Transport endpoint is not connected)
2009-02-26 12:46:30 E [ib-verbs.c:392:ib_verbs_writev] remote2: ib-verbs is
not connected to post a send request
2009-02-26 12:46:30 E [dht-common.c:1294:dht_err_cbk] distribute0:
subvolume remote2 returned -1 (Transport endpoint is not connected)
2009-02-26 12:46:30 E [fuse-bridge.c:920:fuse_err_cbk] glusterfs-fuse:
109469: FLUSH() ERR => -1 (Transport endpoint is not connected)
2009-02-26 12:46:30 E [ib-verbs.c:392:ib_verbs_writev] remote2: ib-verbs is
not connected to post a send request
2009-02-26 12:46:30 E [saved-frames.c:149:saved_frames_unwind] remote2:
forced unwinding frame type(2) op((null))
2009-02-26 12:46:30 E [client-protocol.c:522:client_ping_cbk] remote2:
timer must have expired
2009-02-26 12:46:30 N [client-protocol.c:6495:notify] remote2: disconnected
2009-02-26 12:46:40 N [client-protocol.c:5853:client_setvolume_cbk]
remote2: connection and handshake succeeded

Configuration, hardware and s.o. are same at rc1, that work without
problems!

Config file:

#### SERVER ####
volume brick
  type storage/posix # POSIX FS translator
  option directory /mnt/glusterfs1/export # Export this directory
end-volume

volume posix-locks
  type features/locks
  option mandatory-locks on
  subvolumes brick
end-volume

volume iot
 type performance/io-threads
  option thread-count 12 # deafult is 1
 subvolumes posix-locks
end-volume

volume wb
 type performance/write-behind
  option page-size 1MB
  option cache-size 4MB
  option flush-behind on # default is 'off'
 subvolumes iot
end-volume

volume ra
 type performance/read-ahead
  option page-size  1MB # is the default option
  option page-count 8 # 2 is default option
  option force-atime-update off # default is off
 subvolumes wb
end-volume

volume ioc
 type performance/io-cache
  option cache-size 64MB             # default is 32MB
  option page-size 512kB               #128KB is default option
  #option priority *.h:3,*.html:2,*:1 # default is '*:0'
  option cache-timeout 2  # default is 1
 subvolumes ra
end-volume

### Add network serving capability to above brick.
volume server
  type protocol/server
  option transport.ib-verbs.work-request-send-size 1048576 
  option transport.ib-verbs.work-request-send-count 16
  option transport.ib-verbs.work-request-recv-size 1048576 
  option transport.ib-verbs.work-request-recv-count 8
  option transport-type ib-verbs/server # For ib-verbs transport
  option transport.ib-verbs.listen-path 10.0.0.11
  option transport.ib-verbs.listen-port 6996 # Default is 6996
  subvolumes brick
# NOTE: Access to any volume through protocol/server is denied by
# default. You need to explicitly grant access through # "auth"
# option.
  option auth.addr.brick.allow * # Allow access to "brick" volume
end-volume

#### CLIENT ####
volume remote1
  type protocol/client
  option transport-type ib-verbs/client # For ib-verbs transport
  option transport.ib-verbs.work-request-send-size 1048576 
  option transport.ib-verbs.work-request-send-count 16
  option transport.ib-verbs.work-request-recv-size 1048576 
  option transport.ib-verbs.work-request-recv-count 8
  option transport.ib-verbs.remote-port 6996
  option transport.address-family inet/inet6
  option remote-host 10.0.0.10
  option remote-subvolume brick
end-volume

volume remote2
  type protocol/client
  option transport-type ib-verbs/client # For ib-verbs transport
  option transport.ib-verbs.work-request-send-size 1048576 
  option transport.ib-verbs.work-request-send-count 16
  option transport.ib-verbs.work-request-recv-size 1048576
  option transport.ib-verbs.work-request-recv-count 8
  option transport.ib-verbs.remote-port 6996
  option transport.address-family inet/inet6
  option remote-host 10.0.0.11
  option remote-subvolume brick
end-volume

volume distribute0
  type cluster/distribute
  subvolumes remote1 remote2
end-volume

volume iot
 type performance/io-threads
  option thread-count 12  # deault is 1
 subvolumes distribute0
end-volume

volume wb
 type performance/write-behind
  option aggregate-size 2MB #1048576
  option window-size 2MB #1048576
  option flush-behind on    # default is 'off'
 subvolumes iot
end-volume

volume ioc
 type performance/io-cache
  option cache-size 128MB             # default is 32MB
  option page-size 1MB               #128KB is default option
  option cache-timeout 1  # default is 1
 subvolumes wb
end-volume

volume ra
 type performance/read-ahead
  option page-size 1048576      # is the default option
  option page-count 8           # 2 is default option
  option force-atime-update off # default is off
 subvolumes ioc
end-volume