[Gluster-users] read-subvolume

Tue Jul 2 11:38:34 UTC 2013

Hi everyone

I have installed 3.3.1-1 from the Debian repository you provide.

I am using a simple 2 node cluster and running in replication mode. The
connection between the nodes is limited to 100MB/sec (that's bits not
bytes!). Usage will be mainly for read access and since there is always
a local copy available [ exactly 2 replicas on exactly 2 machines ] I
expect very fast read performance. Writes are low volume and very
infrequent - performance is not an issue.

Almost everything works as I would expect.

Write speed is limited to 10Mb (bytes) per second which is what I would
expect and is adequate for the application.

But read speed is either super fast or 10Mb/sec. i.e. read operations
take place on the local copy or the remote seemingly at random.

This not the 'small files problem'. I am aware that Gluster must use
network access for stat() etc. This is all about where the data comes
from on a read(). If I do an m5dum on a 200Mb file it takes either half
a second or 18 seconds.

There is an option read-subvolume.

I have tried to understand how this works from the documentation
available and from the few examples on the web.

I have added the option using:

gluster volume set X read-subvolume Y

It has no effect even after stopping and starting the volume,
remounting, restarting gluster servers etc.

What's more I fail to see how this option could ever work at all. The
configuration changes caused by the above command are rolled out to both
nodes - but what is right for one node is exactly the wrong
configuration for the other node.

Configs attached are in /var/lib/glusterd/vols/shared except
glusterd.vol which is in /etc/glusterfs.

Here is the output of the mount command filtered to just the glusterfs
mount:

10.255.255.1:/shared on /gluster/rw/shared type fuse.glusterfs
(rw,relatime,user_id=0,group_id=0,default_permissions,allow_other,max_read=131072)

10.255.255.1 is local to this host.

I would be very thankful if someone can enlighten me. I am obviously
configuring this wrong. I may have missed something important.

Best regards to all

Allan

-------------- next part --------------
volume shared-client-0
    type protocol/client
    option remote-host 10.255.255.1
    option remote-subvolume /gluster/vol/shared1/brick
    option transport-type tcp
    option username 1bd771cd-3506-48fc-95de-2fd7605cc012
    option password cc801d8d-305b-45c4-9820-bb396ce65d9b
end-volume

volume shared-client-1
    type protocol/client
    option remote-host 10.255.255.2
    option remote-subvolume /gluster/vol/shared2/brick
    option transport-type tcp
    option username 1bd771cd-3506-48fc-95de-2fd7605cc012
    option password cc801d8d-305b-45c4-9820-bb396ce65d9b
end-volume

volume shared-replicate-0
    type cluster/replicate
    option read-subvolume shared-client-1
    subvolumes shared-client-0 shared-client-1
end-volume

volume shared-write-behind
    type performance/write-behind
    subvolumes shared-replicate-0
end-volume

volume shared-read-ahead
    type performance/read-ahead
    subvolumes shared-write-behind
end-volume

volume shared-io-cache
    type performance/io-cache
    subvolumes shared-read-ahead
end-volume

volume shared-quick-read
    type performance/quick-read
    subvolumes shared-io-cache
end-volume

volume shared-md-cache
    type performance/md-cache
    subvolumes shared-quick-read
end-volume

volume shared
    type debug/io-stats
    option latency-measurement off
    option count-fop-hits off
    subvolumes shared-md-cache
end-volume
-------------- next part --------------
volume shared-posix
    type storage/posix
    option directory /gluster/vol/shared2/brick
    option volume-id df592c0a-8190-42d8-b0d5-e50458a5368e
end-volume

volume shared-access-control
    type features/access-control
    subvolumes shared-posix
end-volume

volume shared-locks
    type features/locks
    subvolumes shared-access-control
end-volume

volume shared-io-threads
    type performance/io-threads
    subvolumes shared-locks
end-volume

volume shared-index
    type features/index
    option index-base /gluster/vol/shared2/brick/.glusterfs/indices
    subvolumes shared-io-threads
end-volume

volume shared-marker
    type features/marker
    option volume-uuid df592c0a-8190-42d8-b0d5-e50458a5368e
    option timestamp-file /var/lib/glusterd/vols/shared/marker.tstamp
    option xtime off
    option quota off
    subvolumes shared-index
end-volume

volume /gluster/vol/shared2/brick
    type debug/io-stats
    option latency-measurement off
    option count-fop-hits off
    subvolumes shared-marker
end-volume

volume shared-server
    type protocol/server
    option transport-type tcp
    option auth.login./gluster/vol/shared2/brick.allow 1bd771cd-3506-48fc-95de-2fd7605cc012
    option auth.login.1bd771cd-3506-48fc-95de-2fd7605cc012.password cc801d8d-305b-45c4-9820-bb396ce65d9b
    option auth.addr./gluster/vol/shared2/brick.allow *
    subvolumes /gluster/vol/shared2/brick
end-volume
-------------- next part --------------
volume shared-posix
    type storage/posix
    option directory /gluster/vol/shared1/brick
    option volume-id df592c0a-8190-42d8-b0d5-e50458a5368e
end-volume

volume shared-access-control
    type features/access-control
    subvolumes shared-posix
end-volume

volume shared-locks
    type features/locks
    subvolumes shared-access-control
end-volume

volume shared-io-threads
    type performance/io-threads
    subvolumes shared-locks
end-volume

volume shared-index
    type features/index
    option index-base /gluster/vol/shared1/brick/.glusterfs/indices
    subvolumes shared-io-threads
end-volume

volume shared-marker
    type features/marker
    option volume-uuid df592c0a-8190-42d8-b0d5-e50458a5368e
    option timestamp-file /var/lib/glusterd/vols/shared/marker.tstamp
    option xtime off
    option quota off
    subvolumes shared-index
end-volume

volume /gluster/vol/shared1/brick
    type debug/io-stats
    option latency-measurement off
    option count-fop-hits off
    subvolumes shared-marker
end-volume

volume shared-server
    type protocol/server
    option transport-type tcp
    option auth.login./gluster/vol/shared1/brick.allow 1bd771cd-3506-48fc-95de-2fd7605cc012
    option auth.login.1bd771cd-3506-48fc-95de-2fd7605cc012.password cc801d8d-305b-45c4-9820-bb396ce65d9b
    option auth.addr./gluster/vol/shared1/brick.allow *
    subvolumes /gluster/vol/shared1/brick
end-volume
-------------- next part --------------
volume shared-client-0
    type protocol/client
    option remote-host 10.255.255.1
    option remote-subvolume /gluster/vol/shared1/brick
    option transport-type tcp
end-volume

volume shared-client-1
    type protocol/client
    option remote-host 10.255.255.2
    option remote-subvolume /gluster/vol/shared2/brick
    option transport-type tcp
end-volume

volume shared-replicate-0
    type cluster/replicate
    option read-subvolume shared-client-1
    subvolumes shared-client-0 shared-client-1
end-volume

volume shared-write-behind
    type performance/write-behind
    subvolumes shared-replicate-0
end-volume

volume shared-read-ahead
    type performance/read-ahead
    subvolumes shared-write-behind
end-volume

volume shared-io-cache
    type performance/io-cache
    subvolumes shared-read-ahead
end-volume

volume shared-quick-read
    type performance/quick-read
    subvolumes shared-io-cache
end-volume

volume shared-md-cache
    type performance/md-cache
    subvolumes shared-quick-read
end-volume

volume shared
    type debug/io-stats
    option latency-measurement off
    option count-fop-hits off
    subvolumes shared-md-cache
end-volume
-------------- next part --------------
volume management
    type mgmt/glusterd
    option working-directory /var/lib/glusterd
    option transport-type socket,rdma
    option transport.socket.keepalive-time 10
    option transport.socket.keepalive-interval 2
    option transport.socket.read-fail-log off
end-volume