[Gluster-devel] Infiniband help

[Nathan Allen Stratton]

Can anyone offer any help with infiniband? I can't find much at all on how
to configure server configs with ib-sdp or ib-verbs. I have basic
infiniband working:

root at vs1.voilaip.net# ibhosts
Ca      : 0x0002c90200001884 ports 2 "vs2 HCA-1"
Ca      : 0x0005ad00000505f8 ports 2 "vs0 HCA-1"
Ca      : 0x00066a0098003fe2 ports 2 "vs1 HCA-1"

I am not sure if IP over IB is required, but I turned that on and can ping
over infiniband also. I am not quite sure how the addressing works if you
dont use IP over IB.

I have tried the config below that works fine with tcp with ib-sdp and
ip-verbs and no matter what I try I just get connection refused. Do I need
to use the 0x0002c90200001884 addresses instead of the server IP addresses?
I have also tried using the ethernet hosts IPs as well as the infiniband
ib0 IPs.

# Server 0
volume brick-a-ns
        type protocol/client
        option transport-type ib-sdp/client
        option remote-host 192.168.0.10
        option transport-timeout 2
        option remote-subvolume brick-a-ns
end-volume
volume brick-a
        type protocol/client
        option transport-type ib-sdp/client
        option remote-host 192.168.0.10
        option transport-timeout 2
        option remote-subvolume brick-a
end-volume
volume mirror-c
        type protocol/client
        option transport-type ib-sdp/client
        option remote-host 192.168.0.10
        option transport-timeout 2
        option remote-subvolume mirror-c
end-volume

# Server 1
volume brick-b-ns
        type storage/posix
        option directory /ns
end-volume
volume brick-b
        type storage/posix
        option directory /raid/brick-b
end-volume
volume mirror-a
        type storage/posix
        option directory /raid/mirror-a
end-volume

# Server 2
volume brick-c-ns
        type protocol/client
        option transport-type ib-sdp/client
        option remote-host 192.168.0.12
        option transport-timeout 2
        option remote-subvolume brick-c-ns
end-volume
volume brick-c
        type protocol/client
        option transport-type ib-sdp/client
        option remote-host 192.168.0.12
        option transport-timeout 2
        option remote-subvolume brick-c
end-volume
volume mirror-b
        type protocol/client
        option transport-type ib-sdp/client
        option remote-host 192.168.0.12
        option transport-timeout 2
        option remote-subvolume mirror-b
end-volume

# NameSpace AFR
volume block-ns-afr
        type cluster/afr
        subvolumes brick-a-ns brick-b-ns brick-c-ns
        option replicate *:3
        option self-heal on
end-volume

# Block AFR
volume block-a-afr
        type cluster/afr
        subvolumes brick-a mirror-a
        option replicate *:2
        option self-heal on
end-volume
volume block-b-afr
        type cluster/afr
        subvolumes brick-b mirror-b
        option replicate *:2
        option self-heal on
end-volume
volume block-c-afr
        type cluster/afr
        subvolumes brick-c mirror-c
        option replicate *:2
        option self-heal on
end-volume

# Unify Blocks into Share
volume share-unify
        type cluster/unify
        subvolumes block-a-afr block-b-afr block-c-afr
        option namespace block-ns-afr
        option scheduler rr
end-volume

# Final Share with performance
volume share
        type performance/io-threads
        option thread-count 12
        option cache-size 32MB
        subvolumes share-unify
end-volume

# Server settings
volume server
        type protocol/server
        option transport-type ib-sdp/server
        subvolumes share
        option auth.ip.brick-b-ns.allow 192.168.0.*,127.0.0.1
        option auth.ip.brick-b.allow 192.168.0.*,127.0.0.1
        option auth.ip.mirror-a.allow 192.168.0.*,127.0.0.1
        option auth.ip.share.allow *
end-volume