[Gluster-devel] lots of errors from latest TLA on HA

Mickey Mazarick mic at digitaltadpole.com
Mon Jan 12 18:45:22 UTC 2009


They are the same transports. (this is just an automated test and it 
worked in ver ~800) It's getting confused in later versions and sending 
tcp requests to the ibverbs daemon even though I've specified the port 
explicitly in both the client and server. This is only for HA, it works 
fine if I just use ibverbs or tcp.


<SERVER>
volume br1
 type storage/posix
 option directory /mnt/gluster/system
end-volume

volume locks
  type features/posix-locks
  subvolumes br1
end-volume

volume brick
  type performance/io-threads
  option thread-count 2  # deault is 1
  option cache-size 128MB
  subvolumes locks
end-volume

volume brick-ns1
 type storage/posix
 option directory /mnt/gluster/system-ns
end-volume

volume brick-ns
  type features/posix-locks
  subvolumes brick-ns1
end-volume

volume main1s
 type storage/posix
 option directory /mnt/gluster/main1/export1
end-volume

volume main1
  type features/posix-locks
  option mandatory on
  subvolumes main1s
end-volume

volume main2s
 type storage/posix
 option directory /mnt/gluster/main2
end-volume

volume main2
  type features/posix-locks
  option mandatory on
  subvolumes main2s
end-volume


volume server
 type protocol/server
 option transport-type ib-verbs
 subvolumes brick brick-ns main2 main1
 option listen-port 6997
 option auth.addr.main1.allow *
 option auth.addr.main2.allow *
 option auth.addr.brick.allow *
 option auth.addr.brick-ns.allow *
 #option auth.ip.brick.allow *
 #option auth.ip.brick-ns.allow *
 #option auth.ip.main1.allow *
 #option auth.ip.main2.allow *
end-volume



<CLIENT>
####TCP######
volume brick1tcp
 type protocol/client
 option transport-type tcp     # for TCP/IP transport
 option remote-host RTPST201      # IP address of the remote brick
 option remote-subvolume brick        # name of the remote volume
end-volume

volume brick2tcp
 type protocol/client
 option transport-type tcp
 option remote-host RTPST202
 option remote-subvolume brick
end-volume

volume brick3tcp
 type protocol/client
 option transport-type tcp
 option remote-host RTPST203
 option remote-subvolume brick
end-volume

volume brick4tcp
 type protocol/client
 option transport-type tcp
 option remote-host RTPST204
 option remote-subvolume brick
end-volume

volume brick5tcp
 type protocol/client
 option transport-type tcp
 option remote-host RTPST205
 option remote-subvolume brick
end-volume

volume brick6tcp
 type protocol/client
 option transport-type tcp
 option remote-host RTPST206
 option remote-subvolume brick
end-volume

volume brick-ns1tcp
 type protocol/client
 option transport-type tcp
 option remote-host RTPST201
 option remote-subvolume brick-ns  # Note the different remote volume name.
end-volume

volume brick-ns2tcp
 type protocol/client
 option transport-type tcp
 option remote-host RTPST202
 option remote-subvolume brick-ns  # Note the different remote volume name.
end-volume

###IBVERBS####

volume brick1ib
 type protocol/client
 option transport-type ib-verbs
 option transport-timeout 2
 option remote-port 6997
 option remote-host RTPST201
 option remote-subvolume brick
end-volume

volume brick2ib
 type protocol/client
 option transport-type ib-verbs
 option transport-timeout 2
 option remote-port 6997
 option remote-host RTPST202
 option remote-subvolume brick
end-volume

volume brick3ib
 type protocol/client
 option transport-type ib-verbs
 option transport-timeout 2
 option remote-port 6997
 option remote-host RTPST203
 option remote-subvolume brick
end-volume

volume brick4ib
 type protocol/client
 option transport-type ib-verbs
 option transport-timeout 2
 option remote-port 6997
 option remote-host RTPST204
 option remote-subvolume brick
end-volume

volume brick5ib
 type protocol/client
 option transport-type ib-verbs
 option transport-timeout 2
 option remote-port 6997
 option remote-host RTPST205
 option remote-subvolume brick
end-volume

volume brick6ib
 type protocol/client
 option transport-type ib-verbs
 option transport-timeout 2
 option remote-port 6997
 option remote-host RTPST206
 option remote-subvolume brick
end-volume

volume brick-ns1ib
 type protocol/client
 option transport-type ib-verbs
 option transport-timeout 2
 option remote-port 6997
 option remote-host RTPST201
 option remote-subvolume brick-ns  # Note the different remote volume name.
end-volume

volume brick-ns2ib
 type protocol/client
 option transport-type ib-verbs
 option transport-timeout 2
 option remote-port 6997
 option remote-host RTPST202
 option remote-subvolume brick-ns  # Note the different remote volume name.
end-volume


###HA Section
volume brick1
  type cluster/ha
  subvolumes brick1ib brick1tcp
end-volume

volume brick2
  type cluster/ha
  subvolumes brick2ib brick2tcp
end-volume

volume brick3
  type cluster/ha
  subvolumes brick3ib brick3tcp
end-volume

volume brick4
  type cluster/ha
  subvolumes brick4ib brick4tcp
end-volume

volume brick5
  type cluster/ha
  subvolumes brick5ib brick5tcp
end-volume

volume brick6
  type cluster/ha
  subvolumes brick6ib brick6tcp
end-volume

volume brick-ns1
  type cluster/ha
  subvolumes brick-ns1ib brick-ns1tcp
end-volume

volume brick-ns2
  type cluster/ha
  subvolumes brick-ns2ib brick-ns2tcp
end-volume


###AFR SECTION

volume afr1
 type cluster/afr
 subvolumes brick1 brick4
end-volume

volume afr2
 type cluster/afr
 subvolumes brick2 brick5
end-volume

volume afr3
 type cluster/afr
 subvolumes brick3 brick6
end-volume

volume afr-ns
 type cluster/afr
 subvolumes brick-ns1 brick-ns2
end-volume

volume unify
 type cluster/unify
 option namespace afr-ns
 option scheduler rr
 subvolumes afr1 afr2 afr3
end-volume

###PERFORMANCE SECTION

volume io-cache
  type performance/io-cache
  subvolumes unify
end-volume

volume readahead
  type performance/read-ahead
  option force-atime-update on # default is off
  subvolumes io-cache
end-volume

volume writebehind
  type performance/write-behind
  option aggregate-size 1MB # default is 0bytes
  #option flush-behind on    # default is 'off'
  subvolumes readahead
end-volume


Raghavendra G wrote:
> Hi Mickey,
>
> what are the transeports of client and serverts ? you seem to be using 
> different transports on client and server side? If so, can you use the 
> same transport on both sides?
>
> regards,
> On Mon, Jan 12, 2009 at 7:42 PM, Mickey Mazarick 
> <mic at digitaltadpole.com <mailto:mic at digitaltadpole.com>> wrote:
>
>     I just updated our test cluster to the newest tla and it works
>     fine but throws a lot of errors on the HA/AFR/unify portion over
>     ib-verbs/tcp. The errors only happen when HA is enabled (HA is
>     setup ibverbs<->tcp )
>
>     The client errors:
>     2009-01-12 10:34:44 E [saved-frames.c:148:saved_frames_unwind]
>     brick6tcp: forced unwinding frame type(2) op(SETVOLUME)
>     2009-01-12 10:34:44 E
>     [client-protocol.c:5583:client_setvolume_cbk] brick6tcp: setvolume
>     failed (Transport endpoint is not connected)
>     2009-01-12 10:34:54 E
>     [socket.c:473:__socket_proto_validate_header] brick-ns1tcp: socket
>     header signature does not match :O (51.50.31)
>     2009-01-12 10:34:54 E [socket.c:588:socket_proto_state_machine]
>     brick-ns1tcp: socket header validation failed (192.168.1.201:6997
>     <http://192.168.1.201:6997>). possible mis-match of
>     'transport-type' in server and client volumes, or different
>     versions of glusterfs used
>     2009-01-12 10:34:54 E [saved-frames.c:148:saved_frames_unwind]
>     brick-ns1tcp: forced unwinding frame type(2) op(SETVOLUME)
>     2009-01-12 10:34:54 E
>     [client-protocol.c:5583:client_setvolume_cbk] brick-ns1tcp:
>     setvolume failed (Transport endpoint is not connected)
>     2009-01-12 10:34:54 E
>     [socket.c:473:__socket_proto_validate_header] brick-ns2tcp: socket
>     header signature does not match :O (51.50.31)
>     2009-01-12 10:34:54 E [socket.c:588:socket_proto_state_machine]
>     brick-ns2tcp: socket header validation failed (192.168.1.202:6997
>     <http://192.168.1.202:6997>). possibl
>
>     The server errors:
>     2009-01-12 10:38:16 C [ib-verbs.c:1671:ib_verbs_handshake_pollin]
>     transport/ib-verbs: server: remote-host(192.168.1.74:993)'s
>     transport type is different
>     2009-01-12 10:38:16 E [ib-verbs.c:213:__ib_verbs_disconnect]
>     transport/ib-verbs: shutdown () - error: Transport endpoint is not
>     connected
>     2009-01-12 10:38:16 E
>     [server-protocol.c:7354:server_protocol_cleanup] server: invalid
>     argument: cprivate
>     2009-01-12 10:38:18 C [ib-verbs.c:1671:ib_verbs_handshake_pollin]
>     transport/ib-verbs: server: remote-host(127.0.0.1:1023)'s
>     transport type is different
>     2009-01-12 10:38:18 E [ib-verbs.c:213:__ib_verbs_disconnect]
>     transport/ib-verbs: shutdown () - error: Transport endpoint is not
>     connected
>     2009-01-12 10:38:18 E [ib-verbs.c:1940:tcp_connect_finish] server:
>     tcp connect to 127.0.0.1:1023 <http://127.0.0.1:1023> failed
>     (Connection reset by peer)
>     2009-01-12 10:38:18 E
>     [server-protocol.c:7354:server_protocol_cleanup] server: invalid
>     argument: cprivate
>     2009-01-12 10:38:18 C [ib-verbs.c:1671:ib_verbs_handshake_pollin]
>     transport/ib-verbs: server: remote-host(127.0.0.1:1023)'s
>     transport type is different
>     2009-01-12 10:38:18 C [ib-verbs.c:1671:ib_verbs_handshake_pollin]
>     transport/ib-verbs: server: remote-host(127.0.0.1:1023)'s
>     transport type is different
>     2009-01-12 10:38:18 E [ib-verbs.c:213:__ib_verbs_disconnect]
>     transport/ib-verbs: shutdown () - error: Transport endpoint is not
>     connected
>
>     Thanks for all the great work!
>     -- 
>
>
>     _______________________________________________
>     Gluster-devel mailing list
>     Gluster-devel at nongnu.org <mailto:Gluster-devel at nongnu.org>
>     http://lists.nongnu.org/mailman/listinfo/gluster-devel
>
>
>
>
> -- 
> Raghavendra G
>


-- 





More information about the Gluster-devel mailing list