[Gluster-devel] Files being double created on bricks

[Brian Taber]

I am using glusterfs 1.3.9 (glusterfs--mainline--2.5--patch-770)  I am
using Dovecot for email retrieval.  Dovecot creates files called
dovecot-uidlist which is a listing of all messages int he mailbox. 
somehow, the file is getting created on more than one brick, which
causes gluster to go nuts and deny access to the file.  Here is the
errors in the client log:

2008-05-30 11:31:43 E [unify.c:873:unify_open] brick:
/domain.com/usermailbox/dovecot-uidlist: entry_count is 3
2008-05-30 11:31:43 E [unify.c:876:unify_open] brick:
/domain.com/usermailbox/dovecot-uidlist: found on mail
2008-05-30 11:31:43 E [unify.c:876:unify_open] brick:
/domain.com/usermailbox/dovecot-uidlist: found on mail2
2008-05-30 11:31:43 E [unify.c:876:unify_open] brick:
/domain.com/usermailbox/dovecot-uidlist: found on ns-mail
2008-05-30 11:31:43 E [fuse-bridge.c:692:fuse_fd_cbk] glusterfs-fuse:
1530174: (12) /domain.com/usermailbox/dovecot-uidlist => -1 (5)

On the server there is stranger errors:
2008-05-29 11:57:27 E [posix.c:1984:posix_setdents] ns-mail: Error
creating file
/data2/gluster-index/mail/domain.com/usermailbox/dovecot-uidlist with
mode (0100600)
2008-05-29 13:30:23 E [posix.c:1984:posix_setdents] ns-mail: Error
creating file
/data2/gluster-index/mail/domain.com/usermailbox/dovecot-uidlist with
mode (0100600)
2008-05-29 13:30:23 E [posix.c:1984:posix_setdents] ns-mail: Error
creating file
/data2/gluster-index/mail/domain.com/usermailbox/courierpop3dsizelist
with mode (0100644)
2008-05-29 13:30:23 E [posix.c:1984:posix_setdents] ns-mail: Error
creating file
/data2/gluster-index/mail/domain.com/usermailbox/maildirsize with mode
(0100644)
2008-05-29 13:30:23 E [posix.c:1984:posix_setdents] ns-mail: Error
creating file
/data2/gluster-index/mail/domain.com/usermailbox/dovecot-keywords with
mode (0100600)
2008-05-29 13:30:23 E [posix.c:1984:posix_setdents] ns-mail: Error
creating file
/data2/gluster-index/mail/domain.com/usermailbox/dovecot.index.cache
with mode (0100600)



How can this file be created on more than one brick? 
What are all these errors I cam getting on the server?  The files the
server is yelling about exist.
Am I doing something wrong?



My configs looks like this:

# namespace
volume ns-mail
  type protocol/client
  option transport-type tcp/client
  option remote-host 192.168.200.200
  option remote-port 7010
  option remote-subvolume ns-mail
end-volume

# first storage vol
volume mail
  type protocol/client
  option transport-type tcp/client
  option remote-host 192.168.200.200
  option remote-port 7011
  option remote-subvolume io-threads-mail
end-volume

# second storage vol
volume mail2
  type protocol/client
  option transport-type tcp/client
  option remote-host 192.168.200.200
  option remote-port 7012
  option remote-subvolume io-threads-mail2
end-volume

volume brick
  type cluster/unify
  subvolumes mail mail2
  option namespace ns-mail
  option scheduler alu
  option alu.limits.min-free-disk  5%      # Don't create files one a
volume with less than 5% free diskspace
  option alu.limits.max-open-files 10000   # Don't create files on a
volume with more than 10000 files open

  option alu.order
disk-usage:read-usage:write-usage:open-files-usage:disk-speed-usage
  option alu.disk-usage.entry-threshold 2GB   # Kick in if the
discrepancy in disk-usage between volumes is more than 2GB
  option alu.disk-usage.exit-threshold  60MB   # Don't stop writing to
the least-used volume until the discrepancy is 1988MB
  option alu.open-files-usage.entry-threshold 1024   # Kick in if the
discrepancy in open files is 1024
  option alu.open-files-usage.exit-threshold 32   # Don't stop until 992
files have been written the least-used volume
  option alu.stat-refresh.interval 10sec   # Refresh the statistics used
for decision-making every 10 seconds
end-volume


my 3 server configs are:
1:
volume ns-mail
  type storage/posix
  option directory /data2/gluster-index/mail
end-volume

volume server
  type protocol/server
  subvolumes ns-mail
  option transport-type tcp/server     # For TCP/IP transport
  option listen-port 7010
  option auth.ip.ns-mail.allow 192.168.*
end-volume


2:
volume mail
  type storage/posix
  option directory /data/mail
end-volume

volume posix-locks-mail
  type features/posix-locks
  option mandatory on
  subvolumes mail
end-volume

volume io-threads-mail
  type performance/io-threads
  option thread-count 8
  option cache-size 4096MB
  subvolumes posix-locks-mail
end-volume

volume server
  type protocol/server
  subvolumes posix-locks-mail
  option transport-type tcp/server     # For TCP/IP transport
  option listen-port 7011
  option auth.ip.io-threads-mail.allow 192.168.*
  option auth.ip.posix-locks-mail.allow 192.168.*
end-volume

3:
volume mail2
  type storage/posix
  option directory /data3/mail
end-volume

volume posix-locks-mail2
  type features/posix-locks
  option mandatory on
  subvolumes mail2
end-volume

volume io-threads-mail2
  type performance/io-threads
  option thread-count 8
  option cache-size 4096MB
  subvolumes posix-locks-mail2
end-volume

volume server
  type protocol/server
  subvolumes posix-locks-mail2
  option transport-type tcp/server     # For TCP/IP transport
  option listen-port 7012
  option auth.ip.io-threads-mail2.allow 192.168.*
  option auth.ip.posix-locks-mail2.allow 192.168.*
end-volume


-- 

-------------------------
Brian Taber
Owner/IT Specialist
Diverse Computer Group
Office: 774-206-5592
Cell: 508-496-9221
btaber at diversecg.com