[Gluster-devel] Re: client cannot maintain mount of unified AFR
Anand Avati
avati at zresearch.com
Thu Jun 28 13:10:28 UTC 2007
> This happened when the first client in the client config
> rebooted. Normally, the second client in the afr group would have picked up
> the slack, but instead I was getting connection refused from the client. I
> am assuming this is a locking issue?
You would get a 'connection refused' if the server is not running. Can you
please check if glusterfsd was running at that moment?
Also have you upgraded all the servers and clients?
thanks,
avati
Majied Najjar
>
> On Wed, 27 Jun 2007 12:01:35 -0400
> Majied Najjar <majied.najjar at nationalnet.com> wrote:
>
> > Hi,
> >
> > With the new release, I ran into a bit of a problem. I have six servers
> running a unified AFR mounted by a single client and after upgrading, I
> maintain the mount with the client. I get the following error when running
> "df" against the mountpoint:
> >
> > df: `/mnt/glusterfs': Invalid argument
> >
> > Here is an strace:
> >
> > execve("/usr/sbin/glusterfs", ["glusterfs", "-f",
> "/etc/glusterfs/glusterfs-client."..., "/mnt/glusterfs/"], [/* 15 vars */])
> = 0
> > uname({sys="Linux", node="nat1621", ...}) = 0
> > brk(0) = 0x8054000
> > access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or
> directory)
> > mmap2(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1,
> 0) = 0xb7fc1000
> > access("/etc/ld.so.preload", R_OK) = -1 ENOENT (No such file or
> directory)
> > open("/etc/ld.so.cache", O_RDONLY) = 3
> > fstat64(3, {st_mode=S_IFREG|0644, st_size=28812, ...}) = 0
> > mmap2(NULL, 28812, PROT_READ, MAP_PRIVATE, 3, 0) = 0xb7fb9000
> > close(3) = 0
> > access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or
> directory)
> > open("/usr/lib/libglusterfs.so.0", O_RDONLY) = 3
> > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\3601\0"...,
> 512) = 512
> > fstat64(3, {st_mode=S_IFREG|0644, st_size=58048, ...}) = 0
> > mmap2(NULL, 61216, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0)
> = 0xb7faa000
> > mmap2(0xb7fb8000, 4096, PROT_READ|PROT_WRITE,
> MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xd) = 0xb7fb8000
> > close(3) = 0
> > access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or
> directory)
> > open("/usr/lib/libibverbs.so.1", O_RDONLY) = 3
> > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0 \34\0\000"...,
> 512) = 512
> > fstat64(3, {st_mode=S_IFREG|0644, st_size=27696, ...}) = 0
> > mmap2(NULL, 30700, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0)
> = 0xb7fa2000
> > mmap2(0xb7fa9000, 4096, PROT_READ|PROT_WRITE,
> MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x6) = 0xb7fa9000
> > close(3) = 0
> > access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or
> directory)
> > open("/usr/lib/libfuse.so.2", O_RDONLY) = 3
> > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\00005\0\000"...,
> 512) = 512
> > fstat64(3, {st_mode=S_IFREG|0644, st_size=73704, ...}) = 0
> > mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1,
> 0) = 0xb7fa1000
> > mmap2(NULL, 76644, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0)
> = 0xb7f8e000
> > mmap2(0xb7fa0000, 4096, PROT_READ|PROT_WRITE,
> MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x11) = 0xb7fa0000
> > close(3) = 0
> > access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or
> directory)
> > open("/lib/tls/libdl.so.2", O_RDONLY) = 3
> > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\20\f\0"...,
> 512) = 512
> > fstat64(3, {st_mode=S_IFREG|0644, st_size=9592, ...}) = 0
> > mmap2(NULL, 12404, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0)
> = 0xb7f8a000
> > mmap2(0xb7f8c000, 8192, PROT_READ|PROT_WRITE,
> MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1) = 0xb7f8c000
> > close(3) = 0
> > access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or
> directory)
> > open("/lib/tls/libpthread.so.0", O_RDONLY) = 3
> > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\360G\0"...,
> 512) = 512
> > fstat64(3, {st_mode=S_IFREG|0755, st_size=85010, ...}) = 0
> > mmap2(NULL, 70104, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0)
> = 0xb7f78000
> > mmap2(0xb7f86000, 8192, PROT_READ|PROT_WRITE,
> MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xd) = 0xb7f86000
> > mmap2(0xb7f88000, 4568, PROT_READ|PROT_WRITE,
> MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xb7f88000
> > close(3) = 0
> > access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or
> directory)
> > open("/lib/tls/libc.so.6", O_RDONLY) = 3
> > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\240O\1"...,
> 512) = 512
> > fstat64(3, {st_mode=S_IFREG|0644, st_size=1241392, ...}) = 0
> > mmap2(NULL, 1251484, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3,
> 0) = 0xb7e46000
> > mmap2(0xb7f6e000, 28672, PROT_READ|PROT_WRITE,
> MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x127) = 0xb7f6e000
> > mmap2(0xb7f75000, 10396, PROT_READ|PROT_WRITE,
> MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xb7f75000
> > close(3) = 0
> > access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or
> directory)
> > open("/lib/libsysfs.so.2", O_RDONLY) = 3
> > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\0\37\0"...,
> 512) = 512
> > fstat64(3, {st_mode=S_IFREG|0644, st_size=37496, ...}) = 0
> > mmap2(NULL, 40508, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0)
> = 0xb7e3c000
> > mmap2(0xb7e45000, 4096, PROT_READ|PROT_WRITE,
> MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x8) = 0xb7e45000
> > close(3) = 0
> > access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or
> directory)
> > open("/lib/tls/librt.so.1", O_RDONLY) = 3
> > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0`\35\0\000"...,
> 512) = 512
> > fstat64(3, {st_mode=S_IFREG|0644, st_size=26516, ...}) = 0
> > mmap2(NULL, 29264, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0)
> = 0xb7e34000
> > mmap2(0xb7e3a000, 8192, PROT_READ|PROT_WRITE,
> MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x5) = 0xb7e3a000
> > close(3) = 0
> > mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1,
> 0) = 0xb7e33000
> > mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1,
> 0) = 0xb7e32000
> > mprotect(0xb7f6e000, 20480, PROT_READ) = 0
> > set_thread_area({entry_number:-1 -> 6, base_addr:0xb7e326c0,
> limit:1048575, seg_32bit:1, contents:0, read_exec_only:0, limit_in_pages:1,
> seg_not_present:0, useable:1}) = 0
> > munmap(0xb7fb9000, 28812) = 0
> > set_tid_address(0xb7e32708) = 31997
> > rt_sigaction(SIGRTMIN, {0xb7f7c450, [], SA_SIGINFO}, NULL, 8) = 0
> > rt_sigaction(SIGRT_1, {0xb7f7c3c0, [], SA_RESTART|SA_SIGINFO}, NULL, 8)
> = 0
> > rt_sigprocmask(SIG_UNBLOCK, [RTMIN RT_1], NULL, 8) = 0
> > getrlimit(RLIMIT_STACK, {rlim_cur=8192*1024, rlim_max=RLIM_INFINITY}) =
> 0
> > uname({sys="Linux", node="nat1621", ...}) = 0
> > setrlimit(RLIMIT_CORE, {rlim_cur=RLIM_INFINITY, rlim_max=RLIM_INFINITY})
> = 0
> > setrlimit(RLIMIT_NOFILE, {rlim_cur=RLIM_INFINITY,
> rlim_max=RLIM_INFINITY}) = -1 EPERM (Operation not permitted)
> > brk(0) = 0x8054000
> > brk(0x8075000) = 0x8075000
> > open("/var/log/glusterfs/glusterfs.log",
> O_WRONLY|O_APPEND|O_CREAT|O_LARGEFILE, 0666) = 3
> > fstat64(3, {st_mode=S_IFREG|0644, st_size=29947462, ...}) = 0
> > mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1,
> 0) = 0xb7fc0000
> > fstat64(3, {st_mode=S_IFREG|0644, st_size=29947462, ...}) = 0
> > _llseek(3, 29947462, [29947462], SEEK_SET) = 0
> > open("/etc/glusterfs/glusterfs-client.vol", O_RDONLY|O_LARGEFILE) = 4
> > rt_sigaction(SIGPIPE, {SIG_IGN}, {SIG_DFL}, 8) = 0
> > rt_sigaction(SIGSEGV, {0xb7fb36b0, [SEGV], SA_RESTART}, {SIG_DFL}, 8) =
> 0
> > rt_sigaction(SIGABRT, {0xb7fb36b0, [ABRT], SA_RESTART}, {SIG_DFL}, 8) =
> 0
> > socketpair(PF_FILE, SOCK_STREAM, 0, [5, 6]) = 0
> > clone(child_stack=0,
> flags=CLONE_CHILD_CLEARTID|CLONE_CHILD_SETTID|SIGCHLD,
> child_tidptr=0xb7e32708) = 31998
> > close(5) = 0
> > recvmsg(6, {msg_name(0)=NULL, msg_iov(1)=[{"\0", 1}], msg_controllen=16,
> {cmsg_len=16, cmsg_level=SOL_SOCKET, cmsg_type=SCM_RIGHTS, {5}},
> msg_flags=0}, 0) = 1
> > --- SIGCHLD (Child exited) @ 0 (0) ---
> > close(6) = 0
> > waitpid(31998, NULL, 0) = 31998
> > getuid32() = 0
> > rt_sigaction(SIGHUP, NULL, {SIG_DFL}, 8) = 0
> > rt_sigaction(SIGHUP, {0xb7f9ca80, [], 0}, NULL, 8) = 0
> > rt_sigaction(SIGINT, NULL, {SIG_DFL}, 8) = 0
> > rt_sigaction(SIGINT, {0xb7f9ca80, [], 0}, NULL, 8) = 0
> > rt_sigaction(SIGTERM, NULL, {SIG_DFL}, 8) = 0
> > rt_sigaction(SIGTERM, {0xb7f9ca80, [], 0}, NULL, 8) = 0
> > rt_sigaction(SIGPIPE, NULL, {SIG_IGN}, 8) = 0
> > epoll_create(1024) = 6
> > epoll_ctl(6, EPOLL_CTL_ADD, 5, {EPOLLIN|EPOLLPRI|EPOLLERR|EPOLLHUP,
> {u32=134562696, u64=134562696}}) = 0
> > clone(child_stack=0,
> flags=CLONE_CHILD_CLEARTID|CLONE_CHILD_SETTID|SIGCHLD,
> child_tidptr=0xb7e32708) = 31999
> > exit_group(0) = ?
> > Process 31997 detached
> >
> > ---------------------------------------
> >
> > and here is a copy of my client and server config:
> >
> > ----------------------------------------
> >
> > client.vol
> >
> > volume client1a
> > type protocol/client
> > option transport-type tcp/client
> > option remote-host 10.100.0.1
> > option remote-port 6996
> > option remote-subvolume iothreads1
> > end-volume
> >
> > volume client1b
> > type protocol/client
> > option transport-type tcp/client
> > option remote-host 10.100.0.1
> > option remote-port 6996
> > option remote-subvolume iothreads2
> > end-volume
> >
> > volume client1c
> > type protocol/client
> > option transport-type tcp/client
> > option remote-host 10.100.0.1
> > option remote-port 6996
> > option remote-subvolume iothreads3
> > end-volume
> >
> > volume client1d
> > type protocol/client
> > option transport-type tcp/client
> > option remote-host 10.100.0.1
> > option remote-port 6996
> > option remote-subvolume iothreads4
> > end-volume
> >
> > volume client2a
> > type protocol/client
> > option transport-type tcp/client
> > option remote-host 10.100.0.2
> > option remote-port 6996
> > option remote-subvolume iothreads1
> > end-volume
> >
> > volume client2b
> > type protocol/client
> > option transport-type tcp/client
> > option remote-host 10.100.0.2
> > option remote-port 6996
> > option remote-subvolume iothreads2
> > end-volume
> >
> > volume client2c
> > type protocol/client
> > option transport-type tcp/client
> > option remote-host 10.100.0.2
> > option remote-port 6996
> > option remote-subvolume iothreads3
> > end-volume
> >
> > volume client2d
> > type protocol/client
> > option transport-type tcp/client
> > option remote-host 10.100.0.2
> > option remote-port 6996
> > option remote-subvolume iothreads4
> > end-volume
> >
> > volume client3a
> > type protocol/client
> > option transport-type tcp/client
> > option remote-host 10.100.0.3
> > option remote-port 6996
> > option remote-subvolume iothreads1
> > end-volume
> >
> > volume client3b
> > type protocol/client
> > option transport-type tcp/client
> > option remote-host 10.100.0.3
> > option remote-port 6996
> > option remote-subvolume iothreads2
> > end-volume
> >
> > volume client3c
> > type protocol/client
> > option transport-type tcp/client
> > option remote-host 10.100.0.3
> > option remote-port 6996
> > option remote-subvolume iothreads3
> > end-volume
> >
> > volume client3d
> > type protocol/client
> > option transport-type tcp/client
> > option remote-host 10.100.0.3
> > option remote-port 6996
> > option remote-subvolume iothreads4
> > end-volume
> >
> > volume client4a
> > type protocol/client
> > option transport-type tcp/client
> > option remote-host 10.100.0.4
> > option remote-port 6996
> > option remote-subvolume iothreads1
> > end-volume
> >
> > volume client4b
> > type protocol/client
> > option transport-type tcp/client
> > option remote-host 10.100.0.4
> > option remote-port 6996
> > option remote-subvolume iothreads2
> > end-volume
> >
> > volume client4c
> > type protocol/client
> > option transport-type tcp/client
> > option remote-host 10.100.0.4
> > option remote-port 6996
> > option remote-subvolume iothreads3
> > end-volume
> >
> > volume client4d
> > type protocol/client
> > option transport-type tcp/client
> > option remote-host 10.100.0.4
> > option remote-port 6996
> > option remote-subvolume iothreads4
> > end-volume
> >
> > volume client5a
> > type protocol/client
> > option transport-type tcp/client
> > option remote-host 10.100.0.5
> > option remote-port 6996
> > option remote-subvolume iothreads1
> > end-volume
> >
> > volume client5b
> > type protocol/client
> > option transport-type tcp/client
> > option remote-host 10.100.0.5
> > option remote-port 6996
> > option remote-subvolume iothreads2
> > end-volume
> >
> > volume client5c
> > type protocol/client
> > option transport-type tcp/client
> > option remote-host 10.100.0.5
> > option remote-port 6996
> > option remote-subvolume iothreads3
> > end-volume
> >
> > volume client5d
> > type protocol/client
> > option transport-type tcp/client
> > option remote-host 10.100.0.5
> > option remote-port 6996
> > option remote-subvolume iothreads4
> > end-volume
> >
> > volume client6a
> > type protocol/client
> > option transport-type tcp/client
> > option remote-host 10.100.0.6
> > option remote-port 6996
> > option remote-subvolume iothreads1
> > end-volume
> >
> > volume client6b
> > type protocol/client
> > option transport-type tcp/client
> > option remote-host 10.100.0.6
> > option remote-port 6996
> > option remote-subvolume iothreads2
> > end-volume
> >
> > volume client6c
> > type protocol/client
> > option transport-type tcp/client
> > option remote-host 10.100.0.6
> > option remote-port 6996
> > option remote-subvolume iothreads3
> > end-volume
> >
> > volume client6d
> > type protocol/client
> > option transport-type tcp/client
> > option remote-host 10.100.0.6
> > option remote-port 6996
> > option remote-subvolume iothreads4
> > end-volume
> >
> > volume afr1
> > type cluster/afr
> > subvolumes client1a client2a
> > option replicate *:2
> > end-volume
> >
> > volume afr2
> > type cluster/afr
> > subvolumes client1b client2b
> > option replicate *:2
> > end-volume
> >
> > volume afr3
> > type cluster/afr
> > subvolumes client1c client2c
> > option replicate *:2
> > end-volume
> >
> > volume afr4
> > type cluster/afr
> > subvolumes client1d client2d
> > option replicate *:2
> > end-volume
> >
> > volume afr5
> > type cluster/afr
> > subvolumes client3a client4a
> > option replicate *:2
> > end-volume
> >
> > volume afr6
> > type cluster/afr
> > subvolumes client3b client4b
> > option replicate *:2
> > end-volume
> >
> > volume afr7
> > type cluster/afr
> > subvolumes client3c client4c
> > option replicate *:2
> > end-volume
> >
> > volume afr8
> > type cluster/afr
> > subvolumes client3d client4d
> > option replicate *:2
> > end-volume
> >
> > volume afr9
> > type cluster/afr
> > subvolumes client5a client6a
> > option replicate *:2
> > end-volume
> >
> > volume afr10
> > type cluster/afr
> > subvolumes client5b client6b
> > option replicate *:2
> > end-volume
> >
> > volume afr11
> > type cluster/afr
> > subvolumes client5c client6c
> > option replicate *:2
> > end-volume
> >
> > volume afr12
> > type cluster/afr
> > subvolumes client5d client6d
> > option replicate *:2
> > end-volume
> >
> > volume unify
> > type cluster/unify
> > subvolumes afr1 afr2 afr3 afr4 afr5 afr6 afr7 afr8 afr9 afr10 afr11
> afr12
> > option scheduler alu
> > option alu.limits.min-free-disk 60G
> > option alu.limits.max-open-files 10000
> > option alu.orderdisk-usage:read-usage:write-usage:open-files-usage:disk-speed-usage
> > option alu.disk-usage.entry-threshold 2GB
> > option alu.disk-usage.exit-threshold 64MB
> > option alu.open-files-usage.entry-threshold 1024
> > option alu.open-files-usage.exit-threshold 32
> > option alu.read-usage.entry-threshold 20%
> > option alu.read-usage.exit-threshold 4%
> > option alu.write-usage.entry-threshold 20%
> > option alu.write-usage.exit-threshold 4%
> > option alu.stat-refresh.interval 10sec
> > option alu.stat-refresh.num-file-create 10
> > option readdir-force-success on
> > end-volume
> >
> > #volume writebehind
> > # type performance/write-behind
> > # option aggregate-size 131072
> > # subvolumes unify
> > #end-volume
> >
> > volume readahead
> > type performance/read-ahead
> > option page-size 131072
> > option page-count 16
> > subvolumes unify
> > end-volume
> >
> > volume statprefetch
> > type performance/stat-prefetch
> > option cache-seconds 2
> > subvolumes readahead
> > end-volume
> >
> > ---------------------
> >
> > server.vol
> >
> > volume volume1
> > type storage/posix
> > option directory /storage/vol1
> > end-volume
> >
> > volume readahead1
> > type performance/read-ahead
> > option page-size 131072
> > option page-count 16
> > subvolumes volume1
> > end-volume
> >
> > volume iothreads1
> > type performance/io-threads
> > option thread-count 8
> > subvolumes readahead1
> > end-volume
> >
> > volume volume2
> > type storage/posix
> > option directory /storage/vol2
> > end-volume
> >
> > volume readahead2
> > type performance/read-ahead
> > option page-size 131072
> > option page-count 16
> > subvolumes volume2
> > end-volume
> >
> > volume iothreads2
> > type performance/io-threads
> > option thread-count 8
> > subvolumes readahead2
> > end-volume
> >
> > volume volume3
> > type storage/posix
> > option directory /storage/vol3
> > end-volume
> >
> > volume readahead3
> > type performance/read-ahead
> > option page-size 131072
> > option page-count 16
> > subvolumes volume3
> > end-volume
> >
> > volume iothreads3
> > type performance/io-threads
> > option thread-count 8
> > subvolumes readahead3
> > end-volume
> >
> > volume volume4
> > type storage/posix
> > option directory /storage/vol4
> > end-volume
> >
> > volume readahead4
> > type performance/read-ahead
> > option page-size 131072
> > option page-count 16
> > subvolumes volume4
> > end-volume
> >
> > volume iothreads4
> > type performance/io-threads
> > option thread-count 8
> > subvolumes readahead4
> > end-volume
> >
> > volume server
> > type protocol/server
> > option transport-type tcp/server
> > subvolumes iothreads1 iothreads2 iothreads3 iothreads4
> > option auth.ip.iothreads1.allow 10.100.0.*
> > option auth.ip.iothreads2.allow 10.100.0.*
> > option auth.ip.iothreads3.allow 10.100.0.*
> > option auth.ip.iothreads4.allow 10.100.0.*
> > end-volume
> > -------------------------------------
> >
> > Let me know if you need more information.
> >
> > Thanks,
> > Majied Najjar
>
>
> _______________________________________________
> Gluster-devel mailing list
> Gluster-devel at nongnu.org
> http://lists.nongnu.org/mailman/listinfo/gluster-devel
>
--
Anand V. Avati
More information about the Gluster-devel
mailing list