[Gluster-users] 2.0.0rc4 performance/write-behind seems broken with flush-behind=on

[Kirby Zhou]

2.0.0rc4 performance/write-behind seems broken with flush-behind=on

If you use flush-behind=on, it is a acceptable result that other client
machines will not see your operations immediately.
But it is not acceptable that the sequentially executed program cannot see
the result immediately caused by the prior one on the same client machine.

The later situation happens when I run rpmbuild over a glusterfs volume.

If I run following commands with flush-behind = on, some error happens. 
############
[root at xen-727057 BUILD]# ( set -x ; rm -fr openssl-fips-0.9.8e;
/usr/bin/bzip2 -dc /root/rpmbuild/SOURCES/openssl-fips-0.9.8e-usa.tar.bz2 |
tar -xf -; cd openssl-fips-0.9.8e; /root/rpmbuild/SOURCES/hobble-openssl >
/dev/null; patch -p1 -b --suffix .redhat <
/root/rpmbuild/SOURCES/openssl-fips-0.9.8e-redhat.patch; patch -p1 -b
--suffix .defaults < /root/rpmbuild/SOURCES/openssl-0.9.8a-defaults.patch;
patch -p1 -b --suffix .krb5 <
/root/rpmbuild/SOURCES/openssl-0.9.8a-link-krb5.patch; patch -p1 -b --suffix
.soversion < /root/rpmbuild/SOURCES/openssl-0.9.8b-soversion.patch; )
+ rm -i -fr openssl-fips-0.9.8e
+ /usr/bin/bzip2 -dc /root/rpmbuild/SOURCES/openssl-fips-0.9.8e-usa.tar.bz2
+ tar -xf -
+ cd openssl-fips-0.9.8e
+ /root/rpmbuild/SOURCES/hobble-openssl
find: crypto/idea/asm: No such file or directory
find: crypto/mdc2/asm: No such file or directory
+ patch -p1 -b --suffix .redhat
patching file Configure
+ patch -p1 -b --suffix .defaults
patching file apps/openssl.cnf
Hunk #1 succeeded at 107 (offset 8 lines).
Hunk #3 succeeded at 153 (offset 8 lines).
+ patch -p1 -b --suffix .krb5
patching file Makefile.org
Hunk #1 succeeded at 420 (offset 154 lines).
+ patch -p1 -b --suffix .soversion
patching file Makefile.org
Hunk #2 FAILED at 278.
Hunk #3 FAILED at 291.
2 out of 3 hunks FAILED -- saving rejects to file Makefile.org.rej
patching file Configure
Hunk #1 FAILED at 1327.
1 out of 1 hunk FAILED -- saving rejects to file Configure.rej
############
You can see, the last 2 patch operates the same file "Makefile.org".
############ 
[root at xen-727057 BUILD]# lsdiff
/root/rpmbuild/SOURCES/openssl-0.9.8a-link-krb5.patch
openssl-0.9.8a/Makefile.org
[root at xen-727057 BUILD]# lsdiff
/root/rpmbuild/SOURCES/openssl-0.9.8b-soversion.patch
openssl-0.9.8b/Makefile.org
openssl-0.9.8b/Configure
############
If I run each of the command above manually, everything goes right.
If the flush-behind is turned off, everything goes right too. 

#### client vol spec file
volume 10.10.123.32-brick1
  type protocol/client
  option transport-type tcp/client
  option remote-host 10.10.123.32
  option remote-port 6986
  option transport-timeout 5
  option remote-subvolume brick1
end-volume
volume 10.10.123.32-brick2
....
end-volume

volume dht0-raw
  type cluster/distribute
  subvolumes 10.10.123.21-brick1 10.10.123.21-brick2 10.10.123.22-brick1
10.10.123.22-brick2 10.10.123.31-brick1 10.10.123.31-brick2
10.10.123.32-brick1 10.10.123.32-brick2
  option lookup-unhashed yes
end-volume

volume dht0-io-cache
  type performance/io-cache
  option cache-size 128MB            # default is 32MB
  option page-size 128KB             # 128KB is default option
# option priority *.h:3,*.html:2,*:1 # default is '*:0'
# option cache-timeout 2             # default is 1 second
  subvolumes dht0-raw
end-volume

volume dht0-writebehind
  type performance/write-behind
  option aggregate-size 128KB        # default is 0bytes
  option window-size 2MB             # default is equal to aggregate-size
  option flush-behind on             # default is 'off'
  subvolumes dht0-io-cache
end-volume

volume dht0
  type features/filter
  option read-only off
  #option root-squashing enable
  option fixed-uid 99 
  option fixed-gid 99
  #option translate-uid 0=999
  #option translate-gid 0=999
  subvolumes dht0-writebehind
  #subvolumes dht0-io-cache
end-volume 
############################