[Gluster-users] Problem with folders not being created on newly added disks

Roland Rabben roland at jotta.no
Wed Sep 9 09:36:24 UTC 2009


Hi
I am using GlusterFS 2.02 on Ubuntu 9.04 64 bit. I have 4 data-nodes and 3
clients. Se my vol files at the end of this email.

After adding more disks to my data-nodes for more capacity and reconfiguring
GlusterFS to include those drives I am experiencing problems.

I am getting "No such file or directory" if I try to copy a new file into an
existing directory.
Client says: E [unify.c:1554:unify_create_cbk] dfs: create failed on
repl-002-003-04 (file /876/jonesivert/784/bjarne, error No such file or
directory), sending unlink to namespace

However, if I copy a new file into a new directory everyting works fine.

It seems that if I create the folderstructure from the old data-disks on the
new disks, everything works fine.

So my questions are?

1. Am I doing somthing wrong in the upgrade process?
2. Do I need to manually create the existing folders on the new hard drives?
3. Self heal does not fix this. Shouldn't it?
4. Is there a tool that will create the folderstructure on the new disks for
me?

How can I fix?


Client vol file example:
=================
# DN-000
volume dn-000-01
        type protocol/client
        option transport-type tcp
        option remote-host dn-000
        option remote-subvolume brick-01
end-volume

volume dn-000-02
        type protocol/client
        option transport-type tcp
        option remote-host dn-000
        option remote-subvolume brick-02
end-volume

volume dn-000-03
        type protocol/client
        option transport-type tcp
        option remote-host dn-000
        option remote-subvolume brick-03
end-volume

volume dn-000-04
        type protocol/client
        option transport-type tcp
        option remote-host dn-000
        option remote-subvolume brick-04
end-volume


volume dn-000-ns
        type protocol/client
        option transport-type tcp
        option remote-host dn-000
        option remote-subvolume brick-ns
end-volume

# DN-001
volume dn-001-01
        type protocol/client
        option transport-type tcp
        option remote-host dn-001
        option remote-subvolume brick-01
end-volume

volume dn-001-02
        type protocol/client
        option transport-type tcp
        option remote-host dn-001
        option remote-subvolume brick-02
end-volume

volume dn-001-03
        type protocol/client
        option transport-type tcp
        option remote-host dn-001
        option remote-subvolume brick-03
end-volume

volume dn-001-04
        type protocol/client
        option transport-type tcp
        option remote-host dn-001
        option remote-subvolume brick-04
end-volume

volume dn-001-ns
        type protocol/client
        option transport-type tcp
        option remote-host dn-001
        option remote-subvolume brick-ns
end-volume

# DN-002
volume dn-002-01
        type protocol/client
        option transport-type tcp
        option remote-host dn-002
        option remote-subvolume brick-01
end-volume

volume dn-002-02
        type protocol/client
        option transport-type tcp
        option remote-host dn-002
        option remote-subvolume brick-02
end-volume

volume dn-002-03
        type protocol/client
        option transport-type tcp
        option remote-host dn-002
        option remote-subvolume brick-03
end-volume

volume dn-002-04
        type protocol/client
        option transport-type tcp
        option remote-host dn-002
        option remote-subvolume brick-04
end-volume

# DN-003
volume dn-003-01
        type protocol/client
        option transport-type tcp
        option remote-host dn-003
        option remote-subvolume brick-01
end-volume

volume dn-003-02
        type protocol/client
        option transport-type tcp
        option remote-host dn-003
        option remote-subvolume brick-02
end-volume

volume dn-003-03
        type protocol/client
        option transport-type tcp
        option remote-host dn-003
        option remote-subvolume brick-03
end-volume

volume dn-003-04
        type protocol/client
        option transport-type tcp
        option remote-host dn-003
        option remote-subvolume brick-04
end-volume

# Replicate data between the servers
# Use pairs, but swtich the order to distribute read load
volume repl-000-001-01
        type cluster/replicate
        subvolumes dn-000-01 dn-001-01
end-volume

volume repl-000-001-02
        type cluster/replicate
        subvolumes dn-001-02 dn-000-02
end-volume

volume repl-000-001-03
        type cluster/replicate
        subvolumes dn-000-03 dn-001-03
end-volume

volume repl-000-001-04
        type cluster/replicate
        subvolumes dn-001-04 dn-000-04
end-volume


volume repl-002-003-01
        type cluster/replicate
        subvolumes dn-002-01 dn-003-01
end-volume

volume repl-002-003-02
        type cluster/replicate
        subvolumes dn-003-02 dn-002-02
end-volume

volume repl-002-003-03
        type cluster/replicate
        subvolumes dn-002-03 dn-003-03
end-volume

volume repl-002-003-04
        type cluster/replicate
        subvolumes dn-003-04 dn-002-04
end-volume


# Also replicate the namespace
volume repl-ns
        type cluster/replicate
        subvolumes dn-000-ns dn-001-ns
end-volume

# Distribute the data using the "adaptive least usage" scheduler
# We have a 5GB treshold for disk-usage first, then we look at write-usage,
and finanly read-usage
volume dfs
        type cluster/unify
        option namespace repl-ns
        option scheduler alu
        option scheduler.limits.min-free-disk 5%
        option scheduler.alu.order disk-usage:write-usage:read-usage
        option scheduler.alu.disk-usage.entry-threshold 5GB
        option scheduler.alu.disk-usage.exit-threshold 1GB
        option scheduler.alu.write-usage.entry-threshold 25
        option scheduler.alu.write-usage.exit-threshold 5
        option scheduler.alu.read-usage.entry-threshold 25
        option scheduler.alu.read-usage.exit-threshold 5
        subvolumes repl-000-001-01 repl-000-001-02 repl-000-001-03
repl-000-001-04 repl-002-003-01 repl-002-003-02 repl-002-003-03
repl-002-003-04

end-volume

# Enable write-behind to decrease write latency
volume wb
        type performance/write-behind
        option flush-behind off
        option cache-size 128MB
        subvolumes dfs
end-volume

volume cache
        type performance/io-cache
        option cache-size 1024MB
        subvolumes wb
end-volume





Server vol file example:
==================
# The posix volumes
volume posix-01
    type storage/posix
    option directory /mnt/data01
end-volume

volume posix-02
    type storage/posix
    option directory /mnt/data02
end-volume

volume posix-03
        type storage/posix
        option directory /mnt/data03
end-volume

volume posix-04
        type storage/posix
        option directory /mnt/data04
end-volume


volume posix-ns
    type storage/posix
    option directory /var/lib/glusterfs/ns
end-volume

# Add locking capabilities
volume locks-01
    type features/locks
    subvolumes posix-01
end-volume

volume locks-02
    type features/locks
    subvolumes posix-02
end-volume

volume locks-03
        type features/locks
        subvolumes posix-03
end-volume

volume locks-04
        type features/locks
        subvolumes posix-04
end-volume


volume locks-ns
    type features/locks
    subvolumes posix-ns
end-volume

# Finally add threads to the briks
volume brick-01
    type performance/io-threads
    option thread-count 8
    subvolumes locks-01
end-volume

volume brick-02
    type performance/io-threads
    option thread-count 8
    subvolumes locks-02
end-volume

volume brick-03
        type performance/io-threads
        option thread-count 8
        subvolumes locks-03
end-volume

volume brick-04
        type performance/io-threads
        option thread-count 8
        subvolumes locks-04
end-volume


volume brick-ns
    type performance/io-threads
    option thread-count 8
    subvolumes locks-ns
end-volume

# Mount the posix drives as a network drive
volume server
    type protocol/server
    option transport-type tcp
    subvolumes brick-01 brick-02 brick-03 brick-04 brick-ns
    option auth.addr.brick-01.allow 10.0.*
    option auth.addr.brick-02.allow 10.0.*
    option auth.addr.brick-03.allow 10.0.*
    option auth.addr.brick-04.allow 10.0.*
    option auth.addr.brick-ns.allow 10.0.*
end-volume



Regards

Roland Rabben
Founder & CEO Jotta AS
Cell: +47 90 85 85 39
Phone: +47 21 04 29 00
Email: roland at jotta.no


More information about the Gluster-users mailing list