[Gluster-users] mv lost some files ?

Wed Sep 5 08:45:53 UTC 2018

On Wed, Sep 5, 2018 at 12:46 PM, yu sun <sunyu1949 at gmail.com> wrote:

>
> [root at ml-ctl-ser01 sunyuyusun]$ salt "ml-storage-ser2*.nmg01" cmd.run
> "getfattr -d -m. -e hex /data2/bricks/project2/371_37829/face_landmarks"
> ml-storage-ser24.nmg01:
>     getfattr: Removing leading '/' from absolute path names
>     # file: data2/bricks/project2/371_37829/face_landmarks
>     trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429
>     trusted.glusterfs.dht=0xdbde0bb100000000b0c30c2eb3cf3cf0
>     trusted.glusterfs.quota.dirty=0x3000
>     trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=
> 0x00000000000d760000000000000000010000000000000003
>     trusted.glusterfs.quota.size.1=0x00000000000d7600000000000000
> 00010000000000000003
> ml-storage-ser26.nmg01:
>     getfattr: Removing leading '/' from absolute path names
>     # file: data2/bricks/project2/371_37829/face_landmarks
>     trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429
>     trusted.glusterfs.dht=0xdbde0bb100000000d249248fd5555551
>     trusted.glusterfs.quota.dirty=0x3000
>     trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=
> 0x000000000000000000000000000000000000000000000003
>     trusted.glusterfs.quota.size.1=0x0000000000000000000000000000
> 00000000000000000003
> ml-storage-ser23.nmg01:
>     getfattr: Removing leading '/' from absolute path names
>     # file: data2/bricks/project2/371_37829/face_landmarks
>     trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429
>     trusted.glusterfs.dht=0xdbde0bb1000000008f3cf3cd9249248f
>     trusted.glusterfs.quota.dirty=0x3000
>     trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=
> 0x000000000000000000000000000000000000000000000003
>     trusted.glusterfs.quota.size.1=0x0000000000000000000000000000
> 00000000000000000003
> ml-storage-ser27.nmg01:
>     getfattr: Removing leading '/' from absolute path names
>     # file: data2/bricks/project2/371_37829/face_landmarks
>     trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429
>     trusted.glusterfs.dht=0xdbde0bb100000000d249248fd5555551
>     trusted.glusterfs.quota.dirty=0x3000
>     trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=
> 0x000000000000000000000000000000000000000000000003
>     trusted.glusterfs.quota.size.1=0x0000000000000000000000000000
> 00000000000000000003
> ml-storage-ser25.nmg01:
>     getfattr: Removing leading '/' from absolute path names
>     # file: data2/bricks/project2/371_37829/face_landmarks
>     trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429
>     trusted.glusterfs.dht=0xdbde0bb100000000b0c30c2eb3cf3cf0
>     trusted.glusterfs.quota.dirty=0x3000
>     trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=
> 0x00000000000d760000000000000000010000000000000003
>     trusted.glusterfs.quota.size.1=0x00000000000d7600000000000000
> 00010000000000000003
> ml-storage-ser28.nmg01:
>     getfattr: Removing leading '/' from absolute path names
>     # file: data2/bricks/project2/371_37829/face_landmarks
>     trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429
>     trusted.glusterfs.dht=0xdbde0bb100000000f3cf3cf0f6db6db2
>     trusted.glusterfs.quota.dirty=0x3000
>     trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=
> 0x000000000000000000000000000000000000000000000003
>     trusted.glusterfs.quota.size.1=0x0000000000000000000000000000
> 00000000000000000003
> ml-storage-ser29.nmg01:
>     getfattr: Removing leading '/' from absolute path names
>     # file: data2/bricks/project2/371_37829/face_landmarks
>     trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429
>     trusted.glusterfs.dht=0xdbde0bb100000000f3cf3cf0f6db6db2
>     trusted.glusterfs.quota.dirty=0x3000
>     trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=
> 0x000000000000000000000000000000000000000000000003
>     trusted.glusterfs.quota.size.1=0x0000000000000000000000000000
> 00000000000000000003
> ml-storage-ser20.nmg01:
>     getfattr: Removing leading '/' from absolute path names
>     # file: data2/bricks/project2/371_37829/face_landmarks
>     trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429
>     trusted.glusterfs.dht=0xdbde0bb1000000008c30c30a8f3cf3cc
>     trusted.glusterfs.quota.dirty=0x3000
>     trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=
> 0x000000000000000000000000000000000000000000000003
>     trusted.glusterfs.quota.size.1=0x0000000000000000000000000000
> 00000000000000000003
> ml-storage-ser22.nmg01:
>     getfattr: Removing leading '/' from absolute path names
>     # file: data2/bricks/project2/371_37829/face_landmarks
>     trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429
>     trusted.glusterfs.dht=0xdbde0bb1000000008f3cf3cd9249248f
>     trusted.glusterfs.quota.dirty=0x3000
>     trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=
> 0x000000000000000000000000000000000000000000000003
>     trusted.glusterfs.quota.size.1=0x0000000000000000000000000000
> 00000000000000000003
> ml-storage-ser21.nmg01:
>     getfattr: Removing leading '/' from absolute path names
>     # file: data2/bricks/project2/371_37829/face_landmarks
>     trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429
>     trusted.glusterfs.dht=0xdbde0bb1000000008c30c30a8f3cf3cc
>     trusted.glusterfs.quota.dirty=0x3000
>     trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=
> 0x000000000000000000000000000000000000000000000003
>     trusted.glusterfs.quota.size.1=0x0000000000000000000000000000
> 00000000000000000003
> [root at ml-ctl-ser01 sunyuyusun]$ salt "ml-storage-ser2*.nmg01" cmd.run
> "getfattr -d -m. -e hex /data3/bricks/project2/371_37829/face_landmarks"
> ml-storage-ser28.nmg01:
>     getfattr: Removing leading '/' from absolute path names
>     # file: data3/bricks/project2/371_37829/face_landmarks
>     trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429
>     trusted.glusterfs.dht=0xdbde0bb1000000009555555398618615
>     trusted.glusterfs.quota.dirty=0x3000
>     trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=
> 0x00000000001aec0000000000000000020000000000000003
>     trusted.glusterfs.quota.size.1=0x00000000001aec00000000000000
> 00020000000000000003
> ml-storage-ser25.nmg01:
>     getfattr: Removing leading '/' from absolute path names
>     # file: data3/bricks/project2/371_37829/face_landmarks
>     trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429
>     trusted.glusterfs.dht=0xdbde0bb100000000892492478c30c309
>     trusted.glusterfs.quota.dirty=0x3000
>     trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=
> 0x000000000000000000000000000000000000000000000003
>     trusted.glusterfs.quota.size.1=0x0000000000000000000000000000
> 00000000000000000003
> ml-storage-ser24.nmg01:
>     getfattr: Removing leading '/' from absolute path names
>     # file: data3/bricks/project2/371_37829/face_landmarks
>     trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429
>     trusted.glusterfs.dht=0xdbde0bb100000000892492478c30c309
>     trusted.glusterfs.quota.dirty=0x3000
>     trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=
> 0x000000000000000000000000000000000000000000000003
>     trusted.glusterfs.quota.size.1=0x0000000000000000000000000000
> 00000000000000000003
> ml-storage-ser27.nmg01:
>     getfattr: Removing leading '/' from absolute path names
>     # file: data3/bricks/project2/371_37829/face_landmarks
>     trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429
>     trusted.glusterfs.dht=0xdbde0bb1000000009249249095555552
>     trusted.glusterfs.quota.dirty=0x3000
>     trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=
> 0x000000000000000000000000000000000000000000000003
>     trusted.glusterfs.quota.size.1=0x0000000000000000000000000000
> 00000000000000000003
> ml-storage-ser23.nmg01:
>     getfattr: Removing leading '/' from absolute path names
>     # file: data3/bricks/project2/371_37829/face_landmarks
>     trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429
>     trusted.glusterfs.dht=0xdbde0bb10000000079e79e787cf3cf3a
>     trusted.glusterfs.quota.dirty=0x3000
>     trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=
> 0x000000000000000000000000000000000000000000000003
>     trusted.glusterfs.quota.size.1=0x0000000000000000000000000000
> 00000000000000000003
> ml-storage-ser22.nmg01:
>     getfattr: Removing leading '/' from absolute path names
>     # file: data3/bricks/project2/371_37829/face_landmarks
>     trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429
>     trusted.glusterfs.dht=0xdbde0bb10000000079e79e787cf3cf3a
>     trusted.glusterfs.quota.dirty=0x3000
>     trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=
> 0x000000000000000000000000000000000000000000000003
>     trusted.glusterfs.quota.size.1=0x0000000000000000000000000000
> 00000000000000000003
> ml-storage-ser29.nmg01:
>     getfattr: Removing leading '/' from absolute path names
>     # file: data3/bricks/project2/371_37829/face_landmarks
>     trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429
>     trusted.glusterfs.dht=0xdbde0bb1000000009555555398618615
>     trusted.glusterfs.quota.dirty=0x3000
>     trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=
> 0x00000000001aec0000000000000000020000000000000003
>     trusted.glusterfs.quota.size.1=0x00000000001aec00000000000000
> 00020000000000000003
> ml-storage-ser26.nmg01:
>     getfattr: Removing leading '/' from absolute path names
>     # file: data3/bricks/project2/371_37829/face_landmarks
>     trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429
>     trusted.glusterfs.dht=0xdbde0bb1000000009249249095555552
>     trusted.glusterfs.quota.dirty=0x3000
>     trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=
> 0x000000000000000000000000000000000000000000000003
>     trusted.glusterfs.quota.size.1=0x0000000000000000000000000000
> 00000000000000000003
> ml-storage-ser21.nmg01:
>     getfattr: Removing leading '/' from absolute path names
>     # file: data3/bricks/project2/371_37829/face_landmarks
>     trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429
>     trusted.glusterfs.dht=0xdbde0bb100000000586186175b6db6d9
>     trusted.glusterfs.quota.dirty=0x3000
>     trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=
> 0x000000000000000000000000000000000000000000000003
>     trusted.glusterfs.quota.size.1=0x0000000000000000000000000000
> 00000000000000000003
> ml-storage-ser20.nmg01:
>     getfattr: Removing leading '/' from absolute path names
>     # file: data3/bricks/project2/371_37829/face_landmarks
>     trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429
>     trusted.glusterfs.dht=0xdbde0bb100000000586186175b6db6d9
>     trusted.glusterfs.quota.dirty=0x3000
>     trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=
> 0x000000000000000000000000000000000000000000000003
>     trusted.glusterfs.quota.size.1=0x0000000000000000000000000000
> 00000000000000000003
>
>
>
> [root at ml-ctl-ser01 sunyuyusun]$ salt "ml-storage-ser2*.nmg01" cmd.run
> "getfattr -d -m. -e hex /data4/bricks/project2/371_37829/face_landmarks"
> ml-storage-ser25.nmg01:
>     getfattr: Removing leading '/' from absolute path names
>     # file: data4/bricks/project2/371_37829/face_landmarks
>     trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429
>     trusted.glusterfs.dht=0xdbde0bb100000000a4924922a79e79e4
>     trusted.glusterfs.quota.dirty=0x3000
>     trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=
> 0x000000000000000000000000000000000000000000000003
>     trusted.glusterfs.quota.size.1=0x0000000000000000000000000000
> 00000000000000000003
> ml-storage-ser23.nmg01:
>     getfattr: Removing leading '/' from absolute path names
>     # file: data4/bricks/project2/371_37829/face_landmarks
>     trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429
>     trusted.glusterfs.dht=0xdbde0bb100000000a186185fa4924921
>     trusted.glusterfs.quota.dirty=0x3000
>     trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=
> 0x000000000000000000000000000000000000000000000003
>     trusted.glusterfs.quota.size.1=0x0000000000000000000000000000
> 00000000000000000003
> ml-storage-ser24.nmg01:
>     getfattr: Removing leading '/' from absolute path names
>     # file: data4/bricks/project2/371_37829/face_landmarks
>     trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429
>     trusted.glusterfs.dht=0xdbde0bb100000000a4924922a79e79e4
>     trusted.glusterfs.quota.dirty=0x3000
>     trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=
> 0x000000000000000000000000000000000000000000000003
>     trusted.glusterfs.quota.size.1=0x0000000000000000000000000000
> 00000000000000000003
> ml-storage-ser28.nmg01:
>     getfattr: Removing leading '/' from absolute path names
>     # file: data4/bricks/project2/371_37829/face_landmarks
>     trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429
>     trusted.glusterfs.dht=0xdbde0bb100000000aaaaaaa8adb6db6a
>     trusted.glusterfs.quota.dirty=0x3000
>     trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=
> 0x000000000000000000000000000000000000000000000003
>     trusted.glusterfs.quota.size.1=0x0000000000000000000000000000
> 00000000000000000003
> ml-storage-ser27.nmg01:
>     getfattr: Removing leading '/' from absolute path names
>     # file: data4/bricks/project2/371_37829/face_landmarks
>     trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429
>     trusted.glusterfs.dht=0xdbde0bb100000000a79e79e5aaaaaaa7
>     trusted.glusterfs.quota.dirty=0x3000
>     trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=
> 0x000000000000000000000000000000000000000000000003
>     trusted.glusterfs.quota.size.1=0x0000000000000000000000000000
> 00000000000000000003
> ml-storage-ser26.nmg01:
>     getfattr: Removing leading '/' from absolute path names
>     # file: data4/bricks/project2/371_37829/face_landmarks
>     trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429
>     trusted.glusterfs.dht=0xdbde0bb100000000a79e79e5aaaaaaa7
>     trusted.glusterfs.quota.dirty=0x3000
>     trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=
> 0x000000000000000000000000000000000000000000000003
>     trusted.glusterfs.quota.size.1=0x0000000000000000000000000000
> 00000000000000000003
> ml-storage-ser21.nmg01:
>     getfattr: Removing leading '/' from absolute path names
>     # file: data4/bricks/project2/371_37829/face_landmarks
>     trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429
>     trusted.glusterfs.dht=0xdbde0bb1000000009e79e79ca186185e
>     trusted.glusterfs.quota.dirty=0x3000
>     trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=
> 0x000000000000000000000000000000000000000000000003
>     trusted.glusterfs.quota.size.1=0x0000000000000000000000000000
> 00000000000000000003
> ml-storage-ser22.nmg01:
>     getfattr: Removing leading '/' from absolute path names
>     # file: data4/bricks/project2/371_37829/face_landmarks
>     trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429
>     trusted.glusterfs.dht=0xdbde0bb100000000a186185fa4924921
>     trusted.glusterfs.quota.dirty=0x3000
>     trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=
> 0x000000000000000000000000000000000000000000000003
>     trusted.glusterfs.quota.size.1=0x0000000000000000000000000000
> 00000000000000000003
> ml-storage-ser29.nmg01:
>     getfattr: Removing leading '/' from absolute path names
>     # file: data4/bricks/project2/371_37829/face_landmarks
>     trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429
>     trusted.glusterfs.dht=0xdbde0bb100000000aaaaaaa8adb6db6a
>     trusted.glusterfs.quota.dirty=0x3000
>     trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=
> 0x000000000000000000000000000000000000000000000003
>     trusted.glusterfs.quota.size.1=0x0000000000000000000000000000
> 00000000000000000003
> ml-storage-ser20.nmg01:
>     getfattr: Removing leading '/' from absolute path names
>     # file: data4/bricks/project2/371_37829/face_landmarks
>     trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429
>     trusted.glusterfs.dht=0xdbde0bb1000000009e79e79ca186185e
>     trusted.glusterfs.quota.dirty=0x3000
>     trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=
> 0x000000000000000000000000000000000000000000000003
>     trusted.glusterfs.quota.size.1=0x0000000000000000000000000000
> 00000000000000000003
>
>
>
>
> [root at ml-ctl-ser01 sunyuyusun]$ salt "ml-storage-ser2*.nmg01" cmd.run
> "getfattr -d -m. -e hex /data2/bricks/project2/371_37829/test-dir"
> ml-storage-ser23.nmg01:
>     getfattr: Removing leading '/' from absolute path names
>     # file: data2/bricks/project2/371_37829/test-dir
>     trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217
>     trusted.glusterfs.dht=0xdbde0bb1000000000f3cf3cf12492491
>     trusted.glusterfs.quota.dirty=0x3000
>     trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=
> 0x000000000000000000000000000000000000000000000001
>     trusted.glusterfs.quota.size.1=0x0000000000000000000000000000
> 00000000000000000001
> ml-storage-ser24.nmg01:
>     getfattr: Removing leading '/' from absolute path names
>     # file: data2/bricks/project2/371_37829/test-dir
>     trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217
>     trusted.glusterfs.dht=0xdbde0bb10000000030c30c3033cf3cf2
>     trusted.glusterfs.quota.dirty=0x3000
>     trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=
> 0x000000000000000000000000000000000000000000000001
>     trusted.glusterfs.quota.size.1=0x0000000000000000000000000000
> 00000000000000000001
> ml-storage-ser28.nmg01:
>     getfattr: Removing leading '/' from absolute path names
>     # file: data2/bricks/project2/371_37829/test-dir
>     trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217
>     trusted.glusterfs.dht=0xdbde0bb10000000073cf3cf276db6db4
>     trusted.glusterfs.quota.dirty=0x3000
>     trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=
> 0x000000000000000000000000000000000000000000000001
>     trusted.glusterfs.quota.size.1=0x0000000000000000000000000000
> 00000000000000000001
> ml-storage-ser25.nmg01:
>     getfattr: Removing leading '/' from absolute path names
>     # file: data2/bricks/project2/371_37829/test-dir
>     trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217
>     trusted.glusterfs.dht=0xdbde0bb10000000030c30c3033cf3cf2
>     trusted.glusterfs.quota.dirty=0x3000
>     trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=
> 0x000000000000000000000000000000000000000000000001
>     trusted.glusterfs.quota.size.1=0x0000000000000000000000000000
> 00000000000000000001
> ml-storage-ser26.nmg01:
>     getfattr: Removing leading '/' from absolute path names
>     # file: data2/bricks/project2/371_37829/test-dir
>     trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217
>     trusted.glusterfs.dht=0xdbde0bb1000000005249249155555553
>     trusted.glusterfs.quota.dirty=0x3000
>     trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=
> 0x000000000000000000000000000000000000000000000001
>     trusted.glusterfs.quota.size.1=0x0000000000000000000000000000
> 00000000000000000001
> ml-storage-ser27.nmg01:
>     getfattr: Removing leading '/' from absolute path names
>     # file: data2/bricks/project2/371_37829/test-dir
>     trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217
>     trusted.glusterfs.dht=0xdbde0bb1000000005249249155555553
>     trusted.glusterfs.quota.dirty=0x3000
>     trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=
> 0x000000000000000000000000000000000000000000000001
>     trusted.glusterfs.quota.size.1=0x0000000000000000000000000000
> 00000000000000000001
> ml-storage-ser22.nmg01:
>     getfattr: Removing leading '/' from absolute path names
>     # file: data2/bricks/project2/371_37829/test-dir
>     trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217
>     trusted.glusterfs.dht=0xdbde0bb1000000000f3cf3cf12492491
>     trusted.glusterfs.quota.dirty=0x3000
>     trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=
> 0x000000000000000000000000000000000000000000000001
>     trusted.glusterfs.quota.size.1=0x0000000000000000000000000000
> 00000000000000000001
> ml-storage-ser29.nmg01:
>     getfattr: Removing leading '/' from absolute path names
>     # file: data2/bricks/project2/371_37829/test-dir
>     trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217
>     trusted.glusterfs.dht=0xdbde0bb10000000073cf3cf276db6db4
>     trusted.glusterfs.quota.dirty=0x3000
>     trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=
> 0x000000000000000000000000000000000000000000000001
>     trusted.glusterfs.quota.size.1=0x0000000000000000000000000000
> 00000000000000000001
> ml-storage-ser21.nmg01:
>     getfattr: Removing leading '/' from absolute path names
>     # file: data2/bricks/project2/371_37829/test-dir
>     trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217
>     trusted.glusterfs.dht=0xdbde0bb1000000000c30c30c0f3cf3ce
>     trusted.glusterfs.quota.dirty=0x3000
>     trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=
> 0x000000000000000000000000000000000000000000000001
>     trusted.glusterfs.quota.size.1=0x0000000000000000000000000000
> 00000000000000000001
> ml-storage-ser20.nmg01:
>     getfattr: Removing leading '/' from absolute path names
>     # file: data2/bricks/project2/371_37829/test-dir
>     trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217
>     trusted.glusterfs.dht=0xdbde0bb1000000000c30c30c0f3cf3ce
>     trusted.glusterfs.quota.dirty=0x3000
>     trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=
> 0x000000000000000000000000000000000000000000000001
>     trusted.glusterfs.quota.size.1=0x0000000000000000000000000000
> 00000000000000000001
>
>
>
> [root at ml-ctl-ser01 sunyuyusun]$ salt "ml-storage-ser2*.nmg01" cmd.run
> "getfattr -d -m. -e hex /data3/bricks/project2/371_37829/test-dir"
> ml-storage-ser23.nmg01:
>     getfattr: Removing leading '/' from absolute path names
>     # file: data3/bricks/project2/371_37829/test-dir
>     trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217
>     trusted.glusterfs.dht=0xdbde0bb100000000f9e79e76fcf3cf38
>     trusted.glusterfs.quota.dirty=0x3000
>     trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=
> 0x000000000000000000000000000000000000000000000001
>     trusted.glusterfs.quota.size.1=0x0000000000000000000000000000
> 00000000000000000001
> ml-storage-ser26.nmg01:
>     getfattr: Removing leading '/' from absolute path names
>     # file: data3/bricks/project2/371_37829/test-dir
>     trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217
>     trusted.glusterfs.dht=0xdbde0bb1000000001249249215555554
>     trusted.glusterfs.quota.dirty=0x3000
>     trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=
> 0x000000000000000000000000000000000000000000000001
>     trusted.glusterfs.quota.size.1=0x0000000000000000000000000000
> 00000000000000000001
> ml-storage-ser25.nmg01:
>     getfattr: Removing leading '/' from absolute path names
>     # file: data3/bricks/project2/371_37829/test-dir
>     trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217
>     trusted.glusterfs.dht=0xdbde0bb100000000092492490c30c30b
>     trusted.glusterfs.quota.dirty=0x3000
>     trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=
> 0x000000000000000000000000000000000000000000000001
>     trusted.glusterfs.quota.size.1=0x0000000000000000000000000000
> 00000000000000000001
> ml-storage-ser28.nmg01:
>     getfattr: Removing leading '/' from absolute path names
>     # file: data3/bricks/project2/371_37829/test-dir
>     trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217
>     trusted.glusterfs.dht=0xdbde0bb1000000001555555518618617
>     trusted.glusterfs.quota.dirty=0x3000
>     trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=
> 0x000000000000000000000000000000000000000000000001
>     trusted.glusterfs.quota.size.1=0x0000000000000000000000000000
> 00000000000000000001
> ml-storage-ser24.nmg01:
>     getfattr: Removing leading '/' from absolute path names
>     # file: data3/bricks/project2/371_37829/test-dir
>     trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217
>     trusted.glusterfs.dht=0xdbde0bb100000000092492490c30c30b
>     trusted.glusterfs.quota.dirty=0x3000
>     trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=
> 0x000000000000000000000000000000000000000000000001
>     trusted.glusterfs.quota.size.1=0x0000000000000000000000000000
> 00000000000000000001
> ml-storage-ser22.nmg01:
>     getfattr: Removing leading '/' from absolute path names
>     # file: data3/bricks/project2/371_37829/test-dir
>     trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217
>     trusted.glusterfs.dht=0xdbde0bb100000000f9e79e76fcf3cf38
>     trusted.glusterfs.quota.dirty=0x3000
>     trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=
> 0x000000000000000000000000000000000000000000000001
>     trusted.glusterfs.quota.size.1=0x0000000000000000000000000000
> 00000000000000000001
> ml-storage-ser29.nmg01:
>     getfattr: Removing leading '/' from absolute path names
>     # file: data3/bricks/project2/371_37829/test-dir
>     trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217
>     trusted.glusterfs.dht=0xdbde0bb1000000001555555518618617
>     trusted.glusterfs.quota.dirty=0x3000
>     trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=
> 0x000000000000000000000000000000000000000000000001
>     trusted.glusterfs.quota.size.1=0x0000000000000000000000000000
> 00000000000000000001
> ml-storage-ser21.nmg01:
>     getfattr: Removing leading '/' from absolute path names
>     # file: data3/bricks/project2/371_37829/test-dir
>     trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217
>     trusted.glusterfs.dht=0xdbde0bb100000000d8618615db6db6d7
>     trusted.glusterfs.quota.dirty=0x3000
>     trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=
> 0x000000000000000000000000000000000000000000000001
>     trusted.glusterfs.quota.size.1=0x0000000000000000000000000000
> 00000000000000000001
> ml-storage-ser27.nmg01:
>     getfattr: Removing leading '/' from absolute path names
>     # file: data3/bricks/project2/371_37829/test-dir
>     trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217
>     trusted.glusterfs.dht=0xdbde0bb1000000001249249215555554
>     trusted.glusterfs.quota.dirty=0x3000
>     trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=
> 0x000000000000000000000000000000000000000000000001
>     trusted.glusterfs.quota.size.1=0x0000000000000000000000000000
> 00000000000000000001
> ml-storage-ser20.nmg01:
>     getfattr: Removing leading '/' from absolute path names
>     # file: data3/bricks/project2/371_37829/test-dir
>     trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217
>     trusted.glusterfs.dht=0xdbde0bb100000000d8618615db6db6d7
>     trusted.glusterfs.quota.dirty=0x3000
>     trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=
> 0x000000000000000000000000000000000000000000000001
>     trusted.glusterfs.quota.size.1=0x0000000000000000000000000000
> 00000000000000000001
>
>
>
> [root at ml-ctl-ser01 sunyuyusun]$ salt "ml-storage-ser2*.nmg01" cmd.run
> "getfattr -d -m. -e hex /data4/bricks/project2/371_37829/test-dir"
> ml-storage-ser25.nmg01:
>     getfattr: Removing leading '/' from absolute path names
>     # file: data4/bricks/project2/371_37829/test-dir
>     trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217
>     trusted.glusterfs.dht=0xdbde0bb10000000024924924279e79e6
>     trusted.glusterfs.quota.dirty=0x3000
>     trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=
> 0x000000000000000000000000000000000000000000000001
>     trusted.glusterfs.quota.size.1=0x0000000000000000000000000000
> 00000000000000000001
> ml-storage-ser27.nmg01:
>     getfattr: Removing leading '/' from absolute path names
>     # file: data4/bricks/project2/371_37829/test-dir
>     trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217
>     trusted.glusterfs.dht=0xdbde0bb100000000279e79e72aaaaaa9
>     trusted.glusterfs.quota.dirty=0x3000
>     trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=
> 0x000000000000000000000000000000000000000000000001
>     trusted.glusterfs.quota.size.1=0x0000000000000000000000000000
> 00000000000000000001
> ml-storage-ser28.nmg01:
>     getfattr: Removing leading '/' from absolute path names
>     # file: data4/bricks/project2/371_37829/test-dir
>     trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217
>     trusted.glusterfs.dht=0xdbde0bb1000000002aaaaaaa2db6db6c
>     trusted.glusterfs.quota.dirty=0x3000
>     trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=
> 0x000000000000000000000000000000000000000000000001
>     trusted.glusterfs.quota.size.1=0x0000000000000000000000000000
> 00000000000000000001
> ml-storage-ser23.nmg01:
>     getfattr: Removing leading '/' from absolute path names
>     # file: data4/bricks/project2/371_37829/test-dir
>     trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217
>     trusted.glusterfs.dht=0xdbde0bb1000000002186186124924923
>     trusted.glusterfs.quota.dirty=0x3000
>     trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=
> 0x000000000000000000000000000000000000000000000001
>     trusted.glusterfs.quota.size.1=0x0000000000000000000000000000
> 00000000000000000001
> ml-storage-ser26.nmg01:
>     getfattr: Removing leading '/' from absolute path names
>     # file: data4/bricks/project2/371_37829/test-dir
>     trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217
>     trusted.glusterfs.dht=0xdbde0bb100000000279e79e72aaaaaa9
>     trusted.glusterfs.quota.dirty=0x3000
>     trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=
> 0x000000000000000000000000000000000000000000000001
>     trusted.glusterfs.quota.size.1=0x0000000000000000000000000000
> 00000000000000000001
> ml-storage-ser29.nmg01:
>     getfattr: Removing leading '/' from absolute path names
>     # file: data4/bricks/project2/371_37829/test-dir
>     trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217
>     trusted.glusterfs.dht=0xdbde0bb1000000002aaaaaaa2db6db6c
>     trusted.glusterfs.quota.dirty=0x3000
>     trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=
> 0x000000000000000000000000000000000000000000000001
>     trusted.glusterfs.quota.size.1=0x0000000000000000000000000000
> 00000000000000000001
> ml-storage-ser22.nmg01:
>     getfattr: Removing leading '/' from absolute path names
>     # file: data4/bricks/project2/371_37829/test-dir
>     trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217
>     trusted.glusterfs.dht=0xdbde0bb1000000002186186124924923
>     trusted.glusterfs.quota.dirty=0x3000
>     trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=
> 0x000000000000000000000000000000000000000000000001
>     trusted.glusterfs.quota.size.1=0x0000000000000000000000000000
> 00000000000000000001
> ml-storage-ser24.nmg01:
>     getfattr: Removing leading '/' from absolute path names
>     # file: data4/bricks/project2/371_37829/test-dir
>     trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217
>     trusted.glusterfs.dht=0xdbde0bb10000000024924924279e79e6
>     trusted.glusterfs.quota.dirty=0x3000
>     trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=
> 0x000000000000000000000000000000000000000000000001
>     trusted.glusterfs.quota.size.1=0x0000000000000000000000000000
> 00000000000000000001
> ml-storage-ser20.nmg01:
>     getfattr: Removing leading '/' from absolute path names
>     # file: data4/bricks/project2/371_37829/test-dir
>     c=0x57e0a8945a224ab4be2c6c71eada6217
>     trusted.glusterfs.dht=0xdbde0bb1000000001e79e79e21861860
>     trusted.glusterfs.quota.dirty=0x3000
>     trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=
> 0x000000000000000000000000000000000000000000000001
>     trusted.glusterfs.quota.size.1=0x0000000000000000000000000000
> 00000000000000000001
> ml-storage-ser21.nmg01:
>     getfattr: Removing leading '/' from absolute path names
>     # file: data4/bricks/project2/371_37829/test-dir
>     trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217
>     trusted.glusterfs.dht=0xdbde0bb1000000001e79e79e21861860
>     trusted.glusterfs.quota.dirty=0x3000
>     trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=
> 0x000000000000000000000000000000000000000000000001
>     trusted.glusterfs.quota.size.1=0x0000000000000000000000000000
> 00000000000000000001
> [root at ml-ctl-ser01 sunyuyusun]$
>
> Is the infomation listed above what you need? I can see the two directory
> (src and dst) are of diffenent trusted.gfid.
>

Yes. As you observed both directories have different gfids. So, this issue
may not be related to to bugs I pointed out earlier.

> also I have another test and basically certain that when i enable quota,
> this will happen in one mount point(i only mount the test volume in one
> mount point)
>
>
>
> Best Regards
> Yours
> Yu
>
>
>
> Raghavendra Gowdappa <rgowdapp at redhat.com> 于2018年9月5日周三 下午12:40写道：
>
>>
>>
>> On Wed, Sep 5, 2018 at 9:21 AM, yu sun <sunyu1949 at gmail.com> wrote:
>>
>>> thank you , I will investigate and have a try.
>>>
>>
>> Please get all the extended attributes of src and dst from all bricks.
>> That should help us to (dis)prove whether its same bug.
>>
>>
>>> BYW, I found another thing:
>>> 1. I found this only occur when I try to mv dir to a subdir.
>>> 2. I create a similar volume with default option and without quota
>>> enabled, i found its ok, mv seems no problem.
>>>
>>> so I think it's possibly the volume option led to this problem, but I
>>> dont know which options and  how to resolve this, the volume have about 25T
>>> data.
>>>
>>> Best Regards.
>>>
>>> Raghavendra Gowdappa <rgowdapp at redhat.com> 于2018年9月5日周三 上午10:50写道：
>>>
>>>>
>>>>
>>>> On Tue, Sep 4, 2018 at 5:28 PM, yu sun <sunyu1949 at gmail.com> wrote:
>>>>
>>>>> Hi all:
>>>>>
>>>>> I have a replicated volume project2 with info:
>>>>> Volume Name: project2 Type: Distributed-Replicate Volume ID:
>>>>> 60175b8e-de0e-4409-81ae-7bb5eb5cacbf Status: Started Snapshot Count:
>>>>> 0 Number of Bricks: 84 x 2 = 168 Transport-type: tcp Bricks: Brick1:
>>>>> node20:/data2/bricks/project2 Brick2: node21:/data2/bricks/project2 Brick3:
>>>>> node22:/data2/bricks/project2 Brick4: node23:/data2/bricks/project2 Brick5:
>>>>> node24:/data2/bricks/project2 Brick6: node25:/data2/bricks/project2 Brick7:
>>>>> node26:/data2/bricks/project2 Brick8: node27:/data2/bricks/project2 Brick9:
>>>>> node28:/data2/bricks/project2 Brick10: node29:/data2/bricks/project2
>>>>> Brick11: node30:/data2/bricks/project2 Brick12:
>>>>> node31:/data2/bricks/project2 Brick13: node32:/data2/bricks/project2
>>>>> Brick14: node33:/data2/bricks/project2 Brick15:
>>>>> node20:/data3/bricks/project2 Brick16: node21:/data3/bricks/project2
>>>>> Brick17: node22:/data3/bricks/project2 Brick18:
>>>>> node23:/data3/bricks/project2 Brick19: node24:/data3/bricks/project2
>>>>> Brick20: node25:/data3/bricks/project2 Brick21:
>>>>> node26:/data3/bricks/project2 Brick22: node27:/data3/bricks/project2
>>>>> Brick23: node28:/data3/bricks/project2 Brick24:
>>>>> node29:/data3/bricks/project2 Brick25: node30:/data3/bricks/project2
>>>>> Brick26: node31:/data3/bricks/project2 Brick27:
>>>>> node32:/data3/bricks/project2 Brick28: node33:/data3/bricks/project2
>>>>> Brick29: node20:/data4/bricks/project2 Brick30:
>>>>> node21:/data4/bricks/project2 Brick31: node22:/data4/bricks/project2
>>>>> Brick32: node23:/data4/bricks/project2 Brick33:
>>>>> node24:/data4/bricks/project2 Brick34: node25:/data4/bricks/project2
>>>>> Brick35: node26:/data4/bricks/project2 Brick36:
>>>>> node27:/data4/bricks/project2 Brick37: node28:/data4/bricks/project2
>>>>> Brick38: node29:/data4/bricks/project2 Brick39:
>>>>> node30:/data4/bricks/project2 Brick40: node31:/data4/bricks/project2
>>>>> Brick41: node32:/data4/bricks/project2 Brick42:
>>>>> node33:/data4/bricks/project2 Brick43: node20:/data5/bricks/project2
>>>>> Brick44: node21:/data5/bricks/project2 Brick45:
>>>>> node22:/data5/bricks/project2 Brick46: node23:/data5/bricks/project2
>>>>> Brick47: node24:/data5/bricks/project2 Brick48:
>>>>> node25:/data5/bricks/project2 Brick49: node26:/data5/bricks/project2
>>>>> Brick50: node27:/data5/bricks/project2 Brick51:
>>>>> node28:/data5/bricks/project2 Brick52: node29:/data5/bricks/project2
>>>>> Brick53: node30:/data5/bricks/project2 Brick54:
>>>>> node31:/data5/bricks/project2 Brick55: node32:/data5/bricks/project2
>>>>> Brick56: node33:/data5/bricks/project2 Brick57:
>>>>> node20:/data6/bricks/project2 Brick58: node21:/data6/bricks/project2
>>>>> Brick59: node22:/data6/bricks/project2 Brick60:
>>>>> node23:/data6/bricks/project2 Brick61: node24:/data6/bricks/project2
>>>>> Brick62: node25:/data6/bricks/project2 Brick63:
>>>>> node26:/data6/bricks/project2 Brick64: node27:/data6/bricks/project2
>>>>> Brick65: node28:/data6/bricks/project2 Brick66:
>>>>> node29:/data6/bricks/project2 Brick67: node30:/data6/bricks/project2
>>>>> Brick68: node31:/data6/bricks/project2 Brick69:
>>>>> node32:/data6/bricks/project2 Brick70: node33:/data6/bricks/project2
>>>>> Brick71: node20:/data7/bricks/project2 Brick72:
>>>>> node21:/data7/bricks/project2 Brick73: node22:/data7/bricks/project2
>>>>> Brick74: node23:/data7/bricks/project2 Brick75:
>>>>> node24:/data7/bricks/project2 Brick76: node25:/data7/bricks/project2
>>>>> Brick77: node26:/data7/bricks/project2 Brick78:
>>>>> node27:/data7/bricks/project2 Brick79: node28:/data7/bricks/project2
>>>>> Brick80: node29:/data7/bricks/project2 Brick81:
>>>>> node30:/data7/bricks/project2 Brick82: node31:/data7/bricks/project2
>>>>> Brick83: node32:/data7/bricks/project2 Brick84:
>>>>> node33:/data7/bricks/project2 Brick85: node20:/data8/bricks/project2
>>>>> Brick86: node21:/data8/bricks/project2 Brick87:
>>>>> node22:/data8/bricks/project2 Brick88: node23:/data8/bricks/project2
>>>>> Brick89: node24:/data8/bricks/project2 Brick90:
>>>>> node25:/data8/bricks/project2 Brick91: node26:/data8/bricks/project2
>>>>> Brick92: node27:/data8/bricks/project2 Brick93:
>>>>> node28:/data8/bricks/project2 Brick94: node29:/data8/bricks/project2
>>>>> Brick95: node30:/data8/bricks/project2 Brick96:
>>>>> node31:/data8/bricks/project2 Brick97: node32:/data8/bricks/project2
>>>>> Brick98: node33:/data8/bricks/project2 Brick99:
>>>>> node20:/data9/bricks/project2 Brick100: node21:/data9/bricks/project2
>>>>> Brick101: node22:/data9/bricks/project2 Brick102:
>>>>> node23:/data9/bricks/project2 Brick103: node24:/data9/bricks/project2
>>>>> Brick104: node25:/data9/bricks/project2 Brick105:
>>>>> node26:/data9/bricks/project2 Brick106: node27:/data9/bricks/project2
>>>>> Brick107: node28:/data9/bricks/project2 Brick108:
>>>>> node29:/data9/bricks/project2 Brick109: node30:/data9/bricks/project2
>>>>> Brick110: node31:/data9/bricks/project2 Brick111:
>>>>> node32:/data9/bricks/project2 Brick112: node33:/data9/bricks/project2
>>>>> Brick113: node20:/data10/bricks/project2 Brick114:
>>>>> node21:/data10/bricks/project2 Brick115: node22:/data10/bricks/project2
>>>>> Brick116: node23:/data10/bricks/project2 Brick117:
>>>>> node24:/data10/bricks/project2 Brick118: node25:/data10/bricks/project2
>>>>> Brick119: node26:/data10/bricks/project2 Brick120:
>>>>> node27:/data10/bricks/project2 Brick121: node28:/data10/bricks/project2
>>>>> Brick122: node29:/data10/bricks/project2 Brick123:
>>>>> node30:/data10/bricks/project2 Brick124: node31:/data10/bricks/project2
>>>>> Brick125: node32:/data10/bricks/project2 Brick126:
>>>>> node33:/data10/bricks/project2 Brick127: node20:/data11/bricks/project2
>>>>> Brick128: node21:/data11/bricks/project2 Brick129:
>>>>> node22:/data11/bricks/project2 Brick130: node23:/data11/bricks/project2
>>>>> Brick131: node24:/data11/bricks/project2 Brick132:
>>>>> node25:/data11/bricks/project2 Brick133: node26:/data11/bricks/project2
>>>>> Brick134: node27:/data11/bricks/project2 Brick135:
>>>>> node28:/data11/bricks/project2 Brick136: node29:/data11/bricks/project2
>>>>> Brick137: node30:/data11/bricks/project2 Brick138:
>>>>> node31:/data11/bricks/project2 Brick139: node32:/data11/bricks/project2
>>>>> Brick140: node33:/data11/bricks/project2 Brick141:
>>>>> node20:/data12/bricks/project2 Brick142: node21:/data12/bricks/project2
>>>>> Brick143: node22:/data12/bricks/project2 Brick144:
>>>>> node23:/data12/bricks/project2 Brick145: node24:/data12/bricks/project2
>>>>> Brick146: node25:/data12/bricks/project2 Brick147:
>>>>> node26:/data12/bricks/project2 Brick148: node27:/data12/bricks/project2
>>>>> Brick149: node28:/data12/bricks/project2 Brick150:
>>>>> node29:/data12/bricks/project2 Brick151: node30:/data12/bricks/project2
>>>>> Brick152: node31:/data12/bricks/project2 Brick153:
>>>>> node32:/data12/bricks/project2 Brick154: node33:/data12/bricks/project2
>>>>> Brick155: node20:/data13/bricks/project2 Brick156:
>>>>> node21:/data13/bricks/project2 Brick157: node22:/data13/bricks/project2
>>>>> Brick158: node23:/data13/bricks/project2 Brick159:
>>>>> node24:/data13/bricks/project2 Brick160: node25:/data13/bricks/project2
>>>>> Brick161: node26:/data13/bricks/project2 Brick162:
>>>>> node27:/data13/bricks/project2 Brick163: node28:/data13/bricks/project2
>>>>> Brick164: node29:/data13/bricks/project2 Brick165:
>>>>> node30:/data13/bricks/project2 Brick166: node31:/data13/bricks/project2
>>>>> Brick167: node32:/data13/bricks/project2 Brick168:
>>>>> node33:/data13/bricks/project2 Options Reconfigured:
>>>>> performance.force-readdirp: on performance.write-behind: off
>>>>> performance.stat-prefetch: on performance.client-io-threads: on
>>>>> nfs.disable: on transport.address-family: inet features.quota: on
>>>>> features.inode-quota: on features.quota-deem-statfs: on
>>>>> cluster.readdir-optimize: on cluster.lookup-optimize: on
>>>>> dht.force-readdirp: off client.event-threads: 10 server.event-threads: 10
>>>>> performance.readdir-ahead: on performance.io-cache: on
>>>>> performance.flush-behind: on performance.cache-size: 5GB
>>>>> performance.cache-max-file-size: 1MB performance.write-behind-window-size:
>>>>> 10MB performance.read-ahead: off network.remote-dio: enable
>>>>> performance.strict-o-direct: disable performance.io-thread-count: 25
>>>>>
>>>>>
>>>>> the volume looks ok, and I mount this volume on my client machine:
>>>>> mount -t glusterfs -o oom-score-adj=-999 -o direct-io-mode=disable -o
>>>>> use-readdirp=no node20:/project2 /mnt/project2
>>>>>
>>>>> I have a directory in /mnt/project2/, but when I mv the directory to
>>>>> other dirs, files in the dir lost while tree or ls the dir, some files
>>>>> missing, my operations is list as below:
>>>>>
>>>>
>>>> Looks very similar to:
>>>> https://bugzilla.redhat.com/show_bug.cgi?id=1118762
>>>> https://bugzilla.redhat.com/show_bug.cgi?id=1337394
>>>>
>>>>
>>>>
>>>>> root at ml-gpu-ser129.nmg01:/mnt/project2/371_37829$ mkdir test-dir
>>>>> root at ml-gpu-ser129.nmg01:/mnt/project2/371_37829$ tree
>>>>> .
>>>>> ├── face_landmarks
>>>>> │   └── alive
>>>>> │       └── logs_20180823_28
>>>>> │           ├── info_000000.out
>>>>> │           ├── info_000001.out
>>>>> │           ├── info_000002.out
>>>>> │           ├── info_000003.out
>>>>> │           ├── info_000004.out
>>>>> │           ├── info_000005.out
>>>>> │           ├── info_000006.out
>>>>> │           ├── info_000007.out
>>>>> │           ├── info_000008.out
>>>>> │           ├── info_000009.out
>>>>> │           ├── info_000010.out
>>>>> │           ├── info_000011.out
>>>>> │           ├── info_000012.out
>>>>> │           ├── info_000013.out
>>>>> │           ├── info_000014.out
>>>>> │           ├── info_000015.out
>>>>> │           ├── info_000016.out
>>>>> │           ├── info_000017.out
>>>>> │           ├── info_000018.out
>>>>> │           └── info_000019.out
>>>>> └── test-dir
>>>>>
>>>>> 4 directories, 20 files
>>>>> root at ml-gpu-ser129.nmg01:/mnt/project2/371_37829$ mv face_landmarks/
>>>>> test-dir/
>>>>> root at ml-gpu-ser129.nmg01:/mnt/project2/371_37829$ tree
>>>>> .
>>>>> └── test-dir
>>>>>     └── face_landmarks
>>>>>
>>>>> 2 directories, 0 files
>>>>> root at ml-gpu-ser129.nmg01:/mnt/project2/371_37829$ cd
>>>>> test-dir/face_landmarks/
>>>>> root at ml-gpu-ser129.nmg01:/mnt/project2/371_37829/test-dir/face_landmarks$
>>>>> ls
>>>>> root at ml-gpu-ser129.nmg01:/mnt/project2/371_37829/test-dir/face_landmarks$
>>>>> cd ..
>>>>> root at ml-gpu-ser129.nmg01:/mnt/project2/371_37829/test-dir$ mv
>>>>> face_landmarks/ ..
>>>>> root at ml-gpu-ser129.nmg01:/mnt/project2/371_37829/test-dir$ cd ..
>>>>> root at ml-gpu-ser129.nmg01:/mnt/project2/371_37829$ tree
>>>>> .
>>>>> ├── face_landmarks
>>>>> │   └── alive
>>>>> └── test-dir
>>>>>
>>>>> 3 directories, 0 files
>>>>> root at ml-gpu-ser129.nmg01:/mnt/project2/371_37829$
>>>>>
>>>>> I think i make some mistakes with volume option, buti i am not sure,
>>>>> so how can i find the lost files?  the files seems still int the directory,
>>>>> because i cant remove the directory  and rm tell me "Not empty directory"
>>>>>
>>>>
>>>> Its likely that src and dst of mv having same gfid and that's causing
>>>> the issues. Can you look into both src and dst paths on all bricks? Union
>>>> of contents of both directories should give all the files in the src
>>>> directory before mv. Once found you can,
>>>> * keep a backup of contents of src and dst on all bricks
>>>> * remove trusted.gfid xattr on src and dst from all bricks
>>>> * remove gfid handle (.glusterfs/<first two characters of gfid>/<second
>>>> set of two characters of gfid>/<gfid> on each brick)
>>>> * disable readdirplus in entire stack (maybe you can use a tmp mount
>>>> for this) [1]
>>>> * stat src and dst on a mount point with readdirplus disabled.
>>>> * Now you'll see two directories src and dst on mountpoint. You  can
>>>> copy the contents of both into a new directory
>>>>
>>>> [1] https://lists.gluster.org/pipermail/gluster-users/2017-
>>>> March/030148.html
>>>>
>>>>
>>>>>
>>>>> Any suggestions is appreciated.
>>>>> Many Thanks
>>>>>
>>>>> Best regards
>>>>> Yu
>>>>>
>>>>> _______________________________________________
>>>>> Gluster-users mailing list
>>>>> Gluster-users at gluster.org
>>>>> https://lists.gluster.org/mailman/listinfo/gluster-users
>>>>>
>>>>
>>>>
>>
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.gluster.org/pipermail/gluster-users/attachments/20180905/f2062c84/attachment-0001.html>