[Gluster-users] glusterfsd process thrashing CPU

Tue Nov 18 08:00:43 UTC 2014

On 18 November 2014 17:46, Franco Broi <franco.broi at iongeo.com> wrote:
>
> Try strace -Ff -e file -p 'glusterfsd pid'

Thanks, Attached
-------------- next part --------------
Process 27115 attached with 25 threads - interrupt to quit
[pid 27122] stat("/mnt/gluster-brick1/datastore", {st_mode=S_IFDIR|0755, st_size=4, ...}) = 0
[pid 11840] lstat("/mnt/gluster-brick1/datastore/", {st_mode=S_IFDIR|0755, st_size=4, ...}) = 0
[pid 11840] lgetxattr("/mnt/gluster-brick1/datastore/", "system.posix_acl_default", 0x0, 0) = -1 EOPNOTSUPP (Operation not supported)
[pid 11840] lgetxattr("/mnt/gluster-brick1/datastore/", "system.posix_acl_access", 0x0, 0) = -1 EOPNOTSUPP (Operation not supported)
[pid 11840] lgetxattr("/mnt/gluster-brick1/datastore/", "trusted.glusterfs.dht" <unfinished ...>
[pid 29198] lstat("/mnt/gluster-brick1/datastore/", {st_mode=S_IFDIR|0755, st_size=4, ...}) = 0
[pid 29198] lgetxattr("/mnt/gluster-brick1/datastore/", "system.posix_acl_default", 0x0, 0) = -1 EOPNOTSUPP (Operation not supported)
[pid 29198] lgetxattr("/mnt/gluster-brick1/datastore/", "system.posix_acl_access", 0x0, 0) = -1 EOPNOTSUPP (Operation not supported)
[pid 29198] lgetxattr("/mnt/gluster-brick1/datastore/", "trusted.glusterfs.dht" <unfinished ...>
[pid 29197] lstat("/mnt/gluster-brick1/datastore/", {st_mode=S_IFDIR|0755, st_size=4, ...}) = 0
[pid 29197] lgetxattr("/mnt/gluster-brick1/datastore/", "system.posix_acl_default", 0x0, 0) = -1 EOPNOTSUPP (Operation not supported)
[pid 29197] lgetxattr("/mnt/gluster-brick1/datastore/", "system.posix_acl_access", 0x0, 0) = -1 EOPNOTSUPP (Operation not supported)
[pid 29197] lgetxattr("/mnt/gluster-brick1/datastore/", "trusted.glusterfs.dht" <unfinished ...>
[pid 11840] <... lgetxattr resumed> , 0x0, 0) = 16
[pid 11840] lgetxattr("/mnt/gluster-brick1/datastore/", "trusted.glusterfs.dht", "\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\xff\xff\xff\xff", 16) = 16
[pid 11840] lgetxattr("/mnt/gluster-brick1/datastore/", "missing-gfid-ESTALE", 0x0, 0) = -1 EOPNOTSUPP (Operation not supported)
[pid 11840] lgetxattr("/mnt/gluster-brick1/datastore/", "trusted.afr.datastore1-client-0", 0x0, 0) = -1 ENODATA (No data available)
[pid 11840] lgetxattr("/mnt/gluster-brick1/datastore/", "trusted.afr.datastore1-client-1", 0x0, 0) = -1 ENODATA (No data available)
[pid 11840] llistxattr("/mnt/gluster-brick1/datastore/", (nil), 0) = 63
[pid 11840] llistxattr("/mnt/gluster-brick1/datastore/", 0x7feae3cfda10, 63) = 63
[pid 29198] <... lgetxattr resumed> , 0x0, 0) = 16
[pid 29198] lgetxattr("/mnt/gluster-brick1/datastore/", "trusted.glusterfs.dht", "\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\xff\xff\xff\xff", 16) = 16
[pid 29198] lgetxattr("/mnt/gluster-brick1/datastore/", "missing-gfid-ESTALE", 0x0, 0) = -1 EOPNOTSUPP (Operation not supported)
[pid 29198] lgetxattr("/mnt/gluster-brick1/datastore/", "trusted.afr.datastore1-client-0", 0x0, 0) = -1 ENODATA (No data available)
[pid 29198] lgetxattr("/mnt/gluster-brick1/datastore/", "trusted.afr.datastore1-client-1" <unfinished ...>
[pid 29197] <... lgetxattr resumed> , 0x0, 0) = 16
[pid 29198] <... lgetxattr resumed> , 0x0, 0) = -1 ENODATA (No data available)
[pid 29197] lgetxattr("/mnt/gluster-brick1/datastore/", "trusted.glusterfs.dht" <unfinished ...>
[pid 29198] llistxattr("/mnt/gluster-brick1/datastore/", (nil), 0) = 63
[pid 29198] llistxattr("/mnt/gluster-brick1/datastore/" <unfinished ...>
[pid 29197] <... lgetxattr resumed> , "\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\xff\xff\xff\xff", 16) = 16
[pid 29198] <... llistxattr resumed> , 0x7feae3ffea10, 63) = 63
[pid 29197] lgetxattr("/mnt/gluster-brick1/datastore/", "missing-gfid-ESTALE", 0x0, 0) = -1 EOPNOTSUPP (Operation not supported)
[pid 29197] lgetxattr("/mnt/gluster-brick1/datastore/", "trusted.afr.datastore1-client-0", 0x0, 0) = -1 ENODATA (No data available)
[pid 29197] lgetxattr("/mnt/gluster-brick1/datastore/", "trusted.afr.datastore1-client-1", 0x0, 0) = -1 ENODATA (No data available)
[pid 29197] llistxattr("/mnt/gluster-brick1/datastore/", (nil), 0) = 63
[pid 29197] llistxattr("/mnt/gluster-brick1/datastore/", 0x7feaf0487a10, 63) = 63
[pid 11846] lstat("/mnt/gluster-brick1/datastore/images", {st_mode=S_IFDIR|0755, st_size=27, ...}) = 0
[pid 11846] lgetxattr("/mnt/gluster-brick1/datastore/images", "trusted.gfid" <unfinished ...>
[pid 11844] lstat("/mnt/gluster-brick1/datastore/images", {st_mode=S_IFDIR|0755, st_size=27, ...}) = 0
[pid 11844] lgetxattr("/mnt/gluster-brick1/datastore/images", "trusted.gfid" <unfinished ...>
[pid 11845] lstat("/mnt/gluster-brick1/datastore/images", {st_mode=S_IFDIR|0755, st_size=27, ...}) = 0
[pid 11845] lgetxattr("/mnt/gluster-brick1/datastore/images", "trusted.gfid" <unfinished ...>
[pid 11844] <... lgetxattr resumed> , "\xbe\x7fIlH\xb0C\xbd\xaaA=BJ6\xca\xb1", 16) = 16
[pid 11846] <... lgetxattr resumed> , "\xbe\x7fIlH\xb0C\xbd\xaaA=BJ6\xca\xb1", 16) = 16
[pid 11845] <... lgetxattr resumed> , "\xbe\x7fIlH\xb0C\xbd\xaaA=BJ6\xca\xb1", 16) = 16
[pid 11846] lgetxattr("/mnt/gluster-brick1/datastore/images", "system.posix_acl_default" <unfinished ...>
[pid 11845] lgetxattr("/mnt/gluster-brick1/datastore/images", "system.posix_acl_default" <unfinished ...>
[pid 11846] <... lgetxattr resumed> , 0x0, 0) = -1 EOPNOTSUPP (Operation not supported)
[pid 11845] <... lgetxattr resumed> , 0x0, 0) = -1 EOPNOTSUPP (Operation not supported)
[pid 11846] lgetxattr("/mnt/gluster-brick1/datastore/images", "system.posix_acl_access" <unfinished ...>
[pid 11845] lgetxattr("/mnt/gluster-brick1/datastore/images", "system.posix_acl_access" <unfinished ...>
[pid 11846] <... lgetxattr resumed> , 0x0, 0) = -1 EOPNOTSUPP (Operation not supported)
[pid 11845] <... lgetxattr resumed> , 0x0, 0) = -1 EOPNOTSUPP (Operation not supported)
[pid 11846] lgetxattr("/mnt/gluster-brick1/datastore/images", "trusted.glusterfs.dht" <unfinished ...>
[pid 11845] lgetxattr("/mnt/gluster-brick1/datastore/images", "trusted.glusterfs.dht" <unfinished ...>
[pid 11844] lgetxattr("/mnt/gluster-brick1/datastore/images", "system.posix_acl_default" <unfinished ...>
[pid 11846] <... lgetxattr resumed> , 0x0, 0) = 16
[pid 11845] <... lgetxattr resumed> , 0x0, 0) = 16
[pid 11846] lgetxattr("/mnt/gluster-brick1/datastore/images", "trusted.glusterfs.dht" <unfinished ...>
[pid 11845] lgetxattr("/mnt/gluster-brick1/datastore/images", "trusted.glusterfs.dht" <unfinished ...>
[pid 11844] <... lgetxattr resumed> , 0x0, 0) = -1 EOPNOTSUPP (Operation not supported)
[pid 11846] <... lgetxattr resumed> , "\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\xff\xff\xff\xff", 16) = 16
[pid 11844] lgetxattr("/mnt/gluster-brick1/datastore/images", "system.posix_acl_access" <unfinished ...>
[pid 11846] lgetxattr("/mnt/gluster-brick1/datastore/images", "missing-gfid-ESTALE" <unfinished ...>
[pid 11845] <... lgetxattr resumed> , "\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\xff\xff\xff\xff", 16) = 16
[pid 11846] <... lgetxattr resumed> , 0x0, 0) = -1 EOPNOTSUPP (Operation not supported)
[pid 11845] lgetxattr("/mnt/gluster-brick1/datastore/images", "missing-gfid-ESTALE" <unfinished ...>
[pid 11846] lgetxattr("/mnt/gluster-brick1/datastore/images", "trusted.afr.datastore1-client-0" <unfinished ...>
[pid 11845] <... lgetxattr resumed> , 0x0, 0) = -1 EOPNOTSUPP (Operation not supported)
[pid 11844] <... lgetxattr resumed> , 0x0, 0) = -1 EOPNOTSUPP (Operation not supported)
[pid 11846] <... lgetxattr resumed> , 0x0, 0) = -1 ENODATA (No data available)
[pid 11845] lgetxattr("/mnt/gluster-brick1/datastore/images", "trusted.afr.datastore1-client-0" <unfinished ...>
[pid 11846] lgetxattr("/mnt/gluster-brick1/datastore/images", "trusted.afr.datastore1-client-1" <unfinished ...>
[pid 11845] <... lgetxattr resumed> , 0x0, 0) = -1 ENODATA (No data available)
[pid 11844] lgetxattr("/mnt/gluster-brick1/datastore/images", "trusted.glusterfs.dht" <unfinished ...>
[pid 11846] <... lgetxattr resumed> , 0x0, 0) = -1 ENODATA (No data available)
[pid 11845] lgetxattr("/mnt/gluster-brick1/datastore/images", "trusted.afr.datastore1-client-1" <unfinished ...>
[pid 11846] llistxattr("/mnt/gluster-brick1/datastore/images" <unfinished ...>
[pid 11845] <... lgetxattr resumed> , 0x0, 0) = -1 ENODATA (No data available)
[pid 11844] <... lgetxattr resumed> , 0x0, 0) = 16
[pid 11846] <... llistxattr resumed> , (nil), 0) = 35
[pid 11845] llistxattr("/mnt/gluster-brick1/datastore/images" <unfinished ...>
[pid 11846] llistxattr("/mnt/gluster-brick1/datastore/images" <unfinished ...>
[pid 11845] <... llistxattr resumed> , (nil), 0) = 35
[pid 11844] lgetxattr("/mnt/gluster-brick1/datastore/images", "trusted.glusterfs.dht" <unfinished ...>
[pid 11846] <... llistxattr resumed> , 0x7feae36f79d0, 35) = 35
[pid 11845] llistxattr("/mnt/gluster-brick1/datastore/images" <unfinished ...>
[pid 11846] lstat("/mnt/gluster-brick1/datastore",  <unfinished ...>
[pid 11845] <... llistxattr resumed> , 0x7feae37f89d0, 35) = 35
[pid 11846] <... lstat resumed> {st_mode=S_IFDIR|0755, st_size=4, ...}) = 0
[pid 11845] lstat("/mnt/gluster-brick1/datastore",  <unfinished ...>
[pid 11844] <... lgetxattr resumed> , "\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\xff\xff\xff\xff", 16) = 16
[pid 11845] <... lstat resumed> {st_mode=S_IFDIR|0755, st_size=4, ...}) = 0
[pid 11844] lgetxattr("/mnt/gluster-brick1/datastore/images", "missing-gfid-ESTALE", 0x0, 0) = -1 EOPNOTSUPP (Operation not supported)
[pid 11844] lgetxattr("/mnt/gluster-brick1/datastore/images", "trusted.afr.datastore1-client-0", 0x0, 0) = -1 ENODATA (No data available)
[pid 11844] lgetxattr("/mnt/gluster-brick1/datastore/images", "trusted.afr.datastore1-client-1", 0x0, 0) = -1 ENODATA (No data available)
[pid 11844] llistxattr("/mnt/gluster-brick1/datastore/images", (nil), 0) = 35
[pid 11844] llistxattr("/mnt/gluster-brick1/datastore/images", 0x7feae38f99d0, 35) = 35
[pid 11844] lstat("/mnt/gluster-brick1/datastore", {st_mode=S_IFDIR|0755, st_size=4, ...}) = 0
[pid 11846] lstat("/mnt/gluster-brick1/datastore/", {st_mode=S_IFDIR|0755, st_size=4, ...}) = 0
[pid 11846] open("/mnt/gluster-brick1/datastore/", O_RDONLY|O_NONBLOCK|O_DIRECTORY|O_CLOEXEC) = 47
[pid 11847] lstat("/mnt/gluster-brick1/datastore/", {st_mode=S_IFDIR|0755, st_size=4, ...}) = 0
[pid 11847] open("/mnt/gluster-brick1/datastore/", O_RDONLY|O_NONBLOCK|O_DIRECTORY|O_CLOEXEC) = 57
[pid 11848] lstat("/mnt/gluster-brick1/datastore/", {st_mode=S_IFDIR|0755, st_size=4, ...}) = 0
[pid 11848] open("/mnt/gluster-brick1/datastore/", O_RDONLY|O_NONBLOCK|O_DIRECTORY|O_CLOEXEC) = 58
[pid 28552] lstat("/mnt/gluster-brick1/datastore/", {st_mode=S_IFDIR|0755, st_size=4, ...}) = 0
[pid 28552] statfs("/mnt/gluster-brick1/datastore/", {f_type=0x2fc12fc1, f_bsize=131072, f_blocks=22450028, f_bfree=15434979, f_bavail=15434979, f_files=3951355249, f_ffree=3951354864, f_fsid={700673035, 1354725}, f_namelen=255, f_frsize=131072}) = 0
[pid 28552] stat("/mnt/gluster-brick1/datastore/", {st_mode=S_IFDIR|0755, st_size=4, ...}) = 0
[pid 27990] lstat("/mnt/gluster-brick1/datastore/", {st_mode=S_IFDIR|0755, st_size=4, ...}) = 0
[pid 27990] statfs("/mnt/gluster-brick1/datastore/", {f_type=0x2fc12fc1, f_bsize=131072, f_blocks=22450028, f_bfree=15434979, f_bavail=15434979, f_files=3951355249, f_ffree=3951354864, f_fsid={700673035, 1354725}, f_namelen=255, f_frsize=131072}) = 0
[pid 27990] stat("/mnt/gluster-brick1/datastore/", {st_mode=S_IFDIR|0755, st_size=4, ...}) = 0
[pid 29198] lstat("/mnt/gluster-brick1/datastore/", {st_mode=S_IFDIR|0755, st_size=4, ...}) = 0
[pid 29198] statfs("/mnt/gluster-brick1/datastore/", {f_type=0x2fc12fc1, f_bsize=131072, f_blocks=22450028, f_bfree=15434979, f_bavail=15434979, f_files=3951355249, f_ffree=3951354864, f_fsid={700673035, 1354725}, f_namelen=255, f_frsize=131072}) = 0
[pid 29198] stat("/mnt/gluster-brick1/datastore/", {st_mode=S_IFDIR|0755, st_size=4, ...}) = 0
[pid 11848] lstat("/mnt/gluster-brick1/datastore/", {st_mode=S_IFDIR|0755, st_size=4, ...}) = 0
[pid 11848] open("/mnt/gluster-brick1/datastore/", O_RDONLY|O_NONBLOCK|O_DIRECTORY|O_CLOEXEC) = 47
[pid 11841] lstat("/mnt/gluster-brick1/datastore/", {st_mode=S_IFDIR|0755, st_size=4, ...}) = 0
[pid 11841] open("/mnt/gluster-brick1/datastore/", O_RDONLY|O_NONBLOCK|O_DIRECTORY|O_CLOEXEC) = 57
[pid 11846] lstat("/mnt/gluster-brick1/datastore/", {st_mode=S_IFDIR|0755, st_size=4, ...}) = 0
[pid 11846] open("/mnt/gluster-brick1/datastore/", O_RDONLY|O_NONBLOCK|O_DIRECTORY|O_CLOEXEC) = 47
[pid 27121] lstat("/mnt/gluster-brick1/datastore/", {st_mode=S_IFDIR|0755, st_size=4, ...}) = 0
[pid 27121] statfs("/mnt/gluster-brick1/datastore/", {f_type=0x2fc12fc1, f_bsize=131072, f_blocks=22450028, f_bfree=15434979, f_bavail=15434979, f_files=3951355249, f_ffree=3951354864, f_fsid={700673035, 1354725}, f_namelen=255, f_frsize=131072}) = 0
[pid 27121] stat("/mnt/gluster-brick1/datastore/", {st_mode=S_IFDIR|0755, st_size=4, ...}) = 0
[pid 17924] lstat("/mnt/gluster-brick1/datastore/", {st_mode=S_IFDIR|0755, st_size=4, ...}) = 0
[pid 17924] statfs("/mnt/gluster-brick1/datastore/", {f_type=0x2fc12fc1, f_bsize=131072, f_blocks=22450028, f_bfree=15434979, f_bavail=15434979, f_files=3951355249, f_ffree=3951354864, f_fsid={700673035, 1354725}, f_namelen=255, f_frsize=131072}) = 0
[pid 17924] stat("/mnt/gluster-brick1/datastore/", {st_mode=S_IFDIR|0755, st_size=4, ...}) = 0
[pid 11843] lstat("/mnt/gluster-brick1/datastore/", {st_mode=S_IFDIR|0755, st_size=4, ...}) = 0
[pid 11843] statfs("/mnt/gluster-brick1/datastore/", {f_type=0x2fc12fc1, f_bsize=131072, f_blocks=22450028, f_bfree=15434979, f_bavail=15434979, f_files=3951355249, f_ffree=3951354864, f_fsid={700673035, 1354725}, f_namelen=255, f_frsize=131072}) = 0
[pid 11843] stat("/mnt/gluster-brick1/datastore/", {st_mode=S_IFDIR|0755, st_size=4, ...}) = 0
Process 27115 detached
Process 27116 detached
Process 27117 detached
Process 27118 detached
Process 27119 detached
Process 27120 detached
Process 27121 detached
Process 27122 detached
Process 27123 detached
Process 27124 detached
Process 27990 detached
Process 28552 detached
Process 29197 detached
Process 29198 detached
Process 30325 detached
Process 11840 detached
Process 11841 detached
Process 11842 detached
Process 11843 detached
Process 11844 detached
Process 11845 detached
Process 11846 detached
Process 11847 detached
Process 11848 detached
Process 17924 detached