[Gluster-users] nfs-ganesha locking problems
Bernhard Dübi
1linuxengineer at gmail.com
Mon Oct 2 20:45:10 UTC 2017
Hi Soumya,
what I can say so far:
it is working on a standalone system but not on the clustered system
from reading the ganesha wiki I have the impression that it is
possible to change the log level without restarting ganesha. I was
playing with dbus-send but so far was unsuccessful. if you can help me
with that, this would be great.
here some details about the tested machines. the nfs client was always the same
THIS SYSTEM IS WORKING
root at chvirnfstst01 ~]# uname -a
Linux chvirnfstst01 3.10.0-693.2.2.el7.x86_64 #1 SMP Tue Sep 12
22:26:13 UTC 2017 x86_64 x86_64 x86_64 GNU/Linux
[root at chvirnfstst01 ~]# cd /etc/
[root at chvirnfstst01 etc]# ls -ld *rel*
-rw-r--r--. 1 root root 38 Aug 30 17:53 centos-release
-rw-r--r--. 1 root root 51 Aug 30 17:53 centos-release-upstream
-rw-r--r--. 1 root root 393 Aug 30 17:53 os-release
drwxr-xr-x. 2 root root 78 Oct 1 15:52 prelink.conf.d
lrwxrwxrwx. 1 root root 14 Oct 1 15:51 redhat-release -> centos-release
lrwxrwxrwx. 1 root root 14 Oct 1 15:51 system-release -> centos-release
-rw-r--r--. 1 root root 23 Aug 30 17:53 system-release-cpe
[root at chvirnfstst01 etc]# cat centos-release
CentOS Linux release 7.4.1708 (Core)
[root at chvirnfstst01 etc]# cat os-release
NAME="CentOS Linux"
VERSION="7 (Core)"
ID="centos"
ID_LIKE="rhel fedora"
VERSION_ID="7"
PRETTY_NAME="CentOS Linux 7 (Core)"
ANSI_COLOR="0;31"
CPE_NAME="cpe:/o:centos:centos:7"
HOME_URL="https://www.centos.org/"
BUG_REPORT_URL="https://bugs.centos.org/"
CENTOS_MANTISBT_PROJECT="CentOS-7"
CENTOS_MANTISBT_PROJECT_VERSION="7"
REDHAT_SUPPORT_PRODUCT="centos"
REDHAT_SUPPORT_PRODUCT_VERSION="7"
[root at chvirnfstst01 etc]#
[root at chvirnfstst01 etc]# rpm -qa | grep ganesha | sort
nfs-ganesha-2.3.3-1.el7.x86_64
nfs-ganesha-gluster-2.3.3-1.el7.x86_64
[root at chvirnfstst01 etc]#
[root at chvirnfstst01 etc]# rpm -qa | grep gluster | sort
centos-release-gluster38-1.0-1.el7.centos.noarch
glusterfs-3.8.15-2.el7.x86_64
glusterfs-api-3.8.15-2.el7.x86_64
glusterfs-client-xlators-3.8.15-2.el7.x86_64
glusterfs-libs-3.8.15-2.el7.x86_64
nfs-ganesha-gluster-2.3.3-1.el7.x86_64
[root at chvirnfstst01 etc]#
[root at chvirnfstst01 etc]# cat /etc/ganesha/ganesha.conf
EXPORT
{
# Export Id (mandatory, each EXPORT must have a unique Export_Id)
Export_Id = 77;
# Exported path (mandatory)
Path = /ora_dump;
# Pseudo Path (required for NFS v4)
Pseudo = /ora_dump;
# Exporting FSAL
FSAL {
Name = GLUSTER;
Hostname = 10.30.28.43;
Volume = ora_dump;
}
CLIENT {
# Oracle Servers
Clients =
10.30.29.125,10.30.28.25,10.30.28.64,10.30.29.123,10.30.28.21,10.30.28.81,10.30.29.124,10.30.28.82,10.30.29.111;
Access_Type = RW;
}
}
EXPORT
{
# Export Id (mandatory, each EXPORT must have a unique Export_Id)
Export_Id = 88;
# Exported path (mandatory)
Path = /chzrhcvtprd04;
# Pseudo Path (required for NFS v4)
Pseudo = /chzrhcvtprd04;
# Exporting FSAL
FSAL {
Name = GLUSTER;
Hostname = 10.30.28.43;
Volume = chzrhcvtprd04;
}
CLIENT {
# everybody
Clients = 10.30.0.0/16,10.40.0.0/16,10.50.0.0/16;
Access_Type = RW;
}
}
[root at chvirnfstst01 etc]#
THIS SYSTEM IS NOT WORKING
you can find the details about the shared volume in my previous mail
[root at chvirnfsprd12 ~]# uname -a
Linux chvirnfsprd12 3.10.0-514.26.2.el7.x86_64 #1 SMP Tue Jul 4
15:04:05 UTC 2017 x86_64 x86_64 x86_64 GNU/Linux
[root at chvirnfsprd12 ~]# cd /etc/
[root at chvirnfsprd12 etc]# ls -ld *rel*
-rw-r--r--. 1 root root 38 Nov 29 2016 centos-release
-rw-r--r--. 1 root root 51 Nov 29 2016 centos-release-upstream
-rw-r--r--. 1 root root 393 Nov 29 2016 os-release
drwxr-xr-x. 2 root root 78 Sep 2 08:54 prelink.conf.d
lrwxrwxrwx. 1 root root 14 Sep 2 08:53 redhat-release -> centos-release
lrwxrwxrwx. 1 root root 14 Sep 2 08:53 system-release -> centos-release
-rw-r--r--. 1 root root 23 Nov 29 2016 system-release-cpe
[root at chvirnfsprd12 etc]# cat centos-release
CentOS Linux release 7.3.1611 (Core)
[root at chvirnfsprd12 etc]# cat os-release
NAME="CentOS Linux"
VERSION="7 (Core)"
ID="centos"
ID_LIKE="rhel fedora"
VERSION_ID="7"
PRETTY_NAME="CentOS Linux 7 (Core)"
ANSI_COLOR="0;31"
CPE_NAME="cpe:/o:centos:centos:7"
HOME_URL="https://www.centos.org/"
BUG_REPORT_URL="https://bugs.centos.org/"
CENTOS_MANTISBT_PROJECT="CentOS-7"
CENTOS_MANTISBT_PROJECT_VERSION="7"
REDHAT_SUPPORT_PRODUCT="centos"
REDHAT_SUPPORT_PRODUCT_VERSION="7"
[root at chvirnfsprd12 etc]# rpm -qa | grep ganesha | sort
glusterfs-ganesha-3.8.15-2.el7.x86_64
nfs-ganesha-2.3.3-1.el7.x86_64
nfs-ganesha-gluster-2.3.3-1.el7.x86_64
[root at chvirnfsprd12 etc]# rpm -qa | grep gluster | sort
centos-release-gluster38-1.0-1.el7.centos.noarch
glusterfs-3.8.15-2.el7.x86_64
glusterfs-api-3.8.15-2.el7.x86_64
glusterfs-cli-3.8.15-2.el7.x86_64
glusterfs-client-xlators-3.8.15-2.el7.x86_64
glusterfs-fuse-3.8.15-2.el7.x86_64
glusterfs-ganesha-3.8.15-2.el7.x86_64
glusterfs-libs-3.8.15-2.el7.x86_64
glusterfs-server-3.8.15-2.el7.x86_64
nfs-ganesha-gluster-2.3.3-1.el7.x86_64
[root at chvirnfsprd12 etc]# cat /etc/ganesha/ganesha.conf
NFS_Core_Param {
#Use supplied name other than IP In NSM operations
NSM_Use_Caller_Name = true;
#Copy lock states into "/var/lib/nfs/ganesha" dir
Clustered = true;
#Use a non-privileged port for RQuota
Rquota_Port = 875;
}
%include /etc/ganesha/exports/ora_dump.conf
%include /etc/ganesha/exports/chzrhcvtprd04.conf
[root at chvirnfsprd12 etc]# cat /etc/ganesha/exports/ora_dump.conf
EXPORT
{
# Export Id (mandatory, each EXPORT must have a unique Export_Id)
Export_Id = 77;
# Exported path (mandatory)
Path = /ora_dump;
# Pseudo Path (required for NFS v4)
Pseudo = /ora_dump;
# Exporting FSAL
FSAL {
Name = GLUSTER;
Hostname = 10.30.28.43;
Volume = ora_dump;
}
CLIENT {
# Oracle Servers
Clients =
10.30.29.125,10.30.28.25,10.30.28.64,10.30.29.123,10.30.28.21,10.30.28.81,10.30.29.124,10.30.28.82,10.30.29.111;
Access_Type = RW;
}
}
[root at chvirnfsprd12 etc]#
[root at chvirnfsprd12 log]# grep '^\[2017-10-02 [12]' ganesha-gfapi.log
[2017-10-02 18:49:12.855174] I [MSGID: 104043]
[glfs-mgmt.c:565:glfs_mgmt_getspec_cbk] 0-gfapi: No change in volfile,
continuing
[2017-10-02 18:49:12.862051] I [MSGID: 104043]
[glfs-mgmt.c:565:glfs_mgmt_getspec_cbk] 0-gfapi: No change in volfile,
continuing
[2017-10-02 18:50:05.789064] E [socket.c:2309:socket_connect_finish]
0-gfapi: connection to 10.30.28.43:24007 failed (Connection refused)
[2017-10-02 18:50:06.519516] E [socket.c:2309:socket_connect_finish]
0-gfapi: connection to 10.30.28.43:24007 failed (Connection refused)
[2017-10-02 18:49:13.076308] I [MSGID: 104043]
[glfs-mgmt.c:565:glfs_mgmt_getspec_cbk] 0-gfapi: No change in volfile,
continuing
[2017-10-02 18:50:22.694873] I [MSGID: 104043]
[glfs-mgmt.c:565:glfs_mgmt_getspec_cbk] 0-gfapi: No change in volfile,
continuing
[2017-10-02 19:17:54.733967] E [MSGID: 109040]
[dht-helper.c:1198:dht_migration_complete_check_task] 0-ora_dump-dht:
<gfid:6d0de5f7-293c-4075-b3e2-f85f7902456e>: failed to lookup the file
on ora_dump-dht [Stale file handle]
[2017-10-02 19:17:54.774694] E [MSGID: 109040]
[dht-helper.c:1198:dht_migration_complete_check_task] 0-ora_dump-dht:
<gfid:6b8da639-5a48-42b5-aa9d-a25850eee8bd>: failed to lookup the file
on ora_dump-dht [Stale file handle]
[2017-10-02 19:17:54.825167] E [MSGID: 109040]
[dht-helper.c:1198:dht_migration_complete_check_task] 0-ora_dump-dht:
<gfid:a5ab5eb4-f6fa-419a-87fc-1180f2ef324a>: failed to lookup the file
on ora_dump-dht [Stale file handle]
[2017-10-02 19:17:54.877278] E [MSGID: 109040]
[dht-helper.c:1198:dht_migration_complete_check_task] 0-ora_dump-dht:
<gfid:9d12bd00-75ba-47d5-b513-5cfd9926b061>: failed to lookup the file
on ora_dump-dht [Stale file handle]
[2017-10-02 19:17:54.938806] E [MSGID: 109040]
[dht-helper.c:1198:dht_migration_complete_check_task] 0-ora_dump-dht:
<gfid:2ca21df0-d8bb-4484-b386-438d99f9be0d>: failed to lookup the file
on ora_dump-dht [Stale file handle]
[2017-10-02 19:19:00.865158] E [MSGID: 109040]
[dht-helper.c:1198:dht_migration_complete_check_task] 0-ora_dump-dht:
<gfid:32634781-2a61-45d1-83b9-909fe96ec0ad>: failed to lookup the file
on ora_dump-dht [Stale file handle]
[2017-10-02 19:19:05.619545] E [MSGID: 109040]
[dht-helper.c:1198:dht_migration_complete_check_task] 0-ora_dump-dht:
<gfid:ed9dec37-081a-4062-b626-779f9ce494de>: failed to lookup the file
on ora_dump-dht [Stale file handle]
[2017-10-02 19:19:08.176730] E [MSGID: 109040]
[dht-helper.c:1198:dht_migration_complete_check_task] 0-ora_dump-dht:
<gfid:fb38de03-11c7-4d77-b262-933f5a6e9289>: failed to lookup the file
on ora_dump-dht [Stale file handle]
[2017-10-02 19:19:09.456716] E [MSGID: 109040]
[dht-helper.c:1198:dht_migration_complete_check_task] 0-ora_dump-dht:
<gfid:b252ffdd-b62e-4470-a885-75df67ba4bf5>: failed to lookup the file
on ora_dump-dht [Stale file handle]
[2017-10-02 19:19:10.730761] E [MSGID: 109040]
[dht-helper.c:1198:dht_migration_complete_check_task] 0-ora_dump-dht:
<gfid:10ee40e0-4807-40fe-abdc-590a7ee86de5>: failed to lookup the file
on ora_dump-dht [Stale file handle]
[2017-10-02 19:19:11.425541] E [MSGID: 109040]
[dht-helper.c:1198:dht_migration_complete_check_task] 0-ora_dump-dht:
<gfid:d6f4cffd-eda7-4188-8af1-60eaa67a3724>: failed to lookup the file
on ora_dump-dht [Stale file handle]
[root at chvirnfsprd12 log]# grep '^02/10/2017 21:1[7-9]' ganesha.log
02/10/2017 21:17:43 : epoch 59cbce8c : chvirnfsprd12 :
ganesha.nfsd-8516[work-6] mnt_Mnt :NFS3 :EVENT :MOUNT: Performance
warning: Export entry is not cached
02/10/2017 21:17:54 : epoch 59cbce8c : chvirnfsprd12 :
ganesha.nfsd-8516[work-13] nsm_monitor :NLM :CRIT :Can not monitor
chglboraprd10.fpprod.corp SM_MON status 1
02/10/2017 21:17:54 : epoch 59cbce8c : chvirnfsprd12 :
ganesha.nfsd-8516[work-16] nsm_monitor :NLM :CRIT :Can not monitor
chglboraprd10.fpprod.corp SM_MON status 1
02/10/2017 21:17:54 : epoch 59cbce8c : chvirnfsprd12 :
ganesha.nfsd-8516[work-1] nsm_monitor :NLM :CRIT :Can not monitor
chglboraprd10.fpprod.corp SM_MON status 1
02/10/2017 21:17:54 : epoch 59cbce8c : chvirnfsprd12 :
ganesha.nfsd-8516[work-2] nsm_monitor :NLM :CRIT :Can not monitor
chglboraprd10.fpprod.corp SM_MON status 1
02/10/2017 21:17:54 : epoch 59cbce8c : chvirnfsprd12 :
ganesha.nfsd-8516[work-12] nsm_monitor :NLM :CRIT :Can not monitor
chglboraprd10.fpprod.corp SM_MON status 1
02/10/2017 21:17:54 : epoch 59cbce8c : chvirnfsprd12 :
ganesha.nfsd-8516[work-16] nsm_monitor :NLM :CRIT :Can not monitor
chglboraprd10.fpprod.corp SM_MON status 1
02/10/2017 21:19:05 : epoch 59cbce8c : chvirnfsprd12 :
ganesha.nfsd-8516[work-2] nsm_monitor :NLM :CRIT :Can not monitor
chglboraprd10.fpprod.corp SM_MON status 1
02/10/2017 21:19:08 : epoch 59cbce8c : chvirnfsprd12 :
ganesha.nfsd-8516[work-6] nsm_monitor :NLM :CRIT :Can not monitor
chglboraprd10.fpprod.corp SM_MON status 1
02/10/2017 21:19:09 : epoch 59cbce8c : chvirnfsprd12 :
ganesha.nfsd-8516[work-5] nsm_monitor :NLM :CRIT :Can not monitor
chglboraprd10.fpprod.corp SM_MON status 1
02/10/2017 21:19:10 : epoch 59cbce8c : chvirnfsprd12 :
ganesha.nfsd-8516[work-2] nsm_monitor :NLM :CRIT :Can not monitor
chglboraprd10.fpprod.corp SM_MON status 1
02/10/2017 21:19:11 : epoch 59cbce8c : chvirnfsprd12 :
ganesha.nfsd-8516[work-11] nsm_monitor :NLM :CRIT :Can not monitor
chglboraprd10.fpprod.corp SM_MON status 1
02/10/2017 21:19:15 : epoch 59cbce8c : chvirnfsprd12 :
ganesha.nfsd-8516[work-14] nsm_monitor :NLM :CRIT :Can not monitor
chglboraprd10.fpprod.corp SM_MON status 1
More information about the Gluster-users
mailing list