<html xmlns:v="urn:schemas-microsoft-com:vml" xmlns:o="urn:schemas-microsoft-com:office:office" xmlns:w="urn:schemas-microsoft-com:office:word" xmlns:m="http://schemas.microsoft.com/office/2004/12/omml" xmlns="http://www.w3.org/TR/REC-html40">
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
<meta name="Generator" content="Microsoft Word 15 (filtered medium)">
<style><!--
/* Font Definitions */
@font-face
        {font-family:宋体;
        panose-1:2 1 6 0 3 1 1 1 1 1;}
@font-face
        {font-family:"Cambria Math";
        panose-1:2 4 5 3 5 4 6 3 2 4;}
@font-face
        {font-family:DengXian;
        panose-1:2 1 6 0 3 1 1 1 1 1;}
@font-face
        {font-family:Calibri;
        panose-1:2 15 5 2 2 2 4 3 2 4;}
@font-face
        {font-family:"\@宋体";
        panose-1:2 1 6 0 3 1 1 1 1 1;}
@font-face
        {font-family:"\@等线";
        panose-1:2 1 6 0 3 1 1 1 1 1;}
/* Style Definitions */
p.MsoNormal, li.MsoNormal, div.MsoNormal
        {margin:0cm;
        margin-bottom:.0001pt;
        font-size:12.0pt;
        font-family:宋体;}
a:link, span.MsoHyperlink
        {mso-style-priority:99;
        color:blue;
        text-decoration:underline;}
a:visited, span.MsoHyperlinkFollowed
        {mso-style-priority:99;
        color:purple;
        text-decoration:underline;}
p.MsoListParagraph, li.MsoListParagraph, div.MsoListParagraph
        {mso-style-priority:34;
        margin:0cm;
        margin-bottom:.0001pt;
        text-indent:21.0pt;
        font-size:12.0pt;
        font-family:宋体;}
p.msonormal0, li.msonormal0, div.msonormal0
        {mso-style-name:msonormal;
        mso-margin-top-alt:auto;
        margin-right:0cm;
        mso-margin-bottom-alt:auto;
        margin-left:0cm;
        font-size:12.0pt;
        font-family:宋体;}
span.EmailStyle19
        {mso-style-type:personal;
        font-family:"Calibri",sans-serif;
        color:windowtext;}
span.EmailStyle20
        {mso-style-type:personal;
        font-family:"Calibri",sans-serif;
        color:windowtext;}
span.EmailStyle21
        {mso-style-type:personal;
        font-family:DengXian;
        color:windowtext;}
span.EmailStyle23
        {mso-style-type:personal-reply;
        font-family:DengXian;
        color:windowtext;}
.MsoChpDefault
        {mso-style-type:export-only;
        font-size:10.0pt;}
@page WordSection1
        {size:612.0pt 792.0pt;
        margin:72.0pt 90.0pt 72.0pt 90.0pt;}
div.WordSection1
        {page:WordSection1;}
/* List Definitions */
@list l0
        {mso-list-id:1684160029;
        mso-list-type:hybrid;
        mso-list-template-ids:-1781474878 -2091978634 67698713 67698715 67698703 67698713 67698715 67698703 67698713 67698715;}
@list l0:level1
        {mso-level-text:"%1\)";
        mso-level-tab-stop:none;
        mso-level-number-position:left;
        margin-left:18.0pt;
        text-indent:-18.0pt;}
@list l0:level2
        {mso-level-number-format:alpha-lower;
        mso-level-text:"%2\)";
        mso-level-tab-stop:none;
        mso-level-number-position:left;
        margin-left:42.0pt;
        text-indent:-21.0pt;}
@list l0:level3
        {mso-level-number-format:roman-lower;
        mso-level-tab-stop:none;
        mso-level-number-position:right;
        margin-left:63.0pt;
        text-indent:-21.0pt;}
@list l0:level4
        {mso-level-tab-stop:none;
        mso-level-number-position:left;
        margin-left:84.0pt;
        text-indent:-21.0pt;}
@list l0:level5
        {mso-level-number-format:alpha-lower;
        mso-level-text:"%5\)";
        mso-level-tab-stop:none;
        mso-level-number-position:left;
        margin-left:105.0pt;
        text-indent:-21.0pt;}
@list l0:level6
        {mso-level-number-format:roman-lower;
        mso-level-tab-stop:none;
        mso-level-number-position:right;
        margin-left:126.0pt;
        text-indent:-21.0pt;}
@list l0:level7
        {mso-level-tab-stop:none;
        mso-level-number-position:left;
        margin-left:147.0pt;
        text-indent:-21.0pt;}
@list l0:level8
        {mso-level-number-format:alpha-lower;
        mso-level-text:"%8\)";
        mso-level-tab-stop:none;
        mso-level-number-position:left;
        margin-left:168.0pt;
        text-indent:-21.0pt;}
@list l0:level9
        {mso-level-number-format:roman-lower;
        mso-level-tab-stop:none;
        mso-level-number-position:right;
        margin-left:189.0pt;
        text-indent:-21.0pt;}
@list l1
        {mso-list-id:1739546548;
        mso-list-type:hybrid;
        mso-list-template-ids:-6364430 -610792634 67698713 67698715 67698703 67698713 67698715 67698703 67698713 67698715;}
@list l1:level1
        {mso-level-tab-stop:none;
        mso-level-number-position:left;
        margin-left:42.0pt;
        text-indent:-18.0pt;}
@list l1:level2
        {mso-level-number-format:alpha-lower;
        mso-level-text:"%2\)";
        mso-level-tab-stop:none;
        mso-level-number-position:left;
        margin-left:66.0pt;
        text-indent:-21.0pt;}
@list l1:level3
        {mso-level-number-format:roman-lower;
        mso-level-tab-stop:none;
        mso-level-number-position:right;
        margin-left:87.0pt;
        text-indent:-21.0pt;}
@list l1:level4
        {mso-level-tab-stop:none;
        mso-level-number-position:left;
        margin-left:108.0pt;
        text-indent:-21.0pt;}
@list l1:level5
        {mso-level-number-format:alpha-lower;
        mso-level-text:"%5\)";
        mso-level-tab-stop:none;
        mso-level-number-position:left;
        margin-left:129.0pt;
        text-indent:-21.0pt;}
@list l1:level6
        {mso-level-number-format:roman-lower;
        mso-level-tab-stop:none;
        mso-level-number-position:right;
        margin-left:150.0pt;
        text-indent:-21.0pt;}
@list l1:level7
        {mso-level-tab-stop:none;
        mso-level-number-position:left;
        margin-left:171.0pt;
        text-indent:-21.0pt;}
@list l1:level8
        {mso-level-number-format:alpha-lower;
        mso-level-text:"%8\)";
        mso-level-tab-stop:none;
        mso-level-number-position:left;
        margin-left:192.0pt;
        text-indent:-21.0pt;}
@list l1:level9
        {mso-level-number-format:roman-lower;
        mso-level-tab-stop:none;
        mso-level-number-position:right;
        margin-left:213.0pt;
        text-indent:-21.0pt;}
ol
        {margin-bottom:0cm;}
ul
        {margin-bottom:0cm;}
--></style><!--[if gte mso 9]><xml>
<o:shapedefaults v:ext="edit" spidmax="1026" />
</xml><![endif]--><!--[if gte mso 9]><xml>
<o:shapelayout v:ext="edit">
<o:idmap v:ext="edit" data="1" />
</o:shapelayout></xml><![endif]-->
</head>
<body lang="ZH-CN" link="blue" vlink="purple">
<div class="WordSection1">
<p class="MsoNormal"><span lang="EN-US" style="font-size:10.5pt;font-family:DengXian">Hi, Atin and experts<o:p></o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:10.5pt;font-family:DengXian"><o:p>&nbsp;</o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:10.5pt;font-family:DengXian">&gt;&gt;&gt;</span><span lang="EN-US"> That would be a hack. I think we need to first figure out on a successful brick connect why do we receive a connect, followed by a disconnect
 event and then another connect event. That's where the problem exists. I'm waiting to hear for inputs from our RPC experts on this.<o:p></o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:10.5pt;font-family:DengXian"><o:p>&nbsp;</o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:10.5pt;font-family:DengXian">I can answer the above questions now and suppose the root cause of this issue is clear now.<o:p></o:p></span></p>
<p class="MsoListParagraph" style="margin-left:18.0pt;text-indent:-18.0pt;mso-list:l0 level1 lfo3">
<![if !supportLists]><span lang="EN-US" style="font-size:10.5pt;font-family:DengXian"><span style="mso-list:Ignore">1)<span style="font:7.0pt &quot;Times New Roman&quot;">&nbsp;&nbsp;&nbsp;&nbsp;
</span></span></span><![endif]><span lang="EN-US" style="font-size:10.5pt;font-family:DengXian">Why receive a connect, followed by a disconnect?<o:p></o:p></span></p>
<p class="MsoListParagraph" style="margin-left:18.0pt;text-indent:0cm"><span lang="EN-US" style="font-size:10.5pt;font-family:DengXian">In current implement, the connect,&nbsp; function glusterd_rpc_create called in glusterd_brick_connect not specify option &quot;transport.socket.</span><span lang="EN-US">
</span><span lang="EN-US" style="font-size:10.5pt;font-family:DengXian">ignore-enoent &quot;.<o:p></o:p></span></p>
<p class="MsoListParagraph" style="margin-left:18.0pt;text-indent:0cm"><span lang="EN-US" style="font-size:10.5pt;font-family:DengXian">So in connect function is “socket_connect” in socket.c &nbsp;will use connect instead of connect_loop, in most case it will return
 ret with -1 and errno=ENOENT, and it will treat as SUCCESS!(attach the code below)<o:p></o:p></span></p>
<p class="MsoListParagraph" style="margin-left:18.0pt;text-indent:0cm"><span lang="EN-US" style="font-size:10.5pt;font-family:DengXian">And epoll mechanism will introduce a EPOLLHUP event with a&nbsp; not really connected socket! So it always trigger a EPOLLHUP
 handler which will disconnect the socket, and set the start_triggerred flag to faluse for the brick.<o:p></o:p></span></p>
<p class="MsoListParagraph" style="margin-left:18.0pt;text-indent:0cm"><span lang="EN-US" style="font-size:10.5pt;font-family:DengXian"><o:p>&nbsp;</o:p></span></p>
<p class="MsoListParagraph" style="margin-left:18.0pt;text-indent:0cm"><span lang="EN-US" style="font-size:10.5pt;font-family:DengXian">This disconnect issue is 100% reproduced, You could check the glusterd.log to confirm.<o:p></o:p></span></p>
<p class="MsoListParagraph" style="margin-left:18.0pt;text-indent:0cm"><span lang="EN-US" style="font-size:10.5pt;font-family:DengXian">--------------------------------------------------------------------------------------------------------<o:p></o:p></span></p>
<p class="MsoListParagraph" style="margin-left:18.0pt"><i><span lang="EN-US" style="font-size:10.5pt;font-family:DengXian">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; if (ign_enoent) {<o:p></o:p></span></i></p>
<p class="MsoListParagraph" style="margin-left:18.0pt"><i><span lang="EN-US" style="font-size:10.5pt;font-family:DengXian">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; ret = connect_loop (priv-&gt;sock,<o:p></o:p></span></i></p>
<p class="MsoListParagraph" style="margin-left:18.0pt"><i><span lang="EN-US" style="font-size:10.5pt;font-family:DengXian">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; SA (&amp;this-&gt;peerinfo.sockaddr),<o:p></o:p></span></i></p>
<p class="MsoListParagraph" style="margin-left:18.0pt"><i><span lang="EN-US" style="font-size:10.5pt;font-family:DengXian">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; this-&gt;peerinfo.sockaddr_len);<o:p></o:p></span></i></p>
<p class="MsoListParagraph" style="margin-left:18.0pt"><i><span lang="EN-US" style="font-size:10.5pt;font-family:DengXian">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
<o:p></o:p></span></i></p>
<p class="MsoListParagraph" style="margin-left:18.0pt"><i><span lang="EN-US" style="font-size:10.5pt;font-family:DengXian">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;} else {<o:p></o:p></span></i></p>
<p class="MsoListParagraph" style="margin-left:18.0pt"><i><span lang="EN-US" style="font-size:10.5pt;font-family:DengXian">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; ret = connect (priv-&gt;sock,<o:p></o:p></span></i></p>
<p class="MsoListParagraph" style="margin-left:18.0pt"><i><span lang="EN-US" style="font-size:10.5pt;font-family:DengXian">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; SA (&amp;this-&gt;peerinfo.sockaddr),<o:p></o:p></span></i></p>
<p class="MsoListParagraph" style="margin-left:18.0pt"><i><span lang="EN-US" style="font-size:10.5pt;font-family:DengXian">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; this-&gt;peerinfo.sockaddr_len);<o:p></o:p></span></i></p>
<p class="MsoListParagraph" style="margin-left:18.0pt;text-indent:0cm"><i><span lang="EN-US" style="font-size:10.5pt;font-family:DengXian">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; }<o:p></o:p></span></i></p>
<p class="MsoListParagraph" style="margin-left:18.0pt;text-indent:0cm"><i><span lang="EN-US" style="font-size:10.5pt;font-family:DengXian">……<o:p></o:p></span></i></p>
<p class="MsoListParagraph" style="margin-left:18.0pt"><i><span lang="EN-US" style="font-size:10.5pt;font-family:DengXian">if (ret == -1 &amp;&amp; ((errno != EINPROGRESS) &amp;&amp; (errno != ENOENT))) {<o:p></o:p></span></i></p>
<p class="MsoListParagraph" style="margin-left:18.0pt"><i><span lang="EN-US" style="font-size:10.5pt;font-family:DengXian">&nbsp;&nbsp;&nbsp;&nbsp; …………….<o:p></o:p></span></i></p>
<p class="MsoListParagraph" style="margin-left:18.0pt"><i><span lang="EN-US" style="font-size:10.5pt;font-family:DengXian">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; }<o:p></o:p></span></i></p>
<p class="MsoListParagraph" style="margin-left:18.0pt"><i><span lang="EN-US" style="font-size:10.5pt;font-family:DengXian">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; else {<o:p></o:p></span></i></p>
<p class="MsoListParagraph" style="margin-left:18.0pt"><i><span lang="EN-US" style="font-size:10.5pt;font-family:DengXian">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; /* reset connect_failed so that any previous attempts<o:p></o:p></span></i></p>
<p class="MsoListParagraph" style="margin-left:18.0pt"><i><span lang="EN-US" style="font-size:10.5pt;font-family:DengXian">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; state is not carried forward */<o:p></o:p></span></i></p>
<p class="MsoListParagraph" style="margin-left:18.0pt"><i><span lang="EN-US" style="font-size:10.5pt;font-family:DengXian">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; priv-&gt;connect_failed = 0;<o:p></o:p></span></i></p>
<p class="MsoListParagraph" style="margin-left:18.0pt"><i><span lang="EN-US" style="font-size:10.5pt;font-family:DengXian">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; ret = 0;<o:p></o:p></span></i></p>
<p class="MsoListParagraph" style="margin-left:18.0pt;text-indent:0cm"><i><span lang="EN-US" style="font-size:10.5pt;font-family:DengXian">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; }<o:p></o:p></span></i></p>
<p class="MsoListParagraph" style="margin-left:18.0pt;text-indent:0cm"><span lang="EN-US" style="font-size:10.5pt;font-family:DengXian">--------------------------------------------------------------------------------------------------------<o:p></o:p></span></p>
<p class="MsoListParagraph" style="margin-left:18.0pt;text-indent:-18.0pt;mso-list:l0 level1 lfo3">
<![if !supportLists]><span lang="EN-US" style="font-size:10.5pt;font-family:DengXian"><span style="mso-list:Ignore">2)<span style="font:7.0pt &quot;Times New Roman&quot;">&nbsp;&nbsp;&nbsp;&nbsp;
</span></span></span><![endif]><span lang="EN-US" style="font-size:10.5pt;font-family:DengXian">Why glusterfsd start twice ?<o:p></o:p></span></p>
<p class="MsoListParagraph" style="margin-left:18.0pt;text-indent:0cm"><span lang="EN-US" style="font-size:10.5pt;font-family:DengXian">On cluster SN, the brick started with the event ACC event which triggered by friend storage node, it will finally called
 in function glusterd_friend_sm as the following code <o:p></o:p></span></p>
<p class="MsoListParagraph" style="margin-left:18.0pt;text-indent:0cm"><span lang="EN-US" style="font-size:10.5pt;font-family:DengXian">--------------------------------------------------------------------------------<o:p></o:p></span></p>
<p class="MsoListParagraph" style="margin-left:18.0pt"><i><span lang="EN-US" style="font-size:10.5pt;font-family:DengXian">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; synclock_unlock (&amp;priv-&gt;big_lock);<o:p></o:p></span></i></p>
<p class="MsoListParagraph" style="margin-left:18.0pt"><i><span lang="EN-US" style="font-size:10.5pt;font-family:DengXian">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; glusterd_launch_synctask (glusterd_spawn_daemons, NULL);<o:p></o:p></span></i></p>
<p class="MsoListParagraph" style="margin-left:18.0pt"><i><span lang="EN-US" style="font-size:10.5pt;font-family:DengXian">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; synclock_lock (&amp;priv-&gt;big_lock);<o:p></o:p></span></i></p>
<p class="MsoListParagraph" style="margin-left:18.0pt;text-indent:0cm"><i><span lang="EN-US" style="font-size:10.5pt;font-family:DengXian">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; glusterd_do_quorum_action ();<o:p></o:p></span></i></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:10.5pt;font-family:DengXian">---------------------------------------------------------------------------<o:p></o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:10.5pt;font-family:DengXian">From the above code, the brick start maybe trigger twice, one is glusterd_spawn_saemons, another is glusterd_do_quorum_action.<o:p></o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:10.5pt;font-family:DengXian">And it is protected by the flag “brickinfo-&gt;start_triggered”, which used to prevent one brick started twice.<o:p></o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:10.5pt;font-family:DengXian"><o:p>&nbsp;</o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:10.5pt;font-family:DengXian">BUT, when the disconnect issue is always be there as analyzed as the above, it will be occasionally destroy the protect mechanism,
<o:p></o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:10.5pt;font-family:DengXian">which is to say, before the second “brick start” begin, the first “brick start” on same brick has EPOLLHUP event arrived, and the flag “brickinfo-&gt;start_triggered” has change
 to false , then the second “brick start” will treat it as a fresh brick process, which we has find in the log of “glusterd.log”, abstract one of example as the below:<o:p></o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:10.5pt;font-family:DengXian"><o:p>&nbsp;</o:p></span></p>
<p class="MsoNormal"><i><span lang="EN-US" style="font-size:10.5pt;font-family:DengXian">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; Line 632: [2018-04-15 18:06:00.277399] I [glusterd-utils.c:5928:glusterd_brick_start] 0-management: starting a fresh brick process for brick /mnt/bricks/log/brick<o:p></o:p></span></i></p>
<p class="MsoNormal"><i><span lang="EN-US" style="font-size:10.5pt;font-family:DengXian">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; Line 661: [2018-04-15 18:06:00.764239] I [glusterd-utils.c:5928:glusterd_brick_start] 0-management: starting a fresh brick process for brick /mnt/bricks/log/brick<o:p></o:p></span></i></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:10.5pt;font-family:DengXian"><o:p>&nbsp;</o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:10.5pt;font-family:DengXian">And when this occasional issue happen, the glustershd service will abnormal finally.<o:p></o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:10.5pt;font-family:DengXian"><o:p>&nbsp;</o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:10.5pt;font-family:DengXian">So I suppose it is a defect for current implement, and a suggest change solution is one code change in function “socket_connect” in socket.c as the below:<o:p></o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:10.5pt;font-family:DengXian"><o:p>&nbsp;</o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:10.5pt;font-family:DengXian">/******<o:p></o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:10.5pt;font-family:DengXian">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; ign_enoent = dict_get_str_boolean (this-&gt;options,<o:p></o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:10.5pt;font-family:DengXian">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; &quot;transport.socket.ignore-enoent&quot;, _gf_false);&nbsp;&nbsp;
<o:p></o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:10.5pt;font-family:DengXian">Change to ****/<o:p></o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:10.5pt;font-family:DengXian">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; ign_enoent = dict_get_str_boolean (this-&gt;options,<o:p></o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:10.5pt;font-family:DengXian">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; &quot;transport.socket.ignore-enoent&quot;, _gf_true);&nbsp;&nbsp;
<o:p></o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:10.5pt;font-family:DengXian"><o:p>&nbsp;</o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:10.5pt;font-family:DengXian">After test, the disconnect issue will not happen anymore!<o:p></o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:10.5pt;font-family:DengXian"><o:p>&nbsp;</o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:10.5pt;font-family:DengXian">@all expert from gluster, &nbsp;could you please review my investigation as the above, and share your comments?<o:p></o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:10.5pt;font-family:DengXian">Also, maybe my change will lead to other issue, so could you please give the right solutions?<o:p></o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:10.5pt;font-family:DengXian"><o:p>&nbsp;</o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:10.5pt;font-family:DengXian">Thanks &amp; Best Regards<o:p></o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:10.5pt;font-family:DengXian">George<o:p></o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif"><o:p>&nbsp;</o:p></span></p>
<div>
<div style="border:none;border-top:solid #E1E1E1 1.0pt;padding:3.0pt 0cm 0cm 0cm">
<p class="MsoNormal"><b><span lang="EN-US" style="font-size:11.0pt;font-family:&quot;Calibri&quot;,sans-serif">From:</span></b><span lang="EN-US" style="font-size:11.0pt;font-family:&quot;Calibri&quot;,sans-serif"> Atin Mukherjee [<a href="mailto:amukherj@redhat.com">mailto:amukherj@redhat.com</a>]
<br>
<b>Sent:</b> Thursday, December 14, 2017 12:37 PM<br>
<b>To:</b> Zhou, Cynthia (NSB - CN/Hangzhou) &lt;<a href="mailto:cynthia.zhou@nokia-sbell.com">cynthia.zhou@nokia-sbell.com</a>&gt;<br>
<b>Cc:</b> Milind Changire &lt;<a href="mailto:mchangir@redhat.com">mchangir@redhat.com</a>&gt;; Gaurav Yadav &lt;<a href="mailto:gyadav@redhat.com">gyadav@redhat.com</a>&gt;; Mohit Agrawal &lt;<a href="mailto:moagrawa@redhat.com">moagrawa@redhat.com</a>&gt;; Raghavendra Gowdappa
 &lt;<a href="mailto:rgowdapp@redhat.com">rgowdapp@redhat.com</a>&gt;; Samikshan Bairagya &lt;<a href="mailto:sbairagy@redhat.com">sbairagy@redhat.com</a>&gt;<br>
<b>Subject:</b> Re: glusterfsd start twice leading to glustershd can not connect with glusterfsd</span><span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif"><o:p></o:p></span></p>
</div>
</div>
<p class="MsoNormal"><span lang="EN-US"><o:p>&nbsp;</o:p></span></p>
<div>
<p class="MsoNormal"><span lang="EN-US">That would be a hack. I think we need to first figure out on a successful brick connect why do we receive a connect, followed by a disconnect event and then another connect event. That's where the problem exists. I'm
 waiting to hear for inputs from our RPC experts on this.<o:p></o:p></span></p>
</div>
<div>
<p class="MsoNormal"><span lang="EN-US"><o:p>&nbsp;</o:p></span></p>
<div>
<p class="MsoNormal"><span lang="EN-US">On Thu, Dec 14, 2017 at 8:20 AM, Zhou, Cynthia (NSB - CN/Hangzhou) &lt;<a href="mailto:cynthia.zhou@nokia-sbell.com" target="_blank">cynthia.zhou@nokia-sbell.com</a>&gt; wrote:<o:p></o:p></span></p>
<blockquote style="border:none;border-left:solid #CCCCCC 1.0pt;padding:0cm 0cm 0cm 6.0pt;margin-left:4.8pt;margin-top:5.0pt;margin-right:0cm;margin-bottom:5.0pt">
<div>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">Hi
</span><span lang="EN-US" style="font-size:11.0pt;font-family:&quot;Calibri&quot;,sans-serif">Mukherjee,</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">I will try to add some sleep in glusterd_volume_start_glusterfs, before it execute glusterd_brick_connect,
 to see if this could help to avoid the “unexpected disconnect event”, I think if this works this race condition could be avoided.</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">&nbsp;</span><span lang="EN-US"><o:p></o:p></span></p>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Arial&quot;,sans-serif">Best regards,<br>
<b>Cynthia </b></span><b><span style="font-size:10.5pt">(周琳)</span></b><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Arial&quot;,sans-serif">MBB SM HETRAN SW3 MATRIX&nbsp;
</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Arial&quot;,sans-serif">Storage&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
<b><br>
</b>Mobile: &#43;86 (0)18657188311</span><span lang="EN-US"><o:p></o:p></span></p>
</div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">&nbsp;</span><span lang="EN-US"><o:p></o:p></span></p>
<div>
<div style="border:none;border-top:solid #E1E1E1 1.0pt;padding:3.0pt 0cm 0cm 0cm">
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><b><span lang="EN-US" style="font-size:11.0pt;font-family:&quot;Calibri&quot;,sans-serif">From:</span></b><span lang="EN-US" style="font-size:11.0pt;font-family:&quot;Calibri&quot;,sans-serif"> Zhou,
 Cynthia (NSB - CN/Hangzhou) <br>
<b>Sent:</b> Wednesday, December 13, 2017 4:18 PM<br>
<b>To:</b> 'Atin Mukherjee' &lt;<a href="mailto:amukherj@redhat.com" target="_blank">amukherj@redhat.com</a>&gt;; Milind Changire &lt;<a href="mailto:mchangir@redhat.com" target="_blank">mchangir@redhat.com</a>&gt;<br>
<b>Cc:</b> Gaurav Yadav &lt;<a href="mailto:gyadav@redhat.com" target="_blank">gyadav@redhat.com</a>&gt;; Mohit Agrawal &lt;<a href="mailto:moagrawa@redhat.com" target="_blank">moagrawa@redhat.com</a>&gt;; Raghavendra Gowdappa &lt;<a href="mailto:rgowdapp@redhat.com" target="_blank">rgowdapp@redhat.com</a>&gt;;
 Samikshan Bairagya &lt;<a href="mailto:sbairagy@redhat.com" target="_blank">sbairagy@redhat.com</a>&gt;<br>
<b>Subject:</b> RE: glusterfsd start twice leading to glustershd can not connect with glusterfsd</span><span lang="EN-US"><o:p></o:p></span></p>
</div>
</div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US">&nbsp;<o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">@</span><span lang="EN-US" style="font-size:11.0pt;font-family:&quot;Calibri&quot;,sans-serif"> Mukherjee
</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">When I check the code , I find glusterd_volume_start_glusterfs will call glusterd_brick_connect, is
 it possible that the glusterd process try to connect to glusterfsd process too early so that it will receive disconnect event?</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">&nbsp;</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">&nbsp;</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Arial&quot;,sans-serif">Best regards,<br>
<b>Cynthia </b></span><b><span style="font-size:10.5pt">(周琳)</span></b><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Arial&quot;,sans-serif">MBB SM HETRAN SW3 MATRIX&nbsp;
</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Arial&quot;,sans-serif">Storage&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
<b><br>
</b>Mobile: &#43;86 (0)18657188311</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">&nbsp;</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><b><span lang="EN-US" style="font-size:11.0pt;font-family:&quot;Calibri&quot;,sans-serif">From:</span></b><span lang="EN-US" style="font-size:11.0pt;font-family:&quot;Calibri&quot;,sans-serif"> Atin
 Mukherjee [mailto:<a href="mailto:amukherj@redhat.com" target="_blank">amukherj@redhat.com</a>]
<br>
<b>Sent:</b> Tuesday, December 12, 2017 9:00 PM<br>
<b>To:</b> Milind Changire &lt;<a href="mailto:mchangir@redhat.com" target="_blank">mchangir@redhat.com</a>&gt;<br>
<b>Cc:</b> Gaurav Yadav &lt;<a href="mailto:gyadav@redhat.com" target="_blank">gyadav@redhat.com</a>&gt;; Mohit Agrawal &lt;<a href="mailto:moagrawa@redhat.com" target="_blank">moagrawa@redhat.com</a>&gt;; Raghavendra Gowdappa &lt;<a href="mailto:rgowdapp@redhat.com" target="_blank">rgowdapp@redhat.com</a>&gt;;
 Samikshan Bairagya &lt;<a href="mailto:sbairagy@redhat.com" target="_blank">sbairagy@redhat.com</a>&gt;; Zhou, Cynthia (NSB - CN/Hangzhou) &lt;<a href="mailto:cynthia.zhou@nokia-sbell.com" target="_blank">cynthia.zhou@nokia-sbell.com</a>&gt;<br>
<b>Subject:</b> Re: glusterfsd start twice leading to glustershd can not connect with glusterfsd</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US">&nbsp;<o:p></o:p></span></p>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US">&nbsp;<o:p></o:p></span></p>
<div>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US">On Tue, 12 Dec 2017 at 17:43, Milind Changire &lt;<a href="mailto:mchangir@redhat.com" target="_blank">mchangir@redhat.com</a>&gt; wrote:<o:p></o:p></span></p>
</div>
<blockquote style="border:none;border-left:solid #CCCCCC 1.0pt;padding:0cm 0cm 0cm 6.0pt;margin-left:4.8pt;margin-top:5.0pt;margin-right:0cm;margin-bottom:5.0pt">
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US">@Cynthia:<o:p></o:p></span></p>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US">What is the command(s) used to restart gluster processes ?<o:p></o:p></span></p>
</div>
</div>
</blockquote>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US">&nbsp;<o:p></o:p></span></p>
</div>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US">pkill gluster; glusterd is what I did to simulate this.<o:p></o:p></span></p>
</div>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US">&nbsp;<o:p></o:p></span></p>
</div>
<blockquote style="border:none;border-left:solid #CCCCCC 1.0pt;padding:0cm 0cm 0cm 6.0pt;margin-left:4.8pt;margin-top:5.0pt;margin-right:0cm;margin-bottom:5.0pt">
<div>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;margin-bottom:12.0pt"><span lang="EN-US">&nbsp;<o:p></o:p></span></p>
</div>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US">&nbsp;<o:p></o:p></span></p>
</div>
</div>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US">&nbsp;<o:p></o:p></span></p>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US">On Tue, Dec 12, 2017 at 3:25 PM, Atin Mukherjee &lt;<a href="mailto:amukherj@redhat.com" target="_blank">amukherj@redhat.com</a>&gt; wrote:<o:p></o:p></span></p>
<blockquote style="border:none;border-left:solid #CCCCCC 1.0pt;padding:0cm 0cm 0cm 6.0pt;margin-left:4.8pt;margin-top:5.0pt;margin-right:0cm;margin-bottom:5.0pt">
<div>
<div>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;margin-bottom:12.0pt"><span lang="EN-US">Looks like we have a race here. I can see a brick disconnect (RPC_CLNT_DISCONNECT) event when the brick process was coming up.<o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">[2017-12-09 18:07:34.488767] I [socket.c:2478:socket_event_handler] 0-transport: EPOLLERR - disconnecting now</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">[2017-12-09 18:07:34.489020] I [MSGID: 106005] [glusterd-handler.c:6071:__glusterd_brick_rpc_notify] 0-management: Brick sn-0.local:/mnt/bricks/export/brick has disconnected from glusterd.</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">[2017-12-09 18:07:34.489450] I [socket.c:2478:socket_event_handler] 0-transport: EPOLLERR - disconnecting now</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">[2017-12-09 18:07:34.489717] I [MSGID: 106005] [glusterd-handler.c:6071:__glusterd_brick_rpc_notify] 0-management: Brick sn-0.local:/mnt/bricks/log/brick has disconnected from glusterd.</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">[2017-12-09 18:07:34.490114] I [socket.c:2478:socket_event_handler] 0-transport: EPOLLERR - disconnecting now</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">[2017-12-09 18:07:34.490398] I [MSGID: 106005] [glusterd-handler.c:6071:__glusterd_brick_rpc_notify] 0-management: Brick sn-0.local:/mnt/bricks/mstate/brick has disconnected from glusterd.</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">[2017-12-09 18:07:34.490785] I [socket.c:2478:socket_event_handler] 0-transport: EPOLLERR - disconnecting now</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">&nbsp;</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">What I can see is every time when I restart all the gluster process (simulating a reboot scenario) I could see respective brick disconnect events being received at glusterd.</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;margin-bottom:12.0pt"><span lang="EN-US"><br>
#0&nbsp; __glusterd_brick_rpc_notify (rpc=rpc@entry=0x7fffe8008810, mydata=mydata@entry=0x7fffe800e0d0,
<br>
&nbsp;&nbsp;&nbsp; event=event@entry=RPC_CLNT_DISCONNECT, data=data@entry=0x0) at glusterd-handler.c:6007<br>
#1&nbsp; 0x00007ffff2936169 in glusterd_big_locked_notify (rpc=0x7fffe8008810, mydata=0x7fffe800e0d0, event=RPC_CLNT_DISCONNECT,
<br>
&nbsp;&nbsp;&nbsp; data=0x0, notify_fn=0x7ffff2932440 &lt;__glusterd_brick_rpc_notify&gt;) at glusterd-handler.c:68<br>
#2&nbsp; 0x00007ffff78c0343 in rpc_clnt_handle_disconnect (conn=0x7fffe8008840, clnt=0x7fffe8008810) at rpc-clnt.c:876<br>
#3&nbsp; rpc_clnt_notify (trans=0x7fffe8008ae0, mydata=0x7fffe8008840, event=RPC_TRANSPORT_DISCONNECT, data=&lt;optimized out&gt;)<br>
&nbsp;&nbsp;&nbsp; at rpc-clnt.c:939<br>
#4&nbsp; 0x00007ffff78bca73 in rpc_transport_notify (this=this@entry=0x7fffe8008ae0, event=event@entry=RPC_TRANSPORT_DISCONNECT,
<br>
&nbsp;&nbsp;&nbsp; data=data@entry=0x7fffe8008ae0) at rpc-transport.c:545<br>
#5&nbsp; 0x00007fffefc252bf in socket_event_poll_err (idx=&lt;optimized out&gt;, gen=&lt;optimized out&gt;, this=0x7fffe8008ae0) at socket.c:1210<br>
#6&nbsp; socket_event_handler (fd=9, idx=&lt;optimized out&gt;, gen=&lt;optimized out&gt;, data=0x7fffe8008ae0, poll_in=0, poll_out=&lt;optimized out&gt;,
<br>
&nbsp;&nbsp;&nbsp; poll_err=16) at socket.c:2488<br>
#7&nbsp; 0x00007ffff7b5097c in event_dispatch_epoll_handler (event=0x7fffee9dce84, event_pool=0x64f040) at event-epoll.c:583<br>
#8&nbsp; event_dispatch_epoll_worker (data=0x70b820) at event-epoll.c:659<br>
#9&nbsp; 0x00007ffff693f36d in start_thread () from /lib64/libpthread.so.0<br>
#10 0x00007ffff61ecbbf in clone () from /lib64/libc.so.6<o:p></o:p></span></p>
</div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;margin-bottom:12.0pt"><span lang="EN-US">The question what I have for RPC team is - why do we see a disconnect event in this case?&nbsp; Any pointers, Mohit/Raghu/Milind?<o:p></o:p></span></p>
</div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US">The side effect of this disconnect event is what caused this issue. In __glusterd_brick_rpc_notify we set the brickinfo-&gt;start_triggered to false. So when two
 glusterd_brick_start threads were racing with each other, in between we got a disconnect because of which this flag got reset and we ended up trying to spawn the same brick twice.<o:p></o:p></span></p>
</div>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US">&nbsp;<o:p></o:p></span></p>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US">On Tue, Dec 12, 2017 at 11:05 AM, Zhou, Cynthia (NSB - CN/Hangzhou) &lt;<a href="mailto:cynthia.zhou@nokia-sbell.com" target="_blank">cynthia.zhou@nokia-sbell.com</a>&gt;
 wrote:<o:p></o:p></span></p>
<blockquote style="border:none;border-left:solid #CCCCCC 1.0pt;padding:0cm 0cm 0cm 6.0pt;margin-left:4.8pt;margin-top:5.0pt;margin-right:0cm;margin-bottom:5.0pt">
<div>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">Ok. When this happened.</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">&nbsp;</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">gluster volume info mstate</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">Volume Name: mstate</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">Type: Replicate</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">Volume ID: 5fc29728-b5de-4b5f-8f3c-9654acda2577</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">Status: Started</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">Snapshot Count: 0</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">Number of Bricks: 1 x 2 = 2</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">Transport-type: tcp</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">Bricks:</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">Brick1: sn-0.local:/mnt/bricks/mstate/brick</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">Brick2: sn-1.local:/mnt/bricks/mstate/brick</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">Options Reconfigured:</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">nfs.disable: on</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">transport.address-family: inet</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">cluster.server-quorum-type: none</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">cluster.consistent-metadata: on</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">server.allow-insecure: on</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">network.ping-timeout: 42</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">performance.client-io-threads: off</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">cluster.server-quorum-ratio: 51%</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">[root@sn-0:/root]</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">Also the ps result:</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">&nbsp;</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">ps aux | grep gluster</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">root&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 380&nbsp; 0.0&nbsp; 0.6 377892 13760 ?&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; Ss&nbsp;&nbsp; 20:07&nbsp;&nbsp; 0:00 python /usr/sbin/glustereventsd --pid-file
 /var/run/glustereventsd.pid</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">root&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 409&nbsp; 0.0&nbsp; 0.4&nbsp; 82964&nbsp; 9148 ?&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; S&nbsp;&nbsp;&nbsp; 20:07&nbsp;&nbsp; 0:00 python /usr/sbin/glustereventsd --pid-file
 /var/run/glustereventsd.pid</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">root&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 811&nbsp; 0.5&nbsp; 1.0 528720 21636 ?&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; Ssl&nbsp; 20:07&nbsp;&nbsp; 0:01 /usr/sbin/glusterd --volfile=/opt/nokia/libexec/StorageUtils/etc/glusterd/glusterd.vol
 -p /run/glusterd.pid</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">root&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 1088&nbsp; 0.1&nbsp; 0.6 1036648 13996 ?&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; Ssl&nbsp; 20:07&nbsp;&nbsp; 0:00 /usr/sbin/glusterfsd -s sn-0.local
 --volfile-id export.sn-0.local.mnt-bricks-export-brick -p /var/run/gluster/vols/export/sn-0.local-mnt-bricks-export-brick.pid -S /var/run/gluster/44f098f14fe45b74b61c5a1d98993d15.socket --brick-name /mnt/bricks/export/brick -l /var/log/glusterfs/bricks/mnt-bricks-export-brick.log
 --xlator-option *-posix.glusterd-uuid=f830d809-c935-451f-853a-1364f76a4160 --brick-port 49152 --xlator-option export-server.listen-port=49152 --xlator-option transport.socket.bind-address=sn-0.local</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">root&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 1153&nbsp; 1.8&nbsp; 0.7 1234036 14664 ?&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; Ssl&nbsp; 20:07&nbsp;&nbsp; 0:06 /usr/sbin/glusterfsd -s sn-0.local
 --volfile-id log.sn-0.local.mnt-bricks-log-brick -p /var/run/gluster/vols/log/sn-0.local-mnt-bricks-log-brick.pid -S /var/run/gluster/0af76d9541921cccad34e5fd3041a44d.socket --brick-name /mnt/bricks/log/brick -l /var/log/glusterfs/bricks/mnt-bricks-log-brick.log
 --xlator-option *-posix.glusterd-uuid=f830d809-c935-451f-853a-1364f76a4160 --brick-port 49156 --xlator-option log-server.listen-port=49156 --xlator-option transport.socket.bind-address=sn-0.local</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">root&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 1155&nbsp; 0.0&nbsp; 0.5 772996 10580 ?&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; Ssl&nbsp; 20:07&nbsp;&nbsp; 0:00 /usr/sbin/glusterfsd -s sn-0.local
 --volfile-id mstate.sn-0.local.mnt-bricks-mstate-brick -p /var/run/gluster/vols/mstate/sn-0.local-mnt-bricks-mstate-brick.pid -S /var/run/gluster/79d00394b1085adcf7dc1fd4c57f71bc.socket --brick-name /mnt/bricks/mstate/brick -l /var/log/glusterfs/bricks/mnt-bricks-mstate-brick.log
 --xlator-option *-posix.glusterd-uuid=f830d809-c935-451f-853a-1364f76a4160 --brick-port 49154 --xlator-option mstate-server.listen-port=49154 --xlator-option transport.socket.bind-address=sn-0.local</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">root&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 1167&nbsp; 0.4 &nbsp;0.7 1235512 15488 ?&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; Ssl&nbsp; 20:07&nbsp;&nbsp; 0:01 /usr/sbin/glusterfsd -s sn-0.local
 --volfile-id services.sn-0.local.mnt-bricks-services-brick -p /var/run/gluster/vols/services/sn-0.local-mnt-bricks-services-brick.pid -S /var/run/gluster/fc16d9fe6fd0695555a5592343da7ba3.socket --brick-name /mnt/bricks/services/brick -l /var/log/glusterfs/bricks/mnt-bricks-services-brick.log
 --xlator-option *-posix.glusterd-uuid=f830d809-c935-451f-853a-1364f76a4160 --brick-port 49155 --xlator-option services-server.listen-port=49155 --xlator-option transport.socket.bind-address=sn-0.local</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">root&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 1188&nbsp; 0.0&nbsp; 0.5 644916 11896 ?&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; Ssl&nbsp; 20:07&nbsp;&nbsp; 0:00 /usr/sbin/glusterfs -s sn-0.local --volfile-id
 gluster/glustershd -p /var/run/gluster/glustershd/glustershd.pid -l /var/log/glusterfs/glustershd.log -S /var/run/gluster/c405d28a3a79d88436a826d75cf7153a.socket --xlator-option *replicate*.node-uuid=f830d809-c935-451f-853a-1364f76a4160</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">root&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 2324&nbsp; 0.0&nbsp; 0.6 629784 13932 ?&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; Ssl&nbsp; 20:07&nbsp;&nbsp; 0:00 /usr/sbin/glusterfs --acl --volfile-server=sn-0.local
 --volfile-server=sn-1.local --volfile-id=/services /mnt/services</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">root&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 2422&nbsp; 0.0&nbsp; 0.5 431036 10604 ?&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; Ssl&nbsp; 20:07&nbsp;&nbsp; 0:00 /usr/sbin/glusterfs --acl --volfile-server=sn-0.local
 --volfile-server=sn-1.local --volfile-id=/export /mnt/export</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">root&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 2520&nbsp; 0.0&nbsp; 0.5 498712 11024 ?&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; Ssl&nbsp; 20:07&nbsp;&nbsp; 0:00 /usr/sbin/glusterfs --acl --volfile-server=sn-0.local
 --volfile-server=sn-1.local --volfile-id=/log /mnt/log</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">root&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 2618&nbsp; 0.0&nbsp; 0.5 431036 10476 ?&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; Ssl&nbsp; 20:07&nbsp;&nbsp; 0:00 /usr/sbin/glusterfs --acl --volfile-server=sn-0.local
 --volfile-server=sn-1.local --volfile-id=/mstate /mnt/mstate</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">root&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 8136&nbsp; 0.0&nbsp; 0.0&nbsp;&nbsp; 8224&nbsp;&nbsp; 784 pts/0&nbsp;&nbsp;&nbsp; S&#43;&nbsp;&nbsp; 20:13&nbsp;&nbsp; 0:00 grep gluster</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">[root@sn-0:/root]</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">#</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">&nbsp;</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Arial&quot;,sans-serif">Best regards,<br>
<b>Cynthia </b></span><b><span style="font-size:10.5pt">(周琳)</span></b><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Arial&quot;,sans-serif">MBB SM HETRAN SW3 MATRIX&nbsp;
</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Arial&quot;,sans-serif">Storage&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
<b><br>
</b>Mobile: &#43;86 (0)18657188311</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">&nbsp;</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><b><span lang="EN-US" style="font-size:11.0pt;font-family:&quot;Calibri&quot;,sans-serif">From:</span></b><span lang="EN-US" style="font-size:11.0pt;font-family:&quot;Calibri&quot;,sans-serif"> Atin
 Mukherjee [mailto:</span><span lang="EN-US"><a href="mailto:amukherj@redhat.com" target="_blank"><span style="font-size:11.0pt;font-family:&quot;Calibri&quot;,sans-serif">amukherj@redhat.com</span></a></span><span lang="EN-US" style="font-size:11.0pt;font-family:&quot;Calibri&quot;,sans-serif">]
<br>
<b>Sent:</b> Tuesday, December 12, 2017 1:32 PM<br>
<b>To:</b> Zhou, Cynthia (NSB - CN/Hangzhou) &lt;</span><span lang="EN-US"><a href="mailto:cynthia.zhou@nokia-sbell.com" target="_blank"><span style="font-size:11.0pt;font-family:&quot;Calibri&quot;,sans-serif">cynthia.zhou@nokia-sbell.com</span></a></span><span lang="EN-US" style="font-size:11.0pt;font-family:&quot;Calibri&quot;,sans-serif">&gt;<br>
<b>Cc:</b> Gaurav Yadav &lt;</span><span lang="EN-US"><a href="mailto:gyadav@redhat.com" target="_blank"><span style="font-size:11.0pt;font-family:&quot;Calibri&quot;,sans-serif">gyadav@redhat.com</span></a></span><span lang="EN-US" style="font-size:11.0pt;font-family:&quot;Calibri&quot;,sans-serif">&gt;;
 Samikshan Bairagya &lt;</span><span lang="EN-US"><a href="mailto:sbairagy@redhat.com" target="_blank"><span style="font-size:11.0pt;font-family:&quot;Calibri&quot;,sans-serif">sbairagy@redhat.com</span></a></span><span lang="EN-US" style="font-size:11.0pt;font-family:&quot;Calibri&quot;,sans-serif">&gt;<br>
<b>Subject:</b> Re: glusterfsd start twice leading to glustershd can not connect with glusterfsd</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US">&nbsp;<o:p></o:p></span></p>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US">I was looking for output of gluster volume info to understand what all volume tunables were turned on.<o:p></o:p></span></p>
</div>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US">&nbsp;<o:p></o:p></span></p>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US">On Tue, Dec 12, 2017 at 10:56 AM, Zhou, Cynthia (NSB - CN/Hangzhou) &lt;<a href="mailto:cynthia.zhou@nokia-sbell.com" target="_blank">cynthia.zhou@nokia-sbell.com</a>&gt;
 wrote:<o:p></o:p></span></p>
<blockquote style="border:none;border-left:solid #CCCCCC 1.0pt;padding:0cm 0cm 0cm 6.0pt;margin-left:4.8pt;margin-top:5.0pt;margin-right:0cm;margin-bottom:5.0pt">
<div>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">Hi,</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">From our test case when this issue reproduced:</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">gluster volume status mstate</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">Status of volume: mstate</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">Gluster process&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; TCP Port&nbsp; RDMA Port&nbsp; Online&nbsp; Pid</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">------------------------------------------------------------------------------</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">Brick sn-0.local:/mnt/bricks/mstate/brick&nbsp;&nbsp; 49157&nbsp;&nbsp;&nbsp;&nbsp; 0&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; Y&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 1155
</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">Brick sn-1.local:/mnt/bricks/mstate/brick&nbsp;&nbsp; 49154&nbsp;&nbsp;&nbsp;&nbsp; 0&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; Y&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 1201
</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">Self-heal Daemon on localhost&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; N/A&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; N/A&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; Y&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 1188
</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">Self-heal Daemon on sn-2.local&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; N/A&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; N/A&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; Y&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 27109</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">Self-heal Daemon on sn-1.local&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; N/A&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; N/A&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; Y&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 4836
</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">&nbsp;</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">Task Status of Volume mstate</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">------------------------------------------------------------------------------</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">There are no active volume tasks</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">[root@sn-0:/root]</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">#</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">&nbsp;</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Arial&quot;,sans-serif">Best regards,<br>
<b>Cynthia </b></span><b><span style="font-size:10.5pt">(周琳)</span></b><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Arial&quot;,sans-serif">MBB SM HETRAN SW3 MATRIX&nbsp;
</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Arial&quot;,sans-serif">Storage&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
<b><br>
</b>Mobile: &#43;86 (0)18657188311</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">&nbsp;</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><b><span lang="EN-US" style="font-size:11.0pt;font-family:&quot;Calibri&quot;,sans-serif">From:</span></b><span lang="EN-US" style="font-size:11.0pt;font-family:&quot;Calibri&quot;,sans-serif"> Atin
 Mukherjee [mailto:</span><span lang="EN-US"><a href="mailto:amukherj@redhat.com" target="_blank"><span style="font-size:11.0pt;font-family:&quot;Calibri&quot;,sans-serif">amukherj@redhat.com</span></a></span><span lang="EN-US" style="font-size:11.0pt;font-family:&quot;Calibri&quot;,sans-serif">]
<br>
<b>Sent:</b> Tuesday, December 12, 2017 12:02 PM<br>
<b>To:</b> Zhou, Cynthia (NSB - CN/Hangzhou) &lt;</span><span lang="EN-US"><a href="mailto:cynthia.zhou@nokia-sbell.com" target="_blank"><span style="font-size:11.0pt;font-family:&quot;Calibri&quot;,sans-serif">cynthia.zhou@nokia-sbell.com</span></a></span><span lang="EN-US" style="font-size:11.0pt;font-family:&quot;Calibri&quot;,sans-serif">&gt;<br>
<b>Cc:</b> Gaurav Yadav &lt;</span><span lang="EN-US"><a href="mailto:gyadav@redhat.com" target="_blank"><span style="font-size:11.0pt;font-family:&quot;Calibri&quot;,sans-serif">gyadav@redhat.com</span></a></span><span lang="EN-US" style="font-size:11.0pt;font-family:&quot;Calibri&quot;,sans-serif">&gt;;
 Samikshan Bairagya &lt;</span><span lang="EN-US"><a href="mailto:sbairagy@redhat.com" target="_blank"><span style="font-size:11.0pt;font-family:&quot;Calibri&quot;,sans-serif">sbairagy@redhat.com</span></a></span><span lang="EN-US" style="font-size:11.0pt;font-family:&quot;Calibri&quot;,sans-serif">&gt;<br>
<b>Subject:</b> Re: glusterfsd start twice leading to glustershd can not connect with glusterfsd</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US">&nbsp;<o:p></o:p></span></p>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US">&nbsp;<o:p></o:p></span></p>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US">&nbsp;<o:p></o:p></span></p>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US">On Tue, Dec 12, 2017 at 8:34 AM, Zhou, Cynthia (NSB - CN/Hangzhou) &lt;<a href="mailto:cynthia.zhou@nokia-sbell.com" target="_blank">cynthia.zhou@nokia-sbell.com</a>&gt;
 wrote:<o:p></o:p></span></p>
<blockquote style="border:none;border-left:solid #CCCCCC 1.0pt;padding:0cm 0cm 0cm 6.0pt;margin-left:4.8pt;margin-top:5.0pt;margin-right:0cm;margin-bottom:5.0pt">
<div>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">By the way , when I check the code I find that in</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">glusterd_op_perform_add_bricks and glusterd_start_volume, glusterd_brick_start is not triggered within
 pthread_mutex_lock, do you think we could add pthread_mutex_lock?</span><span lang="EN-US"><o:p></o:p></span></p>
</div>
</div>
</blockquote>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US">&nbsp;<o:p></o:p></span></p>
</div>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US">I don't think that's needed as these functions are triggered through CLI command. We get into this race when the brick start is triggered asynchronously from
 a restart glusterd code path. Do you have the volume info output when this issue occurred?<o:p></o:p></span></p>
</div>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US">&nbsp;<o:p></o:p></span></p>
</div>
<blockquote style="border:none;border-left:solid #CCCCCC 1.0pt;padding:0cm 0cm 0cm 6.0pt;margin-left:4.8pt;margin-top:5.0pt;margin-right:0cm;margin-bottom:5.0pt">
<div>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">&nbsp;</span><span lang="EN-US"><o:p></o:p></span></p>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Arial&quot;,sans-serif">Best regards,<br>
<b>Cynthia </b></span><b><span style="font-size:10.5pt">(周琳)</span></b><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Arial&quot;,sans-serif">MBB SM HETRAN SW3 MATRIX&nbsp;
</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Arial&quot;,sans-serif">Storage&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
<b><br>
</b>Mobile: &#43;86 (0)18657188311</span><span lang="EN-US"><o:p></o:p></span></p>
</div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">&nbsp;</span><span lang="EN-US"><o:p></o:p></span></p>
<div>
<div style="border:none;border-top:solid #E1E1E1 1.0pt;padding:3.0pt 0cm 0cm 0cm">
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><b><span lang="EN-US" style="font-size:11.0pt;font-family:&quot;Calibri&quot;,sans-serif">From:</span></b><span lang="EN-US" style="font-size:11.0pt;font-family:&quot;Calibri&quot;,sans-serif"> Zhou,
 Cynthia (NSB - CN/Hangzhou) <br>
<b>Sent:</b> Tuesday, December 12, 2017 9:23 AM<br>
<b>To:</b> 'Gaurav Yadav' &lt;</span><span lang="EN-US"><a href="mailto:gyadav@redhat.com" target="_blank"><span style="font-size:11.0pt;font-family:&quot;Calibri&quot;,sans-serif">gyadav@redhat.com</span></a></span><span lang="EN-US" style="font-size:11.0pt;font-family:&quot;Calibri&quot;,sans-serif">&gt;;
 Atin Mukherjee &lt;</span><span lang="EN-US"><a href="mailto:amukherj@redhat.com" target="_blank"><span style="font-size:11.0pt;font-family:&quot;Calibri&quot;,sans-serif">amukherj@redhat.com</span></a></span><span lang="EN-US" style="font-size:11.0pt;font-family:&quot;Calibri&quot;,sans-serif">&gt;<br>
<b>Cc:</b> Samikshan Bairagya &lt;</span><span lang="EN-US"><a href="mailto:sbairagy@redhat.com" target="_blank"><span style="font-size:11.0pt;font-family:&quot;Calibri&quot;,sans-serif">sbairagy@redhat.com</span></a></span><span lang="EN-US" style="font-size:11.0pt;font-family:&quot;Calibri&quot;,sans-serif">&gt;<br>
<b>Subject:</b> RE: glusterfsd start twice leading to glustershd can not connect with glusterfsd</span><span lang="EN-US"><o:p></o:p></span></p>
</div>
</div>
<div>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US">&nbsp;<o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">HI,
</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">Thanks for responding!</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">My testing version has include the correction of bug-1506589, I’ve also checked my gluster source code.
 so this is a new issue.</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">Unfortunately this issue rarely appear, It only appeared once until now. it is found after reboot sn
 node.</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">Enclosed is the content in /var/lib/glusterd and /var/log/glusterfs</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">&nbsp;</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">The time when this issue happened is , only mstate volume has this issue.</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">&nbsp;</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">[<span style="color:red">2017-12-09 18:07:02.743046</span>] W [glusterfsd.c:1375:cleanup_and_exit] (--&gt;/lib64/libpthread.so.0(&#43;0x7454)
 [0x7f5df4db6454] --&gt;/usr/sbin/glusterfsd(glusterfs_sigwaiter&#43;0xdc) [0x40a50c] --&gt;/usr/sbin/glusterfsd(cleanup_and_exit&#43;0x77) [0x4089ec] ) 0-: received signum (15), shutting down</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">[<span style="color:red">2017-12-09 18:07:34.416169</span>] I [MSGID: 100030] [glusterfsd.c:2524:main]
 0-/usr/sbin/glusterfsd: Started running /usr/sbin/glusterfsd version 3.12.3 (args: /usr/sbin/glusterfsd -s sn-0.local --volfile-id mstate.sn-0.local.mnt-bricks-mstate-brick -p /var/run/gluster/vols/mstate/sn-0.local-mnt-bricks-mstate-brick.pid -S /var/run/gluster/79d00394b1085adcf7dc1fd4c57f71bc.socket
 --brick-name /mnt/bricks/mstate/brick -l /var/log/glusterfs/bricks/mnt-bricks-mstate-brick.log --xlator-option *-posix.glusterd-uuid=f830d809-c935-451f-853a-1364f76a4160 --brick-port 49154 --xlator-option mstate-server.listen-port=49154 --xlator-option transport.socket.bind-address=sn-0.local)</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif;color:red">[2017-12-09 18:07:34.511598</span><span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">]
 I [MSGID: 100030] [glusterfsd.c:2524:main] 0-/usr/sbin/glusterfsd: Started running /usr/sbin/glusterfsd version 3.12.3 (args: /usr/sbin/glusterfsd -s sn-0.local --volfile-id mstate.sn-0.local.mnt-bricks-mstate-brick -p /var/run/gluster/vols/mstate/sn-0.local-mnt-bricks-mstate-brick.pid
 -S /var/run/gluster/79d00394b1085adcf7dc1fd4c57f71bc.socket --brick-name /mnt/bricks/mstate/brick -l /var/log/glusterfs/bricks/mnt-bricks-mstate-brick.log --xlator-option *-posix.glusterd-uuid=f830d809-c935-451f-853a-1364f76a4160 --brick-port 49157 --xlator-option
 mstate-server.listen-port=49157 --xlator-option transport.socket.bind-address=sn-0.local)</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Arial&quot;,sans-serif">Best regards,<br>
<b>Cynthia </b></span><b><span style="font-size:10.5pt">(周琳)</span></b><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Arial&quot;,sans-serif">MBB SM HETRAN SW3 MATRIX&nbsp;
</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Arial&quot;,sans-serif">Storage&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
<b><br>
</b>Mobile: &#43;86 (0)18657188311</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">&nbsp;</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><b><span lang="EN-US" style="font-size:11.0pt;font-family:&quot;Calibri&quot;,sans-serif">From:</span></b><span lang="EN-US" style="font-size:11.0pt;font-family:&quot;Calibri&quot;,sans-serif"> Gaurav
 Yadav [</span><span lang="EN-US"><a href="mailto:gyadav@redhat.com" target="_blank"><span style="font-size:11.0pt;font-family:&quot;Calibri&quot;,sans-serif">mailto:gyadav@redhat.com</span></a></span><span lang="EN-US" style="font-size:11.0pt;font-family:&quot;Calibri&quot;,sans-serif">]
<br>
<b>Sent:</b> Monday, December 11, 2017 6:58 PM<br>
<b>To:</b> Atin Mukherjee &lt;</span><span lang="EN-US"><a href="mailto:amukherj@redhat.com" target="_blank"><span style="font-size:11.0pt;font-family:&quot;Calibri&quot;,sans-serif">amukherj@redhat.com</span></a></span><span lang="EN-US" style="font-size:11.0pt;font-family:&quot;Calibri&quot;,sans-serif">&gt;<br>
<b>Cc:</b> Samikshan Bairagya &lt;</span><span lang="EN-US"><a href="mailto:sbairagy@redhat.com" target="_blank"><span style="font-size:11.0pt;font-family:&quot;Calibri&quot;,sans-serif">sbairagy@redhat.com</span></a></span><span lang="EN-US" style="font-size:11.0pt;font-family:&quot;Calibri&quot;,sans-serif">&gt;;
 Zhou, Cynthia (NSB - CN/Hangzhou) &lt;</span><span lang="EN-US"><a href="mailto:cynthia.zhou@nokia-sbell.com" target="_blank"><span style="font-size:11.0pt;font-family:&quot;Calibri&quot;,sans-serif">cynthia.zhou@nokia-sbell.com</span></a></span><span lang="EN-US" style="font-size:11.0pt;font-family:&quot;Calibri&quot;,sans-serif">&gt;<br>
<b>Subject:</b> Re: glusterfsd start twice leading to glustershd can not connect with glusterfsd</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US">&nbsp;<o:p></o:p></span></p>
<div>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US"><br>
After seeing logs which you have shared here, it seems it's a port mismatch issue.
<br>
We had a similar issue <a href="https://bugzilla.redhat.com/show_bug.cgi?id=1506589" target="_blank">
bug-1506589,</a> &quot;Please confirm that you too are seeing same behavior&quot;.<o:p></o:p></span></p>
</div>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US"><br>
If not please share setup details along with.. <o:p></o:p></span></p>
</div>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US">* files from /var/lib/glusterd/ directory,<o:p></o:p></span></p>
</div>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US">* log files from /var/log/glusterfs<o:p></o:p></span></p>
</div>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US">&nbsp;<o:p></o:p></span></p>
</div>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US">Thanks<o:p></o:p></span></p>
</div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US">Gaurav<o:p></o:p></span></p>
</div>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US">&nbsp;<o:p></o:p></span></p>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US">On Mon, Dec 11, 2017 at 3:16 PM, Atin Mukherjee &lt;<a href="mailto:amukherj@redhat.com" target="_blank">amukherj@redhat.com</a>&gt; wrote:<o:p></o:p></span></p>
<blockquote style="border:none;border-left:solid #CCCCCC 1.0pt;padding:0cm 0cm 0cm 6.0pt;margin-left:4.8pt;margin-top:5.0pt;margin-right:0cm;margin-bottom:5.0pt">
<div>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US">Adding couple of my colleagues to the thread since I will be busy through out this week. If this is easily reproducible then could you share the exact steps?<o:p></o:p></span></p>
</div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US">&nbsp;<o:p></o:p></span></p>
<div>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US">On Mon, 11 Dec 2017 at 14:56, Zhou, Cynthia (NSB - CN/Hangzhou) &lt;<a href="mailto:cynthia.zhou@nokia-sbell.com" target="_blank">cynthia.zhou@nokia-sbell.com</a>&gt;
 wrote:<o:p></o:p></span></p>
</div>
<blockquote style="border:none;border-left:solid #CCCCCC 1.0pt;padding:0cm 0cm 0cm 6.0pt;margin-left:4.8pt;margin-top:5.0pt;margin-right:0cm;margin-bottom:5.0pt">
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">Hi,</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; Good day!</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Times New Roman&quot;,serif">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
</span><span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">I see your change
</span><span lang="EN-US" style="font-size:8.5pt;font-family:&quot;Courier New&quot;">I292f1e58d6971e111725e1baea1fe98b890b43e2 i</span><span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">n glusterfs code.</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; However from my test glusterfs 3.12.3 (already contain that modification) still has something wrong,</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Times New Roman&quot;,serif">&nbsp;</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">gluster volume status mstate</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">Status of volume: mstate</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">Gluster process&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; TCP Port&nbsp; RDMA Port&nbsp; Online&nbsp; Pid</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">------------------------------------------------------------------------------</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">Brick sn-0.local:/mnt/bricks/mstate/brick&nbsp;&nbsp;
<b><span style="color:red">49157</span></b>&nbsp;&nbsp;&nbsp;&nbsp; 0&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; Y&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 1155 </span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">Brick sn-1.local:/mnt/bricks/mstate/brick&nbsp;&nbsp; 49154&nbsp;&nbsp;&nbsp;&nbsp; 0&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; Y&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 1201
</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">Self-heal Daemon on localhost&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; N/A&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; N/A&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; Y&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 1188
</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">Self-heal Daemon on sn-2.local&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; N/A&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; N/A&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; Y&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 27109</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">Self-heal Daemon on sn-1.local&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; N/A&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; N/A&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; Y&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 4836
</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">&nbsp;</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">Task Status of Volume mstate</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">------------------------------------------------------------------------------</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">There are no active volume tasks</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Times New Roman&quot;,serif">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">But ps shows</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">ps aux | grep gluster</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">root&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 380&nbsp; 0.0&nbsp; 0.6 377892 13760 ?&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; Ss&nbsp;&nbsp; 20:07&nbsp;&nbsp; 0:00 python /usr/sbin/glustereventsd --pid-file /var/run/glustereventsd.pid</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">root&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 409&nbsp; 0.0&nbsp; 0.4&nbsp; 82964&nbsp; 9148 ?&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; S&nbsp;&nbsp;&nbsp; 20:07&nbsp;&nbsp; 0:00 python /usr/sbin/glustereventsd --pid-file /var/run/glustereventsd.pid</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">root&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 811&nbsp; 0.5&nbsp; 1.0 528720 21636 ?&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; Ssl&nbsp; 20:07&nbsp;&nbsp; 0:01 /usr/sbin/glusterd --volfile=/opt/nokia/libexec/StorageUtils/etc/glusterd/glusterd.vol -p /run/glusterd.pid</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">root&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 1088&nbsp; 0.1&nbsp; 0.6 1036648 13996 ?&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; Ssl&nbsp; 20:07&nbsp;&nbsp; 0:00 /usr/sbin/glusterfsd -s sn-0.local --volfile-id export.sn-0.local.mnt-bricks-export-brick -p /var/run/gluster/vols/export/sn-0.local-mnt-bricks-export-brick.pid
 -S /var/run/gluster/44f098f14fe45b74b61c5a1d98993d15.socket --brick-name /mnt/bricks/export/brick -l /var/log/glusterfs/bricks/mnt-bricks-export-brick.log --xlator-option *-posix.glusterd-uuid=f830d809-c935-451f-853a-1364f76a4160 --brick-port 49152 --xlator-option
 export-server.listen-port=49152 --xlator-option transport.socket.bind-address=sn-0.local</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">root&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 1153&nbsp; 1.8&nbsp; 0.7 1234036 14664 ?&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; Ssl&nbsp; 20:07&nbsp;&nbsp; 0:06 /usr/sbin/glusterfsd -s sn-0.local --volfile-id log.sn-0.local.mnt-bricks-log-brick -p /var/run/gluster/vols/log/sn-0.local-mnt-bricks-log-brick.pid
 -S /var/run/gluster/0af76d9541921cccad34e5fd3041a44d.socket --brick-name /mnt/bricks/log/brick -l /var/log/glusterfs/bricks/mnt-bricks-log-brick.log --xlator-option *-posix.glusterd-uuid=f830d809-c935-451f-853a-1364f76a4160 --brick-port 49156 --xlator-option
 log-server.listen-port=49156 --xlator-option transport.socket.bind-address=sn-0.local</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">root&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 1155&nbsp; 0.0&nbsp; 0.5 772996 10580 ?&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; Ssl&nbsp; 20:07&nbsp;&nbsp; 0:00 /usr/sbin/glusterfsd -s sn-0.local --volfile-id mstate.sn-0.local.mnt-bricks-mstate-brick -p /var/run/gluster/vols/mstate/sn-0.local-mnt-bricks-mstate-brick.pid
 -S /var/run/gluster/79d00394b1085adcf7dc1fd4c57f71bc.socket --brick-name /mnt/bricks/mstate/brick -l /var/log/glusterfs/bricks/mnt-bricks-mstate-brick.log --xlator-option *-posix.glusterd-uuid=f830d809-c935-451f-853a-1364f76a4160 --brick-port 49154 --xlator-option
 mstate-server.listen-port=<b><span style="color:red">49154</span></b> --xlator-option transport.socket.bind-address=sn-0.local</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">root&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 1167&nbsp; 0.4&nbsp; 0.7 1235512 15488 ?&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; Ssl&nbsp; 20:07&nbsp;&nbsp; 0:01 /usr/sbin/glusterfsd -s sn-0.local --volfile-id services.sn-0.local.mnt-bricks-services-brick -p /var/run/gluster/vols/services/sn-0.local-mnt-bricks-services-brick.pid
 -S /var/run/gluster/fc16d9fe6fd0695555a5592343da7ba3.socket --brick-name /mnt/bricks/services/brick -l /var/log/glusterfs/bricks/mnt-bricks-services-brick.log --xlator-option *-posix.glusterd-uuid=f830d809-c935-451f-853a-1364f76a4160 --brick-port 49155 --xlator-option
 services-server.listen-port=49155 --xlator-option transport.socket.bind-address=sn-0.local</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">root&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 1188&nbsp; 0.0&nbsp; 0.5 644916 11896 ?&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; Ssl&nbsp; 20:07&nbsp;&nbsp; 0:00 /usr/sbin/glusterfs -s sn-0.local --volfile-id gluster/glustershd -p /var/run/gluster/glustershd/glustershd.pid -l
 /var/log/glusterfs/glustershd.log -S /var/run/gluster/c405d28a3a79d88436a826d75cf7153a.socket --xlator-option *replicate*.node-uuid=f830d809-c935-451f-853a-1364f76a4160</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">root&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 2324&nbsp; 0.0&nbsp; 0.6 629784 13932 ?&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; Ssl&nbsp; 20:07&nbsp;&nbsp; 0:00 /usr/sbin/glusterfs --acl --volfile-server=sn-0.local --volfile-server=sn-1.local --volfile-id=/services /mnt/services</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">root&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 2422&nbsp; 0.0&nbsp; 0.5 431036 10604 ?&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; Ssl&nbsp; 20:07&nbsp;&nbsp; 0:00 /usr/sbin/glusterfs --acl --volfile-server=sn-0.local --volfile-server=sn-1.local --volfile-id=/export /mnt/export</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">root&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 2520&nbsp; 0.0&nbsp; 0.5 498712 11024 ?&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; Ssl&nbsp; 20:07&nbsp;&nbsp; 0:00 /usr/sbin/glusterfs --acl --volfile-server=sn-0.local --volfile-server=sn-1.local --volfile-id=/log /mnt/log</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">root&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 2618&nbsp; 0.0&nbsp; 0.5 431036 10476 ?&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; Ssl&nbsp; 20:07&nbsp;&nbsp; 0:00 /usr/sbin/glusterfs --acl --volfile-server=sn-0.local --volfile-server=sn-1.local --volfile-id=/mstate /mnt/mstate</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">root&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 8136&nbsp; 0.0&nbsp; 0.0&nbsp;&nbsp; 8224&nbsp;&nbsp; 784 pts/0&nbsp;&nbsp;&nbsp; S&#43;&nbsp;&nbsp; 20:13&nbsp;&nbsp; 0:00 grep gluster</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">[root@sn-0:/root]</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Times New Roman&quot;,serif">&nbsp;</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<b><span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">So glustershd can not connect with mstate brick on sn-0</span></b><span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">:</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Times New Roman&quot;,serif">&nbsp;</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">gluster volume heal mstate info</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">Brick sn-0.local:/mnt/bricks/mstate/brick</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<b><span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif;color:red">Status: Transport endpoint is not connected</span></b><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">Number of entries: -</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">&nbsp;</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">Brick sn-1.local:/mnt/bricks/mstate/brick</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">Status: Connected</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">Number of entries: 0</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Times New Roman&quot;,serif">&nbsp;</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Times New Roman&quot;,serif">&nbsp;</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Times New Roman&quot;,serif">&nbsp;</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">From sn-0 glusterd log I find:</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">[2017-12-09 18:07:34.366851] I [glusterd-utils.c:5926:glusterd_brick_start] 0-management: starting a fresh brick process for brick /mnt/bricks/export/brick</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">[2017-12-09 18:07:34.368484] I [rpc-clnt.c:1044:rpc_clnt_connection_init] 0-management: setting frame-timeout to 600</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">[2017-12-09 18:07:34.406488] I [glusterd-utils.c:5926:glusterd_brick_start] 0-management: starting a fresh brick process for brick /mnt/bricks/log/brick</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">[2017-12-09 18:07:34.410384] I [rpc-clnt.c:1044:rpc_clnt_connection_init] 0-management: setting frame-timeout to 600</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif;color:red">[2017-12-09 18:07:34.410530] I [glusterd-utils.c:5926:glusterd_brick_start] 0-management: starting a fresh brick process for brick /mnt/bricks/mstate/brick</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">[2017-12-09 18:07:34.413819] I [rpc-clnt.c:1044:rpc_clnt_connection_init] 0-management: setting frame-timeout to 600</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">[2017-12-09 18:07:34.413926] C [MSGID: 106003] [glusterd-server-quorum.c:352:glusterd_do_volume_quorum_action] 0-management: Server quorum regained for volume services. Starting local
 bricks.</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">[2017-12-09 18:07:34.414031] I [glusterd-utils.c:5926:glusterd_brick_start] 0-management: starting a fresh brick process for brick /mnt/bricks/services/brick</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">[2017-12-09 18:07:34.417573] I [rpc-clnt.c:1044:rpc_clnt_connection_init] 0-management: setting frame-timeout to 600</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">[2017-12-09 18:07:34.462163] I [MSGID: 106492] [glusterd-handler.c:2718:__glusterd_handle_friend_update] 0-glusterd: Received friend update from uuid: e4ea0550-873b-410c-a29e-3901c4737cd8</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">[2017-12-09 18:07:34.470228] I [MSGID: 106502] [glusterd-handler.c:2763:__glusterd_handle_friend_update] 0-management: Received my uuid as Friend</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">[2017-12-09 18:07:34.470432] I [MSGID: 106492] [glusterd-handler.c:2718:__glusterd_handle_friend_update] 0-glusterd: Received friend update from uuid: feddd74c-c768-4157-b152-31d5dd488dcc</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">[2017-12-09 18:07:34.470493] I [MSGID: 106502] [glusterd-handler.c:2763:__glusterd_handle_friend_update] 0-management: Received my uuid as Friend</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">[2017-12-09 18:07:34.479540] I [MSGID: 106163] [glusterd-handshake.c:1316:__glusterd_mgmt_hndsk_versions_ack] 0-management: using the op-version 31202</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">[2017-12-09 18:07:34.488767] I [socket.c:2478:socket_event_handler] 0-transport: EPOLLERR - disconnecting now</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">[2017-12-09 18:07:34.489020] I [MSGID: 106005] [glusterd-handler.c:6071:__glusterd_brick_rpc_notify] 0-management: Brick sn-0.local:/mnt/bricks/export/brick has disconnected from glusterd.</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">[2017-12-09 18:07:34.489450] I [socket.c:2478:socket_event_handler] 0-transport: EPOLLERR - disconnecting now</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">[2017-12-09 18:07:34.489717] I [MSGID: 106005] [glusterd-handler.c:6071:__glusterd_brick_rpc_notify] 0-management: Brick sn-0.local:/mnt/bricks/log/brick has disconnected from glusterd.</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">[2017-12-09 18:07:34.490114] I [socket.c:2478:socket_event_handler] 0-transport: EPOLLERR - disconnecting now</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">[2017-12-09 18:07:34.490398] I [MSGID: 106005] [glusterd-handler.c:6071:__glusterd_brick_rpc_notify] 0-management: Brick sn-0.local:/mnt/bricks/mstate/brick has disconnected from glusterd.</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">[2017-12-09 18:07:34.490785] I [socket.c:2478:socket_event_handler] 0-transport: EPOLLERR - disconnecting now</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">[2017-12-09 18:07:34.491049] I [MSGID: 106005] [glusterd-handler.c:6071:__glusterd_brick_rpc_notify] 0-management: Brick sn-0.local:/mnt/bricks/services/brick has disconnected from
 glusterd.</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">[2017-12-09 18:07:34.491514] I [MSGID: 106493] [glusterd-rpc-ops.c:486:__glusterd_friend_add_cbk] 0-glusterd: Received ACC from uuid: feddd74c-c768-4157-b152-31d5dd488dcc, host: sn-1.local,
 port: 0</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">[2017-12-09 18:07:34.501902] I [glusterd-utils.c:5926:glusterd_brick_start] 0-management: starting a fresh brick process for brick /mnt/bricks/log/brick</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">[2017-12-09 18:07:34.503376] I [rpc-clnt.c:1044:rpc_clnt_connection_init] 0-management: setting frame-timeout to 600</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif;color:red">[2017-12-09 18:07:34.503502] I [glusterd-utils.c:5926:glusterd_brick_start] 0-management: starting a fresh brick process for brick /mnt/bricks/mstate/brick</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">[2017-12-09 18:07:34.506655] I [rpc-clnt.c:1044:rpc_clnt_connection_init] 0-management: setting frame-timeout to 600</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">[2017-12-09 18:07:34.507887] I [socket.c:2478:socket_event_handler] 0-transport: EPOLLERR - disconnecting now</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">[2017-12-09 18:07:34.508188] I [MSGID: 106005] [glusterd-handler.c:6071:__glusterd_brick_rpc_notify] 0-management: Brick sn-0.local:/mnt/bricks/log/brick has disconnected from glusterd.</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">[2017-12-09 18:07:34.509479] I [socket.c:2478:socket_event_handler] 0-transport: EPOLLERR - disconnecting now</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">……</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">[2017-12-09 18:07:36.023087] I [glusterd-utils.c:5834:glusterd_brick_start] 0-management: discovered already-running brick /mnt/bricks/export/brick</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">[2017-12-09 18:07:36.023106] I [MSGID: 106143] [glusterd-pmap.c:280:pmap_registry_bind] 0-pmap: adding brick /mnt/bricks/export/brick on port 49152</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">[2017-12-09 18:07:36.060419] I [glusterd-utils.c:5834:glusterd_brick_start] 0-management: discovered already-running brick /mnt/bricks/log/brick</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">[2017-12-09 18:07:36.060442] I [MSGID: 106143] [glusterd-pmap.c:280:pmap_registry_bind] 0-pmap: adding brick /mnt/bricks/log/brick on port 49156</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">[2017-12-09 18:07:36.094987] I [glusterd-utils.c:5834:glusterd_brick_start] 0-management: discovered already-running brick /mnt/bricks/mstate/brick</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">[2017-12-09 18:07:36.095008] I [MSGID: 106143] [glusterd-pmap.c:280:pmap_registry_bind] 0-pmap: adding brick /mnt/bricks/mstate/brick on port
<span style="color:red">49157</span></span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">[2017-12-09 18:07:36.129059] I [glusterd-utils.c:5834:glusterd_brick_start] 0-management: discovered already-running brick /mnt/bricks/services/brick</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">[2017-12-09 18:07:36.129108] I [MSGID: 106143] [glusterd-pmap.c:280:pmap_registry_bind] 0-pmap: adding brick /mnt/bricks/services/brick on port 49155</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">[2017-12-09 18:07:36.159408] I [rpc-clnt.c:1044:rpc_clnt_connection_init] 0-snapd: setting frame-timeout to 600</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Times New Roman&quot;,serif">&nbsp;</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">……</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">[2017-12-09 18:07:36.919192] I [MSGID: 106493] [glusterd-rpc-ops.c:701:__glusterd_friend_update_cbk] 0-management: Received ACC from uuid: e4ea0550-873b-410c-a29e-3901c4737cd8</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">[2017-12-09 18:07:36.920269] I [MSGID: 106492] [glusterd-handler.c:2718:__glusterd_handle_friend_update] 0-glusterd: Received friend update from uuid: e4ea0550-873b-410c-a29e-3901c4737cd8</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">[2017-12-09 18:07:36.927413] I [MSGID: 106502] [glusterd-handler.c:2763:__glusterd_handle_friend_update] 0-management: Received my uuid as Friend</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">[2017-12-09 18:07:36.949762] I [MSGID: 106493] [glusterd-rpc-ops.c:701:__glusterd_friend_update_cbk] 0-management: Received ACC from uuid: feddd74c-c768-4157-b152-31d5dd488dcc</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">[2017-12-09 18:07:36.949817] I [MSGID: 106492] [glusterd-handler.c:2718:__glusterd_handle_friend_update] 0-glusterd: Received friend update from uuid: feddd74c-c768-4157-b152-31d5dd488dcc</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">[2017-12-09 18:07:36.949882] I [MSGID: 106502] [glusterd-handler.c:2763:__glusterd_handle_friend_update] 0-management: Received my uuid as Friend</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">[2017-12-09 18:07:37.768326] W [MSGID: 101095] [xlator.c:213:xlator_dynload] 0-xlator: /usr/lib64/glusterfs/3.12.3/xlator/nfs/server.so: cannot open shared object file: No such file
 or directory</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">[2017-12-09 18:07:37.783811] I [MSGID: 106143] [glusterd-pmap.c:280:pmap_registry_bind] 0-pmap: adding brick /mnt/bricks/services/brick on port 49155</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">[2017-12-09 18:07:37.783901] I [MSGID: 106143] [glusterd-pmap.c:280:pmap_registry_bind] 0-pmap: adding brick /mnt/bricks/mstate/brick on port
<span style="color:red">49154</span></span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">[2017-12-09 18:07:37.783966] I [MSGID: 106143] [glusterd-pmap.c:280:pmap_registry_bind] 0-pmap: adding brick /mnt/bricks/log/brick on port 49156</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">[2017-12-09 18:07:37.784032] I [MSGID: 106143] [glusterd-pmap.c:280:pmap_registry_bind] 0-pmap: adding brick /mnt/bricks/export/brick on port 49152</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">[2017-12-09 18:07:37.810305] W [MSGID: 101095] [xlator.c:162:xlator_volopt_dynload] 0-xlator: /usr/lib64/glusterfs/3.12.3/xlator/nfs/server.so: cannot open shared object file: No such
 file or directory</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Times New Roman&quot;,serif">&nbsp;</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">And the mstate brick process on sn-0 shows:</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Times New Roman&quot;,serif">&nbsp;</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Times New Roman&quot;,serif">&nbsp;</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">[2017-12-09 18:07:02.743046] W [glusterfsd.c:1375:cleanup_and_exit] (--&gt;/lib64/libpthread.so.0(&#43;0x7454) [0x7f5df4db6454] --&gt;/usr/sbin/glusterfsd(glusterfs_sigwaiter&#43;0xdc) [0x40a50c]
 --&gt;/usr/sbin/glusterfsd(cleanup_and_exit&#43;0x77) [0x4089ec] ) 0-: received signum (15), shutting down</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">[2017-12-09 18:07:34.416169] I [MSGID: 100030] [glusterfsd.c:2524:main] 0-/usr/sbin/glusterfsd: Started running /usr/sbin/glusterfsd version 3.12.3 (args: /usr/sbin/glusterfsd -s sn-0.local
 --volfile-id mstate.sn-0.local.mnt-bricks-mstate-brick -p /var/run/gluster/vols/mstate/sn-0.local-mnt-bricks-mstate-brick.pid -S /var/run/gluster/79d00394b1085adcf7dc1fd4c57f71bc.socket --brick-name /mnt/bricks/mstate/brick -l /var/log/glusterfs/bricks/mnt-bricks-mstate-brick.log
 --xlator-option *-posix.glusterd-uuid=f830d809-c935-451f-853a-1364f76a4160 --brick-port
<span style="color:red">49154</span> --xlator-option mstate-server.listen-port=49154 --xlator-option transport.socket.bind-address=sn-0.local)</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">[2017-12-09 18:07:34.511598] I [MSGID: 100030] [glusterfsd.c:2524:main] 0-/usr/sbin/glusterfsd: Started running /usr/sbin/glusterfsd version 3.12.3 (args: /usr/sbin/glusterfsd -s sn-0.local
 --volfile-id mstate.sn-0.local.mnt-bricks-mstate-brick -p /var/run/gluster/vols/mstate/sn-0.local-mnt-bricks-mstate-brick.pid -S /var/run/gluster/79d00394b1085adcf7dc1fd4c57f71bc.socket --brick-name /mnt/bricks/mstate/brick -l /var/log/glusterfs/bricks/mnt-bricks-mstate-brick.log
 --xlator-option *-posix.glusterd-uuid=f830d809-c935-451f-853a-1364f76a4160 --brick-port
<span style="color:red">49157</span> --xlator-option mstate-server.listen-port=49157 --xlator-option transport.socket.bind-address=sn-0.local)</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">[2017-12-09 18:07:34.542268] E [MSGID: 100018] [glusterfsd.c:2109:glusterfs_pidfile_update] 0-glusterfsd: pidfile /var/run/gluster/vols/mstate/sn-0.local-mnt-bricks-mstate-brick.pid
 lock failed [Resource temporarily unavailable]</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">[2017-12-09 18:07:34.549242] W [MSGID: 101002] [options.c:995:xl_opt_validate] 0-glusterfs: option 'address-family' is deprecated, preferred is 'transport.address-family', continuing
 with correction</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">[2017-12-09 18:07:34.550119] I [MSGID: 101190] [event-epoll.c:613:event_dispatch_epoll_worker] 0-epoll: Started thread with index 1</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">[2017-12-09 18:07:37.690037] I [rpcsvc.c:2322:rpcsvc_set_outstanding_rpc_limit] 0-rpc-service: Configured rpc.outstanding-rpc-limit with value 64</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">[2017-12-09 18:07:37.690125] W [MSGID: 101002] [options.c:995:xl_opt_validate] 0-mstate-server: option 'listen-port' is deprecated, preferred is 'transport.socket.listen-port', continuing
 with correction</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Times New Roman&quot;,serif">&nbsp;</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">……</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">&nbsp;</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">Final graph:</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">&#43;------------------------------------------------------------------------------&#43;</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">&nbsp; 1: volume mstate-posix</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">&nbsp; 2:&nbsp;&nbsp;&nbsp;&nbsp; type storage/posix</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">&nbsp; 3:&nbsp;&nbsp;&nbsp;&nbsp; option glusterd-uuid f830d809-c935-451f-853a-1364f76a4160</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">&nbsp; 4:&nbsp;&nbsp;&nbsp;&nbsp; option directory /mnt/bricks/mstate/brick</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">&nbsp; 5:&nbsp;&nbsp;&nbsp;&nbsp; option volume-id 5fc29728-b5de-4b5f-8f3c-9654acda2577</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">&nbsp; 6:&nbsp;&nbsp;&nbsp;&nbsp; option shared-brick-count 1</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">&nbsp; 7: end-volume</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">&nbsp; 8:&nbsp;
</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">&nbsp; 9: volume mstate-trash</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">10:&nbsp;&nbsp;&nbsp;&nbsp; type features/trash</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">11:&nbsp;&nbsp;&nbsp;&nbsp; option trash-dir .trashcan</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">12:&nbsp;&nbsp;&nbsp;&nbsp; option brick-path /mnt/bricks/mstate/brick</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">13:&nbsp;&nbsp;&nbsp;&nbsp; option trash-internal-op off</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">14:&nbsp;&nbsp;&nbsp;&nbsp; subvolumes mstate-posix</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">15: end-volume</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">16:&nbsp;
</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">17: volume mstate-changetimerecorder</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">18:&nbsp;&nbsp;&nbsp;&nbsp; type features/changetimerecorder</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">19:&nbsp;&nbsp;&nbsp;&nbsp; option db-type sqlite3</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">20:&nbsp;&nbsp;&nbsp;&nbsp; option hot-brick off</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">21:&nbsp;&nbsp;&nbsp;&nbsp; option db-name brick.db</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">22:&nbsp;&nbsp;&nbsp;&nbsp; option db-path /mnt/bricks/mstate/brick/.glusterfs/</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">23:&nbsp;&nbsp;&nbsp;&nbsp; option record-exit off</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">24:&nbsp;&nbsp;&nbsp;&nbsp; option ctr_link_consistency off</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">25:&nbsp;&nbsp;&nbsp;&nbsp; option ctr_lookupheal_link_timeout 300</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">26:&nbsp;&nbsp;&nbsp;&nbsp; option ctr_lookupheal_inode_timeout 300</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">27:&nbsp;&nbsp;&nbsp;&nbsp; option record-entry on</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">28:&nbsp;&nbsp;&nbsp;&nbsp; option ctr-enabled off</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">29:&nbsp;&nbsp;&nbsp;&nbsp; option record-counters off</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">30:&nbsp;&nbsp;&nbsp;&nbsp; option ctr-record-metadata-heat off</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">31:&nbsp;&nbsp;&nbsp;&nbsp; option sql-db-cachesize 12500</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">32:&nbsp;&nbsp;&nbsp;&nbsp; option sql-db-wal-autocheckpoint 25000</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">33:&nbsp;&nbsp;&nbsp;&nbsp; subvolumes mstate-trash</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">34: end-volume</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">35:&nbsp;
</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">36: volume mstate-changelog</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">37:&nbsp;&nbsp;&nbsp;&nbsp; type features/changelog</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">38:&nbsp;&nbsp;&nbsp;&nbsp; option changelog-brick /mnt/bricks/mstate/brick</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">39:&nbsp;&nbsp;&nbsp;&nbsp; option changelog-dir /mnt/bricks/mstate/brick/.glusterfs/changelogs</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">40:&nbsp;&nbsp;&nbsp;&nbsp; option changelog-barrier-timeout 120</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">41:&nbsp;&nbsp;&nbsp;&nbsp; subvolumes mstate-changetimerecorder</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">42: end-volume</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">43:&nbsp;
</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">44: volume mstate-bitrot-stub</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">45:&nbsp;&nbsp;&nbsp;&nbsp; type features/bitrot-stub</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">46:&nbsp;&nbsp;&nbsp;&nbsp; option export /mnt/bricks/mstate/brick</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">47:&nbsp;&nbsp;&nbsp;&nbsp; option bitrot disable</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">48:&nbsp;&nbsp;&nbsp;&nbsp; subvolumes mstate-changelog</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">49: end-volume</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">50:&nbsp;
</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">51: volume mstate-access-control</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">52:&nbsp;&nbsp;&nbsp;&nbsp; type features/access-control</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">53:&nbsp;&nbsp;&nbsp;&nbsp; subvolumes mstate-bitrot-stub</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">54: end-volume</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">55:&nbsp;
</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">56: volume mstate-locks</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">57:&nbsp;&nbsp;&nbsp;&nbsp; type features/locks</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">58:&nbsp;&nbsp;&nbsp;&nbsp; subvolumes mstate-access-control</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">59: end-volume</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">60:&nbsp;
</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">61: volume mstate-worm</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">62:&nbsp;&nbsp;&nbsp;&nbsp; type features/worm</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">63:&nbsp;&nbsp;&nbsp;&nbsp; option worm off</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">64:&nbsp;&nbsp;&nbsp;&nbsp; option worm-file-level off</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">65:&nbsp;&nbsp;&nbsp;&nbsp; subvolumes mstate-locks</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">66: end-volume</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">67:&nbsp;
</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">68: volume mstate-read-only</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">69:&nbsp;&nbsp;&nbsp;&nbsp; type features/read-only</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">70:&nbsp;&nbsp;&nbsp;&nbsp; option read-only off</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">71:&nbsp;&nbsp;&nbsp;&nbsp; subvolumes mstate-worm</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">72: end-volume</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">73:&nbsp;
</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">74: volume mstate-leases</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">75:&nbsp;&nbsp;&nbsp;&nbsp; type features/leases</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">76:&nbsp;&nbsp;&nbsp;&nbsp; option leases off</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">77:&nbsp;&nbsp;&nbsp;&nbsp; subvolumes mstate-read-only</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">78: end-volume</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">79:&nbsp;
</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">80: volume mstate-upcall</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">81:&nbsp;&nbsp;&nbsp;&nbsp; type features/upcall</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">82:&nbsp;&nbsp;&nbsp;&nbsp; option cache-invalidation off</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">83:&nbsp;&nbsp;&nbsp;&nbsp; subvolumes mstate-leases</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">84: end-volume</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">85:&nbsp;
</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">86: volume mstate-io-threads</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">87:&nbsp;&nbsp;&nbsp;&nbsp; type performance/io-threads</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">88:&nbsp;&nbsp;&nbsp;&nbsp; subvolumes mstate-upcall</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">89: end-volume</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">90:&nbsp;
</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">91: volume mstate-selinux</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">92:&nbsp;&nbsp;&nbsp;&nbsp; type features/selinux</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">93:&nbsp;&nbsp;&nbsp;&nbsp; option selinux on</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">94:&nbsp;&nbsp;&nbsp;&nbsp; subvolumes mstate-io-threads</span><span lang="EN-US"><o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;text-align:justify;text-justify:inter-ideograph">
<span lang="EN-US" style="font-size:10.5pt;font-family:&quot;Calibri&quot;,sans-serif">95: end-volume&lt;</span><span lang="EN-US"><o:p></o:p></span></p>
</div>
</blockquote>
</div>
</div>
</blockquote>
</div>
</div>
</div>
</div>
</div>
</div>
</blockquote>
</div>
</div>
</div>
</div>
</div>
</blockquote>
</div>
</div>
</div>
</div>
</blockquote>
</div>
</div>
</blockquote>
</div>
</div>
</blockquote>
</div>
</div>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US">--
<o:p></o:p></span></p>
</div>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span lang="EN-US">- Atin (atinm)<o:p></o:p></span></p>
</div>
</div>
</div>
</blockquote>
</div>
<p class="MsoNormal"><span lang="EN-US"><o:p>&nbsp;</o:p></span></p>
</div>
</div>
</body>
</html>