[2018-08-01 08:19:18.800900] I [gsyncdstatus(monitor):243:set_worker_status] GeorepStatus: Worker Status Change status=Initializing... [2018-08-01 08:19:18.808379] I [monitor(monitor):158:monitor] Monitor: starting gsyncd worker brick=/urd-gds/gluster slave_node=urd-gds-geo-000 [2018-08-01 08:19:18.843025] I [gsyncd(agent /urd-gds/gluster):297:main] : Using session config file path=/var/lib/glusterd/geo-replication/urd-gds-volume_urd-gds-geo-001_urd-gds-volume/gsyncd.conf [2018-08-01 08:19:18.843529] I [gsyncd(worker /urd-gds/gluster):297:main] : Using session config file path=/var/lib/glusterd/geo-replication/urd-gds-volume_urd-gds-geo-001_urd-gds-volume/gsyncd.conf [2018-08-01 08:19:18.843996] I [changelogagent(agent /urd-gds/gluster):72:__init__] ChangelogAgent: Agent listining... [2018-08-01 08:19:18.852423] I [resource(worker /urd-gds/gluster):1348:connect_remote] SSH: Initializing SSH connection between master and slave... [2018-08-01 08:19:20.486857] I [resource(worker /urd-gds/gluster):1395:connect_remote] SSH: SSH connection between master and slave established. duration=1.6342 [2018-08-01 08:19:20.487375] I [resource(worker /urd-gds/gluster):1067:connect] GLUSTER: Mounting gluster volume locally... [2018-08-01 08:19:21.722038] I [resource(worker /urd-gds/gluster):1090:connect] GLUSTER: Mounted gluster volume duration=1.2343 [2018-08-01 08:19:21.722587] I [subcmds(worker /urd-gds/gluster):70:subcmd_worker] : Worker spawn successful. Acknowledging back to monitor [2018-08-01 08:19:24.668253] I [gsyncd(config-get):297:main] : Using session config file path=/var/lib/glusterd/geo-replication/urd-gds-volume_urd-gds-geo-001_urd-gds-volume/gsyncd.conf [2018-08-01 08:19:24.737204] I [gsyncd(status):297:main] : Using session config file path=/var/lib/glusterd/geo-replication/urd-gds-volume_urd-gds-geo-001_urd-gds-volume/gsyncd.conf [2018-08-01 08:19:33.678735] I [master(worker /urd-gds/gluster):1534:register] _GMaster: Working dir path=/var/lib/misc/gluster/gsyncd/urd-gds-volume_urd-gds-geo-001_urd-gds-volume/urd-gds-gluster [2018-08-01 08:19:33.679353] I [resource(worker /urd-gds/gluster):1253:service_loop] GLUSTER: Register time time=1533111573 [2018-08-01 08:19:33.727123] I [gsyncdstatus(worker /urd-gds/gluster):276:set_active] GeorepStatus: Worker Status Change status=Active [2018-08-01 08:19:33.729835] I [gsyncdstatus(worker /urd-gds/gluster):248:set_worker_crawl_status] GeorepStatus: Crawl Status Change status=History Crawl [2018-08-01 08:19:33.730175] I [master(worker /urd-gds/gluster):1448:crawl] _GMaster: starting history crawl turns=1 stime=(1523903409, 0) entry_stime=None etime=1533111573 [2018-08-01 08:19:34.753371] I [master(worker /urd-gds/gluster):1477:crawl] _GMaster: slave's time stime=(1523903409, 0) [2018-08-01 08:19:35.111413] I [gsyncd(config-get):297:main] : Using session config file path=/var/lib/glusterd/geo-replication/urd-gds-volume_urd-gds-geo-001_urd-gds-volume/gsyncd.conf [2018-08-01 08:19:35.185450] I [gsyncd(status):297:main] : Using session config file path=/var/lib/glusterd/geo-replication/urd-gds-volume_urd-gds-geo-001_urd-gds-volume/gsyncd.conf [2018-08-01 08:24:12.543174] I [master(worker /urd-gds/gluster):1885:syncjob] Syncer: Sync Time Taken duration=277.3017 num_files=14700 job=3 return_code=23 [2018-08-01 08:24:39.763540] W [master(worker /urd-gds/gluster):1287:process] _GMaster: incomplete sync, retrying changelogs files=['CHANGELOG.1523903410', 'CHANGELOG.1523903425', 'CHANGELOG.1523903440', 'CHANGELOG.1523903455', 'CHANGELOG.1523903470', 'CHANGELOG.1523903485', 'CHANGELOG.1523903500', 'CHANGELOG.1523903515', 'CHANGELOG.1523903530'] [2018-08-01 08:25:22.292231] I [master(worker /urd-gds/gluster):1885:syncjob] Syncer: Sync Time Taken duration=41.2383 num_files=5779 job=1 return_code=23 [2018-08-01 08:25:33.849513] W [master(worker /urd-gds/gluster):1287:process] _GMaster: incomplete sync, retrying changelogs files=['CHANGELOG.1523903410', 'CHANGELOG.1523903425', 'CHANGELOG.1523903440', 'CHANGELOG.1523903455', 'CHANGELOG.1523903470', 'CHANGELOG.1523903485', 'CHANGELOG.1523903500', 'CHANGELOG.1523903515', 'CHANGELOG.1523903530'] [2018-08-01 08:26:15.171473] I [master(worker /urd-gds/gluster):1885:syncjob] Syncer: Sync Time Taken duration=40.5618 num_files=5779 job=2 return_code=23 [2018-08-01 08:26:27.953500] W [master(worker /urd-gds/gluster):1287:process] _GMaster: incomplete sync, retrying changelogs files=['CHANGELOG.1523903410', 'CHANGELOG.1523903425', 'CHANGELOG.1523903440', 'CHANGELOG.1523903455', 'CHANGELOG.1523903470', 'CHANGELOG.1523903485', 'CHANGELOG.1523903500', 'CHANGELOG.1523903515', 'CHANGELOG.1523903530'] [2018-08-01 08:27:12.614435] I [master(worker /urd-gds/gluster):1885:syncjob] Syncer: Sync Time Taken duration=43.9321 num_files=5779 job=3 return_code=23 [2018-08-01 08:27:25.407852] W [master(worker /urd-gds/gluster):1287:process] _GMaster: incomplete sync, retrying changelogs files=['CHANGELOG.1523903410', 'CHANGELOG.1523903425', 'CHANGELOG.1523903440', 'CHANGELOG.1523903455', 'CHANGELOG.1523903470', 'CHANGELOG.1523903485', 'CHANGELOG.1523903500', 'CHANGELOG.1523903515', 'CHANGELOG.1523903530'] [2018-08-01 08:27:28.688704] I [master(worker /urd-gds/gluster):1885:syncjob] Syncer: Sync Time Taken duration=2.7667 num_files=409 job=1 return_code=23 [2018-08-01 08:28:06.926116] I [master(worker /urd-gds/gluster):1885:syncjob] Syncer: Sync Time Taken duration=40.7955 num_files=5370 job=3 return_code=23 [2018-08-01 08:28:18.813146] W [master(worker /urd-gds/gluster):1287:process] _GMaster: incomplete sync, retrying changelogs files=['CHANGELOG.1523903410', 'CHANGELOG.1523903425', 'CHANGELOG.1523903440', 'CHANGELOG.1523903455', 'CHANGELOG.1523903470', 'CHANGELOG.1523903485', 'CHANGELOG.1523903500', 'CHANGELOG.1523903515', 'CHANGELOG.1523903530'] [2018-08-01 08:29:02.298345] I [master(worker /urd-gds/gluster):1885:syncjob] Syncer: Sync Time Taken duration=42.8586 num_files=5779 job=3 return_code=23 [2018-08-01 08:29:15.258282] W [master(worker /urd-gds/gluster):1287:process] _GMaster: incomplete sync, retrying changelogs files=['CHANGELOG.1523903410', 'CHANGELOG.1523903425', 'CHANGELOG.1523903440', 'CHANGELOG.1523903455', 'CHANGELOG.1523903470', 'CHANGELOG.1523903485', 'CHANGELOG.1523903500', 'CHANGELOG.1523903515', 'CHANGELOG.1523903530'] [2018-08-01 08:29:16.644232] I [master(worker /urd-gds/gluster):1885:syncjob] Syncer: Sync Time Taken duration=0.8308 num_files=126 job=3 return_code=23 [2018-08-01 08:29:58.745820] I [master(worker /urd-gds/gluster):1885:syncjob] Syncer: Sync Time Taken duration=42.9430 num_files=5653 job=1 return_code=23 [2018-08-01 08:30:11.605155] W [master(worker /urd-gds/gluster):1287:process] _GMaster: incomplete sync, retrying changelogs files=['CHANGELOG.1523903410', 'CHANGELOG.1523903425', 'CHANGELOG.1523903440', 'CHANGELOG.1523903455', 'CHANGELOG.1523903470', 'CHANGELOG.1523903485', 'CHANGELOG.1523903500', 'CHANGELOG.1523903515', 'CHANGELOG.1523903530'] [2018-08-01 08:30:55.456170] I [master(worker /urd-gds/gluster):1885:syncjob] Syncer: Sync Time Taken duration=43.2509 num_files=5779 job=3 return_code=23 [2018-08-01 08:31:08.246109] W [master(worker /urd-gds/gluster):1287:process] _GMaster: incomplete sync, retrying changelogs files=['CHANGELOG.1523903410', 'CHANGELOG.1523903425', 'CHANGELOG.1523903440', 'CHANGELOG.1523903455', 'CHANGELOG.1523903470', 'CHANGELOG.1523903485', 'CHANGELOG.1523903500', 'CHANGELOG.1523903515', 'CHANGELOG.1523903530'] [2018-08-01 08:31:52.905714] I [master(worker /urd-gds/gluster):1885:syncjob] Syncer: Sync Time Taken duration=44.0849 num_files=5779 job=1 return_code=23 [2018-08-01 08:32:05.455931] W [master(worker /urd-gds/gluster):1287:process] _GMaster: incomplete sync, retrying changelogs files=['CHANGELOG.1523903410', 'CHANGELOG.1523903425', 'CHANGELOG.1523903440', 'CHANGELOG.1523903455', 'CHANGELOG.1523903470', 'CHANGELOG.1523903485', 'CHANGELOG.1523903500', 'CHANGELOG.1523903515', 'CHANGELOG.1523903530'] [2018-08-01 08:32:50.4466] I [master(worker /urd-gds/gluster):1885:syncjob] Syncer: Sync Time Taken duration=43.9717 num_files=5779 job=3 return_code=23 [2018-08-01 08:33:02.248553] E [master(worker /urd-gds/gluster):1266:process] _GMaster: changelogs could not be processed completely - moving on... files=['CHANGELOG.1523903410', 'CHANGELOG.1523903425', 'CHANGELOG.1523903440', 'CHANGELOG.1523903455', 'CHANGELOG.1523903470', 'CHANGELOG.1523903485', 'CHANGELOG.1523903500', 'CHANGELOG.1523903515', 'CHANGELOG.1523903530'] [2018-08-01 08:33:02.278516] I [master(worker /urd-gds/gluster):1315:process] _GMaster: Entry Time Taken MKD=0 MKN=0 LIN=0 SYM=0 REN=0 RMD=0 CRE=0 duration=0.0000 UNL=0 [2018-08-01 08:33:02.278882] I [master(worker /urd-gds/gluster):1325:process] _GMaster: Data/Metadata Time Taken SETA=0 SETX=0 meta_duration=0.0000 data_duration=1533112382.2788 DATA=0 XATT=0 [2018-08-01 08:33:02.279510] I [master(worker /urd-gds/gluster):1335:process] _GMaster: Batch Completed changelog_end=1523903530 entry_stime=None changelog_start=1523903410 stime=(1523903529, 0) duration=807.5208 num_changelogs=9 mode=history_changelog [2018-08-01 11:27:53.821345] I [gsyncd(config-get):297:main] : Using session config file path=/var/lib/glusterd/geo-replication/urd-gds-volume_urd-gds-geo-001_urd-gds-volume/gsyncd.conf [2018-08-01 11:27:53.893773] I [gsyncd(status):297:main] : Using session config file path=/var/lib/glusterd/geo-replication/urd-gds-volume_urd-gds-geo-001_urd-gds-volume/gsyncd.conf [2018-08-01 19:32:26.33901] I [gsyncd(config-get):297:main] : Using session config file path=/var/lib/glusterd/geo-replication/urd-gds-volume_urd-gds-geo-001_urd-gds-volume/gsyncd.conf [2018-08-01 19:32:26.103063] I [gsyncd(status):297:main] : Using session config file path=/var/lib/glusterd/geo-replication/urd-gds-volume_urd-gds-geo-001_urd-gds-volume/gsyncd.conf