[root@app2 ~]# docker ps |grep heketi ff3d86bd474d docker.io/heketi/heketi@sha256:b99aacd133b76c97fdaef4a85be075e07b5286fc2877bf5255f35ee369b64771 "/usr/bin/heketi-star" 2 hours ago Up 2 hours k8s_heketi_heketi-storage-6-4m61c_glusterfs_028fc807-1faa-11e9-a468-5254007b74fe_0 52033aa162d9 openshift/origin-pod:v3.6.0 "/usr/bin/pod" 2 hours ago Up 2 hours k8s_POD_heketi-storage-6-4m61c_glusterfs_028fc807-1faa-11e9-a468-5254007b74fe_0 [root@app2 ~]# docker logs ff3d86bd474d Setting up heketi database File: /var/lib/heketi/heketi.db Size: 262144 Blocks: 512 IO Block: 131072 regular file Device: 36h/54d Inode: 13717556549865401166 Links: 1 Access: (0600/-rw-------) Uid: ( 0/ root) Gid: ( 0/ root) Access: 2019-01-21 14:17:53.062782412 +0000 Modify: 2019-01-22 06:04:12.692292458 +0000 Change: 2019-01-22 06:04:12.702292637 +0000 Birth: - Heketi v8.0.0-1-g082b556-release-8 [heketi] INFO 2019/01/24 07:31:36 Loaded kubernetes executor [heketi] INFO 2019/01/24 07:31:36 Block: Auto Create Block Hosting Volume set to true [heketi] INFO 2019/01/24 07:31:36 Block: New Block Hosting Volume size 100 GB [heketi] INFO 2019/01/24 07:31:36 GlusterFS Application Loaded [heketi] INFO 2019/01/24 07:31:36 Started Node Health Cache Monitor Authorization loaded Listening on port 8080 [heketi] INFO 2019/01/24 07:31:46 Starting Node Health Status refresh [cmdexec] INFO 2019/01/24 07:31:46 Check Glusterd service status in node storage.matrix.nokia.com [kubeexec] DEBUG 2019/01/24 07:31:46 /src/github.com/heketi/heketi/executors/kubeexec/kubeexec.go:246: Host: storage.matrix.nokia.com Pod: glusterfs-storage-9h5cf Command: systemctl status glusterd Result: ? glusterd.service - GlusterFS, a clustered file-system server Loaded: loaded (/usr/lib/systemd/system/glusterd.service; enabled; vendor preset: disabled) Active: active (running) since Fri 2019-01-18 13:27:57 UTC; 5 days ago Process: 23951 ExecStart=/usr/sbin/glusterd -p /var/run/glusterd.pid --log-level $LOG_LEVEL $GLUSTERD_OPTIONS (code=exited, status=0/SUCCESS) Main PID: 23952 (glusterd) CGroup: /kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod97b11242_cd44_11e8_a361_5254007b74fe.slice/docker-ce7cfe74053c991b94bda7e05b33316da1af1ce519ce170748f1ece89f266143.scope/system.slice/glusterd.service +- 558 /usr/sbin/glusterfsd -s 192.168.89.220 --volfile-id heketidbstorage.192.168.89.220.var-lib-heketi-mounts-vg_018e77166eb3b38a0c250eb8faede2f7-brick_2b6ed29607d007a75f7a7c1bf0da7bae-brick -p /var/run/gluster/vols/heketidbstorage/192.168.89.220-var-lib-heketi-mounts-vg_018e77166eb3b38a0c250eb8faede2f7-brick_2b6ed29607d007a75f7a7c1bf0da7bae-brick.pid -S /var/run/gluster/88c0da28ee877a32.socket --brick-name /var/lib/heketi/mounts/vg_018e77166eb3b38a0c250eb8faede2f7/brick_2b6ed29607d007a75f7a7c1bf0da7bae/brick -l /var/log/glusterfs/bricks/var-lib-heketi-mounts-vg_018e77166eb3b38a0c250eb8faede2f7-brick_2b6ed29607d007a75f7a7c1bf0da7bae-brick.log --xlator-option *-posix.glusterd-uuid=25b5f142-4890-4315-a352-cf947fdf649c --process-name brick --brick-port 49154 --xlator-option heketidbstorage-server.listen-port=49154 +- 901 /usr/sbin/glusterfsd -s 192.168.89.220 --volfile-id glusterfs-registry-volume.192.168.89.220.var-lib-heketi-mounts-vg_5054cf122b4f54021cb9d67752d23c1d-brick_b22c3ab4d7bbc420e05340e8e187bcf5-brick -p /var/run/gluster/vols/glusterfs-registry-volume/192.168.89.220-var-lib-heketi-mounts-vg_5054cf122b4f54021cb9d67752d23c1d-brick_b22c3ab4d7bbc420e05340e8e187bcf5-brick.pid -S /var/run/gluster/b64892014146a0d5.socket --brick-name /var/lib/heketi/mounts/vg_5054cf122b4f54021cb9d67752d23c1d/brick_b22c3ab4d7bbc420e05340e8e187bcf5/brick -l /var/log/glusterfs/bricks/var-lib-heketi-mounts-vg_5054cf122b4f54021cb9d67752d23c1d-brick_b22c3ab4d7bbc420e05340e8e187bcf5-brick.log --xlator-option *-posix.glusterd-uuid=25b5f142-4890-4315-a352-cf947fdf649c --process-name brick --brick-port 49155 --xlator-option glusterfs-registry-volume-server.listen-port=49155 +- 2130 /usr/sbin/glusterfsd -s 192.168.89.220 --volfile-id vol_2e2e6b9cf174901b370ea79a266c651b.192.168.89.220.var-lib-heketi-mounts-vg_25eb42cb007aab028afd0ee42b72bae9-brick_62c4432fc7a6e4aeb11573e22eaf53d8-brick -p /var/run/gluster/vols/vol_2e2e6b9cf174901b370ea79a266c651b/192.168.89.220-var-lib-heketi-mounts-vg_25eb42cb007aab028afd0ee42b72bae9-brick_62c4432fc7a6e4aeb11573e22eaf53d8-brick.pid -S /var/run/gluster/5ab438223cc6750c.socket --brick-name /var/lib/heketi/mounts/vg_25eb42cb007aab028afd0ee42b72bae9/brick_62c4432fc7a6e4aeb11573e22eaf53d8/brick -l /var/log/glusterfs/bricks/var-lib-heketi-mounts-vg_25eb42cb007aab028afd0ee42b72bae9-brick_62c4432fc7a6e4aeb11573e22eaf53d8-brick.log --xlator-option *-posix.glusterd-uuid=25b5f142-4890-4315-a352-cf947fdf649c --process-name brick --brick-port 49161 --xlator-option vol_2e2e6b9cf174901b370ea79a266c651b-server.listen-port=49161 +- 2713 /usr/sbin/glusterfsd -s 192.168.89.220 --volfile-id vol_c5fa076446479cf414397591c0af1c7f.192.168.89.220.var-lib-heketi-mounts-vg_25eb42cb007aab028afd0ee42b72bae9-brick_ac7b22198b6424785edd32d6f0a078d2-brick -p /var/run/gluster/vols/vol_c5fa076446479cf414397591c0af1c7f/192.168.89.220-var-lib-heketi-mounts-vg_25eb42cb007aab028afd0ee42b72bae9-brick_ac7b22198b6424785edd32d6f0a078d2-brick.pid -S /var/run/gluster/b586da88dc5b2118.socket --brick-name /var/lib/heketi/mounts/vg_25eb42cb007aab028afd0ee42b72bae9/brick_ac7b22198b6424785edd32d6f0a078d2/brick -l /var/log/glusterfs/bricks/var-lib-heketi-mounts-vg_25eb42cb007aab028afd0ee42b72bae9-brick_ac7b22198b6424785edd32d6f0a078d2-brick.log --xlator-option *-posix.glusterd-uuid=25b5f142-4890-4315-a352-cf947fdf649c --process-name brick --brick-port 49162 --xlator-option vol_c5fa076446479cf414397591c0af1c7f-server.listen-port=49162 +- 9064 /usr/sbin/glusterfsd -s 192.168.89.220 --volfile-id vol_bc5012fd39c5b3ab958b9da4b0256d3a.192.168.89.220.var-lib-heketi-mounts-vg_5054cf122b4f54021cb9d67752d23c1d-brick_fded2411a02e03c7e0219eb9a2b08c9a-brick -p /var/run/gluster/vols/vol_bc5012fd39c5b3ab958b9da4b0256d3a/192.168.89.220-var-lib-heketi-mounts-vg_5054cf122b4f54021cb9d67752d23c1d-brick_fded2411a02e03c7e0219eb9a2b08c9a-brick.pid -S /var/run/gluster/8b6fba20e4ec888e.socket --brick-name /var/lib/heketi/mounts/vg_5054cf122b4f54021cb9d67752d23c1d/brick_fded2411a02e03c7e0219eb9a2b08c9a/brick -l /var/log/glusterfs/bricks/var-lib-heketi-mounts-vg_5054cf122b4f54021cb9d67752d23c1d-brick_fded2411a02e03c7e0219eb9a2b08c9a-brick.log --xlator-option *-posix.glusterd-uuid=25b5f142-4890-4315-a352-cf947fdf649c --process-name brick --brick-port 49166 --xlator-option vol_bc5012fd39c5b3ab958b9da4b0256d3a-server.listen-port=49166 +- 23952 /usr/sbin/glusterd -p /var/run/glusterd.pid --log-level INFO +- 24048 /usr/sbin/glusterfs -s localhost --volfile-id gluster/glustershd -p /var/run/gluster/glustershd/glustershd.pid -l /var/log/glusterfs/glustershd.log -S /var/run/gluster/06a8fe3d13500a44.socket --xlator-option *replicate*.node-uuid=25b5f142-4890-4315-a352-cf947fdf649c --process-name glustershd +- 71770 /usr/sbin/glusterfsd -s 192.168.89.220 --volfile-id vol_88fbf36bf87106e549dca765f171cf69.192.168.89.220.var-lib-heketi-mounts-vg_9c007bfbe66a2522ad5dd851bb18d03a-brick_22982c432dd089512ab2bde1e0663f9a-brick -p /var/run/gluster/vols/vol_88fbf36bf87106e549dca765f171cf69/192.168.89.220-var-lib-heketi-mounts-vg_9c007bfbe66a2522ad5dd851bb18d03a-brick_22982c432dd089512ab2bde1e0663f9a-brick.pid -S /var/run/gluster/adbe241d384df1c2.socket --brick-name /var/lib/heketi/mounts/vg_9c007bfbe66a2522ad5dd851bb18d03a/brick_22982c432dd089512ab2bde1e0663f9a/brick -l /var/log/glusterfs/bricks/var-lib-heketi-mounts-vg_9c007bfbe66a2522ad5dd851bb18d03a-brick_22982c432dd089512ab2bde1e0663f9a-brick.log --xlator-option *-posix.glusterd-uuid=25b5f142-4890-4315-a352-cf947fdf649c --process-name brick --brick-port 49167 --xlator-option vol_88fbf36bf87106e549dca765f171cf69-server.listen-port=49167 +- 72776 /usr/sbin/glusterfsd -s 192.168.89.220 --volfile-id vol_47c2af6849ac9ad7d7fbc897ae3ae80c.192.168.89.220.var-lib-heketi-mounts-vg_5054cf122b4f54021cb9d67752d23c1d-brick_025b1bf4bbabea839ef139c4453f9bcd-brick -p /var/run/gluster/vols/vol_47c2af6849ac9ad7d7fbc897ae3ae80c/192.168.89.220-var-lib-heketi-mounts-vg_5054cf122b4f54021cb9d67752d23c1d-brick_025b1bf4bbabea839ef139c4453f9bcd-brick.pid -S /var/run/gluster/e6f521dfc39ee69b.socket --brick-name /var/lib/heketi/mounts/vg_5054cf122b4f54021cb9d67752d23c1d/brick_025b1bf4bbabea839ef139c4453f9bcd/brick -l /var/log/glusterfs/bricks/var-lib-heketi-mounts-vg_5054cf122b4f54021cb9d67752d23c1d-brick_025b1bf4bbabea839ef139c4453f9bcd-brick.log --xlator-option *-posix.glusterd-uuid=25b5f142-4890-4315-a352-cf947fdf649c --process-name brick --brick-port 49158 --xlator-option vol_47c2af6849ac9ad7d7fbc897ae3ae80c-server.listen-port=49158 +- 82920 /usr/sbin/glusterfsd -s 192.168.89.220 --volfile-id vol_63428c902b5d577a7739eb66a050a420.192.168.89.220.var-lib-heketi-mounts-vg_25eb42cb007aab028afd0ee42b72bae9-brick_85c1670a17f956abd0f3dd99cad53a7e-brick -p /var/run/gluster/vols/vol_63428c902b5d577a7739eb66a050a420/192.168.89.220-var-lib-heketi-mounts-vg_25eb42cb007aab028afd0ee42b72bae9-brick_85c1670a17f956abd0f3dd99cad53a7e-brick.pid -S /var/run/gluster/c82b614596a19c6c.socket --brick-name /var/lib/heketi/mounts/vg_25eb42cb007aab028afd0ee42b72bae9/brick_85c1670a17f956abd0f3dd99cad53a7e/brick -l /var/log/glusterfs/bricks/var-lib-heketi-mounts-vg_25eb42cb007aab028afd0ee42b72bae9-brick_85c1670a17f956abd0f3dd99cad53a7e-brick.log --xlator-option *-posix.glusterd-uuid=25b5f142-4890-4315-a352-cf947fdf649c --process-name brick --brick-port 49171 --xlator-option vol_63428c902b5d577a7739eb66a050a420-server.listen-port=49171 +- 86318 /usr/sbin/glusterfsd -s 192.168.89.220 --volfile-id vol_937aa673e0d60126082bbd8c1589e383.192.168.89.220.var-lib-heketi-mounts-vg_018e77166eb3b38a0c250eb8faede2f7-brick_275d68b56cfc42c8b46aa90eb227eb25-brick -p /var/run/gluster/vols/vol_937aa673e0d60126082bbd8c1589e383/192.168.89.220-var-lib-heketi-mounts-vg_018e77166eb3b38a0c250eb8faede2f7-brick_275d68b56cfc42c8b46aa90eb227eb25-brick.pid -S /var/run/gluster/5ae55f9a21c8e488.socket --brick-name /var/lib/heketi/mounts/vg_018e77166eb3b38a0c250eb8faede2f7/brick_275d68b56cfc42c8b46aa90eb227eb25/brick -l /var/log/glusterfs/bricks/var-lib-heketi-mounts-vg_018e77166eb3b38a0c250eb8faede2f7-brick_275d68b56cfc42c8b46aa90eb227eb25-brick.log --xlator-option *-posix.glusterd-uuid=25b5f142-4890-4315-a352-cf947fdf649c --process-name brick --brick-port 49172 --xlator-option vol_937aa673e0d60126082bbd8c1589e383-server.listen-port=49172 +- 87342 /usr/sbin/glusterfsd -s 192.168.89.220 --volfile-id vol_62905ab0e959e7663f4576501c0c9b69.192.168.89.220.var-lib-heketi-mounts-vg_5054cf122b4f54021cb9d67752d23c1d-brick_9684d9e56109e011585af33ecd8039cc-brick -p /var/run/gluster/vols/vol_62905ab0e959e7663f4576501c0c9b69/192.168.89.220-var-lib-heketi-mounts-vg_5054cf122b4f54021cb9d67752d23c1d-brick_9684d9e56109e011585af33ecd8039cc-brick.pid -S /var/run/gluster/e63ba38f2720a00f.socket --brick-name /var/lib/heketi/mounts/vg_5054cf122b4f54021cb9d67752d23c1d/brick_9684d9e56109e011585af33ecd8039cc/brick -l /var/log/glusterfs/bricks/var-lib-heketi-mounts-vg_5054cf122b4f54021cb9d67752d23c1d-brick_9684d9e56109e011585af33ecd8039cc-brick.log --xlator-option *-posix.glusterd-uuid=25b5f142-4890-4315-a352-cf947fdf649c --process-name brick --brick-port 49157 --xlator-option vol_62905ab0e959e7663f4576501c0c9b69-server.listen-port=49157 +- 97945 /usr/sbin/glusterfsd -s 192.168.89.220 --volfile-id vol_dcee4f7f020d47e58fb48612bbba19d1.192.168.89.220.var-lib-heketi-mounts-vg_018e77166eb3b38a0c250eb8faede2f7-brick_16de636f8337f47a87996fca50313260-brick -p /var/run/gluster/vols/vol_dcee4f7f020d47e58fb48612bbba19d1/192.168.89.220-var-lib-heketi-mounts-vg_018e77166eb3b38a0c250eb8faede2f7-brick_16de636f8337f47a87996fca50313260-brick.pid -S /var/run/gluster/5bbb028f813f880f.socket --brick-name /var/lib/heketi/mounts/vg_018e77166eb3b38a0c250eb8faede2f7/brick_16de636f8337f47a87996fca50313260/brick -l /var/log/glusterfs/bricks/var-lib-heketi-mounts-vg_018e77166eb3b38a0c250eb8faede2f7-brick_16de636f8337f47a87996fca50313260-brick.log --xlator-option *-posix.glusterd-uuid=25b5f142-4890-4315-a352-cf947fdf649c --process-name brick --brick-port 49168 --xlator-option vol_dcee4f7f020d47e58fb48612bbba19d1-server.listen-port=49168 +-103617 /usr/sbin/glusterfsd -s 192.168.89.220 --volfile-id vol_5101331e9d33a5d04adab92837b9d5ad.192.168.89.220.var-lib-heketi-mounts-vg_9c007bfbe66a2522ad5dd851bb18d03a-brick_079acb5b4a0b0f3c45ed87f9c7def14b-brick -p /var/run/gluster/vols/vol_5101331e9d33a5d04adab92837b9d5ad/192.168.89.220-var-lib-heketi-mounts-vg_9c007bfbe66a2522ad5dd851bb18d03a-brick_079acb5b4a0b0f3c45ed87f9c7def14b-brick.pid -S /var/run/gluster/20aa83a13cec2563.socket --brick-name /var/lib/heketi/mounts/vg_9c007bfbe66a2522ad5dd851bb18d03a/brick_079acb5b4a0b0f3c45ed87f9c7def14b/brick -l /var/log/glusterfs/bricks/var-lib-heketi-mounts-vg_9c007bfbe66a2522ad5dd851bb18d03a-brick_079acb5b4a0b0f3c45ed87f9c7def14b-brick.log --xlator-option *-posix.glusterd-uuid=25b5f142-4890-4315-a352-cf947fdf649c --process-name brick --brick-port 49159 --xlator-option vol_5101331e9d33a5d04adab92837b9d5ad-server.listen-port=49159 +-103808 /usr/sbin/glusterfsd -s 192.168.89.220 --volfile-id vol_2e6ad6aa03f7fe219807cb135ca1c766.192.168.89.220.var-lib-heketi-mounts-vg_9c007bfbe66a2522ad5dd851bb18d03a-brick_2fd555bc1cbd1b49ef04e7b88d34297b-brick -p /var/run/gluster/vols/vol_2e6ad6aa03f7fe219807cb135ca1c766/192.168.89.220-var-lib-heketi-mounts-vg_9c007bfbe66a2522ad5dd851bb18d03a-brick_2fd555bc1cbd1b49ef04e7b88d34297b-brick.pid -S /var/run/gluster/a375f4b3a8cba7c0.socket --brick-name /var/lib/heketi/mounts/vg_9c007bfbe66a2522ad5dd851bb18d03a/brick_2fd555bc1cbd1b49ef04e7b88d34297b/brick -l /var/log/glusterfs/bricks/var-lib-heketi-mounts-vg_9c007bfbe66a2522ad5dd851bb18d03a-brick_2fd555bc1cbd1b49ef04e7b88d34297b-brick.log --xlator-option *-posix.glusterd-uuid=25b5f142-4890-4315-a352-cf947fdf649c --process-name brick --brick-port 49163 --xlator-option vol_2e6ad6aa03f7fe219807cb135ca1c766-server.listen-port=49163 +-103990 /usr/sbin/glusterfsd -s 192.168.89.220 --volfile-id vol_78d426889b7bd661f3d6c6f7814b6d4f.192.168.89.220.var-lib-heketi-mounts-vg_9c007bfbe66a2522ad5dd851bb18d03a-brick_8dba95329e797beb06eca079b9ae6a4b-brick -p /var/run/gluster/vols/vol_78d426889b7bd661f3d6c6f7814b6d4f/192.168.89.220-var-lib-heketi-mounts-vg_9c007bfbe66a2522ad5dd851bb18d03a-brick_8dba95329e797beb06eca079b9ae6a4b-brick.pid -S /var/run/gluster/8d3f3f80d9e62092.socket --brick-name /var/lib/heketi/mounts/vg_9c007bfbe66a2522ad5dd851bb18d03a/brick_8dba95329e797beb06eca079b9ae6a4b/brick -l /var/log/glusterfs/bricks/var-lib-heketi-mounts-vg_9c007bfbe66a2522ad5dd851bb18d03a-brick_8dba95329e797beb06eca079b9ae6a4b-brick.log --xlator-option *-posix.glusterd-uuid=25b5f142-4890-4315-a352-cf947fdf649c --process-name brick --brick-port 49164 --xlator-option vol_78d426889b7bd661f3d6c6f7814b6d4f-server.listen-port=49164 +-105270 /usr/sbin/glusterfsd -s 192.168.89.220 --volfile-id vol_9f04d82be19e2ea8ee80deb9098cd390.192.168.89.220.var-lib-heketi-mounts-vg_018e77166eb3b38a0c250eb8faede2f7-brick_0bd819f4969e7adbba32e2c77afc202c-brick -p /var/run/gluster/vols/vol_9f04d82be19e2ea8ee80deb9098cd390/192.168.89.220-var-lib-heketi-mounts-vg_018e77166eb3b38a0c250eb8faede2f7-brick_0bd819f4969e7adbba32e2c77afc202c-brick.pid -S /var/run/gluster/0882e30a8d5b1272.socket --brick-name /var/lib/heketi/mounts/vg_018e77166eb3b38a0c250eb8faede2f7/brick_0bd819f4969e7adbba32e2c77afc202c/brick -l /var/log/glusterfs/bricks/var-lib-heketi-mounts-vg_018e77166eb3b38a0c250eb8faede2f7-brick_0bd819f4969e7adbba32e2c77afc202c-brick.log --xlator-option *-posix.glusterd-uuid=25b5f142-4890-4315-a352-cf947fdf649c --process-name brick --brick-port 49165 --xlator-option vol_9f04d82be19e2ea8ee80deb9098cd390-server.listen-port=49165 +-121771 /usr/sbin/glusterfsd -s 192.168.89.220 --volfile-id vol_7e73769a754d7f9f6213ef3b4551af0e.192.168.89.220.var-lib-heketi-mounts-vg_018e77166eb3b38a0c250eb8faede2f7-brick_c2fe3a63636192e85d8adeefff18c92b-brick -p /var/run/gluster/vols/vol_7e73769a754d7f9f6213ef3b4551af0e/192.168.89.220-var-lib-heketi-mounts-vg_018e77166eb3b38a0c250eb8faede2f7-brick_c2fe3a63636192e85d8adeefff18c92b-brick.pid -S /var/run/gluster/130cc033b68aedd6.socket --brick-name /var/lib/heketi/mounts/vg_018e77166eb3b38a0c250eb8faede2f7/brick_c2fe3a63636192e85d8adeefff18c92b/brick -l /var/log/glusterfs/bricks/var-lib-heketi-mounts-vg_018e77166eb3b38a0c250eb8faede2f7-brick_c2fe3a63636192e85d8adeefff18c92b-brick.log --xlator-option *-posix.glusterd-uuid=25b5f142-4890-4315-a352-cf947fdf649c --process-name brick --brick-port 49156 --xlator-option vol_7e73769a754d7f9f6213ef3b4551af0e-server.listen-port=49156 +-130924 /usr/sbin/glusterfsd -s 192.168.89.220 --volfile-id vol_dbc601f7162df326784aea34c4ebe8f2.192.168.89.220.var-lib-heketi-mounts-vg_5054cf122b4f54021cb9d67752d23c1d-brick_4fa8a73223d69de183d8e58618722084-brick -p /var/run/gluster/vols/vol_dbc601f7162df326784aea34c4ebe8f2/192.168.89.220-var-lib-heketi-mounts-vg_5054cf122b4f54021cb9d67752d23c1d-brick_4fa8a73223d69de183d8e58618722084-brick.pid -S /var/run/gluster/88109cbb53cf4280.socket --brick-name /var/lib/heketi/mounts/vg_5054cf122b4f54021cb9d67752d23c1d/brick_4fa8a73223d69de183d8e58618722084/brick -l /var/log/glusterfs/bricks/var-lib-heketi-mounts-vg_5054cf122b4f54021cb9d67752d23c1d-brick_4fa8a73223d69de183d8e58618722084-brick.log --xlator-option *-posix.glusterd-uuid=25b5f142-4890-4315-a352-cf947fdf649c --process-name brick --brick-port 49160 --xlator-option vol_dbc601f7162df326784aea34c4ebe8f2-server.listen-port=49160 Jan 18 13:27:56 storage.matrix.nokia.com systemd[1]: Starting GlusterFS, a clustered file-system server... Jan 18 13:27:57 storage.matrix.nokia.com systemd[1]: Started GlusterFS, a clustered file-system server. [heketi] INFO 2019/01/24 07:31:46 Periodic health check status: node 5f090e0aa757345d22ebdb15897bd68a up=true [cmdexec] INFO 2019/01/24 07:31:46 Check Glusterd service status in node app1.matrix.nokia.com [kubeexec] DEBUG 2019/01/24 07:31:47 /src/github.com/heketi/heketi/executors/kubeexec/kubeexec.go:246: Host: app1.matrix.nokia.com Pod: glusterfs-storage-31fjz Command: systemctl status glusterd Result: ? glusterd.service - GlusterFS, a clustered file-system server Loaded: loaded (/usr/lib/systemd/system/glusterd.service; enabled; vendor preset: disabled) Active: active (running) since Fri 2019-01-18 13:24:52 UTC; 5 days ago Process: 33779 ExecStart=/usr/sbin/glusterd -p /var/run/glusterd.pid --log-level $LOG_LEVEL $GLUSTERD_OPTIONS (code=exited, status=0/SUCCESS) Main PID: 33780 (glusterd) CGroup: /kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod97b10041_cd44_11e8_a361_5254007b74fe.slice/docker-9bc763a8fd6411461604d4d24f0693389c54d9f66d2cff31f672ca7dbcc2660d.scope/system.slice/glusterd.service +- 576 /usr/sbin/glusterfsd -s 192.168.89.218 --volfile-id heketidbstorage.192.168.89.218.var-lib-heketi-mounts-vg_127a3d697690febddfa2a41b3a820bde-brick_79e211c7bee7d61bfdabb0add8e42b33-brick -p /var/run/gluster/vols/heketidbstorage/192.168.89.218-var-lib-heketi-mounts-vg_127a3d697690febddfa2a41b3a820bde-brick_79e211c7bee7d61bfdabb0add8e42b33-brick.pid -S /var/run/gluster/f905b930c13d9816.socket --brick-name /var/lib/heketi/mounts/vg_127a3d697690febddfa2a41b3a820bde/brick_79e211c7bee7d61bfdabb0add8e42b33/brick -l /var/log/glusterfs/bricks/var-lib-heketi-mounts-vg_127a3d697690febddfa2a41b3a820bde-brick_79e211c7bee7d61bfdabb0add8e42b33-brick.log --xlator-option *-posix.glusterd-uuid=0574c4d2-6900-4447-a752-2bf7477b443e --process-name brick --brick-port 49154 --xlator-option heketidbstorage-server.listen-port=49154 +- 869 /usr/sbin/glusterfsd -s 192.168.89.218 --volfile-id glusterfs-registry-volume.192.168.89.218.var-lib-heketi-mounts-vg_cea64ccf8a96a13dacc1cb3ac3ea13de-brick_7ef53adf2bce0e240229941ac7f43661-brick -p /var/run/gluster/vols/glusterfs-registry-volume/192.168.89.218-var-lib-heketi-mounts-vg_cea64ccf8a96a13dacc1cb3ac3ea13de-brick_7ef53adf2bce0e240229941ac7f43661-brick.pid -S /var/run/gluster/f223a3f7986d97b5.socket --brick-name /var/lib/heketi/mounts/vg_cea64ccf8a96a13dacc1cb3ac3ea13de/brick_7ef53adf2bce0e240229941ac7f43661/brick -l /var/log/glusterfs/bricks/var-lib-heketi-mounts-vg_cea64ccf8a96a13dacc1cb3ac3ea13de-brick_7ef53adf2bce0e240229941ac7f43661-brick.log --xlator-option *-posix.glusterd-uuid=0574c4d2-6900-4447-a752-2bf7477b443e --process-name brick --brick-port 49155 --xlator-option glusterfs-registry-volume-server.listen-port=49155 +- 1934 /usr/sbin/glusterfsd -s 192.168.89.218 --volfile-id vol_dbc601f7162df326784aea34c4ebe8f2.192.168.89.218.var-lib-heketi-mounts-vg_e7e08adf3c8cd3dee08dd006b1596074-brick_ff62a0d3e0fe5c53c3725235d2832893-brick -p /var/run/gluster/vols/vol_dbc601f7162df326784aea34c4ebe8f2/192.168.89.218-var-lib-heketi-mounts-vg_e7e08adf3c8cd3dee08dd006b1596074-brick_ff62a0d3e0fe5c53c3725235d2832893-brick.pid -S /var/run/gluster/53bb40a037c87c29.socket --brick-name /var/lib/heketi/mounts/vg_e7e08adf3c8cd3dee08dd006b1596074/brick_ff62a0d3e0fe5c53c3725235d2832893/brick -l /var/log/glusterfs/bricks/var-lib-heketi-mounts-vg_e7e08adf3c8cd3dee08dd006b1596074-brick_ff62a0d3e0fe5c53c3725235d2832893-brick.log --xlator-option *-posix.glusterd-uuid=0574c4d2-6900-4447-a752-2bf7477b443e --process-name brick --brick-port 49160 --xlator-option vol_dbc601f7162df326784aea34c4ebe8f2-server.listen-port=49160 +- 3907 /usr/sbin/glusterfsd -s 192.168.89.218 --volfile-id vol_2e2e6b9cf174901b370ea79a266c651b.192.168.89.218.var-lib-heketi-mounts-vg_88c314d3366471a0dd89ca4d5a3c8ef8-brick_6f738c7cc82f66dbda8bc0b6d74883c0-brick -p /var/run/gluster/vols/vol_2e2e6b9cf174901b370ea79a266c651b/192.168.89.218-var-lib-heketi-mounts-vg_88c314d3366471a0dd89ca4d5a3c8ef8-brick_6f738c7cc82f66dbda8bc0b6d74883c0-brick.pid -S /var/run/gluster/ac938e354448789e.socket --brick-name /var/lib/heketi/mounts/vg_88c314d3366471a0dd89ca4d5a3c8ef8/brick_6f738c7cc82f66dbda8bc0b6d74883c0/brick -l /var/log/glusterfs/bricks/var-lib-heketi-mounts-vg_88c314d3366471a0dd89ca4d5a3c8ef8-brick_6f738c7cc82f66dbda8bc0b6d74883c0-brick.log --xlator-option *-posix.glusterd-uuid=0574c4d2-6900-4447-a752-2bf7477b443e --process-name brick --brick-port 49161 --xlator-option vol_2e2e6b9cf174901b370ea79a266c651b-server.listen-port=49161 +- 4486 /usr/sbin/glusterfsd -s 192.168.89.218 --volfile-id vol_c5fa076446479cf414397591c0af1c7f.192.168.89.218.var-lib-heketi-mounts-vg_cea64ccf8a96a13dacc1cb3ac3ea13de-brick_ae2fa0410160c9fb489af8e5ad063ac3-brick -p /var/run/gluster/vols/vol_c5fa076446479cf414397591c0af1c7f/192.168.89.218-var-lib-heketi-mounts-vg_cea64ccf8a96a13dacc1cb3ac3ea13de-brick_ae2fa0410160c9fb489af8e5ad063ac3-brick.pid -S /var/run/gluster/8bb099144ac69c55.socket --brick-name /var/lib/heketi/mounts/vg_cea64ccf8a96a13dacc1cb3ac3ea13de/brick_ae2fa0410160c9fb489af8e5ad063ac3/brick -l /var/log/glusterfs/bricks/var-lib-heketi-mounts-vg_cea64ccf8a96a13dacc1cb3ac3ea13de-brick_ae2fa0410160c9fb489af8e5ad063ac3-brick.log --xlator-option *-posix.glusterd-uuid=0574c4d2-6900-4447-a752-2bf7477b443e --process-name brick --brick-port 49162 --xlator-option vol_c5fa076446479cf414397591c0af1c7f-server.listen-port=49162 +- 6510 /usr/sbin/glusterfs -s localhost --volfile-id gluster/glustershd -p /var/run/gluster/glustershd/glustershd.pid -l /var/log/glusterfs/glustershd.log -S /var/run/gluster/46eb4bfe4652baa1.socket --xlator-option *replicate*.node-uuid=0574c4d2-6900-4447-a752-2bf7477b443e --process-name glustershd +- 11149 /usr/sbin/glusterfsd -s 192.168.89.218 --volfile-id vol_bc5012fd39c5b3ab958b9da4b0256d3a.192.168.89.218.var-lib-heketi-mounts-vg_cea64ccf8a96a13dacc1cb3ac3ea13de-brick_08599e162e105063a8e2efec868d2a76-brick -p /var/run/gluster/vols/vol_bc5012fd39c5b3ab958b9da4b0256d3a/192.168.89.218-var-lib-heketi-mounts-vg_cea64ccf8a96a13dacc1cb3ac3ea13de-brick_08599e162e105063a8e2efec868d2a76-brick.pid -S /var/run/gluster/42ae4fefd1cb6554.socket --brick-name /var/lib/heketi/mounts/vg_cea64ccf8a96a13dacc1cb3ac3ea13de/brick_08599e162e105063a8e2efec868d2a76/brick -l /var/log/glusterfs/bricks/var-lib-heketi-mounts-vg_cea64ccf8a96a13dacc1cb3ac3ea13de-brick_08599e162e105063a8e2efec868d2a76-brick.log --xlator-option *-posix.glusterd-uuid=0574c4d2-6900-4447-a752-2bf7477b443e --process-name brick --brick-port 49166 --xlator-option vol_bc5012fd39c5b3ab958b9da4b0256d3a-server.listen-port=49166 +- 16908 /usr/sbin/glusterfsd -s 192.168.89.218 --volfile-id vol_9f04d82be19e2ea8ee80deb9098cd390.192.168.89.218.var-lib-heketi-mounts-vg_88c314d3366471a0dd89ca4d5a3c8ef8-brick_281606b3024ac98416fedee94528e3c7-brick -p /var/run/gluster/vols/vol_9f04d82be19e2ea8ee80deb9098cd390/192.168.89.218-var-lib-heketi-mounts-vg_88c314d3366471a0dd89ca4d5a3c8ef8-brick_281606b3024ac98416fedee94528e3c7-brick.pid -S /var/run/gluster/a3fd18961e1f360b.socket --brick-name /var/lib/heketi/mounts/vg_88c314d3366471a0dd89ca4d5a3c8ef8/brick_281606b3024ac98416fedee94528e3c7/brick -l /var/log/glusterfs/bricks/var-lib-heketi-mounts-vg_88c314d3366471a0dd89ca4d5a3c8ef8-brick_281606b3024ac98416fedee94528e3c7-brick.log --xlator-option *-posix.glusterd-uuid=0574c4d2-6900-4447-a752-2bf7477b443e --process-name brick --brick-port 49165 --xlator-option vol_9f04d82be19e2ea8ee80deb9098cd390-server.listen-port=49165 +- 33780 /usr/sbin/glusterd -p /var/run/glusterd.pid --log-level INFO +- 33863 /usr/sbin/glusterfs -s localhost --volfile-id gluster/glustershd -p /var/run/gluster/glustershd/glustershd.pid -l /var/log/glusterfs/glustershd.log -S /var/run/gluster/46eb4bfe4652baa1.socket --xlator-option *replicate*.node-uuid=0574c4d2-6900-4447-a752-2bf7477b443e --process-name glustershd +- 73126 /usr/sbin/glusterfsd -s 192.168.89.218 --volfile-id vol_47c2af6849ac9ad7d7fbc897ae3ae80c.192.168.89.218.var-lib-heketi-mounts-vg_cea64ccf8a96a13dacc1cb3ac3ea13de-brick_1f52101af46329a1d5e758156c8267e0-brick -p /var/run/gluster/vols/vol_47c2af6849ac9ad7d7fbc897ae3ae80c/192.168.89.218-var-lib-heketi-mounts-vg_cea64ccf8a96a13dacc1cb3ac3ea13de-brick_1f52101af46329a1d5e758156c8267e0-brick.pid -S /var/run/gluster/17890cacc5ba5074.socket --brick-name /var/lib/heketi/mounts/vg_cea64ccf8a96a13dacc1cb3ac3ea13de/brick_1f52101af46329a1d5e758156c8267e0/brick -l /var/log/glusterfs/bricks/var-lib-heketi-mounts-vg_cea64ccf8a96a13dacc1cb3ac3ea13de-brick_1f52101af46329a1d5e758156c8267e0-brick.log --xlator-option *-posix.glusterd-uuid=0574c4d2-6900-4447-a752-2bf7477b443e --process-name brick --brick-port 49158 --xlator-option vol_47c2af6849ac9ad7d7fbc897ae3ae80c-server.listen-port=49158 +- 76135 /usr/sbin/glusterfsd -s 192.168.89.218 --volfile-id vol_88fbf36bf87106e549dca765f171cf69.192.168.89.218.var-lib-heketi-mounts-vg_e7e08adf3c8cd3dee08dd006b1596074-brick_ef253aa4b2756fc293aa101c2eb743cb-brick -p /var/run/gluster/vols/vol_88fbf36bf87106e549dca765f171cf69/192.168.89.218-var-lib-heketi-mounts-vg_e7e08adf3c8cd3dee08dd006b1596074-brick_ef253aa4b2756fc293aa101c2eb743cb-brick.pid -S /var/run/gluster/57118ee0d54a4d9c.socket --brick-name /var/lib/heketi/mounts/vg_e7e08adf3c8cd3dee08dd006b1596074/brick_ef253aa4b2756fc293aa101c2eb743cb/brick -l /var/log/glusterfs/bricks/var-lib-heketi-mounts-vg_e7e08adf3c8cd3dee08dd006b1596074-brick_ef253aa4b2756fc293aa101c2eb743cb-brick.log --xlator-option *-posix.glusterd-uuid=0574c4d2-6900-4447-a752-2bf7477b443e --process-name brick --brick-port 49167 --xlator-option vol_88fbf36bf87106e549dca765f171cf69-server.listen-port=49167 +- 87644 /usr/sbin/glusterfsd -s 192.168.89.218 --volfile-id vol_63428c902b5d577a7739eb66a050a420.192.168.89.218.var-lib-heketi-mounts-vg_88c314d3366471a0dd89ca4d5a3c8ef8-brick_586c05c5ce5bd7474cfb9a9cd190225c-brick -p /var/run/gluster/vols/vol_63428c902b5d577a7739eb66a050a420/192.168.89.218-var-lib-heketi-mounts-vg_88c314d3366471a0dd89ca4d5a3c8ef8-brick_586c05c5ce5bd7474cfb9a9cd190225c-brick.pid -S /var/run/gluster/27400e8d3608aa83.socket --brick-name /var/lib/heketi/mounts/vg_88c314d3366471a0dd89ca4d5a3c8ef8/brick_586c05c5ce5bd7474cfb9a9cd190225c/brick -l /var/log/glusterfs/bricks/var-lib-heketi-mounts-vg_88c314d3366471a0dd89ca4d5a3c8ef8-brick_586c05c5ce5bd7474cfb9a9cd190225c-brick.log --xlator-option *-posix.glusterd-uuid=0574c4d2-6900-4447-a752-2bf7477b443e --process-name brick --brick-port 49171 --xlator-option vol_63428c902b5d577a7739eb66a050a420-server.listen-port=49171 +- 91110 /usr/sbin/glusterfsd -s 192.168.89.218 --volfile-id vol_937aa673e0d60126082bbd8c1589e383.192.168.89.218.var-lib-heketi-mounts-vg_127a3d697690febddfa2a41b3a820bde-brick_5f22e9906d3566116ec474bb618213e5-brick -p /var/run/gluster/vols/vol_937aa673e0d60126082bbd8c1589e383/192.168.89.218-var-lib-heketi-mounts-vg_127a3d697690febddfa2a41b3a820bde-brick_5f22e9906d3566116ec474bb618213e5-brick.pid -S /var/run/gluster/926dadf68659a463.socket --brick-name /var/lib/heketi/mounts/vg_127a3d697690febddfa2a41b3a820bde/brick_5f22e9906d3566116ec474bb618213e5/brick -l /var/log/glusterfs/bricks/var-lib-heketi-mounts-vg_127a3d697690febddfa2a41b3a820bde-brick_5f22e9906d3566116ec474bb618213e5-brick.log --xlator-option *-posix.glusterd-uuid=0574c4d2-6900-4447-a752-2bf7477b443e --process-name brick --brick-port 49172 --xlator-option vol_937aa673e0d60126082bbd8c1589e383-server.listen-port=49172 +- 92244 /usr/sbin/glusterfsd -s 192.168.89.218 --volfile-id vol_62905ab0e959e7663f4576501c0c9b69.192.168.89.218.var-lib-heketi-mounts-vg_cea64ccf8a96a13dacc1cb3ac3ea13de-brick_31ed93bf445cf1fd00d34660a80e1078-brick -p /var/run/gluster/vols/vol_62905ab0e959e7663f4576501c0c9b69/192.168.89.218-var-lib-heketi-mounts-vg_cea64ccf8a96a13dacc1cb3ac3ea13de-brick_31ed93bf445cf1fd00d34660a80e1078-brick.pid -S /var/run/gluster/73b2ef5a8bf5baa1.socket --brick-name /var/lib/heketi/mounts/vg_cea64ccf8a96a13dacc1cb3ac3ea13de/brick_31ed93bf445cf1fd00d34660a80e1078/brick -l /var/log/glusterfs/bricks/var-lib-heketi-mounts-vg_cea64ccf8a96a13dacc1cb3ac3ea13de-brick_31ed93bf445cf1fd00d34660a80e1078-brick.log --xlator-option *-posix.glusterd-uuid=0574c4d2-6900-4447-a752-2bf7477b443e --process-name brick --brick-port 49157 --xlator-option vol_62905ab0e959e7663f4576501c0c9b69-server.listen-port=49157 +-103360 /usr/sbin/glusterfsd -s 192.168.89.218 --volfile-id vol_dcee4f7f020d47e58fb48612bbba19d1.192.168.89.218.var-lib-heketi-mounts-vg_e7e08adf3c8cd3dee08dd006b1596074-brick_800e2a99110af93ca69bd950c9ea7ef8-brick -p /var/run/gluster/vols/vol_dcee4f7f020d47e58fb48612bbba19d1/192.168.89.218-var-lib-heketi-mounts-vg_e7e08adf3c8cd3dee08dd006b1596074-brick_800e2a99110af93ca69bd950c9ea7ef8-brick.pid -S /var/run/gluster/fbf6e334c35a83bb.socket --brick-name /var/lib/heketi/mounts/vg_e7e08adf3c8cd3dee08dd006b1596074/brick_800e2a99110af93ca69bd950c9ea7ef8/brick -l /var/log/glusterfs/bricks/var-lib-heketi-mounts-vg_e7e08adf3c8cd3dee08dd006b1596074-brick_800e2a99110af93ca69bd950c9ea7ef8-brick.log --xlator-option *-posix.glusterd-uuid=0574c4d2-6900-4447-a752-2bf7477b443e --process-name brick --brick-port 49168 --xlator-option vol_dcee4f7f020d47e58fb48612bbba19d1-server.listen-port=49168 +-109344 /usr/sbin/glusterfsd -s 192.168.89.218 --volfile-id vol_5101331e9d33a5d04adab92837b9d5ad.192.168.89.218.var-lib-heketi-mounts-vg_127a3d697690febddfa2a41b3a820bde-brick_17f8162940314babf2b7d75976890130-brick -p /var/run/gluster/vols/vol_5101331e9d33a5d04adab92837b9d5ad/192.168.89.218-var-lib-heketi-mounts-vg_127a3d697690febddfa2a41b3a820bde-brick_17f8162940314babf2b7d75976890130-brick.pid -S /var/run/gluster/6d420a71217c335e.socket --brick-name /var/lib/heketi/mounts/vg_127a3d697690febddfa2a41b3a820bde/brick_17f8162940314babf2b7d75976890130/brick -l /var/log/glusterfs/bricks/var-lib-heketi-mounts-vg_127a3d697690febddfa2a41b3a820bde-brick_17f8162940314babf2b7d75976890130-brick.log --xlator-option *-posix.glusterd-uuid=0574c4d2-6900-4447-a752-2bf7477b443e --process-name brick --brick-port 49159 --xlator-option vol_5101331e9d33a5d04adab92837b9d5ad-server.listen-port=49159 +-109591 /usr/sbin/glusterfsd -s 192.168.89.218 --volfile-id vol_2e6ad6aa03f7fe219807cb135ca1c766.192.168.89.218.var-lib-heketi-mounts-vg_e7e08adf3c8cd3dee08dd006b1596074-brick_980bd302213bbdd2d9aa09dbe8abf62b-brick -p /var/run/gluster/vols/vol_2e6ad6aa03f7fe219807cb135ca1c766/192.168.89.218-var-lib-heketi-mounts-vg_e7e08adf3c8cd3dee08dd006b1596074-brick_980bd302213bbdd2d9aa09dbe8abf62b-brick.pid -S /var/run/gluster/db1300311bc7697d.socket --brick-name /var/lib/heketi/mounts/vg_e7e08adf3c8cd3dee08dd006b1596074/brick_980bd302213bbdd2d9aa09dbe8abf62b/brick -l /var/log/glusterfs/bricks/var-lib-heketi-mounts-vg_e7e08adf3c8cd3dee08dd006b1596074-brick_980bd302213bbdd2d9aa09dbe8abf62b-brick.log --xlator-option *-posix.glusterd-uuid=0574c4d2-6900-4447-a752-2bf7477b443e --process-name brick --brick-port 49163 --xlator-option vol_2e6ad6aa03f7fe219807cb135ca1c766-server.listen-port=49163 +-109834 /usr/sbin/glusterfsd -s 192.168.89.218 --volfile-id vol_78d426889b7bd661f3d6c6f7814b6d4f.192.168.89.218.var-lib-heketi-mounts-vg_127a3d697690febddfa2a41b3a820bde-brick_d3230ee5de8fe8300f47eb4bcea6f2e2-brick -p /var/run/gluster/vols/vol_78d426889b7bd661f3d6c6f7814b6d4f/192.168.89.218-var-lib-heketi-mounts-vg_127a3d697690febddfa2a41b3a820bde-brick_d3230ee5de8fe8300f47eb4bcea6f2e2-brick.pid -S /var/run/gluster/87c81a7cb1ad328b.socket --brick-name /var/lib/heketi/mounts/vg_127a3d697690febddfa2a41b3a820bde/brick_d3230ee5de8fe8300f47eb4bcea6f2e2/brick -l /var/log/glusterfs/bricks/var-lib-heketi-mounts-vg_127a3d697690febddfa2a41b3a820bde-brick_d3230ee5de8fe8300f47eb4bcea6f2e2-brick.log --xlator-option *-posix.glusterd-uuid=0574c4d2-6900-4447-a752-2bf7477b443e --process-name brick --brick-port 49164 --xlator-option vol_78d426889b7bd661f3d6c6f7814b6d4f-server.listen-port=49164 +-123388 /usr/sbin/glusterfsd -s 192.168.89.218 --volfile-id vol_7e73769a754d7f9f6213ef3b4551af0e.192.168.89.218.var-lib-heketi-mounts-vg_88c314d3366471a0dd89ca4d5a3c8ef8-brick_84e395f5464cd25dce66b8e60f395982-brick -p /var/run/gluster/vols/vol_7e73769a754d7f9f6213ef3b4551af0e/192.168.89.218-var-lib-heketi-mounts-vg_88c314d3366471a0dd89ca4d5a3c8ef8-brick_84e395f5464cd25dce66b8e60f395982-brick.pid -S /var/run/gluster/c7917e048093731a.socket --brick-name /var/lib/heketi/mounts/vg_88c314d3366471a0dd89ca4d5a3c8ef8/brick_84e395f5464cd25dce66b8e60f395982/brick -l /var/log/glusterfs/bricks/var-lib-heketi-mounts-vg_88c314d3366471a0dd89ca4d5a3c8ef8-brick_84e395f5464cd25dce66b8e60f395982-brick.log --xlator-option *-posix.glusterd-uuid=0574c4d2-6900-4447-a752-2bf7477b443e --process-name brick --brick-port 49156 --xlator-option vol_7e73769a754d7f9f6213ef3b4551af0e-server.listen-port=49156 Jan 18 13:24:50 app1.matrix.nokia.com systemd[1]: Starting GlusterFS, a clustered file-system server... Jan 18 13:24:52 app1.matrix.nokia.com systemd[1]: Started GlusterFS, a clustered file-system server. [heketi] INFO 2019/01/24 07:31:47 Periodic health check status: node b72660bc8e28725f612d33bd675f8341 up=true [cmdexec] INFO 2019/01/24 07:31:47 Check Glusterd service status in node app2.matrix.nokia.com [negroni] Started GET /operations [negroni] Completed 200 OK in 235.37µs [negroni] Started GET /admin [negroni] Completed 200 OK in 160.585µs [negroni] Started POST /volumes [heketi] INFO 2019/01/24 07:37:36 Allocating brick set #0 [negroni] Started GET /admin [negroni] Completed 200 OK in 186.286µs [negroni] Started POST /volumes [negroni] Started GET /operations [negroni] Completed 200 OK in 166.294µs [negroni] Started GET /operations [negroni] Completed 200 OK in 186.411µs [negroni] Started GET /operations [negroni] Completed 200 OK in 179.796µs [negroni] Started POST /volumes [negroni] Started POST /volumes [negroni] Started POST /volumes [negroni] Started POST /volumes [negroni] Started GET /operations [negroni] Completed 200 OK in 131.108µs [negroni] Started POST /volumes [negroni] Started GET /operations [negroni] Completed 200 OK in 111.392µs [negroni] Started GET /operations [negroni] Completed 200 OK in 265.023µs [negroni] Started GET /operations [negroni] Completed 200 OK in 179.364µs [negroni] Started GET /operations [negroni] Completed 200 OK in 295.058µs [negroni] Started GET /operations [negroni] Completed 200 OK in 146.857µs [negroni] Started POST /volumes [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 07:43:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 403.166µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 07:43:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 193.554µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 07:44:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 301.85µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 07:44:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 334.219µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 07:44:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 280.267µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 07:44:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 314.112µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 07:45:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 173.956µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 07:45:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 259.106µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 07:45:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 267.954µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 07:45:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 168.322µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 07:46:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 211.378µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 07:46:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 411.904µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 07:46:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 227.577µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 07:46:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 205.121µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 07:47:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 188.152µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 07:47:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 308.371µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 07:47:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 321.142µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 07:47:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 170.624µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 07:48:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 245.405µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 07:48:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 292.096µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 07:48:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 407.949µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 07:48:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 229.056µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 07:49:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 348.766µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 07:49:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 300.105µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 07:49:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 183.418µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 07:49:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 174.123µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 07:50:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 221.385µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 07:50:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 157.804µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 07:50:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 1.32577ms [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 07:50:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 349.653µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 07:51:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 319.322µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 07:51:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 228.345µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 07:51:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 250.911µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 07:51:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 335.595µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 07:52:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 198.296µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 07:52:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 337.152µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 07:52:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 350.359µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 07:52:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 422.908µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 07:53:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 197.926µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 07:53:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 209.677µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 07:53:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 314.168µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 07:53:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 351.228µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 07:54:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 215.099µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 07:54:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 177.938µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 07:54:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 178.633µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 07:54:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 386.201µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 07:55:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 358.894µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 07:55:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 292.314µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 07:55:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 193.953µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 07:55:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 201.961µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 07:56:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 366.115µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 07:56:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 242.508µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 07:56:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 382.523µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 07:56:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 424.559µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 07:57:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 232.661µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 07:57:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 367.447µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 07:57:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 335.655µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 07:57:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 334.459µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 07:58:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 384.103µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 07:58:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 387.066µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 07:58:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 215.561µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 07:58:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 610.841µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 07:59:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 313.935µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 07:59:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 298.077µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 07:59:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 238.63µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 07:59:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 358.844µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:00:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 300.566µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:00:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 211.034µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:00:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 359.025µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:00:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 646.077µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:01:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 313.224µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:01:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 381.497µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:01:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 245.67µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:01:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 288.393µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:02:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 204.654µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:02:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 218.533µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:02:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 296.205µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:02:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 656.141µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:03:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 347.323µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:03:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 371.61µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:03:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 266.386µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:03:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 228.429µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:04:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 325.348µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:04:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 219.603µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:04:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 262.586µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:04:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 572.003µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:05:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 170.275µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:05:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 199.741µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:05:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 251.27µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:05:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 298.1µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:06:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 330.851µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:06:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 222.358µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:06:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 186.848µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:06:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 830.409µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:07:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 256.492µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:07:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 361.217µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:07:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 280.73µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:07:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 317.837µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:08:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 210.27µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:08:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 175.694µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:08:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 328.832µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:08:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 439.625µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:09:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 189.946µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:09:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 227.874µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:09:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 344.762µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:09:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 322.743µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:10:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 248.714µs [negroni] Started POST /volumes [negroni] PANIC: CHECK: func (github.com/heketi/heketi/pkg/utils.IdSource.ReadUUID) 0xdda29b File /build/golang/src/github.com/heketi/heketi/pkg/utils/uuid.go:34 Info: [12 16] goroutine 1246 [running]: github.com/heketi/heketi/vendor/github.com/urfave/negroni.(*Recovery).ServeHTTP.func1(0x7f9cb259bd48, 0xc4205d5d00, 0xc420583400) /build/golang/src/github.com/heketi/heketi/vendor/github.com/urfave/negroni/recovery.go:34 +0xef panic(0x16a6860, 0xc420332aa0) /usr/lib/golang/src/runtime/panic.go:491 +0x283 github.com/heketi/heketi/vendor/github.com/lpabon/godbc.dbc_panic(0x18a7541, 0x5, 0x0, 0xc4206daba0, 0x2, 0x2) /build/golang/src/github.com/heketi/heketi/vendor/github.com/lpabon/godbc/godbc.go:85 +0x40c github.com/heketi/heketi/vendor/github.com/lpabon/godbc.Check(0x1648100, 0xc4206daba0, 0x2, 0x2) /build/golang/src/github.com/heketi/heketi/vendor/github.com/lpabon/godbc/godbc.go:123 +0x5d github.com/heketi/heketi/pkg/utils.IdSource.ReadUUID(0x2545b00, 0xc4200a8180, 0x1, 0x25ea6b8) /build/golang/src/github.com/heketi/heketi/pkg/utils/uuid.go:34 +0x19c github.com/heketi/heketi/pkg/utils.GenUUID(0x1648ce0, 0x0) /build/golang/src/github.com/heketi/heketi/pkg/utils/uuid.go:42 +0x39 github.com/heketi/heketi/apps/glusterfs.NewVolumeEntryFromRequest(0xc42092c480, 0xc420959380) /build/golang/src/github.com/heketi/heketi/apps/glusterfs/volume_entry.go:90 +0xf2 github.com/heketi/heketi/apps/glusterfs.(*App).VolumeCreate(0xc4202a2a20, 0x7f9cb259bd48, 0xc4205d5d00, 0xc420a32d00) /build/golang/src/github.com/heketi/heketi/apps/glusterfs/app_volume.go:136 +0x300 github.com/heketi/heketi/apps/glusterfs.(*App).VolumeCreate-fm(0x7f9cb259bd48, 0xc4205d5d00, 0xc420a32d00) /build/golang/src/github.com/heketi/heketi/apps/glusterfs/app.go:466 +0x48 net/http.HandlerFunc.ServeHTTP(0xc420072fd0, 0x7f9cb259bd48, 0xc4205d5d00, 0xc420a32d00) /usr/lib/golang/src/net/http/server.go:1918 +0x44 github.com/heketi/heketi/vendor/github.com/gorilla/mux.(*Router).ServeHTTP(0xc420135270, 0x7f9cb259bd48, 0xc4205d5d00, 0xc420a32d00) /build/golang/src/github.com/heketi/heketi/vendor/github.com/gorilla/mux/mux.go:114 +0xdc github.com/heketi/heketi/vendor/github.com/urfave/negroni.Wrap.func1(0x7f9cb259bd48, 0xc4205d5d00, 0xc420a32900, 0xc4206daa80) /build/golang/src/github.com/heketi/heketi/vendor/github.com/urfave/negroni/negroni.go:41 +0x4d github.com/heketi/heketi/vendor/github.com/urfave/negroni.HandlerFunc.ServeHTTP(0xc42079a800, 0x7f9cb259bd48, 0xc4205d5d00, 0xc420a32900, 0xc4206daa80) /build/golang/src/github.com/heketi/heketi/vendor/github.com/urfave/negroni/negroni.go:24 +0x4e github.com/heketi/heketi/vendor/github.com/urfave/negroni.middleware.ServeHTTP(0x254e540, 0xc42079a800, 0xc42079a8c0, 0x7f9cb259bd48, 0xc4205d5d00, 0xc420a32900) /build/golang/src/github.com/heketi/heketi/vendor/github.com/urfave/negroni/negroni.go:33 +0xa5 github.com/heketi/heketi/vendor/github.com/urfave/negroni.(middleware).ServeHTTP-fm(0x7f9cb259bd48, 0xc4205d5d00, 0xc420a32900) /build/golang/src/github.com/heketi/heketi/vendor/github.com/urfave/negroni/negroni.go:33 +0x60 github.com/heketi/heketi/server/admin.(*ServerState).ServeHTTP(0xc42077be30, 0x7f9cb259bd48, 0xc4205d5d00, 0xc420a32900, 0xc4206daa40) /build/golang/src/github.com/heketi/heketi/server/admin/http.go:55 +0x69 github.com/heketi/heketi/vendor/github.com/urfave/negroni.middleware.ServeHTTP(0x2546140, 0xc42077be30, 0xc42079a8a0, 0x7f9cb259bd48, 0xc4205d5d00, 0xc420a32900) /build/golang/src/github.com/heketi/heketi/vendor/github.com/urfave/negroni/negroni.go:33 +0xa5 github.com/heketi/heketi/vendor/github.com/urfave/negroni.(middleware).ServeHTTP-fm(0x7f9cb259bd48, 0xc4205d5d00, 0xc420a32900) /build/golang/src/github.com/heketi/heketi/vendor/github.com/urfave/negroni/negroni.go:33 +0x60 github.com/heketi/heketi/apps/glusterfs.(*App).Auth(0xc4202a2a20, 0x7f9cb259bd48, 0xc4205d5d00, 0xc420a32900, 0xc4206daa20) /build/golang/src/github.com/heketi/heketi/apps/glusterfs/app_middleware.go:45 +0x126 github.com/heketi/heketi/apps/glusterfs.(*App).Auth-fm(0x7f9cb259bd48, 0xc4205d5d00, 0xc420a32900, 0xc4206daa20) /build/golang/src/github.com/heketi/heketi/main.go:374 +0x52 github.com/heketi/heketi/vendor/github.com/urfave/negroni.HandlerFunc.ServeHTTP(0xc4201b67a0, 0x7f9cb259bd48, 0xc4205d5d00, 0xc420a32900, 0xc4206daa20) /build/golang/src/github.com/heketi/heketi/vendor/github.com/urfave/negroni/negroni.go:24 +0x4e github.com/heketi/heketi/vendor/github.com/urfave/negroni.middleware.ServeHTTP(0x254e540, 0xc4201b67a0, 0xc42079a880, 0x7f9cb259bd48, 0xc4205d5d00, 0xc420a32900) /build/golang/src/github.com/heketi/heketi/vendor/github.com/urfave/negroni/negroni.go:33 +0xa5 github.com/heketi/heketi/vendor/github.com/urfave/negroni.(middleware).ServeHTTP-fm(0x7f9cb259bd48, 0xc4205d5d00, 0xc420a32900) /build/golang/src/github.com/heketi/heketi/vendor/github.com/urfave/negroni/negroni.go:33 +0x60 github.com/heketi/heketi/middleware.(*JwtAuth).ServeHTTP(0xc42077be00, 0x7f9cb259bd48, 0xc4205d5d00, 0xc420a32900, 0xc4206da8e0) /build/golang/src/github.com/heketi/heketi/middleware/jwt.go:193 +0x48d github.com/heketi/heketi/vendor/github.com/urfave/negroni.middleware.ServeHTTP(0x2546080, 0xc42077be00, 0xc42079a860, 0x7f9cb259bd48, 0xc4205d5d00, 0xc420a32900) /build/golang/src/github.com/heketi/heketi/vendor/github.com/urfave/negroni/negroni.go:33 +0xa5 github.com/heketi/heketi/vendor/github.com/urfave/negroni.(middleware).ServeHTTP-fm(0x7f9cb259bd48, 0xc4205d5d00, 0xc420a32900) /build/golang/src/github.com/heketi/heketi/vendor/github.com/urfave/negroni/negroni.go:33 +0x60 github.com/heketi/heketi/vendor/github.com/urfave/negroni.(*Logger).ServeHTTP(0xc420586198, 0x7f9cb259bd48, 0xc4205d5d00, 0xc420a32900, 0xc4206da720) /build/golang/src/github.com/heketi/heketi/vendor/github.com/urfave/negroni/logger.go:25 +0x16d github.com/heketi/heketi/vendor/github.com/urfave/negroni.middleware.ServeHTTP(0x2546880, 0xc420586198, 0xc42079a840, 0x7f9cb259bd48, 0xc4205d5d00, 0xc420a32900) /build/golang/src/github.com/heketi/heketi/vendor/github.com/urfave/negroni/negroni.go:33 +0xa5 github.com/heketi/heketi/vendor/github.com/urfave/negroni.(middleware).ServeHTTP-fm(0x7f9cb259bd48, 0xc4205d5d00, 0xc420a32900) /build/golang/src/github.com/heketi/heketi/vendor/github.com/urfave/negroni/negroni.go:33 +0x60 github.com/heketi/heketi/vendor/github.com/urfave/negroni.(*Recovery).ServeHTTP(0xc420583400, 0x7f9cb259bd48, 0xc4205d5d00, 0xc420a32900, 0xc4206da700) /build/golang/src/github.com/heketi/heketi/vendor/github.com/urfave/negroni/recovery.go:45 +0x82 github.com/heketi/heketi/vendor/github.com/urfave/negroni.middleware.ServeHTTP(0x2546900, 0xc420583400, 0xc42079a820, 0x7f9cb259bd48, 0xc4205d5d00, 0xc420a32900) /build/golang/src/github.com/heketi/heketi/vendor/github.com/urfave/negroni/negroni.go:33 +0xa5 github.com/heketi/heketi/vendor/github.com/urfave/negroni.(*Negroni).ServeHTTP(0xc42037edb0, 0x255e5c0, 0xc4206ef180, 0xc420a32900) /build/golang/src/github.com/heketi/heketi/vendor/github.com/urfave/negroni/negroni.go:73 +0x100 github.com/heketi/heketi/vendor/github.com/gorilla/mux.(*Router).ServeHTTP(0xc420135130, 0x255e5c0, 0xc4206ef180, 0xc420a32900) /build/golang/src/github.com/heketi/heketi/vendor/github.com/gorilla/mux/mux.go:114 +0xdc net/http.serverHandler.ServeHTTP(0xc4207b2000, 0x255e5c0, 0xc4206ef180, 0xc420a32600) /usr/lib/golang/src/net/http/server.go:2619 +0xb4 net/http.(*conn).serve(0xc4200bd4a0, 0x255f840, 0xc420842d80) /usr/lib/golang/src/net/http/server.go:1801 +0x71d created by net/http.(*Server).Serve /usr/lib/golang/src/net/http/server.go:2720 +0x288 [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:10:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 595.494µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:10:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 246.878µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:11:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 223.928µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:11:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 232.189µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:11:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 274.92µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:11:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 360.138µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:12:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 419.536µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:12:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 172.297µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:12:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 344.941µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:12:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 302.195µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:13:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 446.799µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:13:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 287.337µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:13:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 193.97µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:13:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 174.162µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:14:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 351.728µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:14:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 231.506µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:14:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 196.076µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:14:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 390.865µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:15:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 900.042µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:15:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 330.65µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:15:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 326.316µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:15:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 230.25µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:16:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 188.501µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:16:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 299.659µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:16:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 391.049µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:16:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 345.609µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:17:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 540.436µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:17:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 406.569µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:17:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 261.787µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:17:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 282.851µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:18:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 197.322µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:18:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 330.954µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:18:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 197.801µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:18:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 262.719µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:19:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 1.43819ms [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:19:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 311.734µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:19:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 228.193µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:19:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 175.639µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:20:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 194.941µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:20:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 168.218µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:20:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 318.348µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:20:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 186.539µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:21:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 577.326µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:21:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 327.039µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:21:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 340.658µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:21:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 200.26µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:22:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 187.678µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:22:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 191.848µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:22:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 343.227µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:22:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 267.344µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:23:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 601.03µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:23:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 331.366µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:23:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 177.519µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:23:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 192.549µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:24:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 361.685µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:24:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 366.671µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:24:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 226.697µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:24:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 374.556µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:25:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 851.056µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:25:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 340.818µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:25:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 372.929µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:25:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 234.305µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:26:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 250.05µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:26:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 185.376µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:26:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 225.9µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:26:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 316.257µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:27:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 404.92µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:27:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 169.021µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:27:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 224.44µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:27:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 228.735µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:28:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 285.645µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:28:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 174.21µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:28:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 352.675µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:28:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 190.191µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:29:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 358.431µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:29:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 485.058µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:29:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 219.83µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:29:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 357.516µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:30:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 419.493µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:30:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 346.75µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:30:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 189.93µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:30:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 168.308µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:31:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 162.139µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:31:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 201.356µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:31:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 167.658µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:31:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 335.324µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:32:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 311.815µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:32:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 322.397µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:32:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 347.744µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:32:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 348.614µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:33:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 399.58µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:33:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 1.303776ms [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:33:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 187.647µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:33:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 343.439µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:34:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 246.635µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:34:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 314.633µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:34:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 213.896µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:34:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 394.351µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:35:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 159.367µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:35:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 768.712µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:35:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 335.249µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:35:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 207.425µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:36:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 232.598µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:36:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 372.168µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:36:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 352.848µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:36:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 167.553µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:37:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 164.163µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:37:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 442.343µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:37:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 318.554µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:37:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 348.799µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:38:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 274.493µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:38:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 178.534µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:38:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 312.227µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:38:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 314.655µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:39:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 219.02µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:39:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 174.781µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:39:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 240.562µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:39:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 239.08µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:40:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 323.2µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:40:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 300.355µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:40:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 365.325µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:40:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 211.858µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:41:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 312.846µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:41:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 231.03µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:41:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 220.229µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:41:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 342.975µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:42:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 316.999µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:42:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 313.755µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:42:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 823.643µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:42:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 162.303µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:43:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 416.99µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:43:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 311.662µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:43:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 398.682µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:43:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 170.511µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:44:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 355.275µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:44:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 207.915µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:44:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 177.304µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:44:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 301.819µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:45:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 297.383µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:45:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 205.897µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:45:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 619.448µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:45:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 320.039µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:46:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 307.617µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:46:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 322.118µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:46:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 198.23µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:46:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 151.18µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:47:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 202.763µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:47:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 334.597µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:47:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 1.045798ms [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:47:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 212.146µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:48:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 338.643µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:48:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 422.411µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:48:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 308.414µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:48:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 214.629µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:49:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 341.923µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:49:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 410.876µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:49:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 189.011µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:49:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 218.657µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:50:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 382.674µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:50:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 212.325µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:50:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 332.068µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:50:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 211.048µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:51:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 314.467µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:51:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 176.347µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:51:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 298.698µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:51:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 413.609µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:52:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 203.063µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:52:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 392.585µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:52:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 249.034µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:52:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 397.922µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:53:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 254.771µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:53:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 340.426µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:53:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 381.07µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:53:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 220.78µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:54:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 193.99µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:54:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 332.131µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:54:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 355.978µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:54:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 152.867µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:55:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 170.786µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:55:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 341.312µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:55:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 267.829µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:55:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 1.059237ms [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:56:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 216.251µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:56:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 162.558µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:56:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 183.002µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:56:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 219.881µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:57:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 189.817µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:57:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 358.16µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:57:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 319.81µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:57:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 231.893µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:58:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 217.724µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:58:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 328.524µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:58:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 272.89µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:58:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 401.954µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:59:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 166.181µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:59:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 372.265µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:59:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 311.143µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 08:59:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 273.911µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:00:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 215.805µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:00:23 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 210.629µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:00:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 373.113µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:00:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 264.618µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:01:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 373.316µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:01:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 334.048µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:01:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 189.863µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:01:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 332.309µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:02:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 232.996µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:02:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 327.961µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:02:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 330.925µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:02:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 327.715µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:03:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 364.433µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:03:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 220.927µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:03:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 398.02µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:03:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 425.151µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:04:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 244.408µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:04:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 201.056µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:04:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 178.377µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:04:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 231.34µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:05:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 180.255µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:05:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 284.924µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:05:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 227.706µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:05:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 146.91µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:06:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 282.689µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:06:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 208.965µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:06:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 244.428µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:06:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 220.955µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:07:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 330.393µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:07:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 206.907µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:07:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 230.743µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:07:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 322.946µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:08:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 199.683µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:08:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 197.851µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:08:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 157.693µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:08:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 397.682µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:09:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 357.573µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:09:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 351.856µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:09:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 250.365µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:09:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 311.372µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:10:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 192.504µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:10:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 566.611µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:10:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 175.954µs [negroni] Started GET /operations [negroni] Completed 401 Unauthorized in 79.912µs [negroni] Started GET /operations [negroni] Completed 200 OK in 166.427µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:10:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 343.008µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:11:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 238.932µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:11:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 238.247µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:11:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 343.677µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:11:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 223.579µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:12:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 173.124µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:12:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 680.294µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:12:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 234.067µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:12:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 231.76µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:13:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 196.788µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:13:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 300.578µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:13:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 265.011µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:13:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 198.857µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:14:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 223.796µs [negroni] Started GET /volumes [negroni] Completed 200 OK in 219.233µs [negroni] Started GET /volumes/2e2e6b9cf174901b370ea79a266c651b [negroni] Completed 200 OK in 1.642036ms [negroni] Started GET /volumes/2e6ad6aa03f7fe219807cb135ca1c766 [negroni] Completed 200 OK in 689.915µs [negroni] Started GET /volumes/47c2af6849ac9ad7d7fbc897ae3ae80c [negroni] Completed 200 OK in 458.031µs [negroni] Started GET /volumes/5101331e9d33a5d04adab92837b9d5ad [negroni] Completed 200 OK in 460.458µs [negroni] Started GET /volumes/62905ab0e959e7663f4576501c0c9b69 [negroni] Completed 200 OK in 369.783µs [negroni] Started GET /volumes/63428c902b5d577a7739eb66a050a420 [negroni] Completed 200 OK in 439.812µs [negroni] Started GET /volumes/78d426889b7bd661f3d6c6f7814b6d4f [negroni] Completed 200 OK in 368.851µs [negroni] Started GET /volumes/7e73769a754d7f9f6213ef3b4551af0e [negroni] Completed 200 OK in 460.146µs [negroni] Started GET /volumes/83dcb62de26517790f155b51526cace5 [negroni] Completed 200 OK in 365.339µs [negroni] Started GET /volumes/88fbf36bf87106e549dca765f171cf69 [negroni] Completed 200 OK in 327.914µs [negroni] Started GET /volumes/937aa673e0d60126082bbd8c1589e383 [negroni] Completed 200 OK in 358.577µs [negroni] Started GET /volumes/9f04d82be19e2ea8ee80deb9098cd390 [negroni] Completed 200 OK in 388.584µs [negroni] Started GET /volumes/bc5012fd39c5b3ab958b9da4b0256d3a [negroni] Completed 200 OK in 328.895µs [negroni] Started GET /volumes/c3b5f711128418ce9a7c68b514e34618 [negroni] Completed 200 OK in 448.594µs [negroni] Started GET /volumes/c5fa076446479cf414397591c0af1c7f [negroni] Completed 200 OK in 372.786µs [negroni] Started GET /volumes/dbc601f7162df326784aea34c4ebe8f2 [negroni] Completed 200 OK in 331.995µs [negroni] Started GET /volumes/dcee4f7f020d47e58fb48612bbba19d1 [negroni] Completed 200 OK in 337.639µs [negroni] Started GET /volumes [negroni] Completed 200 OK in 177.511µs [negroni] Started GET /volumes/2e2e6b9cf174901b370ea79a266c651b [negroni] Completed 200 OK in 383.81µs [negroni] Started GET /volumes/2e6ad6aa03f7fe219807cb135ca1c766 [negroni] Completed 200 OK in 521.168µs [negroni] Started GET /volumes/47c2af6849ac9ad7d7fbc897ae3ae80c [negroni] Completed 200 OK in 473.727µs [negroni] Started GET /volumes/5101331e9d33a5d04adab92837b9d5ad [negroni] Completed 200 OK in 333.262µs [negroni] Started GET /volumes/62905ab0e959e7663f4576501c0c9b69 [negroni] Completed 200 OK in 317.539µs [negroni] Started GET /volumes/63428c902b5d577a7739eb66a050a420 [negroni] Completed 200 OK in 365.87µs [negroni] Started GET /volumes/78d426889b7bd661f3d6c6f7814b6d4f [negroni] Completed 200 OK in 322.688µs [negroni] Started GET /volumes/7e73769a754d7f9f6213ef3b4551af0e [negroni] Completed 200 OK in 319.535µs [negroni] Started GET /volumes/83dcb62de26517790f155b51526cace5 [negroni] Completed 200 OK in 337.907µs [negroni] Started GET /volumes/88fbf36bf87106e549dca765f171cf69 [negroni] Completed 200 OK in 316.617µs [negroni] Started GET /volumes/937aa673e0d60126082bbd8c1589e383 [negroni] Completed 200 OK in 397.347µs [negroni] Started GET /volumes/9f04d82be19e2ea8ee80deb9098cd390 [negroni] Completed 200 OK in 311.492µs [negroni] Started GET /volumes/bc5012fd39c5b3ab958b9da4b0256d3a [negroni] Completed 200 OK in 318.182µs [negroni] Started GET /volumes/c3b5f711128418ce9a7c68b514e34618 [negroni] Completed 200 OK in 360.286µs [negroni] Started GET /volumes/c5fa076446479cf414397591c0af1c7f [negroni] Completed 200 OK in 385.462µs [negroni] Started GET /volumes/dbc601f7162df326784aea34c4ebe8f2 [negroni] Completed 200 OK in 328.744µs [negroni] Started GET /volumes/dcee4f7f020d47e58fb48612bbba19d1 [negroni] Completed 200 OK in 321.539µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:14:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 702.635µs [negroni] Started GET /operations [negroni] Completed 200 OK in 163.737µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:14:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 776.43µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:14:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 213.782µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:15:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 168.199µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:15:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 367.935µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:15:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 204.985µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:15:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 324.554µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:16:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 227.682µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:16:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 303.86µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:16:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 355.329µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:16:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 175.884µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:17:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 209.67µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:17:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 353.411µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:17:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 151.15µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:17:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 342.522µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:18:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 217.934µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:18:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 325.532µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:18:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 448.931µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:18:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 272.9µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:19:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 493.309µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:19:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 225.464µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:19:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 314.797µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:19:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 298.971µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:20:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 525.993µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:20:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 318.084µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:20:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 704.173µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:20:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 233.777µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:21:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 340.802µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:21:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 287.331µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:21:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 368.177µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:21:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 348.916µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:22:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 249.292µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:22:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 189.684µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:22:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 170.041µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:22:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 223.376µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:23:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 320.86µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:23:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 324.62µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:23:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 198.149µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:23:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 226.109µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:24:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 358.887µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:24:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 231.545µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:24:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 372.701µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:24:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 470.962µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:25:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 223.404µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:25:07 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 185.679µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:25:18 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 211.786µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:25:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 337.247µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:25:35 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 191.545µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:25:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 337.048µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:25:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 374.233µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:25:52 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 199.399µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:26:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 262.636µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:26:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 246.025µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:26:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 250.491µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:26:32 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 191.15µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:26:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 319.24µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:26:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 898.699µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:27:00 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 233.576µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:27:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 208.029µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:27:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 251.467µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:27:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 349.458µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:27:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 298.843µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:28:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 335.529µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:28:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 341.053µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:28:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 196.127µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:28:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 468.544µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:29:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 236.939µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:29:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 317.235µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:29:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 296.565µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:29:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 301.693µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:30:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 209.072µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:30:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 207.577µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:30:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 214.255µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:30:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 281.427µs [negroni] Started GET /operations [negroni] Completed 200 OK in 277.974µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:31:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 405.827µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:31:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 369.588µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:31:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 190.032µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:31:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 345.693µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:32:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 195.421µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:32:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 337.884µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:32:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 336.975µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:32:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 180.506µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:33:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 396.412µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:33:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 185.964µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:33:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 299.981µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:33:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 225.913µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:34:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 180.258µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:34:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 183.552µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:34:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 168.84µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:34:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 379.45µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:35:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 378.403µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:35:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 341.641µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:35:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 244.451µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:35:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 324.564µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:36:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 305.151µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:36:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 256.7µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:36:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 212.069µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:36:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 328.516µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:37:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 294.288µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:37:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 171.075µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:37:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 201.229µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:37:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 227.851µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:38:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 309.231µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:38:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 348.649µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:38:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 309.33µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:38:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 195.542µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:39:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 310.479µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:39:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 252.048µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:39:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 294.588µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:39:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 330.372µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:40:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 421.78µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:40:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 181.001µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:40:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 252.115µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:40:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 155.161µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:41:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 371.37µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:41:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 235.222µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:41:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 354.567µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:41:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 225.288µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:42:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 245.328µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:42:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 393.571µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:42:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 355.285µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:42:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 212.775µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:43:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 330.351µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:43:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 753.032µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:43:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 301.122µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:43:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 172.447µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:44:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 205.967µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:44:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 176.951µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:44:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 248.395µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:44:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 514.302µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:45:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 303.846µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:45:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 1.114422ms [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:45:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 265.384µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:45:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 334.444µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:46:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 355.054µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:46:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 396.658µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:46:36 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 388.804µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:46:51 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 380.931µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:47:06 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 308.839µs [negroni] Started POST /volumes [heketi] WARNING 2019/01/24 09:47:21 operations in-flight (8) exceeds limit (8) [negroni] Completed 429 Too Many Requests in 320.256µs