========= iteration 1 -- 1 of 25 ========= -- scale 1 clients 16 INIT_WAIT 0 CLEAN_ONLY= ============================================================================ -- 20170222 17:17:55 -- scale 1 clients 16 INIT_WAIT 0 -- sub_count -ne 0 : deleting sub1 pub_count -ne 0 - deleting pub1 pub_count 0 pub_repl_slot_count 0 sub_count 0 sub_repl_slot_count 0 creating tables... 100000 of 100000 tuples (100%) done (elapsed 0.12 s, remaining 0.00 s) vacuum... set primary keys... done. create publication pub1 for all tables; create subscription sub1 connection 'port=6972' publication pub1 with (disabled); alter subscription sub1 enable; -- pgbench -c 16 -T 10 -P 5 -n progress: 5.0 s, 58.6 tps, lat 259.785 ms stddev 241.722 progress: 10.0 s, 78.0 tps, lat 206.134 ms stddev 141.885 transaction type: scaling factor: 1 query mode: simple number of clients: 16 number of threads: 1 duration: 10 s number of transactions actually processed: 699 latency average = 229.399 ms latency stddev = 191.393 ms tps = 68.525051 (including connections establishing) tps = 68.559004 (excluding connections establishing) -- waiting 4s... (scale * 4) 6972 a,b,t,h: 100000 1 10 699 98fa13bf6 2adcbd100 c87137c0c d19b38ac8 master 6973 a,b,t,h: 100000 1 0 0 d51a318dd 2adcbd100 d41d8cd98 d41d8cd98 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 699 98fa13bf6 2adcbd100 c87137c0c d19b38ac8 master 6973 a,b,t,h: 100000 1 10 699 8b34ad93a 2adcbd100 c87137c0c d19b38ac8 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 699 98fa13bf6 2adcbd100 c87137c0c d19b38ac8 master 6973 a,b,t,h: 100000 1 10 699 98fa13bf6 2adcbd100 c87137c0c d19b38ac8 replica ok -- All is well. ========= iteration 2 -- 2 of 25 ========= -- scale 1 clients 16 INIT_WAIT 0 CLEAN_ONLY= ============================================================================ -- 20170222 17:18:25 -- scale 1 clients 16 INIT_WAIT 0 -- sub_count -ne 0 : deleting sub1 pub_count -ne 0 - deleting pub1 pub_count 0 pub_repl_slot_count 0 sub_count 0 sub_repl_slot_count 0 creating tables... 100000 of 100000 tuples (100%) done (elapsed 0.12 s, remaining 0.00 s) vacuum... set primary keys... done. create publication pub1 for all tables; create subscription sub1 connection 'port=6972' publication pub1 with (disabled); alter subscription sub1 enable; -- pgbench -c 16 -T 10 -P 5 -n progress: 5.0 s, 36.0 tps, lat 412.231 ms stddev 407.886 progress: 10.0 s, 31.2 tps, lat 512.439 ms stddev 358.707 transaction type: scaling factor: 1 query mode: simple number of clients: 16 number of threads: 1 duration: 10 s number of transactions actually processed: 352 latency average = 461.261 ms latency stddev = 385.822 ms tps = 33.777938 (including connections establishing) tps = 33.793989 (excluding connections establishing) -- waiting 4s... (scale * 4) 6972 a,b,t,h: 100000 1 10 352 dec0d04ec 390ee372e 594615da1 fa1903458 master 6973 a,b,t,h: 100000 1 10 329 dec0d04ec 390ee372e 594615da1 a9d4b3d33 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 352 dec0d04ec 390ee372e 594615da1 fa1903458 master 6973 a,b,t,h: 100000 1 10 329 dec0d04ec 390ee372e 594615da1 a9d4b3d33 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 352 dec0d04ec 390ee372e 594615da1 fa1903458 master 6973 a,b,t,h: 100000 1 10 329 dec0d04ec 390ee372e 594615da1 a9d4b3d33 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 352 dec0d04ec 390ee372e 594615da1 fa1903458 master 6973 a,b,t,h: 100000 1 10 329 dec0d04ec 390ee372e 594615da1 a9d4b3d33 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 352 dec0d04ec 390ee372e 594615da1 fa1903458 master 6973 a,b,t,h: 100000 1 10 329 dec0d04ec 390ee372e 594615da1 a9d4b3d33 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 352 dec0d04ec 390ee372e 594615da1 fa1903458 master 6973 a,b,t,h: 100000 1 10 329 dec0d04ec 390ee372e 594615da1 a9d4b3d33 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 352 dec0d04ec 390ee372e 594615da1 fa1903458 master 6973 a,b,t,h: 100000 1 10 329 dec0d04ec 390ee372e 594615da1 a9d4b3d33 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 352 dec0d04ec 390ee372e 594615da1 fa1903458 master 6973 a,b,t,h: 100000 1 10 329 dec0d04ec 390ee372e 594615da1 a9d4b3d33 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 352 dec0d04ec 390ee372e 594615da1 fa1903458 master 6973 a,b,t,h: 100000 1 10 329 dec0d04ec 390ee372e 594615da1 a9d4b3d33 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 352 dec0d04ec 390ee372e 594615da1 fa1903458 master 6973 a,b,t,h: 100000 1 10 329 dec0d04ec 390ee372e 594615da1 a9d4b3d33 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 352 dec0d04ec 390ee372e 594615da1 fa1903458 master 6973 a,b,t,h: 100000 1 10 329 dec0d04ec 390ee372e 594615da1 a9d4b3d33 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 352 dec0d04ec 390ee372e 594615da1 fa1903458 master 6973 a,b,t,h: 100000 1 10 329 dec0d04ec 390ee372e 594615da1 a9d4b3d33 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 352 dec0d04ec 390ee372e 594615da1 fa1903458 master 6973 a,b,t,h: 100000 1 10 329 dec0d04ec 390ee372e 594615da1 a9d4b3d33 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 352 dec0d04ec 390ee372e 594615da1 fa1903458 master 6973 a,b,t,h: 100000 1 10 329 dec0d04ec 390ee372e 594615da1 a9d4b3d33 replica NOK -- Not good, but breaking out of wait (waited more than 60s) ========= iteration 3 -- 3 of 25 ========= -- scale 1 clients 16 INIT_WAIT 0 CLEAN_ONLY= ============================================================================ -- 20170222 17:19:58 -- scale 1 clients 16 INIT_WAIT 0 -- sub_count -ne 0 : deleting sub1 pub_count -ne 0 - deleting pub1 pub_count 0 pub_repl_slot_count 0 sub_count 0 sub_repl_slot_count 0 creating tables... 100000 of 100000 tuples (100%) done (elapsed 0.11 s, remaining 0.00 s) vacuum... set primary keys... done. create publication pub1 for all tables; create subscription sub1 connection 'port=6972' publication pub1 with (disabled); alter subscription sub1 enable; -- pgbench -c 16 -T 10 -P 5 -n progress: 5.0 s, 31.8 tps, lat 469.885 ms stddev 359.256 progress: 10.0 s, 31.6 tps, lat 492.270 ms stddev 394.721 transaction type: scaling factor: 1 query mode: simple number of clients: 16 number of threads: 1 duration: 10 s number of transactions actually processed: 333 latency average = 488.720 ms latency stddev = 384.282 ms tps = 31.775203 (including connections establishing) tps = 31.790507 (excluding connections establishing) -- waiting 4s... (scale * 4) 6972 a,b,t,h: 100000 1 10 333 11f57c86f d57f95b68 0d6428196 94d33c4f8 master 6973 a,b,t,h: 100000 1 10 333 11f57c86f d57f95b68 0d6428196 94d33c4f8 replica ok -- All is well. ========= iteration 4 -- 4 of 25 ========= -- scale 1 clients 16 INIT_WAIT 0 CLEAN_ONLY= ============================================================================ -- 20170222 17:20:17 -- scale 1 clients 16 INIT_WAIT 0 -- sub_count -ne 0 : deleting sub1 pub_count -ne 0 - deleting pub1 pub_count 0 pub_repl_slot_count 0 sub_count 0 sub_repl_slot_count 0 creating tables... 100000 of 100000 tuples (100%) done (elapsed 0.12 s, remaining 0.00 s) vacuum... set primary keys... done. create publication pub1 for all tables; create subscription sub1 connection 'port=6972' publication pub1 with (disabled); alter subscription sub1 enable; -- pgbench -c 16 -T 10 -P 5 -n progress: 5.0 s, 28.4 tps, lat 514.337 ms stddev 363.885 progress: 10.0 s, 36.6 tps, lat 443.849 ms stddev 318.958 transaction type: scaling factor: 1 query mode: simple number of clients: 16 number of threads: 1 duration: 10 s number of transactions actually processed: 341 latency average = 475.087 ms latency stddev = 335.317 ms tps = 32.821439 (including connections establishing) tps = 32.837889 (excluding connections establishing) -- waiting 4s... (scale * 4) 6972 a,b,t,h: 100000 1 10 341 616c4d757 dc542b2ed ab6283467 77694c77b master 6973 a,b,t,h: 100000 1 10 337 616c4d757 dc542b2ed ab6283467 e1ea0ed75 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 341 616c4d757 dc542b2ed ab6283467 77694c77b master 6973 a,b,t,h: 100000 1 10 337 616c4d757 dc542b2ed ab6283467 e1ea0ed75 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 341 616c4d757 dc542b2ed ab6283467 77694c77b master 6973 a,b,t,h: 100000 1 10 337 616c4d757 dc542b2ed ab6283467 e1ea0ed75 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 341 616c4d757 dc542b2ed ab6283467 77694c77b master 6973 a,b,t,h: 100000 1 10 337 616c4d757 dc542b2ed ab6283467 e1ea0ed75 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 341 616c4d757 dc542b2ed ab6283467 77694c77b master 6973 a,b,t,h: 100000 1 10 337 616c4d757 dc542b2ed ab6283467 e1ea0ed75 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 341 616c4d757 dc542b2ed ab6283467 77694c77b master 6973 a,b,t,h: 100000 1 10 337 616c4d757 dc542b2ed ab6283467 e1ea0ed75 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 341 616c4d757 dc542b2ed ab6283467 77694c77b master 6973 a,b,t,h: 100000 1 10 337 616c4d757 dc542b2ed ab6283467 e1ea0ed75 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 341 616c4d757 dc542b2ed ab6283467 77694c77b master 6973 a,b,t,h: 100000 1 10 337 616c4d757 dc542b2ed ab6283467 e1ea0ed75 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 341 616c4d757 dc542b2ed ab6283467 77694c77b master 6973 a,b,t,h: 100000 1 10 337 616c4d757 dc542b2ed ab6283467 e1ea0ed75 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 341 616c4d757 dc542b2ed ab6283467 77694c77b master 6973 a,b,t,h: 100000 1 10 337 616c4d757 dc542b2ed ab6283467 e1ea0ed75 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 341 616c4d757 dc542b2ed ab6283467 77694c77b master 6973 a,b,t,h: 100000 1 10 337 616c4d757 dc542b2ed ab6283467 e1ea0ed75 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 341 616c4d757 dc542b2ed ab6283467 77694c77b master 6973 a,b,t,h: 100000 1 10 337 616c4d757 dc542b2ed ab6283467 e1ea0ed75 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 341 616c4d757 dc542b2ed ab6283467 77694c77b master 6973 a,b,t,h: 100000 1 10 337 616c4d757 dc542b2ed ab6283467 e1ea0ed75 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 341 616c4d757 dc542b2ed ab6283467 77694c77b master 6973 a,b,t,h: 100000 1 10 337 616c4d757 dc542b2ed ab6283467 e1ea0ed75 replica NOK -- Not good, but breaking out of wait (waited more than 60s) ========= iteration 5 -- 5 of 25 ========= -- scale 1 clients 16 INIT_WAIT 0 CLEAN_ONLY= ============================================================================ -- 20170222 17:21:50 -- scale 1 clients 16 INIT_WAIT 0 -- sub_count -ne 0 : deleting sub1 pub_count -ne 0 - deleting pub1 pub_count 0 pub_repl_slot_count 0 sub_count 0 sub_repl_slot_count 0 creating tables... 100000 of 100000 tuples (100%) done (elapsed 0.12 s, remaining 0.00 s) vacuum... set primary keys... done. create publication pub1 for all tables; create subscription sub1 connection 'port=6972' publication pub1 with (disabled); alter subscription sub1 enable; -- pgbench -c 16 -T 10 -P 5 -n progress: 5.0 s, 35.4 tps, lat 404.254 ms stddev 308.388 progress: 10.0 s, 59.6 tps, lat 284.920 ms stddev 264.340 transaction type: scaling factor: 1 query mode: simple number of clients: 16 number of threads: 1 duration: 10 s number of transactions actually processed: 491 latency average = 326.625 ms latency stddev = 283.871 ms tps = 48.185066 (including connections establishing) tps = 48.207582 (excluding connections establishing) -- waiting 4s... (scale * 4) 6972 a,b,t,h: 100000 1 10 491 f7c545dad 85f789851 2856a4d5e a040409b6 master 6973 a,b,t,h: 100000 1 10 482 f7c545dad 85f789851 2856a4d5e 695641540 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 491 f7c545dad 85f789851 2856a4d5e a040409b6 master 6973 a,b,t,h: 100000 1 10 482 f7c545dad 85f789851 2856a4d5e 695641540 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 491 f7c545dad 85f789851 2856a4d5e a040409b6 master 6973 a,b,t,h: 100000 1 10 482 f7c545dad 85f789851 2856a4d5e 695641540 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 491 f7c545dad 85f789851 2856a4d5e a040409b6 master 6973 a,b,t,h: 100000 1 10 482 f7c545dad 85f789851 2856a4d5e 695641540 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 491 f7c545dad 85f789851 2856a4d5e a040409b6 master 6973 a,b,t,h: 100000 1 10 482 f7c545dad 85f789851 2856a4d5e 695641540 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 491 f7c545dad 85f789851 2856a4d5e a040409b6 master 6973 a,b,t,h: 100000 1 10 482 f7c545dad 85f789851 2856a4d5e 695641540 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 491 f7c545dad 85f789851 2856a4d5e a040409b6 master 6973 a,b,t,h: 100000 1 10 482 f7c545dad 85f789851 2856a4d5e 695641540 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 491 f7c545dad 85f789851 2856a4d5e a040409b6 master 6973 a,b,t,h: 100000 1 10 482 f7c545dad 85f789851 2856a4d5e 695641540 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 491 f7c545dad 85f789851 2856a4d5e a040409b6 master 6973 a,b,t,h: 100000 1 10 482 f7c545dad 85f789851 2856a4d5e 695641540 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 491 f7c545dad 85f789851 2856a4d5e a040409b6 master 6973 a,b,t,h: 100000 1 10 482 f7c545dad 85f789851 2856a4d5e 695641540 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 491 f7c545dad 85f789851 2856a4d5e a040409b6 master 6973 a,b,t,h: 100000 1 10 482 f7c545dad 85f789851 2856a4d5e 695641540 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 491 f7c545dad 85f789851 2856a4d5e a040409b6 master 6973 a,b,t,h: 100000 1 10 482 f7c545dad 85f789851 2856a4d5e 695641540 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 491 f7c545dad 85f789851 2856a4d5e a040409b6 master 6973 a,b,t,h: 100000 1 10 482 f7c545dad 85f789851 2856a4d5e 695641540 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 491 f7c545dad 85f789851 2856a4d5e a040409b6 master 6973 a,b,t,h: 100000 1 10 482 f7c545dad 85f789851 2856a4d5e 695641540 replica NOK -- Not good, but breaking out of wait (waited more than 60s) ========= iteration 6 -- 6 of 25 ========= -- scale 1 clients 16 INIT_WAIT 0 CLEAN_ONLY= ============================================================================ -- 20170222 17:23:23 -- scale 1 clients 16 INIT_WAIT 0 -- sub_count -ne 0 : deleting sub1 pub_count -ne 0 - deleting pub1 pub_count 0 pub_repl_slot_count 0 sub_count 0 sub_repl_slot_count 0 creating tables... 100000 of 100000 tuples (100%) done (elapsed 0.11 s, remaining 0.00 s) vacuum... set primary keys... done. create publication pub1 for all tables; create subscription sub1 connection 'port=6972' publication pub1 with (disabled); alter subscription sub1 enable; -- pgbench -c 16 -T 10 -P 5 -n progress: 5.0 s, 34.4 tps, lat 420.032 ms stddev 317.786 progress: 10.0 s, 33.8 tps, lat 483.442 ms stddev 379.011 transaction type: scaling factor: 1 query mode: simple number of clients: 16 number of threads: 1 duration: 10 s number of transactions actually processed: 357 latency average = 453.720 ms latency stddev = 346.757 ms tps = 34.309708 (including connections establishing) tps = 34.326189 (excluding connections establishing) -- waiting 4s... (scale * 4) 6972 a,b,t,h: 100000 1 10 357 a43f7bac1 9c1fb6c04 2c1dfd5a9 6dbb3dbce master 6973 a,b,t,h: 100000 1 10 357 a43f7bac1 9c1fb6c04 2c1dfd5a9 6dbb3dbce replica ok -- All is well. ========= iteration 7 -- 7 of 25 ========= -- scale 1 clients 16 INIT_WAIT 0 CLEAN_ONLY= ============================================================================ -- 20170222 17:23:42 -- scale 1 clients 16 INIT_WAIT 0 -- sub_count -ne 0 : deleting sub1 pub_count -ne 0 - deleting pub1 pub_count 0 pub_repl_slot_count 0 sub_count 0 sub_repl_slot_count 0 creating tables... 100000 of 100000 tuples (100%) done (elapsed 0.12 s, remaining 0.00 s) vacuum... set primary keys... done. create publication pub1 for all tables; create subscription sub1 connection 'port=6972' publication pub1 with (disabled); alter subscription sub1 enable; -- pgbench -c 16 -T 10 -P 5 -n progress: 5.0 s, 36.6 tps, lat 412.817 ms stddev 449.764 progress: 10.0 s, 30.6 tps, lat 502.317 ms stddev 388.423 transaction type: scaling factor: 1 query mode: simple number of clients: 16 number of threads: 1 duration: 10 s number of transactions actually processed: 352 latency average = 459.416 ms latency stddev = 427.646 ms tps = 33.896897 (including connections establishing) tps = 33.913492 (excluding connections establishing) -- waiting 4s... (scale * 4) 6972 a,b,t,h: 100000 1 10 352 9fa2b59d2 7a8fb27ab 3b749933f eda699a21 master 6973 a,b,t,h: 100000 1 10 333 9fa2b59d2 7a8fb27ab 3b749933f 690801655 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 352 9fa2b59d2 7a8fb27ab 3b749933f eda699a21 master 6973 a,b,t,h: 100000 1 10 333 9fa2b59d2 7a8fb27ab 3b749933f 690801655 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 352 9fa2b59d2 7a8fb27ab 3b749933f eda699a21 master 6973 a,b,t,h: 100000 1 10 333 9fa2b59d2 7a8fb27ab 3b749933f 690801655 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 352 9fa2b59d2 7a8fb27ab 3b749933f eda699a21 master 6973 a,b,t,h: 100000 1 10 333 9fa2b59d2 7a8fb27ab 3b749933f 690801655 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 352 9fa2b59d2 7a8fb27ab 3b749933f eda699a21 master 6973 a,b,t,h: 100000 1 10 333 9fa2b59d2 7a8fb27ab 3b749933f 690801655 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 352 9fa2b59d2 7a8fb27ab 3b749933f eda699a21 master 6973 a,b,t,h: 100000 1 10 333 9fa2b59d2 7a8fb27ab 3b749933f 690801655 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 352 9fa2b59d2 7a8fb27ab 3b749933f eda699a21 master 6973 a,b,t,h: 100000 1 10 333 9fa2b59d2 7a8fb27ab 3b749933f 690801655 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 352 9fa2b59d2 7a8fb27ab 3b749933f eda699a21 master 6973 a,b,t,h: 100000 1 10 333 9fa2b59d2 7a8fb27ab 3b749933f 690801655 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 352 9fa2b59d2 7a8fb27ab 3b749933f eda699a21 master 6973 a,b,t,h: 100000 1 10 333 9fa2b59d2 7a8fb27ab 3b749933f 690801655 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 352 9fa2b59d2 7a8fb27ab 3b749933f eda699a21 master 6973 a,b,t,h: 100000 1 10 333 9fa2b59d2 7a8fb27ab 3b749933f 690801655 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 352 9fa2b59d2 7a8fb27ab 3b749933f eda699a21 master 6973 a,b,t,h: 100000 1 10 333 9fa2b59d2 7a8fb27ab 3b749933f 690801655 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 352 9fa2b59d2 7a8fb27ab 3b749933f eda699a21 master 6973 a,b,t,h: 100000 1 10 333 9fa2b59d2 7a8fb27ab 3b749933f 690801655 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 352 9fa2b59d2 7a8fb27ab 3b749933f eda699a21 master 6973 a,b,t,h: 100000 1 10 333 9fa2b59d2 7a8fb27ab 3b749933f 690801655 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 352 9fa2b59d2 7a8fb27ab 3b749933f eda699a21 master 6973 a,b,t,h: 100000 1 10 333 9fa2b59d2 7a8fb27ab 3b749933f 690801655 replica NOK -- Not good, but breaking out of wait (waited more than 60s) ========= iteration 8 -- 8 of 25 ========= -- scale 1 clients 16 INIT_WAIT 0 CLEAN_ONLY= ============================================================================ -- 20170222 17:25:15 -- scale 1 clients 16 INIT_WAIT 0 -- sub_count -ne 0 : deleting sub1 pub_count -ne 0 - deleting pub1 pub_count 0 pub_repl_slot_count 0 sub_count 0 sub_repl_slot_count 0 creating tables... 100000 of 100000 tuples (100%) done (elapsed 0.11 s, remaining 0.00 s) vacuum... set primary keys... done. create publication pub1 for all tables; create subscription sub1 connection 'port=6972' publication pub1 with (disabled); alter subscription sub1 enable; -- pgbench -c 16 -T 10 -P 5 -n progress: 5.0 s, 34.2 tps, lat 403.648 ms stddev 363.201 progress: 10.0 s, 32.8 tps, lat 504.073 ms stddev 409.915 transaction type: scaling factor: 1 query mode: simple number of clients: 16 number of threads: 1 duration: 10 s number of transactions actually processed: 351 latency average = 461.544 ms latency stddev = 392.191 ms tps = 33.813511 (including connections establishing) tps = 33.828790 (excluding connections establishing) -- waiting 4s... (scale * 4) 6972 a,b,t,h: 100000 1 10 351 2686b2020 e00386847 62845ca37 817d830dd master 6973 a,b,t,h: 100000 1 10 330 bdc56dad3 e00386847 62845ca37 bc6cbef6f replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 351 2686b2020 e00386847 62845ca37 817d830dd master 6973 a,b,t,h: 100000 1 10 330 bdc56dad3 e00386847 62845ca37 bc6cbef6f replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 351 2686b2020 e00386847 62845ca37 817d830dd master 6973 a,b,t,h: 100000 1 10 330 bdc56dad3 e00386847 62845ca37 bc6cbef6f replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 351 2686b2020 e00386847 62845ca37 817d830dd master 6973 a,b,t,h: 100000 1 10 330 bdc56dad3 e00386847 62845ca37 bc6cbef6f replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 351 2686b2020 e00386847 62845ca37 817d830dd master 6973 a,b,t,h: 100000 1 10 330 bdc56dad3 e00386847 62845ca37 bc6cbef6f replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 351 2686b2020 e00386847 62845ca37 817d830dd master 6973 a,b,t,h: 100000 1 10 330 bdc56dad3 e00386847 62845ca37 bc6cbef6f replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 351 2686b2020 e00386847 62845ca37 817d830dd master 6973 a,b,t,h: 100000 1 10 330 bdc56dad3 e00386847 62845ca37 bc6cbef6f replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 351 2686b2020 e00386847 62845ca37 817d830dd master 6973 a,b,t,h: 100000 1 10 330 bdc56dad3 e00386847 62845ca37 bc6cbef6f replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 351 2686b2020 e00386847 62845ca37 817d830dd master 6973 a,b,t,h: 100000 1 10 330 bdc56dad3 e00386847 62845ca37 bc6cbef6f replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 351 2686b2020 e00386847 62845ca37 817d830dd master 6973 a,b,t,h: 100000 1 10 330 bdc56dad3 e00386847 62845ca37 bc6cbef6f replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 351 2686b2020 e00386847 62845ca37 817d830dd master 6973 a,b,t,h: 100000 1 10 330 bdc56dad3 e00386847 62845ca37 bc6cbef6f replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 351 2686b2020 e00386847 62845ca37 817d830dd master 6973 a,b,t,h: 100000 1 10 330 bdc56dad3 e00386847 62845ca37 bc6cbef6f replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 351 2686b2020 e00386847 62845ca37 817d830dd master 6973 a,b,t,h: 100000 1 10 330 bdc56dad3 e00386847 62845ca37 bc6cbef6f replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 351 2686b2020 e00386847 62845ca37 817d830dd master 6973 a,b,t,h: 100000 1 10 330 bdc56dad3 e00386847 62845ca37 bc6cbef6f replica NOK -- Not good, but breaking out of wait (waited more than 60s) ========= iteration 9 -- 9 of 25 ========= -- scale 1 clients 16 INIT_WAIT 0 CLEAN_ONLY= ============================================================================ -- 20170222 17:26:47 -- scale 1 clients 16 INIT_WAIT 0 -- sub_count -ne 0 : deleting sub1 pub_count -ne 0 - deleting pub1 pub_count 0 pub_repl_slot_count 0 sub_count 0 sub_repl_slot_count 0 creating tables... 100000 of 100000 tuples (100%) done (elapsed 0.11 s, remaining 0.00 s) vacuum... set primary keys... done. create publication pub1 for all tables; create subscription sub1 connection 'port=6972' publication pub1 with (disabled); alter subscription sub1 enable; -- pgbench -c 16 -T 10 -P 5 -n progress: 5.0 s, 31.2 tps, lat 471.704 ms stddev 360.982 progress: 10.0 s, 35.0 tps, lat 450.637 ms stddev 341.561 transaction type: scaling factor: 1 query mode: simple number of clients: 16 number of threads: 1 duration: 10 s number of transactions actually processed: 347 latency average = 466.516 ms latency stddev = 353.223 ms tps = 33.443149 (including connections establishing) tps = 33.458878 (excluding connections establishing) -- waiting 4s... (scale * 4) 6972 a,b,t,h: 100000 1 10 347 b1dd4021a 08daf99d8 759a726d4 7efe0dc26 master 6973 a,b,t,h: 100000 1 10 329 e2b197539 08daf99d8 759a726d4 4ca3d7367 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 347 b1dd4021a 08daf99d8 759a726d4 7efe0dc26 master 6973 a,b,t,h: 100000 1 10 329 e2b197539 08daf99d8 759a726d4 4ca3d7367 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 347 b1dd4021a 08daf99d8 759a726d4 7efe0dc26 master 6973 a,b,t,h: 100000 1 10 329 e2b197539 08daf99d8 759a726d4 4ca3d7367 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 347 b1dd4021a 08daf99d8 759a726d4 7efe0dc26 master 6973 a,b,t,h: 100000 1 10 329 e2b197539 08daf99d8 759a726d4 4ca3d7367 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 347 b1dd4021a 08daf99d8 759a726d4 7efe0dc26 master 6973 a,b,t,h: 100000 1 10 329 e2b197539 08daf99d8 759a726d4 4ca3d7367 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 347 b1dd4021a 08daf99d8 759a726d4 7efe0dc26 master 6973 a,b,t,h: 100000 1 10 329 e2b197539 08daf99d8 759a726d4 4ca3d7367 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 347 b1dd4021a 08daf99d8 759a726d4 7efe0dc26 master 6973 a,b,t,h: 100000 1 10 329 e2b197539 08daf99d8 759a726d4 4ca3d7367 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 347 b1dd4021a 08daf99d8 759a726d4 7efe0dc26 master 6973 a,b,t,h: 100000 1 10 329 e2b197539 08daf99d8 759a726d4 4ca3d7367 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 347 b1dd4021a 08daf99d8 759a726d4 7efe0dc26 master 6973 a,b,t,h: 100000 1 10 329 e2b197539 08daf99d8 759a726d4 4ca3d7367 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 347 b1dd4021a 08daf99d8 759a726d4 7efe0dc26 master 6973 a,b,t,h: 100000 1 10 329 e2b197539 08daf99d8 759a726d4 4ca3d7367 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 347 b1dd4021a 08daf99d8 759a726d4 7efe0dc26 master 6973 a,b,t,h: 100000 1 10 329 e2b197539 08daf99d8 759a726d4 4ca3d7367 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 347 b1dd4021a 08daf99d8 759a726d4 7efe0dc26 master 6973 a,b,t,h: 100000 1 10 329 e2b197539 08daf99d8 759a726d4 4ca3d7367 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 347 b1dd4021a 08daf99d8 759a726d4 7efe0dc26 master 6973 a,b,t,h: 100000 1 10 329 e2b197539 08daf99d8 759a726d4 4ca3d7367 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 347 b1dd4021a 08daf99d8 759a726d4 7efe0dc26 master 6973 a,b,t,h: 100000 1 10 329 e2b197539 08daf99d8 759a726d4 4ca3d7367 replica NOK -- Not good, but breaking out of wait (waited more than 60s) ========= iteration 10 -- 10 of 25 ========= -- scale 1 clients 16 INIT_WAIT 0 CLEAN_ONLY= ============================================================================ -- 20170222 17:28:20 -- scale 1 clients 16 INIT_WAIT 0 -- sub_count -ne 0 : deleting sub1 pub_count -ne 0 - deleting pub1 pub_count 0 pub_repl_slot_count 0 sub_count 0 sub_repl_slot_count 0 creating tables... 100000 of 100000 tuples (100%) done (elapsed 0.12 s, remaining 0.00 s) vacuum... set primary keys... done. create publication pub1 for all tables; create subscription sub1 connection 'port=6972' publication pub1 with (disabled); alter subscription sub1 enable; -- pgbench -c 16 -T 10 -P 5 -n progress: 5.0 s, 28.4 tps, lat 534.757 ms stddev 451.815 progress: 10.0 s, 33.2 tps, lat 469.060 ms stddev 350.581 transaction type: scaling factor: 1 query mode: simple number of clients: 16 number of threads: 1 duration: 10 s number of transactions actually processed: 324 latency average = 499.584 ms latency stddev = 393.966 ms tps = 31.184591 (including connections establishing) tps = 31.199653 (excluding connections establishing) -- waiting 4s... (scale * 4) 6972 a,b,t,h: 100000 1 10 324 68f3b0df5 09c843e77 55fbe43ad 0bc0b17aa master 6973 a,b,t,h: 100000 1 10 317 68f3b0df5 09c843e77 55fbe43ad 0672b7945 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 324 68f3b0df5 09c843e77 55fbe43ad 0bc0b17aa master 6973 a,b,t,h: 100000 1 10 317 68f3b0df5 09c843e77 55fbe43ad 0672b7945 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 324 68f3b0df5 09c843e77 55fbe43ad 0bc0b17aa master 6973 a,b,t,h: 100000 1 10 317 68f3b0df5 09c843e77 55fbe43ad 0672b7945 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 324 68f3b0df5 09c843e77 55fbe43ad 0bc0b17aa master 6973 a,b,t,h: 100000 1 10 317 68f3b0df5 09c843e77 55fbe43ad 0672b7945 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 324 68f3b0df5 09c843e77 55fbe43ad 0bc0b17aa master 6973 a,b,t,h: 100000 1 10 317 68f3b0df5 09c843e77 55fbe43ad 0672b7945 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 324 68f3b0df5 09c843e77 55fbe43ad 0bc0b17aa master 6973 a,b,t,h: 100000 1 10 317 68f3b0df5 09c843e77 55fbe43ad 0672b7945 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 324 68f3b0df5 09c843e77 55fbe43ad 0bc0b17aa master 6973 a,b,t,h: 100000 1 10 317 68f3b0df5 09c843e77 55fbe43ad 0672b7945 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 324 68f3b0df5 09c843e77 55fbe43ad 0bc0b17aa master 6973 a,b,t,h: 100000 1 10 317 68f3b0df5 09c843e77 55fbe43ad 0672b7945 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 324 68f3b0df5 09c843e77 55fbe43ad 0bc0b17aa master 6973 a,b,t,h: 100000 1 10 317 68f3b0df5 09c843e77 55fbe43ad 0672b7945 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 324 68f3b0df5 09c843e77 55fbe43ad 0bc0b17aa master 6973 a,b,t,h: 100000 1 10 317 68f3b0df5 09c843e77 55fbe43ad 0672b7945 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 324 68f3b0df5 09c843e77 55fbe43ad 0bc0b17aa master 6973 a,b,t,h: 100000 1 10 317 68f3b0df5 09c843e77 55fbe43ad 0672b7945 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 324 68f3b0df5 09c843e77 55fbe43ad 0bc0b17aa master 6973 a,b,t,h: 100000 1 10 317 68f3b0df5 09c843e77 55fbe43ad 0672b7945 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 324 68f3b0df5 09c843e77 55fbe43ad 0bc0b17aa master 6973 a,b,t,h: 100000 1 10 317 68f3b0df5 09c843e77 55fbe43ad 0672b7945 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 324 68f3b0df5 09c843e77 55fbe43ad 0bc0b17aa master 6973 a,b,t,h: 100000 1 10 317 68f3b0df5 09c843e77 55fbe43ad 0672b7945 replica NOK -- Not good, but breaking out of wait (waited more than 60s) ========= iteration 11 -- 11 of 25 ========= -- scale 1 clients 16 INIT_WAIT 0 CLEAN_ONLY= ============================================================================ -- 20170222 17:29:52 -- scale 1 clients 16 INIT_WAIT 0 -- sub_count -ne 0 : deleting sub1 pub_count -ne 0 - deleting pub1 pub_count 0 pub_repl_slot_count 0 sub_count 0 sub_repl_slot_count 0 creating tables... 100000 of 100000 tuples (100%) done (elapsed 0.12 s, remaining 0.00 s) vacuum... set primary keys... done. create publication pub1 for all tables; create subscription sub1 connection 'port=6972' publication pub1 with (disabled); alter subscription sub1 enable; -- pgbench -c 16 -T 10 -P 5 -n progress: 5.0 s, 38.4 tps, lat 346.568 ms stddev 376.305 progress: 10.0 s, 32.0 tps, lat 533.630 ms stddev 426.593 transaction type: scaling factor: 1 query mode: simple number of clients: 16 number of threads: 1 duration: 10 s number of transactions actually processed: 368 latency average = 444.310 ms latency stddev = 425.433 ms tps = 35.119104 (including connections establishing) tps = 35.134025 (excluding connections establishing) -- waiting 4s... (scale * 4) 6972 a,b,t,h: 100000 1 10 368 739951ab1 8b11c02b3 6f30f0b2b 22e604303 master 6973 a,b,t,h: 100000 1 10 358 739951ab1 8b11c02b3 6f30f0b2b 5077534d3 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 368 739951ab1 8b11c02b3 6f30f0b2b 22e604303 master 6973 a,b,t,h: 100000 1 10 358 739951ab1 8b11c02b3 6f30f0b2b 5077534d3 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 368 739951ab1 8b11c02b3 6f30f0b2b 22e604303 master 6973 a,b,t,h: 100000 1 10 358 739951ab1 8b11c02b3 6f30f0b2b 5077534d3 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 368 739951ab1 8b11c02b3 6f30f0b2b 22e604303 master 6973 a,b,t,h: 100000 1 10 358 739951ab1 8b11c02b3 6f30f0b2b 5077534d3 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 368 739951ab1 8b11c02b3 6f30f0b2b 22e604303 master 6973 a,b,t,h: 100000 1 10 358 739951ab1 8b11c02b3 6f30f0b2b 5077534d3 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 368 739951ab1 8b11c02b3 6f30f0b2b 22e604303 master 6973 a,b,t,h: 100000 1 10 358 739951ab1 8b11c02b3 6f30f0b2b 5077534d3 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 368 739951ab1 8b11c02b3 6f30f0b2b 22e604303 master 6973 a,b,t,h: 100000 1 10 358 739951ab1 8b11c02b3 6f30f0b2b 5077534d3 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 368 739951ab1 8b11c02b3 6f30f0b2b 22e604303 master 6973 a,b,t,h: 100000 1 10 358 739951ab1 8b11c02b3 6f30f0b2b 5077534d3 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 368 739951ab1 8b11c02b3 6f30f0b2b 22e604303 master 6973 a,b,t,h: 100000 1 10 358 739951ab1 8b11c02b3 6f30f0b2b 5077534d3 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 368 739951ab1 8b11c02b3 6f30f0b2b 22e604303 master 6973 a,b,t,h: 100000 1 10 358 739951ab1 8b11c02b3 6f30f0b2b 5077534d3 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 368 739951ab1 8b11c02b3 6f30f0b2b 22e604303 master 6973 a,b,t,h: 100000 1 10 358 739951ab1 8b11c02b3 6f30f0b2b 5077534d3 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 368 739951ab1 8b11c02b3 6f30f0b2b 22e604303 master 6973 a,b,t,h: 100000 1 10 358 739951ab1 8b11c02b3 6f30f0b2b 5077534d3 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 368 739951ab1 8b11c02b3 6f30f0b2b 22e604303 master 6973 a,b,t,h: 100000 1 10 358 739951ab1 8b11c02b3 6f30f0b2b 5077534d3 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 368 739951ab1 8b11c02b3 6f30f0b2b 22e604303 master 6973 a,b,t,h: 100000 1 10 358 739951ab1 8b11c02b3 6f30f0b2b 5077534d3 replica NOK -- Not good, but breaking out of wait (waited more than 60s) ========= iteration 12 -- 12 of 25 ========= -- scale 1 clients 16 INIT_WAIT 0 CLEAN_ONLY= ============================================================================ -- 20170222 17:31:25 -- scale 1 clients 16 INIT_WAIT 0 -- sub_count -ne 0 : deleting sub1 pub_count -ne 0 - deleting pub1 pub_count 0 pub_repl_slot_count 0 sub_count 0 sub_repl_slot_count 0 creating tables... 100000 of 100000 tuples (100%) done (elapsed 0.12 s, remaining 0.00 s) vacuum... set primary keys... done. create publication pub1 for all tables; create subscription sub1 connection 'port=6972' publication pub1 with (disabled); alter subscription sub1 enable; -- pgbench -c 16 -T 10 -P 5 -n progress: 5.0 s, 34.0 tps, lat 432.697 ms stddev 405.636 progress: 10.0 s, 35.0 tps, lat 450.697 ms stddev 469.087 transaction type: scaling factor: 1 query mode: simple number of clients: 16 number of threads: 1 duration: 10 s number of transactions actually processed: 361 latency average = 449.872 ms latency stddev = 437.764 ms tps = 34.630405 (including connections establishing) tps = 34.646743 (excluding connections establishing) -- waiting 4s... (scale * 4) 6972 a,b,t,h: 100000 1 10 361 bac3435df 268ef2d7e db95d858e e852652b0 master 6973 a,b,t,h: 100000 1 10 351 bac3435df 268ef2d7e db95d858e 26130f3e9 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 361 bac3435df 268ef2d7e db95d858e e852652b0 master 6973 a,b,t,h: 100000 1 10 351 bac3435df 268ef2d7e db95d858e 26130f3e9 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 361 bac3435df 268ef2d7e db95d858e e852652b0 master 6973 a,b,t,h: 100000 1 10 351 bac3435df 268ef2d7e db95d858e 26130f3e9 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 361 bac3435df 268ef2d7e db95d858e e852652b0 master 6973 a,b,t,h: 100000 1 10 351 bac3435df 268ef2d7e db95d858e 26130f3e9 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 361 bac3435df 268ef2d7e db95d858e e852652b0 master 6973 a,b,t,h: 100000 1 10 351 bac3435df 268ef2d7e db95d858e 26130f3e9 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 361 bac3435df 268ef2d7e db95d858e e852652b0 master 6973 a,b,t,h: 100000 1 10 351 bac3435df 268ef2d7e db95d858e 26130f3e9 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 361 bac3435df 268ef2d7e db95d858e e852652b0 master 6973 a,b,t,h: 100000 1 10 351 bac3435df 268ef2d7e db95d858e 26130f3e9 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 361 bac3435df 268ef2d7e db95d858e e852652b0 master 6973 a,b,t,h: 100000 1 10 351 bac3435df 268ef2d7e db95d858e 26130f3e9 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 361 bac3435df 268ef2d7e db95d858e e852652b0 master 6973 a,b,t,h: 100000 1 10 351 bac3435df 268ef2d7e db95d858e 26130f3e9 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 361 bac3435df 268ef2d7e db95d858e e852652b0 master 6973 a,b,t,h: 100000 1 10 351 bac3435df 268ef2d7e db95d858e 26130f3e9 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 361 bac3435df 268ef2d7e db95d858e e852652b0 master 6973 a,b,t,h: 100000 1 10 351 bac3435df 268ef2d7e db95d858e 26130f3e9 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 361 bac3435df 268ef2d7e db95d858e e852652b0 master 6973 a,b,t,h: 100000 1 10 351 bac3435df 268ef2d7e db95d858e 26130f3e9 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 361 bac3435df 268ef2d7e db95d858e e852652b0 master 6973 a,b,t,h: 100000 1 10 351 bac3435df 268ef2d7e db95d858e 26130f3e9 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 361 bac3435df 268ef2d7e db95d858e e852652b0 master 6973 a,b,t,h: 100000 1 10 351 bac3435df 268ef2d7e db95d858e 26130f3e9 replica NOK -- Not good, but breaking out of wait (waited more than 60s) ========= iteration 13 -- 13 of 25 ========= -- scale 1 clients 16 INIT_WAIT 0 CLEAN_ONLY= ============================================================================ -- 20170222 17:32:58 -- scale 1 clients 16 INIT_WAIT 0 -- sub_count -ne 0 : deleting sub1 pub_count -ne 0 - deleting pub1 pub_count 0 pub_repl_slot_count 0 sub_count 0 sub_repl_slot_count 0 creating tables... 100000 of 100000 tuples (100%) done (elapsed 0.11 s, remaining 0.00 s) vacuum... set primary keys... done. create publication pub1 for all tables; create subscription sub1 connection 'port=6972' publication pub1 with (disabled); alter subscription sub1 enable; -- pgbench -c 16 -T 10 -P 5 -n progress: 5.0 s, 58.6 tps, lat 260.276 ms stddev 235.189 progress: 10.0 s, 75.4 tps, lat 211.703 ms stddev 141.075 transaction type: scaling factor: 1 query mode: simple number of clients: 16 number of threads: 1 duration: 10 s number of transactions actually processed: 686 latency average = 239.055 ms latency stddev = 192.620 ms tps = 65.847250 (including connections establishing) tps = 65.877875 (excluding connections establishing) -- waiting 4s... (scale * 4) 6972 a,b,t,h: 100000 1 10 686 47d037cc2 7145999ce 19783fb88 f4a6c607f master 6973 a,b,t,h: 100000 1 0 0 0ead58d44 7145999ce d41d8cd98 d41d8cd98 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 686 47d037cc2 7145999ce 19783fb88 f4a6c607f master 6973 a,b,t,h: 100000 1 10 686 26f11b98f 7145999ce 19783fb88 f4a6c607f replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 686 47d037cc2 7145999ce 19783fb88 f4a6c607f master 6973 a,b,t,h: 100000 1 10 686 aba60216e 7145999ce 19783fb88 f4a6c607f replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 686 47d037cc2 7145999ce 19783fb88 f4a6c607f master 6973 a,b,t,h: 100000 1 10 686 47d037cc2 7145999ce 19783fb88 f4a6c607f replica ok -- All is well. ========= iteration 14 -- 14 of 25 ========= -- scale 1 clients 16 INIT_WAIT 0 CLEAN_ONLY= ============================================================================ -- 20170222 17:33:34 -- scale 1 clients 16 INIT_WAIT 0 -- sub_count -ne 0 : deleting sub1 pub_count -ne 0 - deleting pub1 pub_count 0 pub_repl_slot_count 0 sub_count 0 sub_repl_slot_count 0 creating tables... 100000 of 100000 tuples (100%) done (elapsed 0.12 s, remaining 0.00 s) vacuum... set primary keys... done. create publication pub1 for all tables; create subscription sub1 connection 'port=6972' publication pub1 with (disabled); alter subscription sub1 enable; -- pgbench -c 16 -T 10 -P 5 -n progress: 5.0 s, 32.8 tps, lat 415.462 ms stddev 354.770 progress: 10.0 s, 27.4 tps, lat 597.199 ms stddev 596.712 transaction type: scaling factor: 1 query mode: simple number of clients: 16 number of threads: 1 duration: 10 s number of transactions actually processed: 317 latency average = 521.671 ms latency stddev = 498.334 ms tps = 30.035785 (including connections establishing) tps = 30.049653 (excluding connections establishing) -- waiting 4s... (scale * 4) 6972 a,b,t,h: 100000 1 10 317 c35d6281d f8287a14c 0f8710532 b0e0d34f0 master 6973 a,b,t,h: 100000 1 10 293 c35d6281d f8287a14c 0f8710532 2b2fd4924 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 317 c35d6281d f8287a14c 0f8710532 b0e0d34f0 master 6973 a,b,t,h: 100000 1 10 293 c35d6281d f8287a14c 0f8710532 2b2fd4924 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 317 c35d6281d f8287a14c 0f8710532 b0e0d34f0 master 6973 a,b,t,h: 100000 1 10 293 c35d6281d f8287a14c 0f8710532 2b2fd4924 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 317 c35d6281d f8287a14c 0f8710532 b0e0d34f0 master 6973 a,b,t,h: 100000 1 10 293 c35d6281d f8287a14c 0f8710532 2b2fd4924 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 317 c35d6281d f8287a14c 0f8710532 b0e0d34f0 master 6973 a,b,t,h: 100000 1 10 293 c35d6281d f8287a14c 0f8710532 2b2fd4924 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 317 c35d6281d f8287a14c 0f8710532 b0e0d34f0 master 6973 a,b,t,h: 100000 1 10 293 c35d6281d f8287a14c 0f8710532 2b2fd4924 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 317 c35d6281d f8287a14c 0f8710532 b0e0d34f0 master 6973 a,b,t,h: 100000 1 10 293 c35d6281d f8287a14c 0f8710532 2b2fd4924 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 317 c35d6281d f8287a14c 0f8710532 b0e0d34f0 master 6973 a,b,t,h: 100000 1 10 293 c35d6281d f8287a14c 0f8710532 2b2fd4924 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 317 c35d6281d f8287a14c 0f8710532 b0e0d34f0 master 6973 a,b,t,h: 100000 1 10 293 c35d6281d f8287a14c 0f8710532 2b2fd4924 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 317 c35d6281d f8287a14c 0f8710532 b0e0d34f0 master 6973 a,b,t,h: 100000 1 10 293 c35d6281d f8287a14c 0f8710532 2b2fd4924 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 317 c35d6281d f8287a14c 0f8710532 b0e0d34f0 master 6973 a,b,t,h: 100000 1 10 293 c35d6281d f8287a14c 0f8710532 2b2fd4924 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 317 c35d6281d f8287a14c 0f8710532 b0e0d34f0 master 6973 a,b,t,h: 100000 1 10 293 c35d6281d f8287a14c 0f8710532 2b2fd4924 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 317 c35d6281d f8287a14c 0f8710532 b0e0d34f0 master 6973 a,b,t,h: 100000 1 10 293 c35d6281d f8287a14c 0f8710532 2b2fd4924 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 317 c35d6281d f8287a14c 0f8710532 b0e0d34f0 master 6973 a,b,t,h: 100000 1 10 293 c35d6281d f8287a14c 0f8710532 2b2fd4924 replica NOK -- Not good, but breaking out of wait (waited more than 60s) ========= iteration 15 -- 15 of 25 ========= -- scale 1 clients 16 INIT_WAIT 0 CLEAN_ONLY= ============================================================================ -- 20170222 17:35:07 -- scale 1 clients 16 INIT_WAIT 0 -- sub_count -ne 0 : deleting sub1 pub_count -ne 0 - deleting pub1 pub_count 0 pub_repl_slot_count 0 sub_count 0 sub_repl_slot_count 0 creating tables... 100000 of 100000 tuples (100%) done (elapsed 0.12 s, remaining 0.00 s) vacuum... set primary keys... done. create publication pub1 for all tables; create subscription sub1 connection 'port=6972' publication pub1 with (disabled); alter subscription sub1 enable; -- pgbench -c 16 -T 10 -P 5 -n progress: 5.0 s, 32.6 tps, lat 439.742 ms stddev 341.768 progress: 10.0 s, 34.0 tps, lat 479.367 ms stddev 359.267 transaction type: scaling factor: 1 query mode: simple number of clients: 16 number of threads: 1 duration: 10 s number of transactions actually processed: 349 latency average = 469.711 ms latency stddev = 354.811 ms tps = 32.693640 (including connections establishing) tps = 32.707337 (excluding connections establishing) -- waiting 4s... (scale * 4) 6972 a,b,t,h: 100000 1 10 349 c046ea5ee d61a48265 dbaf3beae 8d66f89c5 master 6973 a,b,t,h: 100000 1 10 349 e8847073f d61a48265 dbaf3beae 8d66f89c5 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 349 c046ea5ee d61a48265 dbaf3beae 8d66f89c5 master 6973 a,b,t,h: 100000 1 10 349 e8847073f d61a48265 dbaf3beae 8d66f89c5 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 349 c046ea5ee d61a48265 dbaf3beae 8d66f89c5 master 6973 a,b,t,h: 100000 1 10 349 e8847073f d61a48265 dbaf3beae 8d66f89c5 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 349 c046ea5ee d61a48265 dbaf3beae 8d66f89c5 master 6973 a,b,t,h: 100000 1 10 349 e8847073f d61a48265 dbaf3beae 8d66f89c5 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 349 c046ea5ee d61a48265 dbaf3beae 8d66f89c5 master 6973 a,b,t,h: 100000 1 10 349 e8847073f d61a48265 dbaf3beae 8d66f89c5 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 349 c046ea5ee d61a48265 dbaf3beae 8d66f89c5 master 6973 a,b,t,h: 100000 1 10 349 e8847073f d61a48265 dbaf3beae 8d66f89c5 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 349 c046ea5ee d61a48265 dbaf3beae 8d66f89c5 master 6973 a,b,t,h: 100000 1 10 349 e8847073f d61a48265 dbaf3beae 8d66f89c5 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 349 c046ea5ee d61a48265 dbaf3beae 8d66f89c5 master 6973 a,b,t,h: 100000 1 10 349 e8847073f d61a48265 dbaf3beae 8d66f89c5 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 349 c046ea5ee d61a48265 dbaf3beae 8d66f89c5 master 6973 a,b,t,h: 100000 1 10 349 e8847073f d61a48265 dbaf3beae 8d66f89c5 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 349 c046ea5ee d61a48265 dbaf3beae 8d66f89c5 master 6973 a,b,t,h: 100000 1 10 349 e8847073f d61a48265 dbaf3beae 8d66f89c5 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 349 c046ea5ee d61a48265 dbaf3beae 8d66f89c5 master 6973 a,b,t,h: 100000 1 10 349 e8847073f d61a48265 dbaf3beae 8d66f89c5 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 349 c046ea5ee d61a48265 dbaf3beae 8d66f89c5 master 6973 a,b,t,h: 100000 1 10 349 e8847073f d61a48265 dbaf3beae 8d66f89c5 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 349 c046ea5ee d61a48265 dbaf3beae 8d66f89c5 master 6973 a,b,t,h: 100000 1 10 349 e8847073f d61a48265 dbaf3beae 8d66f89c5 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 349 c046ea5ee d61a48265 dbaf3beae 8d66f89c5 master 6973 a,b,t,h: 100000 1 10 349 e8847073f d61a48265 dbaf3beae 8d66f89c5 replica NOK -- Not good, but breaking out of wait (waited more than 60s) ========= iteration 16 -- 16 of 25 ========= -- scale 1 clients 16 INIT_WAIT 0 CLEAN_ONLY= ============================================================================ -- 20170222 17:36:40 -- scale 1 clients 16 INIT_WAIT 0 -- sub_count -ne 0 : deleting sub1 pub_count -ne 0 - deleting pub1 pub_count 0 pub_repl_slot_count 0 sub_count 0 sub_repl_slot_count 0 creating tables... 100000 of 100000 tuples (100%) done (elapsed 0.12 s, remaining 0.00 s) vacuum... set primary keys... done. create publication pub1 for all tables; create subscription sub1 connection 'port=6972' publication pub1 with (disabled); alter subscription sub1 enable; -- pgbench -c 16 -T 10 -P 5 -n progress: 5.0 s, 44.4 tps, lat 344.928 ms stddev 296.375 progress: 10.0 s, 74.4 tps, lat 215.582 ms stddev 134.046 transaction type: scaling factor: 1 query mode: simple number of clients: 16 number of threads: 1 duration: 10 s number of transactions actually processed: 610 latency average = 263.041 ms latency stddev = 216.654 ms tps = 59.724208 (including connections establishing) tps = 59.753464 (excluding connections establishing) -- waiting 4s... (scale * 4) 6972 a,b,t,h: 100000 1 10 610 c94f62918 a00e9ca8c df6f34268 820fd9907 master 6973 a,b,t,h: 100000 1 10 148 d60800217 dd8549027 df6f34268 06cf28fdd replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 610 c94f62918 a00e9ca8c df6f34268 820fd9907 master 6973 a,b,t,h: 100000 1 10 408 1671c182f 05b4077d8 df6f34268 d350a9c47 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 610 c94f62918 a00e9ca8c df6f34268 820fd9907 master 6973 a,b,t,h: 100000 1 10 610 c94f62918 a00e9ca8c df6f34268 820fd9907 replica ok -- All is well. ========= iteration 17 -- 17 of 25 ========= -- scale 1 clients 16 INIT_WAIT 0 CLEAN_ONLY= ============================================================================ -- 20170222 17:37:11 -- scale 1 clients 16 INIT_WAIT 0 -- sub_count -ne 0 : deleting sub1 pub_count -ne 0 - deleting pub1 pub_count 0 pub_repl_slot_count 0 sub_count 0 sub_repl_slot_count 0 creating tables... 100000 of 100000 tuples (100%) done (elapsed 0.12 s, remaining 0.00 s) vacuum... set primary keys... done. create publication pub1 for all tables; create subscription sub1 connection 'port=6972' publication pub1 with (disabled); alter subscription sub1 enable; -- pgbench -c 16 -T 10 -P 5 -n progress: 5.0 s, 30.2 tps, lat 480.175 ms stddev 440.168 progress: 10.0 s, 35.2 tps, lat 463.473 ms stddev 333.200 transaction type: scaling factor: 1 query mode: simple number of clients: 16 number of threads: 1 duration: 10 s number of transactions actually processed: 343 latency average = 474.217 ms latency stddev = 382.618 ms tps = 32.832983 (including connections establishing) tps = 32.848668 (excluding connections establishing) -- waiting 4s... (scale * 4) 6972 a,b,t,h: 100000 1 10 343 cf2eb3f88 8b08a8812 a28ebd27f e84e54091 master 6973 a,b,t,h: 100000 1 10 317 cf2eb3f88 8b08a8812 a28ebd27f fd858a890 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 343 cf2eb3f88 8b08a8812 a28ebd27f e84e54091 master 6973 a,b,t,h: 100000 1 10 317 cf2eb3f88 8b08a8812 a28ebd27f fd858a890 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 343 cf2eb3f88 8b08a8812 a28ebd27f e84e54091 master 6973 a,b,t,h: 100000 1 10 317 cf2eb3f88 8b08a8812 a28ebd27f fd858a890 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 343 cf2eb3f88 8b08a8812 a28ebd27f e84e54091 master 6973 a,b,t,h: 100000 1 10 317 cf2eb3f88 8b08a8812 a28ebd27f fd858a890 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 343 cf2eb3f88 8b08a8812 a28ebd27f e84e54091 master 6973 a,b,t,h: 100000 1 10 317 cf2eb3f88 8b08a8812 a28ebd27f fd858a890 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 343 cf2eb3f88 8b08a8812 a28ebd27f e84e54091 master 6973 a,b,t,h: 100000 1 10 317 cf2eb3f88 8b08a8812 a28ebd27f fd858a890 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 343 cf2eb3f88 8b08a8812 a28ebd27f e84e54091 master 6973 a,b,t,h: 100000 1 10 317 cf2eb3f88 8b08a8812 a28ebd27f fd858a890 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 343 cf2eb3f88 8b08a8812 a28ebd27f e84e54091 master 6973 a,b,t,h: 100000 1 10 317 cf2eb3f88 8b08a8812 a28ebd27f fd858a890 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 343 cf2eb3f88 8b08a8812 a28ebd27f e84e54091 master 6973 a,b,t,h: 100000 1 10 317 cf2eb3f88 8b08a8812 a28ebd27f fd858a890 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 343 cf2eb3f88 8b08a8812 a28ebd27f e84e54091 master 6973 a,b,t,h: 100000 1 10 317 cf2eb3f88 8b08a8812 a28ebd27f fd858a890 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 343 cf2eb3f88 8b08a8812 a28ebd27f e84e54091 master 6973 a,b,t,h: 100000 1 10 317 cf2eb3f88 8b08a8812 a28ebd27f fd858a890 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 343 cf2eb3f88 8b08a8812 a28ebd27f e84e54091 master 6973 a,b,t,h: 100000 1 10 317 cf2eb3f88 8b08a8812 a28ebd27f fd858a890 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 343 cf2eb3f88 8b08a8812 a28ebd27f e84e54091 master 6973 a,b,t,h: 100000 1 10 317 cf2eb3f88 8b08a8812 a28ebd27f fd858a890 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 343 cf2eb3f88 8b08a8812 a28ebd27f e84e54091 master 6973 a,b,t,h: 100000 1 10 317 cf2eb3f88 8b08a8812 a28ebd27f fd858a890 replica NOK -- Not good, but breaking out of wait (waited more than 60s) ========= iteration 18 -- 18 of 25 ========= -- scale 1 clients 16 INIT_WAIT 0 CLEAN_ONLY= ============================================================================ -- 20170222 17:38:44 -- scale 1 clients 16 INIT_WAIT 0 -- sub_count -ne 0 : deleting sub1 pub_count -ne 0 - deleting pub1 pub_count 0 pub_repl_slot_count 0 sub_count 0 sub_repl_slot_count 0 creating tables... 100000 of 100000 tuples (100%) done (elapsed 0.11 s, remaining 0.00 s) vacuum... set primary keys... done. create publication pub1 for all tables; create subscription sub1 connection 'port=6972' publication pub1 with (disabled); alter subscription sub1 enable; -- pgbench -c 16 -T 10 -P 5 -n progress: 5.0 s, 25.0 tps, lat 531.257 ms stddev 476.615 progress: 10.0 s, 63.8 tps, lat 275.205 ms stddev 334.684 transaction type: scaling factor: 1 query mode: simple number of clients: 16 number of threads: 1 duration: 10 s number of transactions actually processed: 460 latency average = 355.093 ms latency stddev = 396.052 ms tps = 44.219310 (including connections establishing) tps = 44.240581 (excluding connections establishing) -- waiting 4s... (scale * 4) 6972 a,b,t,h: 100000 1 10 460 beff98a46 b789bbbc4 536dde875 e86cebe6a master 6973 a,b,t,h: 100000 1 10 164 8b34c765a 2b27fbe9f 536dde875 ce24eb658 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 460 beff98a46 b789bbbc4 536dde875 e86cebe6a master 6973 a,b,t,h: 100000 1 10 440 f25135cad 559f8622f 536dde875 e9709252d replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 460 beff98a46 b789bbbc4 536dde875 e86cebe6a master 6973 a,b,t,h: 100000 1 10 457 beff98a46 b789bbbc4 536dde875 f22ec4b09 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 460 beff98a46 b789bbbc4 536dde875 e86cebe6a master 6973 a,b,t,h: 100000 1 10 457 beff98a46 b789bbbc4 536dde875 f22ec4b09 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 460 beff98a46 b789bbbc4 536dde875 e86cebe6a master 6973 a,b,t,h: 100000 1 10 457 beff98a46 b789bbbc4 536dde875 f22ec4b09 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 460 beff98a46 b789bbbc4 536dde875 e86cebe6a master 6973 a,b,t,h: 100000 1 10 457 beff98a46 b789bbbc4 536dde875 f22ec4b09 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 460 beff98a46 b789bbbc4 536dde875 e86cebe6a master 6973 a,b,t,h: 100000 1 10 457 beff98a46 b789bbbc4 536dde875 f22ec4b09 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 460 beff98a46 b789bbbc4 536dde875 e86cebe6a master 6973 a,b,t,h: 100000 1 10 457 beff98a46 b789bbbc4 536dde875 f22ec4b09 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 460 beff98a46 b789bbbc4 536dde875 e86cebe6a master 6973 a,b,t,h: 100000 1 10 457 beff98a46 b789bbbc4 536dde875 f22ec4b09 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 460 beff98a46 b789bbbc4 536dde875 e86cebe6a master 6973 a,b,t,h: 100000 1 10 457 beff98a46 b789bbbc4 536dde875 f22ec4b09 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 460 beff98a46 b789bbbc4 536dde875 e86cebe6a master 6973 a,b,t,h: 100000 1 10 457 beff98a46 b789bbbc4 536dde875 f22ec4b09 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 460 beff98a46 b789bbbc4 536dde875 e86cebe6a master 6973 a,b,t,h: 100000 1 10 457 beff98a46 b789bbbc4 536dde875 f22ec4b09 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 460 beff98a46 b789bbbc4 536dde875 e86cebe6a master 6973 a,b,t,h: 100000 1 10 457 beff98a46 b789bbbc4 536dde875 f22ec4b09 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 460 beff98a46 b789bbbc4 536dde875 e86cebe6a master 6973 a,b,t,h: 100000 1 10 457 beff98a46 b789bbbc4 536dde875 f22ec4b09 replica NOK -- Not good, but breaking out of wait (waited more than 60s) ========= iteration 19 -- 19 of 25 ========= -- scale 1 clients 16 INIT_WAIT 0 CLEAN_ONLY= ============================================================================ -- 20170222 17:40:17 -- scale 1 clients 16 INIT_WAIT 0 -- sub_count -ne 0 : deleting sub1 pub_count -ne 0 - deleting pub1 pub_count 0 pub_repl_slot_count 0 sub_count 0 sub_repl_slot_count 0 creating tables... 100000 of 100000 tuples (100%) done (elapsed 0.11 s, remaining 0.00 s) vacuum... set primary keys... done. create publication pub1 for all tables; create subscription sub1 connection 'port=6972' publication pub1 with (disabled); alter subscription sub1 enable; -- pgbench -c 16 -T 10 -P 5 -n progress: 5.0 s, 41.4 tps, lat 363.870 ms stddev 322.182 progress: 10.0 s, 75.2 tps, lat 213.732 ms stddev 170.732 transaction type: scaling factor: 1 query mode: simple number of clients: 16 number of threads: 1 duration: 10 s number of transactions actually processed: 599 latency average = 267.543 ms latency stddev = 245.151 ms tps = 58.778363 (including connections establishing) tps = 58.807287 (excluding connections establishing) -- waiting 4s... (scale * 4) 6972 a,b,t,h: 100000 1 10 599 7d0533b22 47a30e1a8 cf1b1df77 7ae1e7b8e master 6973 a,b,t,h: 100000 1 10 599 4e90acb7b 4ace26871 cbd90dd9f 7ae1e7b8e replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 599 7d0533b22 47a30e1a8 cf1b1df77 7ae1e7b8e master 6973 a,b,t,h: 100000 1 10 599 43e5076aa b0a50fe02 aee08327a 7ae1e7b8e replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 599 7d0533b22 47a30e1a8 cf1b1df77 7ae1e7b8e master 6973 a,b,t,h: 100000 1 10 599 7d0533b22 47a30e1a8 cf1b1df77 7ae1e7b8e replica ok -- All is well. ========= iteration 20 -- 20 of 25 ========= -- scale 1 clients 16 INIT_WAIT 0 CLEAN_ONLY= ============================================================================ -- 20170222 17:40:47 -- scale 1 clients 16 INIT_WAIT 0 -- sub_count -ne 0 : deleting sub1 pub_count -ne 0 - deleting pub1 pub_count 0 pub_repl_slot_count 0 sub_count 0 sub_repl_slot_count 0 creating tables... 100000 of 100000 tuples (100%) done (elapsed 0.12 s, remaining 0.00 s) vacuum... set primary keys... done. create publication pub1 for all tables; create subscription sub1 connection 'port=6972' publication pub1 with (disabled); alter subscription sub1 enable; -- pgbench -c 16 -T 10 -P 5 -n progress: 5.0 s, 27.6 tps, lat 548.269 ms stddev 493.923 progress: 10.0 s, 65.8 tps, lat 243.702 ms stddev 240.710 transaction type: scaling factor: 1 query mode: simple number of clients: 16 number of threads: 1 duration: 10 s number of transactions actually processed: 483 latency average = 331.769 ms latency stddev = 359.136 ms tps = 47.410456 (including connections establishing) tps = 47.433984 (excluding connections establishing) -- waiting 4s... (scale * 4) 6972 a,b,t,h: 100000 1 10 483 fcab04a03 3ff238050 99332275c 916d136ec master 6973 a,b,t,h: 100000 0 0 138 1eed140af d41d8cd98 d41d8cd98 14d0f6b01 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 483 fcab04a03 3ff238050 99332275c 916d136ec master 6973 a,b,t,h: 100000 0 0 138 1eed140af d41d8cd98 d41d8cd98 14d0f6b01 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 483 fcab04a03 3ff238050 99332275c 916d136ec master 6973 a,b,t,h: 100000 1 10 138 1eed140af 3ff238050 99332275c 14d0f6b01 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 483 fcab04a03 3ff238050 99332275c 916d136ec master 6973 a,b,t,h: 100000 1 10 370 d5f9738d2 3ff238050 99332275c 8688784fa replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 483 fcab04a03 3ff238050 99332275c 916d136ec master 6973 a,b,t,h: 100000 1 10 483 fcab04a03 3ff238050 99332275c 916d136ec replica ok -- All is well. ========= iteration 21 -- 21 of 25 ========= -- scale 1 clients 16 INIT_WAIT 0 CLEAN_ONLY= ============================================================================ -- 20170222 17:41:29 -- scale 1 clients 16 INIT_WAIT 0 -- sub_count -ne 0 : deleting sub1 pub_count -ne 0 - deleting pub1 pub_count 0 pub_repl_slot_count 0 sub_count 0 sub_repl_slot_count 0 creating tables... 100000 of 100000 tuples (100%) done (elapsed 0.11 s, remaining 0.00 s) vacuum... set primary keys... done. create publication pub1 for all tables; create subscription sub1 connection 'port=6972' publication pub1 with (disabled); alter subscription sub1 enable; -- pgbench -c 16 -T 10 -P 5 -n progress: 5.0 s, 53.6 tps, lat 281.054 ms stddev 271.811 progress: 10.0 s, 76.0 tps, lat 212.278 ms stddev 157.546 transaction type: scaling factor: 1 query mode: simple number of clients: 16 number of threads: 1 duration: 10 s number of transactions actually processed: 664 latency average = 241.413 ms latency stddev = 213.544 ms tps = 65.075698 (including connections establishing) tps = 65.108027 (excluding connections establishing) -- waiting 4s... (scale * 4) 6972 a,b,t,h: 100000 1 10 664 f7e1c6ebf 7804aabe5 21144ca5a 3a4560f21 master 6973 a,b,t,h: 100000 1 0 0 8825f9deb 7804aabe5 d41d8cd98 d41d8cd98 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 664 f7e1c6ebf 7804aabe5 21144ca5a 3a4560f21 master 6973 a,b,t,h: 100000 1 10 664 c1344efef 7804aabe5 21144ca5a 3a4560f21 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 664 f7e1c6ebf 7804aabe5 21144ca5a 3a4560f21 master 6973 a,b,t,h: 100000 1 10 664 f7e1c6ebf 7804aabe5 21144ca5a 3a4560f21 replica ok -- All is well. ========= iteration 22 -- 22 of 25 ========= -- scale 1 clients 16 INIT_WAIT 0 CLEAN_ONLY= ============================================================================ -- 20170222 17:41:59 -- scale 1 clients 16 INIT_WAIT 0 -- sub_count -ne 0 : deleting sub1 pub_count -ne 0 - deleting pub1 pub_count 0 pub_repl_slot_count 0 sub_count 0 sub_repl_slot_count 0 creating tables... 100000 of 100000 tuples (100%) done (elapsed 0.12 s, remaining 0.00 s) vacuum... set primary keys... done. create publication pub1 for all tables; create subscription sub1 connection 'port=6972' publication pub1 with (disabled); alter subscription sub1 enable; -- pgbench -c 16 -T 10 -P 5 -n progress: 5.0 s, 33.2 tps, lat 423.262 ms stddev 352.017 progress: 10.0 s, 35.0 tps, lat 482.311 ms stddev 347.622 transaction type: scaling factor: 1 query mode: simple number of clients: 16 number of threads: 1 duration: 10 s number of transactions actually processed: 357 latency average = 456.237 ms latency stddev = 346.802 ms tps = 34.151763 (including connections establishing) tps = 34.167922 (excluding connections establishing) -- waiting 4s... (scale * 4) 6972 a,b,t,h: 100000 1 10 357 60a43644d d549ac6d1 df72edf6e 27f1df366 master 6973 a,b,t,h: 100000 1 10 354 60a43644d d549ac6d1 df72edf6e 252626539 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 357 60a43644d d549ac6d1 df72edf6e 27f1df366 master 6973 a,b,t,h: 100000 1 10 354 60a43644d d549ac6d1 df72edf6e 252626539 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 357 60a43644d d549ac6d1 df72edf6e 27f1df366 master 6973 a,b,t,h: 100000 1 10 354 60a43644d d549ac6d1 df72edf6e 252626539 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 357 60a43644d d549ac6d1 df72edf6e 27f1df366 master 6973 a,b,t,h: 100000 1 10 354 60a43644d d549ac6d1 df72edf6e 252626539 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 357 60a43644d d549ac6d1 df72edf6e 27f1df366 master 6973 a,b,t,h: 100000 1 10 354 60a43644d d549ac6d1 df72edf6e 252626539 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 357 60a43644d d549ac6d1 df72edf6e 27f1df366 master 6973 a,b,t,h: 100000 1 10 354 60a43644d d549ac6d1 df72edf6e 252626539 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 357 60a43644d d549ac6d1 df72edf6e 27f1df366 master 6973 a,b,t,h: 100000 1 10 354 60a43644d d549ac6d1 df72edf6e 252626539 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 357 60a43644d d549ac6d1 df72edf6e 27f1df366 master 6973 a,b,t,h: 100000 1 10 354 60a43644d d549ac6d1 df72edf6e 252626539 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 357 60a43644d d549ac6d1 df72edf6e 27f1df366 master 6973 a,b,t,h: 100000 1 10 354 60a43644d d549ac6d1 df72edf6e 252626539 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 357 60a43644d d549ac6d1 df72edf6e 27f1df366 master 6973 a,b,t,h: 100000 1 10 354 60a43644d d549ac6d1 df72edf6e 252626539 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 357 60a43644d d549ac6d1 df72edf6e 27f1df366 master 6973 a,b,t,h: 100000 1 10 354 60a43644d d549ac6d1 df72edf6e 252626539 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 357 60a43644d d549ac6d1 df72edf6e 27f1df366 master 6973 a,b,t,h: 100000 1 10 354 60a43644d d549ac6d1 df72edf6e 252626539 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 357 60a43644d d549ac6d1 df72edf6e 27f1df366 master 6973 a,b,t,h: 100000 1 10 354 60a43644d d549ac6d1 df72edf6e 252626539 replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 357 60a43644d d549ac6d1 df72edf6e 27f1df366 master 6973 a,b,t,h: 100000 1 10 354 60a43644d d549ac6d1 df72edf6e 252626539 replica NOK -- Not good, but breaking out of wait (waited more than 60s) ========= iteration 23 -- 23 of 25 ========= -- scale 1 clients 16 INIT_WAIT 0 CLEAN_ONLY= ============================================================================ -- 20170222 17:43:32 -- scale 1 clients 16 INIT_WAIT 0 -- sub_count -ne 0 : deleting sub1 pub_count -ne 0 - deleting pub1 pub_count 0 pub_repl_slot_count 0 sub_count 0 sub_repl_slot_count 0 creating tables... 100000 of 100000 tuples (100%) done (elapsed 0.12 s, remaining 0.00 s) vacuum... set primary keys... done. create publication pub1 for all tables; create subscription sub1 connection 'port=6972' publication pub1 with (disabled); alter subscription sub1 enable; -- pgbench -c 16 -T 10 -P 5 -n progress: 5.0 s, 36.0 tps, lat 411.423 ms stddev 386.615 progress: 10.0 s, 29.8 tps, lat 532.956 ms stddev 410.552 transaction type: scaling factor: 1 query mode: simple number of clients: 16 number of threads: 1 duration: 10 s number of transactions actually processed: 345 latency average = 470.016 ms latency stddev = 397.295 ms tps = 33.195581 (including connections establishing) tps = 33.209671 (excluding connections establishing) -- waiting 4s... (scale * 4) 6972 a,b,t,h: 100000 1 10 345 953a6ec4e 0d1bc1522 59204303a 5318d8e6f master 6973 a,b,t,h: 100000 1 10 345 953a6ec4e 0d1bc1522 59204303a 5318d8e6f replica ok -- All is well. ========= iteration 24 -- 24 of 25 ========= -- scale 1 clients 16 INIT_WAIT 0 CLEAN_ONLY= ============================================================================ -- 20170222 17:43:51 -- scale 1 clients 16 INIT_WAIT 0 -- sub_count -ne 0 : deleting sub1 pub_count -ne 0 - deleting pub1 pub_count 0 pub_repl_slot_count 0 sub_count 0 sub_repl_slot_count 0 creating tables... 100000 of 100000 tuples (100%) done (elapsed 0.12 s, remaining 0.00 s) vacuum... set primary keys... done. create publication pub1 for all tables; create subscription sub1 connection 'port=6972' publication pub1 with (disabled); alter subscription sub1 enable; -- pgbench -c 16 -T 10 -P 5 -n progress: 5.0 s, 37.0 tps, lat 382.205 ms stddev 369.352 progress: 10.0 s, 59.4 tps, lat 286.659 ms stddev 268.318 transaction type: scaling factor: 1 query mode: simple number of clients: 16 number of threads: 1 duration: 10 s number of transactions actually processed: 498 latency average = 321.764 ms latency stddev = 311.433 ms tps = 48.794769 (including connections establishing) tps = 48.817649 (excluding connections establishing) -- waiting 4s... (scale * 4) 6972 a,b,t,h: 100000 1 10 498 ccafc7a6b e451c0bc7 7549e7a6b 402bc0df6 master 6973 a,b,t,h: 100000 1 10 484 ccafc7a6b e451c0bc7 7549e7a6b e9f6e433c replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 498 ccafc7a6b e451c0bc7 7549e7a6b 402bc0df6 master 6973 a,b,t,h: 100000 1 10 484 ccafc7a6b e451c0bc7 7549e7a6b e9f6e433c replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 498 ccafc7a6b e451c0bc7 7549e7a6b 402bc0df6 master 6973 a,b,t,h: 100000 1 10 484 ccafc7a6b e451c0bc7 7549e7a6b e9f6e433c replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 498 ccafc7a6b e451c0bc7 7549e7a6b 402bc0df6 master 6973 a,b,t,h: 100000 1 10 484 ccafc7a6b e451c0bc7 7549e7a6b e9f6e433c replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 498 ccafc7a6b e451c0bc7 7549e7a6b 402bc0df6 master 6973 a,b,t,h: 100000 1 10 484 ccafc7a6b e451c0bc7 7549e7a6b e9f6e433c replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 498 ccafc7a6b e451c0bc7 7549e7a6b 402bc0df6 master 6973 a,b,t,h: 100000 1 10 484 ccafc7a6b e451c0bc7 7549e7a6b e9f6e433c replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 498 ccafc7a6b e451c0bc7 7549e7a6b 402bc0df6 master 6973 a,b,t,h: 100000 1 10 484 ccafc7a6b e451c0bc7 7549e7a6b e9f6e433c replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 498 ccafc7a6b e451c0bc7 7549e7a6b 402bc0df6 master 6973 a,b,t,h: 100000 1 10 484 ccafc7a6b e451c0bc7 7549e7a6b e9f6e433c replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 498 ccafc7a6b e451c0bc7 7549e7a6b 402bc0df6 master 6973 a,b,t,h: 100000 1 10 484 ccafc7a6b e451c0bc7 7549e7a6b e9f6e433c replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 498 ccafc7a6b e451c0bc7 7549e7a6b 402bc0df6 master 6973 a,b,t,h: 100000 1 10 484 ccafc7a6b e451c0bc7 7549e7a6b e9f6e433c replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 498 ccafc7a6b e451c0bc7 7549e7a6b 402bc0df6 master 6973 a,b,t,h: 100000 1 10 484 ccafc7a6b e451c0bc7 7549e7a6b e9f6e433c replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 498 ccafc7a6b e451c0bc7 7549e7a6b 402bc0df6 master 6973 a,b,t,h: 100000 1 10 484 ccafc7a6b e451c0bc7 7549e7a6b e9f6e433c replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 498 ccafc7a6b e451c0bc7 7549e7a6b 402bc0df6 master 6973 a,b,t,h: 100000 1 10 484 ccafc7a6b e451c0bc7 7549e7a6b e9f6e433c replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 498 ccafc7a6b e451c0bc7 7549e7a6b 402bc0df6 master 6973 a,b,t,h: 100000 1 10 484 ccafc7a6b e451c0bc7 7549e7a6b e9f6e433c replica NOK -- Not good, but breaking out of wait (waited more than 60s) ========= iteration 25 -- 25 of 25 ========= -- scale 1 clients 16 INIT_WAIT 0 CLEAN_ONLY= ============================================================================ -- 20170222 17:45:24 -- scale 1 clients 16 INIT_WAIT 0 -- sub_count -ne 0 : deleting sub1 pub_count -ne 0 - deleting pub1 pub_count 0 pub_repl_slot_count 0 sub_count 0 sub_repl_slot_count 0 creating tables... 100000 of 100000 tuples (100%) done (elapsed 0.12 s, remaining 0.00 s) vacuum... set primary keys... done. create publication pub1 for all tables; create subscription sub1 connection 'port=6972' publication pub1 with (disabled); alter subscription sub1 enable; -- pgbench -c 16 -T 10 -P 5 -n progress: 5.0 s, 30.6 tps, lat 454.697 ms stddev 349.306 progress: 10.0 s, 33.6 tps, lat 501.345 ms stddev 431.724 transaction type: scaling factor: 1 query mode: simple number of clients: 16 number of threads: 1 duration: 10 s number of transactions actually processed: 337 latency average = 481.223 ms latency stddev = 389.510 ms tps = 32.340571 (including connections establishing) tps = 32.356054 (excluding connections establishing) -- waiting 4s... (scale * 4) 6972 a,b,t,h: 100000 1 10 337 bf610711f 845a7ea5e 48f2c8994 1d5c83bc5 master 6973 a,b,t,h: 100000 1 10 330 bf610711f 845a7ea5e 48f2c8994 910cca9bd replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 337 bf610711f 845a7ea5e 48f2c8994 1d5c83bc5 master 6973 a,b,t,h: 100000 1 10 330 bf610711f 845a7ea5e 48f2c8994 910cca9bd replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 337 bf610711f 845a7ea5e 48f2c8994 1d5c83bc5 master 6973 a,b,t,h: 100000 1 10 330 bf610711f 845a7ea5e 48f2c8994 910cca9bd replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 337 bf610711f 845a7ea5e 48f2c8994 1d5c83bc5 master 6973 a,b,t,h: 100000 1 10 330 bf610711f 845a7ea5e 48f2c8994 910cca9bd replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 337 bf610711f 845a7ea5e 48f2c8994 1d5c83bc5 master 6973 a,b,t,h: 100000 1 10 330 bf610711f 845a7ea5e 48f2c8994 910cca9bd replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 337 bf610711f 845a7ea5e 48f2c8994 1d5c83bc5 master 6973 a,b,t,h: 100000 1 10 330 bf610711f 845a7ea5e 48f2c8994 910cca9bd replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 337 bf610711f 845a7ea5e 48f2c8994 1d5c83bc5 master 6973 a,b,t,h: 100000 1 10 330 bf610711f 845a7ea5e 48f2c8994 910cca9bd replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 337 bf610711f 845a7ea5e 48f2c8994 1d5c83bc5 master 6973 a,b,t,h: 100000 1 10 330 bf610711f 845a7ea5e 48f2c8994 910cca9bd replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 337 bf610711f 845a7ea5e 48f2c8994 1d5c83bc5 master 6973 a,b,t,h: 100000 1 10 330 bf610711f 845a7ea5e 48f2c8994 910cca9bd replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 337 bf610711f 845a7ea5e 48f2c8994 1d5c83bc5 master 6973 a,b,t,h: 100000 1 10 330 bf610711f 845a7ea5e 48f2c8994 910cca9bd replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 337 bf610711f 845a7ea5e 48f2c8994 1d5c83bc5 master 6973 a,b,t,h: 100000 1 10 330 bf610711f 845a7ea5e 48f2c8994 910cca9bd replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 337 bf610711f 845a7ea5e 48f2c8994 1d5c83bc5 master 6973 a,b,t,h: 100000 1 10 330 bf610711f 845a7ea5e 48f2c8994 910cca9bd replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 337 bf610711f 845a7ea5e 48f2c8994 1d5c83bc5 master 6973 a,b,t,h: 100000 1 10 330 bf610711f 845a7ea5e 48f2c8994 910cca9bd replica NOK -- wait another 5s... 6972 a,b,t,h: 100000 1 10 337 bf610711f 845a7ea5e 48f2c8994 1d5c83bc5 master 6973 a,b,t,h: 100000 1 10 330 bf610711f 845a7ea5e 48f2c8994 910cca9bd replica NOK -- Not good, but breaking out of wait (waited more than 60s)