diff --git a/cts/scheduler/dot/remote-recover-all.dot b/cts/scheduler/dot/remote-recover-all.dot
index d513ce4768..96e974d5f4 100644
--- a/cts/scheduler/dot/remote-recover-all.dot
+++ b/cts/scheduler/dot/remote-recover-all.dot
@@ -1,135 +1,154 @@
digraph "g" {
"galera-0_monitor_20000 controller-2" [ style=bold color="green" fontcolor="black"]
"galera-0_start_0 controller-2" -> "galera-0_monitor_20000 controller-2" [ style = bold]
"galera-0_start_0 controller-2" -> "galera_monitor_10000 galera-0" [ style = bold]
"galera-0_start_0 controller-2" [ style=bold color="green" fontcolor="black"]
"galera-0_stop_0 controller-1" -> "galera-0_start_0 controller-2" [ style = bold]
"galera-0_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"]
"galera-2_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"]
"galera-master_demote_0" -> "galera-master_demoted_0" [ style = bold]
"galera-master_demote_0" -> "galera_demote_0 galera-2" [ style = bold]
"galera-master_demote_0" [ style=bold color="green" fontcolor="orange"]
"galera-master_demoted_0" -> "galera-master_stop_0" [ style = bold]
"galera-master_demoted_0" [ style=bold color="green" fontcolor="orange"]
"galera-master_stop_0" -> "galera-master_stopped_0" [ style = bold]
"galera-master_stop_0" -> "galera_stop_0 galera-2" [ style = bold]
"galera-master_stop_0" [ style=bold color="green" fontcolor="orange"]
"galera-master_stopped_0" [ style=bold color="green" fontcolor="orange"]
"galera_demote_0 galera-2" -> "galera-master_demoted_0" [ style = bold]
"galera_demote_0 galera-2" -> "galera_stop_0 galera-2" [ style = bold]
"galera_demote_0 galera-2" [ style=bold color="green" fontcolor="orange"]
"galera_monitor_10000 galera-0" [ style=bold color="green" fontcolor="black"]
"galera_stop_0 galera-2" -> "galera-master_stopped_0" [ style = bold]
"galera_stop_0 galera-2" [ style=bold color="green" fontcolor="orange"]
"haproxy-clone_stop_0" -> "haproxy-clone_stopped_0" [ style = bold]
"haproxy-clone_stop_0" -> "haproxy_stop_0 controller-1" [ style = bold]
"haproxy-clone_stop_0" [ style=bold color="green" fontcolor="orange"]
"haproxy-clone_stopped_0" -> "ip-172.17.1.14_stop_0 controller-1" [ style = bold]
"haproxy-clone_stopped_0" -> "ip-172.17.1.17_stop_0 controller-1" [ style = bold]
"haproxy-clone_stopped_0" -> "ip-172.17.4.11_stop_0 controller-1" [ style = bold]
"haproxy-clone_stopped_0" [ style=bold color="green" fontcolor="orange"]
"haproxy_stop_0 controller-1" -> "haproxy-clone_stopped_0" [ style = bold]
"haproxy_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"]
"ip-172.17.1.14_monitor_10000 controller-2" [ style=bold color="green" fontcolor="black"]
"ip-172.17.1.14_start_0 controller-2" -> "ip-172.17.1.14_monitor_10000 controller-2" [ style = bold]
"ip-172.17.1.14_start_0 controller-2" [ style=bold color="green" fontcolor="black"]
"ip-172.17.1.14_stop_0 controller-1" -> "ip-172.17.1.14_start_0 controller-2" [ style = bold]
"ip-172.17.1.14_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"]
"ip-172.17.1.17_monitor_10000 controller-2" [ style=bold color="green" fontcolor="black"]
"ip-172.17.1.17_start_0 controller-2" -> "ip-172.17.1.17_monitor_10000 controller-2" [ style = bold]
"ip-172.17.1.17_start_0 controller-2" [ style=bold color="green" fontcolor="black"]
"ip-172.17.1.17_stop_0 controller-1" -> "ip-172.17.1.17_start_0 controller-2" [ style = bold]
"ip-172.17.1.17_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"]
"ip-172.17.4.11_monitor_10000 controller-2" [ style=bold color="green" fontcolor="black"]
"ip-172.17.4.11_start_0 controller-2" -> "ip-172.17.4.11_monitor_10000 controller-2" [ style = bold]
"ip-172.17.4.11_start_0 controller-2" [ style=bold color="green" fontcolor="black"]
"ip-172.17.4.11_stop_0 controller-1" -> "ip-172.17.4.11_start_0 controller-2" [ style = bold]
"ip-172.17.4.11_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"]
"messaging-1_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"]
+"rabbitmq-clone_confirmed-post_notify_stopped_0" [ style=bold color="green" fontcolor="orange"]
+"rabbitmq-clone_confirmed-pre_notify_stop_0" -> "rabbitmq-clone_post_notify_stopped_0" [ style = bold]
+"rabbitmq-clone_confirmed-pre_notify_stop_0" -> "rabbitmq-clone_stop_0" [ style = bold]
+"rabbitmq-clone_confirmed-pre_notify_stop_0" [ style=bold color="green" fontcolor="orange"]
+"rabbitmq-clone_post_notify_stopped_0" -> "rabbitmq-clone_confirmed-post_notify_stopped_0" [ style = bold]
+"rabbitmq-clone_post_notify_stopped_0" -> "rabbitmq_post_notify_stonith_0 messaging-0" [ style = bold]
+"rabbitmq-clone_post_notify_stopped_0" -> "rabbitmq_post_notify_stonith_0 messaging-2" [ style = bold]
+"rabbitmq-clone_post_notify_stopped_0" [ style=bold color="green" fontcolor="orange"]
+"rabbitmq-clone_pre_notify_stop_0" -> "rabbitmq-clone_confirmed-pre_notify_stop_0" [ style = bold]
+"rabbitmq-clone_pre_notify_stop_0" -> "rabbitmq_pre_notify_stop_0 messaging-0" [ style = bold]
+"rabbitmq-clone_pre_notify_stop_0" -> "rabbitmq_pre_notify_stop_0 messaging-2" [ style = bold]
+"rabbitmq-clone_pre_notify_stop_0" [ style=bold color="green" fontcolor="orange"]
"rabbitmq-clone_stop_0" -> "rabbitmq-clone_stopped_0" [ style = bold]
"rabbitmq-clone_stop_0" -> "rabbitmq_stop_0 messaging-1" [ style = bold]
"rabbitmq-clone_stop_0" [ style=bold color="green" fontcolor="orange"]
+"rabbitmq-clone_stopped_0" -> "rabbitmq-clone_post_notify_stopped_0" [ style = bold]
"rabbitmq-clone_stopped_0" [ style=bold color="green" fontcolor="orange"]
"rabbitmq_confirmed-post_notify_stonith_0" [ style=bold color="green" fontcolor="orange"]
+"rabbitmq_post_notify_stonith_0 messaging-0" -> "rabbitmq-clone_confirmed-post_notify_stopped_0" [ style = bold]
"rabbitmq_post_notify_stonith_0 messaging-0" -> "rabbitmq_confirmed-post_notify_stonith_0" [ style = bold]
"rabbitmq_post_notify_stonith_0 messaging-0" [ style=bold color="green" fontcolor="black"]
+"rabbitmq_post_notify_stonith_0 messaging-2" -> "rabbitmq-clone_confirmed-post_notify_stopped_0" [ style = bold]
"rabbitmq_post_notify_stonith_0 messaging-2" -> "rabbitmq_confirmed-post_notify_stonith_0" [ style = bold]
"rabbitmq_post_notify_stonith_0 messaging-2" [ style=bold color="green" fontcolor="black"]
"rabbitmq_post_notify_stonith_0" -> "rabbitmq_confirmed-post_notify_stonith_0" [ style = bold]
"rabbitmq_post_notify_stonith_0" -> "rabbitmq_post_notify_stonith_0 messaging-0" [ style = bold]
"rabbitmq_post_notify_stonith_0" -> "rabbitmq_post_notify_stonith_0 messaging-2" [ style = bold]
"rabbitmq_post_notify_stonith_0" [ style=bold color="green" fontcolor="orange"]
+"rabbitmq_pre_notify_stop_0 messaging-0" -> "rabbitmq-clone_confirmed-pre_notify_stop_0" [ style = bold]
+"rabbitmq_pre_notify_stop_0 messaging-0" [ style=bold color="green" fontcolor="black"]
+"rabbitmq_pre_notify_stop_0 messaging-2" -> "rabbitmq-clone_confirmed-pre_notify_stop_0" [ style = bold]
+"rabbitmq_pre_notify_stop_0 messaging-2" [ style=bold color="green" fontcolor="black"]
"rabbitmq_stop_0 messaging-1" -> "rabbitmq-clone_stopped_0" [ style = bold]
"rabbitmq_stop_0 messaging-1" [ style=bold color="green" fontcolor="orange"]
"redis-master_confirmed-post_notify_stopped_0" [ style=bold color="green" fontcolor="orange"]
"redis-master_confirmed-pre_notify_stop_0" -> "redis-master_post_notify_stopped_0" [ style = bold]
"redis-master_confirmed-pre_notify_stop_0" -> "redis-master_stop_0" [ style = bold]
"redis-master_confirmed-pre_notify_stop_0" [ style=bold color="green" fontcolor="orange"]
"redis-master_post_notify_stopped_0" -> "redis-master_confirmed-post_notify_stopped_0" [ style = bold]
"redis-master_post_notify_stopped_0" -> "redis_post_notify_stonith_0 controller-0" [ style = bold]
"redis-master_post_notify_stopped_0" -> "redis_post_notify_stonith_0 controller-2" [ style = bold]
"redis-master_post_notify_stopped_0" [ style=bold color="green" fontcolor="orange"]
"redis-master_pre_notify_stop_0" -> "redis-master_confirmed-pre_notify_stop_0" [ style = bold]
"redis-master_pre_notify_stop_0" -> "redis_pre_notify_stop_0 controller-0" [ style = bold]
"redis-master_pre_notify_stop_0" -> "redis_pre_notify_stop_0 controller-2" [ style = bold]
"redis-master_pre_notify_stop_0" [ style=bold color="green" fontcolor="orange"]
"redis-master_stop_0" -> "redis-master_stopped_0" [ style = bold]
"redis-master_stop_0" -> "redis_stop_0 controller-1" [ style = bold]
"redis-master_stop_0" [ style=bold color="green" fontcolor="orange"]
"redis-master_stopped_0" -> "redis-master_post_notify_stopped_0" [ style = bold]
"redis-master_stopped_0" [ style=bold color="green" fontcolor="orange"]
"redis_confirmed-post_notify_stonith_0" [ style=bold color="green" fontcolor="orange"]
"redis_post_notify_stonith_0 controller-0" -> "redis-master_confirmed-post_notify_stopped_0" [ style = bold]
"redis_post_notify_stonith_0 controller-0" -> "redis_confirmed-post_notify_stonith_0" [ style = bold]
"redis_post_notify_stonith_0 controller-0" [ style=bold color="green" fontcolor="black"]
"redis_post_notify_stonith_0 controller-2" -> "redis-master_confirmed-post_notify_stopped_0" [ style = bold]
"redis_post_notify_stonith_0 controller-2" -> "redis_confirmed-post_notify_stonith_0" [ style = bold]
"redis_post_notify_stonith_0 controller-2" [ style=bold color="green" fontcolor="black"]
"redis_post_notify_stonith_0" -> "redis_confirmed-post_notify_stonith_0" [ style = bold]
"redis_post_notify_stonith_0" -> "redis_post_notify_stonith_0 controller-0" [ style = bold]
"redis_post_notify_stonith_0" -> "redis_post_notify_stonith_0 controller-2" [ style = bold]
"redis_post_notify_stonith_0" [ style=bold color="green" fontcolor="orange"]
"redis_pre_notify_stop_0 controller-0" -> "redis-master_confirmed-pre_notify_stop_0" [ style = bold]
"redis_pre_notify_stop_0 controller-0" [ style=bold color="green" fontcolor="black"]
"redis_pre_notify_stop_0 controller-2" -> "redis-master_confirmed-pre_notify_stop_0" [ style = bold]
"redis_pre_notify_stop_0 controller-2" [ style=bold color="green" fontcolor="black"]
"redis_stop_0 controller-1" -> "redis-master_stopped_0" [ style = bold]
"redis_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"]
"stonith 'reboot' controller-1" -> "haproxy-clone_stop_0" [ style = bold]
"stonith 'reboot' controller-1" -> "haproxy_stop_0 controller-1" [ style = bold]
"stonith 'reboot' controller-1" -> "ip-172.17.1.14_start_0 controller-2" [ style = bold]
"stonith 'reboot' controller-1" -> "ip-172.17.1.14_stop_0 controller-1" [ style = bold]
"stonith 'reboot' controller-1" -> "ip-172.17.1.17_start_0 controller-2" [ style = bold]
"stonith 'reboot' controller-1" -> "ip-172.17.1.17_stop_0 controller-1" [ style = bold]
"stonith 'reboot' controller-1" -> "ip-172.17.4.11_start_0 controller-2" [ style = bold]
"stonith 'reboot' controller-1" -> "ip-172.17.4.11_stop_0 controller-1" [ style = bold]
"stonith 'reboot' controller-1" -> "redis-master_stop_0" [ style = bold]
"stonith 'reboot' controller-1" -> "redis_post_notify_stonith_0" [ style = bold]
"stonith 'reboot' controller-1" -> "redis_stop_0 controller-1" [ style = bold]
"stonith 'reboot' controller-1" -> "stonith 'reboot' galera-2" [ style = bold]
"stonith 'reboot' controller-1" [ style=bold color="green" fontcolor="black"]
"stonith 'reboot' galera-2" -> "galera-0_start_0 controller-2" [ style = bold]
"stonith 'reboot' galera-2" -> "galera-master_stop_0" [ style = bold]
"stonith 'reboot' galera-2" -> "galera_demote_0 galera-2" [ style = bold]
"stonith 'reboot' galera-2" -> "galera_stop_0 galera-2" [ style = bold]
"stonith 'reboot' galera-2" -> "ip-172.17.1.14_start_0 controller-2" [ style = bold]
"stonith 'reboot' galera-2" -> "ip-172.17.1.17_start_0 controller-2" [ style = bold]
"stonith 'reboot' galera-2" -> "ip-172.17.4.11_start_0 controller-2" [ style = bold]
"stonith 'reboot' galera-2" -> "stonith 'reboot' messaging-1" [ style = bold]
"stonith 'reboot' galera-2" -> "stonith-fence_ipmilan-5254005bdbb5_start_0 controller-2" [ style = bold]
"stonith 'reboot' galera-2" [ style=bold color="green" fontcolor="black"]
"stonith 'reboot' messaging-1" -> "galera-0_start_0 controller-2" [ style = bold]
"stonith 'reboot' messaging-1" -> "ip-172.17.1.14_start_0 controller-2" [ style = bold]
"stonith 'reboot' messaging-1" -> "ip-172.17.1.17_start_0 controller-2" [ style = bold]
"stonith 'reboot' messaging-1" -> "ip-172.17.4.11_start_0 controller-2" [ style = bold]
"stonith 'reboot' messaging-1" -> "rabbitmq-clone_stop_0" [ style = bold]
"stonith 'reboot' messaging-1" -> "rabbitmq_post_notify_stonith_0" [ style = bold]
"stonith 'reboot' messaging-1" -> "rabbitmq_stop_0 messaging-1" [ style = bold]
"stonith 'reboot' messaging-1" -> "stonith-fence_ipmilan-5254005bdbb5_start_0 controller-2" [ style = bold]
"stonith 'reboot' messaging-1" [ style=bold color="green" fontcolor="black"]
"stonith-fence_ipmilan-5254005bdbb5_monitor_60000 controller-2" [ style=bold color="green" fontcolor="black"]
"stonith-fence_ipmilan-5254005bdbb5_start_0 controller-2" -> "stonith-fence_ipmilan-5254005bdbb5_monitor_60000 controller-2" [ style = bold]
"stonith-fence_ipmilan-5254005bdbb5_start_0 controller-2" [ style=bold color="green" fontcolor="black"]
"stonith-fence_ipmilan-5254005bdbb5_stop_0 controller-1" -> "stonith-fence_ipmilan-5254005bdbb5_start_0 controller-2" [ style = bold]
"stonith-fence_ipmilan-5254005bdbb5_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"]
}
diff --git a/cts/scheduler/dot/remote-recover-no-resources.dot b/cts/scheduler/dot/remote-recover-no-resources.dot
index bae902f573..5dc93aa451 100644
--- a/cts/scheduler/dot/remote-recover-no-resources.dot
+++ b/cts/scheduler/dot/remote-recover-no-resources.dot
@@ -1,111 +1,130 @@
digraph "g" {
"galera-0_monitor_20000 controller-2" [ style=bold color="green" fontcolor="black"]
"galera-0_start_0 controller-2" -> "galera-0_monitor_20000 controller-2" [ style = bold]
"galera-0_start_0 controller-2" -> "galera_monitor_10000 galera-0" [ style = bold]
"galera-0_start_0 controller-2" [ style=bold color="green" fontcolor="black"]
"galera-0_stop_0 controller-1" -> "galera-0_start_0 controller-2" [ style = bold]
"galera-0_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"]
"galera-2_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"]
"galera_monitor_10000 galera-0" [ style=bold color="green" fontcolor="black"]
"haproxy-clone_stop_0" -> "haproxy-clone_stopped_0" [ style = bold]
"haproxy-clone_stop_0" -> "haproxy_stop_0 controller-1" [ style = bold]
"haproxy-clone_stop_0" [ style=bold color="green" fontcolor="orange"]
"haproxy-clone_stopped_0" -> "ip-172.17.1.14_stop_0 controller-1" [ style = bold]
"haproxy-clone_stopped_0" -> "ip-172.17.1.17_stop_0 controller-1" [ style = bold]
"haproxy-clone_stopped_0" -> "ip-172.17.4.11_stop_0 controller-1" [ style = bold]
"haproxy-clone_stopped_0" [ style=bold color="green" fontcolor="orange"]
"haproxy_stop_0 controller-1" -> "haproxy-clone_stopped_0" [ style = bold]
"haproxy_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"]
"ip-172.17.1.14_monitor_10000 controller-2" [ style=bold color="green" fontcolor="black"]
"ip-172.17.1.14_start_0 controller-2" -> "ip-172.17.1.14_monitor_10000 controller-2" [ style = bold]
"ip-172.17.1.14_start_0 controller-2" [ style=bold color="green" fontcolor="black"]
"ip-172.17.1.14_stop_0 controller-1" -> "ip-172.17.1.14_start_0 controller-2" [ style = bold]
"ip-172.17.1.14_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"]
"ip-172.17.1.17_monitor_10000 controller-2" [ style=bold color="green" fontcolor="black"]
"ip-172.17.1.17_start_0 controller-2" -> "ip-172.17.1.17_monitor_10000 controller-2" [ style = bold]
"ip-172.17.1.17_start_0 controller-2" [ style=bold color="green" fontcolor="black"]
"ip-172.17.1.17_stop_0 controller-1" -> "ip-172.17.1.17_start_0 controller-2" [ style = bold]
"ip-172.17.1.17_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"]
"ip-172.17.4.11_monitor_10000 controller-2" [ style=bold color="green" fontcolor="black"]
"ip-172.17.4.11_start_0 controller-2" -> "ip-172.17.4.11_monitor_10000 controller-2" [ style = bold]
"ip-172.17.4.11_start_0 controller-2" [ style=bold color="green" fontcolor="black"]
"ip-172.17.4.11_stop_0 controller-1" -> "ip-172.17.4.11_start_0 controller-2" [ style = bold]
"ip-172.17.4.11_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"]
"messaging-1_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"]
+"rabbitmq-clone_confirmed-post_notify_stopped_0" [ style=bold color="green" fontcolor="orange"]
+"rabbitmq-clone_confirmed-pre_notify_stop_0" -> "rabbitmq-clone_post_notify_stopped_0" [ style = bold]
+"rabbitmq-clone_confirmed-pre_notify_stop_0" -> "rabbitmq-clone_stop_0" [ style = bold]
+"rabbitmq-clone_confirmed-pre_notify_stop_0" [ style=bold color="green" fontcolor="orange"]
+"rabbitmq-clone_post_notify_stopped_0" -> "rabbitmq-clone_confirmed-post_notify_stopped_0" [ style = bold]
+"rabbitmq-clone_post_notify_stopped_0" -> "rabbitmq_post_notify_stonith_0 messaging-0" [ style = bold]
+"rabbitmq-clone_post_notify_stopped_0" -> "rabbitmq_post_notify_stonith_0 messaging-2" [ style = bold]
+"rabbitmq-clone_post_notify_stopped_0" [ style=bold color="green" fontcolor="orange"]
+"rabbitmq-clone_pre_notify_stop_0" -> "rabbitmq-clone_confirmed-pre_notify_stop_0" [ style = bold]
+"rabbitmq-clone_pre_notify_stop_0" -> "rabbitmq_pre_notify_stop_0 messaging-0" [ style = bold]
+"rabbitmq-clone_pre_notify_stop_0" -> "rabbitmq_pre_notify_stop_0 messaging-2" [ style = bold]
+"rabbitmq-clone_pre_notify_stop_0" [ style=bold color="green" fontcolor="orange"]
"rabbitmq-clone_stop_0" -> "rabbitmq-clone_stopped_0" [ style = bold]
"rabbitmq-clone_stop_0" -> "rabbitmq_stop_0 messaging-1" [ style = bold]
"rabbitmq-clone_stop_0" [ style=bold color="green" fontcolor="orange"]
+"rabbitmq-clone_stopped_0" -> "rabbitmq-clone_post_notify_stopped_0" [ style = bold]
"rabbitmq-clone_stopped_0" [ style=bold color="green" fontcolor="orange"]
"rabbitmq_confirmed-post_notify_stonith_0" [ style=bold color="green" fontcolor="orange"]
+"rabbitmq_post_notify_stonith_0 messaging-0" -> "rabbitmq-clone_confirmed-post_notify_stopped_0" [ style = bold]
"rabbitmq_post_notify_stonith_0 messaging-0" -> "rabbitmq_confirmed-post_notify_stonith_0" [ style = bold]
"rabbitmq_post_notify_stonith_0 messaging-0" [ style=bold color="green" fontcolor="black"]
+"rabbitmq_post_notify_stonith_0 messaging-2" -> "rabbitmq-clone_confirmed-post_notify_stopped_0" [ style = bold]
"rabbitmq_post_notify_stonith_0 messaging-2" -> "rabbitmq_confirmed-post_notify_stonith_0" [ style = bold]
"rabbitmq_post_notify_stonith_0 messaging-2" [ style=bold color="green" fontcolor="black"]
"rabbitmq_post_notify_stonith_0" -> "rabbitmq_confirmed-post_notify_stonith_0" [ style = bold]
"rabbitmq_post_notify_stonith_0" -> "rabbitmq_post_notify_stonith_0 messaging-0" [ style = bold]
"rabbitmq_post_notify_stonith_0" -> "rabbitmq_post_notify_stonith_0 messaging-2" [ style = bold]
"rabbitmq_post_notify_stonith_0" [ style=bold color="green" fontcolor="orange"]
+"rabbitmq_pre_notify_stop_0 messaging-0" -> "rabbitmq-clone_confirmed-pre_notify_stop_0" [ style = bold]
+"rabbitmq_pre_notify_stop_0 messaging-0" [ style=bold color="green" fontcolor="black"]
+"rabbitmq_pre_notify_stop_0 messaging-2" -> "rabbitmq-clone_confirmed-pre_notify_stop_0" [ style = bold]
+"rabbitmq_pre_notify_stop_0 messaging-2" [ style=bold color="green" fontcolor="black"]
"rabbitmq_stop_0 messaging-1" -> "rabbitmq-clone_stopped_0" [ style = bold]
"rabbitmq_stop_0 messaging-1" [ style=bold color="green" fontcolor="orange"]
"redis-master_confirmed-post_notify_stopped_0" [ style=bold color="green" fontcolor="orange"]
"redis-master_confirmed-pre_notify_stop_0" -> "redis-master_post_notify_stopped_0" [ style = bold]
"redis-master_confirmed-pre_notify_stop_0" -> "redis-master_stop_0" [ style = bold]
"redis-master_confirmed-pre_notify_stop_0" [ style=bold color="green" fontcolor="orange"]
"redis-master_post_notify_stopped_0" -> "redis-master_confirmed-post_notify_stopped_0" [ style = bold]
"redis-master_post_notify_stopped_0" -> "redis_post_notify_stonith_0 controller-0" [ style = bold]
"redis-master_post_notify_stopped_0" -> "redis_post_notify_stonith_0 controller-2" [ style = bold]
"redis-master_post_notify_stopped_0" [ style=bold color="green" fontcolor="orange"]
"redis-master_pre_notify_stop_0" -> "redis-master_confirmed-pre_notify_stop_0" [ style = bold]
"redis-master_pre_notify_stop_0" -> "redis_pre_notify_stop_0 controller-0" [ style = bold]
"redis-master_pre_notify_stop_0" -> "redis_pre_notify_stop_0 controller-2" [ style = bold]
"redis-master_pre_notify_stop_0" [ style=bold color="green" fontcolor="orange"]
"redis-master_stop_0" -> "redis-master_stopped_0" [ style = bold]
"redis-master_stop_0" -> "redis_stop_0 controller-1" [ style = bold]
"redis-master_stop_0" [ style=bold color="green" fontcolor="orange"]
"redis-master_stopped_0" -> "redis-master_post_notify_stopped_0" [ style = bold]
"redis-master_stopped_0" [ style=bold color="green" fontcolor="orange"]
"redis_confirmed-post_notify_stonith_0" [ style=bold color="green" fontcolor="orange"]
"redis_post_notify_stonith_0 controller-0" -> "redis-master_confirmed-post_notify_stopped_0" [ style = bold]
"redis_post_notify_stonith_0 controller-0" -> "redis_confirmed-post_notify_stonith_0" [ style = bold]
"redis_post_notify_stonith_0 controller-0" [ style=bold color="green" fontcolor="black"]
"redis_post_notify_stonith_0 controller-2" -> "redis-master_confirmed-post_notify_stopped_0" [ style = bold]
"redis_post_notify_stonith_0 controller-2" -> "redis_confirmed-post_notify_stonith_0" [ style = bold]
"redis_post_notify_stonith_0 controller-2" [ style=bold color="green" fontcolor="black"]
"redis_post_notify_stonith_0" -> "redis_confirmed-post_notify_stonith_0" [ style = bold]
"redis_post_notify_stonith_0" -> "redis_post_notify_stonith_0 controller-0" [ style = bold]
"redis_post_notify_stonith_0" -> "redis_post_notify_stonith_0 controller-2" [ style = bold]
"redis_post_notify_stonith_0" [ style=bold color="green" fontcolor="orange"]
"redis_pre_notify_stop_0 controller-0" -> "redis-master_confirmed-pre_notify_stop_0" [ style = bold]
"redis_pre_notify_stop_0 controller-0" [ style=bold color="green" fontcolor="black"]
"redis_pre_notify_stop_0 controller-2" -> "redis-master_confirmed-pre_notify_stop_0" [ style = bold]
"redis_pre_notify_stop_0 controller-2" [ style=bold color="green" fontcolor="black"]
"redis_stop_0 controller-1" -> "redis-master_stopped_0" [ style = bold]
"redis_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"]
"stonith 'reboot' controller-1" -> "haproxy-clone_stop_0" [ style = bold]
"stonith 'reboot' controller-1" -> "haproxy_stop_0 controller-1" [ style = bold]
"stonith 'reboot' controller-1" -> "ip-172.17.1.14_start_0 controller-2" [ style = bold]
"stonith 'reboot' controller-1" -> "ip-172.17.1.14_stop_0 controller-1" [ style = bold]
"stonith 'reboot' controller-1" -> "ip-172.17.1.17_start_0 controller-2" [ style = bold]
"stonith 'reboot' controller-1" -> "ip-172.17.1.17_stop_0 controller-1" [ style = bold]
"stonith 'reboot' controller-1" -> "ip-172.17.4.11_start_0 controller-2" [ style = bold]
"stonith 'reboot' controller-1" -> "ip-172.17.4.11_stop_0 controller-1" [ style = bold]
"stonith 'reboot' controller-1" -> "redis-master_stop_0" [ style = bold]
"stonith 'reboot' controller-1" -> "redis_post_notify_stonith_0" [ style = bold]
"stonith 'reboot' controller-1" -> "redis_stop_0 controller-1" [ style = bold]
"stonith 'reboot' controller-1" -> "stonith 'reboot' messaging-1" [ style = bold]
"stonith 'reboot' controller-1" [ style=bold color="green" fontcolor="black"]
"stonith 'reboot' messaging-1" -> "galera-0_start_0 controller-2" [ style = bold]
"stonith 'reboot' messaging-1" -> "ip-172.17.1.14_start_0 controller-2" [ style = bold]
"stonith 'reboot' messaging-1" -> "ip-172.17.1.17_start_0 controller-2" [ style = bold]
"stonith 'reboot' messaging-1" -> "ip-172.17.4.11_start_0 controller-2" [ style = bold]
"stonith 'reboot' messaging-1" -> "rabbitmq-clone_stop_0" [ style = bold]
"stonith 'reboot' messaging-1" -> "rabbitmq_post_notify_stonith_0" [ style = bold]
"stonith 'reboot' messaging-1" -> "rabbitmq_stop_0 messaging-1" [ style = bold]
"stonith 'reboot' messaging-1" -> "stonith-fence_ipmilan-5254005bdbb5_start_0 controller-2" [ style = bold]
"stonith 'reboot' messaging-1" [ style=bold color="green" fontcolor="black"]
"stonith-fence_ipmilan-5254005bdbb5_monitor_60000 controller-2" [ style=bold color="green" fontcolor="black"]
"stonith-fence_ipmilan-5254005bdbb5_start_0 controller-2" -> "stonith-fence_ipmilan-5254005bdbb5_monitor_60000 controller-2" [ style = bold]
"stonith-fence_ipmilan-5254005bdbb5_start_0 controller-2" [ style=bold color="green" fontcolor="black"]
"stonith-fence_ipmilan-5254005bdbb5_stop_0 controller-1" -> "stonith-fence_ipmilan-5254005bdbb5_start_0 controller-2" [ style = bold]
"stonith-fence_ipmilan-5254005bdbb5_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"]
}
diff --git a/cts/scheduler/dot/remote-recover-unknown.dot b/cts/scheduler/dot/remote-recover-unknown.dot
index eea6eea81e..edbdddf1b5 100644
--- a/cts/scheduler/dot/remote-recover-unknown.dot
+++ b/cts/scheduler/dot/remote-recover-unknown.dot
@@ -1,119 +1,138 @@
digraph "g" {
"galera-0_monitor_20000 controller-2" [ style=bold color="green" fontcolor="black"]
"galera-0_start_0 controller-2" -> "galera-0_monitor_20000 controller-2" [ style = bold]
"galera-0_start_0 controller-2" -> "galera_monitor_10000 galera-0" [ style = bold]
"galera-0_start_0 controller-2" [ style=bold color="green" fontcolor="black"]
"galera-0_stop_0 controller-1" -> "galera-0_start_0 controller-2" [ style = bold]
"galera-0_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"]
"galera-2_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"]
"galera_monitor_0 galera-2" [ style=dashed color="red" fontcolor="black"]
"galera_monitor_10000 galera-0" [ style=bold color="green" fontcolor="black"]
"haproxy-clone_stop_0" -> "haproxy-clone_stopped_0" [ style = bold]
"haproxy-clone_stop_0" -> "haproxy_stop_0 controller-1" [ style = bold]
"haproxy-clone_stop_0" [ style=bold color="green" fontcolor="orange"]
"haproxy-clone_stopped_0" -> "ip-172.17.1.14_stop_0 controller-1" [ style = bold]
"haproxy-clone_stopped_0" -> "ip-172.17.1.17_stop_0 controller-1" [ style = bold]
"haproxy-clone_stopped_0" -> "ip-172.17.4.11_stop_0 controller-1" [ style = bold]
"haproxy-clone_stopped_0" [ style=bold color="green" fontcolor="orange"]
"haproxy_stop_0 controller-1" -> "haproxy-clone_stopped_0" [ style = bold]
"haproxy_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"]
"ip-172.17.1.14_monitor_10000 controller-2" [ style=bold color="green" fontcolor="black"]
"ip-172.17.1.14_start_0 controller-2" -> "ip-172.17.1.14_monitor_10000 controller-2" [ style = bold]
"ip-172.17.1.14_start_0 controller-2" [ style=bold color="green" fontcolor="black"]
"ip-172.17.1.14_stop_0 controller-1" -> "ip-172.17.1.14_start_0 controller-2" [ style = bold]
"ip-172.17.1.14_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"]
"ip-172.17.1.17_monitor_10000 controller-2" [ style=bold color="green" fontcolor="black"]
"ip-172.17.1.17_start_0 controller-2" -> "ip-172.17.1.17_monitor_10000 controller-2" [ style = bold]
"ip-172.17.1.17_start_0 controller-2" [ style=bold color="green" fontcolor="black"]
"ip-172.17.1.17_stop_0 controller-1" -> "ip-172.17.1.17_start_0 controller-2" [ style = bold]
"ip-172.17.1.17_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"]
"ip-172.17.4.11_monitor_10000 controller-2" [ style=bold color="green" fontcolor="black"]
"ip-172.17.4.11_start_0 controller-2" -> "ip-172.17.4.11_monitor_10000 controller-2" [ style = bold]
"ip-172.17.4.11_start_0 controller-2" [ style=bold color="green" fontcolor="black"]
"ip-172.17.4.11_stop_0 controller-1" -> "ip-172.17.4.11_start_0 controller-2" [ style = bold]
"ip-172.17.4.11_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"]
"messaging-1_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"]
+"rabbitmq-clone_confirmed-post_notify_stopped_0" [ style=bold color="green" fontcolor="orange"]
+"rabbitmq-clone_confirmed-pre_notify_stop_0" -> "rabbitmq-clone_post_notify_stopped_0" [ style = bold]
+"rabbitmq-clone_confirmed-pre_notify_stop_0" -> "rabbitmq-clone_stop_0" [ style = bold]
+"rabbitmq-clone_confirmed-pre_notify_stop_0" [ style=bold color="green" fontcolor="orange"]
+"rabbitmq-clone_post_notify_stopped_0" -> "rabbitmq-clone_confirmed-post_notify_stopped_0" [ style = bold]
+"rabbitmq-clone_post_notify_stopped_0" -> "rabbitmq_post_notify_stonith_0 messaging-0" [ style = bold]
+"rabbitmq-clone_post_notify_stopped_0" -> "rabbitmq_post_notify_stonith_0 messaging-2" [ style = bold]
+"rabbitmq-clone_post_notify_stopped_0" [ style=bold color="green" fontcolor="orange"]
+"rabbitmq-clone_pre_notify_stop_0" -> "rabbitmq-clone_confirmed-pre_notify_stop_0" [ style = bold]
+"rabbitmq-clone_pre_notify_stop_0" -> "rabbitmq_pre_notify_stop_0 messaging-0" [ style = bold]
+"rabbitmq-clone_pre_notify_stop_0" -> "rabbitmq_pre_notify_stop_0 messaging-2" [ style = bold]
+"rabbitmq-clone_pre_notify_stop_0" [ style=bold color="green" fontcolor="orange"]
"rabbitmq-clone_stop_0" -> "rabbitmq-clone_stopped_0" [ style = bold]
"rabbitmq-clone_stop_0" -> "rabbitmq_stop_0 messaging-1" [ style = bold]
"rabbitmq-clone_stop_0" [ style=bold color="green" fontcolor="orange"]
+"rabbitmq-clone_stopped_0" -> "rabbitmq-clone_post_notify_stopped_0" [ style = bold]
"rabbitmq-clone_stopped_0" [ style=bold color="green" fontcolor="orange"]
"rabbitmq_confirmed-post_notify_stonith_0" [ style=bold color="green" fontcolor="orange"]
+"rabbitmq_post_notify_stonith_0 messaging-0" -> "rabbitmq-clone_confirmed-post_notify_stopped_0" [ style = bold]
"rabbitmq_post_notify_stonith_0 messaging-0" -> "rabbitmq_confirmed-post_notify_stonith_0" [ style = bold]
"rabbitmq_post_notify_stonith_0 messaging-0" [ style=bold color="green" fontcolor="black"]
+"rabbitmq_post_notify_stonith_0 messaging-2" -> "rabbitmq-clone_confirmed-post_notify_stopped_0" [ style = bold]
"rabbitmq_post_notify_stonith_0 messaging-2" -> "rabbitmq_confirmed-post_notify_stonith_0" [ style = bold]
"rabbitmq_post_notify_stonith_0 messaging-2" [ style=bold color="green" fontcolor="black"]
"rabbitmq_post_notify_stonith_0" -> "rabbitmq_confirmed-post_notify_stonith_0" [ style = bold]
"rabbitmq_post_notify_stonith_0" -> "rabbitmq_post_notify_stonith_0 messaging-0" [ style = bold]
"rabbitmq_post_notify_stonith_0" -> "rabbitmq_post_notify_stonith_0 messaging-2" [ style = bold]
"rabbitmq_post_notify_stonith_0" [ style=bold color="green" fontcolor="orange"]
+"rabbitmq_pre_notify_stop_0 messaging-0" -> "rabbitmq-clone_confirmed-pre_notify_stop_0" [ style = bold]
+"rabbitmq_pre_notify_stop_0 messaging-0" [ style=bold color="green" fontcolor="black"]
+"rabbitmq_pre_notify_stop_0 messaging-2" -> "rabbitmq-clone_confirmed-pre_notify_stop_0" [ style = bold]
+"rabbitmq_pre_notify_stop_0 messaging-2" [ style=bold color="green" fontcolor="black"]
"rabbitmq_stop_0 messaging-1" -> "rabbitmq-clone_stopped_0" [ style = bold]
"rabbitmq_stop_0 messaging-1" [ style=bold color="green" fontcolor="orange"]
"redis-master_confirmed-post_notify_stopped_0" [ style=bold color="green" fontcolor="orange"]
"redis-master_confirmed-pre_notify_stop_0" -> "redis-master_post_notify_stopped_0" [ style = bold]
"redis-master_confirmed-pre_notify_stop_0" -> "redis-master_stop_0" [ style = bold]
"redis-master_confirmed-pre_notify_stop_0" [ style=bold color="green" fontcolor="orange"]
"redis-master_post_notify_stopped_0" -> "redis-master_confirmed-post_notify_stopped_0" [ style = bold]
"redis-master_post_notify_stopped_0" -> "redis_post_notify_stonith_0 controller-0" [ style = bold]
"redis-master_post_notify_stopped_0" -> "redis_post_notify_stonith_0 controller-2" [ style = bold]
"redis-master_post_notify_stopped_0" [ style=bold color="green" fontcolor="orange"]
"redis-master_pre_notify_stop_0" -> "redis-master_confirmed-pre_notify_stop_0" [ style = bold]
"redis-master_pre_notify_stop_0" -> "redis_pre_notify_stop_0 controller-0" [ style = bold]
"redis-master_pre_notify_stop_0" -> "redis_pre_notify_stop_0 controller-2" [ style = bold]
"redis-master_pre_notify_stop_0" [ style=bold color="green" fontcolor="orange"]
"redis-master_stop_0" -> "redis-master_stopped_0" [ style = bold]
"redis-master_stop_0" -> "redis_stop_0 controller-1" [ style = bold]
"redis-master_stop_0" [ style=bold color="green" fontcolor="orange"]
"redis-master_stopped_0" -> "redis-master_post_notify_stopped_0" [ style = bold]
"redis-master_stopped_0" [ style=bold color="green" fontcolor="orange"]
"redis_confirmed-post_notify_stonith_0" [ style=bold color="green" fontcolor="orange"]
"redis_post_notify_stonith_0 controller-0" -> "redis-master_confirmed-post_notify_stopped_0" [ style = bold]
"redis_post_notify_stonith_0 controller-0" -> "redis_confirmed-post_notify_stonith_0" [ style = bold]
"redis_post_notify_stonith_0 controller-0" [ style=bold color="green" fontcolor="black"]
"redis_post_notify_stonith_0 controller-2" -> "redis-master_confirmed-post_notify_stopped_0" [ style = bold]
"redis_post_notify_stonith_0 controller-2" -> "redis_confirmed-post_notify_stonith_0" [ style = bold]
"redis_post_notify_stonith_0 controller-2" [ style=bold color="green" fontcolor="black"]
"redis_post_notify_stonith_0" -> "redis_confirmed-post_notify_stonith_0" [ style = bold]
"redis_post_notify_stonith_0" -> "redis_post_notify_stonith_0 controller-0" [ style = bold]
"redis_post_notify_stonith_0" -> "redis_post_notify_stonith_0 controller-2" [ style = bold]
"redis_post_notify_stonith_0" [ style=bold color="green" fontcolor="orange"]
"redis_pre_notify_stop_0 controller-0" -> "redis-master_confirmed-pre_notify_stop_0" [ style = bold]
"redis_pre_notify_stop_0 controller-0" [ style=bold color="green" fontcolor="black"]
"redis_pre_notify_stop_0 controller-2" -> "redis-master_confirmed-pre_notify_stop_0" [ style = bold]
"redis_pre_notify_stop_0 controller-2" [ style=bold color="green" fontcolor="black"]
"redis_stop_0 controller-1" -> "redis-master_stopped_0" [ style = bold]
"redis_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"]
"stonith 'reboot' controller-1" -> "haproxy-clone_stop_0" [ style = bold]
"stonith 'reboot' controller-1" -> "haproxy_stop_0 controller-1" [ style = bold]
"stonith 'reboot' controller-1" -> "ip-172.17.1.14_start_0 controller-2" [ style = bold]
"stonith 'reboot' controller-1" -> "ip-172.17.1.14_stop_0 controller-1" [ style = bold]
"stonith 'reboot' controller-1" -> "ip-172.17.1.17_start_0 controller-2" [ style = bold]
"stonith 'reboot' controller-1" -> "ip-172.17.1.17_stop_0 controller-1" [ style = bold]
"stonith 'reboot' controller-1" -> "ip-172.17.4.11_start_0 controller-2" [ style = bold]
"stonith 'reboot' controller-1" -> "ip-172.17.4.11_stop_0 controller-1" [ style = bold]
"stonith 'reboot' controller-1" -> "redis-master_stop_0" [ style = bold]
"stonith 'reboot' controller-1" -> "redis_post_notify_stonith_0" [ style = bold]
"stonith 'reboot' controller-1" -> "redis_stop_0 controller-1" [ style = bold]
"stonith 'reboot' controller-1" -> "stonith 'reboot' galera-2" [ style = bold]
"stonith 'reboot' controller-1" [ style=bold color="green" fontcolor="black"]
"stonith 'reboot' galera-2" -> "galera-0_start_0 controller-2" [ style = bold]
"stonith 'reboot' galera-2" -> "ip-172.17.1.14_start_0 controller-2" [ style = bold]
"stonith 'reboot' galera-2" -> "ip-172.17.1.17_start_0 controller-2" [ style = bold]
"stonith 'reboot' galera-2" -> "ip-172.17.4.11_start_0 controller-2" [ style = bold]
"stonith 'reboot' galera-2" -> "stonith 'reboot' messaging-1" [ style = bold]
"stonith 'reboot' galera-2" -> "stonith-fence_ipmilan-5254005bdbb5_start_0 controller-2" [ style = bold]
"stonith 'reboot' galera-2" [ style=bold color="green" fontcolor="black"]
"stonith 'reboot' messaging-1" -> "galera-0_start_0 controller-2" [ style = bold]
"stonith 'reboot' messaging-1" -> "ip-172.17.1.14_start_0 controller-2" [ style = bold]
"stonith 'reboot' messaging-1" -> "ip-172.17.1.17_start_0 controller-2" [ style = bold]
"stonith 'reboot' messaging-1" -> "ip-172.17.4.11_start_0 controller-2" [ style = bold]
"stonith 'reboot' messaging-1" -> "rabbitmq-clone_stop_0" [ style = bold]
"stonith 'reboot' messaging-1" -> "rabbitmq_post_notify_stonith_0" [ style = bold]
"stonith 'reboot' messaging-1" -> "rabbitmq_stop_0 messaging-1" [ style = bold]
"stonith 'reboot' messaging-1" -> "stonith-fence_ipmilan-5254005bdbb5_start_0 controller-2" [ style = bold]
"stonith 'reboot' messaging-1" [ style=bold color="green" fontcolor="black"]
"stonith-fence_ipmilan-5254005bdbb5_monitor_60000 controller-2" [ style=bold color="green" fontcolor="black"]
"stonith-fence_ipmilan-5254005bdbb5_start_0 controller-2" -> "stonith-fence_ipmilan-5254005bdbb5_monitor_60000 controller-2" [ style = bold]
"stonith-fence_ipmilan-5254005bdbb5_start_0 controller-2" [ style=bold color="green" fontcolor="black"]
"stonith-fence_ipmilan-5254005bdbb5_stop_0 controller-1" -> "stonith-fence_ipmilan-5254005bdbb5_start_0 controller-2" [ style = bold]
"stonith-fence_ipmilan-5254005bdbb5_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"]
}
diff --git a/cts/scheduler/exp/remote-recover-all.exp b/cts/scheduler/exp/remote-recover-all.exp
index aaaa5bf91c..4d35469680 100644
--- a/cts/scheduler/exp/remote-recover-all.exp
+++ b/cts/scheduler/exp/remote-recover-all.exp
@@ -1,720 +1,814 @@
-
+
-
+
-
+
-
+
-
+
-
+
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
+
-
+
-
+
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
+
-
+
-
+
-
+
-
-
+
+
-
+
-
+
-
+
-
+
-
+
-
+
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
+
-
+
-
-
+
+
-
+
+
+
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
diff --git a/cts/scheduler/exp/remote-recover-connection.exp b/cts/scheduler/exp/remote-recover-connection.exp
index dd93a78829..4e4b274dcd 100644
--- a/cts/scheduler/exp/remote-recover-connection.exp
+++ b/cts/scheduler/exp/remote-recover-connection.exp
@@ -1,554 +1,554 @@
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
diff --git a/cts/scheduler/exp/remote-recover-no-resources.exp b/cts/scheduler/exp/remote-recover-no-resources.exp
index a598cddc33..38fe6f114a 100644
--- a/cts/scheduler/exp/remote-recover-no-resources.exp
+++ b/cts/scheduler/exp/remote-recover-no-resources.exp
@@ -1,604 +1,698 @@
-
+
-
+
-
+
-
+
-
+
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
+
-
+
-
+
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
+
-
+
-
+
-
+
-
-
+
+
-
+
-
+
-
+
-
+
-
+
-
+
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
+
-
+
-
-
+
+
-
+
+
+
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
diff --git a/cts/scheduler/exp/remote-recover-unknown.exp b/cts/scheduler/exp/remote-recover-unknown.exp
index 0d8f5d541b..beaf08ad72 100644
--- a/cts/scheduler/exp/remote-recover-unknown.exp
+++ b/cts/scheduler/exp/remote-recover-unknown.exp
@@ -1,634 +1,728 @@
-
+
-
+
-
+
-
+
-
+
-
+
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
+
-
+
+
+
+
-
+
-
-
+
+
-
+
-
+
-
+
-
+
-
-
+
+
-
+
-
+
-
+
-
+
-
+
-
+
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
+
-
+
-
-
+
+
-
+
+
+
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
diff --git a/cts/scheduler/exp/remote-recovery.exp b/cts/scheduler/exp/remote-recovery.exp
index dd93a78829..4e4b274dcd 100644
--- a/cts/scheduler/exp/remote-recovery.exp
+++ b/cts/scheduler/exp/remote-recovery.exp
@@ -1,554 +1,554 @@
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
diff --git a/cts/scheduler/summary/remote-recover-all.summary b/cts/scheduler/summary/remote-recover-all.summary
index 5052ad7cfc..257301a3d7 100644
--- a/cts/scheduler/summary/remote-recover-all.summary
+++ b/cts/scheduler/summary/remote-recover-all.summary
@@ -1,146 +1,152 @@
Using the original execution date of: 2017-05-03 13:33:24Z
Current cluster status:
* Node List:
* Node controller-1: UNCLEAN (offline)
* Online: [ controller-0 controller-2 ]
* RemoteOnline: [ galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ]
* Full List of Resources:
* messaging-0 (ocf:pacemaker:remote): Started controller-0
* messaging-1 (ocf:pacemaker:remote): Started controller-1 (UNCLEAN)
* messaging-2 (ocf:pacemaker:remote): Started controller-0
* galera-0 (ocf:pacemaker:remote): Started controller-1 (UNCLEAN)
* galera-1 (ocf:pacemaker:remote): Started controller-0
* galera-2 (ocf:pacemaker:remote): Started controller-1 (UNCLEAN)
* Clone Set: rabbitmq-clone [rabbitmq]:
* Started: [ messaging-0 messaging-1 messaging-2 ]
* Stopped: [ controller-0 controller-1 controller-2 galera-0 galera-1 galera-2 ]
* Clone Set: galera-master [galera] (promotable):
* Promoted: [ galera-0 galera-1 galera-2 ]
* Stopped: [ controller-0 controller-1 controller-2 messaging-0 messaging-1 messaging-2 ]
* Clone Set: redis-master [redis] (promotable):
* redis (ocf:heartbeat:redis): Unpromoted controller-1 (UNCLEAN)
* Promoted: [ controller-0 ]
* Unpromoted: [ controller-2 ]
* Stopped: [ galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ]
* ip-192.168.24.6 (ocf:heartbeat:IPaddr2): Started controller-0
* ip-10.0.0.102 (ocf:heartbeat:IPaddr2): Started controller-0
* ip-172.17.1.14 (ocf:heartbeat:IPaddr2): Started controller-1 (UNCLEAN)
* ip-172.17.1.17 (ocf:heartbeat:IPaddr2): Started controller-1 (UNCLEAN)
* ip-172.17.3.15 (ocf:heartbeat:IPaddr2): Started controller-0
* ip-172.17.4.11 (ocf:heartbeat:IPaddr2): Started controller-1 (UNCLEAN)
* Clone Set: haproxy-clone [haproxy]:
* haproxy (systemd:haproxy): Started controller-1 (UNCLEAN)
* Started: [ controller-0 controller-2 ]
* Stopped: [ galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ]
* openstack-cinder-volume (systemd:openstack-cinder-volume): Started controller-0
* stonith-fence_ipmilan-525400bbf613 (stonith:fence_ipmilan): Started controller-0
* stonith-fence_ipmilan-525400b4f6bd (stonith:fence_ipmilan): Started controller-0
* stonith-fence_ipmilan-5254005bdbb5 (stonith:fence_ipmilan): Started controller-1 (UNCLEAN)
Transition Summary:
* Fence (reboot) messaging-1 'resources are active but connection is unrecoverable'
* Fence (reboot) galera-2 'resources are active but connection is unrecoverable'
* Fence (reboot) controller-1 'peer is no longer part of the cluster'
* Stop messaging-1 ( controller-1 ) due to node availability
* Move galera-0 ( controller-1 -> controller-2 )
* Stop galera-2 ( controller-1 ) due to node availability
* Stop rabbitmq:2 ( messaging-1 ) due to node availability
* Stop galera:1 ( Promoted galera-2 ) due to node availability
* Stop redis:0 ( Unpromoted controller-1 ) due to node availability
* Move ip-172.17.1.14 ( controller-1 -> controller-2 )
* Move ip-172.17.1.17 ( controller-1 -> controller-2 )
* Move ip-172.17.4.11 ( controller-1 -> controller-2 )
* Stop haproxy:0 ( controller-1 ) due to node availability
* Move stonith-fence_ipmilan-5254005bdbb5 ( controller-1 -> controller-2 )
Executing Cluster Transition:
* Pseudo action: messaging-1_stop_0
* Pseudo action: galera-0_stop_0
* Pseudo action: galera-2_stop_0
+ * Pseudo action: rabbitmq-clone_pre_notify_stop_0
* Pseudo action: galera-master_demote_0
* Pseudo action: redis-master_pre_notify_stop_0
* Pseudo action: stonith-fence_ipmilan-5254005bdbb5_stop_0
* Fencing controller-1 (reboot)
+ * Resource action: rabbitmq notify on messaging-2
+ * Resource action: rabbitmq notify on messaging-0
+ * Pseudo action: rabbitmq-clone_confirmed-pre_notify_stop_0
* Pseudo action: redis_post_notify_stop_0
* Resource action: redis notify on controller-0
* Resource action: redis notify on controller-2
* Pseudo action: redis-master_confirmed-pre_notify_stop_0
* Pseudo action: redis-master_stop_0
* Pseudo action: haproxy-clone_stop_0
* Fencing galera-2 (reboot)
* Pseudo action: galera_demote_0
* Pseudo action: galera-master_demoted_0
* Pseudo action: galera-master_stop_0
* Pseudo action: redis_stop_0
* Pseudo action: redis-master_stopped_0
* Pseudo action: haproxy_stop_0
* Pseudo action: haproxy-clone_stopped_0
* Fencing messaging-1 (reboot)
* Resource action: galera-0 start on controller-2
* Pseudo action: rabbitmq_post_notify_stop_0
* Pseudo action: rabbitmq-clone_stop_0
* Pseudo action: galera_stop_0
* Resource action: galera monitor=10000 on galera-0
* Pseudo action: galera-master_stopped_0
* Pseudo action: redis-master_post_notify_stopped_0
* Pseudo action: ip-172.17.1.14_stop_0
* Pseudo action: ip-172.17.1.17_stop_0
* Pseudo action: ip-172.17.4.11_stop_0
* Resource action: stonith-fence_ipmilan-5254005bdbb5 start on controller-2
* Resource action: galera-0 monitor=20000 on controller-2
- * Resource action: rabbitmq notify on messaging-2
- * Resource action: rabbitmq notify on messaging-0
- * Pseudo action: rabbitmq_notified_0
* Pseudo action: rabbitmq_stop_0
* Pseudo action: rabbitmq-clone_stopped_0
* Resource action: redis notify on controller-0
* Resource action: redis notify on controller-2
* Pseudo action: redis-master_confirmed-post_notify_stopped_0
* Resource action: ip-172.17.1.14 start on controller-2
* Resource action: ip-172.17.1.17 start on controller-2
* Resource action: ip-172.17.4.11 start on controller-2
* Resource action: stonith-fence_ipmilan-5254005bdbb5 monitor=60000 on controller-2
+ * Pseudo action: rabbitmq-clone_post_notify_stopped_0
* Pseudo action: redis_notified_0
* Resource action: ip-172.17.1.14 monitor=10000 on controller-2
* Resource action: ip-172.17.1.17 monitor=10000 on controller-2
* Resource action: ip-172.17.4.11 monitor=10000 on controller-2
+ * Resource action: rabbitmq notify on messaging-2
+ * Resource action: rabbitmq notify on messaging-0
+ * Pseudo action: rabbitmq_notified_0
+ * Pseudo action: rabbitmq-clone_confirmed-post_notify_stopped_0
Using the original execution date of: 2017-05-03 13:33:24Z
Revised Cluster Status:
* Node List:
* Online: [ controller-0 controller-2 ]
* OFFLINE: [ controller-1 ]
* RemoteOnline: [ galera-0 galera-1 messaging-0 messaging-2 ]
* RemoteOFFLINE: [ galera-2 messaging-1 ]
* Full List of Resources:
* messaging-0 (ocf:pacemaker:remote): Started controller-0
* messaging-1 (ocf:pacemaker:remote): Stopped
* messaging-2 (ocf:pacemaker:remote): Started controller-0
* galera-0 (ocf:pacemaker:remote): Started controller-2
* galera-1 (ocf:pacemaker:remote): Started controller-0
* galera-2 (ocf:pacemaker:remote): Stopped
* Clone Set: rabbitmq-clone [rabbitmq]:
* Started: [ messaging-0 messaging-2 ]
* Stopped: [ controller-0 controller-1 controller-2 galera-0 galera-1 galera-2 messaging-1 ]
* Clone Set: galera-master [galera] (promotable):
* Promoted: [ galera-0 galera-1 ]
* Stopped: [ controller-0 controller-1 controller-2 galera-2 messaging-0 messaging-1 messaging-2 ]
* Clone Set: redis-master [redis] (promotable):
* Promoted: [ controller-0 ]
* Unpromoted: [ controller-2 ]
* Stopped: [ controller-1 galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ]
* ip-192.168.24.6 (ocf:heartbeat:IPaddr2): Started controller-0
* ip-10.0.0.102 (ocf:heartbeat:IPaddr2): Started controller-0
* ip-172.17.1.14 (ocf:heartbeat:IPaddr2): Started controller-2
* ip-172.17.1.17 (ocf:heartbeat:IPaddr2): Started controller-2
* ip-172.17.3.15 (ocf:heartbeat:IPaddr2): Started controller-0
* ip-172.17.4.11 (ocf:heartbeat:IPaddr2): Started controller-2
* Clone Set: haproxy-clone [haproxy]:
* Started: [ controller-0 controller-2 ]
* Stopped: [ controller-1 galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ]
* openstack-cinder-volume (systemd:openstack-cinder-volume): Started controller-0
* stonith-fence_ipmilan-525400bbf613 (stonith:fence_ipmilan): Started controller-0
* stonith-fence_ipmilan-525400b4f6bd (stonith:fence_ipmilan): Started controller-0
* stonith-fence_ipmilan-5254005bdbb5 (stonith:fence_ipmilan): Started controller-2
diff --git a/cts/scheduler/summary/remote-recover-no-resources.summary b/cts/scheduler/summary/remote-recover-no-resources.summary
index 0e2be90a2a..d5978be449 100644
--- a/cts/scheduler/summary/remote-recover-no-resources.summary
+++ b/cts/scheduler/summary/remote-recover-no-resources.summary
@@ -1,137 +1,143 @@
Using the original execution date of: 2017-05-03 13:33:24Z
Current cluster status:
* Node List:
* Node controller-1: UNCLEAN (offline)
* Online: [ controller-0 controller-2 ]
* RemoteOnline: [ galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ]
* Full List of Resources:
* messaging-0 (ocf:pacemaker:remote): Started controller-0
* messaging-1 (ocf:pacemaker:remote): Started controller-1 (UNCLEAN)
* messaging-2 (ocf:pacemaker:remote): Started controller-0
* galera-0 (ocf:pacemaker:remote): Started controller-1 (UNCLEAN)
* galera-1 (ocf:pacemaker:remote): Started controller-0
* galera-2 (ocf:pacemaker:remote): Started controller-1 (UNCLEAN)
* Clone Set: rabbitmq-clone [rabbitmq]:
* Started: [ messaging-0 messaging-1 messaging-2 ]
* Stopped: [ controller-0 controller-1 controller-2 galera-0 galera-1 galera-2 ]
* Clone Set: galera-master [galera] (promotable):
* Promoted: [ galera-0 galera-1 ]
* Stopped: [ controller-0 controller-1 controller-2 galera-2 messaging-0 messaging-1 messaging-2 ]
* Clone Set: redis-master [redis] (promotable):
* redis (ocf:heartbeat:redis): Unpromoted controller-1 (UNCLEAN)
* Promoted: [ controller-0 ]
* Unpromoted: [ controller-2 ]
* Stopped: [ galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ]
* ip-192.168.24.6 (ocf:heartbeat:IPaddr2): Started controller-0
* ip-10.0.0.102 (ocf:heartbeat:IPaddr2): Started controller-0
* ip-172.17.1.14 (ocf:heartbeat:IPaddr2): Started controller-1 (UNCLEAN)
* ip-172.17.1.17 (ocf:heartbeat:IPaddr2): Started controller-1 (UNCLEAN)
* ip-172.17.3.15 (ocf:heartbeat:IPaddr2): Started controller-0
* ip-172.17.4.11 (ocf:heartbeat:IPaddr2): Started controller-1 (UNCLEAN)
* Clone Set: haproxy-clone [haproxy]:
* haproxy (systemd:haproxy): Started controller-1 (UNCLEAN)
* Started: [ controller-0 controller-2 ]
* Stopped: [ galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ]
* openstack-cinder-volume (systemd:openstack-cinder-volume): Started controller-0
* stonith-fence_ipmilan-525400bbf613 (stonith:fence_ipmilan): Started controller-0
* stonith-fence_ipmilan-525400b4f6bd (stonith:fence_ipmilan): Started controller-0
* stonith-fence_ipmilan-5254005bdbb5 (stonith:fence_ipmilan): Started controller-1 (UNCLEAN)
Transition Summary:
* Fence (reboot) messaging-1 'resources are active but connection is unrecoverable'
* Fence (reboot) controller-1 'peer is no longer part of the cluster'
* Stop messaging-1 ( controller-1 ) due to node availability
* Move galera-0 ( controller-1 -> controller-2 )
* Stop galera-2 ( controller-1 ) due to node availability
* Stop rabbitmq:2 ( messaging-1 ) due to node availability
* Stop redis:0 ( Unpromoted controller-1 ) due to node availability
* Move ip-172.17.1.14 ( controller-1 -> controller-2 )
* Move ip-172.17.1.17 ( controller-1 -> controller-2 )
* Move ip-172.17.4.11 ( controller-1 -> controller-2 )
* Stop haproxy:0 ( controller-1 ) due to node availability
* Move stonith-fence_ipmilan-5254005bdbb5 ( controller-1 -> controller-2 )
Executing Cluster Transition:
* Pseudo action: messaging-1_stop_0
* Pseudo action: galera-0_stop_0
* Pseudo action: galera-2_stop_0
+ * Pseudo action: rabbitmq-clone_pre_notify_stop_0
* Pseudo action: redis-master_pre_notify_stop_0
* Pseudo action: stonith-fence_ipmilan-5254005bdbb5_stop_0
* Fencing controller-1 (reboot)
+ * Resource action: rabbitmq notify on messaging-2
+ * Resource action: rabbitmq notify on messaging-0
+ * Pseudo action: rabbitmq-clone_confirmed-pre_notify_stop_0
* Pseudo action: redis_post_notify_stop_0
* Resource action: redis notify on controller-0
* Resource action: redis notify on controller-2
* Pseudo action: redis-master_confirmed-pre_notify_stop_0
* Pseudo action: redis-master_stop_0
* Pseudo action: haproxy-clone_stop_0
* Fencing messaging-1 (reboot)
* Resource action: galera-0 start on controller-2
* Pseudo action: rabbitmq_post_notify_stop_0
* Pseudo action: rabbitmq-clone_stop_0
* Resource action: galera monitor=10000 on galera-0
* Pseudo action: redis_stop_0
* Pseudo action: redis-master_stopped_0
* Pseudo action: haproxy_stop_0
* Pseudo action: haproxy-clone_stopped_0
* Resource action: stonith-fence_ipmilan-5254005bdbb5 start on controller-2
* Resource action: galera-0 monitor=20000 on controller-2
- * Resource action: rabbitmq notify on messaging-2
- * Resource action: rabbitmq notify on messaging-0
- * Pseudo action: rabbitmq_notified_0
* Pseudo action: rabbitmq_stop_0
* Pseudo action: rabbitmq-clone_stopped_0
* Pseudo action: redis-master_post_notify_stopped_0
* Pseudo action: ip-172.17.1.14_stop_0
* Pseudo action: ip-172.17.1.17_stop_0
* Pseudo action: ip-172.17.4.11_stop_0
* Resource action: stonith-fence_ipmilan-5254005bdbb5 monitor=60000 on controller-2
+ * Pseudo action: rabbitmq-clone_post_notify_stopped_0
* Resource action: redis notify on controller-0
* Resource action: redis notify on controller-2
* Pseudo action: redis-master_confirmed-post_notify_stopped_0
* Resource action: ip-172.17.1.14 start on controller-2
* Resource action: ip-172.17.1.17 start on controller-2
* Resource action: ip-172.17.4.11 start on controller-2
+ * Resource action: rabbitmq notify on messaging-2
+ * Resource action: rabbitmq notify on messaging-0
+ * Pseudo action: rabbitmq_notified_0
+ * Pseudo action: rabbitmq-clone_confirmed-post_notify_stopped_0
* Pseudo action: redis_notified_0
* Resource action: ip-172.17.1.14 monitor=10000 on controller-2
* Resource action: ip-172.17.1.17 monitor=10000 on controller-2
* Resource action: ip-172.17.4.11 monitor=10000 on controller-2
Using the original execution date of: 2017-05-03 13:33:24Z
Revised Cluster Status:
* Node List:
* Online: [ controller-0 controller-2 ]
* OFFLINE: [ controller-1 ]
* RemoteOnline: [ galera-0 galera-1 messaging-0 messaging-2 ]
* RemoteOFFLINE: [ galera-2 messaging-1 ]
* Full List of Resources:
* messaging-0 (ocf:pacemaker:remote): Started controller-0
* messaging-1 (ocf:pacemaker:remote): Stopped
* messaging-2 (ocf:pacemaker:remote): Started controller-0
* galera-0 (ocf:pacemaker:remote): Started controller-2
* galera-1 (ocf:pacemaker:remote): Started controller-0
* galera-2 (ocf:pacemaker:remote): Stopped
* Clone Set: rabbitmq-clone [rabbitmq]:
* Started: [ messaging-0 messaging-2 ]
* Stopped: [ controller-0 controller-1 controller-2 galera-0 galera-1 galera-2 messaging-1 ]
* Clone Set: galera-master [galera] (promotable):
* Promoted: [ galera-0 galera-1 ]
* Stopped: [ controller-0 controller-1 controller-2 galera-2 messaging-0 messaging-1 messaging-2 ]
* Clone Set: redis-master [redis] (promotable):
* Promoted: [ controller-0 ]
* Unpromoted: [ controller-2 ]
* Stopped: [ controller-1 galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ]
* ip-192.168.24.6 (ocf:heartbeat:IPaddr2): Started controller-0
* ip-10.0.0.102 (ocf:heartbeat:IPaddr2): Started controller-0
* ip-172.17.1.14 (ocf:heartbeat:IPaddr2): Started controller-2
* ip-172.17.1.17 (ocf:heartbeat:IPaddr2): Started controller-2
* ip-172.17.3.15 (ocf:heartbeat:IPaddr2): Started controller-0
* ip-172.17.4.11 (ocf:heartbeat:IPaddr2): Started controller-2
* Clone Set: haproxy-clone [haproxy]:
* Started: [ controller-0 controller-2 ]
* Stopped: [ controller-1 galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ]
* openstack-cinder-volume (systemd:openstack-cinder-volume): Started controller-0
* stonith-fence_ipmilan-525400bbf613 (stonith:fence_ipmilan): Started controller-0
* stonith-fence_ipmilan-525400b4f6bd (stonith:fence_ipmilan): Started controller-0
* stonith-fence_ipmilan-5254005bdbb5 (stonith:fence_ipmilan): Started controller-2
diff --git a/cts/scheduler/summary/remote-recover-unknown.summary b/cts/scheduler/summary/remote-recover-unknown.summary
index 59e1085c12..c68915878d 100644
--- a/cts/scheduler/summary/remote-recover-unknown.summary
+++ b/cts/scheduler/summary/remote-recover-unknown.summary
@@ -1,139 +1,145 @@
Using the original execution date of: 2017-05-03 13:33:24Z
Current cluster status:
* Node List:
* Node controller-1: UNCLEAN (offline)
* Online: [ controller-0 controller-2 ]
* RemoteOnline: [ galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ]
* Full List of Resources:
* messaging-0 (ocf:pacemaker:remote): Started controller-0
* messaging-1 (ocf:pacemaker:remote): Started controller-1 (UNCLEAN)
* messaging-2 (ocf:pacemaker:remote): Started controller-0
* galera-0 (ocf:pacemaker:remote): Started controller-1 (UNCLEAN)
* galera-1 (ocf:pacemaker:remote): Started controller-0
* galera-2 (ocf:pacemaker:remote): Started controller-1 (UNCLEAN)
* Clone Set: rabbitmq-clone [rabbitmq]:
* Started: [ messaging-0 messaging-1 messaging-2 ]
* Stopped: [ controller-0 controller-1 controller-2 galera-0 galera-1 galera-2 ]
* Clone Set: galera-master [galera] (promotable):
* Promoted: [ galera-0 galera-1 ]
* Stopped: [ controller-0 controller-1 controller-2 galera-2 messaging-0 messaging-1 messaging-2 ]
* Clone Set: redis-master [redis] (promotable):
* redis (ocf:heartbeat:redis): Unpromoted controller-1 (UNCLEAN)
* Promoted: [ controller-0 ]
* Unpromoted: [ controller-2 ]
* Stopped: [ galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ]
* ip-192.168.24.6 (ocf:heartbeat:IPaddr2): Started controller-0
* ip-10.0.0.102 (ocf:heartbeat:IPaddr2): Started controller-0
* ip-172.17.1.14 (ocf:heartbeat:IPaddr2): Started controller-1 (UNCLEAN)
* ip-172.17.1.17 (ocf:heartbeat:IPaddr2): Started controller-1 (UNCLEAN)
* ip-172.17.3.15 (ocf:heartbeat:IPaddr2): Started controller-0
* ip-172.17.4.11 (ocf:heartbeat:IPaddr2): Started controller-1 (UNCLEAN)
* Clone Set: haproxy-clone [haproxy]:
* haproxy (systemd:haproxy): Started controller-1 (UNCLEAN)
* Started: [ controller-0 controller-2 ]
* Stopped: [ galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ]
* openstack-cinder-volume (systemd:openstack-cinder-volume): Started controller-0
* stonith-fence_ipmilan-525400bbf613 (stonith:fence_ipmilan): Started controller-0
* stonith-fence_ipmilan-525400b4f6bd (stonith:fence_ipmilan): Started controller-0
* stonith-fence_ipmilan-5254005bdbb5 (stonith:fence_ipmilan): Started controller-1 (UNCLEAN)
Transition Summary:
* Fence (reboot) galera-2 'resources are in unknown state and connection is unrecoverable'
* Fence (reboot) messaging-1 'resources are active but connection is unrecoverable'
* Fence (reboot) controller-1 'peer is no longer part of the cluster'
* Stop messaging-1 ( controller-1 ) due to node availability
* Move galera-0 ( controller-1 -> controller-2 )
* Stop galera-2 ( controller-1 ) due to node availability
* Stop rabbitmq:2 ( messaging-1 ) due to node availability
* Stop redis:0 ( Unpromoted controller-1 ) due to node availability
* Move ip-172.17.1.14 ( controller-1 -> controller-2 )
* Move ip-172.17.1.17 ( controller-1 -> controller-2 )
* Move ip-172.17.4.11 ( controller-1 -> controller-2 )
* Stop haproxy:0 ( controller-1 ) due to node availability
* Move stonith-fence_ipmilan-5254005bdbb5 ( controller-1 -> controller-2 )
Executing Cluster Transition:
* Pseudo action: messaging-1_stop_0
* Pseudo action: galera-0_stop_0
* Pseudo action: galera-2_stop_0
+ * Pseudo action: rabbitmq-clone_pre_notify_stop_0
* Pseudo action: redis-master_pre_notify_stop_0
* Pseudo action: stonith-fence_ipmilan-5254005bdbb5_stop_0
* Fencing controller-1 (reboot)
+ * Resource action: rabbitmq notify on messaging-2
+ * Resource action: rabbitmq notify on messaging-0
+ * Pseudo action: rabbitmq-clone_confirmed-pre_notify_stop_0
* Pseudo action: redis_post_notify_stop_0
* Resource action: redis notify on controller-0
* Resource action: redis notify on controller-2
* Pseudo action: redis-master_confirmed-pre_notify_stop_0
* Pseudo action: redis-master_stop_0
* Pseudo action: haproxy-clone_stop_0
* Fencing galera-2 (reboot)
* Fencing messaging-1 (reboot)
* Resource action: galera-0 start on controller-2
* Pseudo action: rabbitmq_post_notify_stop_0
* Pseudo action: rabbitmq-clone_stop_0
* Resource action: galera monitor=10000 on galera-0
* Pseudo action: redis_stop_0
* Pseudo action: redis-master_stopped_0
* Pseudo action: haproxy_stop_0
* Pseudo action: haproxy-clone_stopped_0
* Resource action: stonith-fence_ipmilan-5254005bdbb5 start on controller-2
* Resource action: galera-0 monitor=20000 on controller-2
- * Resource action: rabbitmq notify on messaging-2
- * Resource action: rabbitmq notify on messaging-0
- * Pseudo action: rabbitmq_notified_0
* Pseudo action: rabbitmq_stop_0
* Pseudo action: rabbitmq-clone_stopped_0
* Pseudo action: redis-master_post_notify_stopped_0
* Pseudo action: ip-172.17.1.14_stop_0
* Pseudo action: ip-172.17.1.17_stop_0
* Pseudo action: ip-172.17.4.11_stop_0
* Resource action: stonith-fence_ipmilan-5254005bdbb5 monitor=60000 on controller-2
+ * Pseudo action: rabbitmq-clone_post_notify_stopped_0
* Resource action: redis notify on controller-0
* Resource action: redis notify on controller-2
* Pseudo action: redis-master_confirmed-post_notify_stopped_0
* Resource action: ip-172.17.1.14 start on controller-2
* Resource action: ip-172.17.1.17 start on controller-2
* Resource action: ip-172.17.4.11 start on controller-2
+ * Resource action: rabbitmq notify on messaging-2
+ * Resource action: rabbitmq notify on messaging-0
+ * Pseudo action: rabbitmq_notified_0
+ * Pseudo action: rabbitmq-clone_confirmed-post_notify_stopped_0
* Pseudo action: redis_notified_0
* Resource action: ip-172.17.1.14 monitor=10000 on controller-2
* Resource action: ip-172.17.1.17 monitor=10000 on controller-2
* Resource action: ip-172.17.4.11 monitor=10000 on controller-2
Using the original execution date of: 2017-05-03 13:33:24Z
Revised Cluster Status:
* Node List:
* Online: [ controller-0 controller-2 ]
* OFFLINE: [ controller-1 ]
* RemoteOnline: [ galera-0 galera-1 messaging-0 messaging-2 ]
* RemoteOFFLINE: [ galera-2 messaging-1 ]
* Full List of Resources:
* messaging-0 (ocf:pacemaker:remote): Started controller-0
* messaging-1 (ocf:pacemaker:remote): Stopped
* messaging-2 (ocf:pacemaker:remote): Started controller-0
* galera-0 (ocf:pacemaker:remote): Started controller-2
* galera-1 (ocf:pacemaker:remote): Started controller-0
* galera-2 (ocf:pacemaker:remote): Stopped
* Clone Set: rabbitmq-clone [rabbitmq]:
* Started: [ messaging-0 messaging-2 ]
* Stopped: [ controller-0 controller-1 controller-2 galera-0 galera-1 galera-2 messaging-1 ]
* Clone Set: galera-master [galera] (promotable):
* Promoted: [ galera-0 galera-1 ]
* Stopped: [ controller-0 controller-1 controller-2 galera-2 messaging-0 messaging-1 messaging-2 ]
* Clone Set: redis-master [redis] (promotable):
* Promoted: [ controller-0 ]
* Unpromoted: [ controller-2 ]
* Stopped: [ controller-1 galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ]
* ip-192.168.24.6 (ocf:heartbeat:IPaddr2): Started controller-0
* ip-10.0.0.102 (ocf:heartbeat:IPaddr2): Started controller-0
* ip-172.17.1.14 (ocf:heartbeat:IPaddr2): Started controller-2
* ip-172.17.1.17 (ocf:heartbeat:IPaddr2): Started controller-2
* ip-172.17.3.15 (ocf:heartbeat:IPaddr2): Started controller-0
* ip-172.17.4.11 (ocf:heartbeat:IPaddr2): Started controller-2
* Clone Set: haproxy-clone [haproxy]:
* Started: [ controller-0 controller-2 ]
* Stopped: [ controller-1 galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ]
* openstack-cinder-volume (systemd:openstack-cinder-volume): Started controller-0
* stonith-fence_ipmilan-525400bbf613 (stonith:fence_ipmilan): Started controller-0
* stonith-fence_ipmilan-525400b4f6bd (stonith:fence_ipmilan): Started controller-0
* stonith-fence_ipmilan-5254005bdbb5 (stonith:fence_ipmilan): Started controller-2