diff --git a/cts/scheduler/order-expired-failure.dot b/cts/scheduler/order-expired-failure.dot
index 2e9963b9ed..5c21d5d539 100644
--- a/cts/scheduler/order-expired-failure.dot
+++ b/cts/scheduler/order-expired-failure.dot
@@ -1,25 +1,24 @@
digraph "g" {
"compute-unfence-trigger-clone_stop_0" -> "compute-unfence-trigger-clone_stopped_0" [ style = bold]
"compute-unfence-trigger-clone_stop_0" -> "compute-unfence-trigger_stop_0 overcloud-novacompute-1" [ style = bold]
"compute-unfence-trigger-clone_stop_0" [ style=bold color="green" fontcolor="orange"]
"compute-unfence-trigger-clone_stopped_0" [ style=bold color="green" fontcolor="orange"]
"compute-unfence-trigger_stop_0 overcloud-novacompute-1" -> "compute-unfence-trigger-clone_stopped_0" [ style = bold]
-"compute-unfence-trigger_stop_0 overcloud-novacompute-1" -> "overcloud-novacompute-1_stop_0 controller-1" [ style = bold]
"compute-unfence-trigger_stop_0 overcloud-novacompute-1" [ style=bold color="green" fontcolor="orange"]
"ip-10.0.0.110_monitor_10000 controller-1" [ style=bold color="green" fontcolor="black"]
"ip-10.0.0.110_start_0 controller-1" -> "ip-10.0.0.110_monitor_10000 controller-1" [ style = bold]
"ip-10.0.0.110_start_0 controller-1" [ style=bold color="green" fontcolor="black"]
"overcloud-novacompute-1_clear_failcount_0 controller-1" [ style=bold color="green" fontcolor="black"]
"overcloud-novacompute-1_stop_0 controller-1" [ style=bold color="green" fontcolor="black"]
"stonith 'reboot' overcloud-novacompute-1" -> "compute-unfence-trigger-clone_stop_0" [ style = bold]
"stonith 'reboot' overcloud-novacompute-1" -> "compute-unfence-trigger_stop_0 overcloud-novacompute-1" [ style = bold]
"stonith 'reboot' overcloud-novacompute-1" -> "ip-10.0.0.110_start_0 controller-1" [ style = bold]
"stonith 'reboot' overcloud-novacompute-1" -> "overcloud-novacompute-1_clear_failcount_0 controller-1" [ style = bold]
"stonith 'reboot' overcloud-novacompute-1" -> "stonith-fence_compute-fence-nova_start_0 controller-2" [ style = bold]
"stonith 'reboot' overcloud-novacompute-1" [ style=bold color="green" fontcolor="black"]
"stonith-fence_compute-fence-nova_monitor_60000 controller-2" [ style=bold color="green" fontcolor="black"]
"stonith-fence_compute-fence-nova_start_0 controller-2" -> "stonith-fence_compute-fence-nova_monitor_60000 controller-2" [ style = bold]
"stonith-fence_compute-fence-nova_start_0 controller-2" [ style=bold color="green" fontcolor="black"]
"stonith-fence_compute-fence-nova_stop_0 controller-2" -> "stonith-fence_compute-fence-nova_start_0 controller-2" [ style = bold]
"stonith-fence_compute-fence-nova_stop_0 controller-2" [ style=bold color="green" fontcolor="black"]
}
diff --git a/cts/scheduler/order-expired-failure.exp b/cts/scheduler/order-expired-failure.exp
index c476bc2ab4..4a504936b2 100644
--- a/cts/scheduler/order-expired-failure.exp
+++ b/cts/scheduler/order-expired-failure.exp
@@ -1,148 +1,144 @@
-
-
-
-
-
+
diff --git a/cts/scheduler/order-expired-failure.summary b/cts/scheduler/order-expired-failure.summary
index c86bb914cd..2cf43edf03 100644
--- a/cts/scheduler/order-expired-failure.summary
+++ b/cts/scheduler/order-expired-failure.summary
@@ -1,110 +1,110 @@
Using the original execution date of: 2018-04-09 07:55:35Z
Current cluster status:
RemoteNode overcloud-novacompute-1: UNCLEAN (offline)
Online: [ controller-0 controller-1 controller-2 ]
RemoteOnline: [ overcloud-novacompute-0 ]
GuestOnline: [ galera-bundle-0:galera-bundle-docker-0 galera-bundle-1:galera-bundle-docker-1 galera-bundle-2:galera-bundle-docker-2 rabbitmq-bundle-0:rabbitmq-bundle-docker-0 rabbitmq-bundle-1:rabbitmq-bundle-docker-1 rabbitmq-bundle-2:rabbitmq-bundle-docker-2 redis-bundle-0:redis-bundle-docker-0 redis-bundle-1:redis-bundle-docker-1 redis-bundle-2:redis-bundle-docker-2 ]
overcloud-novacompute-0 (ocf::pacemaker:remote): Started controller-0
overcloud-novacompute-1 (ocf::pacemaker:remote): FAILED controller-1
Container bundle set: rabbitmq-bundle [192.168.24.1:8787/rhosp13/openstack-rabbitmq:pcmklatest]
rabbitmq-bundle-0 (ocf::heartbeat:rabbitmq-cluster): Started controller-2
rabbitmq-bundle-1 (ocf::heartbeat:rabbitmq-cluster): Started controller-0
rabbitmq-bundle-2 (ocf::heartbeat:rabbitmq-cluster): Started controller-1
Container bundle set: galera-bundle [192.168.24.1:8787/rhosp13/openstack-mariadb:pcmklatest]
galera-bundle-0 (ocf::heartbeat:galera): Master controller-2
galera-bundle-1 (ocf::heartbeat:galera): Master controller-0
galera-bundle-2 (ocf::heartbeat:galera): Master controller-1
Container bundle set: redis-bundle [192.168.24.1:8787/rhosp13/openstack-redis:pcmklatest]
redis-bundle-0 (ocf::heartbeat:redis): Master controller-2
redis-bundle-1 (ocf::heartbeat:redis): Slave controller-0
redis-bundle-2 (ocf::heartbeat:redis): Slave controller-1
ip-192.168.24.11 (ocf::heartbeat:IPaddr2): Started controller-2
ip-10.0.0.110 (ocf::heartbeat:IPaddr2): Stopped
ip-172.17.1.14 (ocf::heartbeat:IPaddr2): Started controller-1
ip-172.17.1.17 (ocf::heartbeat:IPaddr2): Started controller-2
ip-172.17.3.11 (ocf::heartbeat:IPaddr2): Started controller-0
ip-172.17.4.17 (ocf::heartbeat:IPaddr2): Started controller-1
Container bundle set: haproxy-bundle [192.168.24.1:8787/rhosp13/openstack-haproxy:pcmklatest]
haproxy-bundle-docker-0 (ocf::heartbeat:docker): Started controller-2
haproxy-bundle-docker-1 (ocf::heartbeat:docker): Started controller-0
haproxy-bundle-docker-2 (ocf::heartbeat:docker): Started controller-1
stonith-fence_compute-fence-nova (stonith:fence_compute): FAILED controller-2
Clone Set: compute-unfence-trigger-clone [compute-unfence-trigger]
compute-unfence-trigger (ocf::pacemaker:Dummy): Started overcloud-novacompute-1 (UNCLEAN)
Started: [ overcloud-novacompute-0 ]
Stopped: [ controller-0 controller-1 controller-2 ]
nova-evacuate (ocf::openstack:NovaEvacuate): Started controller-0
stonith-fence_ipmilan-5254008be2cc (stonith:fence_ipmilan): Started controller-1
stonith-fence_ipmilan-525400803f9e (stonith:fence_ipmilan): Started controller-0
stonith-fence_ipmilan-525400fca120 (stonith:fence_ipmilan): Started controller-2
stonith-fence_ipmilan-525400953d48 (stonith:fence_ipmilan): Started controller-2
stonith-fence_ipmilan-525400b02b86 (stonith:fence_ipmilan): Started controller-1
Container bundle: openstack-cinder-volume [192.168.24.1:8787/rhosp13/openstack-cinder-volume:pcmklatest]
openstack-cinder-volume-docker-0 (ocf::heartbeat:docker): Started controller-0
Transition Summary:
* Fence (reboot) overcloud-novacompute-1 'remote connection is unrecoverable'
* Stop overcloud-novacompute-1 ( controller-1 ) due to node availability
* Start ip-10.0.0.110 ( controller-1 )
* Recover stonith-fence_compute-fence-nova ( controller-2 )
* Stop compute-unfence-trigger:1 ( overcloud-novacompute-1 ) due to node availability
Executing cluster transition:
+ * Resource action: overcloud-novacompute-1 stop on controller-1
* Resource action: stonith-fence_compute-fence-nova stop on controller-2
* Fencing overcloud-novacompute-1 (reboot)
* Cluster action: clear_failcount for overcloud-novacompute-1 on controller-1
* Resource action: ip-10.0.0.110 start on controller-1
* Resource action: stonith-fence_compute-fence-nova start on controller-2
* Resource action: stonith-fence_compute-fence-nova monitor=60000 on controller-2
* Pseudo action: compute-unfence-trigger-clone_stop_0
* Resource action: ip-10.0.0.110 monitor=10000 on controller-1
* Pseudo action: compute-unfence-trigger_stop_0
* Pseudo action: compute-unfence-trigger-clone_stopped_0
- * Resource action: overcloud-novacompute-1 stop on controller-1
Using the original execution date of: 2018-04-09 07:55:35Z
Revised cluster status:
RemoteNode overcloud-novacompute-1: UNCLEAN (offline)
Online: [ controller-0 controller-1 controller-2 ]
RemoteOnline: [ overcloud-novacompute-0 ]
GuestOnline: [ galera-bundle-0:galera-bundle-docker-0 galera-bundle-1:galera-bundle-docker-1 galera-bundle-2:galera-bundle-docker-2 rabbitmq-bundle-0:rabbitmq-bundle-docker-0 rabbitmq-bundle-1:rabbitmq-bundle-docker-1 rabbitmq-bundle-2:rabbitmq-bundle-docker-2 redis-bundle-0:redis-bundle-docker-0 redis-bundle-1:redis-bundle-docker-1 redis-bundle-2:redis-bundle-docker-2 ]
overcloud-novacompute-0 (ocf::pacemaker:remote): Started controller-0
overcloud-novacompute-1 (ocf::pacemaker:remote): FAILED
Container bundle set: rabbitmq-bundle [192.168.24.1:8787/rhosp13/openstack-rabbitmq:pcmklatest]
rabbitmq-bundle-0 (ocf::heartbeat:rabbitmq-cluster): Started controller-2
rabbitmq-bundle-1 (ocf::heartbeat:rabbitmq-cluster): Started controller-0
rabbitmq-bundle-2 (ocf::heartbeat:rabbitmq-cluster): Started controller-1
Container bundle set: galera-bundle [192.168.24.1:8787/rhosp13/openstack-mariadb:pcmklatest]
galera-bundle-0 (ocf::heartbeat:galera): Master controller-2
galera-bundle-1 (ocf::heartbeat:galera): Master controller-0
galera-bundle-2 (ocf::heartbeat:galera): Master controller-1
Container bundle set: redis-bundle [192.168.24.1:8787/rhosp13/openstack-redis:pcmklatest]
redis-bundle-0 (ocf::heartbeat:redis): Master controller-2
redis-bundle-1 (ocf::heartbeat:redis): Slave controller-0
redis-bundle-2 (ocf::heartbeat:redis): Slave controller-1
ip-192.168.24.11 (ocf::heartbeat:IPaddr2): Started controller-2
ip-10.0.0.110 (ocf::heartbeat:IPaddr2): Started controller-1
ip-172.17.1.14 (ocf::heartbeat:IPaddr2): Started controller-1
ip-172.17.1.17 (ocf::heartbeat:IPaddr2): Started controller-2
ip-172.17.3.11 (ocf::heartbeat:IPaddr2): Started controller-0
ip-172.17.4.17 (ocf::heartbeat:IPaddr2): Started controller-1
Container bundle set: haproxy-bundle [192.168.24.1:8787/rhosp13/openstack-haproxy:pcmklatest]
haproxy-bundle-docker-0 (ocf::heartbeat:docker): Started controller-2
haproxy-bundle-docker-1 (ocf::heartbeat:docker): Started controller-0
haproxy-bundle-docker-2 (ocf::heartbeat:docker): Started controller-1
stonith-fence_compute-fence-nova (stonith:fence_compute): Started controller-2
Clone Set: compute-unfence-trigger-clone [compute-unfence-trigger]
Started: [ overcloud-novacompute-0 ]
Stopped: [ controller-0 controller-1 controller-2 overcloud-novacompute-1 ]
nova-evacuate (ocf::openstack:NovaEvacuate): Started controller-0
stonith-fence_ipmilan-5254008be2cc (stonith:fence_ipmilan): Started controller-1
stonith-fence_ipmilan-525400803f9e (stonith:fence_ipmilan): Started controller-0
stonith-fence_ipmilan-525400fca120 (stonith:fence_ipmilan): Started controller-2
stonith-fence_ipmilan-525400953d48 (stonith:fence_ipmilan): Started controller-2
stonith-fence_ipmilan-525400b02b86 (stonith:fence_ipmilan): Started controller-1
Container bundle: openstack-cinder-volume [192.168.24.1:8787/rhosp13/openstack-cinder-volume:pcmklatest]
openstack-cinder-volume-docker-0 (ocf::heartbeat:docker): Started controller-0
diff --git a/cts/scheduler/remote-connection-unrecoverable.dot b/cts/scheduler/remote-connection-unrecoverable.dot
index 77284257ce..1017d2bbeb 100644
--- a/cts/scheduler/remote-connection-unrecoverable.dot
+++ b/cts/scheduler/remote-connection-unrecoverable.dot
@@ -1,46 +1,44 @@
digraph "g" {
"killer_monitor_60000 node2" [ style=bold color="green" fontcolor="black"]
"killer_start_0 node2" -> "killer_monitor_60000 node2" [ style = bold]
"killer_start_0 node2" [ style=bold color="green" fontcolor="black"]
"killer_stop_0 node2" -> "killer_start_0 node2" [ style = bold]
"killer_stop_0 node2" [ style=bold color="green" fontcolor="black"]
"remote1_stop_0 node1" [ style=bold color="green" fontcolor="orange"]
"rsc1_delete_0 remote1" -> "rsc1_start_0 node2" [ style = dashed]
"rsc1_delete_0 remote1" [ style=dashed color="red" fontcolor="black"]
-"rsc1_monitor_0 node2" -> "remote1_stop_0 node1" [ style = bold]
"rsc1_monitor_0 node2" -> "rsc1_start_0 node2" [ style = bold]
"rsc1_monitor_0 node2" -> "rsc2-master_demote_0" [ style = bold]
"rsc1_monitor_0 node2" [ style=bold color="green" fontcolor="black"]
"rsc1_monitor_10000 node2" [ style=bold color="green" fontcolor="black"]
"rsc1_start_0 node2" -> "rsc1_monitor_10000 node2" [ style = bold]
"rsc1_start_0 node2" [ style=bold color="green" fontcolor="black"]
-"rsc1_stop_0 remote1" -> "remote1_stop_0 node1" [ style = bold]
"rsc1_stop_0 remote1" -> "rsc1_delete_0 remote1" [ style = dashed]
"rsc1_stop_0 remote1" -> "rsc1_start_0 node2" [ style = bold]
"rsc1_stop_0 remote1" -> "rsc2-master_demote_0" [ style = bold]
"rsc1_stop_0 remote1" [ style=bold color="green" fontcolor="orange"]
"rsc2-master_demote_0" -> "rsc2-master_demoted_0" [ style = bold]
"rsc2-master_demote_0" -> "rsc2_demote_0 node1" [ style = bold]
"rsc2-master_demote_0" [ style=bold color="green" fontcolor="orange"]
"rsc2-master_demoted_0" -> "rsc2-master_stop_0" [ style = bold]
"rsc2-master_demoted_0" [ style=bold color="green" fontcolor="orange"]
"rsc2-master_stop_0" -> "rsc2-master_stopped_0" [ style = bold]
"rsc2-master_stop_0" -> "rsc2_stop_0 node1" [ style = bold]
"rsc2-master_stop_0" [ style=bold color="green" fontcolor="orange"]
"rsc2-master_stopped_0" [ style=bold color="green" fontcolor="orange"]
"rsc2_demote_0 node1" -> "rsc2-master_demoted_0" [ style = bold]
"rsc2_demote_0 node1" -> "rsc2_stop_0 node1" [ style = bold]
"rsc2_demote_0 node1" [ style=bold color="green" fontcolor="orange"]
"rsc2_stop_0 node1" -> "rsc2-master_stopped_0" [ style = bold]
"rsc2_stop_0 node1" [ style=bold color="green" fontcolor="orange"]
"stonith 'reboot' node1" -> "rsc1_start_0 node2" [ style = bold]
"stonith 'reboot' node1" -> "rsc2-master_stop_0" [ style = bold]
"stonith 'reboot' node1" -> "rsc2_demote_0 node1" [ style = bold]
"stonith 'reboot' node1" -> "rsc2_stop_0 node1" [ style = bold]
"stonith 'reboot' node1" -> "stonith 'reboot' remote1" [ style = bold]
"stonith 'reboot' node1" [ style=bold color="green" fontcolor="black"]
"stonith 'reboot' remote1" -> "killer_start_0 node2" [ style = bold]
"stonith 'reboot' remote1" -> "rsc1_start_0 node2" [ style = bold]
"stonith 'reboot' remote1" -> "rsc1_stop_0 remote1" [ style = bold]
"stonith 'reboot' remote1" [ style=bold color="green" fontcolor="black"]
}
diff --git a/cts/scheduler/remote-connection-unrecoverable.exp b/cts/scheduler/remote-connection-unrecoverable.exp
index 2c9357b78a..d57c106627 100644
--- a/cts/scheduler/remote-connection-unrecoverable.exp
+++ b/cts/scheduler/remote-connection-unrecoverable.exp
@@ -1,230 +1,223 @@
-
-
-
-
-
-
-
-
+
diff --git a/cts/scheduler/remote-connection-unrecoverable.summary b/cts/scheduler/remote-connection-unrecoverable.summary
index 23fa9ca6b8..caff564ca1 100644
--- a/cts/scheduler/remote-connection-unrecoverable.summary
+++ b/cts/scheduler/remote-connection-unrecoverable.summary
@@ -1,52 +1,52 @@
Current cluster status:
Node node1 (1): UNCLEAN (offline)
Online: [ node2 ]
RemoteOnline: [ remote1 ]
remote1 (ocf::pacemaker:remote): Started node1 (UNCLEAN)
killer (stonith:fence_xvm): Started node2
rsc1 (ocf::pacemaker:Dummy): Started remote1
Clone Set: rsc2-master [rsc2] (promotable)
rsc2 (ocf::pacemaker:Stateful): Master node1 (UNCLEAN)
Masters: [ node2 ]
Stopped: [ remote1 ]
Transition Summary:
* Fence (reboot) remote1 'resources are active and the connection is unrecoverable'
* Fence (reboot) node1 'peer is no longer part of the cluster'
* Stop remote1 ( node1 ) due to node availability
* Restart killer ( node2 ) due to resource definition change
* Move rsc1 ( remote1 -> node2 )
* Stop rsc2:0 ( Master node1 ) due to node availability
Executing cluster transition:
+ * Pseudo action: remote1_stop_0
* Resource action: killer stop on node2
* Resource action: rsc1 monitor on node2
* Fencing node1 (reboot)
* Fencing remote1 (reboot)
* Resource action: killer start on node2
* Resource action: killer monitor=60000 on node2
* Pseudo action: rsc1_stop_0
* Pseudo action: rsc2-master_demote_0
- * Pseudo action: remote1_stop_0
* Resource action: rsc1 start on node2
* Pseudo action: rsc2_demote_0
* Pseudo action: rsc2-master_demoted_0
* Pseudo action: rsc2-master_stop_0
* Resource action: rsc1 monitor=10000 on node2
* Pseudo action: rsc2_stop_0
* Pseudo action: rsc2-master_stopped_0
Revised cluster status:
Online: [ node2 ]
OFFLINE: [ node1 ]
RemoteOFFLINE: [ remote1 ]
remote1 (ocf::pacemaker:remote): Stopped
killer (stonith:fence_xvm): Started node2
rsc1 (ocf::pacemaker:Dummy): Started node2
Clone Set: rsc2-master [rsc2] (promotable)
Masters: [ node2 ]
Stopped: [ node1 remote1 ]
diff --git a/cts/scheduler/remote-fence-before-reconnect.dot b/cts/scheduler/remote-fence-before-reconnect.dot
index 4ced43eb10..5812b7fd5c 100644
--- a/cts/scheduler/remote-fence-before-reconnect.dot
+++ b/cts/scheduler/remote-fence-before-reconnect.dot
@@ -1,12 +1,11 @@
digraph "g" {
"c7auto4_stop_0 c7auto1" [ style=bold color="green" fontcolor="black"]
"fake2_monitor_10000 c7auto1" [ style=bold color="green" fontcolor="black"]
"fake2_start_0 c7auto1" -> "fake2_monitor_10000 c7auto1" [ style = bold]
"fake2_start_0 c7auto1" [ style=bold color="green" fontcolor="black"]
-"fake2_stop_0 c7auto4" -> "c7auto4_stop_0 c7auto1" [ style = bold]
"fake2_stop_0 c7auto4" -> "fake2_start_0 c7auto1" [ style = bold]
"fake2_stop_0 c7auto4" [ style=bold color="green" fontcolor="orange"]
"stonith 'reboot' c7auto4" -> "fake2_start_0 c7auto1" [ style = bold]
"stonith 'reboot' c7auto4" -> "fake2_stop_0 c7auto4" [ style = bold]
"stonith 'reboot' c7auto4" [ style=bold color="green" fontcolor="black"]
}
diff --git a/cts/scheduler/remote-fence-before-reconnect.exp b/cts/scheduler/remote-fence-before-reconnect.exp
index f99d9ef762..f506f85435 100644
--- a/cts/scheduler/remote-fence-before-reconnect.exp
+++ b/cts/scheduler/remote-fence-before-reconnect.exp
@@ -1,70 +1,66 @@
-
-
-
-
-
+
diff --git a/cts/scheduler/remote-fence-before-reconnect.summary b/cts/scheduler/remote-fence-before-reconnect.summary
index f61e18b4ad..03eac203a1 100644
--- a/cts/scheduler/remote-fence-before-reconnect.summary
+++ b/cts/scheduler/remote-fence-before-reconnect.summary
@@ -1,37 +1,37 @@
Current cluster status:
RemoteNode c7auto4: UNCLEAN (offline)
Online: [ c7auto1 c7auto2 c7auto3 ]
shooter (stonith:fence_phd_kvm): Started c7auto2
c7auto4 (ocf::pacemaker:remote): FAILED c7auto1
fake1 (ocf::heartbeat:Dummy): Started c7auto3
fake2 (ocf::heartbeat:Dummy): Started c7auto4 (UNCLEAN)
fake3 (ocf::heartbeat:Dummy): Started c7auto1
fake4 (ocf::heartbeat:Dummy): Started c7auto2
fake5 (ocf::heartbeat:Dummy): Started c7auto3
Transition Summary:
* Fence (reboot) c7auto4 'remote connection is unrecoverable'
* Stop c7auto4 ( c7auto1 ) due to node availability
* Move fake2 ( c7auto4 -> c7auto1 )
Executing cluster transition:
+ * Resource action: c7auto4 stop on c7auto1
* Fencing c7auto4 (reboot)
* Pseudo action: fake2_stop_0
- * Resource action: c7auto4 stop on c7auto1
* Resource action: fake2 start on c7auto1
* Resource action: fake2 monitor=10000 on c7auto1
Revised cluster status:
RemoteNode c7auto4: UNCLEAN (offline)
Online: [ c7auto1 c7auto2 c7auto3 ]
shooter (stonith:fence_phd_kvm): Started c7auto2
c7auto4 (ocf::pacemaker:remote): FAILED
fake1 (ocf::heartbeat:Dummy): Started c7auto3
fake2 (ocf::heartbeat:Dummy): Started c7auto1
fake3 (ocf::heartbeat:Dummy): Started c7auto1
fake4 (ocf::heartbeat:Dummy): Started c7auto2
fake5 (ocf::heartbeat:Dummy): Started c7auto3
diff --git a/cts/scheduler/remote-recover-all.dot b/cts/scheduler/remote-recover-all.dot
index deed802637..4128b104e1 100644
--- a/cts/scheduler/remote-recover-all.dot
+++ b/cts/scheduler/remote-recover-all.dot
@@ -1,151 +1,149 @@
digraph "g" {
"galera-0_monitor_20000 controller-2" [ style=bold color="green" fontcolor="black"]
"galera-0_start_0 controller-2" -> "galera-0_monitor_20000 controller-2" [ style = bold]
"galera-0_start_0 controller-2" -> "galera_monitor_10000 galera-0" [ style = bold]
"galera-0_start_0 controller-2" [ style=bold color="green" fontcolor="black"]
"galera-0_stop_0 controller-1" -> "galera-0_start_0 controller-2" [ style = bold]
"galera-0_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"]
"galera-2_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"]
"galera-master_demote_0" -> "galera-master_demoted_0" [ style = bold]
"galera-master_demote_0" -> "galera_demote_0 galera-2" [ style = bold]
"galera-master_demote_0" [ style=bold color="green" fontcolor="orange"]
"galera-master_demoted_0" -> "galera-master_stop_0" [ style = bold]
"galera-master_demoted_0" [ style=bold color="green" fontcolor="orange"]
"galera-master_stop_0" -> "galera-master_stopped_0" [ style = bold]
"galera-master_stop_0" -> "galera_stop_0 galera-2" [ style = bold]
"galera-master_stop_0" [ style=bold color="green" fontcolor="orange"]
"galera-master_stopped_0" [ style=bold color="green" fontcolor="orange"]
"galera_demote_0 galera-2" -> "galera-master_demoted_0" [ style = bold]
"galera_demote_0 galera-2" -> "galera_stop_0 galera-2" [ style = bold]
"galera_demote_0 galera-2" [ style=bold color="green" fontcolor="orange"]
"galera_monitor_10000 galera-0" [ style=bold color="green" fontcolor="black"]
-"galera_stop_0 galera-2" -> "galera-2_stop_0 controller-1" [ style = bold]
"galera_stop_0 galera-2" -> "galera-master_stopped_0" [ style = bold]
"galera_stop_0 galera-2" [ style=bold color="green" fontcolor="orange"]
"haproxy-clone_stop_0" -> "haproxy-clone_stopped_0" [ style = bold]
"haproxy-clone_stop_0" -> "haproxy_stop_0 controller-1" [ style = bold]
"haproxy-clone_stop_0" [ style=bold color="green" fontcolor="orange"]
"haproxy-clone_stopped_0" -> "ip-172.17.1.14_stop_0 controller-1" [ style = bold]
"haproxy-clone_stopped_0" -> "ip-172.17.1.17_stop_0 controller-1" [ style = bold]
"haproxy-clone_stopped_0" -> "ip-172.17.4.11_stop_0 controller-1" [ style = bold]
"haproxy-clone_stopped_0" [ style=bold color="green" fontcolor="orange"]
"haproxy_stop_0 controller-1" -> "haproxy-clone_stopped_0" [ style = bold]
"haproxy_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"]
"ip-172.17.1.14_monitor_10000 controller-2" [ style=bold color="green" fontcolor="black"]
"ip-172.17.1.14_start_0 controller-2" -> "ip-172.17.1.14_monitor_10000 controller-2" [ style = bold]
"ip-172.17.1.14_start_0 controller-2" [ style=bold color="green" fontcolor="black"]
"ip-172.17.1.14_stop_0 controller-1" -> "ip-172.17.1.14_start_0 controller-2" [ style = bold]
"ip-172.17.1.14_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"]
"ip-172.17.1.17_monitor_10000 controller-2" [ style=bold color="green" fontcolor="black"]
"ip-172.17.1.17_start_0 controller-2" -> "ip-172.17.1.17_monitor_10000 controller-2" [ style = bold]
"ip-172.17.1.17_start_0 controller-2" [ style=bold color="green" fontcolor="black"]
"ip-172.17.1.17_stop_0 controller-1" -> "ip-172.17.1.17_start_0 controller-2" [ style = bold]
"ip-172.17.1.17_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"]
"ip-172.17.4.11_monitor_10000 controller-2" [ style=bold color="green" fontcolor="black"]
"ip-172.17.4.11_start_0 controller-2" -> "ip-172.17.4.11_monitor_10000 controller-2" [ style = bold]
"ip-172.17.4.11_start_0 controller-2" [ style=bold color="green" fontcolor="black"]
"ip-172.17.4.11_stop_0 controller-1" -> "ip-172.17.4.11_start_0 controller-2" [ style = bold]
"ip-172.17.4.11_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"]
"messaging-1_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"]
"rabbitmq-clone_stop_0" -> "rabbitmq-clone_stopped_0" [ style = bold]
"rabbitmq-clone_stop_0" -> "rabbitmq_stop_0 messaging-1" [ style = bold]
"rabbitmq-clone_stop_0" [ style=bold color="green" fontcolor="orange"]
"rabbitmq-clone_stopped_0" [ style=bold color="green" fontcolor="orange"]
"rabbitmq_confirmed-post_notify_stonith_0" [ style=bold color="green" fontcolor="orange"]
"rabbitmq_post_notify_stonith_0 messaging-0" -> "rabbitmq_confirmed-post_notify_stonith_0" [ style = bold]
"rabbitmq_post_notify_stonith_0 messaging-0" [ style=bold color="green" fontcolor="black"]
"rabbitmq_post_notify_stonith_0 messaging-2" -> "rabbitmq_confirmed-post_notify_stonith_0" [ style = bold]
"rabbitmq_post_notify_stonith_0 messaging-2" [ style=bold color="green" fontcolor="black"]
"rabbitmq_post_notify_stonith_0" -> "rabbitmq_confirmed-post_notify_stonith_0" [ style = bold]
"rabbitmq_post_notify_stonith_0" -> "rabbitmq_post_notify_stonith_0 messaging-0" [ style = bold]
"rabbitmq_post_notify_stonith_0" -> "rabbitmq_post_notify_stonith_0 messaging-2" [ style = bold]
"rabbitmq_post_notify_stonith_0" [ style=bold color="green" fontcolor="orange"]
-"rabbitmq_stop_0 messaging-1" -> "messaging-1_stop_0 controller-1" [ style = bold]
"rabbitmq_stop_0 messaging-1" -> "rabbitmq-clone_stopped_0" [ style = bold]
"rabbitmq_stop_0 messaging-1" [ style=bold color="green" fontcolor="orange"]
"redis-master_confirmed-post_notify_stopped_0" [ style=bold color="green" fontcolor="orange"]
"redis-master_confirmed-pre_notify_stop_0" -> "redis-master_post_notify_stopped_0" [ style = bold]
"redis-master_confirmed-pre_notify_stop_0" -> "redis-master_stop_0" [ style = bold]
"redis-master_confirmed-pre_notify_stop_0" [ style=bold color="green" fontcolor="orange"]
"redis-master_post_notify_stopped_0" -> "redis-master_confirmed-post_notify_stopped_0" [ style = bold]
"redis-master_post_notify_stopped_0" -> "redis_post_notify_stonith_0 controller-0" [ style = bold]
"redis-master_post_notify_stopped_0" -> "redis_post_notify_stonith_0 controller-2" [ style = bold]
"redis-master_post_notify_stopped_0" [ style=bold color="green" fontcolor="orange"]
"redis-master_pre_notify_stop_0" -> "redis-master_confirmed-pre_notify_stop_0" [ style = bold]
"redis-master_pre_notify_stop_0" -> "redis_pre_notify_stop_0 controller-0" [ style = bold]
"redis-master_pre_notify_stop_0" -> "redis_pre_notify_stop_0 controller-2" [ style = bold]
"redis-master_pre_notify_stop_0" [ style=bold color="green" fontcolor="orange"]
"redis-master_stop_0" -> "redis-master_stopped_0" [ style = bold]
"redis-master_stop_0" -> "redis_stop_0 controller-1" [ style = bold]
"redis-master_stop_0" [ style=bold color="green" fontcolor="orange"]
"redis-master_stopped_0" -> "redis-master_post_notify_stopped_0" [ style = bold]
"redis-master_stopped_0" [ style=bold color="green" fontcolor="orange"]
"redis_confirmed-post_notify_stonith_0" [ style=bold color="green" fontcolor="orange"]
"redis_post_notify_stonith_0 controller-0" -> "redis-master_confirmed-post_notify_stopped_0" [ style = bold]
"redis_post_notify_stonith_0 controller-0" -> "redis_confirmed-post_notify_stonith_0" [ style = bold]
"redis_post_notify_stonith_0 controller-0" [ style=bold color="green" fontcolor="black"]
"redis_post_notify_stonith_0 controller-2" -> "redis-master_confirmed-post_notify_stopped_0" [ style = bold]
"redis_post_notify_stonith_0 controller-2" -> "redis_confirmed-post_notify_stonith_0" [ style = bold]
"redis_post_notify_stonith_0 controller-2" [ style=bold color="green" fontcolor="black"]
"redis_post_notify_stonith_0" -> "redis_confirmed-post_notify_stonith_0" [ style = bold]
"redis_post_notify_stonith_0" -> "redis_post_notify_stonith_0 controller-0" [ style = bold]
"redis_post_notify_stonith_0" -> "redis_post_notify_stonith_0 controller-2" [ style = bold]
"redis_post_notify_stonith_0" [ style=bold color="green" fontcolor="orange"]
"redis_pre_notify_stop_0 controller-0" -> "redis-master_confirmed-pre_notify_stop_0" [ style = bold]
"redis_pre_notify_stop_0 controller-0" [ style=bold color="green" fontcolor="black"]
"redis_pre_notify_stop_0 controller-2" -> "redis-master_confirmed-pre_notify_stop_0" [ style = bold]
"redis_pre_notify_stop_0 controller-2" [ style=bold color="green" fontcolor="black"]
"redis_stop_0 controller-1" -> "redis-master_stopped_0" [ style = bold]
"redis_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"]
"stonith 'reboot' controller-1" -> "haproxy-clone_stop_0" [ style = bold]
"stonith 'reboot' controller-1" -> "haproxy_stop_0 controller-1" [ style = bold]
"stonith 'reboot' controller-1" -> "ip-172.17.1.14_start_0 controller-2" [ style = bold]
"stonith 'reboot' controller-1" -> "ip-172.17.1.14_stop_0 controller-1" [ style = bold]
"stonith 'reboot' controller-1" -> "ip-172.17.1.17_start_0 controller-2" [ style = bold]
"stonith 'reboot' controller-1" -> "ip-172.17.1.17_stop_0 controller-1" [ style = bold]
"stonith 'reboot' controller-1" -> "ip-172.17.4.11_start_0 controller-2" [ style = bold]
"stonith 'reboot' controller-1" -> "ip-172.17.4.11_stop_0 controller-1" [ style = bold]
"stonith 'reboot' controller-1" -> "redis-master_stop_0" [ style = bold]
"stonith 'reboot' controller-1" -> "redis_post_notify_stonith_0" [ style = bold]
"stonith 'reboot' controller-1" -> "redis_stop_0 controller-1" [ style = bold]
"stonith 'reboot' controller-1" -> "stonith 'reboot' galera-2" [ style = bold]
"stonith 'reboot' controller-1" [ style=bold color="green" fontcolor="black"]
"stonith 'reboot' galera-2" -> "galera-0_start_0 controller-2" [ style = bold]
"stonith 'reboot' galera-2" -> "galera-master_stop_0" [ style = bold]
"stonith 'reboot' galera-2" -> "galera_demote_0 galera-2" [ style = bold]
"stonith 'reboot' galera-2" -> "galera_stop_0 galera-2" [ style = bold]
"stonith 'reboot' galera-2" -> "ip-172.17.1.14_start_0 controller-2" [ style = bold]
"stonith 'reboot' galera-2" -> "ip-172.17.1.17_start_0 controller-2" [ style = bold]
"stonith 'reboot' galera-2" -> "ip-172.17.4.11_start_0 controller-2" [ style = bold]
"stonith 'reboot' galera-2" -> "stonith 'reboot' messaging-1" [ style = bold]
"stonith 'reboot' galera-2" -> "stonith-fence_ipmilan-5254005bdbb5_start_0 controller-2" [ style = bold]
"stonith 'reboot' galera-2" -> "stonith-fence_ipmilan-525400b4f6bd_start_0 controller-0" [ style = bold]
"stonith 'reboot' galera-2" -> "stonith-fence_ipmilan-525400bbf613_start_0 controller-0" [ style = bold]
"stonith 'reboot' galera-2" [ style=bold color="green" fontcolor="black"]
"stonith 'reboot' messaging-1" -> "galera-0_start_0 controller-2" [ style = bold]
"stonith 'reboot' messaging-1" -> "ip-172.17.1.14_start_0 controller-2" [ style = bold]
"stonith 'reboot' messaging-1" -> "ip-172.17.1.17_start_0 controller-2" [ style = bold]
"stonith 'reboot' messaging-1" -> "ip-172.17.4.11_start_0 controller-2" [ style = bold]
"stonith 'reboot' messaging-1" -> "rabbitmq-clone_stop_0" [ style = bold]
"stonith 'reboot' messaging-1" -> "rabbitmq_post_notify_stonith_0" [ style = bold]
"stonith 'reboot' messaging-1" -> "rabbitmq_stop_0 messaging-1" [ style = bold]
"stonith 'reboot' messaging-1" -> "stonith-fence_ipmilan-5254005bdbb5_start_0 controller-2" [ style = bold]
"stonith 'reboot' messaging-1" -> "stonith-fence_ipmilan-525400b4f6bd_start_0 controller-0" [ style = bold]
"stonith 'reboot' messaging-1" -> "stonith-fence_ipmilan-525400bbf613_start_0 controller-0" [ style = bold]
"stonith 'reboot' messaging-1" [ style=bold color="green" fontcolor="black"]
"stonith-fence_ipmilan-5254005bdbb5_monitor_60000 controller-2" [ style=bold color="green" fontcolor="black"]
"stonith-fence_ipmilan-5254005bdbb5_start_0 controller-2" -> "stonith-fence_ipmilan-5254005bdbb5_monitor_60000 controller-2" [ style = bold]
"stonith-fence_ipmilan-5254005bdbb5_start_0 controller-2" [ style=bold color="green" fontcolor="black"]
"stonith-fence_ipmilan-5254005bdbb5_stop_0 controller-1" -> "stonith-fence_ipmilan-5254005bdbb5_start_0 controller-2" [ style = bold]
"stonith-fence_ipmilan-5254005bdbb5_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"]
"stonith-fence_ipmilan-525400b4f6bd_monitor_60000 controller-0" [ style=bold color="green" fontcolor="black"]
"stonith-fence_ipmilan-525400b4f6bd_start_0 controller-0" -> "stonith-fence_ipmilan-525400b4f6bd_monitor_60000 controller-0" [ style = bold]
"stonith-fence_ipmilan-525400b4f6bd_start_0 controller-0" [ style=bold color="green" fontcolor="black"]
"stonith-fence_ipmilan-525400b4f6bd_stop_0 controller-0" -> "stonith-fence_ipmilan-525400b4f6bd_start_0 controller-0" [ style = bold]
"stonith-fence_ipmilan-525400b4f6bd_stop_0 controller-0" [ style=bold color="green" fontcolor="black"]
"stonith-fence_ipmilan-525400bbf613_monitor_60000 controller-0" [ style=bold color="green" fontcolor="black"]
"stonith-fence_ipmilan-525400bbf613_start_0 controller-0" -> "stonith-fence_ipmilan-525400bbf613_monitor_60000 controller-0" [ style = bold]
"stonith-fence_ipmilan-525400bbf613_start_0 controller-0" [ style=bold color="green" fontcolor="black"]
"stonith-fence_ipmilan-525400bbf613_stop_0 controller-0" -> "stonith-fence_ipmilan-525400bbf613_start_0 controller-0" [ style = bold]
"stonith-fence_ipmilan-525400bbf613_stop_0 controller-0" [ style=bold color="green" fontcolor="black"]
}
diff --git a/cts/scheduler/remote-recover-all.exp b/cts/scheduler/remote-recover-all.exp
index 8137ffba08..0cb51f693c 100644
--- a/cts/scheduler/remote-recover-all.exp
+++ b/cts/scheduler/remote-recover-all.exp
@@ -1,810 +1,802 @@
-
-
-
-
-
+
-
-
-
-
-
+
diff --git a/cts/scheduler/remote-recover-all.summary b/cts/scheduler/remote-recover-all.summary
index 2ac0c6a0cf..d095fddee0 100644
--- a/cts/scheduler/remote-recover-all.summary
+++ b/cts/scheduler/remote-recover-all.summary
@@ -1,152 +1,152 @@
Using the original execution date of: 2017-05-03 13:33:24Z
Current cluster status:
Node controller-1 (2): UNCLEAN (offline)
Online: [ controller-0 controller-2 ]
RemoteOnline: [ galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ]
messaging-0 (ocf::pacemaker:remote): Started controller-0
messaging-1 (ocf::pacemaker:remote): Started controller-1 (UNCLEAN)
messaging-2 (ocf::pacemaker:remote): Started controller-0
galera-0 (ocf::pacemaker:remote): Started controller-1 (UNCLEAN)
galera-1 (ocf::pacemaker:remote): Started controller-0
galera-2 (ocf::pacemaker:remote): Started controller-1 (UNCLEAN)
Clone Set: rabbitmq-clone [rabbitmq]
Started: [ messaging-0 messaging-1 messaging-2 ]
Stopped: [ controller-0 controller-1 controller-2 galera-0 galera-1 galera-2 ]
Clone Set: galera-master [galera] (promotable)
Masters: [ galera-0 galera-1 galera-2 ]
Stopped: [ controller-0 controller-1 controller-2 messaging-0 messaging-1 messaging-2 ]
Clone Set: redis-master [redis] (promotable)
redis (ocf::heartbeat:redis): Slave controller-1 (UNCLEAN)
Masters: [ controller-0 ]
Slaves: [ controller-2 ]
Stopped: [ galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ]
ip-192.168.24.6 (ocf::heartbeat:IPaddr2): Started controller-0
ip-10.0.0.102 (ocf::heartbeat:IPaddr2): Started controller-0
ip-172.17.1.14 (ocf::heartbeat:IPaddr2): Started controller-1 (UNCLEAN)
ip-172.17.1.17 (ocf::heartbeat:IPaddr2): Started controller-1 (UNCLEAN)
ip-172.17.3.15 (ocf::heartbeat:IPaddr2): Started controller-0
ip-172.17.4.11 (ocf::heartbeat:IPaddr2): Started controller-1 (UNCLEAN)
Clone Set: haproxy-clone [haproxy]
haproxy (systemd:haproxy): Started controller-1 (UNCLEAN)
Started: [ controller-0 controller-2 ]
Stopped: [ galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ]
openstack-cinder-volume (systemd:openstack-cinder-volume): Started controller-0
stonith-fence_ipmilan-525400bbf613 (stonith:fence_ipmilan): Started controller-0
stonith-fence_ipmilan-525400b4f6bd (stonith:fence_ipmilan): Started controller-0
stonith-fence_ipmilan-5254005bdbb5 (stonith:fence_ipmilan): Started controller-1 (UNCLEAN)
Transition Summary:
* Fence (reboot) messaging-1 'resources are active and the connection is unrecoverable'
* Fence (reboot) galera-2 'resources are active and the connection is unrecoverable'
* Fence (reboot) controller-1 'peer is no longer part of the cluster'
* Stop messaging-1 (controller-1) due to node availability
* Move galera-0 ( controller-1 -> controller-2 )
* Stop galera-2 (controller-1) due to node availability
* Stop rabbitmq:2 (messaging-1) due to node availability
* Stop galera:1 ( Master galera-2 ) due to node availability
* Stop redis:0 ( Slave controller-1 ) due to node availability
* Move ip-172.17.1.14 ( controller-1 -> controller-2 )
* Move ip-172.17.1.17 ( controller-1 -> controller-2 )
* Move ip-172.17.4.11 ( controller-1 -> controller-2 )
* Stop haproxy:0 (controller-1) due to node availability
* Restart stonith-fence_ipmilan-525400bbf613 ( controller-0 ) due to resource definition change
* Restart stonith-fence_ipmilan-525400b4f6bd ( controller-0 ) due to resource definition change
* Move stonith-fence_ipmilan-5254005bdbb5 ( controller-1 -> controller-2 )
Executing cluster transition:
+ * Pseudo action: messaging-1_stop_0
* Pseudo action: galera-0_stop_0
+ * Pseudo action: galera-2_stop_0
* Pseudo action: galera-master_demote_0
* Pseudo action: redis-master_pre_notify_stop_0
* Resource action: stonith-fence_ipmilan-525400bbf613 stop on controller-0
* Resource action: stonith-fence_ipmilan-525400b4f6bd stop on controller-0
* Pseudo action: stonith-fence_ipmilan-5254005bdbb5_stop_0
* Fencing controller-1 (reboot)
* Pseudo action: redis_post_notify_stop_0
* Resource action: redis notify on controller-0
* Resource action: redis notify on controller-2
* Pseudo action: redis-master_confirmed-pre_notify_stop_0
* Pseudo action: redis-master_stop_0
* Pseudo action: haproxy-clone_stop_0
* Fencing galera-2 (reboot)
* Pseudo action: galera_demote_0
* Pseudo action: galera-master_demoted_0
* Pseudo action: galera-master_stop_0
* Pseudo action: redis_stop_0
* Pseudo action: redis-master_stopped_0
* Pseudo action: haproxy_stop_0
* Pseudo action: haproxy-clone_stopped_0
* Fencing messaging-1 (reboot)
* Resource action: galera-0 start on controller-2
* Pseudo action: rabbitmq_post_notify_stop_0
* Pseudo action: rabbitmq-clone_stop_0
* Pseudo action: galera_stop_0
* Resource action: galera monitor=10000 on galera-0
* Pseudo action: galera-master_stopped_0
* Pseudo action: redis-master_post_notify_stopped_0
* Pseudo action: ip-172.17.1.14_stop_0
* Pseudo action: ip-172.17.1.17_stop_0
* Pseudo action: ip-172.17.4.11_stop_0
* Resource action: stonith-fence_ipmilan-525400bbf613 start on controller-0
* Resource action: stonith-fence_ipmilan-525400bbf613 monitor=60000 on controller-0
* Resource action: stonith-fence_ipmilan-525400b4f6bd start on controller-0
* Resource action: stonith-fence_ipmilan-525400b4f6bd monitor=60000 on controller-0
* Resource action: stonith-fence_ipmilan-5254005bdbb5 start on controller-2
* Resource action: galera-0 monitor=20000 on controller-2
- * Pseudo action: galera-2_stop_0
* Resource action: rabbitmq notify on messaging-2
* Resource action: rabbitmq notify on messaging-0
* Pseudo action: rabbitmq_notified_0
* Pseudo action: rabbitmq_stop_0
* Pseudo action: rabbitmq-clone_stopped_0
* Resource action: redis notify on controller-0
* Resource action: redis notify on controller-2
* Pseudo action: redis-master_confirmed-post_notify_stopped_0
* Resource action: ip-172.17.1.14 start on controller-2
* Resource action: ip-172.17.1.17 start on controller-2
* Resource action: ip-172.17.4.11 start on controller-2
* Resource action: stonith-fence_ipmilan-5254005bdbb5 monitor=60000 on controller-2
- * Pseudo action: messaging-1_stop_0
* Pseudo action: redis_notified_0
* Resource action: ip-172.17.1.14 monitor=10000 on controller-2
* Resource action: ip-172.17.1.17 monitor=10000 on controller-2
* Resource action: ip-172.17.4.11 monitor=10000 on controller-2
Using the original execution date of: 2017-05-03 13:33:24Z
Revised cluster status:
Online: [ controller-0 controller-2 ]
OFFLINE: [ controller-1 ]
RemoteOnline: [ galera-0 galera-1 messaging-0 messaging-2 ]
RemoteOFFLINE: [ galera-2 messaging-1 ]
messaging-0 (ocf::pacemaker:remote): Started controller-0
messaging-1 (ocf::pacemaker:remote): Stopped
messaging-2 (ocf::pacemaker:remote): Started controller-0
galera-0 (ocf::pacemaker:remote): Started controller-2
galera-1 (ocf::pacemaker:remote): Started controller-0
galera-2 (ocf::pacemaker:remote): Stopped
Clone Set: rabbitmq-clone [rabbitmq]
Started: [ messaging-0 messaging-2 ]
Stopped: [ controller-0 controller-1 controller-2 galera-0 galera-1 galera-2 messaging-1 ]
Clone Set: galera-master [galera] (promotable)
Masters: [ galera-0 galera-1 ]
Stopped: [ controller-0 controller-1 controller-2 galera-2 messaging-0 messaging-1 messaging-2 ]
Clone Set: redis-master [redis] (promotable)
Masters: [ controller-0 ]
Slaves: [ controller-2 ]
Stopped: [ controller-1 galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ]
ip-192.168.24.6 (ocf::heartbeat:IPaddr2): Started controller-0
ip-10.0.0.102 (ocf::heartbeat:IPaddr2): Started controller-0
ip-172.17.1.14 (ocf::heartbeat:IPaddr2): Started controller-2
ip-172.17.1.17 (ocf::heartbeat:IPaddr2): Started controller-2
ip-172.17.3.15 (ocf::heartbeat:IPaddr2): Started controller-0
ip-172.17.4.11 (ocf::heartbeat:IPaddr2): Started controller-2
Clone Set: haproxy-clone [haproxy]
Started: [ controller-0 controller-2 ]
Stopped: [ controller-1 galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ]
openstack-cinder-volume (systemd:openstack-cinder-volume): Started controller-0
stonith-fence_ipmilan-525400bbf613 (stonith:fence_ipmilan): Started controller-0
stonith-fence_ipmilan-525400b4f6bd (stonith:fence_ipmilan): Started controller-0
stonith-fence_ipmilan-5254005bdbb5 (stonith:fence_ipmilan): Started controller-2
diff --git a/cts/scheduler/remote-recover-no-resources.dot b/cts/scheduler/remote-recover-no-resources.dot
index ef78aa6e43..a2f8ce00d1 100644
--- a/cts/scheduler/remote-recover-no-resources.dot
+++ b/cts/scheduler/remote-recover-no-resources.dot
@@ -1,124 +1,123 @@
digraph "g" {
"galera-0_monitor_20000 controller-2" [ style=bold color="green" fontcolor="black"]
"galera-0_start_0 controller-2" -> "galera-0_monitor_20000 controller-2" [ style = bold]
"galera-0_start_0 controller-2" -> "galera_monitor_10000 galera-0" [ style = bold]
"galera-0_start_0 controller-2" [ style=bold color="green" fontcolor="black"]
"galera-0_stop_0 controller-1" -> "galera-0_start_0 controller-2" [ style = bold]
"galera-0_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"]
"galera-2_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"]
"galera_monitor_10000 galera-0" [ style=bold color="green" fontcolor="black"]
"haproxy-clone_stop_0" -> "haproxy-clone_stopped_0" [ style = bold]
"haproxy-clone_stop_0" -> "haproxy_stop_0 controller-1" [ style = bold]
"haproxy-clone_stop_0" [ style=bold color="green" fontcolor="orange"]
"haproxy-clone_stopped_0" -> "ip-172.17.1.14_stop_0 controller-1" [ style = bold]
"haproxy-clone_stopped_0" -> "ip-172.17.1.17_stop_0 controller-1" [ style = bold]
"haproxy-clone_stopped_0" -> "ip-172.17.4.11_stop_0 controller-1" [ style = bold]
"haproxy-clone_stopped_0" [ style=bold color="green" fontcolor="orange"]
"haproxy_stop_0 controller-1" -> "haproxy-clone_stopped_0" [ style = bold]
"haproxy_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"]
"ip-172.17.1.14_monitor_10000 controller-2" [ style=bold color="green" fontcolor="black"]
"ip-172.17.1.14_start_0 controller-2" -> "ip-172.17.1.14_monitor_10000 controller-2" [ style = bold]
"ip-172.17.1.14_start_0 controller-2" [ style=bold color="green" fontcolor="black"]
"ip-172.17.1.14_stop_0 controller-1" -> "ip-172.17.1.14_start_0 controller-2" [ style = bold]
"ip-172.17.1.14_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"]
"ip-172.17.1.17_monitor_10000 controller-2" [ style=bold color="green" fontcolor="black"]
"ip-172.17.1.17_start_0 controller-2" -> "ip-172.17.1.17_monitor_10000 controller-2" [ style = bold]
"ip-172.17.1.17_start_0 controller-2" [ style=bold color="green" fontcolor="black"]
"ip-172.17.1.17_stop_0 controller-1" -> "ip-172.17.1.17_start_0 controller-2" [ style = bold]
"ip-172.17.1.17_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"]
"ip-172.17.4.11_monitor_10000 controller-2" [ style=bold color="green" fontcolor="black"]
"ip-172.17.4.11_start_0 controller-2" -> "ip-172.17.4.11_monitor_10000 controller-2" [ style = bold]
"ip-172.17.4.11_start_0 controller-2" [ style=bold color="green" fontcolor="black"]
"ip-172.17.4.11_stop_0 controller-1" -> "ip-172.17.4.11_start_0 controller-2" [ style = bold]
"ip-172.17.4.11_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"]
"messaging-1_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"]
"rabbitmq-clone_stop_0" -> "rabbitmq-clone_stopped_0" [ style = bold]
"rabbitmq-clone_stop_0" -> "rabbitmq_stop_0 messaging-1" [ style = bold]
"rabbitmq-clone_stop_0" [ style=bold color="green" fontcolor="orange"]
"rabbitmq-clone_stopped_0" [ style=bold color="green" fontcolor="orange"]
"rabbitmq_confirmed-post_notify_stonith_0" [ style=bold color="green" fontcolor="orange"]
"rabbitmq_post_notify_stonith_0 messaging-0" -> "rabbitmq_confirmed-post_notify_stonith_0" [ style = bold]
"rabbitmq_post_notify_stonith_0 messaging-0" [ style=bold color="green" fontcolor="black"]
"rabbitmq_post_notify_stonith_0 messaging-2" -> "rabbitmq_confirmed-post_notify_stonith_0" [ style = bold]
"rabbitmq_post_notify_stonith_0 messaging-2" [ style=bold color="green" fontcolor="black"]
"rabbitmq_post_notify_stonith_0" -> "rabbitmq_confirmed-post_notify_stonith_0" [ style = bold]
"rabbitmq_post_notify_stonith_0" -> "rabbitmq_post_notify_stonith_0 messaging-0" [ style = bold]
"rabbitmq_post_notify_stonith_0" -> "rabbitmq_post_notify_stonith_0 messaging-2" [ style = bold]
"rabbitmq_post_notify_stonith_0" [ style=bold color="green" fontcolor="orange"]
-"rabbitmq_stop_0 messaging-1" -> "messaging-1_stop_0 controller-1" [ style = bold]
"rabbitmq_stop_0 messaging-1" -> "rabbitmq-clone_stopped_0" [ style = bold]
"rabbitmq_stop_0 messaging-1" [ style=bold color="green" fontcolor="orange"]
"redis-master_confirmed-post_notify_stopped_0" [ style=bold color="green" fontcolor="orange"]
"redis-master_confirmed-pre_notify_stop_0" -> "redis-master_post_notify_stopped_0" [ style = bold]
"redis-master_confirmed-pre_notify_stop_0" -> "redis-master_stop_0" [ style = bold]
"redis-master_confirmed-pre_notify_stop_0" [ style=bold color="green" fontcolor="orange"]
"redis-master_post_notify_stopped_0" -> "redis-master_confirmed-post_notify_stopped_0" [ style = bold]
"redis-master_post_notify_stopped_0" -> "redis_post_notify_stonith_0 controller-0" [ style = bold]
"redis-master_post_notify_stopped_0" -> "redis_post_notify_stonith_0 controller-2" [ style = bold]
"redis-master_post_notify_stopped_0" [ style=bold color="green" fontcolor="orange"]
"redis-master_pre_notify_stop_0" -> "redis-master_confirmed-pre_notify_stop_0" [ style = bold]
"redis-master_pre_notify_stop_0" -> "redis_pre_notify_stop_0 controller-0" [ style = bold]
"redis-master_pre_notify_stop_0" -> "redis_pre_notify_stop_0 controller-2" [ style = bold]
"redis-master_pre_notify_stop_0" [ style=bold color="green" fontcolor="orange"]
"redis-master_stop_0" -> "redis-master_stopped_0" [ style = bold]
"redis-master_stop_0" -> "redis_stop_0 controller-1" [ style = bold]
"redis-master_stop_0" [ style=bold color="green" fontcolor="orange"]
"redis-master_stopped_0" -> "redis-master_post_notify_stopped_0" [ style = bold]
"redis-master_stopped_0" [ style=bold color="green" fontcolor="orange"]
"redis_confirmed-post_notify_stonith_0" [ style=bold color="green" fontcolor="orange"]
"redis_post_notify_stonith_0 controller-0" -> "redis-master_confirmed-post_notify_stopped_0" [ style = bold]
"redis_post_notify_stonith_0 controller-0" -> "redis_confirmed-post_notify_stonith_0" [ style = bold]
"redis_post_notify_stonith_0 controller-0" [ style=bold color="green" fontcolor="black"]
"redis_post_notify_stonith_0 controller-2" -> "redis-master_confirmed-post_notify_stopped_0" [ style = bold]
"redis_post_notify_stonith_0 controller-2" -> "redis_confirmed-post_notify_stonith_0" [ style = bold]
"redis_post_notify_stonith_0 controller-2" [ style=bold color="green" fontcolor="black"]
"redis_post_notify_stonith_0" -> "redis_confirmed-post_notify_stonith_0" [ style = bold]
"redis_post_notify_stonith_0" -> "redis_post_notify_stonith_0 controller-0" [ style = bold]
"redis_post_notify_stonith_0" -> "redis_post_notify_stonith_0 controller-2" [ style = bold]
"redis_post_notify_stonith_0" [ style=bold color="green" fontcolor="orange"]
"redis_pre_notify_stop_0 controller-0" -> "redis-master_confirmed-pre_notify_stop_0" [ style = bold]
"redis_pre_notify_stop_0 controller-0" [ style=bold color="green" fontcolor="black"]
"redis_pre_notify_stop_0 controller-2" -> "redis-master_confirmed-pre_notify_stop_0" [ style = bold]
"redis_pre_notify_stop_0 controller-2" [ style=bold color="green" fontcolor="black"]
"redis_stop_0 controller-1" -> "redis-master_stopped_0" [ style = bold]
"redis_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"]
"stonith 'reboot' controller-1" -> "haproxy-clone_stop_0" [ style = bold]
"stonith 'reboot' controller-1" -> "haproxy_stop_0 controller-1" [ style = bold]
"stonith 'reboot' controller-1" -> "ip-172.17.1.14_start_0 controller-2" [ style = bold]
"stonith 'reboot' controller-1" -> "ip-172.17.1.14_stop_0 controller-1" [ style = bold]
"stonith 'reboot' controller-1" -> "ip-172.17.1.17_start_0 controller-2" [ style = bold]
"stonith 'reboot' controller-1" -> "ip-172.17.1.17_stop_0 controller-1" [ style = bold]
"stonith 'reboot' controller-1" -> "ip-172.17.4.11_start_0 controller-2" [ style = bold]
"stonith 'reboot' controller-1" -> "ip-172.17.4.11_stop_0 controller-1" [ style = bold]
"stonith 'reboot' controller-1" -> "redis-master_stop_0" [ style = bold]
"stonith 'reboot' controller-1" -> "redis_post_notify_stonith_0" [ style = bold]
"stonith 'reboot' controller-1" -> "redis_stop_0 controller-1" [ style = bold]
"stonith 'reboot' controller-1" -> "stonith 'reboot' messaging-1" [ style = bold]
"stonith 'reboot' controller-1" [ style=bold color="green" fontcolor="black"]
"stonith 'reboot' messaging-1" -> "galera-0_start_0 controller-2" [ style = bold]
"stonith 'reboot' messaging-1" -> "ip-172.17.1.14_start_0 controller-2" [ style = bold]
"stonith 'reboot' messaging-1" -> "ip-172.17.1.17_start_0 controller-2" [ style = bold]
"stonith 'reboot' messaging-1" -> "ip-172.17.4.11_start_0 controller-2" [ style = bold]
"stonith 'reboot' messaging-1" -> "rabbitmq-clone_stop_0" [ style = bold]
"stonith 'reboot' messaging-1" -> "rabbitmq_post_notify_stonith_0" [ style = bold]
"stonith 'reboot' messaging-1" -> "rabbitmq_stop_0 messaging-1" [ style = bold]
"stonith 'reboot' messaging-1" -> "stonith-fence_ipmilan-5254005bdbb5_start_0 controller-2" [ style = bold]
"stonith 'reboot' messaging-1" -> "stonith-fence_ipmilan-525400b4f6bd_start_0 controller-0" [ style = bold]
"stonith 'reboot' messaging-1" -> "stonith-fence_ipmilan-525400bbf613_start_0 controller-0" [ style = bold]
"stonith 'reboot' messaging-1" [ style=bold color="green" fontcolor="black"]
"stonith-fence_ipmilan-5254005bdbb5_monitor_60000 controller-2" [ style=bold color="green" fontcolor="black"]
"stonith-fence_ipmilan-5254005bdbb5_start_0 controller-2" -> "stonith-fence_ipmilan-5254005bdbb5_monitor_60000 controller-2" [ style = bold]
"stonith-fence_ipmilan-5254005bdbb5_start_0 controller-2" [ style=bold color="green" fontcolor="black"]
"stonith-fence_ipmilan-5254005bdbb5_stop_0 controller-1" -> "stonith-fence_ipmilan-5254005bdbb5_start_0 controller-2" [ style = bold]
"stonith-fence_ipmilan-5254005bdbb5_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"]
"stonith-fence_ipmilan-525400b4f6bd_monitor_60000 controller-0" [ style=bold color="green" fontcolor="black"]
"stonith-fence_ipmilan-525400b4f6bd_start_0 controller-0" -> "stonith-fence_ipmilan-525400b4f6bd_monitor_60000 controller-0" [ style = bold]
"stonith-fence_ipmilan-525400b4f6bd_start_0 controller-0" [ style=bold color="green" fontcolor="black"]
"stonith-fence_ipmilan-525400b4f6bd_stop_0 controller-0" -> "stonith-fence_ipmilan-525400b4f6bd_start_0 controller-0" [ style = bold]
"stonith-fence_ipmilan-525400b4f6bd_stop_0 controller-0" [ style=bold color="green" fontcolor="black"]
"stonith-fence_ipmilan-525400bbf613_monitor_60000 controller-0" [ style=bold color="green" fontcolor="black"]
"stonith-fence_ipmilan-525400bbf613_start_0 controller-0" -> "stonith-fence_ipmilan-525400bbf613_monitor_60000 controller-0" [ style = bold]
"stonith-fence_ipmilan-525400bbf613_start_0 controller-0" [ style=bold color="green" fontcolor="black"]
"stonith-fence_ipmilan-525400bbf613_stop_0 controller-0" -> "stonith-fence_ipmilan-525400bbf613_start_0 controller-0" [ style = bold]
"stonith-fence_ipmilan-525400bbf613_stop_0 controller-0" [ style=bold color="green" fontcolor="black"]
}
diff --git a/cts/scheduler/remote-recover-no-resources.exp b/cts/scheduler/remote-recover-no-resources.exp
index 8a67c1144c..90470fbfce 100644
--- a/cts/scheduler/remote-recover-no-resources.exp
+++ b/cts/scheduler/remote-recover-no-resources.exp
@@ -1,684 +1,680 @@
-
-
-
-
-
+
diff --git a/cts/scheduler/remote-recover-no-resources.summary b/cts/scheduler/remote-recover-no-resources.summary
index 89da784570..18a989b4e2 100644
--- a/cts/scheduler/remote-recover-no-resources.summary
+++ b/cts/scheduler/remote-recover-no-resources.summary
@@ -1,143 +1,143 @@
Using the original execution date of: 2017-05-03 13:33:24Z
Current cluster status:
Node controller-1 (2): UNCLEAN (offline)
Online: [ controller-0 controller-2 ]
RemoteOnline: [ galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ]
messaging-0 (ocf::pacemaker:remote): Started controller-0
messaging-1 (ocf::pacemaker:remote): Started controller-1 (UNCLEAN)
messaging-2 (ocf::pacemaker:remote): Started controller-0
galera-0 (ocf::pacemaker:remote): Started controller-1 (UNCLEAN)
galera-1 (ocf::pacemaker:remote): Started controller-0
galera-2 (ocf::pacemaker:remote): Started controller-1 (UNCLEAN)
Clone Set: rabbitmq-clone [rabbitmq]
Started: [ messaging-0 messaging-1 messaging-2 ]
Stopped: [ controller-0 controller-1 controller-2 galera-0 galera-1 galera-2 ]
Clone Set: galera-master [galera] (promotable)
Masters: [ galera-0 galera-1 ]
Stopped: [ controller-0 controller-1 controller-2 galera-2 messaging-0 messaging-1 messaging-2 ]
Clone Set: redis-master [redis] (promotable)
redis (ocf::heartbeat:redis): Slave controller-1 (UNCLEAN)
Masters: [ controller-0 ]
Slaves: [ controller-2 ]
Stopped: [ galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ]
ip-192.168.24.6 (ocf::heartbeat:IPaddr2): Started controller-0
ip-10.0.0.102 (ocf::heartbeat:IPaddr2): Started controller-0
ip-172.17.1.14 (ocf::heartbeat:IPaddr2): Started controller-1 (UNCLEAN)
ip-172.17.1.17 (ocf::heartbeat:IPaddr2): Started controller-1 (UNCLEAN)
ip-172.17.3.15 (ocf::heartbeat:IPaddr2): Started controller-0
ip-172.17.4.11 (ocf::heartbeat:IPaddr2): Started controller-1 (UNCLEAN)
Clone Set: haproxy-clone [haproxy]
haproxy (systemd:haproxy): Started controller-1 (UNCLEAN)
Started: [ controller-0 controller-2 ]
Stopped: [ galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ]
openstack-cinder-volume (systemd:openstack-cinder-volume): Started controller-0
stonith-fence_ipmilan-525400bbf613 (stonith:fence_ipmilan): Started controller-0
stonith-fence_ipmilan-525400b4f6bd (stonith:fence_ipmilan): Started controller-0
stonith-fence_ipmilan-5254005bdbb5 (stonith:fence_ipmilan): Started controller-1 (UNCLEAN)
Transition Summary:
* Fence (reboot) messaging-1 'resources are active and the connection is unrecoverable'
* Fence (reboot) controller-1 'peer is no longer part of the cluster'
* Stop messaging-1 (controller-1) due to node availability
* Move galera-0 ( controller-1 -> controller-2 )
* Stop galera-2 ( controller-1 ) due to node availability
* Stop rabbitmq:2 (messaging-1) due to node availability
* Stop redis:0 ( Slave controller-1 ) due to node availability
* Move ip-172.17.1.14 ( controller-1 -> controller-2 )
* Move ip-172.17.1.17 ( controller-1 -> controller-2 )
* Move ip-172.17.4.11 ( controller-1 -> controller-2 )
* Stop haproxy:0 (controller-1) due to node availability
* Restart stonith-fence_ipmilan-525400bbf613 ( controller-0 ) due to resource definition change
* Restart stonith-fence_ipmilan-525400b4f6bd ( controller-0 ) due to resource definition change
* Move stonith-fence_ipmilan-5254005bdbb5 ( controller-1 -> controller-2 )
Executing cluster transition:
+ * Pseudo action: messaging-1_stop_0
* Pseudo action: galera-0_stop_0
* Pseudo action: galera-2_stop_0
* Pseudo action: redis-master_pre_notify_stop_0
* Resource action: stonith-fence_ipmilan-525400bbf613 stop on controller-0
* Resource action: stonith-fence_ipmilan-525400b4f6bd stop on controller-0
* Pseudo action: stonith-fence_ipmilan-5254005bdbb5_stop_0
* Fencing controller-1 (reboot)
* Pseudo action: redis_post_notify_stop_0
* Resource action: redis notify on controller-0
* Resource action: redis notify on controller-2
* Pseudo action: redis-master_confirmed-pre_notify_stop_0
* Pseudo action: redis-master_stop_0
* Pseudo action: haproxy-clone_stop_0
* Fencing messaging-1 (reboot)
* Resource action: galera-0 start on controller-2
* Pseudo action: rabbitmq_post_notify_stop_0
* Pseudo action: rabbitmq-clone_stop_0
* Resource action: galera monitor=10000 on galera-0
* Pseudo action: redis_stop_0
* Pseudo action: redis-master_stopped_0
* Pseudo action: haproxy_stop_0
* Pseudo action: haproxy-clone_stopped_0
* Resource action: stonith-fence_ipmilan-525400bbf613 start on controller-0
* Resource action: stonith-fence_ipmilan-525400bbf613 monitor=60000 on controller-0
* Resource action: stonith-fence_ipmilan-525400b4f6bd start on controller-0
* Resource action: stonith-fence_ipmilan-525400b4f6bd monitor=60000 on controller-0
* Resource action: stonith-fence_ipmilan-5254005bdbb5 start on controller-2
* Resource action: galera-0 monitor=20000 on controller-2
* Resource action: rabbitmq notify on messaging-2
* Resource action: rabbitmq notify on messaging-0
* Pseudo action: rabbitmq_notified_0
* Pseudo action: rabbitmq_stop_0
* Pseudo action: rabbitmq-clone_stopped_0
* Pseudo action: redis-master_post_notify_stopped_0
* Pseudo action: ip-172.17.1.14_stop_0
* Pseudo action: ip-172.17.1.17_stop_0
* Pseudo action: ip-172.17.4.11_stop_0
* Resource action: stonith-fence_ipmilan-5254005bdbb5 monitor=60000 on controller-2
- * Pseudo action: messaging-1_stop_0
* Resource action: redis notify on controller-0
* Resource action: redis notify on controller-2
* Pseudo action: redis-master_confirmed-post_notify_stopped_0
* Resource action: ip-172.17.1.14 start on controller-2
* Resource action: ip-172.17.1.17 start on controller-2
* Resource action: ip-172.17.4.11 start on controller-2
* Pseudo action: redis_notified_0
* Resource action: ip-172.17.1.14 monitor=10000 on controller-2
* Resource action: ip-172.17.1.17 monitor=10000 on controller-2
* Resource action: ip-172.17.4.11 monitor=10000 on controller-2
Using the original execution date of: 2017-05-03 13:33:24Z
Revised cluster status:
Online: [ controller-0 controller-2 ]
OFFLINE: [ controller-1 ]
RemoteOnline: [ galera-0 galera-1 messaging-0 messaging-2 ]
RemoteOFFLINE: [ galera-2 messaging-1 ]
messaging-0 (ocf::pacemaker:remote): Started controller-0
messaging-1 (ocf::pacemaker:remote): Stopped
messaging-2 (ocf::pacemaker:remote): Started controller-0
galera-0 (ocf::pacemaker:remote): Started controller-2
galera-1 (ocf::pacemaker:remote): Started controller-0
galera-2 (ocf::pacemaker:remote): Stopped
Clone Set: rabbitmq-clone [rabbitmq]
Started: [ messaging-0 messaging-2 ]
Stopped: [ controller-0 controller-1 controller-2 galera-0 galera-1 galera-2 messaging-1 ]
Clone Set: galera-master [galera] (promotable)
Masters: [ galera-0 galera-1 ]
Stopped: [ controller-0 controller-1 controller-2 galera-2 messaging-0 messaging-1 messaging-2 ]
Clone Set: redis-master [redis] (promotable)
Masters: [ controller-0 ]
Slaves: [ controller-2 ]
Stopped: [ controller-1 galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ]
ip-192.168.24.6 (ocf::heartbeat:IPaddr2): Started controller-0
ip-10.0.0.102 (ocf::heartbeat:IPaddr2): Started controller-0
ip-172.17.1.14 (ocf::heartbeat:IPaddr2): Started controller-2
ip-172.17.1.17 (ocf::heartbeat:IPaddr2): Started controller-2
ip-172.17.3.15 (ocf::heartbeat:IPaddr2): Started controller-0
ip-172.17.4.11 (ocf::heartbeat:IPaddr2): Started controller-2
Clone Set: haproxy-clone [haproxy]
Started: [ controller-0 controller-2 ]
Stopped: [ controller-1 galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ]
openstack-cinder-volume (systemd:openstack-cinder-volume): Started controller-0
stonith-fence_ipmilan-525400bbf613 (stonith:fence_ipmilan): Started controller-0
stonith-fence_ipmilan-525400b4f6bd (stonith:fence_ipmilan): Started controller-0
stonith-fence_ipmilan-5254005bdbb5 (stonith:fence_ipmilan): Started controller-2
diff --git a/cts/scheduler/remote-recover-unknown.dot b/cts/scheduler/remote-recover-unknown.dot
index 5cd760bc10..29ab59fd2b 100644
--- a/cts/scheduler/remote-recover-unknown.dot
+++ b/cts/scheduler/remote-recover-unknown.dot
@@ -1,134 +1,133 @@
digraph "g" {
"galera-0_monitor_20000 controller-2" [ style=bold color="green" fontcolor="black"]
"galera-0_start_0 controller-2" -> "galera-0_monitor_20000 controller-2" [ style = bold]
"galera-0_start_0 controller-2" -> "galera_monitor_10000 galera-0" [ style = bold]
"galera-0_start_0 controller-2" [ style=bold color="green" fontcolor="black"]
"galera-0_stop_0 controller-1" -> "galera-0_start_0 controller-2" [ style = bold]
"galera-0_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"]
"galera-2_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"]
"galera_monitor_0 galera-2" [ style=dashed color="red" fontcolor="black"]
"galera_monitor_10000 galera-0" [ style=bold color="green" fontcolor="black"]
"haproxy-clone_stop_0" -> "haproxy-clone_stopped_0" [ style = bold]
"haproxy-clone_stop_0" -> "haproxy_stop_0 controller-1" [ style = bold]
"haproxy-clone_stop_0" [ style=bold color="green" fontcolor="orange"]
"haproxy-clone_stopped_0" -> "ip-172.17.1.14_stop_0 controller-1" [ style = bold]
"haproxy-clone_stopped_0" -> "ip-172.17.1.17_stop_0 controller-1" [ style = bold]
"haproxy-clone_stopped_0" -> "ip-172.17.4.11_stop_0 controller-1" [ style = bold]
"haproxy-clone_stopped_0" [ style=bold color="green" fontcolor="orange"]
"haproxy_stop_0 controller-1" -> "haproxy-clone_stopped_0" [ style = bold]
"haproxy_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"]
"ip-172.17.1.14_monitor_10000 controller-2" [ style=bold color="green" fontcolor="black"]
"ip-172.17.1.14_start_0 controller-2" -> "ip-172.17.1.14_monitor_10000 controller-2" [ style = bold]
"ip-172.17.1.14_start_0 controller-2" [ style=bold color="green" fontcolor="black"]
"ip-172.17.1.14_stop_0 controller-1" -> "ip-172.17.1.14_start_0 controller-2" [ style = bold]
"ip-172.17.1.14_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"]
"ip-172.17.1.17_monitor_10000 controller-2" [ style=bold color="green" fontcolor="black"]
"ip-172.17.1.17_start_0 controller-2" -> "ip-172.17.1.17_monitor_10000 controller-2" [ style = bold]
"ip-172.17.1.17_start_0 controller-2" [ style=bold color="green" fontcolor="black"]
"ip-172.17.1.17_stop_0 controller-1" -> "ip-172.17.1.17_start_0 controller-2" [ style = bold]
"ip-172.17.1.17_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"]
"ip-172.17.4.11_monitor_10000 controller-2" [ style=bold color="green" fontcolor="black"]
"ip-172.17.4.11_start_0 controller-2" -> "ip-172.17.4.11_monitor_10000 controller-2" [ style = bold]
"ip-172.17.4.11_start_0 controller-2" [ style=bold color="green" fontcolor="black"]
"ip-172.17.4.11_stop_0 controller-1" -> "ip-172.17.4.11_start_0 controller-2" [ style = bold]
"ip-172.17.4.11_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"]
"messaging-1_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"]
"rabbitmq-clone_stop_0" -> "rabbitmq-clone_stopped_0" [ style = bold]
"rabbitmq-clone_stop_0" -> "rabbitmq_stop_0 messaging-1" [ style = bold]
"rabbitmq-clone_stop_0" [ style=bold color="green" fontcolor="orange"]
"rabbitmq-clone_stopped_0" [ style=bold color="green" fontcolor="orange"]
"rabbitmq_confirmed-post_notify_stonith_0" [ style=bold color="green" fontcolor="orange"]
"rabbitmq_post_notify_stonith_0 messaging-0" -> "rabbitmq_confirmed-post_notify_stonith_0" [ style = bold]
"rabbitmq_post_notify_stonith_0 messaging-0" [ style=bold color="green" fontcolor="black"]
"rabbitmq_post_notify_stonith_0 messaging-2" -> "rabbitmq_confirmed-post_notify_stonith_0" [ style = bold]
"rabbitmq_post_notify_stonith_0 messaging-2" [ style=bold color="green" fontcolor="black"]
"rabbitmq_post_notify_stonith_0" -> "rabbitmq_confirmed-post_notify_stonith_0" [ style = bold]
"rabbitmq_post_notify_stonith_0" -> "rabbitmq_post_notify_stonith_0 messaging-0" [ style = bold]
"rabbitmq_post_notify_stonith_0" -> "rabbitmq_post_notify_stonith_0 messaging-2" [ style = bold]
"rabbitmq_post_notify_stonith_0" [ style=bold color="green" fontcolor="orange"]
-"rabbitmq_stop_0 messaging-1" -> "messaging-1_stop_0 controller-1" [ style = bold]
"rabbitmq_stop_0 messaging-1" -> "rabbitmq-clone_stopped_0" [ style = bold]
"rabbitmq_stop_0 messaging-1" [ style=bold color="green" fontcolor="orange"]
"redis-master_confirmed-post_notify_stopped_0" [ style=bold color="green" fontcolor="orange"]
"redis-master_confirmed-pre_notify_stop_0" -> "redis-master_post_notify_stopped_0" [ style = bold]
"redis-master_confirmed-pre_notify_stop_0" -> "redis-master_stop_0" [ style = bold]
"redis-master_confirmed-pre_notify_stop_0" [ style=bold color="green" fontcolor="orange"]
"redis-master_post_notify_stopped_0" -> "redis-master_confirmed-post_notify_stopped_0" [ style = bold]
"redis-master_post_notify_stopped_0" -> "redis_post_notify_stonith_0 controller-0" [ style = bold]
"redis-master_post_notify_stopped_0" -> "redis_post_notify_stonith_0 controller-2" [ style = bold]
"redis-master_post_notify_stopped_0" [ style=bold color="green" fontcolor="orange"]
"redis-master_pre_notify_stop_0" -> "redis-master_confirmed-pre_notify_stop_0" [ style = bold]
"redis-master_pre_notify_stop_0" -> "redis_pre_notify_stop_0 controller-0" [ style = bold]
"redis-master_pre_notify_stop_0" -> "redis_pre_notify_stop_0 controller-2" [ style = bold]
"redis-master_pre_notify_stop_0" [ style=bold color="green" fontcolor="orange"]
"redis-master_stop_0" -> "redis-master_stopped_0" [ style = bold]
"redis-master_stop_0" -> "redis_stop_0 controller-1" [ style = bold]
"redis-master_stop_0" [ style=bold color="green" fontcolor="orange"]
"redis-master_stopped_0" -> "redis-master_post_notify_stopped_0" [ style = bold]
"redis-master_stopped_0" [ style=bold color="green" fontcolor="orange"]
"redis_confirmed-post_notify_stonith_0" [ style=bold color="green" fontcolor="orange"]
"redis_post_notify_stonith_0 controller-0" -> "redis-master_confirmed-post_notify_stopped_0" [ style = bold]
"redis_post_notify_stonith_0 controller-0" -> "redis_confirmed-post_notify_stonith_0" [ style = bold]
"redis_post_notify_stonith_0 controller-0" [ style=bold color="green" fontcolor="black"]
"redis_post_notify_stonith_0 controller-2" -> "redis-master_confirmed-post_notify_stopped_0" [ style = bold]
"redis_post_notify_stonith_0 controller-2" -> "redis_confirmed-post_notify_stonith_0" [ style = bold]
"redis_post_notify_stonith_0 controller-2" [ style=bold color="green" fontcolor="black"]
"redis_post_notify_stonith_0" -> "redis_confirmed-post_notify_stonith_0" [ style = bold]
"redis_post_notify_stonith_0" -> "redis_post_notify_stonith_0 controller-0" [ style = bold]
"redis_post_notify_stonith_0" -> "redis_post_notify_stonith_0 controller-2" [ style = bold]
"redis_post_notify_stonith_0" [ style=bold color="green" fontcolor="orange"]
"redis_pre_notify_stop_0 controller-0" -> "redis-master_confirmed-pre_notify_stop_0" [ style = bold]
"redis_pre_notify_stop_0 controller-0" [ style=bold color="green" fontcolor="black"]
"redis_pre_notify_stop_0 controller-2" -> "redis-master_confirmed-pre_notify_stop_0" [ style = bold]
"redis_pre_notify_stop_0 controller-2" [ style=bold color="green" fontcolor="black"]
"redis_stop_0 controller-1" -> "redis-master_stopped_0" [ style = bold]
"redis_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"]
"stonith 'reboot' controller-1" -> "haproxy-clone_stop_0" [ style = bold]
"stonith 'reboot' controller-1" -> "haproxy_stop_0 controller-1" [ style = bold]
"stonith 'reboot' controller-1" -> "ip-172.17.1.14_start_0 controller-2" [ style = bold]
"stonith 'reboot' controller-1" -> "ip-172.17.1.14_stop_0 controller-1" [ style = bold]
"stonith 'reboot' controller-1" -> "ip-172.17.1.17_start_0 controller-2" [ style = bold]
"stonith 'reboot' controller-1" -> "ip-172.17.1.17_stop_0 controller-1" [ style = bold]
"stonith 'reboot' controller-1" -> "ip-172.17.4.11_start_0 controller-2" [ style = bold]
"stonith 'reboot' controller-1" -> "ip-172.17.4.11_stop_0 controller-1" [ style = bold]
"stonith 'reboot' controller-1" -> "redis-master_stop_0" [ style = bold]
"stonith 'reboot' controller-1" -> "redis_post_notify_stonith_0" [ style = bold]
"stonith 'reboot' controller-1" -> "redis_stop_0 controller-1" [ style = bold]
"stonith 'reboot' controller-1" -> "stonith 'reboot' galera-2" [ style = bold]
"stonith 'reboot' controller-1" [ style=bold color="green" fontcolor="black"]
"stonith 'reboot' galera-2" -> "galera-0_start_0 controller-2" [ style = bold]
"stonith 'reboot' galera-2" -> "ip-172.17.1.14_start_0 controller-2" [ style = bold]
"stonith 'reboot' galera-2" -> "ip-172.17.1.17_start_0 controller-2" [ style = bold]
"stonith 'reboot' galera-2" -> "ip-172.17.4.11_start_0 controller-2" [ style = bold]
"stonith 'reboot' galera-2" -> "stonith 'reboot' messaging-1" [ style = bold]
"stonith 'reboot' galera-2" -> "stonith-fence_ipmilan-5254005bdbb5_start_0 controller-2" [ style = bold]
"stonith 'reboot' galera-2" -> "stonith-fence_ipmilan-525400b4f6bd_start_0 controller-0" [ style = bold]
"stonith 'reboot' galera-2" -> "stonith-fence_ipmilan-525400bbf613_start_0 controller-0" [ style = bold]
"stonith 'reboot' galera-2" [ style=bold color="green" fontcolor="black"]
"stonith 'reboot' messaging-1" -> "galera-0_start_0 controller-2" [ style = bold]
"stonith 'reboot' messaging-1" -> "ip-172.17.1.14_start_0 controller-2" [ style = bold]
"stonith 'reboot' messaging-1" -> "ip-172.17.1.17_start_0 controller-2" [ style = bold]
"stonith 'reboot' messaging-1" -> "ip-172.17.4.11_start_0 controller-2" [ style = bold]
"stonith 'reboot' messaging-1" -> "rabbitmq-clone_stop_0" [ style = bold]
"stonith 'reboot' messaging-1" -> "rabbitmq_post_notify_stonith_0" [ style = bold]
"stonith 'reboot' messaging-1" -> "rabbitmq_stop_0 messaging-1" [ style = bold]
"stonith 'reboot' messaging-1" -> "stonith-fence_ipmilan-5254005bdbb5_start_0 controller-2" [ style = bold]
"stonith 'reboot' messaging-1" -> "stonith-fence_ipmilan-525400b4f6bd_start_0 controller-0" [ style = bold]
"stonith 'reboot' messaging-1" -> "stonith-fence_ipmilan-525400bbf613_start_0 controller-0" [ style = bold]
"stonith 'reboot' messaging-1" [ style=bold color="green" fontcolor="black"]
"stonith-fence_ipmilan-5254005bdbb5_monitor_60000 controller-2" [ style=bold color="green" fontcolor="black"]
"stonith-fence_ipmilan-5254005bdbb5_start_0 controller-2" -> "stonith-fence_ipmilan-5254005bdbb5_monitor_60000 controller-2" [ style = bold]
"stonith-fence_ipmilan-5254005bdbb5_start_0 controller-2" [ style=bold color="green" fontcolor="black"]
"stonith-fence_ipmilan-5254005bdbb5_stop_0 controller-1" -> "stonith-fence_ipmilan-5254005bdbb5_start_0 controller-2" [ style = bold]
"stonith-fence_ipmilan-5254005bdbb5_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"]
"stonith-fence_ipmilan-525400b4f6bd_monitor_60000 controller-0" [ style=bold color="green" fontcolor="black"]
"stonith-fence_ipmilan-525400b4f6bd_start_0 controller-0" -> "stonith-fence_ipmilan-525400b4f6bd_monitor_60000 controller-0" [ style = bold]
"stonith-fence_ipmilan-525400b4f6bd_start_0 controller-0" [ style=bold color="green" fontcolor="black"]
"stonith-fence_ipmilan-525400b4f6bd_stop_0 controller-0" -> "stonith-fence_ipmilan-525400b4f6bd_start_0 controller-0" [ style = bold]
"stonith-fence_ipmilan-525400b4f6bd_stop_0 controller-0" [ style=bold color="green" fontcolor="black"]
"stonith-fence_ipmilan-525400bbf613_monitor_60000 controller-0" [ style=bold color="green" fontcolor="black"]
"stonith-fence_ipmilan-525400bbf613_start_0 controller-0" -> "stonith-fence_ipmilan-525400bbf613_monitor_60000 controller-0" [ style = bold]
"stonith-fence_ipmilan-525400bbf613_start_0 controller-0" [ style=bold color="green" fontcolor="black"]
"stonith-fence_ipmilan-525400bbf613_stop_0 controller-0" -> "stonith-fence_ipmilan-525400bbf613_start_0 controller-0" [ style = bold]
"stonith-fence_ipmilan-525400bbf613_stop_0 controller-0" [ style=bold color="green" fontcolor="black"]
}
diff --git a/cts/scheduler/remote-recover-unknown.exp b/cts/scheduler/remote-recover-unknown.exp
index ac6f00486c..82cb65f772 100644
--- a/cts/scheduler/remote-recover-unknown.exp
+++ b/cts/scheduler/remote-recover-unknown.exp
@@ -1,720 +1,716 @@
-
-
-
-
-
+
diff --git a/cts/scheduler/remote-recover-unknown.summary b/cts/scheduler/remote-recover-unknown.summary
index 2c607135cf..4d7a411e70 100644
--- a/cts/scheduler/remote-recover-unknown.summary
+++ b/cts/scheduler/remote-recover-unknown.summary
@@ -1,145 +1,145 @@
Using the original execution date of: 2017-05-03 13:33:24Z
Current cluster status:
Node controller-1 (2): UNCLEAN (offline)
Online: [ controller-0 controller-2 ]
RemoteOnline: [ galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ]
messaging-0 (ocf::pacemaker:remote): Started controller-0
messaging-1 (ocf::pacemaker:remote): Started controller-1 (UNCLEAN)
messaging-2 (ocf::pacemaker:remote): Started controller-0
galera-0 (ocf::pacemaker:remote): Started controller-1 (UNCLEAN)
galera-1 (ocf::pacemaker:remote): Started controller-0
galera-2 (ocf::pacemaker:remote): Started controller-1 (UNCLEAN)
Clone Set: rabbitmq-clone [rabbitmq]
Started: [ messaging-0 messaging-1 messaging-2 ]
Stopped: [ controller-0 controller-1 controller-2 galera-0 galera-1 galera-2 ]
Clone Set: galera-master [galera] (promotable)
Masters: [ galera-0 galera-1 ]
Stopped: [ controller-0 controller-1 controller-2 galera-2 messaging-0 messaging-1 messaging-2 ]
Clone Set: redis-master [redis] (promotable)
redis (ocf::heartbeat:redis): Slave controller-1 (UNCLEAN)
Masters: [ controller-0 ]
Slaves: [ controller-2 ]
Stopped: [ galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ]
ip-192.168.24.6 (ocf::heartbeat:IPaddr2): Started controller-0
ip-10.0.0.102 (ocf::heartbeat:IPaddr2): Started controller-0
ip-172.17.1.14 (ocf::heartbeat:IPaddr2): Started controller-1 (UNCLEAN)
ip-172.17.1.17 (ocf::heartbeat:IPaddr2): Started controller-1 (UNCLEAN)
ip-172.17.3.15 (ocf::heartbeat:IPaddr2): Started controller-0
ip-172.17.4.11 (ocf::heartbeat:IPaddr2): Started controller-1 (UNCLEAN)
Clone Set: haproxy-clone [haproxy]
haproxy (systemd:haproxy): Started controller-1 (UNCLEAN)
Started: [ controller-0 controller-2 ]
Stopped: [ galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ]
openstack-cinder-volume (systemd:openstack-cinder-volume): Started controller-0
stonith-fence_ipmilan-525400bbf613 (stonith:fence_ipmilan): Started controller-0
stonith-fence_ipmilan-525400b4f6bd (stonith:fence_ipmilan): Started controller-0
stonith-fence_ipmilan-5254005bdbb5 (stonith:fence_ipmilan): Started controller-1 (UNCLEAN)
Transition Summary:
* Fence (reboot) galera-2 'resources are in an unknown state and the connection is unrecoverable'
* Fence (reboot) messaging-1 'resources are active and the connection is unrecoverable'
* Fence (reboot) controller-1 'peer is no longer part of the cluster'
* Stop messaging-1 (controller-1) due to node availability
* Move galera-0 ( controller-1 -> controller-2 )
* Stop galera-2 (controller-1) due to node availability
* Stop rabbitmq:2 (messaging-1) due to node availability
* Stop redis:0 ( Slave controller-1 ) due to node availability
* Move ip-172.17.1.14 ( controller-1 -> controller-2 )
* Move ip-172.17.1.17 ( controller-1 -> controller-2 )
* Move ip-172.17.4.11 ( controller-1 -> controller-2 )
* Stop haproxy:0 (controller-1) due to node availability
* Restart stonith-fence_ipmilan-525400bbf613 ( controller-0 ) due to resource definition change
* Restart stonith-fence_ipmilan-525400b4f6bd ( controller-0 ) due to resource definition change
* Move stonith-fence_ipmilan-5254005bdbb5 ( controller-1 -> controller-2 )
Executing cluster transition:
+ * Pseudo action: messaging-1_stop_0
* Pseudo action: galera-0_stop_0
* Pseudo action: galera-2_stop_0
* Pseudo action: redis-master_pre_notify_stop_0
* Resource action: stonith-fence_ipmilan-525400bbf613 stop on controller-0
* Resource action: stonith-fence_ipmilan-525400b4f6bd stop on controller-0
* Pseudo action: stonith-fence_ipmilan-5254005bdbb5_stop_0
* Fencing controller-1 (reboot)
* Pseudo action: redis_post_notify_stop_0
* Resource action: redis notify on controller-0
* Resource action: redis notify on controller-2
* Pseudo action: redis-master_confirmed-pre_notify_stop_0
* Pseudo action: redis-master_stop_0
* Pseudo action: haproxy-clone_stop_0
* Fencing galera-2 (reboot)
* Fencing messaging-1 (reboot)
* Resource action: galera-0 start on controller-2
* Pseudo action: rabbitmq_post_notify_stop_0
* Pseudo action: rabbitmq-clone_stop_0
* Resource action: galera monitor=10000 on galera-0
* Pseudo action: redis_stop_0
* Pseudo action: redis-master_stopped_0
* Pseudo action: haproxy_stop_0
* Pseudo action: haproxy-clone_stopped_0
* Resource action: stonith-fence_ipmilan-525400bbf613 start on controller-0
* Resource action: stonith-fence_ipmilan-525400bbf613 monitor=60000 on controller-0
* Resource action: stonith-fence_ipmilan-525400b4f6bd start on controller-0
* Resource action: stonith-fence_ipmilan-525400b4f6bd monitor=60000 on controller-0
* Resource action: stonith-fence_ipmilan-5254005bdbb5 start on controller-2
* Resource action: galera-0 monitor=20000 on controller-2
* Resource action: rabbitmq notify on messaging-2
* Resource action: rabbitmq notify on messaging-0
* Pseudo action: rabbitmq_notified_0
* Pseudo action: rabbitmq_stop_0
* Pseudo action: rabbitmq-clone_stopped_0
* Pseudo action: redis-master_post_notify_stopped_0
* Pseudo action: ip-172.17.1.14_stop_0
* Pseudo action: ip-172.17.1.17_stop_0
* Pseudo action: ip-172.17.4.11_stop_0
* Resource action: stonith-fence_ipmilan-5254005bdbb5 monitor=60000 on controller-2
- * Pseudo action: messaging-1_stop_0
* Resource action: redis notify on controller-0
* Resource action: redis notify on controller-2
* Pseudo action: redis-master_confirmed-post_notify_stopped_0
* Resource action: ip-172.17.1.14 start on controller-2
* Resource action: ip-172.17.1.17 start on controller-2
* Resource action: ip-172.17.4.11 start on controller-2
* Pseudo action: redis_notified_0
* Resource action: ip-172.17.1.14 monitor=10000 on controller-2
* Resource action: ip-172.17.1.17 monitor=10000 on controller-2
* Resource action: ip-172.17.4.11 monitor=10000 on controller-2
Using the original execution date of: 2017-05-03 13:33:24Z
Revised cluster status:
Online: [ controller-0 controller-2 ]
OFFLINE: [ controller-1 ]
RemoteOnline: [ galera-0 galera-1 messaging-0 messaging-2 ]
RemoteOFFLINE: [ galera-2 messaging-1 ]
messaging-0 (ocf::pacemaker:remote): Started controller-0
messaging-1 (ocf::pacemaker:remote): Stopped
messaging-2 (ocf::pacemaker:remote): Started controller-0
galera-0 (ocf::pacemaker:remote): Started controller-2
galera-1 (ocf::pacemaker:remote): Started controller-0
galera-2 (ocf::pacemaker:remote): Stopped
Clone Set: rabbitmq-clone [rabbitmq]
Started: [ messaging-0 messaging-2 ]
Stopped: [ controller-0 controller-1 controller-2 galera-0 galera-1 galera-2 messaging-1 ]
Clone Set: galera-master [galera] (promotable)
Masters: [ galera-0 galera-1 ]
Stopped: [ controller-0 controller-1 controller-2 galera-2 messaging-0 messaging-1 messaging-2 ]
Clone Set: redis-master [redis] (promotable)
Masters: [ controller-0 ]
Slaves: [ controller-2 ]
Stopped: [ controller-1 galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ]
ip-192.168.24.6 (ocf::heartbeat:IPaddr2): Started controller-0
ip-10.0.0.102 (ocf::heartbeat:IPaddr2): Started controller-0
ip-172.17.1.14 (ocf::heartbeat:IPaddr2): Started controller-2
ip-172.17.1.17 (ocf::heartbeat:IPaddr2): Started controller-2
ip-172.17.3.15 (ocf::heartbeat:IPaddr2): Started controller-0
ip-172.17.4.11 (ocf::heartbeat:IPaddr2): Started controller-2
Clone Set: haproxy-clone [haproxy]
Started: [ controller-0 controller-2 ]
Stopped: [ controller-1 galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ]
openstack-cinder-volume (systemd:openstack-cinder-volume): Started controller-0
stonith-fence_ipmilan-525400bbf613 (stonith:fence_ipmilan): Started controller-0
stonith-fence_ipmilan-525400b4f6bd (stonith:fence_ipmilan): Started controller-0
stonith-fence_ipmilan-5254005bdbb5 (stonith:fence_ipmilan): Started controller-2