diff --git a/cts/scheduler/order-expired-failure.dot b/cts/scheduler/order-expired-failure.dot index 2e9963b9ed..5c21d5d539 100644 --- a/cts/scheduler/order-expired-failure.dot +++ b/cts/scheduler/order-expired-failure.dot @@ -1,25 +1,24 @@ digraph "g" { "compute-unfence-trigger-clone_stop_0" -> "compute-unfence-trigger-clone_stopped_0" [ style = bold] "compute-unfence-trigger-clone_stop_0" -> "compute-unfence-trigger_stop_0 overcloud-novacompute-1" [ style = bold] "compute-unfence-trigger-clone_stop_0" [ style=bold color="green" fontcolor="orange"] "compute-unfence-trigger-clone_stopped_0" [ style=bold color="green" fontcolor="orange"] "compute-unfence-trigger_stop_0 overcloud-novacompute-1" -> "compute-unfence-trigger-clone_stopped_0" [ style = bold] -"compute-unfence-trigger_stop_0 overcloud-novacompute-1" -> "overcloud-novacompute-1_stop_0 controller-1" [ style = bold] "compute-unfence-trigger_stop_0 overcloud-novacompute-1" [ style=bold color="green" fontcolor="orange"] "ip-10.0.0.110_monitor_10000 controller-1" [ style=bold color="green" fontcolor="black"] "ip-10.0.0.110_start_0 controller-1" -> "ip-10.0.0.110_monitor_10000 controller-1" [ style = bold] "ip-10.0.0.110_start_0 controller-1" [ style=bold color="green" fontcolor="black"] "overcloud-novacompute-1_clear_failcount_0 controller-1" [ style=bold color="green" fontcolor="black"] "overcloud-novacompute-1_stop_0 controller-1" [ style=bold color="green" fontcolor="black"] "stonith 'reboot' overcloud-novacompute-1" -> "compute-unfence-trigger-clone_stop_0" [ style = bold] "stonith 'reboot' overcloud-novacompute-1" -> "compute-unfence-trigger_stop_0 overcloud-novacompute-1" [ style = bold] "stonith 'reboot' overcloud-novacompute-1" -> "ip-10.0.0.110_start_0 controller-1" [ style = bold] "stonith 'reboot' overcloud-novacompute-1" -> "overcloud-novacompute-1_clear_failcount_0 controller-1" [ style = bold] "stonith 'reboot' overcloud-novacompute-1" -> "stonith-fence_compute-fence-nova_start_0 controller-2" [ style = bold] "stonith 'reboot' overcloud-novacompute-1" [ style=bold color="green" fontcolor="black"] "stonith-fence_compute-fence-nova_monitor_60000 controller-2" [ style=bold color="green" fontcolor="black"] "stonith-fence_compute-fence-nova_start_0 controller-2" -> "stonith-fence_compute-fence-nova_monitor_60000 controller-2" [ style = bold] "stonith-fence_compute-fence-nova_start_0 controller-2" [ style=bold color="green" fontcolor="black"] "stonith-fence_compute-fence-nova_stop_0 controller-2" -> "stonith-fence_compute-fence-nova_start_0 controller-2" [ style = bold] "stonith-fence_compute-fence-nova_stop_0 controller-2" [ style=bold color="green" fontcolor="black"] } diff --git a/cts/scheduler/order-expired-failure.exp b/cts/scheduler/order-expired-failure.exp index c476bc2ab4..4a504936b2 100644 --- a/cts/scheduler/order-expired-failure.exp +++ b/cts/scheduler/order-expired-failure.exp @@ -1,148 +1,144 @@ - - - - - + diff --git a/cts/scheduler/order-expired-failure.summary b/cts/scheduler/order-expired-failure.summary index c86bb914cd..2cf43edf03 100644 --- a/cts/scheduler/order-expired-failure.summary +++ b/cts/scheduler/order-expired-failure.summary @@ -1,110 +1,110 @@ Using the original execution date of: 2018-04-09 07:55:35Z Current cluster status: RemoteNode overcloud-novacompute-1: UNCLEAN (offline) Online: [ controller-0 controller-1 controller-2 ] RemoteOnline: [ overcloud-novacompute-0 ] GuestOnline: [ galera-bundle-0:galera-bundle-docker-0 galera-bundle-1:galera-bundle-docker-1 galera-bundle-2:galera-bundle-docker-2 rabbitmq-bundle-0:rabbitmq-bundle-docker-0 rabbitmq-bundle-1:rabbitmq-bundle-docker-1 rabbitmq-bundle-2:rabbitmq-bundle-docker-2 redis-bundle-0:redis-bundle-docker-0 redis-bundle-1:redis-bundle-docker-1 redis-bundle-2:redis-bundle-docker-2 ] overcloud-novacompute-0 (ocf::pacemaker:remote): Started controller-0 overcloud-novacompute-1 (ocf::pacemaker:remote): FAILED controller-1 Container bundle set: rabbitmq-bundle [192.168.24.1:8787/rhosp13/openstack-rabbitmq:pcmklatest] rabbitmq-bundle-0 (ocf::heartbeat:rabbitmq-cluster): Started controller-2 rabbitmq-bundle-1 (ocf::heartbeat:rabbitmq-cluster): Started controller-0 rabbitmq-bundle-2 (ocf::heartbeat:rabbitmq-cluster): Started controller-1 Container bundle set: galera-bundle [192.168.24.1:8787/rhosp13/openstack-mariadb:pcmklatest] galera-bundle-0 (ocf::heartbeat:galera): Master controller-2 galera-bundle-1 (ocf::heartbeat:galera): Master controller-0 galera-bundle-2 (ocf::heartbeat:galera): Master controller-1 Container bundle set: redis-bundle [192.168.24.1:8787/rhosp13/openstack-redis:pcmklatest] redis-bundle-0 (ocf::heartbeat:redis): Master controller-2 redis-bundle-1 (ocf::heartbeat:redis): Slave controller-0 redis-bundle-2 (ocf::heartbeat:redis): Slave controller-1 ip-192.168.24.11 (ocf::heartbeat:IPaddr2): Started controller-2 ip-10.0.0.110 (ocf::heartbeat:IPaddr2): Stopped ip-172.17.1.14 (ocf::heartbeat:IPaddr2): Started controller-1 ip-172.17.1.17 (ocf::heartbeat:IPaddr2): Started controller-2 ip-172.17.3.11 (ocf::heartbeat:IPaddr2): Started controller-0 ip-172.17.4.17 (ocf::heartbeat:IPaddr2): Started controller-1 Container bundle set: haproxy-bundle [192.168.24.1:8787/rhosp13/openstack-haproxy:pcmklatest] haproxy-bundle-docker-0 (ocf::heartbeat:docker): Started controller-2 haproxy-bundle-docker-1 (ocf::heartbeat:docker): Started controller-0 haproxy-bundle-docker-2 (ocf::heartbeat:docker): Started controller-1 stonith-fence_compute-fence-nova (stonith:fence_compute): FAILED controller-2 Clone Set: compute-unfence-trigger-clone [compute-unfence-trigger] compute-unfence-trigger (ocf::pacemaker:Dummy): Started overcloud-novacompute-1 (UNCLEAN) Started: [ overcloud-novacompute-0 ] Stopped: [ controller-0 controller-1 controller-2 ] nova-evacuate (ocf::openstack:NovaEvacuate): Started controller-0 stonith-fence_ipmilan-5254008be2cc (stonith:fence_ipmilan): Started controller-1 stonith-fence_ipmilan-525400803f9e (stonith:fence_ipmilan): Started controller-0 stonith-fence_ipmilan-525400fca120 (stonith:fence_ipmilan): Started controller-2 stonith-fence_ipmilan-525400953d48 (stonith:fence_ipmilan): Started controller-2 stonith-fence_ipmilan-525400b02b86 (stonith:fence_ipmilan): Started controller-1 Container bundle: openstack-cinder-volume [192.168.24.1:8787/rhosp13/openstack-cinder-volume:pcmklatest] openstack-cinder-volume-docker-0 (ocf::heartbeat:docker): Started controller-0 Transition Summary: * Fence (reboot) overcloud-novacompute-1 'remote connection is unrecoverable' * Stop overcloud-novacompute-1 ( controller-1 ) due to node availability * Start ip-10.0.0.110 ( controller-1 ) * Recover stonith-fence_compute-fence-nova ( controller-2 ) * Stop compute-unfence-trigger:1 ( overcloud-novacompute-1 ) due to node availability Executing cluster transition: + * Resource action: overcloud-novacompute-1 stop on controller-1 * Resource action: stonith-fence_compute-fence-nova stop on controller-2 * Fencing overcloud-novacompute-1 (reboot) * Cluster action: clear_failcount for overcloud-novacompute-1 on controller-1 * Resource action: ip-10.0.0.110 start on controller-1 * Resource action: stonith-fence_compute-fence-nova start on controller-2 * Resource action: stonith-fence_compute-fence-nova monitor=60000 on controller-2 * Pseudo action: compute-unfence-trigger-clone_stop_0 * Resource action: ip-10.0.0.110 monitor=10000 on controller-1 * Pseudo action: compute-unfence-trigger_stop_0 * Pseudo action: compute-unfence-trigger-clone_stopped_0 - * Resource action: overcloud-novacompute-1 stop on controller-1 Using the original execution date of: 2018-04-09 07:55:35Z Revised cluster status: RemoteNode overcloud-novacompute-1: UNCLEAN (offline) Online: [ controller-0 controller-1 controller-2 ] RemoteOnline: [ overcloud-novacompute-0 ] GuestOnline: [ galera-bundle-0:galera-bundle-docker-0 galera-bundle-1:galera-bundle-docker-1 galera-bundle-2:galera-bundle-docker-2 rabbitmq-bundle-0:rabbitmq-bundle-docker-0 rabbitmq-bundle-1:rabbitmq-bundle-docker-1 rabbitmq-bundle-2:rabbitmq-bundle-docker-2 redis-bundle-0:redis-bundle-docker-0 redis-bundle-1:redis-bundle-docker-1 redis-bundle-2:redis-bundle-docker-2 ] overcloud-novacompute-0 (ocf::pacemaker:remote): Started controller-0 overcloud-novacompute-1 (ocf::pacemaker:remote): FAILED Container bundle set: rabbitmq-bundle [192.168.24.1:8787/rhosp13/openstack-rabbitmq:pcmklatest] rabbitmq-bundle-0 (ocf::heartbeat:rabbitmq-cluster): Started controller-2 rabbitmq-bundle-1 (ocf::heartbeat:rabbitmq-cluster): Started controller-0 rabbitmq-bundle-2 (ocf::heartbeat:rabbitmq-cluster): Started controller-1 Container bundle set: galera-bundle [192.168.24.1:8787/rhosp13/openstack-mariadb:pcmklatest] galera-bundle-0 (ocf::heartbeat:galera): Master controller-2 galera-bundle-1 (ocf::heartbeat:galera): Master controller-0 galera-bundle-2 (ocf::heartbeat:galera): Master controller-1 Container bundle set: redis-bundle [192.168.24.1:8787/rhosp13/openstack-redis:pcmklatest] redis-bundle-0 (ocf::heartbeat:redis): Master controller-2 redis-bundle-1 (ocf::heartbeat:redis): Slave controller-0 redis-bundle-2 (ocf::heartbeat:redis): Slave controller-1 ip-192.168.24.11 (ocf::heartbeat:IPaddr2): Started controller-2 ip-10.0.0.110 (ocf::heartbeat:IPaddr2): Started controller-1 ip-172.17.1.14 (ocf::heartbeat:IPaddr2): Started controller-1 ip-172.17.1.17 (ocf::heartbeat:IPaddr2): Started controller-2 ip-172.17.3.11 (ocf::heartbeat:IPaddr2): Started controller-0 ip-172.17.4.17 (ocf::heartbeat:IPaddr2): Started controller-1 Container bundle set: haproxy-bundle [192.168.24.1:8787/rhosp13/openstack-haproxy:pcmklatest] haproxy-bundle-docker-0 (ocf::heartbeat:docker): Started controller-2 haproxy-bundle-docker-1 (ocf::heartbeat:docker): Started controller-0 haproxy-bundle-docker-2 (ocf::heartbeat:docker): Started controller-1 stonith-fence_compute-fence-nova (stonith:fence_compute): Started controller-2 Clone Set: compute-unfence-trigger-clone [compute-unfence-trigger] Started: [ overcloud-novacompute-0 ] Stopped: [ controller-0 controller-1 controller-2 overcloud-novacompute-1 ] nova-evacuate (ocf::openstack:NovaEvacuate): Started controller-0 stonith-fence_ipmilan-5254008be2cc (stonith:fence_ipmilan): Started controller-1 stonith-fence_ipmilan-525400803f9e (stonith:fence_ipmilan): Started controller-0 stonith-fence_ipmilan-525400fca120 (stonith:fence_ipmilan): Started controller-2 stonith-fence_ipmilan-525400953d48 (stonith:fence_ipmilan): Started controller-2 stonith-fence_ipmilan-525400b02b86 (stonith:fence_ipmilan): Started controller-1 Container bundle: openstack-cinder-volume [192.168.24.1:8787/rhosp13/openstack-cinder-volume:pcmklatest] openstack-cinder-volume-docker-0 (ocf::heartbeat:docker): Started controller-0 diff --git a/cts/scheduler/remote-connection-unrecoverable.dot b/cts/scheduler/remote-connection-unrecoverable.dot index 77284257ce..1017d2bbeb 100644 --- a/cts/scheduler/remote-connection-unrecoverable.dot +++ b/cts/scheduler/remote-connection-unrecoverable.dot @@ -1,46 +1,44 @@ digraph "g" { "killer_monitor_60000 node2" [ style=bold color="green" fontcolor="black"] "killer_start_0 node2" -> "killer_monitor_60000 node2" [ style = bold] "killer_start_0 node2" [ style=bold color="green" fontcolor="black"] "killer_stop_0 node2" -> "killer_start_0 node2" [ style = bold] "killer_stop_0 node2" [ style=bold color="green" fontcolor="black"] "remote1_stop_0 node1" [ style=bold color="green" fontcolor="orange"] "rsc1_delete_0 remote1" -> "rsc1_start_0 node2" [ style = dashed] "rsc1_delete_0 remote1" [ style=dashed color="red" fontcolor="black"] -"rsc1_monitor_0 node2" -> "remote1_stop_0 node1" [ style = bold] "rsc1_monitor_0 node2" -> "rsc1_start_0 node2" [ style = bold] "rsc1_monitor_0 node2" -> "rsc2-master_demote_0" [ style = bold] "rsc1_monitor_0 node2" [ style=bold color="green" fontcolor="black"] "rsc1_monitor_10000 node2" [ style=bold color="green" fontcolor="black"] "rsc1_start_0 node2" -> "rsc1_monitor_10000 node2" [ style = bold] "rsc1_start_0 node2" [ style=bold color="green" fontcolor="black"] -"rsc1_stop_0 remote1" -> "remote1_stop_0 node1" [ style = bold] "rsc1_stop_0 remote1" -> "rsc1_delete_0 remote1" [ style = dashed] "rsc1_stop_0 remote1" -> "rsc1_start_0 node2" [ style = bold] "rsc1_stop_0 remote1" -> "rsc2-master_demote_0" [ style = bold] "rsc1_stop_0 remote1" [ style=bold color="green" fontcolor="orange"] "rsc2-master_demote_0" -> "rsc2-master_demoted_0" [ style = bold] "rsc2-master_demote_0" -> "rsc2_demote_0 node1" [ style = bold] "rsc2-master_demote_0" [ style=bold color="green" fontcolor="orange"] "rsc2-master_demoted_0" -> "rsc2-master_stop_0" [ style = bold] "rsc2-master_demoted_0" [ style=bold color="green" fontcolor="orange"] "rsc2-master_stop_0" -> "rsc2-master_stopped_0" [ style = bold] "rsc2-master_stop_0" -> "rsc2_stop_0 node1" [ style = bold] "rsc2-master_stop_0" [ style=bold color="green" fontcolor="orange"] "rsc2-master_stopped_0" [ style=bold color="green" fontcolor="orange"] "rsc2_demote_0 node1" -> "rsc2-master_demoted_0" [ style = bold] "rsc2_demote_0 node1" -> "rsc2_stop_0 node1" [ style = bold] "rsc2_demote_0 node1" [ style=bold color="green" fontcolor="orange"] "rsc2_stop_0 node1" -> "rsc2-master_stopped_0" [ style = bold] "rsc2_stop_0 node1" [ style=bold color="green" fontcolor="orange"] "stonith 'reboot' node1" -> "rsc1_start_0 node2" [ style = bold] "stonith 'reboot' node1" -> "rsc2-master_stop_0" [ style = bold] "stonith 'reboot' node1" -> "rsc2_demote_0 node1" [ style = bold] "stonith 'reboot' node1" -> "rsc2_stop_0 node1" [ style = bold] "stonith 'reboot' node1" -> "stonith 'reboot' remote1" [ style = bold] "stonith 'reboot' node1" [ style=bold color="green" fontcolor="black"] "stonith 'reboot' remote1" -> "killer_start_0 node2" [ style = bold] "stonith 'reboot' remote1" -> "rsc1_start_0 node2" [ style = bold] "stonith 'reboot' remote1" -> "rsc1_stop_0 remote1" [ style = bold] "stonith 'reboot' remote1" [ style=bold color="green" fontcolor="black"] } diff --git a/cts/scheduler/remote-connection-unrecoverable.exp b/cts/scheduler/remote-connection-unrecoverable.exp index 2c9357b78a..d57c106627 100644 --- a/cts/scheduler/remote-connection-unrecoverable.exp +++ b/cts/scheduler/remote-connection-unrecoverable.exp @@ -1,230 +1,223 @@ - - - - - - - - + diff --git a/cts/scheduler/remote-connection-unrecoverable.summary b/cts/scheduler/remote-connection-unrecoverable.summary index 23fa9ca6b8..caff564ca1 100644 --- a/cts/scheduler/remote-connection-unrecoverable.summary +++ b/cts/scheduler/remote-connection-unrecoverable.summary @@ -1,52 +1,52 @@ Current cluster status: Node node1 (1): UNCLEAN (offline) Online: [ node2 ] RemoteOnline: [ remote1 ] remote1 (ocf::pacemaker:remote): Started node1 (UNCLEAN) killer (stonith:fence_xvm): Started node2 rsc1 (ocf::pacemaker:Dummy): Started remote1 Clone Set: rsc2-master [rsc2] (promotable) rsc2 (ocf::pacemaker:Stateful): Master node1 (UNCLEAN) Masters: [ node2 ] Stopped: [ remote1 ] Transition Summary: * Fence (reboot) remote1 'resources are active and the connection is unrecoverable' * Fence (reboot) node1 'peer is no longer part of the cluster' * Stop remote1 ( node1 ) due to node availability * Restart killer ( node2 ) due to resource definition change * Move rsc1 ( remote1 -> node2 ) * Stop rsc2:0 ( Master node1 ) due to node availability Executing cluster transition: + * Pseudo action: remote1_stop_0 * Resource action: killer stop on node2 * Resource action: rsc1 monitor on node2 * Fencing node1 (reboot) * Fencing remote1 (reboot) * Resource action: killer start on node2 * Resource action: killer monitor=60000 on node2 * Pseudo action: rsc1_stop_0 * Pseudo action: rsc2-master_demote_0 - * Pseudo action: remote1_stop_0 * Resource action: rsc1 start on node2 * Pseudo action: rsc2_demote_0 * Pseudo action: rsc2-master_demoted_0 * Pseudo action: rsc2-master_stop_0 * Resource action: rsc1 monitor=10000 on node2 * Pseudo action: rsc2_stop_0 * Pseudo action: rsc2-master_stopped_0 Revised cluster status: Online: [ node2 ] OFFLINE: [ node1 ] RemoteOFFLINE: [ remote1 ] remote1 (ocf::pacemaker:remote): Stopped killer (stonith:fence_xvm): Started node2 rsc1 (ocf::pacemaker:Dummy): Started node2 Clone Set: rsc2-master [rsc2] (promotable) Masters: [ node2 ] Stopped: [ node1 remote1 ] diff --git a/cts/scheduler/remote-fence-before-reconnect.dot b/cts/scheduler/remote-fence-before-reconnect.dot index 4ced43eb10..5812b7fd5c 100644 --- a/cts/scheduler/remote-fence-before-reconnect.dot +++ b/cts/scheduler/remote-fence-before-reconnect.dot @@ -1,12 +1,11 @@ digraph "g" { "c7auto4_stop_0 c7auto1" [ style=bold color="green" fontcolor="black"] "fake2_monitor_10000 c7auto1" [ style=bold color="green" fontcolor="black"] "fake2_start_0 c7auto1" -> "fake2_monitor_10000 c7auto1" [ style = bold] "fake2_start_0 c7auto1" [ style=bold color="green" fontcolor="black"] -"fake2_stop_0 c7auto4" -> "c7auto4_stop_0 c7auto1" [ style = bold] "fake2_stop_0 c7auto4" -> "fake2_start_0 c7auto1" [ style = bold] "fake2_stop_0 c7auto4" [ style=bold color="green" fontcolor="orange"] "stonith 'reboot' c7auto4" -> "fake2_start_0 c7auto1" [ style = bold] "stonith 'reboot' c7auto4" -> "fake2_stop_0 c7auto4" [ style = bold] "stonith 'reboot' c7auto4" [ style=bold color="green" fontcolor="black"] } diff --git a/cts/scheduler/remote-fence-before-reconnect.exp b/cts/scheduler/remote-fence-before-reconnect.exp index f99d9ef762..f506f85435 100644 --- a/cts/scheduler/remote-fence-before-reconnect.exp +++ b/cts/scheduler/remote-fence-before-reconnect.exp @@ -1,70 +1,66 @@ - - - - - + diff --git a/cts/scheduler/remote-fence-before-reconnect.summary b/cts/scheduler/remote-fence-before-reconnect.summary index f61e18b4ad..03eac203a1 100644 --- a/cts/scheduler/remote-fence-before-reconnect.summary +++ b/cts/scheduler/remote-fence-before-reconnect.summary @@ -1,37 +1,37 @@ Current cluster status: RemoteNode c7auto4: UNCLEAN (offline) Online: [ c7auto1 c7auto2 c7auto3 ] shooter (stonith:fence_phd_kvm): Started c7auto2 c7auto4 (ocf::pacemaker:remote): FAILED c7auto1 fake1 (ocf::heartbeat:Dummy): Started c7auto3 fake2 (ocf::heartbeat:Dummy): Started c7auto4 (UNCLEAN) fake3 (ocf::heartbeat:Dummy): Started c7auto1 fake4 (ocf::heartbeat:Dummy): Started c7auto2 fake5 (ocf::heartbeat:Dummy): Started c7auto3 Transition Summary: * Fence (reboot) c7auto4 'remote connection is unrecoverable' * Stop c7auto4 ( c7auto1 ) due to node availability * Move fake2 ( c7auto4 -> c7auto1 ) Executing cluster transition: + * Resource action: c7auto4 stop on c7auto1 * Fencing c7auto4 (reboot) * Pseudo action: fake2_stop_0 - * Resource action: c7auto4 stop on c7auto1 * Resource action: fake2 start on c7auto1 * Resource action: fake2 monitor=10000 on c7auto1 Revised cluster status: RemoteNode c7auto4: UNCLEAN (offline) Online: [ c7auto1 c7auto2 c7auto3 ] shooter (stonith:fence_phd_kvm): Started c7auto2 c7auto4 (ocf::pacemaker:remote): FAILED fake1 (ocf::heartbeat:Dummy): Started c7auto3 fake2 (ocf::heartbeat:Dummy): Started c7auto1 fake3 (ocf::heartbeat:Dummy): Started c7auto1 fake4 (ocf::heartbeat:Dummy): Started c7auto2 fake5 (ocf::heartbeat:Dummy): Started c7auto3 diff --git a/cts/scheduler/remote-recover-all.dot b/cts/scheduler/remote-recover-all.dot index deed802637..4128b104e1 100644 --- a/cts/scheduler/remote-recover-all.dot +++ b/cts/scheduler/remote-recover-all.dot @@ -1,151 +1,149 @@ digraph "g" { "galera-0_monitor_20000 controller-2" [ style=bold color="green" fontcolor="black"] "galera-0_start_0 controller-2" -> "galera-0_monitor_20000 controller-2" [ style = bold] "galera-0_start_0 controller-2" -> "galera_monitor_10000 galera-0" [ style = bold] "galera-0_start_0 controller-2" [ style=bold color="green" fontcolor="black"] "galera-0_stop_0 controller-1" -> "galera-0_start_0 controller-2" [ style = bold] "galera-0_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"] "galera-2_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"] "galera-master_demote_0" -> "galera-master_demoted_0" [ style = bold] "galera-master_demote_0" -> "galera_demote_0 galera-2" [ style = bold] "galera-master_demote_0" [ style=bold color="green" fontcolor="orange"] "galera-master_demoted_0" -> "galera-master_stop_0" [ style = bold] "galera-master_demoted_0" [ style=bold color="green" fontcolor="orange"] "galera-master_stop_0" -> "galera-master_stopped_0" [ style = bold] "galera-master_stop_0" -> "galera_stop_0 galera-2" [ style = bold] "galera-master_stop_0" [ style=bold color="green" fontcolor="orange"] "galera-master_stopped_0" [ style=bold color="green" fontcolor="orange"] "galera_demote_0 galera-2" -> "galera-master_demoted_0" [ style = bold] "galera_demote_0 galera-2" -> "galera_stop_0 galera-2" [ style = bold] "galera_demote_0 galera-2" [ style=bold color="green" fontcolor="orange"] "galera_monitor_10000 galera-0" [ style=bold color="green" fontcolor="black"] -"galera_stop_0 galera-2" -> "galera-2_stop_0 controller-1" [ style = bold] "galera_stop_0 galera-2" -> "galera-master_stopped_0" [ style = bold] "galera_stop_0 galera-2" [ style=bold color="green" fontcolor="orange"] "haproxy-clone_stop_0" -> "haproxy-clone_stopped_0" [ style = bold] "haproxy-clone_stop_0" -> "haproxy_stop_0 controller-1" [ style = bold] "haproxy-clone_stop_0" [ style=bold color="green" fontcolor="orange"] "haproxy-clone_stopped_0" -> "ip-172.17.1.14_stop_0 controller-1" [ style = bold] "haproxy-clone_stopped_0" -> "ip-172.17.1.17_stop_0 controller-1" [ style = bold] "haproxy-clone_stopped_0" -> "ip-172.17.4.11_stop_0 controller-1" [ style = bold] "haproxy-clone_stopped_0" [ style=bold color="green" fontcolor="orange"] "haproxy_stop_0 controller-1" -> "haproxy-clone_stopped_0" [ style = bold] "haproxy_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"] "ip-172.17.1.14_monitor_10000 controller-2" [ style=bold color="green" fontcolor="black"] "ip-172.17.1.14_start_0 controller-2" -> "ip-172.17.1.14_monitor_10000 controller-2" [ style = bold] "ip-172.17.1.14_start_0 controller-2" [ style=bold color="green" fontcolor="black"] "ip-172.17.1.14_stop_0 controller-1" -> "ip-172.17.1.14_start_0 controller-2" [ style = bold] "ip-172.17.1.14_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"] "ip-172.17.1.17_monitor_10000 controller-2" [ style=bold color="green" fontcolor="black"] "ip-172.17.1.17_start_0 controller-2" -> "ip-172.17.1.17_monitor_10000 controller-2" [ style = bold] "ip-172.17.1.17_start_0 controller-2" [ style=bold color="green" fontcolor="black"] "ip-172.17.1.17_stop_0 controller-1" -> "ip-172.17.1.17_start_0 controller-2" [ style = bold] "ip-172.17.1.17_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"] "ip-172.17.4.11_monitor_10000 controller-2" [ style=bold color="green" fontcolor="black"] "ip-172.17.4.11_start_0 controller-2" -> "ip-172.17.4.11_monitor_10000 controller-2" [ style = bold] "ip-172.17.4.11_start_0 controller-2" [ style=bold color="green" fontcolor="black"] "ip-172.17.4.11_stop_0 controller-1" -> "ip-172.17.4.11_start_0 controller-2" [ style = bold] "ip-172.17.4.11_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"] "messaging-1_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"] "rabbitmq-clone_stop_0" -> "rabbitmq-clone_stopped_0" [ style = bold] "rabbitmq-clone_stop_0" -> "rabbitmq_stop_0 messaging-1" [ style = bold] "rabbitmq-clone_stop_0" [ style=bold color="green" fontcolor="orange"] "rabbitmq-clone_stopped_0" [ style=bold color="green" fontcolor="orange"] "rabbitmq_confirmed-post_notify_stonith_0" [ style=bold color="green" fontcolor="orange"] "rabbitmq_post_notify_stonith_0 messaging-0" -> "rabbitmq_confirmed-post_notify_stonith_0" [ style = bold] "rabbitmq_post_notify_stonith_0 messaging-0" [ style=bold color="green" fontcolor="black"] "rabbitmq_post_notify_stonith_0 messaging-2" -> "rabbitmq_confirmed-post_notify_stonith_0" [ style = bold] "rabbitmq_post_notify_stonith_0 messaging-2" [ style=bold color="green" fontcolor="black"] "rabbitmq_post_notify_stonith_0" -> "rabbitmq_confirmed-post_notify_stonith_0" [ style = bold] "rabbitmq_post_notify_stonith_0" -> "rabbitmq_post_notify_stonith_0 messaging-0" [ style = bold] "rabbitmq_post_notify_stonith_0" -> "rabbitmq_post_notify_stonith_0 messaging-2" [ style = bold] "rabbitmq_post_notify_stonith_0" [ style=bold color="green" fontcolor="orange"] -"rabbitmq_stop_0 messaging-1" -> "messaging-1_stop_0 controller-1" [ style = bold] "rabbitmq_stop_0 messaging-1" -> "rabbitmq-clone_stopped_0" [ style = bold] "rabbitmq_stop_0 messaging-1" [ style=bold color="green" fontcolor="orange"] "redis-master_confirmed-post_notify_stopped_0" [ style=bold color="green" fontcolor="orange"] "redis-master_confirmed-pre_notify_stop_0" -> "redis-master_post_notify_stopped_0" [ style = bold] "redis-master_confirmed-pre_notify_stop_0" -> "redis-master_stop_0" [ style = bold] "redis-master_confirmed-pre_notify_stop_0" [ style=bold color="green" fontcolor="orange"] "redis-master_post_notify_stopped_0" -> "redis-master_confirmed-post_notify_stopped_0" [ style = bold] "redis-master_post_notify_stopped_0" -> "redis_post_notify_stonith_0 controller-0" [ style = bold] "redis-master_post_notify_stopped_0" -> "redis_post_notify_stonith_0 controller-2" [ style = bold] "redis-master_post_notify_stopped_0" [ style=bold color="green" fontcolor="orange"] "redis-master_pre_notify_stop_0" -> "redis-master_confirmed-pre_notify_stop_0" [ style = bold] "redis-master_pre_notify_stop_0" -> "redis_pre_notify_stop_0 controller-0" [ style = bold] "redis-master_pre_notify_stop_0" -> "redis_pre_notify_stop_0 controller-2" [ style = bold] "redis-master_pre_notify_stop_0" [ style=bold color="green" fontcolor="orange"] "redis-master_stop_0" -> "redis-master_stopped_0" [ style = bold] "redis-master_stop_0" -> "redis_stop_0 controller-1" [ style = bold] "redis-master_stop_0" [ style=bold color="green" fontcolor="orange"] "redis-master_stopped_0" -> "redis-master_post_notify_stopped_0" [ style = bold] "redis-master_stopped_0" [ style=bold color="green" fontcolor="orange"] "redis_confirmed-post_notify_stonith_0" [ style=bold color="green" fontcolor="orange"] "redis_post_notify_stonith_0 controller-0" -> "redis-master_confirmed-post_notify_stopped_0" [ style = bold] "redis_post_notify_stonith_0 controller-0" -> "redis_confirmed-post_notify_stonith_0" [ style = bold] "redis_post_notify_stonith_0 controller-0" [ style=bold color="green" fontcolor="black"] "redis_post_notify_stonith_0 controller-2" -> "redis-master_confirmed-post_notify_stopped_0" [ style = bold] "redis_post_notify_stonith_0 controller-2" -> "redis_confirmed-post_notify_stonith_0" [ style = bold] "redis_post_notify_stonith_0 controller-2" [ style=bold color="green" fontcolor="black"] "redis_post_notify_stonith_0" -> "redis_confirmed-post_notify_stonith_0" [ style = bold] "redis_post_notify_stonith_0" -> "redis_post_notify_stonith_0 controller-0" [ style = bold] "redis_post_notify_stonith_0" -> "redis_post_notify_stonith_0 controller-2" [ style = bold] "redis_post_notify_stonith_0" [ style=bold color="green" fontcolor="orange"] "redis_pre_notify_stop_0 controller-0" -> "redis-master_confirmed-pre_notify_stop_0" [ style = bold] "redis_pre_notify_stop_0 controller-0" [ style=bold color="green" fontcolor="black"] "redis_pre_notify_stop_0 controller-2" -> "redis-master_confirmed-pre_notify_stop_0" [ style = bold] "redis_pre_notify_stop_0 controller-2" [ style=bold color="green" fontcolor="black"] "redis_stop_0 controller-1" -> "redis-master_stopped_0" [ style = bold] "redis_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"] "stonith 'reboot' controller-1" -> "haproxy-clone_stop_0" [ style = bold] "stonith 'reboot' controller-1" -> "haproxy_stop_0 controller-1" [ style = bold] "stonith 'reboot' controller-1" -> "ip-172.17.1.14_start_0 controller-2" [ style = bold] "stonith 'reboot' controller-1" -> "ip-172.17.1.14_stop_0 controller-1" [ style = bold] "stonith 'reboot' controller-1" -> "ip-172.17.1.17_start_0 controller-2" [ style = bold] "stonith 'reboot' controller-1" -> "ip-172.17.1.17_stop_0 controller-1" [ style = bold] "stonith 'reboot' controller-1" -> "ip-172.17.4.11_start_0 controller-2" [ style = bold] "stonith 'reboot' controller-1" -> "ip-172.17.4.11_stop_0 controller-1" [ style = bold] "stonith 'reboot' controller-1" -> "redis-master_stop_0" [ style = bold] "stonith 'reboot' controller-1" -> "redis_post_notify_stonith_0" [ style = bold] "stonith 'reboot' controller-1" -> "redis_stop_0 controller-1" [ style = bold] "stonith 'reboot' controller-1" -> "stonith 'reboot' galera-2" [ style = bold] "stonith 'reboot' controller-1" [ style=bold color="green" fontcolor="black"] "stonith 'reboot' galera-2" -> "galera-0_start_0 controller-2" [ style = bold] "stonith 'reboot' galera-2" -> "galera-master_stop_0" [ style = bold] "stonith 'reboot' galera-2" -> "galera_demote_0 galera-2" [ style = bold] "stonith 'reboot' galera-2" -> "galera_stop_0 galera-2" [ style = bold] "stonith 'reboot' galera-2" -> "ip-172.17.1.14_start_0 controller-2" [ style = bold] "stonith 'reboot' galera-2" -> "ip-172.17.1.17_start_0 controller-2" [ style = bold] "stonith 'reboot' galera-2" -> "ip-172.17.4.11_start_0 controller-2" [ style = bold] "stonith 'reboot' galera-2" -> "stonith 'reboot' messaging-1" [ style = bold] "stonith 'reboot' galera-2" -> "stonith-fence_ipmilan-5254005bdbb5_start_0 controller-2" [ style = bold] "stonith 'reboot' galera-2" -> "stonith-fence_ipmilan-525400b4f6bd_start_0 controller-0" [ style = bold] "stonith 'reboot' galera-2" -> "stonith-fence_ipmilan-525400bbf613_start_0 controller-0" [ style = bold] "stonith 'reboot' galera-2" [ style=bold color="green" fontcolor="black"] "stonith 'reboot' messaging-1" -> "galera-0_start_0 controller-2" [ style = bold] "stonith 'reboot' messaging-1" -> "ip-172.17.1.14_start_0 controller-2" [ style = bold] "stonith 'reboot' messaging-1" -> "ip-172.17.1.17_start_0 controller-2" [ style = bold] "stonith 'reboot' messaging-1" -> "ip-172.17.4.11_start_0 controller-2" [ style = bold] "stonith 'reboot' messaging-1" -> "rabbitmq-clone_stop_0" [ style = bold] "stonith 'reboot' messaging-1" -> "rabbitmq_post_notify_stonith_0" [ style = bold] "stonith 'reboot' messaging-1" -> "rabbitmq_stop_0 messaging-1" [ style = bold] "stonith 'reboot' messaging-1" -> "stonith-fence_ipmilan-5254005bdbb5_start_0 controller-2" [ style = bold] "stonith 'reboot' messaging-1" -> "stonith-fence_ipmilan-525400b4f6bd_start_0 controller-0" [ style = bold] "stonith 'reboot' messaging-1" -> "stonith-fence_ipmilan-525400bbf613_start_0 controller-0" [ style = bold] "stonith 'reboot' messaging-1" [ style=bold color="green" fontcolor="black"] "stonith-fence_ipmilan-5254005bdbb5_monitor_60000 controller-2" [ style=bold color="green" fontcolor="black"] "stonith-fence_ipmilan-5254005bdbb5_start_0 controller-2" -> "stonith-fence_ipmilan-5254005bdbb5_monitor_60000 controller-2" [ style = bold] "stonith-fence_ipmilan-5254005bdbb5_start_0 controller-2" [ style=bold color="green" fontcolor="black"] "stonith-fence_ipmilan-5254005bdbb5_stop_0 controller-1" -> "stonith-fence_ipmilan-5254005bdbb5_start_0 controller-2" [ style = bold] "stonith-fence_ipmilan-5254005bdbb5_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"] "stonith-fence_ipmilan-525400b4f6bd_monitor_60000 controller-0" [ style=bold color="green" fontcolor="black"] "stonith-fence_ipmilan-525400b4f6bd_start_0 controller-0" -> "stonith-fence_ipmilan-525400b4f6bd_monitor_60000 controller-0" [ style = bold] "stonith-fence_ipmilan-525400b4f6bd_start_0 controller-0" [ style=bold color="green" fontcolor="black"] "stonith-fence_ipmilan-525400b4f6bd_stop_0 controller-0" -> "stonith-fence_ipmilan-525400b4f6bd_start_0 controller-0" [ style = bold] "stonith-fence_ipmilan-525400b4f6bd_stop_0 controller-0" [ style=bold color="green" fontcolor="black"] "stonith-fence_ipmilan-525400bbf613_monitor_60000 controller-0" [ style=bold color="green" fontcolor="black"] "stonith-fence_ipmilan-525400bbf613_start_0 controller-0" -> "stonith-fence_ipmilan-525400bbf613_monitor_60000 controller-0" [ style = bold] "stonith-fence_ipmilan-525400bbf613_start_0 controller-0" [ style=bold color="green" fontcolor="black"] "stonith-fence_ipmilan-525400bbf613_stop_0 controller-0" -> "stonith-fence_ipmilan-525400bbf613_start_0 controller-0" [ style = bold] "stonith-fence_ipmilan-525400bbf613_stop_0 controller-0" [ style=bold color="green" fontcolor="black"] } diff --git a/cts/scheduler/remote-recover-all.exp b/cts/scheduler/remote-recover-all.exp index 8137ffba08..0cb51f693c 100644 --- a/cts/scheduler/remote-recover-all.exp +++ b/cts/scheduler/remote-recover-all.exp @@ -1,810 +1,802 @@ - - - - - + - - - - - + diff --git a/cts/scheduler/remote-recover-all.summary b/cts/scheduler/remote-recover-all.summary index 2ac0c6a0cf..d095fddee0 100644 --- a/cts/scheduler/remote-recover-all.summary +++ b/cts/scheduler/remote-recover-all.summary @@ -1,152 +1,152 @@ Using the original execution date of: 2017-05-03 13:33:24Z Current cluster status: Node controller-1 (2): UNCLEAN (offline) Online: [ controller-0 controller-2 ] RemoteOnline: [ galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ] messaging-0 (ocf::pacemaker:remote): Started controller-0 messaging-1 (ocf::pacemaker:remote): Started controller-1 (UNCLEAN) messaging-2 (ocf::pacemaker:remote): Started controller-0 galera-0 (ocf::pacemaker:remote): Started controller-1 (UNCLEAN) galera-1 (ocf::pacemaker:remote): Started controller-0 galera-2 (ocf::pacemaker:remote): Started controller-1 (UNCLEAN) Clone Set: rabbitmq-clone [rabbitmq] Started: [ messaging-0 messaging-1 messaging-2 ] Stopped: [ controller-0 controller-1 controller-2 galera-0 galera-1 galera-2 ] Clone Set: galera-master [galera] (promotable) Masters: [ galera-0 galera-1 galera-2 ] Stopped: [ controller-0 controller-1 controller-2 messaging-0 messaging-1 messaging-2 ] Clone Set: redis-master [redis] (promotable) redis (ocf::heartbeat:redis): Slave controller-1 (UNCLEAN) Masters: [ controller-0 ] Slaves: [ controller-2 ] Stopped: [ galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ] ip-192.168.24.6 (ocf::heartbeat:IPaddr2): Started controller-0 ip-10.0.0.102 (ocf::heartbeat:IPaddr2): Started controller-0 ip-172.17.1.14 (ocf::heartbeat:IPaddr2): Started controller-1 (UNCLEAN) ip-172.17.1.17 (ocf::heartbeat:IPaddr2): Started controller-1 (UNCLEAN) ip-172.17.3.15 (ocf::heartbeat:IPaddr2): Started controller-0 ip-172.17.4.11 (ocf::heartbeat:IPaddr2): Started controller-1 (UNCLEAN) Clone Set: haproxy-clone [haproxy] haproxy (systemd:haproxy): Started controller-1 (UNCLEAN) Started: [ controller-0 controller-2 ] Stopped: [ galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ] openstack-cinder-volume (systemd:openstack-cinder-volume): Started controller-0 stonith-fence_ipmilan-525400bbf613 (stonith:fence_ipmilan): Started controller-0 stonith-fence_ipmilan-525400b4f6bd (stonith:fence_ipmilan): Started controller-0 stonith-fence_ipmilan-5254005bdbb5 (stonith:fence_ipmilan): Started controller-1 (UNCLEAN) Transition Summary: * Fence (reboot) messaging-1 'resources are active and the connection is unrecoverable' * Fence (reboot) galera-2 'resources are active and the connection is unrecoverable' * Fence (reboot) controller-1 'peer is no longer part of the cluster' * Stop messaging-1 (controller-1) due to node availability * Move galera-0 ( controller-1 -> controller-2 ) * Stop galera-2 (controller-1) due to node availability * Stop rabbitmq:2 (messaging-1) due to node availability * Stop galera:1 ( Master galera-2 ) due to node availability * Stop redis:0 ( Slave controller-1 ) due to node availability * Move ip-172.17.1.14 ( controller-1 -> controller-2 ) * Move ip-172.17.1.17 ( controller-1 -> controller-2 ) * Move ip-172.17.4.11 ( controller-1 -> controller-2 ) * Stop haproxy:0 (controller-1) due to node availability * Restart stonith-fence_ipmilan-525400bbf613 ( controller-0 ) due to resource definition change * Restart stonith-fence_ipmilan-525400b4f6bd ( controller-0 ) due to resource definition change * Move stonith-fence_ipmilan-5254005bdbb5 ( controller-1 -> controller-2 ) Executing cluster transition: + * Pseudo action: messaging-1_stop_0 * Pseudo action: galera-0_stop_0 + * Pseudo action: galera-2_stop_0 * Pseudo action: galera-master_demote_0 * Pseudo action: redis-master_pre_notify_stop_0 * Resource action: stonith-fence_ipmilan-525400bbf613 stop on controller-0 * Resource action: stonith-fence_ipmilan-525400b4f6bd stop on controller-0 * Pseudo action: stonith-fence_ipmilan-5254005bdbb5_stop_0 * Fencing controller-1 (reboot) * Pseudo action: redis_post_notify_stop_0 * Resource action: redis notify on controller-0 * Resource action: redis notify on controller-2 * Pseudo action: redis-master_confirmed-pre_notify_stop_0 * Pseudo action: redis-master_stop_0 * Pseudo action: haproxy-clone_stop_0 * Fencing galera-2 (reboot) * Pseudo action: galera_demote_0 * Pseudo action: galera-master_demoted_0 * Pseudo action: galera-master_stop_0 * Pseudo action: redis_stop_0 * Pseudo action: redis-master_stopped_0 * Pseudo action: haproxy_stop_0 * Pseudo action: haproxy-clone_stopped_0 * Fencing messaging-1 (reboot) * Resource action: galera-0 start on controller-2 * Pseudo action: rabbitmq_post_notify_stop_0 * Pseudo action: rabbitmq-clone_stop_0 * Pseudo action: galera_stop_0 * Resource action: galera monitor=10000 on galera-0 * Pseudo action: galera-master_stopped_0 * Pseudo action: redis-master_post_notify_stopped_0 * Pseudo action: ip-172.17.1.14_stop_0 * Pseudo action: ip-172.17.1.17_stop_0 * Pseudo action: ip-172.17.4.11_stop_0 * Resource action: stonith-fence_ipmilan-525400bbf613 start on controller-0 * Resource action: stonith-fence_ipmilan-525400bbf613 monitor=60000 on controller-0 * Resource action: stonith-fence_ipmilan-525400b4f6bd start on controller-0 * Resource action: stonith-fence_ipmilan-525400b4f6bd monitor=60000 on controller-0 * Resource action: stonith-fence_ipmilan-5254005bdbb5 start on controller-2 * Resource action: galera-0 monitor=20000 on controller-2 - * Pseudo action: galera-2_stop_0 * Resource action: rabbitmq notify on messaging-2 * Resource action: rabbitmq notify on messaging-0 * Pseudo action: rabbitmq_notified_0 * Pseudo action: rabbitmq_stop_0 * Pseudo action: rabbitmq-clone_stopped_0 * Resource action: redis notify on controller-0 * Resource action: redis notify on controller-2 * Pseudo action: redis-master_confirmed-post_notify_stopped_0 * Resource action: ip-172.17.1.14 start on controller-2 * Resource action: ip-172.17.1.17 start on controller-2 * Resource action: ip-172.17.4.11 start on controller-2 * Resource action: stonith-fence_ipmilan-5254005bdbb5 monitor=60000 on controller-2 - * Pseudo action: messaging-1_stop_0 * Pseudo action: redis_notified_0 * Resource action: ip-172.17.1.14 monitor=10000 on controller-2 * Resource action: ip-172.17.1.17 monitor=10000 on controller-2 * Resource action: ip-172.17.4.11 monitor=10000 on controller-2 Using the original execution date of: 2017-05-03 13:33:24Z Revised cluster status: Online: [ controller-0 controller-2 ] OFFLINE: [ controller-1 ] RemoteOnline: [ galera-0 galera-1 messaging-0 messaging-2 ] RemoteOFFLINE: [ galera-2 messaging-1 ] messaging-0 (ocf::pacemaker:remote): Started controller-0 messaging-1 (ocf::pacemaker:remote): Stopped messaging-2 (ocf::pacemaker:remote): Started controller-0 galera-0 (ocf::pacemaker:remote): Started controller-2 galera-1 (ocf::pacemaker:remote): Started controller-0 galera-2 (ocf::pacemaker:remote): Stopped Clone Set: rabbitmq-clone [rabbitmq] Started: [ messaging-0 messaging-2 ] Stopped: [ controller-0 controller-1 controller-2 galera-0 galera-1 galera-2 messaging-1 ] Clone Set: galera-master [galera] (promotable) Masters: [ galera-0 galera-1 ] Stopped: [ controller-0 controller-1 controller-2 galera-2 messaging-0 messaging-1 messaging-2 ] Clone Set: redis-master [redis] (promotable) Masters: [ controller-0 ] Slaves: [ controller-2 ] Stopped: [ controller-1 galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ] ip-192.168.24.6 (ocf::heartbeat:IPaddr2): Started controller-0 ip-10.0.0.102 (ocf::heartbeat:IPaddr2): Started controller-0 ip-172.17.1.14 (ocf::heartbeat:IPaddr2): Started controller-2 ip-172.17.1.17 (ocf::heartbeat:IPaddr2): Started controller-2 ip-172.17.3.15 (ocf::heartbeat:IPaddr2): Started controller-0 ip-172.17.4.11 (ocf::heartbeat:IPaddr2): Started controller-2 Clone Set: haproxy-clone [haproxy] Started: [ controller-0 controller-2 ] Stopped: [ controller-1 galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ] openstack-cinder-volume (systemd:openstack-cinder-volume): Started controller-0 stonith-fence_ipmilan-525400bbf613 (stonith:fence_ipmilan): Started controller-0 stonith-fence_ipmilan-525400b4f6bd (stonith:fence_ipmilan): Started controller-0 stonith-fence_ipmilan-5254005bdbb5 (stonith:fence_ipmilan): Started controller-2 diff --git a/cts/scheduler/remote-recover-no-resources.dot b/cts/scheduler/remote-recover-no-resources.dot index ef78aa6e43..a2f8ce00d1 100644 --- a/cts/scheduler/remote-recover-no-resources.dot +++ b/cts/scheduler/remote-recover-no-resources.dot @@ -1,124 +1,123 @@ digraph "g" { "galera-0_monitor_20000 controller-2" [ style=bold color="green" fontcolor="black"] "galera-0_start_0 controller-2" -> "galera-0_monitor_20000 controller-2" [ style = bold] "galera-0_start_0 controller-2" -> "galera_monitor_10000 galera-0" [ style = bold] "galera-0_start_0 controller-2" [ style=bold color="green" fontcolor="black"] "galera-0_stop_0 controller-1" -> "galera-0_start_0 controller-2" [ style = bold] "galera-0_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"] "galera-2_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"] "galera_monitor_10000 galera-0" [ style=bold color="green" fontcolor="black"] "haproxy-clone_stop_0" -> "haproxy-clone_stopped_0" [ style = bold] "haproxy-clone_stop_0" -> "haproxy_stop_0 controller-1" [ style = bold] "haproxy-clone_stop_0" [ style=bold color="green" fontcolor="orange"] "haproxy-clone_stopped_0" -> "ip-172.17.1.14_stop_0 controller-1" [ style = bold] "haproxy-clone_stopped_0" -> "ip-172.17.1.17_stop_0 controller-1" [ style = bold] "haproxy-clone_stopped_0" -> "ip-172.17.4.11_stop_0 controller-1" [ style = bold] "haproxy-clone_stopped_0" [ style=bold color="green" fontcolor="orange"] "haproxy_stop_0 controller-1" -> "haproxy-clone_stopped_0" [ style = bold] "haproxy_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"] "ip-172.17.1.14_monitor_10000 controller-2" [ style=bold color="green" fontcolor="black"] "ip-172.17.1.14_start_0 controller-2" -> "ip-172.17.1.14_monitor_10000 controller-2" [ style = bold] "ip-172.17.1.14_start_0 controller-2" [ style=bold color="green" fontcolor="black"] "ip-172.17.1.14_stop_0 controller-1" -> "ip-172.17.1.14_start_0 controller-2" [ style = bold] "ip-172.17.1.14_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"] "ip-172.17.1.17_monitor_10000 controller-2" [ style=bold color="green" fontcolor="black"] "ip-172.17.1.17_start_0 controller-2" -> "ip-172.17.1.17_monitor_10000 controller-2" [ style = bold] "ip-172.17.1.17_start_0 controller-2" [ style=bold color="green" fontcolor="black"] "ip-172.17.1.17_stop_0 controller-1" -> "ip-172.17.1.17_start_0 controller-2" [ style = bold] "ip-172.17.1.17_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"] "ip-172.17.4.11_monitor_10000 controller-2" [ style=bold color="green" fontcolor="black"] "ip-172.17.4.11_start_0 controller-2" -> "ip-172.17.4.11_monitor_10000 controller-2" [ style = bold] "ip-172.17.4.11_start_0 controller-2" [ style=bold color="green" fontcolor="black"] "ip-172.17.4.11_stop_0 controller-1" -> "ip-172.17.4.11_start_0 controller-2" [ style = bold] "ip-172.17.4.11_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"] "messaging-1_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"] "rabbitmq-clone_stop_0" -> "rabbitmq-clone_stopped_0" [ style = bold] "rabbitmq-clone_stop_0" -> "rabbitmq_stop_0 messaging-1" [ style = bold] "rabbitmq-clone_stop_0" [ style=bold color="green" fontcolor="orange"] "rabbitmq-clone_stopped_0" [ style=bold color="green" fontcolor="orange"] "rabbitmq_confirmed-post_notify_stonith_0" [ style=bold color="green" fontcolor="orange"] "rabbitmq_post_notify_stonith_0 messaging-0" -> "rabbitmq_confirmed-post_notify_stonith_0" [ style = bold] "rabbitmq_post_notify_stonith_0 messaging-0" [ style=bold color="green" fontcolor="black"] "rabbitmq_post_notify_stonith_0 messaging-2" -> "rabbitmq_confirmed-post_notify_stonith_0" [ style = bold] "rabbitmq_post_notify_stonith_0 messaging-2" [ style=bold color="green" fontcolor="black"] "rabbitmq_post_notify_stonith_0" -> "rabbitmq_confirmed-post_notify_stonith_0" [ style = bold] "rabbitmq_post_notify_stonith_0" -> "rabbitmq_post_notify_stonith_0 messaging-0" [ style = bold] "rabbitmq_post_notify_stonith_0" -> "rabbitmq_post_notify_stonith_0 messaging-2" [ style = bold] "rabbitmq_post_notify_stonith_0" [ style=bold color="green" fontcolor="orange"] -"rabbitmq_stop_0 messaging-1" -> "messaging-1_stop_0 controller-1" [ style = bold] "rabbitmq_stop_0 messaging-1" -> "rabbitmq-clone_stopped_0" [ style = bold] "rabbitmq_stop_0 messaging-1" [ style=bold color="green" fontcolor="orange"] "redis-master_confirmed-post_notify_stopped_0" [ style=bold color="green" fontcolor="orange"] "redis-master_confirmed-pre_notify_stop_0" -> "redis-master_post_notify_stopped_0" [ style = bold] "redis-master_confirmed-pre_notify_stop_0" -> "redis-master_stop_0" [ style = bold] "redis-master_confirmed-pre_notify_stop_0" [ style=bold color="green" fontcolor="orange"] "redis-master_post_notify_stopped_0" -> "redis-master_confirmed-post_notify_stopped_0" [ style = bold] "redis-master_post_notify_stopped_0" -> "redis_post_notify_stonith_0 controller-0" [ style = bold] "redis-master_post_notify_stopped_0" -> "redis_post_notify_stonith_0 controller-2" [ style = bold] "redis-master_post_notify_stopped_0" [ style=bold color="green" fontcolor="orange"] "redis-master_pre_notify_stop_0" -> "redis-master_confirmed-pre_notify_stop_0" [ style = bold] "redis-master_pre_notify_stop_0" -> "redis_pre_notify_stop_0 controller-0" [ style = bold] "redis-master_pre_notify_stop_0" -> "redis_pre_notify_stop_0 controller-2" [ style = bold] "redis-master_pre_notify_stop_0" [ style=bold color="green" fontcolor="orange"] "redis-master_stop_0" -> "redis-master_stopped_0" [ style = bold] "redis-master_stop_0" -> "redis_stop_0 controller-1" [ style = bold] "redis-master_stop_0" [ style=bold color="green" fontcolor="orange"] "redis-master_stopped_0" -> "redis-master_post_notify_stopped_0" [ style = bold] "redis-master_stopped_0" [ style=bold color="green" fontcolor="orange"] "redis_confirmed-post_notify_stonith_0" [ style=bold color="green" fontcolor="orange"] "redis_post_notify_stonith_0 controller-0" -> "redis-master_confirmed-post_notify_stopped_0" [ style = bold] "redis_post_notify_stonith_0 controller-0" -> "redis_confirmed-post_notify_stonith_0" [ style = bold] "redis_post_notify_stonith_0 controller-0" [ style=bold color="green" fontcolor="black"] "redis_post_notify_stonith_0 controller-2" -> "redis-master_confirmed-post_notify_stopped_0" [ style = bold] "redis_post_notify_stonith_0 controller-2" -> "redis_confirmed-post_notify_stonith_0" [ style = bold] "redis_post_notify_stonith_0 controller-2" [ style=bold color="green" fontcolor="black"] "redis_post_notify_stonith_0" -> "redis_confirmed-post_notify_stonith_0" [ style = bold] "redis_post_notify_stonith_0" -> "redis_post_notify_stonith_0 controller-0" [ style = bold] "redis_post_notify_stonith_0" -> "redis_post_notify_stonith_0 controller-2" [ style = bold] "redis_post_notify_stonith_0" [ style=bold color="green" fontcolor="orange"] "redis_pre_notify_stop_0 controller-0" -> "redis-master_confirmed-pre_notify_stop_0" [ style = bold] "redis_pre_notify_stop_0 controller-0" [ style=bold color="green" fontcolor="black"] "redis_pre_notify_stop_0 controller-2" -> "redis-master_confirmed-pre_notify_stop_0" [ style = bold] "redis_pre_notify_stop_0 controller-2" [ style=bold color="green" fontcolor="black"] "redis_stop_0 controller-1" -> "redis-master_stopped_0" [ style = bold] "redis_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"] "stonith 'reboot' controller-1" -> "haproxy-clone_stop_0" [ style = bold] "stonith 'reboot' controller-1" -> "haproxy_stop_0 controller-1" [ style = bold] "stonith 'reboot' controller-1" -> "ip-172.17.1.14_start_0 controller-2" [ style = bold] "stonith 'reboot' controller-1" -> "ip-172.17.1.14_stop_0 controller-1" [ style = bold] "stonith 'reboot' controller-1" -> "ip-172.17.1.17_start_0 controller-2" [ style = bold] "stonith 'reboot' controller-1" -> "ip-172.17.1.17_stop_0 controller-1" [ style = bold] "stonith 'reboot' controller-1" -> "ip-172.17.4.11_start_0 controller-2" [ style = bold] "stonith 'reboot' controller-1" -> "ip-172.17.4.11_stop_0 controller-1" [ style = bold] "stonith 'reboot' controller-1" -> "redis-master_stop_0" [ style = bold] "stonith 'reboot' controller-1" -> "redis_post_notify_stonith_0" [ style = bold] "stonith 'reboot' controller-1" -> "redis_stop_0 controller-1" [ style = bold] "stonith 'reboot' controller-1" -> "stonith 'reboot' messaging-1" [ style = bold] "stonith 'reboot' controller-1" [ style=bold color="green" fontcolor="black"] "stonith 'reboot' messaging-1" -> "galera-0_start_0 controller-2" [ style = bold] "stonith 'reboot' messaging-1" -> "ip-172.17.1.14_start_0 controller-2" [ style = bold] "stonith 'reboot' messaging-1" -> "ip-172.17.1.17_start_0 controller-2" [ style = bold] "stonith 'reboot' messaging-1" -> "ip-172.17.4.11_start_0 controller-2" [ style = bold] "stonith 'reboot' messaging-1" -> "rabbitmq-clone_stop_0" [ style = bold] "stonith 'reboot' messaging-1" -> "rabbitmq_post_notify_stonith_0" [ style = bold] "stonith 'reboot' messaging-1" -> "rabbitmq_stop_0 messaging-1" [ style = bold] "stonith 'reboot' messaging-1" -> "stonith-fence_ipmilan-5254005bdbb5_start_0 controller-2" [ style = bold] "stonith 'reboot' messaging-1" -> "stonith-fence_ipmilan-525400b4f6bd_start_0 controller-0" [ style = bold] "stonith 'reboot' messaging-1" -> "stonith-fence_ipmilan-525400bbf613_start_0 controller-0" [ style = bold] "stonith 'reboot' messaging-1" [ style=bold color="green" fontcolor="black"] "stonith-fence_ipmilan-5254005bdbb5_monitor_60000 controller-2" [ style=bold color="green" fontcolor="black"] "stonith-fence_ipmilan-5254005bdbb5_start_0 controller-2" -> "stonith-fence_ipmilan-5254005bdbb5_monitor_60000 controller-2" [ style = bold] "stonith-fence_ipmilan-5254005bdbb5_start_0 controller-2" [ style=bold color="green" fontcolor="black"] "stonith-fence_ipmilan-5254005bdbb5_stop_0 controller-1" -> "stonith-fence_ipmilan-5254005bdbb5_start_0 controller-2" [ style = bold] "stonith-fence_ipmilan-5254005bdbb5_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"] "stonith-fence_ipmilan-525400b4f6bd_monitor_60000 controller-0" [ style=bold color="green" fontcolor="black"] "stonith-fence_ipmilan-525400b4f6bd_start_0 controller-0" -> "stonith-fence_ipmilan-525400b4f6bd_monitor_60000 controller-0" [ style = bold] "stonith-fence_ipmilan-525400b4f6bd_start_0 controller-0" [ style=bold color="green" fontcolor="black"] "stonith-fence_ipmilan-525400b4f6bd_stop_0 controller-0" -> "stonith-fence_ipmilan-525400b4f6bd_start_0 controller-0" [ style = bold] "stonith-fence_ipmilan-525400b4f6bd_stop_0 controller-0" [ style=bold color="green" fontcolor="black"] "stonith-fence_ipmilan-525400bbf613_monitor_60000 controller-0" [ style=bold color="green" fontcolor="black"] "stonith-fence_ipmilan-525400bbf613_start_0 controller-0" -> "stonith-fence_ipmilan-525400bbf613_monitor_60000 controller-0" [ style = bold] "stonith-fence_ipmilan-525400bbf613_start_0 controller-0" [ style=bold color="green" fontcolor="black"] "stonith-fence_ipmilan-525400bbf613_stop_0 controller-0" -> "stonith-fence_ipmilan-525400bbf613_start_0 controller-0" [ style = bold] "stonith-fence_ipmilan-525400bbf613_stop_0 controller-0" [ style=bold color="green" fontcolor="black"] } diff --git a/cts/scheduler/remote-recover-no-resources.exp b/cts/scheduler/remote-recover-no-resources.exp index 8a67c1144c..90470fbfce 100644 --- a/cts/scheduler/remote-recover-no-resources.exp +++ b/cts/scheduler/remote-recover-no-resources.exp @@ -1,684 +1,680 @@ - - - - - + diff --git a/cts/scheduler/remote-recover-no-resources.summary b/cts/scheduler/remote-recover-no-resources.summary index 89da784570..18a989b4e2 100644 --- a/cts/scheduler/remote-recover-no-resources.summary +++ b/cts/scheduler/remote-recover-no-resources.summary @@ -1,143 +1,143 @@ Using the original execution date of: 2017-05-03 13:33:24Z Current cluster status: Node controller-1 (2): UNCLEAN (offline) Online: [ controller-0 controller-2 ] RemoteOnline: [ galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ] messaging-0 (ocf::pacemaker:remote): Started controller-0 messaging-1 (ocf::pacemaker:remote): Started controller-1 (UNCLEAN) messaging-2 (ocf::pacemaker:remote): Started controller-0 galera-0 (ocf::pacemaker:remote): Started controller-1 (UNCLEAN) galera-1 (ocf::pacemaker:remote): Started controller-0 galera-2 (ocf::pacemaker:remote): Started controller-1 (UNCLEAN) Clone Set: rabbitmq-clone [rabbitmq] Started: [ messaging-0 messaging-1 messaging-2 ] Stopped: [ controller-0 controller-1 controller-2 galera-0 galera-1 galera-2 ] Clone Set: galera-master [galera] (promotable) Masters: [ galera-0 galera-1 ] Stopped: [ controller-0 controller-1 controller-2 galera-2 messaging-0 messaging-1 messaging-2 ] Clone Set: redis-master [redis] (promotable) redis (ocf::heartbeat:redis): Slave controller-1 (UNCLEAN) Masters: [ controller-0 ] Slaves: [ controller-2 ] Stopped: [ galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ] ip-192.168.24.6 (ocf::heartbeat:IPaddr2): Started controller-0 ip-10.0.0.102 (ocf::heartbeat:IPaddr2): Started controller-0 ip-172.17.1.14 (ocf::heartbeat:IPaddr2): Started controller-1 (UNCLEAN) ip-172.17.1.17 (ocf::heartbeat:IPaddr2): Started controller-1 (UNCLEAN) ip-172.17.3.15 (ocf::heartbeat:IPaddr2): Started controller-0 ip-172.17.4.11 (ocf::heartbeat:IPaddr2): Started controller-1 (UNCLEAN) Clone Set: haproxy-clone [haproxy] haproxy (systemd:haproxy): Started controller-1 (UNCLEAN) Started: [ controller-0 controller-2 ] Stopped: [ galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ] openstack-cinder-volume (systemd:openstack-cinder-volume): Started controller-0 stonith-fence_ipmilan-525400bbf613 (stonith:fence_ipmilan): Started controller-0 stonith-fence_ipmilan-525400b4f6bd (stonith:fence_ipmilan): Started controller-0 stonith-fence_ipmilan-5254005bdbb5 (stonith:fence_ipmilan): Started controller-1 (UNCLEAN) Transition Summary: * Fence (reboot) messaging-1 'resources are active and the connection is unrecoverable' * Fence (reboot) controller-1 'peer is no longer part of the cluster' * Stop messaging-1 (controller-1) due to node availability * Move galera-0 ( controller-1 -> controller-2 ) * Stop galera-2 ( controller-1 ) due to node availability * Stop rabbitmq:2 (messaging-1) due to node availability * Stop redis:0 ( Slave controller-1 ) due to node availability * Move ip-172.17.1.14 ( controller-1 -> controller-2 ) * Move ip-172.17.1.17 ( controller-1 -> controller-2 ) * Move ip-172.17.4.11 ( controller-1 -> controller-2 ) * Stop haproxy:0 (controller-1) due to node availability * Restart stonith-fence_ipmilan-525400bbf613 ( controller-0 ) due to resource definition change * Restart stonith-fence_ipmilan-525400b4f6bd ( controller-0 ) due to resource definition change * Move stonith-fence_ipmilan-5254005bdbb5 ( controller-1 -> controller-2 ) Executing cluster transition: + * Pseudo action: messaging-1_stop_0 * Pseudo action: galera-0_stop_0 * Pseudo action: galera-2_stop_0 * Pseudo action: redis-master_pre_notify_stop_0 * Resource action: stonith-fence_ipmilan-525400bbf613 stop on controller-0 * Resource action: stonith-fence_ipmilan-525400b4f6bd stop on controller-0 * Pseudo action: stonith-fence_ipmilan-5254005bdbb5_stop_0 * Fencing controller-1 (reboot) * Pseudo action: redis_post_notify_stop_0 * Resource action: redis notify on controller-0 * Resource action: redis notify on controller-2 * Pseudo action: redis-master_confirmed-pre_notify_stop_0 * Pseudo action: redis-master_stop_0 * Pseudo action: haproxy-clone_stop_0 * Fencing messaging-1 (reboot) * Resource action: galera-0 start on controller-2 * Pseudo action: rabbitmq_post_notify_stop_0 * Pseudo action: rabbitmq-clone_stop_0 * Resource action: galera monitor=10000 on galera-0 * Pseudo action: redis_stop_0 * Pseudo action: redis-master_stopped_0 * Pseudo action: haproxy_stop_0 * Pseudo action: haproxy-clone_stopped_0 * Resource action: stonith-fence_ipmilan-525400bbf613 start on controller-0 * Resource action: stonith-fence_ipmilan-525400bbf613 monitor=60000 on controller-0 * Resource action: stonith-fence_ipmilan-525400b4f6bd start on controller-0 * Resource action: stonith-fence_ipmilan-525400b4f6bd monitor=60000 on controller-0 * Resource action: stonith-fence_ipmilan-5254005bdbb5 start on controller-2 * Resource action: galera-0 monitor=20000 on controller-2 * Resource action: rabbitmq notify on messaging-2 * Resource action: rabbitmq notify on messaging-0 * Pseudo action: rabbitmq_notified_0 * Pseudo action: rabbitmq_stop_0 * Pseudo action: rabbitmq-clone_stopped_0 * Pseudo action: redis-master_post_notify_stopped_0 * Pseudo action: ip-172.17.1.14_stop_0 * Pseudo action: ip-172.17.1.17_stop_0 * Pseudo action: ip-172.17.4.11_stop_0 * Resource action: stonith-fence_ipmilan-5254005bdbb5 monitor=60000 on controller-2 - * Pseudo action: messaging-1_stop_0 * Resource action: redis notify on controller-0 * Resource action: redis notify on controller-2 * Pseudo action: redis-master_confirmed-post_notify_stopped_0 * Resource action: ip-172.17.1.14 start on controller-2 * Resource action: ip-172.17.1.17 start on controller-2 * Resource action: ip-172.17.4.11 start on controller-2 * Pseudo action: redis_notified_0 * Resource action: ip-172.17.1.14 monitor=10000 on controller-2 * Resource action: ip-172.17.1.17 monitor=10000 on controller-2 * Resource action: ip-172.17.4.11 monitor=10000 on controller-2 Using the original execution date of: 2017-05-03 13:33:24Z Revised cluster status: Online: [ controller-0 controller-2 ] OFFLINE: [ controller-1 ] RemoteOnline: [ galera-0 galera-1 messaging-0 messaging-2 ] RemoteOFFLINE: [ galera-2 messaging-1 ] messaging-0 (ocf::pacemaker:remote): Started controller-0 messaging-1 (ocf::pacemaker:remote): Stopped messaging-2 (ocf::pacemaker:remote): Started controller-0 galera-0 (ocf::pacemaker:remote): Started controller-2 galera-1 (ocf::pacemaker:remote): Started controller-0 galera-2 (ocf::pacemaker:remote): Stopped Clone Set: rabbitmq-clone [rabbitmq] Started: [ messaging-0 messaging-2 ] Stopped: [ controller-0 controller-1 controller-2 galera-0 galera-1 galera-2 messaging-1 ] Clone Set: galera-master [galera] (promotable) Masters: [ galera-0 galera-1 ] Stopped: [ controller-0 controller-1 controller-2 galera-2 messaging-0 messaging-1 messaging-2 ] Clone Set: redis-master [redis] (promotable) Masters: [ controller-0 ] Slaves: [ controller-2 ] Stopped: [ controller-1 galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ] ip-192.168.24.6 (ocf::heartbeat:IPaddr2): Started controller-0 ip-10.0.0.102 (ocf::heartbeat:IPaddr2): Started controller-0 ip-172.17.1.14 (ocf::heartbeat:IPaddr2): Started controller-2 ip-172.17.1.17 (ocf::heartbeat:IPaddr2): Started controller-2 ip-172.17.3.15 (ocf::heartbeat:IPaddr2): Started controller-0 ip-172.17.4.11 (ocf::heartbeat:IPaddr2): Started controller-2 Clone Set: haproxy-clone [haproxy] Started: [ controller-0 controller-2 ] Stopped: [ controller-1 galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ] openstack-cinder-volume (systemd:openstack-cinder-volume): Started controller-0 stonith-fence_ipmilan-525400bbf613 (stonith:fence_ipmilan): Started controller-0 stonith-fence_ipmilan-525400b4f6bd (stonith:fence_ipmilan): Started controller-0 stonith-fence_ipmilan-5254005bdbb5 (stonith:fence_ipmilan): Started controller-2 diff --git a/cts/scheduler/remote-recover-unknown.dot b/cts/scheduler/remote-recover-unknown.dot index 5cd760bc10..29ab59fd2b 100644 --- a/cts/scheduler/remote-recover-unknown.dot +++ b/cts/scheduler/remote-recover-unknown.dot @@ -1,134 +1,133 @@ digraph "g" { "galera-0_monitor_20000 controller-2" [ style=bold color="green" fontcolor="black"] "galera-0_start_0 controller-2" -> "galera-0_monitor_20000 controller-2" [ style = bold] "galera-0_start_0 controller-2" -> "galera_monitor_10000 galera-0" [ style = bold] "galera-0_start_0 controller-2" [ style=bold color="green" fontcolor="black"] "galera-0_stop_0 controller-1" -> "galera-0_start_0 controller-2" [ style = bold] "galera-0_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"] "galera-2_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"] "galera_monitor_0 galera-2" [ style=dashed color="red" fontcolor="black"] "galera_monitor_10000 galera-0" [ style=bold color="green" fontcolor="black"] "haproxy-clone_stop_0" -> "haproxy-clone_stopped_0" [ style = bold] "haproxy-clone_stop_0" -> "haproxy_stop_0 controller-1" [ style = bold] "haproxy-clone_stop_0" [ style=bold color="green" fontcolor="orange"] "haproxy-clone_stopped_0" -> "ip-172.17.1.14_stop_0 controller-1" [ style = bold] "haproxy-clone_stopped_0" -> "ip-172.17.1.17_stop_0 controller-1" [ style = bold] "haproxy-clone_stopped_0" -> "ip-172.17.4.11_stop_0 controller-1" [ style = bold] "haproxy-clone_stopped_0" [ style=bold color="green" fontcolor="orange"] "haproxy_stop_0 controller-1" -> "haproxy-clone_stopped_0" [ style = bold] "haproxy_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"] "ip-172.17.1.14_monitor_10000 controller-2" [ style=bold color="green" fontcolor="black"] "ip-172.17.1.14_start_0 controller-2" -> "ip-172.17.1.14_monitor_10000 controller-2" [ style = bold] "ip-172.17.1.14_start_0 controller-2" [ style=bold color="green" fontcolor="black"] "ip-172.17.1.14_stop_0 controller-1" -> "ip-172.17.1.14_start_0 controller-2" [ style = bold] "ip-172.17.1.14_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"] "ip-172.17.1.17_monitor_10000 controller-2" [ style=bold color="green" fontcolor="black"] "ip-172.17.1.17_start_0 controller-2" -> "ip-172.17.1.17_monitor_10000 controller-2" [ style = bold] "ip-172.17.1.17_start_0 controller-2" [ style=bold color="green" fontcolor="black"] "ip-172.17.1.17_stop_0 controller-1" -> "ip-172.17.1.17_start_0 controller-2" [ style = bold] "ip-172.17.1.17_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"] "ip-172.17.4.11_monitor_10000 controller-2" [ style=bold color="green" fontcolor="black"] "ip-172.17.4.11_start_0 controller-2" -> "ip-172.17.4.11_monitor_10000 controller-2" [ style = bold] "ip-172.17.4.11_start_0 controller-2" [ style=bold color="green" fontcolor="black"] "ip-172.17.4.11_stop_0 controller-1" -> "ip-172.17.4.11_start_0 controller-2" [ style = bold] "ip-172.17.4.11_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"] "messaging-1_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"] "rabbitmq-clone_stop_0" -> "rabbitmq-clone_stopped_0" [ style = bold] "rabbitmq-clone_stop_0" -> "rabbitmq_stop_0 messaging-1" [ style = bold] "rabbitmq-clone_stop_0" [ style=bold color="green" fontcolor="orange"] "rabbitmq-clone_stopped_0" [ style=bold color="green" fontcolor="orange"] "rabbitmq_confirmed-post_notify_stonith_0" [ style=bold color="green" fontcolor="orange"] "rabbitmq_post_notify_stonith_0 messaging-0" -> "rabbitmq_confirmed-post_notify_stonith_0" [ style = bold] "rabbitmq_post_notify_stonith_0 messaging-0" [ style=bold color="green" fontcolor="black"] "rabbitmq_post_notify_stonith_0 messaging-2" -> "rabbitmq_confirmed-post_notify_stonith_0" [ style = bold] "rabbitmq_post_notify_stonith_0 messaging-2" [ style=bold color="green" fontcolor="black"] "rabbitmq_post_notify_stonith_0" -> "rabbitmq_confirmed-post_notify_stonith_0" [ style = bold] "rabbitmq_post_notify_stonith_0" -> "rabbitmq_post_notify_stonith_0 messaging-0" [ style = bold] "rabbitmq_post_notify_stonith_0" -> "rabbitmq_post_notify_stonith_0 messaging-2" [ style = bold] "rabbitmq_post_notify_stonith_0" [ style=bold color="green" fontcolor="orange"] -"rabbitmq_stop_0 messaging-1" -> "messaging-1_stop_0 controller-1" [ style = bold] "rabbitmq_stop_0 messaging-1" -> "rabbitmq-clone_stopped_0" [ style = bold] "rabbitmq_stop_0 messaging-1" [ style=bold color="green" fontcolor="orange"] "redis-master_confirmed-post_notify_stopped_0" [ style=bold color="green" fontcolor="orange"] "redis-master_confirmed-pre_notify_stop_0" -> "redis-master_post_notify_stopped_0" [ style = bold] "redis-master_confirmed-pre_notify_stop_0" -> "redis-master_stop_0" [ style = bold] "redis-master_confirmed-pre_notify_stop_0" [ style=bold color="green" fontcolor="orange"] "redis-master_post_notify_stopped_0" -> "redis-master_confirmed-post_notify_stopped_0" [ style = bold] "redis-master_post_notify_stopped_0" -> "redis_post_notify_stonith_0 controller-0" [ style = bold] "redis-master_post_notify_stopped_0" -> "redis_post_notify_stonith_0 controller-2" [ style = bold] "redis-master_post_notify_stopped_0" [ style=bold color="green" fontcolor="orange"] "redis-master_pre_notify_stop_0" -> "redis-master_confirmed-pre_notify_stop_0" [ style = bold] "redis-master_pre_notify_stop_0" -> "redis_pre_notify_stop_0 controller-0" [ style = bold] "redis-master_pre_notify_stop_0" -> "redis_pre_notify_stop_0 controller-2" [ style = bold] "redis-master_pre_notify_stop_0" [ style=bold color="green" fontcolor="orange"] "redis-master_stop_0" -> "redis-master_stopped_0" [ style = bold] "redis-master_stop_0" -> "redis_stop_0 controller-1" [ style = bold] "redis-master_stop_0" [ style=bold color="green" fontcolor="orange"] "redis-master_stopped_0" -> "redis-master_post_notify_stopped_0" [ style = bold] "redis-master_stopped_0" [ style=bold color="green" fontcolor="orange"] "redis_confirmed-post_notify_stonith_0" [ style=bold color="green" fontcolor="orange"] "redis_post_notify_stonith_0 controller-0" -> "redis-master_confirmed-post_notify_stopped_0" [ style = bold] "redis_post_notify_stonith_0 controller-0" -> "redis_confirmed-post_notify_stonith_0" [ style = bold] "redis_post_notify_stonith_0 controller-0" [ style=bold color="green" fontcolor="black"] "redis_post_notify_stonith_0 controller-2" -> "redis-master_confirmed-post_notify_stopped_0" [ style = bold] "redis_post_notify_stonith_0 controller-2" -> "redis_confirmed-post_notify_stonith_0" [ style = bold] "redis_post_notify_stonith_0 controller-2" [ style=bold color="green" fontcolor="black"] "redis_post_notify_stonith_0" -> "redis_confirmed-post_notify_stonith_0" [ style = bold] "redis_post_notify_stonith_0" -> "redis_post_notify_stonith_0 controller-0" [ style = bold] "redis_post_notify_stonith_0" -> "redis_post_notify_stonith_0 controller-2" [ style = bold] "redis_post_notify_stonith_0" [ style=bold color="green" fontcolor="orange"] "redis_pre_notify_stop_0 controller-0" -> "redis-master_confirmed-pre_notify_stop_0" [ style = bold] "redis_pre_notify_stop_0 controller-0" [ style=bold color="green" fontcolor="black"] "redis_pre_notify_stop_0 controller-2" -> "redis-master_confirmed-pre_notify_stop_0" [ style = bold] "redis_pre_notify_stop_0 controller-2" [ style=bold color="green" fontcolor="black"] "redis_stop_0 controller-1" -> "redis-master_stopped_0" [ style = bold] "redis_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"] "stonith 'reboot' controller-1" -> "haproxy-clone_stop_0" [ style = bold] "stonith 'reboot' controller-1" -> "haproxy_stop_0 controller-1" [ style = bold] "stonith 'reboot' controller-1" -> "ip-172.17.1.14_start_0 controller-2" [ style = bold] "stonith 'reboot' controller-1" -> "ip-172.17.1.14_stop_0 controller-1" [ style = bold] "stonith 'reboot' controller-1" -> "ip-172.17.1.17_start_0 controller-2" [ style = bold] "stonith 'reboot' controller-1" -> "ip-172.17.1.17_stop_0 controller-1" [ style = bold] "stonith 'reboot' controller-1" -> "ip-172.17.4.11_start_0 controller-2" [ style = bold] "stonith 'reboot' controller-1" -> "ip-172.17.4.11_stop_0 controller-1" [ style = bold] "stonith 'reboot' controller-1" -> "redis-master_stop_0" [ style = bold] "stonith 'reboot' controller-1" -> "redis_post_notify_stonith_0" [ style = bold] "stonith 'reboot' controller-1" -> "redis_stop_0 controller-1" [ style = bold] "stonith 'reboot' controller-1" -> "stonith 'reboot' galera-2" [ style = bold] "stonith 'reboot' controller-1" [ style=bold color="green" fontcolor="black"] "stonith 'reboot' galera-2" -> "galera-0_start_0 controller-2" [ style = bold] "stonith 'reboot' galera-2" -> "ip-172.17.1.14_start_0 controller-2" [ style = bold] "stonith 'reboot' galera-2" -> "ip-172.17.1.17_start_0 controller-2" [ style = bold] "stonith 'reboot' galera-2" -> "ip-172.17.4.11_start_0 controller-2" [ style = bold] "stonith 'reboot' galera-2" -> "stonith 'reboot' messaging-1" [ style = bold] "stonith 'reboot' galera-2" -> "stonith-fence_ipmilan-5254005bdbb5_start_0 controller-2" [ style = bold] "stonith 'reboot' galera-2" -> "stonith-fence_ipmilan-525400b4f6bd_start_0 controller-0" [ style = bold] "stonith 'reboot' galera-2" -> "stonith-fence_ipmilan-525400bbf613_start_0 controller-0" [ style = bold] "stonith 'reboot' galera-2" [ style=bold color="green" fontcolor="black"] "stonith 'reboot' messaging-1" -> "galera-0_start_0 controller-2" [ style = bold] "stonith 'reboot' messaging-1" -> "ip-172.17.1.14_start_0 controller-2" [ style = bold] "stonith 'reboot' messaging-1" -> "ip-172.17.1.17_start_0 controller-2" [ style = bold] "stonith 'reboot' messaging-1" -> "ip-172.17.4.11_start_0 controller-2" [ style = bold] "stonith 'reboot' messaging-1" -> "rabbitmq-clone_stop_0" [ style = bold] "stonith 'reboot' messaging-1" -> "rabbitmq_post_notify_stonith_0" [ style = bold] "stonith 'reboot' messaging-1" -> "rabbitmq_stop_0 messaging-1" [ style = bold] "stonith 'reboot' messaging-1" -> "stonith-fence_ipmilan-5254005bdbb5_start_0 controller-2" [ style = bold] "stonith 'reboot' messaging-1" -> "stonith-fence_ipmilan-525400b4f6bd_start_0 controller-0" [ style = bold] "stonith 'reboot' messaging-1" -> "stonith-fence_ipmilan-525400bbf613_start_0 controller-0" [ style = bold] "stonith 'reboot' messaging-1" [ style=bold color="green" fontcolor="black"] "stonith-fence_ipmilan-5254005bdbb5_monitor_60000 controller-2" [ style=bold color="green" fontcolor="black"] "stonith-fence_ipmilan-5254005bdbb5_start_0 controller-2" -> "stonith-fence_ipmilan-5254005bdbb5_monitor_60000 controller-2" [ style = bold] "stonith-fence_ipmilan-5254005bdbb5_start_0 controller-2" [ style=bold color="green" fontcolor="black"] "stonith-fence_ipmilan-5254005bdbb5_stop_0 controller-1" -> "stonith-fence_ipmilan-5254005bdbb5_start_0 controller-2" [ style = bold] "stonith-fence_ipmilan-5254005bdbb5_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"] "stonith-fence_ipmilan-525400b4f6bd_monitor_60000 controller-0" [ style=bold color="green" fontcolor="black"] "stonith-fence_ipmilan-525400b4f6bd_start_0 controller-0" -> "stonith-fence_ipmilan-525400b4f6bd_monitor_60000 controller-0" [ style = bold] "stonith-fence_ipmilan-525400b4f6bd_start_0 controller-0" [ style=bold color="green" fontcolor="black"] "stonith-fence_ipmilan-525400b4f6bd_stop_0 controller-0" -> "stonith-fence_ipmilan-525400b4f6bd_start_0 controller-0" [ style = bold] "stonith-fence_ipmilan-525400b4f6bd_stop_0 controller-0" [ style=bold color="green" fontcolor="black"] "stonith-fence_ipmilan-525400bbf613_monitor_60000 controller-0" [ style=bold color="green" fontcolor="black"] "stonith-fence_ipmilan-525400bbf613_start_0 controller-0" -> "stonith-fence_ipmilan-525400bbf613_monitor_60000 controller-0" [ style = bold] "stonith-fence_ipmilan-525400bbf613_start_0 controller-0" [ style=bold color="green" fontcolor="black"] "stonith-fence_ipmilan-525400bbf613_stop_0 controller-0" -> "stonith-fence_ipmilan-525400bbf613_start_0 controller-0" [ style = bold] "stonith-fence_ipmilan-525400bbf613_stop_0 controller-0" [ style=bold color="green" fontcolor="black"] } diff --git a/cts/scheduler/remote-recover-unknown.exp b/cts/scheduler/remote-recover-unknown.exp index ac6f00486c..82cb65f772 100644 --- a/cts/scheduler/remote-recover-unknown.exp +++ b/cts/scheduler/remote-recover-unknown.exp @@ -1,720 +1,716 @@ - - - - - + diff --git a/cts/scheduler/remote-recover-unknown.summary b/cts/scheduler/remote-recover-unknown.summary index 2c607135cf..4d7a411e70 100644 --- a/cts/scheduler/remote-recover-unknown.summary +++ b/cts/scheduler/remote-recover-unknown.summary @@ -1,145 +1,145 @@ Using the original execution date of: 2017-05-03 13:33:24Z Current cluster status: Node controller-1 (2): UNCLEAN (offline) Online: [ controller-0 controller-2 ] RemoteOnline: [ galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ] messaging-0 (ocf::pacemaker:remote): Started controller-0 messaging-1 (ocf::pacemaker:remote): Started controller-1 (UNCLEAN) messaging-2 (ocf::pacemaker:remote): Started controller-0 galera-0 (ocf::pacemaker:remote): Started controller-1 (UNCLEAN) galera-1 (ocf::pacemaker:remote): Started controller-0 galera-2 (ocf::pacemaker:remote): Started controller-1 (UNCLEAN) Clone Set: rabbitmq-clone [rabbitmq] Started: [ messaging-0 messaging-1 messaging-2 ] Stopped: [ controller-0 controller-1 controller-2 galera-0 galera-1 galera-2 ] Clone Set: galera-master [galera] (promotable) Masters: [ galera-0 galera-1 ] Stopped: [ controller-0 controller-1 controller-2 galera-2 messaging-0 messaging-1 messaging-2 ] Clone Set: redis-master [redis] (promotable) redis (ocf::heartbeat:redis): Slave controller-1 (UNCLEAN) Masters: [ controller-0 ] Slaves: [ controller-2 ] Stopped: [ galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ] ip-192.168.24.6 (ocf::heartbeat:IPaddr2): Started controller-0 ip-10.0.0.102 (ocf::heartbeat:IPaddr2): Started controller-0 ip-172.17.1.14 (ocf::heartbeat:IPaddr2): Started controller-1 (UNCLEAN) ip-172.17.1.17 (ocf::heartbeat:IPaddr2): Started controller-1 (UNCLEAN) ip-172.17.3.15 (ocf::heartbeat:IPaddr2): Started controller-0 ip-172.17.4.11 (ocf::heartbeat:IPaddr2): Started controller-1 (UNCLEAN) Clone Set: haproxy-clone [haproxy] haproxy (systemd:haproxy): Started controller-1 (UNCLEAN) Started: [ controller-0 controller-2 ] Stopped: [ galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ] openstack-cinder-volume (systemd:openstack-cinder-volume): Started controller-0 stonith-fence_ipmilan-525400bbf613 (stonith:fence_ipmilan): Started controller-0 stonith-fence_ipmilan-525400b4f6bd (stonith:fence_ipmilan): Started controller-0 stonith-fence_ipmilan-5254005bdbb5 (stonith:fence_ipmilan): Started controller-1 (UNCLEAN) Transition Summary: * Fence (reboot) galera-2 'resources are in an unknown state and the connection is unrecoverable' * Fence (reboot) messaging-1 'resources are active and the connection is unrecoverable' * Fence (reboot) controller-1 'peer is no longer part of the cluster' * Stop messaging-1 (controller-1) due to node availability * Move galera-0 ( controller-1 -> controller-2 ) * Stop galera-2 (controller-1) due to node availability * Stop rabbitmq:2 (messaging-1) due to node availability * Stop redis:0 ( Slave controller-1 ) due to node availability * Move ip-172.17.1.14 ( controller-1 -> controller-2 ) * Move ip-172.17.1.17 ( controller-1 -> controller-2 ) * Move ip-172.17.4.11 ( controller-1 -> controller-2 ) * Stop haproxy:0 (controller-1) due to node availability * Restart stonith-fence_ipmilan-525400bbf613 ( controller-0 ) due to resource definition change * Restart stonith-fence_ipmilan-525400b4f6bd ( controller-0 ) due to resource definition change * Move stonith-fence_ipmilan-5254005bdbb5 ( controller-1 -> controller-2 ) Executing cluster transition: + * Pseudo action: messaging-1_stop_0 * Pseudo action: galera-0_stop_0 * Pseudo action: galera-2_stop_0 * Pseudo action: redis-master_pre_notify_stop_0 * Resource action: stonith-fence_ipmilan-525400bbf613 stop on controller-0 * Resource action: stonith-fence_ipmilan-525400b4f6bd stop on controller-0 * Pseudo action: stonith-fence_ipmilan-5254005bdbb5_stop_0 * Fencing controller-1 (reboot) * Pseudo action: redis_post_notify_stop_0 * Resource action: redis notify on controller-0 * Resource action: redis notify on controller-2 * Pseudo action: redis-master_confirmed-pre_notify_stop_0 * Pseudo action: redis-master_stop_0 * Pseudo action: haproxy-clone_stop_0 * Fencing galera-2 (reboot) * Fencing messaging-1 (reboot) * Resource action: galera-0 start on controller-2 * Pseudo action: rabbitmq_post_notify_stop_0 * Pseudo action: rabbitmq-clone_stop_0 * Resource action: galera monitor=10000 on galera-0 * Pseudo action: redis_stop_0 * Pseudo action: redis-master_stopped_0 * Pseudo action: haproxy_stop_0 * Pseudo action: haproxy-clone_stopped_0 * Resource action: stonith-fence_ipmilan-525400bbf613 start on controller-0 * Resource action: stonith-fence_ipmilan-525400bbf613 monitor=60000 on controller-0 * Resource action: stonith-fence_ipmilan-525400b4f6bd start on controller-0 * Resource action: stonith-fence_ipmilan-525400b4f6bd monitor=60000 on controller-0 * Resource action: stonith-fence_ipmilan-5254005bdbb5 start on controller-2 * Resource action: galera-0 monitor=20000 on controller-2 * Resource action: rabbitmq notify on messaging-2 * Resource action: rabbitmq notify on messaging-0 * Pseudo action: rabbitmq_notified_0 * Pseudo action: rabbitmq_stop_0 * Pseudo action: rabbitmq-clone_stopped_0 * Pseudo action: redis-master_post_notify_stopped_0 * Pseudo action: ip-172.17.1.14_stop_0 * Pseudo action: ip-172.17.1.17_stop_0 * Pseudo action: ip-172.17.4.11_stop_0 * Resource action: stonith-fence_ipmilan-5254005bdbb5 monitor=60000 on controller-2 - * Pseudo action: messaging-1_stop_0 * Resource action: redis notify on controller-0 * Resource action: redis notify on controller-2 * Pseudo action: redis-master_confirmed-post_notify_stopped_0 * Resource action: ip-172.17.1.14 start on controller-2 * Resource action: ip-172.17.1.17 start on controller-2 * Resource action: ip-172.17.4.11 start on controller-2 * Pseudo action: redis_notified_0 * Resource action: ip-172.17.1.14 monitor=10000 on controller-2 * Resource action: ip-172.17.1.17 monitor=10000 on controller-2 * Resource action: ip-172.17.4.11 monitor=10000 on controller-2 Using the original execution date of: 2017-05-03 13:33:24Z Revised cluster status: Online: [ controller-0 controller-2 ] OFFLINE: [ controller-1 ] RemoteOnline: [ galera-0 galera-1 messaging-0 messaging-2 ] RemoteOFFLINE: [ galera-2 messaging-1 ] messaging-0 (ocf::pacemaker:remote): Started controller-0 messaging-1 (ocf::pacemaker:remote): Stopped messaging-2 (ocf::pacemaker:remote): Started controller-0 galera-0 (ocf::pacemaker:remote): Started controller-2 galera-1 (ocf::pacemaker:remote): Started controller-0 galera-2 (ocf::pacemaker:remote): Stopped Clone Set: rabbitmq-clone [rabbitmq] Started: [ messaging-0 messaging-2 ] Stopped: [ controller-0 controller-1 controller-2 galera-0 galera-1 galera-2 messaging-1 ] Clone Set: galera-master [galera] (promotable) Masters: [ galera-0 galera-1 ] Stopped: [ controller-0 controller-1 controller-2 galera-2 messaging-0 messaging-1 messaging-2 ] Clone Set: redis-master [redis] (promotable) Masters: [ controller-0 ] Slaves: [ controller-2 ] Stopped: [ controller-1 galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ] ip-192.168.24.6 (ocf::heartbeat:IPaddr2): Started controller-0 ip-10.0.0.102 (ocf::heartbeat:IPaddr2): Started controller-0 ip-172.17.1.14 (ocf::heartbeat:IPaddr2): Started controller-2 ip-172.17.1.17 (ocf::heartbeat:IPaddr2): Started controller-2 ip-172.17.3.15 (ocf::heartbeat:IPaddr2): Started controller-0 ip-172.17.4.11 (ocf::heartbeat:IPaddr2): Started controller-2 Clone Set: haproxy-clone [haproxy] Started: [ controller-0 controller-2 ] Stopped: [ controller-1 galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ] openstack-cinder-volume (systemd:openstack-cinder-volume): Started controller-0 stonith-fence_ipmilan-525400bbf613 (stonith:fence_ipmilan): Started controller-0 stonith-fence_ipmilan-525400b4f6bd (stonith:fence_ipmilan): Started controller-0 stonith-fence_ipmilan-5254005bdbb5 (stonith:fence_ipmilan): Started controller-2