diff --git a/cts/cts-scheduler.in b/cts/cts-scheduler.in
index 346ada28d5..9e343795b6 100644
--- a/cts/cts-scheduler.in
+++ b/cts/cts-scheduler.in
@@ -1,1471 +1,1477 @@
#!@PYTHON@
""" Regression tests for Pacemaker's scheduler
"""
# Pacemaker targets compatibility with Python 2.7 and 3.2+
from __future__ import print_function, unicode_literals, absolute_import, division
__copyright__ = "Copyright 2004-2020 the Pacemaker project contributors"
__license__ = "GNU General Public License version 2 or later (GPLv2+) WITHOUT ANY WARRANTY"
import io
import os
import re
import sys
import stat
import shlex
import shutil
import argparse
import subprocess
import platform
DESC = """Regression tests for Pacemaker's scheduler"""
# Each entry in TESTS is a group of tests, where each test consists of a
# test base name, test description, and additional test arguments.
# Test groups will be separated by newlines in output.
TESTS = [
[
[ "simple1", "Offline" ],
[ "simple2", "Start" ],
[ "simple3", "Start 2" ],
[ "simple4", "Start Failed" ],
[ "simple6", "Stop Start" ],
[ "simple7", "Shutdown" ],
#[ "simple8", "Stonith" ],
#[ "simple9", "Lower version" ],
#[ "simple10", "Higher version" ],
[ "simple11", "Priority (ne)" ],
[ "simple12", "Priority (eq)" ],
[ "simple8", "Stickiness" ],
],
[
[ "group1", "Group" ],
[ "group2", "Group + Native" ],
[ "group3", "Group + Group" ],
[ "group4", "Group + Native (nothing)" ],
[ "group5", "Group + Native (move)" ],
[ "group6", "Group + Group (move)" ],
[ "group7", "Group colocation" ],
[ "group13", "Group colocation (cant run)" ],
[ "group8", "Group anti-colocation" ],
[ "group9", "Group recovery" ],
[ "group10", "Group partial recovery" ],
[ "group11", "Group target_role" ],
[ "group14", "Group stop (graph terminated)" ],
[ "group15", "Negative group colocation" ],
[ "bug-1573", "Partial stop of a group with two children" ],
[ "bug-1718", "Mandatory group ordering - Stop group_FUN" ],
[ "bug-lf-2613", "Move group on failure" ],
[ "bug-lf-2619", "Move group on clone failure" ],
[ "group-fail", "Ensure stop order is preserved for partially active groups" ],
[ "group-unmanaged", "No need to restart r115 because r114 is unmanaged" ],
[ "group-unmanaged-stopped", "Make sure r115 is stopped when r114 fails" ],
[ "group-dependents", "Account for the location preferences of things colocated with a group" ],
[ "group-stop-ordering", "Ensure blocked group member stop does not force other member stops" ],
],
[
[ "rsc_dep1", "Must not" ],
[ "rsc_dep3", "Must" ],
[ "rsc_dep5", "Must not 3" ],
[ "rsc_dep7", "Must 3" ],
[ "rsc_dep10", "Must (but cant)" ],
[ "rsc_dep2", "Must (running)" ],
[ "rsc_dep8", "Must (running : alt)" ],
[ "rsc_dep4", "Must (running + move)" ],
[ "asymmetric", "Asymmetric - require explicit location constraints" ],
],
[
[ "orphan-0", "Orphan ignore" ],
[ "orphan-1", "Orphan stop" ],
[ "orphan-2", "Orphan stop, remove failcount" ],
],
[
[ "params-0", "Params: No change" ],
[ "params-1", "Params: Changed" ],
[ "params-2", "Params: Resource definition" ],
+ [ "params-3", "Params: Restart instead of reload if start pending" ],
[ "params-4", "Params: Reload" ],
[ "params-5", "Params: Restart based on probe digest" ],
[ "novell-251689", "Resource definition change + target_role=stopped" ],
[ "bug-lf-2106", "Restart all anonymous clone instances after config change" ],
[ "params-6", "Params: Detect reload in previously migrated resource" ],
[ "nvpair-id-ref", "Support id-ref in nvpair with optional name" ],
[ "not-reschedule-unneeded-monitor",
"Do not reschedule unneeded monitors while resource definitions have changed" ],
[ "reload-becomes-restart", "Cancel reload if restart becomes required" ],
],
[
[ "target-0", "Target Role : baseline" ],
[ "target-1", "Target Role : master" ],
[ "target-2", "Target Role : invalid" ],
],
[
[ "base-score", "Set a node's default score for all nodes" ],
],
[
[ "date-1", "Dates", [ "-t", "2005-020" ] ],
[ "date-2", "Date Spec - Pass", [ "-t", "2005-020T12:30" ] ],
[ "date-3", "Date Spec - Fail", [ "-t", "2005-020T11:30" ] ],
[ "origin", "Timing of recurring operations", [ "-t", "2014-05-07 00:28:00" ] ],
[ "probe-0", "Probe (anon clone)" ],
[ "probe-1", "Pending Probe" ],
[ "probe-2", "Correctly re-probe cloned groups" ],
[ "probe-3", "Probe (pending node)" ],
[ "probe-4", "Probe (pending node + stopped resource)" ],
[ "standby", "Standby" ],
[ "comments", "Comments" ],
],
[
[ "one-or-more-0", "Everything starts" ],
[ "one-or-more-1", "Nothing starts because of A" ],
[ "one-or-more-2", "D can start because of C" ],
[ "one-or-more-3", "D cannot start because of B and C" ],
[ "one-or-more-4", "D cannot start because of target-role" ],
[ "one-or-more-5", "Start A and F even though C and D are stopped" ],
[ "one-or-more-6", "Leave A running even though B is stopped" ],
[ "one-or-more-7", "Leave A running even though C is stopped" ],
[ "bug-5140-require-all-false", "Allow basegrp:0 to stop" ],
[ "clone-require-all-1", "clone B starts node 3 and 4" ],
[ "clone-require-all-2", "clone B remains stopped everywhere" ],
[ "clone-require-all-3", "clone B stops everywhere because A stops everywhere" ],
[ "clone-require-all-4", "clone B remains on node 3 and 4 with only one instance of A remaining" ],
[ "clone-require-all-5", "clone B starts on node 1 3 and 4" ],
[ "clone-require-all-6", "clone B remains active after shutting down instances of A" ],
[ "clone-require-all-7",
"clone A and B both start at the same time. all instances of A start before B" ],
[ "clone-require-all-no-interleave-1", "C starts everywhere after A and B" ],
[ "clone-require-all-no-interleave-2",
"C starts on nodes 1, 2, and 4 with only one active instance of B" ],
[ "clone-require-all-no-interleave-3",
"C remains active when instance of B is stopped on one node and started on another" ],
[ "one-or-more-unrunnable-instances", "Avoid dependencies on instances that won't ever be started" ],
],
[
[ "location-date-rules-1", "Use location constraints with ineffective date-based rules" ],
[ "location-date-rules-2", "Use location constraints with effective date-based rules" ],
[ "nvpair-date-rules-1", "Use nvpair blocks with a variety of date-based rules" ],
],
[
[ "order1", "Order start 1" ],
[ "order2", "Order start 2" ],
[ "order3", "Order stop" ],
[ "order4", "Order (multiple)" ],
[ "order5", "Order (move)" ],
[ "order6", "Order (move w/ restart)" ],
[ "order7", "Order (mandatory)" ],
[ "order-optional", "Order (score=0)" ],
[ "order-required", "Order (score=INFINITY)" ],
[ "bug-lf-2171", "Prevent group start when clone is stopped" ],
[ "order-clone", "Clone ordering should be able to prevent startup of dependent clones" ],
[ "order-sets", "Ordering for resource sets" ],
[ "order-serialize", "Serialize resources without inhibiting migration" ],
[ "order-serialize-set", "Serialize a set of resources without inhibiting migration" ],
[ "clone-order-primitive", "Order clone start after a primitive" ],
[ "clone-order-16instances", "Verify ordering of 16 cloned resources" ],
[ "order-optional-keyword", "Order (optional keyword)" ],
[ "order-mandatory", "Order (mandatory keyword)" ],
[ "bug-lf-2493",
"Don't imply colocation requirements when applying ordering constraints with clones" ],
[ "ordered-set-basic-startup", "Constraint set with default order settings" ],
[ "ordered-set-natural", "Allow natural set ordering" ],
[ "order-wrong-kind", "Order (error)" ],
],
[
[ "coloc-loop", "Colocation - loop" ],
[ "coloc-many-one", "Colocation - many-to-one" ],
[ "coloc-list", "Colocation - many-to-one with list" ],
[ "coloc-group", "Colocation - groups" ],
[ "coloc-slave-anti", "Anti-colocation with slave shouldn't prevent master colocation" ],
[ "coloc-attr", "Colocation based on node attributes" ],
[ "coloc-negative-group", "Negative colocation with a group" ],
[ "coloc-intra-set", "Intra-set colocation" ],
[ "bug-lf-2435", "Colocation sets with a negative score" ],
[ "coloc-clone-stays-active",
"Ensure clones don't get stopped/demoted because a dependent must stop" ],
[ "coloc_fp_logic", "Verify floating point calculations in colocation are working" ],
[ "colo_master_w_native",
"cl#5070 - Verify promotion order is affected when colocating master to native rsc" ],
[ "colo_slave_w_native",
"cl#5070 - Verify promotion order is affected when colocating slave to native rsc" ],
[ "anti-colocation-order",
"cl#5187 - Prevent resources in an anti-colocation from even temporarily running on a same node" ],
[ "anti-colocation-master", "Organize order of actions for master resources in anti-colocations" ],
[ "anti-colocation-slave", "Organize order of actions for slave resources in anti-colocations" ],
[ "enforce-colo1", "Always enforce B with A INFINITY" ],
[ "complex_enforce_colo", "Always enforce B with A INFINITY. (make sure heat-engine stops)" ],
[ "coloc-dependee-should-stay", "Stickiness outweighs group colocation" ],
[ "coloc-dependee-should-move", "Group colocation outweighs stickiness" ],
],
[
[ "rsc-sets-seq-true", "Resource Sets - sequential=false" ],
[ "rsc-sets-seq-false", "Resource Sets - sequential=true" ],
[ "rsc-sets-clone", "Resource Sets - Clone" ],
[ "rsc-sets-master", "Resource Sets - Master" ],
[ "rsc-sets-clone-1", "Resource Sets - Clone (lf#2404)" ],
],
[
[ "attrs1", "string: eq (and)" ],
[ "attrs2", "string: lt / gt (and)" ],
[ "attrs3", "string: ne (or)" ],
[ "attrs4", "string: exists" ],
[ "attrs5", "string: not_exists" ],
[ "attrs6", "is_dc: true" ],
[ "attrs7", "is_dc: false" ],
[ "attrs8", "score_attribute" ],
[ "per-node-attrs", "Per node resource parameters" ],
],
[
[ "mon-rsc-1", "Schedule Monitor - start" ],
[ "mon-rsc-2", "Schedule Monitor - move" ],
[ "mon-rsc-3", "Schedule Monitor - pending start" ],
[ "mon-rsc-4", "Schedule Monitor - move/pending start" ],
],
[
[ "rec-rsc-0", "Resource Recover - no start" ],
[ "rec-rsc-1", "Resource Recover - start" ],
[ "rec-rsc-2", "Resource Recover - monitor" ],
[ "rec-rsc-3", "Resource Recover - stop - ignore" ],
[ "rec-rsc-4", "Resource Recover - stop - block" ],
[ "rec-rsc-5", "Resource Recover - stop - fence" ],
[ "rec-rsc-6", "Resource Recover - multiple - restart" ],
[ "rec-rsc-7", "Resource Recover - multiple - stop" ],
[ "rec-rsc-8", "Resource Recover - multiple - block" ],
[ "rec-rsc-9", "Resource Recover - group/group" ],
[ "monitor-recovery", "on-fail=block + resource recovery detected by recurring monitor" ],
[ "stop-failure-no-quorum", "Stop failure without quorum" ],
[ "stop-failure-no-fencing", "Stop failure without fencing available" ],
[ "stop-failure-with-fencing", "Stop failure with fencing available" ],
[ "multiple-active-block-group", "Support of multiple-active=block for resource groups" ],
[ "multiple-monitor-one-failed",
"Consider resource failed if any of the configured monitor operations failed" ],
],
[
[ "quorum-1", "No quorum - ignore" ],
[ "quorum-2", "No quorum - freeze" ],
[ "quorum-3", "No quorum - stop" ],
[ "quorum-4", "No quorum - start anyway" ],
[ "quorum-5", "No quorum - start anyway (group)" ],
[ "quorum-6", "No quorum - start anyway (clone)" ],
[ "bug-cl-5212", "No promotion with no-quorum-policy=freeze" ],
[ "suicide-needed-inquorate", "no-quorum-policy=suicide: suicide necessary" ],
[ "suicide-not-needed-initial-quorum",
"no-quorum-policy=suicide: suicide not necessary at initial quorum" ],
[ "suicide-not-needed-never-quorate",
"no-quorum-policy=suicide: suicide not necessary if never quorate" ],
[ "suicide-not-needed-quorate", "no-quorum-policy=suicide: suicide necessary if quorate" ],
],
[
[ "rec-node-1", "Node Recover - Startup - no fence" ],
[ "rec-node-2", "Node Recover - Startup - fence" ],
[ "rec-node-3", "Node Recover - HA down - no fence" ],
[ "rec-node-4", "Node Recover - HA down - fence" ],
[ "rec-node-5", "Node Recover - CRM down - no fence" ],
[ "rec-node-6", "Node Recover - CRM down - fence" ],
[ "rec-node-7", "Node Recover - no quorum - ignore" ],
[ "rec-node-8", "Node Recover - no quorum - freeze" ],
[ "rec-node-9", "Node Recover - no quorum - stop" ],
[ "rec-node-10", "Node Recover - no quorum - stop w/fence" ],
[ "rec-node-11", "Node Recover - CRM down w/ group - fence" ],
[ "rec-node-12", "Node Recover - nothing active - fence" ],
[ "rec-node-13", "Node Recover - failed resource + shutdown - fence" ],
[ "rec-node-15", "Node Recover - unknown lrm section" ],
[ "rec-node-14", "Serialize all stonith's" ],
],
[
[ "multi1", "Multiple Active (stop/start)" ],
],
[
[ "migrate-begin", "Normal migration" ],
[ "migrate-success", "Completed migration" ],
[ "migrate-partial-1", "Completed migration, missing stop on source" ],
[ "migrate-partial-2", "Successful migrate_to only" ],
[ "migrate-partial-3", "Successful migrate_to only, target down" ],
[ "migrate-partial-4", "Migrate from the correct host after migrate_to+migrate_from" ],
[ "bug-5186-partial-migrate", "Handle partial migration when src node loses membership" ],
[ "migrate-fail-2", "Failed migrate_from" ],
[ "migrate-fail-3", "Failed migrate_from + stop on source" ],
[ "migrate-fail-4",
"Failed migrate_from + stop on target - ideally we wouldn't need to re-stop on target" ],
[ "migrate-fail-5", "Failed migrate_from + stop on source and target" ],
[ "migrate-fail-6", "Failed migrate_to" ],
[ "migrate-fail-7", "Failed migrate_to + stop on source" ],
[ "migrate-fail-8",
"Failed migrate_to + stop on target - ideally we wouldn't need to re-stop on target" ],
[ "migrate-fail-9", "Failed migrate_to + stop on source and target" ],
[ "migration-ping-pong", "Old migrate_to failure + successful migrate_from on same node" ],
[ "migrate-stop", "Migration in a stopping stack" ],
[ "migrate-start", "Migration in a starting stack" ],
[ "migrate-stop_start", "Migration in a restarting stack" ],
[ "migrate-stop-complex", "Migration in a complex stopping stack" ],
[ "migrate-start-complex", "Migration in a complex starting stack" ],
[ "migrate-stop-start-complex", "Migration in a complex moving stack" ],
[ "migrate-shutdown", "Order the post-migration 'stop' before node shutdown" ],
[ "migrate-1", "Migrate (migrate)" ],
[ "migrate-2", "Migrate (stable)" ],
[ "migrate-3", "Migrate (failed migrate_to)" ],
[ "migrate-4", "Migrate (failed migrate_from)" ],
[ "novell-252693", "Migration in a stopping stack" ],
[ "novell-252693-2", "Migration in a starting stack" ],
[ "novell-252693-3", "Non-Migration in a starting and stopping stack" ],
[ "bug-1820", "Migration in a group" ],
[ "bug-1820-1", "Non-migration in a group" ],
[ "migrate-5", "Primitive migration with a clone" ],
[ "migrate-fencing", "Migration after Fencing" ],
[ "migrate-both-vms", "Migrate two VMs that have no colocation" ],
[ "migration-behind-migrating-remote", "Migrate resource behind migrating remote connection" ],
[ "1-a-then-bm-move-b", "Advanced migrate logic. A then B. migrate B" ],
[ "2-am-then-b-move-a", "Advanced migrate logic, A then B, migrate A without stopping B" ],
[ "3-am-then-bm-both-migrate", "Advanced migrate logic. A then B. migrate both" ],
[ "4-am-then-bm-b-not-migratable", "Advanced migrate logic, A then B, B not migratable" ],
[ "5-am-then-bm-a-not-migratable", "Advanced migrate logic. A then B. move both, a not migratable" ],
[ "6-migrate-group", "Advanced migrate logic, migrate a group" ],
[ "7-migrate-group-one-unmigratable",
"Advanced migrate logic, migrate group mixed with allow-migrate true/false" ],
[ "8-am-then-bm-a-migrating-b-stopping",
"Advanced migrate logic, A then B, A migrating, B stopping" ],
[ "9-am-then-bm-b-migrating-a-stopping",
"Advanced migrate logic, A then B, B migrate, A stopping" ],
[ "10-a-then-bm-b-move-a-clone",
"Advanced migrate logic, A clone then B, migrate B while stopping A" ],
[ "11-a-then-bm-b-move-a-clone-starting",
"Advanced migrate logic, A clone then B, B moving while A is start/stopping" ],
[ "a-promote-then-b-migrate", "A promote then B start. migrate B" ],
[ "a-demote-then-b-migrate", "A demote then B stop. migrate B" ],
# @TODO: If pacemaker implements versioned attributes, uncomment this test
#[ "migrate-versioned", "Disable migration for versioned resources" ],
[ "bug-lf-2422", "Dependency on partially active group - stop ocfs:*" ],
],
[
[ "clone-anon-probe-1", "Probe the correct (anonymous) clone instance for each node" ],
[ "clone-anon-probe-2", "Avoid needless re-probing of anonymous clones" ],
[ "clone-anon-failcount", "Merge failcounts for anonymous clones" ],
[ "force-anon-clone-max", "Update clone-max properly when forcing a clone to be anonymous" ],
[ "anon-instance-pending", "Assign anonymous clone instance numbers properly when action pending" ],
[ "inc0", "Incarnation start" ],
[ "inc1", "Incarnation start order" ],
[ "inc2", "Incarnation silent restart, stop, move" ],
[ "inc3", "Inter-incarnation ordering, silent restart, stop, move" ],
[ "inc4", "Inter-incarnation ordering, silent restart, stop, move (ordered)" ],
[ "inc5", "Inter-incarnation ordering, silent restart, stop, move (restart 1)" ],
[ "inc6", "Inter-incarnation ordering, silent restart, stop, move (restart 2)" ],
[ "inc7", "Clone colocation" ],
[ "inc8", "Clone anti-colocation" ],
[ "inc9", "Non-unique clone" ],
[ "inc10", "Non-unique clone (stop)" ],
[ "inc11", "Primitive colocation with clones" ],
[ "inc12", "Clone shutdown" ],
[ "cloned-group", "Make sure only the correct number of cloned groups are started" ],
[ "cloned-group-stop", "Ensure stopping qpidd also stops glance and cinder" ],
[ "clone-no-shuffle", "Don't prioritize allocation of instances that must be moved" ],
[ "clone-max-zero", "Orphan processing with clone-max=0" ],
[ "clone-anon-dup",
"Bug LF#2087 - Correctly parse the state of anonymous clones that are active more than once per node" ],
[ "bug-lf-2160", "Don't shuffle clones due to colocation" ],
[ "bug-lf-2213", "clone-node-max enforcement for cloned groups" ],
[ "bug-lf-2153", "Clone ordering constraints" ],
[ "bug-lf-2361", "Ensure clones observe mandatory ordering constraints if the LHS is unrunnable" ],
[ "bug-lf-2317", "Avoid needless restart of primitive depending on a clone" ],
[ "clone-colocate-instance-1", "Colocation with a specific clone instance (negative example)" ],
[ "clone-colocate-instance-2", "Colocation with a specific clone instance" ],
[ "clone-order-instance", "Ordering with specific clone instances" ],
[ "bug-lf-2453", "Enforce mandatory clone ordering without colocation" ],
[ "bug-lf-2508", "Correctly reconstruct the status of anonymous cloned groups" ],
[ "bug-lf-2544", "Balanced clone placement" ],
[ "bug-lf-2445", "Redistribute clones with node-max > 1 and stickiness = 0" ],
[ "bug-lf-2574", "Avoid clone shuffle" ],
[ "bug-lf-2581", "Avoid group restart due to unrelated clone (re)start" ],
[ "bug-cl-5168", "Don't shuffle clones" ],
[ "bug-cl-5170", "Prevent clone from starting with on-fail=block" ],
[ "clone-fail-block-colocation", "Move colocated group when failed clone has on-fail=block" ],
[ "clone-interleave-1",
"Clone-3 cannot start on pcmk-1 due to interleaved ordering (no colocation)" ],
[ "clone-interleave-2", "Clone-3 must stop on pcmk-1 due to interleaved ordering (no colocation)" ],
[ "clone-interleave-3",
"Clone-3 must be recovered on pcmk-1 due to interleaved ordering (no colocation)" ],
[ "rebalance-unique-clones", "Rebalance unique clone instances with no stickiness" ],
[ "clone-requires-quorum-recovery", "Clone with requires=quorum on failed node needing recovery" ],
[ "clone-requires-quorum",
"Clone with requires=quorum with presumed-inactive instance on failed node" ],
],
[
[ "cloned_start_one", "order first clone then clone... first clone_min=2" ],
[ "cloned_start_two", "order first clone then clone... first clone_min=2" ],
[ "cloned_stop_one", "order first clone then clone... first clone_min=2" ],
[ "cloned_stop_two", "order first clone then clone... first clone_min=2" ],
[ "clone_min_interleave_start_one",
"order first clone then clone... first clone_min=2 and then has interleave=true" ],
[ "clone_min_interleave_start_two",
"order first clone then clone... first clone_min=2 and then has interleave=true" ],
[ "clone_min_interleave_stop_one",
"order first clone then clone... first clone_min=2 and then has interleave=true" ],
[ "clone_min_interleave_stop_two",
"order first clone then clone... first clone_min=2 and then has interleave=true" ],
[ "clone_min_start_one", "order first clone then primitive... first clone_min=2" ],
[ "clone_min_start_two", "order first clone then primitive... first clone_min=2" ],
[ "clone_min_stop_all", "order first clone then primitive... first clone_min=2" ],
[ "clone_min_stop_one", "order first clone then primitive... first clone_min=2" ],
[ "clone_min_stop_two", "order first clone then primitive... first clone_min=2" ],
],
[
[ "unfence-startup", "Clean unfencing" ],
[ "unfence-definition", "Unfencing when the agent changes" ],
[ "unfence-parameters", "Unfencing when the agent parameters changes" ],
[ "unfence-device", "Unfencing when a cluster has only fence devices" ],
],
[
[ "master-0", "Stopped -> Slave" ],
[ "master-1", "Stopped -> Promote" ],
[ "master-2", "Stopped -> Promote : notify" ],
[ "master-3", "Stopped -> Promote : master location" ],
[ "master-4", "Started -> Promote : master location" ],
[ "master-5", "Promoted -> Promoted" ],
[ "master-6", "Promoted -> Promoted (2)" ],
[ "master-7", "Promoted -> Fenced" ],
[ "master-8", "Promoted -> Fenced -> Moved" ],
[ "master-9", "Stopped + Promotable + No quorum" ],
[ "master-10", "Stopped -> Promotable : notify with monitor" ],
[ "master-11", "Stopped -> Promote : colocation" ],
[ "novell-239082", "Demote/Promote ordering" ],
[ "novell-239087", "Stable master placement" ],
[ "master-12", "Promotion based solely on rsc_location constraints" ],
[ "master-13", "Include preferences of colocated resources when placing master" ],
[ "master-demote", "Ordering when actions depends on demoting a slave resource" ],
[ "master-ordering", "Prevent resources from starting that need a master" ],
[ "bug-1765", "Master-Master Colocation (do not stop the slaves)" ],
[ "master-group", "Promotion of cloned groups" ],
[ "bug-lf-1852", "Don't shuffle master/slave instances unnecessarily" ],
[ "master-failed-demote", "Don't retry failed demote actions" ],
[ "master-failed-demote-2", "Don't retry failed demote actions (notify=false)" ],
[ "master-depend",
"Ensure resources that depend on the master don't get allocated until the master does" ],
[ "master-reattach", "Re-attach to a running master" ],
[ "master-allow-start", "Don't include master score if it would prevent allocation" ],
[ "master-colocation",
"Allow master instances placemaker to be influenced by colocation constraints" ],
[ "master-pseudo", "Make sure promote/demote pseudo actions are created correctly" ],
[ "master-role", "Prevent target-role from promoting more than master-max instances" ],
[ "bug-lf-2358", "Master-Master anti-colocation" ],
[ "master-promotion-constraint", "Mandatory master colocation constraints" ],
[ "unmanaged-master", "Ensure role is preserved for unmanaged resources" ],
[ "master-unmanaged-monitor", "Start the correct monitor operation for unmanaged masters" ],
[ "master-demote-2", "Demote does not clear past failure" ],
[ "master-move", "Move master based on failure of colocated group" ],
[ "master-probed-score", "Observe the promotion score of probed resources" ],
[ "colocation_constraint_stops_master",
"cl#5054 - Ensure master is demoted when stopped by colocation constraint" ],
[ "colocation_constraint_stops_slave",
"cl#5054 - Ensure slave is not demoted when stopped by colocation constraint" ],
[ "order_constraint_stops_master",
"cl#5054 - Ensure master is demoted when stopped by order constraint" ],
[ "order_constraint_stops_slave",
"cl#5054 - Ensure slave is not demoted when stopped by order constraint" ],
[ "master_monitor_restart", "cl#5072 - Ensure master monitor operation will start after promotion" ],
[ "bug-rh-880249", "Handle replacement of an m/s resource with a primitive" ],
[ "bug-5143-ms-shuffle", "Prevent master shuffling due to promotion score" ],
[ "master-demote-block", "Block promotion if demote fails with on-fail=block" ],
[ "master-dependent-ban",
"Don't stop instances from being active because a dependent is banned from that host" ],
[ "master-stop", "Stop instances due to location constraint with role=Started" ],
[ "master-partially-demoted-group", "Allow partially demoted group to finish demoting" ],
[ "bug-cl-5213", "Ensure role colocation with -INFINITY is enforced" ],
[ "bug-cl-5219", "Allow unrelated resources with a common colocation target to remain promoted" ],
[ "master-asymmetrical-order",
"Fix the behaviors of multi-state resources with asymmetrical ordering" ],
[ "master-notify", "Master promotion with notifies" ],
[ "master-score-startup", "Use permanent master scores without LRM history" ],
[ "failed-demote-recovery", "Recover resource in slave role after demote fails" ],
[ "failed-demote-recovery-master", "Recover resource in master role after demote fails" ],
+ [ "on_fail_demote1", "Recovery with on-fail=\"demote\" on healthy cluster, remote, guest, and bundle nodes" ],
+ [ "on_fail_demote2", "Recovery with on-fail=\"demote\" with promotion on different node" ],
+ [ "on_fail_demote3", "Recovery with on-fail=\"demote\" with no promotion" ],
+ [ "on_fail_demote4", "Recovery with on-fail=\"demote\" on failed cluster, remote, guest, and bundle nodes" ],
+ [ "no_quorum_demote", "Promotable demotion and primitive stop with no-quorum-policy=\"demote\"" ],
],
[
[ "history-1", "Correctly parse stateful-1 resource state" ],
],
[
[ "managed-0", "Managed (reference)" ],
[ "managed-1", "Not managed - down" ],
[ "managed-2", "Not managed - up" ],
[ "bug-5028", "Shutdown should block if anything depends on an unmanaged resource" ],
[ "bug-5028-detach", "Ensure detach still works" ],
[ "bug-5028-bottom",
"Ensure shutdown still blocks if the blocked resource is at the bottom of the stack" ],
[ "unmanaged-stop-1",
"cl#5155 - Block the stop of resources if any depending resource is unmanaged" ],
[ "unmanaged-stop-2",
"cl#5155 - Block the stop of resources if the first resource in a mandatory stop order is unmanaged" ],
[ "unmanaged-stop-3",
"cl#5155 - Block the stop of resources if any depending resource in a group is unmanaged" ],
[ "unmanaged-stop-4",
"cl#5155 - Block the stop of resources if any depending resource in the middle of a group is unmanaged" ],
[ "unmanaged-block-restart",
"Block restart of resources if any dependent resource in a group is unmanaged" ],
],
[
[ "interleave-0", "Interleave (reference)" ],
[ "interleave-1", "coloc - not interleaved" ],
[ "interleave-2", "coloc - interleaved" ],
[ "interleave-3", "coloc - interleaved (2)" ],
[ "interleave-pseudo-stop", "Interleaved clone during stonith" ],
[ "interleave-stop", "Interleaved clone during stop" ],
[ "interleave-restart", "Interleaved clone during dependency restart" ],
],
[
[ "notify-0", "Notify reference" ],
[ "notify-1", "Notify simple" ],
[ "notify-2", "Notify simple, confirm" ],
[ "notify-3", "Notify move, confirm" ],
[ "novell-239079", "Notification priority" ],
#[ "notify-2", "Notify - 764" ],
[ "notifs-for-unrunnable", "Don't schedule notifications for an unrunnable action" ],
[ "route-remote-notify", "Route remote notify actions through correct cluster node" ],
[ "notify-behind-stopping-remote", "Don't schedule notifications behind stopped remote" ],
],
[
[ "594", "OSDL #594 - Unrunnable actions scheduled in transition" ],
[ "662", "OSDL #662 - Two resources start on one node when incarnation_node_max = 1" ],
[ "696", "OSDL #696 - CRM starts stonith RA without monitor" ],
[ "726", "OSDL #726 - Attempting to schedule rsc_posic041_monitor_5000 _after_ a stop" ],
[ "735", "OSDL #735 - Correctly detect that rsc_hadev1 is stopped on hadev3" ],
[ "764", "OSDL #764 - Missing monitor op for DoFencing:child_DoFencing:1" ],
[ "797", "OSDL #797 - Assert triggered: task_id_i > max_call_id" ],
[ "829", "OSDL #829" ],
[ "994",
"OSDL #994 - Stopping the last resource in a resource group causes the entire group to be restarted" ],
[ "994-2", "OSDL #994 - with a dependent resource" ],
[ "1360", "OSDL #1360 - Clone stickiness" ],
[ "1484", "OSDL #1484 - on_fail=stop" ],
[ "1494", "OSDL #1494 - Clone stability" ],
[ "unrunnable-1", "Unrunnable" ],
[ "unrunnable-2", "Unrunnable 2" ],
[ "stonith-0", "Stonith loop - 1" ],
[ "stonith-1", "Stonith loop - 2" ],
[ "stonith-2", "Stonith loop - 3" ],
[ "stonith-3", "Stonith startup" ],
[ "stonith-4", "Stonith node state" ],
[ "dc-fence-ordering", "DC needs fencing while other nodes are shutting down" ],
[ "bug-1572-1", "Recovery of groups depending on master/slave" ],
[ "bug-1572-2", "Recovery of groups depending on master/slave when the master is never re-promoted" ],
[ "bug-1685", "Depends-on-master ordering" ],
[ "bug-1822", "Don't promote partially active groups" ],
[ "bug-pm-11", "New resource added to a m/s group" ],
[ "bug-pm-12", "Recover only the failed portion of a cloned group" ],
[ "bug-n-387749", "Don't shuffle clone instances" ],
[ "bug-n-385265",
"Don't ignore the failure stickiness of group children - resource_idvscommon should stay stopped" ],
[ "bug-n-385265-2",
"Ensure groups are migrated instead of remaining partially active on the current node" ],
[ "bug-lf-1920", "Correctly handle probes that find active resources" ],
[ "bnc-515172", "Location constraint with multiple expressions" ],
[ "colocate-primitive-with-clone", "Optional colocation with a clone" ],
[ "use-after-free-merge", "Use-after-free in native_merge_weights" ],
[ "bug-lf-2551", "STONITH ordering for stop" ],
[ "bug-lf-2606", "Stonith implies demote" ],
[ "bug-lf-2474", "Ensure resource op timeout takes precedence over op_defaults" ],
[ "bug-suse-707150", "Prevent vm-01 from starting due to colocation/ordering" ],
[ "bug-5014-A-start-B-start", "Verify when A starts B starts using symmetrical=false" ],
[ "bug-5014-A-stop-B-started",
"Verify when A stops B does not stop if it has already started using symmetric=false" ],
[ "bug-5014-A-stopped-B-stopped",
"Verify when A is stopped and B has not started, B does not start before A using symmetric=false" ],
[ "bug-5014-CthenAthenB-C-stopped",
"Verify when C then A is symmetrical=true, A then B is symmetric=false, and C is stopped that nothing starts" ],
[ "bug-5014-CLONE-A-start-B-start",
"Verify when A starts B starts using clone resources with symmetric=false" ],
[ "bug-5014-CLONE-A-stop-B-started",
"Verify when A stops B does not stop if it has already started using clone resources with symmetric=false" ],
[ "bug-5014-GROUP-A-start-B-start",
"Verify when A starts B starts when using group resources with symmetric=false" ],
[ "bug-5014-GROUP-A-stopped-B-started",
"Verify when A stops B does not stop if it has already started using group resources with symmetric=false" ],
[ "bug-5014-GROUP-A-stopped-B-stopped",
"Verify when A is stopped and B has not started, B does not start before A using group resources with symmetric=false" ],
[ "bug-5014-ordered-set-symmetrical-false",
"Verify ordered sets work with symmetrical=false" ],
[ "bug-5014-ordered-set-symmetrical-true",
"Verify ordered sets work with symmetrical=true" ],
[ "bug-5007-masterslave_colocation",
"Verify use of colocation scores other than INFINITY and -INFINITY work on multi-state resources" ],
[ "bug-5038", "Prevent restart of anonymous clones when clone-max decreases" ],
[ "bug-5025-1", "Automatically clean up failcount after resource config change with reload" ],
[ "bug-5025-2", "Make sure clear failcount action isn't set when config does not change" ],
[ "bug-5025-3", "Automatically clean up failcount after resource config change with restart" ],
[ "bug-5025-4", "Clear failcount when last failure is a start op and rsc attributes changed" ],
[ "failcount", "Ensure failcounts are correctly expired" ],
[ "failcount-block", "Ensure failcounts are not expired when on-fail=block is present" ],
[ "per-op-failcount", "Ensure per-operation failcount is handled and not passed to fence agent" ],
[ "on-fail-ignore", "Ensure on-fail=ignore works even beyond migration-threshold" ],
[ "monitor-onfail-restart", "bug-5058 - Monitor failure with on-fail set to restart" ],
[ "monitor-onfail-stop", "bug-5058 - Monitor failure wiht on-fail set to stop" ],
[ "bug-5059", "No need to restart p_stateful1:*" ],
[ "bug-5069-op-enabled", "Test on-fail=ignore with failure when monitor is enabled" ],
[ "bug-5069-op-disabled", "Test on-fail-ignore with failure when monitor is disabled" ],
[ "obsolete-lrm-resource", "cl#5115 - Do not use obsolete lrm_resource sections" ],
[ "expire-non-blocked-failure",
"Ignore failure-timeout only if the failed operation has on-fail=block" ],
[ "asymmetrical-order-move", "Respect asymmetrical ordering when trying to move resources" ],
[ "asymmetrical-order-restart", "Respect asymmetrical ordering when restarting dependent resource" ],
[ "start-then-stop-with-unfence", "Avoid graph loop with start-then-stop constraint plus unfencing" ],
[ "order-expired-failure", "Order failcount cleanup after remote fencing" ],
[ "ignore_stonith_rsc_order1",
"cl#5056- Ignore order constraint between stonith and non-stonith rsc" ],
[ "ignore_stonith_rsc_order2",
"cl#5056- Ignore order constraint with group rsc containing mixed stonith and non-stonith" ],
[ "ignore_stonith_rsc_order3", "cl#5056- Ignore order constraint, stonith clone and mixed group" ],
[ "ignore_stonith_rsc_order4",
"cl#5056- Ignore order constraint, stonith clone and clone with nested mixed group" ],
[ "honor_stonith_rsc_order1",
"cl#5056- Honor order constraint, stonith clone and pure stonith group(single rsc)" ],
[ "honor_stonith_rsc_order2",
"cl#5056- Honor order constraint, stonith clone and pure stonith group(multiple rsc)" ],
[ "honor_stonith_rsc_order3",
"cl#5056- Honor order constraint, stonith clones with nested pure stonith group" ],
[ "honor_stonith_rsc_order4",
"cl#5056- Honor order constraint, between two native stonith rscs" ],
[ "multiply-active-stonith", "Multiply active stonith" ],
[ "probe-timeout", "cl#5099 - Default probe timeout" ],
[ "order-first-probes",
"cl#5301 - respect order constraints when relevant resources are being probed" ],
[ "concurrent-fencing", "Allow performing fencing operations in parallel" ],
[ "priority-fencing-delay", "Delay fencing targeting the more significant node" ],
],
[
[ "systemhealth1", "System Health () #1" ],
[ "systemhealth2", "System Health () #2" ],
[ "systemhealth3", "System Health () #3" ],
[ "systemhealthn1", "System Health (None) #1" ],
[ "systemhealthn2", "System Health (None) #2" ],
[ "systemhealthn3", "System Health (None) #3" ],
[ "systemhealthm1", "System Health (Migrate On Red) #1" ],
[ "systemhealthm2", "System Health (Migrate On Red) #2" ],
[ "systemhealthm3", "System Health (Migrate On Red) #3" ],
[ "systemhealtho1", "System Health (Only Green) #1" ],
[ "systemhealtho2", "System Health (Only Green) #2" ],
[ "systemhealtho3", "System Health (Only Green) #3" ],
[ "systemhealthp1", "System Health (Progessive) #1" ],
[ "systemhealthp2", "System Health (Progessive) #2" ],
[ "systemhealthp3", "System Health (Progessive) #3" ],
],
[
[ "utilization", "Placement Strategy - utilization" ],
[ "minimal", "Placement Strategy - minimal" ],
[ "balanced", "Placement Strategy - balanced" ],
],
[
[ "placement-stickiness", "Optimized Placement Strategy - stickiness" ],
[ "placement-priority", "Optimized Placement Strategy - priority" ],
[ "placement-location", "Optimized Placement Strategy - location" ],
[ "placement-capacity", "Optimized Placement Strategy - capacity" ],
],
[
[ "utilization-order1", "Utilization Order - Simple" ],
[ "utilization-order2", "Utilization Order - Complex" ],
[ "utilization-order3", "Utilization Order - Migrate" ],
[ "utilization-order4", "Utilization Order - Live Migration (bnc#695440)" ],
[ "utilization-shuffle",
"Don't displace prmExPostgreSQLDB2 on act2, Start prmExPostgreSQLDB1 on act3" ],
[ "load-stopped-loop", "Avoid transition loop due to load_stopped (cl#5044)" ],
[ "load-stopped-loop-2",
"cl#5235 - Prevent graph loops that can be introduced by load_stopped -> migrate_to ordering" ],
],
[
[ "colocated-utilization-primitive-1", "Colocated Utilization - Primitive" ],
[ "colocated-utilization-primitive-2", "Colocated Utilization - Choose the most capable node" ],
[ "colocated-utilization-group", "Colocated Utilization - Group" ],
[ "colocated-utilization-clone", "Colocated Utilization - Clone" ],
[ "utilization-check-allowed-nodes",
"Only check the capacities of the nodes that can run the resource" ],
],
[
[ "reprobe-target_rc", "Ensure correct target_rc for reprobe of inactive resources" ],
[ "node-maintenance-1", "cl#5128 - Node maintenance" ],
[ "node-maintenance-2", "cl#5128 - Node maintenance (coming out of maintenance mode)" ],
[ "shutdown-maintenance-node", "Do not fence a maintenance node if it shuts down cleanly" ],
[ "rsc-maintenance", "Per-resource maintenance" ],
],
[
[ "not-installed-agent", "The resource agent is missing" ],
[ "not-installed-tools", "Something the resource agent needs is missing" ],
],
[
[ "stopped-monitor-00", "Stopped Monitor - initial start" ],
[ "stopped-monitor-01", "Stopped Monitor - failed started" ],
[ "stopped-monitor-02", "Stopped Monitor - started multi-up" ],
[ "stopped-monitor-03", "Stopped Monitor - stop started" ],
[ "stopped-monitor-04", "Stopped Monitor - failed stop" ],
[ "stopped-monitor-05", "Stopped Monitor - start unmanaged" ],
[ "stopped-monitor-06", "Stopped Monitor - unmanaged multi-up" ],
[ "stopped-monitor-07", "Stopped Monitor - start unmanaged multi-up" ],
[ "stopped-monitor-08", "Stopped Monitor - migrate" ],
[ "stopped-monitor-09", "Stopped Monitor - unmanage started" ],
[ "stopped-monitor-10", "Stopped Monitor - unmanaged started multi-up" ],
[ "stopped-monitor-11", "Stopped Monitor - stop unmanaged started" ],
[ "stopped-monitor-12", "Stopped Monitor - unmanaged started multi-up (target-role=Stopped)" ],
[ "stopped-monitor-20", "Stopped Monitor - initial stop" ],
[ "stopped-monitor-21", "Stopped Monitor - stopped single-up" ],
[ "stopped-monitor-22", "Stopped Monitor - stopped multi-up" ],
[ "stopped-monitor-23", "Stopped Monitor - start stopped" ],
[ "stopped-monitor-24", "Stopped Monitor - unmanage stopped" ],
[ "stopped-monitor-25", "Stopped Monitor - unmanaged stopped multi-up" ],
[ "stopped-monitor-26", "Stopped Monitor - start unmanaged stopped" ],
[ "stopped-monitor-27", "Stopped Monitor - unmanaged stopped multi-up (target-role=Started)" ],
[ "stopped-monitor-30", "Stopped Monitor - new node started" ],
[ "stopped-monitor-31", "Stopped Monitor - new node stopped" ],
],
[
# This is a combo test to check:
# - probe timeout defaults to the minimum-interval monitor's
# - duplicate recurring operations are ignored
# - if timeout spec is bad, the default timeout is used
# - failure is blocked with on-fail=block even if ISO8601 interval is specified
# - started/stopped role monitors are started/stopped on right nodes
[ "intervals", "Recurring monitor interval handling" ],
],
[
[ "ticket-primitive-1", "Ticket - Primitive (loss-policy=stop, initial)" ],
[ "ticket-primitive-2", "Ticket - Primitive (loss-policy=stop, granted)" ],
[ "ticket-primitive-3", "Ticket - Primitive (loss-policy-stop, revoked)" ],
[ "ticket-primitive-4", "Ticket - Primitive (loss-policy=demote, initial)" ],
[ "ticket-primitive-5", "Ticket - Primitive (loss-policy=demote, granted)" ],
[ "ticket-primitive-6", "Ticket - Primitive (loss-policy=demote, revoked)" ],
[ "ticket-primitive-7", "Ticket - Primitive (loss-policy=fence, initial)" ],
[ "ticket-primitive-8", "Ticket - Primitive (loss-policy=fence, granted)" ],
[ "ticket-primitive-9", "Ticket - Primitive (loss-policy=fence, revoked)" ],
[ "ticket-primitive-10", "Ticket - Primitive (loss-policy=freeze, initial)" ],
[ "ticket-primitive-11", "Ticket - Primitive (loss-policy=freeze, granted)" ],
[ "ticket-primitive-12", "Ticket - Primitive (loss-policy=freeze, revoked)" ],
[ "ticket-primitive-13", "Ticket - Primitive (loss-policy=stop, standby, granted)" ],
[ "ticket-primitive-14", "Ticket - Primitive (loss-policy=stop, granted, standby)" ],
[ "ticket-primitive-15", "Ticket - Primitive (loss-policy=stop, standby, revoked)" ],
[ "ticket-primitive-16", "Ticket - Primitive (loss-policy=demote, standby, granted)" ],
[ "ticket-primitive-17", "Ticket - Primitive (loss-policy=demote, granted, standby)" ],
[ "ticket-primitive-18", "Ticket - Primitive (loss-policy=demote, standby, revoked)" ],
[ "ticket-primitive-19", "Ticket - Primitive (loss-policy=fence, standby, granted)" ],
[ "ticket-primitive-20", "Ticket - Primitive (loss-policy=fence, granted, standby)" ],
[ "ticket-primitive-21", "Ticket - Primitive (loss-policy=fence, standby, revoked)" ],
[ "ticket-primitive-22", "Ticket - Primitive (loss-policy=freeze, standby, granted)" ],
[ "ticket-primitive-23", "Ticket - Primitive (loss-policy=freeze, granted, standby)" ],
[ "ticket-primitive-24", "Ticket - Primitive (loss-policy=freeze, standby, revoked)" ],
],
[
[ "ticket-group-1", "Ticket - Group (loss-policy=stop, initial)" ],
[ "ticket-group-2", "Ticket - Group (loss-policy=stop, granted)" ],
[ "ticket-group-3", "Ticket - Group (loss-policy-stop, revoked)" ],
[ "ticket-group-4", "Ticket - Group (loss-policy=demote, initial)" ],
[ "ticket-group-5", "Ticket - Group (loss-policy=demote, granted)" ],
[ "ticket-group-6", "Ticket - Group (loss-policy=demote, revoked)" ],
[ "ticket-group-7", "Ticket - Group (loss-policy=fence, initial)" ],
[ "ticket-group-8", "Ticket - Group (loss-policy=fence, granted)" ],
[ "ticket-group-9", "Ticket - Group (loss-policy=fence, revoked)" ],
[ "ticket-group-10", "Ticket - Group (loss-policy=freeze, initial)" ],
[ "ticket-group-11", "Ticket - Group (loss-policy=freeze, granted)" ],
[ "ticket-group-12", "Ticket - Group (loss-policy=freeze, revoked)" ],
[ "ticket-group-13", "Ticket - Group (loss-policy=stop, standby, granted)" ],
[ "ticket-group-14", "Ticket - Group (loss-policy=stop, granted, standby)" ],
[ "ticket-group-15", "Ticket - Group (loss-policy=stop, standby, revoked)" ],
[ "ticket-group-16", "Ticket - Group (loss-policy=demote, standby, granted)" ],
[ "ticket-group-17", "Ticket - Group (loss-policy=demote, granted, standby)" ],
[ "ticket-group-18", "Ticket - Group (loss-policy=demote, standby, revoked)" ],
[ "ticket-group-19", "Ticket - Group (loss-policy=fence, standby, granted)" ],
[ "ticket-group-20", "Ticket - Group (loss-policy=fence, granted, standby)" ],
[ "ticket-group-21", "Ticket - Group (loss-policy=fence, standby, revoked)" ],
[ "ticket-group-22", "Ticket - Group (loss-policy=freeze, standby, granted)" ],
[ "ticket-group-23", "Ticket - Group (loss-policy=freeze, granted, standby)" ],
[ "ticket-group-24", "Ticket - Group (loss-policy=freeze, standby, revoked)" ],
],
[
[ "ticket-clone-1", "Ticket - Clone (loss-policy=stop, initial)" ],
[ "ticket-clone-2", "Ticket - Clone (loss-policy=stop, granted)" ],
[ "ticket-clone-3", "Ticket - Clone (loss-policy-stop, revoked)" ],
[ "ticket-clone-4", "Ticket - Clone (loss-policy=demote, initial)" ],
[ "ticket-clone-5", "Ticket - Clone (loss-policy=demote, granted)" ],
[ "ticket-clone-6", "Ticket - Clone (loss-policy=demote, revoked)" ],
[ "ticket-clone-7", "Ticket - Clone (loss-policy=fence, initial)" ],
[ "ticket-clone-8", "Ticket - Clone (loss-policy=fence, granted)" ],
[ "ticket-clone-9", "Ticket - Clone (loss-policy=fence, revoked)" ],
[ "ticket-clone-10", "Ticket - Clone (loss-policy=freeze, initial)" ],
[ "ticket-clone-11", "Ticket - Clone (loss-policy=freeze, granted)" ],
[ "ticket-clone-12", "Ticket - Clone (loss-policy=freeze, revoked)" ],
[ "ticket-clone-13", "Ticket - Clone (loss-policy=stop, standby, granted)" ],
[ "ticket-clone-14", "Ticket - Clone (loss-policy=stop, granted, standby)" ],
[ "ticket-clone-15", "Ticket - Clone (loss-policy=stop, standby, revoked)" ],
[ "ticket-clone-16", "Ticket - Clone (loss-policy=demote, standby, granted)" ],
[ "ticket-clone-17", "Ticket - Clone (loss-policy=demote, granted, standby)" ],
[ "ticket-clone-18", "Ticket - Clone (loss-policy=demote, standby, revoked)" ],
[ "ticket-clone-19", "Ticket - Clone (loss-policy=fence, standby, granted)" ],
[ "ticket-clone-20", "Ticket - Clone (loss-policy=fence, granted, standby)" ],
[ "ticket-clone-21", "Ticket - Clone (loss-policy=fence, standby, revoked)" ],
[ "ticket-clone-22", "Ticket - Clone (loss-policy=freeze, standby, granted)" ],
[ "ticket-clone-23", "Ticket - Clone (loss-policy=freeze, granted, standby)" ],
[ "ticket-clone-24", "Ticket - Clone (loss-policy=freeze, standby, revoked)" ],
],
[
[ "ticket-master-1", "Ticket - Master (loss-policy=stop, initial)" ],
[ "ticket-master-2", "Ticket - Master (loss-policy=stop, granted)" ],
[ "ticket-master-3", "Ticket - Master (loss-policy-stop, revoked)" ],
[ "ticket-master-4", "Ticket - Master (loss-policy=demote, initial)" ],
[ "ticket-master-5", "Ticket - Master (loss-policy=demote, granted)" ],
[ "ticket-master-6", "Ticket - Master (loss-policy=demote, revoked)" ],
[ "ticket-master-7", "Ticket - Master (loss-policy=fence, initial)" ],
[ "ticket-master-8", "Ticket - Master (loss-policy=fence, granted)" ],
[ "ticket-master-9", "Ticket - Master (loss-policy=fence, revoked)" ],
[ "ticket-master-10", "Ticket - Master (loss-policy=freeze, initial)" ],
[ "ticket-master-11", "Ticket - Master (loss-policy=freeze, granted)" ],
[ "ticket-master-12", "Ticket - Master (loss-policy=freeze, revoked)" ],
[ "ticket-master-13", "Ticket - Master (loss-policy=stop, standby, granted)" ],
[ "ticket-master-14", "Ticket - Master (loss-policy=stop, granted, standby)" ],
[ "ticket-master-15", "Ticket - Master (loss-policy=stop, standby, revoked)" ],
[ "ticket-master-16", "Ticket - Master (loss-policy=demote, standby, granted)" ],
[ "ticket-master-17", "Ticket - Master (loss-policy=demote, granted, standby)" ],
[ "ticket-master-18", "Ticket - Master (loss-policy=demote, standby, revoked)" ],
[ "ticket-master-19", "Ticket - Master (loss-policy=fence, standby, granted)" ],
[ "ticket-master-20", "Ticket - Master (loss-policy=fence, granted, standby)" ],
[ "ticket-master-21", "Ticket - Master (loss-policy=fence, standby, revoked)" ],
[ "ticket-master-22", "Ticket - Master (loss-policy=freeze, standby, granted)" ],
[ "ticket-master-23", "Ticket - Master (loss-policy=freeze, granted, standby)" ],
[ "ticket-master-24", "Ticket - Master (loss-policy=freeze, standby, revoked)" ],
],
[
[ "ticket-rsc-sets-1", "Ticket - Resource sets (1 ticket, initial)" ],
[ "ticket-rsc-sets-2", "Ticket - Resource sets (1 ticket, granted)" ],
[ "ticket-rsc-sets-3", "Ticket - Resource sets (1 ticket, revoked)" ],
[ "ticket-rsc-sets-4", "Ticket - Resource sets (2 tickets, initial)" ],
[ "ticket-rsc-sets-5", "Ticket - Resource sets (2 tickets, granted)" ],
[ "ticket-rsc-sets-6", "Ticket - Resource sets (2 tickets, granted)" ],
[ "ticket-rsc-sets-7", "Ticket - Resource sets (2 tickets, revoked)" ],
[ "ticket-rsc-sets-8", "Ticket - Resource sets (1 ticket, standby, granted)" ],
[ "ticket-rsc-sets-9", "Ticket - Resource sets (1 ticket, granted, standby)" ],
[ "ticket-rsc-sets-10", "Ticket - Resource sets (1 ticket, standby, revoked)" ],
[ "ticket-rsc-sets-11", "Ticket - Resource sets (2 tickets, standby, granted)" ],
[ "ticket-rsc-sets-12", "Ticket - Resource sets (2 tickets, standby, granted)" ],
[ "ticket-rsc-sets-13", "Ticket - Resource sets (2 tickets, granted, standby)" ],
[ "ticket-rsc-sets-14", "Ticket - Resource sets (2 tickets, standby, revoked)" ],
[ "cluster-specific-params", "Cluster-specific instance attributes based on rules" ],
[ "site-specific-params", "Site-specific instance attributes based on rules" ],
],
[
[ "template-1", "Template - 1" ],
[ "template-2", "Template - 2" ],
[ "template-3", "Template - 3 (merge operations)" ],
[ "template-coloc-1", "Template - Colocation 1" ],
[ "template-coloc-2", "Template - Colocation 2" ],
[ "template-coloc-3", "Template - Colocation 3" ],
[ "template-order-1", "Template - Order 1" ],
[ "template-order-2", "Template - Order 2" ],
[ "template-order-3", "Template - Order 3" ],
[ "template-ticket", "Template - Ticket" ],
[ "template-rsc-sets-1", "Template - Resource Sets 1" ],
[ "template-rsc-sets-2", "Template - Resource Sets 2" ],
[ "template-rsc-sets-3", "Template - Resource Sets 3" ],
[ "template-rsc-sets-4", "Template - Resource Sets 4" ],
[ "template-clone-primitive", "Cloned primitive from template" ],
[ "template-clone-group", "Cloned group from template" ],
[ "location-sets-templates", "Resource sets and templates - Location" ],
[ "tags-coloc-order-1", "Tags - Colocation and Order (Simple)" ],
[ "tags-coloc-order-2", "Tags - Colocation and Order (Resource Sets with Templates)" ],
[ "tags-location", "Tags - Location" ],
[ "tags-ticket", "Tags - Ticket" ],
],
[
[ "container-1", "Container - initial" ],
[ "container-2", "Container - monitor failed" ],
[ "container-3", "Container - stop failed" ],
[ "container-4", "Container - reached migration-threshold" ],
[ "container-group-1", "Container in group - initial" ],
[ "container-group-2", "Container in group - monitor failed" ],
[ "container-group-3", "Container in group - stop failed" ],
[ "container-group-4", "Container in group - reached migration-threshold" ],
[ "container-is-remote-node", "Place resource within container when container is remote-node" ],
[ "bug-rh-1097457", "Kill user defined container/contents ordering" ],
[ "bug-cl-5247", "Graph loop when recovering m/s resource in a container" ],
[ "bundle-order-startup", "Bundle startup ordering" ],
[ "bundle-order-partial-start",
"Bundle startup ordering when some dependencies are already running" ],
[ "bundle-order-partial-start-2",
"Bundle startup ordering when some dependencies and the container are already running" ],
[ "bundle-order-stop", "Bundle stop ordering" ],
[ "bundle-order-partial-stop", "Bundle startup ordering when some dependencies are already stopped" ],
[ "bundle-order-stop-on-remote", "Stop nested resource after bringing up the connection" ],
[ "bundle-order-startup-clone", "Prevent startup because bundle isn't promoted" ],
[ "bundle-order-startup-clone-2", "Bundle startup with clones" ],
[ "bundle-order-stop-clone", "Stop bundle because clone is stopping" ],
[ "bundle-nested-colocation", "Colocation of nested connection resources" ],
[ "bundle-order-fencing",
"Order pseudo bundle fencing after parent node fencing if both are happening" ],
[ "bundle-probe-order-1", "order 1" ],
[ "bundle-probe-order-2", "order 2" ],
[ "bundle-probe-order-3", "order 3" ],
[ "bundle-probe-remotes", "Ensure remotes get probed too" ],
[ "bundle-replicas-change", "Change bundle from 1 replica to multiple" ],
[ "nested-remote-recovery", "Recover bundle's container hosted on remote node" ],
],
[
[ "whitebox-fail1", "Fail whitebox container rsc" ],
[ "whitebox-fail2", "Fail cluster connection to guest node" ],
[ "whitebox-fail3", "Failed containers should not run nested on remote nodes" ],
[ "whitebox-start", "Start whitebox container with resources assigned to it" ],
[ "whitebox-stop", "Stop whitebox container with resources assigned to it" ],
[ "whitebox-move", "Move whitebox container with resources assigned to it" ],
[ "whitebox-asymmetric", "Verify connection rsc opts-in based on container resource" ],
[ "whitebox-ms-ordering", "Verify promote/demote can not occur before connection is established" ],
[ "whitebox-ms-ordering-move", "Stop/Start cycle within a moving container" ],
[ "whitebox-orphaned", "Properly shutdown orphaned whitebox container" ],
[ "whitebox-orphan-ms", "Properly tear down orphan ms resources on remote-nodes" ],
[ "whitebox-unexpectedly-running", "Recover container nodes the cluster did not start" ],
[ "whitebox-migrate1", "Migrate both container and connection resource" ],
[ "whitebox-imply-stop-on-fence",
"imply stop action on container node rsc when host node is fenced" ],
[ "whitebox-nested-group", "Verify guest remote-node works nested in a group" ],
[ "guest-node-host-dies", "Verify guest node is recovered if host goes away" ],
[ "guest-node-cleanup", "Order guest node connection recovery after container probe" ],
[ "guest-host-not-fenceable", "Actions on guest node are unrunnable if host is unclean and cannot be fenced" ],
],
[
[ "remote-startup-probes", "Baremetal remote-node startup probes" ],
[ "remote-startup", "Startup a newly discovered remote-nodes with no status" ],
[ "remote-fence-unclean", "Fence unclean baremetal remote-node" ],
[ "remote-fence-unclean2",
"Fence baremetal remote-node after cluster node fails and connection can not be recovered" ],
[ "remote-fence-unclean-3", "Probe failed remote nodes (triggers fencing)" ],
[ "remote-move", "Move remote-node connection resource" ],
[ "remote-disable", "Disable a baremetal remote-node" ],
[ "remote-probe-disable", "Probe then stop a baremetal remote-node" ],
[ "remote-orphaned", "Properly shutdown orphaned connection resource" ],
[ "remote-orphaned2",
"verify we can handle orphaned remote connections with active resources on the remote" ],
[ "remote-recover", "Recover connection resource after cluster-node fails" ],
[ "remote-stale-node-entry",
"Make sure we properly handle leftover remote-node entries in the node section" ],
[ "remote-partial-migrate",
"Make sure partial migrations are handled before ops on the remote node" ],
[ "remote-partial-migrate2",
"Make sure partial migration target is prefered for remote connection" ],
[ "remote-recover-fail", "Make sure start failure causes fencing if rsc are active on remote" ],
[ "remote-start-fail",
"Make sure a start failure does not result in fencing if no active resources are on remote" ],
[ "remote-unclean2",
"Make monitor failure always results in fencing, even if no rsc are active on remote" ],
[ "remote-fence-before-reconnect", "Fence before clearing recurring monitor failure" ],
[ "remote-recovery", "Recover remote connections before attempting demotion" ],
[ "remote-recover-connection", "Optimistically recovery of only the connection" ],
[ "remote-recover-all", "Fencing when the connection has no home" ],
[ "remote-recover-no-resources", "Fencing when the connection has no home and no active resources" ],
[ "remote-recover-unknown",
"Fencing when the connection has no home and the remote has no operation history" ],
[ "remote-reconnect-delay", "Waiting for remote reconnect interval to expire" ],
[ "remote-connection-unrecoverable",
"Remote connection host must be fenced, with connection unrecoverable" ],
],
[
[ "resource-discovery", "Exercises resource-discovery location constraint option" ],
[ "rsc-discovery-per-node", "Disable resource discovery per node" ],
[ "shutdown-lock", "Ensure shutdown lock works properly" ],
[ "shutdown-lock-expiration", "Ensure shutdown lock expiration works properly" ],
],
[
[ "op-defaults", "Test op_defaults conditional expressions" ],
[ "op-defaults-2", "Test op_defaults AND'ed conditional expressions" ],
[ "op-defaults-3", "Test op_defaults precedence" ],
[ "rsc-defaults", "Test rsc_defaults conditional expressions" ],
[ "rsc-defaults-2", "Test rsc_defaults conditional expressions without type" ],
],
# @TODO: If pacemaker implements versioned attributes, uncomment these tests
#[
# [ "versioned-resources", "Start resources with #ra-version rules" ],
# [ "restart-versioned", "Restart resources on #ra-version change" ],
# [ "reload-versioned", "Reload resources on #ra-version change" ],
#],
#[
# [ "versioned-operations-1", "Use #ra-version to configure operations of native resources" ],
# [ "versioned-operations-2", "Use #ra-version to configure operations of stonith resources" ],
# [ "versioned-operations-3", "Use #ra-version to configure operations of master/slave resources" ],
# [ "versioned-operations-4", "Use #ra-version to configure operations of groups of the resources" ],
#],
]
TESTS_64BIT = [
[
[ "year-2038", "Check handling of timestamps beyond 2038-01-19 03:14:08 UTC" ],
],
]
# Constants substituted in the build process
class BuildVars(object):
SBINDIR = "@sbindir@"
BUILDDIR = "@abs_top_builddir@"
CRM_SCHEMA_DIRECTORY = "@CRM_SCHEMA_DIRECTORY@"
# These values must be kept in sync with include/crm/crm.h
class CrmExit(object):
OK = 0
ERROR = 1
NOT_INSTALLED = 5
NOINPUT = 66
def is_executable(path):
""" Check whether a file at a given path is executable. """
try:
return os.stat(path)[stat.ST_MODE] & stat.S_IXUSR
except OSError:
return False
def diff(file1, file2, **kwargs):
""" Call diff on two files """
return subprocess.call([ "diff", "-u", "-N", "--ignore-all-space",
"--ignore-blank-lines", file1, file2 ], **kwargs)
def sort_file(filename):
""" Sort a file alphabetically """
with io.open(filename, "rt") as f:
lines = sorted(f)
with io.open(filename, "wt") as f:
f.writelines(lines)
def remove_files(filenames):
""" Remove a list of files """
for filename in filenames:
try:
os.remove(filename)
except OSError:
pass
def normalize(filename):
""" Remove text from a file that isn't important for comparison """
if not hasattr(normalize, "patterns"):
normalize.patterns = [
re.compile(r'crm_feature_set="[^"]*"'),
re.compile(r'batch-limit="[0-9]*"')
]
if os.path.isfile(filename):
with io.open(filename, "rt") as f:
lines = f.readlines()
with io.open(filename, "wt") as f:
for line in lines:
for pattern in normalize.patterns:
line = pattern.sub("", line)
f.write(line)
def cat(filename, dest=sys.stdout):
""" Copy a file to a destination file descriptor """
with io.open(filename, "rt") as f:
shutil.copyfileobj(f, dest)
class CtsScheduler(object):
""" Regression tests for Pacemaker's scheduler """
def _parse_args(self, argv):
""" Parse command-line arguments """
parser = argparse.ArgumentParser(description=DESC)
parser.add_argument('-V', '--verbose', action='count',
help='Display any differences from expected output')
parser.add_argument('--run', metavar='TEST',
help=('Run only single specified test (any further '
'arguments will be passed to crm_simulate)'))
parser.add_argument('--update', action='store_true',
help='Update expected results with actual results')
parser.add_argument('-b', '--binary', metavar='PATH',
help='Specify path to crm_simulate')
parser.add_argument('-i', '--io-dir', metavar='PATH',
help='Specify path to regression test data directory')
parser.add_argument('-o', '--out-dir', metavar='PATH',
help='Specify where intermediate and output files should go')
parser.add_argument('-v', '--valgrind', action='store_true',
help='Run all commands under valgrind')
parser.add_argument('--valgrind-dhat', action='store_true',
help='Run all commands under valgrind with heap analyzer')
parser.add_argument('--valgrind-skip-output', action='store_true',
help='If running under valgrind, do not display output')
parser.add_argument('--testcmd-options', metavar='OPTIONS', default='',
help='Additional options for command under test')
# argparse can't handle "everything after --run TEST", so grab that
self.single_test_args = []
narg = 0
for arg in argv:
narg = narg + 1
if arg == '--run':
(argv, self.single_test_args) = (argv[:narg+1], argv[narg+1:])
break
self.args = parser.parse_args(argv[1:])
def _error(self, s):
print(" * ERROR: %s" % s)
def _failed(self, s):
print(" * FAILED: %s" % s)
def _get_valgrind_cmd(self):
""" Return command arguments needed (or not) to run valgrind """
if self.args.valgrind:
os.environ['G_SLICE'] = "always-malloc"
return [
"valgrind",
"-q",
"--gen-suppressions=all",
"--time-stamp=yes",
"--trace-children=no",
"--show-reachable=no",
"--leak-check=full",
"--num-callers=20",
"--suppressions=%s/valgrind-pcmk.suppressions" % (self.test_home)
]
if self.args.valgrind_dhat:
os.environ['G_SLICE'] = "always-malloc"
return [
"valgrind",
"--tool=exp-dhat",
"--time-stamp=yes",
"--trace-children=no",
"--show-top-n=100",
"--num-callers=4"
]
return []
def _get_simulator_cmd(self):
""" Locate the simulation binary """
if self.args.binary is None:
self.args.binary = BuildVars.BUILDDIR + "/tools/crm_simulate"
if not is_executable(self.args.binary):
self.args.binary = BuildVars.SBINDIR + "/crm_simulate"
if not is_executable(self.args.binary):
# @TODO it would be more pythonic to raise an exception
self._error("Test binary " + self.args.binary + " not found")
sys.exit(CrmExit.NOT_INSTALLED)
return [ self.args.binary ] + shlex.split(self.args.testcmd_options)
def set_schema_env(self):
""" Ensure schema directory environment variable is set, if possible """
try:
return os.environ['PCMK_schema_directory']
except KeyError:
for d in [ os.path.join(BuildVars.BUILDDIR, "xml"),
BuildVars.CRM_SCHEMA_DIRECTORY ]:
if os.path.isdir(d):
os.environ['PCMK_schema_directory'] = d
return d
return None
def __init__(self, argv=sys.argv):
self._parse_args(argv)
# Where this executable lives
self.test_home = os.path.dirname(os.path.realpath(argv[0]))
# Where test data resides
if self.args.io_dir is None:
self.args.io_dir = os.path.join(self.test_home, "scheduler")
# Where to store generated files
if self.args.out_dir is None:
self.args.out_dir = self.args.io_dir
self.failed_filename = os.path.join(self.test_home, ".regression.failed.diff")
else:
self.failed_filename = os.path.join(self.args.out_dir, ".regression.failed.diff")
os.environ['CIB_shadow_dir'] = self.args.out_dir
self.failed_file = None
# Single test mode (if requested)
try:
# User can give test base name or file name of a test input
self.args.run = os.path.splitext(os.path.basename(self.args.run))[0]
except (AttributeError, TypeError):
pass # --run was not specified
self.set_schema_env()
# Arguments needed (or not) to run commands
self.valgrind_args = self._get_valgrind_cmd()
self.simulate_args = self._get_simulator_cmd()
# Test counters
self.num_failed = 0
self.num_tests = 0
def _compare_files(self, filename1, filename2):
""" Add any file differences to failed results """
with io.open("/dev/null", "wt") as dev_null:
if diff(filename1, filename2, stdout=dev_null) != 0:
diff(filename1, filename2, stdout=self.failed_file, stderr=dev_null)
self.failed_file.write("\n");
return True
return False
def run_one(self, test_name, test_desc, test_args=[]):
""" Run one scheduler test """
print(" Test %-25s %s" % ((test_name + ":"), test_desc))
did_fail = False
self.num_tests = self.num_tests + 1
# Test inputs
input_filename = "%s/%s.xml" % (self.args.io_dir, test_name)
expected_filename = "%s/%s.exp" % (self.args.io_dir, test_name)
dot_expected_filename = "%s/%s.dot" % (self.args.io_dir, test_name)
scores_filename = "%s/%s.scores" % (self.args.io_dir, test_name)
summary_filename = "%s/%s.summary" % (self.args.io_dir, test_name)
stderr_expected_filename = "%s/%s.stderr" % (self.args.io_dir, test_name)
# (Intermediate) test outputs
output_filename = "%s/%s.out" % (self.args.out_dir, test_name)
dot_output_filename = "%s/%s.pe.dot" % (self.args.out_dir, test_name)
score_output_filename = "%s/%s.scores.pe" % (self.args.out_dir, test_name)
summary_output_filename = "%s/%s.summary.pe" % (self.args.out_dir, test_name)
stderr_output_filename = "%s/%s.stderr.pe" % (self.args.out_dir, test_name)
valgrind_output_filename = "%s/%s.valgrind" % (self.args.out_dir, test_name)
# Common arguments for running test
test_cmd = []
if self.valgrind_args:
test_cmd = self.valgrind_args + [ "--log-file=%s" % valgrind_output_filename ]
test_cmd = test_cmd + self.simulate_args
# @TODO It would be more pythonic to raise exceptions for errors,
# then perhaps it would be nice to make a single-test class
# Ensure necessary test inputs exist
if not os.path.isfile(input_filename):
self._error("No input")
self.num_failed = self.num_failed + 1
return CrmExit.NOINPUT
if not self.args.update and not os.path.isfile(expected_filename):
self._error("no stored output")
return CrmExit.NOINPUT
# Run simulation to generate summary output
if self.args.run: # Single test mode
test_cmd_full = test_cmd + [ '-x', input_filename, '-S' ] + test_args
print(" ".join(test_cmd_full))
else:
# @TODO Why isn't test_args added here?
test_cmd_full = test_cmd + [ '-x', input_filename, '-S' ]
with io.open(summary_output_filename, "wt") as f:
subprocess.call(test_cmd_full, stdout=f, stderr=subprocess.STDOUT, env=os.environ)
if self.args.run:
cat(summary_output_filename)
# Re-run simulation to generate dot, graph, and scores
test_cmd_full = test_cmd + [
'-x', input_filename,
'-D', dot_output_filename,
'-G', output_filename,
'-sSQ' ] + test_args
with io.open(stderr_output_filename, "wt") as f_stderr, \
io.open(score_output_filename, "wt") as f_score:
rc = subprocess.call(test_cmd_full, stdout=f_score, stderr=f_stderr, env=os.environ)
# Check for test command failure
if rc != CrmExit.OK:
self._failed("Test returned: %d" % rc)
did_fail = True
print(" ".join(test_cmd_full))
# Check for valgrind errors
if self.valgrind_args and not self.args.valgrind_skip_output:
if os.stat(valgrind_output_filename).st_size > 0:
self._failed("Valgrind reported errors")
did_fail = True
cat(valgrind_output_filename)
remove_files([ valgrind_output_filename ])
# Check for core dump
if os.path.isfile("core"):
self._failed("Core-file detected: core." + test_name)
did_fail = True
os.rename("core", "%s/core.%s" % (self.test_home, test_name))
# Check any stderr output
if os.path.isfile(stderr_expected_filename):
if self._compare_files(stderr_expected_filename, stderr_output_filename):
self._failed("stderr changed")
did_fail = True
elif os.stat(stderr_output_filename).st_size > 0:
self._failed("Output was written to stderr")
did_fail = True
cat(stderr_output_filename)
remove_files([ stderr_output_filename ])
# Check whether output graph exists, and normalize it
if (not os.path.isfile(output_filename)
or os.stat(output_filename).st_size == 0):
self._error("No graph produced")
did_fail = True
self.num_failed = self.num_failed + 1
remove_files([ output_filename ])
return CrmExit.ERROR
normalize(output_filename)
# Check whether dot output exists, and sort it
if (not os.path.isfile(dot_output_filename) or
os.stat(dot_output_filename).st_size == 0):
self._error("No dot-file summary produced")
did_fail = True
self.num_failed = self.num_failed + 1
remove_files([ dot_output_filename, output_filename ])
return CrmExit.ERROR
with io.open(dot_output_filename, "rt") as f:
first_line = f.readline() # "digraph" line with opening brace
lines = f.readlines()
last_line = lines[-1] # closing brace
del lines[-1]
lines = sorted(set(lines)) # unique sort
with io.open(dot_output_filename, "wt") as f:
f.write(first_line)
f.writelines(lines)
f.write(last_line)
# Check whether score output exists, and sort it
if (not os.path.isfile(score_output_filename)
or os.stat(score_output_filename).st_size == 0):
self._error("No allocation scores produced")
did_fail = True
self.num_failed = self.num_failed + 1
remove_files([ score_output_filename, output_filename ])
return CrmExit.ERROR
else:
sort_file(score_output_filename)
if self.args.update:
shutil.copyfile(output_filename, expected_filename)
shutil.copyfile(dot_output_filename, dot_expected_filename)
shutil.copyfile(score_output_filename, scores_filename)
shutil.copyfile(summary_output_filename, summary_filename)
print(" Updated expected outputs")
if self._compare_files(summary_filename, summary_output_filename):
self._failed("summary changed")
did_fail = True
if self._compare_files(dot_expected_filename, dot_output_filename):
self._failed("dot-file summary changed")
did_fail = True
else:
remove_files([ dot_output_filename ])
if self._compare_files(expected_filename, output_filename):
self._failed("xml-file changed")
did_fail = True
if self._compare_files(scores_filename, score_output_filename):
self._failed("scores-file changed")
did_fail = True
remove_files([ output_filename,
score_output_filename,
summary_output_filename])
if did_fail:
self.num_failed = self.num_failed + 1
return CrmExit.ERROR
return CrmExit.OK
def run_all(self):
""" Run all defined tests """
if platform.architecture()[0] == "64bit":
TESTS.extend(TESTS_64BIT)
for group in TESTS:
for test in group:
try:
args = test[2]
except IndexError:
args = []
self.run_one(test[0], test[1], args)
print()
def _print_summary(self):
""" Print a summary of parameters for this test run """
print("Test home is:\t" + self.test_home)
print("Test binary is:\t" + self.args.binary)
if 'PCMK_schema_directory' in os.environ:
print("Schema home is:\t" + os.environ['PCMK_schema_directory'])
if self.valgrind_args != []:
print("Activating memory testing with valgrind")
print()
def _test_results(self):
if self.num_failed == 0:
return CrmExit.OK
if os.path.isfile(self.failed_filename) and os.stat(self.failed_filename).st_size != 0:
if self.args.verbose:
self._error("Results of %d failed tests (out of %d):" %
(self.num_failed, self.num_tests))
cat(self.failed_filename)
else:
self._error("Results of %d failed tests (out of %d) are in %s" %
(self.num_failed, self.num_tests, self.failed_filename))
self._error("Use -V to display them after running the tests")
else:
self._error("%d (of %d) tests failed (no diff results)" %
(self.num_failed, self.num_tests))
if os.path.isfile(self.failed_filename):
os.remove(self.failed_filename)
return CrmExit.ERROR
def run(self):
""" Run test(s) as specified """
self._print_summary()
# Zero out the error log
self.failed_file = io.open(self.failed_filename, "wt")
if self.args.run is None:
print("Performing the following tests from " + self.args.io_dir)
print()
self.run_all()
print()
self.failed_file.close()
rc = self._test_results()
else:
rc = self.run_one(self.args.run, "Single shot", self.single_test_args)
self.failed_file.close()
cat(self.failed_filename)
return rc
if __name__ == "__main__":
sys.exit(CtsScheduler().run())
# vim: set filetype=python expandtab tabstop=4 softtabstop=4 shiftwidth=4 textwidth=120:
diff --git a/cts/scheduler/no_quorum_demote.dot b/cts/scheduler/no_quorum_demote.dot
new file mode 100644
index 0000000000..ea5b30c940
--- /dev/null
+++ b/cts/scheduler/no_quorum_demote.dot
@@ -0,0 +1,22 @@
+ digraph "g" {
+"Cancel rsc1_monitor_10000 rhel7-1" -> "rsc1_demote_0 rhel7-1" [ style = bold]
+"Cancel rsc1_monitor_10000 rhel7-1" [ style=bold color="green" fontcolor="black"]
+"Fencing_monitor_120000 rhel7-1" [ style=dashed color="red" fontcolor="black"]
+"Fencing_start_0 rhel7-1" -> "Fencing_monitor_120000 rhel7-1" [ style = dashed]
+"Fencing_start_0 rhel7-1" [ style=dashed color="red" fontcolor="black"]
+"Fencing_stop_0 rhel7-1" -> "Fencing_start_0 rhel7-1" [ style = dashed]
+"Fencing_stop_0 rhel7-1" [ style=bold color="green" fontcolor="black"]
+"rsc1-clone_demote_0" -> "rsc1-clone_demoted_0" [ style = bold]
+"rsc1-clone_demote_0" -> "rsc1_demote_0 rhel7-1" [ style = bold]
+"rsc1-clone_demote_0" [ style=bold color="green" fontcolor="orange"]
+"rsc1-clone_demoted_0" [ style=bold color="green" fontcolor="orange"]
+"rsc1_demote_0 rhel7-1" -> "rsc1-clone_demoted_0" [ style = bold]
+"rsc1_demote_0 rhel7-1" -> "rsc1_monitor_11000 rhel7-1" [ style = bold]
+"rsc1_demote_0 rhel7-1" [ style=bold color="green" fontcolor="black"]
+"rsc1_monitor_11000 rhel7-1" [ style=bold color="green" fontcolor="black"]
+"rsc2_monitor_10000 rhel7-2" [ style=dashed color="red" fontcolor="black"]
+"rsc2_start_0 rhel7-2" -> "rsc2_monitor_10000 rhel7-2" [ style = dashed]
+"rsc2_start_0 rhel7-2" [ style=dashed color="red" fontcolor="black"]
+"rsc2_stop_0 rhel7-2" -> "rsc2_start_0 rhel7-2" [ style = dashed]
+"rsc2_stop_0 rhel7-2" [ style=bold color="green" fontcolor="black"]
+}
diff --git a/cts/scheduler/no_quorum_demote.exp b/cts/scheduler/no_quorum_demote.exp
new file mode 100644
index 0000000000..245574c35d
--- /dev/null
+++ b/cts/scheduler/no_quorum_demote.exp
@@ -0,0 +1,81 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/cts/scheduler/no_quorum_demote.scores b/cts/scheduler/no_quorum_demote.scores
new file mode 100644
index 0000000000..dddc57b819
--- /dev/null
+++ b/cts/scheduler/no_quorum_demote.scores
@@ -0,0 +1,72 @@
+Allocation scores:
+Using the original execution date of: 2020-06-17 17:26:35Z
+pcmk__clone_allocate: rsc1-clone allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc1-clone allocation score on rhel7-2: 0
+pcmk__clone_allocate: rsc1-clone allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc1-clone allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc1-clone allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc1:0 allocation score on rhel7-1: 11
+pcmk__clone_allocate: rsc1:0 allocation score on rhel7-2: 0
+pcmk__clone_allocate: rsc1:0 allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc1:0 allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc1:0 allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc1:1 allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc1:1 allocation score on rhel7-2: 6
+pcmk__clone_allocate: rsc1:1 allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc1:1 allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc1:1 allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc1:2 allocation score on rhel7-1: 10
+pcmk__clone_allocate: rsc1:2 allocation score on rhel7-2: 5
+pcmk__clone_allocate: rsc1:2 allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc1:2 allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc1:2 allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc1:3 allocation score on rhel7-1: 10
+pcmk__clone_allocate: rsc1:3 allocation score on rhel7-2: 5
+pcmk__clone_allocate: rsc1:3 allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc1:3 allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc1:3 allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc1:4 allocation score on rhel7-1: 10
+pcmk__clone_allocate: rsc1:4 allocation score on rhel7-2: 5
+pcmk__clone_allocate: rsc1:4 allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc1:4 allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc1:4 allocation score on rhel7-5: 0
+pcmk__native_allocate: Fencing allocation score on rhel7-1: 0
+pcmk__native_allocate: Fencing allocation score on rhel7-2: 0
+pcmk__native_allocate: Fencing allocation score on rhel7-3: 0
+pcmk__native_allocate: Fencing allocation score on rhel7-4: 0
+pcmk__native_allocate: Fencing allocation score on rhel7-5: 0
+pcmk__native_allocate: rsc1:0 allocation score on rhel7-1: 11
+pcmk__native_allocate: rsc1:0 allocation score on rhel7-2: 0
+pcmk__native_allocate: rsc1:0 allocation score on rhel7-3: -INFINITY
+pcmk__native_allocate: rsc1:0 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: rsc1:0 allocation score on rhel7-5: -INFINITY
+pcmk__native_allocate: rsc1:1 allocation score on rhel7-1: -INFINITY
+pcmk__native_allocate: rsc1:1 allocation score on rhel7-2: 6
+pcmk__native_allocate: rsc1:1 allocation score on rhel7-3: -INFINITY
+pcmk__native_allocate: rsc1:1 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: rsc1:1 allocation score on rhel7-5: -INFINITY
+pcmk__native_allocate: rsc1:2 allocation score on rhel7-1: -INFINITY
+pcmk__native_allocate: rsc1:2 allocation score on rhel7-2: -INFINITY
+pcmk__native_allocate: rsc1:2 allocation score on rhel7-3: -INFINITY
+pcmk__native_allocate: rsc1:2 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: rsc1:2 allocation score on rhel7-5: -INFINITY
+pcmk__native_allocate: rsc1:3 allocation score on rhel7-1: -INFINITY
+pcmk__native_allocate: rsc1:3 allocation score on rhel7-2: -INFINITY
+pcmk__native_allocate: rsc1:3 allocation score on rhel7-3: -INFINITY
+pcmk__native_allocate: rsc1:3 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: rsc1:3 allocation score on rhel7-5: -INFINITY
+pcmk__native_allocate: rsc1:4 allocation score on rhel7-1: -INFINITY
+pcmk__native_allocate: rsc1:4 allocation score on rhel7-2: -INFINITY
+pcmk__native_allocate: rsc1:4 allocation score on rhel7-3: -INFINITY
+pcmk__native_allocate: rsc1:4 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: rsc1:4 allocation score on rhel7-5: -INFINITY
+pcmk__native_allocate: rsc2 allocation score on rhel7-1: 0
+pcmk__native_allocate: rsc2 allocation score on rhel7-2: 0
+pcmk__native_allocate: rsc2 allocation score on rhel7-3: 0
+pcmk__native_allocate: rsc2 allocation score on rhel7-4: 0
+pcmk__native_allocate: rsc2 allocation score on rhel7-5: 0
+rsc1:0 promotion score on rhel7-1: 10
+rsc1:1 promotion score on rhel7-2: 5
+rsc1:2 promotion score on none: 0
+rsc1:3 promotion score on none: 0
+rsc1:4 promotion score on none: 0
diff --git a/cts/scheduler/no_quorum_demote.summary b/cts/scheduler/no_quorum_demote.summary
new file mode 100644
index 0000000000..9b69ca15eb
--- /dev/null
+++ b/cts/scheduler/no_quorum_demote.summary
@@ -0,0 +1,38 @@
+Using the original execution date of: 2020-06-17 17:26:35Z
+
+Current cluster status:
+Online: [ rhel7-1 rhel7-2 ]
+OFFLINE: [ rhel7-3 rhel7-4 rhel7-5 ]
+
+ Fencing (stonith:fence_xvm): Started rhel7-1
+ Clone Set: rsc1-clone [rsc1] (promotable)
+ Masters: [ rhel7-1 ]
+ Slaves: [ rhel7-2 ]
+ Stopped: [ rhel7-3 rhel7-4 rhel7-5 ]
+ rsc2 (ocf::pacemaker:Dummy): Started rhel7-2
+
+Transition Summary:
+ * Stop Fencing ( rhel7-1 ) due to no quorum
+ * Demote rsc1:0 ( Master -> Slave rhel7-1 )
+ * Stop rsc2 ( rhel7-2 ) due to no quorum
+
+Executing cluster transition:
+ * Resource action: Fencing stop on rhel7-1
+ * Resource action: rsc1 cancel=10000 on rhel7-1
+ * Pseudo action: rsc1-clone_demote_0
+ * Resource action: rsc2 stop on rhel7-2
+ * Resource action: rsc1 demote on rhel7-1
+ * Pseudo action: rsc1-clone_demoted_0
+ * Resource action: rsc1 monitor=11000 on rhel7-1
+Using the original execution date of: 2020-06-17 17:26:35Z
+
+Revised cluster status:
+Online: [ rhel7-1 rhel7-2 ]
+OFFLINE: [ rhel7-3 rhel7-4 rhel7-5 ]
+
+ Fencing (stonith:fence_xvm): Stopped
+ Clone Set: rsc1-clone [rsc1] (promotable)
+ Slaves: [ rhel7-1 rhel7-2 ]
+ Stopped: [ rhel7-3 rhel7-4 rhel7-5 ]
+ rsc2 (ocf::pacemaker:Dummy): Stopped
+
diff --git a/cts/scheduler/no_quorum_demote.xml b/cts/scheduler/no_quorum_demote.xml
new file mode 100644
index 0000000000..8497f0a244
--- /dev/null
+++ b/cts/scheduler/no_quorum_demote.xml
@@ -0,0 +1,224 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/cts/scheduler/on_fail_demote1.dot b/cts/scheduler/on_fail_demote1.dot
new file mode 100644
index 0000000000..d11c1c15a4
--- /dev/null
+++ b/cts/scheduler/on_fail_demote1.dot
@@ -0,0 +1,64 @@
+ digraph "g" {
+"bundled_demote_0 stateful-bundle-0" -> "bundled_promote_0 stateful-bundle-0" [ style = bold]
+"bundled_demote_0 stateful-bundle-0" -> "stateful-bundle-master_demoted_0" [ style = bold]
+"bundled_demote_0 stateful-bundle-0" [ style=bold color="green" fontcolor="black"]
+"bundled_promote_0 stateful-bundle-0" -> "stateful-bundle-master_promoted_0" [ style = bold]
+"bundled_promote_0 stateful-bundle-0" [ style=bold color="green" fontcolor="black"]
+"lxc-ms-master_demote_0" -> "lxc-ms-master_demoted_0" [ style = bold]
+"lxc-ms-master_demote_0" -> "lxc-ms_demote_0 lxc2" [ style = bold]
+"lxc-ms-master_demote_0" [ style=bold color="green" fontcolor="orange"]
+"lxc-ms-master_demoted_0" -> "lxc-ms-master_promote_0" [ style = bold]
+"lxc-ms-master_demoted_0" [ style=bold color="green" fontcolor="orange"]
+"lxc-ms-master_promote_0" -> "lxc-ms_promote_0 lxc2" [ style = bold]
+"lxc-ms-master_promote_0" [ style=bold color="green" fontcolor="orange"]
+"lxc-ms-master_promoted_0" [ style=bold color="green" fontcolor="orange"]
+"lxc-ms_demote_0 lxc2" -> "lxc-ms-master_demoted_0" [ style = bold]
+"lxc-ms_demote_0 lxc2" -> "lxc-ms_promote_0 lxc2" [ style = bold]
+"lxc-ms_demote_0 lxc2" [ style=bold color="green" fontcolor="black"]
+"lxc-ms_promote_0 lxc2" -> "lxc-ms-master_promoted_0" [ style = bold]
+"lxc-ms_promote_0 lxc2" [ style=bold color="green" fontcolor="black"]
+"rsc1-clone_demote_0" -> "rsc1-clone_demoted_0" [ style = bold]
+"rsc1-clone_demote_0" -> "rsc1_demote_0 rhel7-4" [ style = bold]
+"rsc1-clone_demote_0" [ style=bold color="green" fontcolor="orange"]
+"rsc1-clone_demoted_0" -> "rsc1-clone_promote_0" [ style = bold]
+"rsc1-clone_demoted_0" [ style=bold color="green" fontcolor="orange"]
+"rsc1-clone_promote_0" -> "rsc1_promote_0 rhel7-4" [ style = bold]
+"rsc1-clone_promote_0" [ style=bold color="green" fontcolor="orange"]
+"rsc1-clone_promoted_0" [ style=bold color="green" fontcolor="orange"]
+"rsc1_demote_0 rhel7-4" -> "rsc1-clone_demoted_0" [ style = bold]
+"rsc1_demote_0 rhel7-4" -> "rsc1_promote_0 rhel7-4" [ style = bold]
+"rsc1_demote_0 rhel7-4" [ style=bold color="green" fontcolor="black"]
+"rsc1_promote_0 rhel7-4" -> "rsc1-clone_promoted_0" [ style = bold]
+"rsc1_promote_0 rhel7-4" [ style=bold color="green" fontcolor="black"]
+"rsc2-master_demote_0" -> "rsc2-master_demoted_0" [ style = bold]
+"rsc2-master_demote_0" -> "rsc2_demote_0 remote-rhel7-2" [ style = bold]
+"rsc2-master_demote_0" [ style=bold color="green" fontcolor="orange"]
+"rsc2-master_demoted_0" -> "rsc2-master_promote_0" [ style = bold]
+"rsc2-master_demoted_0" [ style=bold color="green" fontcolor="orange"]
+"rsc2-master_promote_0" -> "rsc2_promote_0 remote-rhel7-2" [ style = bold]
+"rsc2-master_promote_0" [ style=bold color="green" fontcolor="orange"]
+"rsc2-master_promoted_0" [ style=bold color="green" fontcolor="orange"]
+"rsc2_demote_0 remote-rhel7-2" -> "rsc2-master_demoted_0" [ style = bold]
+"rsc2_demote_0 remote-rhel7-2" -> "rsc2_promote_0 remote-rhel7-2" [ style = bold]
+"rsc2_demote_0 remote-rhel7-2" [ style=bold color="green" fontcolor="black"]
+"rsc2_promote_0 remote-rhel7-2" -> "rsc2-master_promoted_0" [ style = bold]
+"rsc2_promote_0 remote-rhel7-2" [ style=bold color="green" fontcolor="black"]
+"stateful-bundle-master_demote_0" -> "bundled_demote_0 stateful-bundle-0" [ style = bold]
+"stateful-bundle-master_demote_0" -> "stateful-bundle-master_demoted_0" [ style = bold]
+"stateful-bundle-master_demote_0" [ style=bold color="green" fontcolor="orange"]
+"stateful-bundle-master_demoted_0" -> "stateful-bundle-master_promote_0" [ style = bold]
+"stateful-bundle-master_demoted_0" -> "stateful-bundle_demoted_0" [ style = bold]
+"stateful-bundle-master_demoted_0" [ style=bold color="green" fontcolor="orange"]
+"stateful-bundle-master_promote_0" -> "bundled_promote_0 stateful-bundle-0" [ style = bold]
+"stateful-bundle-master_promote_0" [ style=bold color="green" fontcolor="orange"]
+"stateful-bundle-master_promoted_0" -> "stateful-bundle_promoted_0" [ style = bold]
+"stateful-bundle-master_promoted_0" [ style=bold color="green" fontcolor="orange"]
+"stateful-bundle_demote_0" -> "stateful-bundle-master_demote_0" [ style = bold]
+"stateful-bundle_demote_0" -> "stateful-bundle_demoted_0" [ style = bold]
+"stateful-bundle_demote_0" [ style=bold color="green" fontcolor="orange"]
+"stateful-bundle_demoted_0" -> "stateful-bundle_promote_0" [ style = bold]
+"stateful-bundle_demoted_0" [ style=bold color="green" fontcolor="orange"]
+"stateful-bundle_promote_0" -> "stateful-bundle-master_promote_0" [ style = bold]
+"stateful-bundle_promote_0" [ style=bold color="green" fontcolor="orange"]
+"stateful-bundle_promoted_0" [ style=bold color="green" fontcolor="orange"]
+}
diff --git a/cts/scheduler/on_fail_demote1.exp b/cts/scheduler/on_fail_demote1.exp
new file mode 100644
index 0000000000..ebe1dd59ec
--- /dev/null
+++ b/cts/scheduler/on_fail_demote1.exp
@@ -0,0 +1,360 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/cts/scheduler/on_fail_demote1.scores b/cts/scheduler/on_fail_demote1.scores
new file mode 100644
index 0000000000..7df582f0b5
--- /dev/null
+++ b/cts/scheduler/on_fail_demote1.scores
@@ -0,0 +1,470 @@
+Allocation scores:
+Using the original execution date of: 2020-06-16 19:23:21Z
+bundled:0 promotion score on stateful-bundle-0: 10
+bundled:1 promotion score on stateful-bundle-1: 5
+bundled:2 promotion score on stateful-bundle-2: 5
+lxc-ms:0 promotion score on lxc2: INFINITY
+lxc-ms:1 promotion score on lxc1: INFINITY
+pcmk__bundle_allocate: bundled:0 allocation score on stateful-bundle-0: 501
+pcmk__bundle_allocate: bundled:1 allocation score on stateful-bundle-1: 501
+pcmk__bundle_allocate: bundled:2 allocation score on stateful-bundle-2: 501
+pcmk__bundle_allocate: stateful-bundle allocation score on lxc1: 0
+pcmk__bundle_allocate: stateful-bundle allocation score on lxc2: 0
+pcmk__bundle_allocate: stateful-bundle allocation score on remote-rhel7-2: 0
+pcmk__bundle_allocate: stateful-bundle allocation score on rhel7-1: 0
+pcmk__bundle_allocate: stateful-bundle allocation score on rhel7-3: 0
+pcmk__bundle_allocate: stateful-bundle allocation score on rhel7-4: 0
+pcmk__bundle_allocate: stateful-bundle allocation score on rhel7-5: 0
+pcmk__bundle_allocate: stateful-bundle-0 allocation score on lxc1: -INFINITY
+pcmk__bundle_allocate: stateful-bundle-0 allocation score on lxc2: -INFINITY
+pcmk__bundle_allocate: stateful-bundle-0 allocation score on remote-rhel7-2: -INFINITY
+pcmk__bundle_allocate: stateful-bundle-0 allocation score on rhel7-1: 0
+pcmk__bundle_allocate: stateful-bundle-0 allocation score on rhel7-3: 0
+pcmk__bundle_allocate: stateful-bundle-0 allocation score on rhel7-4: 0
+pcmk__bundle_allocate: stateful-bundle-0 allocation score on rhel7-5: 0
+pcmk__bundle_allocate: stateful-bundle-1 allocation score on lxc1: -INFINITY
+pcmk__bundle_allocate: stateful-bundle-1 allocation score on lxc2: -INFINITY
+pcmk__bundle_allocate: stateful-bundle-1 allocation score on remote-rhel7-2: -INFINITY
+pcmk__bundle_allocate: stateful-bundle-1 allocation score on rhel7-1: 0
+pcmk__bundle_allocate: stateful-bundle-1 allocation score on rhel7-3: 0
+pcmk__bundle_allocate: stateful-bundle-1 allocation score on rhel7-4: 0
+pcmk__bundle_allocate: stateful-bundle-1 allocation score on rhel7-5: 0
+pcmk__bundle_allocate: stateful-bundle-2 allocation score on lxc1: -INFINITY
+pcmk__bundle_allocate: stateful-bundle-2 allocation score on lxc2: -INFINITY
+pcmk__bundle_allocate: stateful-bundle-2 allocation score on remote-rhel7-2: -INFINITY
+pcmk__bundle_allocate: stateful-bundle-2 allocation score on rhel7-1: 0
+pcmk__bundle_allocate: stateful-bundle-2 allocation score on rhel7-3: 0
+pcmk__bundle_allocate: stateful-bundle-2 allocation score on rhel7-4: 0
+pcmk__bundle_allocate: stateful-bundle-2 allocation score on rhel7-5: 0
+pcmk__bundle_allocate: stateful-bundle-docker-0 allocation score on lxc1: 0
+pcmk__bundle_allocate: stateful-bundle-docker-0 allocation score on lxc2: 0
+pcmk__bundle_allocate: stateful-bundle-docker-0 allocation score on remote-rhel7-2: 0
+pcmk__bundle_allocate: stateful-bundle-docker-0 allocation score on rhel7-1: 0
+pcmk__bundle_allocate: stateful-bundle-docker-0 allocation score on rhel7-3: 0
+pcmk__bundle_allocate: stateful-bundle-docker-0 allocation score on rhel7-4: 0
+pcmk__bundle_allocate: stateful-bundle-docker-0 allocation score on rhel7-5: 0
+pcmk__bundle_allocate: stateful-bundle-docker-1 allocation score on lxc1: 0
+pcmk__bundle_allocate: stateful-bundle-docker-1 allocation score on lxc2: 0
+pcmk__bundle_allocate: stateful-bundle-docker-1 allocation score on remote-rhel7-2: 0
+pcmk__bundle_allocate: stateful-bundle-docker-1 allocation score on rhel7-1: 0
+pcmk__bundle_allocate: stateful-bundle-docker-1 allocation score on rhel7-3: 0
+pcmk__bundle_allocate: stateful-bundle-docker-1 allocation score on rhel7-4: 0
+pcmk__bundle_allocate: stateful-bundle-docker-1 allocation score on rhel7-5: 0
+pcmk__bundle_allocate: stateful-bundle-docker-2 allocation score on lxc1: 0
+pcmk__bundle_allocate: stateful-bundle-docker-2 allocation score on lxc2: 0
+pcmk__bundle_allocate: stateful-bundle-docker-2 allocation score on remote-rhel7-2: 0
+pcmk__bundle_allocate: stateful-bundle-docker-2 allocation score on rhel7-1: 0
+pcmk__bundle_allocate: stateful-bundle-docker-2 allocation score on rhel7-3: 0
+pcmk__bundle_allocate: stateful-bundle-docker-2 allocation score on rhel7-4: 0
+pcmk__bundle_allocate: stateful-bundle-docker-2 allocation score on rhel7-5: 0
+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.131 allocation score on lxc1: 0
+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.131 allocation score on lxc2: 0
+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.131 allocation score on remote-rhel7-2: 0
+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.131 allocation score on rhel7-1: 0
+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.131 allocation score on rhel7-3: 0
+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.131 allocation score on rhel7-4: 0
+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.131 allocation score on rhel7-5: 0
+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.132 allocation score on lxc1: 0
+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.132 allocation score on lxc2: 0
+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.132 allocation score on remote-rhel7-2: 0
+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.132 allocation score on rhel7-1: 0
+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.132 allocation score on rhel7-3: 0
+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.132 allocation score on rhel7-4: 0
+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.132 allocation score on rhel7-5: 0
+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.133 allocation score on lxc1: 0
+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.133 allocation score on lxc2: 0
+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.133 allocation score on remote-rhel7-2: 0
+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.133 allocation score on rhel7-1: 0
+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.133 allocation score on rhel7-3: 0
+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.133 allocation score on rhel7-4: 0
+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.133 allocation score on rhel7-5: 0
+pcmk__bundle_allocate: stateful-bundle-master allocation score on lxc1: 0
+pcmk__bundle_allocate: stateful-bundle-master allocation score on lxc2: 0
+pcmk__bundle_allocate: stateful-bundle-master allocation score on remote-rhel7-2: 0
+pcmk__bundle_allocate: stateful-bundle-master allocation score on rhel7-1: 0
+pcmk__bundle_allocate: stateful-bundle-master allocation score on rhel7-3: 0
+pcmk__bundle_allocate: stateful-bundle-master allocation score on rhel7-4: 0
+pcmk__bundle_allocate: stateful-bundle-master allocation score on rhel7-5: 0
+pcmk__bundle_allocate: stateful-bundle-master allocation score on stateful-bundle-0: -INFINITY
+pcmk__bundle_allocate: stateful-bundle-master allocation score on stateful-bundle-1: -INFINITY
+pcmk__bundle_allocate: stateful-bundle-master allocation score on stateful-bundle-2: -INFINITY
+pcmk__clone_allocate: bundled:0 allocation score on stateful-bundle-0: INFINITY
+pcmk__clone_allocate: bundled:1 allocation score on stateful-bundle-1: INFINITY
+pcmk__clone_allocate: bundled:2 allocation score on stateful-bundle-2: INFINITY
+pcmk__clone_allocate: lxc-ms-master allocation score on lxc1: INFINITY
+pcmk__clone_allocate: lxc-ms-master allocation score on lxc2: INFINITY
+pcmk__clone_allocate: lxc-ms-master allocation score on remote-rhel7-2: 0
+pcmk__clone_allocate: lxc-ms-master allocation score on rhel7-1: 0
+pcmk__clone_allocate: lxc-ms-master allocation score on rhel7-3: 0
+pcmk__clone_allocate: lxc-ms-master allocation score on rhel7-4: 0
+pcmk__clone_allocate: lxc-ms-master allocation score on rhel7-5: 0
+pcmk__clone_allocate: lxc-ms:0 allocation score on lxc1: INFINITY
+pcmk__clone_allocate: lxc-ms:0 allocation score on lxc2: INFINITY
+pcmk__clone_allocate: lxc-ms:0 allocation score on remote-rhel7-2: 0
+pcmk__clone_allocate: lxc-ms:0 allocation score on rhel7-1: 0
+pcmk__clone_allocate: lxc-ms:0 allocation score on rhel7-3: 0
+pcmk__clone_allocate: lxc-ms:0 allocation score on rhel7-4: 0
+pcmk__clone_allocate: lxc-ms:0 allocation score on rhel7-5: 0
+pcmk__clone_allocate: lxc-ms:1 allocation score on lxc1: INFINITY
+pcmk__clone_allocate: lxc-ms:1 allocation score on lxc2: INFINITY
+pcmk__clone_allocate: lxc-ms:1 allocation score on remote-rhel7-2: 0
+pcmk__clone_allocate: lxc-ms:1 allocation score on rhel7-1: 0
+pcmk__clone_allocate: lxc-ms:1 allocation score on rhel7-3: 0
+pcmk__clone_allocate: lxc-ms:1 allocation score on rhel7-4: 0
+pcmk__clone_allocate: lxc-ms:1 allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc1-clone allocation score on lxc1: 0
+pcmk__clone_allocate: rsc1-clone allocation score on lxc2: 0
+pcmk__clone_allocate: rsc1-clone allocation score on remote-rhel7-2: 0
+pcmk__clone_allocate: rsc1-clone allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc1-clone allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc1-clone allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc1-clone allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc1:0 allocation score on lxc1: 0
+pcmk__clone_allocate: rsc1:0 allocation score on lxc2: 0
+pcmk__clone_allocate: rsc1:0 allocation score on remote-rhel7-2: 0
+pcmk__clone_allocate: rsc1:0 allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc1:0 allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc1:0 allocation score on rhel7-4: 11
+pcmk__clone_allocate: rsc1:0 allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc1:1 allocation score on lxc1: 0
+pcmk__clone_allocate: rsc1:1 allocation score on lxc2: 0
+pcmk__clone_allocate: rsc1:1 allocation score on remote-rhel7-2: 0
+pcmk__clone_allocate: rsc1:1 allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc1:1 allocation score on rhel7-3: 6
+pcmk__clone_allocate: rsc1:1 allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc1:1 allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc1:2 allocation score on lxc1: 0
+pcmk__clone_allocate: rsc1:2 allocation score on lxc2: 0
+pcmk__clone_allocate: rsc1:2 allocation score on remote-rhel7-2: 0
+pcmk__clone_allocate: rsc1:2 allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc1:2 allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc1:2 allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc1:2 allocation score on rhel7-5: 6
+pcmk__clone_allocate: rsc1:3 allocation score on lxc1: 0
+pcmk__clone_allocate: rsc1:3 allocation score on lxc2: 0
+pcmk__clone_allocate: rsc1:3 allocation score on remote-rhel7-2: 0
+pcmk__clone_allocate: rsc1:3 allocation score on rhel7-1: 6
+pcmk__clone_allocate: rsc1:3 allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc1:3 allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc1:3 allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc1:4 allocation score on lxc1: 0
+pcmk__clone_allocate: rsc1:4 allocation score on lxc2: 0
+pcmk__clone_allocate: rsc1:4 allocation score on remote-rhel7-2: 6
+pcmk__clone_allocate: rsc1:4 allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc1:4 allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc1:4 allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc1:4 allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc1:5 allocation score on lxc1: 0
+pcmk__clone_allocate: rsc1:5 allocation score on lxc2: 6
+pcmk__clone_allocate: rsc1:5 allocation score on remote-rhel7-2: 0
+pcmk__clone_allocate: rsc1:5 allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc1:5 allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc1:5 allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc1:5 allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc1:6 allocation score on lxc1: 6
+pcmk__clone_allocate: rsc1:6 allocation score on lxc2: 0
+pcmk__clone_allocate: rsc1:6 allocation score on remote-rhel7-2: 0
+pcmk__clone_allocate: rsc1:6 allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc1:6 allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc1:6 allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc1:6 allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc2-master allocation score on lxc1: 0
+pcmk__clone_allocate: rsc2-master allocation score on lxc2: 0
+pcmk__clone_allocate: rsc2-master allocation score on remote-rhel7-2: 0
+pcmk__clone_allocate: rsc2-master allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc2-master allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc2-master allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc2-master allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc2:0 allocation score on lxc1: 0
+pcmk__clone_allocate: rsc2:0 allocation score on lxc2: 0
+pcmk__clone_allocate: rsc2:0 allocation score on remote-rhel7-2: 0
+pcmk__clone_allocate: rsc2:0 allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc2:0 allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc2:0 allocation score on rhel7-4: 11
+pcmk__clone_allocate: rsc2:0 allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc2:1 allocation score on lxc1: 0
+pcmk__clone_allocate: rsc2:1 allocation score on lxc2: 0
+pcmk__clone_allocate: rsc2:1 allocation score on remote-rhel7-2: 0
+pcmk__clone_allocate: rsc2:1 allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc2:1 allocation score on rhel7-3: 6
+pcmk__clone_allocate: rsc2:1 allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc2:1 allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc2:2 allocation score on lxc1: 0
+pcmk__clone_allocate: rsc2:2 allocation score on lxc2: 0
+pcmk__clone_allocate: rsc2:2 allocation score on remote-rhel7-2: 0
+pcmk__clone_allocate: rsc2:2 allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc2:2 allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc2:2 allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc2:2 allocation score on rhel7-5: 6
+pcmk__clone_allocate: rsc2:3 allocation score on lxc1: 0
+pcmk__clone_allocate: rsc2:3 allocation score on lxc2: 0
+pcmk__clone_allocate: rsc2:3 allocation score on remote-rhel7-2: 0
+pcmk__clone_allocate: rsc2:3 allocation score on rhel7-1: 6
+pcmk__clone_allocate: rsc2:3 allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc2:3 allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc2:3 allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc2:4 allocation score on lxc1: 0
+pcmk__clone_allocate: rsc2:4 allocation score on lxc2: 0
+pcmk__clone_allocate: rsc2:4 allocation score on remote-rhel7-2: 11
+pcmk__clone_allocate: rsc2:4 allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc2:4 allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc2:4 allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc2:4 allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc2:5 allocation score on lxc1: 0
+pcmk__clone_allocate: rsc2:5 allocation score on lxc2: 6
+pcmk__clone_allocate: rsc2:5 allocation score on remote-rhel7-2: 0
+pcmk__clone_allocate: rsc2:5 allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc2:5 allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc2:5 allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc2:5 allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc2:6 allocation score on lxc1: 6
+pcmk__clone_allocate: rsc2:6 allocation score on lxc2: 0
+pcmk__clone_allocate: rsc2:6 allocation score on remote-rhel7-2: 0
+pcmk__clone_allocate: rsc2:6 allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc2:6 allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc2:6 allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc2:6 allocation score on rhel7-5: 0
+pcmk__clone_allocate: stateful-bundle-master allocation score on lxc1: -INFINITY
+pcmk__clone_allocate: stateful-bundle-master allocation score on lxc2: -INFINITY
+pcmk__clone_allocate: stateful-bundle-master allocation score on remote-rhel7-2: -INFINITY
+pcmk__clone_allocate: stateful-bundle-master allocation score on rhel7-1: -INFINITY
+pcmk__clone_allocate: stateful-bundle-master allocation score on rhel7-3: -INFINITY
+pcmk__clone_allocate: stateful-bundle-master allocation score on rhel7-4: -INFINITY
+pcmk__clone_allocate: stateful-bundle-master allocation score on rhel7-5: -INFINITY
+pcmk__clone_allocate: stateful-bundle-master allocation score on stateful-bundle-0: 0
+pcmk__clone_allocate: stateful-bundle-master allocation score on stateful-bundle-1: 0
+pcmk__clone_allocate: stateful-bundle-master allocation score on stateful-bundle-2: 0
+pcmk__native_allocate: Fencing allocation score on lxc1: -INFINITY
+pcmk__native_allocate: Fencing allocation score on lxc2: -INFINITY
+pcmk__native_allocate: Fencing allocation score on remote-rhel7-2: -INFINITY
+pcmk__native_allocate: Fencing allocation score on rhel7-1: 0
+pcmk__native_allocate: Fencing allocation score on rhel7-3: 0
+pcmk__native_allocate: Fencing allocation score on rhel7-4: 0
+pcmk__native_allocate: Fencing allocation score on rhel7-5: 0
+pcmk__native_allocate: bundled:0 allocation score on stateful-bundle-0: INFINITY
+pcmk__native_allocate: bundled:1 allocation score on stateful-bundle-1: INFINITY
+pcmk__native_allocate: bundled:2 allocation score on stateful-bundle-2: INFINITY
+pcmk__native_allocate: container1 allocation score on lxc1: -INFINITY
+pcmk__native_allocate: container1 allocation score on lxc2: -INFINITY
+pcmk__native_allocate: container1 allocation score on remote-rhel7-2: -INFINITY
+pcmk__native_allocate: container1 allocation score on rhel7-1: 0
+pcmk__native_allocate: container1 allocation score on rhel7-3: INFINITY
+pcmk__native_allocate: container1 allocation score on rhel7-4: 0
+pcmk__native_allocate: container1 allocation score on rhel7-5: 0
+pcmk__native_allocate: container2 allocation score on lxc1: -INFINITY
+pcmk__native_allocate: container2 allocation score on lxc2: -INFINITY
+pcmk__native_allocate: container2 allocation score on remote-rhel7-2: -INFINITY
+pcmk__native_allocate: container2 allocation score on rhel7-1: 0
+pcmk__native_allocate: container2 allocation score on rhel7-3: INFINITY
+pcmk__native_allocate: container2 allocation score on rhel7-4: 0
+pcmk__native_allocate: container2 allocation score on rhel7-5: 0
+pcmk__native_allocate: lxc-ms:0 allocation score on lxc1: -INFINITY
+pcmk__native_allocate: lxc-ms:0 allocation score on lxc2: INFINITY
+pcmk__native_allocate: lxc-ms:0 allocation score on remote-rhel7-2: 0
+pcmk__native_allocate: lxc-ms:0 allocation score on rhel7-1: 0
+pcmk__native_allocate: lxc-ms:0 allocation score on rhel7-3: 0
+pcmk__native_allocate: lxc-ms:0 allocation score on rhel7-4: 0
+pcmk__native_allocate: lxc-ms:0 allocation score on rhel7-5: 0
+pcmk__native_allocate: lxc-ms:1 allocation score on lxc1: INFINITY
+pcmk__native_allocate: lxc-ms:1 allocation score on lxc2: INFINITY
+pcmk__native_allocate: lxc-ms:1 allocation score on remote-rhel7-2: 0
+pcmk__native_allocate: lxc-ms:1 allocation score on rhel7-1: 0
+pcmk__native_allocate: lxc-ms:1 allocation score on rhel7-3: 0
+pcmk__native_allocate: lxc-ms:1 allocation score on rhel7-4: 0
+pcmk__native_allocate: lxc-ms:1 allocation score on rhel7-5: 0
+pcmk__native_allocate: lxc1 allocation score on lxc1: -INFINITY
+pcmk__native_allocate: lxc1 allocation score on lxc2: -INFINITY
+pcmk__native_allocate: lxc1 allocation score on remote-rhel7-2: -INFINITY
+pcmk__native_allocate: lxc1 allocation score on rhel7-1: -INFINITY
+pcmk__native_allocate: lxc1 allocation score on rhel7-3: 0
+pcmk__native_allocate: lxc1 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: lxc1 allocation score on rhel7-5: -INFINITY
+pcmk__native_allocate: lxc2 allocation score on lxc1: -INFINITY
+pcmk__native_allocate: lxc2 allocation score on lxc2: -INFINITY
+pcmk__native_allocate: lxc2 allocation score on remote-rhel7-2: -INFINITY
+pcmk__native_allocate: lxc2 allocation score on rhel7-1: -INFINITY
+pcmk__native_allocate: lxc2 allocation score on rhel7-3: 0
+pcmk__native_allocate: lxc2 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: lxc2 allocation score on rhel7-5: -INFINITY
+pcmk__native_allocate: remote-rhel7-2 allocation score on lxc1: -INFINITY
+pcmk__native_allocate: remote-rhel7-2 allocation score on lxc2: -INFINITY
+pcmk__native_allocate: remote-rhel7-2 allocation score on remote-rhel7-2: -INFINITY
+pcmk__native_allocate: remote-rhel7-2 allocation score on rhel7-1: 0
+pcmk__native_allocate: remote-rhel7-2 allocation score on rhel7-3: 0
+pcmk__native_allocate: remote-rhel7-2 allocation score on rhel7-4: 0
+pcmk__native_allocate: remote-rhel7-2 allocation score on rhel7-5: 0
+pcmk__native_allocate: rsc1:0 allocation score on lxc1: -INFINITY
+pcmk__native_allocate: rsc1:0 allocation score on lxc2: -INFINITY
+pcmk__native_allocate: rsc1:0 allocation score on remote-rhel7-2: -INFINITY
+pcmk__native_allocate: rsc1:0 allocation score on rhel7-1: -INFINITY
+pcmk__native_allocate: rsc1:0 allocation score on rhel7-3: -INFINITY
+pcmk__native_allocate: rsc1:0 allocation score on rhel7-4: 11
+pcmk__native_allocate: rsc1:0 allocation score on rhel7-5: -INFINITY
+pcmk__native_allocate: rsc1:1 allocation score on lxc1: 0
+pcmk__native_allocate: rsc1:1 allocation score on lxc2: 0
+pcmk__native_allocate: rsc1:1 allocation score on remote-rhel7-2: 0
+pcmk__native_allocate: rsc1:1 allocation score on rhel7-1: 0
+pcmk__native_allocate: rsc1:1 allocation score on rhel7-3: 6
+pcmk__native_allocate: rsc1:1 allocation score on rhel7-4: 0
+pcmk__native_allocate: rsc1:1 allocation score on rhel7-5: 0
+pcmk__native_allocate: rsc1:2 allocation score on lxc1: 0
+pcmk__native_allocate: rsc1:2 allocation score on lxc2: 0
+pcmk__native_allocate: rsc1:2 allocation score on remote-rhel7-2: 0
+pcmk__native_allocate: rsc1:2 allocation score on rhel7-1: 0
+pcmk__native_allocate: rsc1:2 allocation score on rhel7-3: -INFINITY
+pcmk__native_allocate: rsc1:2 allocation score on rhel7-4: 0
+pcmk__native_allocate: rsc1:2 allocation score on rhel7-5: 6
+pcmk__native_allocate: rsc1:3 allocation score on lxc1: 0
+pcmk__native_allocate: rsc1:3 allocation score on lxc2: 0
+pcmk__native_allocate: rsc1:3 allocation score on remote-rhel7-2: 0
+pcmk__native_allocate: rsc1:3 allocation score on rhel7-1: 6
+pcmk__native_allocate: rsc1:3 allocation score on rhel7-3: -INFINITY
+pcmk__native_allocate: rsc1:3 allocation score on rhel7-4: 0
+pcmk__native_allocate: rsc1:3 allocation score on rhel7-5: -INFINITY
+pcmk__native_allocate: rsc1:4 allocation score on lxc1: 0
+pcmk__native_allocate: rsc1:4 allocation score on lxc2: 0
+pcmk__native_allocate: rsc1:4 allocation score on remote-rhel7-2: 6
+pcmk__native_allocate: rsc1:4 allocation score on rhel7-1: -INFINITY
+pcmk__native_allocate: rsc1:4 allocation score on rhel7-3: -INFINITY
+pcmk__native_allocate: rsc1:4 allocation score on rhel7-4: 0
+pcmk__native_allocate: rsc1:4 allocation score on rhel7-5: -INFINITY
+pcmk__native_allocate: rsc1:5 allocation score on lxc1: 0
+pcmk__native_allocate: rsc1:5 allocation score on lxc2: 6
+pcmk__native_allocate: rsc1:5 allocation score on remote-rhel7-2: -INFINITY
+pcmk__native_allocate: rsc1:5 allocation score on rhel7-1: -INFINITY
+pcmk__native_allocate: rsc1:5 allocation score on rhel7-3: -INFINITY
+pcmk__native_allocate: rsc1:5 allocation score on rhel7-4: 0
+pcmk__native_allocate: rsc1:5 allocation score on rhel7-5: -INFINITY
+pcmk__native_allocate: rsc1:6 allocation score on lxc1: 6
+pcmk__native_allocate: rsc1:6 allocation score on lxc2: -INFINITY
+pcmk__native_allocate: rsc1:6 allocation score on remote-rhel7-2: -INFINITY
+pcmk__native_allocate: rsc1:6 allocation score on rhel7-1: -INFINITY
+pcmk__native_allocate: rsc1:6 allocation score on rhel7-3: -INFINITY
+pcmk__native_allocate: rsc1:6 allocation score on rhel7-4: 0
+pcmk__native_allocate: rsc1:6 allocation score on rhel7-5: -INFINITY
+pcmk__native_allocate: rsc2:0 allocation score on lxc1: 0
+pcmk__native_allocate: rsc2:0 allocation score on lxc2: 0
+pcmk__native_allocate: rsc2:0 allocation score on remote-rhel7-2: 0
+pcmk__native_allocate: rsc2:0 allocation score on rhel7-1: 0
+pcmk__native_allocate: rsc2:0 allocation score on rhel7-3: 0
+pcmk__native_allocate: rsc2:0 allocation score on rhel7-4: 11
+pcmk__native_allocate: rsc2:0 allocation score on rhel7-5: 0
+pcmk__native_allocate: rsc2:1 allocation score on lxc1: 0
+pcmk__native_allocate: rsc2:1 allocation score on lxc2: 0
+pcmk__native_allocate: rsc2:1 allocation score on remote-rhel7-2: 0
+pcmk__native_allocate: rsc2:1 allocation score on rhel7-1: 0
+pcmk__native_allocate: rsc2:1 allocation score on rhel7-3: 6
+pcmk__native_allocate: rsc2:1 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: rsc2:1 allocation score on rhel7-5: 0
+pcmk__native_allocate: rsc2:2 allocation score on lxc1: 0
+pcmk__native_allocate: rsc2:2 allocation score on lxc2: 0
+pcmk__native_allocate: rsc2:2 allocation score on remote-rhel7-2: 0
+pcmk__native_allocate: rsc2:2 allocation score on rhel7-1: 0
+pcmk__native_allocate: rsc2:2 allocation score on rhel7-3: -INFINITY
+pcmk__native_allocate: rsc2:2 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: rsc2:2 allocation score on rhel7-5: 6
+pcmk__native_allocate: rsc2:3 allocation score on lxc1: 0
+pcmk__native_allocate: rsc2:3 allocation score on lxc2: 0
+pcmk__native_allocate: rsc2:3 allocation score on remote-rhel7-2: 0
+pcmk__native_allocate: rsc2:3 allocation score on rhel7-1: 6
+pcmk__native_allocate: rsc2:3 allocation score on rhel7-3: -INFINITY
+pcmk__native_allocate: rsc2:3 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: rsc2:3 allocation score on rhel7-5: -INFINITY
+pcmk__native_allocate: rsc2:4 allocation score on lxc1: -INFINITY
+pcmk__native_allocate: rsc2:4 allocation score on lxc2: -INFINITY
+pcmk__native_allocate: rsc2:4 allocation score on remote-rhel7-2: 11
+pcmk__native_allocate: rsc2:4 allocation score on rhel7-1: -INFINITY
+pcmk__native_allocate: rsc2:4 allocation score on rhel7-3: -INFINITY
+pcmk__native_allocate: rsc2:4 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: rsc2:4 allocation score on rhel7-5: -INFINITY
+pcmk__native_allocate: rsc2:5 allocation score on lxc1: 0
+pcmk__native_allocate: rsc2:5 allocation score on lxc2: 6
+pcmk__native_allocate: rsc2:5 allocation score on remote-rhel7-2: 0
+pcmk__native_allocate: rsc2:5 allocation score on rhel7-1: -INFINITY
+pcmk__native_allocate: rsc2:5 allocation score on rhel7-3: -INFINITY
+pcmk__native_allocate: rsc2:5 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: rsc2:5 allocation score on rhel7-5: -INFINITY
+pcmk__native_allocate: rsc2:6 allocation score on lxc1: 6
+pcmk__native_allocate: rsc2:6 allocation score on lxc2: -INFINITY
+pcmk__native_allocate: rsc2:6 allocation score on remote-rhel7-2: 0
+pcmk__native_allocate: rsc2:6 allocation score on rhel7-1: -INFINITY
+pcmk__native_allocate: rsc2:6 allocation score on rhel7-3: -INFINITY
+pcmk__native_allocate: rsc2:6 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: rsc2:6 allocation score on rhel7-5: -INFINITY
+pcmk__native_allocate: stateful-bundle-0 allocation score on lxc1: -INFINITY
+pcmk__native_allocate: stateful-bundle-0 allocation score on lxc2: -INFINITY
+pcmk__native_allocate: stateful-bundle-0 allocation score on remote-rhel7-2: -INFINITY
+pcmk__native_allocate: stateful-bundle-0 allocation score on rhel7-1: 0
+pcmk__native_allocate: stateful-bundle-0 allocation score on rhel7-3: 0
+pcmk__native_allocate: stateful-bundle-0 allocation score on rhel7-4: 0
+pcmk__native_allocate: stateful-bundle-0 allocation score on rhel7-5: 10000
+pcmk__native_allocate: stateful-bundle-1 allocation score on lxc1: -INFINITY
+pcmk__native_allocate: stateful-bundle-1 allocation score on lxc2: -INFINITY
+pcmk__native_allocate: stateful-bundle-1 allocation score on remote-rhel7-2: -INFINITY
+pcmk__native_allocate: stateful-bundle-1 allocation score on rhel7-1: 10000
+pcmk__native_allocate: stateful-bundle-1 allocation score on rhel7-3: 0
+pcmk__native_allocate: stateful-bundle-1 allocation score on rhel7-4: 0
+pcmk__native_allocate: stateful-bundle-1 allocation score on rhel7-5: 0
+pcmk__native_allocate: stateful-bundle-2 allocation score on lxc1: -INFINITY
+pcmk__native_allocate: stateful-bundle-2 allocation score on lxc2: -INFINITY
+pcmk__native_allocate: stateful-bundle-2 allocation score on remote-rhel7-2: -INFINITY
+pcmk__native_allocate: stateful-bundle-2 allocation score on rhel7-1: 0
+pcmk__native_allocate: stateful-bundle-2 allocation score on rhel7-3: 0
+pcmk__native_allocate: stateful-bundle-2 allocation score on rhel7-4: 10000
+pcmk__native_allocate: stateful-bundle-2 allocation score on rhel7-5: 0
+pcmk__native_allocate: stateful-bundle-docker-0 allocation score on lxc1: -10000
+pcmk__native_allocate: stateful-bundle-docker-0 allocation score on lxc2: -10000
+pcmk__native_allocate: stateful-bundle-docker-0 allocation score on remote-rhel7-2: -10000
+pcmk__native_allocate: stateful-bundle-docker-0 allocation score on rhel7-1: 0
+pcmk__native_allocate: stateful-bundle-docker-0 allocation score on rhel7-3: 0
+pcmk__native_allocate: stateful-bundle-docker-0 allocation score on rhel7-4: 0
+pcmk__native_allocate: stateful-bundle-docker-0 allocation score on rhel7-5: 0
+pcmk__native_allocate: stateful-bundle-docker-1 allocation score on lxc1: -10000
+pcmk__native_allocate: stateful-bundle-docker-1 allocation score on lxc2: -10000
+pcmk__native_allocate: stateful-bundle-docker-1 allocation score on remote-rhel7-2: -10000
+pcmk__native_allocate: stateful-bundle-docker-1 allocation score on rhel7-1: 0
+pcmk__native_allocate: stateful-bundle-docker-1 allocation score on rhel7-3: 0
+pcmk__native_allocate: stateful-bundle-docker-1 allocation score on rhel7-4: 0
+pcmk__native_allocate: stateful-bundle-docker-1 allocation score on rhel7-5: -INFINITY
+pcmk__native_allocate: stateful-bundle-docker-2 allocation score on lxc1: -10000
+pcmk__native_allocate: stateful-bundle-docker-2 allocation score on lxc2: -10000
+pcmk__native_allocate: stateful-bundle-docker-2 allocation score on remote-rhel7-2: -10000
+pcmk__native_allocate: stateful-bundle-docker-2 allocation score on rhel7-1: -INFINITY
+pcmk__native_allocate: stateful-bundle-docker-2 allocation score on rhel7-3: 0
+pcmk__native_allocate: stateful-bundle-docker-2 allocation score on rhel7-4: 0
+pcmk__native_allocate: stateful-bundle-docker-2 allocation score on rhel7-5: -INFINITY
+pcmk__native_allocate: stateful-bundle-ip-192.168.122.131 allocation score on lxc1: -INFINITY
+pcmk__native_allocate: stateful-bundle-ip-192.168.122.131 allocation score on lxc2: -INFINITY
+pcmk__native_allocate: stateful-bundle-ip-192.168.122.131 allocation score on remote-rhel7-2: -INFINITY
+pcmk__native_allocate: stateful-bundle-ip-192.168.122.131 allocation score on rhel7-1: -INFINITY
+pcmk__native_allocate: stateful-bundle-ip-192.168.122.131 allocation score on rhel7-3: -INFINITY
+pcmk__native_allocate: stateful-bundle-ip-192.168.122.131 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: stateful-bundle-ip-192.168.122.131 allocation score on rhel7-5: 0
+pcmk__native_allocate: stateful-bundle-ip-192.168.122.132 allocation score on lxc1: -INFINITY
+pcmk__native_allocate: stateful-bundle-ip-192.168.122.132 allocation score on lxc2: -INFINITY
+pcmk__native_allocate: stateful-bundle-ip-192.168.122.132 allocation score on remote-rhel7-2: -INFINITY
+pcmk__native_allocate: stateful-bundle-ip-192.168.122.132 allocation score on rhel7-1: 0
+pcmk__native_allocate: stateful-bundle-ip-192.168.122.132 allocation score on rhel7-3: -INFINITY
+pcmk__native_allocate: stateful-bundle-ip-192.168.122.132 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: stateful-bundle-ip-192.168.122.132 allocation score on rhel7-5: -INFINITY
+pcmk__native_allocate: stateful-bundle-ip-192.168.122.133 allocation score on lxc1: -INFINITY
+pcmk__native_allocate: stateful-bundle-ip-192.168.122.133 allocation score on lxc2: -INFINITY
+pcmk__native_allocate: stateful-bundle-ip-192.168.122.133 allocation score on remote-rhel7-2: -INFINITY
+pcmk__native_allocate: stateful-bundle-ip-192.168.122.133 allocation score on rhel7-1: -INFINITY
+pcmk__native_allocate: stateful-bundle-ip-192.168.122.133 allocation score on rhel7-3: -INFINITY
+pcmk__native_allocate: stateful-bundle-ip-192.168.122.133 allocation score on rhel7-4: 0
+pcmk__native_allocate: stateful-bundle-ip-192.168.122.133 allocation score on rhel7-5: -INFINITY
+rsc1:0 promotion score on rhel7-4: 10
+rsc1:1 promotion score on rhel7-3: 5
+rsc1:2 promotion score on rhel7-5: 5
+rsc1:3 promotion score on rhel7-1: 5
+rsc1:4 promotion score on remote-rhel7-2: 5
+rsc1:5 promotion score on lxc2: 5
+rsc1:6 promotion score on lxc1: 5
+rsc2:0 promotion score on rhel7-4: 10
+rsc2:1 promotion score on rhel7-3: 5
+rsc2:2 promotion score on rhel7-5: 5
+rsc2:3 promotion score on rhel7-1: 5
+rsc2:4 promotion score on remote-rhel7-2: 110
+rsc2:5 promotion score on lxc2: 5
+rsc2:6 promotion score on lxc1: 5
diff --git a/cts/scheduler/on_fail_demote1.summary b/cts/scheduler/on_fail_demote1.summary
new file mode 100644
index 0000000000..b1735829d2
--- /dev/null
+++ b/cts/scheduler/on_fail_demote1.summary
@@ -0,0 +1,86 @@
+Using the original execution date of: 2020-06-16 19:23:21Z
+
+Current cluster status:
+Online: [ rhel7-1 rhel7-3 rhel7-4 rhel7-5 ]
+RemoteOnline: [ remote-rhel7-2 ]
+GuestOnline: [ lxc1:container1 lxc2:container2 stateful-bundle-0:stateful-bundle-docker-0 stateful-bundle-1:stateful-bundle-docker-1 stateful-bundle-2:stateful-bundle-docker-2 ]
+
+ Fencing (stonith:fence_xvm): Started rhel7-4
+ Clone Set: rsc1-clone [rsc1] (promotable)
+ rsc1 (ocf::pacemaker:Stateful): FAILED Master rhel7-4
+ Slaves: [ lxc1 lxc2 remote-rhel7-2 rhel7-1 rhel7-3 rhel7-5 ]
+ Clone Set: rsc2-master [rsc2] (promotable)
+ rsc2 (ocf::pacemaker:Stateful): FAILED Master remote-rhel7-2
+ Slaves: [ lxc1 lxc2 rhel7-1 rhel7-3 rhel7-4 rhel7-5 ]
+ remote-rhel7-2 (ocf::pacemaker:remote): Started rhel7-1
+ container1 (ocf::heartbeat:VirtualDomain): Started rhel7-3
+ container2 (ocf::heartbeat:VirtualDomain): Started rhel7-3
+ Clone Set: lxc-ms-master [lxc-ms] (promotable)
+ lxc-ms (ocf::pacemaker:Stateful): FAILED Master lxc2
+ Slaves: [ lxc1 ]
+ Stopped: [ remote-rhel7-2 rhel7-1 rhel7-3 rhel7-4 rhel7-5 ]
+ Container bundle set: stateful-bundle [pcmktest:http]
+ stateful-bundle-0 (192.168.122.131) (ocf::pacemaker:Stateful): FAILED Master rhel7-5
+ stateful-bundle-1 (192.168.122.132) (ocf::pacemaker:Stateful): Slave rhel7-1
+ stateful-bundle-2 (192.168.122.133) (ocf::pacemaker:Stateful): Slave rhel7-4
+
+Transition Summary:
+ * Re-promote rsc1:0 ( Master rhel7-4 )
+ * Re-promote rsc2:4 ( Master remote-rhel7-2 )
+ * Re-promote lxc-ms:0 ( Master lxc2 )
+ * Re-promote bundled:0 ( Master stateful-bundle-0 )
+
+Executing cluster transition:
+ * Pseudo action: rsc1-clone_demote_0
+ * Pseudo action: rsc2-master_demote_0
+ * Pseudo action: lxc-ms-master_demote_0
+ * Pseudo action: stateful-bundle_demote_0
+ * Resource action: rsc1 demote on rhel7-4
+ * Pseudo action: rsc1-clone_demoted_0
+ * Pseudo action: rsc1-clone_promote_0
+ * Resource action: rsc2 demote on remote-rhel7-2
+ * Pseudo action: rsc2-master_demoted_0
+ * Pseudo action: rsc2-master_promote_0
+ * Resource action: lxc-ms demote on lxc2
+ * Pseudo action: lxc-ms-master_demoted_0
+ * Pseudo action: lxc-ms-master_promote_0
+ * Pseudo action: stateful-bundle-master_demote_0
+ * Resource action: rsc1 promote on rhel7-4
+ * Pseudo action: rsc1-clone_promoted_0
+ * Resource action: rsc2 promote on remote-rhel7-2
+ * Pseudo action: rsc2-master_promoted_0
+ * Resource action: lxc-ms promote on lxc2
+ * Pseudo action: lxc-ms-master_promoted_0
+ * Resource action: bundled demote on stateful-bundle-0
+ * Pseudo action: stateful-bundle-master_demoted_0
+ * Pseudo action: stateful-bundle_demoted_0
+ * Pseudo action: stateful-bundle_promote_0
+ * Pseudo action: stateful-bundle-master_promote_0
+ * Resource action: bundled promote on stateful-bundle-0
+ * Pseudo action: stateful-bundle-master_promoted_0
+ * Pseudo action: stateful-bundle_promoted_0
+Using the original execution date of: 2020-06-16 19:23:21Z
+
+Revised cluster status:
+Online: [ rhel7-1 rhel7-3 rhel7-4 rhel7-5 ]
+RemoteOnline: [ remote-rhel7-2 ]
+GuestOnline: [ lxc1:container1 lxc2:container2 stateful-bundle-0:stateful-bundle-docker-0 stateful-bundle-1:stateful-bundle-docker-1 stateful-bundle-2:stateful-bundle-docker-2 ]
+
+ Fencing (stonith:fence_xvm): Started rhel7-4
+ Clone Set: rsc1-clone [rsc1] (promotable)
+ Masters: [ rhel7-4 ]
+ Slaves: [ lxc1 lxc2 remote-rhel7-2 rhel7-1 rhel7-3 rhel7-5 ]
+ Clone Set: rsc2-master [rsc2] (promotable)
+ Masters: [ remote-rhel7-2 ]
+ Slaves: [ lxc1 lxc2 rhel7-1 rhel7-3 rhel7-4 rhel7-5 ]
+ remote-rhel7-2 (ocf::pacemaker:remote): Started rhel7-1
+ container1 (ocf::heartbeat:VirtualDomain): Started rhel7-3
+ container2 (ocf::heartbeat:VirtualDomain): Started rhel7-3
+ Clone Set: lxc-ms-master [lxc-ms] (promotable)
+ Masters: [ lxc2 ]
+ Slaves: [ lxc1 ]
+ Container bundle set: stateful-bundle [pcmktest:http]
+ stateful-bundle-0 (192.168.122.131) (ocf::pacemaker:Stateful): Master rhel7-5
+ stateful-bundle-1 (192.168.122.132) (ocf::pacemaker:Stateful): Slave rhel7-1
+ stateful-bundle-2 (192.168.122.133) (ocf::pacemaker:Stateful): Slave rhel7-4
+
diff --git a/cts/scheduler/on_fail_demote1.xml b/cts/scheduler/on_fail_demote1.xml
new file mode 100644
index 0000000000..9f3ff20eaa
--- /dev/null
+++ b/cts/scheduler/on_fail_demote1.xml
@@ -0,0 +1,616 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/cts/scheduler/on_fail_demote2.dot b/cts/scheduler/on_fail_demote2.dot
new file mode 100644
index 0000000000..06193cb72a
--- /dev/null
+++ b/cts/scheduler/on_fail_demote2.dot
@@ -0,0 +1,22 @@
+ digraph "g" {
+"Cancel rsc1_monitor_10000 rhel7-4" -> "rsc1_demote_0 rhel7-4" [ style = bold]
+"Cancel rsc1_monitor_10000 rhel7-4" [ style=bold color="green" fontcolor="black"]
+"Cancel rsc1_monitor_11000 rhel7-3" -> "rsc1_promote_0 rhel7-3" [ style = bold]
+"Cancel rsc1_monitor_11000 rhel7-3" [ style=bold color="green" fontcolor="black"]
+"rsc1-clone_demote_0" -> "rsc1-clone_demoted_0" [ style = bold]
+"rsc1-clone_demote_0" -> "rsc1_demote_0 rhel7-4" [ style = bold]
+"rsc1-clone_demote_0" [ style=bold color="green" fontcolor="orange"]
+"rsc1-clone_demoted_0" -> "rsc1-clone_promote_0" [ style = bold]
+"rsc1-clone_demoted_0" [ style=bold color="green" fontcolor="orange"]
+"rsc1-clone_promote_0" -> "rsc1_promote_0 rhel7-3" [ style = bold]
+"rsc1-clone_promote_0" [ style=bold color="green" fontcolor="orange"]
+"rsc1-clone_promoted_0" [ style=bold color="green" fontcolor="orange"]
+"rsc1_demote_0 rhel7-4" -> "rsc1-clone_demoted_0" [ style = bold]
+"rsc1_demote_0 rhel7-4" -> "rsc1_monitor_11000 rhel7-4" [ style = bold]
+"rsc1_demote_0 rhel7-4" [ style=bold color="green" fontcolor="black"]
+"rsc1_monitor_10000 rhel7-3" [ style=bold color="green" fontcolor="black"]
+"rsc1_monitor_11000 rhel7-4" [ style=bold color="green" fontcolor="black"]
+"rsc1_promote_0 rhel7-3" -> "rsc1-clone_promoted_0" [ style = bold]
+"rsc1_promote_0 rhel7-3" -> "rsc1_monitor_10000 rhel7-3" [ style = bold]
+"rsc1_promote_0 rhel7-3" [ style=bold color="green" fontcolor="black"]
+}
diff --git a/cts/scheduler/on_fail_demote2.exp b/cts/scheduler/on_fail_demote2.exp
new file mode 100644
index 0000000000..492e86fe5b
--- /dev/null
+++ b/cts/scheduler/on_fail_demote2.exp
@@ -0,0 +1,125 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/cts/scheduler/on_fail_demote2.scores b/cts/scheduler/on_fail_demote2.scores
new file mode 100644
index 0000000000..25aea90728
--- /dev/null
+++ b/cts/scheduler/on_fail_demote2.scores
@@ -0,0 +1,127 @@
+Allocation scores:
+Using the original execution date of: 2020-06-16 19:23:21Z
+pcmk__clone_allocate: rsc1-clone allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc1-clone allocation score on rhel7-2: 0
+pcmk__clone_allocate: rsc1-clone allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc1-clone allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc1-clone allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc1:0 allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc1:0 allocation score on rhel7-2: 0
+pcmk__clone_allocate: rsc1:0 allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc1:0 allocation score on rhel7-4: 11
+pcmk__clone_allocate: rsc1:0 allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc1:1 allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc1:1 allocation score on rhel7-2: 0
+pcmk__clone_allocate: rsc1:1 allocation score on rhel7-3: 6
+pcmk__clone_allocate: rsc1:1 allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc1:1 allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc1:2 allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc1:2 allocation score on rhel7-2: 0
+pcmk__clone_allocate: rsc1:2 allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc1:2 allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc1:2 allocation score on rhel7-5: 6
+pcmk__clone_allocate: rsc1:3 allocation score on rhel7-1: 6
+pcmk__clone_allocate: rsc1:3 allocation score on rhel7-2: 0
+pcmk__clone_allocate: rsc1:3 allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc1:3 allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc1:3 allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc1:4 allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc1:4 allocation score on rhel7-2: 6
+pcmk__clone_allocate: rsc1:4 allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc1:4 allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc1:4 allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc2-master allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc2-master allocation score on rhel7-2: 0
+pcmk__clone_allocate: rsc2-master allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc2-master allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc2-master allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc2:0 allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc2:0 allocation score on rhel7-2: 0
+pcmk__clone_allocate: rsc2:0 allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc2:0 allocation score on rhel7-4: 11
+pcmk__clone_allocate: rsc2:0 allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc2:1 allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc2:1 allocation score on rhel7-2: 0
+pcmk__clone_allocate: rsc2:1 allocation score on rhel7-3: 6
+pcmk__clone_allocate: rsc2:1 allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc2:1 allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc2:2 allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc2:2 allocation score on rhel7-2: 0
+pcmk__clone_allocate: rsc2:2 allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc2:2 allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc2:2 allocation score on rhel7-5: 6
+pcmk__clone_allocate: rsc2:3 allocation score on rhel7-1: 6
+pcmk__clone_allocate: rsc2:3 allocation score on rhel7-2: 0
+pcmk__clone_allocate: rsc2:3 allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc2:3 allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc2:3 allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc2:4 allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc2:4 allocation score on rhel7-2: 6
+pcmk__clone_allocate: rsc2:4 allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc2:4 allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc2:4 allocation score on rhel7-5: 0
+pcmk__native_allocate: Fencing allocation score on rhel7-1: 0
+pcmk__native_allocate: Fencing allocation score on rhel7-2: 0
+pcmk__native_allocate: Fencing allocation score on rhel7-3: 0
+pcmk__native_allocate: Fencing allocation score on rhel7-4: 0
+pcmk__native_allocate: Fencing allocation score on rhel7-5: 0
+pcmk__native_allocate: rsc1:0 allocation score on rhel7-1: -INFINITY
+pcmk__native_allocate: rsc1:0 allocation score on rhel7-2: -INFINITY
+pcmk__native_allocate: rsc1:0 allocation score on rhel7-3: -INFINITY
+pcmk__native_allocate: rsc1:0 allocation score on rhel7-4: 11
+pcmk__native_allocate: rsc1:0 allocation score on rhel7-5: -INFINITY
+pcmk__native_allocate: rsc1:1 allocation score on rhel7-1: 0
+pcmk__native_allocate: rsc1:1 allocation score on rhel7-2: 0
+pcmk__native_allocate: rsc1:1 allocation score on rhel7-3: 6
+pcmk__native_allocate: rsc1:1 allocation score on rhel7-4: 0
+pcmk__native_allocate: rsc1:1 allocation score on rhel7-5: 0
+pcmk__native_allocate: rsc1:2 allocation score on rhel7-1: 0
+pcmk__native_allocate: rsc1:2 allocation score on rhel7-2: 0
+pcmk__native_allocate: rsc1:2 allocation score on rhel7-3: -INFINITY
+pcmk__native_allocate: rsc1:2 allocation score on rhel7-4: 0
+pcmk__native_allocate: rsc1:2 allocation score on rhel7-5: 6
+pcmk__native_allocate: rsc1:3 allocation score on rhel7-1: 6
+pcmk__native_allocate: rsc1:3 allocation score on rhel7-2: 0
+pcmk__native_allocate: rsc1:3 allocation score on rhel7-3: -INFINITY
+pcmk__native_allocate: rsc1:3 allocation score on rhel7-4: 0
+pcmk__native_allocate: rsc1:3 allocation score on rhel7-5: -INFINITY
+pcmk__native_allocate: rsc1:4 allocation score on rhel7-1: -INFINITY
+pcmk__native_allocate: rsc1:4 allocation score on rhel7-2: 6
+pcmk__native_allocate: rsc1:4 allocation score on rhel7-3: -INFINITY
+pcmk__native_allocate: rsc1:4 allocation score on rhel7-4: 0
+pcmk__native_allocate: rsc1:4 allocation score on rhel7-5: -INFINITY
+pcmk__native_allocate: rsc2:0 allocation score on rhel7-1: 0
+pcmk__native_allocate: rsc2:0 allocation score on rhel7-2: 0
+pcmk__native_allocate: rsc2:0 allocation score on rhel7-3: 0
+pcmk__native_allocate: rsc2:0 allocation score on rhel7-4: 11
+pcmk__native_allocate: rsc2:0 allocation score on rhel7-5: 0
+pcmk__native_allocate: rsc2:1 allocation score on rhel7-1: 0
+pcmk__native_allocate: rsc2:1 allocation score on rhel7-2: 0
+pcmk__native_allocate: rsc2:1 allocation score on rhel7-3: 6
+pcmk__native_allocate: rsc2:1 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: rsc2:1 allocation score on rhel7-5: 0
+pcmk__native_allocate: rsc2:2 allocation score on rhel7-1: 0
+pcmk__native_allocate: rsc2:2 allocation score on rhel7-2: 0
+pcmk__native_allocate: rsc2:2 allocation score on rhel7-3: -INFINITY
+pcmk__native_allocate: rsc2:2 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: rsc2:2 allocation score on rhel7-5: 6
+pcmk__native_allocate: rsc2:3 allocation score on rhel7-1: 6
+pcmk__native_allocate: rsc2:3 allocation score on rhel7-2: 0
+pcmk__native_allocate: rsc2:3 allocation score on rhel7-3: -INFINITY
+pcmk__native_allocate: rsc2:3 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: rsc2:3 allocation score on rhel7-5: -INFINITY
+pcmk__native_allocate: rsc2:4 allocation score on rhel7-1: -INFINITY
+pcmk__native_allocate: rsc2:4 allocation score on rhel7-2: 6
+pcmk__native_allocate: rsc2:4 allocation score on rhel7-3: -INFINITY
+pcmk__native_allocate: rsc2:4 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: rsc2:4 allocation score on rhel7-5: -INFINITY
+rsc1:0 promotion score on rhel7-4: -INFINITY
+rsc1:1 promotion score on rhel7-3: 5
+rsc1:2 promotion score on rhel7-5: 5
+rsc1:3 promotion score on rhel7-1: 5
+rsc1:4 promotion score on rhel7-2: 5
+rsc2:0 promotion score on rhel7-4: 10
+rsc2:1 promotion score on rhel7-3: 5
+rsc2:2 promotion score on rhel7-5: 5
+rsc2:3 promotion score on rhel7-1: 5
+rsc2:4 promotion score on rhel7-2: 5
diff --git a/cts/scheduler/on_fail_demote2.summary b/cts/scheduler/on_fail_demote2.summary
new file mode 100644
index 0000000000..795a11ddfd
--- /dev/null
+++ b/cts/scheduler/on_fail_demote2.summary
@@ -0,0 +1,41 @@
+Using the original execution date of: 2020-06-16 19:23:21Z
+
+Current cluster status:
+Online: [ rhel7-1 rhel7-2 rhel7-3 rhel7-4 rhel7-5 ]
+
+ Fencing (stonith:fence_xvm): Started rhel7-1
+ Clone Set: rsc1-clone [rsc1] (promotable)
+ rsc1 (ocf::pacemaker:Stateful): FAILED Master rhel7-4
+ Slaves: [ rhel7-1 rhel7-2 rhel7-3 rhel7-5 ]
+ Clone Set: rsc2-master [rsc2] (promotable)
+ Masters: [ rhel7-4 ]
+ Slaves: [ rhel7-1 rhel7-2 rhel7-3 rhel7-5 ]
+
+Transition Summary:
+ * Demote rsc1:0 ( Master -> Slave rhel7-4 )
+ * Promote rsc1:1 ( Slave -> Master rhel7-3 )
+
+Executing cluster transition:
+ * Resource action: rsc1 cancel=10000 on rhel7-4
+ * Resource action: rsc1 cancel=11000 on rhel7-3
+ * Pseudo action: rsc1-clone_demote_0
+ * Resource action: rsc1 demote on rhel7-4
+ * Pseudo action: rsc1-clone_demoted_0
+ * Pseudo action: rsc1-clone_promote_0
+ * Resource action: rsc1 monitor=11000 on rhel7-4
+ * Resource action: rsc1 promote on rhel7-3
+ * Pseudo action: rsc1-clone_promoted_0
+ * Resource action: rsc1 monitor=10000 on rhel7-3
+Using the original execution date of: 2020-06-16 19:23:21Z
+
+Revised cluster status:
+Online: [ rhel7-1 rhel7-2 rhel7-3 rhel7-4 rhel7-5 ]
+
+ Fencing (stonith:fence_xvm): Started rhel7-1
+ Clone Set: rsc1-clone [rsc1] (promotable)
+ Masters: [ rhel7-3 ]
+ Slaves: [ rhel7-1 rhel7-2 rhel7-4 rhel7-5 ]
+ Clone Set: rsc2-master [rsc2] (promotable)
+ Masters: [ rhel7-4 ]
+ Slaves: [ rhel7-1 rhel7-2 rhel7-3 rhel7-5 ]
+
diff --git a/cts/scheduler/on_fail_demote2.xml b/cts/scheduler/on_fail_demote2.xml
new file mode 100644
index 0000000000..ae9163327c
--- /dev/null
+++ b/cts/scheduler/on_fail_demote2.xml
@@ -0,0 +1,221 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/cts/scheduler/on_fail_demote3.dot b/cts/scheduler/on_fail_demote3.dot
new file mode 100644
index 0000000000..e78325b14a
--- /dev/null
+++ b/cts/scheduler/on_fail_demote3.dot
@@ -0,0 +1,12 @@
+ digraph "g" {
+"Cancel rsc1_monitor_10000 rhel7-4" -> "rsc1_demote_0 rhel7-4" [ style = bold]
+"Cancel rsc1_monitor_10000 rhel7-4" [ style=bold color="green" fontcolor="black"]
+"rsc1-clone_demote_0" -> "rsc1-clone_demoted_0" [ style = bold]
+"rsc1-clone_demote_0" -> "rsc1_demote_0 rhel7-4" [ style = bold]
+"rsc1-clone_demote_0" [ style=bold color="green" fontcolor="orange"]
+"rsc1-clone_demoted_0" [ style=bold color="green" fontcolor="orange"]
+"rsc1_demote_0 rhel7-4" -> "rsc1-clone_demoted_0" [ style = bold]
+"rsc1_demote_0 rhel7-4" -> "rsc1_monitor_11000 rhel7-4" [ style = bold]
+"rsc1_demote_0 rhel7-4" [ style=bold color="green" fontcolor="black"]
+"rsc1_monitor_11000 rhel7-4" [ style=bold color="green" fontcolor="black"]
+}
diff --git a/cts/scheduler/on_fail_demote3.exp b/cts/scheduler/on_fail_demote3.exp
new file mode 100644
index 0000000000..ed6bd6dc10
--- /dev/null
+++ b/cts/scheduler/on_fail_demote3.exp
@@ -0,0 +1,63 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/cts/scheduler/on_fail_demote3.scores b/cts/scheduler/on_fail_demote3.scores
new file mode 100644
index 0000000000..a85639a51e
--- /dev/null
+++ b/cts/scheduler/on_fail_demote3.scores
@@ -0,0 +1,127 @@
+Allocation scores:
+Using the original execution date of: 2020-06-16 19:23:21Z
+pcmk__clone_allocate: rsc1-clone allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc1-clone allocation score on rhel7-2: 0
+pcmk__clone_allocate: rsc1-clone allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc1-clone allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc1-clone allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc1:0 allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc1:0 allocation score on rhel7-2: 0
+pcmk__clone_allocate: rsc1:0 allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc1:0 allocation score on rhel7-4: 11
+pcmk__clone_allocate: rsc1:0 allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc1:1 allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc1:1 allocation score on rhel7-2: 0
+pcmk__clone_allocate: rsc1:1 allocation score on rhel7-3: 6
+pcmk__clone_allocate: rsc1:1 allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc1:1 allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc1:2 allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc1:2 allocation score on rhel7-2: 0
+pcmk__clone_allocate: rsc1:2 allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc1:2 allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc1:2 allocation score on rhel7-5: 6
+pcmk__clone_allocate: rsc1:3 allocation score on rhel7-1: 6
+pcmk__clone_allocate: rsc1:3 allocation score on rhel7-2: 0
+pcmk__clone_allocate: rsc1:3 allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc1:3 allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc1:3 allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc1:4 allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc1:4 allocation score on rhel7-2: 6
+pcmk__clone_allocate: rsc1:4 allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc1:4 allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc1:4 allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc2-master allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc2-master allocation score on rhel7-2: 0
+pcmk__clone_allocate: rsc2-master allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc2-master allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc2-master allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc2:0 allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc2:0 allocation score on rhel7-2: 0
+pcmk__clone_allocate: rsc2:0 allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc2:0 allocation score on rhel7-4: 11
+pcmk__clone_allocate: rsc2:0 allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc2:1 allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc2:1 allocation score on rhel7-2: 0
+pcmk__clone_allocate: rsc2:1 allocation score on rhel7-3: 6
+pcmk__clone_allocate: rsc2:1 allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc2:1 allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc2:2 allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc2:2 allocation score on rhel7-2: 0
+pcmk__clone_allocate: rsc2:2 allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc2:2 allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc2:2 allocation score on rhel7-5: 6
+pcmk__clone_allocate: rsc2:3 allocation score on rhel7-1: 6
+pcmk__clone_allocate: rsc2:3 allocation score on rhel7-2: 0
+pcmk__clone_allocate: rsc2:3 allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc2:3 allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc2:3 allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc2:4 allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc2:4 allocation score on rhel7-2: 6
+pcmk__clone_allocate: rsc2:4 allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc2:4 allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc2:4 allocation score on rhel7-5: 0
+pcmk__native_allocate: Fencing allocation score on rhel7-1: 0
+pcmk__native_allocate: Fencing allocation score on rhel7-2: 0
+pcmk__native_allocate: Fencing allocation score on rhel7-3: 0
+pcmk__native_allocate: Fencing allocation score on rhel7-4: 0
+pcmk__native_allocate: Fencing allocation score on rhel7-5: 0
+pcmk__native_allocate: rsc1:0 allocation score on rhel7-1: -INFINITY
+pcmk__native_allocate: rsc1:0 allocation score on rhel7-2: -INFINITY
+pcmk__native_allocate: rsc1:0 allocation score on rhel7-3: -INFINITY
+pcmk__native_allocate: rsc1:0 allocation score on rhel7-4: 11
+pcmk__native_allocate: rsc1:0 allocation score on rhel7-5: -INFINITY
+pcmk__native_allocate: rsc1:1 allocation score on rhel7-1: 0
+pcmk__native_allocate: rsc1:1 allocation score on rhel7-2: 0
+pcmk__native_allocate: rsc1:1 allocation score on rhel7-3: 6
+pcmk__native_allocate: rsc1:1 allocation score on rhel7-4: 0
+pcmk__native_allocate: rsc1:1 allocation score on rhel7-5: 0
+pcmk__native_allocate: rsc1:2 allocation score on rhel7-1: 0
+pcmk__native_allocate: rsc1:2 allocation score on rhel7-2: 0
+pcmk__native_allocate: rsc1:2 allocation score on rhel7-3: -INFINITY
+pcmk__native_allocate: rsc1:2 allocation score on rhel7-4: 0
+pcmk__native_allocate: rsc1:2 allocation score on rhel7-5: 6
+pcmk__native_allocate: rsc1:3 allocation score on rhel7-1: 6
+pcmk__native_allocate: rsc1:3 allocation score on rhel7-2: 0
+pcmk__native_allocate: rsc1:3 allocation score on rhel7-3: -INFINITY
+pcmk__native_allocate: rsc1:3 allocation score on rhel7-4: 0
+pcmk__native_allocate: rsc1:3 allocation score on rhel7-5: -INFINITY
+pcmk__native_allocate: rsc1:4 allocation score on rhel7-1: -INFINITY
+pcmk__native_allocate: rsc1:4 allocation score on rhel7-2: 6
+pcmk__native_allocate: rsc1:4 allocation score on rhel7-3: -INFINITY
+pcmk__native_allocate: rsc1:4 allocation score on rhel7-4: 0
+pcmk__native_allocate: rsc1:4 allocation score on rhel7-5: -INFINITY
+pcmk__native_allocate: rsc2:0 allocation score on rhel7-1: 0
+pcmk__native_allocate: rsc2:0 allocation score on rhel7-2: 0
+pcmk__native_allocate: rsc2:0 allocation score on rhel7-3: 0
+pcmk__native_allocate: rsc2:0 allocation score on rhel7-4: 11
+pcmk__native_allocate: rsc2:0 allocation score on rhel7-5: 0
+pcmk__native_allocate: rsc2:1 allocation score on rhel7-1: 0
+pcmk__native_allocate: rsc2:1 allocation score on rhel7-2: 0
+pcmk__native_allocate: rsc2:1 allocation score on rhel7-3: 6
+pcmk__native_allocate: rsc2:1 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: rsc2:1 allocation score on rhel7-5: 0
+pcmk__native_allocate: rsc2:2 allocation score on rhel7-1: 0
+pcmk__native_allocate: rsc2:2 allocation score on rhel7-2: 0
+pcmk__native_allocate: rsc2:2 allocation score on rhel7-3: -INFINITY
+pcmk__native_allocate: rsc2:2 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: rsc2:2 allocation score on rhel7-5: 6
+pcmk__native_allocate: rsc2:3 allocation score on rhel7-1: 6
+pcmk__native_allocate: rsc2:3 allocation score on rhel7-2: 0
+pcmk__native_allocate: rsc2:3 allocation score on rhel7-3: -INFINITY
+pcmk__native_allocate: rsc2:3 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: rsc2:3 allocation score on rhel7-5: -INFINITY
+pcmk__native_allocate: rsc2:4 allocation score on rhel7-1: -INFINITY
+pcmk__native_allocate: rsc2:4 allocation score on rhel7-2: 6
+pcmk__native_allocate: rsc2:4 allocation score on rhel7-3: -INFINITY
+pcmk__native_allocate: rsc2:4 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: rsc2:4 allocation score on rhel7-5: -INFINITY
+rsc1:0 promotion score on rhel7-4: -INFINITY
+rsc1:1 promotion score on rhel7-3: -INFINITY
+rsc1:2 promotion score on rhel7-5: -INFINITY
+rsc1:3 promotion score on rhel7-1: -INFINITY
+rsc1:4 promotion score on rhel7-2: -INFINITY
+rsc2:0 promotion score on rhel7-4: 10
+rsc2:1 promotion score on rhel7-3: 5
+rsc2:2 promotion score on rhel7-5: 5
+rsc2:3 promotion score on rhel7-1: 5
+rsc2:4 promotion score on rhel7-2: 5
diff --git a/cts/scheduler/on_fail_demote3.summary b/cts/scheduler/on_fail_demote3.summary
new file mode 100644
index 0000000000..f1173fd08d
--- /dev/null
+++ b/cts/scheduler/on_fail_demote3.summary
@@ -0,0 +1,34 @@
+Using the original execution date of: 2020-06-16 19:23:21Z
+
+Current cluster status:
+Online: [ rhel7-1 rhel7-2 rhel7-3 rhel7-4 rhel7-5 ]
+
+ Fencing (stonith:fence_xvm): Started rhel7-1
+ Clone Set: rsc1-clone [rsc1] (promotable)
+ rsc1 (ocf::pacemaker:Stateful): FAILED Master rhel7-4
+ Slaves: [ rhel7-1 rhel7-2 rhel7-3 rhel7-5 ]
+ Clone Set: rsc2-master [rsc2] (promotable)
+ Masters: [ rhel7-4 ]
+ Slaves: [ rhel7-1 rhel7-2 rhel7-3 rhel7-5 ]
+
+Transition Summary:
+ * Demote rsc1:0 ( Master -> Slave rhel7-4 )
+
+Executing cluster transition:
+ * Resource action: rsc1 cancel=10000 on rhel7-4
+ * Pseudo action: rsc1-clone_demote_0
+ * Resource action: rsc1 demote on rhel7-4
+ * Pseudo action: rsc1-clone_demoted_0
+ * Resource action: rsc1 monitor=11000 on rhel7-4
+Using the original execution date of: 2020-06-16 19:23:21Z
+
+Revised cluster status:
+Online: [ rhel7-1 rhel7-2 rhel7-3 rhel7-4 rhel7-5 ]
+
+ Fencing (stonith:fence_xvm): Started rhel7-1
+ Clone Set: rsc1-clone [rsc1] (promotable)
+ Slaves: [ rhel7-1 rhel7-2 rhel7-3 rhel7-4 rhel7-5 ]
+ Clone Set: rsc2-master [rsc2] (promotable)
+ Masters: [ rhel7-4 ]
+ Slaves: [ rhel7-1 rhel7-2 rhel7-3 rhel7-5 ]
+
diff --git a/cts/scheduler/on_fail_demote3.xml b/cts/scheduler/on_fail_demote3.xml
new file mode 100644
index 0000000000..a7b68067e8
--- /dev/null
+++ b/cts/scheduler/on_fail_demote3.xml
@@ -0,0 +1,221 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/cts/scheduler/on_fail_demote4.dot b/cts/scheduler/on_fail_demote4.dot
new file mode 100644
index 0000000000..4715cd3849
--- /dev/null
+++ b/cts/scheduler/on_fail_demote4.dot
@@ -0,0 +1,383 @@
+ digraph "g" {
+"Cancel rsc1_monitor_11000 rhel7-3" -> "rsc1_promote_0 rhel7-3" [ style = bold]
+"Cancel rsc1_monitor_11000 rhel7-3" [ style=bold color="green" fontcolor="black"]
+"Cancel rsc2_monitor_11000 rhel7-3" -> "rsc2_promote_0 rhel7-3" [ style = bold]
+"Cancel rsc2_monitor_11000 rhel7-3" [ style=bold color="green" fontcolor="black"]
+"Fencing_monitor_120000 rhel7-5" [ style=bold color="green" fontcolor="black"]
+"Fencing_start_0 rhel7-5" -> "Fencing_monitor_120000 rhel7-5" [ style = bold]
+"Fencing_start_0 rhel7-5" [ style=bold color="green" fontcolor="black"]
+"Fencing_stop_0 rhel7-4" -> "Fencing_start_0 rhel7-5" [ style = bold]
+"Fencing_stop_0 rhel7-4" [ style=bold color="green" fontcolor="orange"]
+"bundled_demote_0 stateful-bundle-0" -> "bundled_promote_0 stateful-bundle-0" [ style = bold]
+"bundled_demote_0 stateful-bundle-0" -> "bundled_stop_0 stateful-bundle-0" [ style = bold]
+"bundled_demote_0 stateful-bundle-0" -> "stateful-bundle-master_demoted_0" [ style = bold]
+"bundled_demote_0 stateful-bundle-0" [ style=bold color="green" fontcolor="orange"]
+"bundled_monitor_10000 stateful-bundle-0" [ style=bold color="green" fontcolor="black"]
+"bundled_monitor_11000 stateful-bundle-2" [ style=bold color="green" fontcolor="black"]
+"bundled_promote_0 stateful-bundle-0" -> "bundled_monitor_10000 stateful-bundle-0" [ style = bold]
+"bundled_promote_0 stateful-bundle-0" -> "stateful-bundle-master_promoted_0" [ style = bold]
+"bundled_promote_0 stateful-bundle-0" [ style=bold color="green" fontcolor="black"]
+"bundled_start_0 stateful-bundle-0" -> "bundled_monitor_10000 stateful-bundle-0" [ style = bold]
+"bundled_start_0 stateful-bundle-0" -> "bundled_promote_0 stateful-bundle-0" [ style = bold]
+"bundled_start_0 stateful-bundle-0" -> "bundled_start_0 stateful-bundle-2" [ style = bold]
+"bundled_start_0 stateful-bundle-0" -> "stateful-bundle-master_running_0" [ style = bold]
+"bundled_start_0 stateful-bundle-0" [ style=bold color="green" fontcolor="black"]
+"bundled_start_0 stateful-bundle-2" -> "bundled_monitor_11000 stateful-bundle-2" [ style = bold]
+"bundled_start_0 stateful-bundle-2" -> "stateful-bundle-master_running_0" [ style = bold]
+"bundled_start_0 stateful-bundle-2" [ style=bold color="green" fontcolor="black"]
+"bundled_stop_0 stateful-bundle-0" -> "bundled_start_0 stateful-bundle-0" [ style = bold]
+"bundled_stop_0 stateful-bundle-0" -> "stateful-bundle-master_stopped_0" [ style = bold]
+"bundled_stop_0 stateful-bundle-0" [ style=bold color="green" fontcolor="orange"]
+"bundled_stop_0 stateful-bundle-2" -> "bundled_start_0 stateful-bundle-2" [ style = bold]
+"bundled_stop_0 stateful-bundle-2" -> "bundled_stop_0 stateful-bundle-0" [ style = bold]
+"bundled_stop_0 stateful-bundle-2" -> "stateful-bundle-master_stopped_0" [ style = bold]
+"bundled_stop_0 stateful-bundle-2" [ style=bold color="green" fontcolor="orange"]
+"container2_monitor_20000 rhel7-3" [ style=bold color="green" fontcolor="black"]
+"container2_start_0 rhel7-3" -> "container2_monitor_20000 rhel7-3" [ style = bold]
+"container2_start_0 rhel7-3" -> "lxc-ms_promote_0 lxc2" [ style = bold]
+"container2_start_0 rhel7-3" -> "lxc-ms_start_0 lxc2" [ style = bold]
+"container2_start_0 rhel7-3" -> "lxc2_start_0 rhel7-3" [ style = bold]
+"container2_start_0 rhel7-3" -> "rsc1_start_0 lxc2" [ style = bold]
+"container2_start_0 rhel7-3" -> "rsc2_start_0 lxc2" [ style = bold]
+"container2_start_0 rhel7-3" [ style=bold color="green" fontcolor="black"]
+"container2_stop_0 rhel7-3" -> "container2_start_0 rhel7-3" [ style = bold]
+"container2_stop_0 rhel7-3" -> "stonith 'reboot' lxc2" [ style = bold]
+"container2_stop_0 rhel7-3" [ style=bold color="green" fontcolor="black"]
+"lxc-ms-master_demote_0" -> "lxc-ms-master_demoted_0" [ style = bold]
+"lxc-ms-master_demote_0" -> "lxc-ms_demote_0 lxc2" [ style = bold]
+"lxc-ms-master_demote_0" [ style=bold color="green" fontcolor="orange"]
+"lxc-ms-master_demoted_0" -> "lxc-ms-master_promote_0" [ style = bold]
+"lxc-ms-master_demoted_0" -> "lxc-ms-master_start_0" [ style = bold]
+"lxc-ms-master_demoted_0" -> "lxc-ms-master_stop_0" [ style = bold]
+"lxc-ms-master_demoted_0" [ style=bold color="green" fontcolor="orange"]
+"lxc-ms-master_promote_0" -> "lxc-ms_promote_0 lxc2" [ style = bold]
+"lxc-ms-master_promote_0" [ style=bold color="green" fontcolor="orange"]
+"lxc-ms-master_promoted_0" [ style=bold color="green" fontcolor="orange"]
+"lxc-ms-master_running_0" -> "lxc-ms-master_promote_0" [ style = bold]
+"lxc-ms-master_running_0" [ style=bold color="green" fontcolor="orange"]
+"lxc-ms-master_start_0" -> "lxc-ms-master_running_0" [ style = bold]
+"lxc-ms-master_start_0" -> "lxc-ms_start_0 lxc2" [ style = bold]
+"lxc-ms-master_start_0" [ style=bold color="green" fontcolor="orange"]
+"lxc-ms-master_stop_0" -> "lxc-ms-master_stopped_0" [ style = bold]
+"lxc-ms-master_stop_0" -> "lxc-ms_stop_0 lxc2" [ style = bold]
+"lxc-ms-master_stop_0" [ style=bold color="green" fontcolor="orange"]
+"lxc-ms-master_stopped_0" -> "lxc-ms-master_promote_0" [ style = bold]
+"lxc-ms-master_stopped_0" -> "lxc-ms-master_start_0" [ style = bold]
+"lxc-ms-master_stopped_0" [ style=bold color="green" fontcolor="orange"]
+"lxc-ms_demote_0 lxc2" -> "lxc-ms-master_demoted_0" [ style = bold]
+"lxc-ms_demote_0 lxc2" -> "lxc-ms_promote_0 lxc2" [ style = bold]
+"lxc-ms_demote_0 lxc2" -> "lxc-ms_stop_0 lxc2" [ style = bold]
+"lxc-ms_demote_0 lxc2" [ style=bold color="green" fontcolor="orange"]
+"lxc-ms_monitor_10000 lxc2" [ style=bold color="green" fontcolor="black"]
+"lxc-ms_promote_0 lxc2" -> "lxc-ms-master_promoted_0" [ style = bold]
+"lxc-ms_promote_0 lxc2" -> "lxc-ms_monitor_10000 lxc2" [ style = bold]
+"lxc-ms_promote_0 lxc2" [ style=bold color="green" fontcolor="black"]
+"lxc-ms_start_0 lxc2" -> "lxc-ms-master_running_0" [ style = bold]
+"lxc-ms_start_0 lxc2" -> "lxc-ms_monitor_10000 lxc2" [ style = bold]
+"lxc-ms_start_0 lxc2" -> "lxc-ms_promote_0 lxc2" [ style = bold]
+"lxc-ms_start_0 lxc2" [ style=bold color="green" fontcolor="black"]
+"lxc-ms_stop_0 lxc2" -> "lxc-ms-master_stopped_0" [ style = bold]
+"lxc-ms_stop_0 lxc2" -> "lxc-ms_start_0 lxc2" [ style = bold]
+"lxc-ms_stop_0 lxc2" [ style=bold color="green" fontcolor="orange"]
+"lxc2_monitor_30000 rhel7-3" [ style=bold color="green" fontcolor="black"]
+"lxc2_start_0 rhel7-3" -> "lxc-ms_monitor_10000 lxc2" [ style = bold]
+"lxc2_start_0 rhel7-3" -> "lxc-ms_promote_0 lxc2" [ style = bold]
+"lxc2_start_0 rhel7-3" -> "lxc-ms_start_0 lxc2" [ style = bold]
+"lxc2_start_0 rhel7-3" -> "lxc2_monitor_30000 rhel7-3" [ style = bold]
+"lxc2_start_0 rhel7-3" -> "rsc1_monitor_11000 lxc2" [ style = bold]
+"lxc2_start_0 rhel7-3" -> "rsc1_start_0 lxc2" [ style = bold]
+"lxc2_start_0 rhel7-3" -> "rsc2_monitor_11000 lxc2" [ style = bold]
+"lxc2_start_0 rhel7-3" -> "rsc2_start_0 lxc2" [ style = bold]
+"lxc2_start_0 rhel7-3" [ style=bold color="green" fontcolor="black"]
+"lxc2_stop_0 rhel7-3" -> "container2_stop_0 rhel7-3" [ style = bold]
+"lxc2_stop_0 rhel7-3" -> "lxc2_start_0 rhel7-3" [ style = bold]
+"lxc2_stop_0 rhel7-3" [ style=bold color="green" fontcolor="black"]
+"remote-rhel7-2_monitor_60000 rhel7-1" [ style=bold color="green" fontcolor="black"]
+"remote-rhel7-2_start_0 rhel7-1" -> "remote-rhel7-2_monitor_60000 rhel7-1" [ style = bold]
+"remote-rhel7-2_start_0 rhel7-1" [ style=bold color="green" fontcolor="black"]
+"remote-rhel7-2_stop_0 rhel7-1" -> "remote-rhel7-2_start_0 rhel7-1" [ style = bold]
+"remote-rhel7-2_stop_0 rhel7-1" [ style=bold color="green" fontcolor="black"]
+"rsc1-clone_demote_0" -> "rsc1-clone_demoted_0" [ style = bold]
+"rsc1-clone_demote_0" -> "rsc1_demote_0 rhel7-4" [ style = bold]
+"rsc1-clone_demote_0" [ style=bold color="green" fontcolor="orange"]
+"rsc1-clone_demoted_0" -> "rsc1-clone_promote_0" [ style = bold]
+"rsc1-clone_demoted_0" -> "rsc1-clone_start_0" [ style = bold]
+"rsc1-clone_demoted_0" -> "rsc1-clone_stop_0" [ style = bold]
+"rsc1-clone_demoted_0" [ style=bold color="green" fontcolor="orange"]
+"rsc1-clone_promote_0" -> "rsc1_promote_0 rhel7-3" [ style = bold]
+"rsc1-clone_promote_0" [ style=bold color="green" fontcolor="orange"]
+"rsc1-clone_promoted_0" [ style=bold color="green" fontcolor="orange"]
+"rsc1-clone_running_0" -> "rsc1-clone_promote_0" [ style = bold]
+"rsc1-clone_running_0" [ style=bold color="green" fontcolor="orange"]
+"rsc1-clone_start_0" -> "rsc1-clone_running_0" [ style = bold]
+"rsc1-clone_start_0" -> "rsc1_start_0 lxc2" [ style = bold]
+"rsc1-clone_start_0" [ style=bold color="green" fontcolor="orange"]
+"rsc1-clone_stop_0" -> "rsc1-clone_stopped_0" [ style = bold]
+"rsc1-clone_stop_0" -> "rsc1_stop_0 lxc2" [ style = bold]
+"rsc1-clone_stop_0" -> "rsc1_stop_0 remote-rhel7-2" [ style = bold]
+"rsc1-clone_stop_0" -> "rsc1_stop_0 rhel7-4" [ style = bold]
+"rsc1-clone_stop_0" [ style=bold color="green" fontcolor="orange"]
+"rsc1-clone_stopped_0" -> "rsc1-clone_promote_0" [ style = bold]
+"rsc1-clone_stopped_0" -> "rsc1-clone_start_0" [ style = bold]
+"rsc1-clone_stopped_0" [ style=bold color="green" fontcolor="orange"]
+"rsc1_demote_0 rhel7-4" -> "rsc1-clone_demoted_0" [ style = bold]
+"rsc1_demote_0 rhel7-4" -> "rsc1_stop_0 rhel7-4" [ style = bold]
+"rsc1_demote_0 rhel7-4" [ style=bold color="green" fontcolor="orange"]
+"rsc1_monitor_10000 rhel7-3" [ style=bold color="green" fontcolor="black"]
+"rsc1_monitor_11000 lxc2" [ style=bold color="green" fontcolor="black"]
+"rsc1_promote_0 rhel7-3" -> "rsc1-clone_promoted_0" [ style = bold]
+"rsc1_promote_0 rhel7-3" -> "rsc1_monitor_10000 rhel7-3" [ style = bold]
+"rsc1_promote_0 rhel7-3" [ style=bold color="green" fontcolor="black"]
+"rsc1_start_0 lxc2" -> "rsc1-clone_running_0" [ style = bold]
+"rsc1_start_0 lxc2" -> "rsc1_monitor_11000 lxc2" [ style = bold]
+"rsc1_start_0 lxc2" [ style=bold color="green" fontcolor="black"]
+"rsc1_stop_0 lxc2" -> "rsc1-clone_stopped_0" [ style = bold]
+"rsc1_stop_0 lxc2" -> "rsc1_start_0 lxc2" [ style = bold]
+"rsc1_stop_0 lxc2" [ style=bold color="green" fontcolor="orange"]
+"rsc1_stop_0 remote-rhel7-2" -> "remote-rhel7-2_stop_0 rhel7-1" [ style = bold]
+"rsc1_stop_0 remote-rhel7-2" -> "rsc1-clone_stopped_0" [ style = bold]
+"rsc1_stop_0 remote-rhel7-2" [ style=bold color="green" fontcolor="orange"]
+"rsc1_stop_0 rhel7-4" -> "rsc1-clone_stopped_0" [ style = bold]
+"rsc1_stop_0 rhel7-4" [ style=bold color="green" fontcolor="orange"]
+"rsc2-master_demote_0" -> "rsc2-master_demoted_0" [ style = bold]
+"rsc2-master_demote_0" -> "rsc2_demote_0 remote-rhel7-2" [ style = bold]
+"rsc2-master_demote_0" [ style=bold color="green" fontcolor="orange"]
+"rsc2-master_demoted_0" -> "rsc2-master_promote_0" [ style = bold]
+"rsc2-master_demoted_0" -> "rsc2-master_start_0" [ style = bold]
+"rsc2-master_demoted_0" -> "rsc2-master_stop_0" [ style = bold]
+"rsc2-master_demoted_0" [ style=bold color="green" fontcolor="orange"]
+"rsc2-master_promote_0" -> "rsc2_promote_0 rhel7-3" [ style = bold]
+"rsc2-master_promote_0" [ style=bold color="green" fontcolor="orange"]
+"rsc2-master_promoted_0" [ style=bold color="green" fontcolor="orange"]
+"rsc2-master_running_0" -> "rsc2-master_promote_0" [ style = bold]
+"rsc2-master_running_0" [ style=bold color="green" fontcolor="orange"]
+"rsc2-master_start_0" -> "rsc2-master_running_0" [ style = bold]
+"rsc2-master_start_0" -> "rsc2_start_0 lxc2" [ style = bold]
+"rsc2-master_start_0" [ style=bold color="green" fontcolor="orange"]
+"rsc2-master_stop_0" -> "rsc2-master_stopped_0" [ style = bold]
+"rsc2-master_stop_0" -> "rsc2_stop_0 lxc2" [ style = bold]
+"rsc2-master_stop_0" -> "rsc2_stop_0 remote-rhel7-2" [ style = bold]
+"rsc2-master_stop_0" -> "rsc2_stop_0 rhel7-4" [ style = bold]
+"rsc2-master_stop_0" [ style=bold color="green" fontcolor="orange"]
+"rsc2-master_stopped_0" -> "rsc2-master_promote_0" [ style = bold]
+"rsc2-master_stopped_0" -> "rsc2-master_start_0" [ style = bold]
+"rsc2-master_stopped_0" [ style=bold color="green" fontcolor="orange"]
+"rsc2_demote_0 remote-rhel7-2" -> "rsc2-master_demoted_0" [ style = bold]
+"rsc2_demote_0 remote-rhel7-2" -> "rsc2_stop_0 remote-rhel7-2" [ style = bold]
+"rsc2_demote_0 remote-rhel7-2" [ style=bold color="green" fontcolor="orange"]
+"rsc2_monitor_10000 rhel7-3" [ style=bold color="green" fontcolor="black"]
+"rsc2_monitor_11000 lxc2" [ style=bold color="green" fontcolor="black"]
+"rsc2_promote_0 rhel7-3" -> "rsc2-master_promoted_0" [ style = bold]
+"rsc2_promote_0 rhel7-3" -> "rsc2_monitor_10000 rhel7-3" [ style = bold]
+"rsc2_promote_0 rhel7-3" [ style=bold color="green" fontcolor="black"]
+"rsc2_start_0 lxc2" -> "rsc2-master_running_0" [ style = bold]
+"rsc2_start_0 lxc2" -> "rsc2_monitor_11000 lxc2" [ style = bold]
+"rsc2_start_0 lxc2" [ style=bold color="green" fontcolor="black"]
+"rsc2_stop_0 lxc2" -> "rsc2-master_stopped_0" [ style = bold]
+"rsc2_stop_0 lxc2" -> "rsc2_start_0 lxc2" [ style = bold]
+"rsc2_stop_0 lxc2" [ style=bold color="green" fontcolor="orange"]
+"rsc2_stop_0 remote-rhel7-2" -> "remote-rhel7-2_stop_0 rhel7-1" [ style = bold]
+"rsc2_stop_0 remote-rhel7-2" -> "rsc2-master_stopped_0" [ style = bold]
+"rsc2_stop_0 remote-rhel7-2" [ style=bold color="green" fontcolor="orange"]
+"rsc2_stop_0 rhel7-4" -> "rsc2-master_stopped_0" [ style = bold]
+"rsc2_stop_0 rhel7-4" [ style=bold color="green" fontcolor="orange"]
+"stateful-bundle-0_monitor_30000 rhel7-5" [ style=bold color="green" fontcolor="black"]
+"stateful-bundle-0_start_0 rhel7-5" -> "bundled_monitor_10000 stateful-bundle-0" [ style = bold]
+"stateful-bundle-0_start_0 rhel7-5" -> "bundled_promote_0 stateful-bundle-0" [ style = bold]
+"stateful-bundle-0_start_0 rhel7-5" -> "bundled_start_0 stateful-bundle-0" [ style = bold]
+"stateful-bundle-0_start_0 rhel7-5" -> "stateful-bundle-0_monitor_30000 rhel7-5" [ style = bold]
+"stateful-bundle-0_start_0 rhel7-5" [ style=bold color="green" fontcolor="black"]
+"stateful-bundle-0_stop_0 rhel7-5" -> "stateful-bundle-0_start_0 rhel7-5" [ style = bold]
+"stateful-bundle-0_stop_0 rhel7-5" -> "stateful-bundle-docker-0_stop_0 rhel7-5" [ style = bold]
+"stateful-bundle-0_stop_0 rhel7-5" [ style=bold color="green" fontcolor="black"]
+"stateful-bundle-2_monitor_30000 rhel7-3" [ style=bold color="green" fontcolor="black"]
+"stateful-bundle-2_start_0 rhel7-3" -> "bundled_monitor_11000 stateful-bundle-2" [ style = bold]
+"stateful-bundle-2_start_0 rhel7-3" -> "bundled_start_0 stateful-bundle-2" [ style = bold]
+"stateful-bundle-2_start_0 rhel7-3" -> "stateful-bundle-2_monitor_30000 rhel7-3" [ style = bold]
+"stateful-bundle-2_start_0 rhel7-3" [ style=bold color="green" fontcolor="black"]
+"stateful-bundle-2_stop_0 rhel7-4" -> "stateful-bundle-2_start_0 rhel7-3" [ style = bold]
+"stateful-bundle-2_stop_0 rhel7-4" -> "stateful-bundle-docker-2_stop_0 rhel7-4" [ style = bold]
+"stateful-bundle-2_stop_0 rhel7-4" [ style=bold color="green" fontcolor="orange"]
+"stateful-bundle-docker-0_monitor_60000 rhel7-5" [ style=bold color="green" fontcolor="black"]
+"stateful-bundle-docker-0_start_0 rhel7-5" -> "bundled_promote_0 stateful-bundle-0" [ style = bold]
+"stateful-bundle-docker-0_start_0 rhel7-5" -> "bundled_start_0 stateful-bundle-0" [ style = bold]
+"stateful-bundle-docker-0_start_0 rhel7-5" -> "stateful-bundle-0_start_0 rhel7-5" [ style = bold]
+"stateful-bundle-docker-0_start_0 rhel7-5" -> "stateful-bundle-docker-0_monitor_60000 rhel7-5" [ style = bold]
+"stateful-bundle-docker-0_start_0 rhel7-5" -> "stateful-bundle_running_0" [ style = bold]
+"stateful-bundle-docker-0_start_0 rhel7-5" [ style=bold color="green" fontcolor="black"]
+"stateful-bundle-docker-0_stop_0 rhel7-5" -> "stateful-bundle-docker-0_start_0 rhel7-5" [ style = bold]
+"stateful-bundle-docker-0_stop_0 rhel7-5" -> "stateful-bundle_stopped_0" [ style = bold]
+"stateful-bundle-docker-0_stop_0 rhel7-5" -> "stonith 'reboot' stateful-bundle-0" [ style = bold]
+"stateful-bundle-docker-0_stop_0 rhel7-5" [ style=bold color="green" fontcolor="black"]
+"stateful-bundle-docker-2_monitor_60000 rhel7-3" [ style=bold color="green" fontcolor="black"]
+"stateful-bundle-docker-2_start_0 rhel7-3" -> "bundled_start_0 stateful-bundle-2" [ style = bold]
+"stateful-bundle-docker-2_start_0 rhel7-3" -> "stateful-bundle-2_start_0 rhel7-3" [ style = bold]
+"stateful-bundle-docker-2_start_0 rhel7-3" -> "stateful-bundle-docker-2_monitor_60000 rhel7-3" [ style = bold]
+"stateful-bundle-docker-2_start_0 rhel7-3" -> "stateful-bundle_running_0" [ style = bold]
+"stateful-bundle-docker-2_start_0 rhel7-3" [ style=bold color="green" fontcolor="black"]
+"stateful-bundle-docker-2_stop_0 rhel7-4" -> "stateful-bundle-docker-2_start_0 rhel7-3" [ style = bold]
+"stateful-bundle-docker-2_stop_0 rhel7-4" -> "stateful-bundle-ip-192.168.122.133_stop_0 rhel7-4" [ style = bold]
+"stateful-bundle-docker-2_stop_0 rhel7-4" -> "stateful-bundle_stopped_0" [ style = bold]
+"stateful-bundle-docker-2_stop_0 rhel7-4" [ style=bold color="green" fontcolor="orange"]
+"stateful-bundle-ip-192.168.122.133_monitor_60000 rhel7-3" [ style=bold color="green" fontcolor="black"]
+"stateful-bundle-ip-192.168.122.133_start_0 rhel7-3" -> "stateful-bundle-docker-2_start_0 rhel7-3" [ style = bold]
+"stateful-bundle-ip-192.168.122.133_start_0 rhel7-3" -> "stateful-bundle-ip-192.168.122.133_monitor_60000 rhel7-3" [ style = bold]
+"stateful-bundle-ip-192.168.122.133_start_0 rhel7-3" [ style=bold color="green" fontcolor="black"]
+"stateful-bundle-ip-192.168.122.133_stop_0 rhel7-4" -> "stateful-bundle-ip-192.168.122.133_start_0 rhel7-3" [ style = bold]
+"stateful-bundle-ip-192.168.122.133_stop_0 rhel7-4" [ style=bold color="green" fontcolor="orange"]
+"stateful-bundle-master_demote_0" -> "bundled_demote_0 stateful-bundle-0" [ style = bold]
+"stateful-bundle-master_demote_0" -> "stateful-bundle-master_demoted_0" [ style = bold]
+"stateful-bundle-master_demote_0" [ style=bold color="green" fontcolor="orange"]
+"stateful-bundle-master_demoted_0" -> "stateful-bundle-master_promote_0" [ style = bold]
+"stateful-bundle-master_demoted_0" -> "stateful-bundle-master_start_0" [ style = bold]
+"stateful-bundle-master_demoted_0" -> "stateful-bundle-master_stop_0" [ style = bold]
+"stateful-bundle-master_demoted_0" -> "stateful-bundle_demoted_0" [ style = bold]
+"stateful-bundle-master_demoted_0" [ style=bold color="green" fontcolor="orange"]
+"stateful-bundle-master_promote_0" -> "bundled_promote_0 stateful-bundle-0" [ style = bold]
+"stateful-bundle-master_promote_0" [ style=bold color="green" fontcolor="orange"]
+"stateful-bundle-master_promoted_0" -> "stateful-bundle_promoted_0" [ style = bold]
+"stateful-bundle-master_promoted_0" [ style=bold color="green" fontcolor="orange"]
+"stateful-bundle-master_running_0" -> "stateful-bundle-master_promote_0" [ style = bold]
+"stateful-bundle-master_running_0" -> "stateful-bundle_running_0" [ style = bold]
+"stateful-bundle-master_running_0" [ style=bold color="green" fontcolor="orange"]
+"stateful-bundle-master_start_0" -> "bundled_start_0 stateful-bundle-0" [ style = bold]
+"stateful-bundle-master_start_0" -> "bundled_start_0 stateful-bundle-2" [ style = bold]
+"stateful-bundle-master_start_0" -> "stateful-bundle-master_running_0" [ style = bold]
+"stateful-bundle-master_start_0" [ style=bold color="green" fontcolor="orange"]
+"stateful-bundle-master_stop_0" -> "bundled_stop_0 stateful-bundle-0" [ style = bold]
+"stateful-bundle-master_stop_0" -> "bundled_stop_0 stateful-bundle-2" [ style = bold]
+"stateful-bundle-master_stop_0" -> "stateful-bundle-master_stopped_0" [ style = bold]
+"stateful-bundle-master_stop_0" [ style=bold color="green" fontcolor="orange"]
+"stateful-bundle-master_stopped_0" -> "stateful-bundle-master_promote_0" [ style = bold]
+"stateful-bundle-master_stopped_0" -> "stateful-bundle-master_start_0" [ style = bold]
+"stateful-bundle-master_stopped_0" -> "stateful-bundle_stopped_0" [ style = bold]
+"stateful-bundle-master_stopped_0" [ style=bold color="green" fontcolor="orange"]
+"stateful-bundle_demote_0" -> "stateful-bundle-master_demote_0" [ style = bold]
+"stateful-bundle_demote_0" -> "stateful-bundle_demoted_0" [ style = bold]
+"stateful-bundle_demote_0" [ style=bold color="green" fontcolor="orange"]
+"stateful-bundle_demoted_0" -> "stateful-bundle_promote_0" [ style = bold]
+"stateful-bundle_demoted_0" -> "stateful-bundle_start_0" [ style = bold]
+"stateful-bundle_demoted_0" -> "stateful-bundle_stop_0" [ style = bold]
+"stateful-bundle_demoted_0" [ style=bold color="green" fontcolor="orange"]
+"stateful-bundle_promote_0" -> "stateful-bundle-master_promote_0" [ style = bold]
+"stateful-bundle_promote_0" [ style=bold color="green" fontcolor="orange"]
+"stateful-bundle_promoted_0" [ style=bold color="green" fontcolor="orange"]
+"stateful-bundle_running_0" -> "stateful-bundle_promote_0" [ style = bold]
+"stateful-bundle_running_0" [ style=bold color="green" fontcolor="orange"]
+"stateful-bundle_start_0" -> "stateful-bundle-docker-0_start_0 rhel7-5" [ style = bold]
+"stateful-bundle_start_0" -> "stateful-bundle-docker-2_start_0 rhel7-3" [ style = bold]
+"stateful-bundle_start_0" -> "stateful-bundle-master_start_0" [ style = bold]
+"stateful-bundle_start_0" [ style=bold color="green" fontcolor="orange"]
+"stateful-bundle_stop_0" -> "bundled_stop_0 stateful-bundle-0" [ style = bold]
+"stateful-bundle_stop_0" -> "bundled_stop_0 stateful-bundle-2" [ style = bold]
+"stateful-bundle_stop_0" -> "stateful-bundle-docker-0_stop_0 rhel7-5" [ style = bold]
+"stateful-bundle_stop_0" -> "stateful-bundle-docker-2_stop_0 rhel7-4" [ style = bold]
+"stateful-bundle_stop_0" -> "stateful-bundle-master_stop_0" [ style = bold]
+"stateful-bundle_stop_0" [ style=bold color="green" fontcolor="orange"]
+"stateful-bundle_stopped_0" -> "stateful-bundle_promote_0" [ style = bold]
+"stateful-bundle_stopped_0" -> "stateful-bundle_start_0" [ style = bold]
+"stateful-bundle_stopped_0" [ style=bold color="green" fontcolor="orange"]
+"stonith 'reboot' lxc2" -> "Fencing_start_0 rhel7-5" [ style = bold]
+"stonith 'reboot' lxc2" -> "bundled_promote_0 stateful-bundle-0" [ style = bold]
+"stonith 'reboot' lxc2" -> "bundled_start_0 stateful-bundle-0" [ style = bold]
+"stonith 'reboot' lxc2" -> "bundled_start_0 stateful-bundle-2" [ style = bold]
+"stonith 'reboot' lxc2" -> "container2_start_0 rhel7-3" [ style = bold]
+"stonith 'reboot' lxc2" -> "lxc-ms-master_stop_0" [ style = bold]
+"stonith 'reboot' lxc2" -> "lxc-ms_demote_0 lxc2" [ style = bold]
+"stonith 'reboot' lxc2" -> "lxc-ms_promote_0 lxc2" [ style = bold]
+"stonith 'reboot' lxc2" -> "lxc-ms_start_0 lxc2" [ style = bold]
+"stonith 'reboot' lxc2" -> "lxc-ms_stop_0 lxc2" [ style = bold]
+"stonith 'reboot' lxc2" -> "lxc2_start_0 rhel7-3" [ style = bold]
+"stonith 'reboot' lxc2" -> "remote-rhel7-2_start_0 rhel7-1" [ style = bold]
+"stonith 'reboot' lxc2" -> "rsc1-clone_stop_0" [ style = bold]
+"stonith 'reboot' lxc2" -> "rsc1_promote_0 rhel7-3" [ style = bold]
+"stonith 'reboot' lxc2" -> "rsc1_start_0 lxc2" [ style = bold]
+"stonith 'reboot' lxc2" -> "rsc1_stop_0 lxc2" [ style = bold]
+"stonith 'reboot' lxc2" -> "rsc2-master_stop_0" [ style = bold]
+"stonith 'reboot' lxc2" -> "rsc2_promote_0 rhel7-3" [ style = bold]
+"stonith 'reboot' lxc2" -> "rsc2_start_0 lxc2" [ style = bold]
+"stonith 'reboot' lxc2" -> "rsc2_stop_0 lxc2" [ style = bold]
+"stonith 'reboot' lxc2" -> "stateful-bundle-0_start_0 rhel7-5" [ style = bold]
+"stonith 'reboot' lxc2" -> "stateful-bundle-2_start_0 rhel7-3" [ style = bold]
+"stonith 'reboot' lxc2" -> "stateful-bundle-docker-0_start_0 rhel7-5" [ style = bold]
+"stonith 'reboot' lxc2" -> "stateful-bundle-docker-2_start_0 rhel7-3" [ style = bold]
+"stonith 'reboot' lxc2" -> "stateful-bundle-ip-192.168.122.133_start_0 rhel7-3" [ style = bold]
+"stonith 'reboot' lxc2" [ style=bold color="green" fontcolor="orange"]
+"stonith 'reboot' remote-rhel7-2" -> "Fencing_start_0 rhel7-5" [ style = bold]
+"stonith 'reboot' remote-rhel7-2" -> "bundled_promote_0 stateful-bundle-0" [ style = bold]
+"stonith 'reboot' remote-rhel7-2" -> "bundled_start_0 stateful-bundle-0" [ style = bold]
+"stonith 'reboot' remote-rhel7-2" -> "bundled_start_0 stateful-bundle-2" [ style = bold]
+"stonith 'reboot' remote-rhel7-2" -> "container2_start_0 rhel7-3" [ style = bold]
+"stonith 'reboot' remote-rhel7-2" -> "lxc-ms_promote_0 lxc2" [ style = bold]
+"stonith 'reboot' remote-rhel7-2" -> "lxc-ms_start_0 lxc2" [ style = bold]
+"stonith 'reboot' remote-rhel7-2" -> "lxc2_start_0 rhel7-3" [ style = bold]
+"stonith 'reboot' remote-rhel7-2" -> "remote-rhel7-2_start_0 rhel7-1" [ style = bold]
+"stonith 'reboot' remote-rhel7-2" -> "rsc1-clone_stop_0" [ style = bold]
+"stonith 'reboot' remote-rhel7-2" -> "rsc1_promote_0 rhel7-3" [ style = bold]
+"stonith 'reboot' remote-rhel7-2" -> "rsc1_start_0 lxc2" [ style = bold]
+"stonith 'reboot' remote-rhel7-2" -> "rsc1_stop_0 remote-rhel7-2" [ style = bold]
+"stonith 'reboot' remote-rhel7-2" -> "rsc2-master_stop_0" [ style = bold]
+"stonith 'reboot' remote-rhel7-2" -> "rsc2_demote_0 remote-rhel7-2" [ style = bold]
+"stonith 'reboot' remote-rhel7-2" -> "rsc2_promote_0 rhel7-3" [ style = bold]
+"stonith 'reboot' remote-rhel7-2" -> "rsc2_start_0 lxc2" [ style = bold]
+"stonith 'reboot' remote-rhel7-2" -> "rsc2_stop_0 remote-rhel7-2" [ style = bold]
+"stonith 'reboot' remote-rhel7-2" -> "stateful-bundle-0_start_0 rhel7-5" [ style = bold]
+"stonith 'reboot' remote-rhel7-2" -> "stateful-bundle-2_start_0 rhel7-3" [ style = bold]
+"stonith 'reboot' remote-rhel7-2" -> "stateful-bundle-docker-0_start_0 rhel7-5" [ style = bold]
+"stonith 'reboot' remote-rhel7-2" -> "stateful-bundle-docker-2_start_0 rhel7-3" [ style = bold]
+"stonith 'reboot' remote-rhel7-2" -> "stateful-bundle-ip-192.168.122.133_start_0 rhel7-3" [ style = bold]
+"stonith 'reboot' remote-rhel7-2" -> "stonith 'reboot' rhel7-4" [ style = bold]
+"stonith 'reboot' remote-rhel7-2" [ style=bold color="green" fontcolor="black"]
+"stonith 'reboot' rhel7-4" -> "bundled_promote_0 stateful-bundle-0" [ style = bold]
+"stonith 'reboot' rhel7-4" -> "bundled_start_0 stateful-bundle-0" [ style = bold]
+"stonith 'reboot' rhel7-4" -> "bundled_start_0 stateful-bundle-2" [ style = bold]
+"stonith 'reboot' rhel7-4" -> "container2_start_0 rhel7-3" [ style = bold]
+"stonith 'reboot' rhel7-4" -> "lxc-ms_promote_0 lxc2" [ style = bold]
+"stonith 'reboot' rhel7-4" -> "lxc-ms_start_0 lxc2" [ style = bold]
+"stonith 'reboot' rhel7-4" -> "rsc1-clone_stop_0" [ style = bold]
+"stonith 'reboot' rhel7-4" -> "rsc1_demote_0 rhel7-4" [ style = bold]
+"stonith 'reboot' rhel7-4" -> "rsc1_promote_0 rhel7-3" [ style = bold]
+"stonith 'reboot' rhel7-4" -> "rsc1_start_0 lxc2" [ style = bold]
+"stonith 'reboot' rhel7-4" -> "rsc1_stop_0 rhel7-4" [ style = bold]
+"stonith 'reboot' rhel7-4" -> "rsc2-master_stop_0" [ style = bold]
+"stonith 'reboot' rhel7-4" -> "rsc2_promote_0 rhel7-3" [ style = bold]
+"stonith 'reboot' rhel7-4" -> "rsc2_start_0 lxc2" [ style = bold]
+"stonith 'reboot' rhel7-4" -> "rsc2_stop_0 rhel7-4" [ style = bold]
+"stonith 'reboot' rhel7-4" -> "stateful-bundle-docker-0_start_0 rhel7-5" [ style = bold]
+"stonith 'reboot' rhel7-4" -> "stateful-bundle-docker-2_start_0 rhel7-3" [ style = bold]
+"stonith 'reboot' rhel7-4" -> "stateful-bundle-docker-2_stop_0 rhel7-4" [ style = bold]
+"stonith 'reboot' rhel7-4" -> "stateful-bundle-ip-192.168.122.133_start_0 rhel7-3" [ style = bold]
+"stonith 'reboot' rhel7-4" -> "stateful-bundle-ip-192.168.122.133_stop_0 rhel7-4" [ style = bold]
+"stonith 'reboot' rhel7-4" -> "stonith 'reboot' stateful-bundle-2" [ style = bold]
+"stonith 'reboot' rhel7-4" [ style=bold color="green" fontcolor="black"]
+"stonith 'reboot' stateful-bundle-0" -> "bundled_promote_0 stateful-bundle-0" [ style = bold]
+"stonith 'reboot' stateful-bundle-0" -> "bundled_start_0 stateful-bundle-0" [ style = bold]
+"stonith 'reboot' stateful-bundle-0" -> "bundled_start_0 stateful-bundle-2" [ style = bold]
+"stonith 'reboot' stateful-bundle-0" -> "container2_start_0 rhel7-3" [ style = bold]
+"stonith 'reboot' stateful-bundle-0" -> "lxc-ms_promote_0 lxc2" [ style = bold]
+"stonith 'reboot' stateful-bundle-0" -> "lxc-ms_start_0 lxc2" [ style = bold]
+"stonith 'reboot' stateful-bundle-0" -> "rsc1_promote_0 rhel7-3" [ style = bold]
+"stonith 'reboot' stateful-bundle-0" -> "rsc1_start_0 lxc2" [ style = bold]
+"stonith 'reboot' stateful-bundle-0" -> "rsc2_promote_0 rhel7-3" [ style = bold]
+"stonith 'reboot' stateful-bundle-0" -> "rsc2_start_0 lxc2" [ style = bold]
+"stonith 'reboot' stateful-bundle-0" -> "stateful-bundle-docker-0_start_0 rhel7-5" [ style = bold]
+"stonith 'reboot' stateful-bundle-0" -> "stateful-bundle-docker-2_start_0 rhel7-3" [ style = bold]
+"stonith 'reboot' stateful-bundle-0" -> "stateful-bundle-ip-192.168.122.133_start_0 rhel7-3" [ style = bold]
+"stonith 'reboot' stateful-bundle-0" -> "stateful-bundle-master_stop_0" [ style = bold]
+"stonith 'reboot' stateful-bundle-0" [ style=bold color="green" fontcolor="orange"]
+"stonith 'reboot' stateful-bundle-2" -> "bundled_promote_0 stateful-bundle-0" [ style = bold]
+"stonith 'reboot' stateful-bundle-2" -> "bundled_start_0 stateful-bundle-0" [ style = bold]
+"stonith 'reboot' stateful-bundle-2" -> "bundled_start_0 stateful-bundle-2" [ style = bold]
+"stonith 'reboot' stateful-bundle-2" -> "container2_start_0 rhel7-3" [ style = bold]
+"stonith 'reboot' stateful-bundle-2" -> "lxc-ms_promote_0 lxc2" [ style = bold]
+"stonith 'reboot' stateful-bundle-2" -> "lxc-ms_start_0 lxc2" [ style = bold]
+"stonith 'reboot' stateful-bundle-2" -> "rsc1_promote_0 rhel7-3" [ style = bold]
+"stonith 'reboot' stateful-bundle-2" -> "rsc1_start_0 lxc2" [ style = bold]
+"stonith 'reboot' stateful-bundle-2" -> "rsc2_promote_0 rhel7-3" [ style = bold]
+"stonith 'reboot' stateful-bundle-2" -> "rsc2_start_0 lxc2" [ style = bold]
+"stonith 'reboot' stateful-bundle-2" -> "stateful-bundle-docker-0_start_0 rhel7-5" [ style = bold]
+"stonith 'reboot' stateful-bundle-2" -> "stateful-bundle-docker-2_start_0 rhel7-3" [ style = bold]
+"stonith 'reboot' stateful-bundle-2" -> "stateful-bundle-ip-192.168.122.133_start_0 rhel7-3" [ style = bold]
+"stonith 'reboot' stateful-bundle-2" -> "stateful-bundle-master_stop_0" [ style = bold]
+"stonith 'reboot' stateful-bundle-2" [ style=bold color="green" fontcolor="orange"]
+}
diff --git a/cts/scheduler/on_fail_demote4.exp b/cts/scheduler/on_fail_demote4.exp
new file mode 100644
index 0000000000..0789a12284
--- /dev/null
+++ b/cts/scheduler/on_fail_demote4.exp
@@ -0,0 +1,1818 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/cts/scheduler/on_fail_demote4.scores b/cts/scheduler/on_fail_demote4.scores
new file mode 100644
index 0000000000..cde3fece39
--- /dev/null
+++ b/cts/scheduler/on_fail_demote4.scores
@@ -0,0 +1,470 @@
+Allocation scores:
+Using the original execution date of: 2020-06-16 19:23:21Z
+bundled:0 promotion score on stateful-bundle-0: 10
+bundled:1 promotion score on stateful-bundle-1: 5
+bundled:2 promotion score on stateful-bundle-2: 5
+lxc-ms:0 promotion score on lxc2: INFINITY
+lxc-ms:1 promotion score on lxc1: INFINITY
+pcmk__bundle_allocate: bundled:0 allocation score on stateful-bundle-0: 501
+pcmk__bundle_allocate: bundled:1 allocation score on stateful-bundle-1: 501
+pcmk__bundle_allocate: bundled:2 allocation score on stateful-bundle-2: 501
+pcmk__bundle_allocate: stateful-bundle allocation score on lxc1: 0
+pcmk__bundle_allocate: stateful-bundle allocation score on lxc2: 0
+pcmk__bundle_allocate: stateful-bundle allocation score on remote-rhel7-2: 0
+pcmk__bundle_allocate: stateful-bundle allocation score on rhel7-1: 0
+pcmk__bundle_allocate: stateful-bundle allocation score on rhel7-3: 0
+pcmk__bundle_allocate: stateful-bundle allocation score on rhel7-4: 0
+pcmk__bundle_allocate: stateful-bundle allocation score on rhel7-5: 0
+pcmk__bundle_allocate: stateful-bundle-0 allocation score on lxc1: -INFINITY
+pcmk__bundle_allocate: stateful-bundle-0 allocation score on lxc2: -INFINITY
+pcmk__bundle_allocate: stateful-bundle-0 allocation score on remote-rhel7-2: -INFINITY
+pcmk__bundle_allocate: stateful-bundle-0 allocation score on rhel7-1: 0
+pcmk__bundle_allocate: stateful-bundle-0 allocation score on rhel7-3: 0
+pcmk__bundle_allocate: stateful-bundle-0 allocation score on rhel7-4: 0
+pcmk__bundle_allocate: stateful-bundle-0 allocation score on rhel7-5: 0
+pcmk__bundle_allocate: stateful-bundle-1 allocation score on lxc1: -INFINITY
+pcmk__bundle_allocate: stateful-bundle-1 allocation score on lxc2: -INFINITY
+pcmk__bundle_allocate: stateful-bundle-1 allocation score on remote-rhel7-2: -INFINITY
+pcmk__bundle_allocate: stateful-bundle-1 allocation score on rhel7-1: 0
+pcmk__bundle_allocate: stateful-bundle-1 allocation score on rhel7-3: 0
+pcmk__bundle_allocate: stateful-bundle-1 allocation score on rhel7-4: 0
+pcmk__bundle_allocate: stateful-bundle-1 allocation score on rhel7-5: 0
+pcmk__bundle_allocate: stateful-bundle-2 allocation score on lxc1: -INFINITY
+pcmk__bundle_allocate: stateful-bundle-2 allocation score on lxc2: -INFINITY
+pcmk__bundle_allocate: stateful-bundle-2 allocation score on remote-rhel7-2: -INFINITY
+pcmk__bundle_allocate: stateful-bundle-2 allocation score on rhel7-1: 0
+pcmk__bundle_allocate: stateful-bundle-2 allocation score on rhel7-3: 0
+pcmk__bundle_allocate: stateful-bundle-2 allocation score on rhel7-4: 0
+pcmk__bundle_allocate: stateful-bundle-2 allocation score on rhel7-5: 0
+pcmk__bundle_allocate: stateful-bundle-docker-0 allocation score on lxc1: 0
+pcmk__bundle_allocate: stateful-bundle-docker-0 allocation score on lxc2: 0
+pcmk__bundle_allocate: stateful-bundle-docker-0 allocation score on remote-rhel7-2: 0
+pcmk__bundle_allocate: stateful-bundle-docker-0 allocation score on rhel7-1: 0
+pcmk__bundle_allocate: stateful-bundle-docker-0 allocation score on rhel7-3: 0
+pcmk__bundle_allocate: stateful-bundle-docker-0 allocation score on rhel7-4: 0
+pcmk__bundle_allocate: stateful-bundle-docker-0 allocation score on rhel7-5: 0
+pcmk__bundle_allocate: stateful-bundle-docker-1 allocation score on lxc1: 0
+pcmk__bundle_allocate: stateful-bundle-docker-1 allocation score on lxc2: 0
+pcmk__bundle_allocate: stateful-bundle-docker-1 allocation score on remote-rhel7-2: 0
+pcmk__bundle_allocate: stateful-bundle-docker-1 allocation score on rhel7-1: 0
+pcmk__bundle_allocate: stateful-bundle-docker-1 allocation score on rhel7-3: 0
+pcmk__bundle_allocate: stateful-bundle-docker-1 allocation score on rhel7-4: 0
+pcmk__bundle_allocate: stateful-bundle-docker-1 allocation score on rhel7-5: 0
+pcmk__bundle_allocate: stateful-bundle-docker-2 allocation score on lxc1: 0
+pcmk__bundle_allocate: stateful-bundle-docker-2 allocation score on lxc2: 0
+pcmk__bundle_allocate: stateful-bundle-docker-2 allocation score on remote-rhel7-2: 0
+pcmk__bundle_allocate: stateful-bundle-docker-2 allocation score on rhel7-1: 0
+pcmk__bundle_allocate: stateful-bundle-docker-2 allocation score on rhel7-3: 0
+pcmk__bundle_allocate: stateful-bundle-docker-2 allocation score on rhel7-4: 0
+pcmk__bundle_allocate: stateful-bundle-docker-2 allocation score on rhel7-5: 0
+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.131 allocation score on lxc1: 0
+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.131 allocation score on lxc2: 0
+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.131 allocation score on remote-rhel7-2: 0
+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.131 allocation score on rhel7-1: 0
+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.131 allocation score on rhel7-3: 0
+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.131 allocation score on rhel7-4: 0
+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.131 allocation score on rhel7-5: 0
+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.132 allocation score on lxc1: 0
+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.132 allocation score on lxc2: 0
+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.132 allocation score on remote-rhel7-2: 0
+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.132 allocation score on rhel7-1: 0
+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.132 allocation score on rhel7-3: 0
+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.132 allocation score on rhel7-4: 0
+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.132 allocation score on rhel7-5: 0
+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.133 allocation score on lxc1: 0
+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.133 allocation score on lxc2: 0
+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.133 allocation score on remote-rhel7-2: 0
+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.133 allocation score on rhel7-1: 0
+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.133 allocation score on rhel7-3: 0
+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.133 allocation score on rhel7-4: 0
+pcmk__bundle_allocate: stateful-bundle-ip-192.168.122.133 allocation score on rhel7-5: 0
+pcmk__bundle_allocate: stateful-bundle-master allocation score on lxc1: 0
+pcmk__bundle_allocate: stateful-bundle-master allocation score on lxc2: 0
+pcmk__bundle_allocate: stateful-bundle-master allocation score on remote-rhel7-2: 0
+pcmk__bundle_allocate: stateful-bundle-master allocation score on rhel7-1: 0
+pcmk__bundle_allocate: stateful-bundle-master allocation score on rhel7-3: 0
+pcmk__bundle_allocate: stateful-bundle-master allocation score on rhel7-4: 0
+pcmk__bundle_allocate: stateful-bundle-master allocation score on rhel7-5: 0
+pcmk__bundle_allocate: stateful-bundle-master allocation score on stateful-bundle-0: -INFINITY
+pcmk__bundle_allocate: stateful-bundle-master allocation score on stateful-bundle-1: -INFINITY
+pcmk__bundle_allocate: stateful-bundle-master allocation score on stateful-bundle-2: -INFINITY
+pcmk__clone_allocate: bundled:0 allocation score on stateful-bundle-0: INFINITY
+pcmk__clone_allocate: bundled:1 allocation score on stateful-bundle-1: INFINITY
+pcmk__clone_allocate: bundled:2 allocation score on stateful-bundle-2: INFINITY
+pcmk__clone_allocate: lxc-ms-master allocation score on lxc1: INFINITY
+pcmk__clone_allocate: lxc-ms-master allocation score on lxc2: INFINITY
+pcmk__clone_allocate: lxc-ms-master allocation score on remote-rhel7-2: 0
+pcmk__clone_allocate: lxc-ms-master allocation score on rhel7-1: 0
+pcmk__clone_allocate: lxc-ms-master allocation score on rhel7-3: 0
+pcmk__clone_allocate: lxc-ms-master allocation score on rhel7-4: 0
+pcmk__clone_allocate: lxc-ms-master allocation score on rhel7-5: 0
+pcmk__clone_allocate: lxc-ms:0 allocation score on lxc1: INFINITY
+pcmk__clone_allocate: lxc-ms:0 allocation score on lxc2: INFINITY
+pcmk__clone_allocate: lxc-ms:0 allocation score on remote-rhel7-2: 0
+pcmk__clone_allocate: lxc-ms:0 allocation score on rhel7-1: 0
+pcmk__clone_allocate: lxc-ms:0 allocation score on rhel7-3: 0
+pcmk__clone_allocate: lxc-ms:0 allocation score on rhel7-4: 0
+pcmk__clone_allocate: lxc-ms:0 allocation score on rhel7-5: 0
+pcmk__clone_allocate: lxc-ms:1 allocation score on lxc1: INFINITY
+pcmk__clone_allocate: lxc-ms:1 allocation score on lxc2: INFINITY
+pcmk__clone_allocate: lxc-ms:1 allocation score on remote-rhel7-2: 0
+pcmk__clone_allocate: lxc-ms:1 allocation score on rhel7-1: 0
+pcmk__clone_allocate: lxc-ms:1 allocation score on rhel7-3: 0
+pcmk__clone_allocate: lxc-ms:1 allocation score on rhel7-4: 0
+pcmk__clone_allocate: lxc-ms:1 allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc1-clone allocation score on lxc1: 0
+pcmk__clone_allocate: rsc1-clone allocation score on lxc2: 0
+pcmk__clone_allocate: rsc1-clone allocation score on remote-rhel7-2: 0
+pcmk__clone_allocate: rsc1-clone allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc1-clone allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc1-clone allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc1-clone allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc1:0 allocation score on lxc1: 0
+pcmk__clone_allocate: rsc1:0 allocation score on lxc2: 0
+pcmk__clone_allocate: rsc1:0 allocation score on remote-rhel7-2: 0
+pcmk__clone_allocate: rsc1:0 allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc1:0 allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc1:0 allocation score on rhel7-4: 1
+pcmk__clone_allocate: rsc1:0 allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc1:1 allocation score on lxc1: 0
+pcmk__clone_allocate: rsc1:1 allocation score on lxc2: 0
+pcmk__clone_allocate: rsc1:1 allocation score on remote-rhel7-2: 0
+pcmk__clone_allocate: rsc1:1 allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc1:1 allocation score on rhel7-3: 6
+pcmk__clone_allocate: rsc1:1 allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc1:1 allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc1:2 allocation score on lxc1: 0
+pcmk__clone_allocate: rsc1:2 allocation score on lxc2: 0
+pcmk__clone_allocate: rsc1:2 allocation score on remote-rhel7-2: 0
+pcmk__clone_allocate: rsc1:2 allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc1:2 allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc1:2 allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc1:2 allocation score on rhel7-5: 6
+pcmk__clone_allocate: rsc1:3 allocation score on lxc1: 0
+pcmk__clone_allocate: rsc1:3 allocation score on lxc2: 0
+pcmk__clone_allocate: rsc1:3 allocation score on remote-rhel7-2: 0
+pcmk__clone_allocate: rsc1:3 allocation score on rhel7-1: 6
+pcmk__clone_allocate: rsc1:3 allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc1:3 allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc1:3 allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc1:4 allocation score on lxc1: 0
+pcmk__clone_allocate: rsc1:4 allocation score on lxc2: 0
+pcmk__clone_allocate: rsc1:4 allocation score on remote-rhel7-2: 1
+pcmk__clone_allocate: rsc1:4 allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc1:4 allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc1:4 allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc1:4 allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc1:5 allocation score on lxc1: 0
+pcmk__clone_allocate: rsc1:5 allocation score on lxc2: 6
+pcmk__clone_allocate: rsc1:5 allocation score on remote-rhel7-2: 0
+pcmk__clone_allocate: rsc1:5 allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc1:5 allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc1:5 allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc1:5 allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc1:6 allocation score on lxc1: 6
+pcmk__clone_allocate: rsc1:6 allocation score on lxc2: 0
+pcmk__clone_allocate: rsc1:6 allocation score on remote-rhel7-2: 0
+pcmk__clone_allocate: rsc1:6 allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc1:6 allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc1:6 allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc1:6 allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc2-master allocation score on lxc1: 0
+pcmk__clone_allocate: rsc2-master allocation score on lxc2: 0
+pcmk__clone_allocate: rsc2-master allocation score on remote-rhel7-2: 0
+pcmk__clone_allocate: rsc2-master allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc2-master allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc2-master allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc2-master allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc2:0 allocation score on lxc1: 0
+pcmk__clone_allocate: rsc2:0 allocation score on lxc2: 0
+pcmk__clone_allocate: rsc2:0 allocation score on remote-rhel7-2: 0
+pcmk__clone_allocate: rsc2:0 allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc2:0 allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc2:0 allocation score on rhel7-4: 1
+pcmk__clone_allocate: rsc2:0 allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc2:1 allocation score on lxc1: 0
+pcmk__clone_allocate: rsc2:1 allocation score on lxc2: 0
+pcmk__clone_allocate: rsc2:1 allocation score on remote-rhel7-2: 0
+pcmk__clone_allocate: rsc2:1 allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc2:1 allocation score on rhel7-3: 6
+pcmk__clone_allocate: rsc2:1 allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc2:1 allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc2:2 allocation score on lxc1: 0
+pcmk__clone_allocate: rsc2:2 allocation score on lxc2: 0
+pcmk__clone_allocate: rsc2:2 allocation score on remote-rhel7-2: 0
+pcmk__clone_allocate: rsc2:2 allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc2:2 allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc2:2 allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc2:2 allocation score on rhel7-5: 6
+pcmk__clone_allocate: rsc2:3 allocation score on lxc1: 0
+pcmk__clone_allocate: rsc2:3 allocation score on lxc2: 0
+pcmk__clone_allocate: rsc2:3 allocation score on remote-rhel7-2: 0
+pcmk__clone_allocate: rsc2:3 allocation score on rhel7-1: 6
+pcmk__clone_allocate: rsc2:3 allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc2:3 allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc2:3 allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc2:4 allocation score on lxc1: 0
+pcmk__clone_allocate: rsc2:4 allocation score on lxc2: 0
+pcmk__clone_allocate: rsc2:4 allocation score on remote-rhel7-2: 1
+pcmk__clone_allocate: rsc2:4 allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc2:4 allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc2:4 allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc2:4 allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc2:5 allocation score on lxc1: 0
+pcmk__clone_allocate: rsc2:5 allocation score on lxc2: 6
+pcmk__clone_allocate: rsc2:5 allocation score on remote-rhel7-2: 0
+pcmk__clone_allocate: rsc2:5 allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc2:5 allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc2:5 allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc2:5 allocation score on rhel7-5: 0
+pcmk__clone_allocate: rsc2:6 allocation score on lxc1: 6
+pcmk__clone_allocate: rsc2:6 allocation score on lxc2: 0
+pcmk__clone_allocate: rsc2:6 allocation score on remote-rhel7-2: 0
+pcmk__clone_allocate: rsc2:6 allocation score on rhel7-1: 0
+pcmk__clone_allocate: rsc2:6 allocation score on rhel7-3: 0
+pcmk__clone_allocate: rsc2:6 allocation score on rhel7-4: 0
+pcmk__clone_allocate: rsc2:6 allocation score on rhel7-5: 0
+pcmk__clone_allocate: stateful-bundle-master allocation score on lxc1: -INFINITY
+pcmk__clone_allocate: stateful-bundle-master allocation score on lxc2: -INFINITY
+pcmk__clone_allocate: stateful-bundle-master allocation score on remote-rhel7-2: -INFINITY
+pcmk__clone_allocate: stateful-bundle-master allocation score on rhel7-1: -INFINITY
+pcmk__clone_allocate: stateful-bundle-master allocation score on rhel7-3: -INFINITY
+pcmk__clone_allocate: stateful-bundle-master allocation score on rhel7-4: -INFINITY
+pcmk__clone_allocate: stateful-bundle-master allocation score on rhel7-5: -INFINITY
+pcmk__clone_allocate: stateful-bundle-master allocation score on stateful-bundle-0: 0
+pcmk__clone_allocate: stateful-bundle-master allocation score on stateful-bundle-1: 0
+pcmk__clone_allocate: stateful-bundle-master allocation score on stateful-bundle-2: 0
+pcmk__native_allocate: Fencing allocation score on lxc1: -INFINITY
+pcmk__native_allocate: Fencing allocation score on lxc2: -INFINITY
+pcmk__native_allocate: Fencing allocation score on remote-rhel7-2: -INFINITY
+pcmk__native_allocate: Fencing allocation score on rhel7-1: 0
+pcmk__native_allocate: Fencing allocation score on rhel7-3: 0
+pcmk__native_allocate: Fencing allocation score on rhel7-4: 0
+pcmk__native_allocate: Fencing allocation score on rhel7-5: 0
+pcmk__native_allocate: bundled:0 allocation score on stateful-bundle-0: INFINITY
+pcmk__native_allocate: bundled:1 allocation score on stateful-bundle-1: INFINITY
+pcmk__native_allocate: bundled:2 allocation score on stateful-bundle-2: INFINITY
+pcmk__native_allocate: container1 allocation score on lxc1: -INFINITY
+pcmk__native_allocate: container1 allocation score on lxc2: -INFINITY
+pcmk__native_allocate: container1 allocation score on remote-rhel7-2: -INFINITY
+pcmk__native_allocate: container1 allocation score on rhel7-1: 0
+pcmk__native_allocate: container1 allocation score on rhel7-3: INFINITY
+pcmk__native_allocate: container1 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: container1 allocation score on rhel7-5: 0
+pcmk__native_allocate: container2 allocation score on lxc1: -INFINITY
+pcmk__native_allocate: container2 allocation score on lxc2: -INFINITY
+pcmk__native_allocate: container2 allocation score on remote-rhel7-2: -INFINITY
+pcmk__native_allocate: container2 allocation score on rhel7-1: 0
+pcmk__native_allocate: container2 allocation score on rhel7-3: INFINITY
+pcmk__native_allocate: container2 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: container2 allocation score on rhel7-5: 0
+pcmk__native_allocate: lxc-ms:0 allocation score on lxc1: -INFINITY
+pcmk__native_allocate: lxc-ms:0 allocation score on lxc2: INFINITY
+pcmk__native_allocate: lxc-ms:0 allocation score on remote-rhel7-2: -INFINITY
+pcmk__native_allocate: lxc-ms:0 allocation score on rhel7-1: 0
+pcmk__native_allocate: lxc-ms:0 allocation score on rhel7-3: 0
+pcmk__native_allocate: lxc-ms:0 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: lxc-ms:0 allocation score on rhel7-5: 0
+pcmk__native_allocate: lxc-ms:1 allocation score on lxc1: INFINITY
+pcmk__native_allocate: lxc-ms:1 allocation score on lxc2: INFINITY
+pcmk__native_allocate: lxc-ms:1 allocation score on remote-rhel7-2: -INFINITY
+pcmk__native_allocate: lxc-ms:1 allocation score on rhel7-1: 0
+pcmk__native_allocate: lxc-ms:1 allocation score on rhel7-3: 0
+pcmk__native_allocate: lxc-ms:1 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: lxc-ms:1 allocation score on rhel7-5: 0
+pcmk__native_allocate: lxc1 allocation score on lxc1: -INFINITY
+pcmk__native_allocate: lxc1 allocation score on lxc2: -INFINITY
+pcmk__native_allocate: lxc1 allocation score on remote-rhel7-2: -INFINITY
+pcmk__native_allocate: lxc1 allocation score on rhel7-1: -INFINITY
+pcmk__native_allocate: lxc1 allocation score on rhel7-3: 0
+pcmk__native_allocate: lxc1 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: lxc1 allocation score on rhel7-5: -INFINITY
+pcmk__native_allocate: lxc2 allocation score on lxc1: -INFINITY
+pcmk__native_allocate: lxc2 allocation score on lxc2: -INFINITY
+pcmk__native_allocate: lxc2 allocation score on remote-rhel7-2: -INFINITY
+pcmk__native_allocate: lxc2 allocation score on rhel7-1: -INFINITY
+pcmk__native_allocate: lxc2 allocation score on rhel7-3: 0
+pcmk__native_allocate: lxc2 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: lxc2 allocation score on rhel7-5: -INFINITY
+pcmk__native_allocate: remote-rhel7-2 allocation score on lxc1: -INFINITY
+pcmk__native_allocate: remote-rhel7-2 allocation score on lxc2: -INFINITY
+pcmk__native_allocate: remote-rhel7-2 allocation score on remote-rhel7-2: -INFINITY
+pcmk__native_allocate: remote-rhel7-2 allocation score on rhel7-1: 0
+pcmk__native_allocate: remote-rhel7-2 allocation score on rhel7-3: 0
+pcmk__native_allocate: remote-rhel7-2 allocation score on rhel7-4: 0
+pcmk__native_allocate: remote-rhel7-2 allocation score on rhel7-5: 0
+pcmk__native_allocate: rsc1:0 allocation score on lxc1: -INFINITY
+pcmk__native_allocate: rsc1:0 allocation score on lxc2: -INFINITY
+pcmk__native_allocate: rsc1:0 allocation score on remote-rhel7-2: -INFINITY
+pcmk__native_allocate: rsc1:0 allocation score on rhel7-1: -INFINITY
+pcmk__native_allocate: rsc1:0 allocation score on rhel7-3: -INFINITY
+pcmk__native_allocate: rsc1:0 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: rsc1:0 allocation score on rhel7-5: -INFINITY
+pcmk__native_allocate: rsc1:1 allocation score on lxc1: 0
+pcmk__native_allocate: rsc1:1 allocation score on lxc2: 0
+pcmk__native_allocate: rsc1:1 allocation score on remote-rhel7-2: -INFINITY
+pcmk__native_allocate: rsc1:1 allocation score on rhel7-1: 0
+pcmk__native_allocate: rsc1:1 allocation score on rhel7-3: 6
+pcmk__native_allocate: rsc1:1 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: rsc1:1 allocation score on rhel7-5: 0
+pcmk__native_allocate: rsc1:2 allocation score on lxc1: 0
+pcmk__native_allocate: rsc1:2 allocation score on lxc2: 0
+pcmk__native_allocate: rsc1:2 allocation score on remote-rhel7-2: -INFINITY
+pcmk__native_allocate: rsc1:2 allocation score on rhel7-1: 0
+pcmk__native_allocate: rsc1:2 allocation score on rhel7-3: -INFINITY
+pcmk__native_allocate: rsc1:2 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: rsc1:2 allocation score on rhel7-5: 6
+pcmk__native_allocate: rsc1:3 allocation score on lxc1: 0
+pcmk__native_allocate: rsc1:3 allocation score on lxc2: 0
+pcmk__native_allocate: rsc1:3 allocation score on remote-rhel7-2: -INFINITY
+pcmk__native_allocate: rsc1:3 allocation score on rhel7-1: 6
+pcmk__native_allocate: rsc1:3 allocation score on rhel7-3: -INFINITY
+pcmk__native_allocate: rsc1:3 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: rsc1:3 allocation score on rhel7-5: -INFINITY
+pcmk__native_allocate: rsc1:4 allocation score on lxc1: -INFINITY
+pcmk__native_allocate: rsc1:4 allocation score on lxc2: -INFINITY
+pcmk__native_allocate: rsc1:4 allocation score on remote-rhel7-2: -INFINITY
+pcmk__native_allocate: rsc1:4 allocation score on rhel7-1: -INFINITY
+pcmk__native_allocate: rsc1:4 allocation score on rhel7-3: -INFINITY
+pcmk__native_allocate: rsc1:4 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: rsc1:4 allocation score on rhel7-5: -INFINITY
+pcmk__native_allocate: rsc1:5 allocation score on lxc1: -INFINITY
+pcmk__native_allocate: rsc1:5 allocation score on lxc2: 6
+pcmk__native_allocate: rsc1:5 allocation score on remote-rhel7-2: -INFINITY
+pcmk__native_allocate: rsc1:5 allocation score on rhel7-1: -INFINITY
+pcmk__native_allocate: rsc1:5 allocation score on rhel7-3: -INFINITY
+pcmk__native_allocate: rsc1:5 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: rsc1:5 allocation score on rhel7-5: -INFINITY
+pcmk__native_allocate: rsc1:6 allocation score on lxc1: 6
+pcmk__native_allocate: rsc1:6 allocation score on lxc2: 0
+pcmk__native_allocate: rsc1:6 allocation score on remote-rhel7-2: -INFINITY
+pcmk__native_allocate: rsc1:6 allocation score on rhel7-1: -INFINITY
+pcmk__native_allocate: rsc1:6 allocation score on rhel7-3: -INFINITY
+pcmk__native_allocate: rsc1:6 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: rsc1:6 allocation score on rhel7-5: -INFINITY
+pcmk__native_allocate: rsc2:0 allocation score on lxc1: -INFINITY
+pcmk__native_allocate: rsc2:0 allocation score on lxc2: -INFINITY
+pcmk__native_allocate: rsc2:0 allocation score on remote-rhel7-2: -INFINITY
+pcmk__native_allocate: rsc2:0 allocation score on rhel7-1: -INFINITY
+pcmk__native_allocate: rsc2:0 allocation score on rhel7-3: -INFINITY
+pcmk__native_allocate: rsc2:0 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: rsc2:0 allocation score on rhel7-5: -INFINITY
+pcmk__native_allocate: rsc2:1 allocation score on lxc1: 0
+pcmk__native_allocate: rsc2:1 allocation score on lxc2: 0
+pcmk__native_allocate: rsc2:1 allocation score on remote-rhel7-2: -INFINITY
+pcmk__native_allocate: rsc2:1 allocation score on rhel7-1: 0
+pcmk__native_allocate: rsc2:1 allocation score on rhel7-3: 6
+pcmk__native_allocate: rsc2:1 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: rsc2:1 allocation score on rhel7-5: 0
+pcmk__native_allocate: rsc2:2 allocation score on lxc1: 0
+pcmk__native_allocate: rsc2:2 allocation score on lxc2: 0
+pcmk__native_allocate: rsc2:2 allocation score on remote-rhel7-2: -INFINITY
+pcmk__native_allocate: rsc2:2 allocation score on rhel7-1: 0
+pcmk__native_allocate: rsc2:2 allocation score on rhel7-3: -INFINITY
+pcmk__native_allocate: rsc2:2 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: rsc2:2 allocation score on rhel7-5: 6
+pcmk__native_allocate: rsc2:3 allocation score on lxc1: 0
+pcmk__native_allocate: rsc2:3 allocation score on lxc2: 0
+pcmk__native_allocate: rsc2:3 allocation score on remote-rhel7-2: -INFINITY
+pcmk__native_allocate: rsc2:3 allocation score on rhel7-1: 6
+pcmk__native_allocate: rsc2:3 allocation score on rhel7-3: -INFINITY
+pcmk__native_allocate: rsc2:3 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: rsc2:3 allocation score on rhel7-5: -INFINITY
+pcmk__native_allocate: rsc2:4 allocation score on lxc1: -INFINITY
+pcmk__native_allocate: rsc2:4 allocation score on lxc2: -INFINITY
+pcmk__native_allocate: rsc2:4 allocation score on remote-rhel7-2: -INFINITY
+pcmk__native_allocate: rsc2:4 allocation score on rhel7-1: -INFINITY
+pcmk__native_allocate: rsc2:4 allocation score on rhel7-3: -INFINITY
+pcmk__native_allocate: rsc2:4 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: rsc2:4 allocation score on rhel7-5: -INFINITY
+pcmk__native_allocate: rsc2:5 allocation score on lxc1: -INFINITY
+pcmk__native_allocate: rsc2:5 allocation score on lxc2: 6
+pcmk__native_allocate: rsc2:5 allocation score on remote-rhel7-2: -INFINITY
+pcmk__native_allocate: rsc2:5 allocation score on rhel7-1: -INFINITY
+pcmk__native_allocate: rsc2:5 allocation score on rhel7-3: -INFINITY
+pcmk__native_allocate: rsc2:5 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: rsc2:5 allocation score on rhel7-5: -INFINITY
+pcmk__native_allocate: rsc2:6 allocation score on lxc1: 6
+pcmk__native_allocate: rsc2:6 allocation score on lxc2: 0
+pcmk__native_allocate: rsc2:6 allocation score on remote-rhel7-2: -INFINITY
+pcmk__native_allocate: rsc2:6 allocation score on rhel7-1: -INFINITY
+pcmk__native_allocate: rsc2:6 allocation score on rhel7-3: -INFINITY
+pcmk__native_allocate: rsc2:6 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: rsc2:6 allocation score on rhel7-5: -INFINITY
+pcmk__native_allocate: stateful-bundle-0 allocation score on lxc1: -INFINITY
+pcmk__native_allocate: stateful-bundle-0 allocation score on lxc2: -INFINITY
+pcmk__native_allocate: stateful-bundle-0 allocation score on remote-rhel7-2: -INFINITY
+pcmk__native_allocate: stateful-bundle-0 allocation score on rhel7-1: 0
+pcmk__native_allocate: stateful-bundle-0 allocation score on rhel7-3: 0
+pcmk__native_allocate: stateful-bundle-0 allocation score on rhel7-4: 0
+pcmk__native_allocate: stateful-bundle-0 allocation score on rhel7-5: 10000
+pcmk__native_allocate: stateful-bundle-1 allocation score on lxc1: -INFINITY
+pcmk__native_allocate: stateful-bundle-1 allocation score on lxc2: -INFINITY
+pcmk__native_allocate: stateful-bundle-1 allocation score on remote-rhel7-2: -INFINITY
+pcmk__native_allocate: stateful-bundle-1 allocation score on rhel7-1: 10000
+pcmk__native_allocate: stateful-bundle-1 allocation score on rhel7-3: 0
+pcmk__native_allocate: stateful-bundle-1 allocation score on rhel7-4: 0
+pcmk__native_allocate: stateful-bundle-1 allocation score on rhel7-5: 0
+pcmk__native_allocate: stateful-bundle-2 allocation score on lxc1: -INFINITY
+pcmk__native_allocate: stateful-bundle-2 allocation score on lxc2: -INFINITY
+pcmk__native_allocate: stateful-bundle-2 allocation score on remote-rhel7-2: -INFINITY
+pcmk__native_allocate: stateful-bundle-2 allocation score on rhel7-1: 0
+pcmk__native_allocate: stateful-bundle-2 allocation score on rhel7-3: 10000
+pcmk__native_allocate: stateful-bundle-2 allocation score on rhel7-4: 0
+pcmk__native_allocate: stateful-bundle-2 allocation score on rhel7-5: 0
+pcmk__native_allocate: stateful-bundle-docker-0 allocation score on lxc1: -10000
+pcmk__native_allocate: stateful-bundle-docker-0 allocation score on lxc2: -10000
+pcmk__native_allocate: stateful-bundle-docker-0 allocation score on remote-rhel7-2: -INFINITY
+pcmk__native_allocate: stateful-bundle-docker-0 allocation score on rhel7-1: -INFINITY
+pcmk__native_allocate: stateful-bundle-docker-0 allocation score on rhel7-3: 0
+pcmk__native_allocate: stateful-bundle-docker-0 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: stateful-bundle-docker-0 allocation score on rhel7-5: 0
+pcmk__native_allocate: stateful-bundle-docker-1 allocation score on lxc1: -10000
+pcmk__native_allocate: stateful-bundle-docker-1 allocation score on lxc2: -10000
+pcmk__native_allocate: stateful-bundle-docker-1 allocation score on remote-rhel7-2: -INFINITY
+pcmk__native_allocate: stateful-bundle-docker-1 allocation score on rhel7-1: 0
+pcmk__native_allocate: stateful-bundle-docker-1 allocation score on rhel7-3: 0
+pcmk__native_allocate: stateful-bundle-docker-1 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: stateful-bundle-docker-1 allocation score on rhel7-5: 0
+pcmk__native_allocate: stateful-bundle-docker-2 allocation score on lxc1: -10000
+pcmk__native_allocate: stateful-bundle-docker-2 allocation score on lxc2: -10000
+pcmk__native_allocate: stateful-bundle-docker-2 allocation score on remote-rhel7-2: -INFINITY
+pcmk__native_allocate: stateful-bundle-docker-2 allocation score on rhel7-1: -INFINITY
+pcmk__native_allocate: stateful-bundle-docker-2 allocation score on rhel7-3: 0
+pcmk__native_allocate: stateful-bundle-docker-2 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: stateful-bundle-docker-2 allocation score on rhel7-5: -INFINITY
+pcmk__native_allocate: stateful-bundle-ip-192.168.122.131 allocation score on lxc1: -INFINITY
+pcmk__native_allocate: stateful-bundle-ip-192.168.122.131 allocation score on lxc2: -INFINITY
+pcmk__native_allocate: stateful-bundle-ip-192.168.122.131 allocation score on remote-rhel7-2: -INFINITY
+pcmk__native_allocate: stateful-bundle-ip-192.168.122.131 allocation score on rhel7-1: -INFINITY
+pcmk__native_allocate: stateful-bundle-ip-192.168.122.131 allocation score on rhel7-3: -INFINITY
+pcmk__native_allocate: stateful-bundle-ip-192.168.122.131 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: stateful-bundle-ip-192.168.122.131 allocation score on rhel7-5: 0
+pcmk__native_allocate: stateful-bundle-ip-192.168.122.132 allocation score on lxc1: -INFINITY
+pcmk__native_allocate: stateful-bundle-ip-192.168.122.132 allocation score on lxc2: -INFINITY
+pcmk__native_allocate: stateful-bundle-ip-192.168.122.132 allocation score on remote-rhel7-2: -INFINITY
+pcmk__native_allocate: stateful-bundle-ip-192.168.122.132 allocation score on rhel7-1: 0
+pcmk__native_allocate: stateful-bundle-ip-192.168.122.132 allocation score on rhel7-3: -INFINITY
+pcmk__native_allocate: stateful-bundle-ip-192.168.122.132 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: stateful-bundle-ip-192.168.122.132 allocation score on rhel7-5: -INFINITY
+pcmk__native_allocate: stateful-bundle-ip-192.168.122.133 allocation score on lxc1: -INFINITY
+pcmk__native_allocate: stateful-bundle-ip-192.168.122.133 allocation score on lxc2: -INFINITY
+pcmk__native_allocate: stateful-bundle-ip-192.168.122.133 allocation score on remote-rhel7-2: -INFINITY
+pcmk__native_allocate: stateful-bundle-ip-192.168.122.133 allocation score on rhel7-1: -INFINITY
+pcmk__native_allocate: stateful-bundle-ip-192.168.122.133 allocation score on rhel7-3: 0
+pcmk__native_allocate: stateful-bundle-ip-192.168.122.133 allocation score on rhel7-4: -INFINITY
+pcmk__native_allocate: stateful-bundle-ip-192.168.122.133 allocation score on rhel7-5: -INFINITY
+rsc1:0 promotion score on none: 0
+rsc1:1 promotion score on rhel7-3: 5
+rsc1:2 promotion score on rhel7-5: 5
+rsc1:3 promotion score on rhel7-1: 5
+rsc1:4 promotion score on none: 0
+rsc1:5 promotion score on lxc2: 5
+rsc1:6 promotion score on lxc1: 5
+rsc2:0 promotion score on none: 0
+rsc2:1 promotion score on rhel7-3: 5
+rsc2:2 promotion score on rhel7-5: 5
+rsc2:3 promotion score on rhel7-1: 5
+rsc2:4 promotion score on none: 0
+rsc2:5 promotion score on lxc2: 5
+rsc2:6 promotion score on lxc1: 5
diff --git a/cts/scheduler/on_fail_demote4.summary b/cts/scheduler/on_fail_demote4.summary
new file mode 100644
index 0000000000..20520ff763
--- /dev/null
+++ b/cts/scheduler/on_fail_demote4.summary
@@ -0,0 +1,187 @@
+Using the original execution date of: 2020-06-16 19:23:21Z
+
+Current cluster status:
+RemoteNode remote-rhel7-2: UNCLEAN (offline)
+Node rhel7-4 (4): UNCLEAN (offline)
+Online: [ rhel7-1 rhel7-3 rhel7-5 ]
+GuestOnline: [ lxc1:container1 stateful-bundle-1:stateful-bundle-docker-1 ]
+
+ Fencing (stonith:fence_xvm): Started rhel7-4 (UNCLEAN)
+ Clone Set: rsc1-clone [rsc1] (promotable)
+ rsc1 (ocf::pacemaker:Stateful): Master rhel7-4 (UNCLEAN)
+ rsc1 (ocf::pacemaker:Stateful): Slave remote-rhel7-2 (UNCLEAN)
+ Slaves: [ lxc1 rhel7-1 rhel7-3 rhel7-5 ]
+ Clone Set: rsc2-master [rsc2] (promotable)
+ rsc2 (ocf::pacemaker:Stateful): Slave rhel7-4 (UNCLEAN)
+ rsc2 (ocf::pacemaker:Stateful): Master remote-rhel7-2 (UNCLEAN)
+ Slaves: [ lxc1 rhel7-1 rhel7-3 rhel7-5 ]
+ remote-rhel7-2 (ocf::pacemaker:remote): FAILED rhel7-1
+ container1 (ocf::heartbeat:VirtualDomain): Started rhel7-3
+ container2 (ocf::heartbeat:VirtualDomain): FAILED rhel7-3
+ Clone Set: lxc-ms-master [lxc-ms] (promotable)
+ Slaves: [ lxc1 ]
+ Stopped: [ remote-rhel7-2 rhel7-1 rhel7-3 rhel7-4 rhel7-5 ]
+ Container bundle set: stateful-bundle [pcmktest:http]
+ stateful-bundle-0 (192.168.122.131) (ocf::pacemaker:Stateful): FAILED Master rhel7-5
+ stateful-bundle-1 (192.168.122.132) (ocf::pacemaker:Stateful): Slave rhel7-1
+ stateful-bundle-2 (192.168.122.133) (ocf::pacemaker:Stateful): FAILED rhel7-4 (UNCLEAN)
+
+Transition Summary:
+ * Fence (reboot) stateful-bundle-2 (resource: stateful-bundle-docker-2) 'guest is unclean'
+ * Fence (reboot) stateful-bundle-0 (resource: stateful-bundle-docker-0) 'guest is unclean'
+ * Fence (reboot) lxc2 (resource: container2) 'guest is unclean'
+ * Fence (reboot) remote-rhel7-2 'remote connection is unrecoverable'
+ * Fence (reboot) rhel7-4 'peer is no longer part of the cluster'
+ * Move Fencing ( rhel7-4 -> rhel7-5 )
+ * Stop rsc1:0 ( Master rhel7-4 ) due to node availability
+ * Promote rsc1:1 ( Slave -> Master rhel7-3 )
+ * Stop rsc1:4 ( Slave remote-rhel7-2 ) due to node availability
+ * Recover rsc1:5 ( Slave lxc2 )
+ * Stop rsc2:0 ( Slave rhel7-4 ) due to node availability
+ * Promote rsc2:1 ( Slave -> Master rhel7-3 )
+ * Stop rsc2:4 ( Master remote-rhel7-2 ) due to node availability
+ * Recover rsc2:5 ( Slave lxc2 )
+ * Recover remote-rhel7-2 ( rhel7-1 )
+ * Recover container2 ( rhel7-3 )
+ * Recover lxc-ms:0 ( Master lxc2 )
+ * Recover stateful-bundle-docker-0 ( rhel7-5 )
+ * Restart stateful-bundle-0 ( rhel7-5 ) due to required stateful-bundle-docker-0 start
+ * Recover bundled:0 ( Master stateful-bundle-0 )
+ * Move stateful-bundle-ip-192.168.122.133 ( rhel7-4 -> rhel7-3 )
+ * Recover stateful-bundle-docker-2 ( rhel7-4 -> rhel7-3 )
+ * Move stateful-bundle-2 ( rhel7-4 -> rhel7-3 )
+ * Recover bundled:2 ( Slave stateful-bundle-2 )
+ * Restart lxc2 ( rhel7-3 ) due to required container2 start
+
+Executing cluster transition:
+ * Pseudo action: Fencing_stop_0
+ * Resource action: rsc1 cancel=11000 on rhel7-3
+ * Pseudo action: rsc1-clone_demote_0
+ * Resource action: rsc2 cancel=11000 on rhel7-3
+ * Pseudo action: rsc2-master_demote_0
+ * Pseudo action: lxc-ms-master_demote_0
+ * Resource action: stateful-bundle-0 stop on rhel7-5
+ * Pseudo action: stateful-bundle-2_stop_0
+ * Resource action: lxc2 stop on rhel7-3
+ * Pseudo action: stateful-bundle_demote_0
+ * Fencing remote-rhel7-2 (reboot)
+ * Fencing rhel7-4 (reboot)
+ * Pseudo action: rsc1_demote_0
+ * Pseudo action: rsc1-clone_demoted_0
+ * Pseudo action: rsc2_demote_0
+ * Pseudo action: rsc2-master_demoted_0
+ * Resource action: container2 stop on rhel7-3
+ * Pseudo action: stateful-bundle-master_demote_0
+ * Pseudo action: stonith-stateful-bundle-2-reboot on stateful-bundle-2
+ * Pseudo action: stonith-lxc2-reboot on lxc2
+ * Resource action: Fencing start on rhel7-5
+ * Pseudo action: rsc1-clone_stop_0
+ * Pseudo action: rsc2-master_stop_0
+ * Pseudo action: lxc-ms_demote_0
+ * Pseudo action: lxc-ms-master_demoted_0
+ * Pseudo action: lxc-ms-master_stop_0
+ * Pseudo action: bundled_demote_0
+ * Pseudo action: stateful-bundle-master_demoted_0
+ * Pseudo action: stateful-bundle_demoted_0
+ * Pseudo action: stateful-bundle_stop_0
+ * Resource action: Fencing monitor=120000 on rhel7-5
+ * Pseudo action: rsc1_stop_0
+ * Pseudo action: rsc1_stop_0
+ * Pseudo action: rsc1_stop_0
+ * Pseudo action: rsc1-clone_stopped_0
+ * Pseudo action: rsc1-clone_start_0
+ * Pseudo action: rsc2_stop_0
+ * Pseudo action: rsc2_stop_0
+ * Pseudo action: rsc2_stop_0
+ * Pseudo action: rsc2-master_stopped_0
+ * Pseudo action: rsc2-master_start_0
+ * Resource action: remote-rhel7-2 stop on rhel7-1
+ * Pseudo action: lxc-ms_stop_0
+ * Pseudo action: lxc-ms-master_stopped_0
+ * Pseudo action: lxc-ms-master_start_0
+ * Resource action: stateful-bundle-docker-0 stop on rhel7-5
+ * Pseudo action: stateful-bundle-docker-2_stop_0
+ * Pseudo action: stonith-stateful-bundle-0-reboot on stateful-bundle-0
+ * Resource action: remote-rhel7-2 start on rhel7-1
+ * Resource action: remote-rhel7-2 monitor=60000 on rhel7-1
+ * Resource action: container2 start on rhel7-3
+ * Resource action: container2 monitor=20000 on rhel7-3
+ * Pseudo action: stateful-bundle-master_stop_0
+ * Pseudo action: stateful-bundle-ip-192.168.122.133_stop_0
+ * Resource action: lxc2 start on rhel7-3
+ * Resource action: lxc2 monitor=30000 on rhel7-3
+ * Resource action: rsc1 start on lxc2
+ * Pseudo action: rsc1-clone_running_0
+ * Resource action: rsc2 start on lxc2
+ * Pseudo action: rsc2-master_running_0
+ * Resource action: lxc-ms start on lxc2
+ * Pseudo action: lxc-ms-master_running_0
+ * Pseudo action: bundled_stop_0
+ * Resource action: stateful-bundle-ip-192.168.122.133 start on rhel7-3
+ * Resource action: rsc1 monitor=11000 on lxc2
+ * Pseudo action: rsc1-clone_promote_0
+ * Resource action: rsc2 monitor=11000 on lxc2
+ * Pseudo action: rsc2-master_promote_0
+ * Pseudo action: lxc-ms-master_promote_0
+ * Pseudo action: bundled_stop_0
+ * Pseudo action: stateful-bundle-master_stopped_0
+ * Resource action: stateful-bundle-ip-192.168.122.133 monitor=60000 on rhel7-3
+ * Pseudo action: stateful-bundle_stopped_0
+ * Pseudo action: stateful-bundle_start_0
+ * Resource action: rsc1 promote on rhel7-3
+ * Pseudo action: rsc1-clone_promoted_0
+ * Resource action: rsc2 promote on rhel7-3
+ * Pseudo action: rsc2-master_promoted_0
+ * Resource action: lxc-ms promote on lxc2
+ * Pseudo action: lxc-ms-master_promoted_0
+ * Pseudo action: stateful-bundle-master_start_0
+ * Resource action: stateful-bundle-docker-0 start on rhel7-5
+ * Resource action: stateful-bundle-docker-0 monitor=60000 on rhel7-5
+ * Resource action: stateful-bundle-0 start on rhel7-5
+ * Resource action: stateful-bundle-0 monitor=30000 on rhel7-5
+ * Resource action: stateful-bundle-docker-2 start on rhel7-3
+ * Resource action: stateful-bundle-2 start on rhel7-3
+ * Resource action: rsc1 monitor=10000 on rhel7-3
+ * Resource action: rsc2 monitor=10000 on rhel7-3
+ * Resource action: lxc-ms monitor=10000 on lxc2
+ * Resource action: bundled start on stateful-bundle-0
+ * Resource action: bundled start on stateful-bundle-2
+ * Pseudo action: stateful-bundle-master_running_0
+ * Resource action: stateful-bundle-docker-2 monitor=60000 on rhel7-3
+ * Resource action: stateful-bundle-2 monitor=30000 on rhel7-3
+ * Pseudo action: stateful-bundle_running_0
+ * Resource action: bundled monitor=11000 on stateful-bundle-2
+ * Pseudo action: stateful-bundle_promote_0
+ * Pseudo action: stateful-bundle-master_promote_0
+ * Resource action: bundled promote on stateful-bundle-0
+ * Pseudo action: stateful-bundle-master_promoted_0
+ * Pseudo action: stateful-bundle_promoted_0
+ * Resource action: bundled monitor=10000 on stateful-bundle-0
+Using the original execution date of: 2020-06-16 19:23:21Z
+
+Revised cluster status:
+Online: [ rhel7-1 rhel7-3 rhel7-5 ]
+OFFLINE: [ rhel7-4 ]
+RemoteOnline: [ remote-rhel7-2 ]
+GuestOnline: [ lxc1:container1 lxc2:container2 stateful-bundle-0:stateful-bundle-docker-0 stateful-bundle-1:stateful-bundle-docker-1 stateful-bundle-2:stateful-bundle-docker-2 ]
+
+ Fencing (stonith:fence_xvm): Started rhel7-5
+ Clone Set: rsc1-clone [rsc1] (promotable)
+ Masters: [ rhel7-3 ]
+ Slaves: [ lxc1 lxc2 rhel7-1 rhel7-5 ]
+ Stopped: [ remote-rhel7-2 rhel7-4 ]
+ Clone Set: rsc2-master [rsc2] (promotable)
+ Masters: [ rhel7-3 ]
+ Slaves: [ lxc1 lxc2 rhel7-1 rhel7-5 ]
+ Stopped: [ remote-rhel7-2 rhel7-4 ]
+ remote-rhel7-2 (ocf::pacemaker:remote): Started rhel7-1
+ container1 (ocf::heartbeat:VirtualDomain): Started rhel7-3
+ container2 (ocf::heartbeat:VirtualDomain): Started rhel7-3
+ Clone Set: lxc-ms-master [lxc-ms] (promotable)
+ Masters: [ lxc2 ]
+ Slaves: [ lxc1 ]
+ Container bundle set: stateful-bundle [pcmktest:http]
+ stateful-bundle-0 (192.168.122.131) (ocf::pacemaker:Stateful): Master rhel7-5
+ stateful-bundle-1 (192.168.122.132) (ocf::pacemaker:Stateful): Slave rhel7-1
+ stateful-bundle-2 (192.168.122.133) (ocf::pacemaker:Stateful): Slave rhel7-3
+
diff --git a/cts/scheduler/on_fail_demote4.xml b/cts/scheduler/on_fail_demote4.xml
new file mode 100644
index 0000000000..eb4c4cc134
--- /dev/null
+++ b/cts/scheduler/on_fail_demote4.xml
@@ -0,0 +1,625 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/cts/scheduler/params-3.dot b/cts/scheduler/params-3.dot
new file mode 100644
index 0000000000..d681ee52ce
--- /dev/null
+++ b/cts/scheduler/params-3.dot
@@ -0,0 +1,28 @@
+ digraph "g" {
+"Cancel rsc_c001n02_monitor_5000 c001n02" [ style=bold color="green" fontcolor="black"]
+"DcIPaddr_monitor_0 c001n01" -> "DcIPaddr_start_0 c001n02" [ style = bold]
+"DcIPaddr_monitor_0 c001n01" [ style=bold color="green" fontcolor="black"]
+"DcIPaddr_monitor_0 c001n03" -> "DcIPaddr_start_0 c001n02" [ style = bold]
+"DcIPaddr_monitor_0 c001n03" [ style=bold color="green" fontcolor="black"]
+"DcIPaddr_monitor_0 c001n08" -> "DcIPaddr_start_0 c001n02" [ style = bold]
+"DcIPaddr_monitor_0 c001n08" [ style=bold color="green" fontcolor="black"]
+"DcIPaddr_monitor_5000 c001n02" [ style=bold color="green" fontcolor="black"]
+"DcIPaddr_start_0 c001n02" -> "DcIPaddr_monitor_5000 c001n02" [ style = bold]
+"DcIPaddr_start_0 c001n02" [ style=bold color="green" fontcolor="black"]
+"DcIPaddr_stop_0 c001n02" -> "DcIPaddr_start_0 c001n02" [ style = bold]
+"DcIPaddr_stop_0 c001n02" [ style=bold color="green" fontcolor="black"]
+"rsc_c001n01_monitor_0 c001n02" [ style=bold color="green" fontcolor="black"]
+"rsc_c001n01_monitor_0 c001n03" [ style=bold color="green" fontcolor="black"]
+"rsc_c001n01_monitor_0 c001n08" [ style=bold color="green" fontcolor="black"]
+"rsc_c001n02_monitor_0 c001n01" [ style=bold color="green" fontcolor="black"]
+"rsc_c001n02_monitor_0 c001n03" [ style=bold color="green" fontcolor="black"]
+"rsc_c001n02_monitor_0 c001n08" [ style=bold color="green" fontcolor="black"]
+"rsc_c001n02_monitor_6000 c001n02" [ style=bold color="green" fontcolor="black"]
+"rsc_c001n03_monitor_0 c001n01" [ style=bold color="green" fontcolor="black"]
+"rsc_c001n03_monitor_0 c001n02" [ style=bold color="green" fontcolor="black"]
+"rsc_c001n03_monitor_0 c001n08" [ style=bold color="green" fontcolor="black"]
+"rsc_c001n08_monitor_0 c001n01" [ style=bold color="green" fontcolor="black"]
+"rsc_c001n08_monitor_0 c001n02" [ style=bold color="green" fontcolor="black"]
+"rsc_c001n08_monitor_0 c001n03" [ style=bold color="green" fontcolor="black"]
+"rsc_c001n08_monitor_5000 c001n08" [ style=bold color="green" fontcolor="black"]
+}
diff --git a/cts/scheduler/params-3.exp b/cts/scheduler/params-3.exp
new file mode 100644
index 0000000000..5cccdec2c8
--- /dev/null
+++ b/cts/scheduler/params-3.exp
@@ -0,0 +1,208 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/cts/scheduler/params-3.scores b/cts/scheduler/params-3.scores
new file mode 100644
index 0000000000..00417eafc2
--- /dev/null
+++ b/cts/scheduler/params-3.scores
@@ -0,0 +1,21 @@
+Allocation scores:
+pcmk__native_allocate: DcIPaddr allocation score on c001n01: -INFINITY
+pcmk__native_allocate: DcIPaddr allocation score on c001n02: 0
+pcmk__native_allocate: DcIPaddr allocation score on c001n03: -INFINITY
+pcmk__native_allocate: DcIPaddr allocation score on c001n08: -INFINITY
+pcmk__native_allocate: rsc_c001n01 allocation score on c001n01: 100
+pcmk__native_allocate: rsc_c001n01 allocation score on c001n02: 0
+pcmk__native_allocate: rsc_c001n01 allocation score on c001n03: 0
+pcmk__native_allocate: rsc_c001n01 allocation score on c001n08: 0
+pcmk__native_allocate: rsc_c001n02 allocation score on c001n01: 0
+pcmk__native_allocate: rsc_c001n02 allocation score on c001n02: 100
+pcmk__native_allocate: rsc_c001n02 allocation score on c001n03: 0
+pcmk__native_allocate: rsc_c001n02 allocation score on c001n08: 0
+pcmk__native_allocate: rsc_c001n03 allocation score on c001n01: 0
+pcmk__native_allocate: rsc_c001n03 allocation score on c001n02: 0
+pcmk__native_allocate: rsc_c001n03 allocation score on c001n03: 100
+pcmk__native_allocate: rsc_c001n03 allocation score on c001n08: 0
+pcmk__native_allocate: rsc_c001n08 allocation score on c001n01: 0
+pcmk__native_allocate: rsc_c001n08 allocation score on c001n02: 0
+pcmk__native_allocate: rsc_c001n08 allocation score on c001n03: 0
+pcmk__native_allocate: rsc_c001n08 allocation score on c001n08: 100
diff --git a/cts/scheduler/params-3.summary b/cts/scheduler/params-3.summary
new file mode 100644
index 0000000000..257f8babcb
--- /dev/null
+++ b/cts/scheduler/params-3.summary
@@ -0,0 +1,45 @@
+
+Current cluster status:
+Online: [ c001n01 c001n02 c001n03 c001n08 ]
+
+ DcIPaddr (ocf::heartbeat:IPaddr): Starting c001n02
+ rsc_c001n08 (ocf::heartbeat:IPaddr): Started c001n08
+ rsc_c001n02 (ocf::heartbeat:IPaddr): Started c001n02
+ rsc_c001n03 (ocf::heartbeat:IPaddr): Started c001n03
+ rsc_c001n01 (ocf::heartbeat:IPaddr): Started c001n01
+
+Transition Summary:
+ * Restart DcIPaddr ( c001n02 )
+
+Executing cluster transition:
+ * Resource action: DcIPaddr monitor on c001n08
+ * Resource action: DcIPaddr monitor on c001n03
+ * Resource action: DcIPaddr monitor on c001n01
+ * Resource action: DcIPaddr stop on c001n02
+ * Resource action: rsc_c001n08 monitor on c001n03
+ * Resource action: rsc_c001n08 monitor on c001n02
+ * Resource action: rsc_c001n08 monitor on c001n01
+ * Resource action: rsc_c001n08 monitor=5000 on c001n08
+ * Resource action: rsc_c001n02 monitor=6000 on c001n02
+ * Resource action: rsc_c001n02 monitor on c001n08
+ * Resource action: rsc_c001n02 monitor on c001n03
+ * Resource action: rsc_c001n02 monitor on c001n01
+ * Resource action: rsc_c001n02 cancel=5000 on c001n02
+ * Resource action: rsc_c001n03 monitor on c001n08
+ * Resource action: rsc_c001n03 monitor on c001n02
+ * Resource action: rsc_c001n03 monitor on c001n01
+ * Resource action: rsc_c001n01 monitor on c001n08
+ * Resource action: rsc_c001n01 monitor on c001n03
+ * Resource action: rsc_c001n01 monitor on c001n02
+ * Resource action: DcIPaddr start on c001n02
+ * Resource action: DcIPaddr monitor=5000 on c001n02
+
+Revised cluster status:
+Online: [ c001n01 c001n02 c001n03 c001n08 ]
+
+ DcIPaddr (ocf::heartbeat:IPaddr): Started c001n02
+ rsc_c001n08 (ocf::heartbeat:IPaddr): Started c001n08
+ rsc_c001n02 (ocf::heartbeat:IPaddr): Started c001n02
+ rsc_c001n03 (ocf::heartbeat:IPaddr): Started c001n03
+ rsc_c001n01 (ocf::heartbeat:IPaddr): Started c001n01
+
diff --git a/cts/scheduler/params-3.xml b/cts/scheduler/params-3.xml
new file mode 100644
index 0000000000..ee6e157fb2
--- /dev/null
+++ b/cts/scheduler/params-3.xml
@@ -0,0 +1,154 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/daemons/controld/controld_control.c b/daemons/controld/controld_control.c
index 8ed4b70754..5db7f5bca5 100644
--- a/daemons/controld/controld_control.c
+++ b/daemons/controld/controld_control.c
@@ -1,818 +1,818 @@
/*
* Copyright 2004-2020 the Pacemaker project contributors
*
* The version control history for this file may have further details.
*
* This source code is licensed under the GNU General Public License version 2
* or later (GPLv2+) WITHOUT ANY WARRANTY.
*/
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
qb_ipcs_service_t *ipcs = NULL;
#if SUPPORT_COROSYNC
extern gboolean crm_connect_corosync(crm_cluster_t * cluster);
#endif
void crm_shutdown(int nsig);
gboolean crm_read_options(gpointer user_data);
gboolean fsa_has_quorum = FALSE;
crm_trigger_t *fsa_source = NULL;
crm_trigger_t *config_read = NULL;
bool no_quorum_suicide_escalation = FALSE;
bool controld_shutdown_lock_enabled = false;
/* A_HA_CONNECT */
void
do_ha_control(long long action,
enum crmd_fsa_cause cause,
enum crmd_fsa_state cur_state,
enum crmd_fsa_input current_input, fsa_data_t * msg_data)
{
gboolean registered = FALSE;
static crm_cluster_t *cluster = NULL;
if (cluster == NULL) {
cluster = calloc(1, sizeof(crm_cluster_t));
}
if (action & A_HA_DISCONNECT) {
crm_cluster_disconnect(cluster);
crm_info("Disconnected from the cluster");
set_bit(fsa_input_register, R_HA_DISCONNECTED);
}
if (action & A_HA_CONNECT) {
crm_set_status_callback(&peer_update_callback);
crm_set_autoreap(FALSE);
if (is_corosync_cluster()) {
#if SUPPORT_COROSYNC
registered = crm_connect_corosync(cluster);
#endif
}
if (registered == TRUE) {
controld_election_init(cluster->uname);
fsa_our_uname = cluster->uname;
fsa_our_uuid = cluster->uuid;
if(cluster->uuid == NULL) {
crm_err("Could not obtain local uuid");
registered = FALSE;
}
}
if (registered == FALSE) {
set_bit(fsa_input_register, R_HA_DISCONNECTED);
register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
return;
}
populate_cib_nodes(node_update_none, __FUNCTION__);
clear_bit(fsa_input_register, R_HA_DISCONNECTED);
crm_info("Connected to the cluster");
}
if (action & ~(A_HA_CONNECT | A_HA_DISCONNECT)) {
crm_err("Unexpected action %s in %s", fsa_action2string(action), __FUNCTION__);
}
}
/* A_SHUTDOWN */
void
do_shutdown(long long action,
enum crmd_fsa_cause cause,
enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data)
{
/* just in case */
set_bit(fsa_input_register, R_SHUTDOWN);
controld_disconnect_fencer(FALSE);
}
/* A_SHUTDOWN_REQ */
void
do_shutdown_req(long long action,
enum crmd_fsa_cause cause,
enum crmd_fsa_state cur_state,
enum crmd_fsa_input current_input, fsa_data_t * msg_data)
{
xmlNode *msg = NULL;
set_bit(fsa_input_register, R_SHUTDOWN);
crm_info("Sending shutdown request to all peers (DC is %s)",
(fsa_our_dc? fsa_our_dc : "not set"));
msg = create_request(CRM_OP_SHUTDOWN_REQ, NULL, NULL, CRM_SYSTEM_CRMD, CRM_SYSTEM_CRMD, NULL);
/* set_bit(fsa_input_register, R_STAYDOWN); */
if (send_cluster_message(NULL, crm_msg_crmd, msg, TRUE) == FALSE) {
register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
}
free_xml(msg);
}
extern char *max_generation_from;
extern xmlNode *max_generation_xml;
extern GHashTable *resource_history;
extern GHashTable *voted;
void
crmd_fast_exit(crm_exit_t exit_code)
{
if (is_set(fsa_input_register, R_STAYDOWN)) {
crm_warn("Inhibiting respawn "CRM_XS" remapping exit code %d to %d",
exit_code, CRM_EX_FATAL);
exit_code = CRM_EX_FATAL;
} else if ((exit_code == CRM_EX_OK)
&& is_set(fsa_input_register, R_IN_RECOVERY)) {
crm_err("Could not recover from internal error");
exit_code = CRM_EX_ERROR;
}
crm_exit(exit_code);
}
crm_exit_t
crmd_exit(crm_exit_t exit_code)
{
GListPtr gIter = NULL;
GMainLoop *mloop = crmd_mainloop;
static bool in_progress = FALSE;
if (in_progress && (exit_code == CRM_EX_OK)) {
crm_debug("Exit is already in progress");
return exit_code;
} else if(in_progress) {
crm_notice("Error during shutdown process, exiting now with status %d (%s)",
exit_code, crm_exit_str(exit_code));
crm_write_blackbox(SIGTRAP, NULL);
crmd_fast_exit(exit_code);
}
in_progress = TRUE;
crm_trace("Preparing to exit with status %d (%s)",
exit_code, crm_exit_str(exit_code));
/* Suppress secondary errors resulting from us disconnecting everything */
set_bit(fsa_input_register, R_HA_DISCONNECTED);
/* Close all IPC servers and clients to ensure any and all shared memory files are cleaned up */
if(ipcs) {
crm_trace("Closing IPC server");
mainloop_del_ipc_server(ipcs);
ipcs = NULL;
}
controld_close_attrd_ipc();
pe_subsystem_free();
controld_disconnect_fencer(TRUE);
if ((exit_code == CRM_EX_OK) && (crmd_mainloop == NULL)) {
crm_debug("No mainloop detected");
exit_code = CRM_EX_ERROR;
}
/* On an error, just get out.
*
* Otherwise, make the effort to have mainloop exit gracefully so
* that it (mostly) cleans up after itself and valgrind has less
* to report on - allowing real errors stand out
*/
if (exit_code != CRM_EX_OK) {
crm_notice("Forcing immediate exit with status %d (%s)",
exit_code, crm_exit_str(exit_code));
crm_write_blackbox(SIGTRAP, NULL);
crmd_fast_exit(exit_code);
}
/* Clean up as much memory as possible for valgrind */
for (gIter = fsa_message_queue; gIter != NULL; gIter = gIter->next) {
fsa_data_t *fsa_data = gIter->data;
crm_info("Dropping %s: [ state=%s cause=%s origin=%s ]",
fsa_input2string(fsa_data->fsa_input),
fsa_state2string(fsa_state),
fsa_cause2string(fsa_data->fsa_cause), fsa_data->origin);
delete_fsa_input(fsa_data);
}
clear_bit(fsa_input_register, R_MEMBERSHIP);
g_list_free(fsa_message_queue); fsa_message_queue = NULL;
metadata_cache_fini();
controld_election_fini();
/* Tear down the CIB manager connection, but don't free it yet -- it could
* be used when we drain the mainloop later.
*/
cib_free_callbacks(fsa_cib_conn);
fsa_cib_conn->cmds->signoff(fsa_cib_conn);
verify_stopped(fsa_state, LOG_WARNING);
clear_bit(fsa_input_register, R_LRM_CONNECTED);
lrm_state_destroy_all();
/* This basically will not work, since mainloop has a reference to it */
mainloop_destroy_trigger(fsa_source); fsa_source = NULL;
mainloop_destroy_trigger(config_read); config_read = NULL;
mainloop_destroy_trigger(transition_trigger); transition_trigger = NULL;
pcmk__client_cleanup();
crm_peer_destroy();
controld_free_fsa_timers();
te_cleanup_stonith_history_sync(NULL, TRUE);
controld_free_sched_timer();
free(fsa_our_dc_version); fsa_our_dc_version = NULL;
free(fsa_our_uname); fsa_our_uname = NULL;
free(fsa_our_uuid); fsa_our_uuid = NULL;
free(fsa_our_dc); fsa_our_dc = NULL;
free(fsa_cluster_name); fsa_cluster_name = NULL;
free(te_uuid); te_uuid = NULL;
free(failed_stop_offset); failed_stop_offset = NULL;
free(failed_start_offset); failed_start_offset = NULL;
free(max_generation_from); max_generation_from = NULL;
free_xml(max_generation_xml); max_generation_xml = NULL;
mainloop_destroy_signal(SIGPIPE);
mainloop_destroy_signal(SIGUSR1);
mainloop_destroy_signal(SIGTERM);
mainloop_destroy_signal(SIGTRAP);
/* leave SIGCHLD engaged as we might still want to drain some service-actions */
if (mloop) {
GMainContext *ctx = g_main_loop_get_context(crmd_mainloop);
/* Don't re-enter this block */
crmd_mainloop = NULL;
/* no signals on final draining anymore */
mainloop_destroy_signal(SIGCHLD);
crm_trace("Draining mainloop %d %d", g_main_loop_is_running(mloop), g_main_context_pending(ctx));
{
int lpc = 0;
while((g_main_context_pending(ctx) && lpc < 10)) {
lpc++;
crm_trace("Iteration %d", lpc);
g_main_context_dispatch(ctx);
}
}
crm_trace("Closing mainloop %d %d", g_main_loop_is_running(mloop), g_main_context_pending(ctx));
g_main_loop_quit(mloop);
/* Won't do anything yet, since we're inside it now */
g_main_loop_unref(mloop);
} else {
mainloop_destroy_signal(SIGCHLD);
}
cib_delete(fsa_cib_conn);
fsa_cib_conn = NULL;
throttle_fini();
/* Graceful */
crm_trace("Done preparing for exit with status %d (%s)",
exit_code, crm_exit_str(exit_code));
return exit_code;
}
/* A_EXIT_0, A_EXIT_1 */
void
do_exit(long long action,
enum crmd_fsa_cause cause,
enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data)
{
crm_exit_t exit_code = CRM_EX_OK;
int log_level = LOG_INFO;
const char *exit_type = "gracefully";
if (action & A_EXIT_1) {
log_level = LOG_ERR;
exit_type = "forcefully";
exit_code = CRM_EX_ERROR;
}
verify_stopped(cur_state, LOG_ERR);
do_crm_log(log_level, "Performing %s - %s exiting the controller",
fsa_action2string(action), exit_type);
crm_info("[%s] stopped (%d)", crm_system_name, exit_code);
crmd_exit(exit_code);
}
static void sigpipe_ignore(int nsig) { return; }
/* A_STARTUP */
void
do_startup(long long action,
enum crmd_fsa_cause cause,
enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data)
{
crm_debug("Registering Signal Handlers");
mainloop_add_signal(SIGTERM, crm_shutdown);
mainloop_add_signal(SIGPIPE, sigpipe_ignore);
fsa_source = mainloop_add_trigger(G_PRIORITY_HIGH, crm_fsa_trigger, NULL);
config_read = mainloop_add_trigger(G_PRIORITY_HIGH, crm_read_options, NULL);
transition_trigger = mainloop_add_trigger(G_PRIORITY_LOW, te_graph_trigger, NULL);
crm_debug("Creating CIB manager and executor objects");
fsa_cib_conn = cib_new();
lrm_state_init_local();
if (controld_init_fsa_timers() == FALSE) {
register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
}
}
// \return libqb error code (0 on success, -errno on error)
static int32_t
accept_controller_client(qb_ipcs_connection_t *c, uid_t uid, gid_t gid)
{
crm_trace("Accepting new IPC client connection");
if (pcmk__new_client(c, uid, gid) == NULL) {
return -EIO;
}
return 0;
}
// \return libqb error code (0 on success, -errno on error)
static int32_t
dispatch_controller_ipc(qb_ipcs_connection_t * c, void *data, size_t size)
{
uint32_t id = 0;
uint32_t flags = 0;
pcmk__client_t *client = pcmk__find_client(c);
xmlNode *msg = pcmk__client_data2xml(client, data, &id, &flags);
pcmk__ipc_send_ack(client, id, flags, "ack");
if (msg == NULL) {
return 0;
}
#if ENABLE_ACL
CRM_ASSERT(client->user != NULL);
pcmk__update_acl_user(msg, F_CRM_USER, client->user);
#endif
crm_xml_add(msg, F_CRM_SYS_FROM, client->id);
if (controld_authorize_ipc_message(msg, client, NULL)) {
crm_trace("Processing IPC message from %s", pcmk__client_name(client));
route_message(C_IPC_MESSAGE, msg);
}
trigger_fsa(fsa_source);
free_xml(msg);
return 0;
}
static int32_t
crmd_ipc_closed(qb_ipcs_connection_t * c)
{
pcmk__client_t *client = pcmk__find_client(c);
if (client) {
crm_trace("Disconnecting %sregistered client %s (%p/%p)",
(client->userdata? "" : "un"), pcmk__client_name(client),
c, client);
free(client->userdata);
pcmk__free_client(client);
trigger_fsa(fsa_source);
}
return 0;
}
static void
crmd_ipc_destroy(qb_ipcs_connection_t * c)
{
crm_trace("Connection %p", c);
crmd_ipc_closed(c);
}
/* A_STOP */
void
do_stop(long long action,
enum crmd_fsa_cause cause,
enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data)
{
crm_trace("Closing IPC server");
mainloop_del_ipc_server(ipcs); ipcs = NULL;
register_fsa_input(C_FSA_INTERNAL, I_TERMINATE, NULL);
}
/* A_STARTED */
void
do_started(long long action,
enum crmd_fsa_cause cause,
enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data)
{
static struct qb_ipcs_service_handlers crmd_callbacks = {
.connection_accept = accept_controller_client,
.connection_created = NULL,
.msg_process = dispatch_controller_ipc,
.connection_closed = crmd_ipc_closed,
.connection_destroyed = crmd_ipc_destroy
};
if (cur_state != S_STARTING) {
crm_err("Start cancelled... %s", fsa_state2string(cur_state));
return;
} else if (is_set(fsa_input_register, R_MEMBERSHIP) == FALSE) {
crm_info("Delaying start, no membership data (%.16llx)", R_MEMBERSHIP);
crmd_fsa_stall(TRUE);
return;
} else if (is_set(fsa_input_register, R_LRM_CONNECTED) == FALSE) {
crm_info("Delaying start, not connected to executor (%.16llx)", R_LRM_CONNECTED);
crmd_fsa_stall(TRUE);
return;
} else if (is_set(fsa_input_register, R_CIB_CONNECTED) == FALSE) {
crm_info("Delaying start, CIB not connected (%.16llx)", R_CIB_CONNECTED);
crmd_fsa_stall(TRUE);
return;
} else if (is_set(fsa_input_register, R_READ_CONFIG) == FALSE) {
crm_info("Delaying start, Config not read (%.16llx)", R_READ_CONFIG);
crmd_fsa_stall(TRUE);
return;
} else if (is_set(fsa_input_register, R_PEER_DATA) == FALSE) {
crm_info("Delaying start, No peer data (%.16llx)", R_PEER_DATA);
crmd_fsa_stall(TRUE);
return;
}
crm_debug("Init server comms");
ipcs = pcmk__serve_controld_ipc(&crmd_callbacks);
if (ipcs == NULL) {
crm_err("Failed to create IPC server: shutting down and inhibiting respawn");
register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
} else {
crm_notice("Pacemaker controller successfully started and accepting connections");
}
controld_trigger_fencer_connect();
clear_bit(fsa_input_register, R_STARTING);
register_fsa_input(msg_data->fsa_cause, I_PENDING, NULL);
}
/* A_RECOVER */
void
do_recover(long long action,
enum crmd_fsa_cause cause,
enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data)
{
set_bit(fsa_input_register, R_IN_RECOVERY);
crm_warn("Fast-tracking shutdown in response to errors");
register_fsa_input(C_FSA_INTERNAL, I_TERMINATE, NULL);
}
static pcmk__cluster_option_t crmd_opts[] = {
/* name, old name, type, allowed values,
* default value, validator,
* short description,
* long description
*/
{
"dc-version", NULL, "string", NULL, "none", NULL,
"Pacemaker version on cluster node elected Designated Controller (DC)",
"Includes a hash which identifies the exact changeset the code was "
"built from. Used for diagnostic purposes."
},
{
"cluster-infrastructure", NULL, "string", NULL, "corosync", NULL,
"The messaging stack on which Pacemaker is currently running",
"Used for informational and diagnostic purposes."
},
{
"cluster-name", NULL, "string", NULL, NULL, NULL,
"An arbitrary name for the cluster",
"This optional value is mostly for users' convenience as desired "
"in administration, but may also be used in Pacemaker "
"configuration rules via the #cluster-name node attribute, and "
"by higher-level tools and resource agents."
},
{
XML_CONFIG_ATTR_DC_DEADTIME, NULL, "time",
NULL, "20s", pcmk__valid_interval_spec,
"How long to wait for a response from other nodes during start-up",
"The optimal value will depend on the speed and load of your network "
"and the type of switches used."
},
{
XML_CONFIG_ATTR_RECHECK, NULL, "time",
"Zero disables polling, while positive values are an interval in seconds"
"(unless other units are specified, for example \"5min\")",
"15min", pcmk__valid_interval_spec,
"Polling interval to recheck cluster state and evaluate rules "
"with date specifications",
"Pacemaker is primarily event-driven, and looks ahead to know when to "
"recheck cluster state for failure timeouts and most time-based "
"rules. However, it will also recheck the cluster after this "
"amount of inactivity, to evaluate rules with date specifications "
"and serve as a fail-safe for certain types of scheduler bugs."
},
{
"load-threshold", NULL, "percentage", NULL,
"80%", pcmk__valid_utilization,
"Maximum amount of system load that should be used by cluster nodes",
"The cluster will slow down its recovery process when the amount of "
"system resources used (currently CPU) approaches this limit",
},
{
"node-action-limit", NULL, "integer", NULL,
"0", pcmk__valid_number,
"Maximum number of jobs that can be scheduled per node "
"(defaults to 2x cores)"
},
{ XML_CONFIG_ATTR_FENCE_REACTION, NULL, "string", NULL, "stop", NULL,
"How a cluster node should react if notified of its own fencing",
"A cluster node may receive notification of its own fencing if fencing "
"is misconfigured, or if fabric fencing is in use that doesn't cut "
"cluster communication. Allowed values are \"stop\" to attempt to "
"immediately stop pacemaker and stay stopped, or \"panic\" to attempt "
"to immediately reboot the local node, falling back to stop on failure."
},
{
XML_CONFIG_ATTR_ELECTION_FAIL, NULL, "time", NULL,
"2min", pcmk__valid_interval_spec,
"*** Advanced Use Only ***",
"Declare an election failed if it is not decided within this much "
"time. If you need to adjust this value, it probably indicates "
"the presence of a bug."
},
{
XML_CONFIG_ATTR_FORCE_QUIT, NULL, "time", NULL,
"20min", pcmk__valid_interval_spec,
"*** Advanced Use Only ***",
"Exit immediately if shutdown does not complete within this much "
"time. If you need to adjust this value, it probably indicates "
"the presence of a bug."
},
{
"join-integration-timeout", "crmd-integration-timeout", "time", NULL,
"3min", pcmk__valid_interval_spec,
"*** Advanced Use Only ***",
"If you need to adjust this value, it probably indicates "
"the presence of a bug."
},
{
"join-finalization-timeout", "crmd-finalization-timeout", "time", NULL,
"30min", pcmk__valid_interval_spec,
"*** Advanced Use Only ***",
"If you need to adjust this value, it probably indicates "
"the presence of a bug."
},
{
"transition-delay", "crmd-transition-delay", "time", NULL,
"0s", pcmk__valid_interval_spec,
"*** Advanced Use Only *** Enabling this option will slow down "
"cluster recovery under all conditions",
"Delay cluster recovery for this much time to allow for additional "
"events to occur. Useful if your configuration is sensitive to "
"the order in which ping updates arrive."
},
{
"stonith-watchdog-timeout", NULL, "time", NULL,
NULL, pcmk__valid_sbd_timeout,
"How long to wait before we can assume nodes are safely down "
"when sbd is in use",
NULL
},
{
"stonith-max-attempts", NULL, "integer", NULL,
"10", pcmk__valid_positive_number,
"How many times fencing can fail before it will no longer be "
"immediately re-attempted on a target"
},
// Already documented in libpe_status (other values must be kept identical)
{
- "no-quorum-policy", NULL, "enum", "stop, freeze, ignore, suicide",
+ "no-quorum-policy", NULL, "enum", "stop, freeze, ignore, demote, suicide",
"stop", pcmk__valid_quorum, NULL, NULL
},
{
XML_CONFIG_ATTR_SHUTDOWN_LOCK, NULL, "boolean", NULL,
"false", pcmk__valid_boolean, NULL, NULL
},
};
void
crmd_metadata(void)
{
pcmk__print_option_metadata("pacemaker-controld", "1.0",
"Pacemaker controller options",
"Cluster options used by Pacemaker's "
"controller (formerly called crmd)",
crmd_opts, DIMOF(crmd_opts));
}
static void
verify_crmd_options(GHashTable * options)
{
pcmk__validate_cluster_options(options, crmd_opts, DIMOF(crmd_opts));
}
static const char *
crmd_pref(GHashTable * options, const char *name)
{
return pcmk__cluster_option(options, crmd_opts, DIMOF(crmd_opts), name);
}
static void
config_query_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
{
const char *value = NULL;
GHashTable *config_hash = NULL;
crm_time_t *now = crm_time_new(NULL);
xmlNode *crmconfig = NULL;
xmlNode *alerts = NULL;
if (rc != pcmk_ok) {
fsa_data_t *msg_data = NULL;
crm_err("Local CIB query resulted in an error: %s", pcmk_strerror(rc));
register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
if (rc == -EACCES || rc == -pcmk_err_schema_validation) {
crm_err("The cluster is mis-configured - shutting down and staying down");
set_bit(fsa_input_register, R_STAYDOWN);
}
goto bail;
}
crmconfig = output;
if ((crmconfig) &&
(crm_element_name(crmconfig)) &&
(strcmp(crm_element_name(crmconfig), XML_CIB_TAG_CRMCONFIG) != 0)) {
crmconfig = first_named_child(crmconfig, XML_CIB_TAG_CRMCONFIG);
}
if (!crmconfig) {
fsa_data_t *msg_data = NULL;
crm_err("Local CIB query for " XML_CIB_TAG_CRMCONFIG " section failed");
register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
goto bail;
}
crm_debug("Call %d : Parsing CIB options", call_id);
config_hash = crm_str_table_new();
pe_unpack_nvpairs(crmconfig, crmconfig, XML_CIB_TAG_PROPSET, NULL,
config_hash, CIB_OPTIONS_FIRST, FALSE, now, NULL);
verify_crmd_options(config_hash);
value = crmd_pref(config_hash, XML_CONFIG_ATTR_DC_DEADTIME);
election_trigger->period_ms = crm_parse_interval_spec(value);
value = crmd_pref(config_hash, "node-action-limit"); /* Also checks migration-limit */
throttle_update_job_max(value);
value = crmd_pref(config_hash, "load-threshold");
if(value) {
throttle_set_load_target(strtof(value, NULL) / 100.0);
}
value = crmd_pref(config_hash, "no-quorum-policy");
if (safe_str_eq(value, "suicide") && pcmk_locate_sbd()) {
no_quorum_suicide_escalation = TRUE;
}
set_fence_reaction(crmd_pref(config_hash, XML_CONFIG_ATTR_FENCE_REACTION));
value = crmd_pref(config_hash,"stonith-max-attempts");
update_stonith_max_attempts(value);
value = crmd_pref(config_hash, XML_CONFIG_ATTR_FORCE_QUIT);
shutdown_escalation_timer->period_ms = crm_parse_interval_spec(value);
crm_debug("Shutdown escalation occurs if DC has not responded to request in %ums",
shutdown_escalation_timer->period_ms);
value = crmd_pref(config_hash, XML_CONFIG_ATTR_ELECTION_FAIL);
controld_set_election_period(value);
value = crmd_pref(config_hash, XML_CONFIG_ATTR_RECHECK);
recheck_interval_ms = crm_parse_interval_spec(value);
crm_debug("Re-run scheduler after %dms of inactivity", recheck_interval_ms);
value = crmd_pref(config_hash, "transition-delay");
transition_timer->period_ms = crm_parse_interval_spec(value);
value = crmd_pref(config_hash, "join-integration-timeout");
integration_timer->period_ms = crm_parse_interval_spec(value);
value = crmd_pref(config_hash, "join-finalization-timeout");
finalization_timer->period_ms = crm_parse_interval_spec(value);
value = crmd_pref(config_hash, XML_CONFIG_ATTR_SHUTDOWN_LOCK);
controld_shutdown_lock_enabled = crm_is_true(value);
free(fsa_cluster_name);
fsa_cluster_name = NULL;
value = g_hash_table_lookup(config_hash, "cluster-name");
if (value) {
fsa_cluster_name = strdup(value);
}
alerts = first_named_child(output, XML_CIB_TAG_ALERTS);
crmd_unpack_alerts(alerts);
set_bit(fsa_input_register, R_READ_CONFIG);
crm_trace("Triggering FSA: %s", __FUNCTION__);
mainloop_set_trigger(fsa_source);
g_hash_table_destroy(config_hash);
bail:
crm_time_free(now);
}
gboolean
crm_read_options(gpointer user_data)
{
int call_id =
fsa_cib_conn->cmds->query(fsa_cib_conn,
"//" XML_CIB_TAG_CRMCONFIG " | //" XML_CIB_TAG_ALERTS,
NULL, cib_xpath | cib_scope_local);
fsa_register_cib_callback(call_id, FALSE, NULL, config_query_callback);
crm_trace("Querying the CIB... call %d", call_id);
return TRUE;
}
/* A_READCONFIG */
void
do_read_config(long long action,
enum crmd_fsa_cause cause,
enum crmd_fsa_state cur_state,
enum crmd_fsa_input current_input, fsa_data_t * msg_data)
{
throttle_init();
mainloop_set_trigger(config_read);
}
void
crm_shutdown(int nsig)
{
if ((crmd_mainloop == NULL) || !g_main_loop_is_running(crmd_mainloop)) {
crmd_exit(CRM_EX_OK);
return;
}
if (is_set(fsa_input_register, R_SHUTDOWN)) {
crm_err("Escalating shutdown");
register_fsa_input_before(C_SHUTDOWN, I_ERROR, NULL);
return;
}
set_bit(fsa_input_register, R_SHUTDOWN);
register_fsa_input(C_SHUTDOWN, I_SHUTDOWN, NULL);
if (shutdown_escalation_timer->period_ms == 0) {
const char *value = crmd_pref(NULL, XML_CONFIG_ATTR_FORCE_QUIT);
shutdown_escalation_timer->period_ms = crm_parse_interval_spec(value);
}
crm_notice("Initiating controller shutdown sequence " CRM_XS
" limit=%ums", shutdown_escalation_timer->period_ms);
controld_start_timer(shutdown_escalation_timer);
}
diff --git a/doc/Pacemaker_Explained/en-US/Ch-Options.txt b/doc/Pacemaker_Explained/en-US/Ch-Options.txt
index faefe7c888..b158f00765 100644
--- a/doc/Pacemaker_Explained/en-US/Ch-Options.txt
+++ b/doc/Pacemaker_Explained/en-US/Ch-Options.txt
@@ -1,482 +1,484 @@
:compat-mode: legacy
= Cluster-Wide Configuration =
== Configuration Layout ==
The cluster is defined by the Cluster Information Base (CIB),
which uses XML notation. The simplest CIB, an empty one, looks like this:
.An empty configuration
======
[source,XML]
-------
-------
======
The empty configuration above contains the major sections that make up a CIB:
* +cib+: The entire CIB is enclosed with a +cib+ tag. Certain fundamental settings
are defined as attributes of this tag.
** +configuration+: This section -- the primary focus of this document --
contains traditional configuration information such as what resources the
cluster serves and the relationships among them.
*** +crm_config+: cluster-wide configuration options
*** +nodes+: the machines that host the cluster
*** +resources+: the services run by the cluster
*** +constraints+: indications of how resources should be placed
** +status+: This section contains the history of each resource on each node.
Based on this data, the cluster can construct the complete current
state of the cluster. The authoritative source for this section
is the local executor (pacemaker-execd process) on each cluster node, and
the cluster will occasionally repopulate the entire section. For this
reason, it is never written to disk, and administrators are advised
against modifying it in any way.
In this document, configuration settings will be described as 'properties' or 'options'
based on how they are defined in the CIB:
* Properties are XML attributes of an XML element.
* Options are name-value pairs expressed as +nvpair+ child elements of an XML element.
Normally, you will use command-line tools that abstract the XML, so the
distinction will be unimportant; both properties and options are
cluster settings you can tweak.
== CIB Properties ==
Certain settings are defined by CIB properties (that is, attributes of the
+cib+ tag) rather than with the rest of the cluster configuration in the
+configuration+ section.
The reason is simply a matter of parsing. These options are used by the
configuration database which is, by design, mostly ignorant of the content it
holds. So the decision was made to place them in an easy-to-find location.
.CIB Properties
[width="95%",cols="2m,<5",options="header",align="center"]
|=========================================================
|Field |Description
| admin_epoch |
indexterm:[Configuration Version,Cluster]
indexterm:[Cluster,Option,Configuration Version]
indexterm:[admin_epoch,Cluster Option]
indexterm:[Cluster,Option,admin_epoch]
When a node joins the cluster, the cluster performs a check to see
which node has the best configuration. It asks the node with the highest
(+admin_epoch+, +epoch+, +num_updates+) tuple to replace the configuration on
all the nodes -- which makes setting them, and setting them correctly, very
important. +admin_epoch+ is never modified by the cluster; you can use this
to make the configurations on any inactive nodes obsolete. _Never set this
value to zero_. In such cases, the cluster cannot tell the difference between
your configuration and the "empty" one used when nothing is found on disk.
| epoch |
indexterm:[epoch,Cluster Option]
indexterm:[Cluster,Option,epoch]
The cluster increments this every time the configuration is updated (usually by
the administrator).
| num_updates |
indexterm:[num_updates,Cluster Option]
indexterm:[Cluster,Option,num_updates]
The cluster increments this every time the configuration or status is updated
(usually by the cluster) and resets it to 0 when epoch changes.
| validate-with |
indexterm:[validate-with,Cluster Option]
indexterm:[Cluster,Option,validate-with]
Determines the type of XML validation that will be done on the configuration.
If set to +none+, the cluster will not verify that updates conform to the
DTD (nor reject ones that don't). This option can be useful when
operating a mixed-version cluster during an upgrade.
|cib-last-written |
indexterm:[cib-last-written,Cluster Property]
indexterm:[Cluster,Property,cib-last-written]
Indicates when the configuration was last written to disk. Maintained by the
cluster; for informational purposes only.
|have-quorum |
indexterm:[have-quorum,Cluster Property]
indexterm:[Cluster,Property,have-quorum]
Indicates if the cluster has quorum. If false, this may mean that the
cluster cannot start resources or fence other nodes (see
+no-quorum-policy+ below). Maintained by the cluster.
|dc-uuid |
indexterm:[dc-uuid,Cluster Property]
indexterm:[Cluster,Property,dc-uuid]
Indicates which cluster node is the current leader. Used by the
cluster when placing resources and determining the order of some
events. Maintained by the cluster.
|=========================================================
[[s-cluster-options]]
== Cluster Options ==
Cluster options, as you might expect, control how the cluster behaves
when confronted with certain situations.
They are grouped into sets within the +crm_config+ section, and, in advanced
configurations, there may be more than one set. (This will be described later
in the section on <> where we will show how to have the cluster use
different sets of options during working hours than during weekends.) For now,
we will describe the simple case where each option is present at most once.
You can obtain an up-to-date list of cluster options, including
their default values, by running the `man pacemaker-schedulerd` and
`man pacemaker-controld` commands.
.Cluster Options
[width="95%",cols="5m,2,<11",options="header",align="center"]
|=========================================================
|Option |Default |Description
| cluster-name | |
indexterm:[cluster-name,Cluster Property]
indexterm:[Cluster,Property,cluster-name]
An (optional) name for the cluster as a whole. This is mostly for users'
convenience for use as desired in administration, but this can be used
in the Pacemaker configuration in <> (as the
+#cluster-name+ <>). It may
also be used by higher-level tools when displaying cluster information, and by
certain resource agents (for example, the +ocf:heartbeat:GFS2+ agent stores the
cluster name in filesystem meta-data).
| dc-version | |
indexterm:[dc-version,Cluster Property]
indexterm:[Cluster,Property,dc-version]
Version of Pacemaker on the cluster's DC.
Determined automatically by the cluster.
Often includes the hash which identifies the exact Git changeset it was built
from. Used for diagnostic purposes.
| cluster-infrastructure | |
indexterm:[cluster-infrastructure,Cluster Property]
indexterm:[Cluster,Property,cluster-infrastructure]
The messaging stack on which Pacemaker is currently running.
Determined automatically by the cluster.
Used for informational and diagnostic purposes.
| no-quorum-policy | stop
a|
indexterm:[no-quorum-policy,Cluster Option]
indexterm:[Cluster,Option,no-quorum-policy]
What to do when the cluster does not have quorum. Allowed values:
* +ignore:+ continue all resource management
* +freeze:+ continue resource management, but don't recover resources from nodes not in the affected partition
* +stop:+ stop all resources in the affected cluster partition
+* +demote:+ demote promotable resources and stop all other resources in the
+ affected cluster partition
* +suicide:+ fence all nodes in the affected cluster partition
| batch-limit | 0 |
indexterm:[batch-limit,Cluster Option]
indexterm:[Cluster,Option,batch-limit]
The maximum number of actions that the cluster may execute in parallel across
all nodes. The "correct" value will depend on the speed and load of your
network and cluster nodes. If zero, the cluster will impose a dynamically
calculated limit only when any node has high load.
| migration-limit | -1 |
indexterm:[migration-limit,Cluster Option]
indexterm:[Cluster,Option,migration-limit]
The number of <> actions that the cluster
is allowed to execute in parallel on a node. A value of -1 means unlimited.
| symmetric-cluster | TRUE |
indexterm:[symmetric-cluster,Cluster Option]
indexterm:[Cluster,Option,symmetric-cluster]
Can all resources run on any node by default?
| stop-all-resources | FALSE |
indexterm:[stop-all-resources,Cluster Option]
indexterm:[Cluster,Option,stop-all-resources]
Should the cluster stop all resources?
| stop-orphan-resources | TRUE |
indexterm:[stop-orphan-resources,Cluster Option]
indexterm:[Cluster,Option,stop-orphan-resources]
Should deleted resources be stopped? This value takes precedence over
+is-managed+ (i.e. even unmanaged resources will be stopped if deleted from
the configuration when this value is TRUE).
| stop-orphan-actions | TRUE |
indexterm:[stop-orphan-actions,Cluster Option]
indexterm:[Cluster,Option,stop-orphan-actions]
Should deleted actions be cancelled?
| start-failure-is-fatal | TRUE |
indexterm:[start-failure-is-fatal,Cluster Option]
indexterm:[Cluster,Option,start-failure-is-fatal]
Should a failure to start a resource on a particular node prevent further start
attempts on that node? If FALSE, the cluster will decide whether the same
node is still eligible based on the resource's current failure count
and +migration-threshold+ (see <>).
| enable-startup-probes | TRUE |
indexterm:[enable-startup-probes,Cluster Option]
indexterm:[Cluster,Option,enable-startup-probes]
Should the cluster check for active resources during startup?
| maintenance-mode | FALSE |
indexterm:[maintenance-mode,Cluster Option]
indexterm:[Cluster,Option,maintenance-mode]
Should the cluster refrain from monitoring, starting and stopping resources?
| stonith-enabled | TRUE |
indexterm:[stonith-enabled,Cluster Option]
indexterm:[Cluster,Option,stonith-enabled]
Should failed nodes and nodes with resources that can't be stopped be
shot? If you value your data, set up a STONITH device and enable this.
If true, or unset, the cluster will refuse to start resources unless
one or more STONITH resources have been configured.
If false, unresponsive nodes are immediately assumed to be running no
resources, and resource takeover to online nodes starts without any
further protection (which means _data loss_ if the unresponsive node
still accesses shared storage, for example). See also the +requires+
meta-attribute in <>.
| stonith-action | reboot |
indexterm:[stonith-action,Cluster Option]
indexterm:[Cluster,Option,stonith-action]
Action to send to STONITH device. Allowed values are +reboot+ and +off+.
The value +poweroff+ is also allowed, but is only used for
legacy devices.
| stonith-timeout | 60s |
indexterm:[stonith-timeout,Cluster Option]
indexterm:[Cluster,Option,stonith-timeout]
How long to wait for STONITH actions (reboot, on, off) to complete
| stonith-max-attempts | 10 |
indexterm:[stonith-max-attempts,Cluster Option]
indexterm:[Cluster,Option,stonith-max-attempts]
How many times fencing can fail for a target before the cluster will no longer
immediately re-attempt it.
| stonith-watchdog-timeout | 0 |
indexterm:[stonith-watchdog-timeout,Cluster Option]
indexterm:[Cluster,Option,stonith-watchdog-timeout]
If nonzero, rely on hardware watchdog self-fencing. If positive, assume unseen
nodes self-fence within this much time. If negative, and the
SBD_WATCHDOG_TIMEOUT environment variable is set, use twice that value.
| concurrent-fencing | FALSE |
indexterm:[concurrent-fencing,Cluster Option]
indexterm:[Cluster,Option,concurrent-fencing]
Is the cluster allowed to initiate multiple fence actions concurrently?
| fence-reaction | stop |
indexterm:[fence-reaction,Cluster Option]
indexterm:[Cluster,Option,fence-reaction]
How should a cluster node react if notified of its own fencing? A cluster node
may receive notification of its own fencing if fencing is misconfigured, or if
fabric fencing is in use that doesn't cut cluster communication. Allowed values
are +stop+ to attempt to immediately stop pacemaker and stay stopped, or
+panic+ to attempt to immediately reboot the local node, falling back to stop
on failure. The default is likely to be changed to +panic+ in a future release.
'(since 2.0.3)'
| priority-fencing-delay | 0 |
indexterm:[priority-fencing-delay,Cluster Option]
indexterm:[Cluster,Option,priority-fencing-delay]
Apply specified delay for the fencings that are targeting the lost
nodes with the highest total resource priority in case we don't
have the majority of the nodes in our cluster partition, so that
the more significant nodes potentially win any fencing match,
which is especially meaningful under split-brain of 2-node
cluster. A promoted resource instance takes the base priority + 1
on calculation if the base priority is not 0. Any static/random
delays that are introduced by `pcmk_delay_base/max` configured
for the corresponding fencing resources will be added to this
delay. This delay should be significantly greater than, safely
twice, the maximum `pcmk_delay_base/max`. By default, priority
fencing delay is disabled. '(since 2.0.4)'
| cluster-delay | 60s |
indexterm:[cluster-delay,Cluster Option]
indexterm:[Cluster,Option,cluster-delay]
Estimated maximum round-trip delay over the network (excluding action
execution). If the DC requires an action to be executed on another
node, it will consider the action failed if it does not get a response
from the other node in this time (after considering the action's
own timeout). The "correct" value will depend on the speed and load of your
network and cluster nodes.
| dc-deadtime | 20s |
indexterm:[dc-deadtime,Cluster Option]
indexterm:[Cluster,Option,dc-deadtime]
How long to wait for a response from other nodes during startup.
The "correct" value will depend on the speed/load of your network and the type of switches used.
| cluster-ipc-limit | 500 |
indexterm:[cluster-ipc-limit,Cluster Option]
indexterm:[Cluster,Option,cluster-ipc-limit]
The maximum IPC message backlog before one cluster daemon will disconnect
another. This is of use in large clusters, for which a good value is the number
of resources in the cluster multiplied by the number of nodes. The default of
500 is also the minimum. Raise this if you see "Evicting client" messages for
cluster daemon PIDs in the logs.
| pe-error-series-max | -1 |
indexterm:[pe-error-series-max,Cluster Option]
indexterm:[Cluster,Option,pe-error-series-max]
The number of PE inputs resulting in ERRORs to save. Used when reporting problems.
A value of -1 means unlimited (report all).
| pe-warn-series-max | -1 |
indexterm:[pe-warn-series-max,Cluster Option]
indexterm:[Cluster,Option,pe-warn-series-max]
The number of PE inputs resulting in WARNINGs to save. Used when reporting problems.
A value of -1 means unlimited (report all).
| pe-input-series-max | -1 |
indexterm:[pe-input-series-max,Cluster Option]
indexterm:[Cluster,Option,pe-input-series-max]
The number of "normal" PE inputs to save. Used when reporting problems.
A value of -1 means unlimited (report all).
| placement-strategy | default |
indexterm:[placement-strategy,Cluster Option]
indexterm:[Cluster,Option,placement-strategy]
How the cluster should allocate resources to nodes (see <>).
Allowed values are +default+, +utilization+, +balanced+, and +minimal+.
| node-health-strategy | none |
indexterm:[node-health-strategy,Cluster Option]
indexterm:[Cluster,Option,node-health-strategy]
How the cluster should react to node health attributes (see <>).
Allowed values are +none+, +migrate-on-red+, +only-green+, +progressive+, and
+custom+.
| enable-acl | FALSE |
indexterm:[enable-acl,Cluster Option]
indexterm:[Cluster,Option,enable-acl]
Whether access control lists (ACLs) (see <>) can be used to authorize
modifications to the CIB.
| node-health-base | 0 |
indexterm:[node-health-base,Cluster Option]
indexterm:[Cluster,Option,node-health-base]
The base health score assigned to a node. Only used when
+node-health-strategy+ is +progressive+.
| node-health-green | 0 |
indexterm:[node-health-green,Cluster Option]
indexterm:[Cluster,Option,node-health-green]
The score to use for a node health attribute whose value is +green+.
Only used when +node-health-strategy+ is +progressive+ or +custom+.
| node-health-yellow | 0 |
indexterm:[node-health-yellow,Cluster Option]
indexterm:[Cluster,Option,node-health-yellow]
The score to use for a node health attribute whose value is +yellow+.
Only used when +node-health-strategy+ is +progressive+ or +custom+.
| node-health-red | 0 |
indexterm:[node-health-red,Cluster Option]
indexterm:[Cluster,Option,node-health-red]
The score to use for a node health attribute whose value is +red+.
Only used when +node-health-strategy+ is +progressive+ or +custom+.
| cluster-recheck-interval | 15min |
indexterm:[cluster-recheck-interval,Cluster Option]
indexterm:[Cluster,Option,cluster-recheck-interval]
Pacemaker is primarily event-driven, and looks ahead to know when to recheck
the cluster for failure timeouts and most time-based rules. However, it will
also recheck the cluster after this amount of inactivity. This has two goals:
rules with +date_spec+ are only guaranteed to be checked this often, and it
also serves as a fail-safe for certain classes of scheduler bugs. A value of 0
disables this polling; positive values are a time interval.
| shutdown-lock | false |
The default of false allows active resources to be recovered elsewhere when
their node is cleanly shut down, which is what the vast majority of users will
want. However, some users prefer to make resources highly available only for
failures, with no recovery for clean shutdowns. If this option is true,
resources active on a node when it is cleanly shut down are kept "locked" to
that node (not allowed to run elsewhere) until they start again on that node
after it rejoins (or for at most shutdown-lock-limit, if set). Stonith
resources and Pacemaker Remote connections are never locked. Clone and bundle
instances and the master role of promotable clones are currently never locked,
though support could be added in a future release. Locks may be manually
cleared using the `--refresh` option of `crm_resource` (both the resource and
node must be specified; this works with remote nodes if their connection
resource's target-role is set to Stopped, but not if Pacemaker Remote is
stopped on the remote node without disabling the connection resource).
'(since 2.0.4)'
indexterm:[shutdown-lock,Cluster Option]
indexterm:[Cluster,Option,shutdown-lock]
| shutdown-lock-limit | 0 |
If shutdown-lock is true, and this is set to a nonzero time duration, locked
resources will be allowed to start after this much time has passed since the
node shutdown was initiated, even if the node has not rejoined. (This works
with remote nodes only if their connection resource's target-role is set to
Stopped.) '(since 2.0.4)'
indexterm:[shutdown-lock-limit,Cluster Option]
indexterm:[Cluster,Option,shutdown-lock-limit]
| remove-after-stop | FALSE |
indexterm:[remove-after-stop,Cluster Option]
indexterm:[Cluster,Option,remove-after-stop]
_Advanced Use Only:_ Should the cluster remove resources from the LRM after
they are stopped? Values other than the default are, at best, poorly tested and
potentially dangerous.
| startup-fencing | TRUE |
indexterm:[startup-fencing,Cluster Option]
indexterm:[Cluster,Option,startup-fencing]
_Advanced Use Only:_ Should the cluster shoot unseen nodes?
Not using the default is very unsafe!
| election-timeout | 2min |
indexterm:[election-timeout,Cluster Option]
indexterm:[Cluster,Option,election-timeout]
_Advanced Use Only:_ If you need to adjust this value, it probably indicates
the presence of a bug.
| shutdown-escalation | 20min |
indexterm:[shutdown-escalation,Cluster Option]
indexterm:[Cluster,Option,shutdown-escalation]
_Advanced Use Only:_ If you need to adjust this value, it probably indicates
the presence of a bug.
| join-integration-timeout | 3min |
indexterm:[join-integration-timeout,Cluster Option]
indexterm:[Cluster,Option,join-integration-timeout]
_Advanced Use Only:_ If you need to adjust this value, it probably indicates
the presence of a bug.
| join-finalization-timeout | 30min |
indexterm:[join-finalization-timeout,Cluster Option]
indexterm:[Cluster,Option,join-finalization-timeout]
_Advanced Use Only:_ If you need to adjust this value, it probably indicates
the presence of a bug.
| transition-delay | 0s |
indexterm:[transition-delay,Cluster Option]
indexterm:[Cluster,Option,transition-delay]
_Advanced Use Only:_ Delay cluster recovery for the configured interval to
allow for additional/related events to occur. Useful if your configuration is
sensitive to the order in which ping updates arrive.
Enabling this option will slow down cluster recovery under
all conditions.
|=========================================================
diff --git a/doc/Pacemaker_Explained/en-US/Ch-Resources.txt b/doc/Pacemaker_Explained/en-US/Ch-Resources.txt
index d8e7115dbd..88892db342 100644
--- a/doc/Pacemaker_Explained/en-US/Ch-Resources.txt
+++ b/doc/Pacemaker_Explained/en-US/Ch-Resources.txt
@@ -1,907 +1,948 @@
:compat-mode: legacy
= Cluster Resources =
[[s-resource-primitive]]
== What is a Cluster Resource? ==
indexterm:[Resource]
A resource is a service made highly available by a cluster.
The simplest type of resource, a 'primitive' resource, is described
in this chapter. More complex forms, such as groups and clones,
are described in later chapters.
Every primitive resource has a 'resource agent'. A resource agent is an
external program that abstracts the service it provides and present a
consistent view to the cluster.
This allows the cluster to be agnostic about the resources it manages.
The cluster doesn't need to understand how the resource works because
it relies on the resource agent to do the right thing when given a
`start`, `stop` or `monitor` command. For this reason, it is crucial that
resource agents are well-tested.
Typically, resource agents come in the form of shell scripts. However,
they can be written using any technology (such as C, Python or Perl)
that the author is comfortable with.
[[s-resource-supported]]
== Resource Classes ==
indexterm:[Resource,class]
Pacemaker supports several classes of agents:
* OCF
* LSB
* Upstart
* Systemd
* Service
* Fencing
* Nagios Plugins
=== Open Cluster Framework ===
indexterm:[Resource,OCF]
indexterm:[OCF,Resources]
indexterm:[Open Cluster Framework,Resources]
The OCF standard
footnote:[See https://github.com/ClusterLabs/OCF-spec/tree/master/ra . The
Pacemaker implementation has been somewhat extended from the OCF specs.]
is basically an extension of the Linux Standard Base conventions for
init scripts to:
* support parameters,
* make them self-describing, and
* make them extensible
OCF specs have strict definitions of the exit codes that actions must return.
footnote:[
The resource-agents source code includes the `ocf-tester` script, which
can be useful in this regard.
]
The cluster follows these specifications exactly, and giving the wrong
exit code will cause the cluster to behave in ways you will likely
find puzzling and annoying. In particular, the cluster needs to
distinguish a completely stopped resource from one which is in some
erroneous and indeterminate state.
Parameters are passed to the resource agent as environment variables, with the
special prefix +OCF_RESKEY_+. So, a parameter which the user thinks
of as +ip+ will be passed to the resource agent as +OCF_RESKEY_ip+. The
number and purpose of the parameters is left to the resource agent; however,
the resource agent should use the `meta-data` command to advertise any that it
supports.
The OCF class is the most preferred as it is an industry standard,
highly flexible (allowing parameters to be passed to agents in a
non-positional manner) and self-describing.
For more information, see the
http://www.linux-ha.org/wiki/OCF_Resource_Agents[reference] and
the 'Resource Agents' chapter of 'Pacemaker Administration'.
=== Linux Standard Base ===
indexterm:[Resource,LSB]
indexterm:[LSB,Resources]
indexterm:[Linux Standard Base,Resources]
'LSB' resource agents are more commonly known as 'init scripts'. If a full path
is not given, they are assumed to be located in +/etc/init.d+.
Commonly, they are provided by the OS distribution. In order to be used
with a Pacemaker cluster, they must conform to the LSB specification.
footnote:[
See
http://refspecs.linux-foundation.org/LSB_3.0.0/LSB-Core-generic/LSB-Core-generic/iniscrptact.html
for the LSB Spec as it relates to init scripts.
]
[WARNING]
====
Many distributions or particular software packages claim LSB compliance
but ship with broken init scripts. For details on how to check whether
your init script is LSB-compatible, see the 'Resource Agents' chapter of
'Pacemaker Administration'. Common problematic violations of the LSB
standard include:
* Not implementing the +status+ operation at all
* Not observing the correct exit status codes for
+start+/+stop+/+status+ actions
* Starting a started resource returns an error
* Stopping a stopped resource returns an error
====
[IMPORTANT]
====
Remember to make sure the computer is _not_ configured to start any
services at boot time -- that should be controlled by the cluster.
====
[[s-resource-supported-systemd]]
=== Systemd ===
indexterm:[Resource,Systemd]
indexterm:[Systemd,Resources]
Some newer distributions have replaced the old
http://en.wikipedia.org/wiki/Init#SysV-style["SysV"] style of
initialization daemons and scripts with an alternative called
http://www.freedesktop.org/wiki/Software/systemd[Systemd].
Pacemaker is able to manage these services _if they are present_.
Instead of init scripts, systemd has 'unit files'. Generally, the
services (unit files) are provided by the OS distribution, but there
are online guides for converting from init scripts.
footnote:[For example,
http://0pointer.de/blog/projects/systemd-for-admins-3.html]
[IMPORTANT]
====
Remember to make sure the computer is _not_ configured to start any
services at boot time -- that should be controlled by the cluster.
====
=== Upstart ===
indexterm:[Resource,Upstart]
indexterm:[Upstart,Resources]
Some newer distributions have replaced the old
http://en.wikipedia.org/wiki/Init#SysV-style["SysV"] style of
initialization daemons (and scripts) with an alternative called
http://upstart.ubuntu.com/[Upstart].
Pacemaker is able to manage these services _if they are present_.
Instead of init scripts, upstart has 'jobs'. Generally, the
services (jobs) are provided by the OS distribution.
[IMPORTANT]
====
Remember to make sure the computer is _not_ configured to start any
services at boot time -- that should be controlled by the cluster.
====
=== System Services ===
indexterm:[Resource,System Services]
indexterm:[System Service,Resources]
Since there are various types of system services (+systemd+,
+upstart+, and +lsb+), Pacemaker supports a special +service+ alias which
intelligently figures out which one applies to a given cluster node.
This is particularly useful when the cluster contains a mix of
+systemd+, +upstart+, and +lsb+.
In order, Pacemaker will try to find the named service as:
. an LSB init script
. a Systemd unit file
. an Upstart job
=== STONITH ===
indexterm:[Resource,STONITH]
indexterm:[STONITH,Resources]
The STONITH class is used exclusively for fencing-related resources. This is
discussed later in <>.
=== Nagios Plugins ===
indexterm:[Resource,Nagios Plugins]
indexterm:[Nagios Plugins,Resources]
Nagios Plugins
footnote:[The project has two independent forks, hosted at
https://www.nagios-plugins.org/ and https://www.monitoring-plugins.org/. Output
from both projects' plugins is similar, so plugins from either project can be
used with pacemaker.]
allow us to monitor services on remote hosts.
Pacemaker is able to do remote monitoring with the plugins _if they are
present_.
A common use case is to configure them as resources belonging to a resource
container (usually a virtual machine), and the container will be restarted
if any of them has failed. Another use is to configure them as ordinary
resources to be used for monitoring hosts or services via the network.
The supported parameters are same as the long options of the plugin.
[[primitive-resource]]
== Resource Properties ==
These values tell the cluster which resource agent to use for the resource,
where to find that resource agent and what standards it conforms to.
.Properties of a Primitive Resource
[width="95%",cols="1m,<6",options="header",align="center"]
|=========================================================
|Field
|Description
|id
|Your name for the resource
indexterm:[id,Resource]
indexterm:[Resource,Property,id]
|class
|The standard the resource agent conforms to. Allowed values:
+lsb+, +nagios+, +ocf+, +service+, +stonith+, +systemd+, +upstart+
indexterm:[class,Resource]
indexterm:[Resource,Property,class]
|type
|The name of the Resource Agent you wish to use. E.g. +IPaddr+ or +Filesystem+
indexterm:[type,Resource]
indexterm:[Resource,Property,type]
|provider
|The OCF spec allows multiple vendors to supply the same
resource agent. To use the OCF resource agents supplied by
the Heartbeat project, you would specify +heartbeat+ here.
indexterm:[provider,Resource]
indexterm:[Resource,Property,provider]
|=========================================================
The XML definition of a resource can be queried with the `crm_resource` tool.
For example:
----
# crm_resource --resource Email --query-xml
----
might produce:
.A system resource definition
=====
[source,XML]
=====
[NOTE]
=====
One of the main drawbacks to system services (LSB, systemd or
Upstart) resources is that they do not allow any parameters!
=====
////
See https://tools.ietf.org/html/rfc5737 for choice of example IP address
////
.An OCF resource definition
=====
[source,XML]
-------
-------
=====
[[s-resource-options]]
== Resource Options ==
Resources have two types of options: 'meta-attributes' and 'instance attributes'.
Meta-attributes apply to any type of resource, while instance attributes
are specific to each resource agent.
=== Resource Meta-Attributes ===
Meta-attributes are used by the cluster to decide how a resource should
behave and can be easily set using the `--meta` option of the
`crm_resource` command.
.Meta-attributes of a Primitive Resource
[width="95%",cols="2m,2,<5",options="header",align="center"]
|=========================================================
|Field
|Default
|Description
|priority
|0
|If not all resources can be active, the cluster will stop lower
priority resources in order to keep higher priority ones active.
indexterm:[priority,Resource Option]
indexterm:[Resource,Option,priority]
|target-role
|Started
a|What state should the cluster attempt to keep this resource in? Allowed values:
* +Stopped:+ Force the resource to be stopped
* +Started:+ Allow the resource to be started (and in the case of
<>, promoted to master if
appropriate)
* +Slave:+ Allow the resource to be started, but only in Slave mode if
the resource is <>
* +Master:+ Equivalent to +Started+
indexterm:[target-role,Resource Option]
indexterm:[Resource,Option,target-role]
|is-managed
|TRUE
|Is the cluster allowed to start and stop the resource? Allowed
values: +true+, +false+
indexterm:[is-managed,Resource Option]
indexterm:[Resource,Option,is-managed]
|maintenance
|FALSE
|Similar to the +maintenance-mode+ <>, but for
a single resource. If true, the resource will not be started, stopped, or
monitored on any node. This differs from +is-managed+ in that monitors will
not be run. Allowed values: +true+, +false+
indexterm:[maintenance,Resource Option]
indexterm:[Resource,Option,maintenance]
|resource-stickiness
|1 for individual clone instances, 0 for all other resources
|A score that will be added to the current node when a resource is already
active. This allows running resources to stay where they are, even if
they would be placed elsewhere if they were being started from a stopped
state.
indexterm:[resource-stickiness,Resource Option]
indexterm:[Resource,Option,resource-stickiness]
|requires
|+quorum+ for resources with a +class+ of +stonith+,
otherwise +unfencing+ if unfencing is active in the cluster,
otherwise +fencing+ if +stonith-enabled+ is true, otherwise +quorum+
a|Conditions under which the resource can be started
Allowed values:
* +nothing:+ can always be started
* +quorum:+ The cluster can only start this resource if a majority of
the configured nodes are active
* +fencing:+ The cluster can only start this resource if a majority
of the configured nodes are active _and_ any failed or unknown nodes
have been <>
* +unfencing:+
The cluster can only start this resource if a majority
of the configured nodes are active _and_ any failed or unknown nodes
have been fenced _and_ only on nodes that have been
<>
indexterm:[requires,Resource Option]
indexterm:[Resource,Option,requires]
|migration-threshold
|INFINITY
|How many failures may occur for this resource on a node, before this
node is marked ineligible to host this resource. A value of 0 indicates that
this feature is disabled (the node will never be marked ineligible); by
constrast, the cluster treats INFINITY (the default) as a very large but
finite number. This option has an effect only if the failed operation
specifies +on-fail+ as +restart+ (the default), and additionally for
failed +start+ operations, if the cluster property +start-failure-is-fatal+
is +false+.
indexterm:[migration-threshold,Resource Option]
indexterm:[Resource,Option,migration-threshold]
|failure-timeout
|0
|How many seconds to wait before acting as if the failure had not
occurred, and potentially allowing the resource back to the node on
which it failed. A value of 0 indicates that this feature is disabled.
indexterm:[failure-timeout,Resource Option]
indexterm:[Resource,Option,failure-timeout]
|multiple-active
|stop_start
a|What should the cluster do if it ever finds the resource active on
more than one node? Allowed values:
* +block:+ mark the resource as unmanaged
* +stop_only:+ stop all active instances and leave them that way
* +stop_start:+ stop all active instances and start the resource in
one location only
indexterm:[multiple-active,Resource Option]
indexterm:[Resource,Option,multiple-active]
|allow-migrate
|TRUE for ocf:pacemaker:remote resources, FALSE otherwise
|Whether the cluster should try to "live migrate" this resource when it needs
to be moved (see <>)
|container-attribute-target
|
|Specific to bundle resources; see <>
|remote-node
|
|The name of the Pacemaker Remote guest node this resource is associated with,
if any. If specified, this both enables the resource as a guest node and
defines the unique name used to identify the guest node. The guest must be
configured to run the Pacemaker Remote daemon when it is started. +WARNING:+
This value cannot overlap with any resource or node IDs.
|remote-port
|3121
|If +remote-node+ is specified, the port on the guest used for its
Pacemaker Remote connection. The Pacemaker Remote daemon on the guest must be
configured to listen on this port.
|remote-addr
|value of +remote-node+
|If +remote-node+ is specified, the IP address or hostname used to connect to
the guest via Pacemaker Remote. The Pacemaker Remote daemon on the guest
must be configured to accept connections on this address.
|remote-connect-timeout
|60s
|If +remote-node+ is specified, how long before a pending guest connection will
time out.
|=========================================================
As an example of setting resource options, if you performed the following
commands on an LSB Email resource:
-------
# crm_resource --meta --resource Email --set-parameter priority --parameter-value 100
# crm_resource -m -r Email -p multiple-active -v block
-------
the resulting resource definition might be:
.An LSB resource with cluster options
=====
[source,XML]
-------
-------
=====
In addition to the cluster-defined meta-attributes described above, you may
also configure arbitrary meta-attributes of your own choosing. Most commonly,
this would be done for use in <>. For example, an IT department
might define a custom meta-attribute to indicate which company department each
resource is intended for. To reduce the chance of name collisions with
cluster-defined meta-attributes added in the future, it is recommended to use
a unique, organization-specific prefix for such attributes.
[[s-resource-defaults]]
=== Setting Global Defaults for Resource Meta-Attributes ===
To set a default value for a resource option, add it to the
+rsc_defaults+ section with `crm_attribute`. For example,
----
# crm_attribute --type rsc_defaults --name is-managed --update false
----
would prevent the cluster from starting or stopping any of the
resources in the configuration (unless of course the individual
resources were specifically enabled by having their +is-managed+ set to
+true+).
=== Resource Instance Attributes ===
The resource agents of some resource classes (lsb, systemd and upstart 'not' among them)
can be given parameters which determine how they behave and which instance
of a service they control.
If your resource agent supports parameters, you can add them with the
`crm_resource` command. For example,
----
# crm_resource --resource Public-IP --set-parameter ip --parameter-value 192.0.2.2
----
would create an entry in the resource like this:
.An example OCF resource with instance attributes
=====
[source,XML]
-------
-------
=====
For an OCF resource, the result would be an environment variable
called +OCF_RESKEY_ip+ with a value of +192.0.2.2+.
The list of instance attributes supported by an OCF resource agent can be
found by calling the resource agent with the `meta-data` command.
The output contains an XML description of all the supported
attributes, their purpose and default values.
.Displaying the metadata for the Dummy resource agent template
=====
----
# export OCF_ROOT=/usr/lib/ocf
# $OCF_ROOT/resource.d/pacemaker/Dummy meta-data
----
[source,XML]
-------
1.0
This is a Dummy Resource Agent. It does absolutely nothing except
keep track of whether its running or not.
Its purpose in life is for testing and to serve as a template for RA writers.
NB: Please pay attention to the timeouts specified in the actions
section below. They should be meaningful for the kind of resource
the agent manages. They should be the minimum advised timeouts,
but they shouldn't/cannot cover _all_ possible resource
instances. So, try to be neither overly generous nor too stingy,
but moderate. The minimum timeouts should never be below 10 seconds.
Example stateless resource agent
Location to store the resource state in.
State file
Fake attribute that can be changed to cause a reload
Fake attribute that can be changed to cause a reload
Number of seconds to sleep during operations. This can be used to test how
the cluster reacts to operation timeouts.
Operation sleep duration in seconds.
-------
=====
== Resource Operations ==
indexterm:[Resource,Action]
'Operations' are actions the cluster can perform on a resource by calling the
resource agent. Resource agents must support certain common operations such as
start, stop, and monitor, and may implement any others.
Operations may be explicitly configured for two purposes: to override defaults
for options (such as timeout) that the cluster will use whenever it initiates
the operation, and to run an operation on a recurring basis (for example, to
monitor the resource for failure).
.An OCF resource with a non-default start timeout
=====
[source,XML]
-------
-------
=====
Pacemaker identifies operations by a combination of name and interval, so this
combination must be unique for each resource. That is, you should not configure
two operations for the same resource with the same name and interval.
[[s-operation-properties]]
=== Operation Properties ===
Operation properties may be specified directly in the +op+ element as
XML attributes, or in a separate +meta_attributes+ block as +nvpair+ elements.
XML attributes take precedence over +nvpair+ elements if both are specified.
.Properties of an Operation
[width="95%",cols="2m,3,<6",options="header",align="center"]
|=========================================================
|Field
|Default
|Description
|id
|
|A unique name for the operation.
indexterm:[id,Action Property]
indexterm:[Action,Property,id]
|name
|
|The action to perform. This can be any action supported by the agent; common
values include +monitor+, +start+, and +stop+.
indexterm:[name,Action Property]
indexterm:[Action,Property,name]
|interval
|0
|How frequently (in seconds) to perform the operation. A value of 0 means "when
needed". A positive value defines a 'recurring action', which is typically
used with <>.
indexterm:[interval,Action Property]
indexterm:[Action,Property,interval]
|timeout
|
|How long to wait before declaring the action has failed
indexterm:[timeout,Action Property]
indexterm:[Action,Property,timeout]
|on-fail
-|restart '(except for +stop+ operations, which default to' fence 'when
- STONITH is enabled and' block 'otherwise)'
+a|Varies by action:
+
+* +stop+: +fence+ if +stonith-enabled+ is true or +block+ otherwise
+* +demote+: +on-fail+ of the +monitor+ action with +role+ set to +Master+, if
+ present, enabled, and configured to a value other than +demote+, or +restart+
+ otherwise
+* all other actions: +restart+
a|The action to take if this action ever fails. Allowed values:
* +ignore:+ Pretend the resource did not fail.
* +block:+ Don't perform any further operations on the resource.
* +stop:+ Stop the resource and do not start it elsewhere.
+* +demote:+ Demote the resource, without a full restart. This is valid only for
+ +promote+ actions, and for +monitor+ actions with both a nonzero +interval+
+ and +role+ set to +Master+; for any other action, a configuration error will
+ be logged, and the default behavior will be used.
* +restart:+ Stop the resource and start it again (possibly on a different node).
* +fence:+ STONITH the node on which the resource failed.
* +standby:+ Move _all_ resources away from the node on which the resource failed.
indexterm:[on-fail,Action Property]
indexterm:[Action,Property,on-fail]
|enabled
|TRUE
|If +false+, ignore this operation definition. This is typically used to pause
a particular recurring +monitor+ operation; for instance, it can complement
the respective resource being unmanaged (+is-managed=false+), as this alone
will <>.
Disabling the operation does not suppress all actions of the given type.
Allowed values: +true+, +false+.
indexterm:[enabled,Action Property]
indexterm:[Action,Property,enabled]
|record-pending
|TRUE
|If +true+, the intention to perform the operation is recorded so that
GUIs and CLI tools can indicate that an operation is in progress.
This is best set as an _operation default_ (see <>).
Allowed values: +true+, +false+.
indexterm:[enabled,Action Property]
indexterm:[Action,Property,enabled]
|role
|
|Run the operation only on node(s) that the cluster thinks should be in
the specified role. This only makes sense for recurring +monitor+ operations.
Allowed (case-sensitive) values: +Stopped+, +Started+, and in the
case of <>, +Slave+ and +Master+.
indexterm:[role,Action Property]
indexterm:[Action,Property,role]
|=========================================================
+[NOTE]
+====
+When +on-fail+ is set to +demote+, recovery from failure by a successful demote
+causes the cluster to recalculate whether and where a new instance should be
+promoted. The node with the failure is eligible, so if master scores have not
+changed, it will be promoted again.
+
+There is no direct equivalent of +migration-threshold+ for the master role, but
+the same effect can be achieved with a location constraint using a
+<> with a node attribute expression for the resource's fail
+count.
+
+For example, to immediately ban the master role from a node with any failed
+promote or master monitor:
+[source,XML]
+----
+
+
+
+
+
+
+----
+
+This example assumes that there is a promotable clone of the +my_primitive+
+resource (note that the primitive name, not the clone name, is used in the
+rule), and that there is a recurring 10-second-interval monitor configured for
+the master role (fail count attributes specify the interval in milliseconds).
+====
+
[[s-resource-monitoring]]
=== Monitoring Resources for Failure ===
When Pacemaker first starts a resource, it runs one-time +monitor+ operations
(referred to as 'probes') to ensure the resource is running where it's
supposed to be, and not running where it's not supposed to be. (This behavior
can be affected by the +resource-discovery+ location constraint property.)
Other than those initial probes, Pacemaker will 'not' (by default) check that
the resource continues to stay healthy.
footnote:[Currently, anyway. Automatic monitoring operations may be
added in a future version of Pacemaker.]
You must configure +monitor+ operations explicitly to perform these checks.
.An OCF resource with a recurring health check
=====
[source,XML]
-------
-------
=====
By default, a +monitor+ operation will ensure that the resource is running
where it is supposed to. The +target-role+ property can be used for further
checking.
For example, if a resource has one +monitor+ operation with
+interval=10 role=Started+ and a second +monitor+ operation with
+interval=11 role=Stopped+, the cluster will run the first monitor on any nodes
it thinks 'should' be running the resource, and the second monitor on any nodes
that it thinks 'should not' be running the resource (for the truly paranoid,
who want to know when an administrator manually starts a service by mistake).
[NOTE]
====
Currently, monitors with +role=Stopped+ are not implemented for
<> resources.
====
[[s-monitoring-unmanaged]]
=== Monitoring Resources When Administration is Disabled ===
Recurring +monitor+ operations behave differently under various administrative
settings:
* When a resource is unmanaged (by setting +is-managed=false+): No monitors
will be stopped.
+
If the unmanaged resource is stopped on a node where the cluster thinks it
should be running, the cluster will detect and report that it is not, but it
will not consider the monitor failed, and will not try to start the resource
until it is managed again.
+
Starting the unmanaged resource on a different node is strongly discouraged
and will at least cause the cluster to consider the resource failed, and
may require the resource's +target-role+ to be set to +Stopped+ then +Started+
to be recovered.
* When a node is put into standby: All resources will be moved away from the
node, and all +monitor+ operations will be stopped on the node, except those
specifying +role+ as +Stopped+ (which will be newly initiated if
appropriate).
* When the cluster is put into maintenance mode: All resources will be marked
as unmanaged. All monitor operations will be stopped, except those
specifying +role+ as +Stopped+ (which will be newly initiated if
appropriate). As with single unmanaged resources, starting
a resource on a node other than where the cluster expects it to be will
cause problems.
[[s-operation-defaults]]
=== Setting Global Defaults for Operations ===
You can change the global default values for operation properties
in a given cluster. These are defined in an +op_defaults+ section
of the CIB's +configuration+ section, and can be set with `crm_attribute`.
For example,
----
# crm_attribute --type op_defaults --name timeout --update 20s
----
would default each operation's +timeout+ to 20 seconds. If an
operation's definition also includes a value for +timeout+, then that
value would be used for that operation instead.
=== When Implicit Operations Take a Long Time ===
The cluster will always perform a number of implicit operations: +start+,
+stop+ and a non-recurring +monitor+ operation used at startup to check
whether the resource is already active. If one of these is taking too long,
then you can create an entry for them and specify a longer timeout.
.An OCF resource with custom timeouts for its implicit actions
=====
[source,XML]
-------
-------
=====
=== Multiple Monitor Operations ===
Provided no two operations (for a single resource) have the same name
and interval, you can have as many +monitor+ operations as you like.
In this way, you can do a superficial health check every minute and
progressively more intense ones at higher intervals.
To tell the resource agent what kind of check to perform, you need to
provide each monitor with a different value for a common parameter.
The OCF standard creates a special parameter called +OCF_CHECK_LEVEL+
for this purpose and dictates that it is "made available to the
resource agent without the normal +OCF_RESKEY+ prefix".
Whatever name you choose, you can specify it by adding an
+instance_attributes+ block to the +op+ tag. It is up to each
resource agent to look for the parameter and decide how to use it.
.An OCF resource with two recurring health checks, performing different levels of checks specified via +OCF_CHECK_LEVEL+.
=====
[source,XML]
-------
-------
=====
=== Disabling a Monitor Operation ===
The easiest way to stop a recurring monitor is to just delete it.
However, there can be times when you only want to disable it
temporarily. In such cases, simply add +enabled=false+ to the
operation's definition.
.Example of an OCF resource with a disabled health check
=====
[source,XML]
-------
-------
=====
This can be achieved from the command line by executing:
----
# cibadmin --modify --xml-text ''
----
Once you've done whatever you needed to do, you can then re-enable it with
----
# cibadmin --modify --xml-text ''
----
diff --git a/include/crm/crm.h b/include/crm/crm.h
index d2ffb610df..dc2adc12c9 100644
--- a/include/crm/crm.h
+++ b/include/crm/crm.h
@@ -1,231 +1,231 @@
/*
* Copyright 2004-2020 the Pacemaker project contributors
*
* The version control history for this file may have further details.
*
* This source code is licensed under the GNU Lesser General Public License
* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
*/
#ifndef CRM__H
# define CRM__H
#ifdef __cplusplus
extern "C" {
#endif
/**
* \file
* \brief A dumping ground
* \ingroup core
*/
# include
# include
# include
# include
# include
# include
/*!
* The CRM feature set assists with compatibility in mixed-version clusters.
* The major version number increases when nodes with different versions
* would not work (rolling upgrades are not allowed). The minor version
* number increases when mixed-version clusters are allowed only during
* rolling upgrades (a node with the oldest feature set will be elected DC). The
* minor-minor version number is ignored, but allows resource agents to detect
* cluster support for various features.
*
* The feature set also affects the processing of old saved CIBs (such as for
* many scheduler regression tests).
*
* Particular feature points currently used by pacemaker:
*
* >2.1: Operation updates include timing data
* >=3.0.5: XML v2 digests are created
* >=3.0.8: Peers do not need acks for cancellations
* >=3.0.9: DC will send its own shutdown request to all peers
* XML v2 patchsets are created by default
* >=3.0.13: Fail counts include operation name and interval
* >=3.2.0: DC supports PCMK_LRM_OP_INVALID and PCMK_LRM_OP_NOT_CONNECTED
*/
-# define CRM_FEATURE_SET "3.3.0"
+# define CRM_FEATURE_SET "3.4.0"
# define EOS '\0'
# define DIMOF(a) ((int) (sizeof(a)/sizeof(a[0])) )
# ifndef MAX_NAME
# define MAX_NAME 256
# endif
# ifndef __GNUC__
# define __builtin_expect(expr, result) (expr)
# endif
/* Some handy macros used by the Linux kernel */
# define __likely(expr) __builtin_expect(expr, 1)
# define __unlikely(expr) __builtin_expect(expr, 0)
# define CRM_META "CRM_meta"
extern char *crm_system_name;
/* *INDENT-OFF* */
// Used for some internal IPC timeouts (maybe should be configurable option)
# define MAX_IPC_DELAY 120
// How we represent "infinite" scores
# define CRM_SCORE_INFINITY 1000000
# define CRM_INFINITY_S "INFINITY"
# define CRM_PLUS_INFINITY_S "+" CRM_INFINITY_S
# define CRM_MINUS_INFINITY_S "-" CRM_INFINITY_S
/* @COMPAT API < 2.0.0 Deprecated "infinity" aliases
*
* INFINITY might be defined elsewhere (e.g. math.h), so undefine it first.
* This, of course, complicates any attempt to use the other definition in any
* code that includes this header.
*/
# undef INFINITY
# define INFINITY_S "INFINITY"
# define MINUS_INFINITY_S "-INFINITY"
# define INFINITY 1000000
/* Sub-systems */
# define CRM_SYSTEM_DC "dc"
# define CRM_SYSTEM_DCIB "dcib"
/* The master CIB */
# define CRM_SYSTEM_CIB "cib"
# define CRM_SYSTEM_CRMD "crmd"
# define CRM_SYSTEM_LRMD "lrmd"
# define CRM_SYSTEM_PENGINE "pengine"
# define CRM_SYSTEM_TENGINE "tengine"
# define CRM_SYSTEM_STONITHD "stonithd"
# define CRM_SYSTEM_MCP "pacemakerd"
// Names of internally generated node attributes
# define CRM_ATTR_UNAME "#uname"
# define CRM_ATTR_ID "#id"
# define CRM_ATTR_KIND "#kind"
# define CRM_ATTR_ROLE "#role"
# define CRM_ATTR_IS_DC "#is_dc"
# define CRM_ATTR_CLUSTER_NAME "#cluster-name"
# define CRM_ATTR_SITE_NAME "#site-name"
# define CRM_ATTR_UNFENCED "#node-unfenced"
# define CRM_ATTR_DIGESTS_ALL "#digests-all"
# define CRM_ATTR_DIGESTS_SECURE "#digests-secure"
# define CRM_ATTR_RA_VERSION "#ra-version"
# define CRM_ATTR_PROTOCOL "#attrd-protocol"
/* Valid operations */
# define CRM_OP_NOOP "noop"
# define CRM_OP_JOIN_ANNOUNCE "join_announce"
# define CRM_OP_JOIN_OFFER "join_offer"
# define CRM_OP_JOIN_REQUEST "join_request"
# define CRM_OP_JOIN_ACKNAK "join_ack_nack"
# define CRM_OP_JOIN_CONFIRM "join_confirm"
# define CRM_OP_PING "ping"
# define CRM_OP_NODE_INFO "node-info"
# define CRM_OP_THROTTLE "throttle"
# define CRM_OP_VOTE "vote"
# define CRM_OP_NOVOTE "no-vote"
# define CRM_OP_HELLO "hello"
# define CRM_OP_PECALC "pe_calc"
# define CRM_OP_QUIT "quit"
# define CRM_OP_LOCAL_SHUTDOWN "start_shutdown"
# define CRM_OP_SHUTDOWN_REQ "req_shutdown"
# define CRM_OP_SHUTDOWN "do_shutdown"
# define CRM_OP_FENCE "stonith"
# define CRM_OP_REGISTER "register"
# define CRM_OP_IPC_FWD "ipc_fwd"
# define CRM_OP_INVOKE_LRM "lrm_invoke"
# define CRM_OP_LRM_REFRESH "lrm_refresh" /* Deprecated */
# define CRM_OP_LRM_QUERY "lrm_query"
# define CRM_OP_LRM_DELETE "lrm_delete"
# define CRM_OP_LRM_FAIL "lrm_fail"
# define CRM_OP_PROBED "probe_complete"
# define CRM_OP_REPROBE "probe_again"
# define CRM_OP_CLEAR_FAILCOUNT "clear_failcount"
# define CRM_OP_REMOTE_STATE "remote_state"
# define CRM_OP_RELAXED_SET "one-or-more"
# define CRM_OP_RELAXED_CLONE "clone-one-or-more"
# define CRM_OP_RM_NODE_CACHE "rm_node_cache"
# define CRM_OP_MAINTENANCE_NODES "maintenance_nodes"
/* Possible cluster membership states */
# define CRMD_JOINSTATE_DOWN "down"
# define CRMD_JOINSTATE_PENDING "pending"
# define CRMD_JOINSTATE_MEMBER "member"
# define CRMD_JOINSTATE_NACK "banned"
# define CRMD_ACTION_DELETE "delete"
# define CRMD_ACTION_CANCEL "cancel"
# define CRMD_ACTION_RELOAD "reload"
# define CRMD_ACTION_MIGRATE "migrate_to"
# define CRMD_ACTION_MIGRATED "migrate_from"
# define CRMD_ACTION_START "start"
# define CRMD_ACTION_STARTED "running"
# define CRMD_ACTION_STOP "stop"
# define CRMD_ACTION_STOPPED "stopped"
# define CRMD_ACTION_PROMOTE "promote"
# define CRMD_ACTION_PROMOTED "promoted"
# define CRMD_ACTION_DEMOTE "demote"
# define CRMD_ACTION_DEMOTED "demoted"
# define CRMD_ACTION_NOTIFY "notify"
# define CRMD_ACTION_NOTIFIED "notified"
# define CRMD_ACTION_STATUS "monitor"
# define CRMD_ACTION_METADATA "meta-data"
# define CRMD_METADATA_CALL_TIMEOUT 30000
/* short names */
# define RSC_DELETE CRMD_ACTION_DELETE
# define RSC_CANCEL CRMD_ACTION_CANCEL
# define RSC_MIGRATE CRMD_ACTION_MIGRATE
# define RSC_MIGRATED CRMD_ACTION_MIGRATED
# define RSC_START CRMD_ACTION_START
# define RSC_STARTED CRMD_ACTION_STARTED
# define RSC_STOP CRMD_ACTION_STOP
# define RSC_STOPPED CRMD_ACTION_STOPPED
# define RSC_PROMOTE CRMD_ACTION_PROMOTE
# define RSC_PROMOTED CRMD_ACTION_PROMOTED
# define RSC_DEMOTE CRMD_ACTION_DEMOTE
# define RSC_DEMOTED CRMD_ACTION_DEMOTED
# define RSC_NOTIFY CRMD_ACTION_NOTIFY
# define RSC_NOTIFIED CRMD_ACTION_NOTIFIED
# define RSC_STATUS CRMD_ACTION_STATUS
# define RSC_METADATA CRMD_ACTION_METADATA
/* *INDENT-ON* */
typedef GList *GListPtr;
# include
# include
static inline const char *
crm_action_str(const char *task, guint interval_ms) {
if(safe_str_eq(task, RSC_STATUS) && !interval_ms) {
return "probe";
}
return task;
}
#ifdef __cplusplus
}
#endif
#endif
diff --git a/include/crm/pengine/common.h b/include/crm/pengine/common.h
index 3a770b7997..2737b2ec30 100644
--- a/include/crm/pengine/common.h
+++ b/include/crm/pengine/common.h
@@ -1,173 +1,185 @@
/*
* Copyright 2004-2020 the Pacemaker project contributors
*
* The version control history for this file may have further details.
*
* This source code is licensed under the GNU Lesser General Public License
* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
*/
#ifndef PE_COMMON__H
# define PE_COMMON__H
#ifdef __cplusplus
extern "C" {
#endif
# include
# include
# include
extern gboolean was_processing_error;
extern gboolean was_processing_warning;
-/* order is significant here
- * items listed in order of accending severeness
- * more severe actions take precedent over lower ones
+/* The order is (partially) significant here; the values from action_fail_ignore
+ * through action_fail_fence are in order of increasing severity.
+ *
+ * @COMPAT The values should be ordered and numbered per the "TODO" comments
+ * below, so all values are in order of severity and there is room for
+ * future additions, but that would break API compatibility.
+ * @TODO For now, we just use a function to compare the values specially, but
+ * at the next compatibility break, we should arrange things properly.
*/
enum action_fail_response {
- action_fail_ignore,
- action_fail_recover,
- action_fail_migrate, /* recover by moving it somewhere else */
- action_fail_block,
- action_fail_stop,
- action_fail_standby,
- action_fail_fence,
+ action_fail_ignore, // @TODO = 10
+ // @TODO action_fail_demote = 20,
+ action_fail_recover, // @TODO = 30
+ // @TODO action_fail_reset_remote = 40,
+ // @TODO action_fail_restart_container = 50,
+ action_fail_migrate, // @TODO = 60
+ action_fail_block, // @TODO = 70
+ action_fail_stop, // @TODO = 80
+ action_fail_standby, // @TODO = 90
+ action_fail_fence, // @TODO = 100
+
+ // @COMPAT Values below here are out of order for API compatibility
+
action_fail_restart_container,
/* This is reserved for internal use for remote node connection resources.
* Fence the remote node if stonith is enabled, otherwise attempt to recover
* the connection resource. This allows us to specify types of connection
* resource failures that should result in fencing the remote node
* (for example, recurring monitor failures).
*/
action_fail_reset_remote,
+ action_fail_demote,
};
/* the "done" action must be the "pre" action +1 */
enum action_tasks {
no_action,
monitor_rsc,
stop_rsc,
stopped_rsc,
start_rsc,
started_rsc,
action_notify,
action_notified,
action_promote,
action_promoted,
action_demote,
action_demoted,
shutdown_crm,
stonith_node
};
enum rsc_recovery_type {
recovery_stop_start,
recovery_stop_only,
recovery_block
};
enum rsc_start_requirement {
rsc_req_nothing, /* Allowed by custom_action() */
rsc_req_quorum, /* Enforced by custom_action() */
rsc_req_stonith /* Enforced by native_start_constraints() */
};
enum rsc_role_e {
RSC_ROLE_UNKNOWN,
RSC_ROLE_STOPPED,
RSC_ROLE_STARTED,
RSC_ROLE_SLAVE,
RSC_ROLE_MASTER,
};
# define RSC_ROLE_MAX RSC_ROLE_MASTER+1
# define RSC_ROLE_UNKNOWN_S "Unknown"
# define RSC_ROLE_STOPPED_S "Stopped"
# define RSC_ROLE_STARTED_S "Started"
# define RSC_ROLE_SLAVE_S "Slave"
# define RSC_ROLE_MASTER_S "Master"
enum pe_print_options {
pe_print_log = 0x0001,
pe_print_html = 0x0002,
pe_print_ncurses = 0x0004,
pe_print_printf = 0x0008,
pe_print_dev = 0x0010, // Debugging (@COMPAT probably not useful)
pe_print_details = 0x0020,
pe_print_max_details = 0x0040,
pe_print_rsconly = 0x0080,
pe_print_ops = 0x0100,
pe_print_suppres_nl = 0x0200,
pe_print_xml = 0x0400,
pe_print_brief = 0x0800,
pe_print_pending = 0x1000,
pe_print_clone_details = 0x2000,
pe_print_clone_active = 0x4000, // Print clone instances only if active
pe_print_implicit = 0x8000, // Print implicitly created resources
};
const char *task2text(enum action_tasks task);
enum action_tasks text2task(const char *task);
enum rsc_role_e text2role(const char *role);
const char *role2text(enum rsc_role_e role);
const char *fail2text(enum action_fail_response fail);
const char *pe_pref(GHashTable * options, const char *name);
void calculate_active_ops(GList * sorted_op_list, int *start_index, int *stop_index);
static inline const char *
recovery2text(enum rsc_recovery_type type)
{
switch (type) {
case recovery_stop_only:
return "shutting it down";
case recovery_stop_start:
return "attempting recovery";
case recovery_block:
return "waiting for an administrator";
}
return "Unknown";
}
typedef struct pe_re_match_data {
char *string;
int nregs;
regmatch_t *pmatch;
} pe_re_match_data_t;
typedef struct pe_match_data {
pe_re_match_data_t *re;
GHashTable *params;
GHashTable *meta;
} pe_match_data_t;
typedef struct pe_rsc_eval_data {
const char *standard;
const char *provider;
const char *agent;
} pe_rsc_eval_data_t;
typedef struct pe_op_eval_data {
const char *op_name;
guint interval;
} pe_op_eval_data_t;
typedef struct pe_rule_eval_data {
GHashTable *node_hash;
enum rsc_role_e role;
crm_time_t *now;
pe_match_data_t *match_data;
pe_rsc_eval_data_t *rsc_data;
pe_op_eval_data_t *op_data;
} pe_rule_eval_data_t;
#ifdef __cplusplus
}
#endif
#endif
diff --git a/include/crm/pengine/pe_types.h b/include/crm/pengine/pe_types.h
index ba8849181f..f3cb4eff83 100644
--- a/include/crm/pengine/pe_types.h
+++ b/include/crm/pengine/pe_types.h
@@ -1,536 +1,538 @@
/*
* Copyright 2004-2020 the Pacemaker project contributors
*
* The version control history for this file may have further details.
*
* This source code is licensed under the GNU Lesser General Public License
* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
*/
#ifndef PE_TYPES__H
# define PE_TYPES__H
#ifdef __cplusplus
extern "C" {
#endif
/*!
* \file
* \brief Data types for cluster status
* \ingroup pengine
*/
# include // bool
# include // time_t
# include // gboolean, guint, GList, GHashTable
# include // GListPtr
# include
# include
typedef struct pe_node_s pe_node_t;
typedef struct pe_action_s pe_action_t;
typedef struct pe_resource_s pe_resource_t;
typedef struct pe_working_set_s pe_working_set_t;
enum pe_obj_types {
pe_unknown = -1,
pe_native = 0,
pe_group = 1,
pe_clone = 2,
pe_container = 3,
};
typedef struct resource_object_functions_s {
gboolean (*unpack) (pe_resource_t*, pe_working_set_t*);
pe_resource_t *(*find_rsc) (pe_resource_t *parent, const char *search,
const pe_node_t *node, int flags);
/* parameter result must be free'd */
char *(*parameter) (pe_resource_t*, pe_node_t*, gboolean, const char*,
pe_working_set_t*);
void (*print) (pe_resource_t*, const char*, long, void*);
gboolean (*active) (pe_resource_t*, gboolean);
enum rsc_role_e (*state) (const pe_resource_t*, gboolean);
pe_node_t *(*location) (const pe_resource_t*, GList**, int);
void (*free) (pe_resource_t*);
void (*count) (pe_resource_t*);
} resource_object_functions_t;
typedef struct resource_alloc_functions_s resource_alloc_functions_t;
enum pe_quorum_policy {
no_quorum_freeze,
no_quorum_stop,
no_quorum_ignore,
- no_quorum_suicide
+ no_quorum_suicide,
+ no_quorum_demote
};
enum node_type {
node_ping,
node_member,
node_remote
};
//! \deprecated will be removed in a future release
enum pe_restart {
pe_restart_restart,
pe_restart_ignore
};
//! Determine behavior of pe_find_resource_with_flags()
enum pe_find {
pe_find_renamed = 0x001, //!< match resource ID or LRM history ID
pe_find_anon = 0x002, //!< match base name of anonymous clone instances
pe_find_clone = 0x004, //!< match only clone instances
pe_find_current = 0x008, //!< match resource active on specified node
pe_find_inactive = 0x010, //!< match resource not running anywhere
pe_find_any = 0x020, //!< match base name of any clone instance
};
// @TODO Make these an enum
# define pe_flag_have_quorum 0x00000001ULL
# define pe_flag_symmetric_cluster 0x00000002ULL
# define pe_flag_maintenance_mode 0x00000008ULL
# define pe_flag_stonith_enabled 0x00000010ULL
# define pe_flag_have_stonith_resource 0x00000020ULL
# define pe_flag_enable_unfencing 0x00000040ULL
# define pe_flag_concurrent_fencing 0x00000080ULL
# define pe_flag_stop_rsc_orphans 0x00000100ULL
# define pe_flag_stop_action_orphans 0x00000200ULL
# define pe_flag_stop_everything 0x00000400ULL
# define pe_flag_start_failure_fatal 0x00001000ULL
# define pe_flag_remove_after_stop 0x00002000ULL
# define pe_flag_startup_fencing 0x00004000ULL
# define pe_flag_shutdown_lock 0x00008000ULL
# define pe_flag_startup_probes 0x00010000ULL
# define pe_flag_have_status 0x00020000ULL
# define pe_flag_have_remote_nodes 0x00040000ULL
# define pe_flag_quick_location 0x00100000ULL
# define pe_flag_sanitized 0x00200000ULL
# define pe_flag_stdout 0x00400000ULL
//! Don't count total, disabled and blocked resource instances
# define pe_flag_no_counts 0x00800000ULL
/*! Skip deprecated code that is kept solely for backward API compatibility.
* (Internal code should always set this.)
*/
# define pe_flag_no_compat 0x01000000ULL
struct pe_working_set_s {
xmlNode *input;
crm_time_t *now;
/* options extracted from the input */
char *dc_uuid;
pe_node_t *dc_node;
const char *stonith_action;
const char *placement_strategy;
unsigned long long flags;
int stonith_timeout;
enum pe_quorum_policy no_quorum_policy;
GHashTable *config_hash;
GHashTable *tickets;
// Actions for which there can be only one (e.g. fence nodeX)
GHashTable *singletons;
GListPtr nodes;
GListPtr resources;
GListPtr placement_constraints;
GListPtr ordering_constraints;
GListPtr colocation_constraints;
GListPtr ticket_constraints;
GListPtr actions;
xmlNode *failed;
xmlNode *op_defaults;
xmlNode *rsc_defaults;
/* stats */
int num_synapse;
int max_valid_nodes; //! Deprecated (will be removed in a future release)
int order_id;
int action_id;
/* final output */
xmlNode *graph;
GHashTable *template_rsc_sets;
const char *localhost;
GHashTable *tags;
int blocked_resources;
int disabled_resources;
GList *param_check; // History entries that need to be checked
GList *stop_needed; // Containers that need stop actions
time_t recheck_by; // Hint to controller to re-run scheduler by this time
int ninstances; // Total number of resource instances
guint shutdown_lock;// How long (seconds) to lock resources to shutdown node
int priority_fencing_delay; // Priority fencing delay
};
enum pe_check_parameters {
/* Clear fail count if parameters changed for un-expired start or monitor
* last_failure.
*/
pe_check_last_failure,
/* Clear fail count if parameters changed for start, monitor, promote, or
* migrate_from actions for active resources.
*/
pe_check_active,
};
struct pe_node_shared_s {
const char *id;
const char *uname;
enum node_type type;
/* @TODO convert these flags into a bitfield */
gboolean online;
gboolean standby;
gboolean standby_onfail;
gboolean pending;
gboolean unclean;
gboolean unseen;
gboolean shutdown;
gboolean expected_up;
gboolean is_dc;
gboolean maintenance;
gboolean rsc_discovery_enabled;
gboolean remote_requires_reset;
gboolean remote_was_fenced;
gboolean remote_maintenance; /* what the remote-rsc is thinking */
gboolean unpacked;
int num_resources;
pe_resource_t *remote_rsc;
GListPtr running_rsc; /* pe_resource_t* */
GListPtr allocated_rsc; /* pe_resource_t* */
GHashTable *attrs; /* char* => char* */
GHashTable *utilization;
GHashTable *digest_cache; //!< cache of calculated resource digests
int priority; // calculated based on the priority of resources running on the node
};
struct pe_node_s {
int weight;
gboolean fixed;
int count;
struct pe_node_shared_s *details;
int rsc_discover_mode;
};
# define pe_rsc_orphan 0x00000001ULL
# define pe_rsc_managed 0x00000002ULL
# define pe_rsc_block 0x00000004ULL
# define pe_rsc_orphan_container_filler 0x00000008ULL
# define pe_rsc_notify 0x00000010ULL
# define pe_rsc_unique 0x00000020ULL
# define pe_rsc_fence_device 0x00000040ULL
# define pe_rsc_promotable 0x00000080ULL
# define pe_rsc_provisional 0x00000100ULL
# define pe_rsc_allocating 0x00000200ULL
# define pe_rsc_merging 0x00000400ULL
+# define pe_rsc_stop 0x00001000ULL
# define pe_rsc_reload 0x00002000ULL
# define pe_rsc_allow_remote_remotes 0x00004000ULL
# define pe_rsc_failed 0x00010000ULL
# define pe_rsc_runnable 0x00040000ULL
# define pe_rsc_start_pending 0x00080000ULL
# define pe_rsc_starting 0x00100000ULL
# define pe_rsc_stopping 0x00200000ULL
# define pe_rsc_allow_migrate 0x00800000ULL
# define pe_rsc_failure_ignored 0x01000000ULL
# define pe_rsc_maintenance 0x04000000ULL
# define pe_rsc_is_container 0x08000000ULL
# define pe_rsc_needs_quorum 0x10000000ULL
# define pe_rsc_needs_fencing 0x20000000ULL
# define pe_rsc_needs_unfencing 0x40000000ULL
enum pe_graph_flags {
pe_graph_none = 0x00000,
pe_graph_updated_first = 0x00001,
pe_graph_updated_then = 0x00002,
pe_graph_disable = 0x00004,
};
/* *INDENT-OFF* */
enum pe_action_flags {
pe_action_pseudo = 0x00001,
pe_action_runnable = 0x00002,
pe_action_optional = 0x00004,
pe_action_print_always = 0x00008,
pe_action_have_node_attrs = 0x00010,
pe_action_implied_by_stonith = 0x00040,
pe_action_migrate_runnable = 0x00080,
pe_action_dumped = 0x00100,
pe_action_processed = 0x00200,
pe_action_clear = 0x00400,
pe_action_dangle = 0x00800,
/* This action requires one or more of its dependencies to be runnable.
* We use this to clear the runnable flag before checking dependencies.
*/
pe_action_requires_any = 0x01000,
pe_action_reschedule = 0x02000,
pe_action_tracking = 0x04000,
pe_action_dedup = 0x08000, //! Internal state tracking when creating graph
pe_action_dc = 0x10000, //! Action may run on DC instead of target
};
/* *INDENT-ON* */
struct pe_resource_s {
char *id;
char *clone_name;
xmlNode *xml;
xmlNode *orig_xml;
xmlNode *ops_xml;
pe_working_set_t *cluster;
pe_resource_t *parent;
enum pe_obj_types variant;
void *variant_opaque;
resource_object_functions_t *fns;
resource_alloc_functions_t *cmds;
enum rsc_recovery_type recovery_type;
// @TODO only pe_restart_restart is of interest, so merge into flags
enum pe_restart restart_type; //!< \deprecated will be removed in future release
int priority;
int stickiness;
int sort_index;
int failure_timeout;
int migration_threshold;
guint remote_reconnect_ms;
char *pending_task;
unsigned long long flags;
// @TODO merge these into flags
gboolean is_remote_node;
gboolean exclusive_discover;
//!@{
//! This field should be treated as internal to Pacemaker
GListPtr rsc_cons_lhs; // List of rsc_colocation_t*
GListPtr rsc_cons; // List of rsc_colocation_t*
GListPtr rsc_location; // List of pe__location_t*
GListPtr actions; // List of pe_action_t*
GListPtr rsc_tickets; // List of rsc_ticket*
//!@}
pe_node_t *allocated_to;
pe_node_t *partial_migration_target;
pe_node_t *partial_migration_source;
GListPtr running_on; /* pe_node_t* */
GHashTable *known_on; /* pe_node_t* */
GHashTable *allowed_nodes; /* pe_node_t* */
enum rsc_role_e role;
enum rsc_role_e next_role;
GHashTable *meta;
GHashTable *parameters;
GHashTable *utilization;
GListPtr children; /* pe_resource_t* */
GListPtr dangling_migrations; /* pe_node_t* */
pe_resource_t *container;
GListPtr fillers;
pe_node_t *pending_node; // Node on which pending_task is happening
pe_node_t *lock_node; // Resource is shutdown-locked to this node
time_t lock_time; // When shutdown lock started
#if ENABLE_VERSIONED_ATTRS
xmlNode *versioned_parameters;
#endif
};
#if ENABLE_VERSIONED_ATTRS
// Used as action->action_details if action->rsc is not NULL
typedef struct pe_rsc_action_details_s {
xmlNode *versioned_parameters;
xmlNode *versioned_meta;
} pe_rsc_action_details_t;
#endif
struct pe_action_s {
int id;
int priority;
pe_resource_t *rsc;
pe_node_t *node;
xmlNode *op_entry;
char *task;
char *uuid;
char *cancel_task;
char *reason;
enum pe_action_flags flags;
enum rsc_start_requirement needs;
enum action_fail_response on_fail;
enum rsc_role_e fail_role;
GHashTable *meta;
GHashTable *extra;
/*
* These two varables are associated with the constraint logic
* that involves first having one or more actions runnable before
* then allowing this action to execute.
*
* These varables are used with features such as 'clone-min' which
* requires at minimum X number of cloned instances to be running
* before an order dependency can run. Another option that uses
* this is 'require-all=false' in ordering constrants. This option
* says "only require one instance of a resource to start before
* allowing dependencies to start" -- basically, require-all=false is
* the same as clone-min=1.
*/
/* current number of known runnable actions in the before list. */
int runnable_before;
/* the number of "before" runnable actions required for this action
* to be considered runnable */
int required_runnable_before;
GListPtr actions_before; /* pe_action_wrapper_t* */
GListPtr actions_after; /* pe_action_wrapper_t* */
/* Some of the above fields could be moved to the details,
* except for API backward compatibility.
*/
void *action_details; // varies by type of action
};
typedef struct pe_ticket_s {
char *id;
gboolean granted;
time_t last_granted;
gboolean standby;
GHashTable *state;
} pe_ticket_t;
typedef struct pe_tag_s {
char *id;
GListPtr refs;
} pe_tag_t;
//! Internal tracking for transition graph creation
enum pe_link_state {
pe_link_not_dumped, //! Internal tracking for transition graph creation
pe_link_dumped, //! Internal tracking for transition graph creation
pe_link_dup, //! \deprecated No longer used by Pacemaker
};
enum pe_discover_e {
pe_discover_always = 0,
pe_discover_never,
pe_discover_exclusive,
};
/* *INDENT-OFF* */
enum pe_ordering {
pe_order_none = 0x0, /* deleted */
pe_order_optional = 0x1, /* pure ordering, nothing implied */
pe_order_apply_first_non_migratable = 0x2, /* Only apply this constraint's ordering if first is not migratable. */
pe_order_implies_first = 0x10, /* If 'then' is required, ensure 'first' is too */
pe_order_implies_then = 0x20, /* If 'first' is required, ensure 'then' is too */
pe_order_implies_first_master = 0x40, /* Imply 'first' is required when 'then' is required and then's rsc holds Master role. */
/* first requires then to be both runnable and migrate runnable. */
pe_order_implies_first_migratable = 0x80,
pe_order_runnable_left = 0x100, /* 'then' requires 'first' to be runnable */
pe_order_pseudo_left = 0x200, /* 'then' can only be pseudo if 'first' is runnable */
pe_order_implies_then_on_node = 0x400, /* If 'first' is required on 'nodeX',
* ensure instances of 'then' on 'nodeX' are too.
* Only really useful if 'then' is a clone and 'first' is not
*/
pe_order_probe = 0x800, /* If 'first->rsc' is
* - running but about to stop, ignore the constraint
* - otherwise, behave as runnable_left
*/
pe_order_restart = 0x1000, /* 'then' is runnable if 'first' is optional or runnable */
pe_order_stonith_stop = 0x2000, /* only applies if the action is non-pseudo */
pe_order_serialize_only = 0x4000, /* serialize */
pe_order_same_node = 0x8000, /* applies only if 'first' and 'then' are on same node */
pe_order_implies_first_printed = 0x10000, /* Like ..implies_first but only ensures 'first' is printed, not mandatory */
pe_order_implies_then_printed = 0x20000, /* Like ..implies_then but only ensures 'then' is printed, not mandatory */
pe_order_asymmetrical = 0x100000, /* Indicates asymmetrical one way ordering constraint. */
pe_order_load = 0x200000, /* Only relevant if... */
pe_order_one_or_more = 0x400000, /* 'then' is runnable only if one or more of its dependencies are too */
pe_order_anti_colocation = 0x800000,
pe_order_preserve = 0x1000000, /* Hack for breaking user ordering constraints with container resources */
pe_order_then_cancels_first = 0x2000000, // if 'then' becomes required, 'first' becomes optional
pe_order_trace = 0x4000000, /* test marker */
};
/* *INDENT-ON* */
typedef struct pe_action_wrapper_s {
enum pe_ordering type;
enum pe_link_state state;
pe_action_t *action;
} pe_action_wrapper_t;
#ifndef PCMK__NO_COMPAT
/* Everything here is deprecated and kept only for public API backward
* compatibility. It will be moved to compatibility.h when 2.1.0 is released.
*/
//!< \deprecated Use pe_action_t instead
typedef struct pe_action_s action_t;
//!< \deprecated Use pe_action_wrapper_t instead
typedef struct pe_action_wrapper_s action_wrapper_t;
//!< \deprecated Use pe_node_t instead
typedef struct pe_node_s node_t;
//!< \deprecated Use enum pe_quorum_policy instead
typedef enum pe_quorum_policy no_quorum_policy_t;
//!< \deprecated use pe_resource_t instead
typedef struct pe_resource_s resource_t;
//!< \deprecated Use pe_tag_t instead
typedef struct pe_tag_s tag_t;
//!< \deprecated Use pe_ticket_t instead
typedef struct pe_ticket_s ticket_t;
#endif
#ifdef __cplusplus
}
#endif
#endif // PE_TYPES__H
diff --git a/lib/common/options.c b/lib/common/options.c
index 939964279b..9e041c9b30 100644
--- a/lib/common/options.c
+++ b/lib/common/options.c
@@ -1,605 +1,606 @@
/*
* Copyright 2004-2020 the Pacemaker project contributors
*
* The version control history for this file may have further details.
*
* This source code is licensed under the GNU Lesser General Public License
* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
*/
#ifndef _GNU_SOURCE
# define _GNU_SOURCE
#endif
#include
#include
#include
#include
#include
#include
#ifdef HAVE_GETOPT_H
# include
#endif
#include
/*
* Command-line option handling
*/
static char *crm_short_options = NULL;
static pcmk__cli_option_t *crm_long_options = NULL;
static const char *crm_app_description = NULL;
static const char *crm_app_usage = NULL;
void
pcmk__cli_option_cleanup()
{
free(crm_short_options);
crm_short_options = NULL;
}
static struct option *
create_long_opts(pcmk__cli_option_t *long_options)
{
struct option *long_opts = NULL;
#ifdef HAVE_GETOPT_H
int index = 0, lpc = 0;
/*
* A previous, possibly poor, choice of '?' as the short form of --help
* means that getopt_long() returns '?' for both --help and for "unknown option"
*
* This dummy entry allows us to differentiate between the two in
* pcmk__next_cli_option() and exit with the correct error code.
*/
long_opts = realloc_safe(long_opts, (index + 1) * sizeof(struct option));
long_opts[index].name = "__dummmy__";
long_opts[index].has_arg = 0;
long_opts[index].flag = 0;
long_opts[index].val = '_';
index++;
// cppcheck seems not to understand the abort-logic in realloc_safe
// cppcheck-suppress memleak
for (lpc = 0; long_options[lpc].name != NULL; lpc++) {
if (long_options[lpc].name[0] == '-') {
continue;
}
long_opts = realloc_safe(long_opts, (index + 1) * sizeof(struct option));
/*fprintf(stderr, "Creating %d %s = %c\n", index,
* long_options[lpc].name, long_options[lpc].val); */
long_opts[index].name = long_options[lpc].name;
long_opts[index].has_arg = long_options[lpc].has_arg;
long_opts[index].flag = long_options[lpc].flag;
long_opts[index].val = long_options[lpc].val;
index++;
}
/* Now create the list terminator */
long_opts = realloc_safe(long_opts, (index + 1) * sizeof(struct option));
long_opts[index].name = NULL;
long_opts[index].has_arg = 0;
long_opts[index].flag = 0;
long_opts[index].val = 0;
#endif
return long_opts;
}
/*!
* \internal
* \brief Define the command-line options a daemon or tool accepts
*
* \param[in] short_options getopt(3)-style short option list
* \param[in] app_usage summary of how command is invoked (for help)
* \param[in] long_options definition of options accepted
* \param[in] app_desc brief command description (for help)
*/
void
pcmk__set_cli_options(const char *short_options, const char *app_usage,
pcmk__cli_option_t *long_options, const char *app_desc)
{
if (short_options) {
crm_short_options = strdup(short_options);
} else if (long_options) {
int lpc = 0;
int opt_string_len = 0;
char *local_short_options = NULL;
for (lpc = 0; long_options[lpc].name != NULL; lpc++) {
if (long_options[lpc].val && long_options[lpc].val != '-' && long_options[lpc].val < UCHAR_MAX) {
local_short_options = realloc_safe(local_short_options, opt_string_len + 4);
local_short_options[opt_string_len++] = long_options[lpc].val;
/* getopt(3) says: Two colons mean an option takes an optional arg; */
if (long_options[lpc].has_arg == optional_argument) {
local_short_options[opt_string_len++] = ':';
}
if (long_options[lpc].has_arg >= required_argument) {
local_short_options[opt_string_len++] = ':';
}
local_short_options[opt_string_len] = 0;
}
}
crm_short_options = local_short_options;
crm_trace("Generated short option string: '%s'", local_short_options);
}
if (long_options) {
crm_long_options = long_options;
}
if (app_desc) {
crm_app_description = app_desc;
}
if (app_usage) {
crm_app_usage = app_usage;
}
}
int
pcmk__next_cli_option(int argc, char **argv, int *index, const char **longname)
{
#ifdef HAVE_GETOPT_H
static struct option *long_opts = NULL;
if (long_opts == NULL && crm_long_options) {
long_opts = create_long_opts(crm_long_options);
}
*index = 0;
if (long_opts) {
int flag = getopt_long(argc, argv, crm_short_options, long_opts, index);
switch (flag) {
case 0:
if (long_opts[*index].val) {
return long_opts[*index].val;
} else if (longname) {
*longname = long_opts[*index].name;
} else {
crm_notice("Unhandled option --%s", long_opts[*index].name);
return flag;
}
case -1: /* End of option processing */
break;
case ':':
crm_trace("Missing argument");
pcmk__cli_help('?', CRM_EX_USAGE);
break;
case '?':
pcmk__cli_help('?', (*index? CRM_EX_OK : CRM_EX_USAGE));
break;
}
return flag;
}
#endif
if (crm_short_options) {
return getopt(argc, argv, crm_short_options);
}
return -1;
}
void
pcmk__cli_help(char cmd, crm_exit_t exit_code)
{
int i = 0;
FILE *stream = (exit_code ? stderr : stdout);
if (cmd == 'v' || cmd == '$') {
fprintf(stream, "Pacemaker %s\n", PACEMAKER_VERSION);
fprintf(stream, "Written by Andrew Beekhof\n");
goto out;
}
if (cmd == '!') {
fprintf(stream, "Pacemaker %s (Build: %s): %s\n", PACEMAKER_VERSION, BUILD_VERSION, CRM_FEATURES);
goto out;
}
fprintf(stream, "%s - %s\n", crm_system_name, crm_app_description);
if (crm_app_usage) {
fprintf(stream, "Usage: %s %s\n", crm_system_name, crm_app_usage);
}
if (crm_long_options) {
fprintf(stream, "Options:\n");
for (i = 0; crm_long_options[i].name != NULL; i++) {
if (crm_long_options[i].flags & pcmk__option_hidden) {
} else if (crm_long_options[i].flags & pcmk__option_paragraph) {
fprintf(stream, "%s\n\n", crm_long_options[i].desc);
} else if (crm_long_options[i].flags & pcmk__option_example) {
fprintf(stream, "\t#%s\n\n", crm_long_options[i].desc);
} else if (crm_long_options[i].val == '-' && crm_long_options[i].desc) {
fprintf(stream, "%s\n", crm_long_options[i].desc);
} else {
/* is val printable as char ? */
if (crm_long_options[i].val && crm_long_options[i].val <= UCHAR_MAX) {
fprintf(stream, " -%c,", crm_long_options[i].val);
} else {
fputs(" ", stream);
}
fprintf(stream, " --%s%s\t%s\n", crm_long_options[i].name,
crm_long_options[i].has_arg == optional_argument ? "[=value]" :
crm_long_options[i].has_arg == required_argument ? "=value" : "",
crm_long_options[i].desc ? crm_long_options[i].desc : "");
}
}
} else if (crm_short_options) {
fprintf(stream, "Usage: %s - %s\n", crm_system_name, crm_app_description);
for (i = 0; crm_short_options[i] != 0; i++) {
int has_arg = no_argument /* 0 */;
if (crm_short_options[i + 1] == ':') {
if (crm_short_options[i + 2] == ':')
has_arg = optional_argument /* 2 */;
else
has_arg = required_argument /* 1 */;
}
fprintf(stream, " -%c %s\n", crm_short_options[i],
has_arg == optional_argument ? "[value]" :
has_arg == required_argument ? "{value}" : "");
i += has_arg;
}
}
fprintf(stream, "\nReport bugs to %s\n", PACKAGE_BUGREPORT);
out:
crm_exit(exit_code);
while(1); // above does not return
}
/*
* Environment variable option handling
*/
/*!
* \internal
* \brief Get the value of a Pacemaker environment variable option
*
* If an environment variable option is set, with either a PCMK_ or (for
* backward compatibility) HA_ prefix, log and return the value.
*
* \param[in] option Environment variable name (without prefix)
*
* \return Value of environment variable option
*/
const char *
pcmk__env_option(const char *option)
{
char env_name[NAME_MAX];
const char *value = NULL;
snprintf(env_name, NAME_MAX, "PCMK_%s", option);
value = getenv(env_name);
if (value != NULL) {
crm_trace("Found %s = %s", env_name, value);
return value;
}
snprintf(env_name, NAME_MAX, "HA_%s", option);
value = getenv(env_name);
if (value != NULL) {
crm_trace("Found %s = %s", env_name, value);
return value;
}
crm_trace("Nothing found for %s", option);
return NULL;
}
/*!
* \brief Set or unset a Pacemaker environment variable option
*
* Set an environment variable option with both a PCMK_ and (for
* backward compatibility) HA_ prefix.
*
* \param[in] option Environment variable name (without prefix)
* \param[in] value New value (or NULL to unset)
*/
void
pcmk__set_env_option(const char *option, const char *value)
{
char env_name[NAME_MAX];
snprintf(env_name, NAME_MAX, "PCMK_%s", option);
if (value) {
crm_trace("Setting %s to %s", env_name, value);
setenv(env_name, value, 1);
} else {
crm_trace("Unsetting %s", env_name);
unsetenv(env_name);
}
snprintf(env_name, NAME_MAX, "HA_%s", option);
if (value) {
crm_trace("Setting %s to %s", env_name, value);
setenv(env_name, value, 1);
} else {
crm_trace("Unsetting %s", env_name);
unsetenv(env_name);
}
}
/*!
* \internal
* \brief Check whether Pacemaker environment variable option is enabled
*
* Given a Pacemaker environment variable option that can either be boolean
* or a list of daemon names, return true if the option is enabled for a given
* daemon.
*
* \param[in] daemon Daemon name
* \param[in] option Pacemaker environment variable name
*
* \return true if variable is enabled for daemon, otherwise false
*/
bool
pcmk__env_option_enabled(const char *daemon, const char *option)
{
const char *value = pcmk__env_option(option);
return (value != NULL) && (crm_is_true(value) || strstr(value, daemon));
}
/*
* Cluster option handling
*/
bool
pcmk__valid_interval_spec(const char *value)
{
(void) crm_parse_interval_spec(value);
return errno == 0;
}
bool
pcmk__valid_boolean(const char *value)
{
int tmp;
return crm_str_to_boolean(value, &tmp) == 1;
}
bool
pcmk__valid_number(const char *value)
{
if (value == NULL) {
return false;
} else if (pcmk_str_is_minus_infinity(value) ||
pcmk_str_is_infinity(value)) {
return true;
}
errno = 0;
crm_parse_ll(value, NULL);
return errno == 0;
}
bool
pcmk__valid_positive_number(const char *value)
{
return pcmk_str_is_infinity(value) ||
(crm_parse_ll(value, NULL) > 0);
}
bool
pcmk__valid_quorum(const char *value)
{
return safe_str_eq(value, "stop")
|| safe_str_eq(value, "freeze")
|| safe_str_eq(value, "ignore")
+ || safe_str_eq(value, "demote")
|| safe_str_eq(value, "suicide");
}
bool
pcmk__valid_script(const char *value)
{
struct stat st;
if (safe_str_eq(value, "/dev/null")) {
return true;
}
if (stat(value, &st) != 0) {
crm_err("Script %s does not exist", value);
return false;
}
if (S_ISREG(st.st_mode) == 0) {
crm_err("Script %s is not a regular file", value);
return false;
}
if ((st.st_mode & (S_IXUSR | S_IXGRP)) == 0) {
crm_err("Script %s is not executable", value);
return false;
}
return true;
}
bool
pcmk__valid_utilization(const char *value)
{
char *end = NULL;
long number = strtol(value, &end, 10);
if (end && (end[0] != '%')) {
return false;
}
return number >= 0;
}
/*!
* \internal
* \brief Check a table of configured options for a particular option
*
* \param[in] options Name/value pairs for configured options
* \param[in] validate If not NULL, validator function for option value
* \param[in] name Option name to look for
* \param[in] old_name Alternative option name to look for
* \param[in] def_value Default to use if option not configured
*
* \return Option value (from supplied options table or default value)
*/
static const char *
cluster_option_value(GHashTable *options, bool (*validate)(const char *),
const char *name, const char *old_name,
const char *def_value)
{
const char *value = NULL;
char *new_value = NULL;
CRM_ASSERT(name != NULL);
if (options) {
value = g_hash_table_lookup(options, name);
if ((value == NULL) && old_name) {
value = g_hash_table_lookup(options, old_name);
if (value != NULL) {
pcmk__config_warn("Support for legacy name '%s' for cluster "
"option '%s' is deprecated and will be "
"removed in a future release",
old_name, name);
// Inserting copy with current name ensures we only warn once
new_value = strdup(value);
g_hash_table_insert(options, strdup(name), new_value);
value = new_value;
}
}
if (value && validate && (validate(value) == FALSE)) {
pcmk__config_err("Using default value for cluster option '%s' "
"because '%s' is invalid", name, value);
value = NULL;
}
if (value) {
return value;
}
}
// No value found, use default
value = def_value;
if (value == NULL) {
crm_trace("No value or default provided for cluster option '%s'",
name);
return NULL;
}
if (validate) {
CRM_CHECK(validate(value) != FALSE,
crm_err("Bug: default value for cluster option '%s' is invalid", name);
return NULL);
}
crm_trace("Using default value '%s' for cluster option '%s'",
value, name);
if (options) {
new_value = strdup(value);
g_hash_table_insert(options, strdup(name), new_value);
value = new_value;
}
return value;
}
/*!
* \internal
* \brief Get the value of a cluster option
*
* \param[in] options Name/value pairs for configured options
* \param[in] option_list Possible cluster options
* \param[in] name (Primary) option name to look for
*
* \return Option value
*/
const char *
pcmk__cluster_option(GHashTable *options, pcmk__cluster_option_t *option_list,
int len, const char *name)
{
const char *value = NULL;
for (int lpc = 0; lpc < len; lpc++) {
if (safe_str_eq(name, option_list[lpc].name)) {
value = cluster_option_value(options, option_list[lpc].is_valid,
option_list[lpc].name,
option_list[lpc].alt_name,
option_list[lpc].default_value);
return value;
}
}
CRM_CHECK(FALSE, crm_err("Bug: looking for unknown option '%s'", name));
return NULL;
}
void
pcmk__print_option_metadata(const char *name, const char *version,
const char *desc_short, const char *desc_long,
pcmk__cluster_option_t *option_list, int len)
{
int lpc = 0;
fprintf(stdout, ""
"\n"
"\n"
" %s\n"
" %s\n"
" %s\n"
" \n", name, version, desc_long, desc_short);
for (lpc = 0; lpc < len; lpc++) {
if ((option_list[lpc].description_long == NULL)
&& (option_list[lpc].description_short == NULL)) {
continue;
}
fprintf(stdout, " \n"
" %s\n"
" \n"
" %s%s%s\n"
" \n",
option_list[lpc].name,
option_list[lpc].description_short,
option_list[lpc].type,
option_list[lpc].default_value,
option_list[lpc].description_long?
option_list[lpc].description_long :
option_list[lpc].description_short,
(option_list[lpc].values? " Allowed values: " : ""),
(option_list[lpc].values? option_list[lpc].values : ""));
}
fprintf(stdout, " \n\n");
}
void
pcmk__validate_cluster_options(GHashTable *options,
pcmk__cluster_option_t *option_list, int len)
{
for (int lpc = 0; lpc < len; lpc++) {
cluster_option_value(options, option_list[lpc].is_valid,
option_list[lpc].name,
option_list[lpc].alt_name,
option_list[lpc].default_value);
}
}
diff --git a/lib/pacemaker/pcmk_sched_native.c b/lib/pacemaker/pcmk_sched_native.c
index bd8a0b5418..4e3bd7c714 100644
--- a/lib/pacemaker/pcmk_sched_native.c
+++ b/lib/pacemaker/pcmk_sched_native.c
@@ -1,3418 +1,3465 @@
/*
* Copyright 2004-2020 the Pacemaker project contributors
*
* The version control history for this file may have further details.
*
* This source code is licensed under the GNU General Public License version 2
* or later (GPLv2+) WITHOUT ANY WARRANTY.
*/
#include
#include
#include
#include
#include
// The controller removes the resource from the CIB, making this redundant
// #define DELETE_THEN_REFRESH 1
#define INFINITY_HACK (INFINITY * -100)
#define VARIANT_NATIVE 1
#include
static void Recurring(pe_resource_t *rsc, pe_action_t *start, pe_node_t *node,
pe_working_set_t *data_set);
static void RecurringOp(pe_resource_t *rsc, pe_action_t *start, pe_node_t *node,
xmlNode *operation, pe_working_set_t *data_set);
static void Recurring_Stopped(pe_resource_t *rsc, pe_action_t *start, pe_node_t *node,
pe_working_set_t *data_set);
static void RecurringOp_Stopped(pe_resource_t *rsc, pe_action_t *start, pe_node_t *node,
xmlNode *operation, pe_working_set_t *data_set);
void ReloadRsc(pe_resource_t * rsc, pe_node_t *node, pe_working_set_t * data_set);
gboolean DeleteRsc(pe_resource_t * rsc, pe_node_t * node, gboolean optional, pe_working_set_t * data_set);
gboolean StopRsc(pe_resource_t * rsc, pe_node_t * next, gboolean optional, pe_working_set_t * data_set);
gboolean StartRsc(pe_resource_t * rsc, pe_node_t * next, gboolean optional, pe_working_set_t * data_set);
gboolean DemoteRsc(pe_resource_t * rsc, pe_node_t * next, gboolean optional, pe_working_set_t * data_set);
gboolean PromoteRsc(pe_resource_t * rsc, pe_node_t * next, gboolean optional,
pe_working_set_t * data_set);
gboolean RoleError(pe_resource_t * rsc, pe_node_t * next, gboolean optional, pe_working_set_t * data_set);
gboolean NullOp(pe_resource_t * rsc, pe_node_t * next, gboolean optional, pe_working_set_t * data_set);
-/* *INDENT-OFF* */
-enum rsc_role_e rsc_state_matrix[RSC_ROLE_MAX][RSC_ROLE_MAX] = {
-/* Current State */
-/* Next State: Unknown Stopped Started Slave Master */
+/* This array says what the *next* role should be when transitioning from one
+ * role to another. For example going from Stopped to Master, the next role is
+ * RSC_ROLE_SLAVE, because the resource must be started before being promoted.
+ * The current state then becomes Started, which is fed into this array again,
+ * giving a next role of RSC_ROLE_MASTER.
+ */
+static enum rsc_role_e rsc_state_matrix[RSC_ROLE_MAX][RSC_ROLE_MAX] = {
+ /* Current state Next state*/
+ /* Unknown Stopped Started Slave Master */
/* Unknown */ { RSC_ROLE_UNKNOWN, RSC_ROLE_STOPPED, RSC_ROLE_STOPPED, RSC_ROLE_STOPPED, RSC_ROLE_STOPPED, },
/* Stopped */ { RSC_ROLE_STOPPED, RSC_ROLE_STOPPED, RSC_ROLE_STARTED, RSC_ROLE_SLAVE, RSC_ROLE_SLAVE, },
/* Started */ { RSC_ROLE_STOPPED, RSC_ROLE_STOPPED, RSC_ROLE_STARTED, RSC_ROLE_SLAVE, RSC_ROLE_MASTER, },
- /* Slave */ { RSC_ROLE_STOPPED, RSC_ROLE_STOPPED, RSC_ROLE_STOPPED, RSC_ROLE_SLAVE, RSC_ROLE_MASTER, },
- /* Master */ { RSC_ROLE_STOPPED, RSC_ROLE_SLAVE, RSC_ROLE_SLAVE, RSC_ROLE_SLAVE, RSC_ROLE_MASTER, },
+ /* Slave */ { RSC_ROLE_STOPPED, RSC_ROLE_STOPPED, RSC_ROLE_STOPPED, RSC_ROLE_SLAVE, RSC_ROLE_MASTER, },
+ /* Master */ { RSC_ROLE_STOPPED, RSC_ROLE_SLAVE, RSC_ROLE_SLAVE, RSC_ROLE_SLAVE, RSC_ROLE_MASTER, },
};
-gboolean (*rsc_action_matrix[RSC_ROLE_MAX][RSC_ROLE_MAX])(pe_resource_t*,pe_node_t*,gboolean,pe_working_set_t*) = {
-/* Current State */
-/* Next State: Unknown Stopped Started Slave Master */
- /* Unknown */ { RoleError, StopRsc, RoleError, RoleError, RoleError, },
- /* Stopped */ { RoleError, NullOp, StartRsc, StartRsc, RoleError, },
- /* Started */ { RoleError, StopRsc, NullOp, NullOp, PromoteRsc, },
- /* Slave */ { RoleError, StopRsc, StopRsc, NullOp, PromoteRsc, },
- /* Master */ { RoleError, DemoteRsc, DemoteRsc, DemoteRsc, NullOp, },
+typedef gboolean (*rsc_transition_fn)(pe_resource_t *rsc, pe_node_t *next,
+ gboolean optional,
+ pe_working_set_t *data_set);
+
+// This array picks the function needed to transition from one role to another
+static rsc_transition_fn rsc_action_matrix[RSC_ROLE_MAX][RSC_ROLE_MAX] = {
+ /* Current state Next state */
+ /* Unknown Stopped Started Slave Master */
+ /* Unknown */ { RoleError, StopRsc, RoleError, RoleError, RoleError, },
+ /* Stopped */ { RoleError, NullOp, StartRsc, StartRsc, RoleError, },
+ /* Started */ { RoleError, StopRsc, NullOp, NullOp, PromoteRsc, },
+ /* Slave */ { RoleError, StopRsc, StopRsc, NullOp, PromoteRsc, },
+ /* Master */ { RoleError, DemoteRsc, DemoteRsc, DemoteRsc, NullOp , },
};
-/* *INDENT-ON* */
static gboolean
native_choose_node(pe_resource_t * rsc, pe_node_t * prefer, pe_working_set_t * data_set)
{
GListPtr nodes = NULL;
pe_node_t *chosen = NULL;
pe_node_t *best = NULL;
int multiple = 1;
int length = 0;
gboolean result = FALSE;
process_utilization(rsc, &prefer, data_set);
if (is_not_set(rsc->flags, pe_rsc_provisional)) {
return rsc->allocated_to ? TRUE : FALSE;
}
// Sort allowed nodes by weight
if (rsc->allowed_nodes) {
length = g_hash_table_size(rsc->allowed_nodes);
}
if (length > 0) {
nodes = g_hash_table_get_values(rsc->allowed_nodes);
nodes = sort_nodes_by_weight(nodes, pe__current_node(rsc), data_set);
// First node in sorted list has the best score
best = g_list_nth_data(nodes, 0);
}
if (prefer && nodes) {
chosen = g_hash_table_lookup(rsc->allowed_nodes, prefer->details->id);
if (chosen == NULL) {
pe_rsc_trace(rsc, "Preferred node %s for %s was unknown",
prefer->details->uname, rsc->id);
/* Favor the preferred node as long as its weight is at least as good as
* the best allowed node's.
*
* An alternative would be to favor the preferred node even if the best
* node is better, when the best node's weight is less than INFINITY.
*/
} else if ((chosen->weight < 0) || (chosen->weight < best->weight)) {
pe_rsc_trace(rsc, "Preferred node %s for %s was unsuitable",
chosen->details->uname, rsc->id);
chosen = NULL;
} else if (!can_run_resources(chosen)) {
pe_rsc_trace(rsc, "Preferred node %s for %s was unavailable",
chosen->details->uname, rsc->id);
chosen = NULL;
} else {
pe_rsc_trace(rsc,
"Chose preferred node %s for %s (ignoring %d candidates)",
chosen->details->uname, rsc->id, length);
}
}
if ((chosen == NULL) && nodes) {
/* Either there is no preferred node, or the preferred node is not
* available, but there are other nodes allowed to run the resource.
*/
chosen = best;
pe_rsc_trace(rsc, "Chose node %s for %s from %d candidates",
chosen ? chosen->details->uname : "", rsc->id, length);
if (!pe_rsc_is_unique_clone(rsc->parent)
&& chosen && (chosen->weight > 0) && can_run_resources(chosen)) {
/* If the resource is already running on a node, prefer that node if
* it is just as good as the chosen node.
*
* We don't do this for unique clone instances, because
* distribute_children() has already assigned instances to their
* running nodes when appropriate, and if we get here, we don't want
* remaining unallocated instances to prefer a node that's already
* running another instance.
*/
pe_node_t *running = pe__current_node(rsc);
if (running && (can_run_resources(running) == FALSE)) {
pe_rsc_trace(rsc, "Current node for %s (%s) can't run resources",
rsc->id, running->details->uname);
} else if (running) {
for (GList *iter = nodes->next; iter; iter = iter->next) {
pe_node_t *tmp = (pe_node_t *) iter->data;
if (tmp->weight != chosen->weight) {
// The nodes are sorted by weight, so no more are equal
break;
}
if (tmp->details == running->details) {
// Scores are equal, so prefer the current node
chosen = tmp;
}
multiple++;
}
}
}
}
if (multiple > 1) {
static char score[33];
int log_level = (chosen->weight >= INFINITY)? LOG_WARNING : LOG_INFO;
score2char_stack(chosen->weight, score, sizeof(score));
do_crm_log(log_level,
"Chose node %s for %s from %d nodes with score %s",
chosen->details->uname, rsc->id, multiple, score);
}
result = native_assign_node(rsc, nodes, chosen, FALSE);
g_list_free(nodes);
return result;
}
/*!
* \internal
* \brief Find score of highest-scored node that matches colocation attribute
*
* \param[in] rsc Resource whose allowed nodes should be searched
* \param[in] attr Colocation attribute name (must not be NULL)
* \param[in] value Colocation attribute value to require
*/
static int
best_node_score_matching_attr(const pe_resource_t *rsc, const char *attr,
const char *value)
{
GHashTableIter iter;
pe_node_t *node = NULL;
int best_score = -INFINITY;
const char *best_node = NULL;
// Find best allowed node with matching attribute
g_hash_table_iter_init(&iter, rsc->allowed_nodes);
while (g_hash_table_iter_next(&iter, NULL, (void **) &node)) {
if ((node->weight > best_score) && can_run_resources(node)
&& safe_str_eq(value, pe_node_attribute_raw(node, attr))) {
best_score = node->weight;
best_node = node->details->uname;
}
}
if (safe_str_neq(attr, CRM_ATTR_UNAME)) {
if (best_node == NULL) {
crm_info("No allowed node for %s matches node attribute %s=%s",
rsc->id, attr, value);
} else {
crm_info("Allowed node %s for %s had best score (%d) "
"of those matching node attribute %s=%s",
best_node, rsc->id, best_score, attr, value);
}
}
return best_score;
}
/*!
* \internal
* \brief Add resource's colocation matches to current node allocation scores
*
* For each node in a given table, if any of a given resource's allowed nodes
* have a matching value for the colocation attribute, add the highest of those
* nodes' scores to the node's score.
*
* \param[in,out] nodes Hash table of nodes with allocation scores so far
* \param[in] rsc Resource whose allowed nodes should be compared
* \param[in] attr Colocation attribute that must match (NULL for default)
* \param[in] factor Factor by which to multiply scores being added
* \param[in] only_positive Whether to add only positive scores
*/
static void
add_node_scores_matching_attr(GHashTable *nodes, const pe_resource_t *rsc,
const char *attr, float factor,
bool only_positive)
{
GHashTableIter iter;
pe_node_t *node = NULL;
if (attr == NULL) {
attr = CRM_ATTR_UNAME;
}
// Iterate through each node
g_hash_table_iter_init(&iter, nodes);
while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
float weight_f = 0;
int weight = 0;
int score = 0;
int new_score = 0;
score = best_node_score_matching_attr(rsc, attr,
pe_node_attribute_raw(node, attr));
if ((factor < 0) && (score < 0)) {
/* Negative preference for a node with a negative score
* should not become a positive preference.
*
* @TODO Consider filtering only if weight is -INFINITY
*/
crm_trace("%s: Filtering %d + %f * %d (double negative disallowed)",
node->details->uname, node->weight, factor, score);
continue;
}
if (node->weight == INFINITY_HACK) {
crm_trace("%s: Filtering %d + %f * %d (node was marked unusable)",
node->details->uname, node->weight, factor, score);
continue;
}
weight_f = factor * score;
// Round the number; see http://c-faq.com/fp/round.html
weight = (int) ((weight_f < 0)? (weight_f - 0.5) : (weight_f + 0.5));
/* Small factors can obliterate the small scores that are often actually
* used in configurations. If the score and factor are nonzero, ensure
* that the result is nonzero as well.
*/
if ((weight == 0) && (score != 0)) {
if (factor > 0.0) {
weight = 1;
} else if (factor < 0.0) {
weight = -1;
}
}
new_score = pe__add_scores(weight, node->weight);
if (only_positive && (new_score < 0) && (node->weight > 0)) {
crm_trace("%s: Filtering %d + %f * %d = %d "
"(negative disallowed, marking node unusable)",
node->details->uname, node->weight, factor, score,
new_score);
node->weight = INFINITY_HACK;
continue;
}
if (only_positive && (new_score < 0) && (node->weight == 0)) {
crm_trace("%s: Filtering %d + %f * %d = %d (negative disallowed)",
node->details->uname, node->weight, factor, score,
new_score);
continue;
}
crm_trace("%s: %d + %f * %d = %d", node->details->uname,
node->weight, factor, score, new_score);
node->weight = new_score;
}
}
static inline bool
is_nonempty_group(pe_resource_t *rsc)
{
return rsc && (rsc->variant == pe_group) && (rsc->children != NULL);
}
/*!
* \internal
* \brief Incorporate colocation constraint scores into node weights
*
* \param[in,out] rsc Resource being placed
* \param[in] rhs ID of 'with' resource
* \param[in,out] nodes Nodes, with scores as of this point
* \param[in] attr Colocation attribute (ID by default)
* \param[in] factor Incorporate scores multiplied by this factor
* \param[in] flags Bitmask of enum pe_weights values
*
* \return Nodes, with scores modified by this constraint
* \note This function assumes ownership of the nodes argument. The caller
* should free the returned copy rather than the original.
*/
GHashTable *
pcmk__native_merge_weights(pe_resource_t *rsc, const char *rhs,
GHashTable *nodes, const char *attr, float factor,
uint32_t flags)
{
GHashTable *work = NULL;
// Avoid infinite recursion
if (is_set(rsc->flags, pe_rsc_merging)) {
pe_rsc_info(rsc, "%s: Breaking dependency loop at %s", rhs, rsc->id);
return nodes;
}
set_bit(rsc->flags, pe_rsc_merging);
if (is_set(flags, pe_weights_init)) {
if (is_nonempty_group(rsc)) {
GList *last = g_list_last(rsc->children);
pe_resource_t *last_rsc = last->data;
pe_rsc_trace(rsc, "%s: Merging scores from group %s "
"using last member %s (at %.6f)",
rhs, rsc->id, last_rsc->id, factor);
work = pcmk__native_merge_weights(last_rsc, rhs, NULL, attr, factor,
flags);
} else {
work = pcmk__copy_node_table(rsc->allowed_nodes);
}
clear_bit(flags, pe_weights_init);
} else if (is_nonempty_group(rsc)) {
/* The first member of the group will recursively incorporate any
* constraints involving other members (including the group internal
* colocation).
*
* @TODO The indirect colocations from the dependent group's other
* members will be incorporated at full strength rather than by
* factor, so the group's combined stickiness will be treated as
* (factor + (#members - 1)) * stickiness. It is questionable what
* the right approach should be.
*/
pe_rsc_trace(rsc, "%s: Merging scores from first member of group %s "
"(at %.6f)", rhs, rsc->id, factor);
work = pcmk__copy_node_table(nodes);
work = pcmk__native_merge_weights(rsc->children->data, rhs, work, attr,
factor, flags);
} else {
pe_rsc_trace(rsc, "%s: Merging scores from %s (at %.6f)",
rhs, rsc->id, factor);
work = pcmk__copy_node_table(nodes);
add_node_scores_matching_attr(work, rsc, attr, factor,
is_set(flags, pe_weights_positive));
}
if (can_run_any(work)) {
GListPtr gIter = NULL;
int multiplier = (factor < 0)? -1 : 1;
if (is_set(flags, pe_weights_forward)) {
gIter = rsc->rsc_cons;
pe_rsc_trace(rsc,
"Checking additional %d optional '%s with' constraints",
g_list_length(gIter), rsc->id);
} else if (is_nonempty_group(rsc)) {
pe_resource_t *last_rsc = g_list_last(rsc->children)->data;
gIter = last_rsc->rsc_cons_lhs;
pe_rsc_trace(rsc, "Checking additional %d optional 'with group %s' "
"constraints using last member %s",
g_list_length(gIter), rsc->id, last_rsc->id);
} else {
gIter = rsc->rsc_cons_lhs;
pe_rsc_trace(rsc,
"Checking additional %d optional 'with %s' constraints",
g_list_length(gIter), rsc->id);
}
for (; gIter != NULL; gIter = gIter->next) {
pe_resource_t *other = NULL;
rsc_colocation_t *constraint = (rsc_colocation_t *) gIter->data;
if (constraint->score == 0) {
continue;
}
if (is_set(flags, pe_weights_forward)) {
other = constraint->rsc_rh;
} else {
other = constraint->rsc_lh;
}
pe_rsc_trace(rsc, "Optionally merging score of '%s' constraint (%s with %s)",
constraint->id, constraint->rsc_lh->id,
constraint->rsc_rh->id);
work = pcmk__native_merge_weights(other, rhs, work,
constraint->node_attribute,
multiplier * constraint->score / (float) INFINITY,
flags|pe_weights_rollback);
pe__show_node_weights(true, NULL, rhs, work);
}
} else if (is_set(flags, pe_weights_rollback)) {
pe_rsc_info(rsc, "%s: Rolling back optional scores from %s",
rhs, rsc->id);
g_hash_table_destroy(work);
clear_bit(rsc->flags, pe_rsc_merging);
return nodes;
}
if (is_set(flags, pe_weights_positive)) {
pe_node_t *node = NULL;
GHashTableIter iter;
g_hash_table_iter_init(&iter, work);
while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
if (node->weight == INFINITY_HACK) {
node->weight = 1;
}
}
}
if (nodes) {
g_hash_table_destroy(nodes);
}
clear_bit(rsc->flags, pe_rsc_merging);
return work;
}
static inline bool
node_has_been_unfenced(pe_node_t *node)
{
const char *unfenced = pe_node_attribute_raw(node, CRM_ATTR_UNFENCED);
return unfenced && strcmp("0", unfenced);
}
static inline bool
is_unfence_device(pe_resource_t *rsc, pe_working_set_t *data_set)
{
return is_set(rsc->flags, pe_rsc_fence_device)
&& is_set(data_set->flags, pe_flag_enable_unfencing);
}
pe_node_t *
pcmk__native_allocate(pe_resource_t *rsc, pe_node_t *prefer,
pe_working_set_t *data_set)
{
GListPtr gIter = NULL;
if (rsc->parent && is_not_set(rsc->parent->flags, pe_rsc_allocating)) {
/* never allocate children on their own */
pe_rsc_debug(rsc, "Escalating allocation of %s to its parent: %s", rsc->id,
rsc->parent->id);
rsc->parent->cmds->allocate(rsc->parent, prefer, data_set);
}
if (is_not_set(rsc->flags, pe_rsc_provisional)) {
return rsc->allocated_to;
}
if (is_set(rsc->flags, pe_rsc_allocating)) {
pe_rsc_debug(rsc, "Dependency loop detected involving %s", rsc->id);
return NULL;
}
set_bit(rsc->flags, pe_rsc_allocating);
pe__show_node_weights(true, rsc, "Pre-alloc", rsc->allowed_nodes);
for (gIter = rsc->rsc_cons; gIter != NULL; gIter = gIter->next) {
rsc_colocation_t *constraint = (rsc_colocation_t *) gIter->data;
GHashTable *archive = NULL;
pe_resource_t *rsc_rh = constraint->rsc_rh;
if (constraint->score == 0) {
continue;
}
if (constraint->role_lh >= RSC_ROLE_MASTER
|| (constraint->score < 0 && constraint->score > -INFINITY)) {
archive = pcmk__copy_node_table(rsc->allowed_nodes);
}
pe_rsc_trace(rsc,
"%s: Allocating %s first (constraint=%s score=%d role=%s)",
rsc->id, rsc_rh->id, constraint->id,
constraint->score, role2text(constraint->role_lh));
rsc_rh->cmds->allocate(rsc_rh, NULL, data_set);
rsc->cmds->rsc_colocation_lh(rsc, rsc_rh, constraint, data_set);
if (archive && can_run_any(rsc->allowed_nodes) == FALSE) {
pe_rsc_info(rsc, "%s: Rolling back scores from %s", rsc->id, rsc_rh->id);
g_hash_table_destroy(rsc->allowed_nodes);
rsc->allowed_nodes = archive;
archive = NULL;
}
if (archive) {
g_hash_table_destroy(archive);
}
}
pe__show_node_weights(true, rsc, "Post-coloc", rsc->allowed_nodes);
for (gIter = rsc->rsc_cons_lhs; gIter != NULL; gIter = gIter->next) {
rsc_colocation_t *constraint = (rsc_colocation_t *) gIter->data;
if (constraint->score == 0) {
continue;
}
pe_rsc_trace(rsc, "Merging score of '%s' constraint (%s with %s)",
constraint->id, constraint->rsc_lh->id,
constraint->rsc_rh->id);
rsc->allowed_nodes =
constraint->rsc_lh->cmds->merge_weights(constraint->rsc_lh, rsc->id, rsc->allowed_nodes,
constraint->node_attribute,
(float)constraint->score / INFINITY,
pe_weights_rollback);
}
if (rsc->next_role == RSC_ROLE_STOPPED) {
pe_rsc_trace(rsc, "Making sure %s doesn't get allocated", rsc->id);
/* make sure it doesn't come up again */
resource_location(rsc, NULL, -INFINITY, XML_RSC_ATTR_TARGET_ROLE, data_set);
} else if(rsc->next_role > rsc->role
&& is_set(data_set->flags, pe_flag_have_quorum) == FALSE
&& data_set->no_quorum_policy == no_quorum_freeze) {
crm_notice("Resource %s cannot be elevated from %s to %s: no-quorum-policy=freeze",
rsc->id, role2text(rsc->role), role2text(rsc->next_role));
rsc->next_role = rsc->role;
}
pe__show_node_weights(!show_scores, rsc, __FUNCTION__, rsc->allowed_nodes);
if (is_set(data_set->flags, pe_flag_stonith_enabled)
&& is_set(data_set->flags, pe_flag_have_stonith_resource) == FALSE) {
clear_bit(rsc->flags, pe_rsc_managed);
}
if (is_not_set(rsc->flags, pe_rsc_managed)) {
const char *reason = NULL;
pe_node_t *assign_to = NULL;
rsc->next_role = rsc->role;
assign_to = pe__current_node(rsc);
if (assign_to == NULL) {
reason = "inactive";
} else if (rsc->role == RSC_ROLE_MASTER) {
reason = "master";
} else if (is_set(rsc->flags, pe_rsc_failed)) {
reason = "failed";
} else {
reason = "active";
}
pe_rsc_info(rsc, "Unmanaged resource %s allocated to %s: %s", rsc->id,
(assign_to? assign_to->details->uname : "no node"), reason);
native_assign_node(rsc, NULL, assign_to, TRUE);
} else if (is_set(data_set->flags, pe_flag_stop_everything)) {
pe_rsc_debug(rsc, "Forcing %s to stop", rsc->id);
native_assign_node(rsc, NULL, NULL, TRUE);
} else if (is_set(rsc->flags, pe_rsc_provisional)
&& native_choose_node(rsc, prefer, data_set)) {
pe_rsc_trace(rsc, "Allocated resource %s to %s", rsc->id,
rsc->allocated_to->details->uname);
} else if (rsc->allocated_to == NULL) {
if (is_not_set(rsc->flags, pe_rsc_orphan)) {
pe_rsc_info(rsc, "Resource %s cannot run anywhere", rsc->id);
} else if (rsc->running_on != NULL) {
pe_rsc_info(rsc, "Stopping orphan resource %s", rsc->id);
}
} else {
pe_rsc_debug(rsc, "Pre-Allocated resource %s to %s", rsc->id,
rsc->allocated_to->details->uname);
}
clear_bit(rsc->flags, pe_rsc_allocating);
if (rsc->is_remote_node) {
pe_node_t *remote_node = pe_find_node(data_set->nodes, rsc->id);
CRM_ASSERT(remote_node != NULL);
if (rsc->allocated_to && rsc->next_role != RSC_ROLE_STOPPED) {
crm_trace("Setting Pacemaker Remote node %s to ONLINE",
remote_node->details->id);
remote_node->details->online = TRUE;
/* We shouldn't consider an unseen remote-node unclean if we are going
* to try and connect to it. Otherwise we get an unnecessary fence */
if (remote_node->details->unseen == TRUE) {
remote_node->details->unclean = FALSE;
}
} else {
crm_trace("Setting Pacemaker Remote node %s to SHUTDOWN (next role %s, %sallocated)",
remote_node->details->id, role2text(rsc->next_role),
(rsc->allocated_to? "" : "un"));
remote_node->details->shutdown = TRUE;
}
}
return rsc->allocated_to;
}
static gboolean
is_op_dup(pe_resource_t *rsc, const char *name, guint interval_ms)
{
gboolean dup = FALSE;
const char *id = NULL;
const char *value = NULL;
xmlNode *operation = NULL;
guint interval2_ms = 0;
CRM_ASSERT(rsc);
for (operation = __xml_first_child_element(rsc->ops_xml); operation != NULL;
operation = __xml_next_element(operation)) {
if (crm_str_eq((const char *)operation->name, "op", TRUE)) {
value = crm_element_value(operation, "name");
if (safe_str_neq(value, name)) {
continue;
}
value = crm_element_value(operation, XML_LRM_ATTR_INTERVAL);
interval2_ms = crm_parse_interval_spec(value);
if (interval_ms != interval2_ms) {
continue;
}
if (id == NULL) {
id = ID(operation);
} else {
pcmk__config_err("Operation %s is duplicate of %s (do not use "
"same name and interval combination more "
"than once per resource)", ID(operation), id);
dup = TRUE;
}
}
}
return dup;
}
static bool
op_cannot_recur(const char *name)
{
return safe_str_eq(name, RSC_STOP)
|| safe_str_eq(name, RSC_START)
|| safe_str_eq(name, RSC_DEMOTE)
|| safe_str_eq(name, RSC_PROMOTE);
}
static void
RecurringOp(pe_resource_t * rsc, pe_action_t * start, pe_node_t * node,
xmlNode * operation, pe_working_set_t * data_set)
{
char *key = NULL;
const char *name = NULL;
const char *role = NULL;
const char *interval_spec = NULL;
const char *node_uname = node? node->details->uname : "n/a";
guint interval_ms = 0;
pe_action_t *mon = NULL;
gboolean is_optional = TRUE;
GListPtr possible_matches = NULL;
CRM_ASSERT(rsc);
/* Only process for the operations without role="Stopped" */
role = crm_element_value(operation, "role");
if (role && text2role(role) == RSC_ROLE_STOPPED) {
return;
}
interval_spec = crm_element_value(operation, XML_LRM_ATTR_INTERVAL);
interval_ms = crm_parse_interval_spec(interval_spec);
if (interval_ms == 0) {
return;
}
name = crm_element_value(operation, "name");
if (is_op_dup(rsc, name, interval_ms)) {
crm_trace("Not creating duplicate recurring action %s for %dms %s",
ID(operation), interval_ms, name);
return;
}
if (op_cannot_recur(name)) {
pcmk__config_err("Ignoring %s because action '%s' cannot be recurring",
ID(operation), name);
return;
}
key = pcmk__op_key(rsc->id, name, interval_ms);
if (find_rsc_op_entry(rsc, key) == NULL) {
crm_trace("Not creating recurring action %s for disabled resource %s",
ID(operation), rsc->id);
free(key);
return;
}
pe_rsc_trace(rsc, "Creating recurring action %s for %s in role %s on %s",
ID(operation), rsc->id, role2text(rsc->next_role), node_uname);
if (start != NULL) {
pe_rsc_trace(rsc, "Marking %s %s due to %s",
key, is_set(start->flags, pe_action_optional) ? "optional" : "mandatory",
start->uuid);
is_optional = (rsc->cmds->action_flags(start, NULL) & pe_action_optional);
} else {
pe_rsc_trace(rsc, "Marking %s optional", key);
is_optional = TRUE;
}
/* start a monitor for an already active resource */
possible_matches = find_actions_exact(rsc->actions, key, node);
if (possible_matches == NULL) {
is_optional = FALSE;
pe_rsc_trace(rsc, "Marking %s mandatory: not active", key);
} else {
GListPtr gIter = NULL;
for (gIter = possible_matches; gIter != NULL; gIter = gIter->next) {
pe_action_t *op = (pe_action_t *) gIter->data;
if (is_set(op->flags, pe_action_reschedule)) {
is_optional = FALSE;
break;
}
}
g_list_free(possible_matches);
}
if ((rsc->next_role == RSC_ROLE_MASTER && role == NULL)
|| (role != NULL && text2role(role) != rsc->next_role)) {
int log_level = LOG_TRACE;
const char *result = "Ignoring";
if (is_optional) {
char *after_key = NULL;
pe_action_t *cancel_op = NULL;
// It's running, so cancel it
log_level = LOG_INFO;
result = "Cancelling";
cancel_op = pe_cancel_op(rsc, name, interval_ms, node, data_set);
switch (rsc->role) {
case RSC_ROLE_SLAVE:
case RSC_ROLE_STARTED:
if (rsc->next_role == RSC_ROLE_MASTER) {
after_key = promote_key(rsc);
} else if (rsc->next_role == RSC_ROLE_STOPPED) {
after_key = stop_key(rsc);
}
break;
case RSC_ROLE_MASTER:
after_key = demote_key(rsc);
break;
default:
break;
}
if (after_key) {
custom_action_order(rsc, NULL, cancel_op, rsc, after_key, NULL,
pe_order_runnable_left, data_set);
}
}
do_crm_log(log_level, "%s action %s (%s vs. %s)",
result, key, role ? role : role2text(RSC_ROLE_SLAVE),
role2text(rsc->next_role));
free(key);
return;
}
mon = custom_action(rsc, key, name, node, is_optional, TRUE, data_set);
key = mon->uuid;
if (is_optional) {
pe_rsc_trace(rsc, "%s\t %s (optional)", node_uname, mon->uuid);
}
if (start == NULL || is_set(start->flags, pe_action_runnable) == FALSE) {
pe_rsc_debug(rsc, "%s\t %s (cancelled : start un-runnable)",
node_uname, mon->uuid);
update_action_flags(mon, pe_action_runnable | pe_action_clear, __FUNCTION__, __LINE__);
} else if (node == NULL || node->details->online == FALSE || node->details->unclean) {
pe_rsc_debug(rsc, "%s\t %s (cancelled : no node available)",
node_uname, mon->uuid);
update_action_flags(mon, pe_action_runnable | pe_action_clear, __FUNCTION__, __LINE__);
} else if (is_set(mon->flags, pe_action_optional) == FALSE) {
pe_rsc_info(rsc, " Start recurring %s (%us) for %s on %s",
mon->task, interval_ms / 1000, rsc->id, node_uname);
}
if (rsc->next_role == RSC_ROLE_MASTER) {
char *running_master = crm_itoa(PCMK_OCF_RUNNING_MASTER);
add_hash_param(mon->meta, XML_ATTR_TE_TARGET_RC, running_master);
free(running_master);
}
if (node == NULL || is_set(rsc->flags, pe_rsc_managed)) {
custom_action_order(rsc, start_key(rsc), NULL,
NULL, strdup(key), mon,
pe_order_implies_then | pe_order_runnable_left, data_set);
custom_action_order(rsc, reload_key(rsc), NULL,
NULL, strdup(key), mon,
pe_order_implies_then | pe_order_runnable_left, data_set);
if (rsc->next_role == RSC_ROLE_MASTER) {
custom_action_order(rsc, promote_key(rsc), NULL,
rsc, NULL, mon,
pe_order_optional | pe_order_runnable_left, data_set);
} else if (rsc->role == RSC_ROLE_MASTER) {
custom_action_order(rsc, demote_key(rsc), NULL,
rsc, NULL, mon,
pe_order_optional | pe_order_runnable_left, data_set);
}
}
}
static void
Recurring(pe_resource_t * rsc, pe_action_t * start, pe_node_t * node, pe_working_set_t * data_set)
{
if (is_not_set(rsc->flags, pe_rsc_maintenance) &&
(node == NULL || node->details->maintenance == FALSE)) {
xmlNode *operation = NULL;
for (operation = __xml_first_child_element(rsc->ops_xml);
operation != NULL;
operation = __xml_next_element(operation)) {
if (crm_str_eq((const char *)operation->name, "op", TRUE)) {
RecurringOp(rsc, start, node, operation, data_set);
}
}
}
}
static void
RecurringOp_Stopped(pe_resource_t * rsc, pe_action_t * start, pe_node_t * node,
xmlNode * operation, pe_working_set_t * data_set)
{
char *key = NULL;
const char *name = NULL;
const char *role = NULL;
const char *interval_spec = NULL;
const char *node_uname = node? node->details->uname : "n/a";
guint interval_ms = 0;
GListPtr possible_matches = NULL;
GListPtr gIter = NULL;
/* Only process for the operations with role="Stopped" */
role = crm_element_value(operation, "role");
if (role == NULL || text2role(role) != RSC_ROLE_STOPPED) {
return;
}
interval_spec = crm_element_value(operation, XML_LRM_ATTR_INTERVAL);
interval_ms = crm_parse_interval_spec(interval_spec);
if (interval_ms == 0) {
return;
}
name = crm_element_value(operation, "name");
if (is_op_dup(rsc, name, interval_ms)) {
crm_trace("Not creating duplicate recurring action %s for %dms %s",
ID(operation), interval_ms, name);
return;
}
if (op_cannot_recur(name)) {
pcmk__config_err("Ignoring %s because action '%s' cannot be recurring",
ID(operation), name);
return;
}
key = pcmk__op_key(rsc->id, name, interval_ms);
if (find_rsc_op_entry(rsc, key) == NULL) {
crm_trace("Not creating recurring action %s for disabled resource %s",
ID(operation), rsc->id);
free(key);
return;
}
// @TODO add support
if (is_set(rsc->flags, pe_rsc_unique) == FALSE) {
crm_notice("Ignoring %s (recurring monitors for Stopped role are "
"not supported for anonymous clones)",
ID(operation));
return;
}
pe_rsc_trace(rsc,
"Creating recurring action %s for %s in role %s on nodes where it should not be running",
ID(operation), rsc->id, role2text(rsc->next_role));
/* if the monitor exists on the node where the resource will be running, cancel it */
if (node != NULL) {
possible_matches = find_actions_exact(rsc->actions, key, node);
if (possible_matches) {
pe_action_t *cancel_op = NULL;
g_list_free(possible_matches);
cancel_op = pe_cancel_op(rsc, name, interval_ms, node, data_set);
if (rsc->next_role == RSC_ROLE_STARTED || rsc->next_role == RSC_ROLE_SLAVE) {
/* rsc->role == RSC_ROLE_STOPPED: cancel the monitor before start */
/* rsc->role == RSC_ROLE_STARTED: for a migration, cancel the monitor on the target node before start */
custom_action_order(rsc, NULL, cancel_op, rsc, start_key(rsc), NULL,
pe_order_runnable_left, data_set);
}
pe_rsc_info(rsc, "Cancel action %s (%s vs. %s) on %s",
key, role, role2text(rsc->next_role), node_uname);
}
}
for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
pe_node_t *stop_node = (pe_node_t *) gIter->data;
const char *stop_node_uname = stop_node->details->uname;
gboolean is_optional = TRUE;
gboolean probe_is_optional = TRUE;
gboolean stop_is_optional = TRUE;
pe_action_t *stopped_mon = NULL;
char *rc_inactive = NULL;
GListPtr probe_complete_ops = NULL;
GListPtr stop_ops = NULL;
GListPtr local_gIter = NULL;
if (node && safe_str_eq(stop_node_uname, node_uname)) {
continue;
}
pe_rsc_trace(rsc, "Creating recurring action %s for %s on %s",
ID(operation), rsc->id, crm_str(stop_node_uname));
/* start a monitor for an already stopped resource */
possible_matches = find_actions_exact(rsc->actions, key, stop_node);
if (possible_matches == NULL) {
pe_rsc_trace(rsc, "Marking %s mandatory on %s: not active", key,
crm_str(stop_node_uname));
is_optional = FALSE;
} else {
pe_rsc_trace(rsc, "Marking %s optional on %s: already active", key,
crm_str(stop_node_uname));
is_optional = TRUE;
g_list_free(possible_matches);
}
stopped_mon = custom_action(rsc, strdup(key), name, stop_node, is_optional, TRUE, data_set);
rc_inactive = crm_itoa(PCMK_OCF_NOT_RUNNING);
add_hash_param(stopped_mon->meta, XML_ATTR_TE_TARGET_RC, rc_inactive);
free(rc_inactive);
if (is_set(rsc->flags, pe_rsc_managed)) {
GList *probes = pe__resource_actions(rsc, stop_node, RSC_STATUS,
FALSE);
GListPtr pIter = NULL;
for (pIter = probes; pIter != NULL; pIter = pIter->next) {
pe_action_t *probe = (pe_action_t *) pIter->data;
order_actions(probe, stopped_mon, pe_order_runnable_left);
crm_trace("%s then %s on %s", probe->uuid, stopped_mon->uuid, stop_node->details->uname);
}
g_list_free(probes);
}
if (probe_complete_ops) {
g_list_free(probe_complete_ops);
}
stop_ops = pe__resource_actions(rsc, stop_node, RSC_STOP, TRUE);
for (local_gIter = stop_ops; local_gIter != NULL; local_gIter = local_gIter->next) {
pe_action_t *stop = (pe_action_t *) local_gIter->data;
if (is_set(stop->flags, pe_action_optional) == FALSE) {
stop_is_optional = FALSE;
}
if (is_set(stop->flags, pe_action_runnable) == FALSE) {
crm_debug("%s\t %s (cancelled : stop un-runnable)",
crm_str(stop_node_uname), stopped_mon->uuid);
update_action_flags(stopped_mon, pe_action_runnable | pe_action_clear, __FUNCTION__, __LINE__);
}
if (is_set(rsc->flags, pe_rsc_managed)) {
custom_action_order(rsc, stop_key(rsc), stop,
NULL, strdup(key), stopped_mon,
pe_order_implies_then | pe_order_runnable_left, data_set);
}
}
if (stop_ops) {
g_list_free(stop_ops);
}
if (is_optional == FALSE && probe_is_optional && stop_is_optional
&& is_set(rsc->flags, pe_rsc_managed) == FALSE) {
pe_rsc_trace(rsc, "Marking %s optional on %s due to unmanaged",
key, crm_str(stop_node_uname));
update_action_flags(stopped_mon, pe_action_optional, __FUNCTION__, __LINE__);
}
if (is_set(stopped_mon->flags, pe_action_optional)) {
pe_rsc_trace(rsc, "%s\t %s (optional)", crm_str(stop_node_uname), stopped_mon->uuid);
}
if (stop_node->details->online == FALSE || stop_node->details->unclean) {
pe_rsc_debug(rsc, "%s\t %s (cancelled : no node available)",
crm_str(stop_node_uname), stopped_mon->uuid);
update_action_flags(stopped_mon, pe_action_runnable | pe_action_clear, __FUNCTION__, __LINE__);
}
if (is_set(stopped_mon->flags, pe_action_runnable)
&& is_set(stopped_mon->flags, pe_action_optional) == FALSE) {
crm_notice(" Start recurring %s (%us) for %s on %s", stopped_mon->task,
interval_ms / 1000, rsc->id, crm_str(stop_node_uname));
}
}
free(key);
}
static void
Recurring_Stopped(pe_resource_t * rsc, pe_action_t * start, pe_node_t * node, pe_working_set_t * data_set)
{
if (is_not_set(rsc->flags, pe_rsc_maintenance) &&
(node == NULL || node->details->maintenance == FALSE)) {
xmlNode *operation = NULL;
for (operation = __xml_first_child_element(rsc->ops_xml);
operation != NULL;
operation = __xml_next_element(operation)) {
if (crm_str_eq((const char *)operation->name, "op", TRUE)) {
RecurringOp_Stopped(rsc, start, node, operation, data_set);
}
}
}
}
static void
handle_migration_actions(pe_resource_t * rsc, pe_node_t *current, pe_node_t *chosen, pe_working_set_t * data_set)
{
pe_action_t *migrate_to = NULL;
pe_action_t *migrate_from = NULL;
pe_action_t *start = NULL;
pe_action_t *stop = NULL;
gboolean partial = rsc->partial_migration_target ? TRUE : FALSE;
pe_rsc_trace(rsc, "Processing migration actions %s moving from %s to %s . partial migration = %s",
rsc->id, current->details->id, chosen->details->id, partial ? "TRUE" : "FALSE");
start = start_action(rsc, chosen, TRUE);
stop = stop_action(rsc, current, TRUE);
if (partial == FALSE) {
migrate_to = custom_action(rsc, pcmk__op_key(rsc->id, RSC_MIGRATE, 0),
RSC_MIGRATE, current, TRUE, TRUE, data_set);
}
migrate_from = custom_action(rsc, pcmk__op_key(rsc->id, RSC_MIGRATED, 0),
RSC_MIGRATED, chosen, TRUE, TRUE, data_set);
if ((migrate_to && migrate_from) || (migrate_from && partial)) {
set_bit(start->flags, pe_action_migrate_runnable);
set_bit(stop->flags, pe_action_migrate_runnable);
update_action_flags(start, pe_action_pseudo, __FUNCTION__, __LINE__); /* easier than trying to delete it from the graph */
/* order probes before migrations */
if (partial) {
set_bit(migrate_from->flags, pe_action_migrate_runnable);
migrate_from->needs = start->needs;
custom_action_order(rsc, pcmk__op_key(rsc->id, RSC_STATUS, 0), NULL,
rsc, pcmk__op_key(rsc->id, RSC_MIGRATED, 0),
NULL, pe_order_optional, data_set);
} else {
set_bit(migrate_from->flags, pe_action_migrate_runnable);
set_bit(migrate_to->flags, pe_action_migrate_runnable);
migrate_to->needs = start->needs;
custom_action_order(rsc, pcmk__op_key(rsc->id, RSC_STATUS, 0), NULL,
rsc, pcmk__op_key(rsc->id, RSC_MIGRATE, 0),
NULL, pe_order_optional, data_set);
custom_action_order(rsc, pcmk__op_key(rsc->id, RSC_MIGRATE, 0),
NULL, rsc,
pcmk__op_key(rsc->id, RSC_MIGRATED, 0), NULL,
pe_order_optional|pe_order_implies_first_migratable,
data_set);
}
custom_action_order(rsc, pcmk__op_key(rsc->id, RSC_MIGRATED, 0), NULL,
rsc, pcmk__op_key(rsc->id, RSC_STOP, 0), NULL,
pe_order_optional|pe_order_implies_first_migratable,
data_set);
custom_action_order(rsc, pcmk__op_key(rsc->id, RSC_MIGRATED, 0), NULL,
rsc, pcmk__op_key(rsc->id, RSC_START, 0), NULL,
pe_order_optional|pe_order_implies_first_migratable|pe_order_pseudo_left,
data_set);
}
if (migrate_to) {
add_hash_param(migrate_to->meta, XML_LRM_ATTR_MIGRATE_SOURCE, current->details->uname);
add_hash_param(migrate_to->meta, XML_LRM_ATTR_MIGRATE_TARGET, chosen->details->uname);
/* Pacemaker Remote connections don't require pending to be recorded in
* the CIB. We can reduce CIB writes by not setting PENDING for them.
*/
if (rsc->is_remote_node == FALSE) {
/* migrate_to takes place on the source node, but can
* have an effect on the target node depending on how
* the agent is written. Because of this, we have to maintain
* a record that the migrate_to occurred, in case the source node
* loses membership while the migrate_to action is still in-flight.
*/
add_hash_param(migrate_to->meta, XML_OP_ATTR_PENDING, "true");
}
}
if (migrate_from) {
add_hash_param(migrate_from->meta, XML_LRM_ATTR_MIGRATE_SOURCE, current->details->uname);
add_hash_param(migrate_from->meta, XML_LRM_ATTR_MIGRATE_TARGET, chosen->details->uname);
}
}
void
native_create_actions(pe_resource_t * rsc, pe_working_set_t * data_set)
{
pe_action_t *start = NULL;
pe_node_t *chosen = NULL;
pe_node_t *current = NULL;
gboolean need_stop = FALSE;
+ bool need_promote = FALSE;
gboolean is_moving = FALSE;
gboolean allow_migrate = is_set(rsc->flags, pe_rsc_allow_migrate) ? TRUE : FALSE;
GListPtr gIter = NULL;
unsigned int num_all_active = 0;
unsigned int num_clean_active = 0;
bool multiply_active = FALSE;
enum rsc_role_e role = RSC_ROLE_UNKNOWN;
enum rsc_role_e next_role = RSC_ROLE_UNKNOWN;
CRM_ASSERT(rsc);
chosen = rsc->allocated_to;
if (chosen != NULL && rsc->next_role == RSC_ROLE_UNKNOWN) {
rsc->next_role = RSC_ROLE_STARTED;
pe_rsc_trace(rsc, "Fixed next_role: unknown -> %s", role2text(rsc->next_role));
} else if (rsc->next_role == RSC_ROLE_UNKNOWN) {
rsc->next_role = RSC_ROLE_STOPPED;
pe_rsc_trace(rsc, "Fixed next_role: unknown -> %s", role2text(rsc->next_role));
}
pe_rsc_trace(rsc, "Processing state transition for %s %p: %s->%s", rsc->id, rsc,
role2text(rsc->role), role2text(rsc->next_role));
current = pe__find_active_on(rsc, &num_all_active, &num_clean_active);
for (gIter = rsc->dangling_migrations; gIter != NULL; gIter = gIter->next) {
pe_node_t *dangling_source = (pe_node_t *) gIter->data;
pe_action_t *stop = stop_action(rsc, dangling_source, FALSE);
set_bit(stop->flags, pe_action_dangle);
pe_rsc_trace(rsc, "Forcing a cleanup of %s on %s",
rsc->id, dangling_source->details->uname);
if (is_set(data_set->flags, pe_flag_remove_after_stop)) {
DeleteRsc(rsc, dangling_source, FALSE, data_set);
}
}
if ((num_all_active == 2) && (num_clean_active == 2) && chosen
&& rsc->partial_migration_source && rsc->partial_migration_target
&& (current->details == rsc->partial_migration_source->details)
&& (chosen->details == rsc->partial_migration_target->details)) {
/* The chosen node is still the migration target from a partial
* migration. Attempt to continue the migration instead of recovering
* by stopping the resource everywhere and starting it on a single node.
*/
pe_rsc_trace(rsc,
"Will attempt to continue with a partial migration to target %s from %s",
rsc->partial_migration_target->details->id,
rsc->partial_migration_source->details->id);
} else if (is_not_set(rsc->flags, pe_rsc_needs_fencing)) {
/* If a resource has "requires" set to nothing or quorum, don't consider
* it active on unclean nodes (similar to how all resources behave when
* stonith-enabled is false). We can start such resources elsewhere
* before fencing completes, and if we considered the resource active on
* the failed node, we would attempt recovery for being active on
* multiple nodes.
*/
multiply_active = (num_clean_active > 1);
} else {
multiply_active = (num_all_active > 1);
}
if (multiply_active) {
if (rsc->partial_migration_target && rsc->partial_migration_source) {
// Migration was in progress, but we've chosen a different target
crm_notice("Resource %s can no longer migrate to %s. Stopping on %s too",
rsc->id, rsc->partial_migration_target->details->uname,
rsc->partial_migration_source->details->uname);
} else {
// Resource was incorrectly multiply active
pe_proc_err("Resource %s is active on %u nodes (%s)",
rsc->id, num_all_active,
recovery2text(rsc->recovery_type));
crm_notice("See https://wiki.clusterlabs.org/wiki/FAQ#Resource_is_Too_Active for more information");
}
if (rsc->recovery_type == recovery_stop_start) {
need_stop = TRUE;
}
/* If by chance a partial migration is in process, but the migration
* target is not chosen still, clear all partial migration data.
*/
rsc->partial_migration_source = rsc->partial_migration_target = NULL;
allow_migrate = FALSE;
}
if (is_set(rsc->flags, pe_rsc_start_pending)) {
start = start_action(rsc, chosen, TRUE);
set_bit(start->flags, pe_action_print_always);
}
if (current && chosen && current->details != chosen->details) {
pe_rsc_trace(rsc, "Moving %s", rsc->id);
is_moving = TRUE;
need_stop = TRUE;
} else if (is_set(rsc->flags, pe_rsc_failed)) {
- pe_rsc_trace(rsc, "Recovering %s", rsc->id);
- need_stop = TRUE;
+ if (is_set(rsc->flags, pe_rsc_stop)) {
+ need_stop = TRUE;
+ pe_rsc_trace(rsc, "Recovering %s", rsc->id);
+ } else {
+ pe_rsc_trace(rsc, "Recovering %s by demotion", rsc->id);
+ if (rsc->next_role == RSC_ROLE_MASTER) {
+ need_promote = TRUE;
+ }
+ }
} else if (is_set(rsc->flags, pe_rsc_block)) {
pe_rsc_trace(rsc, "Block %s", rsc->id);
need_stop = TRUE;
} else if (rsc->role > RSC_ROLE_STARTED && current != NULL && chosen != NULL) {
/* Recovery of a promoted resource */
start = start_action(rsc, chosen, TRUE);
if (is_set(start->flags, pe_action_optional) == FALSE) {
pe_rsc_trace(rsc, "Forced start %s", rsc->id);
need_stop = TRUE;
}
}
pe_rsc_trace(rsc, "Creating actions for %s: %s->%s", rsc->id,
role2text(rsc->role), role2text(rsc->next_role));
/* Create any additional actions required when bringing resource down and
* back up to same level.
*/
role = rsc->role;
while (role != RSC_ROLE_STOPPED) {
next_role = rsc_state_matrix[role][RSC_ROLE_STOPPED];
pe_rsc_trace(rsc, "Down: Executing: %s->%s (%s)%s", role2text(role), role2text(next_role),
rsc->id, need_stop ? " required" : "");
if (rsc_action_matrix[role][next_role] (rsc, current, !need_stop, data_set) == FALSE) {
break;
}
role = next_role;
}
while (rsc->role <= rsc->next_role && role != rsc->role && is_not_set(rsc->flags, pe_rsc_block)) {
+ bool required = need_stop;
+
next_role = rsc_state_matrix[role][rsc->role];
+ if ((next_role == RSC_ROLE_MASTER) && need_promote) {
+ required = true;
+ }
pe_rsc_trace(rsc, "Up: Executing: %s->%s (%s)%s", role2text(role), role2text(next_role),
- rsc->id, need_stop ? " required" : "");
- if (rsc_action_matrix[role][next_role] (rsc, chosen, !need_stop, data_set) == FALSE) {
+ rsc->id, (required? " required" : ""));
+ if (rsc_action_matrix[role][next_role](rsc, chosen, !required,
+ data_set) == FALSE) {
break;
}
role = next_role;
}
role = rsc->role;
/* Required steps from this role to the next */
while (role != rsc->next_role) {
next_role = rsc_state_matrix[role][rsc->next_role];
pe_rsc_trace(rsc, "Role: Executing: %s->%s = (%s on %s)", role2text(role), role2text(next_role), rsc->id, chosen?chosen->details->uname:"NA");
if (rsc_action_matrix[role][next_role] (rsc, chosen, FALSE, data_set) == FALSE) {
break;
}
role = next_role;
}
if(is_set(rsc->flags, pe_rsc_block)) {
pe_rsc_trace(rsc, "No monitor additional ops for blocked resource");
} else if (rsc->next_role != RSC_ROLE_STOPPED || is_set(rsc->flags, pe_rsc_managed) == FALSE) {
pe_rsc_trace(rsc, "Monitor ops for active resource");
start = start_action(rsc, chosen, TRUE);
Recurring(rsc, start, chosen, data_set);
Recurring_Stopped(rsc, start, chosen, data_set);
} else {
pe_rsc_trace(rsc, "Monitor ops for inactive resource");
Recurring_Stopped(rsc, NULL, NULL, data_set);
}
/* if we are stuck in a partial migration, where the target
* of the partial migration no longer matches the chosen target.
* A full stop/start is required */
if (rsc->partial_migration_target && (chosen == NULL || rsc->partial_migration_target->details != chosen->details)) {
pe_rsc_trace(rsc, "Not allowing partial migration to continue. %s", rsc->id);
allow_migrate = FALSE;
} else if (is_moving == FALSE ||
is_not_set(rsc->flags, pe_rsc_managed) ||
is_set(rsc->flags, pe_rsc_failed) ||
is_set(rsc->flags, pe_rsc_start_pending) ||
(current && current->details->unclean) ||
rsc->next_role < RSC_ROLE_STARTED) {
allow_migrate = FALSE;
}
if (allow_migrate) {
handle_migration_actions(rsc, current, chosen, data_set);
}
}
static void
rsc_avoids_remote_nodes(pe_resource_t *rsc)
{
GHashTableIter iter;
pe_node_t *node = NULL;
g_hash_table_iter_init(&iter, rsc->allowed_nodes);
while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
if (node->details->remote_rsc) {
node->weight = -INFINITY;
}
}
}
/*!
* \internal
* \brief Return allowed nodes as (possibly sorted) list
*
* Convert a resource's hash table of allowed nodes to a list. If printing to
* stdout, sort the list, to keep action ID numbers consistent for regression
* test output (while avoiding the performance hit on a live cluster).
*
* \param[in] rsc Resource to check for allowed nodes
* \param[in] data_set Cluster working set
*
* \return List of resource's allowed nodes
* \note Callers should take care not to rely on the list being sorted.
*/
static GList *
allowed_nodes_as_list(pe_resource_t *rsc, pe_working_set_t *data_set)
{
GList *allowed_nodes = NULL;
if (rsc->allowed_nodes) {
allowed_nodes = g_hash_table_get_values(rsc->allowed_nodes);
}
if (is_set(data_set->flags, pe_flag_stdout)) {
allowed_nodes = g_list_sort(allowed_nodes, sort_node_uname);
}
return allowed_nodes;
}
void
native_internal_constraints(pe_resource_t * rsc, pe_working_set_t * data_set)
{
/* This function is on the critical path and worth optimizing as much as possible */
pe_resource_t *top = NULL;
GList *allowed_nodes = NULL;
bool check_unfencing = FALSE;
bool check_utilization = FALSE;
if (is_not_set(rsc->flags, pe_rsc_managed)) {
pe_rsc_trace(rsc,
"Skipping native constraints for unmanaged resource: %s",
rsc->id);
return;
}
top = uber_parent(rsc);
// Whether resource requires unfencing
check_unfencing = is_not_set(rsc->flags, pe_rsc_fence_device)
&& is_set(data_set->flags, pe_flag_enable_unfencing)
&& is_set(rsc->flags, pe_rsc_needs_unfencing);
// Whether a non-default placement strategy is used
check_utilization = (g_hash_table_size(rsc->utilization) > 0)
&& safe_str_neq(data_set->placement_strategy, "default");
// Order stops before starts (i.e. restart)
custom_action_order(rsc, pcmk__op_key(rsc->id, RSC_STOP, 0), NULL,
rsc, pcmk__op_key(rsc->id, RSC_START, 0), NULL,
pe_order_optional|pe_order_implies_then|pe_order_restart,
data_set);
// Promotable ordering: demote before stop, start before promote
if (is_set(top->flags, pe_rsc_promotable) || (rsc->role > RSC_ROLE_SLAVE)) {
custom_action_order(rsc, pcmk__op_key(rsc->id, RSC_DEMOTE, 0), NULL,
rsc, pcmk__op_key(rsc->id, RSC_STOP, 0), NULL,
pe_order_implies_first_master, data_set);
custom_action_order(rsc, pcmk__op_key(rsc->id, RSC_START, 0), NULL,
rsc, pcmk__op_key(rsc->id, RSC_PROMOTE, 0), NULL,
pe_order_runnable_left, data_set);
}
// Don't clear resource history if probing on same node
custom_action_order(rsc, pcmk__op_key(rsc->id, CRM_OP_LRM_DELETE, 0),
NULL, rsc, pcmk__op_key(rsc->id, RSC_STATUS, 0),
NULL, pe_order_same_node|pe_order_then_cancels_first,
data_set);
// Certain checks need allowed nodes
if (check_unfencing || check_utilization || rsc->container) {
allowed_nodes = allowed_nodes_as_list(rsc, data_set);
}
if (check_unfencing) {
/* Check if the node needs to be unfenced first */
for (GList *item = allowed_nodes; item; item = item->next) {
pe_node_t *node = item->data;
pe_action_t *unfence = pe_fence_op(node, "on", TRUE, NULL, FALSE, data_set);
crm_debug("Ordering any stops of %s before %s, and any starts after",
rsc->id, unfence->uuid);
/*
* It would be more efficient to order clone resources once,
* rather than order each instance, but ordering the instance
* allows us to avoid unnecessary dependencies that might conflict
* with user constraints.
*
* @TODO: This constraint can still produce a transition loop if the
* resource has a stop scheduled on the node being unfenced, and
* there is a user ordering constraint to start some other resource
* (which will be ordered after the unfence) before stopping this
* resource. An example is "start some slow-starting cloned service
* before stopping an associated virtual IP that may be moving to
* it":
* stop this -> unfencing -> start that -> stop this
*/
custom_action_order(rsc, stop_key(rsc), NULL,
NULL, strdup(unfence->uuid), unfence,
pe_order_optional|pe_order_same_node, data_set);
custom_action_order(NULL, strdup(unfence->uuid), unfence,
rsc, start_key(rsc), NULL,
pe_order_implies_then_on_node|pe_order_same_node,
data_set);
}
}
if (check_utilization) {
GListPtr gIter = NULL;
pe_rsc_trace(rsc, "Creating utilization constraints for %s - strategy: %s",
rsc->id, data_set->placement_strategy);
for (gIter = rsc->running_on; gIter != NULL; gIter = gIter->next) {
pe_node_t *current = (pe_node_t *) gIter->data;
char *load_stopped_task = crm_strdup_printf(LOAD_STOPPED "_%s",
current->details->uname);
pe_action_t *load_stopped = get_pseudo_op(load_stopped_task, data_set);
if (load_stopped->node == NULL) {
load_stopped->node = pe__copy_node(current);
update_action_flags(load_stopped, pe_action_optional | pe_action_clear, __FUNCTION__, __LINE__);
}
custom_action_order(rsc, stop_key(rsc), NULL,
NULL, load_stopped_task, load_stopped, pe_order_load, data_set);
}
for (GList *item = allowed_nodes; item; item = item->next) {
pe_node_t *next = item->data;
char *load_stopped_task = crm_strdup_printf(LOAD_STOPPED "_%s",
next->details->uname);
pe_action_t *load_stopped = get_pseudo_op(load_stopped_task, data_set);
if (load_stopped->node == NULL) {
load_stopped->node = pe__copy_node(next);
update_action_flags(load_stopped, pe_action_optional | pe_action_clear, __FUNCTION__, __LINE__);
}
custom_action_order(NULL, strdup(load_stopped_task), load_stopped,
rsc, start_key(rsc), NULL, pe_order_load, data_set);
custom_action_order(NULL, strdup(load_stopped_task), load_stopped,
rsc, pcmk__op_key(rsc->id, RSC_MIGRATE, 0),
NULL, pe_order_load, data_set);
free(load_stopped_task);
}
}
if (rsc->container) {
pe_resource_t *remote_rsc = NULL;
if (rsc->is_remote_node) {
// rsc is the implicit remote connection for a guest or bundle node
/* Do not allow a guest resource to live on a Pacemaker Remote node,
* to avoid nesting remotes. However, allow bundles to run on remote
* nodes.
*/
if (is_not_set(rsc->flags, pe_rsc_allow_remote_remotes)) {
rsc_avoids_remote_nodes(rsc->container);
}
/* If someone cleans up a guest or bundle node's container, we will
* likely schedule a (re-)probe of the container and recovery of the
* connection. Order the connection stop after the container probe,
* so that if we detect the container running, we will trigger a new
* transition and avoid the unnecessary recovery.
*/
new_rsc_order(rsc->container, RSC_STATUS, rsc, RSC_STOP,
pe_order_optional, data_set);
/* A user can specify that a resource must start on a Pacemaker Remote
* node by explicitly configuring it with the container=NODENAME
* meta-attribute. This is of questionable merit, since location
* constraints can accomplish the same thing. But we support it, so here
* we check whether a resource (that is not itself a remote connection)
* has container set to a remote node or guest node resource.
*/
} else if (rsc->container->is_remote_node) {
remote_rsc = rsc->container;
} else {
remote_rsc = pe__resource_contains_guest_node(data_set,
rsc->container);
}
if (remote_rsc) {
/* Force the resource on the Pacemaker Remote node instead of
* colocating the resource with the container resource.
*/
for (GList *item = allowed_nodes; item; item = item->next) {
pe_node_t *node = item->data;
if (node->details->remote_rsc != remote_rsc) {
node->weight = -INFINITY;
}
}
} else {
/* This resource is either a filler for a container that does NOT
* represent a Pacemaker Remote node, or a Pacemaker Remote
* connection resource for a guest node or bundle.
*/
int score;
crm_trace("Order and colocate %s relative to its container %s",
rsc->id, rsc->container->id);
custom_action_order(rsc->container,
pcmk__op_key(rsc->container->id, RSC_START, 0),
NULL, rsc, pcmk__op_key(rsc->id, RSC_START, 0),
NULL,
pe_order_implies_then|pe_order_runnable_left,
data_set);
custom_action_order(rsc, pcmk__op_key(rsc->id, RSC_STOP, 0), NULL,
rsc->container,
pcmk__op_key(rsc->container->id, RSC_STOP, 0),
NULL, pe_order_implies_first, data_set);
if (is_set(rsc->flags, pe_rsc_allow_remote_remotes)) {
score = 10000; /* Highly preferred but not essential */
} else {
score = INFINITY; /* Force them to run on the same host */
}
rsc_colocation_new("resource-with-container", NULL, score, rsc,
rsc->container, NULL, NULL, data_set);
}
}
if (rsc->is_remote_node || is_set(rsc->flags, pe_rsc_fence_device)) {
/* don't allow remote nodes to run stonith devices
* or remote connection resources.*/
rsc_avoids_remote_nodes(rsc);
}
g_list_free(allowed_nodes);
}
void
native_rsc_colocation_lh(pe_resource_t *rsc_lh, pe_resource_t *rsc_rh,
rsc_colocation_t *constraint,
pe_working_set_t *data_set)
{
if (rsc_lh == NULL) {
pe_err("rsc_lh was NULL for %s", constraint->id);
return;
} else if (constraint->rsc_rh == NULL) {
pe_err("rsc_rh was NULL for %s", constraint->id);
return;
}
if (constraint->score == 0) {
return;
}
pe_rsc_trace(rsc_lh, "Processing colocation constraint between %s and %s", rsc_lh->id,
rsc_rh->id);
rsc_rh->cmds->rsc_colocation_rh(rsc_lh, rsc_rh, constraint, data_set);
}
enum filter_colocation_res
filter_colocation_constraint(pe_resource_t * rsc_lh, pe_resource_t * rsc_rh,
rsc_colocation_t * constraint, gboolean preview)
{
if (constraint->score == 0) {
return influence_nothing;
}
/* rh side must be allocated before we can process constraint */
if (preview == FALSE && is_set(rsc_rh->flags, pe_rsc_provisional)) {
return influence_nothing;
}
if ((constraint->role_lh >= RSC_ROLE_SLAVE) &&
rsc_lh->parent && is_set(rsc_lh->parent->flags, pe_rsc_promotable)
&& is_not_set(rsc_lh->flags, pe_rsc_provisional)) {
/* LH and RH resources have already been allocated, place the correct
* priority on LH rsc for the given promotable clone resource role */
return influence_rsc_priority;
}
if (preview == FALSE && is_not_set(rsc_lh->flags, pe_rsc_provisional)) {
// Log an error if we violated a mandatory colocation constraint
const pe_node_t *rh_node = rsc_rh->allocated_to;
if (rsc_lh->allocated_to == NULL) {
// Dependent resource isn't allocated, so constraint doesn't matter
return influence_nothing;
}
if (constraint->score >= INFINITY) {
// Dependent resource must colocate with rh_node
if ((rh_node == NULL)
|| (rh_node->details != rsc_lh->allocated_to->details)) {
crm_err("%s must be colocated with %s but is not (%s vs. %s)",
rsc_lh->id, rsc_rh->id,
rsc_lh->allocated_to->details->uname,
(rh_node? rh_node->details->uname : "unallocated"));
}
} else if (constraint->score <= -INFINITY) {
// Dependent resource must anti-colocate with rh_node
if ((rh_node != NULL)
&& (rsc_lh->allocated_to->details == rh_node->details)) {
crm_err("%s and %s must be anti-colocated but are allocated "
"to the same node (%s)",
rsc_lh->id, rsc_rh->id, rh_node->details->uname);
}
}
return influence_nothing;
}
if (constraint->score > 0
&& constraint->role_lh != RSC_ROLE_UNKNOWN && constraint->role_lh != rsc_lh->next_role) {
crm_trace("LH: Skipping constraint: \"%s\" state filter nextrole is %s",
role2text(constraint->role_lh), role2text(rsc_lh->next_role));
return influence_nothing;
}
if (constraint->score > 0
&& constraint->role_rh != RSC_ROLE_UNKNOWN && constraint->role_rh != rsc_rh->next_role) {
crm_trace("RH: Skipping constraint: \"%s\" state filter", role2text(constraint->role_rh));
return influence_nothing;
}
if (constraint->score < 0
&& constraint->role_lh != RSC_ROLE_UNKNOWN && constraint->role_lh == rsc_lh->next_role) {
crm_trace("LH: Skipping negative constraint: \"%s\" state filter",
role2text(constraint->role_lh));
return influence_nothing;
}
if (constraint->score < 0
&& constraint->role_rh != RSC_ROLE_UNKNOWN && constraint->role_rh == rsc_rh->next_role) {
crm_trace("RH: Skipping negative constraint: \"%s\" state filter",
role2text(constraint->role_rh));
return influence_nothing;
}
return influence_rsc_location;
}
static void
influence_priority(pe_resource_t * rsc_lh, pe_resource_t * rsc_rh, rsc_colocation_t * constraint)
{
const char *rh_value = NULL;
const char *lh_value = NULL;
const char *attribute = CRM_ATTR_ID;
int score_multiplier = 1;
if (constraint->score == 0) {
return;
}
if (!rsc_rh->allocated_to || !rsc_lh->allocated_to) {
return;
}
if (constraint->node_attribute != NULL) {
attribute = constraint->node_attribute;
}
lh_value = pe_node_attribute_raw(rsc_lh->allocated_to, attribute);
rh_value = pe_node_attribute_raw(rsc_rh->allocated_to, attribute);
if (!safe_str_eq(lh_value, rh_value)) {
if(constraint->score == INFINITY && constraint->role_lh == RSC_ROLE_MASTER) {
rsc_lh->priority = -INFINITY;
}
return;
}
if (constraint->role_rh && (constraint->role_rh != rsc_rh->next_role)) {
return;
}
if (constraint->role_lh == RSC_ROLE_SLAVE) {
score_multiplier = -1;
}
rsc_lh->priority = pe__add_scores(score_multiplier * constraint->score,
rsc_lh->priority);
}
static void
colocation_match(pe_resource_t * rsc_lh, pe_resource_t * rsc_rh, rsc_colocation_t * constraint)
{
const char *attribute = CRM_ATTR_ID;
const char *value = NULL;
GHashTable *work = NULL;
GHashTableIter iter;
pe_node_t *node = NULL;
if (constraint->score == 0) {
return;
}
if (constraint->node_attribute != NULL) {
attribute = constraint->node_attribute;
}
if (rsc_rh->allocated_to) {
value = pe_node_attribute_raw(rsc_rh->allocated_to, attribute);
} else if (constraint->score < 0) {
// Nothing to do (anti-colocation with something that is not running)
return;
}
work = pcmk__copy_node_table(rsc_lh->allowed_nodes);
g_hash_table_iter_init(&iter, work);
while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
if (rsc_rh->allocated_to == NULL) {
pe_rsc_trace(rsc_lh, "%s: %s@%s -= %d (%s inactive)",
constraint->id, rsc_lh->id, node->details->uname,
constraint->score, rsc_rh->id);
node->weight = pe__add_scores(-constraint->score, node->weight);
} else if (safe_str_eq(pe_node_attribute_raw(node, attribute), value)) {
if (constraint->score < CRM_SCORE_INFINITY) {
pe_rsc_trace(rsc_lh, "%s: %s@%s += %d",
constraint->id, rsc_lh->id,
node->details->uname, constraint->score);
node->weight = pe__add_scores(constraint->score, node->weight);
}
} else if (constraint->score >= CRM_SCORE_INFINITY) {
pe_rsc_trace(rsc_lh, "%s: %s@%s -= %d (%s mismatch)",
constraint->id, rsc_lh->id, node->details->uname,
constraint->score, attribute);
node->weight = pe__add_scores(-constraint->score, node->weight);
}
}
if (can_run_any(work)
|| constraint->score <= -INFINITY || constraint->score >= INFINITY) {
g_hash_table_destroy(rsc_lh->allowed_nodes);
rsc_lh->allowed_nodes = work;
work = NULL;
} else {
pe_rsc_info(rsc_lh,
"%s: Rolling back scores from %s (no available nodes)",
rsc_lh->id, rsc_rh->id);
}
if (work) {
g_hash_table_destroy(work);
}
}
void
native_rsc_colocation_rh(pe_resource_t *rsc_lh, pe_resource_t *rsc_rh,
rsc_colocation_t *constraint,
pe_working_set_t *data_set)
{
enum filter_colocation_res filter_results;
CRM_ASSERT(rsc_lh);
CRM_ASSERT(rsc_rh);
filter_results = filter_colocation_constraint(rsc_lh, rsc_rh, constraint, FALSE);
pe_rsc_trace(rsc_lh, "%s %s with %s (%s, score=%d, filter=%d)",
((constraint->score >= 0)? "Colocating" : "Anti-colocating"),
rsc_lh->id, rsc_rh->id, constraint->id, constraint->score, filter_results);
switch (filter_results) {
case influence_rsc_priority:
influence_priority(rsc_lh, rsc_rh, constraint);
break;
case influence_rsc_location:
colocation_match(rsc_lh, rsc_rh, constraint);
break;
case influence_nothing:
default:
return;
}
}
static gboolean
filter_rsc_ticket(pe_resource_t * rsc_lh, rsc_ticket_t * rsc_ticket)
{
if (rsc_ticket->role_lh != RSC_ROLE_UNKNOWN && rsc_ticket->role_lh != rsc_lh->role) {
pe_rsc_trace(rsc_lh, "LH: Skipping constraint: \"%s\" state filter",
role2text(rsc_ticket->role_lh));
return FALSE;
}
return TRUE;
}
void
rsc_ticket_constraint(pe_resource_t * rsc_lh, rsc_ticket_t * rsc_ticket, pe_working_set_t * data_set)
{
if (rsc_ticket == NULL) {
pe_err("rsc_ticket was NULL");
return;
}
if (rsc_lh == NULL) {
pe_err("rsc_lh was NULL for %s", rsc_ticket->id);
return;
}
if (rsc_ticket->ticket->granted && rsc_ticket->ticket->standby == FALSE) {
return;
}
if (rsc_lh->children) {
GListPtr gIter = rsc_lh->children;
pe_rsc_trace(rsc_lh, "Processing ticket dependencies from %s", rsc_lh->id);
for (; gIter != NULL; gIter = gIter->next) {
pe_resource_t *child_rsc = (pe_resource_t *) gIter->data;
rsc_ticket_constraint(child_rsc, rsc_ticket, data_set);
}
return;
}
pe_rsc_trace(rsc_lh, "%s: Processing ticket dependency on %s (%s, %s)",
rsc_lh->id, rsc_ticket->ticket->id, rsc_ticket->id,
role2text(rsc_ticket->role_lh));
if ((rsc_ticket->ticket->granted == FALSE)
&& (rsc_lh->running_on != NULL)) {
GListPtr gIter = NULL;
switch (rsc_ticket->loss_policy) {
case loss_ticket_stop:
resource_location(rsc_lh, NULL, -INFINITY, "__loss_of_ticket__", data_set);
break;
case loss_ticket_demote:
// Promotion score will be set to -INFINITY in promotion_order()
if (rsc_ticket->role_lh != RSC_ROLE_MASTER) {
resource_location(rsc_lh, NULL, -INFINITY, "__loss_of_ticket__", data_set);
}
break;
case loss_ticket_fence:
if (filter_rsc_ticket(rsc_lh, rsc_ticket) == FALSE) {
return;
}
resource_location(rsc_lh, NULL, -INFINITY, "__loss_of_ticket__", data_set);
for (gIter = rsc_lh->running_on; gIter != NULL; gIter = gIter->next) {
pe_node_t *node = (pe_node_t *) gIter->data;
pe_fence_node(data_set, node, "deadman ticket was lost", FALSE);
}
break;
case loss_ticket_freeze:
if (filter_rsc_ticket(rsc_lh, rsc_ticket) == FALSE) {
return;
}
if (rsc_lh->running_on != NULL) {
clear_bit(rsc_lh->flags, pe_rsc_managed);
set_bit(rsc_lh->flags, pe_rsc_block);
}
break;
}
} else if (rsc_ticket->ticket->granted == FALSE) {
if (rsc_ticket->role_lh != RSC_ROLE_MASTER || rsc_ticket->loss_policy == loss_ticket_stop) {
resource_location(rsc_lh, NULL, -INFINITY, "__no_ticket__", data_set);
}
} else if (rsc_ticket->ticket->standby) {
if (rsc_ticket->role_lh != RSC_ROLE_MASTER || rsc_ticket->loss_policy == loss_ticket_stop) {
resource_location(rsc_lh, NULL, -INFINITY, "__ticket_standby__", data_set);
}
}
}
enum pe_action_flags
native_action_flags(pe_action_t * action, pe_node_t * node)
{
return action->flags;
}
static inline bool
is_primitive_action(pe_action_t *action)
{
return action && action->rsc && (action->rsc->variant == pe_native);
}
/*!
* \internal
* \brief Set action bits appropriately when pe_restart_order is used
*
* \param[in] first 'First' action in an ordering with pe_restart_order
* \param[in] then 'Then' action in an ordering with pe_restart_order
* \param[in] filter What ordering flags to care about
*
* \note pe_restart_order is set for "stop resource before starting it" and
* "stop later group member before stopping earlier group member"
*/
static void
handle_restart_ordering(pe_action_t *first, pe_action_t *then,
enum pe_action_flags filter)
{
const char *reason = NULL;
CRM_ASSERT(is_primitive_action(first));
CRM_ASSERT(is_primitive_action(then));
// We need to update the action in two cases:
// ... if 'then' is required
if (is_set(filter, pe_action_optional)
&& is_not_set(then->flags, pe_action_optional)) {
reason = "restart";
}
/* ... if 'then' is unrunnable start of managed resource (if a resource
* should restart but can't start, we still want to stop)
*/
if (is_set(filter, pe_action_runnable)
&& is_not_set(then->flags, pe_action_runnable)
&& is_set(then->rsc->flags, pe_rsc_managed)
&& safe_str_eq(then->task, RSC_START)) {
reason = "stop";
}
if (reason == NULL) {
return;
}
pe_rsc_trace(first->rsc, "Handling %s -> %s for %s",
first->uuid, then->uuid, reason);
// Make 'first' required if it is runnable
if (is_set(first->flags, pe_action_runnable)) {
pe_action_implies(first, then, pe_action_optional);
}
// Make 'first' required if 'then' is required
if (is_not_set(then->flags, pe_action_optional)) {
pe_action_implies(first, then, pe_action_optional);
}
// Make 'first' unmigratable if 'then' is unmigratable
if (is_not_set(then->flags, pe_action_migrate_runnable)) {
pe_action_implies(first, then, pe_action_migrate_runnable);
}
// Make 'then' unrunnable if 'first' is required but unrunnable
if (is_not_set(first->flags, pe_action_optional)
&& is_not_set(first->flags, pe_action_runnable)) {
pe_action_implies(then, first, pe_action_runnable);
}
}
enum pe_graph_flags
native_update_actions(pe_action_t *first, pe_action_t *then, pe_node_t *node,
enum pe_action_flags flags, enum pe_action_flags filter,
enum pe_ordering type, pe_working_set_t *data_set)
{
/* flags == get_action_flags(first, then_node) called from update_action() */
enum pe_graph_flags changed = pe_graph_none;
enum pe_action_flags then_flags = then->flags;
enum pe_action_flags first_flags = first->flags;
crm_trace( "Testing %s on %s (0x%.6x) with %s 0x%.6x",
first->uuid, first->node ? first->node->details->uname : "[none]",
first->flags, then->uuid, then->flags);
if (type & pe_order_asymmetrical) {
pe_resource_t *then_rsc = then->rsc;
enum rsc_role_e then_rsc_role = then_rsc ? then_rsc->fns->state(then_rsc, TRUE) : 0;
if (!then_rsc) {
/* ignore */
} else if ((then_rsc_role == RSC_ROLE_STOPPED) && safe_str_eq(then->task, RSC_STOP)) {
/* ignore... if 'then' is supposed to be stopped after 'first', but
* then is already stopped, there is nothing to be done when non-symmetrical. */
} else if ((then_rsc_role >= RSC_ROLE_STARTED)
&& safe_str_eq(then->task, RSC_START)
&& is_set(then->flags, pe_action_optional)
&& then->node
&& pcmk__list_of_1(then_rsc->running_on)
&& then->node->details == ((pe_node_t *) then_rsc->running_on->data)->details) {
/* Ignore. If 'then' is supposed to be started after 'first', but
* 'then' is already started, there is nothing to be done when
* asymmetrical -- unless the start is mandatory, which indicates
* the resource is restarting, and the ordering is still needed.
*/
} else if (!(first->flags & pe_action_runnable)) {
/* prevent 'then' action from happening if 'first' is not runnable and
* 'then' has not yet occurred. */
pe_action_implies(then, first, pe_action_optional);
pe_action_implies(then, first, pe_action_runnable);
pe_rsc_trace(then->rsc, "Unset optional and runnable on %s", then->uuid);
} else {
/* ignore... then is allowed to start/stop if it wants to. */
}
}
if (type & pe_order_implies_first) {
if (is_set(filter, pe_action_optional) && is_not_set(flags /* Should be then_flags? */, pe_action_optional)) {
// Needs is_set(first_flags, pe_action_optional) too?
pe_rsc_trace(first->rsc, "Unset optional on %s because of %s", first->uuid, then->uuid);
pe_action_implies(first, then, pe_action_optional);
}
if (is_set(flags, pe_action_migrate_runnable) &&
is_set(then->flags, pe_action_migrate_runnable) == FALSE &&
is_set(then->flags, pe_action_optional) == FALSE) {
pe_rsc_trace(first->rsc, "Unset migrate runnable on %s because of %s",
first->uuid, then->uuid);
pe_action_implies(first, then, pe_action_migrate_runnable);
}
}
if (type & pe_order_implies_first_master) {
if ((filter & pe_action_optional) &&
((then->flags & pe_action_optional) == FALSE) &&
then->rsc && (then->rsc->role == RSC_ROLE_MASTER)) {
pe_action_implies(first, then, pe_action_optional);
if (is_set(first->flags, pe_action_migrate_runnable) &&
is_set(then->flags, pe_action_migrate_runnable) == FALSE) {
pe_rsc_trace(first->rsc, "Unset migrate runnable on %s because of %s", first->uuid, then->uuid);
pe_action_implies(first, then, pe_action_migrate_runnable);
}
pe_rsc_trace(then->rsc, "Unset optional on %s because of %s", first->uuid, then->uuid);
}
}
if ((type & pe_order_implies_first_migratable)
&& is_set(filter, pe_action_optional)) {
if (((then->flags & pe_action_migrate_runnable) == FALSE) ||
((then->flags & pe_action_runnable) == FALSE)) {
pe_rsc_trace(then->rsc, "Unset runnable on %s because %s is neither runnable or migratable", first->uuid, then->uuid);
pe_action_implies(first, then, pe_action_runnable);
}
if ((then->flags & pe_action_optional) == 0) {
pe_rsc_trace(then->rsc, "Unset optional on %s because %s is not optional", first->uuid, then->uuid);
pe_action_implies(first, then, pe_action_optional);
}
}
if ((type & pe_order_pseudo_left)
&& is_set(filter, pe_action_optional)) {
if ((first->flags & pe_action_runnable) == FALSE) {
pe_action_implies(then, first, pe_action_migrate_runnable);
pe_clear_action_bit(then, pe_action_pseudo);
pe_rsc_trace(then->rsc, "Unset pseudo on %s because %s is not runnable", then->uuid, first->uuid);
}
}
if (is_set(type, pe_order_runnable_left)
&& is_set(filter, pe_action_runnable)
&& is_set(then->flags, pe_action_runnable)
&& is_set(flags, pe_action_runnable) == FALSE) {
pe_rsc_trace(then->rsc, "Unset runnable on %s because of %s", then->uuid, first->uuid);
pe_action_implies(then, first, pe_action_runnable);
pe_action_implies(then, first, pe_action_migrate_runnable);
}
if (is_set(type, pe_order_implies_then)
&& is_set(filter, pe_action_optional)
&& is_set(then->flags, pe_action_optional)
&& is_set(flags, pe_action_optional) == FALSE) {
/* in this case, treat migrate_runnable as if first is optional */
if (is_set(first->flags, pe_action_migrate_runnable) == FALSE) {
pe_rsc_trace(then->rsc, "Unset optional on %s because of %s", then->uuid, first->uuid);
pe_action_implies(then, first, pe_action_optional);
}
}
if (is_set(type, pe_order_restart)) {
handle_restart_ordering(first, then, filter);
}
if (then_flags != then->flags) {
changed |= pe_graph_updated_then;
pe_rsc_trace(then->rsc,
"Then: Flags for %s on %s are now 0x%.6x (was 0x%.6x) because of %s 0x%.6x",
then->uuid, then->node ? then->node->details->uname : "[none]", then->flags,
then_flags, first->uuid, first->flags);
if(then->rsc && then->rsc->parent) {
/* "X_stop then X_start" doesn't get handled for cloned groups unless we do this */
update_action(then, data_set);
}
}
if (first_flags != first->flags) {
changed |= pe_graph_updated_first;
pe_rsc_trace(first->rsc,
"First: Flags for %s on %s are now 0x%.6x (was 0x%.6x) because of %s 0x%.6x",
first->uuid, first->node ? first->node->details->uname : "[none]",
first->flags, first_flags, then->uuid, then->flags);
}
return changed;
}
void
native_rsc_location(pe_resource_t *rsc, pe__location_t *constraint)
{
GListPtr gIter = NULL;
GHashTableIter iter;
pe_node_t *node = NULL;
if (constraint == NULL) {
pe_err("Constraint is NULL");
return;
} else if (rsc == NULL) {
pe_err("LHS of rsc_to_node (%s) is NULL", constraint->id);
return;
}
pe_rsc_trace(rsc, "Applying %s (%s) to %s", constraint->id,
role2text(constraint->role_filter), rsc->id);
/* take "lifetime" into account */
if (constraint->role_filter > RSC_ROLE_UNKNOWN && constraint->role_filter != rsc->next_role) {
pe_rsc_debug(rsc, "Constraint (%s) is not active (role : %s vs. %s)",
constraint->id, role2text(constraint->role_filter), role2text(rsc->next_role));
return;
}
if (constraint->node_list_rh == NULL) {
pe_rsc_trace(rsc, "RHS of constraint %s is NULL", constraint->id);
return;
}
for (gIter = constraint->node_list_rh; gIter != NULL; gIter = gIter->next) {
pe_node_t *node = (pe_node_t *) gIter->data;
pe_node_t *other_node = NULL;
other_node = (pe_node_t *) pe_hash_table_lookup(rsc->allowed_nodes, node->details->id);
if (other_node != NULL) {
pe_rsc_trace(rsc, "%s + %s: %d + %d",
node->details->uname,
other_node->details->uname, node->weight, other_node->weight);
other_node->weight = pe__add_scores(other_node->weight,
node->weight);
} else {
other_node = pe__copy_node(node);
pe_rsc_trace(rsc, "%s: %d (insert %d)", other_node->details->uname, other_node->weight, constraint->discover_mode);
g_hash_table_insert(rsc->allowed_nodes, (gpointer) other_node->details->id, other_node);
}
if (other_node->rsc_discover_mode < constraint->discover_mode) {
if (constraint->discover_mode == pe_discover_exclusive) {
rsc->exclusive_discover = TRUE;
}
/* exclusive > never > always... always is default */
other_node->rsc_discover_mode = constraint->discover_mode;
}
}
g_hash_table_iter_init(&iter, rsc->allowed_nodes);
while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
pe_rsc_trace(rsc, "%s + %s : %d", rsc->id, node->details->uname, node->weight);
}
}
void
native_expand(pe_resource_t * rsc, pe_working_set_t * data_set)
{
GListPtr gIter = NULL;
CRM_ASSERT(rsc);
pe_rsc_trace(rsc, "Processing actions from %s", rsc->id);
for (gIter = rsc->actions; gIter != NULL; gIter = gIter->next) {
pe_action_t *action = (pe_action_t *) gIter->data;
crm_trace("processing action %d for rsc=%s", action->id, rsc->id);
graph_element_from_action(action, data_set);
}
for (gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
pe_resource_t *child_rsc = (pe_resource_t *) gIter->data;
child_rsc->cmds->expand(child_rsc, data_set);
}
}
#define log_change(a, fmt, args...) do { \
if(a && a->reason && terminal) { \
printf(" * "fmt" \tdue to %s\n", ##args, a->reason); \
} else if(a && a->reason) { \
crm_notice(fmt" \tdue to %s", ##args, a->reason); \
} else if(terminal) { \
printf(" * "fmt"\n", ##args); \
} else { \
crm_notice(fmt, ##args); \
} \
} while(0)
#define STOP_SANITY_ASSERT(lineno) do { \
if(current && current->details->unclean) { \
/* It will be a pseudo op */ \
} else if(stop == NULL) { \
crm_err("%s:%d: No stop action exists for %s", __FUNCTION__, lineno, rsc->id); \
CRM_ASSERT(stop != NULL); \
} else if(is_set(stop->flags, pe_action_optional)) { \
crm_err("%s:%d: Action %s is still optional", __FUNCTION__, lineno, stop->uuid); \
CRM_ASSERT(is_not_set(stop->flags, pe_action_optional)); \
} \
} while(0)
-static int rsc_width = 5;
-static int detail_width = 5;
static void
LogAction(const char *change, pe_resource_t *rsc, pe_node_t *origin, pe_node_t *destination, pe_action_t *action, pe_action_t *source, gboolean terminal)
{
int len = 0;
char *reason = NULL;
char *details = NULL;
bool same_host = FALSE;
bool same_role = FALSE;
bool need_role = FALSE;
+ static int rsc_width = 5;
+ static int detail_width = 5;
+
CRM_ASSERT(action);
CRM_ASSERT(destination != NULL || origin != NULL);
if(source == NULL) {
source = action;
}
len = strlen(rsc->id);
if(len > rsc_width) {
rsc_width = len + 2;
}
if(rsc->role > RSC_ROLE_STARTED || rsc->next_role > RSC_ROLE_SLAVE) {
need_role = TRUE;
}
if(origin != NULL && destination != NULL && origin->details == destination->details) {
same_host = TRUE;
}
if(rsc->role == rsc->next_role) {
same_role = TRUE;
}
- if(need_role && origin == NULL) {
- /* Promoting from Stopped */
+ if (need_role && (origin == NULL)) {
+ /* Starting and promoting a promotable clone instance */
details = crm_strdup_printf("%s -> %s %s", role2text(rsc->role), role2text(rsc->next_role), destination->details->uname);
- } else if(need_role && destination == NULL) {
- /* Demoting a Master or Stopping a Slave */
+ } else if (origin == NULL) {
+ /* Starting a resource */
+ details = crm_strdup_printf("%s", destination->details->uname);
+
+ } else if (need_role && (destination == NULL)) {
+ /* Stopping a promotable clone instance */
details = crm_strdup_printf("%s %s", role2text(rsc->role), origin->details->uname);
- } else if(origin == NULL || destination == NULL) {
- /* Starting or stopping a resource */
- details = crm_strdup_printf("%s", origin?origin->details->uname:destination->details->uname);
+ } else if (destination == NULL) {
+ /* Stopping a resource */
+ details = crm_strdup_printf("%s", origin->details->uname);
- } else if(need_role && same_role && same_host) {
- /* Recovering or restarting a promotable clone resource */
+ } else if (need_role && same_role && same_host) {
+ /* Recovering, restarting or re-promoting a promotable clone instance */
details = crm_strdup_printf("%s %s", role2text(rsc->role), origin->details->uname);
- } else if(same_role && same_host) {
+ } else if (same_role && same_host) {
/* Recovering or Restarting a normal resource */
details = crm_strdup_printf("%s", origin->details->uname);
- } else if(same_role && need_role) {
- /* Moving a promotable clone resource */
+ } else if (need_role && same_role) {
+ /* Moving a promotable clone instance */
details = crm_strdup_printf("%s -> %s %s", origin->details->uname, destination->details->uname, role2text(rsc->role));
- } else if(same_role) {
+ } else if (same_role) {
/* Moving a normal resource */
details = crm_strdup_printf("%s -> %s", origin->details->uname, destination->details->uname);
- } else if(same_host) {
- /* Promoting or demoting a promotable clone resource */
+ } else if (same_host) {
+ /* Promoting or demoting a promotable clone instance */
details = crm_strdup_printf("%s -> %s %s", role2text(rsc->role), role2text(rsc->next_role), origin->details->uname);
} else {
/* Moving and promoting/demoting */
details = crm_strdup_printf("%s %s -> %s %s", role2text(rsc->role), origin->details->uname, role2text(rsc->next_role), destination->details->uname);
}
len = strlen(details);
if(len > detail_width) {
detail_width = len;
}
if(source->reason && is_not_set(action->flags, pe_action_runnable)) {
reason = crm_strdup_printf(" due to %s (blocked)", source->reason);
} else if(source->reason) {
reason = crm_strdup_printf(" due to %s", source->reason);
} else if(is_not_set(action->flags, pe_action_runnable)) {
reason = strdup(" blocked");
} else {
reason = strdup("");
}
if(terminal) {
printf(" * %-8s %-*s ( %*s ) %s\n", change, rsc_width, rsc->id, detail_width, details, reason);
} else {
crm_notice(" * %-8s %-*s ( %*s ) %s", change, rsc_width, rsc->id, detail_width, details, reason);
}
free(details);
free(reason);
}
void
LogActions(pe_resource_t * rsc, pe_working_set_t * data_set, gboolean terminal)
{
pe_node_t *next = NULL;
pe_node_t *current = NULL;
pe_node_t *start_node = NULL;
pe_action_t *stop = NULL;
pe_action_t *start = NULL;
pe_action_t *demote = NULL;
pe_action_t *promote = NULL;
char *key = NULL;
gboolean moving = FALSE;
GListPtr possible_matches = NULL;
if(rsc->variant == pe_container) {
pcmk__bundle_log_actions(rsc, data_set, terminal);
return;
}
if (rsc->children) {
GListPtr gIter = NULL;
for (gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
pe_resource_t *child_rsc = (pe_resource_t *) gIter->data;
LogActions(child_rsc, data_set, terminal);
}
return;
}
next = rsc->allocated_to;
if (rsc->running_on) {
current = pe__current_node(rsc);
if (rsc->role == RSC_ROLE_STOPPED) {
/*
* This can occur when resources are being recovered
* We fiddle with the current role in native_create_actions()
*/
rsc->role = RSC_ROLE_STARTED;
}
}
if (current == NULL && is_set(rsc->flags, pe_rsc_orphan)) {
/* Don't log stopped orphans */
return;
}
if (is_not_set(rsc->flags, pe_rsc_managed)
|| (current == NULL && next == NULL)) {
pe_rsc_info(rsc, "Leave %s\t(%s%s)",
rsc->id, role2text(rsc->role), is_not_set(rsc->flags,
pe_rsc_managed) ? " unmanaged" : "");
return;
}
if (current != NULL && next != NULL && safe_str_neq(current->details->id, next->details->id)) {
moving = TRUE;
}
possible_matches = pe__resource_actions(rsc, next, RSC_START, FALSE);
if (possible_matches) {
start = possible_matches->data;
g_list_free(possible_matches);
}
if ((start == NULL) || is_not_set(start->flags, pe_action_runnable)) {
start_node = NULL;
} else {
start_node = current;
}
possible_matches = pe__resource_actions(rsc, start_node, RSC_STOP, FALSE);
if (possible_matches) {
stop = possible_matches->data;
g_list_free(possible_matches);
}
possible_matches = pe__resource_actions(rsc, next, RSC_PROMOTE, FALSE);
if (possible_matches) {
promote = possible_matches->data;
g_list_free(possible_matches);
}
possible_matches = pe__resource_actions(rsc, next, RSC_DEMOTE, FALSE);
if (possible_matches) {
demote = possible_matches->data;
g_list_free(possible_matches);
}
if (rsc->role == rsc->next_role) {
pe_action_t *migrate_op = NULL;
possible_matches = pe__resource_actions(rsc, next, RSC_MIGRATED, FALSE);
if (possible_matches) {
migrate_op = possible_matches->data;
}
CRM_CHECK(next != NULL,);
if (next == NULL) {
} else if (migrate_op && is_set(migrate_op->flags, pe_action_runnable) && current) {
LogAction("Migrate", rsc, current, next, start, NULL, terminal);
} else if (is_set(rsc->flags, pe_rsc_reload)) {
LogAction("Reload", rsc, current, next, start, NULL, terminal);
+
} else if (start == NULL || is_set(start->flags, pe_action_optional)) {
- pe_rsc_info(rsc, "Leave %s\t(%s %s)", rsc->id, role2text(rsc->role),
- next->details->uname);
+ if ((demote != NULL) && (promote != NULL)
+ && is_not_set(demote->flags, pe_action_optional)
+ && is_not_set(promote->flags, pe_action_optional)) {
+ LogAction("Re-promote", rsc, current, next, promote, demote,
+ terminal);
+ } else {
+ pe_rsc_info(rsc, "Leave %s\t(%s %s)", rsc->id,
+ role2text(rsc->role), next->details->uname);
+ }
- } else if (start && is_set(start->flags, pe_action_runnable) == FALSE) {
+ } else if (is_not_set(start->flags, pe_action_runnable)) {
LogAction("Stop", rsc, current, NULL, stop,
(stop && stop->reason)? stop : start, terminal);
STOP_SANITY_ASSERT(__LINE__);
} else if (moving && current) {
LogAction(is_set(rsc->flags, pe_rsc_failed) ? "Recover" : "Move",
rsc, current, next, stop, NULL, terminal);
} else if (is_set(rsc->flags, pe_rsc_failed)) {
LogAction("Recover", rsc, current, NULL, stop, NULL, terminal);
STOP_SANITY_ASSERT(__LINE__);
} else {
LogAction("Restart", rsc, current, next, start, NULL, terminal);
/* STOP_SANITY_ASSERT(__LINE__); False positive for migrate-fail-7 */
}
g_list_free(possible_matches);
return;
}
if(stop
&& (rsc->next_role == RSC_ROLE_STOPPED
|| (start && is_not_set(start->flags, pe_action_runnable)))) {
GListPtr gIter = NULL;
key = stop_key(rsc);
for (gIter = rsc->running_on; gIter != NULL; gIter = gIter->next) {
pe_node_t *node = (pe_node_t *) gIter->data;
pe_action_t *stop_op = NULL;
possible_matches = find_actions(rsc->actions, key, node);
if (possible_matches) {
stop_op = possible_matches->data;
g_list_free(possible_matches);
}
if (stop_op && (stop_op->flags & pe_action_runnable)) {
STOP_SANITY_ASSERT(__LINE__);
}
LogAction("Stop", rsc, node, NULL, stop_op,
(stop_op && stop_op->reason)? stop_op : start, terminal);
}
free(key);
- } else if (stop && is_set(rsc->flags, pe_rsc_failed)) {
+ } else if (stop && is_set(rsc->flags, pe_rsc_failed)
+ && is_set(rsc->flags, pe_rsc_stop)) {
/* 'stop' may be NULL if the failure was ignored */
LogAction("Recover", rsc, current, next, stop, start, terminal);
STOP_SANITY_ASSERT(__LINE__);
} else if (moving) {
LogAction("Move", rsc, current, next, stop, NULL, terminal);
STOP_SANITY_ASSERT(__LINE__);
} else if (is_set(rsc->flags, pe_rsc_reload)) {
LogAction("Reload", rsc, current, next, start, NULL, terminal);
} else if (stop != NULL && is_not_set(stop->flags, pe_action_optional)) {
LogAction("Restart", rsc, current, next, start, NULL, terminal);
STOP_SANITY_ASSERT(__LINE__);
} else if (rsc->role == RSC_ROLE_MASTER) {
CRM_LOG_ASSERT(current != NULL);
LogAction("Demote", rsc, current, next, demote, NULL, terminal);
} else if(rsc->next_role == RSC_ROLE_MASTER) {
CRM_LOG_ASSERT(next);
LogAction("Promote", rsc, current, next, promote, NULL, terminal);
} else if (rsc->role == RSC_ROLE_STOPPED && rsc->next_role > RSC_ROLE_STOPPED) {
LogAction("Start", rsc, current, next, start, NULL, terminal);
}
}
gboolean
StopRsc(pe_resource_t * rsc, pe_node_t * next, gboolean optional, pe_working_set_t * data_set)
{
GListPtr gIter = NULL;
CRM_ASSERT(rsc);
pe_rsc_trace(rsc, "%s", rsc->id);
for (gIter = rsc->running_on; gIter != NULL; gIter = gIter->next) {
pe_node_t *current = (pe_node_t *) gIter->data;
pe_action_t *stop;
if (rsc->partial_migration_target) {
if (rsc->partial_migration_target->details == current->details) {
pe_rsc_trace(rsc, "Filtered %s -> %s %s", current->details->uname,
next->details->uname, rsc->id);
continue;
} else {
pe_rsc_trace(rsc, "Forced on %s %s", current->details->uname, rsc->id);
optional = FALSE;
}
}
pe_rsc_trace(rsc, "%s on %s", rsc->id, current->details->uname);
stop = stop_action(rsc, current, optional);
if(rsc->allocated_to == NULL) {
pe_action_set_reason(stop, "node availability", TRUE);
}
if (is_not_set(rsc->flags, pe_rsc_managed)) {
update_action_flags(stop, pe_action_runnable | pe_action_clear, __FUNCTION__, __LINE__);
}
if (is_set(data_set->flags, pe_flag_remove_after_stop)) {
DeleteRsc(rsc, current, optional, data_set);
}
if(is_set(rsc->flags, pe_rsc_needs_unfencing)) {
pe_action_t *unfence = pe_fence_op(current, "on", TRUE, NULL, FALSE, data_set);
order_actions(stop, unfence, pe_order_implies_first);
if (!node_has_been_unfenced(current)) {
pe_proc_err("Stopping %s until %s can be unfenced", rsc->id, current->details->uname);
}
}
}
return TRUE;
}
static void
order_after_unfencing(pe_resource_t *rsc, pe_node_t *node, pe_action_t *action,
enum pe_ordering order, pe_working_set_t *data_set)
{
/* When unfencing is in use, we order unfence actions before any probe or
* start of resources that require unfencing, and also of fence devices.
*
* This might seem to violate the principle that fence devices require
* only quorum. However, fence agents that unfence often don't have enough
* information to even probe or start unless the node is first unfenced.
*/
if (is_unfence_device(rsc, data_set)
|| is_set(rsc->flags, pe_rsc_needs_unfencing)) {
/* Start with an optional ordering. Requiring unfencing would result in
* the node being unfenced, and all its resources being stopped,
* whenever a new resource is added -- which would be highly suboptimal.
*/
pe_action_t *unfence = pe_fence_op(node, "on", TRUE, NULL, FALSE, data_set);
order_actions(unfence, action, order);
if (!node_has_been_unfenced(node)) {
// But unfencing is required if it has never been done
char *reason = crm_strdup_printf("required by %s %s",
rsc->id, action->task);
trigger_unfencing(NULL, node, reason, NULL, data_set);
free(reason);
}
}
}
gboolean
StartRsc(pe_resource_t * rsc, pe_node_t * next, gboolean optional, pe_working_set_t * data_set)
{
pe_action_t *start = NULL;
CRM_ASSERT(rsc);
pe_rsc_trace(rsc, "%s on %s %d %d", rsc->id, next ? next->details->uname : "N/A", optional, next ? next->weight : 0);
start = start_action(rsc, next, TRUE);
order_after_unfencing(rsc, next, start, pe_order_implies_then, data_set);
if (is_set(start->flags, pe_action_runnable) && optional == FALSE) {
update_action_flags(start, pe_action_optional | pe_action_clear, __FUNCTION__, __LINE__);
}
return TRUE;
}
gboolean
PromoteRsc(pe_resource_t * rsc, pe_node_t * next, gboolean optional, pe_working_set_t * data_set)
{
GListPtr gIter = NULL;
gboolean runnable = TRUE;
GListPtr action_list = NULL;
CRM_ASSERT(rsc);
CRM_CHECK(next != NULL, return FALSE);
pe_rsc_trace(rsc, "%s on %s", rsc->id, next->details->uname);
action_list = pe__resource_actions(rsc, next, RSC_START, TRUE);
for (gIter = action_list; gIter != NULL; gIter = gIter->next) {
pe_action_t *start = (pe_action_t *) gIter->data;
if (is_set(start->flags, pe_action_runnable) == FALSE) {
runnable = FALSE;
}
}
g_list_free(action_list);
if (runnable) {
promote_action(rsc, next, optional);
return TRUE;
}
pe_rsc_debug(rsc, "%s\tPromote %s (canceled)", next->details->uname, rsc->id);
action_list = pe__resource_actions(rsc, next, RSC_PROMOTE, TRUE);
for (gIter = action_list; gIter != NULL; gIter = gIter->next) {
pe_action_t *promote = (pe_action_t *) gIter->data;
update_action_flags(promote, pe_action_runnable | pe_action_clear, __FUNCTION__, __LINE__);
}
g_list_free(action_list);
return TRUE;
}
gboolean
DemoteRsc(pe_resource_t * rsc, pe_node_t * next, gboolean optional, pe_working_set_t * data_set)
{
GListPtr gIter = NULL;
CRM_ASSERT(rsc);
pe_rsc_trace(rsc, "%s", rsc->id);
/* CRM_CHECK(rsc->next_role == RSC_ROLE_SLAVE, return FALSE); */
for (gIter = rsc->running_on; gIter != NULL; gIter = gIter->next) {
pe_node_t *current = (pe_node_t *) gIter->data;
pe_rsc_trace(rsc, "%s on %s", rsc->id, next ? next->details->uname : "N/A");
demote_action(rsc, current, optional);
}
return TRUE;
}
gboolean
RoleError(pe_resource_t * rsc, pe_node_t * next, gboolean optional, pe_working_set_t * data_set)
{
CRM_ASSERT(rsc);
crm_err("%s on %s", rsc->id, next ? next->details->uname : "N/A");
CRM_CHECK(FALSE, return FALSE);
return FALSE;
}
gboolean
NullOp(pe_resource_t * rsc, pe_node_t * next, gboolean optional, pe_working_set_t * data_set)
{
CRM_ASSERT(rsc);
pe_rsc_trace(rsc, "%s", rsc->id);
return FALSE;
}
gboolean
DeleteRsc(pe_resource_t * rsc, pe_node_t * node, gboolean optional, pe_working_set_t * data_set)
{
if (is_set(rsc->flags, pe_rsc_failed)) {
pe_rsc_trace(rsc, "Resource %s not deleted from %s: failed", rsc->id, node->details->uname);
return FALSE;
} else if (node == NULL) {
pe_rsc_trace(rsc, "Resource %s not deleted: NULL node", rsc->id);
return FALSE;
} else if (node->details->unclean || node->details->online == FALSE) {
pe_rsc_trace(rsc, "Resource %s not deleted from %s: unrunnable", rsc->id,
node->details->uname);
return FALSE;
}
crm_notice("Removing %s from %s", rsc->id, node->details->uname);
delete_action(rsc, node, optional);
new_rsc_order(rsc, RSC_STOP, rsc, RSC_DELETE,
optional ? pe_order_implies_then : pe_order_optional, data_set);
new_rsc_order(rsc, RSC_DELETE, rsc, RSC_START,
optional ? pe_order_implies_then : pe_order_optional, data_set);
return TRUE;
}
gboolean
native_create_probe(pe_resource_t * rsc, pe_node_t * node, pe_action_t * complete,
gboolean force, pe_working_set_t * data_set)
{
enum pe_ordering flags = pe_order_optional;
char *key = NULL;
pe_action_t *probe = NULL;
pe_node_t *running = NULL;
pe_node_t *allowed = NULL;
pe_resource_t *top = uber_parent(rsc);
static const char *rc_master = NULL;
static const char *rc_inactive = NULL;
if (rc_inactive == NULL) {
rc_inactive = crm_itoa(PCMK_OCF_NOT_RUNNING);
rc_master = crm_itoa(PCMK_OCF_RUNNING_MASTER);
}
CRM_CHECK(node != NULL, return FALSE);
if (force == FALSE && is_not_set(data_set->flags, pe_flag_startup_probes)) {
pe_rsc_trace(rsc, "Skipping active resource detection for %s", rsc->id);
return FALSE;
}
if (pe__is_guest_or_remote_node(node)) {
const char *class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
if (safe_str_eq(class, PCMK_RESOURCE_CLASS_STONITH)) {
pe_rsc_trace(rsc,
"Skipping probe for %s on %s because Pacemaker Remote nodes cannot run stonith agents",
rsc->id, node->details->id);
return FALSE;
} else if (pe__is_guest_node(node)
&& pe__resource_contains_guest_node(data_set, rsc)) {
pe_rsc_trace(rsc,
"Skipping probe for %s on %s because guest nodes cannot run resources containing guest nodes",
rsc->id, node->details->id);
return FALSE;
} else if (rsc->is_remote_node) {
pe_rsc_trace(rsc,
"Skipping probe for %s on %s because Pacemaker Remote nodes cannot host remote connections",
rsc->id, node->details->id);
return FALSE;
}
}
if (rsc->children) {
GListPtr gIter = NULL;
gboolean any_created = FALSE;
for (gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
pe_resource_t *child_rsc = (pe_resource_t *) gIter->data;
any_created = child_rsc->cmds->create_probe(child_rsc, node, complete, force, data_set)
|| any_created;
}
return any_created;
} else if ((rsc->container) && (!rsc->is_remote_node)) {
pe_rsc_trace(rsc, "Skipping %s: it is within container %s", rsc->id, rsc->container->id);
return FALSE;
}
if (is_set(rsc->flags, pe_rsc_orphan)) {
pe_rsc_trace(rsc, "Skipping orphan: %s", rsc->id);
return FALSE;
}
// Check whether resource is already known on node
if (!force && g_hash_table_lookup(rsc->known_on, node->details->id)) {
pe_rsc_trace(rsc, "Skipping known: %s on %s", rsc->id, node->details->uname);
return FALSE;
}
allowed = g_hash_table_lookup(rsc->allowed_nodes, node->details->id);
if (rsc->exclusive_discover || top->exclusive_discover) {
if (allowed == NULL) {
/* exclusive discover is enabled and this node is not in the allowed list. */
pe_rsc_trace(rsc, "Skipping probe for %s on node %s, A", rsc->id, node->details->id);
return FALSE;
} else if (allowed->rsc_discover_mode != pe_discover_exclusive) {
/* exclusive discover is enabled and this node is not marked
* as a node this resource should be discovered on */
pe_rsc_trace(rsc, "Skipping probe for %s on node %s, B", rsc->id, node->details->id);
return FALSE;
}
}
if(allowed == NULL && node->rsc_discover_mode == pe_discover_never) {
/* If this node was allowed to host this resource it would
* have been explicitly added to the 'allowed_nodes' list.
* However it wasn't and the node has discovery disabled, so
* no need to probe for this resource.
*/
pe_rsc_trace(rsc, "Skipping probe for %s on node %s, C", rsc->id, node->details->id);
return FALSE;
}
if (allowed && allowed->rsc_discover_mode == pe_discover_never) {
/* this resource is marked as not needing to be discovered on this node */
pe_rsc_trace(rsc, "Skipping probe for %s on node %s, discovery mode", rsc->id, node->details->id);
return FALSE;
}
if (pe__is_guest_node(node)) {
pe_resource_t *remote = node->details->remote_rsc->container;
if(remote->role == RSC_ROLE_STOPPED) {
/* If the container is stopped, then we know anything that
* might have been inside it is also stopped and there is
* no need to probe.
*
* If we don't know the container's state on the target
* either:
*
* - the container is running, the transition will abort
* and we'll end up in a different case next time, or
*
* - the container is stopped
*
* Either way there is no need to probe.
*
*/
if(remote->allocated_to
&& g_hash_table_lookup(remote->known_on, remote->allocated_to->details->id) == NULL) {
/* For safety, we order the 'rsc' start after 'remote'
* has been probed.
*
* Using 'top' helps for groups, but we may need to
* follow the start's ordering chain backwards.
*/
custom_action_order(remote,
pcmk__op_key(remote->id, RSC_STATUS, 0),
NULL, top,
pcmk__op_key(top->id, RSC_START, 0), NULL,
pe_order_optional, data_set);
}
pe_rsc_trace(rsc, "Skipping probe for %s on node %s, %s is stopped",
rsc->id, node->details->id, remote->id);
return FALSE;
/* Here we really we want to check if remote->stop is required,
* but that information doesn't exist yet
*/
} else if(node->details->remote_requires_reset
|| node->details->unclean
|| is_set(remote->flags, pe_rsc_failed)
|| remote->next_role == RSC_ROLE_STOPPED
|| (remote->allocated_to
&& pe_find_node(remote->running_on, remote->allocated_to->details->uname) == NULL)
) {
/* The container is stopping or restarting, don't start
* 'rsc' until 'remote' stops as this also implies that
* 'rsc' is stopped - avoiding the need to probe
*/
custom_action_order(remote, pcmk__op_key(remote->id, RSC_STOP, 0),
NULL, top, pcmk__op_key(top->id, RSC_START, 0),
NULL, pe_order_optional, data_set);
pe_rsc_trace(rsc, "Skipping probe for %s on node %s, %s is stopping, restarting or moving",
rsc->id, node->details->id, remote->id);
return FALSE;
/* } else {
* The container is running so there is no problem probing it
*/
}
}
key = pcmk__op_key(rsc->id, RSC_STATUS, 0);
probe = custom_action(rsc, key, RSC_STATUS, node, FALSE, TRUE, data_set);
update_action_flags(probe, pe_action_optional | pe_action_clear, __FUNCTION__, __LINE__);
order_after_unfencing(rsc, node, probe, pe_order_optional, data_set);
/*
* We need to know if it's running_on (not just known_on) this node
* to correctly determine the target rc.
*/
running = pe_find_node_id(rsc->running_on, node->details->id);
if (running == NULL) {
add_hash_param(probe->meta, XML_ATTR_TE_TARGET_RC, rc_inactive);
} else if (rsc->role == RSC_ROLE_MASTER) {
add_hash_param(probe->meta, XML_ATTR_TE_TARGET_RC, rc_master);
}
crm_debug("Probing %s on %s (%s) %d %p", rsc->id, node->details->uname, role2text(rsc->role),
is_set(probe->flags, pe_action_runnable), rsc->running_on);
if (is_unfence_device(rsc, data_set) || !pe_rsc_is_clone(top)) {
top = rsc;
} else {
crm_trace("Probing %s on %s (%s) as %s", rsc->id, node->details->uname, role2text(rsc->role), top->id);
}
if(is_not_set(probe->flags, pe_action_runnable) && rsc->running_on == NULL) {
/* Prevent the start from occurring if rsc isn't active, but
* don't cause it to stop if it was active already
*/
flags |= pe_order_runnable_left;
}
custom_action_order(rsc, NULL, probe,
top, pcmk__op_key(top->id, RSC_START, 0), NULL,
flags, data_set);
/* Before any reloads, if they exist */
custom_action_order(rsc, NULL, probe,
top, reload_key(rsc), NULL,
pe_order_optional, data_set);
#if 0
// complete is always null currently
if (!is_unfence_device(rsc, data_set)) {
/* Normally rsc.start depends on probe complete which depends
* on rsc.probe. But this can't be the case for fence devices
* with unfencing, as it would create graph loops.
*
* So instead we explicitly order 'rsc.probe then rsc.start'
*/
order_actions(probe, complete, pe_order_implies_then);
}
#endif
return TRUE;
}
/*!
* \internal
* \brief Check whether a resource is known on a particular node
*
* \param[in] rsc Resource to check
* \param[in] node Node to check
*
* \return TRUE if resource (or parent if an anonymous clone) is known
*/
static bool
rsc_is_known_on(pe_resource_t *rsc, const pe_node_t *node)
{
if (pe_hash_table_lookup(rsc->known_on, node->details->id)) {
return TRUE;
} else if ((rsc->variant == pe_native)
&& pe_rsc_is_anon_clone(rsc->parent)
&& pe_hash_table_lookup(rsc->parent->known_on, node->details->id)) {
/* We check only the parent, not the uber-parent, because we cannot
* assume that the resource is known if it is in an anonymously cloned
* group (which may be only partially known).
*/
return TRUE;
}
return FALSE;
}
/*!
* \internal
* \brief Order a resource's start and promote actions relative to fencing
*
* \param[in] rsc Resource to be ordered
* \param[in] stonith_op Fence action
* \param[in] data_set Cluster information
*/
static void
native_start_constraints(pe_resource_t * rsc, pe_action_t * stonith_op, pe_working_set_t * data_set)
{
pe_node_t *target;
GListPtr gIter = NULL;
CRM_CHECK(stonith_op && stonith_op->node, return);
target = stonith_op->node;
for (gIter = rsc->actions; gIter != NULL; gIter = gIter->next) {
pe_action_t *action = (pe_action_t *) gIter->data;
switch (action->needs) {
case rsc_req_nothing:
// Anything other than start or promote requires nothing
break;
case rsc_req_stonith:
order_actions(stonith_op, action, pe_order_optional);
break;
case rsc_req_quorum:
if (safe_str_eq(action->task, RSC_START)
&& pe_hash_table_lookup(rsc->allowed_nodes, target->details->id)
&& !rsc_is_known_on(rsc, target)) {
/* If we don't know the status of the resource on the node
* we're about to shoot, we have to assume it may be active
* there. Order the resource start after the fencing. This
* is analogous to waiting for all the probes for a resource
* to complete before starting it.
*
* The most likely explanation is that the DC died and took
* its status with it.
*/
pe_rsc_debug(rsc, "Ordering %s after %s recovery", action->uuid,
target->details->uname);
order_actions(stonith_op, action,
pe_order_optional | pe_order_runnable_left);
}
break;
}
}
}
static void
native_stop_constraints(pe_resource_t * rsc, pe_action_t * stonith_op, pe_working_set_t * data_set)
{
GListPtr gIter = NULL;
GListPtr action_list = NULL;
bool order_implicit = false;
pe_resource_t *top = uber_parent(rsc);
pe_action_t *parent_stop = NULL;
pe_node_t *target;
CRM_CHECK(stonith_op && stonith_op->node, return);
target = stonith_op->node;
/* Get a list of stop actions potentially implied by the fencing */
action_list = pe__resource_actions(rsc, target, RSC_STOP, FALSE);
/* If resource requires fencing, implicit actions must occur after fencing.
*
* Implied stops and demotes of resources running on guest nodes are always
* ordered after fencing, even if the resource does not require fencing,
* because guest node "fencing" is actually just a resource stop.
*/
if (is_set(rsc->flags, pe_rsc_needs_fencing) || pe__is_guest_node(target)) {
order_implicit = true;
}
if (action_list && order_implicit) {
parent_stop = find_first_action(top->actions, NULL, RSC_STOP, NULL);
}
for (gIter = action_list; gIter != NULL; gIter = gIter->next) {
pe_action_t *action = (pe_action_t *) gIter->data;
// The stop would never complete, so convert it into a pseudo-action.
update_action_flags(action, pe_action_pseudo|pe_action_runnable,
__FUNCTION__, __LINE__);
if (order_implicit) {
update_action_flags(action, pe_action_implied_by_stonith,
__FUNCTION__, __LINE__);
/* Order the stonith before the parent stop (if any).
*
* Also order the stonith before the resource stop, unless the
* resource is inside a bundle -- that would cause a graph loop.
* We can rely on the parent stop's ordering instead.
*
* User constraints must not order a resource in a guest node
* relative to the guest node container resource. The
* pe_order_preserve flag marks constraints as generated by the
* cluster and thus immune to that check (and is irrelevant if
* target is not a guest).
*/
if (!pe_rsc_is_bundled(rsc)) {
order_actions(stonith_op, action, pe_order_preserve);
}
order_actions(stonith_op, parent_stop, pe_order_preserve);
}
if (is_set(rsc->flags, pe_rsc_failed)) {
crm_notice("Stop of failed resource %s is implicit %s %s is fenced",
rsc->id, (order_implicit? "after" : "because"),
target->details->uname);
} else {
crm_info("%s is implicit %s %s is fenced",
action->uuid, (order_implicit? "after" : "because"),
target->details->uname);
}
if (is_set(rsc->flags, pe_rsc_notify)) {
/* Create a second notification that will be delivered
* immediately after the node is fenced
*
* Basic problem:
* - C is a clone active on the node to be shot and stopping on another
* - R is a resource that depends on C
*
* + C.stop depends on R.stop
* + C.stopped depends on STONITH
* + C.notify depends on C.stopped
* + C.healthy depends on C.notify
* + R.stop depends on C.healthy
*
* The extra notification here changes
* + C.healthy depends on C.notify
* into:
* + C.healthy depends on C.notify'
* + C.notify' depends on STONITH'
* thus breaking the loop
*/
create_secondary_notification(action, rsc, stonith_op, data_set);
}
/* From Bug #1601, successful fencing must be an input to a failed resources stop action.
However given group(rA, rB) running on nodeX and B.stop has failed,
A := stop healthy resource (rA.stop)
B := stop failed resource (pseudo operation B.stop)
C := stonith nodeX
A requires B, B requires C, C requires A
This loop would prevent the cluster from making progress.
This block creates the "C requires A" dependency and therefore must (at least
for now) be disabled.
Instead, run the block above and treat all resources on nodeX as B would be
(marked as a pseudo op depending on the STONITH).
TODO: Break the "A requires B" dependency in update_action() and re-enable this block
} else if(is_stonith == FALSE) {
crm_info("Moving healthy resource %s"
" off %s before fencing",
rsc->id, node->details->uname);
* stop healthy resources before the
* stonith op
*
custom_action_order(
rsc, stop_key(rsc), NULL,
NULL,strdup(CRM_OP_FENCE),stonith_op,
pe_order_optional, data_set);
*/
}
g_list_free(action_list);
/* Get a list of demote actions potentially implied by the fencing */
action_list = pe__resource_actions(rsc, target, RSC_DEMOTE, FALSE);
for (gIter = action_list; gIter != NULL; gIter = gIter->next) {
pe_action_t *action = (pe_action_t *) gIter->data;
if (action->node->details->online == FALSE || action->node->details->unclean == TRUE
|| is_set(rsc->flags, pe_rsc_failed)) {
if (is_set(rsc->flags, pe_rsc_failed)) {
pe_rsc_info(rsc,
"Demote of failed resource %s is implicit after %s is fenced",
rsc->id, target->details->uname);
} else {
pe_rsc_info(rsc, "%s is implicit after %s is fenced",
action->uuid, target->details->uname);
}
/* The demote would never complete and is now implied by the
* fencing, so convert it into a pseudo-action.
*/
update_action_flags(action, pe_action_pseudo|pe_action_runnable,
__FUNCTION__, __LINE__);
if (pe_rsc_is_bundled(rsc)) {
/* Do nothing, let the recovery be ordered after the parent's implied stop */
} else if (order_implicit) {
order_actions(stonith_op, action, pe_order_preserve|pe_order_optional);
}
}
}
g_list_free(action_list);
}
void
rsc_stonith_ordering(pe_resource_t * rsc, pe_action_t * stonith_op, pe_working_set_t * data_set)
{
if (rsc->children) {
GListPtr gIter = NULL;
for (gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
pe_resource_t *child_rsc = (pe_resource_t *) gIter->data;
rsc_stonith_ordering(child_rsc, stonith_op, data_set);
}
} else if (is_not_set(rsc->flags, pe_rsc_managed)) {
pe_rsc_trace(rsc, "Skipping fencing constraints for unmanaged resource: %s", rsc->id);
} else {
native_start_constraints(rsc, stonith_op, data_set);
native_stop_constraints(rsc, stonith_op, data_set);
}
}
void
ReloadRsc(pe_resource_t * rsc, pe_node_t *node, pe_working_set_t * data_set)
{
GListPtr gIter = NULL;
pe_action_t *reload = NULL;
if (rsc->children) {
for (gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
pe_resource_t *child_rsc = (pe_resource_t *) gIter->data;
ReloadRsc(child_rsc, node, data_set);
}
return;
} else if (rsc->variant > pe_native) {
/* Complex resource with no children */
return;
} else if (is_not_set(rsc->flags, pe_rsc_managed)) {
pe_rsc_trace(rsc, "%s: unmanaged", rsc->id);
return;
- } else if (is_set(rsc->flags, pe_rsc_failed) || is_set(rsc->flags, pe_rsc_start_pending)) {
- pe_rsc_trace(rsc, "%s: general resource state: flags=0x%.16llx", rsc->id, rsc->flags);
- stop_action(rsc, node, FALSE); /* Force a full restart, overkill? */
+ } else if (is_set(rsc->flags, pe_rsc_failed)) {
+ /* We don't need to specify any particular actions here, normal failure
+ * recovery will apply.
+ */
+ pe_rsc_trace(rsc, "%s: preventing reload because failed", rsc->id);
+ return;
+
+ } else if (is_set(rsc->flags, pe_rsc_start_pending)) {
+ /* If a resource's configuration changed while a start was pending,
+ * force a full restart.
+ */
+ pe_rsc_trace(rsc, "%s: preventing reload because start pending", rsc->id);
+ stop_action(rsc, node, FALSE);
return;
} else if (node == NULL) {
pe_rsc_trace(rsc, "%s: not active", rsc->id);
return;
}
pe_rsc_trace(rsc, "Processing %s", rsc->id);
set_bit(rsc->flags, pe_rsc_reload);
reload = custom_action(
rsc, reload_key(rsc), CRMD_ACTION_RELOAD, node, FALSE, TRUE, data_set);
pe_action_set_reason(reload, "resource definition change", FALSE);
custom_action_order(NULL, NULL, reload, rsc, stop_key(rsc), NULL,
pe_order_optional|pe_order_then_cancels_first,
data_set);
custom_action_order(NULL, NULL, reload, rsc, demote_key(rsc), NULL,
pe_order_optional|pe_order_then_cancels_first,
data_set);
}
void
native_append_meta(pe_resource_t * rsc, xmlNode * xml)
{
char *value = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_INCARNATION);
pe_resource_t *parent;
if (value) {
char *name = NULL;
name = crm_meta_name(XML_RSC_ATTR_INCARNATION);
crm_xml_add(xml, name, value);
free(name);
}
value = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_REMOTE_NODE);
if (value) {
char *name = NULL;
name = crm_meta_name(XML_RSC_ATTR_REMOTE_NODE);
crm_xml_add(xml, name, value);
free(name);
}
for (parent = rsc; parent != NULL; parent = parent->parent) {
if (parent->container) {
crm_xml_add(xml, CRM_META"_"XML_RSC_ATTR_CONTAINER, parent->container->id);
}
}
}
diff --git a/lib/pengine/common.c b/lib/pengine/common.c
index ded6df80bb..37f287b8ba 100644
--- a/lib/pengine/common.c
+++ b/lib/pengine/common.c
@@ -1,635 +1,638 @@
/*
* Copyright 2004-2020 the Pacemaker project contributors
*
* The version control history for this file may have further details.
*
* This source code is licensed under the GNU Lesser General Public License
* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
*/
#include
#include
#include
#include
#include
#include
#include
gboolean was_processing_error = FALSE;
gboolean was_processing_warning = FALSE;
static bool
check_health(const char *value)
{
return safe_str_eq(value, "none")
|| safe_str_eq(value, "custom")
|| safe_str_eq(value, "only-green")
|| safe_str_eq(value, "progressive")
|| safe_str_eq(value, "migrate-on-red");
}
static bool
check_stonith_action(const char *value)
{
return safe_str_eq(value, "reboot")
|| safe_str_eq(value, "poweroff")
|| safe_str_eq(value, "off");
}
static bool
check_placement_strategy(const char *value)
{
return safe_str_eq(value, "default")
|| safe_str_eq(value, "utilization")
|| safe_str_eq(value, "minimal")
|| safe_str_eq(value, "balanced");
}
static pcmk__cluster_option_t pe_opts[] = {
/* name, old name, type, allowed values,
* default value, validator,
* short description,
* long description
*/
{
- "no-quorum-policy", NULL, "enum", "stop, freeze, ignore, suicide",
+ "no-quorum-policy", NULL, "enum", "stop, freeze, ignore, demote, suicide",
"stop", pcmk__valid_quorum,
"What to do when the cluster does not have quorum",
NULL
},
{
"symmetric-cluster", NULL, "boolean", NULL,
"true", pcmk__valid_boolean,
"Whether resources can run on any node by default",
NULL
},
{
"maintenance-mode", NULL, "boolean", NULL,
"false", pcmk__valid_boolean,
"Whether the cluster should refrain from monitoring, starting, "
"and stopping resources",
NULL
},
{
"start-failure-is-fatal", NULL, "boolean", NULL,
"true", pcmk__valid_boolean,
"Whether a start failure should prevent a resource from being "
"recovered on the same node",
"When true, the cluster will immediately ban a resource from a node "
"if it fails to start there. When false, the cluster will instead "
"check the resource's fail count against its migration-threshold."
},
{
"enable-startup-probes", NULL, "boolean", NULL,
"true", pcmk__valid_boolean,
"Whether the cluster should check for active resources during start-up",
NULL
},
{
XML_CONFIG_ATTR_SHUTDOWN_LOCK, NULL, "boolean", NULL,
"false", pcmk__valid_boolean,
"Whether to lock resources to a cleanly shut down node",
"When true, resources active on a node when it is cleanly shut down "
"are kept \"locked\" to that node (not allowed to run elsewhere) "
"until they start again on that node after it rejoins (or for at "
"most shutdown-lock-limit, if set). Stonith resources and "
"Pacemaker Remote connections are never locked. Clone and bundle "
"instances and the master role of promotable clones are currently "
"never locked, though support could be added in a future release."
},
{
XML_CONFIG_ATTR_SHUTDOWN_LOCK_LIMIT, NULL, "time", NULL,
"0", pcmk__valid_interval_spec,
"Do not lock resources to a cleanly shut down node longer than this",
"If shutdown-lock is true and this is set to a nonzero time duration, "
"shutdown locks will expire after this much time has passed since "
"the shutdown was initiated, even if the node has not rejoined."
},
// Fencing-related options
{
"stonith-enabled", NULL, "boolean", NULL,
"true", pcmk__valid_boolean,
"*** Advanced Use Only *** "
"Whether nodes may be fenced as part of recovery",
"If false, unresponsive nodes are immediately assumed to be harmless, "
"and resources that were active on them may be recovered "
"elsewhere. This can result in a \"split-brain\" situation, "
"potentially leading to data loss and/or service unavailability."
},
{
"stonith-action", NULL, "enum", "reboot, off, poweroff",
"reboot", check_stonith_action,
"Action to send to fence device when a node needs to be fenced "
"(\"poweroff\" is a deprecated alias for \"off\")",
NULL
},
{
"stonith-timeout", NULL, "time", NULL,
"60s", pcmk__valid_interval_spec,
"*** Advanced Use Only *** Unused by Pacemaker",
"This value is not used by Pacemaker, but is kept for backward "
"compatibility, and certain legacy fence agents might use it."
},
{
XML_ATTR_HAVE_WATCHDOG, NULL, "boolean", NULL,
"false", pcmk__valid_boolean,
"Whether watchdog integration is enabled",
"This is set automatically by the cluster according to whether SBD "
"is detected to be in use. User-configured values are ignored."
},
{
"concurrent-fencing", NULL, "boolean", NULL,
#ifdef DEFAULT_CONCURRENT_FENCING_TRUE
"true",
#else
"false",
#endif
pcmk__valid_boolean,
"Allow performing fencing operations in parallel",
NULL
},
{
"startup-fencing", NULL, "boolean", NULL,
"true", pcmk__valid_boolean,
"*** Advanced Use Only *** Whether to fence unseen nodes at start-up",
"Setting this to false may lead to a \"split-brain\" situation,"
"potentially leading to data loss and/or service unavailability."
},
{
XML_CONFIG_ATTR_PRIORITY_FENCING_DELAY, NULL, "time", NULL,
"0", pcmk__valid_interval_spec,
"Apply fencing delay targeting the lost nodes with the highest total resource priority",
"Apply specified delay for the fencings that are targeting the lost "
"nodes with the highest total resource priority in case we don't "
"have the majority of the nodes in our cluster partition, so that "
"the more significant nodes potentially win any fencing match, "
"which is especially meaningful under split-brain of 2-node "
"cluster. A promoted resource instance takes the base priority + 1 "
"on calculation if the base priority is not 0. Any static/random "
"delays that are introduced by `pcmk_delay_base/max` configured "
"for the corresponding fencing resources will be added to this "
"delay. This delay should be significantly greater than, safely "
"twice, the maximum `pcmk_delay_base/max`. By default, priority "
"fencing delay is disabled."
},
{
"cluster-delay", NULL, "time", NULL,
"60s", pcmk__valid_interval_spec,
"Maximum time for node-to-node communication",
"The node elected Designated Controller (DC) will consider an action "
"failed if it does not get a response from the node executing the "
"action within this time (after considering the action's own "
"timeout). The \"correct\" value will depend on the speed and "
"load of your network and cluster nodes."
},
{
"batch-limit", NULL, "integer", NULL,
"0", pcmk__valid_number,
"Maximum number of jobs that the cluster may execute in parallel "
"across all nodes",
"The \"correct\" value will depend on the speed and load of your "
"network and cluster nodes. If set to 0, the cluster will "
"impose a dynamically calculated limit when any node has a "
"high load."
},
{
"migration-limit", NULL, "integer", NULL,
"-1", pcmk__valid_number,
"The number of live migration actions that the cluster is allowed "
"to execute in parallel on a node (-1 means no limit)"
},
/* Orphans and stopping */
{
"stop-all-resources", NULL, "boolean", NULL,
"false", pcmk__valid_boolean,
"Whether the cluster should stop all active resources",
NULL
},
{
"stop-orphan-resources", NULL, "boolean", NULL,
"true", pcmk__valid_boolean,
"Whether to stop resources that were removed from the configuration",
NULL
},
{
"stop-orphan-actions", NULL, "boolean", NULL,
"true", pcmk__valid_boolean,
"Whether to cancel recurring actions removed from the configuration",
NULL
},
{
"remove-after-stop", NULL, "boolean", NULL,
"false", pcmk__valid_boolean,
"*** Advanced Use Only *** Whether to remove stopped resources from "
"the executor",
"Values other than default are poorly tested and potentially dangerous."
},
/* Storing inputs */
{
"pe-error-series-max", NULL, "integer", NULL,
"-1", pcmk__valid_number,
"The number of scheduler inputs resulting in errors to save",
"Zero to disable, -1 to store unlimited."
},
{
"pe-warn-series-max", NULL, "integer", NULL,
"5000", pcmk__valid_number,
"The number of scheduler inputs resulting in warnings to save",
"Zero to disable, -1 to store unlimited."
},
{
"pe-input-series-max", NULL, "integer", NULL,
"4000", pcmk__valid_number,
"The number of scheduler inputs without errors or warnings to save",
"Zero to disable, -1 to store unlimited."
},
/* Node health */
{
"node-health-strategy", NULL, "enum",
"none, migrate-on-red, only-green, progressive, custom",
"none", check_health,
"How cluster should react to node health attributes",
"Requires external entities to create node attributes (named with "
"the prefix \"#health\") with values \"red\", \"yellow\" or "
"\"green\"."
},
{
"node-health-base", NULL, "integer", NULL,
"0", pcmk__valid_number,
"Base health score assigned to a node",
"Only used when node-health-strategy is set to progressive."
},
{
"node-health-green", NULL, "integer", NULL,
"0", pcmk__valid_number,
"The score to use for a node health attribute whose value is \"green\"",
"Only used when node-health-strategy is set to custom or progressive."
},
{
"node-health-yellow", NULL, "integer", NULL,
"0", pcmk__valid_number,
"The score to use for a node health attribute whose value is \"yellow\"",
"Only used when node-health-strategy is set to custom or progressive."
},
{
"node-health-red", NULL, "integer", NULL,
"-INFINITY", pcmk__valid_number,
"The score to use for a node health attribute whose value is \"red\"",
"Only used when node-health-strategy is set to custom or progressive."
},
/*Placement Strategy*/
{
"placement-strategy", NULL, "enum",
"default, utilization, minimal, balanced",
"default", check_placement_strategy,
"How the cluster should allocate resources to nodes",
NULL
},
};
void
pe_metadata(void)
{
pcmk__print_option_metadata("pacemaker-schedulerd", "1.0",
"Pacemaker scheduler options",
"Cluster options used by Pacemaker's scheduler"
" (formerly called pengine)",
pe_opts, DIMOF(pe_opts));
}
void
verify_pe_options(GHashTable * options)
{
pcmk__validate_cluster_options(options, pe_opts, DIMOF(pe_opts));
}
const char *
pe_pref(GHashTable * options, const char *name)
{
return pcmk__cluster_option(options, pe_opts, DIMOF(pe_opts), name);
}
const char *
fail2text(enum action_fail_response fail)
{
const char *result = "";
switch (fail) {
case action_fail_ignore:
result = "ignore";
break;
+ case action_fail_demote:
+ result = "demote";
+ break;
case action_fail_block:
result = "block";
break;
case action_fail_recover:
result = "recover";
break;
case action_fail_migrate:
result = "migrate";
break;
case action_fail_stop:
result = "stop";
break;
case action_fail_fence:
result = "fence";
break;
case action_fail_standby:
result = "standby";
break;
case action_fail_restart_container:
result = "restart-container";
break;
case action_fail_reset_remote:
result = "reset-remote";
break;
}
return result;
}
enum action_tasks
text2task(const char *task)
{
if (safe_str_eq(task, CRMD_ACTION_STOP)) {
return stop_rsc;
} else if (safe_str_eq(task, CRMD_ACTION_STOPPED)) {
return stopped_rsc;
} else if (safe_str_eq(task, CRMD_ACTION_START)) {
return start_rsc;
} else if (safe_str_eq(task, CRMD_ACTION_STARTED)) {
return started_rsc;
} else if (safe_str_eq(task, CRM_OP_SHUTDOWN)) {
return shutdown_crm;
} else if (safe_str_eq(task, CRM_OP_FENCE)) {
return stonith_node;
} else if (safe_str_eq(task, CRMD_ACTION_STATUS)) {
return monitor_rsc;
} else if (safe_str_eq(task, CRMD_ACTION_NOTIFY)) {
return action_notify;
} else if (safe_str_eq(task, CRMD_ACTION_NOTIFIED)) {
return action_notified;
} else if (safe_str_eq(task, CRMD_ACTION_PROMOTE)) {
return action_promote;
} else if (safe_str_eq(task, CRMD_ACTION_DEMOTE)) {
return action_demote;
} else if (safe_str_eq(task, CRMD_ACTION_PROMOTED)) {
return action_promoted;
} else if (safe_str_eq(task, CRMD_ACTION_DEMOTED)) {
return action_demoted;
}
#if SUPPORT_TRACING
if (safe_str_eq(task, CRMD_ACTION_CANCEL)) {
return no_action;
} else if (safe_str_eq(task, CRMD_ACTION_DELETE)) {
return no_action;
} else if (safe_str_eq(task, CRMD_ACTION_STATUS)) {
return no_action;
} else if (safe_str_eq(task, CRM_OP_PROBED)) {
return no_action;
} else if (safe_str_eq(task, CRM_OP_LRM_REFRESH)) {
return no_action;
} else if (safe_str_eq(task, CRMD_ACTION_MIGRATE)) {
return no_action;
} else if (safe_str_eq(task, CRMD_ACTION_MIGRATED)) {
return no_action;
}
crm_trace("Unsupported action: %s", task);
#endif
return no_action;
}
const char *
task2text(enum action_tasks task)
{
const char *result = "";
switch (task) {
case no_action:
result = "no_action";
break;
case stop_rsc:
result = CRMD_ACTION_STOP;
break;
case stopped_rsc:
result = CRMD_ACTION_STOPPED;
break;
case start_rsc:
result = CRMD_ACTION_START;
break;
case started_rsc:
result = CRMD_ACTION_STARTED;
break;
case shutdown_crm:
result = CRM_OP_SHUTDOWN;
break;
case stonith_node:
result = CRM_OP_FENCE;
break;
case monitor_rsc:
result = CRMD_ACTION_STATUS;
break;
case action_notify:
result = CRMD_ACTION_NOTIFY;
break;
case action_notified:
result = CRMD_ACTION_NOTIFIED;
break;
case action_promote:
result = CRMD_ACTION_PROMOTE;
break;
case action_promoted:
result = CRMD_ACTION_PROMOTED;
break;
case action_demote:
result = CRMD_ACTION_DEMOTE;
break;
case action_demoted:
result = CRMD_ACTION_DEMOTED;
break;
}
return result;
}
const char *
role2text(enum rsc_role_e role)
{
switch (role) {
case RSC_ROLE_UNKNOWN:
return RSC_ROLE_UNKNOWN_S;
case RSC_ROLE_STOPPED:
return RSC_ROLE_STOPPED_S;
case RSC_ROLE_STARTED:
return RSC_ROLE_STARTED_S;
case RSC_ROLE_SLAVE:
return RSC_ROLE_SLAVE_S;
case RSC_ROLE_MASTER:
return RSC_ROLE_MASTER_S;
}
CRM_CHECK(role >= RSC_ROLE_UNKNOWN, return RSC_ROLE_UNKNOWN_S);
CRM_CHECK(role < RSC_ROLE_MAX, return RSC_ROLE_UNKNOWN_S);
// coverity[dead_error_line]
return RSC_ROLE_UNKNOWN_S;
}
enum rsc_role_e
text2role(const char *role)
{
CRM_ASSERT(role != NULL);
if (safe_str_eq(role, RSC_ROLE_STOPPED_S)) {
return RSC_ROLE_STOPPED;
} else if (safe_str_eq(role, RSC_ROLE_STARTED_S)) {
return RSC_ROLE_STARTED;
} else if (safe_str_eq(role, RSC_ROLE_SLAVE_S)) {
return RSC_ROLE_SLAVE;
} else if (safe_str_eq(role, RSC_ROLE_MASTER_S)) {
return RSC_ROLE_MASTER;
} else if (safe_str_eq(role, RSC_ROLE_UNKNOWN_S)) {
return RSC_ROLE_UNKNOWN;
}
crm_err("Unknown role: %s", role);
return RSC_ROLE_UNKNOWN;
}
/*!
* \internal
* \brief Add two scores (bounding to +/- INFINITY)
*
* \param[in] score1 First score to add
* \param[in] score2 Second score to add
*/
int
pe__add_scores(int score1, int score2)
{
int result = score1 + score2;
// First handle the cases where one or both is infinite
if (score1 <= -CRM_SCORE_INFINITY) {
if (score2 <= -CRM_SCORE_INFINITY) {
crm_trace("-INFINITY + -INFINITY = -INFINITY");
} else if (score2 >= CRM_SCORE_INFINITY) {
crm_trace("-INFINITY + +INFINITY = -INFINITY");
} else {
crm_trace("-INFINITY + %d = -INFINITY", score2);
}
return -CRM_SCORE_INFINITY;
} else if (score2 <= -CRM_SCORE_INFINITY) {
if (score1 >= CRM_SCORE_INFINITY) {
crm_trace("+INFINITY + -INFINITY = -INFINITY");
} else {
crm_trace("%d + -INFINITY = -INFINITY", score1);
}
return -CRM_SCORE_INFINITY;
} else if (score1 >= CRM_SCORE_INFINITY) {
if (score2 >= CRM_SCORE_INFINITY) {
crm_trace("+INFINITY + +INFINITY = +INFINITY");
} else {
crm_trace("+INFINITY + %d = +INFINITY", score2);
}
return CRM_SCORE_INFINITY;
} else if (score2 >= CRM_SCORE_INFINITY) {
crm_trace("%d + +INFINITY = +INFINITY", score1);
return CRM_SCORE_INFINITY;
}
/* As long as CRM_SCORE_INFINITY is less than half of the maximum integer,
* we can ignore the possibility of integer overflow
*/
// Bound result to infinity
if (result >= CRM_SCORE_INFINITY) {
crm_trace("%d + %d = +INFINITY", score1, score2);
return CRM_SCORE_INFINITY;
} else if (result <= -CRM_SCORE_INFINITY) {
crm_trace("%d + %d = -INFINITY", score1, score2);
return -CRM_SCORE_INFINITY;
}
crm_trace("%d + %d = %d", score1, score2, result);
return result;
}
void
add_hash_param(GHashTable * hash, const char *name, const char *value)
{
CRM_CHECK(hash != NULL, return);
crm_trace("adding: name=%s value=%s", crm_str(name), crm_str(value));
if (name == NULL || value == NULL) {
return;
} else if (safe_str_eq(value, "#default")) {
return;
} else if (g_hash_table_lookup(hash, name) == NULL) {
g_hash_table_insert(hash, strdup(name), strdup(value));
}
}
const char *
pe_node_attribute_calculated(const pe_node_t *node, const char *name,
const pe_resource_t *rsc)
{
const char *source;
if(node == NULL) {
return NULL;
} else if(rsc == NULL) {
return g_hash_table_lookup(node->details->attrs, name);
}
source = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_TARGET);
if(source == NULL || safe_str_eq("host", source) == FALSE) {
return g_hash_table_lookup(node->details->attrs, name);
}
/* Use attributes set for the containers location
* instead of for the container itself
*
* Useful when the container is using the host's local
* storage
*/
CRM_ASSERT(node->details->remote_rsc);
CRM_ASSERT(node->details->remote_rsc->container);
if(node->details->remote_rsc->container->running_on) {
pe_node_t *host = node->details->remote_rsc->container->running_on->data;
pe_rsc_trace(rsc, "%s: Looking for %s on the container host %s", rsc->id, name, host->details->uname);
return g_hash_table_lookup(host->details->attrs, name);
}
pe_rsc_trace(rsc, "%s: Not looking for %s on the container host: %s is inactive",
rsc->id, name, node->details->remote_rsc->container->id);
return NULL;
}
const char *
pe_node_attribute_raw(pe_node_t *node, const char *name)
{
if(node == NULL) {
return NULL;
}
return g_hash_table_lookup(node->details->attrs, name);
}
diff --git a/lib/pengine/native.c b/lib/pengine/native.c
index f0d83d7cd4..20658a0932 100644
--- a/lib/pengine/native.c
+++ b/lib/pengine/native.c
@@ -1,1451 +1,1451 @@
/*
* Copyright 2004-2020 the Pacemaker project contributors
*
* The version control history for this file may have further details.
*
* This source code is licensed under the GNU Lesser General Public License
* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
*/
#include
#include
#include
#include
#include
#include
#include
#define VARIANT_NATIVE 1
#include "./variant.h"
/*!
* \internal
* \brief Check whether a resource is active on multiple nodes
*/
static bool
is_multiply_active(pe_resource_t *rsc)
{
unsigned int count = 0;
if (rsc->variant == pe_native) {
pe__find_active_requires(rsc, &count);
}
return count > 1;
}
static void
native_priority_to_node(pe_resource_t * rsc, pe_node_t * node)
{
int priority = 0;
if (rsc->priority == 0) {
return;
}
if (rsc->role == RSC_ROLE_MASTER) {
// Promoted instance takes base priority + 1
priority = rsc->priority + 1;
} else {
priority = rsc->priority;
}
node->details->priority += priority;
pe_rsc_trace(rsc, "Node '%s' now has priority %d with %s'%s' (priority: %d%s)",
node->details->uname, node->details->priority,
rsc->role == RSC_ROLE_MASTER ? "promoted " : "",
rsc->id, rsc->priority,
rsc->role == RSC_ROLE_MASTER ? " + 1" : "");
/* Priority of a resource running on a guest node is added to the cluster
* node as well. */
if (node->details->remote_rsc
&& node->details->remote_rsc->container) {
GListPtr gIter = node->details->remote_rsc->container->running_on;
for (; gIter != NULL; gIter = gIter->next) {
pe_node_t *a_node = gIter->data;
a_node->details->priority += priority;
pe_rsc_trace(rsc, "Node '%s' now has priority %d with %s'%s' (priority: %d%s) "
"from guest node '%s'",
a_node->details->uname, a_node->details->priority,
rsc->role == RSC_ROLE_MASTER ? "promoted " : "",
rsc->id, rsc->priority,
rsc->role == RSC_ROLE_MASTER ? " + 1" : "",
node->details->uname);
}
}
}
void
native_add_running(pe_resource_t * rsc, pe_node_t * node, pe_working_set_t * data_set)
{
GListPtr gIter = rsc->running_on;
CRM_CHECK(node != NULL, return);
for (; gIter != NULL; gIter = gIter->next) {
pe_node_t *a_node = (pe_node_t *) gIter->data;
CRM_CHECK(a_node != NULL, return);
if (safe_str_eq(a_node->details->id, node->details->id)) {
return;
}
}
pe_rsc_trace(rsc, "Adding %s to %s %s", rsc->id, node->details->uname,
is_set(rsc->flags, pe_rsc_managed)?"":"(unmanaged)");
rsc->running_on = g_list_append(rsc->running_on, node);
if (rsc->variant == pe_native) {
node->details->running_rsc = g_list_append(node->details->running_rsc, rsc);
native_priority_to_node(rsc, node);
}
if (rsc->variant == pe_native && node->details->maintenance) {
clear_bit(rsc->flags, pe_rsc_managed);
}
if (is_not_set(rsc->flags, pe_rsc_managed)) {
pe_resource_t *p = rsc->parent;
pe_rsc_info(rsc, "resource %s isn't managed", rsc->id);
resource_location(rsc, node, INFINITY, "not_managed_default", data_set);
while(p && node->details->online) {
/* add without the additional location constraint */
p->running_on = g_list_append(p->running_on, node);
p = p->parent;
}
return;
}
if (is_multiply_active(rsc)) {
switch (rsc->recovery_type) {
case recovery_stop_only:
{
GHashTableIter gIter;
pe_node_t *local_node = NULL;
/* make sure it doesn't come up again */
if (rsc->allowed_nodes != NULL) {
g_hash_table_destroy(rsc->allowed_nodes);
}
rsc->allowed_nodes = pe__node_list2table(data_set->nodes);
g_hash_table_iter_init(&gIter, rsc->allowed_nodes);
while (g_hash_table_iter_next(&gIter, NULL, (void **)&local_node)) {
local_node->weight = -INFINITY;
}
}
break;
case recovery_stop_start:
break;
case recovery_block:
clear_bit(rsc->flags, pe_rsc_managed);
set_bit(rsc->flags, pe_rsc_block);
/* If the resource belongs to a group or bundle configured with
* multiple-active=block, block the entire entity.
*/
if (rsc->parent
&& (rsc->parent->variant == pe_group || rsc->parent->variant == pe_container)
&& rsc->parent->recovery_type == recovery_block) {
GListPtr gIter = rsc->parent->children;
for (; gIter != NULL; gIter = gIter->next) {
pe_resource_t *child = (pe_resource_t *) gIter->data;
clear_bit(child->flags, pe_rsc_managed);
set_bit(child->flags, pe_rsc_block);
}
}
break;
}
crm_debug("%s is active on multiple nodes including %s: %s",
rsc->id, node->details->uname,
recovery2text(rsc->recovery_type));
} else {
pe_rsc_trace(rsc, "Resource %s is active on: %s", rsc->id, node->details->uname);
}
if (rsc->parent != NULL) {
native_add_running(rsc->parent, node, data_set);
}
}
static void
recursive_clear_unique(pe_resource_t *rsc)
{
clear_bit(rsc->flags, pe_rsc_unique);
add_hash_param(rsc->meta, XML_RSC_ATTR_UNIQUE, XML_BOOLEAN_FALSE);
for (GList *child = rsc->children; child != NULL; child = child->next) {
recursive_clear_unique((pe_resource_t *) child->data);
}
}
gboolean
native_unpack(pe_resource_t * rsc, pe_working_set_t * data_set)
{
pe_resource_t *parent = uber_parent(rsc);
native_variant_data_t *native_data = NULL;
const char *standard = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
uint32_t ra_caps = pcmk_get_ra_caps(standard);
pe_rsc_trace(rsc, "Processing resource %s...", rsc->id);
native_data = calloc(1, sizeof(native_variant_data_t));
rsc->variant_opaque = native_data;
// Only some agent standards support unique and promotable clones
if (is_not_set(ra_caps, pcmk_ra_cap_unique)
&& is_set(rsc->flags, pe_rsc_unique) && pe_rsc_is_clone(parent)) {
/* @COMPAT We should probably reject this situation as an error (as we
* do for promotable below) rather than warn and convert, but that would
* be a backward-incompatible change that we should probably do with a
* transform at a schema major version bump.
*/
pe__force_anon(standard, parent, rsc->id, data_set);
/* Clear globally-unique on the parent and all its descendents unpacked
* so far (clearing the parent should make any future children unpacking
* correct). We have to clear this resource explicitly because it isn't
* hooked into the parent's children yet.
*/
recursive_clear_unique(parent);
recursive_clear_unique(rsc);
}
if (is_not_set(ra_caps, pcmk_ra_cap_promotable)
&& is_set(parent->flags, pe_rsc_promotable)) {
pe_err("Resource %s is of type %s and therefore "
"cannot be used as a promotable clone resource",
rsc->id, standard);
return FALSE;
}
return TRUE;
}
static bool
rsc_is_on_node(pe_resource_t *rsc, const pe_node_t *node, int flags)
{
pe_rsc_trace(rsc, "Checking whether %s is on %s",
rsc->id, node->details->uname);
if (is_set(flags, pe_find_current) && rsc->running_on) {
for (GListPtr iter = rsc->running_on; iter; iter = iter->next) {
pe_node_t *loc = (pe_node_t *) iter->data;
if (loc->details == node->details) {
return TRUE;
}
}
} else if (is_set(flags, pe_find_inactive) && (rsc->running_on == NULL)) {
return TRUE;
} else if (is_not_set(flags, pe_find_current) && rsc->allocated_to
&& (rsc->allocated_to->details == node->details)) {
return TRUE;
}
return FALSE;
}
pe_resource_t *
native_find_rsc(pe_resource_t * rsc, const char *id, const pe_node_t *on_node,
int flags)
{
bool match = FALSE;
pe_resource_t *result = NULL;
CRM_CHECK(id && rsc && rsc->id, return NULL);
if (flags & pe_find_clone) {
const char *rid = ID(rsc->xml);
if (!pe_rsc_is_clone(uber_parent(rsc))) {
match = FALSE;
} else if (!strcmp(id, rsc->id) || safe_str_eq(id, rid)) {
match = TRUE;
}
} else if (!strcmp(id, rsc->id)) {
match = TRUE;
} else if (is_set(flags, pe_find_renamed)
&& rsc->clone_name && strcmp(rsc->clone_name, id) == 0) {
match = TRUE;
} else if (is_set(flags, pe_find_any)
|| (is_set(flags, pe_find_anon)
&& is_not_set(rsc->flags, pe_rsc_unique))) {
match = pe_base_name_eq(rsc, id);
}
if (match && on_node) {
bool match_node = rsc_is_on_node(rsc, on_node, flags);
if (match_node == FALSE) {
match = FALSE;
}
}
if (match) {
return rsc;
}
for (GListPtr gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
pe_resource_t *child = (pe_resource_t *) gIter->data;
result = rsc->fns->find_rsc(child, id, on_node, flags);
if (result) {
return result;
}
}
return NULL;
}
char *
native_parameter(pe_resource_t * rsc, pe_node_t * node, gboolean create, const char *name,
pe_working_set_t * data_set)
{
char *value_copy = NULL;
const char *value = NULL;
GHashTable *hash = NULL;
GHashTable *local_hash = NULL;
CRM_CHECK(rsc != NULL, return NULL);
CRM_CHECK(name != NULL && strlen(name) != 0, return NULL);
pe_rsc_trace(rsc, "Looking up %s in %s", name, rsc->id);
if (create || g_hash_table_size(rsc->parameters) == 0) {
if (node != NULL) {
pe_rsc_trace(rsc, "Creating hash with node %s", node->details->uname);
} else {
pe_rsc_trace(rsc, "Creating default hash");
}
local_hash = crm_str_table_new();
get_rsc_attributes(local_hash, rsc, node, data_set);
hash = local_hash;
} else {
hash = rsc->parameters;
}
value = g_hash_table_lookup(hash, name);
if (value == NULL) {
/* try meta attributes instead */
value = g_hash_table_lookup(rsc->meta, name);
}
if (value != NULL) {
value_copy = strdup(value);
}
if (local_hash != NULL) {
g_hash_table_destroy(local_hash);
}
return value_copy;
}
gboolean
native_active(pe_resource_t * rsc, gboolean all)
{
- GListPtr gIter = rsc->running_on;
-
- for (; gIter != NULL; gIter = gIter->next) {
+ for (GList *gIter = rsc->running_on; gIter != NULL; gIter = gIter->next) {
pe_node_t *a_node = (pe_node_t *) gIter->data;
if (a_node->details->unclean) {
- crm_debug("Resource %s: node %s is unclean", rsc->id, a_node->details->uname);
+ pe_rsc_trace(rsc, "Resource %s: node %s is unclean",
+ rsc->id, a_node->details->uname);
return TRUE;
} else if (a_node->details->online == FALSE) {
- crm_debug("Resource %s: node %s is offline", rsc->id, a_node->details->uname);
+ pe_rsc_trace(rsc, "Resource %s: node %s is offline",
+ rsc->id, a_node->details->uname);
} else {
- crm_debug("Resource %s active on %s", rsc->id, a_node->details->uname);
+ pe_rsc_trace(rsc, "Resource %s active on %s",
+ rsc->id, a_node->details->uname);
return TRUE;
}
}
-
return FALSE;
}
struct print_data_s {
long options;
void *print_data;
};
static void
native_print_attr(gpointer key, gpointer value, gpointer user_data)
{
long options = ((struct print_data_s *)user_data)->options;
void *print_data = ((struct print_data_s *)user_data)->print_data;
status_print("Option: %s = %s\n", (char *)key, (char *)value);
}
static const char *
native_pending_state(pe_resource_t * rsc)
{
const char *pending_state = NULL;
if (safe_str_eq(rsc->pending_task, CRMD_ACTION_START)) {
pending_state = "Starting";
} else if (safe_str_eq(rsc->pending_task, CRMD_ACTION_STOP)) {
pending_state = "Stopping";
} else if (safe_str_eq(rsc->pending_task, CRMD_ACTION_MIGRATE)) {
pending_state = "Migrating";
} else if (safe_str_eq(rsc->pending_task, CRMD_ACTION_MIGRATED)) {
/* Work might be done in here. */
pending_state = "Migrating";
} else if (safe_str_eq(rsc->pending_task, CRMD_ACTION_PROMOTE)) {
pending_state = "Promoting";
} else if (safe_str_eq(rsc->pending_task, CRMD_ACTION_DEMOTE)) {
pending_state = "Demoting";
}
return pending_state;
}
static const char *
native_pending_task(pe_resource_t * rsc)
{
const char *pending_task = NULL;
if (safe_str_eq(rsc->pending_task, CRMD_ACTION_STATUS)) {
pending_task = "Monitoring";
/* Pending probes are not printed, even if pending
* operations are requested. If someone ever requests that
* behavior, uncomment this and the corresponding part of
* unpack.c:unpack_rsc_op().
*/
/*
} else if (safe_str_eq(rsc->pending_task, "probe")) {
pending_task = "Checking";
*/
}
return pending_task;
}
static enum rsc_role_e
native_displayable_role(pe_resource_t *rsc)
{
enum rsc_role_e role = rsc->role;
if ((role == RSC_ROLE_STARTED)
&& is_set(uber_parent(rsc)->flags, pe_rsc_promotable)) {
role = RSC_ROLE_SLAVE;
}
return role;
}
static const char *
native_displayable_state(pe_resource_t *rsc, long options)
{
const char *rsc_state = NULL;
if (options & pe_print_pending) {
rsc_state = native_pending_state(rsc);
}
if (rsc_state == NULL) {
rsc_state = role2text(native_displayable_role(rsc));
}
return rsc_state;
}
static void
native_print_xml(pe_resource_t * rsc, const char *pre_text, long options, void *print_data)
{
const char *class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
const char *prov = crm_element_value(rsc->xml, XML_AGENT_ATTR_PROVIDER);
const char *rsc_state = native_displayable_state(rsc, options);
const char *target_role = NULL;
/* resource information. */
status_print("%sxml, XML_ATTR_TYPE));
status_print("role=\"%s\" ", rsc_state);
if (rsc->meta) {
target_role = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_TARGET_ROLE);
}
if (target_role) {
status_print("target_role=\"%s\" ", target_role);
}
status_print("active=\"%s\" ", rsc->fns->active(rsc, TRUE) ? "true" : "false");
status_print("orphaned=\"%s\" ", is_set(rsc->flags, pe_rsc_orphan) ? "true" : "false");
status_print("blocked=\"%s\" ", is_set(rsc->flags, pe_rsc_block) ? "true" : "false");
status_print("managed=\"%s\" ", is_set(rsc->flags, pe_rsc_managed) ? "true" : "false");
status_print("failed=\"%s\" ", is_set(rsc->flags, pe_rsc_failed) ? "true" : "false");
status_print("failure_ignored=\"%s\" ",
is_set(rsc->flags, pe_rsc_failure_ignored) ? "true" : "false");
status_print("nodes_running_on=\"%d\" ", g_list_length(rsc->running_on));
if (options & pe_print_pending) {
const char *pending_task = native_pending_task(rsc);
if (pending_task) {
status_print("pending=\"%s\" ", pending_task);
}
}
if (options & pe_print_dev) {
status_print("provisional=\"%s\" ",
is_set(rsc->flags, pe_rsc_provisional) ? "true" : "false");
status_print("runnable=\"%s\" ", is_set(rsc->flags, pe_rsc_runnable) ? "true" : "false");
status_print("priority=\"%f\" ", (double)rsc->priority);
status_print("variant=\"%s\" ", crm_element_name(rsc->xml));
}
/* print out the nodes this resource is running on */
if (options & pe_print_rsconly) {
status_print("/>\n");
/* do nothing */
} else if (rsc->running_on != NULL) {
GListPtr gIter = rsc->running_on;
status_print(">\n");
for (; gIter != NULL; gIter = gIter->next) {
pe_node_t *node = (pe_node_t *) gIter->data;
status_print("%s \n", pre_text,
node->details->uname, node->details->id,
node->details->online ? "false" : "true");
}
status_print("%s\n", pre_text);
} else {
status_print("/>\n");
}
}
// Append a flag to resource description string's flags list
static bool
add_output_flag(GString *s, const char *flag_desc, bool have_flags)
{
g_string_append(s, (have_flags? ", " : " ("));
g_string_append(s, flag_desc);
return true;
}
// Append a node name to resource description string's node list
static bool
add_output_node(GString *s, const char *node, bool have_nodes)
{
g_string_append(s, (have_nodes? " " : " [ "));
g_string_append(s, node);
return true;
}
/*!
* \internal
* \brief Create a string description of a resource
*
* \param[in] rsc Resource to describe
* \param[in] name Desired identifier for the resource
* \param[in] node If not NULL, node that resource is "on"
* \param[in] options Bitmask of pe_print_*
* \param[in] target_role Resource's target role
* \param[in] show_nodes Whether to display nodes when multiply active
*
* \return Newly allocated string description of resource
* \note Caller must free the result with g_free().
*/
static gchar *
native_output_string(pe_resource_t *rsc, const char *name, pe_node_t *node,
long options, const char *target_role, bool show_nodes)
{
const char *class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
const char *provider = NULL;
const char *kind = crm_element_value(rsc->xml, XML_ATTR_TYPE);
char *retval = NULL;
GString *outstr = NULL;
bool have_flags = false;
CRM_CHECK(name != NULL, name = "unknown");
CRM_CHECK(kind != NULL, kind = "unknown");
CRM_CHECK(class != NULL, class = "unknown");
if (is_set(pcmk_get_ra_caps(class), pcmk_ra_cap_provider)) {
provider = crm_element_value(rsc->xml, XML_AGENT_ATTR_PROVIDER);
}
if ((node == NULL) && (rsc->lock_node != NULL)) {
node = rsc->lock_node;
}
if (is_set(options, pe_print_rsconly)
|| pcmk__list_of_multiple(rsc->running_on)) {
node = NULL;
}
// We need a string of at least this size
outstr = g_string_sized_new(strlen(name) + strlen(class) + strlen(kind)
+ (provider? (strlen(provider) + 2) : 0)
+ (node? strlen(node->details->uname) + 1 : 0)
+ 11);
// Resource name and agent
g_string_printf(outstr, "%s\t(%s%s%s:%s):\t", name, class,
/* @COMPAT This should be a single ':' (see CLBZ#5395) but
* to avoid breaking anything relying on it, we're keeping
* it like this until the next minor version bump.
*/
(provider? "::" : ""), (provider? provider : ""), kind);
// State on node
if (is_set(rsc->flags, pe_rsc_orphan)) {
g_string_append(outstr, " ORPHANED");
}
if (is_set(rsc->flags, pe_rsc_failed)) {
enum rsc_role_e role = native_displayable_role(rsc);
if (role > RSC_ROLE_SLAVE) {
g_string_append_printf(outstr, " FAILED %s", role2text(role));
} else {
g_string_append(outstr, " FAILED");
}
} else {
g_string_append_printf(outstr, " %s", native_displayable_state(rsc, options));
}
if (node) {
g_string_append_printf(outstr, " %s", node->details->uname);
}
// Flags, as: ( [...])
if (node && !(node->details->online) && node->details->unclean) {
have_flags = add_output_flag(outstr, "UNCLEAN", have_flags);
}
if (node && (node == rsc->lock_node)) {
have_flags = add_output_flag(outstr, "LOCKED", have_flags);
}
if (is_set(options, pe_print_pending)) {
const char *pending_task = native_pending_task(rsc);
if (pending_task) {
have_flags = add_output_flag(outstr, pending_task, have_flags);
}
}
if (target_role) {
enum rsc_role_e target_role_e = text2role(target_role);
/* Only show target role if it limits our abilities (i.e. ignore
* Started, as it is the default anyways, and doesn't prevent the
* resource from becoming Master).
*/
if (target_role_e == RSC_ROLE_STOPPED) {
have_flags = add_output_flag(outstr, "disabled", have_flags);
} else if (is_set(uber_parent(rsc)->flags, pe_rsc_promotable)
&& target_role_e == RSC_ROLE_SLAVE) {
have_flags = add_output_flag(outstr, "target-role:", have_flags);
g_string_append(outstr, target_role);
}
}
if (is_set(rsc->flags, pe_rsc_block)) {
have_flags = add_output_flag(outstr, "blocked", have_flags);
} else if (is_not_set(rsc->flags, pe_rsc_managed)) {
have_flags = add_output_flag(outstr, "unmanaged", have_flags);
}
if (is_set(rsc->flags, pe_rsc_failure_ignored)) {
have_flags = add_output_flag(outstr, "failure ignored", have_flags);
}
if (is_set(options, pe_print_dev)) {
if (is_set(options, pe_rsc_provisional)) {
have_flags = add_output_flag(outstr, "provisional", have_flags);
}
if (is_not_set(options, pe_rsc_runnable)) {
have_flags = add_output_flag(outstr, "non-startable", have_flags);
}
have_flags = add_output_flag(outstr, "variant:", have_flags);
g_string_append_printf(outstr, "%s priority:%f",
crm_element_name(rsc->xml),
(double) (rsc->priority));
}
if (have_flags) {
g_string_append(outstr, ")");
}
// User-supplied description
if (is_set(options, pe_print_rsconly)
|| pcmk__list_of_multiple(rsc->running_on)) {
const char *desc = crm_element_value(rsc->xml, XML_ATTR_DESC);
if (desc) {
g_string_append_printf(outstr, " %s", desc);
}
}
if (show_nodes && is_not_set(options, pe_print_rsconly)
&& pcmk__list_of_multiple(rsc->running_on)) {
bool have_nodes = false;
for (GList *iter = rsc->running_on; iter != NULL; iter = iter->next) {
pe_node_t *n = (pe_node_t *) iter->data;
have_nodes = add_output_node(outstr, n->details->uname, have_nodes);
}
if (have_nodes) {
g_string_append(outstr, " ]");
}
}
retval = outstr->str;
g_string_free(outstr, FALSE);
return retval;
}
int
pe__common_output_html(pcmk__output_t *out, pe_resource_t * rsc,
const char *name, pe_node_t *node, long options)
{
const char *kind = crm_element_value(rsc->xml, XML_ATTR_TYPE);
const char *target_role = NULL;
xmlNodePtr list_node = NULL;
const char *cl = NULL;
CRM_ASSERT(rsc->variant == pe_native);
CRM_ASSERT(kind != NULL);
if (rsc->meta) {
const char *is_internal = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_INTERNAL_RSC);
if (crm_is_true(is_internal) && is_not_set(options, pe_print_implicit)) {
crm_trace("skipping print of internal resource %s", rsc->id);
return pcmk_rc_no_output;
}
target_role = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_TARGET_ROLE);
}
if (is_not_set(rsc->flags, pe_rsc_managed)) {
cl = "rsc-managed";
} else if (is_set(rsc->flags, pe_rsc_failed)) {
cl = "rsc-failed";
} else if (rsc->variant == pe_native && (rsc->running_on == NULL)) {
cl = "rsc-failed";
} else if (pcmk__list_of_multiple(rsc->running_on)) {
cl = "rsc-multiple";
} else if (is_set(rsc->flags, pe_rsc_failure_ignored)) {
cl = "rsc-failure-ignored";
} else {
cl = "rsc-ok";
}
{
gchar *s = native_output_string(rsc, name, node, options, target_role,
true);
list_node = pcmk__output_create_html_node(out, "li", NULL, NULL, NULL);
pcmk_create_html_node(list_node, "span", NULL, cl, s);
g_free(s);
}
if (is_set(options, pe_print_details)) {
GHashTableIter iter;
gpointer key, value;
out->begin_list(out, NULL, NULL, "Options");
g_hash_table_iter_init(&iter, rsc->parameters);
while (g_hash_table_iter_next(&iter, &key, &value)) {
out->list_item(out, NULL, "Option: %s = %s", (char *) key, (char *) value);
}
out->end_list(out);
}
if (is_set(options, pe_print_dev)) {
GHashTableIter iter;
pe_node_t *n = NULL;
out->begin_list(out, NULL, NULL, "Allowed Nodes");
g_hash_table_iter_init(&iter, rsc->allowed_nodes);
while (g_hash_table_iter_next(&iter, NULL, (void **)&n)) {
out->list_item(out, NULL, "%s %d", n->details->uname, n->weight);
}
out->end_list(out);
}
if (is_set(options, pe_print_max_details)) {
GHashTableIter iter;
pe_node_t *n = NULL;
out->begin_list(out, NULL, NULL, "=== Allowed Nodes");
g_hash_table_iter_init(&iter, rsc->allowed_nodes);
while (g_hash_table_iter_next(&iter, NULL, (void **)&n)) {
pe__output_node(n, FALSE, out);
}
out->end_list(out);
}
return pcmk_rc_ok;
}
int
pe__common_output_text(pcmk__output_t *out, pe_resource_t * rsc,
const char *name, pe_node_t *node, long options)
{
const char *target_role = NULL;
CRM_ASSERT(rsc->variant == pe_native);
if (rsc->meta) {
const char *is_internal = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_INTERNAL_RSC);
if (crm_is_true(is_internal) && is_not_set(options, pe_print_implicit)) {
crm_trace("skipping print of internal resource %s", rsc->id);
return pcmk_rc_no_output;
}
target_role = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_TARGET_ROLE);
}
{
gchar *s = native_output_string(rsc, name, node, options, target_role,
true);
out->list_item(out, NULL, "%s", s);
g_free(s);
}
if (is_set(options, pe_print_details)) {
GHashTableIter iter;
gpointer key, value;
out->begin_list(out, NULL, NULL, "Options");
g_hash_table_iter_init(&iter, rsc->parameters);
while (g_hash_table_iter_next(&iter, &key, &value)) {
out->list_item(out, NULL, "Option: %s = %s", (char *) key, (char *) value);
}
out->end_list(out);
}
if (is_set(options, pe_print_dev)) {
GHashTableIter iter;
pe_node_t *n = NULL;
out->begin_list(out, NULL, NULL, "Allowed Nodes");
g_hash_table_iter_init(&iter, rsc->allowed_nodes);
while (g_hash_table_iter_next(&iter, NULL, (void **)&n)) {
out->list_item(out, NULL, "%s %d", n->details->uname, n->weight);
}
out->end_list(out);
}
if (is_set(options, pe_print_max_details)) {
GHashTableIter iter;
pe_node_t *n = NULL;
out->begin_list(out, NULL, NULL, "=== Allowed Nodes");
g_hash_table_iter_init(&iter, rsc->allowed_nodes);
while (g_hash_table_iter_next(&iter, NULL, (void **)&n)) {
pe__output_node(n, FALSE, out);
}
out->end_list(out);
}
return pcmk_rc_ok;
}
void
common_print(pe_resource_t * rsc, const char *pre_text, const char *name, pe_node_t *node, long options, void *print_data)
{
const char *target_role = NULL;
CRM_ASSERT(rsc->variant == pe_native);
if (rsc->meta) {
const char *is_internal = g_hash_table_lookup(rsc->meta,
XML_RSC_ATTR_INTERNAL_RSC);
if (crm_is_true(is_internal) && is_not_set(options, pe_print_implicit)) {
crm_trace("skipping print of internal resource %s", rsc->id);
return;
}
target_role = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_TARGET_ROLE);
}
if (options & pe_print_xml) {
native_print_xml(rsc, pre_text, options, print_data);
return;
}
if ((pre_text == NULL) && (options & pe_print_printf)) {
pre_text = " ";
}
if (options & pe_print_html) {
if (is_not_set(rsc->flags, pe_rsc_managed)) {
status_print("");
} else if (is_set(rsc->flags, pe_rsc_failed)) {
status_print("");
} else if (rsc->running_on == NULL) {
status_print("");
} else if (pcmk__list_of_multiple(rsc->running_on)) {
status_print("");
} else if (is_set(rsc->flags, pe_rsc_failure_ignored)) {
status_print("");
} else {
status_print("");
}
}
{
gchar *resource_s = native_output_string(rsc, name, node, options,
target_role, false);
status_print("%s%s", (pre_text? pre_text : ""), resource_s);
g_free(resource_s);
}
#if CURSES_ENABLED
if (is_set(options, pe_print_ncurses)
&& is_not_set(options, pe_print_rsconly)
&& !pcmk__list_of_multiple(rsc->running_on)) {
/* coverity[negative_returns] False positive */
move(-1, 0);
}
#endif
if (is_set(options, pe_print_html)) {
status_print(" ");
}
if (is_not_set(options, pe_print_rsconly)
&& pcmk__list_of_multiple(rsc->running_on)) {
GListPtr gIter = rsc->running_on;
int counter = 0;
if (options & pe_print_html) {
status_print("\n");
} else if ((options & pe_print_printf)
|| (options & pe_print_ncurses)) {
status_print("[");
}
for (; gIter != NULL; gIter = gIter->next) {
pe_node_t *n = (pe_node_t *) gIter->data;
counter++;
if (options & pe_print_html) {
status_print("- \n%s", n->details->uname);
} else if ((options & pe_print_printf)
|| (options & pe_print_ncurses)) {
status_print(" %s", n->details->uname);
} else if ((options & pe_print_log)) {
status_print("\t%d : %s", counter, n->details->uname);
} else {
status_print("%s", n->details->uname);
}
if (options & pe_print_html) {
status_print("
\n");
}
}
if (options & pe_print_html) {
status_print("
\n");
} else if ((options & pe_print_printf)
|| (options & pe_print_ncurses)) {
status_print(" ]");
}
}
if (options & pe_print_html) {
status_print("
\n");
} else if (options & pe_print_suppres_nl) {
/* nothing */
} else if ((options & pe_print_printf) || (options & pe_print_ncurses)) {
status_print("\n");
}
if (options & pe_print_details) {
struct print_data_s pdata;
pdata.options = options;
pdata.print_data = print_data;
g_hash_table_foreach(rsc->parameters, native_print_attr, &pdata);
}
if (options & pe_print_dev) {
GHashTableIter iter;
pe_node_t *n = NULL;
status_print("%s\tAllowed Nodes", pre_text);
g_hash_table_iter_init(&iter, rsc->allowed_nodes);
while (g_hash_table_iter_next(&iter, NULL, (void **)&n)) {
status_print("%s\t * %s %d", pre_text, n->details->uname, n->weight);
}
}
if (options & pe_print_max_details) {
GHashTableIter iter;
pe_node_t *n = NULL;
status_print("%s\t=== Allowed Nodes\n", pre_text);
g_hash_table_iter_init(&iter, rsc->allowed_nodes);
while (g_hash_table_iter_next(&iter, NULL, (void **)&n)) {
print_node("\t", n, FALSE);
}
}
}
void
native_print(pe_resource_t * rsc, const char *pre_text, long options, void *print_data)
{
pe_node_t *node = NULL;
CRM_ASSERT(rsc->variant == pe_native);
if (options & pe_print_xml) {
native_print_xml(rsc, pre_text, options, print_data);
return;
}
node = pe__current_node(rsc);
if (node == NULL) {
// This is set only if a non-probe action is pending on this node
node = rsc->pending_node;
}
common_print(rsc, pre_text, rsc_printable_id(rsc), node, options, print_data);
}
PCMK__OUTPUT_ARGS("primitive", "unsigned int", "struct pe_resource_t *", "GListPtr")
int
pe__resource_xml(pcmk__output_t *out, va_list args)
{
unsigned int options = va_arg(args, unsigned int);
pe_resource_t *rsc = va_arg(args, pe_resource_t *);
GListPtr only_show G_GNUC_UNUSED = va_arg(args, GListPtr);
const char *class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
const char *prov = crm_element_value(rsc->xml, XML_AGENT_ATTR_PROVIDER);
const char *rsc_state = native_displayable_state(rsc, options);
long is_print_pending = options & pe_print_pending;
long is_print_dev = options & pe_print_dev;
char ra_name[LINE_MAX];
char *nodes_running_on = NULL;
char *priority = NULL;
int rc = pcmk_rc_no_output;
CRM_ASSERT(rsc->variant == pe_native);
/* resource information. */
sprintf(ra_name, "%s%s%s:%s", class, prov ? "::" : "", prov ? prov : ""
, crm_element_value(rsc->xml, XML_ATTR_TYPE));
nodes_running_on = crm_itoa(g_list_length(rsc->running_on));
priority = crm_ftoa(rsc->priority);
rc = pe__name_and_nvpairs_xml(out, true, "resource", 16
, "id", rsc_printable_id(rsc)
, "resource_agent", ra_name
, "role", rsc_state
, "target_role", (rsc->meta ? g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_TARGET_ROLE) : NULL)
, "active", BOOL2STR(rsc->fns->active(rsc, TRUE))
, "orphaned", BOOL2STR(is_set(rsc->flags, pe_rsc_orphan))
, "blocked", BOOL2STR(is_set(rsc->flags, pe_rsc_block))
, "managed", BOOL2STR(is_set(rsc->flags, pe_rsc_managed))
, "failed", BOOL2STR(is_set(rsc->flags, pe_rsc_failed))
, "failure_ignored", BOOL2STR(is_set(rsc->flags, pe_rsc_failure_ignored))
, "nodes_running_on", nodes_running_on
, "pending", (is_print_pending ? native_pending_task(rsc) : NULL)
, "provisional", (is_print_dev ? BOOL2STR(is_set(rsc->flags, pe_rsc_provisional)) : NULL)
, "runnable", (is_print_dev ? BOOL2STR(is_set(rsc->flags, pe_rsc_runnable)) : NULL)
, "priority", (is_print_dev ? priority : NULL)
, "variant", (is_print_dev ? crm_element_name(rsc->xml) : NULL));
free(priority);
free(nodes_running_on);
CRM_ASSERT(rc == pcmk_rc_ok);
if (rsc->running_on != NULL) {
GListPtr gIter = rsc->running_on;
for (; gIter != NULL; gIter = gIter->next) {
pe_node_t *node = (pe_node_t *) gIter->data;
rc = pe__name_and_nvpairs_xml(out, false, "node", 3
, "name", node->details->uname
, "id", node->details->id
, "cached", BOOL2STR(node->details->online));
CRM_ASSERT(rc == pcmk_rc_ok);
}
}
pcmk__output_xml_pop_parent(out);
return rc;
}
PCMK__OUTPUT_ARGS("primitive", "unsigned int", "struct pe_resource_t *", "GListPtr")
int
pe__resource_html(pcmk__output_t *out, va_list args)
{
unsigned int options = va_arg(args, unsigned int);
pe_resource_t *rsc = va_arg(args, pe_resource_t *);
GListPtr only_show G_GNUC_UNUSED = va_arg(args, GListPtr);
pe_node_t *node = pe__current_node(rsc);
CRM_ASSERT(rsc->variant == pe_native);
if (node == NULL) {
// This is set only if a non-probe action is pending on this node
node = rsc->pending_node;
}
return pe__common_output_html(out, rsc, rsc_printable_id(rsc), node, options);
}
PCMK__OUTPUT_ARGS("primitive", "unsigned int", "struct pe_resource_t *", "GListPtr")
int
pe__resource_text(pcmk__output_t *out, va_list args)
{
unsigned int options = va_arg(args, unsigned int);
pe_resource_t *rsc = va_arg(args, pe_resource_t *);
GListPtr only_show G_GNUC_UNUSED = va_arg(args, GListPtr);
pe_node_t *node = pe__current_node(rsc);
CRM_ASSERT(rsc->variant == pe_native);
if (node == NULL) {
// This is set only if a non-probe action is pending on this node
node = rsc->pending_node;
}
return pe__common_output_text(out, rsc, rsc_printable_id(rsc), node, options);
}
void
native_free(pe_resource_t * rsc)
{
pe_rsc_trace(rsc, "Freeing resource action list (not the data)");
common_free(rsc);
}
enum rsc_role_e
native_resource_state(const pe_resource_t * rsc, gboolean current)
{
enum rsc_role_e role = rsc->next_role;
if (current) {
role = rsc->role;
}
pe_rsc_trace(rsc, "%s state: %s", rsc->id, role2text(role));
return role;
}
/*!
* \internal
* \brief List nodes where a resource (or any of its children) is
*
* \param[in] rsc Resource to check
* \param[out] list List to add result to
* \param[in] current 0 = where known, 1 = running, 2 = running or pending
*
* \return If list contains only one node, that node
*/
pe_node_t *
native_location(const pe_resource_t *rsc, GList **list, int current)
{
pe_node_t *one = NULL;
GListPtr result = NULL;
if (rsc->children) {
GListPtr gIter = rsc->children;
for (; gIter != NULL; gIter = gIter->next) {
pe_resource_t *child = (pe_resource_t *) gIter->data;
child->fns->location(child, &result, current);
}
} else if (current) {
if (rsc->running_on) {
result = g_list_copy(rsc->running_on);
}
if ((current == 2) && rsc->pending_node
&& !pe_find_node_id(result, rsc->pending_node->details->id)) {
result = g_list_append(result, rsc->pending_node);
}
} else if (current == FALSE && rsc->allocated_to) {
result = g_list_append(NULL, rsc->allocated_to);
}
if (result && (result->next == NULL)) {
one = result->data;
}
if (list) {
GListPtr gIter = result;
for (; gIter != NULL; gIter = gIter->next) {
pe_node_t *node = (pe_node_t *) gIter->data;
if (*list == NULL || pe_find_node_id(*list, node->details->id) == NULL) {
*list = g_list_append(*list, node);
}
}
}
g_list_free(result);
return one;
}
static void
get_rscs_brief(GListPtr rsc_list, GHashTable * rsc_table, GHashTable * active_table)
{
GListPtr gIter = rsc_list;
for (; gIter != NULL; gIter = gIter->next) {
pe_resource_t *rsc = (pe_resource_t *) gIter->data;
const char *class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
const char *kind = crm_element_value(rsc->xml, XML_ATTR_TYPE);
int offset = 0;
char buffer[LINE_MAX];
int *rsc_counter = NULL;
int *active_counter = NULL;
if (rsc->variant != pe_native) {
continue;
}
offset += snprintf(buffer + offset, LINE_MAX - offset, "%s", class);
if (is_set(pcmk_get_ra_caps(class), pcmk_ra_cap_provider)) {
const char *prov = crm_element_value(rsc->xml, XML_AGENT_ATTR_PROVIDER);
offset += snprintf(buffer + offset, LINE_MAX - offset, "::%s", prov);
}
offset += snprintf(buffer + offset, LINE_MAX - offset, ":%s", kind);
CRM_LOG_ASSERT(offset > 0);
if (rsc_table) {
rsc_counter = g_hash_table_lookup(rsc_table, buffer);
if (rsc_counter == NULL) {
rsc_counter = calloc(1, sizeof(int));
*rsc_counter = 0;
g_hash_table_insert(rsc_table, strdup(buffer), rsc_counter);
}
(*rsc_counter)++;
}
if (active_table) {
GListPtr gIter2 = rsc->running_on;
for (; gIter2 != NULL; gIter2 = gIter2->next) {
pe_node_t *node = (pe_node_t *) gIter2->data;
GHashTable *node_table = NULL;
if (node->details->unclean == FALSE && node->details->online == FALSE) {
continue;
}
node_table = g_hash_table_lookup(active_table, node->details->uname);
if (node_table == NULL) {
node_table = crm_str_table_new();
g_hash_table_insert(active_table, strdup(node->details->uname), node_table);
}
active_counter = g_hash_table_lookup(node_table, buffer);
if (active_counter == NULL) {
active_counter = calloc(1, sizeof(int));
*active_counter = 0;
g_hash_table_insert(node_table, strdup(buffer), active_counter);
}
(*active_counter)++;
}
}
}
}
static void
destroy_node_table(gpointer data)
{
GHashTable *node_table = data;
if (node_table) {
g_hash_table_destroy(node_table);
}
}
void
print_rscs_brief(GListPtr rsc_list, const char *pre_text, long options,
void *print_data, gboolean print_all)
{
GHashTable *rsc_table = crm_str_table_new();
GHashTable *active_table = g_hash_table_new_full(crm_str_hash, g_str_equal,
free, destroy_node_table);
GHashTableIter hash_iter;
char *type = NULL;
int *rsc_counter = NULL;
get_rscs_brief(rsc_list, rsc_table, active_table);
g_hash_table_iter_init(&hash_iter, rsc_table);
while (g_hash_table_iter_next(&hash_iter, (gpointer *)&type, (gpointer *)&rsc_counter)) {
GHashTableIter hash_iter2;
char *node_name = NULL;
GHashTable *node_table = NULL;
int active_counter_all = 0;
g_hash_table_iter_init(&hash_iter2, active_table);
while (g_hash_table_iter_next(&hash_iter2, (gpointer *)&node_name, (gpointer *)&node_table)) {
int *active_counter = g_hash_table_lookup(node_table, type);
if (active_counter == NULL || *active_counter == 0) {
continue;
} else {
active_counter_all += *active_counter;
}
if (options & pe_print_rsconly) {
node_name = NULL;
}
if (options & pe_print_html) {
status_print("\n");
}
if (print_all) {
status_print("%s%d/%d\t(%s):\tActive %s\n", pre_text ? pre_text : "",
active_counter ? *active_counter : 0,
rsc_counter ? *rsc_counter : 0, type,
active_counter && (*active_counter > 0) && node_name ? node_name : "");
} else {
status_print("%s%d\t(%s):\tActive %s\n", pre_text ? pre_text : "",
active_counter ? *active_counter : 0, type,
active_counter && (*active_counter > 0) && node_name ? node_name : "");
}
if (options & pe_print_html) {
status_print("\n");
}
}
if (print_all && active_counter_all == 0) {
if (options & pe_print_html) {
status_print("\n");
}
status_print("%s%d/%d\t(%s):\tActive\n", pre_text ? pre_text : "",
active_counter_all,
rsc_counter ? *rsc_counter : 0, type);
if (options & pe_print_html) {
status_print("\n");
}
}
}
if (rsc_table) {
g_hash_table_destroy(rsc_table);
rsc_table = NULL;
}
if (active_table) {
g_hash_table_destroy(active_table);
active_table = NULL;
}
}
int
pe__rscs_brief_output(pcmk__output_t *out, GListPtr rsc_list, long options, gboolean print_all)
{
GHashTable *rsc_table = crm_str_table_new();
GHashTable *active_table = g_hash_table_new_full(crm_str_hash, g_str_equal,
free, destroy_node_table);
GHashTableIter hash_iter;
char *type = NULL;
int *rsc_counter = NULL;
int rc = pcmk_rc_no_output;
get_rscs_brief(rsc_list, rsc_table, active_table);
g_hash_table_iter_init(&hash_iter, rsc_table);
while (g_hash_table_iter_next(&hash_iter, (gpointer *)&type, (gpointer *)&rsc_counter)) {
GHashTableIter hash_iter2;
char *node_name = NULL;
GHashTable *node_table = NULL;
int active_counter_all = 0;
g_hash_table_iter_init(&hash_iter2, active_table);
while (g_hash_table_iter_next(&hash_iter2, (gpointer *)&node_name, (gpointer *)&node_table)) {
int *active_counter = g_hash_table_lookup(node_table, type);
if (active_counter == NULL || *active_counter == 0) {
continue;
} else {
active_counter_all += *active_counter;
}
if (options & pe_print_rsconly) {
node_name = NULL;
}
if (print_all) {
out->list_item(out, NULL, " %d/%d\t(%s):\tActive %s",
*active_counter,
rsc_counter ? *rsc_counter : 0, type,
(*active_counter > 0) && node_name ? node_name : "");
} else {
out->list_item(out, NULL, " %d\t(%s):\tActive %s",
*active_counter, type,
(*active_counter > 0) && node_name ? node_name : "");
}
rc = pcmk_rc_ok;
}
if (print_all && active_counter_all == 0) {
out->list_item(out, NULL, " %d/%d\t(%s):\tActive",
active_counter_all,
rsc_counter ? *rsc_counter : 0, type);
rc = pcmk_rc_ok;
}
}
if (rsc_table) {
g_hash_table_destroy(rsc_table);
rsc_table = NULL;
}
if (active_table) {
g_hash_table_destroy(active_table);
active_table = NULL;
}
return rc;
}
diff --git a/lib/pengine/pe_output.c b/lib/pengine/pe_output.c
index 75bf0d5a14..ad469aba4a 100644
--- a/lib/pengine/pe_output.c
+++ b/lib/pengine/pe_output.c
@@ -1,1795 +1,1809 @@
/*
* Copyright 2019-2020 the Pacemaker project contributors
*
* The version control history for this file may have further details.
*
* This source code is licensed under the GNU Lesser General Public License
* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
*/
#include
#include
#include
#include
#define SUMMARY_HEADER(rc, out) do { \
if (rc == pcmk_rc_no_output) { \
out->begin_list(out, NULL, NULL, "Cluster Summary"); \
rc = pcmk_rc_ok; \
} \
} while (0)
static char *
failed_action_string(xmlNodePtr xml_op) {
const char *op_key = crm_element_value(xml_op, XML_LRM_ATTR_TASK_KEY);
int rc = crm_parse_int(crm_element_value(xml_op, XML_LRM_ATTR_RC), "0");
int status = crm_parse_int(crm_element_value(xml_op, XML_LRM_ATTR_OPSTATUS), "0");
const char *exit_reason = crm_element_value(xml_op, XML_LRM_ATTR_EXIT_REASON);
time_t last_change = 0;
if (crm_element_value_epoch(xml_op, XML_RSC_OP_LAST_CHANGE,
&last_change) == pcmk_ok) {
crm_time_t *crm_when = crm_time_new(NULL);
char *time_s = NULL;
char *buf = NULL;
crm_time_set_timet(crm_when, &last_change);
time_s = crm_time_as_string(crm_when, crm_time_log_date | crm_time_log_timeofday | crm_time_log_with_timezone);
buf = crm_strdup_printf("%s on %s '%s' (%d): call=%s, status='%s', "
"exitreason='%s', " XML_RSC_OP_LAST_CHANGE
"='%s', queued=%sms, exec=%sms",
op_key ? op_key : ID(xml_op),
crm_element_value(xml_op, XML_ATTR_UNAME),
services_ocf_exitcode_str(rc), rc,
crm_element_value(xml_op, XML_LRM_ATTR_CALLID),
services_lrm_status_str(status),
exit_reason ? exit_reason : "none",
time_s,
crm_element_value(xml_op, XML_RSC_OP_T_QUEUE),
crm_element_value(xml_op, XML_RSC_OP_T_EXEC));
crm_time_free(crm_when);
free(time_s);
return buf;
} else {
return crm_strdup_printf("%s on %s '%s' (%d): call=%s, status=%s, exitreason='%s'",
op_key ? op_key : ID(xml_op),
crm_element_value(xml_op, XML_ATTR_UNAME),
services_ocf_exitcode_str(rc), rc,
crm_element_value(xml_op, XML_LRM_ATTR_CALLID),
services_lrm_status_str(status),
exit_reason ? exit_reason : "none");
}
}
static const char *
get_cluster_stack(pe_working_set_t *data_set)
{
xmlNode *stack = get_xpath_object("//nvpair[@name='cluster-infrastructure']",
data_set->input, LOG_DEBUG);
return stack? crm_element_value(stack, XML_NVPAIR_ATTR_VALUE) : "unknown";
}
static char *
last_changed_string(const char *last_written, const char *user,
const char *client, const char *origin) {
if (last_written != NULL || user != NULL || client != NULL || origin != NULL) {
return crm_strdup_printf("%s%s%s%s%s%s%s",
last_written ? last_written : "",
user ? " by " : "",
user ? user : "",
client ? " via " : "",
client ? client : "",
origin ? " on " : "",
origin ? origin : "");
} else {
return strdup("");
}
}
static char *
op_history_string(xmlNode *xml_op, const char *task, const char *interval_ms_s,
int rc, gboolean print_timing) {
const char *call = crm_element_value(xml_op, XML_LRM_ATTR_CALLID);
char *interval_str = NULL;
char *buf = NULL;
if (interval_ms_s && safe_str_neq(interval_ms_s, "0")) {
char *pair = pcmk_format_nvpair("interval", interval_ms_s, "ms");
interval_str = crm_strdup_printf(" %s", pair);
free(pair);
}
if (print_timing) {
char *last_change_str = NULL;
char *last_run_str = NULL;
char *exec_str = NULL;
char *queue_str = NULL;
const char *value = NULL;
time_t epoch = 0;
if ((crm_element_value_epoch(xml_op, XML_RSC_OP_LAST_CHANGE, &epoch) == pcmk_ok)
&& (epoch > 0)) {
char *time = pcmk_format_named_time(XML_RSC_OP_LAST_CHANGE, epoch);
last_change_str = crm_strdup_printf(" %s", time);
free(time);
}
if ((crm_element_value_epoch(xml_op, XML_RSC_OP_LAST_RUN, &epoch) == pcmk_ok)
&& (epoch > 0)) {
char *time = pcmk_format_named_time(XML_RSC_OP_LAST_RUN, epoch);
last_run_str = crm_strdup_printf(" %s", time);
free(time);
}
value = crm_element_value(xml_op, XML_RSC_OP_T_EXEC);
if (value) {
char *pair = pcmk_format_nvpair(XML_RSC_OP_T_EXEC, value, "ms");
exec_str = crm_strdup_printf(" %s", pair);
free(pair);
}
value = crm_element_value(xml_op, XML_RSC_OP_T_QUEUE);
if (value) {
char *pair = pcmk_format_nvpair(XML_RSC_OP_T_QUEUE, value, "ms");
queue_str = crm_strdup_printf(" %s", pair);
free(pair);
}
buf = crm_strdup_printf("(%s) %s:%s%s%s%s%s rc=%d (%s)", call, task,
interval_str ? interval_str : "",
last_change_str ? last_change_str : "",
last_run_str ? last_run_str : "",
exec_str ? exec_str : "",
queue_str ? queue_str : "",
rc, services_ocf_exitcode_str(rc));
if (last_change_str) {
free(last_change_str);
}
if (last_run_str) {
free(last_run_str);
}
if (exec_str) {
free(exec_str);
}
if (queue_str) {
free(queue_str);
}
} else {
buf = crm_strdup_printf("(%s) %s%s%s", call, task,
interval_str ? ":" : "",
interval_str ? interval_str : "");
}
if (interval_str) {
free(interval_str);
}
return buf;
}
static char *
resource_history_string(pe_resource_t *rsc, const char *rsc_id, gboolean all,
int failcount, time_t last_failure) {
char *buf = NULL;
if (rsc == NULL) {
buf = crm_strdup_printf("%s: orphan", rsc_id);
} else if (all || failcount || last_failure > 0) {
char *failcount_s = NULL;
char *lastfail_s = NULL;
if (failcount > 0) {
failcount_s = crm_strdup_printf(" %s=%d", PCMK__FAIL_COUNT_PREFIX,
failcount);
} else {
failcount_s = strdup("");
}
if (last_failure > 0) {
lastfail_s = crm_strdup_printf(" %s='%s'",
PCMK__LAST_FAILURE_PREFIX,
pcmk__epoch2str(&last_failure));
}
buf = crm_strdup_printf("%s: migration-threshold=%d%s%s",
rsc_id, rsc->migration_threshold, failcount_s,
lastfail_s? lastfail_s : "");
free(failcount_s);
free(lastfail_s);
} else {
buf = crm_strdup_printf("%s:", rsc_id);
}
return buf;
}
PCMK__OUTPUT_ARGS("cluster-summary", "struct pe_working_set_t *", "gboolean", "gboolean", "gboolean",
"gboolean", "gboolean", "gboolean")
int
pe__cluster_summary(pcmk__output_t *out, va_list args) {
pe_working_set_t *data_set = va_arg(args, pe_working_set_t *);
gboolean print_clone_detail = va_arg(args, gboolean);
gboolean show_stack = va_arg(args, gboolean);
gboolean show_dc = va_arg(args, gboolean);
gboolean show_times = va_arg(args, gboolean);
gboolean show_counts = va_arg(args, gboolean);
gboolean show_options = va_arg(args, gboolean);
int rc = pcmk_rc_no_output;
const char *stack_s = get_cluster_stack(data_set);
if (show_stack) {
SUMMARY_HEADER(rc, out);
out->message(out, "cluster-stack", stack_s);
}
/* Always print DC if none, even if not requested */
if (data_set->dc_node == NULL || show_dc) {
xmlNode *dc_version = get_xpath_object("//nvpair[@name='dc-version']",
data_set->input, LOG_DEBUG);
const char *dc_version_s = dc_version?
crm_element_value(dc_version, XML_NVPAIR_ATTR_VALUE)
: NULL;
const char *quorum = crm_element_value(data_set->input, XML_ATTR_HAVE_QUORUM);
char *dc_name = data_set->dc_node ? pe__node_display_name(data_set->dc_node, print_clone_detail) : NULL;
SUMMARY_HEADER(rc, out);
out->message(out, "cluster-dc", data_set->dc_node, quorum, dc_version_s, dc_name);
free(dc_name);
}
if (show_times) {
const char *last_written = crm_element_value(data_set->input, XML_CIB_ATTR_WRITTEN);
const char *user = crm_element_value(data_set->input, XML_ATTR_UPDATE_USER);
const char *client = crm_element_value(data_set->input, XML_ATTR_UPDATE_CLIENT);
const char *origin = crm_element_value(data_set->input, XML_ATTR_UPDATE_ORIG);
SUMMARY_HEADER(rc, out);
out->message(out, "cluster-times", last_written, user, client, origin);
}
if (show_counts) {
SUMMARY_HEADER(rc, out);
out->message(out, "cluster-counts", g_list_length(data_set->nodes),
data_set->ninstances, data_set->disabled_resources,
data_set->blocked_resources);
}
if (show_options) {
SUMMARY_HEADER(rc, out);
out->message(out, "cluster-options", data_set);
}
if (rc == pcmk_rc_ok) {
out->end_list(out);
}
if (is_set(data_set->flags, pe_flag_maintenance_mode)) {
out->message(out, "maint-mode");
rc = pcmk_rc_ok;
}
return rc;
}
PCMK__OUTPUT_ARGS("cluster-summary", "struct pe_working_set_t *", "gboolean", "gboolean", "gboolean",
"gboolean", "gboolean", "gboolean")
int
pe__cluster_summary_html(pcmk__output_t *out, va_list args) {
pe_working_set_t *data_set = va_arg(args, pe_working_set_t *);
gboolean print_clone_detail = va_arg(args, gboolean);
gboolean show_stack = va_arg(args, gboolean);
gboolean show_dc = va_arg(args, gboolean);
gboolean show_times = va_arg(args, gboolean);
gboolean show_counts = va_arg(args, gboolean);
gboolean show_options = va_arg(args, gboolean);
int rc = pcmk_rc_no_output;
const char *stack_s = get_cluster_stack(data_set);
if (show_stack) {
SUMMARY_HEADER(rc, out);
out->message(out, "cluster-stack", stack_s);
}
/* Always print DC if none, even if not requested */
if (data_set->dc_node == NULL || show_dc) {
xmlNode *dc_version = get_xpath_object("//nvpair[@name='dc-version']",
data_set->input, LOG_DEBUG);
const char *dc_version_s = dc_version?
crm_element_value(dc_version, XML_NVPAIR_ATTR_VALUE)
: NULL;
const char *quorum = crm_element_value(data_set->input, XML_ATTR_HAVE_QUORUM);
char *dc_name = data_set->dc_node ? pe__node_display_name(data_set->dc_node, print_clone_detail) : NULL;
SUMMARY_HEADER(rc, out);
out->message(out, "cluster-dc", data_set->dc_node, quorum, dc_version_s, dc_name);
free(dc_name);
}
if (show_times) {
const char *last_written = crm_element_value(data_set->input, XML_CIB_ATTR_WRITTEN);
const char *user = crm_element_value(data_set->input, XML_ATTR_UPDATE_USER);
const char *client = crm_element_value(data_set->input, XML_ATTR_UPDATE_CLIENT);
const char *origin = crm_element_value(data_set->input, XML_ATTR_UPDATE_ORIG);
SUMMARY_HEADER(rc, out);
out->message(out, "cluster-times", last_written, user, client, origin);
}
if (show_counts) {
SUMMARY_HEADER(rc, out);
out->message(out, "cluster-counts", g_list_length(data_set->nodes),
data_set->ninstances, data_set->disabled_resources,
data_set->blocked_resources);
}
if (show_options) {
/* Kind of a hack - close the list we may have opened earlier in this
* function so we can put all the options into their own list. We
* only want to do this on HTML output, though.
*/
if (rc == pcmk_rc_ok) {
out->end_list(out);
}
out->begin_list(out, NULL, NULL, "Config Options");
out->message(out, "cluster-options", data_set);
}
if (rc == pcmk_rc_ok) {
out->end_list(out);
}
if (is_set(data_set->flags, pe_flag_maintenance_mode)) {
out->message(out, "maint-mode");
rc = pcmk_rc_ok;
}
return rc;
}
char *
pe__node_display_name(pe_node_t *node, bool print_detail)
{
char *node_name;
const char *node_host = NULL;
const char *node_id = NULL;
int name_len;
CRM_ASSERT((node != NULL) && (node->details != NULL) && (node->details->uname != NULL));
/* Host is displayed only if this is a guest node */
if (pe__is_guest_node(node)) {
pe_node_t *host_node = pe__current_node(node->details->remote_rsc);
if (host_node && host_node->details) {
node_host = host_node->details->uname;
}
if (node_host == NULL) {
node_host = ""; /* so we at least get "uname@" to indicate guest */
}
}
/* Node ID is displayed if different from uname and detail is requested */
if (print_detail && safe_str_neq(node->details->uname, node->details->id)) {
node_id = node->details->id;
}
/* Determine name length */
name_len = strlen(node->details->uname) + 1;
if (node_host) {
name_len += strlen(node_host) + 1; /* "@node_host" */
}
if (node_id) {
name_len += strlen(node_id) + 3; /* + " (node_id)" */
}
/* Allocate and populate display name */
node_name = malloc(name_len);
CRM_ASSERT(node_name != NULL);
strcpy(node_name, node->details->uname);
if (node_host) {
strcat(node_name, "@");
strcat(node_name, node_host);
}
if (node_id) {
strcat(node_name, " (");
strcat(node_name, node_id);
strcat(node_name, ")");
}
return node_name;
}
int
pe__name_and_nvpairs_xml(pcmk__output_t *out, bool is_list, const char *tag_name
, size_t pairs_count, ...)
{
xmlNodePtr xml_node = NULL;
va_list args;
CRM_ASSERT(tag_name != NULL);
xml_node = pcmk__output_xml_peek_parent(out);
CRM_ASSERT(xml_node != NULL);
xml_node = is_list
? create_xml_node(xml_node, tag_name)
: xmlNewChild(xml_node, NULL, (pcmkXmlStr) tag_name, NULL);
va_start(args, pairs_count);
while(pairs_count--) {
const char *param_name = va_arg(args, const char *);
const char *param_value = va_arg(args, const char *);
if (param_name && param_value) {
xmlSetProp(xml_node, (pcmkXmlStr)param_name, (pcmkXmlStr)param_value);
}
};
va_end(args);
if (is_list) {
pcmk__output_xml_push_parent(out, xml_node);
}
return pcmk_rc_ok;
}
PCMK__OUTPUT_ARGS("ban", "struct pe_node_t *", "struct pe__location_t *", "gboolean")
int
pe__ban_html(pcmk__output_t *out, va_list args) {
pe_node_t *pe_node = va_arg(args, pe_node_t *);
pe__location_t *location = va_arg(args, pe__location_t *);
gboolean print_clone_detail = va_arg(args, gboolean);
char *node_name = pe__node_display_name(pe_node, print_clone_detail);
char *buf = crm_strdup_printf("%s\tprevents %s from running %son %s",
location->id, location->rsc_lh->id,
location->role_filter == RSC_ROLE_MASTER ? "as Master " : "",
node_name);
pcmk__output_create_html_node(out, "li", NULL, NULL, buf);
free(node_name);
free(buf);
return pcmk_rc_ok;
}
PCMK__OUTPUT_ARGS("ban", "struct pe_node_t *", "struct pe__location_t *", "gboolean")
int
pe__ban_text(pcmk__output_t *out, va_list args) {
pe_node_t *pe_node = va_arg(args, pe_node_t *);
pe__location_t *location = va_arg(args, pe__location_t *);
gboolean print_clone_detail = va_arg(args, gboolean);
char *node_name = pe__node_display_name(pe_node, print_clone_detail);
out->list_item(out, NULL, "%s\tprevents %s from running %son %s",
location->id, location->rsc_lh->id,
location->role_filter == RSC_ROLE_MASTER ? "as Master " : "",
node_name);
free(node_name);
return pcmk_rc_ok;
}
PCMK__OUTPUT_ARGS("ban", "struct pe_node_t *", "struct pe__location_t *", "gboolean")
int
pe__ban_xml(pcmk__output_t *out, va_list args) {
xmlNodePtr node = pcmk__output_create_xml_node(out, "ban");
pe_node_t *pe_node = va_arg(args, pe_node_t *);
pe__location_t *location = va_arg(args, pe__location_t *);
gboolean print_clone_detail G_GNUC_UNUSED = va_arg(args, gboolean);
char *weight_s = crm_itoa(pe_node->weight);
xmlSetProp(node, (pcmkXmlStr) "id", (pcmkXmlStr) location->id);
xmlSetProp(node, (pcmkXmlStr) "resource", (pcmkXmlStr) location->rsc_lh->id);
xmlSetProp(node, (pcmkXmlStr) "node", (pcmkXmlStr) pe_node->details->uname);
xmlSetProp(node, (pcmkXmlStr) "weight", (pcmkXmlStr) weight_s);
xmlSetProp(node, (pcmkXmlStr) "master_only",
(pcmkXmlStr) (location->role_filter == RSC_ROLE_MASTER ? "true" : "false"));
free(weight_s);
return pcmk_rc_ok;
}
PCMK__OUTPUT_ARGS("cluster-counts", "unsigned int", "int", "int", "int")
int
pe__cluster_counts_html(pcmk__output_t *out, va_list args) {
xmlNodePtr nodes_node = pcmk__output_create_xml_node(out, "li");
xmlNodePtr resources_node = pcmk__output_create_xml_node(out, "li");
unsigned int nnodes = va_arg(args, unsigned int);
int nresources = va_arg(args, int);
int ndisabled = va_arg(args, int);
int nblocked = va_arg(args, int);
char *nnodes_str = crm_strdup_printf("%d node%s configured",
nnodes, pcmk__plural_s(nnodes));
pcmk_create_html_node(nodes_node, "span", NULL, NULL, nnodes_str);
free(nnodes_str);
if (ndisabled && nblocked) {
char *s = crm_strdup_printf("%d resource instance%s configured (%d ",
nresources, pcmk__plural_s(nresources),
ndisabled);
pcmk_create_html_node(resources_node, "span", NULL, NULL, s);
free(s);
pcmk_create_html_node(resources_node, "span", NULL, "bold", "DISABLED");
s = crm_strdup_printf(", %d ", nblocked);
pcmk_create_html_node(resources_node, "span", NULL, NULL, s);
free(s);
pcmk_create_html_node(resources_node, "span", NULL, "bold", "BLOCKED");
pcmk_create_html_node(resources_node, "span", NULL, NULL,
" from further action due to failure)");
} else if (ndisabled && !nblocked) {
char *s = crm_strdup_printf("%d resource instance%s configured (%d ",
nresources, pcmk__plural_s(nresources),
ndisabled);
pcmk_create_html_node(resources_node, "span", NULL, NULL, s);
free(s);
pcmk_create_html_node(resources_node, "span", NULL, "bold", "DISABLED");
pcmk_create_html_node(resources_node, "span", NULL, NULL, ")");
} else if (!ndisabled && nblocked) {
char *s = crm_strdup_printf("%d resource instance%s configured (%d ",
nresources, pcmk__plural_s(nresources),
nblocked);
pcmk_create_html_node(resources_node, "span", NULL, NULL, s);
free(s);
pcmk_create_html_node(resources_node, "span", NULL, "bold", "BLOCKED");
pcmk_create_html_node(resources_node, "span", NULL, NULL,
" from further action due to failure)");
} else {
char *s = crm_strdup_printf("%d resource instance%s configured",
nresources, pcmk__plural_s(nresources));
pcmk_create_html_node(resources_node, "span", NULL, NULL, s);
free(s);
}
return pcmk_rc_ok;
}
PCMK__OUTPUT_ARGS("cluster-counts", "unsigned int", "int", "int", "int")
int
pe__cluster_counts_text(pcmk__output_t *out, va_list args) {
unsigned int nnodes = va_arg(args, unsigned int);
int nresources = va_arg(args, int);
int ndisabled = va_arg(args, int);
int nblocked = va_arg(args, int);
out->list_item(out, NULL, "%d node%s configured",
nnodes, pcmk__plural_s(nnodes));
if (ndisabled && nblocked) {
out->list_item(out, NULL, "%d resource instance%s configured "
"(%d DISABLED, %d BLOCKED from "
"further action due to failure)",
nresources, pcmk__plural_s(nresources), ndisabled,
nblocked);
} else if (ndisabled && !nblocked) {
out->list_item(out, NULL, "%d resource instance%s configured "
"(%d DISABLED)",
nresources, pcmk__plural_s(nresources), ndisabled);
} else if (!ndisabled && nblocked) {
out->list_item(out, NULL, "%d resource instance%s configured "
"(%d BLOCKED from further action "
"due to failure)",
nresources, pcmk__plural_s(nresources), nblocked);
} else {
out->list_item(out, NULL, "%d resource instance%s configured",
nresources, pcmk__plural_s(nresources));
}
return pcmk_rc_ok;
}
PCMK__OUTPUT_ARGS("cluster-counts", "unsigned int", "int", "int", "int")
int
pe__cluster_counts_xml(pcmk__output_t *out, va_list args) {
xmlNodePtr nodes_node = pcmk__output_create_xml_node(out, "nodes_configured");
xmlNodePtr resources_node = pcmk__output_create_xml_node(out, "resources_configured");
unsigned int nnodes = va_arg(args, unsigned int);
int nresources = va_arg(args, int);
int ndisabled = va_arg(args, int);
int nblocked = va_arg(args, int);
char *s = crm_itoa(nnodes);
xmlSetProp(nodes_node, (pcmkXmlStr) "number", (pcmkXmlStr) s);
free(s);
s = crm_itoa(nresources);
xmlSetProp(resources_node, (pcmkXmlStr) "number", (pcmkXmlStr) s);
free(s);
s = crm_itoa(ndisabled);
xmlSetProp(resources_node, (pcmkXmlStr) "disabled", (pcmkXmlStr) s);
free(s);
s = crm_itoa(nblocked);
xmlSetProp(resources_node, (pcmkXmlStr) "blocked", (pcmkXmlStr) s);
free(s);
return pcmk_rc_ok;
}
PCMK__OUTPUT_ARGS("cluster-dc", "struct pe_node_t *", "const char *", "const char *", "char *")
int
pe__cluster_dc_html(pcmk__output_t *out, va_list args) {
xmlNodePtr node = pcmk__output_create_xml_node(out, "li");
pe_node_t *dc = va_arg(args, pe_node_t *);
const char *quorum = va_arg(args, const char *);
const char *dc_version_s = va_arg(args, const char *);
char *dc_name = va_arg(args, char *);
pcmk_create_html_node(node, "span", NULL, "bold", "Current DC: ");
if (dc) {
if (crm_is_true(quorum)) {
char *buf = crm_strdup_printf("%s (version %s) - partition with quorum",
dc_name, dc_version_s ? dc_version_s : "unknown");
pcmk_create_html_node(node, "span", NULL, NULL, buf);
free(buf);
} else {
char *buf = crm_strdup_printf("%s (version %s) - partition",
dc_name, dc_version_s ? dc_version_s : "unknown");
pcmk_create_html_node(node, "span", NULL, NULL, buf);
free(buf);
pcmk_create_html_node(node, "span", NULL, "warning", "WITHOUT");
pcmk_create_html_node(node, "span", NULL, NULL, "quorum");
}
} else {
pcmk_create_html_node(node ,"span", NULL, "warning", "NONE");
}
return pcmk_rc_ok;
}
PCMK__OUTPUT_ARGS("cluster-dc", "struct pe_node_t *", "const char *", "const char *", "char *")
int
pe__cluster_dc_text(pcmk__output_t *out, va_list args) {
pe_node_t *dc = va_arg(args, pe_node_t *);
const char *quorum = va_arg(args, const char *);
const char *dc_version_s = va_arg(args, const char *);
char *dc_name = va_arg(args, char *);
if (dc) {
out->list_item(out, "Current DC", "%s (version %s) - partition %s quorum",
dc_name, dc_version_s ? dc_version_s : "unknown",
crm_is_true(quorum) ? "with" : "WITHOUT");
} else {
out->list_item(out, "Current DC", "NONE");
}
return pcmk_rc_ok;
}
PCMK__OUTPUT_ARGS("cluster-dc", "struct pe_node_t *", "const char *", "const char *", "char *")
int
pe__cluster_dc_xml(pcmk__output_t *out, va_list args) {
xmlNodePtr node = pcmk__output_create_xml_node(out, "current_dc");
pe_node_t *dc = va_arg(args, pe_node_t *);
const char *quorum = va_arg(args, const char *);
const char *dc_version_s = va_arg(args, const char *);
char *dc_name G_GNUC_UNUSED = va_arg(args, char *);
if (dc) {
xmlSetProp(node, (pcmkXmlStr) "present", (pcmkXmlStr) "true");
xmlSetProp(node, (pcmkXmlStr) "version", (pcmkXmlStr) (dc_version_s ? dc_version_s : ""));
xmlSetProp(node, (pcmkXmlStr) "name", (pcmkXmlStr) dc->details->uname);
xmlSetProp(node, (pcmkXmlStr) "id", (pcmkXmlStr) dc->details->id);
xmlSetProp(node, (pcmkXmlStr) "with_quorum", (pcmkXmlStr) (crm_is_true(quorum) ? "true" : "false"));
} else {
xmlSetProp(node, (pcmkXmlStr) "present", (pcmkXmlStr) "false");
}
return pcmk_rc_ok;
}
PCMK__OUTPUT_ARGS("maint-mode")
int
pe__cluster_maint_mode_text(pcmk__output_t *out, va_list args) {
fprintf(out->dest, "\n *** Resource management is DISABLED ***");
fprintf(out->dest, "\n The cluster will not attempt to start, stop or recover services");
fprintf(out->dest, "\n");
return pcmk_rc_ok;
}
PCMK__OUTPUT_ARGS("cluster-options", "struct pe_working_set_t *")
int
pe__cluster_options_html(pcmk__output_t *out, va_list args) {
pe_working_set_t *data_set = va_arg(args, pe_working_set_t *);
out->list_item(out, NULL, "STONITH of failed nodes %s",
is_set(data_set->flags, pe_flag_stonith_enabled) ? "enabled" : "disabled");
out->list_item(out, NULL, "Cluster is %s",
is_set(data_set->flags, pe_flag_symmetric_cluster) ? "symmetric" : "asymmetric");
switch (data_set->no_quorum_policy) {
case no_quorum_freeze:
out->list_item(out, NULL, "No quorum policy: Freeze resources");
break;
case no_quorum_stop:
out->list_item(out, NULL, "No quorum policy: Stop ALL resources");
break;
+ case no_quorum_demote:
+ out->list_item(out, NULL, "No quorum policy: Demote promotable "
+ "resources and stop all other resources");
+ break;
+
case no_quorum_ignore:
out->list_item(out, NULL, "No quorum policy: Ignore");
break;
case no_quorum_suicide:
out->list_item(out, NULL, "No quorum policy: Suicide");
break;
}
if (is_set(data_set->flags, pe_flag_maintenance_mode)) {
xmlNodePtr node = pcmk__output_create_xml_node(out, "li");
pcmk_create_html_node(node, "span", NULL, NULL, "Resource management: ");
pcmk_create_html_node(node, "span", NULL, "bold", "DISABLED");
pcmk_create_html_node(node, "span", NULL, NULL,
" (the cluster will not attempt to start, stop, or recover services)");
} else {
out->list_item(out, NULL, "Resource management: enabled");
}
return pcmk_rc_ok;
}
PCMK__OUTPUT_ARGS("cluster-options", "struct pe_working_set_t *")
int
pe__cluster_options_log(pcmk__output_t *out, va_list args) {
pe_working_set_t *data_set = va_arg(args, pe_working_set_t *);
if (is_set(data_set->flags, pe_flag_maintenance_mode)) {
out->info(out, "Resource management is DISABLED. The cluster will not attempt to start, stop or recover services.");
return pcmk_rc_ok;
} else {
return pcmk_rc_no_output;
}
}
PCMK__OUTPUT_ARGS("cluster-options", "struct pe_working_set_t *")
int
pe__cluster_options_text(pcmk__output_t *out, va_list args) {
pe_working_set_t *data_set = va_arg(args, pe_working_set_t *);
out->list_item(out, NULL, "STONITH of failed nodes %s",
is_set(data_set->flags, pe_flag_stonith_enabled) ? "enabled" : "disabled");
out->list_item(out, NULL, "Cluster is %s",
is_set(data_set->flags, pe_flag_symmetric_cluster) ? "symmetric" : "asymmetric");
switch (data_set->no_quorum_policy) {
case no_quorum_freeze:
out->list_item(out, NULL, "No quorum policy: Freeze resources");
break;
case no_quorum_stop:
out->list_item(out, NULL, "No quorum policy: Stop ALL resources");
break;
+ case no_quorum_demote:
+ out->list_item(out, NULL, "No quorum policy: Demote promotable "
+ "resources and stop all other resources");
+ break;
+
case no_quorum_ignore:
out->list_item(out, NULL, "No quorum policy: Ignore");
break;
case no_quorum_suicide:
out->list_item(out, NULL, "No quorum policy: Suicide");
break;
}
return pcmk_rc_ok;
}
PCMK__OUTPUT_ARGS("cluster-options", "struct pe_working_set_t *")
int
pe__cluster_options_xml(pcmk__output_t *out, va_list args) {
xmlNodePtr node = pcmk__output_create_xml_node(out, "cluster_options");
pe_working_set_t *data_set = va_arg(args, pe_working_set_t *);
xmlSetProp(node, (pcmkXmlStr) "stonith-enabled",
(pcmkXmlStr) (is_set(data_set->flags, pe_flag_stonith_enabled) ? "true" : "false"));
xmlSetProp(node, (pcmkXmlStr) "symmetric-cluster",
(pcmkXmlStr) (is_set(data_set->flags, pe_flag_symmetric_cluster) ? "true" : "false"));
switch (data_set->no_quorum_policy) {
case no_quorum_freeze:
xmlSetProp(node, (pcmkXmlStr) "no-quorum-policy", (pcmkXmlStr) "freeze");
break;
case no_quorum_stop:
xmlSetProp(node, (pcmkXmlStr) "no-quorum-policy", (pcmkXmlStr) "stop");
break;
+ case no_quorum_demote:
+ xmlSetProp(node, (pcmkXmlStr) "no-quorum-policy", (pcmkXmlStr) "demote");
+ break;
+
case no_quorum_ignore:
xmlSetProp(node, (pcmkXmlStr) "no-quorum-policy", (pcmkXmlStr) "ignore");
break;
case no_quorum_suicide:
xmlSetProp(node, (pcmkXmlStr) "no-quorum-policy", (pcmkXmlStr) "suicide");
break;
}
xmlSetProp(node, (pcmkXmlStr) "maintenance-mode",
(pcmkXmlStr) (is_set(data_set->flags, pe_flag_maintenance_mode) ? "true" : "false"));
return pcmk_rc_ok;
}
PCMK__OUTPUT_ARGS("cluster-stack", "const char *")
int
pe__cluster_stack_html(pcmk__output_t *out, va_list args) {
xmlNodePtr node = pcmk__output_create_xml_node(out, "li");
const char *stack_s = va_arg(args, const char *);
pcmk_create_html_node(node, "span", NULL, "bold", "Stack: ");
pcmk_create_html_node(node, "span", NULL, NULL, stack_s);
return pcmk_rc_ok;
}
PCMK__OUTPUT_ARGS("cluster-stack", "const char *")
int
pe__cluster_stack_text(pcmk__output_t *out, va_list args) {
const char *stack_s = va_arg(args, const char *);
out->list_item(out, "Stack", "%s", stack_s);
return pcmk_rc_ok;
}
PCMK__OUTPUT_ARGS("cluster-stack", "const char *")
int
pe__cluster_stack_xml(pcmk__output_t *out, va_list args) {
xmlNodePtr node = pcmk__output_create_xml_node(out, "stack");
const char *stack_s = va_arg(args, const char *);
xmlSetProp(node, (pcmkXmlStr) "type", (pcmkXmlStr) stack_s);
return pcmk_rc_ok;
}
PCMK__OUTPUT_ARGS("cluster-times", "const char *", "const char *", "const char *", "const char *")
int
pe__cluster_times_html(pcmk__output_t *out, va_list args) {
xmlNodePtr updated_node = pcmk__output_create_xml_node(out, "li");
xmlNodePtr changed_node = pcmk__output_create_xml_node(out, "li");
const char *last_written = va_arg(args, const char *);
const char *user = va_arg(args, const char *);
const char *client = va_arg(args, const char *);
const char *origin = va_arg(args, const char *);
char *buf = last_changed_string(last_written, user, client, origin);
pcmk_create_html_node(updated_node, "span", NULL, "bold", "Last updated: ");
pcmk_create_html_node(updated_node, "span", NULL, NULL,
pcmk__epoch2str(NULL));
pcmk_create_html_node(changed_node, "span", NULL, "bold", "Last change: ");
pcmk_create_html_node(changed_node, "span", NULL, NULL, buf);
free(buf);
return pcmk_rc_ok;
}
PCMK__OUTPUT_ARGS("cluster-times", "const char *", "const char *", "const char *", "const char *")
int
pe__cluster_times_xml(pcmk__output_t *out, va_list args) {
xmlNodePtr updated_node = pcmk__output_create_xml_node(out, "last_update");
xmlNodePtr changed_node = pcmk__output_create_xml_node(out, "last_change");
const char *last_written = va_arg(args, const char *);
const char *user = va_arg(args, const char *);
const char *client = va_arg(args, const char *);
const char *origin = va_arg(args, const char *);
xmlSetProp(updated_node, (pcmkXmlStr) "time",
(pcmkXmlStr) pcmk__epoch2str(NULL));
xmlSetProp(changed_node, (pcmkXmlStr) "time", (pcmkXmlStr) (last_written ? last_written : ""));
xmlSetProp(changed_node, (pcmkXmlStr) "user", (pcmkXmlStr) (user ? user : ""));
xmlSetProp(changed_node, (pcmkXmlStr) "client", (pcmkXmlStr) (client ? client : ""));
xmlSetProp(changed_node, (pcmkXmlStr) "origin", (pcmkXmlStr) (origin ? origin : ""));
return pcmk_rc_ok;
}
PCMK__OUTPUT_ARGS("cluster-times", "const char *", "const char *", "const char *", "const char *")
int
pe__cluster_times_text(pcmk__output_t *out, va_list args) {
const char *last_written = va_arg(args, const char *);
const char *user = va_arg(args, const char *);
const char *client = va_arg(args, const char *);
const char *origin = va_arg(args, const char *);
char *buf = last_changed_string(last_written, user, client, origin);
out->list_item(out, "Last updated", "%s", pcmk__epoch2str(NULL));
out->list_item(out, "Last change", " %s", buf);
free(buf);
return pcmk_rc_ok;
}
PCMK__OUTPUT_ARGS("failed-action", "xmlNodePtr")
int
pe__failed_action_text(pcmk__output_t *out, va_list args) {
xmlNodePtr xml_op = va_arg(args, xmlNodePtr);
char *s = failed_action_string(xml_op);
out->list_item(out, NULL, "%s", s);
free(s);
return pcmk_rc_ok;
}
PCMK__OUTPUT_ARGS("failed-action", "xmlNodePtr")
int
pe__failed_action_xml(pcmk__output_t *out, va_list args) {
xmlNodePtr xml_op = va_arg(args, xmlNodePtr);
const char *op_key = crm_element_value(xml_op, XML_LRM_ATTR_TASK_KEY);
const char *last = crm_element_value(xml_op, XML_RSC_OP_LAST_CHANGE);
int rc = crm_parse_int(crm_element_value(xml_op, XML_LRM_ATTR_RC), "0");
int status = crm_parse_int(crm_element_value(xml_op, XML_LRM_ATTR_OPSTATUS), "0");
const char *exit_reason = crm_element_value(xml_op, XML_LRM_ATTR_EXIT_REASON);
char *rc_s = crm_itoa(rc);
char *reason_s = crm_xml_escape(exit_reason ? exit_reason : "none");
xmlNodePtr node = pcmk__output_create_xml_node(out, "failure");
xmlSetProp(node, (pcmkXmlStr) (op_key ? "op_key" : "id"),
(pcmkXmlStr) (op_key ? op_key : "id"));
xmlSetProp(node, (pcmkXmlStr) "node",
(pcmkXmlStr) crm_element_value(xml_op, XML_ATTR_UNAME));
xmlSetProp(node, (pcmkXmlStr) "exitstatus",
(pcmkXmlStr) services_ocf_exitcode_str(rc));
xmlSetProp(node, (pcmkXmlStr) "exitreason", (pcmkXmlStr) reason_s);
xmlSetProp(node, (pcmkXmlStr) "exitcode", (pcmkXmlStr) rc_s);
xmlSetProp(node, (pcmkXmlStr) "call",
(pcmkXmlStr) crm_element_value(xml_op, XML_LRM_ATTR_CALLID));
xmlSetProp(node, (pcmkXmlStr) "status",
(pcmkXmlStr) services_lrm_status_str(status));
if (last) {
guint interval_ms = 0;
char *s = NULL;
time_t when = crm_parse_int(last, "0");
crm_time_t *crm_when = crm_time_new(NULL);
char *rc_change = NULL;
crm_element_value_ms(xml_op, XML_LRM_ATTR_INTERVAL_MS, &interval_ms);
s = crm_itoa(interval_ms);
crm_time_set_timet(crm_when, &when);
rc_change = crm_time_as_string(crm_when, crm_time_log_date | crm_time_log_timeofday | crm_time_log_with_timezone);
xmlSetProp(node, (pcmkXmlStr) XML_RSC_OP_LAST_CHANGE, (pcmkXmlStr) rc_change);
xmlSetProp(node, (pcmkXmlStr) "queued",
(pcmkXmlStr) crm_element_value(xml_op, XML_RSC_OP_T_QUEUE));
xmlSetProp(node, (pcmkXmlStr) "exec",
(pcmkXmlStr) crm_element_value(xml_op, XML_RSC_OP_T_EXEC));
xmlSetProp(node, (pcmkXmlStr) "interval", (pcmkXmlStr) s);
xmlSetProp(node, (pcmkXmlStr) "task",
(pcmkXmlStr) crm_element_value(xml_op, XML_LRM_ATTR_TASK));
free(s);
free(rc_change);
crm_time_free(crm_when);
}
free(reason_s);
free(rc_s);
return pcmk_rc_ok;
}
PCMK__OUTPUT_ARGS("node", "struct pe_node_t *", "unsigned int", "gboolean", "const char *", "gboolean", "gboolean", "gboolean", "GListPtr")
int
pe__node_html(pcmk__output_t *out, va_list args) {
pe_node_t *node = va_arg(args, pe_node_t *);
unsigned int print_opts = va_arg(args, unsigned int);
gboolean full = va_arg(args, gboolean);
const char *node_mode G_GNUC_UNUSED = va_arg(args, const char *);
gboolean print_clone_detail = va_arg(args, gboolean);
gboolean print_brief = va_arg(args, gboolean);
gboolean group_by_node = va_arg(args, gboolean);
GListPtr only_show = va_arg(args, GListPtr);
char *node_name = pe__node_display_name(node, print_clone_detail);
char *buf = crm_strdup_printf("Node: %s", node_name);
if (full) {
xmlNodePtr item_node = pcmk__output_create_xml_node(out, "li");
pcmk_create_html_node(item_node, "span", NULL, NULL, buf);
if (node->details->standby_onfail && node->details->online) {
pcmk_create_html_node(item_node, "span", NULL, "standby", " standby (on-fail)");
} else if (node->details->standby && node->details->online) {
char *s = crm_strdup_printf(" standby%s", node->details->running_rsc ? " (with active resources)" : "");
pcmk_create_html_node(item_node, "span", NULL, " standby", s);
free(s);
} else if (node->details->standby) {
pcmk_create_html_node(item_node, "span", NULL, "offline", " OFFLINE (standby)");
} else if (node->details->maintenance && node->details->online) {
pcmk_create_html_node(item_node, "span", NULL, "maint", " maintenance");
} else if (node->details->maintenance) {
pcmk_create_html_node(item_node, "span", NULL, "offline", " OFFLINE (maintenance)");
} else if (node->details->online) {
pcmk_create_html_node(item_node, "span", NULL, "online", " online");
} else {
pcmk_create_html_node(item_node, "span", NULL, "offline", " OFFLINE");
}
if (print_brief && group_by_node) {
out->begin_list(out, NULL, NULL, NULL);
pe__rscs_brief_output(out, node->details->running_rsc, print_opts | pe_print_rsconly,
FALSE);
out->end_list(out);
} else if (group_by_node) {
GListPtr lpc2 = NULL;
out->begin_list(out, NULL, NULL, NULL);
for (lpc2 = node->details->running_rsc; lpc2 != NULL; lpc2 = lpc2->next) {
pe_resource_t *rsc = (pe_resource_t *) lpc2->data;
out->message(out, crm_map_element_name(rsc->xml), print_opts | pe_print_rsconly, rsc, only_show);
}
out->end_list(out);
}
} else {
out->begin_list(out, NULL, NULL, "%s", buf);
}
free(buf);
free(node_name);
return pcmk_rc_ok;
}
PCMK__OUTPUT_ARGS("node", "struct pe_node_t *", "unsigned int", "gboolean", "const char *", "gboolean", "gboolean", "gboolean", "GListPtr")
int
pe__node_text(pcmk__output_t *out, va_list args) {
pe_node_t *node = va_arg(args, pe_node_t *);
unsigned int print_opts = va_arg(args, unsigned int);
gboolean full = va_arg(args, gboolean);
const char *node_mode = va_arg(args, const char *);
gboolean print_clone_detail = va_arg(args, gboolean);
gboolean print_brief = va_arg(args, gboolean);
gboolean group_by_node = va_arg(args, gboolean);
GListPtr only_show = va_arg(args, GListPtr);
if (full) {
char *node_name = pe__node_display_name(node, print_clone_detail);
char *buf = NULL;
/* Print the node name and status */
if (pe__is_guest_node(node)) {
buf = crm_strdup_printf("GuestNode %s: %s", node_name, node_mode);
} else if (pe__is_remote_node(node)) {
buf = crm_strdup_printf("RemoteNode %s: %s", node_name, node_mode);
} else {
buf = crm_strdup_printf("Node %s: %s", node_name, node_mode);
}
/* If we're grouping by node, print its resources */
if (group_by_node) {
out->begin_list(out, NULL, NULL, "%s", buf);
out->begin_list(out, NULL, NULL, "Resources");
if (print_brief) {
pe__rscs_brief_output(out, node->details->running_rsc,
print_opts | pe_print_rsconly, FALSE);
} else {
GListPtr gIter2 = NULL;
for (gIter2 = node->details->running_rsc; gIter2 != NULL; gIter2 = gIter2->next) {
pe_resource_t *rsc = (pe_resource_t *) gIter2->data;
out->message(out, crm_map_element_name(rsc->xml), print_opts | pe_print_rsconly, rsc, only_show);
}
}
out->end_list(out);
out->end_list(out);
} else {
out->list_item(out, NULL, "%s", buf);
}
free(buf);
free(node_name);
} else {
out->begin_list(out, NULL, NULL, "Node: %s", pe__node_display_name(node, print_clone_detail));
}
return pcmk_rc_ok;
}
PCMK__OUTPUT_ARGS("node", "struct pe_node_t *", "unsigned int", "gboolean", "const char *", "gboolean", "gboolean", "gboolean", "GListPtr")
int
pe__node_xml(pcmk__output_t *out, va_list args) {
pe_node_t *node = va_arg(args, pe_node_t *);
unsigned int print_opts = va_arg(args, unsigned int);
gboolean full = va_arg(args, gboolean);
const char *node_mode G_GNUC_UNUSED = va_arg(args, const char *);
gboolean print_clone_detail G_GNUC_UNUSED = va_arg(args, gboolean);
gboolean print_brief G_GNUC_UNUSED = va_arg(args, gboolean);
gboolean group_by_node = va_arg(args, gboolean);
GListPtr only_show = va_arg(args, GListPtr);
if (full) {
const char *node_type = "unknown";
char *length_s = crm_itoa(g_list_length(node->details->running_rsc));
switch (node->details->type) {
case node_member:
node_type = "member";
break;
case node_remote:
node_type = "remote";
break;
case node_ping:
node_type = "ping";
break;
}
pe__name_and_nvpairs_xml(out, true, "node", 13,
"name", node->details->uname,
"id", node->details->id,
"online", node->details->online ? "true" : "false",
"standby", node->details->standby ? "true" : "false",
"standby_onfail", node->details->standby_onfail ? "true" : "false",
"maintenance", node->details->maintenance ? "true" : "false",
"pending", node->details->pending ? "true" : "false",
"unclean", node->details->unclean ? "true" : "false",
"shutdown", node->details->shutdown ? "true" : "false",
"expected_up", node->details->expected_up ? "true" : "false",
"is_dc", node->details->is_dc ? "true" : "false",
"resources_running", length_s,
"type", node_type);
if (pe__is_guest_node(node)) {
xmlNodePtr xml_node = pcmk__output_xml_peek_parent(out);
xmlSetProp(xml_node, (pcmkXmlStr) "id_as_resource",
(pcmkXmlStr) node->details->remote_rsc->container->id);
}
if (group_by_node) {
GListPtr lpc = NULL;
for (lpc = node->details->running_rsc; lpc != NULL; lpc = lpc->next) {
pe_resource_t *rsc = (pe_resource_t *) lpc->data;
out->message(out, crm_map_element_name(rsc->xml), print_opts | pe_print_rsconly, rsc, only_show);
}
}
free(length_s);
out->end_list(out);
} else {
xmlNodePtr parent = pcmk__output_xml_create_parent(out, "node");
xmlSetProp(parent, (pcmkXmlStr) "name", (pcmkXmlStr) node->details->uname);
}
return pcmk_rc_ok;
}
PCMK__OUTPUT_ARGS("node-attribute", "const char *", "const char *", "gboolean", "int")
int
pe__node_attribute_text(pcmk__output_t *out, va_list args) {
const char *name = va_arg(args, const char *);
const char *value = va_arg(args, const char *);
gboolean add_extra = va_arg(args, gboolean);
int expected_score = va_arg(args, int);
if (add_extra) {
int v = crm_parse_int(value, "0");
if (v <= 0) {
out->list_item(out, NULL, "%-32s\t: %-10s\t: Connectivity is lost", name, value);
} else if (v < expected_score) {
out->list_item(out, NULL, "%-32s\t: %-10s\t: Connectivity is degraded (Expected=%d)", name, value, expected_score);
} else {
out->list_item(out, NULL, "%-32s\t: %-10s", name, value);
}
} else {
out->list_item(out, NULL, "%-32s\t: %-10s", name, value);
}
return pcmk_rc_ok;
}
PCMK__OUTPUT_ARGS("node-attribute", "const char *", "const char *", "gboolean", "int")
int
pe__node_attribute_html(pcmk__output_t *out, va_list args) {
const char *name = va_arg(args, const char *);
const char *value = va_arg(args, const char *);
gboolean add_extra = va_arg(args, gboolean);
int expected_score = va_arg(args, int);
if (add_extra) {
int v = crm_parse_int(value, "0");
char *s = crm_strdup_printf("%s: %s", name, value);
xmlNodePtr item_node = pcmk__output_create_xml_node(out, "li");
pcmk_create_html_node(item_node, "span", NULL, NULL, s);
free(s);
if (v <= 0) {
pcmk_create_html_node(item_node, "span", NULL, "bold", "(connectivity is lost)");
} else if (v < expected_score) {
char *buf = crm_strdup_printf("(connectivity is degraded -- expected %d", expected_score);
pcmk_create_html_node(item_node, "span", NULL, "bold", buf);
free(buf);
}
} else {
out->list_item(out, NULL, "%s: %s", name, value);
}
return pcmk_rc_ok;
}
PCMK__OUTPUT_ARGS("node-attribute", "const char *", "const char *", "gboolean", "int")
int
pe__node_attribute_xml(pcmk__output_t *out, va_list args) {
const char *name = va_arg(args, const char *);
const char *value = va_arg(args, const char *);
gboolean add_extra = va_arg(args, gboolean);
int expected_score = va_arg(args, int);
xmlNodePtr node = pcmk__output_create_xml_node(out, "attribute");
xmlSetProp(node, (pcmkXmlStr) "name", (pcmkXmlStr) name);
xmlSetProp(node, (pcmkXmlStr) "value", (pcmkXmlStr) value);
if (add_extra) {
char *buf = crm_itoa(expected_score);
xmlSetProp(node, (pcmkXmlStr) "expected", (pcmkXmlStr) buf);
free(buf);
}
return pcmk_rc_ok;
}
PCMK__OUTPUT_ARGS("node-list", "GListPtr", "GListPtr", "unsigned int", "gboolean", "gboolean", "gboolean")
int
pe__node_list_html(pcmk__output_t *out, va_list args) {
GListPtr nodes = va_arg(args, GListPtr);
GListPtr only_show = va_arg(args, GListPtr);
unsigned int print_opts = va_arg(args, unsigned int);
gboolean print_clone_detail = va_arg(args, gboolean);
gboolean print_brief = va_arg(args, gboolean);
gboolean group_by_node = va_arg(args, gboolean);
gboolean printed_header = FALSE;
for (GListPtr gIter = nodes; gIter != NULL; gIter = gIter->next) {
pe_node_t *node = (pe_node_t *) gIter->data;
if (!pcmk__str_in_list(only_show, node->details->uname)) {
continue;
}
if (printed_header == FALSE) {
printed_header = TRUE;
out->begin_list(out, NULL, NULL, "Node List");
}
out->message(out, "node", node, print_opts, TRUE, NULL, print_clone_detail,
print_brief, group_by_node, only_show);
}
if (printed_header == TRUE) {
out->end_list(out);
}
return pcmk_rc_ok;
}
PCMK__OUTPUT_ARGS("node-list", "GListPtr", "GListPtr", "unsigned int", "gboolean", "gboolean", "gboolean")
int
pe__node_list_text(pcmk__output_t *out, va_list args) {
GListPtr nodes = va_arg(args, GListPtr);
GListPtr only_show = va_arg(args, GListPtr);
unsigned int print_opts = va_arg(args, unsigned int);
gboolean print_clone_detail = va_arg(args, gboolean);
gboolean print_brief = va_arg(args, gboolean);
gboolean group_by_node = va_arg(args, gboolean);
/* space-separated lists of node names */
char *online_nodes = NULL;
char *online_remote_nodes = NULL;
char *online_guest_nodes = NULL;
char *offline_nodes = NULL;
char *offline_remote_nodes = NULL;
int rc = pcmk_rc_no_output;
gboolean printed_header = FALSE;
for (GListPtr gIter = nodes; gIter != NULL; gIter = gIter->next) {
pe_node_t *node = (pe_node_t *) gIter->data;
const char *node_mode = NULL;
char *node_name = pe__node_display_name(node, print_clone_detail);
if (!pcmk__str_in_list(only_show, node->details->uname)) {
free(node_name);
continue;
}
if (printed_header == FALSE) {
out->begin_list(out, NULL, NULL, "Node List");
rc = pcmk_rc_ok;
printed_header = TRUE;
}
/* Get node mode */
if (node->details->unclean) {
if (node->details->online) {
node_mode = "UNCLEAN (online)";
} else if (node->details->pending) {
node_mode = "UNCLEAN (pending)";
} else {
node_mode = "UNCLEAN (offline)";
}
} else if (node->details->pending) {
node_mode = "pending";
} else if (node->details->standby_onfail && node->details->online) {
node_mode = "standby (on-fail)";
} else if (node->details->standby) {
if (node->details->online) {
if (node->details->running_rsc) {
node_mode = "standby (with active resources)";
} else {
node_mode = "standby";
}
} else {
node_mode = "OFFLINE (standby)";
}
} else if (node->details->maintenance) {
if (node->details->online) {
node_mode = "maintenance";
} else {
node_mode = "OFFLINE (maintenance)";
}
} else if (node->details->online) {
node_mode = "online";
if (group_by_node == FALSE) {
if (pe__is_guest_node(node)) {
online_guest_nodes = pcmk__add_word(online_guest_nodes,
node_name);
} else if (pe__is_remote_node(node)) {
online_remote_nodes = pcmk__add_word(online_remote_nodes,
node_name);
} else {
online_nodes = pcmk__add_word(online_nodes, node_name);
}
free(node_name);
continue;
}
} else {
node_mode = "OFFLINE";
if (group_by_node == FALSE) {
if (pe__is_remote_node(node)) {
offline_remote_nodes = pcmk__add_word(offline_remote_nodes,
node_name);
} else if (pe__is_guest_node(node)) {
/* ignore offline guest nodes */
} else {
offline_nodes = pcmk__add_word(offline_nodes, node_name);
}
free(node_name);
continue;
}
}
/* If we get here, node is in bad state, or we're grouping by node */
out->message(out, "node", node, print_opts, TRUE, node_mode, print_clone_detail,
print_brief, group_by_node, only_show);
free(node_name);
}
/* If we're not grouping by node, summarize nodes by status */
if (online_nodes) {
out->list_item(out, "Online", "[%s ]", online_nodes);
free(online_nodes);
}
if (offline_nodes) {
out->list_item(out, "OFFLINE", "[%s ]", offline_nodes);
free(offline_nodes);
}
if (online_remote_nodes) {
out->list_item(out, "RemoteOnline", "[%s ]", online_remote_nodes);
free(online_remote_nodes);
}
if (offline_remote_nodes) {
out->list_item(out, "RemoteOFFLINE", "[%s ]", offline_remote_nodes);
free(offline_remote_nodes);
}
if (online_guest_nodes) {
out->list_item(out, "GuestOnline", "[%s ]", online_guest_nodes);
free(online_guest_nodes);
}
if (printed_header == TRUE) {
out->end_list(out);
}
return rc;
}
PCMK__OUTPUT_ARGS("node-list", "GListPtr", "GListPtr", "unsigned int", "gboolean", "gboolean", "gboolean")
int
pe__node_list_xml(pcmk__output_t *out, va_list args) {
GListPtr nodes = va_arg(args, GListPtr);
GListPtr only_show = va_arg(args, GListPtr);
unsigned int print_opts = va_arg(args, unsigned int);
gboolean print_clone_detail = va_arg(args, gboolean);
gboolean print_brief = va_arg(args, gboolean);
gboolean group_by_node = va_arg(args, gboolean);
out->begin_list(out, NULL, NULL, "nodes");
for (GListPtr gIter = nodes; gIter != NULL; gIter = gIter->next) {
pe_node_t *node = (pe_node_t *) gIter->data;
if (!pcmk__str_in_list(only_show, node->details->uname)) {
continue;
}
out->message(out, "node", node, print_opts, TRUE, NULL, print_clone_detail,
print_brief, group_by_node, only_show);
}
out->end_list(out);
return pcmk_rc_ok;
}
PCMK__OUTPUT_ARGS("op-history", "struct xmlNode *", "const char *", "const char *", "int", "gboolean")
int
pe__op_history_text(pcmk__output_t *out, va_list args) {
xmlNode *xml_op = va_arg(args, xmlNode *);
const char *task = va_arg(args, const char *);
const char *interval_ms_s = va_arg(args, const char *);
int rc = va_arg(args, int);
gboolean print_timing = va_arg(args, gboolean);
char *buf = op_history_string(xml_op, task, interval_ms_s, rc, print_timing);
out->list_item(out, NULL, "%s", buf);
free(buf);
return pcmk_rc_ok;
}
PCMK__OUTPUT_ARGS("op-history", "struct xmlNode *", "const char *", "const char *", "int", "gboolean")
int
pe__op_history_xml(pcmk__output_t *out, va_list args) {
xmlNode *xml_op = va_arg(args, xmlNode *);
const char *task = va_arg(args, const char *);
const char *interval_ms_s = va_arg(args, const char *);
int rc = va_arg(args, int);
gboolean print_timing = va_arg(args, gboolean);
char *rc_s = NULL;
xmlNodePtr node = pcmk__output_create_xml_node(out, "operation_history");
xmlSetProp(node, (pcmkXmlStr) "call",
(pcmkXmlStr) crm_element_value(xml_op, XML_LRM_ATTR_CALLID));
xmlSetProp(node, (pcmkXmlStr) "task", (pcmkXmlStr) task);
if (interval_ms_s && safe_str_neq(interval_ms_s, "0")) {
char *s = crm_strdup_printf("%sms", interval_ms_s);
xmlSetProp(node, (pcmkXmlStr) "interval", (pcmkXmlStr) s);
free(s);
}
if (print_timing) {
const char *value = NULL;
value = crm_element_value(xml_op, XML_RSC_OP_LAST_CHANGE);
if (value) {
time_t int_value = (time_t) crm_parse_int(value, NULL);
if (int_value > 0) {
xmlSetProp(node, (pcmkXmlStr) XML_RSC_OP_LAST_CHANGE,
(pcmkXmlStr) pcmk__epoch2str(&int_value));
}
}
value = crm_element_value(xml_op, XML_RSC_OP_LAST_RUN);
if (value) {
time_t int_value = (time_t) crm_parse_int(value, NULL);
if (int_value > 0) {
xmlSetProp(node, (pcmkXmlStr) XML_RSC_OP_LAST_RUN,
(pcmkXmlStr) pcmk__epoch2str(&int_value));
}
}
value = crm_element_value(xml_op, XML_RSC_OP_T_EXEC);
if (value) {
char *s = crm_strdup_printf("%sms", value);
xmlSetProp(node, (pcmkXmlStr) XML_RSC_OP_T_EXEC, (pcmkXmlStr) s);
free(s);
}
value = crm_element_value(xml_op, XML_RSC_OP_T_QUEUE);
if (value) {
char *s = crm_strdup_printf("%sms", value);
xmlSetProp(node, (pcmkXmlStr) XML_RSC_OP_T_QUEUE, (pcmkXmlStr) s);
free(s);
}
}
rc_s = crm_itoa(rc);
xmlSetProp(node, (pcmkXmlStr) "rc", (pcmkXmlStr) rc_s);
xmlSetProp(node, (pcmkXmlStr) "rc_text", (pcmkXmlStr) services_ocf_exitcode_str(rc));
free(rc_s);
return pcmk_rc_ok;
}
PCMK__OUTPUT_ARGS("resource-history", "struct pe_resource_t *", "const char *", "gboolean", "int", "time_t", "gboolean")
int
pe__resource_history_text(pcmk__output_t *out, va_list args) {
pe_resource_t *rsc = va_arg(args, pe_resource_t *);
const char *rsc_id = va_arg(args, const char *);
gboolean all = va_arg(args, gboolean);
int failcount = va_arg(args, int);
time_t last_failure = va_arg(args, int);
gboolean as_header = va_arg(args, gboolean);
char *buf = resource_history_string(rsc, rsc_id, all, failcount, last_failure);
if (as_header) {
out->begin_list(out, NULL, NULL, "%s", buf);
} else {
out->list_item(out, NULL, "%s", buf);
}
free(buf);
return pcmk_rc_ok;
}
PCMK__OUTPUT_ARGS("resource-history", "struct pe_resource_t *", "const char *", "gboolean", "int", "time_t", "gboolean")
int
pe__resource_history_xml(pcmk__output_t *out, va_list args) {
pe_resource_t *rsc = va_arg(args, pe_resource_t *);
const char *rsc_id = va_arg(args, const char *);
gboolean all = va_arg(args, gboolean);
int failcount = va_arg(args, int);
time_t last_failure = va_arg(args, int);
gboolean as_header = va_arg(args, gboolean);
xmlNodePtr node = pcmk__output_xml_create_parent(out, "resource_history");
xmlSetProp(node, (pcmkXmlStr) "id", (pcmkXmlStr) rsc_id);
if (rsc == NULL) {
xmlSetProp(node, (pcmkXmlStr) "orphan", (pcmkXmlStr) "true");
} else if (all || failcount || last_failure > 0) {
char *migration_s = crm_itoa(rsc->migration_threshold);
xmlSetProp(node, (pcmkXmlStr) "orphan", (pcmkXmlStr) "false");
xmlSetProp(node, (pcmkXmlStr) "migration-threshold",
(pcmkXmlStr) migration_s);
free(migration_s);
if (failcount > 0) {
char *s = crm_itoa(failcount);
xmlSetProp(node, (pcmkXmlStr) PCMK__FAIL_COUNT_PREFIX,
(pcmkXmlStr) s);
free(s);
}
if (last_failure > 0) {
xmlSetProp(node, (pcmkXmlStr) PCMK__LAST_FAILURE_PREFIX,
(pcmkXmlStr) pcmk__epoch2str(&last_failure));
}
}
if (as_header == FALSE) {
pcmk__output_xml_pop_parent(out);
}
return pcmk_rc_ok;
}
PCMK__OUTPUT_ARGS("ticket", "struct pe_ticket_t *")
int
pe__ticket_html(pcmk__output_t *out, va_list args) {
pe_ticket_t *ticket = va_arg(args, pe_ticket_t *);
if (ticket->last_granted > -1) {
char *time = pcmk_format_named_time("last-granted", ticket->last_granted);
out->list_item(out, NULL, "%s:\t%s%s %s", ticket->id,
ticket->granted ? "granted" : "revoked",
ticket->standby ? " [standby]" : "",
time);
free(time);
} else {
out->list_item(out, NULL, "%s:\t%s%s", ticket->id,
ticket->granted ? "granted" : "revoked",
ticket->standby ? " [standby]" : "");
}
return pcmk_rc_ok;
}
PCMK__OUTPUT_ARGS("ticket", "struct pe_ticket_t *")
int
pe__ticket_text(pcmk__output_t *out, va_list args) {
pe_ticket_t *ticket = va_arg(args, pe_ticket_t *);
if (ticket->last_granted > -1) {
char *time = pcmk_format_named_time("last-granted", ticket->last_granted);
out->list_item(out, ticket->id, "\t%s%s %s",
ticket->granted ? "granted" : "revoked",
ticket->standby ? " [standby]" : "",
time);
free(time);
} else {
out->list_item(out, ticket->id, "\t%s%s",
ticket->granted ? "granted" : "revoked",
ticket->standby ? " [standby]" : "");
}
return pcmk_rc_ok;
}
PCMK__OUTPUT_ARGS("ticket", "struct pe_ticket_t *")
int
pe__ticket_xml(pcmk__output_t *out, va_list args) {
xmlNodePtr node = NULL;
pe_ticket_t *ticket = va_arg(args, pe_ticket_t *);
node = pcmk__output_create_xml_node(out, "ticket");
xmlSetProp(node, (pcmkXmlStr) "id", (pcmkXmlStr) ticket->id);
xmlSetProp(node, (pcmkXmlStr) "status", (pcmkXmlStr) (ticket->granted ? "granted" : "revoked"));
xmlSetProp(node, (pcmkXmlStr) "standby", (pcmkXmlStr) (ticket->standby ? "true" : "false"));
if (ticket->last_granted > -1) {
xmlSetProp(node, (pcmkXmlStr) "last-granted",
(pcmkXmlStr) pcmk__epoch2str(&ticket->last_granted));
}
return pcmk_rc_ok;
}
static pcmk__message_entry_t fmt_functions[] = {
{ "ban", "html", pe__ban_html },
{ "ban", "log", pe__ban_text },
{ "ban", "text", pe__ban_text },
{ "ban", "xml", pe__ban_xml },
{ "bundle", "xml", pe__bundle_xml },
{ "bundle", "html", pe__bundle_html },
{ "bundle", "text", pe__bundle_text },
{ "bundle", "log", pe__bundle_text },
{ "clone", "xml", pe__clone_xml },
{ "clone", "html", pe__clone_html },
{ "clone", "text", pe__clone_text },
{ "clone", "log", pe__clone_text },
{ "cluster-counts", "html", pe__cluster_counts_html },
{ "cluster-counts", "log", pe__cluster_counts_text },
{ "cluster-counts", "text", pe__cluster_counts_text },
{ "cluster-counts", "xml", pe__cluster_counts_xml },
{ "cluster-dc", "html", pe__cluster_dc_html },
{ "cluster-dc", "log", pe__cluster_dc_text },
{ "cluster-dc", "text", pe__cluster_dc_text },
{ "cluster-dc", "xml", pe__cluster_dc_xml },
{ "cluster-options", "html", pe__cluster_options_html },
{ "cluster-options", "log", pe__cluster_options_log },
{ "cluster-options", "text", pe__cluster_options_text },
{ "cluster-options", "xml", pe__cluster_options_xml },
{ "cluster-summary", "html", pe__cluster_summary_html },
{ "cluster-summary", "log", pe__cluster_summary },
{ "cluster-summary", "text", pe__cluster_summary },
{ "cluster-summary", "xml", pe__cluster_summary },
{ "cluster-stack", "html", pe__cluster_stack_html },
{ "cluster-stack", "log", pe__cluster_stack_text },
{ "cluster-stack", "text", pe__cluster_stack_text },
{ "cluster-stack", "xml", pe__cluster_stack_xml },
{ "cluster-times", "html", pe__cluster_times_html },
{ "cluster-times", "log", pe__cluster_times_text },
{ "cluster-times", "text", pe__cluster_times_text },
{ "cluster-times", "xml", pe__cluster_times_xml },
{ "failed-action", "html", pe__failed_action_text },
{ "failed-action", "log", pe__failed_action_text },
{ "failed-action", "text", pe__failed_action_text },
{ "failed-action", "xml", pe__failed_action_xml },
{ "group", "xml", pe__group_xml },
{ "group", "html", pe__group_html },
{ "group", "text", pe__group_text },
{ "group", "log", pe__group_text },
/* maint-mode only exists for text and log. Other formatters output it as
* part of the cluster-options handler.
*/
{ "maint-mode", "log", pe__cluster_maint_mode_text },
{ "maint-mode", "text", pe__cluster_maint_mode_text },
{ "node", "html", pe__node_html },
{ "node", "log", pe__node_text },
{ "node", "text", pe__node_text },
{ "node", "xml", pe__node_xml },
{ "node-list", "html", pe__node_list_html },
{ "node-list", "log", pe__node_list_text },
{ "node-list", "text", pe__node_list_text },
{ "node-list", "xml", pe__node_list_xml },
{ "node-attribute", "html", pe__node_attribute_html },
{ "node-attribute", "log", pe__node_attribute_text },
{ "node-attribute", "text", pe__node_attribute_text },
{ "node-attribute", "xml", pe__node_attribute_xml },
{ "op-history", "html", pe__op_history_text },
{ "op-history", "log", pe__op_history_text },
{ "op-history", "text", pe__op_history_text },
{ "op-history", "xml", pe__op_history_xml },
{ "primitive", "xml", pe__resource_xml },
{ "primitive", "html", pe__resource_html },
{ "primitive", "text", pe__resource_text },
{ "primitive", "log", pe__resource_text },
{ "resource-history", "html", pe__resource_history_text },
{ "resource-history", "log", pe__resource_history_text },
{ "resource-history", "text", pe__resource_history_text },
{ "resource-history", "xml", pe__resource_history_xml },
{ "ticket", "html", pe__ticket_html },
{ "ticket", "log", pe__ticket_text },
{ "ticket", "text", pe__ticket_text },
{ "ticket", "xml", pe__ticket_xml },
{ NULL, NULL, NULL }
};
void
pe__register_messages(pcmk__output_t *out) {
pcmk__register_messages(out, fmt_functions);
}
void
pe__output_node(pe_node_t *node, gboolean details, pcmk__output_t *out)
{
if (node == NULL) {
crm_trace("");
return;
}
CRM_ASSERT(node->details);
crm_trace("%sNode %s: (weight=%d, fixed=%s)",
node->details->online ? "" : "Unavailable/Unclean ",
node->details->uname, node->weight, node->fixed ? "True" : "False");
if (details) {
char *pe_mutable = strdup("\t\t");
GListPtr gIter = node->details->running_rsc;
GListPtr unames = NULL;
unames = g_list_prepend(unames, strdup("*"));
crm_trace("\t\t===Node Attributes");
g_hash_table_foreach(node->details->attrs, print_str_str, pe_mutable);
free(pe_mutable);
crm_trace("\t\t=== Resources");
for (; gIter != NULL; gIter = gIter->next) {
pe_resource_t *rsc = (pe_resource_t *) gIter->data;
out->message(out, crm_map_element_name(rsc->xml),
pe_print_pending, rsc, unames);
}
g_list_free_full(unames, free);
}
}
diff --git a/lib/pengine/status.c b/lib/pengine/status.c
index 8dc5095c15..ca34639014 100644
--- a/lib/pengine/status.c
+++ b/lib/pengine/status.c
@@ -1,440 +1,440 @@
/*
* Copyright 2004-2020 the Pacemaker project contributors
*
* The version control history for this file may have further details.
*
* This source code is licensed under the GNU Lesser General Public License
* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
*/
#include
#include
#include
#include
#include
#include
#include
#include
/*!
* \brief Create a new working set
*
* \return New, initialized working set on success, else NULL (and set errno)
* \note Only pe_working_set_t objects created with this function (as opposed
* to statically declared or directly allocated) should be used with the
* functions in this library, to allow for future extensions to the
* data type. The caller is responsible for freeing the memory with
* pe_free_working_set() when the instance is no longer needed.
*/
pe_working_set_t *
pe_new_working_set()
{
pe_working_set_t *data_set = calloc(1, sizeof(pe_working_set_t));
if (data_set != NULL) {
set_working_set_defaults(data_set);
}
return data_set;
}
/*!
* \brief Free a working set
*
* \param[in] data_set Working set to free
*/
void
pe_free_working_set(pe_working_set_t *data_set)
{
if (data_set != NULL) {
pe_reset_working_set(data_set);
free(data_set);
}
}
/*
* Unpack everything
* At the end you'll have:
* - A list of nodes
* - A list of resources (each with any dependencies on other resources)
* - A list of constraints between resources and nodes
* - A list of constraints between start/stop actions
* - A list of nodes that need to be stonith'd
* - A list of nodes that need to be shutdown
* - A list of the possible stop/start actions (without dependencies)
*/
gboolean
cluster_status(pe_working_set_t * data_set)
{
xmlNode *config = get_xpath_object("//"XML_CIB_TAG_CRMCONFIG, data_set->input, LOG_TRACE);
xmlNode *cib_nodes = get_xpath_object("//"XML_CIB_TAG_NODES, data_set->input, LOG_TRACE);
xmlNode *cib_resources = get_xpath_object("//"XML_CIB_TAG_RESOURCES, data_set->input, LOG_TRACE);
xmlNode *cib_status = get_xpath_object("//"XML_CIB_TAG_STATUS, data_set->input, LOG_TRACE);
xmlNode *cib_tags = get_xpath_object("//" XML_CIB_TAG_TAGS, data_set->input,
LOG_NEVER);
const char *value = crm_element_value(data_set->input, XML_ATTR_HAVE_QUORUM);
crm_trace("Beginning unpack");
/* reset remaining global variables */
data_set->failed = create_xml_node(NULL, "failed-ops");
if (data_set->input == NULL) {
return FALSE;
}
if (data_set->now == NULL) {
data_set->now = crm_time_new(NULL);
}
if (data_set->dc_uuid == NULL) {
data_set->dc_uuid = crm_element_value_copy(data_set->input,
XML_ATTR_DC_UUID);
}
clear_bit(data_set->flags, pe_flag_have_quorum);
if (crm_is_true(value)) {
set_bit(data_set->flags, pe_flag_have_quorum);
}
data_set->op_defaults = get_xpath_object("//" XML_CIB_TAG_OPCONFIG,
data_set->input, LOG_NEVER);
data_set->rsc_defaults = get_xpath_object("//" XML_CIB_TAG_RSCCONFIG,
data_set->input, LOG_NEVER);
unpack_config(config, data_set);
if (is_not_set(data_set->flags, pe_flag_quick_location)
&& is_not_set(data_set->flags, pe_flag_have_quorum)
&& data_set->no_quorum_policy != no_quorum_ignore) {
crm_warn("Fencing and resource management disabled due to lack of quorum");
}
unpack_nodes(cib_nodes, data_set);
if(is_not_set(data_set->flags, pe_flag_quick_location)) {
unpack_remote_nodes(cib_resources, data_set);
}
unpack_resources(cib_resources, data_set);
unpack_tags(cib_tags, data_set);
if(is_not_set(data_set->flags, pe_flag_quick_location)) {
unpack_status(cib_status, data_set);
}
if (is_not_set(data_set->flags, pe_flag_no_counts)) {
for (GList *item = data_set->resources; item != NULL;
item = item->next) {
((pe_resource_t *) (item->data))->fns->count(item->data);
}
}
set_bit(data_set->flags, pe_flag_have_status);
return TRUE;
}
/*!
* \internal
* \brief Free a list of pe_resource_t
*
* \param[in] resources List to free
*
* \note When a working set's resource list is freed, that includes the original
* storage for the uname and id of any Pacemaker Remote nodes in the
* working set's node list, so take care not to use those afterward.
* \todo Refactor pe_node_t to strdup() the node name.
*/
static void
pe_free_resources(GListPtr resources)
{
pe_resource_t *rsc = NULL;
GListPtr iterator = resources;
while (iterator != NULL) {
rsc = (pe_resource_t *) iterator->data;
iterator = iterator->next;
rsc->fns->free(rsc);
}
if (resources != NULL) {
g_list_free(resources);
}
}
static void
pe_free_actions(GListPtr actions)
{
GListPtr iterator = actions;
while (iterator != NULL) {
pe_free_action(iterator->data);
iterator = iterator->next;
}
if (actions != NULL) {
g_list_free(actions);
}
}
static void
pe_free_nodes(GListPtr nodes)
{
for (GList *iterator = nodes; iterator != NULL; iterator = iterator->next) {
pe_node_t *node = (pe_node_t *) iterator->data;
// Shouldn't be possible, but to be safe ...
if (node == NULL) {
continue;
}
if (node->details == NULL) {
free(node);
continue;
}
/* This is called after pe_free_resources(), which means that we can't
* use node->details->uname for Pacemaker Remote nodes.
*/
crm_trace("Freeing node %s", (pe__is_guest_or_remote_node(node)?
"(guest or remote)" : node->details->uname));
if (node->details->attrs != NULL) {
g_hash_table_destroy(node->details->attrs);
}
if (node->details->utilization != NULL) {
g_hash_table_destroy(node->details->utilization);
}
if (node->details->digest_cache != NULL) {
g_hash_table_destroy(node->details->digest_cache);
}
g_list_free(node->details->running_rsc);
g_list_free(node->details->allocated_rsc);
free(node->details);
free(node);
}
if (nodes != NULL) {
g_list_free(nodes);
}
}
static void
pe__free_ordering(GListPtr constraints)
{
GListPtr iterator = constraints;
while (iterator != NULL) {
pe__ordering_t *order = iterator->data;
iterator = iterator->next;
free(order->lh_action_task);
free(order->rh_action_task);
free(order);
}
if (constraints != NULL) {
g_list_free(constraints);
}
}
static void
pe__free_location(GListPtr constraints)
{
GListPtr iterator = constraints;
while (iterator != NULL) {
pe__location_t *cons = iterator->data;
iterator = iterator->next;
g_list_free_full(cons->node_list_rh, free);
free(cons->id);
free(cons);
}
if (constraints != NULL) {
g_list_free(constraints);
}
}
/*!
* \brief Reset working set to default state without freeing it or constraints
*
* \param[in,out] data_set Working set to reset
*
* \deprecated This function is deprecated as part of the API;
* pe_reset_working_set() should be used instead.
*/
void
cleanup_calculations(pe_working_set_t * data_set)
{
if (data_set == NULL) {
return;
}
clear_bit(data_set->flags, pe_flag_have_status);
if (data_set->config_hash != NULL) {
g_hash_table_destroy(data_set->config_hash);
}
if (data_set->singletons != NULL) {
g_hash_table_destroy(data_set->singletons);
}
if (data_set->tickets) {
g_hash_table_destroy(data_set->tickets);
}
if (data_set->template_rsc_sets) {
g_hash_table_destroy(data_set->template_rsc_sets);
}
if (data_set->tags) {
g_hash_table_destroy(data_set->tags);
}
free(data_set->dc_uuid);
crm_trace("deleting resources");
pe_free_resources(data_set->resources);
crm_trace("deleting actions");
pe_free_actions(data_set->actions);
crm_trace("deleting nodes");
pe_free_nodes(data_set->nodes);
pe__free_param_checks(data_set);
g_list_free(data_set->stop_needed);
free_xml(data_set->graph);
crm_time_free(data_set->now);
free_xml(data_set->input);
free_xml(data_set->failed);
set_working_set_defaults(data_set);
CRM_CHECK(data_set->ordering_constraints == NULL,;
);
CRM_CHECK(data_set->placement_constraints == NULL,;
);
}
/*!
* \brief Reset a working set to default state without freeing it
*
* \param[in,out] data_set Working set to reset
*/
void
pe_reset_working_set(pe_working_set_t *data_set)
{
if (data_set == NULL) {
return;
}
crm_trace("Deleting %d ordering constraints",
g_list_length(data_set->ordering_constraints));
pe__free_ordering(data_set->ordering_constraints);
data_set->ordering_constraints = NULL;
crm_trace("Deleting %d location constraints",
g_list_length(data_set->placement_constraints));
pe__free_location(data_set->placement_constraints);
data_set->placement_constraints = NULL;
crm_trace("Deleting %d colocation constraints",
g_list_length(data_set->colocation_constraints));
g_list_free_full(data_set->colocation_constraints, free);
data_set->colocation_constraints = NULL;
crm_trace("Deleting %d ticket constraints",
g_list_length(data_set->ticket_constraints));
g_list_free_full(data_set->ticket_constraints, free);
data_set->ticket_constraints = NULL;
cleanup_calculations(data_set);
}
void
set_working_set_defaults(pe_working_set_t * data_set)
{
memset(data_set, 0, sizeof(pe_working_set_t));
data_set->order_id = 1;
data_set->action_id = 1;
- data_set->no_quorum_policy = no_quorum_freeze;
+ data_set->no_quorum_policy = no_quorum_stop;
data_set->flags = 0x0ULL;
set_bit(data_set->flags, pe_flag_stop_rsc_orphans);
set_bit(data_set->flags, pe_flag_symmetric_cluster);
set_bit(data_set->flags, pe_flag_stop_action_orphans);
#ifdef DEFAULT_CONCURRENT_FENCING_TRUE
set_bit(data_set->flags, pe_flag_concurrent_fencing);
#endif
}
pe_resource_t *
pe_find_resource(GListPtr rsc_list, const char *id)
{
return pe_find_resource_with_flags(rsc_list, id, pe_find_renamed);
}
pe_resource_t *
pe_find_resource_with_flags(GListPtr rsc_list, const char *id, enum pe_find flags)
{
GListPtr rIter = NULL;
for (rIter = rsc_list; id && rIter; rIter = rIter->next) {
pe_resource_t *parent = rIter->data;
pe_resource_t *match =
parent->fns->find_rsc(parent, id, NULL, flags);
if (match != NULL) {
return match;
}
}
crm_trace("No match for %s", id);
return NULL;
}
pe_node_t *
pe_find_node_any(GListPtr nodes, const char *id, const char *uname)
{
pe_node_t *match = pe_find_node_id(nodes, id);
if (match) {
return match;
}
crm_trace("Looking up %s via its uname instead", uname);
return pe_find_node(nodes, uname);
}
pe_node_t *
pe_find_node_id(GListPtr nodes, const char *id)
{
GListPtr gIter = nodes;
for (; gIter != NULL; gIter = gIter->next) {
pe_node_t *node = (pe_node_t *) gIter->data;
if (node && safe_str_eq(node->details->id, id)) {
return node;
}
}
/* error */
return NULL;
}
pe_node_t *
pe_find_node(GListPtr nodes, const char *uname)
{
GListPtr gIter = nodes;
for (; gIter != NULL; gIter = gIter->next) {
pe_node_t *node = (pe_node_t *) gIter->data;
if (node && safe_str_eq(node->details->uname, uname)) {
return node;
}
}
/* error */
return NULL;
}
diff --git a/lib/pengine/unpack.c b/lib/pengine/unpack.c
index 3c6606b92d..a480680fb7 100644
--- a/lib/pengine/unpack.c
+++ b/lib/pengine/unpack.c
@@ -1,3896 +1,4017 @@
/*
* Copyright 2004-2020 the Pacemaker project contributors
*
* The version control history for this file may have further details.
*
* This source code is licensed under the GNU Lesser General Public License
* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
*/
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
CRM_TRACE_INIT_DATA(pe_status);
#define set_config_flag(data_set, option, flag) do { \
const char *tmp = pe_pref(data_set->config_hash, option); \
if(tmp) { \
if(crm_is_true(tmp)) { \
set_bit(data_set->flags, flag); \
} else { \
clear_bit(data_set->flags, flag); \
} \
} \
} while(0)
static void unpack_rsc_op(pe_resource_t *rsc, pe_node_t *node, xmlNode *xml_op,
xmlNode **last_failure,
enum action_fail_response *failed,
pe_working_set_t *data_set);
static void determine_remote_online_status(pe_working_set_t *data_set,
pe_node_t *this_node);
static void add_node_attrs(xmlNode *attrs, pe_node_t *node, bool overwrite,
pe_working_set_t *data_set);
static void determine_online_status(xmlNode *node_state, pe_node_t *this_node,
pe_working_set_t *data_set);
static void unpack_lrm_resources(pe_node_t *node, xmlNode *lrm_state,
pe_working_set_t *data_set);
// Bitmask for warnings we only want to print once
uint32_t pe_wo = 0;
static gboolean
is_dangling_guest_node(pe_node_t *node)
{
/* we are looking for a remote-node that was supposed to be mapped to a
* container resource, but all traces of that container have disappeared
* from both the config and the status section. */
if (pe__is_guest_or_remote_node(node) &&
node->details->remote_rsc &&
node->details->remote_rsc->container == NULL &&
is_set(node->details->remote_rsc->flags, pe_rsc_orphan_container_filler)) {
return TRUE;
}
return FALSE;
}
/*!
* \brief Schedule a fence action for a node
*
* \param[in,out] data_set Current working set of cluster
* \param[in,out] node Node to fence
* \param[in] reason Text description of why fencing is needed
* \param[in] priority_delay Whether to consider `priority-fencing-delay`
*/
void
pe_fence_node(pe_working_set_t * data_set, pe_node_t * node,
const char *reason, bool priority_delay)
{
CRM_CHECK(node, return);
/* A guest node is fenced by marking its container as failed */
if (pe__is_guest_node(node)) {
pe_resource_t *rsc = node->details->remote_rsc->container;
if (is_set(rsc->flags, pe_rsc_failed) == FALSE) {
if (!is_set(rsc->flags, pe_rsc_managed)) {
crm_notice("Not fencing guest node %s "
"(otherwise would because %s): "
"its guest resource %s is unmanaged",
node->details->uname, reason, rsc->id);
} else {
crm_warn("Guest node %s will be fenced "
"(by recovering its guest resource %s): %s",
node->details->uname, rsc->id, reason);
/* We don't mark the node as unclean because that would prevent the
* node from running resources. We want to allow it to run resources
* in this transition if the recovery succeeds.
*/
node->details->remote_requires_reset = TRUE;
set_bit(rsc->flags, pe_rsc_failed);
+ set_bit(rsc->flags, pe_rsc_stop);
}
}
} else if (is_dangling_guest_node(node)) {
crm_info("Cleaning up dangling connection for guest node %s: "
"fencing was already done because %s, "
"and guest resource no longer exists",
node->details->uname, reason);
set_bit(node->details->remote_rsc->flags, pe_rsc_failed);
+ set_bit(node->details->remote_rsc->flags, pe_rsc_stop);
} else if (pe__is_remote_node(node)) {
pe_resource_t *rsc = node->details->remote_rsc;
if (rsc && (!is_set(rsc->flags, pe_rsc_managed))) {
crm_notice("Not fencing remote node %s "
"(otherwise would because %s): connection is unmanaged",
node->details->uname, reason);
} else if(node->details->remote_requires_reset == FALSE) {
node->details->remote_requires_reset = TRUE;
crm_warn("Remote node %s %s: %s",
node->details->uname,
pe_can_fence(data_set, node)? "will be fenced" : "is unclean",
reason);
}
node->details->unclean = TRUE;
// No need to apply `priority-fencing-delay` for remote nodes
pe_fence_op(node, NULL, TRUE, reason, FALSE, data_set);
} else if (node->details->unclean) {
crm_trace("Cluster node %s %s because %s",
node->details->uname,
pe_can_fence(data_set, node)? "would also be fenced" : "also is unclean",
reason);
} else {
crm_warn("Cluster node %s %s: %s",
node->details->uname,
pe_can_fence(data_set, node)? "will be fenced" : "is unclean",
reason);
node->details->unclean = TRUE;
pe_fence_op(node, NULL, TRUE, reason, priority_delay, data_set);
}
}
// @TODO xpaths can't handle templates, rules, or id-refs
// nvpair with provides or requires set to unfencing
#define XPATH_UNFENCING_NVPAIR XML_CIB_TAG_NVPAIR \
"[(@" XML_NVPAIR_ATTR_NAME "='" XML_RSC_ATTR_PROVIDES "'" \
"or @" XML_NVPAIR_ATTR_NAME "='" XML_RSC_ATTR_REQUIRES "') " \
"and @" XML_NVPAIR_ATTR_VALUE "='unfencing']"
// unfencing in rsc_defaults or any resource
#define XPATH_ENABLE_UNFENCING \
"/" XML_TAG_CIB "/" XML_CIB_TAG_CONFIGURATION "/" XML_CIB_TAG_RESOURCES \
"//" XML_TAG_META_SETS "/" XPATH_UNFENCING_NVPAIR \
"|/" XML_TAG_CIB "/" XML_CIB_TAG_CONFIGURATION "/" XML_CIB_TAG_RSCCONFIG \
"/" XML_TAG_META_SETS "/" XPATH_UNFENCING_NVPAIR
static
void set_if_xpath(unsigned long long flag, const char *xpath,
pe_working_set_t *data_set)
{
xmlXPathObjectPtr result = NULL;
if (is_not_set(data_set->flags, flag)) {
result = xpath_search(data_set->input, xpath);
if (result && (numXpathResults(result) > 0)) {
set_bit(data_set->flags, flag);
}
freeXpathObject(result);
}
}
gboolean
unpack_config(xmlNode * config, pe_working_set_t * data_set)
{
const char *value = NULL;
GHashTable *config_hash = crm_str_table_new();
pe_rule_eval_data_t rule_data = {
.node_hash = NULL,
.role = RSC_ROLE_UNKNOWN,
.now = data_set->now,
.match_data = NULL,
.rsc_data = NULL,
.op_data = NULL
};
data_set->config_hash = config_hash;
pe__unpack_dataset_nvpairs(config, XML_CIB_TAG_PROPSET, &rule_data, config_hash,
CIB_OPTIONS_FIRST, FALSE, data_set);
verify_pe_options(data_set->config_hash);
set_config_flag(data_set, "enable-startup-probes", pe_flag_startup_probes);
if(is_not_set(data_set->flags, pe_flag_startup_probes)) {
crm_info("Startup probes: disabled (dangerous)");
}
value = pe_pref(data_set->config_hash, XML_ATTR_HAVE_WATCHDOG);
if (value && crm_is_true(value)) {
crm_notice("Watchdog will be used via SBD if fencing is required "
"and stonith-watchdog-timeout is nonzero");
set_bit(data_set->flags, pe_flag_have_stonith_resource);
}
/* Set certain flags via xpath here, so they can be used before the relevant
* configuration sections are unpacked.
*/
set_if_xpath(pe_flag_enable_unfencing, XPATH_ENABLE_UNFENCING, data_set);
value = pe_pref(data_set->config_hash, "stonith-timeout");
data_set->stonith_timeout = (int) crm_parse_interval_spec(value);
crm_debug("STONITH timeout: %d", data_set->stonith_timeout);
set_config_flag(data_set, "stonith-enabled", pe_flag_stonith_enabled);
crm_debug("STONITH of failed nodes is %s",
is_set(data_set->flags, pe_flag_stonith_enabled) ? "enabled" : "disabled");
data_set->stonith_action = pe_pref(data_set->config_hash, "stonith-action");
if (!strcmp(data_set->stonith_action, "poweroff")) {
pe_warn_once(pe_wo_poweroff,
"Support for stonith-action of 'poweroff' is deprecated "
"and will be removed in a future release (use 'off' instead)");
data_set->stonith_action = "off";
}
crm_trace("STONITH will %s nodes", data_set->stonith_action);
set_config_flag(data_set, "concurrent-fencing", pe_flag_concurrent_fencing);
crm_debug("Concurrent fencing is %s",
is_set(data_set->flags, pe_flag_concurrent_fencing) ? "enabled" : "disabled");
value = pe_pref(data_set->config_hash,
XML_CONFIG_ATTR_PRIORITY_FENCING_DELAY);
if (value) {
data_set->priority_fencing_delay = crm_parse_interval_spec(value) / 1000;
crm_trace("Priority fencing delay is %ds", data_set->priority_fencing_delay);
}
set_config_flag(data_set, "stop-all-resources", pe_flag_stop_everything);
crm_debug("Stop all active resources: %s",
is_set(data_set->flags, pe_flag_stop_everything) ? "true" : "false");
set_config_flag(data_set, "symmetric-cluster", pe_flag_symmetric_cluster);
if (is_set(data_set->flags, pe_flag_symmetric_cluster)) {
crm_debug("Cluster is symmetric" " - resources can run anywhere by default");
}
value = pe_pref(data_set->config_hash, "no-quorum-policy");
if (safe_str_eq(value, "ignore")) {
data_set->no_quorum_policy = no_quorum_ignore;
} else if (safe_str_eq(value, "freeze")) {
data_set->no_quorum_policy = no_quorum_freeze;
+ } else if (safe_str_eq(value, "demote")) {
+ data_set->no_quorum_policy = no_quorum_demote;
+
} else if (safe_str_eq(value, "suicide")) {
if (is_set(data_set->flags, pe_flag_stonith_enabled)) {
int do_panic = 0;
crm_element_value_int(data_set->input, XML_ATTR_QUORUM_PANIC,
&do_panic);
if (do_panic || is_set(data_set->flags, pe_flag_have_quorum)) {
data_set->no_quorum_policy = no_quorum_suicide;
} else {
crm_notice("Resetting no-quorum-policy to 'stop': cluster has never had quorum");
data_set->no_quorum_policy = no_quorum_stop;
}
} else {
pcmk__config_err("Resetting no-quorum-policy to 'stop' because "
"fencing is disabled");
data_set->no_quorum_policy = no_quorum_stop;
}
} else {
data_set->no_quorum_policy = no_quorum_stop;
}
switch (data_set->no_quorum_policy) {
case no_quorum_freeze:
crm_debug("On loss of quorum: Freeze resources");
break;
case no_quorum_stop:
crm_debug("On loss of quorum: Stop ALL resources");
break;
+ case no_quorum_demote:
+ crm_debug("On loss of quorum: "
+ "Demote promotable resources and stop other resources");
+ break;
case no_quorum_suicide:
crm_notice("On loss of quorum: Fence all remaining nodes");
break;
case no_quorum_ignore:
crm_notice("On loss of quorum: Ignore");
break;
}
set_config_flag(data_set, "stop-orphan-resources", pe_flag_stop_rsc_orphans);
crm_trace("Orphan resources are %s",
is_set(data_set->flags, pe_flag_stop_rsc_orphans) ? "stopped" : "ignored");
set_config_flag(data_set, "stop-orphan-actions", pe_flag_stop_action_orphans);
crm_trace("Orphan resource actions are %s",
is_set(data_set->flags, pe_flag_stop_action_orphans) ? "stopped" : "ignored");
set_config_flag(data_set, "remove-after-stop", pe_flag_remove_after_stop);
crm_trace("Stopped resources are removed from the status section: %s",
is_set(data_set->flags, pe_flag_remove_after_stop) ? "true" : "false");
set_config_flag(data_set, "maintenance-mode", pe_flag_maintenance_mode);
crm_trace("Maintenance mode: %s",
is_set(data_set->flags, pe_flag_maintenance_mode) ? "true" : "false");
set_config_flag(data_set, "start-failure-is-fatal", pe_flag_start_failure_fatal);
crm_trace("Start failures are %s",
is_set(data_set->flags,
pe_flag_start_failure_fatal) ? "always fatal" : "handled by failcount");
if (is_set(data_set->flags, pe_flag_stonith_enabled)) {
set_config_flag(data_set, "startup-fencing", pe_flag_startup_fencing);
}
if (is_set(data_set->flags, pe_flag_startup_fencing)) {
crm_trace("Unseen nodes will be fenced");
} else {
pe_warn_once(pe_wo_blind, "Blind faith: not fencing unseen nodes");
}
pcmk__score_red = char2score(pe_pref(data_set->config_hash, "node-health-red"));
pcmk__score_green = char2score(pe_pref(data_set->config_hash, "node-health-green"));
pcmk__score_yellow = char2score(pe_pref(data_set->config_hash, "node-health-yellow"));
crm_debug("Node scores: 'red' = %s, 'yellow' = %s, 'green' = %s",
pe_pref(data_set->config_hash, "node-health-red"),
pe_pref(data_set->config_hash, "node-health-yellow"),
pe_pref(data_set->config_hash, "node-health-green"));
data_set->placement_strategy = pe_pref(data_set->config_hash, "placement-strategy");
crm_trace("Placement strategy: %s", data_set->placement_strategy);
set_config_flag(data_set, "shutdown-lock", pe_flag_shutdown_lock);
crm_trace("Resources will%s be locked to cleanly shut down nodes",
(is_set(data_set->flags, pe_flag_shutdown_lock)? "" : " not"));
if (is_set(data_set->flags, pe_flag_shutdown_lock)) {
value = pe_pref(data_set->config_hash,
XML_CONFIG_ATTR_SHUTDOWN_LOCK_LIMIT);
data_set->shutdown_lock = crm_parse_interval_spec(value) / 1000;
crm_trace("Shutdown locks expire after %us", data_set->shutdown_lock);
}
return TRUE;
}
static void
destroy_digest_cache(gpointer ptr)
{
op_digest_cache_t *data = ptr;
free_xml(data->params_all);
free_xml(data->params_secure);
free_xml(data->params_restart);
free(data->digest_all_calc);
free(data->digest_restart_calc);
free(data->digest_secure_calc);
free(data);
}
pe_node_t *
pe_create_node(const char *id, const char *uname, const char *type,
const char *score, pe_working_set_t * data_set)
{
pe_node_t *new_node = NULL;
if (pe_find_node(data_set->nodes, uname) != NULL) {
pcmk__config_warn("More than one node entry has name '%s'", uname);
}
new_node = calloc(1, sizeof(pe_node_t));
if (new_node == NULL) {
return NULL;
}
new_node->weight = char2score(score);
new_node->fixed = FALSE;
new_node->details = calloc(1, sizeof(struct pe_node_shared_s));
if (new_node->details == NULL) {
free(new_node);
return NULL;
}
crm_trace("Creating node for entry %s/%s", uname, id);
new_node->details->id = id;
new_node->details->uname = uname;
new_node->details->online = FALSE;
new_node->details->shutdown = FALSE;
new_node->details->rsc_discovery_enabled = TRUE;
new_node->details->running_rsc = NULL;
new_node->details->type = node_ping;
if (safe_str_eq(type, "remote")) {
new_node->details->type = node_remote;
set_bit(data_set->flags, pe_flag_have_remote_nodes);
} else if ((type == NULL) || safe_str_eq(type, "member")) {
new_node->details->type = node_member;
}
new_node->details->attrs = crm_str_table_new();
if (pe__is_guest_or_remote_node(new_node)) {
g_hash_table_insert(new_node->details->attrs, strdup(CRM_ATTR_KIND),
strdup("remote"));
} else {
g_hash_table_insert(new_node->details->attrs, strdup(CRM_ATTR_KIND),
strdup("cluster"));
}
new_node->details->utilization = crm_str_table_new();
new_node->details->digest_cache = g_hash_table_new_full(crm_str_hash,
g_str_equal, free,
destroy_digest_cache);
data_set->nodes = g_list_insert_sorted(data_set->nodes, new_node, sort_node_uname);
return new_node;
}
static const char *
expand_remote_rsc_meta(xmlNode *xml_obj, xmlNode *parent, pe_working_set_t *data)
{
xmlNode *attr_set = NULL;
xmlNode *attr = NULL;
const char *container_id = ID(xml_obj);
const char *remote_name = NULL;
const char *remote_server = NULL;
const char *remote_port = NULL;
const char *connect_timeout = "60s";
const char *remote_allow_migrate=NULL;
const char *is_managed = NULL;
for (attr_set = __xml_first_child_element(xml_obj); attr_set != NULL;
attr_set = __xml_next_element(attr_set)) {
if (safe_str_neq((const char *)attr_set->name, XML_TAG_META_SETS)) {
continue;
}
for (attr = __xml_first_child_element(attr_set); attr != NULL;
attr = __xml_next_element(attr)) {
const char *value = crm_element_value(attr, XML_NVPAIR_ATTR_VALUE);
const char *name = crm_element_value(attr, XML_NVPAIR_ATTR_NAME);
if (safe_str_eq(name, XML_RSC_ATTR_REMOTE_NODE)) {
remote_name = value;
} else if (safe_str_eq(name, "remote-addr")) {
remote_server = value;
} else if (safe_str_eq(name, "remote-port")) {
remote_port = value;
} else if (safe_str_eq(name, "remote-connect-timeout")) {
connect_timeout = value;
} else if (safe_str_eq(name, "remote-allow-migrate")) {
remote_allow_migrate=value;
} else if (safe_str_eq(name, XML_RSC_ATTR_MANAGED)) {
is_managed = value;
}
}
}
if (remote_name == NULL) {
return NULL;
}
if (pe_find_resource(data->resources, remote_name) != NULL) {
return NULL;
}
pe_create_remote_xml(parent, remote_name, container_id,
remote_allow_migrate, is_managed,
connect_timeout, remote_server, remote_port);
return remote_name;
}
static void
handle_startup_fencing(pe_working_set_t *data_set, pe_node_t *new_node)
{
if ((new_node->details->type == node_remote) && (new_node->details->remote_rsc == NULL)) {
/* Ignore fencing for remote nodes that don't have a connection resource
* associated with them. This happens when remote node entries get left
* in the nodes section after the connection resource is removed.
*/
return;
}
if (is_set(data_set->flags, pe_flag_startup_fencing)) {
// All nodes are unclean until we've seen their status entry
new_node->details->unclean = TRUE;
} else {
// Blind faith ...
new_node->details->unclean = FALSE;
}
/* We need to be able to determine if a node's status section
* exists or not separate from whether the node is unclean. */
new_node->details->unseen = TRUE;
}
gboolean
unpack_nodes(xmlNode * xml_nodes, pe_working_set_t * data_set)
{
xmlNode *xml_obj = NULL;
pe_node_t *new_node = NULL;
const char *id = NULL;
const char *uname = NULL;
const char *type = NULL;
const char *score = NULL;
pe_rule_eval_data_t rule_data = {
.node_hash = NULL,
.role = RSC_ROLE_UNKNOWN,
.now = data_set->now,
.match_data = NULL,
.rsc_data = NULL,
.op_data = NULL
};
for (xml_obj = __xml_first_child_element(xml_nodes); xml_obj != NULL;
xml_obj = __xml_next_element(xml_obj)) {
if (crm_str_eq((const char *)xml_obj->name, XML_CIB_TAG_NODE, TRUE)) {
new_node = NULL;
id = crm_element_value(xml_obj, XML_ATTR_ID);
uname = crm_element_value(xml_obj, XML_ATTR_UNAME);
type = crm_element_value(xml_obj, XML_ATTR_TYPE);
score = crm_element_value(xml_obj, XML_RULE_ATTR_SCORE);
crm_trace("Processing node %s/%s", uname, id);
if (id == NULL) {
pcmk__config_err("Ignoring <" XML_CIB_TAG_NODE
"> entry in configuration without id");
continue;
}
new_node = pe_create_node(id, uname, type, score, data_set);
if (new_node == NULL) {
return FALSE;
}
/* if(data_set->have_quorum == FALSE */
/* && data_set->no_quorum_policy == no_quorum_stop) { */
/* /\* start shutting resources down *\/ */
/* new_node->weight = -INFINITY; */
/* } */
handle_startup_fencing(data_set, new_node);
add_node_attrs(xml_obj, new_node, FALSE, data_set);
pe__unpack_dataset_nvpairs(xml_obj, XML_TAG_UTILIZATION, &rule_data,
new_node->details->utilization, NULL,
FALSE, data_set);
crm_trace("Done with node %s", crm_element_value(xml_obj, XML_ATTR_UNAME));
}
}
if (data_set->localhost && pe_find_node(data_set->nodes, data_set->localhost) == NULL) {
crm_info("Creating a fake local node");
pe_create_node(data_set->localhost, data_set->localhost, NULL, 0,
data_set);
}
return TRUE;
}
static void
setup_container(pe_resource_t * rsc, pe_working_set_t * data_set)
{
const char *container_id = NULL;
if (rsc->children) {
GListPtr gIter = rsc->children;
for (; gIter != NULL; gIter = gIter->next) {
pe_resource_t *child_rsc = (pe_resource_t *) gIter->data;
setup_container(child_rsc, data_set);
}
return;
}
container_id = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_CONTAINER);
if (container_id && safe_str_neq(container_id, rsc->id)) {
pe_resource_t *container = pe_find_resource(data_set->resources, container_id);
if (container) {
rsc->container = container;
set_bit(container->flags, pe_rsc_is_container);
container->fillers = g_list_append(container->fillers, rsc);
pe_rsc_trace(rsc, "Resource %s's container is %s", rsc->id, container_id);
} else {
pe_err("Resource %s: Unknown resource container (%s)", rsc->id, container_id);
}
}
}
gboolean
unpack_remote_nodes(xmlNode * xml_resources, pe_working_set_t * data_set)
{
xmlNode *xml_obj = NULL;
/* Create remote nodes and guest nodes from the resource configuration
* before unpacking resources.
*/
for (xml_obj = __xml_first_child_element(xml_resources); xml_obj != NULL;
xml_obj = __xml_next_element(xml_obj)) {
const char *new_node_id = NULL;
/* Check for remote nodes, which are defined by ocf:pacemaker:remote
* primitives.
*/
if (xml_contains_remote_node(xml_obj)) {
new_node_id = ID(xml_obj);
/* The "pe_find_node" check is here to make sure we don't iterate over
* an expanded node that has already been added to the node list. */
if (new_node_id && pe_find_node(data_set->nodes, new_node_id) == NULL) {
crm_trace("Found remote node %s defined by resource %s",
new_node_id, ID(xml_obj));
pe_create_node(new_node_id, new_node_id, "remote", NULL,
data_set);
}
continue;
}
/* Check for guest nodes, which are defined by special meta-attributes
* of a primitive of any type (for example, VirtualDomain or Xen).
*/
if (crm_str_eq((const char *)xml_obj->name, XML_CIB_TAG_RESOURCE, TRUE)) {
/* This will add an ocf:pacemaker:remote primitive to the
* configuration for the guest node's connection, to be unpacked
* later.
*/
new_node_id = expand_remote_rsc_meta(xml_obj, xml_resources, data_set);
if (new_node_id && pe_find_node(data_set->nodes, new_node_id) == NULL) {
crm_trace("Found guest node %s in resource %s",
new_node_id, ID(xml_obj));
pe_create_node(new_node_id, new_node_id, "remote", NULL,
data_set);
}
continue;
}
/* Check for guest nodes inside a group. Clones are currently not
* supported as guest nodes.
*/
if (crm_str_eq((const char *)xml_obj->name, XML_CIB_TAG_GROUP, TRUE)) {
xmlNode *xml_obj2 = NULL;
for (xml_obj2 = __xml_first_child_element(xml_obj); xml_obj2 != NULL;
xml_obj2 = __xml_next_element(xml_obj2)) {
new_node_id = expand_remote_rsc_meta(xml_obj2, xml_resources, data_set);
if (new_node_id && pe_find_node(data_set->nodes, new_node_id) == NULL) {
crm_trace("Found guest node %s in resource %s inside group %s",
new_node_id, ID(xml_obj2), ID(xml_obj));
pe_create_node(new_node_id, new_node_id, "remote", NULL,
data_set);
}
}
}
}
return TRUE;
}
/* Call this after all the nodes and resources have been
* unpacked, but before the status section is read.
*
* A remote node's online status is reflected by the state
* of the remote node's connection resource. We need to link
* the remote node to this connection resource so we can have
* easy access to the connection resource during the scheduler calculations.
*/
static void
link_rsc2remotenode(pe_working_set_t *data_set, pe_resource_t *new_rsc)
{
pe_node_t *remote_node = NULL;
if (new_rsc->is_remote_node == FALSE) {
return;
}
if (is_set(data_set->flags, pe_flag_quick_location)) {
/* remote_nodes and remote_resources are not linked in quick location calculations */
return;
}
remote_node = pe_find_node(data_set->nodes, new_rsc->id);
CRM_CHECK(remote_node != NULL, return;);
pe_rsc_trace(new_rsc, "Linking remote connection resource %s to node %s",
new_rsc->id, remote_node->details->uname);
remote_node->details->remote_rsc = new_rsc;
if (new_rsc->container == NULL) {
/* Handle start-up fencing for remote nodes (as opposed to guest nodes)
* the same as is done for cluster nodes.
*/
handle_startup_fencing(data_set, remote_node);
} else {
/* pe_create_node() marks the new node as "remote" or "cluster"; now
* that we know the node is a guest node, update it correctly.
*/
g_hash_table_replace(remote_node->details->attrs, strdup(CRM_ATTR_KIND),
strdup("container"));
}
}
static void
destroy_tag(gpointer data)
{
pe_tag_t *tag = data;
if (tag) {
free(tag->id);
g_list_free_full(tag->refs, free);
free(tag);
}
}
/*!
* \internal
* \brief Parse configuration XML for resource information
*
* \param[in] xml_resources Top of resource configuration XML
* \param[in,out] data_set Where to put resource information
*
* \return TRUE
*
* \note unpack_remote_nodes() MUST be called before this, so that the nodes can
* be used when common_unpack() calls resource_location()
*/
gboolean
unpack_resources(xmlNode * xml_resources, pe_working_set_t * data_set)
{
xmlNode *xml_obj = NULL;
GListPtr gIter = NULL;
data_set->template_rsc_sets = g_hash_table_new_full(crm_str_hash,
g_str_equal, free,
destroy_tag);
for (xml_obj = __xml_first_child_element(xml_resources); xml_obj != NULL;
xml_obj = __xml_next_element(xml_obj)) {
pe_resource_t *new_rsc = NULL;
if (crm_str_eq((const char *)xml_obj->name, XML_CIB_TAG_RSC_TEMPLATE, TRUE)) {
const char *template_id = ID(xml_obj);
if (template_id && g_hash_table_lookup_extended(data_set->template_rsc_sets,
template_id, NULL, NULL) == FALSE) {
/* Record the template's ID for the knowledge of its existence anyway. */
g_hash_table_insert(data_set->template_rsc_sets, strdup(template_id), NULL);
}
continue;
}
crm_trace("Beginning unpack... <%s id=%s... >", crm_element_name(xml_obj), ID(xml_obj));
if (common_unpack(xml_obj, &new_rsc, NULL, data_set) && (new_rsc != NULL)) {
data_set->resources = g_list_append(data_set->resources, new_rsc);
pe_rsc_trace(new_rsc, "Added resource %s", new_rsc->id);
} else {
pcmk__config_err("Ignoring <%s> resource '%s' "
"because configuration is invalid",
crm_element_name(xml_obj), crm_str(ID(xml_obj)));
if (new_rsc != NULL && new_rsc->fns != NULL) {
new_rsc->fns->free(new_rsc);
}
}
}
for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
pe_resource_t *rsc = (pe_resource_t *) gIter->data;
setup_container(rsc, data_set);
link_rsc2remotenode(data_set, rsc);
}
data_set->resources = g_list_sort(data_set->resources, sort_rsc_priority);
if (is_set(data_set->flags, pe_flag_quick_location)) {
/* Ignore */
} else if (is_set(data_set->flags, pe_flag_stonith_enabled)
&& is_set(data_set->flags, pe_flag_have_stonith_resource) == FALSE) {
pcmk__config_err("Resource start-up disabled since no STONITH resources have been defined");
pcmk__config_err("Either configure some or disable STONITH with the stonith-enabled option");
pcmk__config_err("NOTE: Clusters with shared data need STONITH to ensure data integrity");
}
return TRUE;
}
gboolean
unpack_tags(xmlNode * xml_tags, pe_working_set_t * data_set)
{
xmlNode *xml_tag = NULL;
data_set->tags = g_hash_table_new_full(crm_str_hash, g_str_equal, free,
destroy_tag);
for (xml_tag = __xml_first_child_element(xml_tags); xml_tag != NULL;
xml_tag = __xml_next_element(xml_tag)) {
xmlNode *xml_obj_ref = NULL;
const char *tag_id = ID(xml_tag);
if (crm_str_eq((const char *)xml_tag->name, XML_CIB_TAG_TAG, TRUE) == FALSE) {
continue;
}
if (tag_id == NULL) {
pcmk__config_err("Ignoring <%s> without " XML_ATTR_ID,
crm_element_name(xml_tag));
continue;
}
for (xml_obj_ref = __xml_first_child_element(xml_tag); xml_obj_ref != NULL;
xml_obj_ref = __xml_next_element(xml_obj_ref)) {
const char *obj_ref = ID(xml_obj_ref);
if (crm_str_eq((const char *)xml_obj_ref->name, XML_CIB_TAG_OBJ_REF, TRUE) == FALSE) {
continue;
}
if (obj_ref == NULL) {
pcmk__config_err("Ignoring <%s> for tag '%s' without " XML_ATTR_ID,
crm_element_name(xml_obj_ref), tag_id);
continue;
}
if (add_tag_ref(data_set->tags, tag_id, obj_ref) == FALSE) {
return FALSE;
}
}
}
return TRUE;
}
/* The ticket state section:
* "/cib/status/tickets/ticket_state" */
static gboolean
unpack_ticket_state(xmlNode * xml_ticket, pe_working_set_t * data_set)
{
const char *ticket_id = NULL;
const char *granted = NULL;
const char *last_granted = NULL;
const char *standby = NULL;
xmlAttrPtr xIter = NULL;
pe_ticket_t *ticket = NULL;
ticket_id = ID(xml_ticket);
if (ticket_id == NULL || strlen(ticket_id) == 0) {
return FALSE;
}
crm_trace("Processing ticket state for %s", ticket_id);
ticket = g_hash_table_lookup(data_set->tickets, ticket_id);
if (ticket == NULL) {
ticket = ticket_new(ticket_id, data_set);
if (ticket == NULL) {
return FALSE;
}
}
for (xIter = xml_ticket->properties; xIter; xIter = xIter->next) {
const char *prop_name = (const char *)xIter->name;
const char *prop_value = crm_element_value(xml_ticket, prop_name);
if (crm_str_eq(prop_name, XML_ATTR_ID, TRUE)) {
continue;
}
g_hash_table_replace(ticket->state, strdup(prop_name), strdup(prop_value));
}
granted = g_hash_table_lookup(ticket->state, "granted");
if (granted && crm_is_true(granted)) {
ticket->granted = TRUE;
crm_info("We have ticket '%s'", ticket->id);
} else {
ticket->granted = FALSE;
crm_info("We do not have ticket '%s'", ticket->id);
}
last_granted = g_hash_table_lookup(ticket->state, "last-granted");
if (last_granted) {
ticket->last_granted = crm_parse_int(last_granted, 0);
}
standby = g_hash_table_lookup(ticket->state, "standby");
if (standby && crm_is_true(standby)) {
ticket->standby = TRUE;
if (ticket->granted) {
crm_info("Granted ticket '%s' is in standby-mode", ticket->id);
}
} else {
ticket->standby = FALSE;
}
crm_trace("Done with ticket state for %s", ticket_id);
return TRUE;
}
static gboolean
unpack_tickets_state(xmlNode * xml_tickets, pe_working_set_t * data_set)
{
xmlNode *xml_obj = NULL;
for (xml_obj = __xml_first_child_element(xml_tickets); xml_obj != NULL;
xml_obj = __xml_next_element(xml_obj)) {
if (crm_str_eq((const char *)xml_obj->name, XML_CIB_TAG_TICKET_STATE, TRUE) == FALSE) {
continue;
}
unpack_ticket_state(xml_obj, data_set);
}
return TRUE;
}
static void
unpack_handle_remote_attrs(pe_node_t *this_node, xmlNode *state, pe_working_set_t * data_set)
{
const char *resource_discovery_enabled = NULL;
xmlNode *attrs = NULL;
pe_resource_t *rsc = NULL;
if (crm_str_eq((const char *)state->name, XML_CIB_TAG_STATE, TRUE) == FALSE) {
return;
}
if ((this_node == NULL) || !pe__is_guest_or_remote_node(this_node)) {
return;
}
crm_trace("Processing remote node id=%s, uname=%s", this_node->details->id, this_node->details->uname);
this_node->details->remote_maintenance =
crm_atoi(crm_element_value(state, XML_NODE_IS_MAINTENANCE), "0");
rsc = this_node->details->remote_rsc;
if (this_node->details->remote_requires_reset == FALSE) {
this_node->details->unclean = FALSE;
this_node->details->unseen = FALSE;
}
attrs = find_xml_node(state, XML_TAG_TRANSIENT_NODEATTRS, FALSE);
add_node_attrs(attrs, this_node, TRUE, data_set);
if (pe__shutdown_requested(this_node)) {
crm_info("Node %s is shutting down", this_node->details->uname);
this_node->details->shutdown = TRUE;
if (rsc) {
rsc->next_role = RSC_ROLE_STOPPED;
}
}
if (crm_is_true(pe_node_attribute_raw(this_node, "standby"))) {
crm_info("Node %s is in standby-mode", this_node->details->uname);
this_node->details->standby = TRUE;
}
if (crm_is_true(pe_node_attribute_raw(this_node, "maintenance")) ||
(rsc && !is_set(rsc->flags, pe_rsc_managed))) {
crm_info("Node %s is in maintenance-mode", this_node->details->uname);
this_node->details->maintenance = TRUE;
}
resource_discovery_enabled = pe_node_attribute_raw(this_node, XML_NODE_ATTR_RSC_DISCOVERY);
if (resource_discovery_enabled && !crm_is_true(resource_discovery_enabled)) {
if (pe__is_remote_node(this_node)
&& is_not_set(data_set->flags, pe_flag_stonith_enabled)) {
crm_warn("Ignoring %s attribute on remote node %s because stonith is disabled",
XML_NODE_ATTR_RSC_DISCOVERY, this_node->details->uname);
} else {
/* This is either a remote node with fencing enabled, or a guest
* node. We don't care whether fencing is enabled when fencing guest
* nodes, because they are "fenced" by recovering their containing
* resource.
*/
crm_info("Node %s has resource discovery disabled", this_node->details->uname);
this_node->details->rsc_discovery_enabled = FALSE;
}
}
}
static bool
unpack_node_loop(xmlNode * status, bool fence, pe_working_set_t * data_set)
{
bool changed = false;
xmlNode *lrm_rsc = NULL;
for (xmlNode *state = __xml_first_child_element(status); state != NULL;
state = __xml_next_element(state)) {
const char *id = NULL;
const char *uname = NULL;
pe_node_t *this_node = NULL;
bool process = FALSE;
if (crm_str_eq((const char *)state->name, XML_CIB_TAG_STATE, TRUE) == FALSE) {
continue;
}
id = crm_element_value(state, XML_ATTR_ID);
uname = crm_element_value(state, XML_ATTR_UNAME);
this_node = pe_find_node_any(data_set->nodes, id, uname);
if (this_node == NULL) {
crm_info("Node %s is unknown", id);
continue;
} else if (this_node->details->unpacked) {
crm_trace("Node %s was already processed", id);
continue;
} else if (!pe__is_guest_or_remote_node(this_node)
&& is_set(data_set->flags, pe_flag_stonith_enabled)) {
// A redundant test, but preserves the order for regression tests
process = TRUE;
} else if (pe__is_guest_or_remote_node(this_node)) {
bool check = FALSE;
pe_resource_t *rsc = this_node->details->remote_rsc;
if(fence) {
check = TRUE;
} else if(rsc == NULL) {
/* Not ready yet */
} else if (pe__is_guest_node(this_node)
&& rsc->role == RSC_ROLE_STARTED
&& rsc->container->role == RSC_ROLE_STARTED) {
/* Both the connection and its containing resource need to be
* known to be up before we process resources running in it.
*/
check = TRUE;
crm_trace("Checking node %s/%s/%s status %d/%d/%d", id, rsc->id, rsc->container->id, fence, rsc->role, RSC_ROLE_STARTED);
} else if (!pe__is_guest_node(this_node)
&& ((rsc->role == RSC_ROLE_STARTED)
|| is_set(data_set->flags, pe_flag_shutdown_lock))) {
check = TRUE;
crm_trace("Checking node %s/%s status %d/%d/%d", id, rsc->id, fence, rsc->role, RSC_ROLE_STARTED);
}
if (check) {
determine_remote_online_status(data_set, this_node);
unpack_handle_remote_attrs(this_node, state, data_set);
process = TRUE;
}
} else if (this_node->details->online) {
process = TRUE;
} else if (fence) {
process = TRUE;
} else if (is_set(data_set->flags, pe_flag_shutdown_lock)) {
process = TRUE;
}
if(process) {
crm_trace("Processing lrm resource entries on %shealthy%s node: %s",
fence?"un":"",
(pe__is_guest_or_remote_node(this_node)? " remote" : ""),
this_node->details->uname);
changed = TRUE;
this_node->details->unpacked = TRUE;
lrm_rsc = find_xml_node(state, XML_CIB_TAG_LRM, FALSE);
lrm_rsc = find_xml_node(lrm_rsc, XML_LRM_TAG_RESOURCES, FALSE);
unpack_lrm_resources(this_node, lrm_rsc, data_set);
}
}
return changed;
}
/* remove nodes that are down, stopping */
/* create positive rsc_to_node constraints between resources and the nodes they are running on */
/* anything else? */
gboolean
unpack_status(xmlNode * status, pe_working_set_t * data_set)
{
const char *id = NULL;
const char *uname = NULL;
xmlNode *state = NULL;
pe_node_t *this_node = NULL;
crm_trace("Beginning unpack");
if (data_set->tickets == NULL) {
data_set->tickets = g_hash_table_new_full(crm_str_hash, g_str_equal,
free, destroy_ticket);
}
for (state = __xml_first_child_element(status); state != NULL;
state = __xml_next_element(state)) {
if (crm_str_eq((const char *)state->name, XML_CIB_TAG_TICKETS, TRUE)) {
unpack_tickets_state((xmlNode *) state, data_set);
} else if (crm_str_eq((const char *)state->name, XML_CIB_TAG_STATE, TRUE)) {
xmlNode *attrs = NULL;
const char *resource_discovery_enabled = NULL;
id = crm_element_value(state, XML_ATTR_ID);
uname = crm_element_value(state, XML_ATTR_UNAME);
this_node = pe_find_node_any(data_set->nodes, id, uname);
if (uname == NULL) {
/* error */
continue;
} else if (this_node == NULL) {
pcmk__config_warn("Ignoring recorded node status for '%s' "
"because no longer in configuration", uname);
continue;
} else if (pe__is_guest_or_remote_node(this_node)) {
/* online state for remote nodes is determined by the
* rsc state after all the unpacking is done. we do however
* need to mark whether or not the node has been fenced as this plays
* a role during unpacking cluster node resource state */
this_node->details->remote_was_fenced =
crm_atoi(crm_element_value(state, XML_NODE_IS_FENCED), "0");
continue;
}
crm_trace("Processing node id=%s, uname=%s", id, uname);
/* Mark the node as provisionally clean
* - at least we have seen it in the current cluster's lifetime
*/
this_node->details->unclean = FALSE;
this_node->details->unseen = FALSE;
attrs = find_xml_node(state, XML_TAG_TRANSIENT_NODEATTRS, FALSE);
add_node_attrs(attrs, this_node, TRUE, data_set);
if (crm_is_true(pe_node_attribute_raw(this_node, "standby"))) {
crm_info("Node %s is in standby-mode", this_node->details->uname);
this_node->details->standby = TRUE;
}
if (crm_is_true(pe_node_attribute_raw(this_node, "maintenance"))) {
crm_info("Node %s is in maintenance-mode", this_node->details->uname);
this_node->details->maintenance = TRUE;
}
resource_discovery_enabled = pe_node_attribute_raw(this_node, XML_NODE_ATTR_RSC_DISCOVERY);
if (resource_discovery_enabled && !crm_is_true(resource_discovery_enabled)) {
crm_warn("ignoring %s attribute on node %s, disabling resource discovery is not allowed on cluster nodes",
XML_NODE_ATTR_RSC_DISCOVERY, this_node->details->uname);
}
crm_trace("determining node state");
determine_online_status(state, this_node, data_set);
if (is_not_set(data_set->flags, pe_flag_have_quorum)
&& this_node->details->online
&& (data_set->no_quorum_policy == no_quorum_suicide)) {
/* Everything else should flow from this automatically
* (at least until the scheduler becomes able to migrate off
* healthy resources)
*/
pe_fence_node(data_set, this_node, "cluster does not have quorum", FALSE);
}
}
}
while(unpack_node_loop(status, FALSE, data_set)) {
crm_trace("Start another loop");
}
// Now catch any nodes we didn't see
unpack_node_loop(status, is_set(data_set->flags, pe_flag_stonith_enabled), data_set);
/* Now that we know where resources are, we can schedule stops of containers
* with failed bundle connections
*/
if (data_set->stop_needed != NULL) {
for (GList *item = data_set->stop_needed; item; item = item->next) {
pe_resource_t *container = item->data;
pe_node_t *node = pe__current_node(container);
if (node) {
stop_action(container, node, FALSE);
}
}
g_list_free(data_set->stop_needed);
data_set->stop_needed = NULL;
}
for (GListPtr gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
pe_node_t *this_node = gIter->data;
if (this_node == NULL) {
continue;
} else if (!pe__is_guest_or_remote_node(this_node)) {
continue;
} else if(this_node->details->unpacked) {
continue;
}
determine_remote_online_status(data_set, this_node);
}
return TRUE;
}
static gboolean
determine_online_status_no_fencing(pe_working_set_t * data_set, xmlNode * node_state,
pe_node_t * this_node)
{
gboolean online = FALSE;
const char *join = crm_element_value(node_state, XML_NODE_JOIN_STATE);
const char *is_peer = crm_element_value(node_state, XML_NODE_IS_PEER);
const char *in_cluster = crm_element_value(node_state, XML_NODE_IN_CLUSTER);
const char *exp_state = crm_element_value(node_state, XML_NODE_EXPECTED);
if (!crm_is_true(in_cluster)) {
crm_trace("Node is down: in_cluster=%s", crm_str(in_cluster));
} else if (safe_str_eq(is_peer, ONLINESTATUS)) {
if (safe_str_eq(join, CRMD_JOINSTATE_MEMBER)) {
online = TRUE;
} else {
crm_debug("Node is not ready to run resources: %s", join);
}
} else if (this_node->details->expected_up == FALSE) {
crm_trace("Controller is down: in_cluster=%s", crm_str(in_cluster));
crm_trace("\tis_peer=%s, join=%s, expected=%s",
crm_str(is_peer), crm_str(join), crm_str(exp_state));
} else {
/* mark it unclean */
pe_fence_node(data_set, this_node, "peer is unexpectedly down", FALSE);
crm_info("\tin_cluster=%s, is_peer=%s, join=%s, expected=%s",
crm_str(in_cluster), crm_str(is_peer), crm_str(join), crm_str(exp_state));
}
return online;
}
static gboolean
determine_online_status_fencing(pe_working_set_t * data_set, xmlNode * node_state,
pe_node_t * this_node)
{
gboolean online = FALSE;
gboolean do_terminate = FALSE;
bool crmd_online = FALSE;
const char *join = crm_element_value(node_state, XML_NODE_JOIN_STATE);
const char *is_peer = crm_element_value(node_state, XML_NODE_IS_PEER);
const char *in_cluster = crm_element_value(node_state, XML_NODE_IN_CLUSTER);
const char *exp_state = crm_element_value(node_state, XML_NODE_EXPECTED);
const char *terminate = pe_node_attribute_raw(this_node, "terminate");
/*
- XML_NODE_IN_CLUSTER ::= true|false
- XML_NODE_IS_PEER ::= online|offline
- XML_NODE_JOIN_STATE ::= member|down|pending|banned
- XML_NODE_EXPECTED ::= member|down
*/
if (crm_is_true(terminate)) {
do_terminate = TRUE;
} else if (terminate != NULL && strlen(terminate) > 0) {
/* could be a time() value */
char t = terminate[0];
if (t != '0' && isdigit(t)) {
do_terminate = TRUE;
}
}
crm_trace("%s: in_cluster=%s, is_peer=%s, join=%s, expected=%s, term=%d",
this_node->details->uname, crm_str(in_cluster), crm_str(is_peer),
crm_str(join), crm_str(exp_state), do_terminate);
online = crm_is_true(in_cluster);
crmd_online = safe_str_eq(is_peer, ONLINESTATUS);
if (exp_state == NULL) {
exp_state = CRMD_JOINSTATE_DOWN;
}
if (this_node->details->shutdown) {
crm_debug("%s is shutting down", this_node->details->uname);
/* Slightly different criteria since we can't shut down a dead peer */
online = crmd_online;
} else if (in_cluster == NULL) {
pe_fence_node(data_set, this_node, "peer has not been seen by the cluster", FALSE);
} else if (safe_str_eq(join, CRMD_JOINSTATE_NACK)) {
pe_fence_node(data_set, this_node, "peer failed the pacemaker membership criteria", FALSE);
} else if (do_terminate == FALSE && safe_str_eq(exp_state, CRMD_JOINSTATE_DOWN)) {
if (crm_is_true(in_cluster) || crmd_online) {
crm_info("- Node %s is not ready to run resources", this_node->details->uname);
this_node->details->standby = TRUE;
this_node->details->pending = TRUE;
} else {
crm_trace("%s is down or still coming up", this_node->details->uname);
}
} else if (do_terminate && safe_str_eq(join, CRMD_JOINSTATE_DOWN)
&& crm_is_true(in_cluster) == FALSE && !crmd_online) {
crm_info("Node %s was just shot", this_node->details->uname);
online = FALSE;
} else if (crm_is_true(in_cluster) == FALSE) {
// Consider `priority-fencing-delay` for lost nodes
pe_fence_node(data_set, this_node, "peer is no longer part of the cluster", TRUE);
} else if (!crmd_online) {
pe_fence_node(data_set, this_node, "peer process is no longer available", FALSE);
/* Everything is running at this point, now check join state */
} else if (do_terminate) {
pe_fence_node(data_set, this_node, "termination was requested", FALSE);
} else if (safe_str_eq(join, CRMD_JOINSTATE_MEMBER)) {
crm_info("Node %s is active", this_node->details->uname);
} else if (safe_str_eq(join, CRMD_JOINSTATE_PENDING)
|| safe_str_eq(join, CRMD_JOINSTATE_DOWN)) {
crm_info("Node %s is not ready to run resources", this_node->details->uname);
this_node->details->standby = TRUE;
this_node->details->pending = TRUE;
} else {
pe_fence_node(data_set, this_node, "peer was in an unknown state", FALSE);
crm_warn("%s: in-cluster=%s, is-peer=%s, join=%s, expected=%s, term=%d, shutdown=%d",
this_node->details->uname, crm_str(in_cluster), crm_str(is_peer),
crm_str(join), crm_str(exp_state), do_terminate, this_node->details->shutdown);
}
return online;
}
static void
determine_remote_online_status(pe_working_set_t * data_set, pe_node_t * this_node)
{
pe_resource_t *rsc = this_node->details->remote_rsc;
pe_resource_t *container = NULL;
pe_node_t *host = NULL;
/* If there is a node state entry for a (former) Pacemaker Remote node
* but no resource creating that node, the node's connection resource will
* be NULL. Consider it an offline remote node in that case.
*/
if (rsc == NULL) {
this_node->details->online = FALSE;
goto remote_online_done;
}
container = rsc->container;
if (container && pcmk__list_of_1(rsc->running_on)) {
host = rsc->running_on->data;
}
/* If the resource is currently started, mark it online. */
if (rsc->role == RSC_ROLE_STARTED) {
crm_trace("%s node %s presumed ONLINE because connection resource is started",
(container? "Guest" : "Remote"), this_node->details->id);
this_node->details->online = TRUE;
}
/* consider this node shutting down if transitioning start->stop */
if (rsc->role == RSC_ROLE_STARTED && rsc->next_role == RSC_ROLE_STOPPED) {
crm_trace("%s node %s shutting down because connection resource is stopping",
(container? "Guest" : "Remote"), this_node->details->id);
this_node->details->shutdown = TRUE;
}
/* Now check all the failure conditions. */
if(container && is_set(container->flags, pe_rsc_failed)) {
crm_trace("Guest node %s UNCLEAN because guest resource failed",
this_node->details->id);
this_node->details->online = FALSE;
this_node->details->remote_requires_reset = TRUE;
} else if(is_set(rsc->flags, pe_rsc_failed)) {
crm_trace("%s node %s OFFLINE because connection resource failed",
(container? "Guest" : "Remote"), this_node->details->id);
this_node->details->online = FALSE;
} else if (rsc->role == RSC_ROLE_STOPPED
|| (container && container->role == RSC_ROLE_STOPPED)) {
crm_trace("%s node %s OFFLINE because its resource is stopped",
(container? "Guest" : "Remote"), this_node->details->id);
this_node->details->online = FALSE;
this_node->details->remote_requires_reset = FALSE;
} else if (host && (host->details->online == FALSE)
&& host->details->unclean) {
crm_trace("Guest node %s UNCLEAN because host is unclean",
this_node->details->id);
this_node->details->online = FALSE;
this_node->details->remote_requires_reset = TRUE;
}
remote_online_done:
crm_trace("Remote node %s online=%s",
this_node->details->id, this_node->details->online ? "TRUE" : "FALSE");
}
static void
determine_online_status(xmlNode * node_state, pe_node_t * this_node, pe_working_set_t * data_set)
{
gboolean online = FALSE;
const char *exp_state = crm_element_value(node_state, XML_NODE_EXPECTED);
CRM_CHECK(this_node != NULL, return);
this_node->details->shutdown = FALSE;
this_node->details->expected_up = FALSE;
if (pe__shutdown_requested(this_node)) {
this_node->details->shutdown = TRUE;
} else if (safe_str_eq(exp_state, CRMD_JOINSTATE_MEMBER)) {
this_node->details->expected_up = TRUE;
}
if (this_node->details->type == node_ping) {
this_node->details->unclean = FALSE;
online = FALSE; /* As far as resource management is concerned,
* the node is safely offline.
* Anyone caught abusing this logic will be shot
*/
} else if (is_set(data_set->flags, pe_flag_stonith_enabled) == FALSE) {
online = determine_online_status_no_fencing(data_set, node_state, this_node);
} else {
online = determine_online_status_fencing(data_set, node_state, this_node);
}
if (online) {
this_node->details->online = TRUE;
} else {
/* remove node from contention */
this_node->fixed = TRUE;
this_node->weight = -INFINITY;
}
if (online && this_node->details->shutdown) {
/* don't run resources here */
this_node->fixed = TRUE;
this_node->weight = -INFINITY;
}
if (this_node->details->type == node_ping) {
crm_info("Node %s is not a pacemaker node", this_node->details->uname);
} else if (this_node->details->unclean) {
pe_proc_warn("Node %s is unclean", this_node->details->uname);
} else if (this_node->details->online) {
crm_info("Node %s is %s", this_node->details->uname,
this_node->details->shutdown ? "shutting down" :
this_node->details->pending ? "pending" :
this_node->details->standby ? "standby" :
this_node->details->maintenance ? "maintenance" : "online");
} else {
crm_trace("Node %s is offline", this_node->details->uname);
}
}
/*!
* \internal
* \brief Find the end of a resource's name, excluding any clone suffix
*
* \param[in] id Resource ID to check
*
* \return Pointer to last character of resource's base name
*/
const char *
pe_base_name_end(const char *id)
{
if (!pcmk__str_empty(id)) {
const char *end = id + strlen(id) - 1;
for (const char *s = end; s > id; --s) {
switch (*s) {
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9':
break;
case ':':
return (s == end)? s : (s - 1);
default:
return end;
}
}
return end;
}
return NULL;
}
/*!
* \internal
* \brief Get a resource name excluding any clone suffix
*
* \param[in] last_rsc_id Resource ID to check
*
* \return Pointer to newly allocated string with resource's base name
* \note It is the caller's responsibility to free() the result.
* This asserts on error, so callers can assume result is not NULL.
*/
char *
clone_strip(const char *last_rsc_id)
{
const char *end = pe_base_name_end(last_rsc_id);
char *basename = NULL;
CRM_ASSERT(end);
basename = strndup(last_rsc_id, end - last_rsc_id + 1);
CRM_ASSERT(basename);
return basename;
}
/*!
* \internal
* \brief Get the name of the first instance of a cloned resource
*
* \param[in] last_rsc_id Resource ID to check
*
* \return Pointer to newly allocated string with resource's base name plus :0
* \note It is the caller's responsibility to free() the result.
* This asserts on error, so callers can assume result is not NULL.
*/
char *
clone_zero(const char *last_rsc_id)
{
const char *end = pe_base_name_end(last_rsc_id);
size_t base_name_len = end - last_rsc_id + 1;
char *zero = NULL;
CRM_ASSERT(end);
zero = calloc(base_name_len + 3, sizeof(char));
CRM_ASSERT(zero);
memcpy(zero, last_rsc_id, base_name_len);
zero[base_name_len] = ':';
zero[base_name_len + 1] = '0';
return zero;
}
static pe_resource_t *
create_fake_resource(const char *rsc_id, xmlNode * rsc_entry, pe_working_set_t * data_set)
{
pe_resource_t *rsc = NULL;
xmlNode *xml_rsc = create_xml_node(NULL, XML_CIB_TAG_RESOURCE);
copy_in_properties(xml_rsc, rsc_entry);
crm_xml_add(xml_rsc, XML_ATTR_ID, rsc_id);
crm_log_xml_debug(xml_rsc, "Orphan resource");
if (!common_unpack(xml_rsc, &rsc, NULL, data_set)) {
return NULL;
}
if (xml_contains_remote_node(xml_rsc)) {
pe_node_t *node;
crm_debug("Detected orphaned remote node %s", rsc_id);
node = pe_find_node(data_set->nodes, rsc_id);
if (node == NULL) {
node = pe_create_node(rsc_id, rsc_id, "remote", NULL, data_set);
}
link_rsc2remotenode(data_set, rsc);
if (node) {
crm_trace("Setting node %s as shutting down due to orphaned connection resource", rsc_id);
node->details->shutdown = TRUE;
}
}
if (crm_element_value(rsc_entry, XML_RSC_ATTR_CONTAINER)) {
/* This orphaned rsc needs to be mapped to a container. */
crm_trace("Detected orphaned container filler %s", rsc_id);
set_bit(rsc->flags, pe_rsc_orphan_container_filler);
}
set_bit(rsc->flags, pe_rsc_orphan);
data_set->resources = g_list_append(data_set->resources, rsc);
return rsc;
}
/*!
* \internal
* \brief Create orphan instance for anonymous clone resource history
*/
static pe_resource_t *
create_anonymous_orphan(pe_resource_t *parent, const char *rsc_id,
pe_node_t *node, pe_working_set_t *data_set)
{
pe_resource_t *top = pe__create_clone_child(parent, data_set);
// find_rsc() because we might be a cloned group
pe_resource_t *orphan = top->fns->find_rsc(top, rsc_id, NULL, pe_find_clone);
pe_rsc_debug(parent, "Created orphan %s for %s: %s on %s",
top->id, parent->id, rsc_id, node->details->uname);
return orphan;
}
/*!
* \internal
* \brief Check a node for an instance of an anonymous clone
*
* Return a child instance of the specified anonymous clone, in order of
* preference: (1) the instance running on the specified node, if any;
* (2) an inactive instance (i.e. within the total of clone-max instances);
* (3) a newly created orphan (i.e. clone-max instances are already active).
*
* \param[in] data_set Cluster information
* \param[in] node Node on which to check for instance
* \param[in] parent Clone to check
* \param[in] rsc_id Name of cloned resource in history (without instance)
*/
static pe_resource_t *
find_anonymous_clone(pe_working_set_t * data_set, pe_node_t * node, pe_resource_t * parent,
const char *rsc_id)
{
GListPtr rIter = NULL;
pe_resource_t *rsc = NULL;
pe_resource_t *inactive_instance = NULL;
gboolean skip_inactive = FALSE;
CRM_ASSERT(parent != NULL);
CRM_ASSERT(pe_rsc_is_clone(parent));
CRM_ASSERT(is_not_set(parent->flags, pe_rsc_unique));
// Check for active (or partially active, for cloned groups) instance
pe_rsc_trace(parent, "Looking for %s on %s in %s", rsc_id, node->details->uname, parent->id);
for (rIter = parent->children; rsc == NULL && rIter; rIter = rIter->next) {
GListPtr locations = NULL;
pe_resource_t *child = rIter->data;
/* Check whether this instance is already known to be active or pending
* anywhere, at this stage of unpacking. Because this function is called
* for a resource before the resource's individual operation history
* entries are unpacked, locations will generally not contain the
* desired node.
*
* However, there are three exceptions:
* (1) when child is a cloned group and we have already unpacked the
* history of another member of the group on the same node;
* (2) when we've already unpacked the history of another numbered
* instance on the same node (which can happen if globally-unique
* was flipped from true to false); and
* (3) when we re-run calculations on the same data set as part of a
* simulation.
*/
child->fns->location(child, &locations, 2);
if (locations) {
/* We should never associate the same numbered anonymous clone
* instance with multiple nodes, and clone instances can't migrate,
* so there must be only one location, regardless of history.
*/
CRM_LOG_ASSERT(locations->next == NULL);
if (((pe_node_t *)locations->data)->details == node->details) {
/* This child instance is active on the requested node, so check
* for a corresponding configured resource. We use find_rsc()
* instead of child because child may be a cloned group, and we
* need the particular member corresponding to rsc_id.
*
* If the history entry is orphaned, rsc will be NULL.
*/
rsc = parent->fns->find_rsc(child, rsc_id, NULL, pe_find_clone);
if (rsc) {
/* If there are multiple instance history entries for an
* anonymous clone in a single node's history (which can
* happen if globally-unique is switched from true to
* false), we want to consider the instances beyond the
* first as orphans, even if there are inactive instance
* numbers available.
*/
if (rsc->running_on) {
crm_notice("Active (now-)anonymous clone %s has "
"multiple (orphan) instance histories on %s",
parent->id, node->details->uname);
skip_inactive = TRUE;
rsc = NULL;
} else {
pe_rsc_trace(parent, "Resource %s, active", rsc->id);
}
}
}
g_list_free(locations);
} else {
pe_rsc_trace(parent, "Resource %s, skip inactive", child->id);
if (!skip_inactive && !inactive_instance
&& is_not_set(child->flags, pe_rsc_block)) {
// Remember one inactive instance in case we don't find active
inactive_instance = parent->fns->find_rsc(child, rsc_id, NULL,
pe_find_clone);
/* ... but don't use it if it was already associated with a
* pending action on another node
*/
if (inactive_instance && inactive_instance->pending_node
&& (inactive_instance->pending_node->details != node->details)) {
inactive_instance = NULL;
}
}
}
}
if ((rsc == NULL) && !skip_inactive && (inactive_instance != NULL)) {
pe_rsc_trace(parent, "Resource %s, empty slot", inactive_instance->id);
rsc = inactive_instance;
}
/* If the resource has "requires" set to "quorum" or "nothing", and we don't
* have a clone instance for every node, we don't want to consume a valid
* instance number for unclean nodes. Such instances may appear to be active
* according to the history, but should be considered inactive, so we can
* start an instance elsewhere. Treat such instances as orphans.
*
* An exception is instances running on guest nodes -- since guest node
* "fencing" is actually just a resource stop, requires shouldn't apply.
*
* @TODO Ideally, we'd use an inactive instance number if it is not needed
* for any clean instances. However, we don't know that at this point.
*/
if ((rsc != NULL) && is_not_set(rsc->flags, pe_rsc_needs_fencing)
&& (!node->details->online || node->details->unclean)
&& !pe__is_guest_node(node)
&& !pe__is_universal_clone(parent, data_set)) {
rsc = NULL;
}
if (rsc == NULL) {
rsc = create_anonymous_orphan(parent, rsc_id, node, data_set);
pe_rsc_trace(parent, "Resource %s, orphan", rsc->id);
}
return rsc;
}
static pe_resource_t *
unpack_find_resource(pe_working_set_t * data_set, pe_node_t * node, const char *rsc_id,
xmlNode * rsc_entry)
{
pe_resource_t *rsc = NULL;
pe_resource_t *parent = NULL;
crm_trace("looking for %s", rsc_id);
rsc = pe_find_resource(data_set->resources, rsc_id);
if (rsc == NULL) {
/* If we didn't find the resource by its name in the operation history,
* check it again as a clone instance. Even when clone-max=0, we create
* a single :0 orphan to match against here.
*/
char *clone0_id = clone_zero(rsc_id);
pe_resource_t *clone0 = pe_find_resource(data_set->resources, clone0_id);
if (clone0 && is_not_set(clone0->flags, pe_rsc_unique)) {
rsc = clone0;
parent = uber_parent(clone0);
crm_trace("%s found as %s (%s)", rsc_id, clone0_id, parent->id);
} else {
crm_trace("%s is not known as %s either (orphan)",
rsc_id, clone0_id);
}
free(clone0_id);
} else if (rsc->variant > pe_native) {
crm_trace("Resource history for %s is orphaned because it is no longer primitive",
rsc_id);
return NULL;
} else {
parent = uber_parent(rsc);
}
if (pe_rsc_is_anon_clone(parent)) {
if (pe_rsc_is_bundled(parent)) {
rsc = pe__find_bundle_replica(parent->parent, node);
} else {
char *base = clone_strip(rsc_id);
rsc = find_anonymous_clone(data_set, node, parent, base);
free(base);
CRM_ASSERT(rsc != NULL);
}
}
if (rsc && safe_str_neq(rsc_id, rsc->id)
&& safe_str_neq(rsc_id, rsc->clone_name)) {
free(rsc->clone_name);
rsc->clone_name = strdup(rsc_id);
pe_rsc_debug(rsc, "Internally renamed %s on %s to %s%s",
rsc_id, node->details->uname, rsc->id,
(is_set(rsc->flags, pe_rsc_orphan)? " (ORPHAN)" : ""));
}
return rsc;
}
static pe_resource_t *
process_orphan_resource(xmlNode * rsc_entry, pe_node_t * node, pe_working_set_t * data_set)
{
pe_resource_t *rsc = NULL;
const char *rsc_id = crm_element_value(rsc_entry, XML_ATTR_ID);
crm_debug("Detected orphan resource %s on %s", rsc_id, node->details->uname);
rsc = create_fake_resource(rsc_id, rsc_entry, data_set);
if (is_set(data_set->flags, pe_flag_stop_rsc_orphans) == FALSE) {
clear_bit(rsc->flags, pe_rsc_managed);
} else {
CRM_CHECK(rsc != NULL, return NULL);
pe_rsc_trace(rsc, "Added orphan %s", rsc->id);
resource_location(rsc, NULL, -INFINITY, "__orphan_do_not_run__", data_set);
}
return rsc;
}
static void
process_rsc_state(pe_resource_t * rsc, pe_node_t * node,
enum action_fail_response on_fail,
xmlNode * migrate_op, pe_working_set_t * data_set)
{
pe_node_t *tmpnode = NULL;
char *reason = NULL;
CRM_ASSERT(rsc);
pe_rsc_trace(rsc, "Resource %s is %s on %s: on_fail=%s",
rsc->id, role2text(rsc->role), node->details->uname, fail2text(on_fail));
/* process current state */
if (rsc->role != RSC_ROLE_UNKNOWN) {
pe_resource_t *iter = rsc;
while (iter) {
if (g_hash_table_lookup(iter->known_on, node->details->id) == NULL) {
pe_node_t *n = pe__copy_node(node);
pe_rsc_trace(rsc, "%s (aka. %s) known on %s", rsc->id, rsc->clone_name,
n->details->uname);
g_hash_table_insert(iter->known_on, (gpointer) n->details->id, n);
}
if (is_set(iter->flags, pe_rsc_unique)) {
break;
}
iter = iter->parent;
}
}
/* If a managed resource is believed to be running, but node is down ... */
if (rsc->role > RSC_ROLE_STOPPED
&& node->details->online == FALSE
&& node->details->maintenance == FALSE
&& is_set(rsc->flags, pe_rsc_managed)) {
gboolean should_fence = FALSE;
/* If this is a guest node, fence it (regardless of whether fencing is
* enabled, because guest node fencing is done by recovery of the
* container resource rather than by the fencer). Mark the resource
* we're processing as failed. When the guest comes back up, its
* operation history in the CIB will be cleared, freeing the affected
* resource to run again once we are sure we know its state.
*/
if (pe__is_guest_node(node)) {
set_bit(rsc->flags, pe_rsc_failed);
+ set_bit(rsc->flags, pe_rsc_stop);
should_fence = TRUE;
} else if (is_set(data_set->flags, pe_flag_stonith_enabled)) {
if (pe__is_remote_node(node) && node->details->remote_rsc
&& is_not_set(node->details->remote_rsc->flags, pe_rsc_failed)) {
/* Setting unseen means that fencing of the remote node will
* occur only if the connection resource is not going to start
* somewhere. This allows connection resources on a failed
* cluster node to move to another node without requiring the
* remote nodes to be fenced as well.
*/
node->details->unseen = TRUE;
reason = crm_strdup_printf("%s is active there (fencing will be"
" revoked if remote connection can "
"be re-established elsewhere)",
rsc->id);
}
should_fence = TRUE;
}
if (should_fence) {
if (reason == NULL) {
reason = crm_strdup_printf("%s is thought to be active there", rsc->id);
}
pe_fence_node(data_set, node, reason, FALSE);
}
free(reason);
}
if (node->details->unclean) {
/* No extra processing needed
* Also allows resources to be started again after a node is shot
*/
on_fail = action_fail_ignore;
}
switch (on_fail) {
case action_fail_ignore:
/* nothing to do */
break;
+ case action_fail_demote:
+ set_bit(rsc->flags, pe_rsc_failed);
+ demote_action(rsc, node, FALSE);
+ break;
+
case action_fail_fence:
/* treat it as if it is still running
* but also mark the node as unclean
*/
reason = crm_strdup_printf("%s failed there", rsc->id);
pe_fence_node(data_set, node, reason, FALSE);
free(reason);
break;
case action_fail_standby:
node->details->standby = TRUE;
node->details->standby_onfail = TRUE;
break;
case action_fail_block:
/* is_managed == FALSE will prevent any
* actions being sent for the resource
*/
clear_bit(rsc->flags, pe_rsc_managed);
set_bit(rsc->flags, pe_rsc_block);
break;
case action_fail_migrate:
/* make sure it comes up somewhere else
* or not at all
*/
resource_location(rsc, node, -INFINITY, "__action_migration_auto__", data_set);
break;
case action_fail_stop:
rsc->next_role = RSC_ROLE_STOPPED;
break;
case action_fail_recover:
if (rsc->role != RSC_ROLE_STOPPED && rsc->role != RSC_ROLE_UNKNOWN) {
set_bit(rsc->flags, pe_rsc_failed);
+ set_bit(rsc->flags, pe_rsc_stop);
stop_action(rsc, node, FALSE);
}
break;
case action_fail_restart_container:
set_bit(rsc->flags, pe_rsc_failed);
+ set_bit(rsc->flags, pe_rsc_stop);
if (rsc->container && pe_rsc_is_bundled(rsc)) {
/* A bundle's remote connection can run on a different node than
* the bundle's container. We don't necessarily know where the
* container is running yet, so remember it and add a stop
* action for it later.
*/
data_set->stop_needed = g_list_prepend(data_set->stop_needed,
rsc->container);
} else if (rsc->container) {
stop_action(rsc->container, node, FALSE);
} else if (rsc->role != RSC_ROLE_STOPPED && rsc->role != RSC_ROLE_UNKNOWN) {
stop_action(rsc, node, FALSE);
}
break;
case action_fail_reset_remote:
set_bit(rsc->flags, pe_rsc_failed);
+ set_bit(rsc->flags, pe_rsc_stop);
if (is_set(data_set->flags, pe_flag_stonith_enabled)) {
tmpnode = NULL;
if (rsc->is_remote_node) {
tmpnode = pe_find_node(data_set->nodes, rsc->id);
}
if (tmpnode &&
pe__is_remote_node(tmpnode) &&
tmpnode->details->remote_was_fenced == 0) {
/* The remote connection resource failed in a way that
* should result in fencing the remote node.
*/
pe_fence_node(data_set, tmpnode,
"remote connection is unrecoverable", FALSE);
}
}
/* require the stop action regardless if fencing is occurring or not. */
if (rsc->role > RSC_ROLE_STOPPED) {
stop_action(rsc, node, FALSE);
}
/* if reconnect delay is in use, prevent the connection from exiting the
* "STOPPED" role until the failure is cleared by the delay timeout. */
if (rsc->remote_reconnect_ms) {
rsc->next_role = RSC_ROLE_STOPPED;
}
break;
}
/* ensure a remote-node connection failure forces an unclean remote-node
* to be fenced. By setting unseen = FALSE, the remote-node failure will
* result in a fencing operation regardless if we're going to attempt to
* reconnect to the remote-node in this transition or not. */
if (is_set(rsc->flags, pe_rsc_failed) && rsc->is_remote_node) {
tmpnode = pe_find_node(data_set->nodes, rsc->id);
if (tmpnode && tmpnode->details->unclean) {
tmpnode->details->unseen = FALSE;
}
}
if (rsc->role != RSC_ROLE_STOPPED && rsc->role != RSC_ROLE_UNKNOWN) {
if (is_set(rsc->flags, pe_rsc_orphan)) {
if (is_set(rsc->flags, pe_rsc_managed)) {
pcmk__config_warn("Detected active orphan %s running on %s",
rsc->id, node->details->uname);
} else {
pcmk__config_warn("Resource '%s' must be stopped manually on "
"%s because cluster is configured not to "
"stop active orphans",
rsc->id, node->details->uname);
}
}
native_add_running(rsc, node, data_set);
- if (on_fail != action_fail_ignore) {
- set_bit(rsc->flags, pe_rsc_failed);
+ switch (on_fail) {
+ case action_fail_ignore:
+ break;
+ case action_fail_demote:
+ case action_fail_block:
+ set_bit(rsc->flags, pe_rsc_failed);
+ break;
+ default:
+ set_bit(rsc->flags, pe_rsc_failed);
+ set_bit(rsc->flags, pe_rsc_stop);
+ break;
}
} else if (rsc->clone_name && strchr(rsc->clone_name, ':') != NULL) {
/* Only do this for older status sections that included instance numbers
* Otherwise stopped instances will appear as orphans
*/
pe_rsc_trace(rsc, "Resetting clone_name %s for %s (stopped)", rsc->clone_name, rsc->id);
free(rsc->clone_name);
rsc->clone_name = NULL;
} else {
GList *possible_matches = pe__resource_actions(rsc, node, RSC_STOP,
FALSE);
GListPtr gIter = possible_matches;
for (; gIter != NULL; gIter = gIter->next) {
pe_action_t *stop = (pe_action_t *) gIter->data;
stop->flags |= pe_action_optional;
}
g_list_free(possible_matches);
}
}
/* create active recurring operations as optional */
static void
process_recurring(pe_node_t * node, pe_resource_t * rsc,
int start_index, int stop_index,
GListPtr sorted_op_list, pe_working_set_t * data_set)
{
int counter = -1;
const char *task = NULL;
const char *status = NULL;
GListPtr gIter = sorted_op_list;
CRM_ASSERT(rsc);
pe_rsc_trace(rsc, "%s: Start index %d, stop index = %d", rsc->id, start_index, stop_index);
for (; gIter != NULL; gIter = gIter->next) {
xmlNode *rsc_op = (xmlNode *) gIter->data;
guint interval_ms = 0;
char *key = NULL;
const char *id = ID(rsc_op);
counter++;
if (node->details->online == FALSE) {
pe_rsc_trace(rsc, "Skipping %s/%s: node is offline", rsc->id, node->details->uname);
break;
/* Need to check if there's a monitor for role="Stopped" */
} else if (start_index < stop_index && counter <= stop_index) {
pe_rsc_trace(rsc, "Skipping %s/%s: resource is not active", id, node->details->uname);
continue;
} else if (counter < start_index) {
pe_rsc_trace(rsc, "Skipping %s/%s: old %d", id, node->details->uname, counter);
continue;
}
crm_element_value_ms(rsc_op, XML_LRM_ATTR_INTERVAL_MS, &interval_ms);
if (interval_ms == 0) {
pe_rsc_trace(rsc, "Skipping %s/%s: non-recurring", id, node->details->uname);
continue;
}
status = crm_element_value(rsc_op, XML_LRM_ATTR_OPSTATUS);
if (safe_str_eq(status, "-1")) {
pe_rsc_trace(rsc, "Skipping %s/%s: status", id, node->details->uname);
continue;
}
task = crm_element_value(rsc_op, XML_LRM_ATTR_TASK);
/* create the action */
key = pcmk__op_key(rsc->id, task, interval_ms);
pe_rsc_trace(rsc, "Creating %s/%s", key, node->details->uname);
custom_action(rsc, key, task, node, TRUE, TRUE, data_set);
}
}
void
calculate_active_ops(GListPtr sorted_op_list, int *start_index, int *stop_index)
{
int counter = -1;
int implied_monitor_start = -1;
int implied_clone_start = -1;
const char *task = NULL;
const char *status = NULL;
GListPtr gIter = sorted_op_list;
*stop_index = -1;
*start_index = -1;
for (; gIter != NULL; gIter = gIter->next) {
xmlNode *rsc_op = (xmlNode *) gIter->data;
counter++;
task = crm_element_value(rsc_op, XML_LRM_ATTR_TASK);
status = crm_element_value(rsc_op, XML_LRM_ATTR_OPSTATUS);
if (safe_str_eq(task, CRMD_ACTION_STOP)
&& safe_str_eq(status, "0")) {
*stop_index = counter;
} else if (safe_str_eq(task, CRMD_ACTION_START) || safe_str_eq(task, CRMD_ACTION_MIGRATED)) {
*start_index = counter;
} else if ((implied_monitor_start <= *stop_index) && safe_str_eq(task, CRMD_ACTION_STATUS)) {
const char *rc = crm_element_value(rsc_op, XML_LRM_ATTR_RC);
if (safe_str_eq(rc, "0") || safe_str_eq(rc, "8")) {
implied_monitor_start = counter;
}
} else if (safe_str_eq(task, CRMD_ACTION_PROMOTE) || safe_str_eq(task, CRMD_ACTION_DEMOTE)) {
implied_clone_start = counter;
}
}
if (*start_index == -1) {
if (implied_clone_start != -1) {
*start_index = implied_clone_start;
} else if (implied_monitor_start != -1) {
*start_index = implied_monitor_start;
}
}
}
// If resource history entry has shutdown lock, remember lock node and time
static void
unpack_shutdown_lock(xmlNode *rsc_entry, pe_resource_t *rsc, pe_node_t *node,
pe_working_set_t *data_set)
{
time_t lock_time = 0; // When lock started (i.e. node shutdown time)
if ((crm_element_value_epoch(rsc_entry, XML_CONFIG_ATTR_SHUTDOWN_LOCK,
&lock_time) == pcmk_ok) && (lock_time != 0)) {
if ((data_set->shutdown_lock > 0)
&& (get_effective_time(data_set)
> (lock_time + data_set->shutdown_lock))) {
pe_rsc_info(rsc, "Shutdown lock for %s on %s expired",
rsc->id, node->details->uname);
pe__clear_resource_history(rsc, node, data_set);
} else {
rsc->lock_node = node;
rsc->lock_time = lock_time;
}
}
}
static pe_resource_t *
unpack_lrm_rsc_state(pe_node_t * node, xmlNode * rsc_entry, pe_working_set_t * data_set)
{
GListPtr gIter = NULL;
int stop_index = -1;
int start_index = -1;
enum rsc_role_e req_role = RSC_ROLE_UNKNOWN;
const char *task = NULL;
const char *rsc_id = crm_element_value(rsc_entry, XML_ATTR_ID);
pe_resource_t *rsc = NULL;
GListPtr op_list = NULL;
GListPtr sorted_op_list = NULL;
xmlNode *migrate_op = NULL;
xmlNode *rsc_op = NULL;
xmlNode *last_failure = NULL;
- enum action_fail_response on_fail = FALSE;
+ enum action_fail_response on_fail = action_fail_ignore;
enum rsc_role_e saved_role = RSC_ROLE_UNKNOWN;
crm_trace("[%s] Processing %s on %s",
crm_element_name(rsc_entry), rsc_id, node->details->uname);
/* extract operations */
op_list = NULL;
sorted_op_list = NULL;
for (rsc_op = __xml_first_child_element(rsc_entry); rsc_op != NULL;
rsc_op = __xml_next_element(rsc_op)) {
if (crm_str_eq((const char *)rsc_op->name, XML_LRM_TAG_RSC_OP, TRUE)) {
op_list = g_list_prepend(op_list, rsc_op);
}
}
if (is_not_set(data_set->flags, pe_flag_shutdown_lock)) {
if (op_list == NULL) {
// If there are no operations, there is nothing to do
return NULL;
}
}
/* find the resource */
rsc = unpack_find_resource(data_set, node, rsc_id, rsc_entry);
if (rsc == NULL) {
if (op_list == NULL) {
// If there are no operations, there is nothing to do
return NULL;
} else {
rsc = process_orphan_resource(rsc_entry, node, data_set);
}
}
CRM_ASSERT(rsc != NULL);
// Check whether the resource is "shutdown-locked" to this node
if (is_set(data_set->flags, pe_flag_shutdown_lock)) {
unpack_shutdown_lock(rsc_entry, rsc, node, data_set);
}
/* process operations */
saved_role = rsc->role;
- on_fail = action_fail_ignore;
rsc->role = RSC_ROLE_UNKNOWN;
sorted_op_list = g_list_sort(op_list, sort_op_by_callid);
for (gIter = sorted_op_list; gIter != NULL; gIter = gIter->next) {
xmlNode *rsc_op = (xmlNode *) gIter->data;
task = crm_element_value(rsc_op, XML_LRM_ATTR_TASK);
if (safe_str_eq(task, CRMD_ACTION_MIGRATED)) {
migrate_op = rsc_op;
}
unpack_rsc_op(rsc, node, rsc_op, &last_failure, &on_fail, data_set);
}
/* create active recurring operations as optional */
calculate_active_ops(sorted_op_list, &start_index, &stop_index);
process_recurring(node, rsc, start_index, stop_index, sorted_op_list, data_set);
/* no need to free the contents */
g_list_free(sorted_op_list);
process_rsc_state(rsc, node, on_fail, migrate_op, data_set);
if (get_target_role(rsc, &req_role)) {
if (rsc->next_role == RSC_ROLE_UNKNOWN || req_role < rsc->next_role) {
pe_rsc_debug(rsc, "%s: Overwriting calculated next role %s"
" with requested next role %s",
rsc->id, role2text(rsc->next_role), role2text(req_role));
rsc->next_role = req_role;
} else if (req_role > rsc->next_role) {
pe_rsc_info(rsc, "%s: Not overwriting calculated next role %s"
" with requested next role %s",
rsc->id, role2text(rsc->next_role), role2text(req_role));
}
}
if (saved_role > rsc->role) {
rsc->role = saved_role;
}
return rsc;
}
static void
handle_orphaned_container_fillers(xmlNode * lrm_rsc_list, pe_working_set_t * data_set)
{
xmlNode *rsc_entry = NULL;
for (rsc_entry = __xml_first_child_element(lrm_rsc_list); rsc_entry != NULL;
rsc_entry = __xml_next_element(rsc_entry)) {
pe_resource_t *rsc;
pe_resource_t *container;
const char *rsc_id;
const char *container_id;
if (safe_str_neq((const char *)rsc_entry->name, XML_LRM_TAG_RESOURCE)) {
continue;
}
container_id = crm_element_value(rsc_entry, XML_RSC_ATTR_CONTAINER);
rsc_id = crm_element_value(rsc_entry, XML_ATTR_ID);
if (container_id == NULL || rsc_id == NULL) {
continue;
}
container = pe_find_resource(data_set->resources, container_id);
if (container == NULL) {
continue;
}
rsc = pe_find_resource(data_set->resources, rsc_id);
if (rsc == NULL ||
is_set(rsc->flags, pe_rsc_orphan_container_filler) == FALSE ||
rsc->container != NULL) {
continue;
}
pe_rsc_trace(rsc, "Mapped container of orphaned resource %s to %s",
rsc->id, container_id);
rsc->container = container;
container->fillers = g_list_append(container->fillers, rsc);
}
}
static void
unpack_lrm_resources(pe_node_t *node, xmlNode *lrm_rsc_list,
pe_working_set_t *data_set)
{
xmlNode *rsc_entry = NULL;
gboolean found_orphaned_container_filler = FALSE;
for (rsc_entry = __xml_first_child_element(lrm_rsc_list); rsc_entry != NULL;
rsc_entry = __xml_next_element(rsc_entry)) {
if (crm_str_eq((const char *)rsc_entry->name, XML_LRM_TAG_RESOURCE, TRUE)) {
pe_resource_t *rsc = unpack_lrm_rsc_state(node, rsc_entry, data_set);
if (!rsc) {
continue;
}
if (is_set(rsc->flags, pe_rsc_orphan_container_filler)) {
found_orphaned_container_filler = TRUE;
}
}
}
/* now that all the resource state has been unpacked for this node
* we have to go back and map any orphaned container fillers to their
* container resource */
if (found_orphaned_container_filler) {
handle_orphaned_container_fillers(lrm_rsc_list, data_set);
}
}
static void
set_active(pe_resource_t * rsc)
{
pe_resource_t *top = uber_parent(rsc);
if (top && is_set(top->flags, pe_rsc_promotable)) {
rsc->role = RSC_ROLE_SLAVE;
} else {
rsc->role = RSC_ROLE_STARTED;
}
}
static void
set_node_score(gpointer key, gpointer value, gpointer user_data)
{
pe_node_t *node = value;
int *score = user_data;
node->weight = *score;
}
#define STATUS_PATH_MAX 1024
static xmlNode *
find_lrm_op(const char *resource, const char *op, const char *node, const char *source,
bool success_only, pe_working_set_t *data_set)
{
int offset = 0;
char xpath[STATUS_PATH_MAX];
xmlNode *xml = NULL;
offset += snprintf(xpath + offset, STATUS_PATH_MAX - offset, "//node_state[@uname='%s']", node);
offset +=
snprintf(xpath + offset, STATUS_PATH_MAX - offset, "//" XML_LRM_TAG_RESOURCE "[@id='%s']",
resource);
/* Need to check against transition_magic too? */
if (source && safe_str_eq(op, CRMD_ACTION_MIGRATE)) {
offset +=
snprintf(xpath + offset, STATUS_PATH_MAX - offset,
"/" XML_LRM_TAG_RSC_OP "[@operation='%s' and @migrate_target='%s']", op,
source);
} else if (source && safe_str_eq(op, CRMD_ACTION_MIGRATED)) {
offset +=
snprintf(xpath + offset, STATUS_PATH_MAX - offset,
"/" XML_LRM_TAG_RSC_OP "[@operation='%s' and @migrate_source='%s']", op,
source);
} else {
offset +=
snprintf(xpath + offset, STATUS_PATH_MAX - offset,
"/" XML_LRM_TAG_RSC_OP "[@operation='%s']", op);
}
CRM_LOG_ASSERT(offset > 0);
xml = get_xpath_object(xpath, data_set->input, LOG_DEBUG);
if (xml && success_only) {
int rc = PCMK_OCF_UNKNOWN_ERROR;
int status = PCMK_LRM_OP_ERROR;
crm_element_value_int(xml, XML_LRM_ATTR_RC, &rc);
crm_element_value_int(xml, XML_LRM_ATTR_OPSTATUS, &status);
if ((rc != PCMK_OCF_OK) || (status != PCMK_LRM_OP_DONE)) {
return NULL;
}
}
return xml;
}
static int
pe__call_id(xmlNode *op_xml)
{
int id = 0;
if (op_xml) {
crm_element_value_int(op_xml, XML_LRM_ATTR_CALLID, &id);
}
return id;
}
/*!
* \brief Check whether a stop happened on the same node after some event
*
* \param[in] rsc Resource being checked
* \param[in] node Node being checked
* \param[in] xml_op Event that stop is being compared to
* \param[in] data_set Cluster working set
*
* \return TRUE if stop happened after event, FALSE otherwise
*
* \note This is really unnecessary, but kept as a safety mechanism. We
* currently don't save more than one successful event in history, so this
* only matters when processing really old CIB files that we don't
* technically support anymore, or as preparation for logging an extended
* history in the future.
*/
static bool
stop_happened_after(pe_resource_t *rsc, pe_node_t *node, xmlNode *xml_op,
pe_working_set_t *data_set)
{
xmlNode *stop_op = find_lrm_op(rsc->id, CRMD_ACTION_STOP,
node->details->uname, NULL, TRUE, data_set);
return (stop_op && (pe__call_id(stop_op) > pe__call_id(xml_op)));
}
static void
unpack_migrate_to_success(pe_resource_t *rsc, pe_node_t *node, xmlNode *xml_op,
pe_working_set_t *data_set)
{
/* A successful migration sequence is:
* migrate_to on source node
* migrate_from on target node
* stop on source node
*
* If a migrate_to is followed by a stop, the entire migration (successful
* or failed) is complete, and we don't care what happened on the target.
*
* If no migrate_from has happened, the migration is considered to be
* "partial". If the migrate_from failed, make sure the resource gets
* stopped on both source and target (if up).
*
* If the migrate_to and migrate_from both succeeded (which also implies the
* resource is no longer running on the source), but there is no stop, the
* migration is considered to be "dangling". Schedule a stop on the source
* in this case.
*/
int from_rc = 0;
int from_status = 0;
pe_node_t *target_node = NULL;
pe_node_t *source_node = NULL;
xmlNode *migrate_from = NULL;
const char *source = crm_element_value(xml_op, XML_LRM_ATTR_MIGRATE_SOURCE);
const char *target = crm_element_value(xml_op, XML_LRM_ATTR_MIGRATE_TARGET);
// Sanity check
CRM_CHECK(source && target && !strcmp(source, node->details->uname), return);
if (stop_happened_after(rsc, node, xml_op, data_set)) {
return;
}
// Clones are not allowed to migrate, so role can't be master
rsc->role = RSC_ROLE_STARTED;
target_node = pe_find_node(data_set->nodes, target);
source_node = pe_find_node(data_set->nodes, source);
// Check whether there was a migrate_from action on the target
migrate_from = find_lrm_op(rsc->id, CRMD_ACTION_MIGRATED, target,
source, FALSE, data_set);
if (migrate_from) {
crm_element_value_int(migrate_from, XML_LRM_ATTR_RC, &from_rc);
crm_element_value_int(migrate_from, XML_LRM_ATTR_OPSTATUS, &from_status);
pe_rsc_trace(rsc, "%s op on %s exited with status=%d, rc=%d",
ID(migrate_from), target, from_status, from_rc);
}
if (migrate_from && from_rc == PCMK_OCF_OK
&& from_status == PCMK_LRM_OP_DONE) {
/* The migrate_to and migrate_from both succeeded, so mark the migration
* as "dangling". This will be used to schedule a stop action on the
* source without affecting the target.
*/
pe_rsc_trace(rsc, "Detected dangling migration op: %s on %s", ID(xml_op),
source);
rsc->role = RSC_ROLE_STOPPED;
rsc->dangling_migrations = g_list_prepend(rsc->dangling_migrations, node);
} else if (migrate_from && (from_status != PCMK_LRM_OP_PENDING)) { // Failed
if (target_node && target_node->details->online) {
pe_rsc_trace(rsc, "Marking active on %s %p %d", target, target_node,
target_node->details->online);
native_add_running(rsc, target_node, data_set);
}
} else { // Pending, or complete but erased
if (target_node && target_node->details->online) {
pe_rsc_trace(rsc, "Marking active on %s %p %d", target, target_node,
target_node->details->online);
native_add_running(rsc, target_node, data_set);
if (source_node && source_node->details->online) {
/* This is a partial migration: the migrate_to completed
* successfully on the source, but the migrate_from has not
* completed. Remember the source and target; if the newly
* chosen target remains the same when we schedule actions
* later, we may continue with the migration.
*/
rsc->partial_migration_target = target_node;
rsc->partial_migration_source = source_node;
}
} else {
/* Consider it failed here - forces a restart, prevents migration */
set_bit(rsc->flags, pe_rsc_failed);
+ set_bit(rsc->flags, pe_rsc_stop);
clear_bit(rsc->flags, pe_rsc_allow_migrate);
}
}
}
static void
unpack_migrate_to_failure(pe_resource_t *rsc, pe_node_t *node, xmlNode *xml_op,
pe_working_set_t *data_set)
{
int target_stop_id = 0;
int target_migrate_from_id = 0;
xmlNode *target_stop = NULL;
xmlNode *target_migrate_from = NULL;
const char *source = crm_element_value(xml_op, XML_LRM_ATTR_MIGRATE_SOURCE);
const char *target = crm_element_value(xml_op, XML_LRM_ATTR_MIGRATE_TARGET);
// Sanity check
CRM_CHECK(source && target && !strcmp(source, node->details->uname), return);
/* If a migration failed, we have to assume the resource is active. Clones
* are not allowed to migrate, so role can't be master.
*/
rsc->role = RSC_ROLE_STARTED;
// Check for stop on the target
target_stop = find_lrm_op(rsc->id, CRMD_ACTION_STOP, target, NULL,
TRUE, data_set);
target_stop_id = pe__call_id(target_stop);
// Check for migrate_from on the target
target_migrate_from = find_lrm_op(rsc->id, CRMD_ACTION_MIGRATED, target,
source, TRUE, data_set);
target_migrate_from_id = pe__call_id(target_migrate_from);
if ((target_stop == NULL) || (target_stop_id < target_migrate_from_id)) {
/* There was no stop on the source, or a stop that happened before a
* migrate_from, so assume the resource is still active on the target
* (if it is up).
*/
pe_node_t *target_node = pe_find_node(data_set->nodes, target);
pe_rsc_trace(rsc, "stop (%d) + migrate_from (%d)",
target_stop_id, target_migrate_from_id);
if (target_node && target_node->details->online) {
native_add_running(rsc, target_node, data_set);
}
} else if (target_migrate_from == NULL) {
/* We know there was a stop on the target, but there may not have been a
* migrate_from (the stop could have happened before migrate_from was
* scheduled or attempted).
*
* That means this could be a "dangling" migration. But first, check
* whether there is a newer migrate_from or start on the source node --
* it's possible the failed migration was followed by a successful
* full restart or migration in the reverse direction, in which case we
* don't want to force it to stop.
*/
xmlNode *source_migrate_from = NULL;
xmlNode *source_start = NULL;
int source_migrate_to_id = pe__call_id(xml_op);
source_migrate_from = find_lrm_op(rsc->id, CRMD_ACTION_MIGRATED, source,
NULL, TRUE, data_set);
if (pe__call_id(source_migrate_from) > source_migrate_to_id) {
return;
}
source_start = find_lrm_op(rsc->id, CRMD_ACTION_START, source, NULL,
TRUE, data_set);
if (pe__call_id(source_start) > source_migrate_to_id) {
return;
}
// Mark node as having dangling migration so we can force a stop later
rsc->dangling_migrations = g_list_prepend(rsc->dangling_migrations, node);
}
}
static void
unpack_migrate_from_failure(pe_resource_t *rsc, pe_node_t *node,
xmlNode *xml_op, pe_working_set_t *data_set)
{
xmlNode *source_stop = NULL;
xmlNode *source_migrate_to = NULL;
const char *source = crm_element_value(xml_op, XML_LRM_ATTR_MIGRATE_SOURCE);
const char *target = crm_element_value(xml_op, XML_LRM_ATTR_MIGRATE_TARGET);
// Sanity check
CRM_CHECK(source && target && !strcmp(target, node->details->uname), return);
/* If a migration failed, we have to assume the resource is active. Clones
* are not allowed to migrate, so role can't be master.
*/
rsc->role = RSC_ROLE_STARTED;
// Check for a stop on the source
source_stop = find_lrm_op(rsc->id, CRMD_ACTION_STOP, source, NULL,
TRUE, data_set);
// Check for a migrate_to on the source
source_migrate_to = find_lrm_op(rsc->id, CRMD_ACTION_MIGRATE,
source, target, TRUE, data_set);
if ((source_stop == NULL)
|| (pe__call_id(source_stop) < pe__call_id(source_migrate_to))) {
/* There was no stop on the source, or a stop that happened before
* migrate_to, so assume the resource is still active on the source (if
* it is up).
*/
pe_node_t *source_node = pe_find_node(data_set->nodes, source);
if (source_node && source_node->details->online) {
native_add_running(rsc, source_node, data_set);
}
}
}
static void
record_failed_op(xmlNode *op, const pe_node_t *node,
const pe_resource_t *rsc, pe_working_set_t *data_set)
{
xmlNode *xIter = NULL;
const char *op_key = crm_element_value(op, XML_LRM_ATTR_TASK_KEY);
if (node->details->online == FALSE) {
return;
}
for (xIter = data_set->failed->children; xIter; xIter = xIter->next) {
const char *key = crm_element_value(xIter, XML_LRM_ATTR_TASK_KEY);
const char *uname = crm_element_value(xIter, XML_ATTR_UNAME);
if(safe_str_eq(op_key, key) && safe_str_eq(uname, node->details->uname)) {
crm_trace("Skipping duplicate entry %s on %s", op_key, node->details->uname);
return;
}
}
crm_trace("Adding entry %s on %s", op_key, node->details->uname);
crm_xml_add(op, XML_ATTR_UNAME, node->details->uname);
crm_xml_add(op, XML_LRM_ATTR_RSCID, rsc->id);
add_node_copy(data_set->failed, op);
}
static const char *get_op_key(xmlNode *xml_op)
{
const char *key = crm_element_value(xml_op, XML_LRM_ATTR_TASK_KEY);
if(key == NULL) {
key = ID(xml_op);
}
return key;
}
static const char *
last_change_str(xmlNode *xml_op)
{
time_t when;
const char *when_s = NULL;
if (crm_element_value_epoch(xml_op, XML_RSC_OP_LAST_CHANGE,
&when) == pcmk_ok) {
when_s = pcmk__epoch2str(&when);
if (when_s) {
// Skip day of week to make message shorter
when_s = strchr(when_s, ' ');
if (when_s) {
++when_s;
}
}
}
return ((when_s && *when_s)? when_s : "unknown time");
}
+/*!
+ * \internal
+ * \brief Compare two on-fail values
+ *
+ * \param[in] first One on-fail value to compare
+ * \param[in] second The other on-fail value to compare
+ *
+ * \return A negative number if second is more severe than first, zero if they
+ * are equal, or a positive number if first is more severe than second.
+ * \note This is only needed until the action_fail_response values can be
+ * renumbered at the next API compatibility break.
+ */
+static int
+cmp_on_fail(enum action_fail_response first, enum action_fail_response second)
+{
+ switch (first) {
+ case action_fail_demote:
+ switch (second) {
+ case action_fail_ignore:
+ return 1;
+ case action_fail_demote:
+ return 0;
+ default:
+ return -1;
+ }
+ break;
+
+ case action_fail_reset_remote:
+ switch (second) {
+ case action_fail_ignore:
+ case action_fail_demote:
+ case action_fail_recover:
+ return 1;
+ case action_fail_reset_remote:
+ return 0;
+ default:
+ return -1;
+ }
+ break;
+
+ case action_fail_restart_container:
+ switch (second) {
+ case action_fail_ignore:
+ case action_fail_demote:
+ case action_fail_recover:
+ case action_fail_reset_remote:
+ return 1;
+ case action_fail_restart_container:
+ return 0;
+ default:
+ return -1;
+ }
+ break;
+
+ default:
+ break;
+ }
+ switch (second) {
+ case action_fail_demote:
+ return (first == action_fail_ignore)? -1 : 1;
+
+ case action_fail_reset_remote:
+ switch (first) {
+ case action_fail_ignore:
+ case action_fail_demote:
+ case action_fail_recover:
+ return -1;
+ default:
+ return 1;
+ }
+ break;
+
+ case action_fail_restart_container:
+ switch (first) {
+ case action_fail_ignore:
+ case action_fail_demote:
+ case action_fail_recover:
+ case action_fail_reset_remote:
+ return -1;
+ default:
+ return 1;
+ }
+ break;
+
+ default:
+ break;
+ }
+ return first - second;
+}
+
static void
unpack_rsc_op_failure(pe_resource_t * rsc, pe_node_t * node, int rc, xmlNode * xml_op, xmlNode ** last_failure,
enum action_fail_response * on_fail, pe_working_set_t * data_set)
{
guint interval_ms = 0;
bool is_probe = false;
pe_action_t *action = NULL;
const char *key = get_op_key(xml_op);
const char *task = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
const char *exit_reason = crm_element_value(xml_op,
XML_LRM_ATTR_EXIT_REASON);
CRM_ASSERT(rsc);
CRM_CHECK(task != NULL, return);
*last_failure = xml_op;
crm_element_value_ms(xml_op, XML_LRM_ATTR_INTERVAL_MS, &interval_ms);
if ((interval_ms == 0) && !strcmp(task, CRMD_ACTION_STATUS)) {
is_probe = true;
}
if (exit_reason == NULL) {
exit_reason = "";
}
if (is_not_set(data_set->flags, pe_flag_symmetric_cluster)
&& (rc == PCMK_OCF_NOT_INSTALLED)) {
crm_trace("Unexpected result (%s%s%s) was recorded for "
"%s of %s on %s at %s " CRM_XS " rc=%d id=%s",
services_ocf_exitcode_str(rc),
(*exit_reason? ": " : ""), exit_reason,
(is_probe? "probe" : task), rsc->id, node->details->uname,
last_change_str(xml_op), rc, ID(xml_op));
} else {
crm_warn("Unexpected result (%s%s%s) was recorded for "
"%s of %s on %s at %s " CRM_XS " rc=%d id=%s",
services_ocf_exitcode_str(rc),
(*exit_reason? ": " : ""), exit_reason,
(is_probe? "probe" : task), rsc->id, node->details->uname,
last_change_str(xml_op), rc, ID(xml_op));
if (is_probe && (rc != PCMK_OCF_OK)
&& (rc != PCMK_OCF_NOT_RUNNING)
&& (rc != PCMK_OCF_RUNNING_MASTER)) {
/* A failed (not just unexpected) probe result could mean the user
* didn't know resources will be probed even where they can't run.
*/
crm_notice("If it is not possible for %s to run on %s, see "
"the resource-discovery option for location constraints",
rsc->id, node->details->uname);
}
record_failed_op(xml_op, node, rsc, data_set);
}
action = custom_action(rsc, strdup(key), task, NULL, TRUE, FALSE, data_set);
- if ((action->on_fail <= action_fail_fence && *on_fail < action->on_fail) ||
- (action->on_fail == action_fail_reset_remote && *on_fail <= action_fail_recover) ||
- (action->on_fail == action_fail_restart_container && *on_fail <= action_fail_recover) ||
- (*on_fail == action_fail_restart_container && action->on_fail >= action_fail_migrate)) {
+ if (cmp_on_fail(*on_fail, action->on_fail) < 0) {
pe_rsc_trace(rsc, "on-fail %s -> %s for %s (%s)", fail2text(*on_fail),
fail2text(action->on_fail), action->uuid, key);
*on_fail = action->on_fail;
}
if (!strcmp(task, CRMD_ACTION_STOP)) {
resource_location(rsc, node, -INFINITY, "__stop_fail__", data_set);
} else if (!strcmp(task, CRMD_ACTION_MIGRATE)) {
unpack_migrate_to_failure(rsc, node, xml_op, data_set);
} else if (!strcmp(task, CRMD_ACTION_MIGRATED)) {
unpack_migrate_from_failure(rsc, node, xml_op, data_set);
} else if (!strcmp(task, CRMD_ACTION_PROMOTE)) {
rsc->role = RSC_ROLE_MASTER;
} else if (!strcmp(task, CRMD_ACTION_DEMOTE)) {
if (action->on_fail == action_fail_block) {
rsc->role = RSC_ROLE_MASTER;
rsc->next_role = RSC_ROLE_STOPPED;
} else if(rc == PCMK_OCF_NOT_RUNNING) {
rsc->role = RSC_ROLE_STOPPED;
} else {
/* Staying in master role would put the scheduler and controller
* into a loop. Setting slave role is not dangerous because the
* resource will be stopped as part of recovery, and any master
* promotion will be ordered after that stop.
*/
rsc->role = RSC_ROLE_SLAVE;
}
}
if(is_probe && rc == PCMK_OCF_NOT_INSTALLED) {
/* leave stopped */
pe_rsc_trace(rsc, "Leaving %s stopped", rsc->id);
rsc->role = RSC_ROLE_STOPPED;
} else if (rsc->role < RSC_ROLE_STARTED) {
pe_rsc_trace(rsc, "Setting %s active", rsc->id);
set_active(rsc);
}
pe_rsc_trace(rsc, "Resource %s: role=%s, unclean=%s, on_fail=%s, fail_role=%s",
rsc->id, role2text(rsc->role),
node->details->unclean ? "true" : "false",
fail2text(action->on_fail), role2text(action->fail_role));
if (action->fail_role != RSC_ROLE_STARTED && rsc->next_role < action->fail_role) {
rsc->next_role = action->fail_role;
}
if (action->fail_role == RSC_ROLE_STOPPED) {
int score = -INFINITY;
pe_resource_t *fail_rsc = rsc;
if (fail_rsc->parent) {
pe_resource_t *parent = uber_parent(fail_rsc);
if (pe_rsc_is_clone(parent)
&& is_not_set(parent->flags, pe_rsc_unique)) {
/* For clone resources, if a child fails on an operation
* with on-fail = stop, all the resources fail. Do this by preventing
* the parent from coming up again. */
fail_rsc = parent;
}
}
crm_notice("%s will not be started under current conditions",
fail_rsc->id);
/* make sure it doesn't come up again */
if (fail_rsc->allowed_nodes != NULL) {
g_hash_table_destroy(fail_rsc->allowed_nodes);
}
fail_rsc->allowed_nodes = pe__node_list2table(data_set->nodes);
g_hash_table_foreach(fail_rsc->allowed_nodes, set_node_score, &score);
}
pe_free_action(action);
}
/*!
* \internal
* \brief Remap operation status based on action result
*
* Given an action result, determine an appropriate operation status for the
* purposes of responding to the action (the status provided by the executor is
* not directly usable since the executor does not know what was expected).
*
* \param[in,out] rsc Resource that operation history entry is for
* \param[in] rc Actual return code of operation
* \param[in] target_rc Expected return code of operation
* \param[in] node Node where operation was executed
* \param[in] xml_op Operation history entry XML from CIB status
* \param[in,out] on_fail What should be done about the result
* \param[in] data_set Current cluster working set
*
* \return Operation status based on return code and action info
* \note This may update the resource's current and next role.
*/
static int
determine_op_status(
pe_resource_t *rsc, int rc, int target_rc, pe_node_t * node, xmlNode * xml_op, enum action_fail_response * on_fail, pe_working_set_t * data_set)
{
guint interval_ms = 0;
bool is_probe = false;
int result = PCMK_LRM_OP_DONE;
const char *key = get_op_key(xml_op);
const char *task = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
const char *exit_reason = crm_element_value(xml_op,
XML_LRM_ATTR_EXIT_REASON);
CRM_ASSERT(rsc);
CRM_CHECK(task != NULL, return PCMK_LRM_OP_ERROR);
if (exit_reason == NULL) {
exit_reason = "";
}
crm_element_value_ms(xml_op, XML_LRM_ATTR_INTERVAL_MS, &interval_ms);
if ((interval_ms == 0) && !strcmp(task, CRMD_ACTION_STATUS)) {
is_probe = true;
task = "probe";
}
if (target_rc < 0) {
/* Pre-1.0 Pacemaker versions, and Pacemaker 1.1.6 or earlier with
* Heartbeat 2.0.7 or earlier as the cluster layer, did not include the
* target_rc in the transition key, which (along with the similar case
* of a corrupted transition key in the CIB) will be reported to this
* function as -1. Pacemaker 2.0+ does not support rolling upgrades from
* those versions or processing of saved CIB files from those versions,
* so we do not need to care much about this case.
*/
result = PCMK_LRM_OP_ERROR;
crm_warn("Expected result not found for %s on %s (corrupt or obsolete CIB?)",
key, node->details->uname);
} else if (target_rc != rc) {
result = PCMK_LRM_OP_ERROR;
pe_rsc_debug(rsc, "%s on %s: expected %d (%s), got %d (%s%s%s)",
key, node->details->uname,
target_rc, services_ocf_exitcode_str(target_rc),
rc, services_ocf_exitcode_str(rc),
(*exit_reason? ": " : ""), exit_reason);
}
switch (rc) {
case PCMK_OCF_OK:
if (is_probe && (target_rc == PCMK_OCF_NOT_RUNNING)) {
result = PCMK_LRM_OP_DONE;
pe_rsc_info(rsc, "Probe found %s active on %s at %s",
rsc->id, node->details->uname,
last_change_str(xml_op));
}
break;
case PCMK_OCF_NOT_RUNNING:
if (is_probe || target_rc == rc || is_not_set(rsc->flags, pe_rsc_managed)) {
result = PCMK_LRM_OP_DONE;
rsc->role = RSC_ROLE_STOPPED;
/* clear any previous failure actions */
*on_fail = action_fail_ignore;
rsc->next_role = RSC_ROLE_UNKNOWN;
}
break;
case PCMK_OCF_RUNNING_MASTER:
if (is_probe && (rc != target_rc)) {
result = PCMK_LRM_OP_DONE;
pe_rsc_info(rsc,
"Probe found %s active and promoted on %s at %s",
rsc->id, node->details->uname,
last_change_str(xml_op));
}
rsc->role = RSC_ROLE_MASTER;
break;
case PCMK_OCF_DEGRADED_MASTER:
case PCMK_OCF_FAILED_MASTER:
rsc->role = RSC_ROLE_MASTER;
result = PCMK_LRM_OP_ERROR;
break;
case PCMK_OCF_NOT_CONFIGURED:
result = PCMK_LRM_OP_ERROR_FATAL;
break;
case PCMK_OCF_UNIMPLEMENT_FEATURE:
if (interval_ms > 0) {
result = PCMK_LRM_OP_NOTSUPPORTED;
break;
}
// fall through
case PCMK_OCF_NOT_INSTALLED:
case PCMK_OCF_INVALID_PARAM:
case PCMK_OCF_INSUFFICIENT_PRIV:
if (!pe_can_fence(data_set, node)
&& !strcmp(task, CRMD_ACTION_STOP)) {
/* If a stop fails and we can't fence, there's nothing else we can do */
pe_proc_err("No further recovery can be attempted for %s "
"because %s on %s failed (%s%s%s) at %s "
CRM_XS " rc=%d id=%s", rsc->id, task,
node->details->uname, services_ocf_exitcode_str(rc),
(*exit_reason? ": " : ""), exit_reason,
last_change_str(xml_op), rc, ID(xml_op));
clear_bit(rsc->flags, pe_rsc_managed);
set_bit(rsc->flags, pe_rsc_block);
}
result = PCMK_LRM_OP_ERROR_HARD;
break;
default:
if (result == PCMK_LRM_OP_DONE) {
crm_info("Treating unknown exit status %d from %s of %s "
"on %s at %s as failure",
rc, task, rsc->id, node->details->uname,
last_change_str(xml_op));
result = PCMK_LRM_OP_ERROR;
}
break;
}
return result;
}
// return TRUE if start or monitor last failure but parameters changed
static bool
should_clear_for_param_change(xmlNode *xml_op, const char *task,
pe_resource_t *rsc, pe_node_t *node,
pe_working_set_t *data_set)
{
if (!strcmp(task, "start") || !strcmp(task, "monitor")) {
if (pe__bundle_needs_remote_name(rsc)) {
/* We haven't allocated resources yet, so we can't reliably
* substitute addr parameters for the REMOTE_CONTAINER_HACK.
* When that's needed, defer the check until later.
*/
pe__add_param_check(xml_op, rsc, node, pe_check_last_failure,
data_set);
} else {
op_digest_cache_t *digest_data = NULL;
digest_data = rsc_action_digest_cmp(rsc, xml_op, node, data_set);
switch (digest_data->rc) {
case RSC_DIGEST_UNKNOWN:
crm_trace("Resource %s history entry %s on %s"
" has no digest to compare",
rsc->id, get_op_key(xml_op), node->details->id);
break;
case RSC_DIGEST_MATCH:
break;
default:
return TRUE;
}
}
}
return FALSE;
}
// Order action after fencing of remote node, given connection rsc
static void
order_after_remote_fencing(pe_action_t *action, pe_resource_t *remote_conn,
pe_working_set_t *data_set)
{
pe_node_t *remote_node = pe_find_node(data_set->nodes, remote_conn->id);
if (remote_node) {
pe_action_t *fence = pe_fence_op(remote_node, NULL, TRUE, NULL,
FALSE, data_set);
order_actions(fence, action, pe_order_implies_then);
}
}
static bool
should_ignore_failure_timeout(pe_resource_t *rsc, xmlNode *xml_op,
const char *task, guint interval_ms,
bool is_last_failure, pe_working_set_t *data_set)
{
/* Clearing failures of recurring monitors has special concerns. The
* executor reports only changes in the monitor result, so if the
* monitor is still active and still getting the same failure result,
* that will go undetected after the failure is cleared.
*
* Also, the operation history will have the time when the recurring
* monitor result changed to the given code, not the time when the
* result last happened.
*
* @TODO We probably should clear such failures only when the failure
* timeout has passed since the last occurrence of the failed result.
* However we don't record that information. We could maybe approximate
* that by clearing only if there is a more recent successful monitor or
* stop result, but we don't even have that information at this point
* since we are still unpacking the resource's operation history.
*
* This is especially important for remote connection resources with a
* reconnect interval, so in that case, we skip clearing failures
* if the remote node hasn't been fenced.
*/
if (rsc->remote_reconnect_ms
&& is_set(data_set->flags, pe_flag_stonith_enabled)
&& (interval_ms != 0) && safe_str_eq(task, CRMD_ACTION_STATUS)) {
pe_node_t *remote_node = pe_find_node(data_set->nodes, rsc->id);
if (remote_node && !remote_node->details->remote_was_fenced) {
if (is_last_failure) {
crm_info("Waiting to clear monitor failure for remote node %s"
" until fencing has occurred", rsc->id);
}
return TRUE;
}
}
return FALSE;
}
/*!
* \internal
* \brief Check operation age and schedule failure clearing when appropriate
*
* This function has two distinct purposes. The first is to check whether an
* operation history entry is expired (i.e. the resource has a failure timeout,
* the entry is older than the timeout, and the resource either has no fail
* count or its fail count is entirely older than the timeout). The second is to
* schedule fail count clearing when appropriate (i.e. the operation is expired
* and either the resource has an expired fail count or the operation is a
* last_failure for a remote connection resource with a reconnect interval,
* or the operation is a last_failure for a start or monitor operation and the
* resource's parameters have changed since the operation).
*
* \param[in] rsc Resource that operation happened to
* \param[in] node Node that operation happened on
* \param[in] rc Actual result of operation
* \param[in] xml_op Operation history entry XML
* \param[in] data_set Current working set
*
* \return TRUE if operation history entry is expired, FALSE otherwise
*/
static bool
check_operation_expiry(pe_resource_t *rsc, pe_node_t *node, int rc,
xmlNode *xml_op, pe_working_set_t *data_set)
{
bool expired = FALSE;
bool is_last_failure = pcmk__ends_with(ID(xml_op), "_last_failure_0");
time_t last_run = 0;
guint interval_ms = 0;
int unexpired_fail_count = 0;
const char *task = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
const char *clear_reason = NULL;
crm_element_value_ms(xml_op, XML_LRM_ATTR_INTERVAL_MS, &interval_ms);
if ((rsc->failure_timeout > 0)
&& (crm_element_value_epoch(xml_op, XML_RSC_OP_LAST_CHANGE,
&last_run) == 0)) {
// Resource has a failure-timeout, and history entry has a timestamp
time_t now = get_effective_time(data_set);
time_t last_failure = 0;
// Is this particular operation history older than the failure timeout?
if ((now >= (last_run + rsc->failure_timeout))
&& !should_ignore_failure_timeout(rsc, xml_op, task, interval_ms,
is_last_failure, data_set)) {
expired = TRUE;
}
// Does the resource as a whole have an unexpired fail count?
unexpired_fail_count = pe_get_failcount(node, rsc, &last_failure,
pe_fc_effective, xml_op,
data_set);
// Update scheduler recheck time according to *last* failure
crm_trace("%s@%lld is %sexpired @%lld with unexpired_failures=%d timeout=%ds"
" last-failure@%lld",
ID(xml_op), (long long) last_run, (expired? "" : "not "),
(long long) now, unexpired_fail_count, rsc->failure_timeout,
(long long) last_failure);
last_failure += rsc->failure_timeout + 1;
if (unexpired_fail_count && (now < last_failure)) {
pe__update_recheck_time(last_failure, data_set);
}
}
if (expired) {
if (pe_get_failcount(node, rsc, NULL, pe_fc_default, xml_op, data_set)) {
// There is a fail count ignoring timeout
if (unexpired_fail_count == 0) {
// There is no fail count considering timeout
clear_reason = "it expired";
} else {
/* This operation is old, but there is an unexpired fail count.
* In a properly functioning cluster, this should only be
* possible if this operation is not a failure (otherwise the
* fail count should be expired too), so this is really just a
* failsafe.
*/
expired = FALSE;
}
} else if (is_last_failure && rsc->remote_reconnect_ms) {
/* Clear any expired last failure when reconnect interval is set,
* even if there is no fail count.
*/
clear_reason = "reconnect interval is set";
}
}
if (!expired && is_last_failure
&& should_clear_for_param_change(xml_op, task, rsc, node, data_set)) {
clear_reason = "resource parameters have changed";
}
if (clear_reason != NULL) {
// Schedule clearing of the fail count
pe_action_t *clear_op = pe__clear_failcount(rsc, node, clear_reason,
data_set);
if (is_set(data_set->flags, pe_flag_stonith_enabled)
&& rsc->remote_reconnect_ms) {
/* If we're clearing a remote connection due to a reconnect
* interval, we want to wait until any scheduled fencing
* completes.
*
* We could limit this to remote_node->details->unclean, but at
* this point, that's always true (it won't be reliable until
* after unpack_node_loop() is done).
*/
crm_info("Clearing %s failure will wait until any scheduled "
"fencing of %s completes", task, rsc->id);
order_after_remote_fencing(clear_op, rsc, data_set);
}
}
if (expired && (interval_ms == 0) && safe_str_eq(task, CRMD_ACTION_STATUS)) {
switch(rc) {
case PCMK_OCF_OK:
case PCMK_OCF_NOT_RUNNING:
case PCMK_OCF_RUNNING_MASTER:
case PCMK_OCF_DEGRADED:
case PCMK_OCF_DEGRADED_MASTER:
// Don't expire probes that return these values
expired = FALSE;
break;
}
}
return expired;
}
int pe__target_rc_from_xml(xmlNode *xml_op)
{
int target_rc = 0;
const char *key = crm_element_value(xml_op, XML_ATTR_TRANSITION_KEY);
if (key == NULL) {
return -1;
}
decode_transition_key(key, NULL, NULL, NULL, &target_rc);
return target_rc;
}
static enum action_fail_response
get_action_on_fail(pe_resource_t *rsc, const char *key, const char *task, pe_working_set_t * data_set)
{
- int result = action_fail_recover;
+ enum action_fail_response result = action_fail_recover;
pe_action_t *action = custom_action(rsc, strdup(key), task, NULL, TRUE, FALSE, data_set);
result = action->on_fail;
pe_free_action(action);
return result;
}
static void
update_resource_state(pe_resource_t * rsc, pe_node_t * node, xmlNode * xml_op, const char * task, int rc,
xmlNode * last_failure, enum action_fail_response * on_fail, pe_working_set_t * data_set)
{
gboolean clear_past_failure = FALSE;
CRM_ASSERT(rsc);
CRM_ASSERT(xml_op);
if (rc == PCMK_OCF_NOT_RUNNING) {
clear_past_failure = TRUE;
} else if (rc == PCMK_OCF_NOT_INSTALLED) {
rsc->role = RSC_ROLE_STOPPED;
} else if (safe_str_eq(task, CRMD_ACTION_STATUS)) {
if (last_failure) {
const char *op_key = get_op_key(xml_op);
const char *last_failure_key = get_op_key(last_failure);
if (safe_str_eq(op_key, last_failure_key)) {
clear_past_failure = TRUE;
}
}
if (rsc->role < RSC_ROLE_STARTED) {
set_active(rsc);
}
} else if (safe_str_eq(task, CRMD_ACTION_START)) {
rsc->role = RSC_ROLE_STARTED;
clear_past_failure = TRUE;
} else if (safe_str_eq(task, CRMD_ACTION_STOP)) {
rsc->role = RSC_ROLE_STOPPED;
clear_past_failure = TRUE;
} else if (safe_str_eq(task, CRMD_ACTION_PROMOTE)) {
rsc->role = RSC_ROLE_MASTER;
clear_past_failure = TRUE;
} else if (safe_str_eq(task, CRMD_ACTION_DEMOTE)) {
- /* Demote from Master does not clear an error */
+
+ if (*on_fail == action_fail_demote) {
+ // Demote clears an error only if on-fail=demote
+ clear_past_failure = TRUE;
+ }
rsc->role = RSC_ROLE_SLAVE;
} else if (safe_str_eq(task, CRMD_ACTION_MIGRATED)) {
rsc->role = RSC_ROLE_STARTED;
clear_past_failure = TRUE;
} else if (safe_str_eq(task, CRMD_ACTION_MIGRATE)) {
unpack_migrate_to_success(rsc, node, xml_op, data_set);
} else if (rsc->role < RSC_ROLE_STARTED) {
pe_rsc_trace(rsc, "%s active on %s", rsc->id, node->details->uname);
set_active(rsc);
}
/* clear any previous failure actions */
if (clear_past_failure) {
switch (*on_fail) {
case action_fail_stop:
case action_fail_fence:
case action_fail_migrate:
case action_fail_standby:
pe_rsc_trace(rsc, "%s.%s is not cleared by a completed stop",
rsc->id, fail2text(*on_fail));
break;
case action_fail_block:
case action_fail_ignore:
+ case action_fail_demote:
case action_fail_recover:
case action_fail_restart_container:
*on_fail = action_fail_ignore;
rsc->next_role = RSC_ROLE_UNKNOWN;
break;
case action_fail_reset_remote:
if (rsc->remote_reconnect_ms == 0) {
/* With no reconnect interval, the connection is allowed to
* start again after the remote node is fenced and
* completely stopped. (With a reconnect interval, we wait
* for the failure to be cleared entirely before attempting
* to reconnect.)
*/
*on_fail = action_fail_ignore;
rsc->next_role = RSC_ROLE_UNKNOWN;
}
break;
}
}
}
/*!
* \internal
* \brief Remap informational monitor results to usual values
*
* Certain OCF result codes are for providing extended information to the
* user about services that aren't yet failed but not entirely healthy either.
* These must be treated as the "normal" result by pacemaker.
*
* \param[in] rc Actual result of a monitor action
* \param[in] xml_op Operation history XML
* \param[in] node Node that operation happened on
* \param[in] rsc Resource that operation happened to
* \param[in] data_set Cluster working set
*
* \return Result code that pacemaker should use
*
* \note If the result is remapped, and the node is not shutting down or failed,
* the operation will be recorded in the data set's list of failed
* operations, to highlight it for the user.
*/
static int
remap_monitor_rc(int rc, xmlNode *xml_op, const pe_node_t *node,
const pe_resource_t *rsc, pe_working_set_t *data_set)
{
int remapped_rc = rc;
switch (rc) {
case PCMK_OCF_DEGRADED:
remapped_rc = PCMK_OCF_OK;
break;
case PCMK_OCF_DEGRADED_MASTER:
remapped_rc = PCMK_OCF_RUNNING_MASTER;
break;
default:
break;
}
if (rc != remapped_rc) {
crm_trace("Remapping monitor result %d to %d", rc, remapped_rc);
if (!node->details->shutdown || node->details->online) {
record_failed_op(xml_op, node, rsc, data_set);
}
}
return remapped_rc;
}
static void
unpack_rsc_op(pe_resource_t *rsc, pe_node_t *node, xmlNode *xml_op,
xmlNode **last_failure, enum action_fail_response *on_fail,
pe_working_set_t *data_set)
{
int rc = 0;
int task_id = 0;
int target_rc = 0;
int status = PCMK_LRM_OP_UNKNOWN;
guint interval_ms = 0;
const char *task = NULL;
const char *task_key = NULL;
const char *exit_reason = NULL;
bool expired = FALSE;
pe_resource_t *parent = rsc;
enum action_fail_response failure_strategy = action_fail_recover;
CRM_CHECK(rsc && node && xml_op, return);
target_rc = pe__target_rc_from_xml(xml_op);
task_key = get_op_key(xml_op);
task = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
exit_reason = crm_element_value(xml_op, XML_LRM_ATTR_EXIT_REASON);
if (exit_reason == NULL) {
exit_reason = "";
}
crm_element_value_int(xml_op, XML_LRM_ATTR_RC, &rc);
crm_element_value_int(xml_op, XML_LRM_ATTR_CALLID, &task_id);
crm_element_value_int(xml_op, XML_LRM_ATTR_OPSTATUS, &status);
crm_element_value_ms(xml_op, XML_LRM_ATTR_INTERVAL_MS, &interval_ms);
CRM_CHECK(task != NULL, return);
CRM_CHECK(status <= PCMK_LRM_OP_INVALID, return);
CRM_CHECK(status >= PCMK_LRM_OP_PENDING, return);
if (!strcmp(task, CRMD_ACTION_NOTIFY) ||
!strcmp(task, CRMD_ACTION_METADATA)) {
/* safe to ignore these */
return;
}
if (is_not_set(rsc->flags, pe_rsc_unique)) {
parent = uber_parent(rsc);
}
pe_rsc_trace(rsc, "Unpacking task %s/%s (call_id=%d, status=%d, rc=%d) on %s (role=%s)",
task_key, task, task_id, status, rc, node->details->uname, role2text(rsc->role));
if (node->details->unclean) {
pe_rsc_trace(rsc, "Node %s (where %s is running) is unclean."
" Further action depends on the value of the stop's on-fail attribute",
node->details->uname, rsc->id);
}
/* It should be possible to call remap_monitor_rc() first then call
* check_operation_expiry() only if rc != target_rc, because there should
* never be a fail count without at least one unexpected result in the
* resource history. That would be more efficient by avoiding having to call
* check_operation_expiry() for expected results.
*
* However, we do have such configurations in the scheduler regression
* tests, even if it shouldn't be possible with the current code. It's
* probably a good idea anyway, but that would require updating the test
* inputs to something currently possible.
*/
if ((status != PCMK_LRM_OP_NOT_INSTALLED)
&& check_operation_expiry(rsc, node, rc, xml_op, data_set)) {
expired = TRUE;
}
if (!strcmp(task, CRMD_ACTION_STATUS)) {
rc = remap_monitor_rc(rc, xml_op, node, rsc, data_set);
}
if (expired && (rc != target_rc)) {
const char *magic = crm_element_value(xml_op, XML_ATTR_TRANSITION_MAGIC);
if (interval_ms == 0) {
crm_notice("Ignoring expired %s failure on %s "
CRM_XS " actual=%d expected=%d magic=%s",
task_key, node->details->uname, rc, target_rc, magic);
goto done;
} else if(node->details->online && node->details->unclean == FALSE) {
/* Reschedule the recurring monitor. CancelXmlOp() won't work at
* this stage, so as a hacky workaround, forcibly change the restart
* digest so check_action_definition() does what we want later.
*
* @TODO We should skip this if there is a newer successful monitor.
* Also, this causes rescheduling only if the history entry
* has an op-digest (which the expire-non-blocked-failure
* scheduler regression test doesn't, but that may not be a
* realistic scenario in production).
*/
crm_notice("Rescheduling %s after failure expired on %s "
CRM_XS " actual=%d expected=%d magic=%s",
task_key, node->details->uname, rc, target_rc, magic);
crm_xml_add(xml_op, XML_LRM_ATTR_RESTART_DIGEST, "calculated-failure-timeout");
goto done;
}
}
/* If the executor reported an operation status of anything but done or
* error, consider that final. But for done or error, we know better whether
* it should be treated as a failure or not, because we know the expected
* result.
*/
if(status == PCMK_LRM_OP_DONE || status == PCMK_LRM_OP_ERROR) {
status = determine_op_status(rsc, rc, target_rc, node, xml_op, on_fail, data_set);
pe_rsc_trace(rsc, "Remapped %s status to %d", task_key, status);
}
switch (status) {
case PCMK_LRM_OP_CANCELLED:
// Should never happen
pe_err("Resource history contains cancellation '%s' "
"(%s of %s on %s at %s)",
ID(xml_op), task, rsc->id, node->details->uname,
last_change_str(xml_op));
break;
case PCMK_LRM_OP_PENDING:
if (!strcmp(task, CRMD_ACTION_START)) {
set_bit(rsc->flags, pe_rsc_start_pending);
set_active(rsc);
} else if (!strcmp(task, CRMD_ACTION_PROMOTE)) {
rsc->role = RSC_ROLE_MASTER;
} else if (!strcmp(task, CRMD_ACTION_MIGRATE) && node->details->unclean) {
/* If a pending migrate_to action is out on a unclean node,
* we have to force the stop action on the target. */
const char *migrate_target = crm_element_value(xml_op, XML_LRM_ATTR_MIGRATE_TARGET);
pe_node_t *target = pe_find_node(data_set->nodes, migrate_target);
if (target) {
stop_action(rsc, target, FALSE);
}
}
if (rsc->pending_task == NULL) {
if ((interval_ms != 0) || strcmp(task, CRMD_ACTION_STATUS)) {
rsc->pending_task = strdup(task);
rsc->pending_node = node;
} else {
/* Pending probes are not printed, even if pending
* operations are requested. If someone ever requests that
* behavior, enable the below and the corresponding part of
* native.c:native_pending_task().
*/
#if 0
rsc->pending_task = strdup("probe");
rsc->pending_node = node;
#endif
}
}
break;
case PCMK_LRM_OP_DONE:
pe_rsc_trace(rsc, "%s of %s on %s completed at %s " CRM_XS " id=%s",
task, rsc->id, node->details->uname,
last_change_str(xml_op), ID(xml_op));
update_resource_state(rsc, node, xml_op, task, rc, *last_failure, on_fail, data_set);
break;
case PCMK_LRM_OP_NOT_INSTALLED:
failure_strategy = get_action_on_fail(rsc, task_key, task, data_set);
if (failure_strategy == action_fail_ignore) {
crm_warn("Cannot ignore failed %s of %s on %s: "
"Resource agent doesn't exist "
CRM_XS " status=%d rc=%d id=%s",
task, rsc->id, node->details->uname, status, rc,
ID(xml_op));
/* Also for printing it as "FAILED" by marking it as pe_rsc_failed later */
*on_fail = action_fail_migrate;
}
resource_location(parent, node, -INFINITY, "hard-error", data_set);
unpack_rsc_op_failure(rsc, node, rc, xml_op, last_failure, on_fail, data_set);
break;
case PCMK_LRM_OP_NOT_CONNECTED:
if (pe__is_guest_or_remote_node(node)
&& is_set(node->details->remote_rsc->flags, pe_rsc_managed)) {
/* We should never get into a situation where a managed remote
* connection resource is considered OK but a resource action
* behind the connection gets a "not connected" status. But as a
* fail-safe in case a bug or unusual circumstances do lead to
* that, ensure the remote connection is considered failed.
*/
set_bit(node->details->remote_rsc->flags, pe_rsc_failed);
+ set_bit(node->details->remote_rsc->flags, pe_rsc_stop);
}
// fall through
case PCMK_LRM_OP_ERROR:
case PCMK_LRM_OP_ERROR_HARD:
case PCMK_LRM_OP_ERROR_FATAL:
case PCMK_LRM_OP_TIMEOUT:
case PCMK_LRM_OP_NOTSUPPORTED:
case PCMK_LRM_OP_INVALID:
failure_strategy = get_action_on_fail(rsc, task_key, task, data_set);
if ((failure_strategy == action_fail_ignore)
|| (failure_strategy == action_fail_restart_container
&& !strcmp(task, CRMD_ACTION_STOP))) {
crm_warn("Pretending failed %s (%s%s%s) of %s on %s at %s "
"succeeded " CRM_XS " rc=%d id=%s",
task, services_ocf_exitcode_str(rc),
(*exit_reason? ": " : ""), exit_reason, rsc->id,
node->details->uname, last_change_str(xml_op), rc,
ID(xml_op));
update_resource_state(rsc, node, xml_op, task, target_rc, *last_failure, on_fail, data_set);
crm_xml_add(xml_op, XML_ATTR_UNAME, node->details->uname);
set_bit(rsc->flags, pe_rsc_failure_ignored);
record_failed_op(xml_op, node, rsc, data_set);
- if (failure_strategy == action_fail_restart_container && *on_fail <= action_fail_recover) {
+ if ((failure_strategy == action_fail_restart_container)
+ && cmp_on_fail(*on_fail, action_fail_recover) <= 0) {
*on_fail = failure_strategy;
}
} else {
unpack_rsc_op_failure(rsc, node, rc, xml_op, last_failure, on_fail, data_set);
if(status == PCMK_LRM_OP_ERROR_HARD) {
do_crm_log(rc != PCMK_OCF_NOT_INSTALLED?LOG_ERR:LOG_NOTICE,
"Preventing %s from restarting on %s because "
"of hard failure (%s%s%s)" CRM_XS " rc=%d id=%s",
parent->id, node->details->uname,
services_ocf_exitcode_str(rc),
(*exit_reason? ": " : ""), exit_reason,
rc, ID(xml_op));
resource_location(parent, node, -INFINITY, "hard-error", data_set);
} else if(status == PCMK_LRM_OP_ERROR_FATAL) {
crm_err("Preventing %s from restarting anywhere because "
"of fatal failure (%s%s%s) " CRM_XS " rc=%d id=%s",
parent->id, services_ocf_exitcode_str(rc),
(*exit_reason? ": " : ""), exit_reason,
rc, ID(xml_op));
resource_location(parent, NULL, -INFINITY, "fatal-error", data_set);
}
}
break;
}
done:
pe_rsc_trace(rsc, "Resource %s after %s: role=%s, next=%s",
rsc->id, task, role2text(rsc->role),
role2text(rsc->next_role));
}
static void
add_node_attrs(xmlNode *xml_obj, pe_node_t *node, bool overwrite,
pe_working_set_t *data_set)
{
const char *cluster_name = NULL;
pe_rule_eval_data_t rule_data = {
.node_hash = NULL,
.role = RSC_ROLE_UNKNOWN,
.now = data_set->now,
.match_data = NULL,
.rsc_data = NULL,
.op_data = NULL
};
g_hash_table_insert(node->details->attrs,
strdup(CRM_ATTR_UNAME), strdup(node->details->uname));
g_hash_table_insert(node->details->attrs, strdup(CRM_ATTR_ID),
strdup(node->details->id));
if (safe_str_eq(node->details->id, data_set->dc_uuid)) {
data_set->dc_node = node;
node->details->is_dc = TRUE;
g_hash_table_insert(node->details->attrs,
strdup(CRM_ATTR_IS_DC), strdup(XML_BOOLEAN_TRUE));
} else {
g_hash_table_insert(node->details->attrs,
strdup(CRM_ATTR_IS_DC), strdup(XML_BOOLEAN_FALSE));
}
cluster_name = g_hash_table_lookup(data_set->config_hash, "cluster-name");
if (cluster_name) {
g_hash_table_insert(node->details->attrs, strdup(CRM_ATTR_CLUSTER_NAME),
strdup(cluster_name));
}
pe__unpack_dataset_nvpairs(xml_obj, XML_TAG_ATTR_SETS, &rule_data,
node->details->attrs, NULL, overwrite, data_set);
if (pe_node_attribute_raw(node, CRM_ATTR_SITE_NAME) == NULL) {
const char *site_name = pe_node_attribute_raw(node, "site-name");
if (site_name) {
g_hash_table_insert(node->details->attrs,
strdup(CRM_ATTR_SITE_NAME),
strdup(site_name));
} else if (cluster_name) {
/* Default to cluster-name if unset */
g_hash_table_insert(node->details->attrs,
strdup(CRM_ATTR_SITE_NAME),
strdup(cluster_name));
}
}
}
static GListPtr
extract_operations(const char *node, const char *rsc, xmlNode * rsc_entry, gboolean active_filter)
{
int counter = -1;
int stop_index = -1;
int start_index = -1;
xmlNode *rsc_op = NULL;
GListPtr gIter = NULL;
GListPtr op_list = NULL;
GListPtr sorted_op_list = NULL;
/* extract operations */
op_list = NULL;
sorted_op_list = NULL;
for (rsc_op = __xml_first_child_element(rsc_entry);
rsc_op != NULL; rsc_op = __xml_next_element(rsc_op)) {
if (crm_str_eq((const char *)rsc_op->name, XML_LRM_TAG_RSC_OP, TRUE)) {
crm_xml_add(rsc_op, "resource", rsc);
crm_xml_add(rsc_op, XML_ATTR_UNAME, node);
op_list = g_list_prepend(op_list, rsc_op);
}
}
if (op_list == NULL) {
/* if there are no operations, there is nothing to do */
return NULL;
}
sorted_op_list = g_list_sort(op_list, sort_op_by_callid);
/* create active recurring operations as optional */
if (active_filter == FALSE) {
return sorted_op_list;
}
op_list = NULL;
calculate_active_ops(sorted_op_list, &start_index, &stop_index);
for (gIter = sorted_op_list; gIter != NULL; gIter = gIter->next) {
xmlNode *rsc_op = (xmlNode *) gIter->data;
counter++;
if (start_index < stop_index) {
crm_trace("Skipping %s: not active", ID(rsc_entry));
break;
} else if (counter < start_index) {
crm_trace("Skipping %s: old", ID(rsc_op));
continue;
}
op_list = g_list_append(op_list, rsc_op);
}
g_list_free(sorted_op_list);
return op_list;
}
GListPtr
find_operations(const char *rsc, const char *node, gboolean active_filter,
pe_working_set_t * data_set)
{
GListPtr output = NULL;
GListPtr intermediate = NULL;
xmlNode *tmp = NULL;
xmlNode *status = find_xml_node(data_set->input, XML_CIB_TAG_STATUS, TRUE);
pe_node_t *this_node = NULL;
xmlNode *node_state = NULL;
for (node_state = __xml_first_child_element(status); node_state != NULL;
node_state = __xml_next_element(node_state)) {
if (crm_str_eq((const char *)node_state->name, XML_CIB_TAG_STATE, TRUE)) {
const char *uname = crm_element_value(node_state, XML_ATTR_UNAME);
if (node != NULL && safe_str_neq(uname, node)) {
continue;
}
this_node = pe_find_node(data_set->nodes, uname);
if(this_node == NULL) {
CRM_LOG_ASSERT(this_node != NULL);
continue;
} else if (pe__is_guest_or_remote_node(this_node)) {
determine_remote_online_status(data_set, this_node);
} else {
determine_online_status(node_state, this_node, data_set);
}
if (this_node->details->online || is_set(data_set->flags, pe_flag_stonith_enabled)) {
/* offline nodes run no resources...
* unless stonith is enabled in which case we need to
* make sure rsc start events happen after the stonith
*/
xmlNode *lrm_rsc = NULL;
tmp = find_xml_node(node_state, XML_CIB_TAG_LRM, FALSE);
tmp = find_xml_node(tmp, XML_LRM_TAG_RESOURCES, FALSE);
for (lrm_rsc = __xml_first_child_element(tmp); lrm_rsc != NULL;
lrm_rsc = __xml_next_element(lrm_rsc)) {
if (crm_str_eq((const char *)lrm_rsc->name, XML_LRM_TAG_RESOURCE, TRUE)) {
const char *rsc_id = crm_element_value(lrm_rsc, XML_ATTR_ID);
if (rsc != NULL && safe_str_neq(rsc_id, rsc)) {
continue;
}
intermediate = extract_operations(uname, rsc_id, lrm_rsc, active_filter);
output = g_list_concat(output, intermediate);
}
}
}
}
}
return output;
}
diff --git a/lib/pengine/utils.c b/lib/pengine/utils.c
index 20a8db5cbd..f8b631aa62 100644
--- a/lib/pengine/utils.c
+++ b/lib/pengine/utils.c
@@ -1,2800 +1,2858 @@
/*
* Copyright 2004-2020 the Pacemaker project contributors
*
* The version control history for this file may have further details.
*
* This source code is licensed under the GNU Lesser General Public License
* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
*/
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
extern xmlNode *get_object_root(const char *object_type, xmlNode * the_root);
void print_str_str(gpointer key, gpointer value, gpointer user_data);
gboolean ghash_free_str_str(gpointer key, gpointer value, gpointer user_data);
static void unpack_operation(pe_action_t * action, xmlNode * xml_obj, pe_resource_t * container,
pe_working_set_t * data_set, guint interval_ms);
static xmlNode *find_rsc_op_entry_helper(pe_resource_t * rsc, const char *key,
gboolean include_disabled);
#if ENABLE_VERSIONED_ATTRS
pe_rsc_action_details_t *
pe_rsc_action_details(pe_action_t *action)
{
pe_rsc_action_details_t *details;
CRM_CHECK(action != NULL, return NULL);
if (action->action_details == NULL) {
action->action_details = calloc(1, sizeof(pe_rsc_action_details_t));
CRM_CHECK(action->action_details != NULL, return NULL);
}
details = (pe_rsc_action_details_t *) action->action_details;
if (details->versioned_parameters == NULL) {
details->versioned_parameters = create_xml_node(NULL,
XML_TAG_OP_VER_ATTRS);
}
if (details->versioned_meta == NULL) {
details->versioned_meta = create_xml_node(NULL, XML_TAG_OP_VER_META);
}
return details;
}
static void
pe_free_rsc_action_details(pe_action_t *action)
{
pe_rsc_action_details_t *details;
if ((action == NULL) || (action->action_details == NULL)) {
return;
}
details = (pe_rsc_action_details_t *) action->action_details;
if (details->versioned_parameters) {
free_xml(details->versioned_parameters);
}
if (details->versioned_meta) {
free_xml(details->versioned_meta);
}
action->action_details = NULL;
}
#endif
/*!
* \internal
* \brief Check whether we can fence a particular node
*
* \param[in] data_set Working set for cluster
* \param[in] node Name of node to check
*
* \return true if node can be fenced, false otherwise
*/
bool
pe_can_fence(pe_working_set_t *data_set, pe_node_t *node)
{
if (pe__is_guest_node(node)) {
/* Guest nodes are fenced by stopping their container resource. We can
* do that if the container's host is either online or fenceable.
*/
pe_resource_t *rsc = node->details->remote_rsc->container;
for (GList *n = rsc->running_on; n != NULL; n = n->next) {
pe_node_t *container_node = n->data;
if (!container_node->details->online
&& !pe_can_fence(data_set, container_node)) {
return false;
}
}
return true;
} else if(is_not_set(data_set->flags, pe_flag_stonith_enabled)) {
return false; /* Turned off */
} else if (is_not_set(data_set->flags, pe_flag_have_stonith_resource)) {
return false; /* No devices */
} else if (is_set(data_set->flags, pe_flag_have_quorum)) {
return true;
} else if (data_set->no_quorum_policy == no_quorum_ignore) {
return true;
} else if(node == NULL) {
return false;
} else if(node->details->online) {
crm_notice("We can fence %s without quorum because they're in our membership", node->details->uname);
return true;
}
crm_trace("Cannot fence %s", node->details->uname);
return false;
}
/*!
* \internal
* \brief Copy a node object
*
* \param[in] this_node Node object to copy
*
* \return Newly allocated shallow copy of this_node
* \note This function asserts on errors and is guaranteed to return non-NULL.
*/
pe_node_t *
pe__copy_node(const pe_node_t *this_node)
{
pe_node_t *new_node = NULL;
CRM_ASSERT(this_node != NULL);
new_node = calloc(1, sizeof(pe_node_t));
CRM_ASSERT(new_node != NULL);
new_node->rsc_discover_mode = this_node->rsc_discover_mode;
new_node->weight = this_node->weight;
new_node->fixed = this_node->fixed;
new_node->details = this_node->details;
return new_node;
}
/* any node in list1 or list2 and not in the other gets a score of -INFINITY */
void
node_list_exclude(GHashTable * hash, GListPtr list, gboolean merge_scores)
{
GHashTable *result = hash;
pe_node_t *other_node = NULL;
GListPtr gIter = list;
GHashTableIter iter;
pe_node_t *node = NULL;
g_hash_table_iter_init(&iter, hash);
while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
other_node = pe_find_node_id(list, node->details->id);
if (other_node == NULL) {
node->weight = -INFINITY;
} else if (merge_scores) {
node->weight = pe__add_scores(node->weight, other_node->weight);
}
}
for (; gIter != NULL; gIter = gIter->next) {
pe_node_t *node = (pe_node_t *) gIter->data;
other_node = pe_hash_table_lookup(result, node->details->id);
if (other_node == NULL) {
pe_node_t *new_node = pe__copy_node(node);
new_node->weight = -INFINITY;
g_hash_table_insert(result, (gpointer) new_node->details->id, new_node);
}
}
}
/*!
* \internal
* \brief Create a node hash table from a node list
*
* \param[in] list Node list
*
* \return Hash table equivalent of node list
*/
GHashTable *
pe__node_list2table(GList *list)
{
GHashTable *result = NULL;
result = g_hash_table_new_full(crm_str_hash, g_str_equal, NULL, free);
for (GList *gIter = list; gIter != NULL; gIter = gIter->next) {
pe_node_t *new_node = pe__copy_node((pe_node_t *) gIter->data);
g_hash_table_insert(result, (gpointer) new_node->details->id, new_node);
}
return result;
}
gint
sort_node_uname(gconstpointer a, gconstpointer b)
{
const char *name_a = ((const pe_node_t *) a)->details->uname;
const char *name_b = ((const pe_node_t *) b)->details->uname;
while (*name_a && *name_b) {
if (isdigit(*name_a) && isdigit(*name_b)) {
// If node names contain a number, sort numerically
char *end_a = NULL;
char *end_b = NULL;
long num_a = strtol(name_a, &end_a, 10);
long num_b = strtol(name_b, &end_b, 10);
// allow ordering e.g. 007 > 7
size_t len_a = end_a - name_a;
size_t len_b = end_b - name_b;
if (num_a < num_b) {
return -1;
} else if (num_a > num_b) {
return 1;
} else if (len_a < len_b) {
return -1;
} else if (len_a > len_b) {
return 1;
}
name_a = end_a;
name_b = end_b;
} else {
// Compare non-digits case-insensitively
int lower_a = tolower(*name_a);
int lower_b = tolower(*name_b);
if (lower_a < lower_b) {
return -1;
} else if (lower_a > lower_b) {
return 1;
}
++name_a;
++name_b;
}
}
if (!*name_a && *name_b) {
return -1;
} else if (*name_a && !*name_b) {
return 1;
}
return 0;
}
/*!
* \internal
* \brief Output node weights to stdout
*
* \param[in] rsc Use allowed nodes for this resource
* \param[in] comment Text description to prefix lines with
* \param[in] nodes If rsc is not specified, use these nodes
*/
static void
pe__output_node_weights(pe_resource_t *rsc, const char *comment,
GHashTable *nodes)
{
char score[128]; // Stack-allocated since this is called frequently
// Sort the nodes so the output is consistent for regression tests
GList *list = g_list_sort(g_hash_table_get_values(nodes), sort_node_uname);
for (GList *gIter = list; gIter != NULL; gIter = gIter->next) {
pe_node_t *node = (pe_node_t *) gIter->data;
score2char_stack(node->weight, score, sizeof(score));
if (rsc) {
printf("%s: %s allocation score on %s: %s\n",
comment, rsc->id, node->details->uname, score);
} else {
printf("%s: %s = %s\n", comment, node->details->uname, score);
}
}
g_list_free(list);
}
/*!
* \internal
* \brief Log node weights at trace level
*
* \param[in] file Caller's filename
* \param[in] function Caller's function name
* \param[in] line Caller's line number
* \param[in] rsc Use allowed nodes for this resource
* \param[in] comment Text description to prefix lines with
* \param[in] nodes If rsc is not specified, use these nodes
*/
static void
pe__log_node_weights(const char *file, const char *function, int line,
pe_resource_t *rsc, const char *comment, GHashTable *nodes)
{
GHashTableIter iter;
pe_node_t *node = NULL;
char score[128]; // Stack-allocated since this is called frequently
// Don't waste time if we're not tracing at this point
pcmk__log_else(LOG_TRACE, return);
g_hash_table_iter_init(&iter, nodes);
while (g_hash_table_iter_next(&iter, NULL, (void **) &node)) {
score2char_stack(node->weight, score, sizeof(score));
if (rsc) {
qb_log_from_external_source(function, file,
"%s: %s allocation score on %s: %s",
LOG_TRACE, line, 0,
comment, rsc->id,
node->details->uname, score);
} else {
qb_log_from_external_source(function, file, "%s: %s = %s",
LOG_TRACE, line, 0,
comment, node->details->uname,
score);
}
}
}
/*!
* \internal
* \brief Log or output node weights
*
* \param[in] file Caller's filename
* \param[in] function Caller's function name
* \param[in] line Caller's line number
* \param[in] to_log Log if true, otherwise output
* \param[in] rsc Use allowed nodes for this resource
* \param[in] comment Text description to prefix lines with
* \param[in] nodes If rsc is not specified, use these nodes
*/
void
pe__show_node_weights_as(const char *file, const char *function, int line,
bool to_log, pe_resource_t *rsc, const char *comment,
GHashTable *nodes)
{
if (rsc != NULL) {
if (is_set(rsc->flags, pe_rsc_orphan)) {
// Don't show allocation scores for orphans
return;
}
nodes = rsc->allowed_nodes;
}
if (nodes == NULL) {
// Nothing to show
return;
}
if (to_log) {
pe__log_node_weights(file, function, line, rsc, comment, nodes);
} else {
pe__output_node_weights(rsc, comment, nodes);
}
// If this resource has children, repeat recursively for each
if (rsc && rsc->children) {
for (GList *gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
pe_resource_t *child = (pe_resource_t *) gIter->data;
pe__show_node_weights_as(file, function, line, to_log, child,
comment, nodes);
}
}
}
static void
append_dump_text(gpointer key, gpointer value, gpointer user_data)
{
char **dump_text = user_data;
char *new_text = crm_strdup_printf("%s %s=%s",
*dump_text, (char *)key, (char *)value);
free(*dump_text);
*dump_text = new_text;
}
void
dump_node_capacity(int level, const char *comment, pe_node_t * node)
{
char *dump_text = crm_strdup_printf("%s: %s capacity:",
comment, node->details->uname);
g_hash_table_foreach(node->details->utilization, append_dump_text, &dump_text);
if (level == LOG_STDOUT) {
fprintf(stdout, "%s\n", dump_text);
} else {
crm_trace("%s", dump_text);
}
free(dump_text);
}
void
dump_rsc_utilization(int level, const char *comment, pe_resource_t * rsc, pe_node_t * node)
{
char *dump_text = crm_strdup_printf("%s: %s utilization on %s:",
comment, rsc->id, node->details->uname);
g_hash_table_foreach(rsc->utilization, append_dump_text, &dump_text);
switch (level) {
case LOG_STDOUT:
fprintf(stdout, "%s\n", dump_text);
break;
case LOG_NEVER:
break;
default:
crm_trace("%s", dump_text);
}
free(dump_text);
}
gint
sort_rsc_index(gconstpointer a, gconstpointer b)
{
const pe_resource_t *resource1 = (const pe_resource_t *)a;
const pe_resource_t *resource2 = (const pe_resource_t *)b;
if (a == NULL && b == NULL) {
return 0;
}
if (a == NULL) {
return 1;
}
if (b == NULL) {
return -1;
}
if (resource1->sort_index > resource2->sort_index) {
return -1;
}
if (resource1->sort_index < resource2->sort_index) {
return 1;
}
return 0;
}
gint
sort_rsc_priority(gconstpointer a, gconstpointer b)
{
const pe_resource_t *resource1 = (const pe_resource_t *)a;
const pe_resource_t *resource2 = (const pe_resource_t *)b;
if (a == NULL && b == NULL) {
return 0;
}
if (a == NULL) {
return 1;
}
if (b == NULL) {
return -1;
}
if (resource1->priority > resource2->priority) {
return -1;
}
if (resource1->priority < resource2->priority) {
return 1;
}
return 0;
}
+static enum pe_quorum_policy
+effective_quorum_policy(pe_resource_t *rsc, pe_working_set_t *data_set)
+{
+ enum pe_quorum_policy policy = data_set->no_quorum_policy;
+
+ if (is_set(data_set->flags, pe_flag_have_quorum)) {
+ policy = no_quorum_ignore;
+
+ } else if (data_set->no_quorum_policy == no_quorum_demote) {
+ switch (rsc->role) {
+ case RSC_ROLE_MASTER:
+ case RSC_ROLE_SLAVE:
+ if (rsc->next_role > RSC_ROLE_SLAVE) {
+ rsc->next_role = RSC_ROLE_SLAVE;
+ }
+ policy = no_quorum_ignore;
+ break;
+ default:
+ policy = no_quorum_stop;
+ break;
+ }
+ }
+ return policy;
+}
+
pe_action_t *
custom_action(pe_resource_t * rsc, char *key, const char *task,
pe_node_t * on_node, gboolean optional, gboolean save_action,
pe_working_set_t * data_set)
{
pe_action_t *action = NULL;
GListPtr possible_matches = NULL;
CRM_CHECK(key != NULL, return NULL);
CRM_CHECK(task != NULL, free(key); return NULL);
if (save_action && rsc != NULL) {
possible_matches = find_actions(rsc->actions, key, on_node);
} else if(save_action) {
#if 0
action = g_hash_table_lookup(data_set->singletons, key);
#else
/* More expensive but takes 'node' into account */
possible_matches = find_actions(data_set->actions, key, on_node);
#endif
}
if(data_set->singletons == NULL) {
data_set->singletons = g_hash_table_new_full(crm_str_hash, g_str_equal, NULL, NULL);
}
if (possible_matches != NULL) {
if (pcmk__list_of_multiple(possible_matches)) {
pe_warn("Action %s for %s on %s exists %d times",
task, rsc ? rsc->id : "",
on_node ? on_node->details->uname : "", g_list_length(possible_matches));
}
action = g_list_nth_data(possible_matches, 0);
pe_rsc_trace(rsc, "Found existing action %d (%s) for %s (%s) on %s",
action->id, action->uuid,
(rsc? rsc->id : "no resource"), task,
(on_node? on_node->details->uname : "no node"));
g_list_free(possible_matches);
}
if (action == NULL) {
if (save_action) {
pe_rsc_trace(rsc, "Creating %s action %d: %s for %s (%s) on %s",
(optional? "optional" : "mandatory"),
data_set->action_id, key,
(rsc? rsc->id : "no resource"), task,
(on_node? on_node->details->uname : "no node"));
}
action = calloc(1, sizeof(pe_action_t));
if (save_action) {
action->id = data_set->action_id++;
} else {
action->id = 0;
}
action->rsc = rsc;
CRM_ASSERT(task != NULL);
action->task = strdup(task);
if (on_node) {
action->node = pe__copy_node(on_node);
}
action->uuid = strdup(key);
if (safe_str_eq(task, CRM_OP_LRM_DELETE)) {
// Resource history deletion for a node can be done on the DC
pe_set_action_bit(action, pe_action_dc);
}
pe_set_action_bit(action, pe_action_runnable);
if (optional) {
pe_set_action_bit(action, pe_action_optional);
} else {
pe_clear_action_bit(action, pe_action_optional);
}
action->extra = crm_str_table_new();
action->meta = crm_str_table_new();
if (save_action) {
data_set->actions = g_list_prepend(data_set->actions, action);
if(rsc == NULL) {
g_hash_table_insert(data_set->singletons, action->uuid, action);
}
}
if (rsc != NULL) {
guint interval_ms = 0;
action->op_entry = find_rsc_op_entry_helper(rsc, key, TRUE);
parse_op_key(key, NULL, NULL, &interval_ms);
unpack_operation(action, action->op_entry, rsc->container, data_set,
interval_ms);
if (save_action) {
rsc->actions = g_list_prepend(rsc->actions, action);
}
}
if (save_action) {
pe_rsc_trace(rsc, "Action %d created", action->id);
}
}
if (!optional && is_set(action->flags, pe_action_optional)) {
pe_rsc_trace(rsc, "Unset optional on action %d", action->id);
pe_clear_action_bit(action, pe_action_optional);
}
if (rsc != NULL) {
enum action_tasks a_task = text2task(action->task);
+ enum pe_quorum_policy quorum_policy = effective_quorum_policy(rsc, data_set);
int warn_level = LOG_TRACE;
if (save_action) {
warn_level = LOG_WARNING;
}
if (is_set(action->flags, pe_action_have_node_attrs) == FALSE
&& action->node != NULL && action->op_entry != NULL) {
pe_rule_eval_data_t rule_data = {
.node_hash = action->node->details->attrs,
.role = RSC_ROLE_UNKNOWN,
.now = data_set->now,
.match_data = NULL,
.rsc_data = NULL,
.op_data = NULL
};
pe_set_action_bit(action, pe_action_have_node_attrs);
pe__unpack_dataset_nvpairs(action->op_entry, XML_TAG_ATTR_SETS,
&rule_data, action->extra, NULL,
FALSE, data_set);
}
if (is_set(action->flags, pe_action_pseudo)) {
/* leave untouched */
} else if (action->node == NULL) {
pe_rsc_trace(rsc, "Unset runnable on %s", action->uuid);
pe_clear_action_bit(action, pe_action_runnable);
} else if (is_not_set(rsc->flags, pe_rsc_managed)
&& g_hash_table_lookup(action->meta,
XML_LRM_ATTR_INTERVAL_MS) == NULL) {
crm_debug("Action %s (unmanaged)", action->uuid);
pe_rsc_trace(rsc, "Set optional on %s", action->uuid);
pe_set_action_bit(action, pe_action_optional);
/* action->runnable = FALSE; */
} else if (is_not_set(action->flags, pe_action_dc)
&& !(action->node->details->online)
&& (!pe__is_guest_node(action->node)
|| action->node->details->remote_requires_reset)) {
pe_clear_action_bit(action, pe_action_runnable);
do_crm_log(warn_level, "Action %s on %s is unrunnable (offline)",
action->uuid, action->node->details->uname);
if (is_set(action->rsc->flags, pe_rsc_managed)
&& save_action && a_task == stop_rsc
&& action->node->details->unclean == FALSE) {
pe_fence_node(data_set, action->node, "resource actions are unrunnable", FALSE);
}
} else if (is_not_set(action->flags, pe_action_dc)
&& action->node->details->pending) {
pe_clear_action_bit(action, pe_action_runnable);
do_crm_log(warn_level, "Action %s on %s is unrunnable (pending)",
action->uuid, action->node->details->uname);
} else if (action->needs == rsc_req_nothing) {
pe_rsc_trace(rsc, "Action %s does not require anything", action->uuid);
pe_action_set_reason(action, NULL, TRUE);
if (pe__is_guest_node(action->node)
&& !pe_can_fence(data_set, action->node)) {
/* An action that requires nothing usually does not require any
* fencing in order to be runnable. However, there is an
* exception: an action cannot be completed if it is on a guest
* node whose host is unclean and cannot be fenced.
*/
pe_clear_action_bit(action, pe_action_runnable);
crm_debug("%s\t%s (cancelled : host cannot be fenced)",
action->node->details->uname, action->uuid);
} else {
pe_set_action_bit(action, pe_action_runnable);
}
#if 0
/*
* No point checking this
* - if we don't have quorum we can't stonith anyway
*/
} else if (action->needs == rsc_req_stonith) {
crm_trace("Action %s requires only stonith", action->uuid);
action->runnable = TRUE;
#endif
- } else if (is_set(data_set->flags, pe_flag_have_quorum) == FALSE
- && data_set->no_quorum_policy == no_quorum_stop) {
+ } else if (quorum_policy == no_quorum_stop) {
pe_action_set_flag_reason(__FUNCTION__, __LINE__, action, NULL, "no quorum", pe_action_runnable, TRUE);
crm_debug("%s\t%s (cancelled : quorum)", action->node->details->uname, action->uuid);
- } else if (is_set(data_set->flags, pe_flag_have_quorum) == FALSE
- && data_set->no_quorum_policy == no_quorum_freeze) {
+ } else if (quorum_policy == no_quorum_freeze) {
pe_rsc_trace(rsc, "Check resource is already active: %s %s %s %s", rsc->id, action->uuid, role2text(rsc->next_role), role2text(rsc->role));
if (rsc->fns->active(rsc, TRUE) == FALSE || rsc->next_role > rsc->role) {
pe_action_set_flag_reason(__FUNCTION__, __LINE__, action, NULL, "quorum freeze", pe_action_runnable, TRUE);
pe_rsc_debug(rsc, "%s\t%s (cancelled : quorum freeze)",
action->node->details->uname, action->uuid);
}
} else if(is_not_set(action->flags, pe_action_runnable)) {
pe_rsc_trace(rsc, "Action %s is runnable", action->uuid);
//pe_action_set_reason(action, NULL, TRUE);
pe_set_action_bit(action, pe_action_runnable);
}
if (save_action) {
switch (a_task) {
case stop_rsc:
set_bit(rsc->flags, pe_rsc_stopping);
break;
case start_rsc:
clear_bit(rsc->flags, pe_rsc_starting);
if (is_set(action->flags, pe_action_runnable)) {
set_bit(rsc->flags, pe_rsc_starting);
}
break;
default:
break;
}
}
}
free(key);
return action;
}
+static bool
+valid_stop_on_fail(const char *value)
+{
+ return safe_str_neq(value, "standby")
+ && safe_str_neq(value, "demote")
+ && safe_str_neq(value, "stop");
+}
+
static const char *
unpack_operation_on_fail(pe_action_t * action)
{
+ const char *name = NULL;
+ const char *role = NULL;
+ const char *on_fail = NULL;
+ const char *interval_spec = NULL;
+ const char *enabled = NULL;
const char *value = g_hash_table_lookup(action->meta, XML_OP_ATTR_ON_FAIL);
- if (safe_str_eq(action->task, CRMD_ACTION_STOP) && safe_str_eq(value, "standby")) {
+ if (safe_str_eq(action->task, CRMD_ACTION_STOP)
+ && !valid_stop_on_fail(value)) {
+
pcmk__config_err("Resetting '" XML_OP_ATTR_ON_FAIL "' for %s stop "
- "action to default value because 'standby' is not "
- "allowed for stop", action->rsc->id);
+ "action to default value because '%s' is not "
+ "allowed for stop", action->rsc->id, value);
return NULL;
+
} else if (safe_str_eq(action->task, CRMD_ACTION_DEMOTE) && !value) {
/* demote on_fail defaults to master monitor value if present */
xmlNode *operation = NULL;
- const char *name = NULL;
- const char *role = NULL;
- const char *on_fail = NULL;
- const char *interval_spec = NULL;
- const char *enabled = NULL;
CRM_CHECK(action->rsc != NULL, return NULL);
for (operation = __xml_first_child_element(action->rsc->ops_xml);
operation && !value; operation = __xml_next_element(operation)) {
if (!crm_str_eq((const char *)operation->name, "op", TRUE)) {
continue;
}
name = crm_element_value(operation, "name");
role = crm_element_value(operation, "role");
on_fail = crm_element_value(operation, XML_OP_ATTR_ON_FAIL);
enabled = crm_element_value(operation, "enabled");
interval_spec = crm_element_value(operation, XML_LRM_ATTR_INTERVAL);
if (!on_fail) {
continue;
} else if (enabled && !crm_is_true(enabled)) {
continue;
} else if (safe_str_neq(name, "monitor") || safe_str_neq(role, "Master")) {
continue;
} else if (crm_parse_interval_spec(interval_spec) == 0) {
continue;
+ } else if (safe_str_eq(on_fail, "demote")) {
+ continue;
}
value = on_fail;
}
} else if (safe_str_eq(action->task, CRM_OP_LRM_DELETE)) {
value = "ignore";
+
+ } else if (safe_str_eq(value, "demote")) {
+ name = crm_element_value(action->op_entry, "name");
+ role = crm_element_value(action->op_entry, "role");
+ on_fail = crm_element_value(action->op_entry, XML_OP_ATTR_ON_FAIL);
+ interval_spec = crm_element_value(action->op_entry,
+ XML_LRM_ATTR_INTERVAL);
+
+ if (safe_str_neq(name, CRMD_ACTION_PROMOTE)
+ && (safe_str_neq(name, CRMD_ACTION_STATUS)
+ || safe_str_neq(role, "Master")
+ || (crm_parse_interval_spec(interval_spec) == 0))) {
+ pcmk__config_err("Resetting '" XML_OP_ATTR_ON_FAIL "' for %s %s "
+ "action to default value because 'demote' is not "
+ "allowed for it", action->rsc->id, name);
+ return NULL;
+ }
}
return value;
}
static xmlNode *
find_min_interval_mon(pe_resource_t * rsc, gboolean include_disabled)
{
guint interval_ms = 0;
guint min_interval_ms = G_MAXUINT;
const char *name = NULL;
const char *value = NULL;
const char *interval_spec = NULL;
xmlNode *op = NULL;
xmlNode *operation = NULL;
for (operation = __xml_first_child_element(rsc->ops_xml); operation != NULL;
operation = __xml_next_element(operation)) {
if (crm_str_eq((const char *)operation->name, "op", TRUE)) {
name = crm_element_value(operation, "name");
interval_spec = crm_element_value(operation, XML_LRM_ATTR_INTERVAL);
value = crm_element_value(operation, "enabled");
if (!include_disabled && value && crm_is_true(value) == FALSE) {
continue;
}
if (safe_str_neq(name, RSC_STATUS)) {
continue;
}
interval_ms = crm_parse_interval_spec(interval_spec);
if (interval_ms && (interval_ms < min_interval_ms)) {
min_interval_ms = interval_ms;
op = operation;
}
}
}
return op;
}
static int
unpack_start_delay(const char *value, GHashTable *meta)
{
int start_delay = 0;
if (value != NULL) {
start_delay = crm_get_msec(value);
if (start_delay < 0) {
start_delay = 0;
}
if (meta) {
g_hash_table_replace(meta, strdup(XML_OP_ATTR_START_DELAY), crm_itoa(start_delay));
}
}
return start_delay;
}
// true if value contains valid, non-NULL interval origin for recurring op
static bool
unpack_interval_origin(const char *value, xmlNode *xml_obj, guint interval_ms,
crm_time_t *now, long long *start_delay)
{
long long result = 0;
guint interval_sec = interval_ms / 1000;
crm_time_t *origin = NULL;
// Ignore unspecified values and non-recurring operations
if ((value == NULL) || (interval_ms == 0) || (now == NULL)) {
return false;
}
// Parse interval origin from text
origin = crm_time_new(value);
if (origin == NULL) {
pcmk__config_err("Ignoring '" XML_OP_ATTR_ORIGIN "' for operation "
"'%s' because '%s' is not valid",
(ID(xml_obj)? ID(xml_obj) : "(missing ID)"), value);
return false;
}
// Get seconds since origin (negative if origin is in the future)
result = crm_time_get_seconds(now) - crm_time_get_seconds(origin);
crm_time_free(origin);
// Calculate seconds from closest interval to now
result = result % interval_sec;
// Calculate seconds remaining until next interval
result = ((result <= 0)? 0 : interval_sec) - result;
crm_info("Calculated a start delay of %llds for operation '%s'",
result,
(ID(xml_obj)? ID(xml_obj) : "(unspecified)"));
if (start_delay != NULL) {
*start_delay = result * 1000; // milliseconds
}
return true;
}
static int
unpack_timeout(const char *value)
{
int timeout = crm_get_msec(value);
if (timeout < 0) {
timeout = crm_get_msec(CRM_DEFAULT_OP_TIMEOUT_S);
}
return timeout;
}
int
pe_get_configured_timeout(pe_resource_t *rsc, const char *action, pe_working_set_t *data_set)
{
xmlNode *child = NULL;
const char *timeout = NULL;
int timeout_ms = 0;
pe_rule_eval_data_t rule_data = {
.node_hash = NULL,
.role = RSC_ROLE_UNKNOWN,
.now = data_set->now,
.match_data = NULL,
.rsc_data = NULL,
.op_data = NULL
};
for (child = first_named_child(rsc->ops_xml, XML_ATTR_OP);
child != NULL; child = crm_next_same_xml(child)) {
if (safe_str_eq(action, crm_element_value(child, XML_NVPAIR_ATTR_NAME))) {
timeout = crm_element_value(child, XML_ATTR_TIMEOUT);
break;
}
}
if (timeout == NULL && data_set->op_defaults) {
GHashTable *action_meta = crm_str_table_new();
pe__unpack_dataset_nvpairs(data_set->op_defaults, XML_TAG_META_SETS,
&rule_data, action_meta, NULL, FALSE, data_set);
timeout = g_hash_table_lookup(action_meta, XML_ATTR_TIMEOUT);
}
// @TODO check meta-attributes (including versioned meta-attributes)
// @TODO maybe use min-interval monitor timeout as default for monitors
timeout_ms = crm_get_msec(timeout);
if (timeout_ms < 0) {
timeout_ms = crm_get_msec(CRM_DEFAULT_OP_TIMEOUT_S);
}
return timeout_ms;
}
#if ENABLE_VERSIONED_ATTRS
static void
unpack_versioned_meta(xmlNode *versioned_meta, xmlNode *xml_obj,
guint interval_ms, crm_time_t *now)
{
xmlNode *attrs = NULL;
xmlNode *attr = NULL;
for (attrs = __xml_first_child_element(versioned_meta); attrs != NULL;
attrs = __xml_next_element(attrs)) {
for (attr = __xml_first_child_element(attrs); attr != NULL;
attr = __xml_next_element(attr)) {
const char *name = crm_element_value(attr, XML_NVPAIR_ATTR_NAME);
const char *value = crm_element_value(attr, XML_NVPAIR_ATTR_VALUE);
if (safe_str_eq(name, XML_OP_ATTR_START_DELAY)) {
int start_delay = unpack_start_delay(value, NULL);
crm_xml_add_int(attr, XML_NVPAIR_ATTR_VALUE, start_delay);
} else if (safe_str_eq(name, XML_OP_ATTR_ORIGIN)) {
long long start_delay = 0;
if (unpack_interval_origin(value, xml_obj, interval_ms, now,
&start_delay)) {
crm_xml_add(attr, XML_NVPAIR_ATTR_NAME,
XML_OP_ATTR_START_DELAY);
crm_xml_add_ll(attr, XML_NVPAIR_ATTR_VALUE, start_delay);
}
} else if (safe_str_eq(name, XML_ATTR_TIMEOUT)) {
int timeout = unpack_timeout(value);
crm_xml_add_int(attr, XML_NVPAIR_ATTR_VALUE, timeout);
}
}
}
}
#endif
/*!
* \brief Unpack operation XML into an action structure
*
* Unpack an operation's meta-attributes (normalizing the interval, timeout,
* and start delay values as integer milliseconds), requirements, and
* failure policy.
*
* \param[in,out] action Action to unpack into
* \param[in] xml_obj Operation XML (or NULL if all defaults)
* \param[in] container Resource that contains affected resource, if any
* \param[in] data_set Cluster state
* \param[in] interval_ms How frequently to perform the operation
*/
static void
unpack_operation(pe_action_t * action, xmlNode * xml_obj, pe_resource_t * container,
pe_working_set_t * data_set, guint interval_ms)
{
int timeout = 0;
char *value_ms = NULL;
const char *value = NULL;
const char *field = XML_LRM_ATTR_INTERVAL;
char *default_timeout = NULL;
#if ENABLE_VERSIONED_ATTRS
pe_rsc_action_details_t *rsc_details = NULL;
#endif
pe_rsc_eval_data_t rsc_rule_data = {
.standard = crm_element_value(action->rsc->xml, XML_AGENT_ATTR_CLASS),
.provider = crm_element_value(action->rsc->xml, XML_AGENT_ATTR_PROVIDER),
.agent = crm_element_value(action->rsc->xml, XML_EXPR_ATTR_TYPE)
};
pe_op_eval_data_t op_rule_data = {
.op_name = action->task,
.interval = interval_ms
};
pe_rule_eval_data_t rule_data = {
.node_hash = NULL,
.role = RSC_ROLE_UNKNOWN,
.now = data_set->now,
.match_data = NULL,
.rsc_data = &rsc_rule_data,
.op_data = &op_rule_data
};
CRM_CHECK(action && action->rsc, return);
// Cluster-wide
pe__unpack_dataset_nvpairs(data_set->op_defaults, XML_TAG_META_SETS, &rule_data,
action->meta, NULL, FALSE, data_set);
// Probe timeouts default differently, so handle timeout default later
default_timeout = g_hash_table_lookup(action->meta, XML_ATTR_TIMEOUT);
if (default_timeout) {
default_timeout = strdup(default_timeout);
g_hash_table_remove(action->meta, XML_ATTR_TIMEOUT);
}
if (xml_obj) {
xmlAttrPtr xIter = NULL;
// take precedence over defaults
pe__unpack_dataset_nvpairs(xml_obj, XML_TAG_META_SETS, &rule_data,
action->meta, NULL, TRUE, data_set);
#if ENABLE_VERSIONED_ATTRS
rsc_details = pe_rsc_action_details(action);
pe_eval_versioned_attributes(data_set->input, xml_obj,
XML_TAG_ATTR_SETS, &rule_data,
rsc_details->versioned_parameters,
NULL);
pe_eval_versioned_attributes(data_set->input, xml_obj,
XML_TAG_META_SETS, &rule_data,
rsc_details->versioned_meta,
NULL);
#endif
/* Anything set as an XML property has highest precedence.
* This ensures we use the name and interval from the tag.
*/
for (xIter = xml_obj->properties; xIter; xIter = xIter->next) {
const char *prop_name = (const char *)xIter->name;
const char *prop_value = crm_element_value(xml_obj, prop_name);
g_hash_table_replace(action->meta, strdup(prop_name), strdup(prop_value));
}
}
g_hash_table_remove(action->meta, "id");
// Normalize interval to milliseconds
if (interval_ms > 0) {
value_ms = crm_strdup_printf("%u", interval_ms);
g_hash_table_replace(action->meta, strdup(field), value_ms);
} else if (g_hash_table_lookup(action->meta, field) != NULL) {
g_hash_table_remove(action->meta, field);
}
// Handle timeout default, now that we know the interval
if (g_hash_table_lookup(action->meta, XML_ATTR_TIMEOUT)) {
free(default_timeout);
} else {
// Probe timeouts default to minimum-interval monitor's
if (safe_str_eq(action->task, RSC_STATUS) && (interval_ms == 0)) {
xmlNode *min_interval_mon = find_min_interval_mon(action->rsc, FALSE);
if (min_interval_mon) {
value = crm_element_value(min_interval_mon, XML_ATTR_TIMEOUT);
if (value) {
crm_trace("\t%s defaults to minimum-interval monitor's timeout '%s'",
action->uuid, value);
free(default_timeout);
default_timeout = strdup(value);
}
}
}
if (default_timeout) {
g_hash_table_insert(action->meta, strdup(XML_ATTR_TIMEOUT),
default_timeout);
}
}
if (safe_str_neq(action->task, RSC_START)
&& safe_str_neq(action->task, RSC_PROMOTE)) {
action->needs = rsc_req_nothing;
value = "nothing (not start/promote)";
} else if (is_set(action->rsc->flags, pe_rsc_needs_fencing)) {
action->needs = rsc_req_stonith;
value = "fencing (resource)";
} else if (is_set(action->rsc->flags, pe_rsc_needs_quorum)) {
action->needs = rsc_req_quorum;
value = "quorum (resource)";
} else {
action->needs = rsc_req_nothing;
value = "nothing (resource)";
}
pe_rsc_trace(action->rsc, "\tAction %s requires: %s", action->uuid, value);
value = unpack_operation_on_fail(action);
if (value == NULL) {
} else if (safe_str_eq(value, "block")) {
action->on_fail = action_fail_block;
g_hash_table_insert(action->meta, strdup(XML_OP_ATTR_ON_FAIL), strdup("block"));
value = "block"; // The above could destroy the original string
} else if (safe_str_eq(value, "fence")) {
action->on_fail = action_fail_fence;
value = "node fencing";
if (is_set(data_set->flags, pe_flag_stonith_enabled) == FALSE) {
pcmk__config_err("Resetting '" XML_OP_ATTR_ON_FAIL "' for "
"operation '%s' to 'stop' because 'fence' is not "
"valid when fencing is disabled", action->uuid);
action->on_fail = action_fail_stop;
action->fail_role = RSC_ROLE_STOPPED;
value = "stop resource";
}
} else if (safe_str_eq(value, "standby")) {
action->on_fail = action_fail_standby;
value = "node standby";
} else if (safe_str_eq(value, "ignore")
|| safe_str_eq(value, "nothing")) {
action->on_fail = action_fail_ignore;
value = "ignore";
} else if (safe_str_eq(value, "migrate")) {
action->on_fail = action_fail_migrate;
value = "force migration";
} else if (safe_str_eq(value, "stop")) {
action->on_fail = action_fail_stop;
action->fail_role = RSC_ROLE_STOPPED;
value = "stop resource";
} else if (safe_str_eq(value, "restart")) {
action->on_fail = action_fail_recover;
value = "restart (and possibly migrate)";
} else if (safe_str_eq(value, "restart-container")) {
if (container) {
action->on_fail = action_fail_restart_container;
value = "restart container (and possibly migrate)";
} else {
value = NULL;
}
+ } else if (safe_str_eq(value, "demote")) {
+ action->on_fail = action_fail_demote;
+ value = "demote instance";
+
} else {
pe_err("Resource %s: Unknown failure type (%s)", action->rsc->id, value);
value = NULL;
}
/* defaults */
if (value == NULL && container) {
action->on_fail = action_fail_restart_container;
value = "restart container (and possibly migrate) (default)";
/* For remote nodes, ensure that any failure that results in dropping an
* active connection to the node results in fencing of the node.
*
* There are only two action failures that don't result in fencing.
* 1. probes - probe failures are expected.
* 2. start - a start failure indicates that an active connection does not already
* exist. The user can set op on-fail=fence if they really want to fence start
* failures. */
} else if (((value == NULL) || !is_set(action->rsc->flags, pe_rsc_managed)) &&
(pe__resource_is_remote_conn(action->rsc, data_set) &&
!(safe_str_eq(action->task, CRMD_ACTION_STATUS) && (interval_ms == 0)) &&
(safe_str_neq(action->task, CRMD_ACTION_START)))) {
if (!is_set(action->rsc->flags, pe_rsc_managed)) {
action->on_fail = action_fail_stop;
action->fail_role = RSC_ROLE_STOPPED;
value = "stop unmanaged remote node (enforcing default)";
} else {
if (is_set(data_set->flags, pe_flag_stonith_enabled)) {
value = "fence remote node (default)";
} else {
value = "recover remote node connection (default)";
}
if (action->rsc->remote_reconnect_ms) {
action->fail_role = RSC_ROLE_STOPPED;
}
action->on_fail = action_fail_reset_remote;
}
} else if (value == NULL && safe_str_eq(action->task, CRMD_ACTION_STOP)) {
if (is_set(data_set->flags, pe_flag_stonith_enabled)) {
action->on_fail = action_fail_fence;
value = "resource fence (default)";
} else {
action->on_fail = action_fail_block;
value = "resource block (default)";
}
} else if (value == NULL) {
action->on_fail = action_fail_recover;
value = "restart (and possibly migrate) (default)";
}
pe_rsc_trace(action->rsc, "\t%s failure handling: %s", action->task, value);
value = NULL;
if (xml_obj != NULL) {
value = g_hash_table_lookup(action->meta, "role_after_failure");
if (value) {
pe_warn_once(pe_wo_role_after,
"Support for role_after_failure is deprecated and will be removed in a future release");
}
}
if (value != NULL && action->fail_role == RSC_ROLE_UNKNOWN) {
action->fail_role = text2role(value);
}
/* defaults */
if (action->fail_role == RSC_ROLE_UNKNOWN) {
if (safe_str_eq(action->task, CRMD_ACTION_PROMOTE)) {
action->fail_role = RSC_ROLE_SLAVE;
} else {
action->fail_role = RSC_ROLE_STARTED;
}
}
pe_rsc_trace(action->rsc, "\t%s failure results in: %s", action->task,
role2text(action->fail_role));
value = g_hash_table_lookup(action->meta, XML_OP_ATTR_START_DELAY);
if (value) {
unpack_start_delay(value, action->meta);
} else {
long long start_delay = 0;
value = g_hash_table_lookup(action->meta, XML_OP_ATTR_ORIGIN);
if (unpack_interval_origin(value, xml_obj, interval_ms, data_set->now,
&start_delay)) {
g_hash_table_replace(action->meta, strdup(XML_OP_ATTR_START_DELAY),
crm_strdup_printf("%lld", start_delay));
}
}
value = g_hash_table_lookup(action->meta, XML_ATTR_TIMEOUT);
timeout = unpack_timeout(value);
g_hash_table_replace(action->meta, strdup(XML_ATTR_TIMEOUT), crm_itoa(timeout));
#if ENABLE_VERSIONED_ATTRS
unpack_versioned_meta(rsc_details->versioned_meta, xml_obj, interval_ms,
data_set->now);
#endif
}
static xmlNode *
find_rsc_op_entry_helper(pe_resource_t * rsc, const char *key, gboolean include_disabled)
{
guint interval_ms = 0;
gboolean do_retry = TRUE;
char *local_key = NULL;
const char *name = NULL;
const char *value = NULL;
const char *interval_spec = NULL;
char *match_key = NULL;
xmlNode *op = NULL;
xmlNode *operation = NULL;
retry:
for (operation = __xml_first_child_element(rsc->ops_xml); operation != NULL;
operation = __xml_next_element(operation)) {
if (crm_str_eq((const char *)operation->name, "op", TRUE)) {
name = crm_element_value(operation, "name");
interval_spec = crm_element_value(operation, XML_LRM_ATTR_INTERVAL);
value = crm_element_value(operation, "enabled");
if (!include_disabled && value && crm_is_true(value) == FALSE) {
continue;
}
interval_ms = crm_parse_interval_spec(interval_spec);
match_key = pcmk__op_key(rsc->id, name, interval_ms);
if (safe_str_eq(key, match_key)) {
op = operation;
}
free(match_key);
if (rsc->clone_name) {
match_key = pcmk__op_key(rsc->clone_name, name, interval_ms);
if (safe_str_eq(key, match_key)) {
op = operation;
}
free(match_key);
}
if (op != NULL) {
free(local_key);
return op;
}
}
}
free(local_key);
if (do_retry == FALSE) {
return NULL;
}
do_retry = FALSE;
if (strstr(key, CRMD_ACTION_MIGRATE) || strstr(key, CRMD_ACTION_MIGRATED)) {
local_key = pcmk__op_key(rsc->id, "migrate", 0);
key = local_key;
goto retry;
} else if (strstr(key, "_notify_")) {
local_key = pcmk__op_key(rsc->id, "notify", 0);
key = local_key;
goto retry;
}
return NULL;
}
xmlNode *
find_rsc_op_entry(pe_resource_t * rsc, const char *key)
{
return find_rsc_op_entry_helper(rsc, key, FALSE);
}
void
print_node(const char *pre_text, pe_node_t * node, gboolean details)
{
if (node == NULL) {
crm_trace("%s%s: ", pre_text == NULL ? "" : pre_text, pre_text == NULL ? "" : ": ");
return;
}
CRM_ASSERT(node->details);
crm_trace("%s%s%sNode %s: (weight=%d, fixed=%s)",
pre_text == NULL ? "" : pre_text,
pre_text == NULL ? "" : ": ",
node->details->online ? "" : "Unavailable/Unclean ",
node->details->uname, node->weight, node->fixed ? "True" : "False");
if (details) {
int log_level = LOG_TRACE;
char *pe_mutable = strdup("\t\t");
GListPtr gIter = node->details->running_rsc;
crm_trace("\t\t===Node Attributes");
g_hash_table_foreach(node->details->attrs, print_str_str, pe_mutable);
free(pe_mutable);
crm_trace("\t\t=== Resources");
for (; gIter != NULL; gIter = gIter->next) {
pe_resource_t *rsc = (pe_resource_t *) gIter->data;
rsc->fns->print(rsc, "\t\t", pe_print_log|pe_print_pending,
&log_level);
}
}
}
/*
* Used by the HashTable for-loop
*/
void
print_str_str(gpointer key, gpointer value, gpointer user_data)
{
crm_trace("%s%s %s ==> %s",
user_data == NULL ? "" : (char *)user_data,
user_data == NULL ? "" : ": ", (char *)key, (char *)value);
}
void
pe_free_action(pe_action_t * action)
{
if (action == NULL) {
return;
}
g_list_free_full(action->actions_before, free); /* pe_action_wrapper_t* */
g_list_free_full(action->actions_after, free); /* pe_action_wrapper_t* */
if (action->extra) {
g_hash_table_destroy(action->extra);
}
if (action->meta) {
g_hash_table_destroy(action->meta);
}
#if ENABLE_VERSIONED_ATTRS
if (action->rsc) {
pe_free_rsc_action_details(action);
}
#endif
free(action->cancel_task);
free(action->reason);
free(action->task);
free(action->uuid);
free(action->node);
free(action);
}
GListPtr
find_recurring_actions(GListPtr input, pe_node_t * not_on_node)
{
const char *value = NULL;
GListPtr result = NULL;
GListPtr gIter = input;
CRM_CHECK(input != NULL, return NULL);
for (; gIter != NULL; gIter = gIter->next) {
pe_action_t *action = (pe_action_t *) gIter->data;
value = g_hash_table_lookup(action->meta, XML_LRM_ATTR_INTERVAL_MS);
if (value == NULL) {
/* skip */
} else if (safe_str_eq(value, "0")) {
/* skip */
} else if (safe_str_eq(CRMD_ACTION_CANCEL, action->task)) {
/* skip */
} else if (not_on_node == NULL) {
crm_trace("(null) Found: %s", action->uuid);
result = g_list_prepend(result, action);
} else if (action->node == NULL) {
/* skip */
} else if (action->node->details != not_on_node->details) {
crm_trace("Found: %s", action->uuid);
result = g_list_prepend(result, action);
}
}
return result;
}
enum action_tasks
get_complex_task(pe_resource_t * rsc, const char *name, gboolean allow_non_atomic)
{
enum action_tasks task = text2task(name);
if (rsc == NULL) {
return task;
} else if (allow_non_atomic == FALSE || rsc->variant == pe_native) {
switch (task) {
case stopped_rsc:
case started_rsc:
case action_demoted:
case action_promoted:
crm_trace("Folding %s back into its atomic counterpart for %s", name, rsc->id);
return task - 1;
break;
default:
break;
}
}
return task;
}
pe_action_t *
find_first_action(GListPtr input, const char *uuid, const char *task, pe_node_t * on_node)
{
GListPtr gIter = NULL;
CRM_CHECK(uuid || task, return NULL);
for (gIter = input; gIter != NULL; gIter = gIter->next) {
pe_action_t *action = (pe_action_t *) gIter->data;
if (uuid != NULL && safe_str_neq(uuid, action->uuid)) {
continue;
} else if (task != NULL && safe_str_neq(task, action->task)) {
continue;
} else if (on_node == NULL) {
return action;
} else if (action->node == NULL) {
continue;
} else if (on_node->details == action->node->details) {
return action;
}
}
return NULL;
}
GListPtr
find_actions(GListPtr input, const char *key, const pe_node_t *on_node)
{
GListPtr gIter = input;
GListPtr result = NULL;
CRM_CHECK(key != NULL, return NULL);
for (; gIter != NULL; gIter = gIter->next) {
pe_action_t *action = (pe_action_t *) gIter->data;
if (safe_str_neq(key, action->uuid)) {
crm_trace("%s does not match action %s", key, action->uuid);
continue;
} else if (on_node == NULL) {
crm_trace("Action %s matches (ignoring node)", key);
result = g_list_prepend(result, action);
} else if (action->node == NULL) {
crm_trace("Action %s matches (unallocated, assigning to %s)",
key, on_node->details->uname);
action->node = pe__copy_node(on_node);
result = g_list_prepend(result, action);
} else if (on_node->details == action->node->details) {
crm_trace("Action %s on %s matches", key, on_node->details->uname);
result = g_list_prepend(result, action);
} else {
crm_trace("Action %s on node %s does not match requested node %s",
key, action->node->details->uname,
on_node->details->uname);
}
}
return result;
}
GList *
find_actions_exact(GList *input, const char *key, const pe_node_t *on_node)
{
GList *result = NULL;
CRM_CHECK(key != NULL, return NULL);
if (on_node == NULL) {
crm_trace("Not searching for action %s because node not specified",
key);
return NULL;
}
for (GList *gIter = input; gIter != NULL; gIter = gIter->next) {
pe_action_t *action = (pe_action_t *) gIter->data;
if (action->node == NULL) {
crm_trace("Skipping comparison of %s vs action %s without node",
key, action->uuid);
} else if (safe_str_neq(key, action->uuid)) {
crm_trace("Desired action %s doesn't match %s", key, action->uuid);
} else if (safe_str_neq(on_node->details->id,
action->node->details->id)) {
crm_trace("Action %s desired node ID %s doesn't match %s",
key, on_node->details->id, action->node->details->id);
} else {
crm_trace("Action %s matches", key);
result = g_list_prepend(result, action);
}
}
return result;
}
/*!
* \brief Find all actions of given type for a resource
*
* \param[in] rsc Resource to search
* \param[in] node Find only actions scheduled on this node
* \param[in] task Action name to search for
* \param[in] require_node If TRUE, NULL node or action node will not match
*
* \return List of actions found (or NULL if none)
* \note If node is not NULL and require_node is FALSE, matching actions
* without a node will be assigned to node.
*/
GList *
pe__resource_actions(const pe_resource_t *rsc, const pe_node_t *node,
const char *task, bool require_node)
{
GList *result = NULL;
char *key = pcmk__op_key(rsc->id, task, 0);
if (require_node) {
result = find_actions_exact(rsc->actions, key, node);
} else {
result = find_actions(rsc->actions, key, node);
}
free(key);
return result;
}
static void
resource_node_score(pe_resource_t * rsc, pe_node_t * node, int score, const char *tag)
{
pe_node_t *match = NULL;
if ((rsc->exclusive_discover || (node->rsc_discover_mode == pe_discover_never))
&& safe_str_eq(tag, "symmetric_default")) {
/* This string comparision may be fragile, but exclusive resources and
* exclusive nodes should not have the symmetric_default constraint
* applied to them.
*/
return;
} else if (rsc->children) {
GListPtr gIter = rsc->children;
for (; gIter != NULL; gIter = gIter->next) {
pe_resource_t *child_rsc = (pe_resource_t *) gIter->data;
resource_node_score(child_rsc, node, score, tag);
}
}
pe_rsc_trace(rsc, "Setting %s for %s on %s: %d", tag, rsc->id, node->details->uname, score);
match = pe_hash_table_lookup(rsc->allowed_nodes, node->details->id);
if (match == NULL) {
match = pe__copy_node(node);
g_hash_table_insert(rsc->allowed_nodes, (gpointer) match->details->id, match);
}
match->weight = pe__add_scores(match->weight, score);
}
void
resource_location(pe_resource_t * rsc, pe_node_t * node, int score, const char *tag,
pe_working_set_t * data_set)
{
if (node != NULL) {
resource_node_score(rsc, node, score, tag);
} else if (data_set != NULL) {
GListPtr gIter = data_set->nodes;
for (; gIter != NULL; gIter = gIter->next) {
pe_node_t *node_iter = (pe_node_t *) gIter->data;
resource_node_score(rsc, node_iter, score, tag);
}
} else {
GHashTableIter iter;
pe_node_t *node_iter = NULL;
g_hash_table_iter_init(&iter, rsc->allowed_nodes);
while (g_hash_table_iter_next(&iter, NULL, (void **)&node_iter)) {
resource_node_score(rsc, node_iter, score, tag);
}
}
if (node == NULL && score == -INFINITY) {
if (rsc->allocated_to) {
crm_info("Deallocating %s from %s", rsc->id, rsc->allocated_to->details->uname);
free(rsc->allocated_to);
rsc->allocated_to = NULL;
}
}
}
#define sort_return(an_int, why) do { \
free(a_uuid); \
free(b_uuid); \
crm_trace("%s (%d) %c %s (%d) : %s", \
a_xml_id, a_call_id, an_int>0?'>':an_int<0?'<':'=', \
b_xml_id, b_call_id, why); \
return an_int; \
} while(0)
gint
sort_op_by_callid(gconstpointer a, gconstpointer b)
{
int a_call_id = -1;
int b_call_id = -1;
char *a_uuid = NULL;
char *b_uuid = NULL;
const xmlNode *xml_a = a;
const xmlNode *xml_b = b;
const char *a_xml_id = crm_element_value(xml_a, XML_ATTR_ID);
const char *b_xml_id = crm_element_value(xml_b, XML_ATTR_ID);
if (safe_str_eq(a_xml_id, b_xml_id)) {
/* We have duplicate lrm_rsc_op entries in the status
* section which is unlikely to be a good thing
* - we can handle it easily enough, but we need to get
* to the bottom of why it's happening.
*/
pe_err("Duplicate lrm_rsc_op entries named %s", a_xml_id);
sort_return(0, "duplicate");
}
crm_element_value_int(xml_a, XML_LRM_ATTR_CALLID, &a_call_id);
crm_element_value_int(xml_b, XML_LRM_ATTR_CALLID, &b_call_id);
if (a_call_id == -1 && b_call_id == -1) {
/* both are pending ops so it doesn't matter since
* stops are never pending
*/
sort_return(0, "pending");
} else if (a_call_id >= 0 && a_call_id < b_call_id) {
sort_return(-1, "call id");
} else if (b_call_id >= 0 && a_call_id > b_call_id) {
sort_return(1, "call id");
} else if (b_call_id >= 0 && a_call_id == b_call_id) {
/*
* The op and last_failed_op are the same
* Order on last-rc-change
*/
time_t last_a = -1;
time_t last_b = -1;
crm_element_value_epoch(xml_a, XML_RSC_OP_LAST_CHANGE, &last_a);
crm_element_value_epoch(xml_b, XML_RSC_OP_LAST_CHANGE, &last_b);
crm_trace("rc-change: %lld vs %lld",
(long long) last_a, (long long) last_b);
if (last_a >= 0 && last_a < last_b) {
sort_return(-1, "rc-change");
} else if (last_b >= 0 && last_a > last_b) {
sort_return(1, "rc-change");
}
sort_return(0, "rc-change");
} else {
/* One of the inputs is a pending operation
* Attempt to use XML_ATTR_TRANSITION_MAGIC to determine its age relative to the other
*/
int a_id = -1;
int b_id = -1;
const char *a_magic = crm_element_value(xml_a, XML_ATTR_TRANSITION_MAGIC);
const char *b_magic = crm_element_value(xml_b, XML_ATTR_TRANSITION_MAGIC);
CRM_CHECK(a_magic != NULL && b_magic != NULL, sort_return(0, "No magic"));
if (!decode_transition_magic(a_magic, &a_uuid, &a_id, NULL, NULL, NULL,
NULL)) {
sort_return(0, "bad magic a");
}
if (!decode_transition_magic(b_magic, &b_uuid, &b_id, NULL, NULL, NULL,
NULL)) {
sort_return(0, "bad magic b");
}
/* try to determine the relative age of the operation...
* some pending operations (e.g. a start) may have been superseded
* by a subsequent stop
*
* [a|b]_id == -1 means it's a shutdown operation and _always_ comes last
*/
if (safe_str_neq(a_uuid, b_uuid) || a_id == b_id) {
/*
* some of the logic in here may be redundant...
*
* if the UUID from the TE doesn't match then one better
* be a pending operation.
* pending operations don't survive between elections and joins
* because we query the LRM directly
*/
if (b_call_id == -1) {
sort_return(-1, "transition + call");
} else if (a_call_id == -1) {
sort_return(1, "transition + call");
}
} else if ((a_id >= 0 && a_id < b_id) || b_id == -1) {
sort_return(-1, "transition");
} else if ((b_id >= 0 && a_id > b_id) || a_id == -1) {
sort_return(1, "transition");
}
}
/* we should never end up here */
CRM_CHECK(FALSE, sort_return(0, "default"));
}
time_t
get_effective_time(pe_working_set_t * data_set)
{
if(data_set) {
if (data_set->now == NULL) {
crm_trace("Recording a new 'now'");
data_set->now = crm_time_new(NULL);
}
return crm_time_get_seconds_since_epoch(data_set->now);
}
crm_trace("Defaulting to 'now'");
return time(NULL);
}
gboolean
get_target_role(pe_resource_t * rsc, enum rsc_role_e * role)
{
enum rsc_role_e local_role = RSC_ROLE_UNKNOWN;
const char *value = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_TARGET_ROLE);
CRM_CHECK(role != NULL, return FALSE);
if (value == NULL || safe_str_eq("started", value)
|| safe_str_eq("default", value)) {
return FALSE;
}
local_role = text2role(value);
if (local_role == RSC_ROLE_UNKNOWN) {
pcmk__config_err("Ignoring '" XML_RSC_ATTR_TARGET_ROLE "' for %s "
"because '%s' is not valid", rsc->id, value);
return FALSE;
} else if (local_role > RSC_ROLE_STARTED) {
if (is_set(uber_parent(rsc)->flags, pe_rsc_promotable)) {
if (local_role > RSC_ROLE_SLAVE) {
/* This is what we'd do anyway, just leave the default to avoid messing up the placement algorithm */
return FALSE;
}
} else {
pcmk__config_err("Ignoring '" XML_RSC_ATTR_TARGET_ROLE "' for %s "
"because '%s' only makes sense for promotable "
"clones", rsc->id, value);
return FALSE;
}
}
*role = local_role;
return TRUE;
}
gboolean
order_actions(pe_action_t * lh_action, pe_action_t * rh_action, enum pe_ordering order)
{
GListPtr gIter = NULL;
pe_action_wrapper_t *wrapper = NULL;
GListPtr list = NULL;
if (order == pe_order_none) {
return FALSE;
}
if (lh_action == NULL || rh_action == NULL) {
return FALSE;
}
crm_trace("Ordering Action %s before %s", lh_action->uuid, rh_action->uuid);
/* Ensure we never create a dependency on ourselves... it's happened */
CRM_ASSERT(lh_action != rh_action);
/* Filter dups, otherwise update_action_states() has too much work to do */
gIter = lh_action->actions_after;
for (; gIter != NULL; gIter = gIter->next) {
pe_action_wrapper_t *after = (pe_action_wrapper_t *) gIter->data;
if (after->action == rh_action && (after->type & order)) {
return FALSE;
}
}
wrapper = calloc(1, sizeof(pe_action_wrapper_t));
wrapper->action = rh_action;
wrapper->type = order;
list = lh_action->actions_after;
list = g_list_prepend(list, wrapper);
lh_action->actions_after = list;
wrapper = NULL;
/* order |= pe_order_implies_then; */
/* order ^= pe_order_implies_then; */
wrapper = calloc(1, sizeof(pe_action_wrapper_t));
wrapper->action = lh_action;
wrapper->type = order;
list = rh_action->actions_before;
list = g_list_prepend(list, wrapper);
rh_action->actions_before = list;
return TRUE;
}
pe_action_t *
get_pseudo_op(const char *name, pe_working_set_t * data_set)
{
pe_action_t *op = NULL;
if(data_set->singletons) {
op = g_hash_table_lookup(data_set->singletons, name);
}
if (op == NULL) {
op = custom_action(NULL, strdup(name), name, NULL, TRUE, TRUE, data_set);
set_bit(op->flags, pe_action_pseudo);
set_bit(op->flags, pe_action_runnable);
}
return op;
}
void
destroy_ticket(gpointer data)
{
pe_ticket_t *ticket = data;
if (ticket->state) {
g_hash_table_destroy(ticket->state);
}
free(ticket->id);
free(ticket);
}
pe_ticket_t *
ticket_new(const char *ticket_id, pe_working_set_t * data_set)
{
pe_ticket_t *ticket = NULL;
if (ticket_id == NULL || strlen(ticket_id) == 0) {
return NULL;
}
if (data_set->tickets == NULL) {
data_set->tickets =
g_hash_table_new_full(crm_str_hash, g_str_equal, free,
destroy_ticket);
}
ticket = g_hash_table_lookup(data_set->tickets, ticket_id);
if (ticket == NULL) {
ticket = calloc(1, sizeof(pe_ticket_t));
if (ticket == NULL) {
crm_err("Cannot allocate ticket '%s'", ticket_id);
return NULL;
}
crm_trace("Creaing ticket entry for %s", ticket_id);
ticket->id = strdup(ticket_id);
ticket->granted = FALSE;
ticket->last_granted = -1;
ticket->standby = FALSE;
ticket->state = crm_str_table_new();
g_hash_table_insert(data_set->tickets, strdup(ticket->id), ticket);
}
return ticket;
}
static void
filter_parameters(xmlNode * param_set, const char *param_string, bool need_present)
{
if (param_set && param_string) {
xmlAttrPtr xIter = param_set->properties;
while (xIter) {
const char *prop_name = (const char *)xIter->name;
char *name = crm_strdup_printf(" %s ", prop_name);
char *match = strstr(param_string, name);
free(name);
// Do now, because current entry might get removed below
xIter = xIter->next;
if (need_present && match == NULL) {
crm_trace("%s not found in %s", prop_name, param_string);
xml_remove_prop(param_set, prop_name);
} else if (need_present == FALSE && match) {
crm_trace("%s found in %s", prop_name, param_string);
xml_remove_prop(param_set, prop_name);
}
}
}
}
#if ENABLE_VERSIONED_ATTRS
static void
append_versioned_params(xmlNode *versioned_params, const char *ra_version, xmlNode *params)
{
GHashTable *hash = pe_unpack_versioned_parameters(versioned_params, ra_version);
char *key = NULL;
char *value = NULL;
GHashTableIter iter;
g_hash_table_iter_init(&iter, hash);
while (g_hash_table_iter_next(&iter, (gpointer *) &key, (gpointer *) &value)) {
crm_xml_add(params, key, value);
}
g_hash_table_destroy(hash);
}
#endif
/*!
* \internal
* \brief Calculate action digests and store in node's digest cache
*
* \param[in] rsc Resource that action was for
* \param[in] task Name of action performed
* \param[in] key Action's task key
* \param[in] node Node action was performed on
* \param[in] xml_op XML of operation in CIB status (if available)
* \param[in] calc_secure Whether to calculate secure digest
* \param[in] data_set Cluster working set
*
* \return Pointer to node's digest cache entry
*/
static op_digest_cache_t *
rsc_action_digest(pe_resource_t *rsc, const char *task, const char *key,
pe_node_t *node, xmlNode *xml_op, bool calc_secure,
pe_working_set_t *data_set)
{
op_digest_cache_t *data = NULL;
data = g_hash_table_lookup(node->details->digest_cache, key);
if (data == NULL) {
GHashTable *local_rsc_params = crm_str_table_new();
pe_action_t *action = custom_action(rsc, strdup(key), task, node, TRUE, FALSE, data_set);
#if ENABLE_VERSIONED_ATTRS
xmlNode *local_versioned_params = create_xml_node(NULL, XML_TAG_RSC_VER_ATTRS);
const char *ra_version = NULL;
#endif
const char *op_version;
const char *restart_list = NULL;
const char *secure_list = " passwd password ";
data = calloc(1, sizeof(op_digest_cache_t));
CRM_ASSERT(data != NULL);
get_rsc_attributes(local_rsc_params, rsc, node, data_set);
#if ENABLE_VERSIONED_ATTRS
pe_get_versioned_attributes(local_versioned_params, rsc, node, data_set);
#endif
data->params_all = create_xml_node(NULL, XML_TAG_PARAMS);
// REMOTE_CONTAINER_HACK: Allow remote nodes that start containers with pacemaker remote inside
if (pe__add_bundle_remote_name(rsc, data->params_all,
XML_RSC_ATTR_REMOTE_RA_ADDR)) {
crm_trace("Set address for bundle connection %s (on %s)",
rsc->id, node->details->uname);
}
g_hash_table_foreach(local_rsc_params, hash2field, data->params_all);
g_hash_table_foreach(action->extra, hash2field, data->params_all);
g_hash_table_foreach(rsc->parameters, hash2field, data->params_all);
g_hash_table_foreach(action->meta, hash2metafield, data->params_all);
if(xml_op) {
secure_list = crm_element_value(xml_op, XML_LRM_ATTR_OP_SECURE);
restart_list = crm_element_value(xml_op, XML_LRM_ATTR_OP_RESTART);
op_version = crm_element_value(xml_op, XML_ATTR_CRM_VERSION);
#if ENABLE_VERSIONED_ATTRS
ra_version = crm_element_value(xml_op, XML_ATTR_RA_VERSION);
#endif
} else {
op_version = CRM_FEATURE_SET;
}
#if ENABLE_VERSIONED_ATTRS
append_versioned_params(local_versioned_params, ra_version, data->params_all);
append_versioned_params(rsc->versioned_parameters, ra_version, data->params_all);
{
pe_rsc_action_details_t *details = pe_rsc_action_details(action);
append_versioned_params(details->versioned_parameters, ra_version, data->params_all);
}
#endif
pcmk__filter_op_for_digest(data->params_all);
g_hash_table_destroy(local_rsc_params);
pe_free_action(action);
data->digest_all_calc = calculate_operation_digest(data->params_all, op_version);
if (calc_secure) {
data->params_secure = copy_xml(data->params_all);
if(secure_list) {
filter_parameters(data->params_secure, secure_list, FALSE);
}
data->digest_secure_calc = calculate_operation_digest(data->params_secure, op_version);
}
if(xml_op && crm_element_value(xml_op, XML_LRM_ATTR_RESTART_DIGEST) != NULL) {
data->params_restart = copy_xml(data->params_all);
if (restart_list) {
filter_parameters(data->params_restart, restart_list, TRUE);
}
data->digest_restart_calc = calculate_operation_digest(data->params_restart, op_version);
}
g_hash_table_insert(node->details->digest_cache, strdup(key), data);
}
return data;
}
op_digest_cache_t *
rsc_action_digest_cmp(pe_resource_t * rsc, xmlNode * xml_op, pe_node_t * node,
pe_working_set_t * data_set)
{
op_digest_cache_t *data = NULL;
char *key = NULL;
guint interval_ms = 0;
const char *op_version;
const char *task = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
const char *digest_all;
const char *digest_restart;
CRM_ASSERT(node != NULL);
op_version = crm_element_value(xml_op, XML_ATTR_CRM_VERSION);
digest_all = crm_element_value(xml_op, XML_LRM_ATTR_OP_DIGEST);
digest_restart = crm_element_value(xml_op, XML_LRM_ATTR_RESTART_DIGEST);
crm_element_value_ms(xml_op, XML_LRM_ATTR_INTERVAL_MS, &interval_ms);
key = pcmk__op_key(rsc->id, task, interval_ms);
data = rsc_action_digest(rsc, task, key, node, xml_op,
is_set(data_set->flags, pe_flag_sanitized),
data_set);
data->rc = RSC_DIGEST_MATCH;
if (digest_restart && data->digest_restart_calc && strcmp(data->digest_restart_calc, digest_restart) != 0) {
pe_rsc_info(rsc, "Parameters to %s on %s changed: was %s vs. now %s (restart:%s) %s",
key, node->details->uname,
crm_str(digest_restart), data->digest_restart_calc,
op_version, crm_element_value(xml_op, XML_ATTR_TRANSITION_MAGIC));
data->rc = RSC_DIGEST_RESTART;
} else if (digest_all == NULL) {
/* it is unknown what the previous op digest was */
data->rc = RSC_DIGEST_UNKNOWN;
} else if (strcmp(digest_all, data->digest_all_calc) != 0) {
pe_rsc_info(rsc, "Parameters to %s on %s changed: was %s vs. now %s (%s:%s) %s",
key, node->details->uname,
crm_str(digest_all), data->digest_all_calc,
(interval_ms > 0)? "reschedule" : "reload",
op_version, crm_element_value(xml_op, XML_ATTR_TRANSITION_MAGIC));
data->rc = RSC_DIGEST_ALL;
}
free(key);
return data;
}
/*!
* \internal
* \brief Create an unfencing summary for use in special node attribute
*
* Create a string combining a fence device's resource ID, agent type, and
* parameter digest (whether for all parameters or just non-private parameters).
* This can be stored in a special node attribute, allowing us to detect changes
* in either the agent type or parameters, to know whether unfencing must be
* redone or can be safely skipped when the device's history is cleaned.
*
* \param[in] rsc_id Fence device resource ID
* \param[in] agent_type Fence device agent
* \param[in] param_digest Fence device parameter digest
*
* \return Newly allocated string with unfencing digest
* \note The caller is responsible for freeing the result.
*/
static inline char *
create_unfencing_summary(const char *rsc_id, const char *agent_type,
const char *param_digest)
{
return crm_strdup_printf("%s:%s:%s", rsc_id, agent_type, param_digest);
}
/*!
* \internal
* \brief Check whether a node can skip unfencing
*
* Check whether a fence device's current definition matches a node's
* stored summary of when it was last unfenced by the device.
*
* \param[in] rsc_id Fence device's resource ID
* \param[in] agent Fence device's agent type
* \param[in] digest_calc Fence device's current parameter digest
* \param[in] node_summary Value of node's special unfencing node attribute
* (a comma-separated list of unfencing summaries for
* all devices that have unfenced this node)
*
* \return TRUE if digest matches, FALSE otherwise
*/
static bool
unfencing_digest_matches(const char *rsc_id, const char *agent,
const char *digest_calc, const char *node_summary)
{
bool matches = FALSE;
if (rsc_id && agent && digest_calc && node_summary) {
char *search_secure = create_unfencing_summary(rsc_id, agent,
digest_calc);
/* The digest was calculated including the device ID and agent,
* so there is no risk of collision using strstr().
*/
matches = (strstr(node_summary, search_secure) != NULL);
crm_trace("Calculated unfencing digest '%s' %sfound in '%s'",
search_secure, matches? "" : "not ", node_summary);
free(search_secure);
}
return matches;
}
/* Magic string to use as action name for digest cache entries used for
* unfencing checks. This is not a real action name (i.e. "on"), so
* check_action_definition() won't confuse these entries with real actions.
*/
#define STONITH_DIGEST_TASK "stonith-on"
/*!
* \internal
* \brief Calculate fence device digests and digest comparison result
*
* \param[in] rsc Fence device resource
* \param[in] agent Fence device's agent type
* \param[in] node Node with digest cache to use
* \param[in] data_set Cluster working set
*
* \return Node's digest cache entry
*/
static op_digest_cache_t *
fencing_action_digest_cmp(pe_resource_t *rsc, const char *agent,
pe_node_t *node, pe_working_set_t *data_set)
{
const char *node_summary = NULL;
// Calculate device's current parameter digests
char *key = pcmk__op_key(rsc->id, STONITH_DIGEST_TASK, 0);
op_digest_cache_t *data = rsc_action_digest(rsc, STONITH_DIGEST_TASK, key,
node, NULL, TRUE, data_set);
free(key);
// Check whether node has special unfencing summary node attribute
node_summary = pe_node_attribute_raw(node, CRM_ATTR_DIGESTS_ALL);
if (node_summary == NULL) {
data->rc = RSC_DIGEST_UNKNOWN;
return data;
}
// Check whether full parameter digest matches
if (unfencing_digest_matches(rsc->id, agent, data->digest_all_calc,
node_summary)) {
data->rc = RSC_DIGEST_MATCH;
return data;
}
// Check whether secure parameter digest matches
node_summary = pe_node_attribute_raw(node, CRM_ATTR_DIGESTS_SECURE);
if (unfencing_digest_matches(rsc->id, agent, data->digest_secure_calc,
node_summary)) {
data->rc = RSC_DIGEST_MATCH;
if (is_set(data_set->flags, pe_flag_stdout)) {
printf("Only 'private' parameters to %s for unfencing %s changed\n",
rsc->id, node->details->uname);
}
return data;
}
// Parameters don't match
data->rc = RSC_DIGEST_ALL;
if (is_set(data_set->flags, (pe_flag_sanitized|pe_flag_stdout))
&& data->digest_secure_calc) {
char *digest = create_unfencing_summary(rsc->id, agent,
data->digest_secure_calc);
printf("Parameters to %s for unfencing %s changed, try '%s'\n",
rsc->id, node->details->uname, digest);
free(digest);
}
return data;
}
const char *rsc_printable_id(pe_resource_t *rsc)
{
if (is_not_set(rsc->flags, pe_rsc_unique)) {
return ID(rsc->xml);
}
return rsc->id;
}
void
clear_bit_recursive(pe_resource_t * rsc, unsigned long long flag)
{
GListPtr gIter = rsc->children;
clear_bit(rsc->flags, flag);
for (; gIter != NULL; gIter = gIter->next) {
pe_resource_t *child_rsc = (pe_resource_t *) gIter->data;
clear_bit_recursive(child_rsc, flag);
}
}
void
set_bit_recursive(pe_resource_t * rsc, unsigned long long flag)
{
GListPtr gIter = rsc->children;
set_bit(rsc->flags, flag);
for (; gIter != NULL; gIter = gIter->next) {
pe_resource_t *child_rsc = (pe_resource_t *) gIter->data;
set_bit_recursive(child_rsc, flag);
}
}
static GListPtr
find_unfencing_devices(GListPtr candidates, GListPtr matches)
{
for (GListPtr gIter = candidates; gIter != NULL; gIter = gIter->next) {
pe_resource_t *candidate = gIter->data;
const char *provides = g_hash_table_lookup(candidate->meta, XML_RSC_ATTR_PROVIDES);
const char *requires = g_hash_table_lookup(candidate->meta, XML_RSC_ATTR_REQUIRES);
if(candidate->children) {
matches = find_unfencing_devices(candidate->children, matches);
} else if (is_not_set(candidate->flags, pe_rsc_fence_device)) {
continue;
} else if (crm_str_eq(provides, "unfencing", FALSE) || crm_str_eq(requires, "unfencing", FALSE)) {
matches = g_list_prepend(matches, candidate);
}
}
return matches;
}
static int
node_priority_fencing_delay(pe_node_t * node, pe_working_set_t * data_set)
{
int member_count = 0;
int online_count = 0;
int top_priority = 0;
int lowest_priority = 0;
GListPtr gIter = NULL;
// `priority-fencing-delay` is disabled
if (data_set->priority_fencing_delay <= 0) {
return 0;
}
/* No need to request a delay if the fencing target is not a normal cluster
* member, for example if it's a remote node or a guest node. */
if (node->details->type != node_member) {
return 0;
}
// No need to request a delay if the fencing target is in our partition
if (node->details->online) {
return 0;
}
for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
pe_node_t *n = gIter->data;
if (n->details->type != node_member) {
continue;
}
member_count ++;
if (n->details->online) {
online_count++;
}
if (member_count == 1
|| n->details->priority > top_priority) {
top_priority = n->details->priority;
}
if (member_count == 1
|| n->details->priority < lowest_priority) {
lowest_priority = n->details->priority;
}
}
// No need to delay if we have more than half of the cluster members
if (online_count > member_count / 2) {
return 0;
}
/* All the nodes have equal priority.
* Any configured corresponding `pcmk_delay_base/max` will be applied. */
if (lowest_priority == top_priority) {
return 0;
}
if (node->details->priority < top_priority) {
return 0;
}
return data_set->priority_fencing_delay;
}
pe_action_t *
pe_fence_op(pe_node_t * node, const char *op, bool optional, const char *reason,
bool priority_delay, pe_working_set_t * data_set)
{
char *op_key = NULL;
pe_action_t *stonith_op = NULL;
if(op == NULL) {
op = data_set->stonith_action;
}
op_key = crm_strdup_printf("%s-%s-%s", CRM_OP_FENCE, node->details->uname, op);
if(data_set->singletons) {
stonith_op = g_hash_table_lookup(data_set->singletons, op_key);
}
if(stonith_op == NULL) {
stonith_op = custom_action(NULL, op_key, CRM_OP_FENCE, node, TRUE, TRUE, data_set);
add_hash_param(stonith_op->meta, XML_LRM_ATTR_TARGET, node->details->uname);
add_hash_param(stonith_op->meta, XML_LRM_ATTR_TARGET_UUID, node->details->id);
add_hash_param(stonith_op->meta, "stonith_action", op);
if (pe__is_guest_or_remote_node(node)
&& is_set(data_set->flags, pe_flag_enable_unfencing)) {
/* Extra work to detect device changes on remotes
*
* We may do this for all nodes in the future, but for now
* the check_action_definition() based stuff works fine.
*/
long max = 1024;
long digests_all_offset = 0;
long digests_secure_offset = 0;
char *digests_all = calloc(max, sizeof(char));
char *digests_secure = calloc(max, sizeof(char));
GListPtr matches = find_unfencing_devices(data_set->resources, NULL);
for (GListPtr gIter = matches; gIter != NULL; gIter = gIter->next) {
pe_resource_t *match = gIter->data;
const char *agent = g_hash_table_lookup(match->meta,
XML_ATTR_TYPE);
op_digest_cache_t *data = NULL;
data = fencing_action_digest_cmp(match, agent, node, data_set);
if(data->rc == RSC_DIGEST_ALL) {
optional = FALSE;
crm_notice("Unfencing %s (remote): because the definition of %s changed", node->details->uname, match->id);
if (is_set(data_set->flags, pe_flag_stdout)) {
fprintf(stdout, " notice: Unfencing %s (remote): because the definition of %s changed\n", node->details->uname, match->id);
}
}
digests_all_offset += snprintf(
digests_all+digests_all_offset, max-digests_all_offset,
"%s:%s:%s,", match->id, agent, data->digest_all_calc);
digests_secure_offset += snprintf(
digests_secure+digests_secure_offset, max-digests_secure_offset,
"%s:%s:%s,", match->id, agent, data->digest_secure_calc);
}
g_hash_table_insert(stonith_op->meta,
strdup(XML_OP_ATTR_DIGESTS_ALL),
digests_all);
g_hash_table_insert(stonith_op->meta,
strdup(XML_OP_ATTR_DIGESTS_SECURE),
digests_secure);
}
} else {
free(op_key);
}
if (data_set->priority_fencing_delay > 0
/* It's a suitable case where `priority-fencing-delay` applies.
* At least add `priority-fencing-delay` field as an indicator. */
&& (priority_delay
/* Re-calculate priority delay for the suitable case when
* pe_fence_op() is called again by stage6() after node priority has
* been actually calculated with native_add_running() */
|| g_hash_table_lookup(stonith_op->meta,
XML_CONFIG_ATTR_PRIORITY_FENCING_DELAY) != NULL)) {
/* Add `priority-fencing-delay` to the fencing op even if it's 0 for
* the targeting node. So that it takes precedence over any possible
* `pcmk_delay_base/max`.
*/
char *delay_s = crm_itoa(node_priority_fencing_delay(node, data_set));
g_hash_table_insert(stonith_op->meta,
strdup(XML_CONFIG_ATTR_PRIORITY_FENCING_DELAY),
delay_s);
}
if(optional == FALSE && pe_can_fence(data_set, node)) {
pe_action_required(stonith_op, NULL, reason);
} else if(reason && stonith_op->reason == NULL) {
stonith_op->reason = strdup(reason);
}
return stonith_op;
}
void
trigger_unfencing(
pe_resource_t * rsc, pe_node_t *node, const char *reason, pe_action_t *dependency, pe_working_set_t * data_set)
{
if(is_not_set(data_set->flags, pe_flag_enable_unfencing)) {
/* No resources require it */
return;
} else if (rsc != NULL && is_not_set(rsc->flags, pe_rsc_fence_device)) {
/* Wasn't a stonith device */
return;
} else if(node
&& node->details->online
&& node->details->unclean == FALSE
&& node->details->shutdown == FALSE) {
pe_action_t *unfence = pe_fence_op(node, "on", FALSE, reason, FALSE, data_set);
if(dependency) {
order_actions(unfence, dependency, pe_order_optional);
}
} else if(rsc) {
GHashTableIter iter;
g_hash_table_iter_init(&iter, rsc->allowed_nodes);
while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
if(node->details->online && node->details->unclean == FALSE && node->details->shutdown == FALSE) {
trigger_unfencing(rsc, node, reason, dependency, data_set);
}
}
}
}
gboolean
add_tag_ref(GHashTable * tags, const char * tag_name, const char * obj_ref)
{
pe_tag_t *tag = NULL;
GListPtr gIter = NULL;
gboolean is_existing = FALSE;
CRM_CHECK(tags && tag_name && obj_ref, return FALSE);
tag = g_hash_table_lookup(tags, tag_name);
if (tag == NULL) {
tag = calloc(1, sizeof(pe_tag_t));
if (tag == NULL) {
return FALSE;
}
tag->id = strdup(tag_name);
tag->refs = NULL;
g_hash_table_insert(tags, strdup(tag_name), tag);
}
for (gIter = tag->refs; gIter != NULL; gIter = gIter->next) {
const char *existing_ref = (const char *) gIter->data;
if (crm_str_eq(existing_ref, obj_ref, TRUE)){
is_existing = TRUE;
break;
}
}
if (is_existing == FALSE) {
tag->refs = g_list_append(tag->refs, strdup(obj_ref));
crm_trace("Added: tag=%s ref=%s", tag->id, obj_ref);
}
return TRUE;
}
void pe_action_set_flag_reason(const char *function, long line,
pe_action_t *action, pe_action_t *reason, const char *text,
enum pe_action_flags flags, bool overwrite)
{
bool unset = FALSE;
bool update = FALSE;
const char *change = NULL;
if(is_set(flags, pe_action_runnable)) {
unset = TRUE;
change = "unrunnable";
} else if(is_set(flags, pe_action_optional)) {
unset = TRUE;
change = "required";
} else if(is_set(flags, pe_action_migrate_runnable)) {
unset = TRUE;
overwrite = TRUE;
change = "unrunnable";
} else if(is_set(flags, pe_action_dangle)) {
change = "dangling";
} else if(is_set(flags, pe_action_requires_any)) {
change = "required";
} else {
crm_err("Unknown flag change to %x by %s: 0x%s",
flags, action->uuid, (reason? reason->uuid : "0"));
}
if(unset) {
if(is_set(action->flags, flags)) {
action->flags = crm_clear_bit(function, line, action->uuid, action->flags, flags);
update = TRUE;
}
} else {
if(is_not_set(action->flags, flags)) {
action->flags = crm_set_bit(function, line, action->uuid, action->flags, flags);
update = TRUE;
}
}
if((change && update) || text) {
char *reason_text = NULL;
if(reason == NULL) {
pe_action_set_reason(action, text, overwrite);
} else if(reason->rsc == NULL) {
reason_text = crm_strdup_printf("%s %s%c %s", change, reason->task, text?':':0, text?text:"");
} else {
reason_text = crm_strdup_printf("%s %s %s%c %s", change, reason->rsc->id, reason->task, text?':':0, text?text:"NA");
}
if(reason_text && action->rsc != reason->rsc) {
pe_action_set_reason(action, reason_text, overwrite);
}
free(reason_text);
}
}
void pe_action_set_reason(pe_action_t *action, const char *reason, bool overwrite)
{
if (action->reason != NULL && overwrite) {
pe_rsc_trace(action->rsc, "Changing %s reason from '%s' to '%s'",
action->uuid, action->reason, crm_str(reason));
free(action->reason);
} else if (action->reason == NULL) {
pe_rsc_trace(action->rsc, "Set %s reason to '%s'",
action->uuid, crm_str(reason));
} else {
// crm_assert(action->reason != NULL && !overwrite);
return;
}
if (reason != NULL) {
action->reason = strdup(reason);
} else {
action->reason = NULL;
}
}
/*!
* \internal
* \brief Check whether shutdown has been requested for a node
*
* \param[in] node Node to check
*
* \return TRUE if node has shutdown attribute set and nonzero, FALSE otherwise
* \note This differs from simply using node->details->shutdown in that it can
* be used before that has been determined (and in fact to determine it),
* and it can also be used to distinguish requested shutdown from implicit
* shutdown of remote nodes by virtue of their connection stopping.
*/
bool
pe__shutdown_requested(pe_node_t *node)
{
const char *shutdown = pe_node_attribute_raw(node, XML_CIB_ATTR_SHUTDOWN);
return shutdown && strcmp(shutdown, "0");
}
/*!
* \internal
* \brief Update a data set's "recheck by" time
*
* \param[in] recheck Epoch time when recheck should happen
* \param[in,out] data_set Current working set
*/
void
pe__update_recheck_time(time_t recheck, pe_working_set_t *data_set)
{
if ((recheck > get_effective_time(data_set))
&& ((data_set->recheck_by == 0)
|| (data_set->recheck_by > recheck))) {
data_set->recheck_by = recheck;
}
}
/*!
* \internal
* \brief Wrapper for pe_unpack_nvpairs() using a cluster working set
*/
void
pe__unpack_dataset_nvpairs(xmlNode *xml_obj, const char *set_name,
pe_rule_eval_data_t *rule_data, GHashTable *hash,
const char *always_first, gboolean overwrite,
pe_working_set_t *data_set)
{
crm_time_t *next_change = crm_time_new_undefined();
pe_eval_nvpairs(data_set->input, xml_obj, set_name, rule_data, hash,
always_first, overwrite, next_change);
if (crm_time_is_defined(next_change)) {
time_t recheck = (time_t) crm_time_get_seconds_since_epoch(next_change);
pe__update_recheck_time(recheck, data_set);
}
crm_time_free(next_change);
}
bool
pe__resource_is_disabled(pe_resource_t *rsc)
{
const char *target_role = NULL;
CRM_CHECK(rsc != NULL, return false);
target_role = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_TARGET_ROLE);
if (target_role) {
enum rsc_role_e target_role_e = text2role(target_role);
if ((target_role_e == RSC_ROLE_STOPPED)
|| ((target_role_e == RSC_ROLE_SLAVE)
&& is_set(uber_parent(rsc)->flags, pe_rsc_promotable))) {
return true;
}
}
return false;
}
/*!
* \internal
* \brief Create an action to clear a resource's history from CIB
*
* \param[in] rsc Resource to clear
* \param[in] node Node to clear history on
*
* \return New action to clear resource history
*/
pe_action_t *
pe__clear_resource_history(pe_resource_t *rsc, pe_node_t *node,
pe_working_set_t *data_set)
{
char *key = NULL;
CRM_ASSERT(rsc && node);
key = pcmk__op_key(rsc->id, CRM_OP_LRM_DELETE, 0);
return custom_action(rsc, key, CRM_OP_LRM_DELETE, node, FALSE, TRUE,
data_set);
}
bool
pe__rsc_running_on_any_node_in_list(pe_resource_t *rsc, GListPtr node_list)
{
for (GListPtr ele = rsc->running_on; ele; ele = ele->next) {
pe_node_t *node = (pe_node_t *) ele->data;
if (pcmk__str_in_list(node_list, node->details->uname)) {
return true;
}
}
return false;
}
bool
pcmk__rsc_is_filtered(pe_resource_t *rsc, GListPtr only_show)
{
return (rsc->fns->active(rsc, FALSE) && !pe__rsc_running_on_any_node_in_list(rsc, only_show));
}
diff --git a/xml/resources-3.4.rng b/xml/resources-3.4.rng
index fbb4b6505a..887dc1c210 100644
--- a/xml/resources-3.4.rng
+++ b/xml/resources-3.4.rng
@@ -1,425 +1,426 @@