diff --git a/cts/cts-scheduler.in b/cts/cts-scheduler.in
index 7ba2415804..23bcc17230 100644
--- a/cts/cts-scheduler.in
+++ b/cts/cts-scheduler.in
@@ -1,1588 +1,1589 @@
 #!@PYTHON@
 """ Regression tests for Pacemaker's scheduler
 """
 
 __copyright__ = "Copyright 2004-2021 the Pacemaker project contributors"
 __license__ = "GNU General Public License version 2 or later (GPLv2+) WITHOUT ANY WARRANTY"
 
 import io
 import os
 import re
 import sys
 import stat
 import shlex
 import shutil
 import argparse
 import subprocess
 import platform
 
 DESC = """Regression tests for Pacemaker's scheduler"""
 
 # Each entry in TESTS is a group of tests, where each test consists of a
 # test base name, test description, and additional test arguments.
 # Test groups will be separated by newlines in output.
 TESTS = [
     [
         [ "simple1", "Offline" ],
         [ "simple2", "Start" ],
         [ "simple3", "Start 2" ],
         [ "simple4", "Start Failed" ],
         [ "simple6", "Stop Start" ],
         [ "simple7", "Shutdown" ],
         #[ "simple8", "Stonith" ],
         #[ "simple9", "Lower version" ],
         #[ "simple10", "Higher version" ],
         [ "simple11", "Priority (ne)" ],
         [ "simple12", "Priority (eq)" ],
         [ "simple8", "Stickiness" ],
     ],
     [
         [ "group1", "Group" ],
         [ "group2", "Group + Native" ],
         [ "group3", "Group + Group" ],
         [ "group4", "Group + Native (nothing)" ],
         [ "group5", "Group + Native (move)" ],
         [ "group6", "Group + Group (move)" ],
         [ "group7", "Group colocation" ],
         [ "group13", "Group colocation (cant run)" ],
         [ "group8", "Group anti-colocation" ],
         [ "group9", "Group recovery" ],
         [ "group10", "Group partial recovery" ],
         [ "group11", "Group target_role" ],
         [ "group14", "Group stop (graph terminated)" ],
         [ "group15", "Negative group colocation" ],
         [ "bug-1573", "Partial stop of a group with two children" ],
         [ "bug-1718", "Mandatory group ordering - Stop group_FUN" ],
         [ "bug-lf-2613", "Move group on failure" ],
         [ "bug-lf-2619", "Move group on clone failure" ],
         [ "group-fail", "Ensure stop order is preserved for partially active groups" ],
         [ "group-unmanaged", "No need to restart r115 because r114 is unmanaged" ],
         [ "group-unmanaged-stopped", "Make sure r115 is stopped when r114 fails" ],
         [ "group-dependents", "Account for the location preferences of things colocated with a group" ],
         [ "group-stop-ordering", "Ensure blocked group member stop does not force other member stops" ],
+        [ "colocate-unmanaged-group", "Respect mandatory colocations even if earlier group member is unmanaged" ],
     ],
     [
         [ "rsc_dep1", "Must not" ],
         [ "rsc_dep3", "Must" ],
         [ "rsc_dep5", "Must not 3" ],
         [ "rsc_dep7", "Must 3" ],
         [ "rsc_dep10", "Must (but cant)" ],
         [ "rsc_dep2", "Must (running)" ],
         [ "rsc_dep8", "Must (running : alt)" ],
         [ "rsc_dep4", "Must (running + move)" ],
         [ "asymmetric", "Asymmetric - require explicit location constraints" ],
     ],
     [
         [ "orphan-0", "Orphan ignore" ],
         [ "orphan-1", "Orphan stop" ],
         [ "orphan-2", "Orphan stop, remove failcount" ],
     ],
     [
         [ "params-0", "Params: No change" ],
         [ "params-1", "Params: Changed" ],
         [ "params-2", "Params: Resource definition" ],
         [ "params-3", "Params: Restart instead of reload if start pending" ],
         [ "params-4", "Params: Reload" ],
         [ "params-5", "Params: Restart based on probe digest" ],
         [ "novell-251689", "Resource definition change + target_role=stopped" ],
         [ "bug-lf-2106", "Restart all anonymous clone instances after config change" ],
         [ "params-6", "Params: Detect reload in previously migrated resource" ],
         [ "nvpair-id-ref", "Support id-ref in nvpair with optional name" ],
         [ "not-reschedule-unneeded-monitor",
                      "Do not reschedule unneeded monitors while resource definitions have changed" ],
         [ "reload-becomes-restart", "Cancel reload if restart becomes required" ],
     ],
     [
         [ "target-0", "Target Role : baseline" ],
         [ "target-1", "Target Role : promoted" ],
         [ "target-2", "Target Role : invalid" ],
     ],
     [
         [ "base-score", "Set a node's default score for all nodes" ],
     ],
     [
         [ "date-1", "Dates", [ "-t",  "2005-020" ] ],
         [ "date-2", "Date Spec - Pass", [ "-t", "2005-020T12:30" ] ],
         [ "date-3", "Date Spec - Fail", [ "-t", "2005-020T11:30" ] ],
         [ "origin", "Timing of recurring operations", [ "-t", "2014-05-07 00:28:00" ] ],
         [ "probe-0", "Probe (anon clone)" ],
         [ "probe-1", "Pending Probe" ],
         [ "probe-2", "Correctly re-probe cloned groups" ],
         [ "probe-3", "Probe (pending node)" ],
         [ "probe-4", "Probe (pending node + stopped resource)" ],
         [ "probe-pending-node", "Probe (pending node + unmanaged resource)" ],
         [ "standby", "Standby" ],
         [ "comments", "Comments" ],
     ],
     [
         [ "one-or-more-0", "Everything starts" ],
         [ "one-or-more-1", "Nothing starts because of A" ],
         [ "one-or-more-2", "D can start because of C" ],
         [ "one-or-more-3", "D cannot start because of B and C" ],
         [ "one-or-more-4", "D cannot start because of target-role" ],
         [ "one-or-more-5", "Start A and F even though C and D are stopped" ],
         [ "one-or-more-6", "Leave A running even though B is stopped" ],
         [ "one-or-more-7", "Leave A running even though C is stopped" ],
         [ "bug-5140-require-all-false", "Allow basegrp:0 to stop" ],
         [ "clone-require-all-1", "clone B starts node 3 and 4" ],
         [ "clone-require-all-2", "clone B remains stopped everywhere" ],
         [ "clone-require-all-3", "clone B stops everywhere because A stops everywhere" ],
         [ "clone-require-all-4", "clone B remains on node 3 and 4 with only one instance of A remaining" ],
         [ "clone-require-all-5", "clone B starts on node 1 3 and 4" ],
         [ "clone-require-all-6", "clone B remains active after shutting down instances of A" ],
         [ "clone-require-all-7",
           "clone A and B both start at the same time. all instances of A start before B" ],
         [ "clone-require-all-no-interleave-1", "C starts everywhere after A and B" ],
         [ "clone-require-all-no-interleave-2",
           "C starts on nodes 1, 2, and 4 with only one active instance of B" ],
         [ "clone-require-all-no-interleave-3",
           "C remains active when instance of B is stopped on one node and started on another" ],
         [ "one-or-more-unrunnable-instances", "Avoid dependencies on instances that won't ever be started" ],
     ],
     [
         [ "location-date-rules-1", "Use location constraints with ineffective date-based rules" ],
         [ "location-date-rules-2", "Use location constraints with effective date-based rules" ],
         [ "nvpair-date-rules-1", "Use nvpair blocks with a variety of date-based rules" ],
         [ "value-source", "Use location constraints with node attribute expressions using value-source" ],
         [ "rule-dbl-as-auto-number-match",
           "Floating-point rule values default to number comparison: match" ],
         [ "rule-dbl-as-auto-number-no-match",
           "Floating-point rule values default to number comparison: no "
                   "match" ],
         [ "rule-dbl-as-integer-match",
           "Floating-point rule values set to integer comparison: match" ],
         [ "rule-dbl-as-integer-no-match",
           "Floating-point rule values set to integer comparison: no match" ],
         [ "rule-dbl-as-number-match",
           "Floating-point rule values set to number comparison: match" ],
         [ "rule-dbl-as-number-no-match",
           "Floating-point rule values set to number comparison: no match" ],
         [ "rule-dbl-parse-fail-default-str-match",
           "Floating-point rule values fail to parse, default to string "
 	          "comparison: match" ],
         [ "rule-dbl-parse-fail-default-str-no-match",
           "Floating-point rule values fail to parse, default to string "
 	          "comparison: no match" ],
         [ "rule-int-as-auto-integer-match",
           "Integer rule values default to integer comparison: match" ],
         [ "rule-int-as-auto-integer-no-match",
           "Integer rule values default to integer comparison: no match" ],
         [ "rule-int-as-integer-match",
           "Integer rule values set to integer comparison: match" ],
         [ "rule-int-as-integer-no-match",
           "Integer rule values set to integer comparison: no match" ],
         [ "rule-int-as-number-match",
           "Integer rule values set to number comparison: match" ],
         [ "rule-int-as-number-no-match",
           "Integer rule values set to number comparison: no match" ],
         [ "rule-int-parse-fail-default-str-match",
           "Integer rule values fail to parse, default to string "
 	          "comparison: match" ],
         [ "rule-int-parse-fail-default-str-no-match",
           "Integer rule values fail to parse, default to string "
 	          "comparison: no match" ],
     ],
     [
         [ "order1", "Order start 1" ],
         [ "order2", "Order start 2" ],
         [ "order3", "Order stop" ],
         [ "order4", "Order (multiple)" ],
         [ "order5", "Order (move)" ],
         [ "order6", "Order (move w/ restart)" ],
         [ "order7", "Order (mandatory)" ],
         [ "order-optional", "Order (score=0)" ],
         [ "order-required", "Order (score=INFINITY)" ],
         [ "bug-lf-2171", "Prevent group start when clone is stopped" ],
         [ "order-clone", "Clone ordering should be able to prevent startup of dependent clones" ],
         [ "order-sets", "Ordering for resource sets" ],
         [ "order-serialize", "Serialize resources without inhibiting migration" ],
         [ "order-serialize-set", "Serialize a set of resources without inhibiting migration" ],
         [ "clone-order-primitive", "Order clone start after a primitive" ],
         [ "clone-order-16instances", "Verify ordering of 16 cloned resources" ],
         [ "order-optional-keyword", "Order (optional keyword)" ],
         [ "order-mandatory", "Order (mandatory keyword)" ],
         [ "bug-lf-2493",
           "Don't imply colocation requirements when applying ordering constraints with clones" ],
         [ "ordered-set-basic-startup", "Constraint set with default order settings" ],
         [ "ordered-set-natural", "Allow natural set ordering" ],
         [ "order-wrong-kind", "Order (error)" ],
     ],
     [
         [ "coloc-loop", "Colocation - loop" ],
         [ "coloc-many-one", "Colocation - many-to-one" ],
         [ "coloc-list", "Colocation - many-to-one with list" ],
         [ "coloc-group", "Colocation - groups" ],
         [ "coloc-unpromoted-anti", "Anti-colocation with unpromoted shouldn't prevent promoted colocation" ],
         [ "coloc-attr", "Colocation based on node attributes" ],
         [ "coloc-negative-group", "Negative colocation with a group" ],
         [ "coloc-intra-set", "Intra-set colocation" ],
         [ "bug-lf-2435", "Colocation sets with a negative score" ],
         [ "coloc-clone-stays-active",
           "Ensure clones don't get stopped/demoted because a dependent must stop" ],
         [ "coloc_fp_logic", "Verify floating point calculations in colocation are working" ],
         [ "colo_promoted_w_native",
           "cl#5070 - Verify promotion order is affected when colocating promoted with primitive" ],
         [ "colo_unpromoted_w_native",
           "cl#5070 - Verify promotion order is affected when colocating unpromoted with primitive" ],
         [ "anti-colocation-order",
           "cl#5187 - Prevent resources in an anti-colocation from even temporarily running on a same node" ],
         [ "anti-colocation-promoted", "Organize order of actions for promoted resources in anti-colocations" ],
         [ "anti-colocation-unpromoted", "Organize order of actions for unpromoted resources in anti-colocations" ],
         [ "enforce-colo1", "Always enforce B with A INFINITY" ],
         [ "complex_enforce_colo", "Always enforce B with A INFINITY. (make sure heat-engine stops)" ],
         [ "coloc-dependee-should-stay", "Stickiness outweighs group colocation" ],
         [ "coloc-dependee-should-move", "Group colocation outweighs stickiness" ],
         [ "colocation-influence", "Respect colocation influence" ],
     ],
     [
         [ "rsc-sets-seq-true", "Resource Sets - sequential=false" ],
         [ "rsc-sets-seq-false", "Resource Sets - sequential=true" ],
         [ "rsc-sets-clone", "Resource Sets - Clone" ],
         [ "rsc-sets-promoted", "Resource Sets - Promoted" ],
         [ "rsc-sets-clone-1", "Resource Sets - Clone (lf#2404)" ],
     ],
     [
         [ "attrs1", "string: eq (and)" ],
         [ "attrs2", "string: lt / gt (and)" ],
         [ "attrs3", "string: ne (or)" ],
         [ "attrs4", "string: exists" ],
         [ "attrs5", "string: not_exists" ],
         [ "attrs6", "is_dc: true" ],
         [ "attrs7", "is_dc: false" ],
         [ "attrs8", "score_attribute" ],
         [ "per-node-attrs", "Per node resource parameters" ],
     ],
     [
         [ "mon-rsc-1", "Schedule Monitor - start" ],
         [ "mon-rsc-2", "Schedule Monitor - move" ],
         [ "mon-rsc-3", "Schedule Monitor - pending start" ],
         [ "mon-rsc-4", "Schedule Monitor - move/pending start" ],
     ],
     [
         [ "rec-rsc-0", "Resource Recover - no start" ],
         [ "rec-rsc-1", "Resource Recover - start" ],
         [ "rec-rsc-2", "Resource Recover - monitor" ],
         [ "rec-rsc-3", "Resource Recover - stop - ignore" ],
         [ "rec-rsc-4", "Resource Recover - stop - block" ],
         [ "rec-rsc-5", "Resource Recover - stop - fence" ],
         [ "rec-rsc-6", "Resource Recover - multiple - restart" ],
         [ "rec-rsc-7", "Resource Recover - multiple - stop" ],
         [ "rec-rsc-8", "Resource Recover - multiple - block" ],
         [ "rec-rsc-9", "Resource Recover - group/group" ],
         [ "monitor-recovery", "on-fail=block + resource recovery detected by recurring monitor" ],
         [ "stop-failure-no-quorum", "Stop failure without quorum" ],
         [ "stop-failure-no-fencing", "Stop failure without fencing available" ],
         [ "stop-failure-with-fencing", "Stop failure with fencing available" ],
         [ "multiple-active-block-group", "Support of multiple-active=block for resource groups" ],
         [ "multiple-monitor-one-failed",
           "Consider resource failed if any of the configured monitor operations failed" ],
     ],
     [
         [ "quorum-1", "No quorum - ignore" ],
         [ "quorum-2", "No quorum - freeze" ],
         [ "quorum-3", "No quorum - stop" ],
         [ "quorum-4", "No quorum - start anyway" ],
         [ "quorum-5", "No quorum - start anyway (group)" ],
         [ "quorum-6", "No quorum - start anyway (clone)" ],
         [ "bug-cl-5212", "No promotion with no-quorum-policy=freeze" ],
         [ "suicide-needed-inquorate", "no-quorum-policy=suicide: suicide necessary" ],
         [ "suicide-not-needed-initial-quorum",
           "no-quorum-policy=suicide: suicide not necessary at initial quorum" ],
         [ "suicide-not-needed-never-quorate",
           "no-quorum-policy=suicide: suicide not necessary if never quorate" ],
         [ "suicide-not-needed-quorate", "no-quorum-policy=suicide: suicide necessary if quorate" ],
     ],
     [
         [ "rec-node-1", "Node Recover - Startup   - no fence" ],
         [ "rec-node-2", "Node Recover - Startup   - fence" ],
         [ "rec-node-3", "Node Recover - HA down   - no fence" ],
         [ "rec-node-4", "Node Recover - HA down   - fence" ],
         [ "rec-node-5", "Node Recover - CRM down  - no fence" ],
         [ "rec-node-6", "Node Recover - CRM down  - fence" ],
         [ "rec-node-7", "Node Recover - no quorum - ignore" ],
         [ "rec-node-8", "Node Recover - no quorum - freeze" ],
         [ "rec-node-9", "Node Recover - no quorum - stop" ],
         [ "rec-node-10", "Node Recover - no quorum - stop w/fence" ],
         [ "rec-node-11", "Node Recover - CRM down w/ group - fence" ],
         [ "rec-node-12", "Node Recover - nothing active - fence" ],
         [ "rec-node-13", "Node Recover - failed resource + shutdown - fence" ],
         [ "rec-node-15", "Node Recover - unknown lrm section" ],
         [ "rec-node-14", "Serialize all stonith's" ],
     ],
     [
         [ "multi1", "Multiple Active (stop/start)" ],
     ],
     [
         [ "migrate-begin", "Normal migration" ],
         [ "migrate-success", "Completed migration" ],
         [ "migrate-partial-1", "Completed migration, missing stop on source" ],
         [ "migrate-partial-2", "Successful migrate_to only" ],
         [ "migrate-partial-3", "Successful migrate_to only, target down" ],
         [ "migrate-partial-4", "Migrate from the correct host after migrate_to+migrate_from" ],
         [ "bug-5186-partial-migrate", "Handle partial migration when src node loses membership" ],
         [ "migrate-fail-2", "Failed migrate_from" ],
         [ "migrate-fail-3", "Failed migrate_from + stop on source" ],
         [ "migrate-fail-4",
           "Failed migrate_from + stop on target - ideally we wouldn't need to re-stop on target" ],
         [ "migrate-fail-5", "Failed migrate_from + stop on source and target" ],
         [ "migrate-fail-6", "Failed migrate_to" ],
         [ "migrate-fail-7", "Failed migrate_to + stop on source" ],
         [ "migrate-fail-8",
           "Failed migrate_to + stop on target - ideally we wouldn't need to re-stop on target" ],
         [ "migrate-fail-9", "Failed migrate_to + stop on source and target" ],
         [ "migration-ping-pong", "Old migrate_to failure + successful migrate_from on same node" ],
         [ "migrate-stop", "Migration in a stopping stack" ],
         [ "migrate-start", "Migration in a starting stack" ],
         [ "migrate-stop_start", "Migration in a restarting stack" ],
         [ "migrate-stop-complex", "Migration in a complex stopping stack" ],
         [ "migrate-start-complex", "Migration in a complex starting stack" ],
         [ "migrate-stop-start-complex", "Migration in a complex moving stack" ],
         [ "migrate-shutdown", "Order the post-migration 'stop' before node shutdown" ],
         [ "migrate-1", "Migrate (migrate)" ],
         [ "migrate-2", "Migrate (stable)" ],
         [ "migrate-3", "Migrate (failed migrate_to)" ],
         [ "migrate-4", "Migrate (failed migrate_from)" ],
         [ "novell-252693", "Migration in a stopping stack" ],
         [ "novell-252693-2", "Migration in a starting stack" ],
         [ "novell-252693-3", "Non-Migration in a starting and stopping stack" ],
         [ "bug-1820", "Migration in a group" ],
         [ "bug-1820-1", "Non-migration in a group" ],
         [ "migrate-5", "Primitive migration with a clone" ],
         [ "migrate-fencing", "Migration after Fencing" ],
         [ "migrate-both-vms", "Migrate two VMs that have no colocation" ],
         [ "migration-behind-migrating-remote", "Migrate resource behind migrating remote connection" ],
         [ "1-a-then-bm-move-b", "Advanced migrate logic. A then B. migrate B" ],
         [ "2-am-then-b-move-a", "Advanced migrate logic, A then B, migrate A without stopping B" ],
         [ "3-am-then-bm-both-migrate", "Advanced migrate logic. A then B. migrate both" ],
         [ "4-am-then-bm-b-not-migratable", "Advanced migrate logic, A then B, B not migratable" ],
         [ "5-am-then-bm-a-not-migratable", "Advanced migrate logic. A then B. move both, a not migratable" ],
         [ "6-migrate-group", "Advanced migrate logic, migrate a group" ],
         [ "7-migrate-group-one-unmigratable",
           "Advanced migrate logic, migrate group mixed with allow-migrate true/false" ],
         [ "8-am-then-bm-a-migrating-b-stopping",
           "Advanced migrate logic, A then B, A migrating, B stopping" ],
         [ "9-am-then-bm-b-migrating-a-stopping",
           "Advanced migrate logic, A then B, B migrate, A stopping" ],
         [ "10-a-then-bm-b-move-a-clone",
           "Advanced migrate logic, A clone then B, migrate B while stopping A" ],
         [ "11-a-then-bm-b-move-a-clone-starting",
           "Advanced migrate logic, A clone then B, B moving while A is start/stopping" ],
         [ "a-promote-then-b-migrate", "A promote then B start. migrate B" ],
         [ "a-demote-then-b-migrate", "A demote then B stop. migrate B" ],
 
         # @TODO: If pacemaker implements versioned attributes, uncomment this test
         #[ "migrate-versioned", "Disable migration for versioned resources" ],
 
         [ "bug-lf-2422", "Dependency on partially active group - stop ocfs:*" ],
     ],
     [
         [ "clone-anon-probe-1", "Probe the correct (anonymous) clone instance for each node" ],
         [ "clone-anon-probe-2", "Avoid needless re-probing of anonymous clones" ],
         [ "clone-anon-failcount", "Merge failcounts for anonymous clones" ],
         [ "force-anon-clone-max", "Update clone-max properly when forcing a clone to be anonymous" ],
         [ "anon-instance-pending", "Assign anonymous clone instance numbers properly when action pending" ],
         [ "inc0", "Incarnation start" ],
         [ "inc1", "Incarnation start order" ],
         [ "inc2", "Incarnation silent restart, stop, move" ],
         [ "inc3", "Inter-incarnation ordering, silent restart, stop, move" ],
         [ "inc4", "Inter-incarnation ordering, silent restart, stop, move (ordered)" ],
         [ "inc5", "Inter-incarnation ordering, silent restart, stop, move (restart 1)" ],
         [ "inc6", "Inter-incarnation ordering, silent restart, stop, move (restart 2)" ],
         [ "inc7", "Clone colocation" ],
         [ "inc8", "Clone anti-colocation" ],
         [ "inc9", "Non-unique clone" ],
         [ "inc10", "Non-unique clone (stop)" ],
         [ "inc11", "Primitive colocation with clones" ],
         [ "inc12", "Clone shutdown" ],
         [ "cloned-group", "Make sure only the correct number of cloned groups are started" ],
         [ "cloned-group-stop", "Ensure stopping qpidd also stops glance and cinder" ],
         [ "clone-no-shuffle", "Don't prioritize allocation of instances that must be moved" ],
         [ "clone-max-zero", "Orphan processing with clone-max=0" ],
         [ "clone-anon-dup",
           "Bug LF#2087 - Correctly parse the state of anonymous clones that are active more than once per node" ],
         [ "bug-lf-2160", "Don't shuffle clones due to colocation" ],
         [ "bug-lf-2213", "clone-node-max enforcement for cloned groups" ],
         [ "bug-lf-2153", "Clone ordering constraints" ],
         [ "bug-lf-2361", "Ensure clones observe mandatory ordering constraints if the LHS is unrunnable" ],
         [ "bug-lf-2317", "Avoid needless restart of primitive depending on a clone" ],
         [ "clone-colocate-instance-1", "Colocation with a specific clone instance (negative example)" ],
         [ "clone-colocate-instance-2", "Colocation with a specific clone instance" ],
         [ "clone-order-instance", "Ordering with specific clone instances" ],
         [ "bug-lf-2453", "Enforce mandatory clone ordering without colocation" ],
         [ "bug-lf-2508", "Correctly reconstruct the status of anonymous cloned groups" ],
         [ "bug-lf-2544", "Balanced clone placement" ],
         [ "bug-lf-2445", "Redistribute clones with node-max > 1 and stickiness = 0" ],
         [ "bug-lf-2574", "Avoid clone shuffle" ],
         [ "bug-lf-2581", "Avoid group restart due to unrelated clone (re)start" ],
         [ "bug-cl-5168", "Don't shuffle clones" ],
         [ "bug-cl-5170", "Prevent clone from starting with on-fail=block" ],
         [ "clone-fail-block-colocation", "Move colocated group when failed clone has on-fail=block" ],
         [ "clone-interleave-1",
           "Clone-3 cannot start on pcmk-1 due to interleaved ordering (no colocation)" ],
         [ "clone-interleave-2", "Clone-3 must stop on pcmk-1 due to interleaved ordering (no colocation)" ],
         [ "clone-interleave-3",
           "Clone-3 must be recovered on pcmk-1 due to interleaved ordering (no colocation)" ],
         [ "rebalance-unique-clones", "Rebalance unique clone instances with no stickiness" ],
         [ "clone-requires-quorum-recovery", "Clone with requires=quorum on failed node needing recovery" ],
         [ "clone-requires-quorum",
           "Clone with requires=quorum with presumed-inactive instance on failed node" ],
     ],
     [
         [ "cloned_start_one", "order first clone then clone... first clone_min=2" ],
         [ "cloned_start_two", "order first clone then clone... first clone_min=2" ],
         [ "cloned_stop_one", "order first clone then clone... first clone_min=2" ],
         [ "cloned_stop_two", "order first clone then clone... first clone_min=2" ],
         [ "clone_min_interleave_start_one",
           "order first clone then clone... first clone_min=2 and then has interleave=true" ],
         [ "clone_min_interleave_start_two",
           "order first clone then clone... first clone_min=2 and then has interleave=true" ],
         [ "clone_min_interleave_stop_one",
           "order first clone then clone... first clone_min=2 and then has interleave=true" ],
         [ "clone_min_interleave_stop_two",
           "order first clone then clone... first clone_min=2 and then has interleave=true" ],
         [ "clone_min_start_one", "order first clone then primitive... first clone_min=2" ],
         [ "clone_min_start_two", "order first clone then primitive... first clone_min=2" ],
         [ "clone_min_stop_all", "order first clone then primitive... first clone_min=2" ],
         [ "clone_min_stop_one", "order first clone then primitive... first clone_min=2" ],
         [ "clone_min_stop_two", "order first clone then primitive... first clone_min=2" ],
     ],
     [
         [ "unfence-startup", "Clean unfencing" ],
         [ "unfence-definition", "Unfencing when the agent changes" ],
         [ "unfence-parameters", "Unfencing when the agent parameters changes" ],
         [ "unfence-device", "Unfencing when a cluster has only fence devices" ],
     ],
     [
         [ "promoted-0", "Stopped -> Unpromoted" ],
         [ "promoted-1", "Stopped -> Promote" ],
         [ "promoted-2", "Stopped -> Promote : notify" ],
         [ "promoted-3", "Stopped -> Promote : promoted location" ],
         [ "promoted-4", "Started -> Promote : promoted location" ],
         [ "promoted-5", "Promoted -> Promoted" ],
         [ "promoted-6", "Promoted -> Promoted (2)" ],
         [ "promoted-7", "Promoted -> Fenced" ],
         [ "promoted-8", "Promoted -> Fenced -> Moved" ],
         [ "promoted-9", "Stopped + Promotable + No quorum" ],
         [ "promoted-10", "Stopped -> Promotable : notify with monitor" ],
         [ "promoted-11", "Stopped -> Promote : colocation" ],
         [ "novell-239082", "Demote/Promote ordering" ],
         [ "novell-239087", "Stable promoted placement" ],
         [ "promoted-12", "Promotion based solely on rsc_location constraints" ],
         [ "promoted-13", "Include preferences of colocated resources when placing promoted" ],
         [ "promoted-demote", "Ordering when actions depends on demoting an unpromoted resource" ],
         [ "promoted-ordering", "Prevent resources from starting that need a promoted" ],
         [ "bug-1765", "Verify promoted-with-promoted colocation does not stop unpromoted instances" ],
         [ "promoted-group", "Promotion of cloned groups" ],
         [ "bug-lf-1852", "Don't shuffle promotable instances unnecessarily" ],
         [ "promoted-failed-demote", "Don't retry failed demote actions" ],
         [ "promoted-failed-demote-2", "Don't retry failed demote actions (notify=false)" ],
         [ "promoted-depend",
           "Ensure resources that depend on promoted instance don't get allocated until that does" ],
         [ "promoted-reattach", "Re-attach to a running promoted" ],
         [ "promoted-allow-start", "Don't include promoted score if it would prevent allocation" ],
         [ "promoted-colocation",
           "Allow promoted instances placemaker to be influenced by colocation constraints" ],
         [ "promoted-pseudo", "Make sure promote/demote pseudo actions are created correctly" ],
         [ "promoted-role", "Prevent target-role from promoting more than promoted-max instances" ],
         [ "bug-lf-2358", "Anti-colocation of promoted instances" ],
         [ "promoted-promotion-constraint", "Mandatory promoted colocation constraints" ],
         [ "unmanaged-promoted", "Ensure role is preserved for unmanaged resources" ],
         [ "promoted-unmanaged-monitor", "Start correct monitor for unmanaged promoted instances" ],
         [ "promoted-demote-2", "Demote does not clear past failure" ],
         [ "promoted-move", "Move promoted based on failure of colocated group" ],
         [ "promoted-probed-score", "Observe the promotion score of probed resources" ],
         [ "colocation_constraint_stops_promoted",
           "cl#5054 - Ensure promoted is demoted when stopped by colocation constraint" ],
         [ "colocation_constraint_stops_unpromoted",
           "cl#5054 - Ensure unpromoted is not demoted when stopped by colocation constraint" ],
         [ "order_constraint_stops_promoted",
           "cl#5054 - Ensure promoted is demoted when stopped by order constraint" ],
         [ "order_constraint_stops_unpromoted",
           "cl#5054 - Ensure unpromoted is not demoted when stopped by order constraint" ],
         [ "promoted_monitor_restart", "cl#5072 - Ensure promoted monitor operation will start after promotion" ],
         [ "bug-rh-880249", "Handle replacement of an m/s resource with a primitive" ],
         [ "bug-5143-ms-shuffle", "Prevent promoted instance shuffling due to promotion score" ],
         [ "promoted-demote-block", "Block promotion if demote fails with on-fail=block" ],
         [ "promoted-dependent-ban",
           "Don't stop instances from being active because a dependent is banned from that host" ],
         [ "promoted-stop", "Stop instances due to location constraint with role=Started" ],
         [ "promoted-partially-demoted-group", "Allow partially demoted group to finish demoting" ],
         [ "bug-cl-5213", "Ensure role colocation with -INFINITY is enforced" ],
         [ "bug-cl-5219", "Allow unrelated resources with a common colocation target to remain promoted" ],
         [ "promoted-asymmetrical-order",
           "Fix the behaviors of multi-state resources with asymmetrical ordering" ],
         [ "promoted-notify", "Promotion with notifications" ],
         [ "promoted-score-startup", "Use permanent promoted scores without LRM history" ],
         [ "failed-demote-recovery", "Recover resource in unpromoted role after demote fails" ],
         [ "failed-demote-recovery-promoted", "Recover resource in promoted role after demote fails" ],
         [ "on_fail_demote1", "Recovery with on-fail=\"demote\" on healthy cluster, remote, guest, and bundle nodes" ],
         [ "on_fail_demote2", "Recovery with on-fail=\"demote\" with promotion on different node" ],
         [ "on_fail_demote3", "Recovery with on-fail=\"demote\" with no promotion" ],
         [ "on_fail_demote4", "Recovery with on-fail=\"demote\" on failed cluster, remote, guest, and bundle nodes" ],
         [ "no_quorum_demote", "Promotable demotion and primitive stop with no-quorum-policy=\"demote\"" ],
         [ "no-promote-on-unrunnable-guest", "Don't select bundle instance for promotion when container can't run" ],
     ],
     [
         [ "history-1", "Correctly parse stateful-1 resource state" ],
     ],
     [
         [ "managed-0", "Managed (reference)" ],
         [ "managed-1", "Not managed - down" ],
         [ "managed-2", "Not managed - up" ],
         [ "bug-5028", "Shutdown should block if anything depends on an unmanaged resource" ],
         [ "bug-5028-detach", "Ensure detach still works" ],
         [ "bug-5028-bottom",
           "Ensure shutdown still blocks if the blocked resource is at the bottom of the stack" ],
         [ "unmanaged-stop-1",
           "cl#5155 - Block the stop of resources if any depending resource is unmanaged" ],
         [ "unmanaged-stop-2",
           "cl#5155 - Block the stop of resources if the first resource in a mandatory stop order is unmanaged" ],
         [ "unmanaged-stop-3",
           "cl#5155 - Block the stop of resources if any depending resource in a group is unmanaged" ],
         [ "unmanaged-stop-4",
           "cl#5155 - Block the stop of resources if any depending resource in the middle of a group is unmanaged" ],
         [ "unmanaged-block-restart",
           "Block restart of resources if any dependent resource in a group is unmanaged" ],
     ],
     [
         [ "interleave-0", "Interleave (reference)" ],
         [ "interleave-1", "coloc - not interleaved" ],
         [ "interleave-2", "coloc - interleaved" ],
         [ "interleave-3", "coloc - interleaved (2)" ],
         [ "interleave-pseudo-stop", "Interleaved clone during stonith" ],
         [ "interleave-stop", "Interleaved clone during stop" ],
         [ "interleave-restart", "Interleaved clone during dependency restart" ],
     ],
     [
         [ "notify-0", "Notify reference" ],
         [ "notify-1", "Notify simple" ],
         [ "notify-2", "Notify simple, confirm" ],
         [ "notify-3", "Notify move, confirm" ],
         [ "novell-239079", "Notification priority" ],
         #[ "notify-2", "Notify - 764" ],
         [ "notifs-for-unrunnable", "Don't schedule notifications for an unrunnable action" ],
         [ "route-remote-notify", "Route remote notify actions through correct cluster node" ],
         [ "notify-behind-stopping-remote", "Don't schedule notifications behind stopped remote" ],
     ],
     [
         [ "594", "OSDL #594 - Unrunnable actions scheduled in transition" ],
         [ "662", "OSDL #662 - Two resources start on one node when incarnation_node_max = 1" ],
         [ "696", "OSDL #696 - CRM starts stonith RA without monitor" ],
         [ "726", "OSDL #726 - Attempting to schedule rsc_posic041_monitor_5000 _after_ a stop" ],
         [ "735", "OSDL #735 - Correctly detect that rsc_hadev1 is stopped on hadev3" ],
         [ "764", "OSDL #764 - Missing monitor op for DoFencing:child_DoFencing:1" ],
         [ "797", "OSDL #797 - Assert triggered: task_id_i > max_call_id" ],
         [ "829", "OSDL #829" ],
         [ "994",
           "OSDL #994 - Stopping the last resource in a resource group causes the entire group to be restarted" ],
         [ "994-2", "OSDL #994 - with a dependent resource" ],
         [ "1360", "OSDL #1360 - Clone stickiness" ],
         [ "1484", "OSDL #1484 - on_fail=stop" ],
         [ "1494", "OSDL #1494 - Clone stability" ],
         [ "unrunnable-1", "Unrunnable" ],
         [ "unrunnable-2", "Unrunnable 2" ],
         [ "stonith-0", "Stonith loop - 1" ],
         [ "stonith-1", "Stonith loop - 2" ],
         [ "stonith-2", "Stonith loop - 3" ],
         [ "stonith-3", "Stonith startup" ],
         [ "stonith-4", "Stonith node state" ],
         [ "dc-fence-ordering", "DC needs fencing while other nodes are shutting down" ],
         [ "bug-1572-1", "Recovery of groups depending on promotable role" ],
         [ "bug-1572-2", "Recovery of groups depending on promotable role when promoted is not re-promoted" ],
         [ "bug-1685", "Depends-on-promoted ordering" ],
         [ "bug-1822", "Don't promote partially active groups" ],
         [ "bug-pm-11", "New resource added to a m/s group" ],
         [ "bug-pm-12", "Recover only the failed portion of a cloned group" ],
         [ "bug-n-387749", "Don't shuffle clone instances" ],
         [ "bug-n-385265",
           "Don't ignore the failure stickiness of group children - resource_idvscommon should stay stopped" ],
         [ "bug-n-385265-2",
           "Ensure groups are migrated instead of remaining partially active on the current node" ],
         [ "bug-lf-1920", "Correctly handle probes that find active resources" ],
         [ "bnc-515172", "Location constraint with multiple expressions" ],
         [ "colocate-primitive-with-clone", "Optional colocation with a clone" ],
         [ "use-after-free-merge", "Use-after-free in native_merge_weights" ],
         [ "bug-lf-2551", "STONITH ordering for stop" ],
         [ "bug-lf-2606", "Stonith implies demote" ],
         [ "bug-lf-2474", "Ensure resource op timeout takes precedence over op_defaults" ],
         [ "bug-suse-707150", "Prevent vm-01 from starting due to colocation/ordering" ],
         [ "bug-5014-A-start-B-start", "Verify when A starts B starts using symmetrical=false" ],
         [ "bug-5014-A-stop-B-started",
           "Verify when A stops B does not stop if it has already started using symmetric=false" ],
         [ "bug-5014-A-stopped-B-stopped",
           "Verify when A is stopped and B has not started, B does not start before A using symmetric=false" ],
         [ "bug-5014-CthenAthenB-C-stopped",
           "Verify when C then A is symmetrical=true, A then B is symmetric=false, and C is stopped that nothing starts" ],
         [ "bug-5014-CLONE-A-start-B-start",
           "Verify when A starts B starts using clone resources with symmetric=false" ],
         [ "bug-5014-CLONE-A-stop-B-started",
           "Verify when A stops B does not stop if it has already started using clone resources with symmetric=false" ],
         [ "bug-5014-GROUP-A-start-B-start",
           "Verify when A starts B starts when using group resources with symmetric=false" ],
         [ "bug-5014-GROUP-A-stopped-B-started",
           "Verify when A stops B does not stop if it has already started using group resources with symmetric=false" ],
         [ "bug-5014-GROUP-A-stopped-B-stopped",
           "Verify when A is stopped and B has not started, B does not start before A using group resources with symmetric=false" ],
         [ "bug-5014-ordered-set-symmetrical-false",
           "Verify ordered sets work with symmetrical=false" ],
         [ "bug-5014-ordered-set-symmetrical-true",
           "Verify ordered sets work with symmetrical=true" ],
         [ "clbz5007-promotable-colocation",
           "Verify use of colocation scores other than INFINITY and -INFINITY work on multi-state resources" ],
         [ "bug-5038", "Prevent restart of anonymous clones when clone-max decreases" ],
         [ "bug-5025-1", "Automatically clean up failcount after resource config change with reload" ],
         [ "bug-5025-2", "Make sure clear failcount action isn't set when config does not change" ],
         [ "bug-5025-3", "Automatically clean up failcount after resource config change with restart" ],
         [ "bug-5025-4", "Clear failcount when last failure is a start op and rsc attributes changed" ],
         [ "failcount", "Ensure failcounts are correctly expired" ],
         [ "failcount-block", "Ensure failcounts are not expired when on-fail=block is present" ],
         [ "per-op-failcount", "Ensure per-operation failcount is handled and not passed to fence agent" ],
         [ "on-fail-ignore", "Ensure on-fail=ignore works even beyond migration-threshold" ],
         [ "monitor-onfail-restart", "bug-5058 - Monitor failure with on-fail set to restart" ],
         [ "monitor-onfail-stop", "bug-5058 - Monitor failure wiht on-fail set to stop" ],
         [ "bug-5059", "No need to restart p_stateful1:*" ],
         [ "bug-5069-op-enabled", "Test on-fail=ignore with failure when monitor is enabled" ],
         [ "bug-5069-op-disabled", "Test on-fail-ignore with failure when monitor is disabled" ],
         [ "obsolete-lrm-resource", "cl#5115 - Do not use obsolete lrm_resource sections" ],
         [ "expire-non-blocked-failure",
           "Ignore failure-timeout only if the failed operation has on-fail=block" ],
         [ "asymmetrical-order-move", "Respect asymmetrical ordering when trying to move resources" ],
         [ "asymmetrical-order-restart", "Respect asymmetrical ordering when restarting dependent resource" ],
         [ "start-then-stop-with-unfence", "Avoid graph loop with start-then-stop constraint plus unfencing" ],
         [ "order-expired-failure", "Order failcount cleanup after remote fencing" ],
     
         [ "ignore_stonith_rsc_order1",
           "cl#5056- Ignore order constraint between stonith and non-stonith rsc" ],
         [ "ignore_stonith_rsc_order2",
           "cl#5056- Ignore order constraint with group rsc containing mixed stonith and non-stonith" ],
         [ "ignore_stonith_rsc_order3", "cl#5056- Ignore order constraint, stonith clone and mixed group" ],
         [ "ignore_stonith_rsc_order4",
           "cl#5056- Ignore order constraint, stonith clone and clone with nested mixed group" ],
         [ "honor_stonith_rsc_order1",
           "cl#5056- Honor order constraint, stonith clone and pure stonith group(single rsc)" ],
         [ "honor_stonith_rsc_order2",
           "cl#5056- Honor order constraint, stonith clone and pure stonith group(multiple rsc)" ],
         [ "honor_stonith_rsc_order3",
           "cl#5056- Honor order constraint, stonith clones with nested pure stonith group" ],
         [ "honor_stonith_rsc_order4",
           "cl#5056- Honor order constraint, between two native stonith rscs" ],
         [ "multiply-active-stonith", "Multiply active stonith" ],
         [ "probe-timeout", "cl#5099 - Default probe timeout" ],
         [ "order-first-probes",
           "cl#5301 - respect order constraints when relevant resources are being probed" ],
         [ "concurrent-fencing", "Allow performing fencing operations in parallel" ],
         [ "priority-fencing-delay", "Delay fencing targeting the more significant node" ],
     ],
     [
         [ "systemhealth1", "System Health ()               #1" ],
         [ "systemhealth2", "System Health ()               #2" ],
         [ "systemhealth3", "System Health ()               #3" ],
         [ "systemhealthn1", "System Health (None)           #1" ],
         [ "systemhealthn2", "System Health (None)           #2" ],
         [ "systemhealthn3", "System Health (None)           #3" ],
         [ "systemhealthm1", "System Health (Migrate On Red) #1" ],
         [ "systemhealthm2", "System Health (Migrate On Red) #2" ],
         [ "systemhealthm3", "System Health (Migrate On Red) #3" ],
         [ "systemhealtho1", "System Health (Only Green)     #1" ],
         [ "systemhealtho2", "System Health (Only Green)     #2" ],
         [ "systemhealtho3", "System Health (Only Green)     #3" ],
         [ "systemhealthp1", "System Health (Progessive)     #1" ],
         [ "systemhealthp2", "System Health (Progessive)     #2" ],
         [ "systemhealthp3", "System Health (Progessive)     #3" ],
     ],
     [
         [ "utilization", "Placement Strategy - utilization" ],
         [ "minimal", "Placement Strategy - minimal" ],
         [ "balanced", "Placement Strategy - balanced" ],
     ],
     [
         [ "placement-stickiness", "Optimized Placement Strategy - stickiness" ],
         [ "placement-priority", "Optimized Placement Strategy - priority" ],
         [ "placement-location", "Optimized Placement Strategy - location" ],
         [ "placement-capacity", "Optimized Placement Strategy - capacity" ],
     ],
     [
         [ "utilization-order1", "Utilization Order - Simple" ],
         [ "utilization-order2", "Utilization Order - Complex" ],
         [ "utilization-order3", "Utilization Order - Migrate" ],
         [ "utilization-order4", "Utilization Order - Live Migration (bnc#695440)" ],
         [ "utilization-shuffle",
           "Don't displace prmExPostgreSQLDB2 on act2, Start prmExPostgreSQLDB1 on act3" ],
         [ "load-stopped-loop", "Avoid transition loop due to load_stopped (cl#5044)" ],
         [ "load-stopped-loop-2",
           "cl#5235 - Prevent graph loops that can be introduced by load_stopped -> migrate_to ordering" ],
     ],
     [
         [ "colocated-utilization-primitive-1", "Colocated Utilization - Primitive" ],
         [ "colocated-utilization-primitive-2", "Colocated Utilization - Choose the most capable node" ],
         [ "colocated-utilization-group", "Colocated Utilization - Group" ],
         [ "colocated-utilization-clone", "Colocated Utilization - Clone" ],
         [ "utilization-check-allowed-nodes",
           "Only check the capacities of the nodes that can run the resource" ],
     ],
     [
         [ "reprobe-target_rc", "Ensure correct target_rc for reprobe of inactive resources" ],
         [ "node-maintenance-1", "cl#5128 - Node maintenance" ],
         [ "node-maintenance-2", "cl#5128 - Node maintenance (coming out of maintenance mode)" ],
         [ "shutdown-maintenance-node", "Do not fence a maintenance node if it shuts down cleanly" ],
         [ "rsc-maintenance", "Per-resource maintenance" ],
     ],
     [
         [ "not-installed-agent", "The resource agent is missing" ],
         [ "not-installed-tools", "Something the resource agent needs is missing" ],
     ],
     [
         [ "stopped-monitor-00", "Stopped Monitor - initial start" ],
         [ "stopped-monitor-01", "Stopped Monitor - failed started" ],
         [ "stopped-monitor-02", "Stopped Monitor - started multi-up" ],
         [ "stopped-monitor-03", "Stopped Monitor - stop started" ],
         [ "stopped-monitor-04", "Stopped Monitor - failed stop" ],
         [ "stopped-monitor-05", "Stopped Monitor - start unmanaged" ],
         [ "stopped-monitor-06", "Stopped Monitor - unmanaged multi-up" ],
         [ "stopped-monitor-07", "Stopped Monitor - start unmanaged multi-up" ],
         [ "stopped-monitor-08", "Stopped Monitor - migrate" ],
         [ "stopped-monitor-09", "Stopped Monitor - unmanage started" ],
         [ "stopped-monitor-10", "Stopped Monitor - unmanaged started multi-up" ],
         [ "stopped-monitor-11", "Stopped Monitor - stop unmanaged started" ],
         [ "stopped-monitor-12", "Stopped Monitor - unmanaged started multi-up (target-role=Stopped)" ],
         [ "stopped-monitor-20", "Stopped Monitor - initial stop" ],
         [ "stopped-monitor-21", "Stopped Monitor - stopped single-up" ],
         [ "stopped-monitor-22", "Stopped Monitor - stopped multi-up" ],
         [ "stopped-monitor-23", "Stopped Monitor - start stopped" ],
         [ "stopped-monitor-24", "Stopped Monitor - unmanage stopped" ],
         [ "stopped-monitor-25", "Stopped Monitor - unmanaged stopped multi-up" ],
         [ "stopped-monitor-26", "Stopped Monitor - start unmanaged stopped" ],
         [ "stopped-monitor-27", "Stopped Monitor - unmanaged stopped multi-up (target-role=Started)" ],
         [ "stopped-monitor-30", "Stopped Monitor - new node started" ],
         [ "stopped-monitor-31", "Stopped Monitor - new node stopped" ],
     ],
     [
         # This is a combo test to check:
         # - probe timeout defaults to the minimum-interval monitor's
         # - duplicate recurring operations are ignored
         # - if timeout spec is bad, the default timeout is used
         # - failure is blocked with on-fail=block even if ISO8601 interval is specified
         # - started/stopped role monitors are started/stopped on right nodes
         [ "intervals", "Recurring monitor interval handling" ],
     ],
     [
         [ "ticket-primitive-1", "Ticket - Primitive (loss-policy=stop, initial)" ],
         [ "ticket-primitive-2", "Ticket - Primitive (loss-policy=stop, granted)" ],
         [ "ticket-primitive-3", "Ticket - Primitive (loss-policy-stop, revoked)" ],
         [ "ticket-primitive-4", "Ticket - Primitive (loss-policy=demote, initial)" ],
         [ "ticket-primitive-5", "Ticket - Primitive (loss-policy=demote, granted)" ],
         [ "ticket-primitive-6", "Ticket - Primitive (loss-policy=demote, revoked)" ],
         [ "ticket-primitive-7", "Ticket - Primitive (loss-policy=fence, initial)" ],
         [ "ticket-primitive-8", "Ticket - Primitive (loss-policy=fence, granted)" ],
         [ "ticket-primitive-9", "Ticket - Primitive (loss-policy=fence, revoked)" ],
         [ "ticket-primitive-10", "Ticket - Primitive (loss-policy=freeze, initial)" ],
         [ "ticket-primitive-11", "Ticket - Primitive (loss-policy=freeze, granted)" ],
         [ "ticket-primitive-12", "Ticket - Primitive (loss-policy=freeze, revoked)" ],
         [ "ticket-primitive-13", "Ticket - Primitive (loss-policy=stop, standby, granted)" ],
         [ "ticket-primitive-14", "Ticket - Primitive (loss-policy=stop, granted, standby)" ],
         [ "ticket-primitive-15", "Ticket - Primitive (loss-policy=stop, standby, revoked)" ],
         [ "ticket-primitive-16", "Ticket - Primitive (loss-policy=demote, standby, granted)" ],
         [ "ticket-primitive-17", "Ticket - Primitive (loss-policy=demote, granted, standby)" ],
         [ "ticket-primitive-18", "Ticket - Primitive (loss-policy=demote, standby, revoked)" ],
         [ "ticket-primitive-19", "Ticket - Primitive (loss-policy=fence, standby, granted)" ],
         [ "ticket-primitive-20", "Ticket - Primitive (loss-policy=fence, granted, standby)" ],
         [ "ticket-primitive-21", "Ticket - Primitive (loss-policy=fence, standby, revoked)" ],
         [ "ticket-primitive-22", "Ticket - Primitive (loss-policy=freeze, standby, granted)" ],
         [ "ticket-primitive-23", "Ticket - Primitive (loss-policy=freeze, granted, standby)" ],
         [ "ticket-primitive-24", "Ticket - Primitive (loss-policy=freeze, standby, revoked)" ],
     ],
     [
         [ "ticket-group-1", "Ticket - Group (loss-policy=stop, initial)" ],
         [ "ticket-group-2", "Ticket - Group (loss-policy=stop, granted)" ],
         [ "ticket-group-3", "Ticket - Group (loss-policy-stop, revoked)" ],
         [ "ticket-group-4", "Ticket - Group (loss-policy=demote, initial)" ],
         [ "ticket-group-5", "Ticket - Group (loss-policy=demote, granted)" ],
         [ "ticket-group-6", "Ticket - Group (loss-policy=demote, revoked)" ],
         [ "ticket-group-7", "Ticket - Group (loss-policy=fence, initial)" ],
         [ "ticket-group-8", "Ticket - Group (loss-policy=fence, granted)" ],
         [ "ticket-group-9", "Ticket - Group (loss-policy=fence, revoked)" ],
         [ "ticket-group-10", "Ticket - Group (loss-policy=freeze, initial)" ],
         [ "ticket-group-11", "Ticket - Group (loss-policy=freeze, granted)" ],
         [ "ticket-group-12", "Ticket - Group (loss-policy=freeze, revoked)" ],
         [ "ticket-group-13", "Ticket - Group (loss-policy=stop, standby, granted)" ],
         [ "ticket-group-14", "Ticket - Group (loss-policy=stop, granted, standby)" ],
         [ "ticket-group-15", "Ticket - Group (loss-policy=stop, standby, revoked)" ],
         [ "ticket-group-16", "Ticket - Group (loss-policy=demote, standby, granted)" ],
         [ "ticket-group-17", "Ticket - Group (loss-policy=demote, granted, standby)" ],
         [ "ticket-group-18", "Ticket - Group (loss-policy=demote, standby, revoked)" ],
         [ "ticket-group-19", "Ticket - Group (loss-policy=fence, standby, granted)" ],
         [ "ticket-group-20", "Ticket - Group (loss-policy=fence, granted, standby)" ],
         [ "ticket-group-21", "Ticket - Group (loss-policy=fence, standby, revoked)" ],
         [ "ticket-group-22", "Ticket - Group (loss-policy=freeze, standby, granted)" ],
         [ "ticket-group-23", "Ticket - Group (loss-policy=freeze, granted, standby)" ],
         [ "ticket-group-24", "Ticket - Group (loss-policy=freeze, standby, revoked)" ],
     ],
     [
         [ "ticket-clone-1", "Ticket - Clone (loss-policy=stop, initial)" ],
         [ "ticket-clone-2", "Ticket - Clone (loss-policy=stop, granted)" ],
         [ "ticket-clone-3", "Ticket - Clone (loss-policy-stop, revoked)" ],
         [ "ticket-clone-4", "Ticket - Clone (loss-policy=demote, initial)" ],
         [ "ticket-clone-5", "Ticket - Clone (loss-policy=demote, granted)" ],
         [ "ticket-clone-6", "Ticket - Clone (loss-policy=demote, revoked)" ],
         [ "ticket-clone-7", "Ticket - Clone (loss-policy=fence, initial)" ],
         [ "ticket-clone-8", "Ticket - Clone (loss-policy=fence, granted)" ],
         [ "ticket-clone-9", "Ticket - Clone (loss-policy=fence, revoked)" ],
         [ "ticket-clone-10", "Ticket - Clone (loss-policy=freeze, initial)" ],
         [ "ticket-clone-11", "Ticket - Clone (loss-policy=freeze, granted)" ],
         [ "ticket-clone-12", "Ticket - Clone (loss-policy=freeze, revoked)" ],
         [ "ticket-clone-13", "Ticket - Clone (loss-policy=stop, standby, granted)" ],
         [ "ticket-clone-14", "Ticket - Clone (loss-policy=stop, granted, standby)" ],
         [ "ticket-clone-15", "Ticket - Clone (loss-policy=stop, standby, revoked)" ],
         [ "ticket-clone-16", "Ticket - Clone (loss-policy=demote, standby, granted)" ],
         [ "ticket-clone-17", "Ticket - Clone (loss-policy=demote, granted, standby)" ],
         [ "ticket-clone-18", "Ticket - Clone (loss-policy=demote, standby, revoked)" ],
         [ "ticket-clone-19", "Ticket - Clone (loss-policy=fence, standby, granted)" ],
         [ "ticket-clone-20", "Ticket - Clone (loss-policy=fence, granted, standby)" ],
         [ "ticket-clone-21", "Ticket - Clone (loss-policy=fence, standby, revoked)" ],
         [ "ticket-clone-22", "Ticket - Clone (loss-policy=freeze, standby, granted)" ],
         [ "ticket-clone-23", "Ticket - Clone (loss-policy=freeze, granted, standby)" ],
         [ "ticket-clone-24", "Ticket - Clone (loss-policy=freeze, standby, revoked)" ],
     ],
     [
         [ "ticket-promoted-1", "Ticket - Promoted (loss-policy=stop, initial)" ],
         [ "ticket-promoted-2", "Ticket - Promoted (loss-policy=stop, granted)" ],
         [ "ticket-promoted-3", "Ticket - Promoted (loss-policy-stop, revoked)" ],
         [ "ticket-promoted-4", "Ticket - Promoted (loss-policy=demote, initial)" ],
         [ "ticket-promoted-5", "Ticket - Promoted (loss-policy=demote, granted)" ],
         [ "ticket-promoted-6", "Ticket - Promoted (loss-policy=demote, revoked)" ],
         [ "ticket-promoted-7", "Ticket - Promoted (loss-policy=fence, initial)" ],
         [ "ticket-promoted-8", "Ticket - Promoted (loss-policy=fence, granted)" ],
         [ "ticket-promoted-9", "Ticket - Promoted (loss-policy=fence, revoked)" ],
         [ "ticket-promoted-10", "Ticket - Promoted (loss-policy=freeze, initial)" ],
         [ "ticket-promoted-11", "Ticket - Promoted (loss-policy=freeze, granted)" ],
         [ "ticket-promoted-12", "Ticket - Promoted (loss-policy=freeze, revoked)" ],
         [ "ticket-promoted-13", "Ticket - Promoted (loss-policy=stop, standby, granted)" ],
         [ "ticket-promoted-14", "Ticket - Promoted (loss-policy=stop, granted, standby)" ],
         [ "ticket-promoted-15", "Ticket - Promoted (loss-policy=stop, standby, revoked)" ],
         [ "ticket-promoted-16", "Ticket - Promoted (loss-policy=demote, standby, granted)" ],
         [ "ticket-promoted-17", "Ticket - Promoted (loss-policy=demote, granted, standby)" ],
         [ "ticket-promoted-18", "Ticket - Promoted (loss-policy=demote, standby, revoked)" ],
         [ "ticket-promoted-19", "Ticket - Promoted (loss-policy=fence, standby, granted)" ],
         [ "ticket-promoted-20", "Ticket - Promoted (loss-policy=fence, granted, standby)" ],
         [ "ticket-promoted-21", "Ticket - Promoted (loss-policy=fence, standby, revoked)" ],
         [ "ticket-promoted-22", "Ticket - Promoted (loss-policy=freeze, standby, granted)" ],
         [ "ticket-promoted-23", "Ticket - Promoted (loss-policy=freeze, granted, standby)" ],
         [ "ticket-promoted-24", "Ticket - Promoted (loss-policy=freeze, standby, revoked)" ],
     ],
     [
         [ "ticket-rsc-sets-1", "Ticket - Resource sets (1 ticket, initial)" ],
         [ "ticket-rsc-sets-2", "Ticket - Resource sets (1 ticket, granted)" ],
         [ "ticket-rsc-sets-3", "Ticket - Resource sets (1 ticket, revoked)" ],
         [ "ticket-rsc-sets-4", "Ticket - Resource sets (2 tickets, initial)" ],
         [ "ticket-rsc-sets-5", "Ticket - Resource sets (2 tickets, granted)" ],
         [ "ticket-rsc-sets-6", "Ticket - Resource sets (2 tickets, granted)" ],
         [ "ticket-rsc-sets-7", "Ticket - Resource sets (2 tickets, revoked)" ],
         [ "ticket-rsc-sets-8", "Ticket - Resource sets (1 ticket, standby, granted)" ],
         [ "ticket-rsc-sets-9", "Ticket - Resource sets (1 ticket, granted, standby)" ],
         [ "ticket-rsc-sets-10", "Ticket - Resource sets (1 ticket, standby, revoked)" ],
         [ "ticket-rsc-sets-11", "Ticket - Resource sets (2 tickets, standby, granted)" ],
         [ "ticket-rsc-sets-12", "Ticket - Resource sets (2 tickets, standby, granted)" ],
         [ "ticket-rsc-sets-13", "Ticket - Resource sets (2 tickets, granted, standby)" ],
         [ "ticket-rsc-sets-14", "Ticket - Resource sets (2 tickets, standby, revoked)" ],
         [ "cluster-specific-params", "Cluster-specific instance attributes based on rules" ],
         [ "site-specific-params", "Site-specific instance attributes based on rules" ],
     ],
     [
         [ "template-1", "Template - 1" ],
         [ "template-2", "Template - 2" ],
         [ "template-3", "Template - 3 (merge operations)" ],
         [ "template-coloc-1", "Template - Colocation 1" ],
         [ "template-coloc-2", "Template - Colocation 2" ],
         [ "template-coloc-3", "Template - Colocation 3" ],
         [ "template-order-1", "Template - Order 1" ],
         [ "template-order-2", "Template - Order 2" ],
         [ "template-order-3", "Template - Order 3" ],
         [ "template-ticket", "Template - Ticket" ],
         [ "template-rsc-sets-1", "Template - Resource Sets 1" ],
         [ "template-rsc-sets-2", "Template - Resource Sets 2" ],
         [ "template-rsc-sets-3", "Template - Resource Sets 3" ],
         [ "template-rsc-sets-4", "Template - Resource Sets 4" ],
         [ "template-clone-primitive", "Cloned primitive from template" ],
         [ "template-clone-group", "Cloned group from template" ],
         [ "location-sets-templates", "Resource sets and templates - Location" ],
         [ "tags-coloc-order-1", "Tags - Colocation and Order (Simple)" ],
         [ "tags-coloc-order-2", "Tags - Colocation and Order (Resource Sets with Templates)" ],
         [ "tags-location", "Tags - Location" ],
         [ "tags-ticket", "Tags - Ticket" ],
     ],
     [
         [ "container-1", "Container - initial" ],
         [ "container-2", "Container - monitor failed" ],
         [ "container-3", "Container - stop failed" ],
         [ "container-4", "Container - reached migration-threshold" ],
         [ "container-group-1", "Container in group - initial" ],
         [ "container-group-2", "Container in group - monitor failed" ],
         [ "container-group-3", "Container in group - stop failed" ],
         [ "container-group-4", "Container in group - reached migration-threshold" ],
         [ "container-is-remote-node", "Place resource within container when container is remote-node" ],
         [ "bug-rh-1097457", "Kill user defined container/contents ordering" ],
         [ "bug-cl-5247", "Graph loop when recovering m/s resource in a container" ],
         [ "bundle-order-startup", "Bundle startup ordering" ],
         [ "bundle-order-partial-start",
           "Bundle startup ordering when some dependencies are already running" ],
         [ "bundle-order-partial-start-2",
           "Bundle startup ordering when some dependencies and the container are already running" ],
         [ "bundle-order-stop", "Bundle stop ordering" ],
         [ "bundle-order-partial-stop", "Bundle startup ordering when some dependencies are already stopped" ],
         [ "bundle-order-stop-on-remote", "Stop nested resource after bringing up the connection" ],
         [ "bundle-order-startup-clone", "Prevent startup because bundle isn't promoted" ],
         [ "bundle-order-startup-clone-2", "Bundle startup with clones" ],
         [ "bundle-order-stop-clone", "Stop bundle because clone is stopping" ],
         [ "bundle-nested-colocation", "Colocation of nested connection resources" ],
         [ "bundle-order-fencing",
           "Order pseudo bundle fencing after parent node fencing if both are happening" ],
         [ "bundle-probe-order-1", "order 1" ],
         [ "bundle-probe-order-2", "order 2" ],
         [ "bundle-probe-order-3", "order 3" ],
         [ "bundle-probe-remotes", "Ensure remotes get probed too" ],
         [ "bundle-replicas-change", "Change bundle from 1 replica to multiple" ],
         [ "nested-remote-recovery", "Recover bundle's container hosted on remote node" ],
     ],
     [
         [ "whitebox-fail1", "Fail whitebox container rsc" ],
         [ "whitebox-fail2", "Fail cluster connection to guest node" ],
         [ "whitebox-fail3", "Failed containers should not run nested on remote nodes" ],
         [ "whitebox-start", "Start whitebox container with resources assigned to it" ],
         [ "whitebox-stop", "Stop whitebox container with resources assigned to it" ],
         [ "whitebox-move", "Move whitebox container with resources assigned to it" ],
         [ "whitebox-asymmetric", "Verify connection rsc opts-in based on container resource" ],
         [ "whitebox-ms-ordering", "Verify promote/demote can not occur before connection is established" ],
         [ "whitebox-ms-ordering-move", "Stop/Start cycle within a moving container" ],
         [ "whitebox-orphaned", "Properly shutdown orphaned whitebox container" ],
         [ "whitebox-orphan-ms", "Properly tear down orphan ms resources on remote-nodes" ],
         [ "whitebox-unexpectedly-running", "Recover container nodes the cluster did not start" ],
         [ "whitebox-migrate1", "Migrate both container and connection resource" ],
         [ "whitebox-imply-stop-on-fence",
           "imply stop action on container node rsc when host node is fenced" ],
         [ "whitebox-nested-group", "Verify guest remote-node works nested in a group" ],
         [ "guest-node-host-dies", "Verify guest node is recovered if host goes away" ],
         [ "guest-node-cleanup", "Order guest node connection recovery after container probe" ],
         [ "guest-host-not-fenceable", "Actions on guest node are unrunnable if host is unclean and cannot be fenced" ],
     ],
     [
         [ "remote-startup-probes", "Baremetal remote-node startup probes" ],
         [ "remote-startup", "Startup a newly discovered remote-nodes with no status" ],
         [ "remote-fence-unclean", "Fence unclean baremetal remote-node" ],
         [ "remote-fence-unclean2",
           "Fence baremetal remote-node after cluster node fails and connection can not be recovered" ],
         [ "remote-fence-unclean-3", "Probe failed remote nodes (triggers fencing)" ],
         [ "remote-move", "Move remote-node connection resource" ],
         [ "remote-disable", "Disable a baremetal remote-node" ],
         [ "remote-probe-disable", "Probe then stop a baremetal remote-node" ],
         [ "remote-orphaned", "Properly shutdown orphaned connection resource" ],
         [ "remote-orphaned2",
           "verify we can handle orphaned remote connections with active resources on the remote" ],
         [ "remote-recover", "Recover connection resource after cluster-node fails" ],
         [ "remote-stale-node-entry",
           "Make sure we properly handle leftover remote-node entries in the node section" ],
         [ "remote-partial-migrate",
           "Make sure partial migrations are handled before ops on the remote node" ],
         [ "remote-partial-migrate2",
           "Make sure partial migration target is prefered for remote connection" ],
         [ "remote-recover-fail", "Make sure start failure causes fencing if rsc are active on remote" ],
         [ "remote-start-fail",
           "Make sure a start failure does not result in fencing if no active resources are on remote" ],
         [ "remote-unclean2",
           "Make monitor failure always results in fencing, even if no rsc are active on remote" ],
         [ "remote-fence-before-reconnect", "Fence before clearing recurring monitor failure" ],
         [ "remote-recovery", "Recover remote connections before attempting demotion" ],
         [ "remote-recover-connection", "Optimistically recovery of only the connection" ],
         [ "remote-recover-all", "Fencing when the connection has no home" ],
         [ "remote-recover-no-resources", "Fencing when the connection has no home and no active resources" ],
         [ "remote-recover-unknown",
           "Fencing when the connection has no home and the remote has no operation history" ],
         [ "remote-reconnect-delay", "Waiting for remote reconnect interval to expire" ],
         [ "remote-connection-unrecoverable",
           "Remote connection host must be fenced, with connection unrecoverable" ],
         [ "cancel-behind-moving-remote",
           "Route recurring monitor cancellations through original node of a moving remote connection" ],
     ],
     [
         [ "resource-discovery", "Exercises resource-discovery location constraint option" ],
         [ "rsc-discovery-per-node", "Disable resource discovery per node" ],
         [ "shutdown-lock", "Ensure shutdown lock works properly" ],
         [ "shutdown-lock-expiration", "Ensure shutdown lock expiration works properly" ],
     ],
     [
         [ "op-defaults", "Test op_defaults conditional expressions" ],
         [ "op-defaults-2", "Test op_defaults AND'ed conditional expressions" ],
         [ "op-defaults-3", "Test op_defaults precedence" ],
         [ "rsc-defaults", "Test rsc_defaults conditional expressions" ],
         [ "rsc-defaults-2", "Test rsc_defaults conditional expressions without type" ],
     ],
     [   [ "stop-all-resources", "Test stop-all-resources=true "],
     ],
     [   [ "ocf_degraded-remap-ocf_ok", "Test degraded remapped to OK" ],
         [ "ocf_degraded_promoted-remap-ocf_ok", "Test degraded promoted remapped to OK"],
     ],
     
     # @TODO: If pacemaker implements versioned attributes, uncomment these tests
     #[
     #    [ "versioned-resources", "Start resources with #ra-version rules" ],
     #    [ "restart-versioned", "Restart resources on #ra-version change" ],
     #    [ "reload-versioned", "Reload resources on #ra-version change" ],
     #],
     #[
     #    [ "versioned-operations-1", "Use #ra-version to configure operations of native resources" ],
     #    [ "versioned-operations-2", "Use #ra-version to configure operations of stonith resources" ],
     #    [ "versioned-operations-3", "Use #ra-version to configure operations of promotable resources" ],
     #    [ "versioned-operations-4", "Use #ra-version to configure operations of groups of the resources" ],
     #],
 ]
 
 TESTS_64BIT = [
     [
         [ "year-2038", "Check handling of timestamps beyond 2038-01-19 03:14:08 UTC" ],
     ],
 ]
 
 # Constants substituted in the build process
 class BuildVars(object):
     SBINDIR = "@sbindir@"
     BUILDDIR = "@abs_top_builddir@"
     CRM_SCHEMA_DIRECTORY = "@CRM_SCHEMA_DIRECTORY@"
 
 
 # These values must be kept in sync with crm_exit_t
 class CrmExit(object):
     OK                   =    0
     ERROR                =    1
     NOT_INSTALLED        =    5
     NOINPUT              =   66
     CANTCREAT            =   73
 
 
 def is_executable(path):
     """ Check whether a file at a given path is executable. """
 
     try:
         return os.stat(path)[stat.ST_MODE] & stat.S_IXUSR
     except OSError:
         return False
 
 
 def diff(file1, file2, **kwargs):
     """ Call diff on two files """
 
     return subprocess.call([ "diff", "-u", "-N", "--ignore-all-space",
                              "--ignore-blank-lines", file1, file2 ], **kwargs)
 
 
 def sort_file(filename):
     """ Sort a file alphabetically """
 
     with io.open(filename, "rt") as f:
         lines = sorted(f)
     with io.open(filename, "wt") as f:
         f.writelines(lines)
 
 
 def remove_files(filenames):
     """ Remove a list of files """
 
     for filename in filenames:
         try:
             os.remove(filename)
         except OSError:
             pass
 
 
 def normalize(filename):
     """ Remove text from a file that isn't important for comparison """
 
     if not hasattr(normalize, "patterns"):
         normalize.patterns = [
             re.compile(r'crm_feature_set="[^"]*"'),
             re.compile(r'batch-limit="[0-9]*"')
         ]
     if os.path.isfile(filename):
         with io.open(filename, "rt") as f:
             lines = f.readlines()
         with io.open(filename, "wt") as f:
             for line in lines:
                 for pattern in normalize.patterns:
                     line = pattern.sub("", line)
                 f.write(line)
 
 
 def cat(filename, dest=sys.stdout):
     """ Copy a file to a destination file descriptor """
 
     with io.open(filename, "rt") as f:
         shutil.copyfileobj(f, dest)
 
 
 class CtsScheduler(object):
     """ Regression tests for Pacemaker's scheduler """
 
     def _parse_args(self, argv):
         """ Parse command-line arguments """
 
         parser = argparse.ArgumentParser(description=DESC)
 
         parser.add_argument('-V', '--verbose', action='count',
                             help='Display any differences from expected output')
 
         parser.add_argument('--run', metavar='TEST',
                             help=('Run only single specified test (any further '
                                   'arguments will be passed to crm_simulate)'))
 
         parser.add_argument('--update', action='store_true',
                             help='Update expected results with actual results')
 
         parser.add_argument('-b', '--binary', metavar='PATH',
                             help='Specify path to crm_simulate')
 
         parser.add_argument('-i', '--io-dir', metavar='PATH',
                             help='Specify path to regression test data directory')
 
         parser.add_argument('-o', '--out-dir', metavar='PATH',
                             help='Specify where intermediate and output files should go')
 
         parser.add_argument('-v', '--valgrind', action='store_true',
                             help='Run all commands under valgrind')
 
         parser.add_argument('--valgrind-dhat', action='store_true',
                             help='Run all commands under valgrind with heap analyzer')
 
         parser.add_argument('--valgrind-skip-output', action='store_true',
                             help='If running under valgrind, do not display output')
 
         parser.add_argument('--testcmd-options', metavar='OPTIONS', default='',
                             help='Additional options for command under test')
 
         # argparse can't handle "everything after --run TEST", so grab that
         self.single_test_args = []
         narg = 0
         for arg in argv:
             narg = narg + 1
             if arg == '--run':
                 (argv, self.single_test_args) = (argv[:narg+1], argv[narg+1:])
                 break
 
         self.args = parser.parse_args(argv[1:])
 
     def _error(self, s):
         print("      * ERROR:   %s" % s)
 
     def _failed(self, s):
         print("      * FAILED:  %s" % s)
 
     def _get_valgrind_cmd(self):
         """ Return command arguments needed (or not) to run valgrind """
 
         if self.args.valgrind:
             os.environ['G_SLICE'] = "always-malloc"
             return [
                 "valgrind",
                 "-q",
                 "--gen-suppressions=all",
                 "--time-stamp=yes",
                 "--trace-children=no",
                 "--show-reachable=no",
                 "--leak-check=full",
                 "--num-callers=20",
                 "--suppressions=%s/valgrind-pcmk.suppressions" % (self.test_home)
             ]
 
         if self.args.valgrind_dhat:
             os.environ['G_SLICE'] = "always-malloc"
             return [
                 "valgrind",
                 "--tool=exp-dhat",
                 "--time-stamp=yes",
                 "--trace-children=no",
                 "--show-top-n=100",
                 "--num-callers=4"
             ]
 
         return []
 
     def _get_simulator_cmd(self):
         """ Locate the simulation binary """
 
         if self.args.binary is None:
             self.args.binary = BuildVars.BUILDDIR + "/tools/crm_simulate"
             if not is_executable(self.args.binary):
                 self.args.binary = BuildVars.SBINDIR + "/crm_simulate"
 
         if not is_executable(self.args.binary):
             # @TODO it would be more pythonic to raise an exception
             self._error("Test binary " + self.args.binary + " not found")
             sys.exit(CrmExit.NOT_INSTALLED)
 
         return [ self.args.binary ] + shlex.split(self.args.testcmd_options)
 
     def set_schema_env(self):
         """ Ensure schema directory environment variable is set, if possible """
 
         try:
             return os.environ['PCMK_schema_directory']
         except KeyError:
             for d in [ os.path.join(BuildVars.BUILDDIR, "xml"),
                        BuildVars.CRM_SCHEMA_DIRECTORY ]:
                 if os.path.isdir(d):
                     os.environ['PCMK_schema_directory'] = d
                     return d
             return None
 
     def __init__(self, argv=sys.argv):
 
         self._parse_args(argv)
 
         # Where this executable lives
         self.test_home = os.path.dirname(os.path.realpath(argv[0]))
 
         # Where test data resides
         if self.args.io_dir is None:
             self.args.io_dir = os.path.join(self.test_home, "scheduler")
 
         self.xml_input_dir = os.path.join(self.args.io_dir, "xml")
         self.expected_dir = os.path.join(self.args.io_dir, "exp")
         self.dot_expected_dir = os.path.join(self.args.io_dir, "dot")
         self.scores_dir = os.path.join(self.args.io_dir, "scores")
         self.summary_dir = os.path.join(self.args.io_dir, "summary")
         self.stderr_expected_dir = os.path.join(self.args.io_dir, "stderr")
 
         # Where to store generated files
         if self.args.out_dir is None:
             self.args.out_dir = self.args.io_dir
             self.failed_filename = os.path.join(self.test_home, ".regression.failed.diff")
         else:
             self.failed_filename = os.path.join(self.args.out_dir, ".regression.failed.diff")
         os.environ['CIB_shadow_dir'] = self.args.out_dir
         self.failed_file = None
 
         self.outfile_out_dir = os.path.join(self.args.out_dir, "out")
         self.dot_out_dir = os.path.join(self.args.out_dir, "dot")
         self.scores_out_dir = os.path.join(self.args.out_dir, "scores")
         self.summary_out_dir = os.path.join(self.args.out_dir, "summary")
         self.stderr_out_dir = os.path.join(self.args.out_dir, "stderr")
         self.valgrind_out_dir = os.path.join(self.args.out_dir, "valgrind")
 
         # Single test mode (if requested)
         try:
             # User can give test base name or file name of a test input
             self.args.run = os.path.splitext(os.path.basename(self.args.run))[0]
         except (AttributeError, TypeError):
             pass # --run was not specified
 
         self.set_schema_env()
 
         # Arguments needed (or not) to run commands
         self.valgrind_args = self._get_valgrind_cmd()
         self.simulate_args = self._get_simulator_cmd()
 
         # Test counters
         self.num_failed = 0
         self.num_tests = 0
 
         # Ensure that the main output directory exists
         # We don't want to create it with os.makedirs below
         if not os.path.isdir(self.args.out_dir):
             self._error("Output directory missing; can't create output files")
             sys.exit(CrmExit.CANTCREAT)
 
         # Create output subdirectories if they don't exist
         try:
             os.makedirs(self.outfile_out_dir, 0o755, True)
             os.makedirs(self.dot_out_dir, 0o755, True)
             os.makedirs(self.scores_out_dir, 0o755, True)
             os.makedirs(self.summary_out_dir, 0o755, True)
             os.makedirs(self.stderr_out_dir, 0o755, True)
             if self.valgrind_args:
                 os.makedirs(self.valgrind_out_dir, 0o755, True)
         except OSError as ex:
             self._error("Unable to create output subdirectory: %s" % ex)
             remove_files([
                 self.outfile_out_dir,
                 self.dot_out_dir,
                 self.scores_out_dir,
                 self.summary_out_dir,
                 self.stderr_out_dir,
             ])
             sys.exit(CrmExit.CANTCREAT)
 
     def _compare_files(self, filename1, filename2):
         """ Add any file differences to failed results """
 
         if diff(filename1, filename2, stdout=subprocess.DEVNULL) != 0:
             diff(filename1, filename2, stdout=self.failed_file, stderr=subprocess.DEVNULL)
             self.failed_file.write("\n");
             return True
         return False
 
     def run_one(self, test_name, test_desc, test_args=[]):
         """ Run one scheduler test """
 
         print("  Test %-25s %s" % ((test_name + ":"), test_desc))
 
         did_fail = False
         self.num_tests = self.num_tests + 1
 
         # Test inputs
         input_filename = os.path.join(
             self.xml_input_dir, "%s.xml" % test_name)
         expected_filename = os.path.join(
             self.expected_dir, "%s.exp" % test_name)
         dot_expected_filename = os.path.join(
             self.dot_expected_dir, "%s.dot" % test_name)
         scores_filename = os.path.join(
             self.scores_dir, "%s.scores" % test_name)
         summary_filename = os.path.join(
             self.summary_dir, "%s.summary" % test_name)
         stderr_expected_filename = os.path.join(
             self.stderr_expected_dir, "%s.stderr" % test_name)
 
         # (Intermediate) test outputs
         output_filename = os.path.join(
             self.outfile_out_dir, "%s.out" % test_name)
         dot_output_filename = os.path.join(
             self.dot_out_dir, "%s.dot.pe" % test_name)
         score_output_filename = os.path.join(
             self.scores_out_dir, "%s.scores.pe" % test_name)
         summary_output_filename = os.path.join(
             self.summary_out_dir, "%s.summary.pe" % test_name)
         stderr_output_filename = os.path.join(
             self.stderr_out_dir, "%s.stderr.pe" % test_name)
         valgrind_output_filename = os.path.join(
             self.valgrind_out_dir, "%s.valgrind" % test_name)
 
         # Common arguments for running test
         test_cmd = []
         if self.valgrind_args:
             test_cmd = self.valgrind_args + [ "--log-file=%s" % valgrind_output_filename ]
         test_cmd = test_cmd + self.simulate_args
 
         # @TODO It would be more pythonic to raise exceptions for errors,
         # then perhaps it would be nice to make a single-test class
 
         # Ensure necessary test inputs exist
         if not os.path.isfile(input_filename):
             self._error("No input")
             self.num_failed = self.num_failed + 1
             return CrmExit.NOINPUT
         if not self.args.update and not os.path.isfile(expected_filename):
             self._error("no stored output")
             return CrmExit.NOINPUT
 
         # Run simulation to generate summary output
         if self.args.run: # Single test mode
             test_cmd_full = test_cmd + [ '-x', input_filename, '-S' ] + test_args
             print(" ".join(test_cmd_full))
         else:
             # @TODO Why isn't test_args added here?
             test_cmd_full = test_cmd + [ '-x', input_filename, '-S' ]
         with io.open(summary_output_filename, "wt") as f:
             simulation = subprocess.Popen(test_cmd_full, stdout=subprocess.PIPE,
                                           stderr=subprocess.STDOUT,
                                           env=os.environ)
             # This makes diff happy regardless of --enable-compat-2.0.
             # Use sed -E to make Linux and BSD special characters more compatible.
             sed = subprocess.Popen(["sed", "-E",
                                     "-e", "s/ocf::/ocf:/g",
                                     "-e", r"s/Masters:/Promoted:/",
                                     "-e", r"s/Slaves:/Unpromoted:/",
                                     "-e", r"s/ Master( |\[|$)/ Promoted\1/",
                                     "-e", r"s/ Slave / Unpromoted /",
                                    ], stdin=simulation.stdout, stdout=f,
                                    stderr=subprocess.STDOUT)
             simulation.stdout.close()
             sed.communicate()
         if self.args.run:
             cat(summary_output_filename)
 
         # Re-run simulation to generate dot, graph, and scores
         test_cmd_full = test_cmd + [
             '-x', input_filename,
             '-D', dot_output_filename,
             '-G', output_filename,
             '-sSQ' ] + test_args
         with io.open(stderr_output_filename, "wt") as f_stderr, \
              io.open(score_output_filename,  "wt") as f_score:
             rc = subprocess.call(test_cmd_full, stdout=f_score, stderr=f_stderr, env=os.environ)
 
         # Check for test command failure
         if rc != CrmExit.OK:
             self._failed("Test returned: %d" % rc)
             did_fail = True
             print(" ".join(test_cmd_full))
 
         # Check for valgrind errors
         if self.valgrind_args and not self.args.valgrind_skip_output:
             if os.stat(valgrind_output_filename).st_size > 0:
                 self._failed("Valgrind reported errors")
                 did_fail = True
                 cat(valgrind_output_filename)
             remove_files([ valgrind_output_filename ])
 
         # Check for core dump
         if os.path.isfile("core"):
             self._failed("Core-file detected: core." + test_name)
             did_fail = True
             os.rename("core", "%s/core.%s" % (self.test_home, test_name))
 
         # Check any stderr output
         if os.path.isfile(stderr_expected_filename):
             if self._compare_files(stderr_expected_filename, stderr_output_filename):
                 self._failed("stderr changed")
                 did_fail = True
         elif os.stat(stderr_output_filename).st_size > 0:
             self._failed("Output was written to stderr")
             did_fail = True
             cat(stderr_output_filename)
         remove_files([ stderr_output_filename ])
 
         # Check whether output graph exists, and normalize it
         if (not os.path.isfile(output_filename)
             or os.stat(output_filename).st_size == 0):
             self._error("No graph produced")
             did_fail = True
             self.num_failed = self.num_failed + 1
             remove_files([ output_filename ])
             return CrmExit.ERROR
         normalize(output_filename)
 
         # Check whether dot output exists, and sort it
         if (not os.path.isfile(dot_output_filename) or
             os.stat(dot_output_filename).st_size == 0):
             self._error("No dot-file summary produced")
             did_fail = True
             self.num_failed = self.num_failed + 1
             remove_files([ dot_output_filename, output_filename ])
             return CrmExit.ERROR
         with io.open(dot_output_filename, "rt") as f:
             first_line = f.readline() # "digraph" line with opening brace
             lines = f.readlines()
             last_line = lines[-1] # closing brace
             del lines[-1]
             lines = sorted(set(lines)) # unique sort
         with io.open(dot_output_filename, "wt") as f:
             f.write(first_line)
             f.writelines(lines)
             f.write(last_line)
 
         # Check whether score output exists, and sort it
         if (not os.path.isfile(score_output_filename)
             or os.stat(score_output_filename).st_size == 0):
             self._error("No allocation scores produced")
             did_fail = True
             self.num_failed = self.num_failed + 1
             remove_files([ score_output_filename, output_filename ])
             return CrmExit.ERROR
         else:
             sort_file(score_output_filename)
 
         if self.args.update:
             shutil.copyfile(output_filename, expected_filename)
             shutil.copyfile(dot_output_filename, dot_expected_filename)
             shutil.copyfile(score_output_filename, scores_filename)
             shutil.copyfile(summary_output_filename, summary_filename)
             print("  Updated expected outputs")
 
         if self._compare_files(summary_filename, summary_output_filename):
             self._failed("summary changed")
             did_fail = True
 
         if self._compare_files(dot_expected_filename, dot_output_filename):
             self._failed("dot-file summary changed")
             did_fail = True
         else:
             remove_files([ dot_output_filename ])
 
         if self._compare_files(expected_filename, output_filename):
             self._failed("xml-file changed")
             did_fail = True
 
         if self._compare_files(scores_filename, score_output_filename):
             self._failed("scores-file changed")
             did_fail = True
 
         remove_files([ output_filename,
                        score_output_filename,
                        summary_output_filename])
 
         if did_fail:
             self.num_failed = self.num_failed + 1
             return CrmExit.ERROR
 
         return CrmExit.OK
 
     def run_all(self):
         """ Run all defined tests """
 
         if platform.architecture()[0] == "64bit":
             TESTS.extend(TESTS_64BIT)
 
         for group in TESTS:
             for test in group:
                 try:
                     args = test[2]
                 except IndexError:
                     args = []
                 self.run_one(test[0], test[1], args)
             print()
 
     def _print_summary(self):
         """ Print a summary of parameters for this test run """
 
         print("Test home is:\t" + self.test_home)
         print("Test binary is:\t" + self.args.binary)
         if 'PCMK_schema_directory' in os.environ:
             print("Schema home is:\t" + os.environ['PCMK_schema_directory'])
         if self.valgrind_args != []:
             print("Activating memory testing with valgrind")
         print()
 
     def _test_results(self):
         if self.num_failed == 0:
             return CrmExit.OK
 
         if os.path.isfile(self.failed_filename) and os.stat(self.failed_filename).st_size != 0:
             if self.args.verbose:
                 self._error("Results of %d failed tests (out of %d):" %
                     (self.num_failed, self.num_tests))
                 cat(self.failed_filename)
             else:
                 self._error("Results of %d failed tests (out of %d) are in %s" %
                     (self.num_failed, self.num_tests, self.failed_filename))
                 self._error("Use -V to display them after running the tests")
         else:
             self._error("%d (of %d) tests failed (no diff results)" %
                 (self.num_failed, self.num_tests))
             if os.path.isfile(self.failed_filename):
                 os.remove(self.failed_filename)
         return CrmExit.ERROR
 
     def run(self):
         """ Run test(s) as specified """
 
         self._print_summary()
 
         # Zero out the error log
         self.failed_file = io.open(self.failed_filename, "wt")
 
         if self.args.run is None:
             print("Performing the following tests from " + self.args.io_dir)
             print()
             self.run_all()
             print()
             self.failed_file.close()
             rc = self._test_results()
         else:
             rc = self.run_one(self.args.run, "Single shot", self.single_test_args)
             self.failed_file.close()
             cat(self.failed_filename)
 
         return rc
 
 
 if __name__ == "__main__":
     sys.exit(CtsScheduler().run())
 
 # vim: set filetype=python expandtab tabstop=4 softtabstop=4 shiftwidth=4 textwidth=120:
diff --git a/cts/scheduler/dot/colocate-unmanaged-group.dot b/cts/scheduler/dot/colocate-unmanaged-group.dot
new file mode 100644
index 0000000000..2e6dd2a885
--- /dev/null
+++ b/cts/scheduler/dot/colocate-unmanaged-group.dot
@@ -0,0 +1,4 @@
+ digraph "g" {
+"prmDummy1_monitor_10000 rh80-test01" [ style=bold color="green" fontcolor="black"]
+"prmDummy3_monitor_0 rh80-test01" [ style=bold color="green" fontcolor="black"]
+}
diff --git a/cts/scheduler/exp/colocate-unmanaged-group.exp b/cts/scheduler/exp/colocate-unmanaged-group.exp
new file mode 100644
index 0000000000..e2c16ab50f
--- /dev/null
+++ b/cts/scheduler/exp/colocate-unmanaged-group.exp
@@ -0,0 +1,20 @@
+<transition_graph cluster-delay="60s" stonith-timeout="60s" failed-stop-offset="INFINITY" failed-start-offset="INFINITY"  transition_id="0">
+  <synapse id="0">
+    <action_set>
+      <rsc_op id="28" operation="monitor" operation_key="prmDummy1_monitor_10000" on_node="rh80-test01" on_node_uuid="1">
+        <primitive id="prmDummy1" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_interval="10000" CRM_meta_name="monitor" CRM_meta_on_fail="restart" CRM_meta_on_node="rh80-test01" CRM_meta_on_node_uuid="1" CRM_meta_timeout="60000" />
+      </rsc_op>
+    </action_set>
+    <inputs/>
+  </synapse>
+  <synapse id="1">
+    <action_set>
+      <rsc_op id="1" operation="monitor" operation_key="prmDummy3_monitor_0" on_node="rh80-test01" on_node_uuid="1">
+        <primitive id="prmDummy3" class="ocf" provider="heartbeat" type="Dummy"/>
+        <attributes CRM_meta_on_node="rh80-test01" CRM_meta_on_node_uuid="1" CRM_meta_op_target_rc="7" CRM_meta_timeout="60000" />
+      </rsc_op>
+    </action_set>
+    <inputs/>
+  </synapse>
+</transition_graph>
diff --git a/cts/scheduler/scores/colocate-unmanaged-group.scores b/cts/scheduler/scores/colocate-unmanaged-group.scores
new file mode 100644
index 0000000000..781f9baf38
--- /dev/null
+++ b/cts/scheduler/scores/colocate-unmanaged-group.scores
@@ -0,0 +1,27 @@
+
+pcmk__clone_allocate: prmStateful-clone allocation score on rh80-test01: -INFINITY
+pcmk__clone_allocate: prmStateful-clone allocation score on rh80-test02: -INFINITY
+pcmk__clone_allocate: prmStateful:0 allocation score on rh80-test01: -INFINITY
+pcmk__clone_allocate: prmStateful:0 allocation score on rh80-test02: -INFINITY
+pcmk__clone_allocate: prmStateful:1 allocation score on rh80-test01: -INFINITY
+pcmk__clone_allocate: prmStateful:1 allocation score on rh80-test02: -INFINITY
+pcmk__group_allocate: grpTest allocation score on rh80-test01: 200
+pcmk__group_allocate: grpTest allocation score on rh80-test02: 100
+pcmk__group_allocate: prmDummy1 allocation score on rh80-test01: INFINITY
+pcmk__group_allocate: prmDummy1 allocation score on rh80-test02: 100
+pcmk__group_allocate: prmDummy2 allocation score on rh80-test01: 0
+pcmk__group_allocate: prmDummy2 allocation score on rh80-test02: 0
+pcmk__group_allocate: prmDummy3 allocation score on rh80-test01: 0
+pcmk__group_allocate: prmDummy3 allocation score on rh80-test02: 0
+pcmk__native_allocate: prmDummy1 allocation score on rh80-test01: -INFINITY
+pcmk__native_allocate: prmDummy1 allocation score on rh80-test02: -INFINITY
+pcmk__native_allocate: prmDummy2 allocation score on rh80-test01: -INFINITY
+pcmk__native_allocate: prmDummy2 allocation score on rh80-test02: -INFINITY
+pcmk__native_allocate: prmDummy3 allocation score on rh80-test01: -INFINITY
+pcmk__native_allocate: prmDummy3 allocation score on rh80-test02: -INFINITY
+pcmk__native_allocate: prmStateful:0 allocation score on rh80-test01: -INFINITY
+pcmk__native_allocate: prmStateful:0 allocation score on rh80-test02: -INFINITY
+pcmk__native_allocate: prmStateful:1 allocation score on rh80-test01: -INFINITY
+pcmk__native_allocate: prmStateful:1 allocation score on rh80-test02: -INFINITY
+prmStateful:0 promotion score on none: 0
+prmStateful:1 promotion score on none: 0
diff --git a/cts/scheduler/scores/unmanaged-promoted.scores b/cts/scheduler/scores/unmanaged-promoted.scores
index aa7543d61c..150861c0f5 100644
--- a/cts/scheduler/scores/unmanaged-promoted.scores
+++ b/cts/scheduler/scores/unmanaged-promoted.scores
@@ -1,165 +1,165 @@
 
 pcmk__clone_allocate: Connectivity allocation score on pcmk-1: 0
 pcmk__clone_allocate: Connectivity allocation score on pcmk-2: 0
 pcmk__clone_allocate: Connectivity allocation score on pcmk-3: 0
 pcmk__clone_allocate: Connectivity allocation score on pcmk-4: 0
 pcmk__clone_allocate: Fencing allocation score on pcmk-1: 0
 pcmk__clone_allocate: Fencing allocation score on pcmk-2: 0
 pcmk__clone_allocate: Fencing allocation score on pcmk-3: 0
 pcmk__clone_allocate: Fencing allocation score on pcmk-4: 0
 pcmk__clone_allocate: FencingChild:0 allocation score on pcmk-1: 0
 pcmk__clone_allocate: FencingChild:0 allocation score on pcmk-2: 0
 pcmk__clone_allocate: FencingChild:0 allocation score on pcmk-3: INFINITY
 pcmk__clone_allocate: FencingChild:0 allocation score on pcmk-4: 0
 pcmk__clone_allocate: FencingChild:1 allocation score on pcmk-1: 0
 pcmk__clone_allocate: FencingChild:1 allocation score on pcmk-2: 0
 pcmk__clone_allocate: FencingChild:1 allocation score on pcmk-3: 0
 pcmk__clone_allocate: FencingChild:1 allocation score on pcmk-4: INFINITY
 pcmk__clone_allocate: FencingChild:2 allocation score on pcmk-1: 0
 pcmk__clone_allocate: FencingChild:2 allocation score on pcmk-2: INFINITY
 pcmk__clone_allocate: FencingChild:2 allocation score on pcmk-3: 0
 pcmk__clone_allocate: FencingChild:2 allocation score on pcmk-4: 0
 pcmk__clone_allocate: FencingChild:3 allocation score on pcmk-1: INFINITY
 pcmk__clone_allocate: FencingChild:3 allocation score on pcmk-2: 0
 pcmk__clone_allocate: FencingChild:3 allocation score on pcmk-3: 0
 pcmk__clone_allocate: FencingChild:3 allocation score on pcmk-4: 0
 pcmk__clone_allocate: master-1 allocation score on pcmk-1: 0
 pcmk__clone_allocate: master-1 allocation score on pcmk-2: 0
 pcmk__clone_allocate: master-1 allocation score on pcmk-3: 0
 pcmk__clone_allocate: master-1 allocation score on pcmk-4: 0
 pcmk__clone_allocate: ping-1:0 allocation score on pcmk-1: 0
 pcmk__clone_allocate: ping-1:0 allocation score on pcmk-2: 0
 pcmk__clone_allocate: ping-1:0 allocation score on pcmk-3: INFINITY
 pcmk__clone_allocate: ping-1:0 allocation score on pcmk-4: 0
 pcmk__clone_allocate: ping-1:1 allocation score on pcmk-1: 0
 pcmk__clone_allocate: ping-1:1 allocation score on pcmk-2: 0
 pcmk__clone_allocate: ping-1:1 allocation score on pcmk-3: 0
 pcmk__clone_allocate: ping-1:1 allocation score on pcmk-4: INFINITY
 pcmk__clone_allocate: ping-1:2 allocation score on pcmk-1: 0
 pcmk__clone_allocate: ping-1:2 allocation score on pcmk-2: INFINITY
 pcmk__clone_allocate: ping-1:2 allocation score on pcmk-3: 0
 pcmk__clone_allocate: ping-1:2 allocation score on pcmk-4: 0
 pcmk__clone_allocate: ping-1:3 allocation score on pcmk-1: INFINITY
 pcmk__clone_allocate: ping-1:3 allocation score on pcmk-2: 0
 pcmk__clone_allocate: ping-1:3 allocation score on pcmk-3: 0
 pcmk__clone_allocate: ping-1:3 allocation score on pcmk-4: 0
 pcmk__clone_allocate: stateful-1:0 allocation score on pcmk-1: 0
 pcmk__clone_allocate: stateful-1:0 allocation score on pcmk-2: 0
 pcmk__clone_allocate: stateful-1:0 allocation score on pcmk-3: INFINITY
 pcmk__clone_allocate: stateful-1:0 allocation score on pcmk-4: 0
 pcmk__clone_allocate: stateful-1:1 allocation score on pcmk-1: 0
 pcmk__clone_allocate: stateful-1:1 allocation score on pcmk-2: 0
 pcmk__clone_allocate: stateful-1:1 allocation score on pcmk-3: 0
 pcmk__clone_allocate: stateful-1:1 allocation score on pcmk-4: INFINITY
 pcmk__clone_allocate: stateful-1:2 allocation score on pcmk-1: 0
 pcmk__clone_allocate: stateful-1:2 allocation score on pcmk-2: INFINITY
 pcmk__clone_allocate: stateful-1:2 allocation score on pcmk-3: 0
 pcmk__clone_allocate: stateful-1:2 allocation score on pcmk-4: 0
 pcmk__clone_allocate: stateful-1:3 allocation score on pcmk-1: INFINITY
 pcmk__clone_allocate: stateful-1:3 allocation score on pcmk-2: 0
 pcmk__clone_allocate: stateful-1:3 allocation score on pcmk-3: 0
 pcmk__clone_allocate: stateful-1:3 allocation score on pcmk-4: 0
 pcmk__group_allocate: group-1 allocation score on pcmk-1: 0
 pcmk__group_allocate: group-1 allocation score on pcmk-2: 0
 pcmk__group_allocate: group-1 allocation score on pcmk-3: 0
 pcmk__group_allocate: group-1 allocation score on pcmk-4: 0
 pcmk__group_allocate: r192.168.122.126 allocation score on pcmk-1: 0
 pcmk__group_allocate: r192.168.122.126 allocation score on pcmk-2: INFINITY
 pcmk__group_allocate: r192.168.122.126 allocation score on pcmk-3: 0
 pcmk__group_allocate: r192.168.122.126 allocation score on pcmk-4: 0
 pcmk__group_allocate: r192.168.122.127 allocation score on pcmk-1: 0
 pcmk__group_allocate: r192.168.122.127 allocation score on pcmk-2: INFINITY
 pcmk__group_allocate: r192.168.122.127 allocation score on pcmk-3: 0
 pcmk__group_allocate: r192.168.122.127 allocation score on pcmk-4: 0
 pcmk__group_allocate: r192.168.122.128 allocation score on pcmk-1: 0
 pcmk__group_allocate: r192.168.122.128 allocation score on pcmk-2: INFINITY
 pcmk__group_allocate: r192.168.122.128 allocation score on pcmk-3: 0
 pcmk__group_allocate: r192.168.122.128 allocation score on pcmk-4: 0
 pcmk__native_allocate: FencingChild:0 allocation score on pcmk-1: -INFINITY
 pcmk__native_allocate: FencingChild:0 allocation score on pcmk-2: -INFINITY
 pcmk__native_allocate: FencingChild:0 allocation score on pcmk-3: -INFINITY
 pcmk__native_allocate: FencingChild:0 allocation score on pcmk-4: -INFINITY
 pcmk__native_allocate: FencingChild:1 allocation score on pcmk-1: -INFINITY
 pcmk__native_allocate: FencingChild:1 allocation score on pcmk-2: -INFINITY
 pcmk__native_allocate: FencingChild:1 allocation score on pcmk-3: -INFINITY
 pcmk__native_allocate: FencingChild:1 allocation score on pcmk-4: -INFINITY
 pcmk__native_allocate: FencingChild:2 allocation score on pcmk-1: -INFINITY
 pcmk__native_allocate: FencingChild:2 allocation score on pcmk-2: -INFINITY
 pcmk__native_allocate: FencingChild:2 allocation score on pcmk-3: -INFINITY
 pcmk__native_allocate: FencingChild:2 allocation score on pcmk-4: -INFINITY
 pcmk__native_allocate: FencingChild:3 allocation score on pcmk-1: -INFINITY
 pcmk__native_allocate: FencingChild:3 allocation score on pcmk-2: -INFINITY
 pcmk__native_allocate: FencingChild:3 allocation score on pcmk-3: -INFINITY
 pcmk__native_allocate: FencingChild:3 allocation score on pcmk-4: -INFINITY
 pcmk__native_allocate: lsb-dummy allocation score on pcmk-1: -INFINITY
 pcmk__native_allocate: lsb-dummy allocation score on pcmk-2: INFINITY
 pcmk__native_allocate: lsb-dummy allocation score on pcmk-3: -INFINITY
 pcmk__native_allocate: lsb-dummy allocation score on pcmk-4: -INFINITY
 pcmk__native_allocate: migrator allocation score on pcmk-1: 0
 pcmk__native_allocate: migrator allocation score on pcmk-2: 0
 pcmk__native_allocate: migrator allocation score on pcmk-3: 0
 pcmk__native_allocate: migrator allocation score on pcmk-4: INFINITY
 pcmk__native_allocate: ping-1:0 allocation score on pcmk-1: -INFINITY
 pcmk__native_allocate: ping-1:0 allocation score on pcmk-2: -INFINITY
 pcmk__native_allocate: ping-1:0 allocation score on pcmk-3: -INFINITY
 pcmk__native_allocate: ping-1:0 allocation score on pcmk-4: -INFINITY
 pcmk__native_allocate: ping-1:1 allocation score on pcmk-1: -INFINITY
 pcmk__native_allocate: ping-1:1 allocation score on pcmk-2: -INFINITY
 pcmk__native_allocate: ping-1:1 allocation score on pcmk-3: -INFINITY
 pcmk__native_allocate: ping-1:1 allocation score on pcmk-4: -INFINITY
 pcmk__native_allocate: ping-1:2 allocation score on pcmk-1: -INFINITY
 pcmk__native_allocate: ping-1:2 allocation score on pcmk-2: -INFINITY
 pcmk__native_allocate: ping-1:2 allocation score on pcmk-3: -INFINITY
 pcmk__native_allocate: ping-1:2 allocation score on pcmk-4: -INFINITY
 pcmk__native_allocate: ping-1:3 allocation score on pcmk-1: -INFINITY
 pcmk__native_allocate: ping-1:3 allocation score on pcmk-2: -INFINITY
 pcmk__native_allocate: ping-1:3 allocation score on pcmk-3: -INFINITY
 pcmk__native_allocate: ping-1:3 allocation score on pcmk-4: -INFINITY
 pcmk__native_allocate: r192.168.122.126 allocation score on pcmk-1: -INFINITY
 pcmk__native_allocate: r192.168.122.126 allocation score on pcmk-2: -INFINITY
 pcmk__native_allocate: r192.168.122.126 allocation score on pcmk-3: -INFINITY
 pcmk__native_allocate: r192.168.122.126 allocation score on pcmk-4: -INFINITY
 pcmk__native_allocate: r192.168.122.127 allocation score on pcmk-1: -INFINITY
-pcmk__native_allocate: r192.168.122.127 allocation score on pcmk-2: INFINITY
+pcmk__native_allocate: r192.168.122.127 allocation score on pcmk-2: -INFINITY
 pcmk__native_allocate: r192.168.122.127 allocation score on pcmk-3: -INFINITY
 pcmk__native_allocate: r192.168.122.127 allocation score on pcmk-4: -INFINITY
 pcmk__native_allocate: r192.168.122.128 allocation score on pcmk-1: -INFINITY
-pcmk__native_allocate: r192.168.122.128 allocation score on pcmk-2: INFINITY
+pcmk__native_allocate: r192.168.122.128 allocation score on pcmk-2: -INFINITY
 pcmk__native_allocate: r192.168.122.128 allocation score on pcmk-3: -INFINITY
 pcmk__native_allocate: r192.168.122.128 allocation score on pcmk-4: -INFINITY
 pcmk__native_allocate: rsc_pcmk-1 allocation score on pcmk-1: INFINITY
 pcmk__native_allocate: rsc_pcmk-1 allocation score on pcmk-2: 0
 pcmk__native_allocate: rsc_pcmk-1 allocation score on pcmk-3: 0
 pcmk__native_allocate: rsc_pcmk-1 allocation score on pcmk-4: 0
 pcmk__native_allocate: rsc_pcmk-2 allocation score on pcmk-1: 0
 pcmk__native_allocate: rsc_pcmk-2 allocation score on pcmk-2: INFINITY
 pcmk__native_allocate: rsc_pcmk-2 allocation score on pcmk-3: 0
 pcmk__native_allocate: rsc_pcmk-2 allocation score on pcmk-4: 0
 pcmk__native_allocate: rsc_pcmk-3 allocation score on pcmk-1: 0
 pcmk__native_allocate: rsc_pcmk-3 allocation score on pcmk-2: 0
 pcmk__native_allocate: rsc_pcmk-3 allocation score on pcmk-3: INFINITY
 pcmk__native_allocate: rsc_pcmk-3 allocation score on pcmk-4: 0
 pcmk__native_allocate: rsc_pcmk-4 allocation score on pcmk-1: 0
 pcmk__native_allocate: rsc_pcmk-4 allocation score on pcmk-2: 0
 pcmk__native_allocate: rsc_pcmk-4 allocation score on pcmk-3: 0
 pcmk__native_allocate: rsc_pcmk-4 allocation score on pcmk-4: INFINITY
 pcmk__native_allocate: stateful-1:0 allocation score on pcmk-1: -INFINITY
 pcmk__native_allocate: stateful-1:0 allocation score on pcmk-2: -INFINITY
 pcmk__native_allocate: stateful-1:0 allocation score on pcmk-3: -INFINITY
 pcmk__native_allocate: stateful-1:0 allocation score on pcmk-4: -INFINITY
 pcmk__native_allocate: stateful-1:1 allocation score on pcmk-1: -INFINITY
 pcmk__native_allocate: stateful-1:1 allocation score on pcmk-2: -INFINITY
 pcmk__native_allocate: stateful-1:1 allocation score on pcmk-3: -INFINITY
 pcmk__native_allocate: stateful-1:1 allocation score on pcmk-4: -INFINITY
 pcmk__native_allocate: stateful-1:2 allocation score on pcmk-1: -INFINITY
 pcmk__native_allocate: stateful-1:2 allocation score on pcmk-2: -INFINITY
 pcmk__native_allocate: stateful-1:2 allocation score on pcmk-3: -INFINITY
 pcmk__native_allocate: stateful-1:2 allocation score on pcmk-4: -INFINITY
 pcmk__native_allocate: stateful-1:3 allocation score on pcmk-1: -INFINITY
 pcmk__native_allocate: stateful-1:3 allocation score on pcmk-2: -INFINITY
 pcmk__native_allocate: stateful-1:3 allocation score on pcmk-3: -INFINITY
 pcmk__native_allocate: stateful-1:3 allocation score on pcmk-4: -INFINITY
 stateful-1:0 promotion score on pcmk-3: -1
 stateful-1:1 promotion score on pcmk-4: -1
 stateful-1:2 promotion score on pcmk-2: INFINITY
 stateful-1:3 promotion score on pcmk-1: -1
diff --git a/cts/scheduler/summary/colocate-unmanaged-group.summary b/cts/scheduler/summary/colocate-unmanaged-group.summary
new file mode 100644
index 0000000000..f29452bfb4
--- /dev/null
+++ b/cts/scheduler/summary/colocate-unmanaged-group.summary
@@ -0,0 +1,31 @@
+Using the original execution date of: 2020-02-26 05:50:16Z
+Current cluster status:
+  * Node List:
+    * Online: [ rh80-test01 rh80-test02 ]
+
+  * Full List of Resources:
+    * Clone Set: prmStateful-clone [prmStateful] (promotable):
+      * Stopped: [ rh80-test01 rh80-test02 ]
+    * Resource Group: grpTest:
+      * prmDummy1	(ocf:heartbeat:Dummy):	 Started rh80-test01 (unmanaged)
+      * prmDummy2	(ocf:heartbeat:Dummy):	 Stopped
+      * prmDummy3	(ocf:heartbeat:Dummy):	 Stopped
+
+Transition Summary:
+
+Executing Cluster Transition:
+  * Resource action: prmDummy1       monitor=10000 on rh80-test01
+  * Resource action: prmDummy3       monitor on rh80-test01
+Using the original execution date of: 2020-02-26 05:50:16Z
+
+Revised Cluster Status:
+  * Node List:
+    * Online: [ rh80-test01 rh80-test02 ]
+
+  * Full List of Resources:
+    * Clone Set: prmStateful-clone [prmStateful] (promotable):
+      * Stopped: [ rh80-test01 rh80-test02 ]
+    * Resource Group: grpTest:
+      * prmDummy1	(ocf:heartbeat:Dummy):	 Started rh80-test01 (unmanaged)
+      * prmDummy2	(ocf:heartbeat:Dummy):	 Stopped
+      * prmDummy3	(ocf:heartbeat:Dummy):	 Stopped
diff --git a/cts/scheduler/xml/colocate-unmanaged-group.xml b/cts/scheduler/xml/colocate-unmanaged-group.xml
new file mode 100644
index 0000000000..b92c903aec
--- /dev/null
+++ b/cts/scheduler/xml/colocate-unmanaged-group.xml
@@ -0,0 +1,129 @@
+<cib crm_feature_set="3.2.0" validate-with="pacemaker-3.2" epoch="18" num_updates="18" admin_epoch="0" cib-last-written="Wed Feb 26 14:49:12 2020" update-origin="rh80-test01" update-client="cibadmin" update-user="root" have-quorum="1" dc-uuid="2" execution-date="1582696216">
+  <configuration>
+    <crm_config>
+      <cluster_property_set id="cib-bootstrap-options">
+        <nvpair id="cib-bootstrap-options-have-watchdog" name="have-watchdog" value="false"/>
+        <nvpair id="cib-bootstrap-options-dc-version" name="dc-version" value="2.0.3-4b1f869f0f"/>
+        <nvpair id="cib-bootstrap-options-cluster-infrastructure" name="cluster-infrastructure" value="corosync"/>
+        <nvpair id="cib-bootstrap-options-cluster-name" name="cluster-name" value="my_cluster"/>
+        <nvpair id="cib-bootstrap-options-stonith-enabled" name="stonith-enabled" value="false"/>
+      </cluster_property_set>
+    </crm_config>
+    <nodes>
+      <node id="1" uname="rh80-test01"/>
+      <node id="2" uname="rh80-test02"/>
+    </nodes>
+    <resources>
+      <clone id="prmStateful-clone">
+        <primitive class="ocf" id="prmStateful" provider="pacemaker" type="Stateful">
+          <operations>
+            <op id="prmStateful-demote-interval-0s" interval="0s" name="demote" on-fail="block" timeout="300s"/>
+            <op id="prmStateful-monitor-interval-10s" interval="10s" name="monitor" on-fail="restart" timeout="60s"/>
+            <op id="prmStateful-monitor-interval-9s" interval="9s" name="monitor" on-fail="restart" role="Master" timeout="60s"/>
+            <op id="prmStateful-notify-interval-0s" interval="0s" name="notify" timeout="60s"/>
+            <op id="prmStateful-promote-interval-0s" interval="0s" name="promote" on-fail="restart" timeout="300s"/>
+            <op id="prmStateful-start-interval-0s" interval="0s" name="start" on-fail="restart" timeout="300s"/>
+            <op id="prmStateful-stop-interval-0s" interval="0s" name="stop" on-fail="block" timeout="300s"/>
+          </operations>
+        </primitive>
+        <meta_attributes id="prmStateful-clone-meta_attributes">
+          <nvpair id="prmStateful-clone-meta_attributes-clone-max" name="clone-max" value="2"/>
+          <nvpair id="prmStateful-clone-meta_attributes-clone-node-max" name="clone-node-max" value="1"/>
+          <nvpair id="prmStateful-clone-meta_attributes-notify" name="notify" value="true"/>
+          <nvpair id="prmStateful-clone-meta_attributes-promotable" name="promotable" value="true"/>
+          <nvpair id="prmStateful-clone-meta_attributes-promoted-max" name="promoted-max" value="1"/>
+          <nvpair id="prmStateful-clone-meta_attributes-promoted-node-max" name="promoted-node-max" value="1"/>
+        </meta_attributes>
+      </clone>
+      <group id="grpTest">
+        <primitive class="ocf" id="prmDummy1" provider="heartbeat" type="Dummy">
+          <meta_attributes id="prmDummy1-meta_attributes">
+            <nvpair id="prmDummy1-meta_attributes-is-managed" name="is-managed" value="false"/>
+          </meta_attributes>
+          <operations>
+            <op id="prmDummy1-migrate_from-interval-0s" interval="0s" name="migrate_from" timeout="20s"/>
+            <op id="prmDummy1-migrate_to-interval-0s" interval="0s" name="migrate_to" timeout="20s"/>
+            <op id="prmDummy1-monitor-interval-10s" interval="10s" name="monitor" on-fail="restart" timeout="60s"/>
+            <op id="prmDummy1-reload-interval-0s" interval="0s" name="reload" timeout="20s"/>
+            <op id="prmDummy1-start-interval-0s" interval="0s" name="start" on-fail="restart" timeout="60s"/>
+            <op id="prmDummy1-stop-interval-0s" interval="0s" name="stop" on-fail="block" timeout="60s"/>
+          </operations>
+        </primitive>
+        <primitive class="ocf" id="prmDummy2" provider="heartbeat" type="Dummy">
+          <operations>
+            <op id="prmDummy2-migrate_from-interval-0s" interval="0s" name="migrate_from" timeout="20s"/>
+            <op id="prmDummy2-migrate_to-interval-0s" interval="0s" name="migrate_to" timeout="20s"/>
+            <op id="prmDummy2-monitor-interval-10s" interval="10s" name="monitor" on-fail="restart" timeout="60s"/>
+            <op id="prmDummy2-reload-interval-0s" interval="0s" name="reload" timeout="20s"/>
+            <op id="prmDummy2-start-interval-0s" interval="0s" name="start" on-fail="restart" timeout="60s"/>
+            <op id="prmDummy2-stop-interval-0s" interval="0s" name="stop" on-fail="block" timeout="60s"/>
+          </operations>
+        </primitive>
+        <primitive class="ocf" id="prmDummy3" provider="heartbeat" type="Dummy">
+          <operations>
+            <op id="prmDummy3-migrate_from-interval-0s" interval="0s" name="migrate_from" timeout="20s"/>
+            <op id="prmDummy3-migrate_to-interval-0s" interval="0s" name="migrate_to" timeout="20s"/>
+            <op id="prmDummy3-monitor-interval-10s" interval="10s" name="monitor" on-fail="restart" timeout="60s"/>
+            <op id="prmDummy3-reload-interval-0s" interval="0s" name="reload" timeout="20s"/>
+            <op id="prmDummy3-start-interval-0s" interval="0s" name="start" on-fail="restart" timeout="60s"/>
+            <op id="prmDummy3-stop-interval-0s" interval="0s" name="stop" on-fail="block" timeout="60s"/>
+          </operations>
+        </primitive>
+      </group>
+    </resources>
+    <constraints>
+      <rsc_location id="location-grpTest-rh80-test01-200" node="rh80-test01" rsc="grpTest" score="200"/>
+      <rsc_location id="location-grpTest-rh80-test02-100" node="rh80-test02" rsc="grpTest" score="100"/>
+      <rsc_location id="location-prmStateful-clone" rsc="prmStateful-clone">
+        <rule boolean-op="or" id="location-prmStateful-clone-rule" score="-INFINITY">
+          <expression attribute="default_ping_set" id="location-prmStateful-clone-rule-expr" operation="lt" value="100"/>
+          <expression attribute="default_ping_set" id="location-prmStateful-clone-rule-expr-1" operation="not_defined"/>
+        </rule>
+      </rsc_location>
+      <rsc_colocation id="colocation-grpTest-prmStateful-clone-INFINITY" rsc="grpTest" rsc-role="Started" score="INFINITY" with-rsc="prmStateful-clone" with-rsc-role="Master"/>
+      <rsc_order first="prmStateful-clone" first-action="promote" id="order-prmStateful-clone-grpTest-Optional" kind="Optional" symmetrical="true" then="grpTest" then-action="start"/>
+    </constraints>
+    <rsc_defaults>
+      <meta_attributes id="rsc_defaults-options">
+        <nvpair id="rsc_defaults-options-migration-threshold" name="migration-threshold" value="1"/>
+        <nvpair id="rsc_defaults-options-resource-stickiness" name="resource-stickiness" value="INFINITY"/>
+      </meta_attributes>
+    </rsc_defaults>
+  </configuration>
+  <status>
+    <node_state id="2" uname="rh80-test02" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
+      <lrm id="2">
+        <lrm_resources>
+          <lrm_resource id="prmStateful" type="Stateful" class="ocf" provider="pacemaker">
+            <lrm_rsc_op id="prmStateful_last_0" operation_key="prmStateful_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.2.0" transition-key="5:0:7:0f75d58f-cd1e-436c-a4e5-cc97fc09899d" transition-magic="0:7;5:0:7:0f75d58f-cd1e-436c-a4e5-cc97fc09899d" exit-reason="" on_node="rh80-test02" call-id="6" rc-code="7" op-status="0" interval="0" last-rc-change="1582696215" last-run="1582696215" exec-time="48" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+          </lrm_resource>
+          <lrm_resource id="prmDummy1" type="Dummy" class="ocf" provider="heartbeat">
+            <lrm_rsc_op id="prmDummy1_last_0" operation_key="prmDummy1_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.2.0" transition-key="6:0:7:0f75d58f-cd1e-436c-a4e5-cc97fc09899d" transition-magic="0:7;6:0:7:0f75d58f-cd1e-436c-a4e5-cc97fc09899d" exit-reason="" on_node="rh80-test02" call-id="10" rc-code="7" op-status="0" interval="0" last-rc-change="1582696215" last-run="1582696215" exec-time="59" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+          </lrm_resource>
+          <lrm_resource id="prmDummy2" type="Dummy" class="ocf" provider="heartbeat">
+            <lrm_rsc_op id="prmDummy2_last_0" operation_key="prmDummy2_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.2.0" transition-key="7:0:7:0f75d58f-cd1e-436c-a4e5-cc97fc09899d" transition-magic="0:7;7:0:7:0f75d58f-cd1e-436c-a4e5-cc97fc09899d" exit-reason="" on_node="rh80-test02" call-id="14" rc-code="7" op-status="0" interval="0" last-rc-change="1582696215" last-run="1582696215" exec-time="39" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+          </lrm_resource>
+          <lrm_resource id="prmDummy3" type="Dummy" class="ocf" provider="heartbeat">
+            <lrm_rsc_op id="prmDummy3_last_0" operation_key="prmDummy3_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.2.0" transition-key="8:0:7:0f75d58f-cd1e-436c-a4e5-cc97fc09899d" transition-magic="0:7;8:0:7:0f75d58f-cd1e-436c-a4e5-cc97fc09899d" exit-reason="" on_node="rh80-test02" call-id="18" rc-code="7" op-status="0" interval="0" last-rc-change="1582696215" last-run="1582696215" exec-time="35" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+          </lrm_resource>
+        </lrm_resources>
+      </lrm>
+    </node_state>
+    <node_state id="1" uname="rh80-test01" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
+      <lrm id="1">
+        <lrm_resources>
+          <lrm_resource id="prmStateful" type="Stateful" class="ocf" provider="pacemaker">
+            <lrm_rsc_op id="prmStateful_last_0" operation_key="prmStateful_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.2.0" transition-key="1:0:7:0f75d58f-cd1e-436c-a4e5-cc97fc09899d" transition-magic="0:7;1:0:7:0f75d58f-cd1e-436c-a4e5-cc97fc09899d" exit-reason="" on_node="rh80-test01" call-id="6" rc-code="7" op-status="0" interval="0" last-rc-change="1582696215" last-run="1582696215" exec-time="54" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+          </lrm_resource>
+          <lrm_resource id="prmDummy1" type="Dummy" class="ocf" provider="heartbeat">
+            <lrm_rsc_op id="prmDummy1_last_0" operation_key="prmDummy1_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.2.0" transition-key="2:0:7:0f75d58f-cd1e-436c-a4e5-cc97fc09899d" transition-magic="0:0;2:0:7:0f75d58f-cd1e-436c-a4e5-cc97fc09899d" exit-reason="" on_node="rh80-test01" call-id="10" rc-code="0" op-status="0" interval="0" last-rc-change="1582696215" last-run="1582696215" exec-time="57" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+            <lrm_rsc_op id="prmDummy1_last_failure_0" operation_key="prmDummy1_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.2.0" transition-key="2:0:7:0f75d58f-cd1e-436c-a4e5-cc97fc09899d" transition-magic="0:0;2:0:7:0f75d58f-cd1e-436c-a4e5-cc97fc09899d" exit-reason="" on_node="rh80-test01" call-id="10" rc-code="0" op-status="0" interval="0" last-rc-change="1582696215" last-run="1582696215" exec-time="57" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+          </lrm_resource>
+          <lrm_resource id="prmDummy2" type="Dummy" class="ocf" provider="heartbeat">
+            <lrm_rsc_op id="prmDummy2_last_0" operation_key="prmDummy2_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.2.0" transition-key="3:0:7:0f75d58f-cd1e-436c-a4e5-cc97fc09899d" transition-magic="0:7;3:0:7:0f75d58f-cd1e-436c-a4e5-cc97fc09899d" exit-reason="" on_node="rh80-test01" call-id="14" rc-code="7" op-status="0" interval="0" last-rc-change="1582696216" last-run="1582696216" exec-time="44" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+          </lrm_resource>
+        </lrm_resources>
+      </lrm>
+    </node_state>
+  </status>
+</cib>
diff --git a/include/crm/pengine/pe_types.h b/include/crm/pengine/pe_types.h
index 32496f3cbd..725b184278 100644
--- a/include/crm/pengine/pe_types.h
+++ b/include/crm/pengine/pe_types.h
@@ -1,544 +1,546 @@
 /*
  * Copyright 2004-2021 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU Lesser General Public License
  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
  */
 
 #ifndef PE_TYPES__H
 #  define PE_TYPES__H
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
 /*!
  * \file
  * \brief Data types for cluster status
  * \ingroup pengine
  */
 
 #  include <stdbool.h>              // bool
 #  include <sys/types.h>            // time_t
 #  include <libxml/tree.h>          // xmlNode
 #  include <glib.h>                 // gboolean, guint, GList, GHashTable
 #  include <crm/common/iso8601.h>
 #  include <crm/pengine/common.h>
 
 typedef struct pe_node_s pe_node_t;
 typedef struct pe_action_s pe_action_t;
 typedef struct pe_resource_s pe_resource_t;
 typedef struct pe_working_set_s pe_working_set_t;
 
 enum pe_obj_types {
     pe_unknown = -1,
     pe_native = 0,
     pe_group = 1,
     pe_clone = 2,
     pe_container = 3,
 };
 
 typedef struct resource_object_functions_s {
     gboolean (*unpack) (pe_resource_t*, pe_working_set_t*);
     pe_resource_t *(*find_rsc) (pe_resource_t *parent, const char *search,
                                 const pe_node_t *node, int flags);
     /* parameter result must be free'd */
     char *(*parameter) (pe_resource_t*, pe_node_t*, gboolean, const char*,
                         pe_working_set_t*);
     //! \deprecated will be removed in a future release
     void (*print) (pe_resource_t*, const char*, long, void*);
     gboolean (*active) (pe_resource_t*, gboolean);
     enum rsc_role_e (*state) (const pe_resource_t*, gboolean);
     pe_node_t *(*location) (const pe_resource_t*, GList**, int);
     void (*free) (pe_resource_t*);
     void (*count) (pe_resource_t*);
     gboolean (*is_filtered) (pe_resource_t*, GList *, gboolean);
 } resource_object_functions_t;
 
 typedef struct resource_alloc_functions_s resource_alloc_functions_t;
 
 enum pe_quorum_policy {
     no_quorum_freeze,
     no_quorum_stop,
     no_quorum_ignore,
     no_quorum_suicide,
     no_quorum_demote
 };
 
 enum node_type {
     node_ping,
     node_member,
     node_remote
 };
 
 //! \deprecated will be removed in a future release
 enum pe_restart {
     pe_restart_restart,
     pe_restart_ignore
 };
 
 //! Determine behavior of pe_find_resource_with_flags()
 enum pe_find {
     pe_find_renamed  = 0x001, //!< match resource ID or LRM history ID
     pe_find_anon     = 0x002, //!< match base name of anonymous clone instances
     pe_find_clone    = 0x004, //!< match only clone instances
     pe_find_current  = 0x008, //!< match resource active on specified node
     pe_find_inactive = 0x010, //!< match resource not running anywhere
     pe_find_any      = 0x020, //!< match base name of any clone instance
 };
 
 // @TODO Make these an enum
 
 #  define pe_flag_have_quorum           0x00000001ULL
 #  define pe_flag_symmetric_cluster     0x00000002ULL
 #  define pe_flag_maintenance_mode      0x00000008ULL
 
 #  define pe_flag_stonith_enabled       0x00000010ULL
 #  define pe_flag_have_stonith_resource 0x00000020ULL
 #  define pe_flag_enable_unfencing      0x00000040ULL
 #  define pe_flag_concurrent_fencing    0x00000080ULL
 
 #  define pe_flag_stop_rsc_orphans      0x00000100ULL
 #  define pe_flag_stop_action_orphans   0x00000200ULL
 #  define pe_flag_stop_everything       0x00000400ULL
 
 #  define pe_flag_start_failure_fatal   0x00001000ULL
 
 //! \deprecated
 #  define pe_flag_remove_after_stop     0x00002000ULL
 
 #  define pe_flag_startup_fencing       0x00004000ULL
 #  define pe_flag_shutdown_lock         0x00008000ULL
 
 #  define pe_flag_startup_probes        0x00010000ULL
 #  define pe_flag_have_status           0x00020000ULL
 #  define pe_flag_have_remote_nodes     0x00040000ULL
 
 #  define pe_flag_quick_location        0x00100000ULL
 #  define pe_flag_sanitized             0x00200000ULL
 
 //! \deprecated
 #  define pe_flag_stdout                0x00400000ULL
 
 //! Don't count total, disabled and blocked resource instances
 #  define pe_flag_no_counts             0x00800000ULL
 
 /*! Skip deprecated code that is kept solely for backward API compatibility.
  * (Internal code should always set this.)
  */
 #  define pe_flag_no_compat             0x01000000ULL
 
 #  define pe_flag_show_scores           0x02000000ULL
 #  define pe_flag_show_utilization      0x04000000ULL
 
 struct pe_working_set_s {
     xmlNode *input;
     crm_time_t *now;
 
     /* options extracted from the input */
     char *dc_uuid;
     pe_node_t *dc_node;
     const char *stonith_action;
     const char *placement_strategy;
 
     unsigned long long flags;
 
     int stonith_timeout;
     enum pe_quorum_policy no_quorum_policy;
 
     GHashTable *config_hash;
     GHashTable *tickets;
 
     // Actions for which there can be only one (e.g. fence nodeX)
     GHashTable *singletons;
 
     GList *nodes;
     GList *resources;
     GList *placement_constraints;
     GList *ordering_constraints;
     GList *colocation_constraints;
     GList *ticket_constraints;
 
     GList *actions;
     xmlNode *failed;
     xmlNode *op_defaults;
     xmlNode *rsc_defaults;
 
     /* stats */
     int num_synapse;
     int max_valid_nodes;    //! Deprecated (will be removed in a future release)
     int order_id;
     int action_id;
 
     /* final output */
     xmlNode *graph;
 
     GHashTable *template_rsc_sets;
     const char *localhost;
     GHashTable *tags;
 
     int blocked_resources;
     int disabled_resources;
 
     GList *param_check; // History entries that need to be checked
     GList *stop_needed; // Containers that need stop actions
     time_t recheck_by;  // Hint to controller to re-run scheduler by this time
     int ninstances;     // Total number of resource instances
     guint shutdown_lock;// How long (seconds) to lock resources to shutdown node
     int priority_fencing_delay; // Priority fencing delay
 
     void *priv;
 };
 
 enum pe_check_parameters {
     /* Clear fail count if parameters changed for un-expired start or monitor
      * last_failure.
      */
     pe_check_last_failure,
 
     /* Clear fail count if parameters changed for start, monitor, promote, or
      * migrate_from actions for active resources.
      */
     pe_check_active,
 };
 
 struct pe_node_shared_s {
     const char *id;
     const char *uname;
     enum node_type type;
 
     /* @TODO convert these flags into a bitfield */
     gboolean online;
     gboolean standby;
     gboolean standby_onfail;
     gboolean pending;
     gboolean unclean;
     gboolean unseen;
     gboolean shutdown;
     gboolean expected_up;
     gboolean is_dc;
     gboolean maintenance;
     gboolean rsc_discovery_enabled;
     gboolean remote_requires_reset;
     gboolean remote_was_fenced;
     gboolean remote_maintenance; /* what the remote-rsc is thinking */
     gboolean unpacked;
 
     int num_resources;
     pe_resource_t *remote_rsc;
     GList *running_rsc;       /* pe_resource_t* */
     GList *allocated_rsc;     /* pe_resource_t* */
 
     GHashTable *attrs;          /* char* => char* */
     GHashTable *utilization;
     GHashTable *digest_cache;   //!< cache of calculated resource digests
     int priority; // calculated based on the priority of resources running on the node
 };
 
 struct pe_node_s {
     int weight;
     gboolean fixed;
     int count;
     struct pe_node_shared_s *details;
     int rsc_discover_mode;
 };
 
 #  define pe_rsc_orphan                     0x00000001ULL
 #  define pe_rsc_managed                    0x00000002ULL
 #  define pe_rsc_block                      0x00000004ULL
 #  define pe_rsc_orphan_container_filler    0x00000008ULL
 
 #  define pe_rsc_notify                     0x00000010ULL
 #  define pe_rsc_unique                     0x00000020ULL
 #  define pe_rsc_fence_device               0x00000040ULL
 #  define pe_rsc_promotable                 0x00000080ULL
 
 #  define pe_rsc_provisional                0x00000100ULL
 #  define pe_rsc_allocating                 0x00000200ULL
 #  define pe_rsc_merging                    0x00000400ULL
 
 #  define pe_rsc_stop                       0x00001000ULL
 #  define pe_rsc_reload                     0x00002000ULL
 #  define pe_rsc_allow_remote_remotes       0x00004000ULL
 #  define pe_rsc_critical                   0x00008000ULL
 
 #  define pe_rsc_failed                     0x00010000ULL
 #  define pe_rsc_runnable                   0x00040000ULL
 #  define pe_rsc_start_pending              0x00080000ULL
 
 #  define pe_rsc_starting                   0x00100000ULL
 #  define pe_rsc_stopping                   0x00200000ULL
 #  define pe_rsc_allow_migrate              0x00800000ULL
 
 #  define pe_rsc_failure_ignored            0x01000000ULL
 #  define pe_rsc_maintenance                0x04000000ULL
 #  define pe_rsc_is_container               0x08000000ULL
 
 #  define pe_rsc_needs_quorum               0x10000000ULL
 #  define pe_rsc_needs_fencing              0x20000000ULL
 #  define pe_rsc_needs_unfencing            0x40000000ULL
 
 enum pe_graph_flags {
     pe_graph_none = 0x00000,
     pe_graph_updated_first = 0x00001,
     pe_graph_updated_then = 0x00002,
     pe_graph_disable = 0x00004,
 };
 
 /* *INDENT-OFF* */
 enum pe_action_flags {
     pe_action_pseudo = 0x00001,
     pe_action_runnable = 0x00002,
     pe_action_optional = 0x00004,
     pe_action_print_always = 0x00008,
 
     pe_action_have_node_attrs = 0x00010,
     pe_action_implied_by_stonith = 0x00040,
     pe_action_migrate_runnable =   0x00080,
 
     pe_action_dumped = 0x00100,
     pe_action_processed = 0x00200,
-    pe_action_clear = 0x00400,
+#if !defined(PCMK_ALLOW_DEPRECATED) || (PCMK_ALLOW_DEPRECATED == 1)
+    pe_action_clear = 0x00400, //! \deprecated Unused
+#endif
     pe_action_dangle = 0x00800,
 
     /* This action requires one or more of its dependencies to be runnable.
      * We use this to clear the runnable flag before checking dependencies.
      */
     pe_action_requires_any = 0x01000,
 
     pe_action_reschedule = 0x02000,
     pe_action_tracking = 0x04000,
     pe_action_dedup = 0x08000, //! Internal state tracking when creating graph
 
     pe_action_dc = 0x10000,         //! Action may run on DC instead of target
 };
 /* *INDENT-ON* */
 
 struct pe_resource_s {
     char *id;
     char *clone_name;
     xmlNode *xml;
     xmlNode *orig_xml;
     xmlNode *ops_xml;
 
     pe_working_set_t *cluster;
     pe_resource_t *parent;
 
     enum pe_obj_types variant;
     void *variant_opaque;
     resource_object_functions_t *fns;
     resource_alloc_functions_t *cmds;
 
     enum rsc_recovery_type recovery_type;
 
     // @TODO only pe_restart_restart is of interest, so merge into flags
     enum pe_restart restart_type; //!< \deprecated will be removed in future release
 
     int priority;
     int stickiness;
     int sort_index;
     int failure_timeout;
     int migration_threshold;
     guint remote_reconnect_ms;
     char *pending_task;
 
     unsigned long long flags;
 
     // @TODO merge these into flags
     gboolean is_remote_node;
     gboolean exclusive_discover;
 
     //!@{
     //! This field should be treated as internal to Pacemaker
     GList *rsc_cons_lhs;      // List of pcmk__colocation_t*
     GList *rsc_cons;          // List of pcmk__colocation_t*
     GList *rsc_location;      // List of pe__location_t*
     GList *actions;           // List of pe_action_t*
     GList *rsc_tickets;       // List of rsc_ticket*
     //!@}
 
     pe_node_t *allocated_to;
     pe_node_t *partial_migration_target;
     pe_node_t *partial_migration_source;
     GList *running_on;        /* pe_node_t*   */
     GHashTable *known_on;       /* pe_node_t*   */
     GHashTable *allowed_nodes;  /* pe_node_t*   */
 
     enum rsc_role_e role;
     enum rsc_role_e next_role;
 
     GHashTable *meta;
     GHashTable *parameters; //! \deprecated Use pe_rsc_params() instead
     GHashTable *utilization;
 
     GList *children;          /* pe_resource_t*   */
     GList *dangling_migrations;       /* pe_node_t*       */
 
     pe_resource_t *container;
     GList *fillers;
 
     pe_node_t *pending_node;    // Node on which pending_task is happening
     pe_node_t *lock_node;       // Resource is shutdown-locked to this node
     time_t lock_time;           // When shutdown lock started
 
     /* Resource parameters may have node-attribute-based rules, which means the
      * values can vary by node. This table is a cache of parameter name/value
      * tables for each node (as needed). Use pe_rsc_params() to get the table
      * for a given node.
      */
     GHashTable *parameter_cache; // Key = node name, value = parameters table
 #if ENABLE_VERSIONED_ATTRS
     xmlNode *versioned_parameters;
 #endif
 };
 
 #if ENABLE_VERSIONED_ATTRS
 // Used as action->action_details if action->rsc is not NULL
 typedef struct pe_rsc_action_details_s {
     xmlNode *versioned_parameters;
     xmlNode *versioned_meta;
 } pe_rsc_action_details_t;
 #endif
 
 struct pe_action_s {
     int id;
     int priority;
 
     pe_resource_t *rsc;
     pe_node_t *node;
     xmlNode *op_entry;
 
     char *task;
     char *uuid;
     char *cancel_task;
     char *reason;
 
     enum pe_action_flags flags;
     enum rsc_start_requirement needs;
     enum action_fail_response on_fail;
     enum rsc_role_e fail_role;
 
     GHashTable *meta;
     GHashTable *extra;
 
     /* 
      * These two varables are associated with the constraint logic
      * that involves first having one or more actions runnable before
      * then allowing this action to execute.
      *
      * These varables are used with features such as 'clone-min' which
      * requires at minimum X number of cloned instances to be running
      * before an order dependency can run. Another option that uses
      * this is 'require-all=false' in ordering constrants. This option
      * says "only require one instance of a resource to start before
      * allowing dependencies to start" -- basically, require-all=false is
      * the same as clone-min=1.
      */
 
     /* current number of known runnable actions in the before list. */
     int runnable_before;
     /* the number of "before" runnable actions required for this action
      * to be considered runnable */ 
     int required_runnable_before;
 
     GList *actions_before;    /* pe_action_wrapper_t* */
     GList *actions_after;     /* pe_action_wrapper_t* */
 
     /* Some of the above fields could be moved to the details,
      * except for API backward compatibility.
      */
     void *action_details; // varies by type of action
 };
 
 typedef struct pe_ticket_s {
     char *id;
     gboolean granted;
     time_t last_granted;
     gboolean standby;
     GHashTable *state;
 } pe_ticket_t;
 
 typedef struct pe_tag_s {
     char *id;
     GList *refs;
 } pe_tag_t;
 
 //! Internal tracking for transition graph creation
 enum pe_link_state {
     pe_link_not_dumped, //! Internal tracking for transition graph creation
     pe_link_dumped,     //! Internal tracking for transition graph creation
     pe_link_dup,        //! \deprecated No longer used by Pacemaker
 };
 
 enum pe_discover_e {
     pe_discover_always = 0,
     pe_discover_never,
     pe_discover_exclusive,
 };
 
 /* *INDENT-OFF* */
 enum pe_ordering {
     pe_order_none                  = 0x0,       /* deleted */
     pe_order_optional              = 0x1,       /* pure ordering, nothing implied */
     pe_order_apply_first_non_migratable = 0x2,  /* Only apply this constraint's ordering if first is not migratable. */
 
     pe_order_implies_first         = 0x10,      /* If 'then' is required, ensure 'first' is too */
     pe_order_implies_then          = 0x20,      /* If 'first' is required, ensure 'then' is too */
     pe_order_promoted_implies_first = 0x40,     /* If 'then' is required and then's rsc is promoted, ensure 'first' becomes required too */
 
     /* first requires then to be both runnable and migrate runnable. */
     pe_order_implies_first_migratable  = 0x80,
 
     pe_order_runnable_left         = 0x100,     /* 'then' requires 'first' to be runnable */
 
     pe_order_pseudo_left           = 0x200,     /* 'then' can only be pseudo if 'first' is runnable */
     pe_order_implies_then_on_node  = 0x400,     /* If 'first' is required on 'nodeX',
                                                  * ensure instances of 'then' on 'nodeX' are too.
                                                  * Only really useful if 'then' is a clone and 'first' is not
                                                  */
     pe_order_probe                 = 0x800,     /* If 'first->rsc' is
                                                  *  - running but about to stop, ignore the constraint
                                                  *  - otherwise, behave as runnable_left
                                                  */
 
     pe_order_restart               = 0x1000,    /* 'then' is runnable if 'first' is optional or runnable */
     pe_order_stonith_stop          = 0x2000,    /* only applies if the action is non-pseudo */
     pe_order_serialize_only        = 0x4000,    /* serialize */
     pe_order_same_node             = 0x8000,    /* applies only if 'first' and 'then' are on same node */
 
     pe_order_implies_first_printed = 0x10000,   /* Like ..implies_first but only ensures 'first' is printed, not mandatory */
     pe_order_implies_then_printed  = 0x20000,   /* Like ..implies_then but only ensures 'then' is printed, not mandatory */
 
     pe_order_asymmetrical          = 0x100000,  /* Indicates asymmetrical one way ordering constraint. */
     pe_order_load                  = 0x200000,  /* Only relevant if... */
     pe_order_one_or_more           = 0x400000,  /* 'then' is runnable only if one or more of its dependencies are too */
     pe_order_anti_colocation       = 0x800000,
 
     pe_order_preserve              = 0x1000000, /* Hack for breaking user ordering constraints with container resources */
     pe_order_then_cancels_first    = 0x2000000, // if 'then' becomes required, 'first' becomes optional
     pe_order_trace                 = 0x4000000, /* test marker */
 
 #if !defined(PCMK_ALLOW_DEPRECATED) || (PCMK_ALLOW_DEPRECATED == 1)
     // \deprecated Use pe_order_promoted_implies_first instead
     pe_order_implies_first_master  = pe_order_promoted_implies_first,
 #endif
 };
 /* *INDENT-ON* */
 
 typedef struct pe_action_wrapper_s {
     enum pe_ordering type;
     enum pe_link_state state;
     pe_action_t *action;
 } pe_action_wrapper_t;
 
 #if !defined(PCMK_ALLOW_DEPRECATED) || (PCMK_ALLOW_DEPRECATED == 1)
 #include <crm/pengine/pe_types_compat.h>
 #endif
 
 #ifdef __cplusplus
 }
 #endif
 
 #endif // PE_TYPES__H
diff --git a/include/pcmki/pcmki_sched_allocate.h b/include/pcmki/pcmki_sched_allocate.h
index 52a11d418a..62fbbeb4d4 100644
--- a/include/pcmki/pcmki_sched_allocate.h
+++ b/include/pcmki/pcmki_sched_allocate.h
@@ -1,179 +1,178 @@
 /*
  * Copyright 2004-2021 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU Lesser General Public License
  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
  */
 
 #ifndef SCHED_ALLOCATE__H
 #  define SCHED_ALLOCATE__H
 
 #  include <glib.h>
 #  include <crm/common/xml.h>
 #  include <crm/pengine/status.h>
 #  include <crm/pengine/complex.h>
 #  include <crm/pengine/internal.h>
 #  include <pcmki/pcmki_scheduler.h>
 
 struct resource_alloc_functions_s {
     GHashTable *(*merge_weights) (pe_resource_t *, const char *, GHashTable *, const char *, float,
                                   enum pe_weights);
     pe_node_t *(*allocate) (pe_resource_t *, pe_node_t *, pe_working_set_t *);
     void (*create_actions) (pe_resource_t *, pe_working_set_t *);
      gboolean(*create_probe) (pe_resource_t *, pe_node_t *, pe_action_t *, gboolean, pe_working_set_t *);
     void (*internal_constraints) (pe_resource_t *, pe_working_set_t *);
 
     void (*rsc_colocation_lh) (pe_resource_t *, pe_resource_t *,
                                pcmk__colocation_t *, pe_working_set_t *);
     void (*rsc_colocation_rh) (pe_resource_t *, pe_resource_t *,
                                pcmk__colocation_t *, pe_working_set_t *);
 
     void (*rsc_location) (pe_resource_t *, pe__location_t *);
 
     enum pe_action_flags (*action_flags) (pe_action_t *, pe_node_t *);
     enum pe_graph_flags (*update_actions) (pe_action_t *, pe_action_t *,
                                            pe_node_t *, enum pe_action_flags,
                                            enum pe_action_flags,
                                            enum pe_ordering,
                                            pe_working_set_t *data_set);
 
     void (*expand) (pe_resource_t *, pe_working_set_t *);
     void (*append_meta) (pe_resource_t * rsc, xmlNode * xml);
 };
 
 GHashTable *pcmk__native_merge_weights(pe_resource_t *rsc, const char *rhs,
                                        GHashTable *nodes, const char *attr,
                                        float factor, uint32_t flags);
 
 GHashTable *pcmk__group_merge_weights(pe_resource_t *rsc, const char *rhs,
                                       GHashTable *nodes, const char *attr,
                                       float factor, uint32_t flags);
 
 pe_node_t *pcmk__native_allocate(pe_resource_t *rsc, pe_node_t *preferred,
                                  pe_working_set_t *data_set);
 extern void native_create_actions(pe_resource_t * rsc, pe_working_set_t * data_set);
 extern void native_internal_constraints(pe_resource_t * rsc, pe_working_set_t * data_set);
 void native_rsc_colocation_lh(pe_resource_t *lh_rsc, pe_resource_t *rh_rsc,
                               pcmk__colocation_t *constraint,
                               pe_working_set_t *data_set);
 void native_rsc_colocation_rh(pe_resource_t *lh_rsc, pe_resource_t *rh_rsc,
                               pcmk__colocation_t *constraint,
                               pe_working_set_t *data_set);
 extern void rsc_ticket_constraint(pe_resource_t * lh_rsc, rsc_ticket_t * rsc_ticket,
                                   pe_working_set_t * data_set);
 extern enum pe_action_flags native_action_flags(pe_action_t * action, pe_node_t * node);
 
 void native_rsc_location(pe_resource_t *rsc, pe__location_t *constraint);
 extern void native_expand(pe_resource_t * rsc, pe_working_set_t * data_set);
 extern gboolean native_create_probe(pe_resource_t * rsc, pe_node_t * node, pe_action_t * complete,
                                     gboolean force, pe_working_set_t * data_set);
 extern void native_append_meta(pe_resource_t * rsc, xmlNode * xml);
 
 pe_node_t *pcmk__group_allocate(pe_resource_t *rsc, pe_node_t *preferred,
                                 pe_working_set_t *data_set);
 extern void group_create_actions(pe_resource_t * rsc, pe_working_set_t * data_set);
 extern void group_internal_constraints(pe_resource_t * rsc, pe_working_set_t * data_set);
 void group_rsc_colocation_lh(pe_resource_t *lh_rsc, pe_resource_t *rh_rsc,
                              pcmk__colocation_t *constraint,
                              pe_working_set_t *data_set);
 void group_rsc_colocation_rh(pe_resource_t *lh_rsc, pe_resource_t *rh_rsc,
                              pcmk__colocation_t *constraint,
                              pe_working_set_t *data_set);
 extern enum pe_action_flags group_action_flags(pe_action_t * action, pe_node_t * node);
 void group_rsc_location(pe_resource_t *rsc, pe__location_t *constraint);
 extern void group_expand(pe_resource_t * rsc, pe_working_set_t * data_set);
 extern void group_append_meta(pe_resource_t * rsc, xmlNode * xml);
 
 pe_node_t *pcmk__bundle_allocate(pe_resource_t *rsc, pe_node_t *preferred,
                                  pe_working_set_t *data_set);
 void pcmk__bundle_create_actions(pe_resource_t *rsc,
                                  pe_working_set_t *data_set);
 gboolean pcmk__bundle_create_probe(pe_resource_t *rsc, pe_node_t *node,
                                    pe_action_t *complete, gboolean force,
                                    pe_working_set_t *data_set);
 void pcmk__bundle_internal_constraints(pe_resource_t *rsc,
                                        pe_working_set_t *data_set);
 void pcmk__bundle_rsc_colocation_lh(pe_resource_t *lh_rsc,
                                     pe_resource_t *rh_rsc,
                                     pcmk__colocation_t *constraint,
                                     pe_working_set_t *data_set);
 void pcmk__bundle_rsc_colocation_rh(pe_resource_t *lh_rsc,
                                     pe_resource_t *rh_rsc,
                                     pcmk__colocation_t *constraint,
                                     pe_working_set_t *data_set);
 void pcmk__bundle_rsc_location(pe_resource_t *rsc, pe__location_t *constraint);
 enum pe_action_flags pcmk__bundle_action_flags(pe_action_t *action,
                                                pe_node_t *node);
 void pcmk__bundle_expand(pe_resource_t *rsc, pe_working_set_t *data_set);
 void pcmk__bundle_append_meta(pe_resource_t *rsc, xmlNode *xml);
 
 pe_node_t *pcmk__clone_allocate(pe_resource_t *rsc, pe_node_t *preferred,
                                 pe_working_set_t *data_set);
 extern void clone_create_actions(pe_resource_t * rsc, pe_working_set_t * data_set);
 extern void clone_internal_constraints(pe_resource_t * rsc, pe_working_set_t * data_set);
 void clone_rsc_colocation_lh(pe_resource_t *lh_rsc, pe_resource_t *rh_rsc,
                              pcmk__colocation_t *constraint,
                              pe_working_set_t *data_set);
 void clone_rsc_colocation_rh(pe_resource_t *lh_rsc, pe_resource_t *rh_rsc,
                              pcmk__colocation_t *constraint,
                              pe_working_set_t *data_set);
 void clone_rsc_location(pe_resource_t *rsc, pe__location_t *constraint);
 extern enum pe_action_flags clone_action_flags(pe_action_t * action, pe_node_t * node);
 extern void clone_expand(pe_resource_t * rsc, pe_working_set_t * data_set);
 extern gboolean clone_create_probe(pe_resource_t * rsc, pe_node_t * node, pe_action_t * complete,
                                    gboolean force, pe_working_set_t * data_set);
 extern void clone_append_meta(pe_resource_t * rsc, xmlNode * xml);
 
 void pcmk__add_promotion_scores(pe_resource_t *rsc);
 pe_node_t *pcmk__set_instance_roles(pe_resource_t *rsc,
                                     pe_working_set_t *data_set);
 void create_promotable_actions(pe_resource_t *rsc, pe_working_set_t *data_set);
 void promote_demote_constraints(pe_resource_t *rsc, pe_working_set_t *data_set);
 void promotable_constraints(pe_resource_t *rsc, pe_working_set_t *data_set);
 void promotable_colocation_rh(pe_resource_t *lh_rsc, pe_resource_t *rh_rsc,
                               pcmk__colocation_t *constraint,
                               pe_working_set_t *data_set);
 
 /* extern resource_object_functions_t resource_variants[]; */
 extern resource_alloc_functions_t resource_class_alloc_functions[];
 
 extern gboolean unpack_rsc_order(xmlNode * xml_obj, pe_working_set_t * data_set);
 
 extern gboolean unpack_rsc_ticket(xmlNode * xml_obj, pe_working_set_t * data_set);
 
 void LogNodeActions(pe_working_set_t * data_set);
 void LogActions(pe_resource_t * rsc, pe_working_set_t * data_set);
 void pcmk__bundle_log_actions(pe_resource_t *rsc, pe_working_set_t *data_set);
 
 extern void rsc_stonith_ordering(pe_resource_t * rsc, pe_action_t * stonith_op,
                                  pe_working_set_t * data_set);
 
 enum pe_graph_flags native_update_actions(pe_action_t *first, pe_action_t *then,
                                           pe_node_t *node,
                                           enum pe_action_flags flags,
                                           enum pe_action_flags filter,
                                           enum pe_ordering type,
                                           pe_working_set_t *data_set);
 enum pe_graph_flags group_update_actions(pe_action_t *first, pe_action_t *then,
                                          pe_node_t *node,
                                          enum pe_action_flags flags,
                                          enum pe_action_flags filter,
                                          enum pe_ordering type,
                                          pe_working_set_t *data_set);
 enum pe_graph_flags pcmk__multi_update_actions(pe_action_t *first,
                                                pe_action_t *then,
                                                pe_node_t *node,
                                                enum pe_action_flags flags,
                                                enum pe_action_flags filter,
                                                enum pe_ordering type,
                                                pe_working_set_t *data_set);
 
-gboolean update_action_flags(pe_action_t * action, enum pe_action_flags flags, const char *source, int line);
 gboolean update_action(pe_action_t *action, pe_working_set_t *data_set);
 void complex_set_cmds(pe_resource_t * rsc);
 void pcmk__log_transition_summary(const char *filename);
 void clone_create_pseudo_actions(
     pe_resource_t * rsc, GList *children, notify_data_t **start_notify, notify_data_t **stop_notify,  pe_working_set_t * data_set);
 #endif
diff --git a/lib/pacemaker/pcmk_sched_allocate.c b/lib/pacemaker/pcmk_sched_allocate.c
index f51955b666..84feec983c 100644
--- a/lib/pacemaker/pcmk_sched_allocate.c
+++ b/lib/pacemaker/pcmk_sched_allocate.c
@@ -1,3093 +1,3063 @@
 /*
  * Copyright 2004-2021 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU General Public License version 2
  * or later (GPLv2+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #include <sys/param.h>
 
 #include <crm/crm.h>
 #include <crm/cib.h>
 #include <crm/msg_xml.h>
 #include <crm/common/xml.h>
 #include <crm/common/xml_internal.h>
 
 #include <glib.h>
 
 #include <crm/pengine/status.h>
 #include <pacemaker-internal.h>
 
 CRM_TRACE_INIT_DATA(pacemaker);
 
 extern bool pcmk__is_daemon;
 
 void set_alloc_actions(pe_working_set_t * data_set);
 extern void ReloadRsc(pe_resource_t * rsc, pe_node_t *node, pe_working_set_t * data_set);
 extern gboolean DeleteRsc(pe_resource_t * rsc, pe_node_t * node, gboolean optional, pe_working_set_t * data_set);
 static void apply_remote_node_ordering(pe_working_set_t *data_set);
 static enum remote_connection_state get_remote_node_state(pe_node_t *node);
 
 enum remote_connection_state {
     remote_state_unknown = 0,
     remote_state_alive = 1,
     remote_state_resting = 2,
     remote_state_failed = 3,
     remote_state_stopped = 4
 };
 
 static const char *
 state2text(enum remote_connection_state state)
 {
     switch (state) {
         case remote_state_unknown:
             return "unknown";
         case remote_state_alive:
             return "alive";
         case remote_state_resting:
             return "resting";
         case remote_state_failed:
             return "failed";
         case remote_state_stopped:
             return "stopped";
     }
 
     return "impossible";
 }
 
 resource_alloc_functions_t resource_class_alloc_functions[] = {
     {
      pcmk__native_merge_weights,
      pcmk__native_allocate,
      native_create_actions,
      native_create_probe,
      native_internal_constraints,
      native_rsc_colocation_lh,
      native_rsc_colocation_rh,
      native_rsc_location,
      native_action_flags,
      native_update_actions,
      native_expand,
      native_append_meta,
      },
     {
      pcmk__group_merge_weights,
      pcmk__group_allocate,
      group_create_actions,
      native_create_probe,
      group_internal_constraints,
      group_rsc_colocation_lh,
      group_rsc_colocation_rh,
      group_rsc_location,
      group_action_flags,
      group_update_actions,
      group_expand,
      group_append_meta,
      },
     {
      pcmk__native_merge_weights,
      pcmk__clone_allocate,
      clone_create_actions,
      clone_create_probe,
      clone_internal_constraints,
      clone_rsc_colocation_lh,
      clone_rsc_colocation_rh,
      clone_rsc_location,
      clone_action_flags,
      pcmk__multi_update_actions,
      clone_expand,
      clone_append_meta,
      },
     {
      pcmk__native_merge_weights,
      pcmk__bundle_allocate,
      pcmk__bundle_create_actions,
      pcmk__bundle_create_probe,
      pcmk__bundle_internal_constraints,
      pcmk__bundle_rsc_colocation_lh,
      pcmk__bundle_rsc_colocation_rh,
      pcmk__bundle_rsc_location,
      pcmk__bundle_action_flags,
      pcmk__multi_update_actions,
      pcmk__bundle_expand,
      pcmk__bundle_append_meta,
      }
 };
 
-gboolean
-update_action_flags(pe_action_t * action, enum pe_action_flags flags, const char *source, int line)
-{
-    static unsigned long calls = 0;
-    gboolean changed = FALSE;
-    gboolean clear = pcmk_is_set(flags, pe_action_clear);
-    enum pe_action_flags last = action->flags;
-
-    if (clear) {
-        pe__clear_action_flags_as(source, line, action, flags);
-    } else {
-        pe__set_action_flags_as(source, line, action, flags);
-    }
-
-    if (last != action->flags) {
-        calls++;
-        changed = TRUE;
-        /* Useful for tracking down _who_ changed a specific flag */
-        /* CRM_ASSERT(calls != 534); */
-        pe__clear_raw_action_flags(flags, "action update", pe_action_clear);
-        crm_trace("%s on %s: %sset flags 0x%.6x (was 0x%.6x, now 0x%.6x, %lu, %s)",
-                  action->uuid, action->node ? action->node->details->uname : "[none]",
-                  clear ? "un-" : "", flags, last, action->flags, calls, source);
-    }
-
-    return changed;
-}
-
 static gboolean
 check_rsc_parameters(pe_resource_t * rsc, pe_node_t * node, xmlNode * rsc_entry,
                      gboolean active_here, pe_working_set_t * data_set)
 {
     int attr_lpc = 0;
     gboolean force_restart = FALSE;
     gboolean delete_resource = FALSE;
     gboolean changed = FALSE;
 
     const char *value = NULL;
     const char *old_value = NULL;
 
     const char *attr_list[] = {
         XML_ATTR_TYPE,
         XML_AGENT_ATTR_CLASS,
         XML_AGENT_ATTR_PROVIDER
     };
 
     for (; attr_lpc < PCMK__NELEM(attr_list); attr_lpc++) {
         value = crm_element_value(rsc->xml, attr_list[attr_lpc]);
         old_value = crm_element_value(rsc_entry, attr_list[attr_lpc]);
         if (value == old_value  /* i.e. NULL */
             || pcmk__str_eq(value, old_value, pcmk__str_none)) {
             continue;
         }
 
         changed = TRUE;
         trigger_unfencing(rsc, node, "Device definition changed", NULL, data_set);
         if (active_here) {
             force_restart = TRUE;
             crm_notice("Forcing restart of %s on %s, %s changed: %s -> %s",
                        rsc->id, node->details->uname, attr_list[attr_lpc],
                        crm_str(old_value), crm_str(value));
         }
     }
     if (force_restart) {
         /* make sure the restart happens */
         stop_action(rsc, node, FALSE);
         pe__set_resource_flags(rsc, pe_rsc_start_pending);
         delete_resource = TRUE;
 
     } else if (changed) {
         delete_resource = TRUE;
     }
     return delete_resource;
 }
 
 static void
 CancelXmlOp(pe_resource_t * rsc, xmlNode * xml_op, pe_node_t * active_node,
             const char *reason, pe_working_set_t * data_set)
 {
     guint interval_ms = 0;
     pe_action_t *cancel = NULL;
 
     const char *task = NULL;
     const char *call_id = NULL;
 
     CRM_CHECK(xml_op != NULL, return);
     CRM_CHECK(active_node != NULL, return);
 
     task = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
     call_id = crm_element_value(xml_op, XML_LRM_ATTR_CALLID);
     crm_element_value_ms(xml_op, XML_LRM_ATTR_INTERVAL_MS, &interval_ms);
 
     crm_info("Action " PCMK__OP_FMT " on %s will be stopped: %s",
              rsc->id, task, interval_ms,
              active_node->details->uname, (reason? reason : "unknown"));
 
     cancel = pe_cancel_op(rsc, task, interval_ms, active_node, data_set);
     add_hash_param(cancel->meta, XML_LRM_ATTR_CALLID, call_id);
     custom_action_order(rsc, stop_key(rsc), NULL, rsc, NULL, cancel, pe_order_optional, data_set);
 }
 
 static gboolean
 check_action_definition(pe_resource_t * rsc, pe_node_t * active_node, xmlNode * xml_op,
                         pe_working_set_t * data_set)
 {
     char *key = NULL;
     guint interval_ms = 0;
     const op_digest_cache_t *digest_data = NULL;
     gboolean did_change = FALSE;
 
     const char *task = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
     const char *digest_secure = NULL;
 
     CRM_CHECK(active_node != NULL, return FALSE);
 
     crm_element_value_ms(xml_op, XML_LRM_ATTR_INTERVAL_MS, &interval_ms);
     if (interval_ms > 0) {
         xmlNode *op_match = NULL;
 
         /* we need to reconstruct the key because of the way we used to construct resource IDs */
         key = pcmk__op_key(rsc->id, task, interval_ms);
 
         pe_rsc_trace(rsc, "Checking parameters for %s", key);
         op_match = find_rsc_op_entry(rsc, key);
 
         if ((op_match == NULL)
             && pcmk_is_set(data_set->flags, pe_flag_stop_action_orphans)) {
             CancelXmlOp(rsc, xml_op, active_node, "orphan", data_set);
             free(key);
             return TRUE;
 
         } else if (op_match == NULL) {
             pe_rsc_debug(rsc, "Orphan action detected: %s on %s", key, active_node->details->uname);
             free(key);
             return TRUE;
         }
         free(key);
         key = NULL;
     }
 
     crm_trace("Testing " PCMK__OP_FMT " on %s",
               rsc->id, task, interval_ms, active_node->details->uname);
     if ((interval_ms == 0) && pcmk__str_eq(task, RSC_STATUS, pcmk__str_casei)) {
         /* Reload based on the start action not a probe */
         task = RSC_START;
 
     } else if ((interval_ms == 0) && pcmk__str_eq(task, RSC_MIGRATED, pcmk__str_casei)) {
         /* Reload based on the start action not a migrate */
         task = RSC_START;
     } else if ((interval_ms == 0) && pcmk__str_eq(task, RSC_PROMOTE, pcmk__str_casei)) {
         /* Reload based on the start action not a promote */
         task = RSC_START;
     }
 
     digest_data = rsc_action_digest_cmp(rsc, xml_op, active_node, data_set);
 
     if (pcmk_is_set(data_set->flags, pe_flag_sanitized)) {
         digest_secure = crm_element_value(xml_op, XML_LRM_ATTR_SECURE_DIGEST);
     }
 
     if(digest_data->rc != RSC_DIGEST_MATCH
        && digest_secure
        && digest_data->digest_secure_calc
        && strcmp(digest_data->digest_secure_calc, digest_secure) == 0) {
         if (!pcmk__is_daemon && data_set->priv != NULL) {
             pcmk__output_t *out = data_set->priv;
             out->info(out, "Only 'private' parameters to "
                       PCMK__OP_FMT " on %s changed: %s", rsc->id, task,
                       interval_ms, active_node->details->uname,
                       crm_element_value(xml_op, XML_ATTR_TRANSITION_MAGIC));
         }
 
     } else if (digest_data->rc == RSC_DIGEST_RESTART) {
         /* Changes that force a restart */
         pe_action_t *required = NULL;
 
         did_change = TRUE;
         key = pcmk__op_key(rsc->id, task, interval_ms);
         crm_log_xml_info(digest_data->params_restart, "params:restart");
         required = custom_action(rsc, key, task, NULL, TRUE, TRUE, data_set);
         pe_action_set_flag_reason(__func__, __LINE__, required, NULL,
                                   "resource definition change", pe_action_optional, TRUE);
 
         trigger_unfencing(rsc, active_node, "Device parameters changed", NULL, data_set);
 
     } else if ((digest_data->rc == RSC_DIGEST_ALL) || (digest_data->rc == RSC_DIGEST_UNKNOWN)) {
         // Changes that can potentially be handled by an agent reload
         const char *digest_restart = crm_element_value(xml_op, XML_LRM_ATTR_RESTART_DIGEST);
 
         did_change = TRUE;
         trigger_unfencing(rsc, active_node, "Device parameters changed (reload)", NULL, data_set);
         crm_log_xml_info(digest_data->params_all, "params:reload");
         key = pcmk__op_key(rsc->id, task, interval_ms);
 
         if (interval_ms > 0) {
             pe_action_t *op = NULL;
 
 #if 0
             /* Always reload/restart the entire resource */
             ReloadRsc(rsc, active_node, data_set);
 #else
             /* Re-sending the recurring op is sufficient - the old one will be cancelled automatically */
             op = custom_action(rsc, key, task, active_node, TRUE, TRUE, data_set);
             pe__set_action_flags(op, pe_action_reschedule);
 #endif
 
         } else if (digest_restart) {
             pe_rsc_trace(rsc, "Reloading '%s' action for resource %s", task, rsc->id);
 
             /* Reload this resource */
             ReloadRsc(rsc, active_node, data_set);
             free(key);
 
         } else {
             pe_action_t *required = NULL;
             pe_rsc_trace(rsc, "Resource %s doesn't support agent reloads",
                          rsc->id);
 
             /* Re-send the start/demote/promote op
              * Recurring ops will be detected independently
              */
             required = custom_action(rsc, key, task, NULL, TRUE, TRUE, data_set);
             pe_action_set_flag_reason(__func__, __LINE__, required, NULL,
                                       "resource definition change", pe_action_optional, TRUE);
         }
     }
 
     return did_change;
 }
 
 /*!
  * \internal
  * \brief Do deferred action checks after allocation
  *
  * \param[in] data_set  Working set for cluster
  */
 static void
 check_params(pe_resource_t *rsc, pe_node_t *node, xmlNode *rsc_op,
              enum pe_check_parameters check, pe_working_set_t *data_set)
 {
     const char *reason = NULL;
     op_digest_cache_t *digest_data = NULL;
 
     switch (check) {
         case pe_check_active:
             if (check_action_definition(rsc, node, rsc_op, data_set)
                 && pe_get_failcount(node, rsc, NULL, pe_fc_effective, NULL,
                                     data_set)) {
 
                 reason = "action definition changed";
             }
             break;
 
         case pe_check_last_failure:
             digest_data = rsc_action_digest_cmp(rsc, rsc_op, node, data_set);
             switch (digest_data->rc) {
                 case RSC_DIGEST_UNKNOWN:
                     crm_trace("Resource %s history entry %s on %s has no digest to compare",
                               rsc->id, ID(rsc_op), node->details->id);
                     break;
                 case RSC_DIGEST_MATCH:
                     break;
                 default:
                     reason = "resource parameters have changed";
                     break;
             }
             break;
     }
 
     if (reason) {
         pe__clear_failcount(rsc, node, reason, data_set);
     }
 }
 
 static void
 check_actions_for(xmlNode * rsc_entry, pe_resource_t * rsc, pe_node_t * node, pe_working_set_t * data_set)
 {
     GList *gIter = NULL;
     int offset = -1;
     int stop_index = 0;
     int start_index = 0;
 
     const char *task = NULL;
 
     xmlNode *rsc_op = NULL;
     GList *op_list = NULL;
     GList *sorted_op_list = NULL;
 
     CRM_CHECK(node != NULL, return);
 
     if (pcmk_is_set(rsc->flags, pe_rsc_orphan)) {
         pe_resource_t *parent = uber_parent(rsc);
         if(parent == NULL
            || pe_rsc_is_clone(parent) == FALSE
            || pcmk_is_set(parent->flags, pe_rsc_unique)) {
             pe_rsc_trace(rsc, "Skipping param check for %s and deleting: orphan", rsc->id);
             DeleteRsc(rsc, node, FALSE, data_set);
         } else {
             pe_rsc_trace(rsc, "Skipping param check for %s (orphan clone)", rsc->id);
         }
         return;
 
     } else if (pe_find_node_id(rsc->running_on, node->details->id) == NULL) {
         if (check_rsc_parameters(rsc, node, rsc_entry, FALSE, data_set)) {
             DeleteRsc(rsc, node, FALSE, data_set);
         }
         pe_rsc_trace(rsc, "Skipping param check for %s: no longer active on %s",
                      rsc->id, node->details->uname);
         return;
     }
 
     pe_rsc_trace(rsc, "Processing %s on %s", rsc->id, node->details->uname);
 
     if (check_rsc_parameters(rsc, node, rsc_entry, TRUE, data_set)) {
         DeleteRsc(rsc, node, FALSE, data_set);
     }
 
     for (rsc_op = pcmk__xe_first_child(rsc_entry); rsc_op != NULL;
          rsc_op = pcmk__xe_next(rsc_op)) {
 
         if (pcmk__str_eq((const char *)rsc_op->name, XML_LRM_TAG_RSC_OP, pcmk__str_none)) {
             op_list = g_list_prepend(op_list, rsc_op);
         }
     }
 
     sorted_op_list = g_list_sort(op_list, sort_op_by_callid);
     calculate_active_ops(sorted_op_list, &start_index, &stop_index);
 
     for (gIter = sorted_op_list; gIter != NULL; gIter = gIter->next) {
         xmlNode *rsc_op = (xmlNode *) gIter->data;
         guint interval_ms = 0;
 
         offset++;
 
         if (start_index < stop_index) {
             /* stopped */
             continue;
         } else if (offset < start_index) {
             /* action occurred prior to a start */
             continue;
         }
 
         task = crm_element_value(rsc_op, XML_LRM_ATTR_TASK);
         crm_element_value_ms(rsc_op, XML_LRM_ATTR_INTERVAL_MS, &interval_ms);
 
         if ((interval_ms > 0) &&
             (pcmk_is_set(rsc->flags, pe_rsc_maintenance) || node->details->maintenance)) {
             // Maintenance mode cancels recurring operations
             CancelXmlOp(rsc, rsc_op, node, "maintenance mode", data_set);
 
         } else if ((interval_ms > 0) || pcmk__strcase_any_of(task, RSC_STATUS, RSC_START,
                                                              RSC_PROMOTE, RSC_MIGRATED, NULL)) {
             /* If a resource operation failed, and the operation's definition
              * has changed, clear any fail count so they can be retried fresh.
              */
 
             if (pe__bundle_needs_remote_name(rsc, data_set)) {
                 /* We haven't allocated resources to nodes yet, so if the
                  * REMOTE_CONTAINER_HACK is used, we may calculate the digest
                  * based on the literal "#uname" value rather than the properly
                  * substituted value. That would mistakenly make the action
                  * definition appear to have been changed. Defer the check until
                  * later in this case.
                  */
                 pe__add_param_check(rsc_op, rsc, node, pe_check_active,
                                     data_set);
 
             } else if (check_action_definition(rsc, node, rsc_op, data_set)
                 && pe_get_failcount(node, rsc, NULL, pe_fc_effective, NULL,
                                     data_set)) {
                 pe__clear_failcount(rsc, node, "action definition changed",
                                     data_set);
             }
         }
     }
     g_list_free(sorted_op_list);
 }
 
 static GList *
 find_rsc_list(GList *result, pe_resource_t * rsc, const char *id, gboolean renamed_clones,
               gboolean partial, pe_working_set_t * data_set)
 {
     GList *gIter = NULL;
     gboolean match = FALSE;
 
     if (id == NULL) {
         return NULL;
     }
 
     if (rsc == NULL) {
         if (data_set == NULL) {
             return NULL;
         }
         for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
             pe_resource_t *child = (pe_resource_t *) gIter->data;
 
             result = find_rsc_list(result, child, id, renamed_clones, partial,
                                    NULL);
         }
         return result;
     }
 
     if (partial) {
         if (strstr(rsc->id, id)) {
             match = TRUE;
 
         } else if (renamed_clones && rsc->clone_name && strstr(rsc->clone_name, id)) {
             match = TRUE;
         }
 
     } else {
         if (strcmp(rsc->id, id) == 0) {
             match = TRUE;
 
         } else if (renamed_clones && rsc->clone_name && strcmp(rsc->clone_name, id) == 0) {
             match = TRUE;
         }
     }
 
     if (match) {
         result = g_list_prepend(result, rsc);
     }
 
     if (rsc->children) {
         gIter = rsc->children;
         for (; gIter != NULL; gIter = gIter->next) {
             pe_resource_t *child = (pe_resource_t *) gIter->data;
 
             result = find_rsc_list(result, child, id, renamed_clones, partial, NULL);
         }
     }
 
     return result;
 }
 
 static void
 check_actions(pe_working_set_t * data_set)
 {
     const char *id = NULL;
     pe_node_t *node = NULL;
     xmlNode *lrm_rscs = NULL;
     xmlNode *status = get_object_root(XML_CIB_TAG_STATUS, data_set->input);
 
     xmlNode *node_state = NULL;
 
     for (node_state = pcmk__xe_first_child(status); node_state != NULL;
          node_state = pcmk__xe_next(node_state)) {
 
         if (pcmk__str_eq((const char *)node_state->name, XML_CIB_TAG_STATE,
                          pcmk__str_none)) {
             id = crm_element_value(node_state, XML_ATTR_ID);
             lrm_rscs = find_xml_node(node_state, XML_CIB_TAG_LRM, FALSE);
             lrm_rscs = find_xml_node(lrm_rscs, XML_LRM_TAG_RESOURCES, FALSE);
 
             node = pe_find_node_id(data_set->nodes, id);
 
             if (node == NULL) {
                 continue;
 
             /* Still need to check actions for a maintenance node to cancel existing monitor operations */
             } else if (can_run_resources(node) == FALSE && node->details->maintenance == FALSE) {
                 crm_trace("Skipping param check for %s: can't run resources",
                           node->details->uname);
                 continue;
             }
 
             crm_trace("Processing node %s", node->details->uname);
             if (node->details->online
                 || pcmk_is_set(data_set->flags, pe_flag_stonith_enabled)) {
                 xmlNode *rsc_entry = NULL;
 
                 for (rsc_entry = pcmk__xe_first_child(lrm_rscs);
                      rsc_entry != NULL;
                      rsc_entry = pcmk__xe_next(rsc_entry)) {
 
                     if (pcmk__str_eq((const char *)rsc_entry->name, XML_LRM_TAG_RESOURCE, pcmk__str_none)) {
 
                         if (xml_has_children(rsc_entry)) {
                             GList *gIter = NULL;
                             GList *result = NULL;
                             const char *rsc_id = ID(rsc_entry);
 
                             CRM_CHECK(rsc_id != NULL, return);
 
                             result = find_rsc_list(NULL, NULL, rsc_id, TRUE, FALSE, data_set);
                             for (gIter = result; gIter != NULL; gIter = gIter->next) {
                                 pe_resource_t *rsc = (pe_resource_t *) gIter->data;
 
                                 if (rsc->variant != pe_native) {
                                     continue;
                                 }
                                 check_actions_for(rsc_entry, rsc, node, data_set);
                             }
                             g_list_free(result);
                         }
                     }
                 }
             }
         }
     }
 }
 
 static void
 apply_placement_constraints(pe_working_set_t * data_set)
 {
     for (GList *gIter = data_set->placement_constraints;
          gIter != NULL; gIter = gIter->next) {
         pe__location_t *cons = gIter->data;
 
         cons->rsc_lh->cmds->rsc_location(cons->rsc_lh, cons);
     }
 }
 
 static gboolean
 failcount_clear_action_exists(pe_node_t * node, pe_resource_t * rsc)
 {
     gboolean rc = FALSE;
     GList *list = pe__resource_actions(rsc, node, CRM_OP_CLEAR_FAILCOUNT, TRUE);
 
     if (list) {
         rc = TRUE;
     }
     g_list_free(list);
     return rc;
 }
 
 /*!
  * \internal
  * \brief Force resource away if failures hit migration threshold
  *
  * \param[in,out] rsc       Resource to check for failures
  * \param[in,out] node      Node to check for failures
  * \param[in,out] data_set  Cluster working set to update
  */
 static void
 check_migration_threshold(pe_resource_t *rsc, pe_node_t *node,
                           pe_working_set_t *data_set)
 {
     int fail_count, countdown;
     pe_resource_t *failed;
 
     /* Migration threshold of 0 means never force away */
     if (rsc->migration_threshold == 0) {
         return;
     }
 
     // If we're ignoring failures, also ignore the migration threshold
     if (pcmk_is_set(rsc->flags, pe_rsc_failure_ignored)) {
         return;
     }
 
     /* If there are no failures, there's no need to force away */
     fail_count = pe_get_failcount(node, rsc, NULL,
                                   pe_fc_effective|pe_fc_fillers, NULL,
                                   data_set);
     if (fail_count <= 0) {
         return;
     }
 
     /* How many more times recovery will be tried on this node */
     countdown = QB_MAX(rsc->migration_threshold - fail_count, 0);
 
     /* If failed resource has a parent, we'll force the parent away */
     failed = rsc;
     if (!pcmk_is_set(rsc->flags, pe_rsc_unique)) {
         failed = uber_parent(rsc);
     }
 
     if (countdown == 0) {
         resource_location(failed, node, -INFINITY, "__fail_limit__", data_set);
         crm_warn("Forcing %s away from %s after %d failures (max=%d)",
                  failed->id, node->details->uname, fail_count,
                  rsc->migration_threshold);
     } else {
         crm_info("%s can fail %d more times on %s before being forced off",
                  failed->id, countdown, node->details->uname);
     }
 }
 
 static void
 common_apply_stickiness(pe_resource_t * rsc, pe_node_t * node, pe_working_set_t * data_set)
 {
     if (rsc->children) {
         GList *gIter = rsc->children;
 
         for (; gIter != NULL; gIter = gIter->next) {
             pe_resource_t *child_rsc = (pe_resource_t *) gIter->data;
 
             common_apply_stickiness(child_rsc, node, data_set);
         }
         return;
     }
 
     if (pcmk_is_set(rsc->flags, pe_rsc_managed)
         && rsc->stickiness != 0 && pcmk__list_of_1(rsc->running_on)) {
         pe_node_t *current = pe_find_node_id(rsc->running_on, node->details->id);
         pe_node_t *match = pe_hash_table_lookup(rsc->allowed_nodes, node->details->id);
 
         if (current == NULL) {
 
         } else if ((match != NULL)
                    || pcmk_is_set(data_set->flags, pe_flag_symmetric_cluster)) {
             pe_resource_t *sticky_rsc = rsc;
 
             resource_location(sticky_rsc, node, rsc->stickiness, "stickiness", data_set);
             pe_rsc_debug(sticky_rsc, "Resource %s: preferring current location"
                          " (node=%s, weight=%d)", sticky_rsc->id,
                          node->details->uname, rsc->stickiness);
         } else {
             GHashTableIter iter;
             pe_node_t *nIter = NULL;
 
             pe_rsc_debug(rsc, "Ignoring stickiness for %s: the cluster is asymmetric"
                          " and node %s is not explicitly allowed", rsc->id, node->details->uname);
             g_hash_table_iter_init(&iter, rsc->allowed_nodes);
             while (g_hash_table_iter_next(&iter, NULL, (void **)&nIter)) {
                 crm_err("%s[%s] = %d", rsc->id, nIter->details->uname, nIter->weight);
             }
         }
     }
 
     /* Check the migration threshold only if a failcount clear action
      * has not already been placed for this resource on the node.
      * There is no sense in potentially forcing the resource from this
      * node if the failcount is being reset anyway.
      *
      * @TODO A clear_failcount operation can be scheduled in stage4() via
      * check_actions_for(), or in stage5() via check_params(). This runs in
      * stage2(), so it cannot detect those, meaning we might check the migration
      * threshold when we shouldn't -- worst case, we stop or move the resource,
      * then move it back next transition.
      */
     if (failcount_clear_action_exists(node, rsc) == FALSE) {
         check_migration_threshold(rsc, node, data_set);
     }
 }
 
 void
 complex_set_cmds(pe_resource_t * rsc)
 {
     GList *gIter = rsc->children;
 
     rsc->cmds = &resource_class_alloc_functions[rsc->variant];
 
     for (; gIter != NULL; gIter = gIter->next) {
         pe_resource_t *child_rsc = (pe_resource_t *) gIter->data;
 
         complex_set_cmds(child_rsc);
     }
 }
 
 void
 set_alloc_actions(pe_working_set_t * data_set)
 {
 
     GList *gIter = data_set->resources;
 
     for (; gIter != NULL; gIter = gIter->next) {
         pe_resource_t *rsc = (pe_resource_t *) gIter->data;
 
         complex_set_cmds(rsc);
     }
 }
 
 static void
 calculate_system_health(gpointer gKey, gpointer gValue, gpointer user_data)
 {
     const char *key = (const char *)gKey;
     const char *value = (const char *)gValue;
     int *system_health = (int *)user_data;
 
     if (!gKey || !gValue || !user_data) {
         return;
     }
 
     if (pcmk__starts_with(key, "#health")) {
         int score;
 
         /* Convert the value into an integer */
         score = char2score(value);
 
         /* Add it to the running total */
         *system_health = pe__add_scores(score, *system_health);
     }
 }
 
 static gboolean
 apply_system_health(pe_working_set_t * data_set)
 {
     GList *gIter = NULL;
     const char *health_strategy = pe_pref(data_set->config_hash, "node-health-strategy");
     int base_health = 0;
 
     if (pcmk__str_eq(health_strategy, "none", pcmk__str_null_matches | pcmk__str_casei)) {
         /* Prevent any accidental health -> score translation */
         pcmk__score_red = 0;
         pcmk__score_yellow = 0;
         pcmk__score_green = 0;
         return TRUE;
 
     } else if (pcmk__str_eq(health_strategy, "migrate-on-red", pcmk__str_casei)) {
 
         /* Resources on nodes which have health values of red are
          * weighted away from that node.
          */
         pcmk__score_red = -INFINITY;
         pcmk__score_yellow = 0;
         pcmk__score_green = 0;
 
     } else if (pcmk__str_eq(health_strategy, "only-green", pcmk__str_casei)) {
 
         /* Resources on nodes which have health values of red or yellow
          * are forced away from that node.
          */
         pcmk__score_red = -INFINITY;
         pcmk__score_yellow = -INFINITY;
         pcmk__score_green = 0;
 
     } else if (pcmk__str_eq(health_strategy, "progressive", pcmk__str_casei)) {
         /* Same as the above, but use the r/y/g scores provided by the user
          * Defaults are provided by the pe_prefs table
          * Also, custom health "base score" can be used
          */
         base_health = char2score(pe_pref(data_set->config_hash,
                                          "node-health-base"));
 
     } else if (pcmk__str_eq(health_strategy, "custom", pcmk__str_casei)) {
 
         /* Requires the admin to configure the rsc_location constaints for
          * processing the stored health scores
          */
         /* TODO: Check for the existence of appropriate node health constraints */
         return TRUE;
 
     } else {
         crm_err("Unknown node health strategy: %s", health_strategy);
         return FALSE;
     }
 
     crm_info("Applying automated node health strategy: %s", health_strategy);
 
     for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
         int system_health = base_health;
         pe_node_t *node = (pe_node_t *) gIter->data;
 
         /* Search through the node hash table for system health entries. */
         g_hash_table_foreach(node->details->attrs, calculate_system_health, &system_health);
 
         crm_info(" Node %s has an combined system health of %d",
                  node->details->uname, system_health);
 
         /* If the health is non-zero, then create a new rsc2node so that the
          * weight will be added later on.
          */
         if (system_health != 0) {
 
             GList *gIter2 = data_set->resources;
 
             for (; gIter2 != NULL; gIter2 = gIter2->next) {
                 pe_resource_t *rsc = (pe_resource_t *) gIter2->data;
 
                 rsc2node_new(health_strategy, rsc, system_health, NULL, node, data_set);
             }
         }
     }
 
     return TRUE;
 }
 
 gboolean
 stage0(pe_working_set_t * data_set)
 {
     xmlNode *cib_constraints = get_object_root(XML_CIB_TAG_CONSTRAINTS, data_set->input);
 
     if (data_set->input == NULL) {
         return FALSE;
     }
 
     if (!pcmk_is_set(data_set->flags, pe_flag_have_status)) {
         crm_trace("Calculating status");
         cluster_status(data_set);
     }
 
     set_alloc_actions(data_set);
     apply_system_health(data_set);
     unpack_constraints(cib_constraints, data_set);
 
     return TRUE;
 }
 
 /*
  * Check nodes for resources started outside of the LRM
  */
 gboolean
 probe_resources(pe_working_set_t * data_set)
 {
     pe_action_t *probe_node_complete = NULL;
 
     for (GList *gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
         pe_node_t *node = (pe_node_t *) gIter->data;
         const char *probed = pe_node_attribute_raw(node, CRM_OP_PROBED);
 
         if (node->details->online == FALSE) {
 
             if (pe__is_remote_node(node) && node->details->remote_rsc
                 && (get_remote_node_state(node) == remote_state_failed)) {
 
                 pe_fence_node(data_set, node, "the connection is unrecoverable", FALSE);
             }
             continue;
 
         } else if (node->details->unclean) {
             continue;
 
         } else if (node->details->rsc_discovery_enabled == FALSE) {
             /* resource discovery is disabled for this node */
             continue;
         }
 
         if (probed != NULL && crm_is_true(probed) == FALSE) {
             pe_action_t *probe_op = custom_action(NULL, crm_strdup_printf("%s-%s", CRM_OP_REPROBE, node->details->uname),
                                                   CRM_OP_REPROBE, node, FALSE, TRUE, data_set);
 
             add_hash_param(probe_op->meta, XML_ATTR_TE_NOWAIT, XML_BOOLEAN_TRUE);
             continue;
         }
 
         for (GList *gIter2 = data_set->resources; gIter2 != NULL; gIter2 = gIter2->next) {
             pe_resource_t *rsc = (pe_resource_t *) gIter2->data;
 
             rsc->cmds->create_probe(rsc, node, probe_node_complete, FALSE, data_set);
         }
     }
     return TRUE;
 }
 
 static void
 rsc_discover_filter(pe_resource_t *rsc, pe_node_t *node)
 {
     GList *gIter = rsc->children;
     pe_resource_t *top = uber_parent(rsc);
     pe_node_t *match;
 
     if (rsc->exclusive_discover == FALSE && top->exclusive_discover == FALSE) {
         return;
     }
 
     for (; gIter != NULL; gIter = gIter->next) {
         pe_resource_t *child_rsc = (pe_resource_t *) gIter->data;
         rsc_discover_filter(child_rsc, node);
     }
 
     match = g_hash_table_lookup(rsc->allowed_nodes, node->details->id);
     if (match && match->rsc_discover_mode != pe_discover_exclusive) {
         match->weight = -INFINITY;
     }
 }
 
 static time_t
 shutdown_time(pe_node_t *node, pe_working_set_t *data_set)
 {
     const char *shutdown = pe_node_attribute_raw(node, XML_CIB_ATTR_SHUTDOWN);
     time_t result = 0;
 
     if (shutdown) {
         long long result_ll;
 
         if (pcmk__scan_ll(shutdown, &result_ll, 0LL) == pcmk_rc_ok) {
             result = (time_t) result_ll;
         }
     }
     return result? result : get_effective_time(data_set);
 }
 
 static void
 apply_shutdown_lock(pe_resource_t *rsc, pe_working_set_t *data_set)
 {
     const char *class;
 
     // Only primitives and (uncloned) groups may be locked
     if (rsc->variant == pe_group) {
         for (GList *item = rsc->children; item != NULL;
              item = item->next) {
             apply_shutdown_lock((pe_resource_t *) item->data, data_set);
         }
     } else if (rsc->variant != pe_native) {
         return;
     }
 
     // Fence devices and remote connections can't be locked
     class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
     if (pcmk__str_eq(class, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_null_matches)
         || pe__resource_is_remote_conn(rsc, data_set)) {
         return;
     }
 
     if (rsc->lock_node != NULL) {
         // The lock was obtained from resource history
 
         if (rsc->running_on != NULL) {
             /* The resource was started elsewhere even though it is now
              * considered locked. This shouldn't be possible, but as a
              * failsafe, we don't want to disturb the resource now.
              */
             pe_rsc_info(rsc,
                         "Cancelling shutdown lock because %s is already active",
                         rsc->id);
             pe__clear_resource_history(rsc, rsc->lock_node, data_set);
             rsc->lock_node = NULL;
             rsc->lock_time = 0;
         }
 
     // Only a resource active on exactly one node can be locked
     } else if (pcmk__list_of_1(rsc->running_on)) {
         pe_node_t *node = rsc->running_on->data;
 
         if (node->details->shutdown) {
             if (node->details->unclean) {
                 pe_rsc_debug(rsc, "Not locking %s to unclean %s for shutdown",
                              rsc->id, node->details->uname);
             } else {
                 rsc->lock_node = node;
                 rsc->lock_time = shutdown_time(node, data_set);
             }
         }
     }
 
     if (rsc->lock_node == NULL) {
         // No lock needed
         return;
     }
 
     if (data_set->shutdown_lock > 0) {
         time_t lock_expiration = rsc->lock_time + data_set->shutdown_lock;
 
         pe_rsc_info(rsc, "Locking %s to %s due to shutdown (expires @%lld)",
                     rsc->id, rsc->lock_node->details->uname,
                     (long long) lock_expiration);
         pe__update_recheck_time(++lock_expiration, data_set);
     } else {
         pe_rsc_info(rsc, "Locking %s to %s due to shutdown",
                     rsc->id, rsc->lock_node->details->uname);
     }
 
     // If resource is locked to one node, ban it from all other nodes
     for (GList *item = data_set->nodes; item != NULL; item = item->next) {
         pe_node_t *node = item->data;
 
         if (strcmp(node->details->uname, rsc->lock_node->details->uname)) {
             resource_location(rsc, node, -CRM_SCORE_INFINITY,
                               XML_CONFIG_ATTR_SHUTDOWN_LOCK, data_set);
         }
     }
 }
 
 /*
  * \internal
  * \brief Stage 2 of cluster status: apply node-specific criteria
  *
  * Count known nodes, and apply location constraints, stickiness, and exclusive
  * resource discovery.
  */
 gboolean
 stage2(pe_working_set_t * data_set)
 {
     GList *gIter = NULL;
 
     if (pcmk_is_set(data_set->flags, pe_flag_shutdown_lock)) {
         for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
             apply_shutdown_lock((pe_resource_t *) gIter->data, data_set);
         }
     }
 
     if (!pcmk_is_set(data_set->flags, pe_flag_no_compat)) {
         // @COMPAT API backward compatibility
         for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
             pe_node_t *node = (pe_node_t *) gIter->data;
 
             if (node && (node->weight >= 0) && node->details->online
                 && (node->details->type != node_ping)) {
                 data_set->max_valid_nodes++;
             }
         }
     }
 
     apply_placement_constraints(data_set);
 
     gIter = data_set->nodes;
     for (; gIter != NULL; gIter = gIter->next) {
         GList *gIter2 = NULL;
         pe_node_t *node = (pe_node_t *) gIter->data;
 
         gIter2 = data_set->resources;
         for (; gIter2 != NULL; gIter2 = gIter2->next) {
             pe_resource_t *rsc = (pe_resource_t *) gIter2->data;
 
             common_apply_stickiness(rsc, node, data_set);
             rsc_discover_filter(rsc, node);
         }
     }
 
     return TRUE;
 }
 
 /*
  * Create internal resource constraints before allocation
  */
 gboolean
 stage3(pe_working_set_t * data_set)
 {
 
     GList *gIter = data_set->resources;
 
     for (; gIter != NULL; gIter = gIter->next) {
         pe_resource_t *rsc = (pe_resource_t *) gIter->data;
 
         rsc->cmds->internal_constraints(rsc, data_set);
     }
 
     return TRUE;
 }
 
 /*
  * Check for orphaned or redefined actions
  */
 gboolean
 stage4(pe_working_set_t * data_set)
 {
     check_actions(data_set);
     return TRUE;
 }
 
 static void *
 convert_const_pointer(const void *ptr)
 {
     /* Worst function ever */
     return (void *)ptr;
 }
 
 static gint
 sort_rsc_process_order(gconstpointer a, gconstpointer b, gpointer data)
 {
     int rc = 0;
     int r1_weight = -INFINITY;
     int r2_weight = -INFINITY;
 
     const char *reason = "existence";
 
     GList *nodes = (GList *) data;
     const pe_resource_t *resource1 = a;
     const pe_resource_t *resource2 = b;
 
     pe_node_t *r1_node = NULL;
     pe_node_t *r2_node = NULL;
     GList *gIter = NULL;
     GHashTable *r1_nodes = NULL;
     GHashTable *r2_nodes = NULL;
 
     reason = "priority";
     r1_weight = resource1->priority;
     r2_weight = resource2->priority;
 
     if (r1_weight > r2_weight) {
         rc = -1;
         goto done;
     }
 
     if (r1_weight < r2_weight) {
         rc = 1;
         goto done;
     }
 
     reason = "no node list";
     if (nodes == NULL) {
         goto done;
     }
 
     r1_nodes = pcmk__native_merge_weights(convert_const_pointer(resource1),
                                           resource1->id, NULL, NULL, 1,
                                           pe_weights_forward | pe_weights_init);
     pe__show_node_weights(true, NULL, resource1->id, r1_nodes,
                           resource1->cluster);
 
     r2_nodes = pcmk__native_merge_weights(convert_const_pointer(resource2),
                                           resource2->id, NULL, NULL, 1,
                                           pe_weights_forward | pe_weights_init);
     pe__show_node_weights(true, NULL, resource2->id, r2_nodes,
                           resource2->cluster);
 
     /* Current location score */
     reason = "current location";
     r1_weight = -INFINITY;
     r2_weight = -INFINITY;
 
     if (resource1->running_on) {
         r1_node = pe__current_node(resource1);
         r1_node = g_hash_table_lookup(r1_nodes, r1_node->details->id);
         if (r1_node != NULL) {
             r1_weight = r1_node->weight;
         }
     }
     if (resource2->running_on) {
         r2_node = pe__current_node(resource2);
         r2_node = g_hash_table_lookup(r2_nodes, r2_node->details->id);
         if (r2_node != NULL) {
             r2_weight = r2_node->weight;
         }
     }
 
     if (r1_weight > r2_weight) {
         rc = -1;
         goto done;
     }
 
     if (r1_weight < r2_weight) {
         rc = 1;
         goto done;
     }
 
     reason = "score";
     for (gIter = nodes; gIter != NULL; gIter = gIter->next) {
         pe_node_t *node = (pe_node_t *) gIter->data;
 
         r1_node = NULL;
         r2_node = NULL;
 
         r1_weight = -INFINITY;
         if (r1_nodes) {
             r1_node = g_hash_table_lookup(r1_nodes, node->details->id);
         }
         if (r1_node) {
             r1_weight = r1_node->weight;
         }
 
         r2_weight = -INFINITY;
         if (r2_nodes) {
             r2_node = g_hash_table_lookup(r2_nodes, node->details->id);
         }
         if (r2_node) {
             r2_weight = r2_node->weight;
         }
 
         if (r1_weight > r2_weight) {
             rc = -1;
             goto done;
         }
 
         if (r1_weight < r2_weight) {
             rc = 1;
             goto done;
         }
     }
 
   done:
     crm_trace("%s (%d) on %s %c %s (%d) on %s: %s",
               resource1->id, r1_weight, r1_node ? r1_node->details->id : "n/a",
               rc < 0 ? '>' : rc > 0 ? '<' : '=',
               resource2->id, r2_weight, r2_node ? r2_node->details->id : "n/a", reason);
 
     if (r1_nodes) {
         g_hash_table_destroy(r1_nodes);
     }
     if (r2_nodes) {
         g_hash_table_destroy(r2_nodes);
     }
 
     return rc;
 }
 
 static void
 allocate_resources(pe_working_set_t * data_set)
 {
     GList *gIter = NULL;
 
     if (pcmk_is_set(data_set->flags, pe_flag_have_remote_nodes)) {
         /* Allocate remote connection resources first (which will also allocate
          * any colocation dependencies). If the connection is migrating, always
          * prefer the partial migration target.
          */
         for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
             pe_resource_t *rsc = (pe_resource_t *) gIter->data;
             if (rsc->is_remote_node == FALSE) {
                 continue;
             }
             pe_rsc_trace(rsc, "Allocating remote connection resource '%s'",
                          rsc->id);
             rsc->cmds->allocate(rsc, rsc->partial_migration_target, data_set);
         }
     }
 
     /* now do the rest of the resources */
     for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
         pe_resource_t *rsc = (pe_resource_t *) gIter->data;
         if (rsc->is_remote_node == TRUE) {
             continue;
         }
         pe_rsc_trace(rsc, "Allocating %s resource '%s'",
                      crm_element_name(rsc->xml), rsc->id);
         rsc->cmds->allocate(rsc, NULL, data_set);
     }
 }
 
 /* We always use pe_order_preserve with these convenience functions to exempt
  * internally generated constraints from the prohibition of user constraints
  * involving remote connection resources.
  *
  * The start ordering additionally uses pe_order_runnable_left so that the
  * specified action is not runnable if the start is not runnable.
  */
 
 static inline void
 order_start_then_action(pe_resource_t *lh_rsc, pe_action_t *rh_action,
                         enum pe_ordering extra, pe_working_set_t *data_set)
 {
     if (lh_rsc && rh_action && data_set) {
         custom_action_order(lh_rsc, start_key(lh_rsc), NULL,
                             rh_action->rsc, NULL, rh_action,
                             pe_order_preserve | pe_order_runnable_left | extra,
                             data_set);
     }
 }
 
 static inline void
 order_action_then_stop(pe_action_t *lh_action, pe_resource_t *rh_rsc,
                        enum pe_ordering extra, pe_working_set_t *data_set)
 {
     if (lh_action && rh_rsc && data_set) {
         custom_action_order(lh_action->rsc, NULL, lh_action,
                             rh_rsc, stop_key(rh_rsc), NULL,
                             pe_order_preserve | extra, data_set);
     }
 }
 
 // Clear fail counts for orphaned rsc on all online nodes
 static void
 cleanup_orphans(pe_resource_t * rsc, pe_working_set_t * data_set)
 {
     GList *gIter = NULL;
 
     for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
         pe_node_t *node = (pe_node_t *) gIter->data;
 
         if (node->details->online
             && pe_get_failcount(node, rsc, NULL, pe_fc_effective, NULL,
                                 data_set)) {
 
             pe_action_t *clear_op = NULL;
 
             clear_op = pe__clear_failcount(rsc, node, "it is orphaned",
                                            data_set);
 
             /* We can't use order_action_then_stop() here because its
              * pe_order_preserve breaks things
              */
             custom_action_order(clear_op->rsc, NULL, clear_op,
                                 rsc, stop_key(rsc), NULL,
                                 pe_order_optional, data_set);
         }
     }
 }
 
 gboolean
 stage5(pe_working_set_t * data_set)
 {
     pcmk__output_t *out = data_set->priv;
     GList *gIter = NULL;
 
     if (!pcmk__str_eq(data_set->placement_strategy, "default", pcmk__str_casei)) {
         GList *nodes = g_list_copy(data_set->nodes);
 
         nodes = sort_nodes_by_weight(nodes, NULL, data_set);
         data_set->resources =
             g_list_sort_with_data(data_set->resources, sort_rsc_process_order, nodes);
 
         g_list_free(nodes);
     }
 
     gIter = data_set->nodes;
     for (; gIter != NULL; gIter = gIter->next) {
         pe_node_t *node = (pe_node_t *) gIter->data;
 
         if (pcmk_is_set(data_set->flags, pe_flag_show_utilization)) {
             out->message(out, "node-capacity", node, "Original");
         }
     }
 
     crm_trace("Allocating services");
     /* Take (next) highest resource, assign it and create its actions */
 
     allocate_resources(data_set);
 
     gIter = data_set->nodes;
     for (; gIter != NULL; gIter = gIter->next) {
         pe_node_t *node = (pe_node_t *) gIter->data;
 
         if (pcmk_is_set(data_set->flags, pe_flag_show_utilization)) {
             out->message(out, "node-capacity", node, "Remaining");
         }
     }
 
     // Process deferred action checks
     pe__foreach_param_check(data_set, check_params);
     pe__free_param_checks(data_set);
 
     if (pcmk_is_set(data_set->flags, pe_flag_startup_probes)) {
         crm_trace("Calculating needed probes");
         /* This code probably needs optimization
          * ptest -x with 100 nodes, 100 clones and clone-max=100:
 
          With probes:
 
          ptest[14781]: 2010/09/27_17:56:46 notice: TRACE: do_calculations: pengine.c:258 Calculate cluster status
          ptest[14781]: 2010/09/27_17:56:46 notice: TRACE: do_calculations: pengine.c:278 Applying placement constraints
          ptest[14781]: 2010/09/27_17:56:47 notice: TRACE: do_calculations: pengine.c:285 Create internal constraints
          ptest[14781]: 2010/09/27_17:56:47 notice: TRACE: do_calculations: pengine.c:292 Check actions
          ptest[14781]: 2010/09/27_17:56:48 notice: TRACE: do_calculations: pengine.c:299 Allocate resources
          ptest[14781]: 2010/09/27_17:56:48 notice: TRACE: stage5: allocate.c:881 Allocating services
          ptest[14781]: 2010/09/27_17:56:49 notice: TRACE: stage5: allocate.c:894 Calculating needed probes
          ptest[14781]: 2010/09/27_17:56:51 notice: TRACE: stage5: allocate.c:899 Creating actions
          ptest[14781]: 2010/09/27_17:56:52 notice: TRACE: stage5: allocate.c:905 Creating done
          ptest[14781]: 2010/09/27_17:56:52 notice: TRACE: do_calculations: pengine.c:306 Processing fencing and shutdown cases
          ptest[14781]: 2010/09/27_17:56:52 notice: TRACE: do_calculations: pengine.c:313 Applying ordering constraints
          36s
          ptest[14781]: 2010/09/27_17:57:28 notice: TRACE: do_calculations: pengine.c:320 Create transition graph
 
          Without probes:
 
          ptest[14637]: 2010/09/27_17:56:21 notice: TRACE: do_calculations: pengine.c:258 Calculate cluster status
          ptest[14637]: 2010/09/27_17:56:22 notice: TRACE: do_calculations: pengine.c:278 Applying placement constraints
          ptest[14637]: 2010/09/27_17:56:22 notice: TRACE: do_calculations: pengine.c:285 Create internal constraints
          ptest[14637]: 2010/09/27_17:56:22 notice: TRACE: do_calculations: pengine.c:292 Check actions
          ptest[14637]: 2010/09/27_17:56:23 notice: TRACE: do_calculations: pengine.c:299 Allocate resources
          ptest[14637]: 2010/09/27_17:56:23 notice: TRACE: stage5: allocate.c:881 Allocating services
          ptest[14637]: 2010/09/27_17:56:24 notice: TRACE: stage5: allocate.c:899 Creating actions
          ptest[14637]: 2010/09/27_17:56:25 notice: TRACE: stage5: allocate.c:905 Creating done
          ptest[14637]: 2010/09/27_17:56:25 notice: TRACE: do_calculations: pengine.c:306 Processing fencing and shutdown cases
          ptest[14637]: 2010/09/27_17:56:25 notice: TRACE: do_calculations: pengine.c:313 Applying ordering constraints
          ptest[14637]: 2010/09/27_17:56:25 notice: TRACE: do_calculations: pengine.c:320 Create transition graph
         */
 
         probe_resources(data_set);
     }
 
     crm_trace("Handle orphans");
     if (pcmk_is_set(data_set->flags, pe_flag_stop_rsc_orphans)) {
         for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
             pe_resource_t *rsc = (pe_resource_t *) gIter->data;
 
             /* There's no need to recurse into rsc->children because those
              * should just be unallocated clone instances.
              */
             if (pcmk_is_set(rsc->flags, pe_rsc_orphan)) {
                 cleanup_orphans(rsc, data_set);
             }
         }
     }
 
     crm_trace("Creating actions");
 
     for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
         pe_resource_t *rsc = (pe_resource_t *) gIter->data;
 
         rsc->cmds->create_actions(rsc, data_set);
     }
 
     crm_trace("Creating done");
     return TRUE;
 }
 
 static gboolean
 is_managed(const pe_resource_t * rsc)
 {
     GList *gIter = rsc->children;
 
     if (pcmk_is_set(rsc->flags, pe_rsc_managed)) {
         return TRUE;
     }
 
     for (; gIter != NULL; gIter = gIter->next) {
         pe_resource_t *child_rsc = (pe_resource_t *) gIter->data;
 
         if (is_managed(child_rsc)) {
             return TRUE;
         }
     }
 
     return FALSE;
 }
 
 static gboolean
 any_managed_resources(pe_working_set_t * data_set)
 {
 
     GList *gIter = data_set->resources;
 
     for (; gIter != NULL; gIter = gIter->next) {
         pe_resource_t *rsc = (pe_resource_t *) gIter->data;
 
         if (is_managed(rsc)) {
             return TRUE;
         }
     }
     return FALSE;
 }
 
 /*!
  * \internal
  * \brief Create pseudo-op for guest node fence, and order relative to it
  *
  * \param[in] node      Guest node to fence
  * \param[in] data_set  Working set of CIB state
  */
 static void
 fence_guest(pe_node_t *node, pe_working_set_t *data_set)
 {
     pe_resource_t *container = node->details->remote_rsc->container;
     pe_action_t *stop = NULL;
     pe_action_t *stonith_op = NULL;
 
     /* The fence action is just a label; we don't do anything differently for
      * off vs. reboot. We specify it explicitly, rather than let it default to
      * cluster's default action, because we are not _initiating_ fencing -- we
      * are creating a pseudo-event to describe fencing that is already occurring
      * by other means (container recovery).
      */
     const char *fence_action = "off";
 
     /* Check whether guest's container resource has any explicit stop or
      * start (the stop may be implied by fencing of the guest's host).
      */
     if (container) {
         stop = find_first_action(container->actions, NULL, CRMD_ACTION_STOP, NULL);
 
         if (find_first_action(container->actions, NULL, CRMD_ACTION_START, NULL)) {
             fence_action = "reboot";
         }
     }
 
     /* Create a fence pseudo-event, so we have an event to order actions
      * against, and the controller can always detect it.
      */
     stonith_op = pe_fence_op(node, fence_action, FALSE, "guest is unclean", FALSE, data_set);
-    update_action_flags(stonith_op, pe_action_pseudo | pe_action_runnable,
-                        __func__, __LINE__);
+    pe__set_action_flags(stonith_op, pe_action_pseudo|pe_action_runnable);
 
     /* We want to imply stops/demotes after the guest is stopped, not wait until
      * it is restarted, so we always order pseudo-fencing after stop, not start
      * (even though start might be closer to what is done for a real reboot).
      */
     if ((stop != NULL) && pcmk_is_set(stop->flags, pe_action_pseudo)) {
         pe_action_t *parent_stonith_op = pe_fence_op(stop->node, NULL, FALSE, NULL, FALSE, data_set);
         crm_info("Implying guest node %s is down (action %d) after %s fencing",
                  node->details->uname, stonith_op->id, stop->node->details->uname);
         order_actions(parent_stonith_op, stonith_op,
                       pe_order_runnable_left|pe_order_implies_then);
 
     } else if (stop) {
         order_actions(stop, stonith_op,
                       pe_order_runnable_left|pe_order_implies_then);
         crm_info("Implying guest node %s is down (action %d) "
                  "after container %s is stopped (action %d)",
                  node->details->uname, stonith_op->id,
                  container->id, stop->id);
     } else {
         /* If we're fencing the guest node but there's no stop for the guest
          * resource, we must think the guest is already stopped. However, we may
          * think so because its resource history was just cleaned. To avoid
          * unnecessarily considering the guest node down if it's really up,
          * order the pseudo-fencing after any stop of the connection resource,
          * which will be ordered after any container (re-)probe.
          */
         stop = find_first_action(node->details->remote_rsc->actions, NULL,
                                  RSC_STOP, NULL);
 
         if (stop) {
             order_actions(stop, stonith_op, pe_order_optional);
             crm_info("Implying guest node %s is down (action %d) "
                      "after connection is stopped (action %d)",
                      node->details->uname, stonith_op->id, stop->id);
         } else {
             /* Not sure why we're fencing, but everything must already be
              * cleanly stopped.
              */
             crm_info("Implying guest node %s is down (action %d) ",
                      node->details->uname, stonith_op->id);
         }
     }
 
     /* Order/imply other actions relative to pseudo-fence as with real fence */
     pcmk__order_vs_fence(stonith_op, data_set);
 }
 
 /*
  * Create dependencies for stonith and shutdown operations
  */
 gboolean
 stage6(pe_working_set_t * data_set)
 {
     pe_action_t *dc_down = NULL;
     pe_action_t *stonith_op = NULL;
     gboolean integrity_lost = FALSE;
     gboolean need_stonith = TRUE;
     GList *gIter;
     GList *stonith_ops = NULL;
     GList *shutdown_ops = NULL;
 
     /* Remote ordering constraints need to happen prior to calculating fencing
      * because it is one more place we will mark the node as dirty.
      *
      * A nice side effect of doing them early is that apply_*_ordering() can be
      * simpler because pe_fence_node() has already done some of the work.
      */
     crm_trace("Creating remote ordering constraints");
     apply_remote_node_ordering(data_set);
 
     crm_trace("Processing fencing and shutdown cases");
     if (any_managed_resources(data_set) == FALSE) {
         crm_notice("Delaying fencing operations until there are resources to manage");
         need_stonith = FALSE;
     }
 
     /* Check each node for stonith/shutdown */
     for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
         pe_node_t *node = (pe_node_t *) gIter->data;
 
         /* Guest nodes are "fenced" by recovering their container resource,
          * so handle them separately.
          */
         if (pe__is_guest_node(node)) {
             if (node->details->remote_requires_reset && need_stonith
                 && pe_can_fence(data_set, node)) {
                 fence_guest(node, data_set);
             }
             continue;
         }
 
         stonith_op = NULL;
 
         if (node->details->unclean
             && need_stonith && pe_can_fence(data_set, node)) {
 
             stonith_op = pe_fence_op(node, NULL, FALSE, "node is unclean", FALSE, data_set);
             pe_warn("Scheduling Node %s for STONITH", node->details->uname);
 
             pcmk__order_vs_fence(stonith_op, data_set);
 
             if (node->details->is_dc) {
                 // Remember if the DC is being fenced
                 dc_down = stonith_op;
 
             } else {
 
                 if (!pcmk_is_set(data_set->flags, pe_flag_concurrent_fencing)
                     && (stonith_ops != NULL)) {
                     /* Concurrent fencing is disabled, so order each non-DC
                      * fencing in a chain. If there is any DC fencing or
                      * shutdown, it will be ordered after the last action in the
                      * chain later.
                      */
                     order_actions((pe_action_t *) stonith_ops->data,
                                   stonith_op, pe_order_optional);
                 }
 
                 // Remember all non-DC fencing actions in a separate list
                 stonith_ops = g_list_prepend(stonith_ops, stonith_op);
             }
 
         } else if (node->details->online && node->details->shutdown &&
                 /* TODO define what a shutdown op means for a remote node.
                  * For now we do not send shutdown operations for remote nodes, but
                  * if we can come up with a good use for this in the future, we will. */
                     pe__is_guest_or_remote_node(node) == FALSE) {
 
             pe_action_t *down_op = sched_shutdown_op(node, data_set);
 
             if (node->details->is_dc) {
                 // Remember if the DC is being shut down
                 dc_down = down_op;
             } else {
                 // Remember non-DC shutdowns for later ordering
                 shutdown_ops = g_list_prepend(shutdown_ops, down_op);
             }
         }
 
         if (node->details->unclean && stonith_op == NULL) {
             integrity_lost = TRUE;
             pe_warn("Node %s is unclean!", node->details->uname);
         }
     }
 
     if (integrity_lost) {
         if (!pcmk_is_set(data_set->flags, pe_flag_stonith_enabled)) {
             pe_warn("YOUR RESOURCES ARE NOW LIKELY COMPROMISED");
             pe_err("ENABLE STONITH TO KEEP YOUR RESOURCES SAFE");
 
         } else if (!pcmk_is_set(data_set->flags, pe_flag_have_quorum)) {
             crm_notice("Cannot fence unclean nodes until quorum is"
                        " attained (or no-quorum-policy is set to ignore)");
         }
     }
 
     if (dc_down != NULL) {
         /* Order any non-DC shutdowns before any DC shutdown, to avoid repeated
          * DC elections. However, we don't want to order non-DC shutdowns before
          * a DC *fencing*, because even though we don't want a node that's
          * shutting down to become DC, the DC fencing could be ordered before a
          * clone stop that's also ordered before the shutdowns, thus leading to
          * a graph loop.
          */
         if (pcmk__str_eq(dc_down->task, CRM_OP_SHUTDOWN, pcmk__str_casei)) {
             for (gIter = shutdown_ops; gIter != NULL; gIter = gIter->next) {
                 pe_action_t *node_stop = (pe_action_t *) gIter->data;
 
                 crm_debug("Ordering shutdown on %s before %s on DC %s",
                           node_stop->node->details->uname,
                           dc_down->task, dc_down->node->details->uname);
 
                 order_actions(node_stop, dc_down, pe_order_optional);
             }
         }
 
         // Order any non-DC fencing before any DC fencing or shutdown
 
         if (pcmk_is_set(data_set->flags, pe_flag_concurrent_fencing)) {
             /* With concurrent fencing, order each non-DC fencing action
              * separately before any DC fencing or shutdown.
              */
             for (gIter = stonith_ops; gIter != NULL; gIter = gIter->next) {
                 order_actions((pe_action_t *) gIter->data, dc_down,
                               pe_order_optional);
             }
         } else if (stonith_ops) {
             /* Without concurrent fencing, the non-DC fencing actions are
              * already ordered relative to each other, so we just need to order
              * the DC fencing after the last action in the chain (which is the
              * first item in the list).
              */
             order_actions((pe_action_t *) stonith_ops->data, dc_down,
                           pe_order_optional);
         }
     }
     g_list_free(stonith_ops);
     g_list_free(shutdown_ops);
     return TRUE;
 }
 
 /*
  * Determine the sets of independent actions and the correct order for the
  *  actions in each set.
  *
  * Mark dependencies of un-runnable actions un-runnable
  *
  */
 static GList *
 find_actions_by_task(GList *actions, pe_resource_t * rsc, const char *original_key)
 {
     GList *list = NULL;
 
     list = find_actions(actions, original_key, NULL);
     if (list == NULL) {
         /* we're potentially searching a child of the original resource */
         char *key = NULL;
         char *task = NULL;
         guint interval_ms = 0;
 
         if (parse_op_key(original_key, NULL, &task, &interval_ms)) {
             key = pcmk__op_key(rsc->id, task, interval_ms);
             list = find_actions(actions, key, NULL);
 
         } else {
             crm_err("search key: %s", original_key);
         }
 
         free(key);
         free(task);
     }
 
     return list;
 }
 
 static void
 rsc_order_then(pe_action_t *lh_action, pe_resource_t *rsc,
                pe__ordering_t *order)
 {
     GList *gIter = NULL;
     GList *rh_actions = NULL;
     pe_action_t *rh_action = NULL;
     enum pe_ordering type;
 
     CRM_CHECK(rsc != NULL, return);
     CRM_CHECK(order != NULL, return);
 
     type = order->type;
     rh_action = order->rh_action;
     crm_trace("Processing RH of ordering constraint %d", order->id);
 
     if (rh_action != NULL) {
         rh_actions = g_list_prepend(NULL, rh_action);
 
     } else if (rsc != NULL) {
         rh_actions = find_actions_by_task(rsc->actions, rsc, order->rh_action_task);
     }
 
     if (rh_actions == NULL) {
         pe_rsc_trace(rsc, "No RH-Side (%s/%s) found for constraint..."
                      " ignoring", rsc->id, order->rh_action_task);
         if (lh_action) {
             pe_rsc_trace(rsc, "LH-Side was: %s", lh_action->uuid);
         }
         return;
     }
 
     if ((lh_action != NULL) && (lh_action->rsc == rsc)
         && pcmk_is_set(lh_action->flags, pe_action_dangle)) {
 
         pe_rsc_trace(rsc, "Detected dangling operation %s -> %s", lh_action->uuid,
                      order->rh_action_task);
         pe__clear_order_flags(type, pe_order_implies_then);
     }
 
     gIter = rh_actions;
     for (; gIter != NULL; gIter = gIter->next) {
         pe_action_t *rh_action_iter = (pe_action_t *) gIter->data;
 
         if (lh_action) {
             order_actions(lh_action, rh_action_iter, type);
 
         } else if (type & pe_order_implies_then) {
-            update_action_flags(rh_action_iter, pe_action_runnable | pe_action_clear,
-                                __func__, __LINE__);
+            pe__clear_action_flags(rh_action_iter, pe_action_runnable);
             crm_warn("Unrunnable %s 0x%.6x", rh_action_iter->uuid, type);
         } else {
             crm_warn("neither %s 0x%.6x", rh_action_iter->uuid, type);
         }
     }
 
     g_list_free(rh_actions);
 }
 
 static void
 rsc_order_first(pe_resource_t *lh_rsc, pe__ordering_t *order,
                 pe_working_set_t *data_set)
 {
     GList *gIter = NULL;
     GList *lh_actions = NULL;
     pe_action_t *lh_action = order->lh_action;
     pe_resource_t *rh_rsc = order->rh_rsc;
 
     crm_trace("Processing LH of ordering constraint %d", order->id);
     CRM_ASSERT(lh_rsc != NULL);
 
     if (lh_action != NULL) {
         lh_actions = g_list_prepend(NULL, lh_action);
 
     } else {
         lh_actions = find_actions_by_task(lh_rsc->actions, lh_rsc, order->lh_action_task);
     }
 
     if (lh_actions == NULL && lh_rsc != rh_rsc) {
         char *key = NULL;
         char *op_type = NULL;
         guint interval_ms = 0;
 
         parse_op_key(order->lh_action_task, NULL, &op_type, &interval_ms);
         key = pcmk__op_key(lh_rsc->id, op_type, interval_ms);
 
         if (lh_rsc->fns->state(lh_rsc, TRUE) == RSC_ROLE_STOPPED && pcmk__str_eq(op_type, RSC_STOP, pcmk__str_casei)) {
             free(key);
             pe_rsc_trace(lh_rsc, "No LH-Side (%s/%s) found for constraint %d with %s - ignoring",
                          lh_rsc->id, order->lh_action_task, order->id, order->rh_action_task);
 
         } else if ((lh_rsc->fns->state(lh_rsc, TRUE) == RSC_ROLE_UNPROMOTED)
                    && pcmk__str_eq(op_type, RSC_DEMOTE, pcmk__str_casei)) {
             free(key);
             pe_rsc_trace(lh_rsc, "No LH-Side (%s/%s) found for constraint %d with %s - ignoring",
                          lh_rsc->id, order->lh_action_task, order->id, order->rh_action_task);
 
         } else {
             pe_rsc_trace(lh_rsc, "No LH-Side (%s/%s) found for constraint %d with %s - creating",
                          lh_rsc->id, order->lh_action_task, order->id, order->rh_action_task);
             lh_action = custom_action(lh_rsc, key, op_type, NULL, TRUE, TRUE, data_set);
             lh_actions = g_list_prepend(NULL, lh_action);
         }
 
         free(op_type);
     }
 
     gIter = lh_actions;
     for (; gIter != NULL; gIter = gIter->next) {
         pe_action_t *lh_action_iter = (pe_action_t *) gIter->data;
 
         if (rh_rsc == NULL && order->rh_action) {
             rh_rsc = order->rh_action->rsc;
         }
         if (rh_rsc) {
             rsc_order_then(lh_action_iter, rh_rsc, order);
 
         } else if (order->rh_action) {
             order_actions(lh_action_iter, order->rh_action, order->type);
         }
     }
 
     g_list_free(lh_actions);
 }
 
 extern void update_colo_start_chain(pe_action_t *action,
                                     pe_working_set_t *data_set);
 
 static int
 is_recurring_action(pe_action_t *action)
 {
     guint interval_ms;
 
     if (pcmk__guint_from_hash(action->meta,
                               XML_LRM_ATTR_INTERVAL_MS, 0,
                               &interval_ms) != pcmk_rc_ok) {
         return 0;
     }
     return (interval_ms > 0);
 }
 
 static void
 apply_container_ordering(pe_action_t *action, pe_working_set_t *data_set)
 {
     /* VMs are also classified as containers for these purposes... in
      * that they both involve a 'thing' running on a real or remote
      * cluster node.
      *
      * This allows us to be smarter about the type and extent of
      * recovery actions required in various scenarios
      */
     pe_resource_t *remote_rsc = NULL;
     pe_resource_t *container = NULL;
     enum action_tasks task = text2task(action->task);
 
     CRM_ASSERT(action->rsc);
     CRM_ASSERT(action->node);
     CRM_ASSERT(pe__is_guest_or_remote_node(action->node));
 
     remote_rsc = action->node->details->remote_rsc;
     CRM_ASSERT(remote_rsc);
 
     container = remote_rsc->container;
     CRM_ASSERT(container);
 
     if (pcmk_is_set(container->flags, pe_rsc_failed)) {
         pe_fence_node(data_set, action->node, "container failed", FALSE);
     }
 
     crm_trace("Order %s action %s relative to %s%s for %s%s",
               action->task, action->uuid,
               pcmk_is_set(remote_rsc->flags, pe_rsc_failed)? "failed " : "",
               remote_rsc->id,
               pcmk_is_set(container->flags, pe_rsc_failed)? "failed " : "",
               container->id);
 
     if (pcmk__strcase_any_of(action->task, CRMD_ACTION_MIGRATE, CRMD_ACTION_MIGRATED, NULL)) {
         /* Migration ops map to "no_action", but we need to apply the same
          * ordering as for stop or demote (see get_router_node()).
          */
         task = stop_rsc;
     }
 
     switch (task) {
         case start_rsc:
         case action_promote:
             /* Force resource recovery if the container is recovered */
             order_start_then_action(container, action, pe_order_implies_then,
                                     data_set);
 
             /* Wait for the connection resource to be up too */
             order_start_then_action(remote_rsc, action, pe_order_none,
                                     data_set);
             break;
 
         case stop_rsc:
         case action_demote:
             if (pcmk_is_set(container->flags, pe_rsc_failed)) {
                 /* When the container representing a guest node fails, any stop
                  * or demote actions for resources running on the guest node
                  * are implied by the container stopping. This is similar to
                  * how fencing operations work for cluster nodes and remote
                  * nodes.
                  */
             } else {
                 /* Ensure the operation happens before the connection is brought
                  * down.
                  *
                  * If we really wanted to, we could order these after the
                  * connection start, IFF the container's current role was
                  * stopped (otherwise we re-introduce an ordering loop when the
                  * connection is restarting).
                  */
                 order_action_then_stop(action, remote_rsc, pe_order_none,
                                        data_set);
             }
             break;
 
         default:
             /* Wait for the connection resource to be up */
             if (is_recurring_action(action)) {
                 /* In case we ever get the recovery logic wrong, force
                  * recurring monitors to be restarted, even if just
                  * the connection was re-established
                  */
                 if(task != no_action) {
                     order_start_then_action(remote_rsc, action,
                                             pe_order_implies_then, data_set);
                 }
             } else {
                 order_start_then_action(remote_rsc, action, pe_order_none,
                                         data_set);
             }
             break;
     }
 }
 
 static enum remote_connection_state
 get_remote_node_state(pe_node_t *node) 
 {
     pe_resource_t *remote_rsc = NULL;
     pe_node_t *cluster_node = NULL;
 
     CRM_ASSERT(node);
 
     remote_rsc = node->details->remote_rsc;
     CRM_ASSERT(remote_rsc);
 
     cluster_node = pe__current_node(remote_rsc);
 
     /* If the cluster node the remote connection resource resides on
      * is unclean or went offline, we can't process any operations
      * on that remote node until after it starts elsewhere.
      */
     if(remote_rsc->next_role == RSC_ROLE_STOPPED || remote_rsc->allocated_to == NULL) {
         /* The connection resource is not going to run anywhere */
 
         if (cluster_node && cluster_node->details->unclean) {
             /* The remote connection is failed because its resource is on a
              * failed node and can't be recovered elsewhere, so we must fence.
              */
             return remote_state_failed;
         }
 
         if (!pcmk_is_set(remote_rsc->flags, pe_rsc_failed)) {
             /* Connection resource is cleanly stopped */
             return remote_state_stopped;
         }
 
         /* Connection resource is failed */
 
         if ((remote_rsc->next_role == RSC_ROLE_STOPPED)
             && remote_rsc->remote_reconnect_ms
             && node->details->remote_was_fenced
             && !pe__shutdown_requested(node)) {
 
             /* We won't know whether the connection is recoverable until the
              * reconnect interval expires and we reattempt connection.
              */
             return remote_state_unknown;
         }
 
         /* The remote connection is in a failed state. If there are any
          * resources known to be active on it (stop) or in an unknown state
          * (probe), we must assume the worst and fence it.
          */
         return remote_state_failed;
 
     } else if (cluster_node == NULL) {
         /* Connection is recoverable but not currently running anywhere, see if we can recover it first */
         return remote_state_unknown;
 
     } else if(cluster_node->details->unclean == TRUE
               || cluster_node->details->online == FALSE) {
         /* Connection is running on a dead node, see if we can recover it first */
         return remote_state_resting;
 
     } else if (pcmk__list_of_multiple(remote_rsc->running_on)
                && remote_rsc->partial_migration_source
                && remote_rsc->partial_migration_target) {
         /* We're in the middle of migrating a connection resource,
          * wait until after the resource migrates before performing
          * any actions.
          */
         return remote_state_resting;
 
     }
     return remote_state_alive;
 }
 
 /*!
  * \internal
  * \brief Order actions on remote node relative to actions for the connection
  */
 static void
 apply_remote_ordering(pe_action_t *action, pe_working_set_t *data_set)
 {
     pe_resource_t *remote_rsc = NULL;
     enum action_tasks task = text2task(action->task);
     enum remote_connection_state state = get_remote_node_state(action->node);
 
     enum pe_ordering order_opts = pe_order_none;
 
     if (action->rsc == NULL) {
         return;
     }
 
     CRM_ASSERT(action->node);
     CRM_ASSERT(pe__is_guest_or_remote_node(action->node));
 
     remote_rsc = action->node->details->remote_rsc;
     CRM_ASSERT(remote_rsc);
 
     crm_trace("Order %s action %s relative to %s%s (state: %s)",
               action->task, action->uuid,
               pcmk_is_set(remote_rsc->flags, pe_rsc_failed)? "failed " : "",
               remote_rsc->id, state2text(state));
 
     if (pcmk__strcase_any_of(action->task, CRMD_ACTION_MIGRATE, CRMD_ACTION_MIGRATED, NULL)) {
         /* Migration ops map to "no_action", but we need to apply the same
          * ordering as for stop or demote (see get_router_node()).
          */
         task = stop_rsc;
     }
 
     switch (task) {
         case start_rsc:
         case action_promote:
             order_opts = pe_order_none;
 
             if (state == remote_state_failed) {
                 /* Force recovery, by making this action required */
                 pe__set_order_flags(order_opts, pe_order_implies_then);
             }
 
             /* Ensure connection is up before running this action */
             order_start_then_action(remote_rsc, action, order_opts, data_set);
             break;
 
         case stop_rsc:
             if(state == remote_state_alive) {
                 order_action_then_stop(action, remote_rsc,
                                        pe_order_implies_first, data_set);
 
             } else if(state == remote_state_failed) {
                 /* The resource is active on the node, but since we don't have a
                  * valid connection, the only way to stop the resource is by
                  * fencing the node. There is no need to order the stop relative
                  * to the remote connection, since the stop will become implied
                  * by the fencing.
                  */
                 pe_fence_node(data_set, action->node, "resources are active and the connection is unrecoverable", FALSE);
 
             } else if(remote_rsc->next_role == RSC_ROLE_STOPPED) {
                 /* State must be remote_state_unknown or remote_state_stopped.
                  * Since the connection is not coming back up in this
                  * transition, stop this resource first.
                  */
                 order_action_then_stop(action, remote_rsc,
                                        pe_order_implies_first, data_set);
 
             } else {
                 /* The connection is going to be started somewhere else, so
                  * stop this resource after that completes.
                  */
                 order_start_then_action(remote_rsc, action, pe_order_none, data_set);
             }
             break;
 
         case action_demote:
             /* Only order this demote relative to the connection start if the
              * connection isn't being torn down. Otherwise, the demote would be
              * blocked because the connection start would not be allowed.
              */
             if(state == remote_state_resting || state == remote_state_unknown) {
                 order_start_then_action(remote_rsc, action, pe_order_none,
                                         data_set);
             } /* Otherwise we can rely on the stop ordering */
             break;
 
         default:
             /* Wait for the connection resource to be up */
             if (is_recurring_action(action)) {
                 /* In case we ever get the recovery logic wrong, force
                  * recurring monitors to be restarted, even if just
                  * the connection was re-established
                  */
                 order_start_then_action(remote_rsc, action,
                                         pe_order_implies_then, data_set);
 
             } else {
                 pe_node_t *cluster_node = pe__current_node(remote_rsc);
 
                 if(task == monitor_rsc && state == remote_state_failed) {
                     /* We would only be here if we do not know the
                      * state of the resource on the remote node.
                      * Since we have no way to find out, it is
                      * necessary to fence the node.
                      */
                     pe_fence_node(data_set, action->node, "resources are in an unknown state and the connection is unrecoverable", FALSE);
                 }
 
                 if(cluster_node && state == remote_state_stopped) {
                     /* The connection is currently up, but is going
                      * down permanently.
                      *
                      * Make sure we check services are actually
                      * stopped _before_ we let the connection get
                      * closed
                      */
                     order_action_then_stop(action, remote_rsc,
                                            pe_order_runnable_left, data_set);
 
                 } else {
                     order_start_then_action(remote_rsc, action, pe_order_none,
                                             data_set);
                 }
             }
             break;
     }
 }
 
 static void
 apply_remote_node_ordering(pe_working_set_t *data_set)
 {
     if (!pcmk_is_set(data_set->flags, pe_flag_have_remote_nodes)) {
         return;
     }
 
     for (GList *gIter = data_set->actions; gIter != NULL; gIter = gIter->next) {
         pe_action_t *action = (pe_action_t *) gIter->data;
         pe_resource_t *remote = NULL;
 
         // We are only interested in resource actions
         if (action->rsc == NULL) {
             continue;
         }
 
         /* Special case: If we are clearing the failcount of an actual
          * remote connection resource, then make sure this happens before
          * any start of the resource in this transition.
          */
         if (action->rsc->is_remote_node &&
             pcmk__str_eq(action->task, CRM_OP_CLEAR_FAILCOUNT, pcmk__str_casei)) {
 
             custom_action_order(action->rsc,
                 NULL,
                 action,
                 action->rsc,
                 pcmk__op_key(action->rsc->id, RSC_START, 0),
                 NULL,
                 pe_order_optional,
                 data_set);
 
             continue;
         }
 
         // We are only interested in actions allocated to a node
         if (action->node == NULL) {
             continue;
         }
 
         if (!pe__is_guest_or_remote_node(action->node)) {
             continue;
         }
 
         /* We are only interested in real actions.
          *
          * @TODO This is probably wrong; pseudo-actions might be converted to
          * real actions and vice versa later in update_actions() at the end of
          * stage7().
          */
         if (pcmk_is_set(action->flags, pe_action_pseudo)) {
             continue;
         }
 
         remote = action->node->details->remote_rsc;
         if (remote == NULL) {
             // Orphaned
             continue;
         }
 
         /* Another special case: if a resource is moving to a Pacemaker Remote
          * node, order the stop on the original node after any start of the
          * remote connection. This ensures that if the connection fails to
          * start, we leave the resource running on the original node.
          */
         if (pcmk__str_eq(action->task, RSC_START, pcmk__str_casei)) {
             for (GList *item = action->rsc->actions; item != NULL;
                  item = item->next) {
                 pe_action_t *rsc_action = item->data;
 
                 if ((rsc_action->node->details != action->node->details)
                     && pcmk__str_eq(rsc_action->task, RSC_STOP, pcmk__str_casei)) {
                     custom_action_order(remote, start_key(remote), NULL,
                                         action->rsc, NULL, rsc_action,
                                         pe_order_optional, data_set);
                 }
             }
         }
 
         /* The action occurs across a remote connection, so create
          * ordering constraints that guarantee the action occurs while the node
          * is active (after start, before stop ... things like that).
          *
          * This is somewhat brittle in that we need to make sure the results of
          * this ordering are compatible with the result of get_router_node().
          * It would probably be better to add XML_LRM_ATTR_ROUTER_NODE as part
          * of this logic rather than action2xml().
          */
         if (remote->container) {
             crm_trace("Container ordering for %s", action->uuid);
             apply_container_ordering(action, data_set);
 
         } else {
             crm_trace("Remote ordering for %s", action->uuid);
             apply_remote_ordering(action, data_set);
         }
     }
 }
 
 static gboolean
 order_first_probe_unneeded(pe_action_t * probe, pe_action_t * rh_action)
 {
     /* No need to probe the resource on the node that is being
      * unfenced. Otherwise it might introduce transition loop
      * since probe will be performed after the node is
      * unfenced.
      */
     if (pcmk__str_eq(rh_action->task, CRM_OP_FENCE, pcmk__str_casei)
          && probe->node && rh_action->node
          && probe->node->details == rh_action->node->details) {
         const char *op = g_hash_table_lookup(rh_action->meta, "stonith_action");
 
         if (pcmk__str_eq(op, "on", pcmk__str_casei)) {
             return TRUE;
         }
     }
 
     // Shutdown waits for probe to complete only if it's on the same node
     if ((pcmk__str_eq(rh_action->task, CRM_OP_SHUTDOWN, pcmk__str_casei))
         && probe->node && rh_action->node
         && probe->node->details != rh_action->node->details) {
         return TRUE;
     }
     return FALSE;
 }
 
 static void
 order_first_probes_imply_stops(pe_working_set_t * data_set)
 {
     GList *gIter = NULL;
 
     for (gIter = data_set->ordering_constraints; gIter != NULL; gIter = gIter->next) {
         pe__ordering_t *order = gIter->data;
         enum pe_ordering order_type = pe_order_optional;
 
         pe_resource_t *lh_rsc = order->lh_rsc;
         pe_resource_t *rh_rsc = order->rh_rsc;
         pe_action_t *lh_action = order->lh_action;
         pe_action_t *rh_action = order->rh_action;
         const char *lh_action_task = order->lh_action_task;
         const char *rh_action_task = order->rh_action_task;
 
         GList *probes = NULL;
         GList *rh_actions = NULL;
 
         GList *pIter = NULL;
 
         if (lh_rsc == NULL) {
             continue;
 
         } else if (rh_rsc && lh_rsc == rh_rsc) {
             continue;
         }
 
         if (lh_action == NULL && lh_action_task == NULL) {
             continue;
         }
 
         if (rh_action == NULL && rh_action_task == NULL) {
             continue;
         }
 
         /* Technically probe is expected to return "not running", which could be
          * the alternative of stop action if the status of the resource is
          * unknown yet.
          */
         if (lh_action && !pcmk__str_eq(lh_action->task, RSC_STOP, pcmk__str_casei)) {
             continue;
 
         } else if (lh_action == NULL
                    && lh_action_task
                    && !pcmk__ends_with(lh_action_task, "_" RSC_STOP "_0")) {
             continue;
         }
 
         /* Do not probe the resource inside of a stopping container. Otherwise
          * it might introduce transition loop since probe will be performed
          * after the container starts again.
          */
         if (rh_rsc && lh_rsc->container == rh_rsc) {
             if (rh_action && pcmk__str_eq(rh_action->task, RSC_STOP, pcmk__str_casei)) {
                 continue;
 
             } else if (rh_action == NULL && rh_action_task
                        && pcmk__ends_with(rh_action_task,"_" RSC_STOP "_0")) {
                 continue;
             }
         }
 
         if (order->type == pe_order_none) {
             continue;
         }
 
         // Preserve the order options for future filtering
         if (pcmk_is_set(order->type, pe_order_apply_first_non_migratable)) {
             pe__set_order_flags(order_type,
                                 pe_order_apply_first_non_migratable);
         }
 
         if (pcmk_is_set(order->type, pe_order_same_node)) {
             pe__set_order_flags(order_type, pe_order_same_node);
         }
 
         // Keep the order types for future filtering
         if (order->type == pe_order_anti_colocation
                    || order->type == pe_order_load) {
             order_type = order->type;
         }
 
         probes = pe__resource_actions(lh_rsc, NULL, RSC_STATUS, FALSE);
         if (probes == NULL) {
             continue;
         }
 
         if (rh_action) {
             rh_actions = g_list_prepend(rh_actions, rh_action);
 
         } else if (rh_rsc && rh_action_task) {
             rh_actions = find_actions(rh_rsc->actions, rh_action_task, NULL);
         }
 
         if (rh_actions == NULL) {
             g_list_free(probes);
             continue;
         }
 
         crm_trace("Processing for LH probe based on ordering constraint %s -> %s"
                   " (id=%d, type=%.6x)",
                   lh_action ? lh_action->uuid : lh_action_task,
                   rh_action ? rh_action->uuid : rh_action_task,
                   order->id, order->type);
 
         for (pIter = probes; pIter != NULL; pIter = pIter->next) {
             pe_action_t *probe = (pe_action_t *) pIter->data;
             GList *rIter = NULL;
 
             for (rIter = rh_actions; rIter != NULL; rIter = rIter->next) {
                 pe_action_t *rh_action_iter = (pe_action_t *) rIter->data;
 
                 if (order_first_probe_unneeded(probe, rh_action_iter)) {
                     continue;
                 }
                 order_actions(probe, rh_action_iter, order_type);
             }
         }
 
         g_list_free(rh_actions);
         g_list_free(probes);
     }
 }
 
 static void
 order_first_probe_then_restart_repromote(pe_action_t * probe,
                                          pe_action_t * after,
                                          pe_working_set_t * data_set)
 {
     GList *gIter = NULL;
     bool interleave = FALSE;
     pe_resource_t *compatible_rsc = NULL;
 
     if (probe == NULL
         || probe->rsc == NULL
         || probe->rsc->variant != pe_native) {
         return;
     }
 
     if (after == NULL
         // Avoid running into any possible loop
         || pcmk_is_set(after->flags, pe_action_tracking)) {
         return;
     }
 
     if (!pcmk__str_eq(probe->task, RSC_STATUS, pcmk__str_casei)) {
         return;
     }
 
     pe__set_action_flags(after, pe_action_tracking);
 
     crm_trace("Processing based on %s %s -> %s %s",
               probe->uuid,
               probe->node ? probe->node->details->uname: "",
               after->uuid,
               after->node ? after->node->details->uname : "");
 
     if (after->rsc
         /* Better not build a dependency directly with a clone/group.
          * We are going to proceed through the ordering chain and build
          * dependencies with its children.
          */
         && after->rsc->variant == pe_native
         && probe->rsc != after->rsc) {
 
             GList *then_actions = NULL;
             enum pe_ordering probe_order_type = pe_order_optional;
 
             if (pcmk__str_eq(after->task, RSC_START, pcmk__str_casei)) {
                 then_actions = pe__resource_actions(after->rsc, NULL, RSC_STOP, FALSE);
 
             } else if (pcmk__str_eq(after->task, RSC_PROMOTE, pcmk__str_casei)) {
                 then_actions = pe__resource_actions(after->rsc, NULL, RSC_DEMOTE, FALSE);
             }
 
             for (gIter = then_actions; gIter != NULL; gIter = gIter->next) {
                 pe_action_t *then = (pe_action_t *) gIter->data;
 
                 // Skip any pseudo action which for example is implied by fencing
                 if (pcmk_is_set(then->flags, pe_action_pseudo)) {
                     continue;
                 }
 
                 order_actions(probe, then, probe_order_type);
             }
             g_list_free(then_actions);
     }
 
     if (after->rsc
         && after->rsc->variant > pe_group) {
         const char *interleave_s = g_hash_table_lookup(after->rsc->meta,
                                                        XML_RSC_ATTR_INTERLEAVE);
 
         interleave = crm_is_true(interleave_s);
 
         if (interleave) {
             /* For an interleaved clone, we should build a dependency only
              * with the relevant clone child.
              */
             compatible_rsc = find_compatible_child(probe->rsc,
                                                    after->rsc,
                                                    RSC_ROLE_UNKNOWN,
                                                    FALSE, data_set);
         }
     }
 
     for (gIter = after->actions_after; gIter != NULL; gIter = gIter->next) {
         pe_action_wrapper_t *after_wrapper = (pe_action_wrapper_t *) gIter->data;
         /* pe_order_implies_then is the reason why a required A.start
          * implies/enforces B.start to be required too, which is the cause of
          * B.restart/re-promote.
          *
          * Not sure about pe_order_implies_then_on_node though. It's now only
          * used for unfencing case, which tends to introduce transition
          * loops...
          */
 
         if (!pcmk_is_set(after_wrapper->type, pe_order_implies_then)) {
             /* The order type between a group/clone and its child such as
              * B.start-> B_child.start is:
              * pe_order_implies_first_printed | pe_order_runnable_left
              *
              * Proceed through the ordering chain and build dependencies with
              * its children.
              */
             if (after->rsc == NULL
                 || after->rsc->variant < pe_group
                 || probe->rsc->parent == after->rsc
                 || after_wrapper->action->rsc == NULL
                 || after_wrapper->action->rsc->variant > pe_group
                 || after->rsc != after_wrapper->action->rsc->parent) {
                 continue;
             }
 
             /* Proceed to the children of a group or a non-interleaved clone.
              * For an interleaved clone, proceed only to the relevant child.
              */
             if (after->rsc->variant > pe_group
                 && interleave == TRUE
                 && (compatible_rsc == NULL
                     || compatible_rsc != after_wrapper->action->rsc)) {
                 continue;
             }
         }
 
         crm_trace("Proceeding through %s %s -> %s %s (type=0x%.6x)",
                   after->uuid,
                   after->node ? after->node->details->uname: "",
                   after_wrapper->action->uuid,
                   after_wrapper->action->node ? after_wrapper->action->node->details->uname : "",
                   after_wrapper->type);
 
         order_first_probe_then_restart_repromote(probe, after_wrapper->action, data_set);
     }
 }
 
 static void clear_actions_tracking_flag(pe_working_set_t * data_set)
 {
     GList *gIter = NULL;
 
     for (gIter = data_set->actions; gIter != NULL; gIter = gIter->next) {
         pe_action_t *action = (pe_action_t *) gIter->data;
 
         if (pcmk_is_set(action->flags, pe_action_tracking)) {
             pe__clear_action_flags(action, pe_action_tracking);
         }
     }
 }
 
 static void
 order_first_rsc_probes(pe_resource_t * rsc, pe_working_set_t * data_set)
 {
     GList *gIter = NULL;
     GList *probes = NULL;
 
     for (gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
         pe_resource_t * child = (pe_resource_t *) gIter->data;
 
         order_first_rsc_probes(child, data_set);
     }
 
     if (rsc->variant != pe_native) {
         return;
     }
 
     probes = pe__resource_actions(rsc, NULL, RSC_STATUS, FALSE);
 
     for (gIter = probes; gIter != NULL; gIter= gIter->next) {
         pe_action_t *probe = (pe_action_t *) gIter->data;
         GList *aIter = NULL;
 
         for (aIter = probe->actions_after; aIter != NULL; aIter = aIter->next) {
             pe_action_wrapper_t *after_wrapper = (pe_action_wrapper_t *) aIter->data;
 
             order_first_probe_then_restart_repromote(probe, after_wrapper->action, data_set);
             clear_actions_tracking_flag(data_set);
         }
     }
 
     g_list_free(probes);
 }
 
 static void
 order_first_probes(pe_working_set_t * data_set)
 {
     GList *gIter = NULL;
 
     for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
         pe_resource_t *rsc = (pe_resource_t *) gIter->data;
 
         order_first_rsc_probes(rsc, data_set);
     }
 
     order_first_probes_imply_stops(data_set);
 }
 
 static void
 order_then_probes(pe_working_set_t * data_set)
 {
 #if 0
     GList *gIter = NULL;
 
     for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
         pe_resource_t *rsc = (pe_resource_t *) gIter->data;
 
         /* Given "A then B", we would prefer to wait for A to be
          * started before probing B.
          *
          * If A was a filesystem on which the binaries and data for B
          * lived, it would have been useful if the author of B's agent
          * could assume that A is running before B.monitor will be
          * called.
          *
          * However we can't _only_ probe once A is running, otherwise
          * we'd not detect the state of B if A could not be started
          * for some reason.
          *
          * In practice however, we cannot even do an opportunistic
          * version of this because B may be moving:
          *
          *   B.probe -> B.start
          *   B.probe -> B.stop
          *   B.stop -> B.start
          *   A.stop -> A.start
          *   A.start -> B.probe
          *
          * So far so good, but if we add the result of this code:
          *
          *   B.stop -> A.stop
          *
          * Then we get a loop:
          *
          *   B.probe -> B.stop -> A.stop -> A.start -> B.probe
          *
          * We could kill the 'B.probe -> B.stop' dependency, but that
          * could mean stopping B "too" soon, because B.start must wait
          * for the probes to complete.
          *
          * Another option is to allow it only if A is a non-unique
          * clone with clone-max == node-max (since we'll never be
          * moving it).  However, we could still be stopping one
          * instance at the same time as starting another.
 
          * The complexity of checking for allowed conditions combined
          * with the ever narrowing usecase suggests that this code
          * should remain disabled until someone gets smarter.
          */
         pe_action_t *start = NULL;
         GList *actions = NULL;
         GList *probes = NULL;
 
         actions = pe__resource_actions(rsc, NULL, RSC_START, FALSE);
 
         if (actions) {
             start = actions->data;
             g_list_free(actions);
         }
 
         if(start == NULL) {
             crm_err("No start action for %s", rsc->id);
             continue;
         }
 
         probes = pe__resource_actions(rsc, NULL, RSC_STATUS, FALSE);
 
         for (actions = start->actions_before; actions != NULL; actions = actions->next) {
             pe_action_wrapper_t *before = (pe_action_wrapper_t *) actions->data;
 
             GList *pIter = NULL;
             pe_action_t *first = before->action;
             pe_resource_t *first_rsc = first->rsc;
 
             if(first->required_runnable_before) {
                 GList *clone_actions = NULL;
                 for (clone_actions = first->actions_before; clone_actions != NULL; clone_actions = clone_actions->next) {
                     before = (pe_action_wrapper_t *) clone_actions->data;
 
                     crm_trace("Testing %s -> %s (%p) for %s", first->uuid, before->action->uuid, before->action->rsc, start->uuid);
 
                     CRM_ASSERT(before->action->rsc);
                     first_rsc = before->action->rsc;
                     break;
                 }
 
             } else if(!pcmk__str_eq(first->task, RSC_START, pcmk__str_casei)) {
                 crm_trace("Not a start op %s for %s", first->uuid, start->uuid);
             }
 
             if(first_rsc == NULL) {
                 continue;
 
             } else if(uber_parent(first_rsc) == uber_parent(start->rsc)) {
                 crm_trace("Same parent %s for %s", first_rsc->id, start->uuid);
                 continue;
 
             } else if(FALSE && pe_rsc_is_clone(uber_parent(first_rsc)) == FALSE) {
                 crm_trace("Not a clone %s for %s", first_rsc->id, start->uuid);
                 continue;
             }
 
             crm_err("Applying %s before %s %d", first->uuid, start->uuid, uber_parent(first_rsc)->variant);
 
             for (pIter = probes; pIter != NULL; pIter = pIter->next) {
                 pe_action_t *probe = (pe_action_t *) pIter->data;
 
                 crm_err("Ordering %s before %s", first->uuid, probe->uuid);
                 order_actions(first, probe, pe_order_optional);
             }
         }
     }
 #endif
 }
 
 static void
 order_probes(pe_working_set_t * data_set)
 {
     order_first_probes(data_set);
     order_then_probes(data_set);
 }
 
 gboolean
 stage7(pe_working_set_t * data_set)
 {
     pcmk__output_t *prev_out = data_set->priv;
     pcmk__output_t *out = NULL;
     GList *gIter = NULL;
 
     crm_trace("Applying ordering constraints");
 
     /* Don't ask me why, but apparently they need to be processed in
      * the order they were created in... go figure
      *
      * Also g_list_append() has horrendous performance characteristics
      * So we need to use g_list_prepend() and then reverse the list here
      */
     data_set->ordering_constraints = g_list_reverse(data_set->ordering_constraints);
 
     for (gIter = data_set->ordering_constraints; gIter != NULL; gIter = gIter->next) {
         pe__ordering_t *order = gIter->data;
         pe_resource_t *rsc = order->lh_rsc;
 
         crm_trace("Applying ordering constraint: %d", order->id);
 
         if (rsc != NULL) {
             crm_trace("rsc_action-to-*");
             rsc_order_first(rsc, order, data_set);
             continue;
         }
 
         rsc = order->rh_rsc;
         if (rsc != NULL) {
             crm_trace("action-to-rsc_action");
             rsc_order_then(order->lh_action, rsc, order);
 
         } else {
             crm_trace("action-to-action");
             order_actions(order->lh_action, order->rh_action, order->type);
         }
     }
 
     for (gIter = data_set->actions; gIter != NULL; gIter = gIter->next) {
         pe_action_t *action = (pe_action_t *) gIter->data;
 
         update_colo_start_chain(action, data_set);
     }
 
     crm_trace("Ordering probes");
     order_probes(data_set);
 
     crm_trace("Updating %d actions", g_list_length(data_set->actions));
     for (gIter = data_set->actions; gIter != NULL; gIter = gIter->next) {
         pe_action_t *action = (pe_action_t *) gIter->data;
 
         update_action(action, data_set);
     }
 
     // Check for invalid orderings
     for (gIter = data_set->actions; gIter != NULL; gIter = gIter->next) {
         pe_action_t *action = (pe_action_t *) gIter->data;
         pe_action_wrapper_t *input = NULL;
 
         for (GList *input_iter = action->actions_before;
              input_iter != NULL; input_iter = input_iter->next) {
 
             input = (pe_action_wrapper_t *) input_iter->data;
             if (pcmk__ordering_is_invalid(action, input)) {
                 input->type = pe_order_none;
             }
         }
     }
 
     /* stage7 only ever outputs to the log, so ignore whatever output object was
      * previously set and just log instead.
      */
     out = pcmk__new_logger();
     if (out == NULL) {
         return FALSE;
     }
 
     pcmk__output_set_log_level(out, LOG_NOTICE);
     data_set->priv = out;
 
     out->begin_list(out, NULL, NULL, "Actions");
     LogNodeActions(data_set);
 
     for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
         pe_resource_t *rsc = (pe_resource_t *) gIter->data;
 
         LogActions(rsc, data_set);
     }
 
     out->end_list(out);
     out->finish(out, CRM_EX_OK, true, NULL);
     pcmk__output_free(out);
 
     data_set->priv = prev_out;
     return TRUE;
 }
 
 static int transition_id = -1;
 
 /*!
  * \internal
  * \brief Log a message after calculating a transition
  *
  * \param[in] filename  Where transition input is stored
  */
 void
 pcmk__log_transition_summary(const char *filename)
 {
     if (was_processing_error) {
         crm_err("Calculated transition %d (with errors)%s%s",
                 transition_id,
                 (filename == NULL)? "" : ", saving inputs in ",
                 (filename == NULL)? "" : filename);
 
     } else if (was_processing_warning) {
         crm_warn("Calculated transition %d (with warnings)%s%s",
                  transition_id,
                  (filename == NULL)? "" : ", saving inputs in ",
                  (filename == NULL)? "" : filename);
 
     } else {
         crm_notice("Calculated transition %d%s%s",
                    transition_id,
                    (filename == NULL)? "" : ", saving inputs in ",
                    (filename == NULL)? "" : filename);
     }
     if (crm_config_error) {
         crm_notice("Configuration errors found during scheduler processing,"
                    "  please run \"crm_verify -L\" to identify issues");
     }
 }
 
 /*
  * Create a dependency graph to send to the transitioner (via the controller)
  */
 gboolean
 stage8(pe_working_set_t * data_set)
 {
     GList *gIter = NULL;
     const char *value = NULL;
     long long limit = 0LL;
 
     transition_id++;
     crm_trace("Creating transition graph %d.", transition_id);
 
     data_set->graph = create_xml_node(NULL, XML_TAG_GRAPH);
 
     value = pe_pref(data_set->config_hash, "cluster-delay");
     crm_xml_add(data_set->graph, "cluster-delay", value);
 
     value = pe_pref(data_set->config_hash, "stonith-timeout");
     crm_xml_add(data_set->graph, "stonith-timeout", value);
 
     crm_xml_add(data_set->graph, "failed-stop-offset", "INFINITY");
 
     if (pcmk_is_set(data_set->flags, pe_flag_start_failure_fatal)) {
         crm_xml_add(data_set->graph, "failed-start-offset", "INFINITY");
     } else {
         crm_xml_add(data_set->graph, "failed-start-offset", "1");
     }
 
     value = pe_pref(data_set->config_hash, "batch-limit");
     crm_xml_add(data_set->graph, "batch-limit", value);
 
     crm_xml_add_int(data_set->graph, "transition_id", transition_id);
 
     value = pe_pref(data_set->config_hash, "migration-limit");
     if ((pcmk__scan_ll(value, &limit, 0LL) == pcmk_rc_ok) && (limit > 0)) {
         crm_xml_add(data_set->graph, "migration-limit", value);
     }
 
     if (data_set->recheck_by > 0) {
         char *recheck_epoch = NULL;
 
         recheck_epoch = crm_strdup_printf("%llu",
                                           (long long) data_set->recheck_by);
         crm_xml_add(data_set->graph, "recheck-by", recheck_epoch);
         free(recheck_epoch);
     }
 
 /* errors...
    slist_iter(action, pe_action_t, action_list, lpc,
    if(action->optional == FALSE && action->runnable == FALSE) {
    print_action("Ignoring", action, TRUE);
    }
    );
 */
 
     /* The following code will de-duplicate action inputs, so nothing past this
      * should rely on the action input type flags retaining their original
      * values.
      */
 
     gIter = data_set->resources;
     for (; gIter != NULL; gIter = gIter->next) {
         pe_resource_t *rsc = (pe_resource_t *) gIter->data;
 
         pe_rsc_trace(rsc, "processing actions for rsc=%s", rsc->id);
         rsc->cmds->expand(rsc, data_set);
     }
 
     crm_log_xml_trace(data_set->graph, "created resource-driven action list");
 
     /* pseudo action to distribute list of nodes with maintenance state update */
     add_maintenance_update(data_set);
 
     /* catch any non-resource specific actions */
     crm_trace("processing non-resource actions");
 
     gIter = data_set->actions;
     for (; gIter != NULL; gIter = gIter->next) {
         pe_action_t *action = (pe_action_t *) gIter->data;
 
         if (action->rsc
             && action->node
             && action->node->details->shutdown
             && !pcmk_is_set(action->rsc->flags, pe_rsc_maintenance)
             && !pcmk_any_flags_set(action->flags,
                                    pe_action_optional|pe_action_runnable)
             && pcmk__str_eq(action->task, RSC_STOP, pcmk__str_none)
             ) {
             /* Eventually we should just ignore the 'fence' case
              * But for now it's the best way to detect (in CTS) when
              * CIB resource updates are being lost
              */
             if (pcmk_is_set(data_set->flags, pe_flag_have_quorum)
                 || data_set->no_quorum_policy == no_quorum_ignore) {
                 crm_crit("Cannot %s node '%s' because of %s:%s%s (%s)",
                          action->node->details->unclean ? "fence" : "shut down",
                          action->node->details->uname, action->rsc->id,
                          pcmk_is_set(action->rsc->flags, pe_rsc_managed)? " blocked" : " unmanaged",
                          pcmk_is_set(action->rsc->flags, pe_rsc_failed)? " failed" : "",
                          action->uuid);
             }
         }
 
         graph_element_from_action(action, data_set);
     }
 
     crm_log_xml_trace(data_set->graph, "created generic action list");
     crm_trace("Created transition graph %d.", transition_id);
 
     return TRUE;
 }
 
 void
 LogNodeActions(pe_working_set_t * data_set)
 {
     pcmk__output_t *out = data_set->priv;
     GList *gIter = NULL;
 
     for (gIter = data_set->actions; gIter != NULL; gIter = gIter->next) {
         char *node_name = NULL;
         char *task = NULL;
         pe_action_t *action = (pe_action_t *) gIter->data;
 
         if (action->rsc != NULL) {
             continue;
         } else if (pcmk_is_set(action->flags, pe_action_optional)) {
             continue;
         }
 
         if (pe__is_guest_node(action->node)) {
             node_name = crm_strdup_printf("%s (resource: %s)", action->node->details->uname, action->node->details->remote_rsc->container->id);
         } else if(action->node) {
             node_name = crm_strdup_printf("%s", action->node->details->uname);
         }
 
 
         if (pcmk__str_eq(action->task, CRM_OP_SHUTDOWN, pcmk__str_casei)) {
             task = strdup("Shutdown");
         } else if (pcmk__str_eq(action->task, CRM_OP_FENCE, pcmk__str_casei)) {
             const char *op = g_hash_table_lookup(action->meta, "stonith_action");
             task = crm_strdup_printf("Fence (%s)", op);
         }
 
         out->message(out, "node-action", task, node_name, action->reason);
 
         free(node_name);
         free(task);
     }
 }
diff --git a/lib/pacemaker/pcmk_sched_clone.c b/lib/pacemaker/pcmk_sched_clone.c
index 7475198830..5ff2274796 100644
--- a/lib/pacemaker/pcmk_sched_clone.c
+++ b/lib/pacemaker/pcmk_sched_clone.c
@@ -1,1568 +1,1567 @@
 /*
  * Copyright 2004-2021 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU General Public License version 2
  * or later (GPLv2+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #include <crm/msg_xml.h>
 #include <pacemaker-internal.h>
 
 #define VARIANT_CLONE 1
 #include <lib/pengine/variant.h>
 
 gint sort_clone_instance(gconstpointer a, gconstpointer b, gpointer data_set);
 static void append_parent_colocation(pe_resource_t * rsc, pe_resource_t * child, gboolean all);
 
 static gint
 sort_rsc_id(gconstpointer a, gconstpointer b)
 {
     const pe_resource_t *resource1 = (const pe_resource_t *)a;
     const pe_resource_t *resource2 = (const pe_resource_t *)b;
     long num1, num2;
 
     CRM_ASSERT(resource1 != NULL);
     CRM_ASSERT(resource2 != NULL);
 
     /*
      * Sort clone instances numerically by instance number, so instance :10
      * comes after :9.
      */
     num1 = strtol(strrchr(resource1->id, ':') + 1, NULL, 10);
     num2 = strtol(strrchr(resource2->id, ':') + 1, NULL, 10);
     if (num1 < num2) {
         return -1;
     } else if (num1 > num2) {
         return 1;
     }
     return 0;
 }
 
 static pe_node_t *
 parent_node_instance(const pe_resource_t * rsc, pe_node_t * node)
 {
     pe_node_t *ret = NULL;
 
     if (node != NULL && rsc->parent) {
         ret = pe_hash_table_lookup(rsc->parent->allowed_nodes, node->details->id);
     } else if(node != NULL) {
         ret = pe_hash_table_lookup(rsc->allowed_nodes, node->details->id);
     }
     return ret;
 }
 
 static gboolean
 did_fail(const pe_resource_t * rsc)
 {
     GList *gIter = rsc->children;
 
     if (pcmk_is_set(rsc->flags, pe_rsc_failed)) {
         return TRUE;
     }
 
     for (; gIter != NULL; gIter = gIter->next) {
         pe_resource_t *child_rsc = (pe_resource_t *) gIter->data;
 
         if (did_fail(child_rsc)) {
             return TRUE;
         }
     }
     return FALSE;
 }
 
 /*!
  * \internal
  * \brief Compare instances based on colocation scores.
  *
  * Determines the relative order in which \c rsc1 and \c rsc2 should be
  * allocated. If one resource compares less than the other, then it
  * should be allocated first.
  *
  * \param[in] rsc1  The first instance to compare.
  * \param[in] rsc2  The second instance to compare.
  * \param[in] data_set  Cluster working set.
  *
  * \return -1 if `rsc1 < rsc2`,
  *          0 if `rsc1 == rsc2`, or
  *          1 if `rsc1 > rsc2`
  */
 static int
 order_instance_by_colocation(const pe_resource_t *rsc1,
                              const pe_resource_t *rsc2,
                              pe_working_set_t *data_set)
 {
     int rc = 0;
     pe_node_t *n = NULL;
     pe_node_t *node1 = NULL;
     pe_node_t *node2 = NULL;
     pe_node_t *current_node1 = pe__current_node(rsc1);
     pe_node_t *current_node2 = pe__current_node(rsc2);
     GList *list1 = NULL;
     GList *list2 = NULL;
     GHashTable *hash1 = pcmk__strkey_table(NULL, free);
     GHashTable *hash2 = pcmk__strkey_table(NULL, free);
 
     /* Clone instances must have parents */
     CRM_ASSERT(rsc1->parent != NULL);
     CRM_ASSERT(rsc2->parent != NULL);
 
     n = pe__copy_node(current_node1);
     g_hash_table_insert(hash1, (gpointer) n->details->id, n);
 
     n = pe__copy_node(current_node2);
     g_hash_table_insert(hash2, (gpointer) n->details->id, n);
 
     /* Apply rsc1's parental colocations */
     for (GList *gIter = rsc1->parent->rsc_cons; gIter != NULL;
          gIter = gIter->next) {
 
         pcmk__colocation_t *constraint = (pcmk__colocation_t *) gIter->data;
 
         crm_trace("Applying %s to %s", constraint->id, rsc1->id);
 
         hash1 = pcmk__native_merge_weights(constraint->rsc_rh, rsc1->id, hash1,
                                            constraint->node_attribute,
                                            constraint->score / (float) INFINITY,
                                            0);
     }
 
     for (GList *gIter = rsc1->parent->rsc_cons_lhs; gIter != NULL;
          gIter = gIter->next) {
 
         pcmk__colocation_t *constraint = (pcmk__colocation_t *) gIter->data;
 
         if (!pcmk__colocation_has_influence(constraint, rsc1)) {
             continue;
         }
         crm_trace("Applying %s to %s", constraint->id, rsc1->id);
 
         hash1 = pcmk__native_merge_weights(constraint->rsc_lh, rsc1->id, hash1,
                                            constraint->node_attribute,
                                            constraint->score / (float) INFINITY,
                                            pe_weights_positive);
     }
 
     /* Apply rsc2's parental colocations */
     for (GList *gIter = rsc2->parent->rsc_cons; gIter != NULL;
          gIter = gIter->next) {
 
         pcmk__colocation_t *constraint = (pcmk__colocation_t *) gIter->data;
 
         crm_trace("Applying %s to %s", constraint->id, rsc2->id);
 
         hash2 = pcmk__native_merge_weights(constraint->rsc_rh, rsc2->id, hash2,
                                            constraint->node_attribute,
                                            constraint->score / (float) INFINITY,
                                            0);
     }
 
     for (GList *gIter = rsc2->parent->rsc_cons_lhs; gIter;
          gIter = gIter->next) {
 
         pcmk__colocation_t *constraint = (pcmk__colocation_t *) gIter->data;
 
         if (!pcmk__colocation_has_influence(constraint, rsc2)) {
             continue;
         }
         crm_trace("Applying %s to %s", constraint->id, rsc2->id);
 
         hash2 = pcmk__native_merge_weights(constraint->rsc_lh, rsc2->id, hash2,
                                            constraint->node_attribute,
                                            constraint->score / (float) INFINITY,
                                            pe_weights_positive);
     }
 
     /* Current location score */
     node1 = g_hash_table_lookup(hash1, current_node1->details->id);
     node2 = g_hash_table_lookup(hash2, current_node2->details->id);
 
     if (node1->weight < node2->weight) {
         if (node1->weight < 0) {
             crm_trace("%s > %s: current score: %d %d",
                       rsc1->id, rsc2->id, node1->weight, node2->weight);
             rc = -1;
             goto out;
 
         } else {
             crm_trace("%s < %s: current score: %d %d",
                       rsc1->id, rsc2->id, node1->weight, node2->weight);
             rc = 1;
             goto out;
         }
 
     } else if (node1->weight > node2->weight) {
         crm_trace("%s > %s: current score: %d %d",
                   rsc1->id, rsc2->id, node1->weight, node2->weight);
         rc = -1;
         goto out;
     }
 
     /* All location scores */
     list1 = g_hash_table_get_values(hash1);
     list2 = g_hash_table_get_values(hash2);
 
     list1 = sort_nodes_by_weight(list1, current_node1, data_set);
     list2 = sort_nodes_by_weight(list2, current_node2, data_set);
 
     for (GList *gIter1 = list1, *gIter2 = list2;
          (gIter1 != NULL) && (gIter2 != NULL);
          gIter1 = gIter1->next, gIter2 = gIter2->next) {
 
         node1 = (pe_node_t *) gIter1->data;
         node2 = (pe_node_t *) gIter2->data;
 
         if (node1 == NULL) {
             crm_trace("%s < %s: colocated score NULL", rsc1->id, rsc2->id);
             rc = 1;
             break;
 
         } else if (node2 == NULL) {
             crm_trace("%s > %s: colocated score NULL", rsc1->id, rsc2->id);
             rc = -1;
             break;
         }
 
         if (node1->weight < node2->weight) {
             crm_trace("%s < %s: colocated score", rsc1->id, rsc2->id);
             rc = 1;
             break;
 
         } else if (node1->weight > node2->weight) {
             crm_trace("%s > %s: colocated score", rsc1->id, rsc2->id);
             rc = -1;
             break;
         }
     }
 
 out:
     g_hash_table_destroy(hash1);
     g_hash_table_destroy(hash2);
     g_list_free(list1);
     g_list_free(list2);
 
     return rc;
 }
 
 gint
 sort_clone_instance(gconstpointer a, gconstpointer b, gpointer data_set)
 {
     int rc = 0;
     pe_node_t *node1 = NULL;
     pe_node_t *node2 = NULL;
     pe_node_t *current_node1 = NULL;
     pe_node_t *current_node2 = NULL;
     unsigned int nnodes1 = 0;
     unsigned int nnodes2 = 0;
 
     gboolean can1 = TRUE;
     gboolean can2 = TRUE;
 
     const pe_resource_t *resource1 = (const pe_resource_t *)a;
     const pe_resource_t *resource2 = (const pe_resource_t *)b;
 
     CRM_ASSERT(resource1 != NULL);
     CRM_ASSERT(resource2 != NULL);
 
     /* allocation order:
      *  - active instances
      *  - instances running on nodes with the least copies
      *  - active instances on nodes that can't support them or are to be fenced
      *  - failed instances
      *  - inactive instances
      */
 
     current_node1 = pe__find_active_on(resource1, &nnodes1, NULL);
     current_node2 = pe__find_active_on(resource2, &nnodes2, NULL);
 
     /* If both instances are running and at least one is multiply
      * active, give precedence to the one that's running on fewer nodes.
      */
     if ((nnodes1 > 0) && (nnodes2 > 0)) {
         if (nnodes1 < nnodes2) {
             crm_trace("%s < %s: running_on", resource1->id, resource2->id);
             return -1;
 
         } else if (nnodes1 > nnodes2) {
             crm_trace("%s > %s: running_on", resource1->id, resource2->id);
             return 1;
         }
     }
 
     /* Instance whose current location is available sorts first */
     node1 = current_node1;
     node2 = current_node2;
     if (node1 != NULL) {
         pe_node_t *match = pe_hash_table_lookup(resource1->allowed_nodes, node1->details->id);
 
         if (match == NULL || match->weight < 0) {
             crm_trace("%s: current location is unavailable", resource1->id);
             node1 = NULL;
             can1 = FALSE;
         }
     }
 
     if (node2 != NULL) {
         pe_node_t *match = pe_hash_table_lookup(resource2->allowed_nodes, node2->details->id);
 
         if (match == NULL || match->weight < 0) {
             crm_trace("%s: current location is unavailable", resource2->id);
             node2 = NULL;
             can2 = FALSE;
         }
     }
 
     if (can1 && !can2) {
         crm_trace("%s < %s: availability of current location", resource1->id,
                   resource2->id);
         return -1;
 
     } else if (!can1 && can2) {
         crm_trace("%s > %s: availability of current location", resource1->id,
                   resource2->id);
         return 1;
     }
 
     /* Higher-priority instance sorts first */
     if (resource1->priority > resource2->priority) {
         crm_trace("%s < %s: priority", resource1->id, resource2->id);
         return -1;
 
     } else if (resource1->priority < resource2->priority) {
         crm_trace("%s > %s: priority", resource1->id, resource2->id);
         return 1;
     }
 
     /* Active instance sorts first */
     if (node1 == NULL && node2 == NULL) {
         crm_trace("%s == %s: not active", resource1->id, resource2->id);
         return 0;
 
     } else if (node1 == NULL) {
         crm_trace("%s > %s: active", resource1->id, resource2->id);
         return 1;
 
     } else if (node2 == NULL) {
         crm_trace("%s < %s: active", resource1->id, resource2->id);
         return -1;
     }
 
     /* Instance whose current node can run resources sorts first */
     can1 = can_run_resources(node1);
     can2 = can_run_resources(node2);
     if (can1 && !can2) {
         crm_trace("%s < %s: can", resource1->id, resource2->id);
         return -1;
 
     } else if (!can1 && can2) {
         crm_trace("%s > %s: can", resource1->id, resource2->id);
         return 1;
     }
 
     /* Is the parent allowed to run on the instance's current node?
      * Instance with parent allowed sorts first.
      */
     node1 = parent_node_instance(resource1, node1);
     node2 = parent_node_instance(resource2, node2);
     if (node1 == NULL && node2 == NULL) {
         crm_trace("%s == %s: not allowed", resource1->id, resource2->id);
         return 0;
 
     } else if (node1 == NULL) {
         crm_trace("%s > %s: not allowed", resource1->id, resource2->id);
         return 1;
 
     } else if (node2 == NULL) {
         crm_trace("%s < %s: not allowed", resource1->id, resource2->id);
         return -1;
     }
 
     /* Does one node have more instances allocated?
      * Instance whose current node has fewer instances sorts first.
      */
     if (node1->count < node2->count) {
         crm_trace("%s < %s: count", resource1->id, resource2->id);
         return -1;
 
     } else if (node1->count > node2->count) {
         crm_trace("%s > %s: count", resource1->id, resource2->id);
         return 1;
     }
 
     /* Failed instance sorts first */
     can1 = did_fail(resource1);
     can2 = did_fail(resource2);
     if (can1 && !can2) {
         crm_trace("%s > %s: failed", resource1->id, resource2->id);
         return 1;
     } else if (!can1 && can2) {
         crm_trace("%s < %s: failed", resource1->id, resource2->id);
         return -1;
     }
 
     rc = order_instance_by_colocation(resource1, resource2, data_set);
     if (rc != 0) {
         return rc;
     }
 
     /* Default to lexicographic order by ID */
     rc = strcmp(resource1->id, resource2->id);
     crm_trace("%s %c %s: default", resource1->id, rc < 0 ? '<' : '>', resource2->id);
     return rc;
 }
 
 static pe_node_t *
 can_run_instance(pe_resource_t * rsc, pe_node_t * node, int limit)
 {
     pe_node_t *local_node = NULL;
 
     if (node == NULL && rsc->allowed_nodes) {
         GHashTableIter iter;
         g_hash_table_iter_init(&iter, rsc->allowed_nodes);
         while (g_hash_table_iter_next(&iter, NULL, (void **)&local_node)) {
             can_run_instance(rsc, local_node, limit);
         }
         return NULL;
     }
 
     if (!node) {
         /* make clang analyzer happy */
         goto bail;
 
     } else if (can_run_resources(node) == FALSE) {
         goto bail;
 
     } else if (pcmk_is_set(rsc->flags, pe_rsc_orphan)) {
         goto bail;
     }
 
     local_node = parent_node_instance(rsc, node);
 
     if (local_node == NULL) {
         crm_warn("%s cannot run on %s: node not allowed", rsc->id, node->details->uname);
         goto bail;
 
     } else if (local_node->weight < 0) {
         common_update_score(rsc, node->details->id, local_node->weight);
         pe_rsc_trace(rsc, "%s cannot run on %s: Parent node weight doesn't allow it.",
                      rsc->id, node->details->uname);
 
     } else if (local_node->count < limit) {
         pe_rsc_trace(rsc, "%s can run on %s (already running %d)",
                      rsc->id, node->details->uname, local_node->count);
         return local_node;
 
     } else {
         pe_rsc_trace(rsc, "%s cannot run on %s: node full (%d >= %d)",
                      rsc->id, node->details->uname, local_node->count, limit);
     }
 
   bail:
     if (node) {
         common_update_score(rsc, node->details->id, -INFINITY);
     }
     return NULL;
 }
 
 static pe_node_t *
 allocate_instance(pe_resource_t *rsc, pe_node_t *prefer, gboolean all_coloc,
                   int limit, pe_working_set_t *data_set)
 {
     pe_node_t *chosen = NULL;
     GHashTable *backup = NULL;
 
     CRM_ASSERT(rsc);
     pe_rsc_trace(rsc, "Checking allocation of %s (preferring %s, using %s parent colocations)",
                  rsc->id, (prefer? prefer->details->uname: "none"),
                  (all_coloc? "all" : "some"));
 
     if (!pcmk_is_set(rsc->flags, pe_rsc_provisional)) {
         return rsc->fns->location(rsc, NULL, FALSE);
 
     } else if (pcmk_is_set(rsc->flags, pe_rsc_allocating)) {
         pe_rsc_debug(rsc, "Dependency loop detected involving %s", rsc->id);
         return NULL;
     }
 
     /* Only include positive colocation preferences of dependent resources
      * if not every node will get a copy of the clone
      */
     append_parent_colocation(rsc->parent, rsc, all_coloc);
 
     if (prefer) {
         pe_node_t *local_prefer = g_hash_table_lookup(rsc->allowed_nodes, prefer->details->id);
 
         if (local_prefer == NULL || local_prefer->weight < 0) {
             pe_rsc_trace(rsc, "Not pre-allocating %s to %s - unavailable", rsc->id,
                          prefer->details->uname);
             return NULL;
         }
     }
 
     can_run_instance(rsc, NULL, limit);
 
     backup = pcmk__copy_node_table(rsc->allowed_nodes);
     pe_rsc_trace(rsc, "Allocating instance %s", rsc->id);
     chosen = rsc->cmds->allocate(rsc, prefer, data_set);
     if (chosen && prefer && (chosen->details != prefer->details)) {
         crm_info("Not pre-allocating %s to %s because %s is better",
                  rsc->id, prefer->details->uname, chosen->details->uname);
         g_hash_table_destroy(rsc->allowed_nodes);
         rsc->allowed_nodes = backup;
         native_deallocate(rsc);
         chosen = NULL;
         backup = NULL;
     }
     if (chosen) {
         pe_node_t *local_node = parent_node_instance(rsc, chosen);
 
         if (local_node) {
             local_node->count++;
 
         } else if (pcmk_is_set(rsc->flags, pe_rsc_managed)) {
             /* what to do? we can't enforce per-node limits in this case */
             pcmk__config_err("%s not found in %s (list of %d)",
                              chosen->details->id, rsc->parent->id,
                              g_hash_table_size(rsc->parent->allowed_nodes));
         }
     }
 
     if(backup) {
         g_hash_table_destroy(backup);
     }
     return chosen;
 }
 
 static void
 append_parent_colocation(pe_resource_t * rsc, pe_resource_t * child, gboolean all)
 {
 
     GList *gIter = NULL;
 
     gIter = rsc->rsc_cons;
     for (; gIter != NULL; gIter = gIter->next) {
         pcmk__colocation_t *cons = (pcmk__colocation_t *) gIter->data;
 
         if (all || cons->score < 0 || cons->score == INFINITY) {
             child->rsc_cons = g_list_prepend(child->rsc_cons, cons);
         }
     }
 
     gIter = rsc->rsc_cons_lhs;
     for (; gIter != NULL; gIter = gIter->next) {
         pcmk__colocation_t *cons = (pcmk__colocation_t *) gIter->data;
 
         if (!pcmk__colocation_has_influence(cons, child)) {
            continue;
         }
         if (all || cons->score < 0) {
             child->rsc_cons_lhs = g_list_prepend(child->rsc_cons_lhs, cons);
         }
     }
 }
 
 
 void
 distribute_children(pe_resource_t *rsc, GList *children, GList *nodes,
                     int max, int per_host_max, pe_working_set_t * data_set);
 
 void
 distribute_children(pe_resource_t *rsc, GList *children, GList *nodes,
                     int max, int per_host_max, pe_working_set_t * data_set) 
 {
     int loop_max = 0;
     int allocated = 0;
     int available_nodes = 0;
     bool all_coloc = false;
 
     /* count now tracks the number of clones currently allocated */
     for(GList *nIter = nodes; nIter != NULL; nIter = nIter->next) {
         pe_node_t *node = nIter->data;
 
         node->count = 0;
         if (can_run_resources(node)) {
             available_nodes++;
         }
     }
 
     all_coloc = (max < available_nodes) ? true : false;
 
     if(available_nodes) {
         loop_max = max / available_nodes;
     }
     if (loop_max < 1) {
         loop_max = 1;
     }
 
     pe_rsc_debug(rsc, "Allocating up to %d %s instances to a possible %d nodes (at most %d per host, %d optimal)",
                  max, rsc->id, available_nodes, per_host_max, loop_max);
 
     /* Pre-allocate as many instances as we can to their current location */
     for (GList *gIter = children; gIter != NULL && allocated < max; gIter = gIter->next) {
         pe_resource_t *child = (pe_resource_t *) gIter->data;
         pe_node_t *child_node = NULL;
         pe_node_t *local_node = NULL;
 
         if ((child->running_on == NULL)
             || !pcmk_is_set(child->flags, pe_rsc_provisional)
             || pcmk_is_set(child->flags, pe_rsc_failed)) {
 
             continue;
         }
 
         child_node = pe__current_node(child);
         local_node = parent_node_instance(child, child_node);
 
         pe_rsc_trace(rsc,
                      "Checking pre-allocation of %s to %s (%d remaining of %d)",
                      child->id, child_node->details->uname, max - allocated,
                      max);
 
         if (!can_run_resources(child_node) || (child_node->weight < 0)) {
             pe_rsc_trace(rsc, "Not pre-allocating because %s can not run %s",
                          child_node->details->uname, child->id);
             continue;
         }
 
         if ((local_node != NULL) && (local_node->count >= loop_max)) {
             pe_rsc_trace(rsc,
                          "Not pre-allocating because %s already allocated "
                          "optimal instances", child_node->details->uname);
             continue;
         }
 
         if (allocate_instance(child, child_node, all_coloc, per_host_max,
                               data_set)) {
             pe_rsc_trace(rsc, "Pre-allocated %s to %s", child->id,
                          child_node->details->uname);
             allocated++;
         }
     }
 
     pe_rsc_trace(rsc, "Done pre-allocating (%d of %d)", allocated, max);
 
     for (GList *gIter = children; gIter != NULL; gIter = gIter->next) {
         pe_resource_t *child = (pe_resource_t *) gIter->data;
 
         if (child->running_on != NULL) {
             pe_node_t *child_node = pe__current_node(child);
             pe_node_t *local_node = parent_node_instance(child, child_node);
 
             if (local_node == NULL) {
                 crm_err("%s is running on %s which isn't allowed",
                         child->id, child_node->details->uname);
             }
         }
 
         if (!pcmk_is_set(child->flags, pe_rsc_provisional)) {
         } else if (allocated >= max) {
             pe_rsc_debug(rsc, "Child %s not allocated - limit reached %d %d", child->id, allocated, max);
             resource_location(child, NULL, -INFINITY, "clone:limit_reached", data_set);
         } else {
             if (allocate_instance(child, NULL, all_coloc, per_host_max,
                                   data_set)) {
                 allocated++;
             }
         }
     }
 
     pe_rsc_debug(rsc, "Allocated %d %s instances of a possible %d",
                  allocated, rsc->id, max);
 }
 
 
 pe_node_t *
 pcmk__clone_allocate(pe_resource_t *rsc, pe_node_t *prefer,
                      pe_working_set_t *data_set)
 {
     GList *nodes = NULL;
     clone_variant_data_t *clone_data = NULL;
 
     get_clone_variant_data(clone_data, rsc);
 
     if (!pcmk_is_set(rsc->flags, pe_rsc_provisional)) {
         return NULL;
 
     } else if (pcmk_is_set(rsc->flags, pe_rsc_allocating)) {
         pe_rsc_debug(rsc, "Dependency loop detected involving %s", rsc->id);
         return NULL;
     }
 
     if (pcmk_is_set(rsc->flags, pe_rsc_promotable)) {
         pcmk__add_promotion_scores(rsc);
     }
 
     pe__set_resource_flags(rsc, pe_rsc_allocating);
 
     /* this information is used by sort_clone_instance() when deciding in which 
      * order to allocate clone instances
      */
     for (GList *gIter = rsc->rsc_cons; gIter != NULL; gIter = gIter->next) {
         pcmk__colocation_t *constraint = (pcmk__colocation_t *) gIter->data;
 
         pe_rsc_trace(rsc, "%s: Allocating %s first",
                      rsc->id, constraint->rsc_rh->id);
         constraint->rsc_rh->cmds->allocate(constraint->rsc_rh, prefer, data_set);
     }
 
     for (GList *gIter = rsc->rsc_cons_lhs; gIter != NULL; gIter = gIter->next) {
         pcmk__colocation_t *constraint = (pcmk__colocation_t *) gIter->data;
 
         if (!pcmk__colocation_has_influence(constraint, NULL)) {
             continue;
         }
         rsc->allowed_nodes =
             constraint->rsc_lh->cmds->merge_weights(constraint->rsc_lh, rsc->id, rsc->allowed_nodes,
                                                     constraint->node_attribute,
                                                     (float)constraint->score / INFINITY,
                                                     (pe_weights_rollback | pe_weights_positive));
     }
 
     pe__show_node_weights(!pcmk_is_set(data_set->flags, pe_flag_show_scores),
                           rsc, __func__, rsc->allowed_nodes, data_set);
 
     nodes = g_hash_table_get_values(rsc->allowed_nodes);
     nodes = sort_nodes_by_weight(nodes, NULL, data_set);
     rsc->children = g_list_sort_with_data(rsc->children, sort_clone_instance, data_set);
     distribute_children(rsc, rsc->children, nodes, clone_data->clone_max, clone_data->clone_node_max, data_set);
     g_list_free(nodes);
 
     if (pcmk_is_set(rsc->flags, pe_rsc_promotable)) {
         pcmk__set_instance_roles(rsc, data_set);
     }
 
     pe__clear_resource_flags(rsc, pe_rsc_provisional|pe_rsc_allocating);
     pe_rsc_trace(rsc, "Done allocating %s", rsc->id);
     return NULL;
 }
 
 static void
 clone_update_pseudo_status(pe_resource_t * rsc, gboolean * stopping, gboolean * starting,
                            gboolean * active)
 {
     GList *gIter = NULL;
 
     if (rsc->children) {
 
         gIter = rsc->children;
         for (; gIter != NULL; gIter = gIter->next) {
             pe_resource_t *child = (pe_resource_t *) gIter->data;
 
             clone_update_pseudo_status(child, stopping, starting, active);
         }
 
         return;
     }
 
     CRM_ASSERT(active != NULL);
     CRM_ASSERT(starting != NULL);
     CRM_ASSERT(stopping != NULL);
 
     if (rsc->running_on) {
         *active = TRUE;
     }
 
     gIter = rsc->actions;
     for (; gIter != NULL; gIter = gIter->next) {
         pe_action_t *action = (pe_action_t *) gIter->data;
 
         if (*starting && *stopping) {
             return;
 
         } else if (pcmk_is_set(action->flags, pe_action_optional)) {
             pe_rsc_trace(rsc, "Skipping optional: %s", action->uuid);
             continue;
 
         } else if (!pcmk_any_flags_set(action->flags,
                                        pe_action_pseudo|pe_action_runnable)) {
             pe_rsc_trace(rsc, "Skipping unrunnable: %s", action->uuid);
             continue;
 
         } else if (pcmk__str_eq(RSC_STOP, action->task, pcmk__str_casei)) {
             pe_rsc_trace(rsc, "Stopping due to: %s", action->uuid);
             *stopping = TRUE;
 
         } else if (pcmk__str_eq(RSC_START, action->task, pcmk__str_casei)) {
             if (!pcmk_is_set(action->flags, pe_action_runnable)) {
                 pe_rsc_trace(rsc, "Skipping pseudo-op: %s run=%d, pseudo=%d",
                              action->uuid,
                              pcmk_is_set(action->flags, pe_action_runnable),
                              pcmk_is_set(action->flags, pe_action_pseudo));
             } else {
                 pe_rsc_trace(rsc, "Starting due to: %s", action->uuid);
                 pe_rsc_trace(rsc, "%s run=%d, pseudo=%d",
                              action->uuid,
                              pcmk_is_set(action->flags, pe_action_runnable),
                              pcmk_is_set(action->flags, pe_action_pseudo));
                 *starting = TRUE;
             }
         }
     }
 }
 
 static pe_action_t *
 find_rsc_action(pe_resource_t *rsc, const char *task, gboolean active_only,
                 GList **list)
 {
     pe_action_t *match = NULL;
     GList *possible = NULL;
     GList *active = NULL;
 
     possible = pe__resource_actions(rsc, NULL, task, FALSE);
 
     if (active_only) {
         GList *gIter = possible;
 
         for (; gIter != NULL; gIter = gIter->next) {
             pe_action_t *op = (pe_action_t *) gIter->data;
 
             if (!pcmk_is_set(op->flags, pe_action_optional)) {
                 active = g_list_prepend(active, op);
             }
         }
 
         if (active && pcmk__list_of_1(active)) {
             match = g_list_nth_data(active, 0);
         }
 
         if (list) {
             *list = active;
             active = NULL;
         }
 
     } else if (possible && pcmk__list_of_1(possible)) {
         match = g_list_nth_data(possible, 0);
 
     }
     if (list) {
         *list = possible;
         possible = NULL;
     }
 
     if (possible) {
         g_list_free(possible);
     }
     if (active) {
         g_list_free(active);
     }
 
     return match;
 }
 
 static void
 child_ordering_constraints(pe_resource_t * rsc, pe_working_set_t * data_set)
 {
     pe_action_t *stop = NULL;
     pe_action_t *start = NULL;
     pe_action_t *last_stop = NULL;
     pe_action_t *last_start = NULL;
     GList *gIter = NULL;
     gboolean active_only = TRUE;        /* change to false to get the old behavior */
     clone_variant_data_t *clone_data = NULL;
 
     get_clone_variant_data(clone_data, rsc);
 
     if (clone_data->ordered == FALSE) {
         return;
     }
     /* we have to maintain a consistent sorted child list when building order constraints */
     rsc->children = g_list_sort(rsc->children, sort_rsc_id);
 
     for (gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
         pe_resource_t *child = (pe_resource_t *) gIter->data;
 
         stop = find_rsc_action(child, RSC_STOP, active_only, NULL);
         if (stop) {
             if (last_stop) {
                 /* child/child relative stop */
                 order_actions(stop, last_stop, pe_order_optional);
             }
             last_stop = stop;
         }
 
         start = find_rsc_action(child, RSC_START, active_only, NULL);
         if (start) {
             if (last_start) {
                 /* child/child relative start */
                 order_actions(last_start, start, pe_order_optional);
             }
             last_start = start;
         }
     }
 }
 
 void
 clone_create_actions(pe_resource_t *rsc, pe_working_set_t *data_set)
 {
     clone_variant_data_t *clone_data = NULL;
 
     get_clone_variant_data(clone_data, rsc);
     clone_create_pseudo_actions(rsc, rsc->children, &clone_data->start_notify, &clone_data->stop_notify,data_set);
     child_ordering_constraints(rsc, data_set);
     if (pcmk_is_set(rsc->flags, pe_rsc_promotable)) {
         create_promotable_actions(rsc, data_set);
     }
 }
 
 void
 clone_create_pseudo_actions(
     pe_resource_t * rsc, GList *children, notify_data_t **start_notify, notify_data_t **stop_notify,  pe_working_set_t * data_set)
 {
     gboolean child_active = FALSE;
     gboolean child_starting = FALSE;
     gboolean child_stopping = FALSE;
     gboolean allow_dependent_migrations = TRUE;
 
     pe_action_t *stop = NULL;
     pe_action_t *stopped = NULL;
 
     pe_action_t *start = NULL;
     pe_action_t *started = NULL;
 
     pe_rsc_trace(rsc, "Creating actions for %s", rsc->id);
 
     for (GList *gIter = children; gIter != NULL; gIter = gIter->next) {
         pe_resource_t *child_rsc = (pe_resource_t *) gIter->data;
         gboolean starting = FALSE;
         gboolean stopping = FALSE;
 
         child_rsc->cmds->create_actions(child_rsc, data_set);
         clone_update_pseudo_status(child_rsc, &stopping, &starting, &child_active);
         if (stopping && starting) {
             allow_dependent_migrations = FALSE;
         }
 
         child_stopping |= stopping;
         child_starting |= starting;
     }
 
     /* start */
     start = create_pseudo_resource_op(rsc, RSC_START, !child_starting, TRUE, data_set);
     started = create_pseudo_resource_op(rsc, RSC_STARTED, !child_starting, FALSE, data_set);
     started->priority = INFINITY;
 
     if (child_active || child_starting) {
-        update_action_flags(started, pe_action_runnable, __func__, __LINE__);
+        pe__set_action_flags(started, pe_action_runnable);
     }
 
     if (start_notify != NULL && *start_notify == NULL) {
         *start_notify = create_notification_boundaries(rsc, RSC_START, start, started, data_set);
     }
 
     /* stop */
     stop = create_pseudo_resource_op(rsc, RSC_STOP, !child_stopping, TRUE, data_set);
     stopped = create_pseudo_resource_op(rsc, RSC_STOPPED, !child_stopping, TRUE, data_set);
     stopped->priority = INFINITY;
     if (allow_dependent_migrations) {
-        update_action_flags(stop, pe_action_migrate_runnable, __func__,
-                            __LINE__);
+        pe__set_action_flags(stop, pe_action_migrate_runnable);
     }
 
     if (stop_notify != NULL && *stop_notify == NULL) {
         *stop_notify = create_notification_boundaries(rsc, RSC_STOP, stop, stopped, data_set);
 
         if (start_notify && *start_notify && *stop_notify) {
             order_actions((*stop_notify)->post_done, (*start_notify)->pre, pe_order_optional);
         }
     }
 }
 
 void
 clone_internal_constraints(pe_resource_t *rsc, pe_working_set_t *data_set)
 {
     pe_resource_t *last_rsc = NULL;
     GList *gIter;
     clone_variant_data_t *clone_data = NULL;
 
     get_clone_variant_data(clone_data, rsc);
 
     pe_rsc_trace(rsc, "Internal constraints for %s", rsc->id);
     new_rsc_order(rsc, RSC_STOPPED, rsc, RSC_START, pe_order_optional, data_set);
     new_rsc_order(rsc, RSC_START, rsc, RSC_STARTED, pe_order_runnable_left, data_set);
     new_rsc_order(rsc, RSC_STOP, rsc, RSC_STOPPED, pe_order_runnable_left, data_set);
 
     if (pcmk_is_set(rsc->flags, pe_rsc_promotable)) {
         new_rsc_order(rsc, RSC_DEMOTED, rsc, RSC_STOP, pe_order_optional, data_set);
         new_rsc_order(rsc, RSC_STARTED, rsc, RSC_PROMOTE, pe_order_runnable_left, data_set);
     }
 
     if (clone_data->ordered) {
         /* we have to maintain a consistent sorted child list when building order constraints */
         rsc->children = g_list_sort(rsc->children, sort_rsc_id);
     }
     for (gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
         pe_resource_t *child_rsc = (pe_resource_t *) gIter->data;
 
         child_rsc->cmds->internal_constraints(child_rsc, data_set);
 
         order_start_start(rsc, child_rsc, pe_order_runnable_left | pe_order_implies_first_printed);
         new_rsc_order(child_rsc, RSC_START, rsc, RSC_STARTED, pe_order_implies_then_printed,
                       data_set);
         if (clone_data->ordered && last_rsc) {
             order_start_start(last_rsc, child_rsc, pe_order_optional);
         }
 
         order_stop_stop(rsc, child_rsc, pe_order_implies_first_printed);
         new_rsc_order(child_rsc, RSC_STOP, rsc, RSC_STOPPED, pe_order_implies_then_printed,
                       data_set);
         if (clone_data->ordered && last_rsc) {
             order_stop_stop(child_rsc, last_rsc, pe_order_optional);
         }
 
         last_rsc = child_rsc;
     }
     if (pcmk_is_set(rsc->flags, pe_rsc_promotable)) {
         promotable_constraints(rsc, data_set);
     }
 }
 
 bool
 assign_node(pe_resource_t * rsc, pe_node_t * node, gboolean force)
 {
     bool changed = FALSE;
 
     if (rsc->children) {
 
         for (GList *gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
             pe_resource_t *child_rsc = (pe_resource_t *) gIter->data;
 
             changed |= assign_node(child_rsc, node, force);
         }
 
         return changed;
     }
 
     if (rsc->allocated_to != NULL) {
         changed = true;
     }
 
     native_assign_node(rsc, node, force);
     return changed;
 }
 
 gboolean
 is_child_compatible(pe_resource_t *child_rsc, pe_node_t * local_node, enum rsc_role_e filter, gboolean current) 
 {
     pe_node_t *node = NULL;
     enum rsc_role_e next_role = child_rsc->fns->state(child_rsc, current);
 
     CRM_CHECK(child_rsc && local_node, return FALSE);
     if (is_set_recursive(child_rsc, pe_rsc_block, TRUE) == FALSE) {
         /* We only want instances that haven't failed */
         node = child_rsc->fns->location(child_rsc, NULL, current);
     }
 
     if (filter != RSC_ROLE_UNKNOWN && next_role != filter) {
         crm_trace("Filtered %s", child_rsc->id);
         return FALSE;
     }
 
     if (node && (node->details == local_node->details)) {
         return TRUE;
 
     } else if (node) {
         crm_trace("%s - %s vs %s", child_rsc->id, node->details->uname,
                   local_node->details->uname);
 
     } else {
         crm_trace("%s - not allocated %d", child_rsc->id, current);
     }
     return FALSE;
 }
 
 pe_resource_t *
 find_compatible_child(pe_resource_t *local_child, pe_resource_t *rsc,
                       enum rsc_role_e filter, gboolean current,
                       pe_working_set_t *data_set)
 {
     pe_resource_t *pair = NULL;
     GList *gIter = NULL;
     GList *scratch = NULL;
     pe_node_t *local_node = NULL;
 
     local_node = local_child->fns->location(local_child, NULL, current);
     if (local_node) {
         return find_compatible_child_by_node(local_child, local_node, rsc, filter, current);
     }
 
     scratch = g_hash_table_get_values(local_child->allowed_nodes);
     scratch = sort_nodes_by_weight(scratch, NULL, data_set);
 
     gIter = scratch;
     for (; gIter != NULL; gIter = gIter->next) {
         pe_node_t *node = (pe_node_t *) gIter->data;
 
         pair = find_compatible_child_by_node(local_child, node, rsc, filter, current);
         if (pair) {
             goto done;
         }
     }
 
     pe_rsc_debug(rsc, "Can't pair %s with %s", local_child->id, rsc->id);
   done:
     g_list_free(scratch);
     return pair;
 }
 
 void
 clone_rsc_colocation_lh(pe_resource_t *rsc_lh, pe_resource_t *rsc_rh,
                         pcmk__colocation_t *constraint,
                         pe_working_set_t *data_set)
 {
     /* -- Never called --
      *
      * Instead we add the colocation constraints to the child and call from there
      */
     CRM_ASSERT(FALSE);
 }
 
 void
 clone_rsc_colocation_rh(pe_resource_t *rsc_lh, pe_resource_t *rsc_rh,
                         pcmk__colocation_t *constraint,
                         pe_working_set_t *data_set)
 {
     GList *gIter = NULL;
     gboolean do_interleave = FALSE;
     const char *interleave_s = NULL;
 
     CRM_CHECK(constraint != NULL, return);
     CRM_CHECK(rsc_lh != NULL, pe_err("rsc_lh was NULL for %s", constraint->id); return);
     CRM_CHECK(rsc_rh != NULL, pe_err("rsc_rh was NULL for %s", constraint->id); return);
     CRM_CHECK(rsc_lh->variant == pe_native, return);
 
     pe_rsc_trace(rsc_rh, "Processing constraint %s: %s -> %s %d",
                  constraint->id, rsc_lh->id, rsc_rh->id, constraint->score);
 
     if (pcmk_is_set(rsc_rh->flags, pe_rsc_promotable)) {
         if (pcmk_is_set(rsc_rh->flags, pe_rsc_provisional)) {
             pe_rsc_trace(rsc_rh, "%s is still provisional", rsc_rh->id);
             return;
         } else if (constraint->role_rh == RSC_ROLE_UNKNOWN) {
             pe_rsc_trace(rsc_rh, "Handling %s as a clone colocation", constraint->id);
         } else {
             promotable_colocation_rh(rsc_lh, rsc_rh, constraint, data_set);
             return;
         }
     }
 
     /* only the LHS side needs to be labeled as interleave */
     interleave_s = g_hash_table_lookup(constraint->rsc_lh->meta, XML_RSC_ATTR_INTERLEAVE);
     if(crm_is_true(interleave_s) && constraint->rsc_lh->variant > pe_group) {
         // TODO: Do we actually care about multiple RH copies sharing a LH copy anymore?
         if (copies_per_node(constraint->rsc_lh) != copies_per_node(constraint->rsc_rh)) {
             pcmk__config_err("Cannot interleave %s and %s because they do not "
                              "support the same number of instances per node",
                              constraint->rsc_lh->id, constraint->rsc_rh->id);
 
         } else {
             do_interleave = TRUE;
         }
     }
 
     if (pcmk_is_set(rsc_rh->flags, pe_rsc_provisional)) {
         pe_rsc_trace(rsc_rh, "%s is still provisional", rsc_rh->id);
         return;
 
     } else if (do_interleave) {
         pe_resource_t *rh_child = NULL;
 
         rh_child = find_compatible_child(rsc_lh, rsc_rh, RSC_ROLE_UNKNOWN,
                                          FALSE, data_set);
 
         if (rh_child) {
             pe_rsc_debug(rsc_rh, "Pairing %s with %s", rsc_lh->id, rh_child->id);
             rsc_lh->cmds->rsc_colocation_lh(rsc_lh, rh_child, constraint,
                                             data_set);
 
         } else if (constraint->score >= INFINITY) {
             crm_notice("Cannot pair %s with instance of %s", rsc_lh->id, rsc_rh->id);
             assign_node(rsc_lh, NULL, TRUE);
 
         } else {
             pe_rsc_debug(rsc_rh, "Cannot pair %s with instance of %s", rsc_lh->id, rsc_rh->id);
         }
 
         return;
 
     } else if (constraint->score >= INFINITY) {
         GList *rhs = NULL;
 
         gIter = rsc_rh->children;
         for (; gIter != NULL; gIter = gIter->next) {
             pe_resource_t *child_rsc = (pe_resource_t *) gIter->data;
             pe_node_t *chosen = child_rsc->fns->location(child_rsc, NULL, FALSE);
 
             if (chosen != NULL && is_set_recursive(child_rsc, pe_rsc_block, TRUE) == FALSE) {
                 pe_rsc_trace(rsc_rh, "Allowing %s: %s %d", constraint->id, chosen->details->uname, chosen->weight);
                 rhs = g_list_prepend(rhs, chosen);
             }
         }
 
         node_list_exclude(rsc_lh->allowed_nodes, rhs, FALSE);
         g_list_free(rhs);
         return;
     }
 
     gIter = rsc_rh->children;
     for (; gIter != NULL; gIter = gIter->next) {
         pe_resource_t *child_rsc = (pe_resource_t *) gIter->data;
 
         child_rsc->cmds->rsc_colocation_rh(rsc_lh, child_rsc, constraint,
                                            data_set);
     }
 }
 
 enum action_tasks
 clone_child_action(pe_action_t * action)
 {
     enum action_tasks result = no_action;
     pe_resource_t *child = (pe_resource_t *) action->rsc->children->data;
 
     if (pcmk__strcase_any_of(action->task, "notify", "notified", NULL)) {
 
         /* Find the action we're notifying about instead */
 
         int stop = 0;
         char *key = action->uuid;
         int lpc = strlen(key);
 
         for (; lpc > 0; lpc--) {
             if (key[lpc] == '_' && stop == 0) {
                 stop = lpc;
 
             } else if (key[lpc] == '_') {
                 char *task_mutable = NULL;
 
                 lpc++;
                 task_mutable = strdup(key + lpc);
                 task_mutable[stop - lpc] = 0;
 
                 crm_trace("Extracted action '%s' from '%s'", task_mutable, key);
                 result = get_complex_task(child, task_mutable, TRUE);
                 free(task_mutable);
                 break;
             }
         }
 
     } else {
         result = get_complex_task(child, action->task, TRUE);
     }
     return result;
 }
 
 #define pe__clear_action_summary_flags(flags, action, flag) do {        \
         flags = pcmk__clear_flags_as(__func__, __LINE__, LOG_TRACE,     \
                                      "Action summary", action->rsc->id, \
                                      flags, flag, #flag);               \
     } while (0)
 
 enum pe_action_flags
 summary_action_flags(pe_action_t * action, GList *children, pe_node_t * node)
 {
     GList *gIter = NULL;
     gboolean any_runnable = FALSE;
     gboolean check_runnable = TRUE;
     enum action_tasks task = clone_child_action(action);
     enum pe_action_flags flags = (pe_action_optional | pe_action_runnable | pe_action_pseudo);
     const char *task_s = task2text(task);
 
     for (gIter = children; gIter != NULL; gIter = gIter->next) {
         pe_action_t *child_action = NULL;
         pe_resource_t *child = (pe_resource_t *) gIter->data;
 
         child_action = find_first_action(child->actions, NULL, task_s, child->children ? NULL : node);
         pe_rsc_trace(action->rsc, "Checking for %s in %s on %s (%s)", task_s, child->id,
                      node ? node->details->uname : "none", child_action?child_action->uuid:"NA");
         if (child_action) {
             enum pe_action_flags child_flags = child->cmds->action_flags(child_action, node);
 
             if (pcmk_is_set(flags, pe_action_optional)
                 && !pcmk_is_set(child_flags, pe_action_optional)) {
                 pe_rsc_trace(child, "%s is mandatory because of %s", action->uuid,
                              child_action->uuid);
                 pe__clear_action_summary_flags(flags, action, pe_action_optional);
                 pe__clear_action_flags(action, pe_action_optional);
             }
             if (pcmk_is_set(child_flags, pe_action_runnable)) {
                 any_runnable = TRUE;
             }
         }
     }
 
     if (check_runnable && any_runnable == FALSE) {
         pe_rsc_trace(action->rsc, "%s is not runnable because no children are", action->uuid);
         pe__clear_action_summary_flags(flags, action, pe_action_runnable);
         if (node == NULL) {
             pe__clear_action_flags(action, pe_action_runnable);
         }
     }
 
     return flags;
 }
 
 enum pe_action_flags
 clone_action_flags(pe_action_t * action, pe_node_t * node)
 {
     return summary_action_flags(action, action->rsc->children, node);
 }
 
 void
 clone_rsc_location(pe_resource_t *rsc, pe__location_t *constraint)
 {
     GList *gIter = rsc->children;
 
     pe_rsc_trace(rsc, "Processing location constraint %s for %s", constraint->id, rsc->id);
 
     native_rsc_location(rsc, constraint);
 
     for (; gIter != NULL; gIter = gIter->next) {
         pe_resource_t *child_rsc = (pe_resource_t *) gIter->data;
 
         child_rsc->cmds->rsc_location(child_rsc, constraint);
     }
 }
 
 void
 clone_expand(pe_resource_t * rsc, pe_working_set_t * data_set)
 {
     GList *gIter = NULL;
     clone_variant_data_t *clone_data = NULL;
 
     get_clone_variant_data(clone_data, rsc);
 
     gIter = rsc->actions;
     for (; gIter != NULL; gIter = gIter->next) {
         pe_action_t *op = (pe_action_t *) gIter->data;
 
         rsc->cmds->action_flags(op, NULL);
     }
 
     if (clone_data->start_notify) {
         collect_notification_data(rsc, TRUE, TRUE, clone_data->start_notify);
         pcmk__create_notification_keys(rsc, clone_data->start_notify, data_set);
         create_notifications(rsc, clone_data->start_notify, data_set);
     }
 
     if (clone_data->stop_notify) {
         collect_notification_data(rsc, TRUE, TRUE, clone_data->stop_notify);
         pcmk__create_notification_keys(rsc, clone_data->stop_notify, data_set);
         create_notifications(rsc, clone_data->stop_notify, data_set);
     }
 
     if (clone_data->promote_notify) {
         collect_notification_data(rsc, TRUE, TRUE, clone_data->promote_notify);
         pcmk__create_notification_keys(rsc, clone_data->promote_notify, data_set);
         create_notifications(rsc, clone_data->promote_notify, data_set);
     }
 
     if (clone_data->demote_notify) {
         collect_notification_data(rsc, TRUE, TRUE, clone_data->demote_notify);
         pcmk__create_notification_keys(rsc, clone_data->demote_notify, data_set);
         create_notifications(rsc, clone_data->demote_notify, data_set);
     }
 
     /* Now that the notifcations have been created we can expand the children */
 
     gIter = rsc->children;
     for (; gIter != NULL; gIter = gIter->next) {
         pe_resource_t *child_rsc = (pe_resource_t *) gIter->data;
 
         child_rsc->cmds->expand(child_rsc, data_set);
     }
 
     native_expand(rsc, data_set);
 
     /* The notifications are in the graph now, we can destroy the notify_data */
     free_notification_data(clone_data->demote_notify);
     clone_data->demote_notify = NULL;
     free_notification_data(clone_data->stop_notify);
     clone_data->stop_notify = NULL;
     free_notification_data(clone_data->start_notify);
     clone_data->start_notify = NULL;
     free_notification_data(clone_data->promote_notify);
     clone_data->promote_notify = NULL;
 }
 
 // Check whether a resource or any of its children is known on node
 static bool
 rsc_known_on(const pe_resource_t *rsc, const pe_node_t *node)
 {
     if (rsc->children) {
         for (GList *child_iter = rsc->children; child_iter != NULL;
              child_iter = child_iter->next) {
 
             pe_resource_t *child = (pe_resource_t *) child_iter->data;
 
             if (rsc_known_on(child, node)) {
                 return TRUE;
             }
         }
 
     } else if (rsc->known_on) {
         GHashTableIter iter;
         pe_node_t *known_node = NULL;
 
         g_hash_table_iter_init(&iter, rsc->known_on);
         while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &known_node)) {
             if (node->details == known_node->details) {
                 return TRUE;
             }
         }
     }
     return FALSE;
 }
 
 // Look for an instance of clone that is known on node
 static pe_resource_t *
 find_instance_on(const pe_resource_t *clone, const pe_node_t *node)
 {
     for (GList *gIter = clone->children; gIter != NULL; gIter = gIter->next) {
         pe_resource_t *child = (pe_resource_t *) gIter->data;
 
         if (rsc_known_on(child, node)) {
             return child;
         }
     }
     return NULL;
 }
 
 // For unique clones, probe each instance separately
 static gboolean
 probe_unique_clone(pe_resource_t *rsc, pe_node_t *node, pe_action_t *complete,
                    gboolean force, pe_working_set_t *data_set)
 {
     gboolean any_created = FALSE;
 
     for (GList *child_iter = rsc->children; child_iter != NULL;
          child_iter = child_iter->next) {
 
         pe_resource_t *child = (pe_resource_t *) child_iter->data;
 
         any_created |= child->cmds->create_probe(child, node, complete, force,
                                                  data_set);
     }
     return any_created;
 }
 
 // For anonymous clones, only a single instance needs to be probed
 static gboolean
 probe_anonymous_clone(pe_resource_t *rsc, pe_node_t *node,
                       pe_action_t *complete, gboolean force,
                       pe_working_set_t *data_set)
 {
     // First, check if we probed an instance on this node last time
     pe_resource_t *child = find_instance_on(rsc, node);
 
     // Otherwise, check if we plan to start an instance on this node
     if (child == NULL) {
         for (GList *child_iter = rsc->children; child_iter && !child;
              child_iter = child_iter->next) {
 
             pe_node_t *local_node = NULL;
             pe_resource_t *child_rsc = (pe_resource_t *) child_iter->data;
 
             if (child_rsc) { /* make clang analyzer happy */
                 local_node = child_rsc->fns->location(child_rsc, NULL, FALSE);
                 if (local_node && (local_node->details == node->details)) {
                     child = child_rsc;
                 }
             }
         }
     }
 
     // Otherwise, use the first clone instance
     if (child == NULL) {
         child = rsc->children->data;
     }
     CRM_ASSERT(child);
     return child->cmds->create_probe(child, node, complete, force, data_set);
 }
 
 gboolean
 clone_create_probe(pe_resource_t * rsc, pe_node_t * node, pe_action_t * complete,
                    gboolean force, pe_working_set_t * data_set)
 {
     gboolean any_created = FALSE;
 
     CRM_ASSERT(rsc);
 
     rsc->children = g_list_sort(rsc->children, sort_rsc_id);
     if (rsc->children == NULL) {
         pe_warn("Clone %s has no children", rsc->id);
         return FALSE;
     }
 
     if (rsc->exclusive_discover) {
         pe_node_t *allowed = g_hash_table_lookup(rsc->allowed_nodes, node->details->id);
         if (allowed && allowed->rsc_discover_mode != pe_discover_exclusive) {
             /* exclusive discover is enabled and this node is not marked
              * as a node this resource should be discovered on
              *
              * remove the node from allowed_nodes so that the
              * notification contains only nodes that we might ever run
              * on
              */
             g_hash_table_remove(rsc->allowed_nodes, node->details->id);
 
             /* Bit of a shortcut - might as well take it */
             return FALSE;
         }
     }
 
     if (pcmk_is_set(rsc->flags, pe_rsc_unique)) {
         any_created = probe_unique_clone(rsc, node, complete, force, data_set);
     } else {
         any_created = probe_anonymous_clone(rsc, node, complete, force,
                                             data_set);
     }
     return any_created;
 }
 
 void
 clone_append_meta(pe_resource_t * rsc, xmlNode * xml)
 {
     char *name = NULL;
     clone_variant_data_t *clone_data = NULL;
 
     get_clone_variant_data(clone_data, rsc);
 
     name = crm_meta_name(XML_RSC_ATTR_UNIQUE);
     crm_xml_add(xml, name, pe__rsc_bool_str(rsc, pe_rsc_unique));
     free(name);
 
     name = crm_meta_name(XML_RSC_ATTR_NOTIFY);
     crm_xml_add(xml, name, pe__rsc_bool_str(rsc, pe_rsc_notify));
     free(name);
 
     name = crm_meta_name(XML_RSC_ATTR_INCARNATION_MAX);
     crm_xml_add_int(xml, name, clone_data->clone_max);
     free(name);
 
     name = crm_meta_name(XML_RSC_ATTR_INCARNATION_NODEMAX);
     crm_xml_add_int(xml, name, clone_data->clone_node_max);
     free(name);
 
     if (pcmk_is_set(rsc->flags, pe_rsc_promotable)) {
         name = crm_meta_name(XML_RSC_ATTR_PROMOTED_MAX);
         crm_xml_add_int(xml, name, clone_data->promoted_max);
         free(name);
 
         name = crm_meta_name(XML_RSC_ATTR_PROMOTED_NODEMAX);
         crm_xml_add_int(xml, name, clone_data->promoted_node_max);
         free(name);
 
         /* @COMPAT Maintain backward compatibility with resource agents that
          * expect the old names (deprecated since 2.0.0).
          */
         name = crm_meta_name(PCMK_XE_PROMOTED_MAX_LEGACY);
         crm_xml_add_int(xml, name, clone_data->promoted_max);
         free(name);
 
         name = crm_meta_name(PCMK_XE_PROMOTED_NODE_MAX_LEGACY);
         crm_xml_add_int(xml, name, clone_data->promoted_node_max);
         free(name);
     }
 }
diff --git a/lib/pacemaker/pcmk_sched_constraints.c b/lib/pacemaker/pcmk_sched_constraints.c
index a6487153f7..ca92c4ffbf 100644
--- a/lib/pacemaker/pcmk_sched_constraints.c
+++ b/lib/pacemaker/pcmk_sched_constraints.c
@@ -1,3132 +1,3130 @@
 /*
  * Copyright 2004-2021 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU General Public License version 2
  * or later (GPLv2+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #include <sys/param.h>
 #include <sys/types.h>
 #include <stdbool.h>
 #include <regex.h>
 #include <glib.h>
 
 #include <crm/crm.h>
 #include <crm/cib.h>
 #include <crm/msg_xml.h>
 #include <crm/common/xml.h>
 #include <crm/common/xml_internal.h>
 #include <crm/common/iso8601.h>
 #include <crm/pengine/status.h>
 #include <crm/pengine/internal.h>
 #include <crm/pengine/rules.h>
 #include <pacemaker-internal.h>
 
 enum pe_order_kind {
     pe_order_kind_optional,
     pe_order_kind_mandatory,
     pe_order_kind_serialize,
 };
 
 #define EXPAND_CONSTRAINT_IDREF(__set, __rsc, __name) do {				\
 	__rsc = pe_find_constraint_resource(data_set->resources, __name);		\
 	if(__rsc == NULL) {						\
 	    pcmk__config_err("%s: No resource found for %s", __set, __name);    \
 	    return FALSE;						\
 	}								\
     } while(0)
 
 enum pe_ordering get_flags(const char *id, enum pe_order_kind kind,
                            const char *action_first, const char *action_then, gboolean invert);
 enum pe_ordering get_asymmetrical_flags(enum pe_order_kind kind);
 static pe__location_t *generate_location_rule(pe_resource_t *rsc,
                                               xmlNode *rule_xml,
                                               const char *discovery,
                                               crm_time_t *next_change,
                                               pe_working_set_t *data_set,
                                               pe_re_match_data_t *match_data);
 static void unpack_location(xmlNode *xml_obj, pe_working_set_t *data_set);
 static void unpack_rsc_colocation(xmlNode *xml_obj, pe_working_set_t *data_set);
 
 static bool
 evaluate_lifetime(xmlNode *lifetime, pe_working_set_t *data_set)
 {
     bool result = FALSE;
     crm_time_t *next_change = crm_time_new_undefined();
 
     result = pe_evaluate_rules(lifetime, NULL, data_set->now, next_change);
     if (crm_time_is_defined(next_change)) {
         time_t recheck = (time_t) crm_time_get_seconds_since_epoch(next_change);
 
         pe__update_recheck_time(recheck, data_set);
     }
     crm_time_free(next_change);
     return result;
 }
 
 gboolean
 unpack_constraints(xmlNode * xml_constraints, pe_working_set_t * data_set)
 {
     xmlNode *xml_obj = NULL;
     xmlNode *lifetime = NULL;
 
     for (xml_obj = pcmk__xe_first_child(xml_constraints); xml_obj != NULL;
          xml_obj = pcmk__xe_next(xml_obj)) {
         const char *id = crm_element_value(xml_obj, XML_ATTR_ID);
         const char *tag = crm_element_name(xml_obj);
 
         if (id == NULL) {
             pcmk__config_err("Ignoring <%s> constraint without "
                              XML_ATTR_ID, tag);
             continue;
         }
 
         crm_trace("Processing constraint %s %s", tag, id);
 
         lifetime = first_named_child(xml_obj, "lifetime");
         if (lifetime) {
             pcmk__config_warn("Support for 'lifetime' attribute (in %s) is "
                               "deprecated (the rules it contains should "
                               "instead be direct descendents of the "
                               "constraint object)", id);
         }
 
         if (lifetime && !evaluate_lifetime(lifetime, data_set)) {
             crm_info("Constraint %s %s is not active", tag, id);
 
         } else if (pcmk__str_eq(XML_CONS_TAG_RSC_ORDER, tag, pcmk__str_casei)) {
             unpack_rsc_order(xml_obj, data_set);
 
         } else if (pcmk__str_eq(XML_CONS_TAG_RSC_DEPEND, tag, pcmk__str_casei)) {
             unpack_rsc_colocation(xml_obj, data_set);
 
         } else if (pcmk__str_eq(XML_CONS_TAG_RSC_LOCATION, tag, pcmk__str_casei)) {
             unpack_location(xml_obj, data_set);
 
         } else if (pcmk__str_eq(XML_CONS_TAG_RSC_TICKET, tag, pcmk__str_casei)) {
             unpack_rsc_ticket(xml_obj, data_set);
 
         } else {
             pe_err("Unsupported constraint type: %s", tag);
         }
     }
 
     return TRUE;
 }
 
 static const char *
 invert_action(const char *action)
 {
     if (pcmk__str_eq(action, RSC_START, pcmk__str_casei)) {
         return RSC_STOP;
 
     } else if (pcmk__str_eq(action, RSC_STOP, pcmk__str_casei)) {
         return RSC_START;
 
     } else if (pcmk__str_eq(action, RSC_PROMOTE, pcmk__str_casei)) {
         return RSC_DEMOTE;
 
     } else if (pcmk__str_eq(action, RSC_DEMOTE, pcmk__str_casei)) {
         return RSC_PROMOTE;
 
     } else if (pcmk__str_eq(action, RSC_PROMOTED, pcmk__str_casei)) {
         return RSC_DEMOTED;
 
     } else if (pcmk__str_eq(action, RSC_DEMOTED, pcmk__str_casei)) {
         return RSC_PROMOTED;
 
     } else if (pcmk__str_eq(action, RSC_STARTED, pcmk__str_casei)) {
         return RSC_STOPPED;
 
     } else if (pcmk__str_eq(action, RSC_STOPPED, pcmk__str_casei)) {
         return RSC_STARTED;
     }
     crm_warn("Unknown action '%s' specified in order constraint", action);
     return NULL;
 }
 
 static enum pe_order_kind
 get_ordering_type(xmlNode * xml_obj)
 {
     enum pe_order_kind kind_e = pe_order_kind_mandatory;
     const char *kind = crm_element_value(xml_obj, XML_ORDER_ATTR_KIND);
 
     if (kind == NULL) {
         const char *score = crm_element_value(xml_obj, XML_RULE_ATTR_SCORE);
 
         kind_e = pe_order_kind_mandatory;
 
         if (score) {
             // @COMPAT deprecated informally since 1.0.7, formally since 2.0.1
             int score_i = char2score(score);
 
             if (score_i == 0) {
                 kind_e = pe_order_kind_optional;
             }
             pe_warn_once(pe_wo_order_score,
                          "Support for 'score' in rsc_order is deprecated "
                          "and will be removed in a future release (use 'kind' instead)");
         }
 
     } else if (pcmk__str_eq(kind, "Mandatory", pcmk__str_casei)) {
         kind_e = pe_order_kind_mandatory;
 
     } else if (pcmk__str_eq(kind, "Optional", pcmk__str_casei)) {
         kind_e = pe_order_kind_optional;
 
     } else if (pcmk__str_eq(kind, "Serialize", pcmk__str_casei)) {
         kind_e = pe_order_kind_serialize;
 
     } else {
         pcmk__config_err("Resetting '" XML_ORDER_ATTR_KIND "' for constraint "
                          "'%s' to Mandatory because '%s' is not valid",
                          crm_str(ID(xml_obj)), kind);
     }
     return kind_e;
 }
 
 static pe_resource_t *
 pe_find_constraint_resource(GList *rsc_list, const char *id)
 {
     GList *rIter = NULL;
 
     for (rIter = rsc_list; id && rIter; rIter = rIter->next) {
         pe_resource_t *parent = rIter->data;
         pe_resource_t *match = parent->fns->find_rsc(parent, id, NULL,
                                                      pe_find_renamed);
 
         if (match != NULL) {
             if(!pcmk__str_eq(match->id, id, pcmk__str_casei)) {
                 /* We found an instance of a clone instead */
                 match = uber_parent(match);
                 crm_debug("Found %s for %s", match->id, id);
             }
             return match;
         }
     }
     crm_trace("No match for %s", id);
     return NULL;
 }
 
 static gboolean
 pe_find_constraint_tag(pe_working_set_t * data_set, const char * id, pe_tag_t ** tag)
 {
     gboolean rc = FALSE;
 
     *tag = NULL;
     rc = g_hash_table_lookup_extended(data_set->template_rsc_sets, id,
                                        NULL, (gpointer*) tag);
 
     if (rc == FALSE) {
         rc = g_hash_table_lookup_extended(data_set->tags, id,
                                           NULL, (gpointer*) tag);
 
         if (rc == FALSE) {
             crm_warn("No template or tag named '%s'", id);
             return FALSE;
 
         } else if (*tag == NULL) {
             crm_warn("No resource is tagged with '%s'", id);
             return FALSE;
         }
 
     } else if (*tag == NULL) {
         crm_warn("No resource is derived from template '%s'", id);
         return FALSE;
     }
 
     return rc;
 }
 
 static gboolean
 valid_resource_or_tag(pe_working_set_t * data_set, const char * id,
                       pe_resource_t ** rsc, pe_tag_t ** tag)
 {
     gboolean rc = FALSE;
 
     if (rsc) {
         *rsc = NULL;
         *rsc = pe_find_constraint_resource(data_set->resources, id);
         if (*rsc) {
             return TRUE;
         }
     }
 
     if (tag) {
         *tag = NULL;
         rc = pe_find_constraint_tag(data_set, id, tag);
     }
 
     return rc;
 }
 
 static gboolean
 order_is_symmetrical(xmlNode * xml_obj,
                      enum pe_order_kind parent_kind, const char * parent_symmetrical_s)
 {
     const char *id = crm_element_value(xml_obj, XML_ATTR_ID);
     const char *kind_s = crm_element_value(xml_obj, XML_ORDER_ATTR_KIND);
     const char *score_s = crm_element_value(xml_obj, XML_RULE_ATTR_SCORE);
     const char *symmetrical_s = crm_element_value(xml_obj, XML_CONS_ATTR_SYMMETRICAL);
     enum pe_order_kind kind = parent_kind;
 
     if (kind_s || score_s) {
         kind = get_ordering_type(xml_obj);
     }
 
     if (symmetrical_s == NULL) {
         symmetrical_s = parent_symmetrical_s;
     }
 
     if (symmetrical_s) {
         gboolean symmetrical = crm_is_true(symmetrical_s);
 
         if (symmetrical && kind == pe_order_kind_serialize) {
             pcmk__config_warn("Ignoring " XML_CONS_ATTR_SYMMETRICAL
                               " for '%s' because not valid with "
                               XML_ORDER_ATTR_KIND " of 'Serialize'", id);
             return FALSE;
         }
 
         return symmetrical;
 
     } else {
         if (kind == pe_order_kind_serialize) {
             return FALSE;
 
         } else {
             return TRUE;
         }
     }
 }
 
 static gboolean
 unpack_simple_rsc_order(xmlNode * xml_obj, pe_working_set_t * data_set)
 {
     int order_id = 0;
     pe_resource_t *rsc_then = NULL;
     pe_resource_t *rsc_first = NULL;
     gboolean invert_bool = TRUE;
     int min_required_before = 0;
     enum pe_order_kind kind = pe_order_kind_mandatory;
     enum pe_ordering cons_weight = pe_order_optional;
 
     const char *id_first = NULL;
     const char *id_then = NULL;
     const char *action_then = NULL;
     const char *action_first = NULL;
     const char *instance_then = NULL;
     const char *instance_first = NULL;
 
     const char *id = NULL;
 
     CRM_CHECK(xml_obj != NULL, return FALSE);
 
     id = crm_element_value(xml_obj, XML_ATTR_ID);
     if (id == NULL) {
         pcmk__config_err("Ignoring <%s> constraint without " XML_ATTR_ID,
                          crm_element_name(xml_obj));
         return FALSE;
     }
 
     invert_bool = order_is_symmetrical(xml_obj, kind, NULL);
 
     id_then = crm_element_value(xml_obj, XML_ORDER_ATTR_THEN);
     id_first = crm_element_value(xml_obj, XML_ORDER_ATTR_FIRST);
 
     action_then = crm_element_value(xml_obj, XML_ORDER_ATTR_THEN_ACTION);
     action_first = crm_element_value(xml_obj, XML_ORDER_ATTR_FIRST_ACTION);
 
     instance_then = crm_element_value(xml_obj, XML_ORDER_ATTR_THEN_INSTANCE);
     instance_first = crm_element_value(xml_obj, XML_ORDER_ATTR_FIRST_INSTANCE);
 
     if (action_first == NULL) {
         action_first = RSC_START;
     }
     if (action_then == NULL) {
         action_then = action_first;
     }
 
     if (id_first == NULL) {
         pcmk__config_err("Ignoring constraint '%s' without "
                          XML_ORDER_ATTR_FIRST, id);
         return FALSE;
     }
     if (id_then == NULL) {
         pcmk__config_err("Ignoring constraint '%s' without "
                          XML_ORDER_ATTR_THEN, id);
         return FALSE;
     }
 
     rsc_then = pe_find_constraint_resource(data_set->resources, id_then);
     rsc_first = pe_find_constraint_resource(data_set->resources, id_first);
 
     if (rsc_then == NULL) {
         pcmk__config_err("Ignoring constraint '%s' because resource '%s' "
                          "does not exist", id, id_then);
         return FALSE;
 
     } else if (rsc_first == NULL) {
         pcmk__config_err("Ignoring constraint '%s' because resource '%s' "
                          "does not exist", id, id_first);
         return FALSE;
 
     } else if (instance_then && pe_rsc_is_clone(rsc_then) == FALSE) {
         pcmk__config_err("Ignoring constraint '%s' because resource '%s' "
                          "is not a clone but instance '%s' was requested",
                          id, id_then, instance_then);
         return FALSE;
 
     } else if (instance_first && pe_rsc_is_clone(rsc_first) == FALSE) {
         pcmk__config_err("Ignoring constraint '%s' because resource '%s' "
                          "is not a clone but instance '%s' was requested",
                          id, id_first, instance_first);
         return FALSE;
     }
 
     if (instance_then) {
         rsc_then = find_clone_instance(rsc_then, instance_then, data_set);
         if (rsc_then == NULL) {
             pcmk__config_warn("Ignoring constraint '%s' because resource '%s' "
                               "does not have an instance '%s'",
                               id, id_then, instance_then);
             return FALSE;
         }
     }
 
     if (instance_first) {
         rsc_first = find_clone_instance(rsc_first, instance_first, data_set);
         if (rsc_first == NULL) {
             pcmk__config_warn("Ignoring constraint '%s' because resource '%s' "
                               "does not have an instance '%s'",
                               "'%s'", id, id_first, instance_first);
             return FALSE;
         }
     }
 
     cons_weight = pe_order_optional;
     kind = get_ordering_type(xml_obj);
 
     if (kind == pe_order_kind_optional && rsc_then->restart_type == pe_restart_restart) {
         crm_trace("Upgrade : recovery - implies right");
         pe__set_order_flags(cons_weight, pe_order_implies_then);
     }
 
     if (invert_bool == FALSE) {
         pe__set_order_flags(cons_weight, get_asymmetrical_flags(kind));
     } else {
         pe__set_order_flags(cons_weight,
                               get_flags(id, kind, action_first, action_then, FALSE));
     }
 
     if (pe_rsc_is_clone(rsc_first)) {
         /* If clone-min is set, require at least that number of instances to be
          * runnable before allowing dependencies to be runnable.
          */
         const char *min_clones_s = g_hash_table_lookup(rsc_first->meta,
                                                        XML_RSC_ATTR_INCARNATION_MIN);
 
         // @COMPAT 1.1.13: deprecated
         const char *require_all_s = crm_element_value(xml_obj, "require-all");
 
         if (min_clones_s) {
             pcmk__scan_min_int(min_clones_s, &min_required_before, 0);
 
         } else if (require_all_s) {
             pe_warn_once(pe_wo_require_all,
                         "Support for require-all in ordering constraints "
                         "is deprecated and will be removed in a future release"
                         " (use clone-min clone meta-attribute instead)");
             if (crm_is_true(require_all_s) == FALSE) {
                 // require-all=false is deprecated equivalent of clone-min=1
                 min_required_before = 1;
             }
         }
     }
 
     /* If there is a minimum number of instances that must be runnable before
      * the 'then' action is runnable, we use a pseudo action as an intermediate step
      * start min number of clones -> pseudo action is runnable -> dependency runnable. */
     if (min_required_before) {
         GList *rIter = NULL;
         char *task = crm_strdup_printf(CRM_OP_RELAXED_CLONE ":%s", id);
         pe_action_t *unordered_action = get_pseudo_op(task, data_set);
         free(task);
 
         /* require the pseudo action to have "min_required_before" number of
          * actions to be considered runnable before allowing the pseudo action
          * to be runnable. */ 
         unordered_action->required_runnable_before = min_required_before;
-        update_action_flags(unordered_action, pe_action_requires_any,
-                            __func__, __LINE__);
+        pe__set_action_flags(unordered_action, pe_action_requires_any);
 
         for (rIter = rsc_first->children; id && rIter; rIter = rIter->next) {
             pe_resource_t *child = rIter->data;
             /* order each clone instance before the pseudo action */
             custom_action_order(child, pcmk__op_key(child->id, action_first, 0),
                                 NULL, NULL, NULL, unordered_action,
                                 pe_order_one_or_more|pe_order_implies_then_printed,
                                 data_set);
         }
 
         /* order the "then" dependency to occur after the pseudo action only if
          * the pseudo action is runnable */ 
         order_id = custom_action_order(NULL, NULL, unordered_action, rsc_then,
                                        pcmk__op_key(rsc_then->id, action_then, 0),
                                        NULL, cons_weight|pe_order_runnable_left,
                                        data_set);
     } else {
         order_id = new_rsc_order(rsc_first, action_first, rsc_then, action_then, cons_weight, data_set);
     }
 
     pe_rsc_trace(rsc_first, "order-%d (%s): %s_%s before %s_%s flags=0x%.6x",
                  order_id, id, rsc_first->id, action_first, rsc_then->id, action_then, cons_weight);
 
     if (invert_bool == FALSE) {
         return TRUE;
     }
 
     action_then = invert_action(action_then);
     action_first = invert_action(action_first);
     if (action_then == NULL || action_first == NULL) {
         pcmk__config_err("Cannot invert constraint '%s' "
                          "(please specify inverse manually)", id);
         return TRUE;
     }
 
     cons_weight = pe_order_optional;
     if (kind == pe_order_kind_optional && rsc_then->restart_type == pe_restart_restart) {
         crm_trace("Upgrade : recovery - implies left");
         pe__set_order_flags(cons_weight, pe_order_implies_first);
     }
 
     pe__set_order_flags(cons_weight,
                           get_flags(id, kind, action_first, action_then, TRUE));
 
     order_id = new_rsc_order(rsc_then, action_then, rsc_first, action_first, cons_weight, data_set);
 
     pe_rsc_trace(rsc_then, "order-%d (%s): %s_%s before %s_%s flags=0x%.6x",
                  order_id, id, rsc_then->id, action_then, rsc_first->id, action_first, cons_weight);
 
     return TRUE;
 }
 
 static gboolean
 expand_tags_in_sets(xmlNode * xml_obj, xmlNode ** expanded_xml, pe_working_set_t * data_set)
 {
     xmlNode *new_xml = NULL;
     xmlNode *set = NULL;
     gboolean any_refs = FALSE;
     const char *cons_id = NULL;
 
     *expanded_xml = NULL;
 
     CRM_CHECK(xml_obj != NULL, return FALSE);
 
     new_xml = copy_xml(xml_obj);
     cons_id = ID(new_xml);
 
     for (set = pcmk__xe_first_child(new_xml); set != NULL;
          set = pcmk__xe_next(set)) {
 
         xmlNode *xml_rsc = NULL;
         GList *tag_refs = NULL;
         GList *gIter = NULL;
 
         if (!pcmk__str_eq((const char *)set->name, XML_CONS_TAG_RSC_SET, pcmk__str_casei)) {
             continue;
         }
 
         for (xml_rsc = pcmk__xe_first_child(set); xml_rsc != NULL;
              xml_rsc = pcmk__xe_next(xml_rsc)) {
 
             pe_resource_t *rsc = NULL;
             pe_tag_t *tag = NULL;
             const char *id = ID(xml_rsc);
 
             if (!pcmk__str_eq((const char *)xml_rsc->name, XML_TAG_RESOURCE_REF, pcmk__str_casei)) {
                 continue;
             }
 
             if (valid_resource_or_tag(data_set, id, &rsc, &tag) == FALSE) {
                 pcmk__config_err("Ignoring resource sets for constraint '%s' "
                                  "because '%s' is not a valid resource or tag",
                                  cons_id, id);
                 free_xml(new_xml);
                 return FALSE;
 
             } else if (rsc) {
                 continue;
 
             } else if (tag) {
                 /* The resource_ref under the resource_set references a template/tag */
                 xmlNode *last_ref = xml_rsc;
 
                 /* A sample:
 
                    Original XML:
 
                    <resource_set id="tag1-colocation-0" sequential="true">
                      <resource_ref id="rsc1"/>
                      <resource_ref id="tag1"/>
                      <resource_ref id="rsc4"/>
                    </resource_set>
 
                    Now we are appending rsc2 and rsc3 which are tagged with tag1 right after it:
 
                    <resource_set id="tag1-colocation-0" sequential="true">
                      <resource_ref id="rsc1"/>
                      <resource_ref id="tag1"/>
                      <resource_ref id="rsc2"/>
                      <resource_ref id="rsc3"/>
                      <resource_ref id="rsc4"/>
                    </resource_set>
 
                  */
 
                 for (gIter = tag->refs; gIter != NULL; gIter = gIter->next) {
                     const char *obj_ref = (const char *) gIter->data;
                     xmlNode *new_rsc_ref = NULL;
 
                     new_rsc_ref = xmlNewDocRawNode(getDocPtr(set), NULL,
                                                    (pcmkXmlStr) XML_TAG_RESOURCE_REF, NULL);
                     crm_xml_add(new_rsc_ref, XML_ATTR_ID, obj_ref);
                     xmlAddNextSibling(last_ref, new_rsc_ref);
 
                     last_ref = new_rsc_ref;
                 }
 
                 any_refs = TRUE;
 
                 /* Do not directly free '<resource_ref id="tag1"/>', which would
                  * break the further pcmk__xe_next(xml_rsc)) and
                  * cause "Invalid read" seen by valgrind. Just remember it for
                  * freeing later.
                  */
                 tag_refs = g_list_append(tag_refs, xml_rsc);
             }
         }
 
         /* Now free '<resource_ref id="tag1"/>', and finally get:
 
            <resource_set id="tag1-colocation-0" sequential="true">
              <resource_ref id="rsc1"/>
              <resource_ref id="rsc2"/>
              <resource_ref id="rsc3"/>
              <resource_ref id="rsc4"/>
            </resource_set>
 
          */
         for (gIter = tag_refs; gIter != NULL; gIter = gIter->next) {
             xmlNode *tag_ref = gIter->data;
 
             free_xml(tag_ref);
         }
         g_list_free(tag_refs);
     }
 
     if (any_refs) {
         *expanded_xml = new_xml;
     } else {
         free_xml(new_xml);
     }
 
     return TRUE;
 }
 
 static gboolean
 tag_to_set(xmlNode * xml_obj, xmlNode ** rsc_set, const char * attr,
                 gboolean convert_rsc, pe_working_set_t * data_set)
 {
     const char *cons_id = NULL;
     const char *id = NULL;
 
     pe_resource_t *rsc = NULL;
     pe_tag_t *tag = NULL;
 
     *rsc_set = NULL;
 
     CRM_CHECK((xml_obj != NULL) && (attr != NULL), return FALSE);
 
     cons_id = ID(xml_obj);
     if (cons_id == NULL) {
         pcmk__config_err("Ignoring <%s> constraint without " XML_ATTR_ID,
                          crm_element_name(xml_obj));
         return FALSE;
     }
 
     id = crm_element_value(xml_obj, attr);
     if (id == NULL) {
         return TRUE;
     }
 
     if (valid_resource_or_tag(data_set, id, &rsc, &tag) == FALSE) {
         pcmk__config_err("Ignoring constraint '%s' because '%s' is not a "
                          "valid resource or tag", cons_id, id);
         return FALSE;
 
     } else if (tag) {
         GList *gIter = NULL;
 
         /* A template/tag is referenced by the "attr" attribute (first, then, rsc or with-rsc).
            Add the template/tag's corresponding "resource_set" which contains the resources derived
            from it or tagged with it under the constraint. */
         *rsc_set = create_xml_node(xml_obj, XML_CONS_TAG_RSC_SET);
         crm_xml_add(*rsc_set, XML_ATTR_ID, id);
 
         for (gIter = tag->refs; gIter != NULL; gIter = gIter->next) {
             const char *obj_ref = (const char *) gIter->data;
             xmlNode *rsc_ref = NULL;
 
             rsc_ref = create_xml_node(*rsc_set, XML_TAG_RESOURCE_REF);
             crm_xml_add(rsc_ref, XML_ATTR_ID, obj_ref);
         }
 
         /* Set sequential="false" for the resource_set */
         crm_xml_add(*rsc_set, "sequential", XML_BOOLEAN_FALSE);
 
     } else if (rsc && convert_rsc) {
         /* Even a regular resource is referenced by "attr", convert it into a resource_set.
            Because the other side of the constraint could be a template/tag reference. */
         xmlNode *rsc_ref = NULL;
 
         *rsc_set = create_xml_node(xml_obj, XML_CONS_TAG_RSC_SET);
         crm_xml_add(*rsc_set, XML_ATTR_ID, id);
 
         rsc_ref = create_xml_node(*rsc_set, XML_TAG_RESOURCE_REF);
         crm_xml_add(rsc_ref, XML_ATTR_ID, id);
 
     } else {
         return TRUE;
     }
 
     /* Remove the "attr" attribute referencing the template/tag */
     if (*rsc_set) {
         xml_remove_prop(xml_obj, attr);
     }
 
     return TRUE;
 }
 
 static void unpack_rsc_location(xmlNode *xml_obj, pe_resource_t *rsc_lh,
                                 const char *role, const char *score,
                                 pe_working_set_t *data_set,
                                 pe_re_match_data_t *match_data);
 
 static void
 unpack_simple_location(xmlNode *xml_obj, pe_working_set_t *data_set)
 {
     const char *id = crm_element_value(xml_obj, XML_ATTR_ID);
     const char *value = crm_element_value(xml_obj, XML_LOC_ATTR_SOURCE);
 
     if(value) {
         pe_resource_t *rsc_lh = pe_find_constraint_resource(data_set->resources, value);
 
         unpack_rsc_location(xml_obj, rsc_lh, NULL, NULL, data_set, NULL);
     }
 
     value = crm_element_value(xml_obj, XML_LOC_ATTR_SOURCE_PATTERN);
     if(value) {
         regex_t *r_patt = calloc(1, sizeof(regex_t));
         bool invert = FALSE;
         GList *rIter = NULL;
 
         if(value[0] == '!') {
             value++;
             invert = TRUE;
         }
 
         if (regcomp(r_patt, value, REG_EXTENDED)) {
             pcmk__config_err("Ignoring constraint '%s' because "
                              XML_LOC_ATTR_SOURCE_PATTERN
                              " has invalid value '%s'", id, value);
             regfree(r_patt);
             free(r_patt);
             return;
         }
 
         for (rIter = data_set->resources; rIter; rIter = rIter->next) {
             pe_resource_t *r = rIter->data;
             int nregs = 0;
             regmatch_t *pmatch = NULL;
             int status;
 
             if(r_patt->re_nsub > 0) {
                 nregs = r_patt->re_nsub + 1;
             } else {
                 nregs = 1;
             }
             pmatch = calloc(nregs, sizeof(regmatch_t));
 
             status = regexec(r_patt, r->id, nregs, pmatch, 0);
 
             if(invert == FALSE && status == 0) {
                 pe_re_match_data_t re_match_data = {
                                                 .string = r->id,
                                                 .nregs = nregs,
                                                 .pmatch = pmatch
                                                };
 
                 crm_debug("'%s' matched '%s' for %s", r->id, value, id);
                 unpack_rsc_location(xml_obj, r, NULL, NULL, data_set, &re_match_data);
 
             } else if (invert && (status != 0)) {
                 crm_debug("'%s' is an inverted match of '%s' for %s", r->id, value, id);
                 unpack_rsc_location(xml_obj, r, NULL, NULL, data_set, NULL);
 
             } else {
                 crm_trace("'%s' does not match '%s' for %s", r->id, value, id);
             }
 
             free(pmatch);
         }
 
         regfree(r_patt);
         free(r_patt);
     }
 }
 
 static void
 unpack_rsc_location(xmlNode *xml_obj, pe_resource_t *rsc_lh, const char *role,
                     const char *score, pe_working_set_t *data_set,
                     pe_re_match_data_t *re_match_data)
 {
     pe__location_t *location = NULL;
     const char *id_lh = crm_element_value(xml_obj, XML_LOC_ATTR_SOURCE);
     const char *id = crm_element_value(xml_obj, XML_ATTR_ID);
     const char *node = crm_element_value(xml_obj, XML_CIB_TAG_NODE);
     const char *discovery = crm_element_value(xml_obj, XML_LOCATION_ATTR_DISCOVERY);
 
     if (rsc_lh == NULL) {
         pcmk__config_warn("Ignoring constraint '%s' because resource '%s' "
                           "does not exist", id, id_lh);
         return;
     }
 
     if (score == NULL) {
         score = crm_element_value(xml_obj, XML_RULE_ATTR_SCORE);
     }
 
     if (node != NULL && score != NULL) {
         int score_i = char2score(score);
         pe_node_t *match = pe_find_node(data_set->nodes, node);
 
         if (!match) {
             return;
         }
         location = rsc2node_new(id, rsc_lh, score_i, discovery, match, data_set);
 
     } else {
         bool empty = TRUE;
         crm_time_t *next_change = crm_time_new_undefined();
 
         /* This loop is logically parallel to pe_evaluate_rules(), except
          * instead of checking whether any rule is active, we set up location
          * constraints for each active rule.
          */
         for (xmlNode *rule_xml = first_named_child(xml_obj, XML_TAG_RULE);
              rule_xml != NULL; rule_xml = crm_next_same_xml(rule_xml)) {
             empty = FALSE;
             crm_trace("Unpacking %s/%s", id, ID(rule_xml));
             generate_location_rule(rsc_lh, rule_xml, discovery, next_change,
                                    data_set, re_match_data);
         }
 
         if (empty) {
             pcmk__config_err("Ignoring constraint '%s' because it contains "
                              "no rules", id);
         }
 
         /* If there is a point in the future when the evaluation of a rule will
          * change, make sure the scheduler is re-run by that time.
          */
         if (crm_time_is_defined(next_change)) {
             time_t t = (time_t) crm_time_get_seconds_since_epoch(next_change);
 
             pe__update_recheck_time(t, data_set);
         }
         crm_time_free(next_change);
         return;
     }
 
     if (role == NULL) {
         role = crm_element_value(xml_obj, XML_RULE_ATTR_ROLE);
     }
 
     if (location && role) {
         if (text2role(role) == RSC_ROLE_UNKNOWN) {
             pe_err("Invalid constraint %s: Bad role %s", id, role);
             return;
 
         } else {
             enum rsc_role_e r = text2role(role);
             switch(r) {
                 case RSC_ROLE_UNKNOWN:
                 case RSC_ROLE_STARTED:
                 case RSC_ROLE_UNPROMOTED:
                     /* Applies to all */
                     location->role_filter = RSC_ROLE_UNKNOWN;
                     break;
                 default:
                     location->role_filter = r;
                     break;
             }
         }
     }
 }
 
 static gboolean
 unpack_location_tags(xmlNode * xml_obj, xmlNode ** expanded_xml, pe_working_set_t * data_set)
 {
     const char *id = NULL;
     const char *id_lh = NULL;
     const char *state_lh = NULL;
 
     pe_resource_t *rsc_lh = NULL;
 
     pe_tag_t *tag_lh = NULL;
 
     xmlNode *new_xml = NULL;
     xmlNode *rsc_set_lh = NULL;
 
     *expanded_xml = NULL;
 
     CRM_CHECK(xml_obj != NULL, return FALSE);
 
     id = ID(xml_obj);
     if (id == NULL) {
         pcmk__config_err("Ignoring <%s> constraint without " XML_ATTR_ID,
                          crm_element_name(xml_obj));
         return FALSE;
     }
 
     /* Attempt to expand any template/tag references in possible resource sets. */
     expand_tags_in_sets(xml_obj, &new_xml, data_set);
     if (new_xml) {
         /* There are resource sets referencing templates. Return with the expanded XML. */
         crm_log_xml_trace(new_xml, "Expanded rsc_location...");
         *expanded_xml = new_xml;
         return TRUE;
     }
 
     id_lh = crm_element_value(xml_obj, XML_LOC_ATTR_SOURCE);
     if (id_lh == NULL) {
         return TRUE;
     }
 
     if (valid_resource_or_tag(data_set, id_lh, &rsc_lh, &tag_lh) == FALSE) {
         pcmk__config_err("Ignoring constraint '%s' because '%s' is not a "
                          "valid resource or tag", id, id_lh);
         return FALSE;
 
     } else if (rsc_lh) {
         /* No template is referenced. */
         return TRUE;
     }
 
     state_lh = crm_element_value(xml_obj, XML_RULE_ATTR_ROLE);
 
     new_xml = copy_xml(xml_obj);
 
     /* Convert the template/tag reference in "rsc" into a resource_set under the rsc_location constraint. */
     if (tag_to_set(new_xml, &rsc_set_lh, XML_LOC_ATTR_SOURCE, FALSE, data_set) == FALSE) {
         free_xml(new_xml);
         return FALSE;
     }
 
     if (rsc_set_lh) {
         if (state_lh) {
             /* A "rsc-role" is specified.
                Move it into the converted resource_set as a "role"" attribute. */
             crm_xml_add(rsc_set_lh, "role", state_lh);
             xml_remove_prop(new_xml, XML_RULE_ATTR_ROLE);
         }
         crm_log_xml_trace(new_xml, "Expanded rsc_location...");
         *expanded_xml = new_xml;
 
     } else {
         /* No sets */
         free_xml(new_xml);
     }
 
     return TRUE;
 }
 
 static gboolean
 unpack_location_set(xmlNode * location, xmlNode * set, pe_working_set_t * data_set)
 {
     xmlNode *xml_rsc = NULL;
     pe_resource_t *resource = NULL;
     const char *set_id;
     const char *role;
     const char *local_score;
 
     CRM_CHECK(set != NULL, return FALSE);
 
     set_id = ID(set);
     if (set_id == NULL) {
         pcmk__config_err("Ignoring " XML_CONS_TAG_RSC_SET " without "
                          XML_ATTR_ID " in constraint '%s'",
                          crm_str(ID(location)));
         return FALSE;
     }
 
     role = crm_element_value(set, "role");
     local_score = crm_element_value(set, XML_RULE_ATTR_SCORE);
 
     for (xml_rsc = pcmk__xe_first_child(set); xml_rsc != NULL;
          xml_rsc = pcmk__xe_next(xml_rsc)) {
 
         if (pcmk__str_eq((const char *)xml_rsc->name, XML_TAG_RESOURCE_REF, pcmk__str_none)) {
             EXPAND_CONSTRAINT_IDREF(set_id, resource, ID(xml_rsc));
             unpack_rsc_location(location, resource, role, local_score, data_set, NULL);
         }
     }
 
     return TRUE;
 }
 
 static void
 unpack_location(xmlNode *xml_obj, pe_working_set_t *data_set)
 {
     xmlNode *set = NULL;
     gboolean any_sets = FALSE;
 
     xmlNode *orig_xml = NULL;
     xmlNode *expanded_xml = NULL;
 
     if (unpack_location_tags(xml_obj, &expanded_xml, data_set) == FALSE) {
         return;
     }
 
     if (expanded_xml) {
         orig_xml = xml_obj;
         xml_obj = expanded_xml;
     }
 
     for (set = pcmk__xe_first_child(xml_obj); set != NULL;
          set = pcmk__xe_next(set)) {
 
         if (pcmk__str_eq((const char *)set->name, XML_CONS_TAG_RSC_SET, pcmk__str_none)) {
             any_sets = TRUE;
             set = expand_idref(set, data_set->input);
             if (unpack_location_set(xml_obj, set, data_set) == FALSE) {
                 if (expanded_xml) {
                     free_xml(expanded_xml);
                 }
                 return;
             }
         }
     }
 
     if (expanded_xml) {
         free_xml(expanded_xml);
         xml_obj = orig_xml;
     }
 
     if (any_sets == FALSE) {
         unpack_simple_location(xml_obj, data_set);
     }
 }
 
 static int
 get_node_score(const char *rule, const char *score, gboolean raw, pe_node_t * node, pe_resource_t *rsc)
 {
     int score_f = 0;
 
     if (score == NULL) {
         pe_err("Rule %s: no score specified.  Assuming 0.", rule);
 
     } else if (raw) {
         score_f = char2score(score);
 
     } else {
         const char *attr_score = pe_node_attribute_calculated(node, score, rsc);
 
         if (attr_score == NULL) {
             crm_debug("Rule %s: node %s did not have a value for %s",
                       rule, node->details->uname, score);
             score_f = -INFINITY;
 
         } else {
             crm_debug("Rule %s: node %s had value %s for %s",
                       rule, node->details->uname, attr_score, score);
             score_f = char2score(attr_score);
         }
     }
     return score_f;
 }
 
 static pe__location_t *
 generate_location_rule(pe_resource_t *rsc, xmlNode *rule_xml,
                        const char *discovery, crm_time_t *next_change,
                        pe_working_set_t *data_set,
                        pe_re_match_data_t *re_match_data)
 {
     const char *rule_id = NULL;
     const char *score = NULL;
     const char *boolean = NULL;
     const char *role = NULL;
 
     GList *gIter = NULL;
     GList *match_L = NULL;
 
     gboolean do_and = TRUE;
     gboolean accept = TRUE;
     gboolean raw_score = TRUE;
     gboolean score_allocated = FALSE;
 
     pe__location_t *location_rule = NULL;
 
     rule_xml = expand_idref(rule_xml, data_set->input);
     rule_id = crm_element_value(rule_xml, XML_ATTR_ID);
     boolean = crm_element_value(rule_xml, XML_RULE_ATTR_BOOLEAN_OP);
     role = crm_element_value(rule_xml, XML_RULE_ATTR_ROLE);
 
     crm_trace("Processing rule: %s", rule_id);
 
     if (role != NULL && text2role(role) == RSC_ROLE_UNKNOWN) {
         pe_err("Bad role specified for %s: %s", rule_id, role);
         return NULL;
     }
 
     score = crm_element_value(rule_xml, XML_RULE_ATTR_SCORE);
     if (score == NULL) {
         score = crm_element_value(rule_xml, XML_RULE_ATTR_SCORE_ATTRIBUTE);
         if (score != NULL) {
             raw_score = FALSE;
         }
     }
     if (pcmk__str_eq(boolean, "or", pcmk__str_casei)) {
         do_and = FALSE;
     }
 
     location_rule = rsc2node_new(rule_id, rsc, 0, discovery, NULL, data_set);
 
     if (location_rule == NULL) {
         return NULL;
     }
 
     if ((re_match_data != NULL) && (re_match_data->nregs > 0)
         && (re_match_data->pmatch[0].rm_so != -1) && !raw_score) {
 
         char *result = pe_expand_re_matches(score, re_match_data);
 
         if (result != NULL) {
             score = result;
             score_allocated = TRUE;
         }
     }
 
     if (role != NULL) {
         crm_trace("Setting role filter: %s", role);
         location_rule->role_filter = text2role(role);
         if (location_rule->role_filter == RSC_ROLE_UNPROMOTED) {
             /* Any promotable clone cannot be promoted without being in the
              * unpromoted role first. Ergo, any constraint for the unpromoted
              * role applies to every role.
              */
             location_rule->role_filter = RSC_ROLE_UNKNOWN;
         }
     }
     if (do_and) {
         GList *gIter = NULL;
 
         match_L = pcmk__copy_node_list(data_set->nodes, true);
         for (gIter = match_L; gIter != NULL; gIter = gIter->next) {
             pe_node_t *node = (pe_node_t *) gIter->data;
 
             node->weight = get_node_score(rule_id, score, raw_score, node, rsc);
         }
     }
 
     for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
         int score_f = 0;
         pe_node_t *node = (pe_node_t *) gIter->data;
         pe_match_data_t match_data = {
             .re = re_match_data,
             .params = pe_rsc_params(rsc, node, data_set),
             .meta = rsc->meta,
         };
 
         accept = pe_test_rule(rule_xml, node->details->attrs, RSC_ROLE_UNKNOWN,
                               data_set->now, next_change, &match_data);
 
         crm_trace("Rule %s %s on %s", ID(rule_xml), accept ? "passed" : "failed",
                   node->details->uname);
 
         score_f = get_node_score(rule_id, score, raw_score, node, rsc);
 /* 			if(accept && score_f == -INFINITY) { */
 /* 				accept = FALSE; */
 /* 			} */
 
         if (accept) {
             pe_node_t *local = pe_find_node_id(match_L, node->details->id);
 
             if (local == NULL && do_and) {
                 continue;
 
             } else if (local == NULL) {
                 local = pe__copy_node(node);
                 match_L = g_list_append(match_L, local);
             }
 
             if (do_and == FALSE) {
                 local->weight = pe__add_scores(local->weight, score_f);
             }
             crm_trace("node %s now has weight %d", node->details->uname, local->weight);
 
         } else if (do_and && !accept) {
             /* remove it */
             pe_node_t *delete = pe_find_node_id(match_L, node->details->id);
 
             if (delete != NULL) {
                 match_L = g_list_remove(match_L, delete);
                 crm_trace("node %s did not match", node->details->uname);
             }
             free(delete);
         }
     }
 
     if (score_allocated == TRUE) {
         free((char *)score);
     }
 
     location_rule->node_list_rh = match_L;
     if (location_rule->node_list_rh == NULL) {
         crm_trace("No matching nodes for rule %s", rule_id);
         return NULL;
     }
 
     crm_trace("%s: %d nodes matched", rule_id, g_list_length(location_rule->node_list_rh));
     return location_rule;
 }
 
 static gint
 sort_cons_priority_lh(gconstpointer a, gconstpointer b)
 {
     const pcmk__colocation_t *rsc_constraint1 = (const pcmk__colocation_t *) a;
     const pcmk__colocation_t *rsc_constraint2 = (const pcmk__colocation_t *) b;
 
     if (a == NULL) {
         return 1;
     }
     if (b == NULL) {
         return -1;
     }
 
     CRM_ASSERT(rsc_constraint1->rsc_lh != NULL);
     CRM_ASSERT(rsc_constraint1->rsc_rh != NULL);
 
     if (rsc_constraint1->rsc_lh->priority > rsc_constraint2->rsc_lh->priority) {
         return -1;
     }
 
     if (rsc_constraint1->rsc_lh->priority < rsc_constraint2->rsc_lh->priority) {
         return 1;
     }
 
     /* Process clones before primitives and groups */
     if (rsc_constraint1->rsc_lh->variant > rsc_constraint2->rsc_lh->variant) {
         return -1;
     } else if (rsc_constraint1->rsc_lh->variant < rsc_constraint2->rsc_lh->variant) {
         return 1;
     }
 
     /* @COMPAT scheduler <2.0.0: Process promotable clones before nonpromotable
      * clones (probably unnecessary, but avoids having to update regression
      * tests)
      */
     if (rsc_constraint1->rsc_lh->variant == pe_clone) {
         if (pcmk_is_set(rsc_constraint1->rsc_lh->flags, pe_rsc_promotable)
             && !pcmk_is_set(rsc_constraint2->rsc_lh->flags, pe_rsc_promotable)) {
             return -1;
         } else if (!pcmk_is_set(rsc_constraint1->rsc_lh->flags, pe_rsc_promotable)
             && pcmk_is_set(rsc_constraint2->rsc_lh->flags, pe_rsc_promotable)) {
             return 1;
         }
     }
 
     return strcmp(rsc_constraint1->rsc_lh->id, rsc_constraint2->rsc_lh->id);
 }
 
 static gint
 sort_cons_priority_rh(gconstpointer a, gconstpointer b)
 {
     const pcmk__colocation_t *rsc_constraint1 = (const pcmk__colocation_t *) a;
     const pcmk__colocation_t *rsc_constraint2 = (const pcmk__colocation_t *) b;
 
     if (a == NULL) {
         return 1;
     }
     if (b == NULL) {
         return -1;
     }
 
     CRM_ASSERT(rsc_constraint1->rsc_lh != NULL);
     CRM_ASSERT(rsc_constraint1->rsc_rh != NULL);
 
     if (rsc_constraint1->rsc_rh->priority > rsc_constraint2->rsc_rh->priority) {
         return -1;
     }
 
     if (rsc_constraint1->rsc_rh->priority < rsc_constraint2->rsc_rh->priority) {
         return 1;
     }
 
     /* Process clones before primitives and groups */
     if (rsc_constraint1->rsc_rh->variant > rsc_constraint2->rsc_rh->variant) {
         return -1;
     } else if (rsc_constraint1->rsc_rh->variant < rsc_constraint2->rsc_rh->variant) {
         return 1;
     }
 
     /* @COMPAT scheduler <2.0.0: Process promotable clones before nonpromotable
      * clones (probably unnecessary, but avoids having to update regression
      * tests)
      */
     if (rsc_constraint1->rsc_rh->variant == pe_clone) {
         if (pcmk_is_set(rsc_constraint1->rsc_rh->flags, pe_rsc_promotable)
             && !pcmk_is_set(rsc_constraint2->rsc_rh->flags, pe_rsc_promotable)) {
             return -1;
         } else if (!pcmk_is_set(rsc_constraint1->rsc_rh->flags, pe_rsc_promotable)
             && pcmk_is_set(rsc_constraint2->rsc_rh->flags, pe_rsc_promotable)) {
             return 1;
         }
     }
 
     return strcmp(rsc_constraint1->rsc_rh->id, rsc_constraint2->rsc_rh->id);
 }
 
 static void
 anti_colocation_order(pe_resource_t * first_rsc, int first_role,
                       pe_resource_t * then_rsc, int then_role,
                       pe_working_set_t * data_set)
 {
     const char *first_tasks[] = { NULL, NULL };
     const char *then_tasks[] = { NULL, NULL };
     int first_lpc = 0;
     int then_lpc = 0;
 
     /* Actions to make first_rsc lose first_role */
     if (first_role == RSC_ROLE_PROMOTED) {
         first_tasks[0] = CRMD_ACTION_DEMOTE;
 
     } else {
         first_tasks[0] = CRMD_ACTION_STOP;
 
         if (first_role == RSC_ROLE_UNPROMOTED) {
             first_tasks[1] = CRMD_ACTION_PROMOTE;
         }
     }
 
     /* Actions to make then_rsc gain then_role */
     if (then_role == RSC_ROLE_PROMOTED) {
         then_tasks[0] = CRMD_ACTION_PROMOTE;
 
     } else {
         then_tasks[0] = CRMD_ACTION_START;
 
         if (then_role == RSC_ROLE_UNPROMOTED) {
             then_tasks[1] = CRMD_ACTION_DEMOTE;
         }
     }
 
     for (first_lpc = 0; first_lpc <= 1 && first_tasks[first_lpc] != NULL; first_lpc++) {
         for (then_lpc = 0; then_lpc <= 1 && then_tasks[then_lpc] != NULL; then_lpc++) {
             new_rsc_order(first_rsc, first_tasks[first_lpc], then_rsc, then_tasks[then_lpc],
                           pe_order_anti_colocation, data_set);
         }
     }
 }
 
 void
 pcmk__new_colocation(const char *id, const char *node_attr, int score,
                      pe_resource_t *rsc_lh, pe_resource_t *rsc_rh,
                      const char *state_lh, const char *state_rh,
                      bool influence, pe_working_set_t *data_set)
 {
     pcmk__colocation_t *new_con = NULL;
 
     if (score == 0) {
         crm_trace("Ignoring colocation '%s' because score is 0", id);
         return;
     }
     if ((rsc_lh == NULL) || (rsc_rh == NULL)) {
         pcmk__config_err("Ignoring colocation '%s' because resource "
                          "does not exist", id);
         return;
     }
 
     new_con = calloc(1, sizeof(pcmk__colocation_t));
     if (new_con == NULL) {
         return;
     }
 
     if (pcmk__str_eq(state_lh, RSC_ROLE_STARTED_S, pcmk__str_null_matches | pcmk__str_casei)) {
         state_lh = RSC_ROLE_UNKNOWN_S;
     }
 
     if (pcmk__str_eq(state_rh, RSC_ROLE_STARTED_S, pcmk__str_null_matches | pcmk__str_casei)) {
         state_rh = RSC_ROLE_UNKNOWN_S;
     }
 
     new_con->id = id;
     new_con->rsc_lh = rsc_lh;
     new_con->rsc_rh = rsc_rh;
     new_con->score = score;
     new_con->role_lh = text2role(state_lh);
     new_con->role_rh = text2role(state_rh);
     new_con->node_attribute = node_attr;
     new_con->influence = influence;
 
     if (node_attr == NULL) {
         node_attr = CRM_ATTR_UNAME;
     }
 
     pe_rsc_trace(rsc_lh, "%s ==> %s (%s %d)", rsc_lh->id, rsc_rh->id, node_attr, score);
 
     rsc_lh->rsc_cons = g_list_insert_sorted(rsc_lh->rsc_cons, new_con, sort_cons_priority_rh);
 
     rsc_rh->rsc_cons_lhs =
         g_list_insert_sorted(rsc_rh->rsc_cons_lhs, new_con, sort_cons_priority_lh);
 
     data_set->colocation_constraints = g_list_append(data_set->colocation_constraints, new_con);
 
     if (score <= -INFINITY) {
         anti_colocation_order(rsc_lh, new_con->role_lh, rsc_rh, new_con->role_rh, data_set);
         anti_colocation_order(rsc_rh, new_con->role_rh, rsc_lh, new_con->role_lh, data_set);
     }
 }
 
 /* LHS before RHS */
 int
 new_rsc_order(pe_resource_t * lh_rsc, const char *lh_task,
               pe_resource_t * rh_rsc, const char *rh_task,
               enum pe_ordering type, pe_working_set_t * data_set)
 {
     char *lh_key = NULL;
     char *rh_key = NULL;
 
     CRM_CHECK(lh_rsc != NULL, return -1);
     CRM_CHECK(lh_task != NULL, return -1);
     CRM_CHECK(rh_rsc != NULL, return -1);
     CRM_CHECK(rh_task != NULL, return -1);
 
 #if 0
     /* We do not need to test if these reference stonith resources
      * because the fencer has access to them even when they're not "running"
      */
     if (validate_order_resources(lh_rsc, lh_task, rh_rsc, rh_task)) {
         return -1;
     }
 #endif
 
     lh_key = pcmk__op_key(lh_rsc->id, lh_task, 0);
     rh_key = pcmk__op_key(rh_rsc->id, rh_task, 0);
 
     return custom_action_order(lh_rsc, lh_key, NULL, rh_rsc, rh_key, NULL, type, data_set);
 }
 
 static char *
 task_from_action_or_key(pe_action_t *action, const char *key)
 {
     char *res = NULL;
 
     if (action) {
         res = strdup(action->task);
     } else if (key) {
         parse_op_key(key, NULL, &res, NULL);
     }
     return res;
 }
 
 /* when order constraints are made between two resources start and stop actions
  * those constraints have to be mirrored against the corresponding
  * migration actions to ensure start/stop ordering is preserved during
  * a migration */
 static void
 handle_migration_ordering(pe__ordering_t *order, pe_working_set_t *data_set)
 {
     char *lh_task = NULL;
     char *rh_task = NULL;
     gboolean rh_migratable;
     gboolean lh_migratable;
 
     if (order->lh_rsc == NULL || order->rh_rsc == NULL) {
         return;
     } else if (order->lh_rsc == order->rh_rsc) {
         return;
     /* don't mess with those constraints built between parent
      * resources and the children */
     } else if (is_parent(order->lh_rsc, order->rh_rsc)) {
         return;
     } else if (is_parent(order->rh_rsc, order->lh_rsc)) {
         return;
     }
 
     lh_migratable = pcmk_is_set(order->lh_rsc->flags, pe_rsc_allow_migrate);
     rh_migratable = pcmk_is_set(order->rh_rsc->flags, pe_rsc_allow_migrate);
 
     /* one of them has to be migratable for
      * the migrate ordering logic to be applied */
     if (lh_migratable == FALSE && rh_migratable == FALSE) {
         return;
     }
 
     /* at this point we have two resources which allow migrations that have an
      * order dependency set between them.  If those order dependencies involve
      * start/stop actions, we need to mirror the corresponding migrate actions
      * so order will be preserved. */
     lh_task = task_from_action_or_key(order->lh_action, order->lh_action_task);
     rh_task = task_from_action_or_key(order->rh_action, order->rh_action_task);
     if (lh_task == NULL || rh_task == NULL) {
         goto cleanup_order;
     }
 
     if (pcmk__str_eq(lh_task, RSC_START, pcmk__str_casei) && pcmk__str_eq(rh_task, RSC_START, pcmk__str_casei)) {
         int flags = pe_order_optional;
 
         if (lh_migratable && rh_migratable) {
             /* A start then B start
              * A migrate_from then B migrate_to */
             custom_action_order(order->lh_rsc,
                                 pcmk__op_key(order->lh_rsc->id, RSC_MIGRATED, 0),
                                 NULL, order->rh_rsc,
                                 pcmk__op_key(order->rh_rsc->id, RSC_MIGRATE, 0),
                                 NULL, flags, data_set);
         }
 
         if (rh_migratable) {
             if (lh_migratable) {
                 pe__set_order_flags(flags, pe_order_apply_first_non_migratable);
             }
 
             /* A start then B start
              * A start then B migrate_to... only if A start is not a part of a migration*/
             custom_action_order(order->lh_rsc,
                                 pcmk__op_key(order->lh_rsc->id, RSC_START, 0),
                                 NULL, order->rh_rsc,
                                 pcmk__op_key(order->rh_rsc->id, RSC_MIGRATE, 0),
                                 NULL, flags, data_set);
         }
 
     } else if (rh_migratable == TRUE && pcmk__str_eq(lh_task, RSC_STOP, pcmk__str_casei) && pcmk__str_eq(rh_task, RSC_STOP, pcmk__str_casei)) {
         int flags = pe_order_optional;
 
         if (lh_migratable) {
             pe__set_order_flags(flags, pe_order_apply_first_non_migratable);
         }
 
         /* rh side is at the bottom of the stack during a stop. If we have a constraint
          * stop B then stop A, if B is migrating via stop/start, and A is migrating using migration actions,
          * we need to enforce that A's migrate_to action occurs after B's stop action. */
         custom_action_order(order->lh_rsc,
                             pcmk__op_key(order->lh_rsc->id, RSC_STOP, 0), NULL,
                             order->rh_rsc,
                             pcmk__op_key(order->rh_rsc->id, RSC_MIGRATE, 0),
                             NULL, flags, data_set);
 
         /* We need to build the stop constraint against migrate_from as well
          * to account for partial migrations. */
         if (order->rh_rsc->partial_migration_target) {
             custom_action_order(order->lh_rsc,
                                 pcmk__op_key(order->lh_rsc->id, RSC_STOP, 0),
                                 NULL, order->rh_rsc,
                                 pcmk__op_key(order->rh_rsc->id, RSC_MIGRATED, 0),
                                 NULL, flags, data_set);
         }
 
     } else if (pcmk__str_eq(lh_task, RSC_PROMOTE, pcmk__str_casei) && pcmk__str_eq(rh_task, RSC_START, pcmk__str_casei)) {
         int flags = pe_order_optional;
 
         if (rh_migratable) {
             /* A promote then B start
              * A promote then B migrate_to */
             custom_action_order(order->lh_rsc,
                                 pcmk__op_key(order->lh_rsc->id, RSC_PROMOTE, 0),
                                 NULL, order->rh_rsc,
                                 pcmk__op_key(order->rh_rsc->id, RSC_MIGRATE, 0),
                                 NULL, flags, data_set);
         }
 
     } else if (pcmk__str_eq(lh_task, RSC_DEMOTE, pcmk__str_casei) && pcmk__str_eq(rh_task, RSC_STOP, pcmk__str_casei)) {
         int flags = pe_order_optional;
 
         if (rh_migratable) {
             /* A demote then B stop
              * A demote then B migrate_to */
             custom_action_order(order->lh_rsc, pcmk__op_key(order->lh_rsc->id, RSC_DEMOTE, 0), NULL,
                                 order->rh_rsc, pcmk__op_key(order->rh_rsc->id, RSC_MIGRATE, 0), NULL,
                                 flags, data_set);
 
             /* We need to build the demote constraint against migrate_from as well
              * to account for partial migrations. */
             if (order->rh_rsc->partial_migration_target) {
                 custom_action_order(order->lh_rsc,
                                     pcmk__op_key(order->lh_rsc->id, RSC_DEMOTE, 0),
                                     NULL, order->rh_rsc,
                                     pcmk__op_key(order->rh_rsc->id, RSC_MIGRATED, 0),
                                     NULL, flags, data_set);
             }
         }
     }
 
 cleanup_order:
     free(lh_task);
     free(rh_task);
 }
 
 /* LHS before RHS */
 int
 custom_action_order(pe_resource_t * lh_rsc, char *lh_action_task, pe_action_t * lh_action,
                     pe_resource_t * rh_rsc, char *rh_action_task, pe_action_t * rh_action,
                     enum pe_ordering type, pe_working_set_t * data_set)
 {
     pe__ordering_t *order = NULL;
 
     if (lh_rsc == NULL && lh_action) {
         lh_rsc = lh_action->rsc;
     }
     if (rh_rsc == NULL && rh_action) {
         rh_rsc = rh_action->rsc;
     }
 
     if ((lh_action == NULL && lh_rsc == NULL)
         || (rh_action == NULL && rh_rsc == NULL)) {
         crm_err("Invalid ordering (bug?)");
         free(lh_action_task);
         free(rh_action_task);
         return -1;
     }
 
     order = calloc(1, sizeof(pe__ordering_t));
 
     crm_trace("Creating[%d] %s %s %s - %s %s %s", data_set->order_id,
               lh_rsc?lh_rsc->id:"NA", lh_action_task?lh_action_task:"NA", lh_action?lh_action->uuid:"NA",
               rh_rsc?rh_rsc->id:"NA", rh_action_task?rh_action_task:"NA", rh_action?rh_action->uuid:"NA");
 
     /* CRM_ASSERT(data_set->order_id != 291); */
 
     order->id = data_set->order_id++;
     order->type = type;
     order->lh_rsc = lh_rsc;
     order->rh_rsc = rh_rsc;
     order->lh_action = lh_action;
     order->rh_action = rh_action;
     order->lh_action_task = lh_action_task;
     order->rh_action_task = rh_action_task;
 
     if (order->lh_action_task == NULL && lh_action) {
         order->lh_action_task = strdup(lh_action->uuid);
     }
 
     if (order->rh_action_task == NULL && rh_action) {
         order->rh_action_task = strdup(rh_action->uuid);
     }
 
     if (order->lh_rsc == NULL && lh_action) {
         order->lh_rsc = lh_action->rsc;
     }
 
     if (order->rh_rsc == NULL && rh_action) {
         order->rh_rsc = rh_action->rsc;
     }
 
     data_set->ordering_constraints = g_list_prepend(data_set->ordering_constraints, order);
     handle_migration_ordering(order, data_set);
 
     return order->id;
 }
 
 enum pe_ordering
 get_asymmetrical_flags(enum pe_order_kind kind)
 {
     enum pe_ordering flags = pe_order_optional;
 
     if (kind == pe_order_kind_mandatory) {
         pe__set_order_flags(flags, pe_order_asymmetrical);
     } else if (kind == pe_order_kind_serialize) {
         pe__set_order_flags(flags, pe_order_serialize_only);
     }
     return flags;
 }
 
 enum pe_ordering
 get_flags(const char *id, enum pe_order_kind kind,
           const char *action_first, const char *action_then, gboolean invert)
 {
     enum pe_ordering flags = pe_order_optional;
 
     if (invert && kind == pe_order_kind_mandatory) {
         crm_trace("Upgrade %s: implies left", id);
         pe__set_order_flags(flags, pe_order_implies_first);
 
     } else if (kind == pe_order_kind_mandatory) {
         crm_trace("Upgrade %s: implies right", id);
         pe__set_order_flags(flags, pe_order_implies_then);
         if (pcmk__strcase_any_of(action_first, RSC_START, RSC_PROMOTE, NULL)) {
             crm_trace("Upgrade %s: runnable", id);
             pe__set_order_flags(flags, pe_order_runnable_left);
         }
 
     } else if (kind == pe_order_kind_serialize) {
         pe__set_order_flags(flags, pe_order_serialize_only);
     }
 
     return flags;
 }
 
 static gboolean
 unpack_order_set(xmlNode * set, enum pe_order_kind parent_kind, pe_resource_t ** rsc,
                  pe_action_t ** begin, pe_action_t ** end, pe_action_t ** inv_begin,
                  pe_action_t ** inv_end, const char *parent_symmetrical_s,
                  pe_working_set_t * data_set)
 {
     xmlNode *xml_rsc = NULL;
     GList *set_iter = NULL;
     GList *resources = NULL;
 
     pe_resource_t *last = NULL;
     pe_resource_t *resource = NULL;
 
     int local_kind = parent_kind;
     gboolean sequential = FALSE;
     enum pe_ordering flags = pe_order_optional;
     gboolean symmetrical = TRUE;
 
     char *key = NULL;
     const char *id = ID(set);
     const char *action = crm_element_value(set, "action");
     const char *sequential_s = crm_element_value(set, "sequential");
     const char *kind_s = crm_element_value(set, XML_ORDER_ATTR_KIND);
 
     /*
        char *pseudo_id = NULL;
        char *end_id    = NULL;
        char *begin_id  = NULL;
      */
 
     if (action == NULL) {
         action = RSC_START;
     }
 
     if (kind_s) {
         local_kind = get_ordering_type(set);
     }
     if (sequential_s == NULL) {
         sequential_s = "1";
     }
 
     sequential = crm_is_true(sequential_s);
 
     symmetrical = order_is_symmetrical(set, parent_kind, parent_symmetrical_s);
     if (symmetrical) {
         flags = get_flags(id, local_kind, action, action, FALSE);
     } else {
         flags = get_asymmetrical_flags(local_kind);
     }
 
     for (xml_rsc = pcmk__xe_first_child(set); xml_rsc != NULL;
          xml_rsc = pcmk__xe_next(xml_rsc)) {
 
         if (pcmk__str_eq((const char *)xml_rsc->name, XML_TAG_RESOURCE_REF, pcmk__str_none)) {
             EXPAND_CONSTRAINT_IDREF(id, resource, ID(xml_rsc));
             resources = g_list_append(resources, resource);
         }
     }
 
     if (pcmk__list_of_1(resources)) {
         crm_trace("Single set: %s", id);
         *rsc = resource;
         *end = NULL;
         *begin = NULL;
         *inv_end = NULL;
         *inv_begin = NULL;
         goto done;
     }
 
     /*
        pseudo_id = crm_strdup_printf("%s-%s", id, action);
        end_id    = crm_strdup_printf("%s-%s", pseudo_id, "end");
        begin_id  = crm_strdup_printf("%s-%s", pseudo_id, "begin");
      */
 
     *rsc = NULL;
     /*
      *end = get_pseudo_op(end_id, data_set);
      *begin = get_pseudo_op(begin_id, data_set);
 
      free(pseudo_id);
      free(begin_id);
      free(end_id);
      */
 
     set_iter = resources;
     while (set_iter != NULL) {
         resource = (pe_resource_t *) set_iter->data;
         set_iter = set_iter->next;
 
         key = pcmk__op_key(resource->id, action, 0);
 
         /*
            custom_action_order(NULL, NULL, *begin, resource, strdup(key), NULL,
            flags|pe_order_implies_first_printed, data_set);
 
            custom_action_order(resource, strdup(key), NULL, NULL, NULL, *end,
            flags|pe_order_implies_then_printed, data_set);
          */
 
         if (local_kind == pe_order_kind_serialize) {
             /* Serialize before everything that comes after */
 
             GList *gIter = NULL;
 
             for (gIter = set_iter; gIter != NULL; gIter = gIter->next) {
                 pe_resource_t *then_rsc = (pe_resource_t *) gIter->data;
                 char *then_key = pcmk__op_key(then_rsc->id, action, 0);
 
                 custom_action_order(resource, strdup(key), NULL, then_rsc, then_key, NULL,
                                     flags, data_set);
             }
 
         } else if (sequential) {
             if (last != NULL) {
                 new_rsc_order(last, action, resource, action, flags, data_set);
             }
             last = resource;
         }
         free(key);
     }
 
     if (symmetrical == FALSE) {
         goto done;
     }
 
     last = NULL;
     action = invert_action(action);
 
     /*
        pseudo_id = crm_strdup_printf("%s-%s", id, action);
        end_id    = crm_strdup_printf("%s-%s", pseudo_id, "end");
        begin_id  = crm_strdup_printf("%s-%s", pseudo_id, "begin");
 
        *inv_end = get_pseudo_op(end_id, data_set);
        *inv_begin = get_pseudo_op(begin_id, data_set);
 
        free(pseudo_id);
        free(begin_id);
        free(end_id);
      */
 
     flags = get_flags(id, local_kind, action, action, TRUE);
 
     set_iter = resources;
     while (set_iter != NULL) {
         resource = (pe_resource_t *) set_iter->data;
         set_iter = set_iter->next;
 
         /*
            key = pcmk__op_key(resource->id, action, 0);
 
            custom_action_order(NULL, NULL, *inv_begin, resource, strdup(key), NULL,
            flags|pe_order_implies_first_printed, data_set);
 
            custom_action_order(resource, key, NULL, NULL, NULL, *inv_end,
            flags|pe_order_implies_then_printed, data_set);
          */
 
         if (sequential) {
             if (last != NULL) {
                 new_rsc_order(resource, action, last, action, flags, data_set);
             }
             last = resource;
         }
     }
 
   done:
     g_list_free(resources);
     return TRUE;
 }
 
 static gboolean
 order_rsc_sets(const char *id, xmlNode * set1, xmlNode * set2, enum pe_order_kind kind,
                pe_working_set_t * data_set, gboolean invert, gboolean symmetrical)
 {
 
     xmlNode *xml_rsc = NULL;
     xmlNode *xml_rsc_2 = NULL;
 
     pe_resource_t *rsc_1 = NULL;
     pe_resource_t *rsc_2 = NULL;
 
     const char *action_1 = crm_element_value(set1, "action");
     const char *action_2 = crm_element_value(set2, "action");
 
     const char *sequential_1 = crm_element_value(set1, "sequential");
     const char *sequential_2 = crm_element_value(set2, "sequential");
 
     const char *require_all_s = crm_element_value(set1, "require-all");
     gboolean require_all = require_all_s ? crm_is_true(require_all_s) : TRUE;
 
     enum pe_ordering flags = pe_order_none;
 
     if (action_1 == NULL) {
         action_1 = RSC_START;
     };
 
     if (action_2 == NULL) {
         action_2 = RSC_START;
     };
 
     if (invert) {
         action_1 = invert_action(action_1);
         action_2 = invert_action(action_2);
     }
 
     if(pcmk__str_eq(RSC_STOP, action_1, pcmk__str_casei) || pcmk__str_eq(RSC_DEMOTE, action_1, pcmk__str_casei)) {
         /* Assuming: A -> ( B || C) -> D
          * The one-or-more logic only applies during the start/promote phase
          * During shutdown neither B nor can shutdown until D is down, so simply turn require_all back on.
          */
         require_all = TRUE;
     }
 
     if (symmetrical == FALSE) {
         flags = get_asymmetrical_flags(kind);
     } else {
         flags = get_flags(id, kind, action_2, action_1, invert);
     }
 
     /* If we have an un-ordered set1, whether it is sequential or not is irrelevant in regards to set2. */
     if (!require_all) {
         char *task = crm_strdup_printf(CRM_OP_RELAXED_SET ":%s", ID(set1));
         pe_action_t *unordered_action = get_pseudo_op(task, data_set);
 
         free(task);
-        update_action_flags(unordered_action, pe_action_requires_any,
-                            __func__, __LINE__);
+        pe__set_action_flags(unordered_action, pe_action_requires_any);
 
         for (xml_rsc = pcmk__xe_first_child(set1); xml_rsc != NULL;
              xml_rsc = pcmk__xe_next(xml_rsc)) {
 
             if (!pcmk__str_eq((const char *)xml_rsc->name, XML_TAG_RESOURCE_REF, pcmk__str_none)) {
                 continue;
             }
 
             EXPAND_CONSTRAINT_IDREF(id, rsc_1, ID(xml_rsc));
 
             /* Add an ordering constraint between every element in set1 and the pseudo action.
              * If any action in set1 is runnable the pseudo action will be runnable. */
             custom_action_order(rsc_1, pcmk__op_key(rsc_1->id, action_1, 0),
                                 NULL, NULL, NULL, unordered_action,
                                 pe_order_one_or_more|pe_order_implies_then_printed,
                                 data_set);
         }
         for (xml_rsc_2 = pcmk__xe_first_child(set2); xml_rsc_2 != NULL;
              xml_rsc_2 = pcmk__xe_next(xml_rsc_2)) {
 
             if (!pcmk__str_eq((const char *)xml_rsc_2->name, XML_TAG_RESOURCE_REF, pcmk__str_none)) {
                 continue;
             }
 
             EXPAND_CONSTRAINT_IDREF(id, rsc_2, ID(xml_rsc_2));
 
             /* Add an ordering constraint between the pseudo action and every element in set2.
              * If the pseudo action is runnable, every action in set2 will be runnable */
             custom_action_order(NULL, NULL, unordered_action,
                                 rsc_2, pcmk__op_key(rsc_2->id, action_2, 0),
                                 NULL, flags|pe_order_runnable_left, data_set);
         }
 
         return TRUE;
     }
 
     if (crm_is_true(sequential_1)) {
         if (invert == FALSE) {
             /* get the last one */
             const char *rid = NULL;
 
             for (xml_rsc = pcmk__xe_first_child(set1); xml_rsc != NULL;
                  xml_rsc = pcmk__xe_next(xml_rsc)) {
 
                 if (pcmk__str_eq((const char *)xml_rsc->name, XML_TAG_RESOURCE_REF, pcmk__str_none)) {
                     rid = ID(xml_rsc);
                 }
             }
             EXPAND_CONSTRAINT_IDREF(id, rsc_1, rid);
 
         } else {
             /* get the first one */
             for (xml_rsc = pcmk__xe_first_child(set1); xml_rsc != NULL;
                  xml_rsc = pcmk__xe_next(xml_rsc)) {
 
                 if (pcmk__str_eq((const char *)xml_rsc->name, XML_TAG_RESOURCE_REF, pcmk__str_none)) {
                     EXPAND_CONSTRAINT_IDREF(id, rsc_1, ID(xml_rsc));
                     break;
                 }
             }
         }
     }
 
     if (crm_is_true(sequential_2)) {
         if (invert == FALSE) {
             /* get the first one */
             for (xml_rsc = pcmk__xe_first_child(set2); xml_rsc != NULL;
                  xml_rsc = pcmk__xe_next(xml_rsc)) {
 
                 if (pcmk__str_eq((const char *)xml_rsc->name, XML_TAG_RESOURCE_REF, pcmk__str_none)) {
                     EXPAND_CONSTRAINT_IDREF(id, rsc_2, ID(xml_rsc));
                     break;
                 }
             }
 
         } else {
             /* get the last one */
             const char *rid = NULL;
 
             for (xml_rsc = pcmk__xe_first_child(set2); xml_rsc != NULL;
                  xml_rsc = pcmk__xe_next(xml_rsc)) {
 
                 if (pcmk__str_eq((const char *)xml_rsc->name, XML_TAG_RESOURCE_REF, pcmk__str_none)) {
                     rid = ID(xml_rsc);
                 }
             }
             EXPAND_CONSTRAINT_IDREF(id, rsc_2, rid);
         }
     }
 
     if (rsc_1 != NULL && rsc_2 != NULL) {
         new_rsc_order(rsc_1, action_1, rsc_2, action_2, flags, data_set);
 
     } else if (rsc_1 != NULL) {
         for (xml_rsc = pcmk__xe_first_child(set2); xml_rsc != NULL;
              xml_rsc = pcmk__xe_next(xml_rsc)) {
 
             if (pcmk__str_eq((const char *)xml_rsc->name, XML_TAG_RESOURCE_REF, pcmk__str_none)) {
                 EXPAND_CONSTRAINT_IDREF(id, rsc_2, ID(xml_rsc));
                 new_rsc_order(rsc_1, action_1, rsc_2, action_2, flags, data_set);
             }
         }
 
     } else if (rsc_2 != NULL) {
         xmlNode *xml_rsc = NULL;
 
         for (xml_rsc = pcmk__xe_first_child(set1); xml_rsc != NULL;
              xml_rsc = pcmk__xe_next(xml_rsc)) {
 
             if (pcmk__str_eq((const char *)xml_rsc->name, XML_TAG_RESOURCE_REF, pcmk__str_none)) {
                 EXPAND_CONSTRAINT_IDREF(id, rsc_1, ID(xml_rsc));
                 new_rsc_order(rsc_1, action_1, rsc_2, action_2, flags, data_set);
             }
         }
 
     } else {
         for (xml_rsc = pcmk__xe_first_child(set1); xml_rsc != NULL;
              xml_rsc = pcmk__xe_next(xml_rsc)) {
 
             if (pcmk__str_eq((const char *)xml_rsc->name, XML_TAG_RESOURCE_REF, pcmk__str_none)) {
                 xmlNode *xml_rsc_2 = NULL;
 
                 EXPAND_CONSTRAINT_IDREF(id, rsc_1, ID(xml_rsc));
 
                 for (xml_rsc_2 = pcmk__xe_first_child(set2);
                      xml_rsc_2 != NULL;
                      xml_rsc_2 = pcmk__xe_next(xml_rsc_2)) {
 
                     if (pcmk__str_eq((const char *)xml_rsc_2->name, XML_TAG_RESOURCE_REF, pcmk__str_none)) {
                         EXPAND_CONSTRAINT_IDREF(id, rsc_2, ID(xml_rsc_2));
                         new_rsc_order(rsc_1, action_1, rsc_2, action_2, flags, data_set);
                     }
                 }
             }
         }
     }
 
     return TRUE;
 }
 
 static gboolean
 unpack_order_tags(xmlNode * xml_obj, xmlNode ** expanded_xml, pe_working_set_t * data_set)
 {
     const char *id = NULL;
     const char *id_first = NULL;
     const char *id_then = NULL;
     const char *action_first = NULL;
     const char *action_then = NULL;
 
     pe_resource_t *rsc_first = NULL;
     pe_resource_t *rsc_then = NULL;
     pe_tag_t *tag_first = NULL;
     pe_tag_t *tag_then = NULL;
 
     xmlNode *new_xml = NULL;
     xmlNode *rsc_set_first = NULL;
     xmlNode *rsc_set_then = NULL;
     gboolean any_sets = FALSE;
 
     *expanded_xml = NULL;
 
     CRM_CHECK(xml_obj != NULL, return FALSE);
 
     id = ID(xml_obj);
     if (id == NULL) {
         pcmk__config_err("Ignoring <%s> constraint without " XML_ATTR_ID,
                          crm_element_name(xml_obj));
         return FALSE;
     }
 
     /* Attempt to expand any template/tag references in possible resource sets. */
     expand_tags_in_sets(xml_obj, &new_xml, data_set);
     if (new_xml) {
         /* There are resource sets referencing templates/tags. Return with the expanded XML. */
         crm_log_xml_trace(new_xml, "Expanded rsc_order...");
         *expanded_xml = new_xml;
         return TRUE;
     }
 
     id_first = crm_element_value(xml_obj, XML_ORDER_ATTR_FIRST);
     id_then = crm_element_value(xml_obj, XML_ORDER_ATTR_THEN);
     if (id_first == NULL || id_then == NULL) {
         return TRUE;
     }
 
     if (valid_resource_or_tag(data_set, id_first, &rsc_first, &tag_first) == FALSE) {
         pcmk__config_err("Ignoring constraint '%s' because '%s' is not a "
                          "valid resource or tag", id, id_first);
         return FALSE;
     }
 
     if (valid_resource_or_tag(data_set, id_then, &rsc_then, &tag_then) == FALSE) {
         pcmk__config_err("Ignoring constraint '%s' because '%s' is not a "
                          "valid resource or tag", id, id_then);
         return FALSE;
     }
 
     if (rsc_first && rsc_then) {
         /* Neither side references any template/tag. */
         return TRUE;
     }
 
     action_first = crm_element_value(xml_obj, XML_ORDER_ATTR_FIRST_ACTION);
     action_then = crm_element_value(xml_obj, XML_ORDER_ATTR_THEN_ACTION);
 
     new_xml = copy_xml(xml_obj);
 
     /* Convert the template/tag reference in "first" into a resource_set under the order constraint. */
     if (tag_to_set(new_xml, &rsc_set_first, XML_ORDER_ATTR_FIRST, TRUE, data_set) == FALSE) {
         free_xml(new_xml);
         return FALSE;
     }
 
     if (rsc_set_first) {
         if (action_first) {
             /* A "first-action" is specified.
                Move it into the converted resource_set as an "action" attribute. */
             crm_xml_add(rsc_set_first, "action", action_first);
             xml_remove_prop(new_xml, XML_ORDER_ATTR_FIRST_ACTION);
         }
         any_sets = TRUE;
     }
 
     /* Convert the template/tag reference in "then" into a resource_set under the order constraint. */
     if (tag_to_set(new_xml, &rsc_set_then, XML_ORDER_ATTR_THEN, TRUE, data_set) == FALSE) {
         free_xml(new_xml);
         return FALSE;
     }
 
     if (rsc_set_then) {
         if (action_then) {
             /* A "then-action" is specified.
                Move it into the converted resource_set as an "action" attribute. */
             crm_xml_add(rsc_set_then, "action", action_then);
             xml_remove_prop(new_xml, XML_ORDER_ATTR_THEN_ACTION);
         }
         any_sets = TRUE;
     }
 
     if (any_sets) {
         crm_log_xml_trace(new_xml, "Expanded rsc_order...");
         *expanded_xml = new_xml;
     } else {
         free_xml(new_xml);
     }
 
     return TRUE;
 }
 
 gboolean
 unpack_rsc_order(xmlNode * xml_obj, pe_working_set_t * data_set)
 {
     gboolean any_sets = FALSE;
 
     pe_resource_t *rsc = NULL;
 
     /*
        pe_resource_t *last_rsc = NULL;
      */
 
     pe_action_t *set_end = NULL;
     pe_action_t *set_begin = NULL;
 
     pe_action_t *set_inv_end = NULL;
     pe_action_t *set_inv_begin = NULL;
 
     xmlNode *set = NULL;
     xmlNode *last = NULL;
 
     xmlNode *orig_xml = NULL;
     xmlNode *expanded_xml = NULL;
 
     /*
        pe_action_t *last_end = NULL;
        pe_action_t *last_begin = NULL;
        pe_action_t *last_inv_end = NULL;
        pe_action_t *last_inv_begin = NULL;
      */
 
     const char *id = crm_element_value(xml_obj, XML_ATTR_ID);
     const char *invert = crm_element_value(xml_obj, XML_CONS_ATTR_SYMMETRICAL);
     enum pe_order_kind kind = get_ordering_type(xml_obj);
 
     gboolean invert_bool = order_is_symmetrical(xml_obj, kind, NULL);
     gboolean rc = TRUE;
 
     rc = unpack_order_tags(xml_obj, &expanded_xml, data_set);
     if (expanded_xml) {
         orig_xml = xml_obj;
         xml_obj = expanded_xml;
 
     } else if (rc == FALSE) {
         return FALSE;
     }
 
     for (set = pcmk__xe_first_child(xml_obj); set != NULL;
          set = pcmk__xe_next(set)) {
 
         if (pcmk__str_eq((const char *)set->name, XML_CONS_TAG_RSC_SET, pcmk__str_none)) {
             any_sets = TRUE;
             set = expand_idref(set, data_set->input);
             if (unpack_order_set(set, kind, &rsc, &set_begin, &set_end,
                                  &set_inv_begin, &set_inv_end, invert, data_set) == FALSE) {
                 return FALSE;
 
                 /* Expand orders in order_rsc_sets() instead of via pseudo actions. */
                 /*
                    } else if(last) {
                    const char *set_action = crm_element_value(set, "action");
                    const char *last_action = crm_element_value(last, "action");
                    enum pe_ordering flags = get_flags(id, kind, last_action, set_action, FALSE);
 
                    if(!set_action) { set_action = RSC_START; }
                    if(!last_action) { last_action = RSC_START; }
 
                    if(rsc == NULL && last_rsc == NULL) {
                    order_actions(last_end, set_begin, flags);
                    } else {
                    custom_action_order(
                    last_rsc, null_or_opkey(last_rsc, last_action), last_end,
                    rsc, null_or_opkey(rsc, set_action), set_begin,
                    flags, data_set);
                    }
 
                    if(crm_is_true(invert)) {
                    set_action = invert_action(set_action);
                    last_action = invert_action(last_action);
 
                    flags = get_flags(id, kind, last_action, set_action, TRUE);
                    if(rsc == NULL && last_rsc == NULL) {
                    order_actions(last_inv_begin, set_inv_end, flags);
 
                    } else {
                    custom_action_order(
                    last_rsc, null_or_opkey(last_rsc, last_action), last_inv_begin,
                    rsc, null_or_opkey(rsc, set_action), set_inv_end,
                    flags, data_set);
                    }
                    }
                  */
 
             } else if (         /* never called -- Now call it for supporting clones in resource sets */
                           last) {
                 if (order_rsc_sets(id, last, set, kind, data_set, FALSE, invert_bool) == FALSE) {
                     return FALSE;
                 }
 
                 if (invert_bool
                     && order_rsc_sets(id, set, last, kind, data_set, TRUE, invert_bool) == FALSE) {
                     return FALSE;
                 }
 
             }
             last = set;
             /*
                last_rsc = rsc;
                last_end = set_end;
                last_begin = set_begin;
                last_inv_end = set_inv_end;
                last_inv_begin = set_inv_begin;
              */
         }
     }
 
     if (expanded_xml) {
         free_xml(expanded_xml);
         xml_obj = orig_xml;
     }
 
     if (any_sets == FALSE) {
         return unpack_simple_rsc_order(xml_obj, data_set);
     }
 
     return TRUE;
 }
 
 /*!
  * \internal
  * \brief Return the boolean influence corresponding to configuration
  *
  * \param[in] coloc_id     Colocation XML ID (for error logging)
  * \param[in] rsc          Resource involved in constraint (for default)
  * \param[in] influence_s  String value of influence option
  *
  * \return true if string evaluates true, false if string evaluates false,
  *         or value of resource's critical option if string is NULL or invalid
  */
 static bool
 unpack_influence(const char *coloc_id, const pe_resource_t *rsc,
                  const char *influence_s)
 {
     if (influence_s != NULL) {
         int influence_i = 0;
 
         if (crm_str_to_boolean(influence_s, &influence_i) < 0) {
             pcmk__config_err("Constraint '%s' has invalid value for "
                              XML_COLOC_ATTR_INFLUENCE " (using default)",
                              coloc_id);
         } else {
             return (influence_i == TRUE);
         }
     }
     return pcmk_is_set(rsc->flags, pe_rsc_critical);
 }
 
 static gboolean
 unpack_colocation_set(xmlNode *set, int score, const char *coloc_id,
                       const char *influence_s, pe_working_set_t *data_set)
 {
     xmlNode *xml_rsc = NULL;
     pe_resource_t *with = NULL;
     pe_resource_t *resource = NULL;
     const char *set_id = ID(set);
     const char *role = crm_element_value(set, "role");
     const char *sequential = crm_element_value(set, "sequential");
     const char *ordering = crm_element_value(set, "ordering");
     int local_score = score;
 
     const char *score_s = crm_element_value(set, XML_RULE_ATTR_SCORE);
 
     if (score_s) {
         local_score = char2score(score_s);
     }
     if (local_score == 0) {
         crm_trace("Ignoring colocation '%s' for set '%s' because score is 0",
                   coloc_id, set_id);
         return TRUE;
     }
 
     if(ordering == NULL) {
         ordering = "group";
     }
 
     if (sequential != NULL && crm_is_true(sequential) == FALSE) {
         return TRUE;
 
     } else if ((local_score > 0)
                && pcmk__str_eq(ordering, "group", pcmk__str_casei)) {
         for (xml_rsc = pcmk__xe_first_child(set); xml_rsc != NULL;
              xml_rsc = pcmk__xe_next(xml_rsc)) {
 
             if (pcmk__str_eq((const char *)xml_rsc->name, XML_TAG_RESOURCE_REF, pcmk__str_none)) {
                 EXPAND_CONSTRAINT_IDREF(set_id, resource, ID(xml_rsc));
                 if (with != NULL) {
                     pe_rsc_trace(resource, "Colocating %s with %s", resource->id, with->id);
                     pcmk__new_colocation(set_id, NULL, local_score, resource,
                                          with, role, role,
                                          unpack_influence(coloc_id, resource,
                                                           influence_s),
                                          data_set);
                 }
 
                 with = resource;
             }
         }
     } else if (local_score > 0) {
         pe_resource_t *last = NULL;
         for (xml_rsc = pcmk__xe_first_child(set); xml_rsc != NULL;
              xml_rsc = pcmk__xe_next(xml_rsc)) {
 
             if (pcmk__str_eq((const char *)xml_rsc->name, XML_TAG_RESOURCE_REF, pcmk__str_none)) {
                 EXPAND_CONSTRAINT_IDREF(set_id, resource, ID(xml_rsc));
                 if (last != NULL) {
                     pe_rsc_trace(resource, "Colocating %s with %s", last->id, resource->id);
                     pcmk__new_colocation(set_id, NULL, local_score, last,
                                          resource, role, role,
                                          unpack_influence(coloc_id, last,
                                                           influence_s),
                                          data_set);
                 }
 
                 last = resource;
             }
         }
 
     } else {
         /* Anti-colocating with every prior resource is
          * the only way to ensure the intuitive result
          * (i.e. that no one in the set can run with anyone else in the set)
          */
 
         for (xml_rsc = pcmk__xe_first_child(set); xml_rsc != NULL;
              xml_rsc = pcmk__xe_next(xml_rsc)) {
 
             if (pcmk__str_eq((const char *)xml_rsc->name, XML_TAG_RESOURCE_REF, pcmk__str_none)) {
                 xmlNode *xml_rsc_with = NULL;
                 bool influence = true;
 
                 EXPAND_CONSTRAINT_IDREF(set_id, resource, ID(xml_rsc));
                 influence = unpack_influence(coloc_id, resource, influence_s);
 
                 for (xml_rsc_with = pcmk__xe_first_child(set);
                      xml_rsc_with != NULL;
                      xml_rsc_with = pcmk__xe_next(xml_rsc_with)) {
 
                     if (pcmk__str_eq((const char *)xml_rsc_with->name, XML_TAG_RESOURCE_REF, pcmk__str_none)) {
                         if (pcmk__str_eq(resource->id, ID(xml_rsc_with), pcmk__str_casei)) {
                             break;
                         }
                         EXPAND_CONSTRAINT_IDREF(set_id, with, ID(xml_rsc_with));
                         pe_rsc_trace(resource, "Anti-Colocating %s with %s", resource->id,
                                      with->id);
                         pcmk__new_colocation(set_id, NULL, local_score,
                                              resource, with, role, role,
                                              influence, data_set);
                     }
                 }
             }
         }
     }
 
     return TRUE;
 }
 
 static gboolean
 colocate_rsc_sets(const char *id, xmlNode * set1, xmlNode * set2, int score,
                   const char *influence_s, pe_working_set_t *data_set)
 {
     xmlNode *xml_rsc = NULL;
     pe_resource_t *rsc_1 = NULL;
     pe_resource_t *rsc_2 = NULL;
 
     const char *role_1 = crm_element_value(set1, "role");
     const char *role_2 = crm_element_value(set2, "role");
 
     const char *sequential_1 = crm_element_value(set1, "sequential");
     const char *sequential_2 = crm_element_value(set2, "sequential");
 
     if (score == 0) {
         crm_trace("Ignoring colocation '%s' between sets because score is 0",
                   id);
         return TRUE;
     }
     if (sequential_1 == NULL || crm_is_true(sequential_1)) {
         /* get the first one */
         for (xml_rsc = pcmk__xe_first_child(set1); xml_rsc != NULL;
              xml_rsc = pcmk__xe_next(xml_rsc)) {
 
             if (pcmk__str_eq((const char *)xml_rsc->name, XML_TAG_RESOURCE_REF, pcmk__str_none)) {
                 EXPAND_CONSTRAINT_IDREF(id, rsc_1, ID(xml_rsc));
                 break;
             }
         }
     }
 
     if (sequential_2 == NULL || crm_is_true(sequential_2)) {
         /* get the last one */
         const char *rid = NULL;
 
         for (xml_rsc = pcmk__xe_first_child(set2); xml_rsc != NULL;
              xml_rsc = pcmk__xe_next(xml_rsc)) {
 
             if (pcmk__str_eq((const char *)xml_rsc->name, XML_TAG_RESOURCE_REF, pcmk__str_none)) {
                 rid = ID(xml_rsc);
             }
         }
         EXPAND_CONSTRAINT_IDREF(id, rsc_2, rid);
     }
 
     if (rsc_1 != NULL && rsc_2 != NULL) {
         pcmk__new_colocation(id, NULL, score, rsc_1, rsc_2, role_1, role_2,
                              unpack_influence(id, rsc_1, influence_s),
                              data_set);
 
     } else if (rsc_1 != NULL) {
         bool influence = unpack_influence(id, rsc_1, influence_s);
 
         for (xml_rsc = pcmk__xe_first_child(set2); xml_rsc != NULL;
              xml_rsc = pcmk__xe_next(xml_rsc)) {
 
             if (pcmk__str_eq((const char *)xml_rsc->name, XML_TAG_RESOURCE_REF, pcmk__str_none)) {
                 EXPAND_CONSTRAINT_IDREF(id, rsc_2, ID(xml_rsc));
                 pcmk__new_colocation(id, NULL, score, rsc_1, rsc_2, role_1,
                                      role_2, influence, data_set);
             }
         }
 
     } else if (rsc_2 != NULL) {
         for (xml_rsc = pcmk__xe_first_child(set1); xml_rsc != NULL;
              xml_rsc = pcmk__xe_next(xml_rsc)) {
 
             if (pcmk__str_eq((const char *)xml_rsc->name, XML_TAG_RESOURCE_REF, pcmk__str_none)) {
                 EXPAND_CONSTRAINT_IDREF(id, rsc_1, ID(xml_rsc));
                 pcmk__new_colocation(id, NULL, score, rsc_1, rsc_2, role_1,
                                      role_2,
                                      unpack_influence(id, rsc_1, influence_s),
                                      data_set);
             }
         }
 
     } else {
         for (xml_rsc = pcmk__xe_first_child(set1); xml_rsc != NULL;
              xml_rsc = pcmk__xe_next(xml_rsc)) {
 
             if (pcmk__str_eq((const char *)xml_rsc->name, XML_TAG_RESOURCE_REF, pcmk__str_none)) {
                 xmlNode *xml_rsc_2 = NULL;
                 bool influence = true;
 
                 EXPAND_CONSTRAINT_IDREF(id, rsc_1, ID(xml_rsc));
                 influence = unpack_influence(id, rsc_1, influence_s);
 
                 for (xml_rsc_2 = pcmk__xe_first_child(set2);
                      xml_rsc_2 != NULL;
                      xml_rsc_2 = pcmk__xe_next(xml_rsc_2)) {
 
                     if (pcmk__str_eq((const char *)xml_rsc_2->name, XML_TAG_RESOURCE_REF, pcmk__str_none)) {
                         EXPAND_CONSTRAINT_IDREF(id, rsc_2, ID(xml_rsc_2));
                         pcmk__new_colocation(id, NULL, score, rsc_1, rsc_2,
                                              role_1, role_2, influence,
                                              data_set);
                     }
                 }
             }
         }
     }
 
     return TRUE;
 }
 
 static void
 unpack_simple_colocation(xmlNode *xml_obj, const char *id,
                          const char *influence_s, pe_working_set_t *data_set)
 {
     int score_i = 0;
 
     const char *score = crm_element_value(xml_obj, XML_RULE_ATTR_SCORE);
     const char *id_lh = crm_element_value(xml_obj, XML_COLOC_ATTR_SOURCE);
     const char *id_rh = crm_element_value(xml_obj, XML_COLOC_ATTR_TARGET);
     const char *state_lh = crm_element_value(xml_obj, XML_COLOC_ATTR_SOURCE_ROLE);
     const char *state_rh = crm_element_value(xml_obj, XML_COLOC_ATTR_TARGET_ROLE);
     const char *attr = crm_element_value(xml_obj, XML_COLOC_ATTR_NODE_ATTR);
     const char *symmetrical = crm_element_value(xml_obj, XML_CONS_ATTR_SYMMETRICAL);
 
     // experimental syntax from pacemaker-next (unlikely to be adopted as-is)
     const char *instance_lh = crm_element_value(xml_obj, XML_COLOC_ATTR_SOURCE_INSTANCE);
     const char *instance_rh = crm_element_value(xml_obj, XML_COLOC_ATTR_TARGET_INSTANCE);
 
     pe_resource_t *rsc_lh = pe_find_constraint_resource(data_set->resources, id_lh);
     pe_resource_t *rsc_rh = pe_find_constraint_resource(data_set->resources, id_rh);
 
     if (rsc_lh == NULL) {
         pcmk__config_err("Ignoring constraint '%s' because resource '%s' "
                          "does not exist", id, id_lh);
         return;
 
     } else if (rsc_rh == NULL) {
         pcmk__config_err("Ignoring constraint '%s' because resource '%s' "
                          "does not exist", id, id_rh);
         return;
 
     } else if (instance_lh && pe_rsc_is_clone(rsc_lh) == FALSE) {
         pcmk__config_err("Ignoring constraint '%s' because resource '%s' "
                          "is not a clone but instance '%s' was requested",
                          id, id_lh, instance_lh);
         return;
 
     } else if (instance_rh && pe_rsc_is_clone(rsc_rh) == FALSE) {
         pcmk__config_err("Ignoring constraint '%s' because resource '%s' "
                          "is not a clone but instance '%s' was requested",
                          id, id_rh, instance_rh);
         return;
     }
 
     if (instance_lh) {
         rsc_lh = find_clone_instance(rsc_lh, instance_lh, data_set);
         if (rsc_lh == NULL) {
             pcmk__config_warn("Ignoring constraint '%s' because resource '%s' "
                               "does not have an instance '%s'",
                               id, id_lh, instance_lh);
             return;
         }
     }
 
     if (instance_rh) {
         rsc_rh = find_clone_instance(rsc_rh, instance_rh, data_set);
         if (rsc_rh == NULL) {
             pcmk__config_warn("Ignoring constraint '%s' because resource '%s' "
                               "does not have an instance '%s'",
                               "'%s'", id, id_rh, instance_rh);
             return;
         }
     }
 
     if (crm_is_true(symmetrical)) {
         pcmk__config_warn("The colocation constraint '"
                           XML_CONS_ATTR_SYMMETRICAL
                           "' attribute has been removed");
     }
 
     if (score) {
         score_i = char2score(score);
     }
 
     pcmk__new_colocation(id, attr, score_i, rsc_lh, rsc_rh, state_lh, state_rh,
                          unpack_influence(id, rsc_lh, influence_s), data_set);
 }
 
 static gboolean
 unpack_colocation_tags(xmlNode * xml_obj, xmlNode ** expanded_xml, pe_working_set_t * data_set)
 {
     const char *id = NULL;
     const char *id_lh = NULL;
     const char *id_rh = NULL;
     const char *state_lh = NULL;
     const char *state_rh = NULL;
 
     pe_resource_t *rsc_lh = NULL;
     pe_resource_t *rsc_rh = NULL;
 
     pe_tag_t *tag_lh = NULL;
     pe_tag_t *tag_rh = NULL;
 
     xmlNode *new_xml = NULL;
     xmlNode *rsc_set_lh = NULL;
     xmlNode *rsc_set_rh = NULL;
     gboolean any_sets = FALSE;
 
     *expanded_xml = NULL;
 
     CRM_CHECK(xml_obj != NULL, return FALSE);
 
     id = ID(xml_obj);
     if (id == NULL) {
         pcmk__config_err("Ignoring <%s> constraint without " XML_ATTR_ID,
                          crm_element_name(xml_obj));
         return FALSE;
     }
 
     /* Attempt to expand any template/tag references in possible resource sets. */
     expand_tags_in_sets(xml_obj, &new_xml, data_set);
     if (new_xml) {
         /* There are resource sets referencing templates/tags. Return with the expanded XML. */
         crm_log_xml_trace(new_xml, "Expanded rsc_colocation...");
         *expanded_xml = new_xml;
         return TRUE;
     }
 
     id_lh = crm_element_value(xml_obj, XML_COLOC_ATTR_SOURCE);
     id_rh = crm_element_value(xml_obj, XML_COLOC_ATTR_TARGET);
     if (id_lh == NULL || id_rh == NULL) {
         return TRUE;
     }
 
     if (valid_resource_or_tag(data_set, id_lh, &rsc_lh, &tag_lh) == FALSE) {
         pcmk__config_err("Ignoring constraint '%s' because '%s' is not a "
                          "valid resource or tag", id, id_lh);
         return FALSE;
     }
 
     if (valid_resource_or_tag(data_set, id_rh, &rsc_rh, &tag_rh) == FALSE) {
         pcmk__config_err("Ignoring constraint '%s' because '%s' is not a "
                          "valid resource or tag", id, id_rh);
         return FALSE;
     }
 
     if (rsc_lh && rsc_rh) {
         /* Neither side references any template/tag. */
         return TRUE;
     }
 
     if (tag_lh && tag_rh) {
         /* A colocation constraint between two templates/tags makes no sense. */
         pcmk__config_err("Ignoring constraint '%s' because two templates or "
                          "tags cannot be colocated", id);
         return FALSE;
     }
 
     state_lh = crm_element_value(xml_obj, XML_COLOC_ATTR_SOURCE_ROLE);
     state_rh = crm_element_value(xml_obj, XML_COLOC_ATTR_TARGET_ROLE);
 
     new_xml = copy_xml(xml_obj);
 
     /* Convert the template/tag reference in "rsc" into a resource_set under the colocation constraint. */
     if (tag_to_set(new_xml, &rsc_set_lh, XML_COLOC_ATTR_SOURCE, TRUE, data_set) == FALSE) {
         free_xml(new_xml);
         return FALSE;
     }
 
     if (rsc_set_lh) {
         if (state_lh) {
             /* A "rsc-role" is specified.
                Move it into the converted resource_set as a "role"" attribute. */
             crm_xml_add(rsc_set_lh, "role", state_lh);
             xml_remove_prop(new_xml, XML_COLOC_ATTR_SOURCE_ROLE);
         }
         any_sets = TRUE;
     }
 
     /* Convert the template/tag reference in "with-rsc" into a resource_set under the colocation constraint. */
     if (tag_to_set(new_xml, &rsc_set_rh, XML_COLOC_ATTR_TARGET, TRUE, data_set) == FALSE) {
         free_xml(new_xml);
         return FALSE;
     }
 
     if (rsc_set_rh) {
         if (state_rh) {
             /* A "with-rsc-role" is specified.
                Move it into the converted resource_set as a "role"" attribute. */
             crm_xml_add(rsc_set_rh, "role", state_rh);
             xml_remove_prop(new_xml, XML_COLOC_ATTR_TARGET_ROLE);
         }
         any_sets = TRUE;
     }
 
     if (any_sets) {
         crm_log_xml_trace(new_xml, "Expanded rsc_colocation...");
         *expanded_xml = new_xml;
     } else {
         free_xml(new_xml);
     }
 
     return TRUE;
 }
 
 static void
 unpack_rsc_colocation(xmlNode *xml_obj, pe_working_set_t *data_set)
 {
     int score_i = 0;
     xmlNode *set = NULL;
     xmlNode *last = NULL;
     gboolean any_sets = FALSE;
 
     xmlNode *orig_xml = NULL;
     xmlNode *expanded_xml = NULL;
 
     const char *id = crm_element_value(xml_obj, XML_ATTR_ID);
     const char *score = crm_element_value(xml_obj, XML_RULE_ATTR_SCORE);
     const char *influence_s = crm_element_value(xml_obj,
                                                 XML_COLOC_ATTR_INFLUENCE);
 
     if (score) {
         score_i = char2score(score);
     }
 
     if (!unpack_colocation_tags(xml_obj, &expanded_xml, data_set)) {
         return;
     }
     if (expanded_xml) {
         orig_xml = xml_obj;
         xml_obj = expanded_xml;
     }
 
     for (set = pcmk__xe_first_child(xml_obj); set != NULL;
          set = pcmk__xe_next(set)) {
 
         if (pcmk__str_eq((const char *)set->name, XML_CONS_TAG_RSC_SET, pcmk__str_none)) {
             any_sets = TRUE;
             set = expand_idref(set, data_set->input);
             if (!unpack_colocation_set(set, score_i, id, influence_s,
                                        data_set)) {
                 return;
             }
             if ((last != NULL) && !colocate_rsc_sets(id, last, set, score_i,
                                                      influence_s, data_set)) {
                 return;
             }
             last = set;
         }
     }
 
     if (expanded_xml) {
         free_xml(expanded_xml);
         xml_obj = orig_xml;
     }
 
     if (!any_sets) {
         unpack_simple_colocation(xml_obj, id, influence_s, data_set);
     }
 }
 
 gboolean
 rsc_ticket_new(const char *id, pe_resource_t * rsc_lh, pe_ticket_t * ticket,
                const char *state_lh, const char *loss_policy, pe_working_set_t * data_set)
 {
     rsc_ticket_t *new_rsc_ticket = NULL;
 
     if (rsc_lh == NULL) {
         pcmk__config_err("Ignoring ticket '%s' because resource "
                          "does not exist", id);
         return FALSE;
     }
 
     new_rsc_ticket = calloc(1, sizeof(rsc_ticket_t));
     if (new_rsc_ticket == NULL) {
         return FALSE;
     }
 
     if (pcmk__str_eq(state_lh, RSC_ROLE_STARTED_S, pcmk__str_null_matches | pcmk__str_casei)) {
         state_lh = RSC_ROLE_UNKNOWN_S;
     }
 
     new_rsc_ticket->id = id;
     new_rsc_ticket->ticket = ticket;
     new_rsc_ticket->rsc_lh = rsc_lh;
     new_rsc_ticket->role_lh = text2role(state_lh);
 
     if (pcmk__str_eq(loss_policy, "fence", pcmk__str_casei)) {
         if (pcmk_is_set(data_set->flags, pe_flag_stonith_enabled)) {
             new_rsc_ticket->loss_policy = loss_ticket_fence;
         } else {
             pcmk__config_err("Resetting '" XML_TICKET_ATTR_LOSS_POLICY
                              "' for ticket '%s' to 'stop' "
                              "because fencing is not configured", ticket->id);
             loss_policy = "stop";
         }
     }
 
     if (new_rsc_ticket->loss_policy == loss_ticket_fence) {
         crm_debug("On loss of ticket '%s': Fence the nodes running %s (%s)",
                   new_rsc_ticket->ticket->id, new_rsc_ticket->rsc_lh->id,
                   role2text(new_rsc_ticket->role_lh));
 
     } else if (pcmk__str_eq(loss_policy, "freeze", pcmk__str_casei)) {
         crm_debug("On loss of ticket '%s': Freeze %s (%s)",
                   new_rsc_ticket->ticket->id, new_rsc_ticket->rsc_lh->id,
                   role2text(new_rsc_ticket->role_lh));
         new_rsc_ticket->loss_policy = loss_ticket_freeze;
 
     } else if (pcmk__str_eq(loss_policy, "demote", pcmk__str_casei)) {
         crm_debug("On loss of ticket '%s': Demote %s (%s)",
                   new_rsc_ticket->ticket->id, new_rsc_ticket->rsc_lh->id,
                   role2text(new_rsc_ticket->role_lh));
         new_rsc_ticket->loss_policy = loss_ticket_demote;
 
     } else if (pcmk__str_eq(loss_policy, "stop", pcmk__str_casei)) {
         crm_debug("On loss of ticket '%s': Stop %s (%s)",
                   new_rsc_ticket->ticket->id, new_rsc_ticket->rsc_lh->id,
                   role2text(new_rsc_ticket->role_lh));
         new_rsc_ticket->loss_policy = loss_ticket_stop;
 
     } else {
         if (new_rsc_ticket->role_lh == RSC_ROLE_PROMOTED) {
             crm_debug("On loss of ticket '%s': Default to demote %s (%s)",
                       new_rsc_ticket->ticket->id, new_rsc_ticket->rsc_lh->id,
                       role2text(new_rsc_ticket->role_lh));
             new_rsc_ticket->loss_policy = loss_ticket_demote;
 
         } else {
             crm_debug("On loss of ticket '%s': Default to stop %s (%s)",
                       new_rsc_ticket->ticket->id, new_rsc_ticket->rsc_lh->id,
                       role2text(new_rsc_ticket->role_lh));
             new_rsc_ticket->loss_policy = loss_ticket_stop;
         }
     }
 
     pe_rsc_trace(rsc_lh, "%s (%s) ==> %s", rsc_lh->id, role2text(new_rsc_ticket->role_lh),
                  ticket->id);
 
     rsc_lh->rsc_tickets = g_list_append(rsc_lh->rsc_tickets, new_rsc_ticket);
 
     data_set->ticket_constraints = g_list_append(data_set->ticket_constraints, new_rsc_ticket);
 
     if (new_rsc_ticket->ticket->granted == FALSE || new_rsc_ticket->ticket->standby) {
         rsc_ticket_constraint(rsc_lh, new_rsc_ticket, data_set);
     }
 
     return TRUE;
 }
 
 static gboolean
 unpack_rsc_ticket_set(xmlNode * set, pe_ticket_t * ticket, const char *loss_policy,
                       pe_working_set_t * data_set)
 {
     xmlNode *xml_rsc = NULL;
     pe_resource_t *resource = NULL;
     const char *set_id = NULL;
     const char *role = NULL;
 
     CRM_CHECK(set != NULL, return FALSE);
     CRM_CHECK(ticket != NULL, return FALSE);
 
     set_id = ID(set);
     if (set_id == NULL) {
         pcmk__config_err("Ignoring <" XML_CONS_TAG_RSC_SET "> without "
                          XML_ATTR_ID);
         return FALSE;
     }
 
     role = crm_element_value(set, "role");
 
     for (xml_rsc = first_named_child(set, XML_TAG_RESOURCE_REF);
          xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {
 
         EXPAND_CONSTRAINT_IDREF(set_id, resource, ID(xml_rsc));
         pe_rsc_trace(resource, "Resource '%s' depends on ticket '%s'",
                      resource->id, ticket->id);
         rsc_ticket_new(set_id, resource, ticket, role, loss_policy, data_set);
     }
 
     return TRUE;
 }
 
 static gboolean
 unpack_simple_rsc_ticket(xmlNode * xml_obj, pe_working_set_t * data_set)
 {
     const char *id = NULL;
     const char *ticket_str = crm_element_value(xml_obj, XML_TICKET_ATTR_TICKET);
     const char *loss_policy = crm_element_value(xml_obj, XML_TICKET_ATTR_LOSS_POLICY);
 
     pe_ticket_t *ticket = NULL;
 
     const char *id_lh = crm_element_value(xml_obj, XML_COLOC_ATTR_SOURCE);
     const char *state_lh = crm_element_value(xml_obj, XML_COLOC_ATTR_SOURCE_ROLE);
 
     // experimental syntax from pacemaker-next (unlikely to be adopted as-is)
     const char *instance_lh = crm_element_value(xml_obj, XML_COLOC_ATTR_SOURCE_INSTANCE);
 
     pe_resource_t *rsc_lh = NULL;
 
     CRM_CHECK(xml_obj != NULL, return FALSE);
 
     id = ID(xml_obj);
     if (id == NULL) {
         pcmk__config_err("Ignoring <%s> constraint without " XML_ATTR_ID,
                          crm_element_name(xml_obj));
         return FALSE;
     }
 
     if (ticket_str == NULL) {
         pcmk__config_err("Ignoring constraint '%s' without ticket specified",
                          id);
         return FALSE;
     } else {
         ticket = g_hash_table_lookup(data_set->tickets, ticket_str);
     }
 
     if (ticket == NULL) {
         pcmk__config_err("Ignoring constraint '%s' because ticket '%s' "
                          "does not exist", id, ticket_str);
         return FALSE;
     }
 
     if (id_lh == NULL) {
         pcmk__config_err("Ignoring constraint '%s' without resource", id);
         return FALSE;
     } else {
         rsc_lh = pe_find_constraint_resource(data_set->resources, id_lh);
     }
 
     if (rsc_lh == NULL) {
         pcmk__config_err("Ignoring constraint '%s' because resource '%s' "
                          "does not exist", id, id_lh);
         return FALSE;
 
     } else if (instance_lh && pe_rsc_is_clone(rsc_lh) == FALSE) {
         pcmk__config_err("Ignoring constraint '%s' because resource '%s' "
                          "is not a clone but instance '%s' was requested",
                          id, id_lh, instance_lh);
         return FALSE;
     }
 
     if (instance_lh) {
         rsc_lh = find_clone_instance(rsc_lh, instance_lh, data_set);
         if (rsc_lh == NULL) {
             pcmk__config_warn("Ignoring constraint '%s' because resource '%s' "
                               "does not have an instance '%s'",
                               "'%s'", id, id_lh, instance_lh);
             return FALSE;
         }
     }
 
     rsc_ticket_new(id, rsc_lh, ticket, state_lh, loss_policy, data_set);
     return TRUE;
 }
 
 static gboolean
 unpack_rsc_ticket_tags(xmlNode * xml_obj, xmlNode ** expanded_xml, pe_working_set_t * data_set)
 {
     const char *id = NULL;
     const char *id_lh = NULL;
     const char *state_lh = NULL;
 
     pe_resource_t *rsc_lh = NULL;
     pe_tag_t *tag_lh = NULL;
 
     xmlNode *new_xml = NULL;
     xmlNode *rsc_set_lh = NULL;
     gboolean any_sets = FALSE;
 
     *expanded_xml = NULL;
 
     CRM_CHECK(xml_obj != NULL, return FALSE);
 
     id = ID(xml_obj);
     if (id == NULL) {
         pcmk__config_err("Ignoring <%s> constraint without " XML_ATTR_ID,
                          crm_element_name(xml_obj));
         return FALSE;
     }
 
     /* Attempt to expand any template/tag references in possible resource sets. */
     expand_tags_in_sets(xml_obj, &new_xml, data_set);
     if (new_xml) {
         /* There are resource sets referencing templates/tags. Return with the expanded XML. */
         crm_log_xml_trace(new_xml, "Expanded rsc_ticket...");
         *expanded_xml = new_xml;
         return TRUE;
     }
 
     id_lh = crm_element_value(xml_obj, XML_COLOC_ATTR_SOURCE);
     if (id_lh == NULL) {
         return TRUE;
     }
 
     if (valid_resource_or_tag(data_set, id_lh, &rsc_lh, &tag_lh) == FALSE) {
         pcmk__config_err("Ignoring constraint '%s' because '%s' is not a "
                          "valid resource or tag", id, id_lh);
         return FALSE;
 
     } else if (rsc_lh) {
         /* No template/tag is referenced. */
         return TRUE;
     }
 
     state_lh = crm_element_value(xml_obj, XML_COLOC_ATTR_SOURCE_ROLE);
 
     new_xml = copy_xml(xml_obj);
 
     /* Convert the template/tag reference in "rsc" into a resource_set under the rsc_ticket constraint. */
     if (tag_to_set(new_xml, &rsc_set_lh, XML_COLOC_ATTR_SOURCE, FALSE, data_set) == FALSE) {
         free_xml(new_xml);
         return FALSE;
     }
 
     if (rsc_set_lh) {
         if (state_lh) {
             /* A "rsc-role" is specified.
                Move it into the converted resource_set as a "role"" attribute. */
             crm_xml_add(rsc_set_lh, "role", state_lh);
             xml_remove_prop(new_xml, XML_COLOC_ATTR_SOURCE_ROLE);
         }
         any_sets = TRUE;
     }
 
     if (any_sets) {
         crm_log_xml_trace(new_xml, "Expanded rsc_ticket...");
         *expanded_xml = new_xml;
     } else {
         free_xml(new_xml);
     }
 
     return TRUE;
 }
 
 gboolean
 unpack_rsc_ticket(xmlNode * xml_obj, pe_working_set_t * data_set)
 {
     xmlNode *set = NULL;
     gboolean any_sets = FALSE;
 
     const char *id = NULL;
     const char *ticket_str = crm_element_value(xml_obj, XML_TICKET_ATTR_TICKET);
     const char *loss_policy = crm_element_value(xml_obj, XML_TICKET_ATTR_LOSS_POLICY);
 
     pe_ticket_t *ticket = NULL;
 
     xmlNode *orig_xml = NULL;
     xmlNode *expanded_xml = NULL;
 
     gboolean rc = TRUE;
 
     CRM_CHECK(xml_obj != NULL, return FALSE);
 
     id = ID(xml_obj);
     if (id == NULL) {
         pcmk__config_err("Ignoring <%s> constraint without " XML_ATTR_ID,
                          crm_element_name(xml_obj));
         return FALSE;
     }
 
     if (data_set->tickets == NULL) {
         data_set->tickets = pcmk__strkey_table(free, destroy_ticket);
     }
 
     if (ticket_str == NULL) {
         pcmk__config_err("Ignoring constraint '%s' without ticket", id);
         return FALSE;
     } else {
         ticket = g_hash_table_lookup(data_set->tickets, ticket_str);
     }
 
     if (ticket == NULL) {
         ticket = ticket_new(ticket_str, data_set);
         if (ticket == NULL) {
             return FALSE;
         }
     }
 
     rc = unpack_rsc_ticket_tags(xml_obj, &expanded_xml, data_set);
     if (expanded_xml) {
         orig_xml = xml_obj;
         xml_obj = expanded_xml;
 
     } else if (rc == FALSE) {
         return FALSE;
     }
 
     for (set = pcmk__xe_first_child(xml_obj); set != NULL;
          set = pcmk__xe_next(set)) {
 
         if (pcmk__str_eq((const char *)set->name, XML_CONS_TAG_RSC_SET, pcmk__str_none)) {
             any_sets = TRUE;
             set = expand_idref(set, data_set->input);
             if (unpack_rsc_ticket_set(set, ticket, loss_policy, data_set) == FALSE) {
                 return FALSE;
             }
         }
     }
 
     if (expanded_xml) {
         free_xml(expanded_xml);
         xml_obj = orig_xml;
     }
 
     if (any_sets == FALSE) {
         return unpack_simple_rsc_ticket(xml_obj, data_set);
     }
 
     return TRUE;
 }
diff --git a/lib/pacemaker/pcmk_sched_graph.c b/lib/pacemaker/pcmk_sched_graph.c
index 1c15556090..e024113f8e 100644
--- a/lib/pacemaker/pcmk_sched_graph.c
+++ b/lib/pacemaker/pcmk_sched_graph.c
@@ -1,1902 +1,1907 @@
 /*
  * Copyright 2004-2021 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU General Public License version 2
  * or later (GPLv2+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #include <sys/param.h>
 #include <crm/crm.h>
 #include <crm/cib.h>
 #include <crm/msg_xml.h>
 #include <crm/common/xml.h>
 
 #include <glib.h>
 
 #include <pacemaker-internal.h>
 
 void update_colo_start_chain(pe_action_t *action, pe_working_set_t *data_set);
 gboolean rsc_update_action(pe_action_t * first, pe_action_t * then, enum pe_ordering type);
 
 static enum pe_action_flags
 get_action_flags(pe_action_t * action, pe_node_t * node)
 {
     enum pe_action_flags flags = action->flags;
 
     if (action->rsc) {
         flags = action->rsc->cmds->action_flags(action, NULL);
 
         if (pe_rsc_is_clone(action->rsc) && node) {
 
             /* We only care about activity on $node */
             enum pe_action_flags clone_flags = action->rsc->cmds->action_flags(action, node);
 
             /* Go to great lengths to ensure the correct value for pe_action_runnable...
              *
              * If we are a clone, then for _ordering_ constraints, it's only relevant
              * if we are runnable _anywhere_.
              *
              * This only applies to _runnable_ though, and only for ordering constraints.
              * If this function is ever used during colocation, then we'll need additional logic
              *
              * Not very satisfying, but it's logical and appears to work well.
              */
             if (!pcmk_is_set(clone_flags, pe_action_runnable)
                 && pcmk_is_set(flags, pe_action_runnable)) {
 
                 pe__set_raw_action_flags(clone_flags, action->rsc->id,
                                          pe_action_runnable);
             }
             flags = clone_flags;
         }
     }
     return flags;
 }
 
 static char *
 convert_non_atomic_uuid(char *old_uuid, pe_resource_t * rsc, gboolean allow_notify,
                         gboolean free_original)
 {
     guint interval_ms = 0;
     char *uuid = NULL;
     char *rid = NULL;
     char *raw_task = NULL;
     int task = no_action;
 
     CRM_ASSERT(rsc);
     pe_rsc_trace(rsc, "Processing %s", old_uuid);
     if (old_uuid == NULL) {
         return NULL;
 
     } else if (strstr(old_uuid, "notify") != NULL) {
         goto done;              /* no conversion */
 
     } else if (rsc->variant < pe_group) {
         goto done;              /* no conversion */
     }
 
     CRM_ASSERT(parse_op_key(old_uuid, &rid, &raw_task, &interval_ms));
     if (interval_ms > 0) {
         goto done;              /* no conversion */
     }
 
     task = text2task(raw_task);
     switch (task) {
         case stop_rsc:
         case start_rsc:
         case action_notify:
         case action_promote:
         case action_demote:
             break;
         case stopped_rsc:
         case started_rsc:
         case action_notified:
         case action_promoted:
         case action_demoted:
             task--;
             break;
         case monitor_rsc:
         case shutdown_crm:
         case stonith_node:
             task = no_action;
             break;
         default:
             crm_err("Unknown action: %s", raw_task);
             task = no_action;
             break;
     }
 
     if (task != no_action) {
         if (pcmk_is_set(rsc->flags, pe_rsc_notify) && allow_notify) {
             uuid = pcmk__notify_key(rid, "confirmed-post", task2text(task + 1));
 
         } else {
             uuid = pcmk__op_key(rid, task2text(task + 1), 0);
         }
         pe_rsc_trace(rsc, "Converted %s -> %s", old_uuid, uuid);
     }
 
   done:
     if (uuid == NULL) {
         uuid = strdup(old_uuid);
     }
 
     if (free_original) {
         free(old_uuid);
     }
 
     free(raw_task);
     free(rid);
     return uuid;
 }
 
 static pe_action_t *
 rsc_expand_action(pe_action_t * action)
 {
     gboolean notify = FALSE;
     pe_action_t *result = action;
     pe_resource_t *rsc = action->rsc;
 
     if (rsc == NULL) {
         return action;
     }
 
     if ((rsc->parent == NULL)
         || (pe_rsc_is_clone(rsc) && (rsc->parent->variant == pe_container))) {
         /* Only outermost resources have notification actions.
          * The exception is those in bundles.
          */
         notify = pcmk_is_set(rsc->flags, pe_rsc_notify);
     }
 
     if (rsc->variant >= pe_group) {
         /* Expand 'start' -> 'started' */
         char *uuid = NULL;
 
         uuid = convert_non_atomic_uuid(action->uuid, rsc, notify, FALSE);
         if (uuid) {
             pe_rsc_trace(rsc, "Converting %s to %s %d", action->uuid, uuid,
                          pcmk_is_set(rsc->flags, pe_rsc_notify));
             result = find_first_action(rsc->actions, uuid, NULL, NULL);
             if (result == NULL) {
                 crm_err("Couldn't expand %s to %s in %s", action->uuid, uuid, rsc->id);
                 result = action;
             }
             free(uuid);
         }
     }
     return result;
 }
 
 static enum pe_graph_flags
 graph_update_action(pe_action_t * first, pe_action_t * then, pe_node_t * node,
                     enum pe_action_flags first_flags, enum pe_action_flags then_flags,
                     pe_action_wrapper_t *order, pe_working_set_t *data_set)
 {
     enum pe_graph_flags changed = pe_graph_none;
     enum pe_ordering type = order->type;
     gboolean processed = FALSE;
 
     /* TODO: Do as many of these in parallel as possible */
 
     if (pcmk_is_set(type, pe_order_implies_then_on_node)) {
         /* Normally we want the _whole_ 'then' clone to
          * restart if 'first' is restarted, so then->node is
          * needed.
          *
          * However for unfencing, we want to limit this to
          * instances on the same node as 'first' (the
          * unfencing operation), so first->node is supplied.
          *
          * Swap the node, from then on we can can treat it
          * like any other 'pe_order_implies_then'
          */
 
         pe__clear_order_flags(type, pe_order_implies_then_on_node);
         pe__set_order_flags(type, pe_order_implies_then);
         node = first->node;
     }
 
     pe__clear_raw_action_flags(first_flags, "first action update",
                                pe_action_pseudo);
 
     if (type & pe_order_implies_then) {
         processed = TRUE;
         if (then->rsc) {
             changed |= then->rsc->cmds->update_actions(first, then, node,
                 first_flags & pe_action_optional, pe_action_optional,
                 pe_order_implies_then, data_set);
 
-        } else if (!pcmk_is_set(first_flags, pe_action_optional)) {
-            if (update_action_flags(then, pe_action_optional | pe_action_clear, __func__, __LINE__)) {
-                pe__set_graph_flags(changed, first, pe_graph_updated_then);
-            }
+        } else if (!pcmk_is_set(first_flags, pe_action_optional)
+                   && pcmk_is_set(then->flags, pe_action_optional)) {
+            pe__clear_action_flags(then, pe_action_optional);
+            pe__set_graph_flags(changed, first, pe_graph_updated_then);
         }
         if (changed) {
             pe_rsc_trace(then->rsc, "implies right: %s then %s: changed", first->uuid, then->uuid);
         } else {
             crm_trace("implies right: %s then %s %p", first->uuid, then->uuid, then->rsc);
         }
     }
 
     if ((type & pe_order_restart) && then->rsc) {
         enum pe_action_flags restart = (pe_action_optional | pe_action_runnable);
 
         processed = TRUE;
         changed |= then->rsc->cmds->update_actions(first, then, node,
                                                    first_flags, restart,
                                                    pe_order_restart, data_set);
         if (changed) {
             pe_rsc_trace(then->rsc, "restart: %s then %s: changed", first->uuid, then->uuid);
         } else {
             crm_trace("restart: %s then %s", first->uuid, then->uuid);
         }
     }
 
     if (type & pe_order_implies_first) {
         processed = TRUE;
         if (first->rsc) {
             changed |= first->rsc->cmds->update_actions(first, then, node,
                 first_flags, pe_action_optional, pe_order_implies_first,
                 data_set);
 
         } else if (!pcmk_is_set(first_flags, pe_action_optional)) {
             pe_rsc_trace(first->rsc, "first unrunnable: %s (%d) then %s (%d)",
                          first->uuid, pcmk_is_set(first_flags, pe_action_optional),
                          then->uuid, pcmk_is_set(then_flags, pe_action_optional));
-            if (update_action_flags(first, pe_action_runnable | pe_action_clear, __func__, __LINE__)) {
+            if (pcmk_is_set(first->flags, pe_action_runnable)) {
+                pe__clear_action_flags(first, pe_action_runnable);
                 pe__set_graph_flags(changed, first, pe_graph_updated_first);
             }
         }
 
         if (changed) {
             pe_rsc_trace(then->rsc, "implies left: %s then %s: changed", first->uuid, then->uuid);
         } else {
             crm_trace("implies left: %s (%d) then %s (%d)",
                       first->uuid, pcmk_is_set(first_flags, pe_action_optional),
                       then->uuid, pcmk_is_set(then_flags, pe_action_optional));
         }
     }
 
     if (type & pe_order_promoted_implies_first) {
         processed = TRUE;
         if (then->rsc) {
             changed |= then->rsc->cmds->update_actions(first, then, node,
                 first_flags & pe_action_optional, pe_action_optional,
                 pe_order_promoted_implies_first, data_set);
         }
 
         if (changed) {
             pe_rsc_trace(then->rsc,
                          "implies left when right resource is promoted: "
                          "%s then %s: changed", first->uuid, then->uuid);
         } else {
             crm_trace("implies left when right resource is promoted: "
                       "%s then %s", first->uuid, then->uuid);
         }
     }
 
     if (type & pe_order_one_or_more) {
         processed = TRUE;
         if (then->rsc) {
             changed |= then->rsc->cmds->update_actions(first, then, node,
                 first_flags, pe_action_runnable, pe_order_one_or_more,
                 data_set);
 
         } else if (pcmk_is_set(first_flags, pe_action_runnable)) {
             // We have another runnable instance of "first"
             then->runnable_before++;
 
             /* Mark "then" as runnable if it requires a certain number of
              * "before" instances to be runnable, and they now are.
              */
-            if (then->runnable_before >= then->required_runnable_before) {
-                if (update_action_flags(then, pe_action_runnable, __func__, __LINE__)) {
-                    pe__set_graph_flags(changed, first, pe_graph_updated_then);
-                }
+            if ((then->runnable_before >= then->required_runnable_before)
+                && !pcmk_is_set(then->flags, pe_action_runnable)) {
+
+                pe__set_action_flags(then, pe_action_runnable);
+                pe__set_graph_flags(changed, first, pe_graph_updated_then);
             }
         }
         if (changed) {
             pe_rsc_trace(then->rsc, "runnable_one_or_more: %s then %s: changed", first->uuid,
                          then->uuid);
         } else {
             crm_trace("runnable_one_or_more: %s then %s", first->uuid, then->uuid);
         }
     }
 
     if (then->rsc && pcmk_is_set(type, pe_order_probe)) {
         processed = TRUE;
 
         if (!pcmk_is_set(first_flags, pe_action_runnable)
             && (first->rsc->running_on != NULL)) {
 
             pe_rsc_trace(then->rsc, "Ignoring %s then %s - %s is about to be stopped",
                          first->uuid, then->uuid, first->rsc->id);
             type = pe_order_none;
             order->type = pe_order_none;
 
         } else {
             pe_rsc_trace(then->rsc, "Enforcing %s then %s", first->uuid, then->uuid);
             changed |= then->rsc->cmds->update_actions(first, then, node,
                 first_flags, pe_action_runnable, pe_order_runnable_left,
                 data_set);
         }
 
         if (changed) {
             pe_rsc_trace(then->rsc, "runnable: %s then %s: changed", first->uuid, then->uuid);
         } else {
             crm_trace("runnable: %s then %s", first->uuid, then->uuid);
         }
     }
 
     if (type & pe_order_runnable_left) {
         processed = TRUE;
         if (then->rsc) {
             changed |= then->rsc->cmds->update_actions(first, then, node,
                 first_flags, pe_action_runnable, pe_order_runnable_left,
                 data_set);
 
-        } else if (!pcmk_is_set(first_flags, pe_action_runnable)) {
-            pe_rsc_trace(then->rsc, "then unrunnable: %s then %s", first->uuid, then->uuid);
-            if (update_action_flags(then, pe_action_runnable | pe_action_clear, __func__, __LINE__)) {
-                pe__set_graph_flags(changed, first, pe_graph_updated_then);
-            }
+        } else if (!pcmk_is_set(first_flags, pe_action_runnable)
+                   && pcmk_is_set(then->flags, pe_action_runnable)) {
+
+            pe__clear_action_flags(then, pe_action_runnable);
+            pe__set_graph_flags(changed, first, pe_graph_updated_then);
         }
         if (changed) {
             pe_rsc_trace(then->rsc, "runnable: %s then %s: changed", first->uuid, then->uuid);
         } else {
             crm_trace("runnable: %s then %s", first->uuid, then->uuid);
         }
     }
 
     if (type & pe_order_implies_first_migratable) {
         processed = TRUE;
         if (then->rsc) {
             changed |= then->rsc->cmds->update_actions(first, then, node,
                 first_flags, pe_action_optional,
                 pe_order_implies_first_migratable, data_set);
         }
         if (changed) {
             pe_rsc_trace(then->rsc, "optional: %s then %s: changed", first->uuid, then->uuid);
         } else {
             crm_trace("optional: %s then %s", first->uuid, then->uuid);
         }
     }
 
     if (type & pe_order_pseudo_left) {
         processed = TRUE;
         if (then->rsc) {
             changed |= then->rsc->cmds->update_actions(first, then, node,
                 first_flags, pe_action_optional, pe_order_pseudo_left,
                 data_set);
         }
         if (changed) {
             pe_rsc_trace(then->rsc, "optional: %s then %s: changed", first->uuid, then->uuid);
         } else {
             crm_trace("optional: %s then %s", first->uuid, then->uuid);
         }
     }
 
     if (type & pe_order_optional) {
         processed = TRUE;
         if (then->rsc) {
             changed |= then->rsc->cmds->update_actions(first, then, node,
                 first_flags, pe_action_runnable, pe_order_optional, data_set);
         }
         if (changed) {
             pe_rsc_trace(then->rsc, "optional: %s then %s: changed", first->uuid, then->uuid);
         } else {
             crm_trace("optional: %s then %s", first->uuid, then->uuid);
         }
     }
 
     if (type & pe_order_asymmetrical) {
         processed = TRUE;
         if (then->rsc) {
             changed |= then->rsc->cmds->update_actions(first, then, node,
                 first_flags, pe_action_runnable, pe_order_asymmetrical,
                 data_set);
         }
 
         if (changed) {
             pe_rsc_trace(then->rsc, "asymmetrical: %s then %s: changed", first->uuid, then->uuid);
         } else {
             crm_trace("asymmetrical: %s then %s", first->uuid, then->uuid);
         }
 
     }
 
     if ((first->flags & pe_action_runnable) && (type & pe_order_implies_then_printed)
         && (first_flags & pe_action_optional) == 0) {
         processed = TRUE;
         crm_trace("%s implies %s printed", first->uuid, then->uuid);
-        update_action_flags(then, pe_action_print_always, __func__, __LINE__);  /* don't care about changed */
+        pe__set_action_flags(then, pe_action_print_always);
+        // Don't bother marking 'then' as changed just for this
     }
 
     if (pcmk_is_set(type, pe_order_implies_first_printed)
         && !pcmk_is_set(then_flags, pe_action_optional)) {
 
         processed = TRUE;
         crm_trace("%s implies %s printed", then->uuid, first->uuid);
-        update_action_flags(first, pe_action_print_always, __func__, __LINE__); /* don't care about changed */
+        pe__set_action_flags(first, pe_action_print_always);
+        // Don't bother marking 'first' as changed just for this
     }
 
     if ((type & pe_order_implies_then
          || type & pe_order_implies_first
          || type & pe_order_restart)
         && first->rsc
         && pcmk__str_eq(first->task, RSC_STOP, pcmk__str_casei)
         && !pcmk_is_set(first->rsc->flags, pe_rsc_managed)
         && pcmk_is_set(first->rsc->flags, pe_rsc_block)
         && !pcmk_is_set(first->flags, pe_action_runnable)) {
 
-        if (update_action_flags(then, pe_action_runnable | pe_action_clear, __func__, __LINE__)) {
+        if (pcmk_is_set(then->flags, pe_action_runnable)) {
+            pe__clear_action_flags(then, pe_action_runnable);
             pe__set_graph_flags(changed, first, pe_graph_updated_then);
         }
 
         if (changed) {
             pe_rsc_trace(then->rsc, "unmanaged left: %s then %s: changed", first->uuid, then->uuid);
         } else {
             crm_trace("unmanaged left: %s then %s", first->uuid, then->uuid);
         }
     }
 
     if (processed == FALSE) {
         crm_trace("Constraint 0x%.6x not applicable", type);
     }
 
     return changed;
 }
 
 static void
 mark_start_blocked(pe_resource_t *rsc, pe_resource_t *reason,
                    pe_working_set_t *data_set)
 {
     GList *gIter = rsc->actions;
     char *reason_text = crm_strdup_printf("colocation with %s", reason->id);
 
     for (; gIter != NULL; gIter = gIter->next) {
         pe_action_t *action = (pe_action_t *) gIter->data;
 
         if (!pcmk__str_eq(action->task, RSC_START, pcmk__str_casei)) {
             continue;
         }
         if (pcmk_is_set(action->flags, pe_action_runnable)) {
             pe_action_set_flag_reason(__func__, __LINE__, action, NULL,
                                       reason_text, pe_action_runnable, FALSE);
             update_colo_start_chain(action, data_set);
             update_action(action, data_set);
         }
     }
     free(reason_text);
 }
 
 void
 update_colo_start_chain(pe_action_t *action, pe_working_set_t *data_set)
 {
     GList *gIter = NULL;
     pe_resource_t *rsc = NULL;
 
     if (!pcmk_is_set(action->flags, pe_action_runnable)
         && pcmk__str_eq(action->task, RSC_START, pcmk__str_casei)) {
 
         rsc = uber_parent(action->rsc);
         if (rsc->parent) {
             /* For bundles, uber_parent() returns the clone, not the bundle, so
              * the existence of rsc->parent implies this is a bundle.
              * In this case, we need the bundle resource, so that we can check
              * if all containers are stopped/stopping.
              */
             rsc = rsc->parent;
         }
     }
 
     if (rsc == NULL || rsc->rsc_cons_lhs == NULL) {
         return;
     }
 
     /* if rsc has children, all the children need to have start set to
      * unrunnable before we follow the colo chain for the parent. */
     for (gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
         pe_resource_t *child = (pe_resource_t *)gIter->data;
         pe_action_t *start = find_first_action(child->actions, NULL, RSC_START, NULL);
         if ((start == NULL) || pcmk_is_set(start->flags, pe_action_runnable)) {
             return;
         }
     }
 
     for (gIter = rsc->rsc_cons_lhs; gIter != NULL; gIter = gIter->next) {
         pcmk__colocation_t *colocate_with = (pcmk__colocation_t *) gIter->data;
 
         if (colocate_with->score == INFINITY) {
             mark_start_blocked(colocate_with->rsc_lh, action->rsc, data_set);
         }
     }
 }
 
 gboolean
 update_action(pe_action_t *then, pe_working_set_t *data_set)
 {
     GList *lpc = NULL;
     enum pe_graph_flags changed = pe_graph_none;
     int last_flags = then->flags;
 
     crm_trace("Processing %s (%s %s %s)",
               then->uuid,
               pcmk_is_set(then->flags, pe_action_optional)? "optional" : "required",
               pcmk_is_set(then->flags, pe_action_runnable)? "runnable" : "unrunnable",
               pcmk_is_set(then->flags, pe_action_pseudo)? "pseudo"
                 : (then->node? then->node->details->uname : ""));
 
     if (pcmk_is_set(then->flags, pe_action_requires_any)) {
         /* initialize current known runnable before actions to 0
          * from here as graph_update_action is called for each of
          * then's before actions, this number will increment as
          * runnable 'first' actions are encountered */
         then->runnable_before = 0;
 
         /* for backwards compatibility with previous options that use
          * the 'requires_any' flag, initialize required to 1 if it is
          * not set. */ 
         if (then->required_runnable_before == 0) {
             then->required_runnable_before = 1;
         }
         pe__clear_action_flags(then, pe_action_runnable);
         /* We are relying on the pe_order_one_or_more clause of
          * graph_update_action(), called as part of the:
          *
          *    'if (first == other->action)'
          *
          * block below, to set this back if appropriate
          */
     }
 
     for (lpc = then->actions_before; lpc != NULL; lpc = lpc->next) {
         pe_action_wrapper_t *other = (pe_action_wrapper_t *) lpc->data;
         pe_action_t *first = other->action;
 
         pe_node_t *then_node = then->node;
         pe_node_t *first_node = first->node;
 
         enum pe_action_flags then_flags = 0;
         enum pe_action_flags first_flags = 0;
 
         if (first->rsc && first->rsc->variant == pe_group && pcmk__str_eq(first->task, RSC_START, pcmk__str_casei)) {
             first_node = first->rsc->fns->location(first->rsc, NULL, FALSE);
             if (first_node) {
                 crm_trace("First: Found node %s for %s", first_node->details->uname, first->uuid);
             }
         }
 
         if (then->rsc && then->rsc->variant == pe_group && pcmk__str_eq(then->task, RSC_START, pcmk__str_casei)) {
             then_node = then->rsc->fns->location(then->rsc, NULL, FALSE);
             if (then_node) {
                 crm_trace("Then: Found node %s for %s", then_node->details->uname, then->uuid);
             }
         }
         /* Disable constraint if it only applies when on same node, but isn't */
         if (pcmk_is_set(other->type, pe_order_same_node)
             && (first_node != NULL) && (then_node != NULL)
             && (first_node->details != then_node->details)) {
 
             crm_trace("Disabled constraint %s on %s -> %s on %s",
                        other->action->uuid, first_node->details->uname,
                        then->uuid, then_node->details->uname);
             other->type = pe_order_none;
             continue;
         }
 
         pe__clear_graph_flags(changed, then, pe_graph_updated_first);
 
         if (first->rsc && pcmk_is_set(other->type, pe_order_then_cancels_first)
             && !pcmk_is_set(then->flags, pe_action_optional)) {
 
             /* 'then' is required, so we must abandon 'first'
              * (e.g. a required stop cancels any agent reload).
              */
             pe__set_action_flags(other->action, pe_action_optional);
             if (!strcmp(first->task, CRMD_ACTION_RELOAD_AGENT)) {
                 pe__clear_resource_flags(first->rsc, pe_rsc_reload);
             }
         }
 
         if (first->rsc && then->rsc && (first->rsc != then->rsc)
             && (is_parent(then->rsc, first->rsc) == FALSE)) {
             first = rsc_expand_action(first);
         }
         if (first != other->action) {
             crm_trace("Ordering %s after %s instead of %s", then->uuid, first->uuid,
                       other->action->uuid);
         }
 
         first_flags = get_action_flags(first, then_node);
         then_flags = get_action_flags(then, first_node);
 
         crm_trace("Checking %s (%s %s %s) against %s (%s %s %s) filter=0x%.6x type=0x%.6x",
                   then->uuid,
                   pcmk_is_set(then_flags, pe_action_optional)? "optional" : "required",
                   pcmk_is_set(then_flags, pe_action_runnable)? "runnable" : "unrunnable",
                   pcmk_is_set(then_flags, pe_action_pseudo)? "pseudo"
                     : (then->node? then->node->details->uname : ""),
                   first->uuid,
                   pcmk_is_set(first_flags, pe_action_optional)? "optional" : "required",
                   pcmk_is_set(first_flags, pe_action_runnable)? "runnable" : "unrunnable",
                   pcmk_is_set(first_flags, pe_action_pseudo)? "pseudo"
                     : (first->node? first->node->details->uname : ""),
                   first_flags, other->type);
 
         if (first == other->action) {
             /*
              * 'first' was not expanded (e.g. from 'start' to 'running'), which could mean it:
              * - has no associated resource,
              * - was a primitive,
              * - was pre-expanded (e.g. 'running' instead of 'start')
              *
              * The third argument here to graph_update_action() is a node which is used under two conditions:
              * - Interleaving, in which case first->node and
              *   then->node are equal (and NULL)
              * - If 'then' is a clone, to limit the scope of the
              *   constraint to instances on the supplied node
              *
              */
             pe_node_t *node = then->node;
             changed |= graph_update_action(first, then, node, first_flags,
                                            then_flags, other, data_set);
 
             /* 'first' was for a complex resource (clone, group, etc),
              * create a new dependency if necessary
              */
         } else if (order_actions(first, then, other->type)) {
             /* This was the first time 'first' and 'then' were associated,
              * start again to get the new actions_before list
              */
             pe__set_graph_flags(changed, then,
                                 pe_graph_updated_then|pe_graph_disable);
         }
 
         if (changed & pe_graph_disable) {
             crm_trace("Disabled constraint %s -> %s in favor of %s -> %s",
                       other->action->uuid, then->uuid, first->uuid, then->uuid);
             pe__clear_graph_flags(changed, then, pe_graph_disable);
             other->type = pe_order_none;
         }
 
         if (changed & pe_graph_updated_first) {
             GList *lpc2 = NULL;
 
             crm_trace("Updated %s (first %s %s %s), processing dependents ",
                       first->uuid,
                       pcmk_is_set(first->flags, pe_action_optional)? "optional" : "required",
                       pcmk_is_set(first->flags, pe_action_runnable)? "runnable" : "unrunnable",
                       pcmk_is_set(first->flags, pe_action_pseudo)? "pseudo"
                         : (first->node? first->node->details->uname : ""));
             for (lpc2 = first->actions_after; lpc2 != NULL; lpc2 = lpc2->next) {
                 pe_action_wrapper_t *other = (pe_action_wrapper_t *) lpc2->data;
 
                 update_action(other->action, data_set);
             }
             update_action(first, data_set);
         }
     }
 
     if (pcmk_is_set(then->flags, pe_action_requires_any)) {
         if (last_flags != then->flags) {
             pe__set_graph_flags(changed, then, pe_graph_updated_then);
         } else {
             pe__clear_graph_flags(changed, then, pe_graph_updated_then);
         }
     }
 
     if (changed & pe_graph_updated_then) {
         crm_trace("Updated %s (then %s %s %s), processing dependents ",
                   then->uuid,
                   pcmk_is_set(then->flags, pe_action_optional)? "optional" : "required",
                   pcmk_is_set(then->flags, pe_action_runnable)? "runnable" : "unrunnable",
                   pcmk_is_set(then->flags, pe_action_pseudo)? "pseudo"
                     : (then->node? then->node->details-> uname : ""));
 
         if (pcmk_is_set(last_flags, pe_action_runnable)
             && !pcmk_is_set(then->flags, pe_action_runnable)) {
             update_colo_start_chain(then, data_set);
         }
         update_action(then, data_set);
         for (lpc = then->actions_after; lpc != NULL; lpc = lpc->next) {
             pe_action_wrapper_t *other = (pe_action_wrapper_t *) lpc->data;
 
             update_action(other->action, data_set);
         }
     }
 
     return FALSE;
 }
 
 gboolean
 shutdown_constraints(pe_node_t * node, pe_action_t * shutdown_op, pe_working_set_t * data_set)
 {
     /* add the stop to the before lists so it counts as a pre-req
      * for the shutdown
      */
     GList *lpc = NULL;
 
     for (lpc = data_set->actions; lpc != NULL; lpc = lpc->next) {
         pe_action_t *action = (pe_action_t *) lpc->data;
 
         if (action->rsc == NULL || action->node == NULL) {
             continue;
         } else if (action->node->details != node->details) {
             continue;
         } else if (pcmk_is_set(action->rsc->flags, pe_rsc_maintenance)) {
             pe_rsc_trace(action->rsc, "Skipping %s: maintenance mode", action->uuid);
             continue;
         } else if (node->details->maintenance) {
             pe_rsc_trace(action->rsc, "Skipping %s: node %s is in maintenance mode",
                          action->uuid, node->details->uname);
             continue;
         } else if (!pcmk__str_eq(action->task, RSC_STOP, pcmk__str_casei)) {
             continue;
         } else if (!pcmk_any_flags_set(action->rsc->flags,
                                        pe_rsc_managed|pe_rsc_block)) {
             /*
              * If another action depends on this one, we may still end up blocking
              */
             pe_rsc_trace(action->rsc, "Skipping %s: unmanaged", action->uuid);
             continue;
         }
 
         pe_rsc_trace(action->rsc, "Ordering %s before shutdown on %s", action->uuid,
                      node->details->uname);
         pe__clear_action_flags(action, pe_action_optional);
         custom_action_order(action->rsc, NULL, action,
                             NULL, strdup(CRM_OP_SHUTDOWN), shutdown_op,
                             pe_order_optional | pe_order_runnable_left, data_set);
     }
 
     return TRUE;
 }
 
 /*!
  * \internal
  * \brief Order all actions appropriately relative to a fencing operation
  *
  * Ensure start operations of affected resources are ordered after fencing,
  * imply stop and demote operations of affected resources by marking them as
  * pseudo-actions, etc.
  *
  * \param[in]     stonith_op  Fencing operation
  * \param[in,out] data_set    Working set of cluster
  */
 void
 pcmk__order_vs_fence(pe_action_t *stonith_op, pe_working_set_t *data_set)
 {
     CRM_CHECK(stonith_op && data_set, return);
     for (GList *r = data_set->resources; r != NULL; r = r->next) {
         rsc_stonith_ordering((pe_resource_t *) r->data, stonith_op, data_set);
     }
 }
 
 static pe_node_t *
 get_router_node(pe_action_t *action)
 {
     pe_node_t *began_on = NULL;
     pe_node_t *ended_on = NULL;
     bool partial_migration = FALSE;
     const char *task = action->task;
 
     if (pcmk__str_eq(task, CRM_OP_FENCE, pcmk__str_casei)
         || !pe__is_guest_or_remote_node(action->node)) {
         return NULL;
     }
 
     CRM_ASSERT(action->node->details->remote_rsc != NULL);
 
     began_on = pe__current_node(action->node->details->remote_rsc);
     ended_on = action->node->details->remote_rsc->allocated_to;
     if (action->node->details->remote_rsc
         && (action->node->details->remote_rsc->container == NULL)
         && action->node->details->remote_rsc->partial_migration_target) {
         partial_migration = TRUE;
     }
 
     if (began_on == NULL) {
         crm_trace("Routing %s for %s through remote connection's "
                   "next node %s (starting)%s",
                   action->task, (action->rsc? action->rsc->id : "no resource"),
                   (ended_on? ended_on->details->uname : "none"),
                   partial_migration? " (partial migration)" : "");
         return ended_on;
     }
 
     if (ended_on == NULL) {
         crm_trace("Routing %s for %s through remote connection's "
                   "current node %s (stopping)%s",
                   action->task, (action->rsc? action->rsc->id : "no resource"),
                   (began_on? began_on->details->uname : "none"),
                   partial_migration? " (partial migration)" : "");
         return began_on;
     }
 
     if (began_on->details == ended_on->details) {
         crm_trace("Routing %s for %s through remote connection's "
                   "current node %s (not moving)%s",
                   action->task, (action->rsc? action->rsc->id : "no resource"),
                   (began_on? began_on->details->uname : "none"),
                   partial_migration? " (partial migration)" : "");
         return began_on;
     }
 
     /* If we get here, the remote connection is moving during this transition.
      * This means some actions for resources behind the connection will get
      * routed through the cluster node the connection reource is currently on,
      * and others are routed through the cluster node the connection will end up
      * on.
      */
 
     if (pcmk__str_eq(task, "notify", pcmk__str_casei)) {
         task = g_hash_table_lookup(action->meta, "notify_operation");
     }
 
     /*
      * Stop, demote, and migration actions must occur before the connection can
      * move (these actions are required before the remote resource can stop). In
      * this case, we know these actions have to be routed through the initial
      * cluster node the connection resource lived on before the move takes
      * place.
      *
      * The exception is a partial migration of a (non-guest) remote connection
      * resource; in that case, all actions (even these) will be ordered after
      * the connection's pseudo-start on the migration target, so the target is
      * the router node.
      */
     if (pcmk__strcase_any_of(task, "cancel", "stop", "demote", "migrate_from",
                              "migrate_to", NULL) && !partial_migration) {
         crm_trace("Routing %s for %s through remote connection's "
                   "current node %s (moving)%s",
                   action->task, (action->rsc? action->rsc->id : "no resource"),
                   (began_on? began_on->details->uname : "none"),
                   partial_migration? " (partial migration)" : "");
         return began_on;
     }
 
     /* Everything else (start, promote, monitor, probe, refresh,
      * clear failcount, delete, ...) must occur after the connection starts on
      * the node it is moving to.
      */
     crm_trace("Routing %s for %s through remote connection's "
               "next node %s (moving)%s",
               action->task, (action->rsc? action->rsc->id : "no resource"),
               (ended_on? ended_on->details->uname : "none"),
               partial_migration? " (partial migration)" : "");
     return ended_on;
 }
 
 /*!
  * \internal
  * \brief Add an XML node tag for a specified ID
  *
  * \param[in]     id      Node UUID to add
  * \param[in,out] xml     Parent XML tag to add to
  */
 static xmlNode*
 add_node_to_xml_by_id(const char *id, xmlNode *xml)
 {
     xmlNode *node_xml;
 
     node_xml = create_xml_node(xml, XML_CIB_TAG_NODE);
     crm_xml_add(node_xml, XML_ATTR_UUID, id);
 
     return node_xml;
 }
 
 /*!
  * \internal
  * \brief Add an XML node tag for a specified node
  *
  * \param[in]     node  Node to add
  * \param[in,out] xml   XML to add node to
  */
 static void
 add_node_to_xml(const pe_node_t *node, void *xml)
 {
     add_node_to_xml_by_id(node->details->id, (xmlNode *) xml);
 }
 
 /*!
  * \internal
  * \brief Add XML with nodes that need an update of their maintenance state
  *
  * \param[in,out] xml       Parent XML tag to add to
  * \param[in]     data_set  Working set for cluster
  */
 static int
 add_maintenance_nodes(xmlNode *xml, const pe_working_set_t *data_set)
 {
     GList *gIter = NULL;
     xmlNode *maintenance =
         xml?create_xml_node(xml, XML_GRAPH_TAG_MAINTENANCE):NULL;
     int count = 0;
 
     for (gIter = data_set->nodes; gIter != NULL;
          gIter = gIter->next) {
         pe_node_t *node = (pe_node_t *) gIter->data;
         struct pe_node_shared_s *details = node->details;
 
         if (!pe__is_guest_or_remote_node(node)) {
             continue; /* just remote nodes need to know atm */
         }
 
         if (details->maintenance != details->remote_maintenance) {
             if (maintenance) {
                 crm_xml_add(
                     add_node_to_xml_by_id(node->details->id, maintenance),
                     XML_NODE_IS_MAINTENANCE, details->maintenance?"1":"0");
             }
             count++;
         }
     }
     crm_trace("%s %d nodes to adjust maintenance-mode "
               "to transition", maintenance?"Added":"Counted", count);
     return count;
 }
 
 /*!
  * \internal
  * \brief Add pseudo action with nodes needing maintenance state update
  *
  * \param[in,out] data_set  Working set for cluster
  */
 void
 add_maintenance_update(pe_working_set_t *data_set)
 {
     pe_action_t *action = NULL;
 
     if (add_maintenance_nodes(NULL, data_set)) {
         crm_trace("adding maintenance state update pseudo action");
         action = get_pseudo_op(CRM_OP_MAINTENANCE_NODES, data_set);
         pe__set_action_flags(action, pe_action_print_always);
     }
 }
 
 /*!
  * \internal
  * \brief Add XML with nodes that an action is expected to bring down
  *
  * If a specified action is expected to bring any nodes down, add an XML block
  * with their UUIDs. When a node is lost, this allows the controller to
  * determine whether it was expected.
  *
  * \param[in,out] xml       Parent XML tag to add to
  * \param[in]     action    Action to check for downed nodes
  * \param[in]     data_set  Working set for cluster
  */
 static void
 add_downed_nodes(xmlNode *xml, const pe_action_t *action,
                  const pe_working_set_t *data_set)
 {
     CRM_CHECK(xml && action && action->node && data_set, return);
 
     if (pcmk__str_eq(action->task, CRM_OP_SHUTDOWN, pcmk__str_casei)) {
 
         /* Shutdown makes the action's node down */
         xmlNode *downed = create_xml_node(xml, XML_GRAPH_TAG_DOWNED);
         add_node_to_xml_by_id(action->node->details->id, downed);
 
     } else if (pcmk__str_eq(action->task, CRM_OP_FENCE, pcmk__str_casei)) {
 
         /* Fencing makes the action's node and any hosted guest nodes down */
         const char *fence = g_hash_table_lookup(action->meta, "stonith_action");
 
         if (pcmk__strcase_any_of(fence, "off", "reboot", NULL)) {
             xmlNode *downed = create_xml_node(xml, XML_GRAPH_TAG_DOWNED);
             add_node_to_xml_by_id(action->node->details->id, downed);
             pe_foreach_guest_node(data_set, action->node, add_node_to_xml, downed);
         }
 
     } else if (action->rsc && action->rsc->is_remote_node
                && pcmk__str_eq(action->task, CRMD_ACTION_STOP, pcmk__str_casei)) {
 
         /* Stopping a remote connection resource makes connected node down,
          * unless it's part of a migration
          */
         GList *iter;
         pe_action_t *input;
         gboolean migrating = FALSE;
 
         for (iter = action->actions_before; iter != NULL; iter = iter->next) {
             input = ((pe_action_wrapper_t *) iter->data)->action;
             if (input->rsc && pcmk__str_eq(action->rsc->id, input->rsc->id, pcmk__str_casei)
                 && pcmk__str_eq(input->task, CRMD_ACTION_MIGRATED, pcmk__str_casei)) {
                 migrating = TRUE;
                 break;
             }
         }
         if (!migrating) {
             xmlNode *downed = create_xml_node(xml, XML_GRAPH_TAG_DOWNED);
             add_node_to_xml_by_id(action->rsc->id, downed);
         }
     }
 }
 
 static bool
 should_lock_action(pe_action_t *action)
 {
     // Only actions taking place on resource's lock node are locked
     if ((action->rsc->lock_node == NULL) || (action->node == NULL)
         || (action->node->details != action->rsc->lock_node->details)) {
         return false;
     }
 
     /* During shutdown, only stops are locked (otherwise, another action such as
      * a demote would cause the controller to clear the lock)
      */
     if (action->node->details->shutdown && action->task
         && strcmp(action->task, RSC_STOP)) {
         return false;
     }
 
     return true;
 }
 
 static xmlNode *
 action2xml(pe_action_t * action, gboolean as_input, pe_working_set_t *data_set)
 {
     gboolean needs_node_info = TRUE;
     gboolean needs_maintenance_info = FALSE;
     xmlNode *action_xml = NULL;
     xmlNode *args_xml = NULL;
 #if ENABLE_VERSIONED_ATTRS
     pe_rsc_action_details_t *rsc_details = NULL;
 #endif
 
     if (action == NULL) {
         return NULL;
     }
 
     if (pcmk__str_eq(action->task, CRM_OP_FENCE, pcmk__str_casei)) {
         /* All fences need node info; guest node fences are pseudo-events */
         action_xml = create_xml_node(NULL,
                                      pcmk_is_set(action->flags, pe_action_pseudo)?
                                      XML_GRAPH_TAG_PSEUDO_EVENT :
                                      XML_GRAPH_TAG_CRM_EVENT);
 
     } else if (pcmk__str_eq(action->task, CRM_OP_SHUTDOWN, pcmk__str_casei)) {
         action_xml = create_xml_node(NULL, XML_GRAPH_TAG_CRM_EVENT);
 
     } else if (pcmk__str_eq(action->task, CRM_OP_CLEAR_FAILCOUNT, pcmk__str_casei)) {
         action_xml = create_xml_node(NULL, XML_GRAPH_TAG_CRM_EVENT);
 
     } else if (pcmk__str_eq(action->task, CRM_OP_LRM_REFRESH, pcmk__str_casei)) {
         action_xml = create_xml_node(NULL, XML_GRAPH_TAG_CRM_EVENT);
 
     } else if (pcmk__str_eq(action->task, CRM_OP_LRM_DELETE, pcmk__str_casei)) {
         // CIB-only clean-up for shutdown locks
         action_xml = create_xml_node(NULL, XML_GRAPH_TAG_CRM_EVENT);
         crm_xml_add(action_xml, PCMK__XA_MODE, XML_TAG_CIB);
 
 /* 	} else if(pcmk__str_eq(action->task, RSC_PROBED, pcmk__str_casei)) { */
 /* 		action_xml = create_xml_node(NULL, XML_GRAPH_TAG_CRM_EVENT); */
 
     } else if (pcmk_is_set(action->flags, pe_action_pseudo)) {
         if (pcmk__str_eq(action->task, CRM_OP_MAINTENANCE_NODES, pcmk__str_casei)) {
             needs_maintenance_info = TRUE;
         }
         action_xml = create_xml_node(NULL, XML_GRAPH_TAG_PSEUDO_EVENT);
         needs_node_info = FALSE;
 
     } else {
         action_xml = create_xml_node(NULL, XML_GRAPH_TAG_RSC_OP);
 
 #if ENABLE_VERSIONED_ATTRS
         rsc_details = pe_rsc_action_details(action);
 #endif
     }
 
     crm_xml_add_int(action_xml, XML_ATTR_ID, action->id);
     crm_xml_add(action_xml, XML_LRM_ATTR_TASK, action->task);
     if (action->rsc != NULL && action->rsc->clone_name != NULL) {
         char *clone_key = NULL;
         guint interval_ms;
 
         if (pcmk__guint_from_hash(action->meta,
                                   XML_LRM_ATTR_INTERVAL_MS, 0,
                                   &interval_ms) != pcmk_rc_ok) {
             interval_ms = 0;
         }
 
         if (pcmk__str_eq(action->task, RSC_NOTIFY, pcmk__str_casei)) {
             const char *n_type = g_hash_table_lookup(action->meta, "notify_type");
             const char *n_task = g_hash_table_lookup(action->meta, "notify_operation");
 
             CRM_CHECK(n_type != NULL, crm_err("No notify type value found for %s", action->uuid));
             CRM_CHECK(n_task != NULL,
                       crm_err("No notify operation value found for %s", action->uuid));
             clone_key = pcmk__notify_key(action->rsc->clone_name,
                                          n_type, n_task);
 
         } else if(action->cancel_task) {
             clone_key = pcmk__op_key(action->rsc->clone_name,
                                      action->cancel_task, interval_ms);
         } else {
             clone_key = pcmk__op_key(action->rsc->clone_name,
                                      action->task, interval_ms);
         }
 
         CRM_CHECK(clone_key != NULL, crm_err("Could not generate a key for %s", action->uuid));
         crm_xml_add(action_xml, XML_LRM_ATTR_TASK_KEY, clone_key);
         crm_xml_add(action_xml, "internal_" XML_LRM_ATTR_TASK_KEY, action->uuid);
         free(clone_key);
 
     } else {
         crm_xml_add(action_xml, XML_LRM_ATTR_TASK_KEY, action->uuid);
     }
 
     if (needs_node_info && action->node != NULL) {
         pe_node_t *router_node = get_router_node(action);
 
         crm_xml_add(action_xml, XML_LRM_ATTR_TARGET, action->node->details->uname);
         crm_xml_add(action_xml, XML_LRM_ATTR_TARGET_UUID, action->node->details->id);
         if (router_node) {
             crm_xml_add(action_xml, XML_LRM_ATTR_ROUTER_NODE, router_node->details->uname);
         }
 
         g_hash_table_insert(action->meta, strdup(XML_LRM_ATTR_TARGET), strdup(action->node->details->uname));
         g_hash_table_insert(action->meta, strdup(XML_LRM_ATTR_TARGET_UUID), strdup(action->node->details->id));
     }
 
     /* No details if this action is only being listed in the inputs section */
     if (as_input) {
         return action_xml;
     }
 
     if (action->rsc && !pcmk_is_set(action->flags, pe_action_pseudo)) {
         int lpc = 0;
         xmlNode *rsc_xml = NULL;
         const char *attr_list[] = {
             XML_AGENT_ATTR_CLASS,
             XML_AGENT_ATTR_PROVIDER,
             XML_ATTR_TYPE
         };
 
         /* If a resource is locked to a node via shutdown-lock, mark its actions
          * so the controller can preserve the lock when the action completes.
          */
         if (should_lock_action(action)) {
             crm_xml_add_ll(action_xml, XML_CONFIG_ATTR_SHUTDOWN_LOCK,
                            (long long) action->rsc->lock_time);
         }
 
         // List affected resource
 
         rsc_xml = create_xml_node(action_xml,
                                   crm_element_name(action->rsc->xml));
         if (pcmk_is_set(action->rsc->flags, pe_rsc_orphan)
             && action->rsc->clone_name) {
             /* Do not use the 'instance free' name here as that
              * might interfere with the instance we plan to keep.
              * Ie. if there are more than two named /anonymous/
              * instances on a given node, we need to make sure the
              * command goes to the right one.
              *
              * Keep this block, even when everyone is using
              * 'instance free' anonymous clone names - it means
              * we'll do the right thing if anyone toggles the
              * unique flag to 'off'
              */
             crm_debug("Using orphan clone name %s instead of %s", action->rsc->id,
                       action->rsc->clone_name);
             crm_xml_add(rsc_xml, XML_ATTR_ID, action->rsc->clone_name);
             crm_xml_add(rsc_xml, XML_ATTR_ID_LONG, action->rsc->id);
 
         } else if (!pcmk_is_set(action->rsc->flags, pe_rsc_unique)) {
             const char *xml_id = ID(action->rsc->xml);
 
             crm_debug("Using anonymous clone name %s for %s (aka. %s)", xml_id, action->rsc->id,
                       action->rsc->clone_name);
 
             /* ID is what we'd like client to use
              * ID_LONG is what they might know it as instead
              *
              * ID_LONG is only strictly needed /here/ during the
              * transition period until all nodes in the cluster
              * are running the new software /and/ have rebooted
              * once (meaning that they've only ever spoken to a DC
              * supporting this feature).
              *
              * If anyone toggles the unique flag to 'on', the
              * 'instance free' name will correspond to an orphan
              * and fall into the clause above instead
              */
             crm_xml_add(rsc_xml, XML_ATTR_ID, xml_id);
             if (action->rsc->clone_name && !pcmk__str_eq(xml_id, action->rsc->clone_name, pcmk__str_casei)) {
                 crm_xml_add(rsc_xml, XML_ATTR_ID_LONG, action->rsc->clone_name);
             } else {
                 crm_xml_add(rsc_xml, XML_ATTR_ID_LONG, action->rsc->id);
             }
 
         } else {
             CRM_ASSERT(action->rsc->clone_name == NULL);
             crm_xml_add(rsc_xml, XML_ATTR_ID, action->rsc->id);
         }
 
         for (lpc = 0; lpc < PCMK__NELEM(attr_list); lpc++) {
             crm_xml_add(rsc_xml, attr_list[lpc],
                         g_hash_table_lookup(action->rsc->meta, attr_list[lpc]));
         }
     }
 
     /* List any attributes in effect */
     args_xml = create_xml_node(NULL, XML_TAG_ATTRS);
     crm_xml_add(args_xml, XML_ATTR_CRM_VERSION, CRM_FEATURE_SET);
 
     g_hash_table_foreach(action->extra, hash2field, args_xml);
     if (action->rsc != NULL && action->node) {
         // Get the resource instance attributes, evaluated properly for node
         GHashTable *params = pe_rsc_params(action->rsc, action->node, data_set);
 
         /* REMOTE_CONTAINER_HACK: If this is a remote connection resource with
          * addr="#uname", pull the actual value from the parameters evaluated
          * without a node (which was put there earlier in stage8() when the
          * bundle's expand() method was called).
          */
         const char *remote_addr = g_hash_table_lookup(params,
                                                       XML_RSC_ATTR_REMOTE_RA_ADDR);
 
         if (pcmk__str_eq(remote_addr, "#uname", pcmk__str_none)) {
             GHashTable *base = pe_rsc_params(action->rsc, NULL, data_set);
 
             remote_addr = g_hash_table_lookup(base,
                                               XML_RSC_ATTR_REMOTE_RA_ADDR);
             if (remote_addr != NULL) {
                 g_hash_table_insert(params, strdup(XML_RSC_ATTR_REMOTE_RA_ADDR),
                                     strdup(remote_addr));
             }
         }
 
         g_hash_table_foreach(params, hash2smartfield, args_xml);
 
 #if ENABLE_VERSIONED_ATTRS
         {
             xmlNode *versioned_parameters = create_xml_node(NULL, XML_TAG_RSC_VER_ATTRS);
 
             pe_get_versioned_attributes(versioned_parameters, action->rsc,
                                         action->node, data_set);
             if (xml_has_children(versioned_parameters)) {
                 add_node_copy(action_xml, versioned_parameters);
             }
             free_xml(versioned_parameters);
         }
 #endif
 
     } else if(action->rsc && action->rsc->variant <= pe_native) {
         GHashTable *params = pe_rsc_params(action->rsc, NULL, data_set);
 
         g_hash_table_foreach(params, hash2smartfield, args_xml);
 
 #if ENABLE_VERSIONED_ATTRS
         if (xml_has_children(action->rsc->versioned_parameters)) {
             add_node_copy(action_xml, action->rsc->versioned_parameters);
         }
 #endif
     }
 
 #if ENABLE_VERSIONED_ATTRS
     if (rsc_details) {
         if (xml_has_children(rsc_details->versioned_parameters)) {
             add_node_copy(action_xml, rsc_details->versioned_parameters);
         }
 
         if (xml_has_children(rsc_details->versioned_meta)) {
             add_node_copy(action_xml, rsc_details->versioned_meta);
         }
     }
 #endif
 
     g_hash_table_foreach(action->meta, hash2metafield, args_xml);
     if (action->rsc != NULL) {
         const char *value = g_hash_table_lookup(action->rsc->meta, "external-ip");
         pe_resource_t *parent = action->rsc;
 
         while (parent != NULL) {
             parent->cmds->append_meta(parent, args_xml);
             parent = parent->parent;
         }
 
         if(value) {
             hash2smartfield((gpointer)"pcmk_external_ip", (gpointer)value, (gpointer)args_xml);
         }
 
         if (action->node && /* make clang analyzer happy */
             pe__is_guest_node(action->node)) {
             pe_node_t *host = NULL;
             enum action_tasks task = text2task(action->task);
 
             if(task == action_notify || task == action_notified) {
                 const char *n_task = g_hash_table_lookup(action->meta, "notify_operation");
                 task = text2task(n_task);
             }
 
             // Differentiate between up and down actions
             switch (task) {
                 case stop_rsc:
                 case stopped_rsc:
                 case action_demote:
                 case action_demoted:
                     host = pe__current_node(action->node->details->remote_rsc->container);
                     break;
                 case start_rsc:
                 case started_rsc:
                 case monitor_rsc:
                 case action_promote:
                 case action_promoted:
                     host = action->node->details->remote_rsc->container->allocated_to;
                     break;
                 default:
                     break;
             }
 
             if(host) {
                 hash2metafield((gpointer)XML_RSC_ATTR_TARGET,
                                (gpointer)g_hash_table_lookup(action->rsc->meta, XML_RSC_ATTR_TARGET), (gpointer)args_xml);
                 hash2metafield((gpointer) PCMK__ENV_PHYSICAL_HOST,
                                (gpointer)host->details->uname,
                                (gpointer)args_xml);
             }
         }
 
     } else if (pcmk__str_eq(action->task, CRM_OP_FENCE, pcmk__str_casei) && action->node) {
         /* Pass the node's attributes as meta-attributes.
          *
          * @TODO: Determine whether it is still necessary to do this. It was
          * added in 33d99707, probably for the libfence-based implementation in
          * c9a90bd, which is no longer used.
          */
         g_hash_table_foreach(action->node->details->attrs, hash2metafield, args_xml);
     }
 
     sorted_xml(args_xml, action_xml, FALSE);
     free_xml(args_xml);
 
     /* List any nodes this action is expected to make down */
     if (needs_node_info && (action->node != NULL)) {
         add_downed_nodes(action_xml, action, data_set);
     }
 
     if (needs_maintenance_info) {
         add_maintenance_nodes(action_xml, data_set);
     }
 
     crm_log_xml_trace(action_xml, "dumped action");
     return action_xml;
 }
 
 static bool
 should_dump_action(pe_action_t *action)
 {
     CRM_CHECK(action != NULL, return false);
 
     if (pcmk_is_set(action->flags, pe_action_dumped)) {
         crm_trace("Action %s (%d) already dumped", action->uuid, action->id);
         return false;
 
     } else if (pcmk_is_set(action->flags, pe_action_pseudo)
                && pcmk__str_eq(action->task, CRM_OP_PROBED, pcmk__str_casei)) {
         GList *lpc = NULL;
 
         /* This is a horrible but convenient hack
          *
          * It mimimizes the number of actions with unsatisfied inputs
          * (i.e. not included in the graph)
          *
          * This in turn, means we can be more concise when printing
          * aborted/incomplete graphs.
          *
          * It also makes it obvious which node is preventing
          * probe_complete from running (presumably because it is only
          * partially up)
          *
          * For these reasons we tolerate such perversions
          */
 
         for (lpc = action->actions_after; lpc != NULL; lpc = lpc->next) {
             pe_action_wrapper_t *wrapper = (pe_action_wrapper_t *) lpc->data;
 
             if (!pcmk_is_set(wrapper->action->flags, pe_action_runnable)) {
                 /* Only interested in runnable operations */
             } else if (!pcmk__str_eq(wrapper->action->task, RSC_START, pcmk__str_casei)) {
                 /* Only interested in start operations */
             } else if (pcmk_is_set(wrapper->action->flags, pe_action_dumped)
                        || should_dump_action(wrapper->action)) {
                 crm_trace("Action %s (%d) should be dumped: "
                           "dependency of %s (%d)",
                           action->uuid, action->id,
                           wrapper->action->uuid, wrapper->action->id);
                 return true;
             }
         }
     }
 
     if (!pcmk_is_set(action->flags, pe_action_runnable)) {
         crm_trace("Ignoring action %s (%d): unrunnable",
                   action->uuid, action->id);
         return false;
 
     } else if (pcmk_is_set(action->flags, pe_action_optional)
                && !pcmk_is_set(action->flags, pe_action_print_always)) {
         crm_trace("Ignoring action %s (%d): optional",
                   action->uuid, action->id);
         return false;
 
     // Monitors should be dumped even for unmanaged resources
     } else if (action->rsc && !pcmk_is_set(action->rsc->flags, pe_rsc_managed)
                && !pcmk__str_eq(action->task, RSC_STATUS, pcmk__str_casei)) {
 
         const char *interval_ms_s = g_hash_table_lookup(action->meta,
                                                         XML_LRM_ATTR_INTERVAL_MS);
 
         // Cancellation of recurring monitors should still be dumped
         if (pcmk__str_eq(interval_ms_s, "0", pcmk__str_null_matches)) {
             crm_trace("Ignoring action %s (%d): for unmanaged resource (%s)",
                       action->uuid, action->id, action->rsc->id);
             return false;
         }
     }
 
     if (pcmk_is_set(action->flags, pe_action_pseudo) ||
         pcmk__strcase_any_of(action->task, CRM_OP_FENCE, CRM_OP_SHUTDOWN, NULL)) {
         /* skip the next checks */
         return true;
     }
 
     if (action->node == NULL) {
         pe_err("Skipping action %s (%d) "
                "because it was not allocated to a node (bug?)",
                action->uuid, action->id);
         log_action(LOG_DEBUG, "Unallocated action", action, false);
         return false;
 
     } else if (pcmk_is_set(action->flags, pe_action_dc)) {
         crm_trace("Action %s (%d) should be dumped: "
                   "can run on DC instead of %s",
                   action->uuid, action->id, action->node->details->uname);
 
     } else if (pe__is_guest_node(action->node)
                && !action->node->details->remote_requires_reset) {
         crm_trace("Action %s (%d) should be dumped: "
                   "assuming will be runnable on guest node %s",
                   action->uuid, action->id, action->node->details->uname);
 
     } else if (action->node->details->online == false) {
         pe_err("Skipping action %s (%d) "
                "because it was scheduled for offline node (bug?)",
                action->uuid, action->id);
         log_action(LOG_DEBUG, "Action for offline node", action, FALSE);
         return false;
 #if 0
         /* but this would also affect resources that can be safely
          *  migrated before a fencing op
          */
     } else if (action->node->details->unclean == false) {
         pe_err("Skipping action %s (%d) "
                "because it was scheduled for unclean node (bug?)",
                action->uuid, action->id);
         log_action(LOG_DEBUG, "Action for unclean node", action, false);
         return false;
 #endif
     }
     return true;
 }
 
 /* lowest to highest */
 static gint
 sort_action_id(gconstpointer a, gconstpointer b)
 {
     const pe_action_wrapper_t *action_wrapper2 = (const pe_action_wrapper_t *)a;
     const pe_action_wrapper_t *action_wrapper1 = (const pe_action_wrapper_t *)b;
 
     if (a == NULL) {
         return 1;
     }
     if (b == NULL) {
         return -1;
     }
 
     if (action_wrapper1->action->id > action_wrapper2->action->id) {
         return -1;
     }
 
     if (action_wrapper1->action->id < action_wrapper2->action->id) {
         return 1;
     }
     return 0;
 }
 
 /*!
  * \internal
  * \brief Check whether an action input should be in the transition graph
  *
  * \param[in]     action  Action to check
  * \param[in,out] input   Action input to check
  *
  * \return true if input should be in graph, false otherwise
  * \note This function may not only check an input, but disable it under certian
  *       circumstances (load or anti-colocation orderings that are not needed).
  */
 static bool
 check_dump_input(pe_action_t *action, pe_action_wrapper_t *input)
 {
     int type = input->type;
 
     if (input->state == pe_link_dumped) {
         return true;
     }
 
     pe__clear_order_flags(type, pe_order_implies_first_printed
                                 |pe_order_implies_then_printed
                                 |pe_order_optional);
 
     if (input->type == pe_order_none) {
         crm_trace("Ignoring %s (%d) input %s (%d): "
                   "ordering disabled",
                   action->uuid, action->id,
                   input->action->uuid, input->action->id);
         return false;
 
     } else if (!pcmk_is_set(input->action->flags, pe_action_runnable)
                && (type == pe_order_none)
                && !pcmk__str_eq(input->action->uuid, CRM_OP_PROBED, pcmk__str_casei)) {
         crm_trace("Ignoring %s (%d) input %s (%d): "
                   "optional and input unrunnable",
                   action->uuid, action->id,
                   input->action->uuid, input->action->id);
         return false;
 
     } else if (!pcmk_is_set(input->action->flags, pe_action_runnable)
                && pcmk_is_set(input->type, pe_order_one_or_more)) {
         crm_trace("Ignoring %s (%d) input %s (%d): "
                   "one-or-more and input unrunnable",
                   action->uuid, action->id,
                   input->action->uuid, input->action->id);
         return false;
 
     } else if (pcmk_is_set(action->flags, pe_action_pseudo)
                && pcmk_is_set(input->type, pe_order_stonith_stop)) {
         crm_trace("Ignoring %s (%d) input %s (%d): "
                   "stonith stop but action is pseudo",
                   action->uuid, action->id,
                   input->action->uuid, input->action->id);
         return false;
 
     } else if (pcmk_is_set(input->type, pe_order_implies_first_migratable)
                && !pcmk_is_set(input->action->flags, pe_action_runnable)) {
         crm_trace("Ignoring %s (%d) input %s (%d): "
                   "implies input migratable but input unrunnable",
                   action->uuid, action->id,
                   input->action->uuid, input->action->id);
         return false;
 
     } else if (pcmk_is_set(input->type, pe_order_apply_first_non_migratable)
                && pcmk_is_set(input->action->flags, pe_action_migrate_runnable)) {
         crm_trace("Ignoring %s (%d) input %s (%d): "
                   "only if input unmigratable but input unrunnable",
                   action->uuid, action->id,
                   input->action->uuid, input->action->id);
         return false;
 
     } else if ((input->type == pe_order_optional)
                && pcmk_is_set(input->action->flags, pe_action_migrate_runnable)
                && pcmk__ends_with(input->action->uuid, "_stop_0")) {
         crm_trace("Ignoring %s (%d) input %s (%d): "
                   "optional but stop in migration",
                   action->uuid, action->id,
                   input->action->uuid, input->action->id);
         return false;
 
     } else if (input->type == pe_order_load) {
         pe_node_t *input_node = input->action->node;
 
         // load orderings are relevant only if actions are for same node
 
         if (action->rsc && pcmk__str_eq(action->task, RSC_MIGRATE, pcmk__str_casei)) {
             pe_node_t *allocated = action->rsc->allocated_to;
 
             /* For load_stopped -> migrate_to orderings, we care about where it
              * has been allocated to, not where it will be executed.
              */
             if ((input_node == NULL) || (allocated == NULL)
                 || (input_node->details != allocated->details)) {
                 crm_trace("Ignoring %s (%d) input %s (%d): "
                           "load ordering node mismatch %s vs %s",
                           action->uuid, action->id,
                           input->action->uuid, input->action->id,
                           (allocated? allocated->details->uname : "<none>"),
                           (input_node? input_node->details->uname : "<none>"));
                 input->type = pe_order_none;
                 return false;
             }
 
         } else if ((input_node == NULL) || (action->node == NULL)
                    || (input_node->details != action->node->details)) {
             crm_trace("Ignoring %s (%d) input %s (%d): "
                       "load ordering node mismatch %s vs %s",
                       action->uuid, action->id,
                       input->action->uuid, input->action->id,
                       (action->node? action->node->details->uname : "<none>"),
                       (input_node? input_node->details->uname : "<none>"));
             input->type = pe_order_none;
             return false;
 
         } else if (pcmk_is_set(input->action->flags, pe_action_optional)) {
             crm_trace("Ignoring %s (%d) input %s (%d): "
                       "load ordering input optional",
                       action->uuid, action->id,
                       input->action->uuid, input->action->id);
             input->type = pe_order_none;
             return false;
         }
 
     } else if (input->type == pe_order_anti_colocation) {
         if (input->action->node && action->node
             && (input->action->node->details != action->node->details)) {
             crm_trace("Ignoring %s (%d) input %s (%d): "
                       "anti-colocation node mismatch %s vs %s",
                       action->uuid, action->id,
                       input->action->uuid, input->action->id,
                       action->node->details->uname,
                       input->action->node->details->uname);
             input->type = pe_order_none;
             return false;
 
         } else if (pcmk_is_set(input->action->flags, pe_action_optional)) {
             crm_trace("Ignoring %s (%d) input %s (%d): "
                       "anti-colocation input optional",
                       action->uuid, action->id,
                       input->action->uuid, input->action->id);
             input->type = pe_order_none;
             return false;
         }
 
     } else if (input->action->rsc
                && input->action->rsc != action->rsc
                && pcmk_is_set(input->action->rsc->flags, pe_rsc_failed)
                && !pcmk_is_set(input->action->rsc->flags, pe_rsc_managed)
                && pcmk__ends_with(input->action->uuid, "_stop_0")
                && action->rsc && pe_rsc_is_clone(action->rsc)) {
         crm_warn("Ignoring requirement that %s complete before %s:"
                  " unmanaged failed resources cannot prevent clone shutdown",
                  input->action->uuid, action->uuid);
         return false;
 
     } else if (pcmk_is_set(input->action->flags, pe_action_optional)
                && !pcmk_any_flags_set(input->action->flags,
                                       pe_action_print_always|pe_action_dumped)
                && !should_dump_action(input->action)) {
         crm_trace("Ignoring %s (%d) input %s (%d): "
                   "input optional",
                   action->uuid, action->id,
                   input->action->uuid, input->action->id);
         return false;
     }
 
     crm_trace("%s (%d) input %s (%d) @ %s should be dumped: %s, %s, %s, 0x%.6x",
               action->uuid, action->id,
               input->action->uuid, input->action->id,
               input->action->node? input->action->node->details->uname : "no node",
               pcmk_is_set(input->action->flags, pe_action_pseudo)? "pseudo" : "real",
               pcmk_is_set(input->action->flags, pe_action_runnable)? "runnable" : "unrunnable",
               pcmk_is_set(input->action->flags, pe_action_optional)? "optional" : "required",
               input->type);
     return true;
 }
 
 static bool
 graph_has_loop(pe_action_t *init_action, pe_action_t *action,
                pe_action_wrapper_t *input)
 {
     bool has_loop = false;
 
     if (pcmk_is_set(input->action->flags, pe_action_tracking)) {
         crm_trace("Breaking tracking loop: %s@%s -> %s@%s (0x%.6x)",
                   input->action->uuid,
                   input->action->node? input->action->node->details->uname : "",
                   action->uuid,
                   action->node? action->node->details->uname : "",
                   input->type);
         return false;
     }
 
     // Don't need to check inputs that won't be used
     if (!check_dump_input(action, input)) {
         return false;
     }
 
     if (input->action == init_action) {
         crm_debug("Input loop found in %s@%s ->...-> %s@%s",
                   action->uuid,
                   action->node? action->node->details->uname : "",
                   init_action->uuid,
                   init_action->node? init_action->node->details->uname : "");
         return true;
     }
 
     pe__set_action_flags(input->action, pe_action_tracking);
 
     crm_trace("Checking inputs of action %s@%s input %s@%s (0x%.6x)"
               "for graph loop with %s@%s ",
               action->uuid,
               action->node? action->node->details->uname : "",
               input->action->uuid,
               input->action->node? input->action->node->details->uname : "",
               input->type,
               init_action->uuid,
               init_action->node? init_action->node->details->uname : "");
 
     // Recursively check input itself for loops
     for (GList *iter = input->action->actions_before;
          iter != NULL; iter = iter->next) {
 
         if (graph_has_loop(init_action, input->action,
                            (pe_action_wrapper_t *) iter->data)) {
             // Recursive call already logged a debug message
             has_loop = true;
             goto done;
         }
     }
 
 done:
     pe__clear_action_flags(input->action, pe_action_tracking);
 
     if (!has_loop) {
         crm_trace("No input loop found in %s@%s -> %s@%s (0x%.6x)",
                   input->action->uuid,
                   input->action->node? input->action->node->details->uname : "",
                   action->uuid,
                   action->node? action->node->details->uname : "",
                   input->type);
     }
     return has_loop;
 }
 
 bool
 pcmk__ordering_is_invalid(pe_action_t *action, pe_action_wrapper_t *input)
 {
     /* Prevent user-defined ordering constraints between resources
      * running in a guest node and the resource that defines that node.
      */
     if (!pcmk_is_set(input->type, pe_order_preserve)
         && action->rsc && action->rsc->fillers
         && input->action->rsc && input->action->node
         && input->action->node->details->remote_rsc
         && (input->action->node->details->remote_rsc->container == action->rsc)) {
         crm_warn("Invalid ordering constraint between %s and %s",
                  input->action->rsc->id, action->rsc->id);
         return true;
     }
 
     /* If there's an order like
      * "rscB_stop node2"-> "load_stopped_node2" -> "rscA_migrate_to node1"
      *
      * then rscA is being migrated from node1 to node2, while rscB is being
      * migrated from node2 to node1. If there would be a graph loop,
      * break the order "load_stopped_node2" -> "rscA_migrate_to node1".
      */
     if ((input->type == pe_order_load) && action->rsc
         && pcmk__str_eq(action->task, RSC_MIGRATE, pcmk__str_casei)
         && graph_has_loop(action, action, input)) {
         return true;
     }
 
     return false;
 }
 
 // Remove duplicate inputs (regardless of flags)
 static void
 deduplicate_inputs(pe_action_t *action)
 {
     GList *item = NULL;
     GList *next = NULL;
     pe_action_wrapper_t *last_input = NULL;
 
     action->actions_before = g_list_sort(action->actions_before,
                                          sort_action_id);
     for (item = action->actions_before; item != NULL; item = next) {
         pe_action_wrapper_t *input = (pe_action_wrapper_t *) item->data;
 
         next = item->next;
         if (last_input && (input->action->id == last_input->action->id)) {
             crm_trace("Input %s (%d) duplicate skipped for action %s (%d)",
                       input->action->uuid, input->action->id,
                       action->uuid, action->id);
 
             /* For the purposes of scheduling, the ordering flags no longer
              * matter, but crm_simulate looks at certain ones when creating a
              * dot graph. Combining the flags is sufficient for that purpose.
              */
             last_input->type |= input->type;
             if (input->state == pe_link_dumped) {
                 last_input->state = pe_link_dumped;
             }
 
             free(item->data);
             action->actions_before = g_list_delete_link(action->actions_before,
                                                         item);
         } else {
             last_input = input;
             input->state = pe_link_not_dumped;
         }
     }
 }
 
 /*!
  * \internal
  * \brief Add an action to the transition graph XML if appropriate
  *
  * \param[in] action    Action to possibly add
  * \param[in] data_set  Cluster working set
  *
  * \note This will de-duplicate the action inputs, meaning that the
  *       pe_action_wrapper_t:type flags can no longer be relied on to retain
  *       their original settings. That means this MUST be called after stage7()
  *       is complete, and nothing after this should rely on those type flags.
  *       (For example, some code looks for type equal to some flag rather than
  *       whether the flag is set, and some code looks for particular
  *       combinations of flags -- such code must be done before stage8().)
  */
 void
 graph_element_from_action(pe_action_t *action, pe_working_set_t *data_set)
 {
     GList *lpc = NULL;
     int synapse_priority = 0;
     xmlNode *syn = NULL;
     xmlNode *set = NULL;
     xmlNode *in = NULL;
     xmlNode *xml_action = NULL;
     pe_action_wrapper_t *input = NULL;
 
     /* If we haven't already, de-duplicate inputs -- even if we won't be dumping
      * the action, so that crm_simulate dot graphs don't have duplicates.
      */
     if (!pcmk_is_set(action->flags, pe_action_dedup)) {
         deduplicate_inputs(action);
         pe__set_action_flags(action, pe_action_dedup);
     }
 
     if (should_dump_action(action) == FALSE) {
         return;
     }
 
     pe__set_action_flags(action, pe_action_dumped);
 
     syn = create_xml_node(data_set->graph, "synapse");
     set = create_xml_node(syn, "action_set");
     in = create_xml_node(syn, "inputs");
 
     crm_xml_add_int(syn, XML_ATTR_ID, data_set->num_synapse);
     data_set->num_synapse++;
 
     if (action->rsc != NULL) {
         synapse_priority = action->rsc->priority;
     }
     if (action->priority > synapse_priority) {
         synapse_priority = action->priority;
     }
     if (synapse_priority > 0) {
         crm_xml_add_int(syn, XML_CIB_ATTR_PRIORITY, synapse_priority);
     }
 
     xml_action = action2xml(action, FALSE, data_set);
     add_node_nocopy(set, crm_element_name(xml_action), xml_action);
 
     for (lpc = action->actions_before; lpc != NULL; lpc = lpc->next) {
         input = (pe_action_wrapper_t *) lpc->data;
         if (check_dump_input(action, input)) {
             xmlNode *input_xml = create_xml_node(in, "trigger");
 
             input->state = pe_link_dumped;
             xml_action = action2xml(input->action, TRUE, data_set);
             add_node_nocopy(input_xml, crm_element_name(xml_action), xml_action);
         }
     }
 }
diff --git a/lib/pacemaker/pcmk_sched_group.c b/lib/pacemaker/pcmk_sched_group.c
index 819fe0f285..c22a560e1b 100644
--- a/lib/pacemaker/pcmk_sched_group.c
+++ b/lib/pacemaker/pcmk_sched_group.c
@@ -1,532 +1,576 @@
 /*
  * Copyright 2004-2021 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU General Public License version 2
  * or later (GPLv2+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #include <stdbool.h>
 
 #include <crm/msg_xml.h>
 
 #include <pacemaker-internal.h>
 
 #define VARIANT_GROUP 1
 #include <lib/pengine/variant.h>
 
+/*!
+ * \internal
+ * \brief Expand a group's colocations to its members
+ *
+ * \param[in,out] rsc  Group resource
+ */
+static void
+expand_group_colocations(pe_resource_t *rsc)
+{
+    group_variant_data_t *group_data = NULL;
+    pe_resource_t *member = NULL;
+    bool any_unmanaged = false;
+
+    get_group_variant_data(group_data, rsc);
+
+    // Treat "group with R" colocations as "first member with R"
+    member = group_data->first_child;
+    member->rsc_cons = g_list_concat(member->rsc_cons, rsc->rsc_cons);
+
+
+    /* The above works for the whole group because each group member is
+     * colocated with the previous one.
+     *
+     * However, there is a special case when a group has a mandatory colocation
+     * with a resource that can't start. In that case, update_colo_start_chain()
+     * will ensure that dependent resources in mandatory colocations (i.e. the
+     * first member for groups) can't start either. But if any group member is
+     * unmanaged and already started, the internal group colocations are no
+     * longer sufficient to make that apply to later members.
+     *
+     * To handle that case, add mandatory colocations to each member after the
+     * first.
+     */
+    any_unmanaged = !pcmk_is_set(member->flags, pe_rsc_managed);
+    for (GList *item = rsc->children->next; item != NULL; item = item->next) {
+        member = item->data;
+        if (any_unmanaged) {
+            for (GList *cons_iter = rsc->rsc_cons; cons_iter != NULL;
+                 cons_iter = cons_iter->next) {
+
+                pcmk__colocation_t *constraint = (pcmk__colocation_t *) cons_iter->data;
+
+                if (constraint->score == INFINITY) {
+                    member->rsc_cons = g_list_prepend(member->rsc_cons, constraint);
+                }
+            }
+        } else if (!pcmk_is_set(member->flags, pe_rsc_managed)) {
+            any_unmanaged = true;
+        }
+    }
+
+    rsc->rsc_cons = NULL;
+
+    // Treat "R with group" colocations as "R with last member"
+    member = group_data->last_child;
+    member->rsc_cons_lhs = g_list_concat(member->rsc_cons_lhs,
+                                         rsc->rsc_cons_lhs);
+    rsc->rsc_cons_lhs = NULL;
+}
+
 pe_node_t *
 pcmk__group_allocate(pe_resource_t *rsc, pe_node_t *prefer,
                      pe_working_set_t *data_set)
 {
     pe_node_t *node = NULL;
     pe_node_t *group_node = NULL;
     GList *gIter = NULL;
     group_variant_data_t *group_data = NULL;
 
     get_group_variant_data(group_data, rsc);
 
     if (!pcmk_is_set(rsc->flags, pe_rsc_provisional)) {
         return rsc->allocated_to;
     }
     if (pcmk_is_set(rsc->flags, pe_rsc_allocating)) {
         pe_rsc_debug(rsc, "Dependency loop detected involving %s", rsc->id);
         return NULL;
     }
 
     if (group_data->first_child == NULL) {
         // Nothing to allocate
         pe__clear_resource_flags(rsc, pe_rsc_provisional);
         return NULL;
     }
 
     pe__set_resource_flags(rsc, pe_rsc_allocating);
     rsc->role = group_data->first_child->role;
 
-    group_data->first_child->rsc_cons =
-        g_list_concat(group_data->first_child->rsc_cons, rsc->rsc_cons);
-    rsc->rsc_cons = NULL;
-
-    group_data->last_child->rsc_cons_lhs =
-        g_list_concat(group_data->last_child->rsc_cons_lhs, rsc->rsc_cons_lhs);
-    rsc->rsc_cons_lhs = NULL;
+    expand_group_colocations(rsc);
 
     pe__show_node_weights(!pcmk_is_set(data_set->flags, pe_flag_show_scores),
                           rsc, __func__, rsc->allowed_nodes, data_set);
 
     gIter = rsc->children;
     for (; gIter != NULL; gIter = gIter->next) {
         pe_resource_t *child_rsc = (pe_resource_t *) gIter->data;
 
         pe_rsc_trace(rsc, "Allocating group %s member %s",
                      rsc->id, child_rsc->id);
         node = child_rsc->cmds->allocate(child_rsc, prefer, data_set);
         if (group_node == NULL) {
             group_node = node;
         }
     }
 
     pe__set_next_role(rsc, group_data->first_child->next_role,
                       "first group member");
     pe__clear_resource_flags(rsc, pe_rsc_allocating|pe_rsc_provisional);
 
     if (group_data->colocated) {
         return group_node;
     }
     return NULL;
 }
 
 void group_update_pseudo_status(pe_resource_t * parent, pe_resource_t * child);
 
 void
 group_create_actions(pe_resource_t * rsc, pe_working_set_t * data_set)
 {
     pe_action_t *op = NULL;
     const char *value = NULL;
     GList *gIter = rsc->children;
 
     pe_rsc_trace(rsc, "Creating actions for %s", rsc->id);
 
     for (; gIter != NULL; gIter = gIter->next) {
         pe_resource_t *child_rsc = (pe_resource_t *) gIter->data;
 
         child_rsc->cmds->create_actions(child_rsc, data_set);
         group_update_pseudo_status(rsc, child_rsc);
     }
 
     op = start_action(rsc, NULL, TRUE /* !group_data->child_starting */ );
     pe__set_action_flags(op, pe_action_pseudo|pe_action_runnable);
 
     op = custom_action(rsc, started_key(rsc),
                        RSC_STARTED, NULL, TRUE /* !group_data->child_starting */ , TRUE, data_set);
     pe__set_action_flags(op, pe_action_pseudo|pe_action_runnable);
 
     op = stop_action(rsc, NULL, TRUE /* !group_data->child_stopping */ );
     pe__set_action_flags(op, pe_action_pseudo|pe_action_runnable);
 
     op = custom_action(rsc, stopped_key(rsc),
                        RSC_STOPPED, NULL, TRUE /* !group_data->child_stopping */ , TRUE, data_set);
     pe__set_action_flags(op, pe_action_pseudo|pe_action_runnable);
 
     value = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_PROMOTABLE);
     if (crm_is_true(value)) {
         op = custom_action(rsc, demote_key(rsc), RSC_DEMOTE, NULL, TRUE, TRUE, data_set);
         pe__set_action_flags(op, pe_action_pseudo|pe_action_runnable);
 
         op = custom_action(rsc, demoted_key(rsc), RSC_DEMOTED, NULL, TRUE, TRUE, data_set);
         pe__set_action_flags(op, pe_action_pseudo|pe_action_runnable);
 
         op = custom_action(rsc, promote_key(rsc), RSC_PROMOTE, NULL, TRUE, TRUE, data_set);
         pe__set_action_flags(op, pe_action_pseudo|pe_action_runnable);
 
         op = custom_action(rsc, promoted_key(rsc), RSC_PROMOTED, NULL, TRUE, TRUE, data_set);
         pe__set_action_flags(op, pe_action_pseudo|pe_action_runnable);
     }
 }
 
 void
 group_update_pseudo_status(pe_resource_t * parent, pe_resource_t * child)
 {
     GList *gIter = child->actions;
     group_variant_data_t *group_data = NULL;
 
     get_group_variant_data(group_data, parent);
 
     if (group_data->ordered == FALSE) {
         /* If this group is not ordered, then leave the meta-actions as optional */
         return;
     }
 
     if (group_data->child_stopping && group_data->child_starting) {
         return;
     }
 
     for (; gIter != NULL; gIter = gIter->next) {
         pe_action_t *action = (pe_action_t *) gIter->data;
 
         if (pcmk_is_set(action->flags, pe_action_optional)) {
             continue;
         }
         if (pcmk__str_eq(RSC_STOP, action->task, pcmk__str_casei)
             && pcmk_is_set(action->flags, pe_action_runnable)) {
 
             group_data->child_stopping = TRUE;
             pe_rsc_trace(action->rsc, "Based on %s the group is stopping", action->uuid);
 
         } else if (pcmk__str_eq(RSC_START, action->task, pcmk__str_casei)
                    && pcmk_is_set(action->flags, pe_action_runnable)) {
             group_data->child_starting = TRUE;
             pe_rsc_trace(action->rsc, "Based on %s the group is starting", action->uuid);
         }
     }
 }
 
 void
 group_internal_constraints(pe_resource_t * rsc, pe_working_set_t * data_set)
 {
     GList *gIter = rsc->children;
     pe_resource_t *last_rsc = NULL;
     pe_resource_t *last_active = NULL;
     pe_resource_t *top = uber_parent(rsc);
     group_variant_data_t *group_data = NULL;
 
     get_group_variant_data(group_data, rsc);
 
     new_rsc_order(rsc, RSC_STOPPED, rsc, RSC_START, pe_order_optional, data_set);
     new_rsc_order(rsc, RSC_START, rsc, RSC_STARTED, pe_order_runnable_left, data_set);
     new_rsc_order(rsc, RSC_STOP, rsc, RSC_STOPPED, pe_order_runnable_left, data_set);
 
     for (; gIter != NULL; gIter = gIter->next) {
         pe_resource_t *child_rsc = (pe_resource_t *) gIter->data;
         int stop = pe_order_none;
         int stopped = pe_order_implies_then_printed;
         int start = pe_order_implies_then | pe_order_runnable_left;
         int started =
             pe_order_runnable_left | pe_order_implies_then | pe_order_implies_then_printed;
 
         child_rsc->cmds->internal_constraints(child_rsc, data_set);
 
         if (last_rsc == NULL) {
             if (group_data->ordered) {
                 pe__set_order_flags(stop, pe_order_optional);
                 stopped = pe_order_implies_then;
             }
 
         } else if (group_data->colocated) {
             pcmk__new_colocation("group:internal_colocation", NULL, INFINITY,
                                  child_rsc, last_rsc, NULL, NULL,
                                  pcmk_is_set(child_rsc->flags, pe_rsc_critical),
                                  data_set);
         }
 
         if (pcmk_is_set(top->flags, pe_rsc_promotable)) {
             new_rsc_order(rsc, RSC_DEMOTE, child_rsc, RSC_DEMOTE,
                           stop | pe_order_implies_first_printed, data_set);
 
             new_rsc_order(child_rsc, RSC_DEMOTE, rsc, RSC_DEMOTED, stopped, data_set);
 
             new_rsc_order(child_rsc, RSC_PROMOTE, rsc, RSC_PROMOTED, started, data_set);
 
             new_rsc_order(rsc, RSC_PROMOTE, child_rsc, RSC_PROMOTE,
                           pe_order_implies_first_printed, data_set);
 
         }
 
         order_start_start(rsc, child_rsc, pe_order_implies_first_printed);
         order_stop_stop(rsc, child_rsc, stop | pe_order_implies_first_printed);
 
         new_rsc_order(child_rsc, RSC_STOP, rsc, RSC_STOPPED, stopped, data_set);
 
         new_rsc_order(child_rsc, RSC_START, rsc, RSC_STARTED, started, data_set);
 
         if (group_data->ordered == FALSE) {
             order_start_start(rsc, child_rsc, start | pe_order_implies_first_printed);
             if (pcmk_is_set(top->flags, pe_rsc_promotable)) {
                 new_rsc_order(rsc, RSC_PROMOTE, child_rsc, RSC_PROMOTE,
                               start | pe_order_implies_first_printed, data_set);
             }
 
         } else if (last_rsc != NULL) {
-            child_rsc->restart_type = pe_restart_restart;
-
             order_start_start(last_rsc, child_rsc, start);
             order_stop_stop(child_rsc, last_rsc, pe_order_optional | pe_order_restart);
 
             if (pcmk_is_set(top->flags, pe_rsc_promotable)) {
                 new_rsc_order(last_rsc, RSC_PROMOTE, child_rsc, RSC_PROMOTE, start, data_set);
                 new_rsc_order(child_rsc, RSC_DEMOTE, last_rsc, RSC_DEMOTE, pe_order_optional,
                               data_set);
             }
 
         } else {
-            /* If anyone in the group is starting, then
-             *  pe_order_implies_then will cause _everyone_ in the group
-             *  to be sent a start action
-             * But this is safe since starting something that is already
-             *  started is required to be "safe"
-             */
-            int flags = pe_order_none;
-
-            order_start_start(rsc, child_rsc, flags);
+            order_start_start(rsc, child_rsc, pe_order_none);
             if (pcmk_is_set(top->flags, pe_rsc_promotable)) {
-                new_rsc_order(rsc, RSC_PROMOTE, child_rsc, RSC_PROMOTE, flags, data_set);
+                new_rsc_order(rsc, RSC_PROMOTE, child_rsc, RSC_PROMOTE,
+                              pe_order_none, data_set);
             }
-
         }
 
         /* Look for partially active groups
          * Make sure they still shut down in sequence
          */
         if (child_rsc->running_on) {
             if (group_data->ordered
                 && last_rsc
                 && last_rsc->running_on == NULL && last_active && last_active->running_on) {
                 order_stop_stop(child_rsc, last_active, pe_order_optional);
             }
             last_active = child_rsc;
         }
 
         last_rsc = child_rsc;
     }
 
     if (group_data->ordered && last_rsc != NULL) {
         int stop_stop_flags = pe_order_implies_then;
         int stop_stopped_flags = pe_order_optional;
 
         order_stop_stop(rsc, last_rsc, stop_stop_flags);
         new_rsc_order(last_rsc, RSC_STOP, rsc, RSC_STOPPED, stop_stopped_flags, data_set);
 
         if (pcmk_is_set(top->flags, pe_rsc_promotable)) {
             new_rsc_order(rsc, RSC_DEMOTE, last_rsc, RSC_DEMOTE, stop_stop_flags, data_set);
             new_rsc_order(last_rsc, RSC_DEMOTE, rsc, RSC_DEMOTED, stop_stopped_flags, data_set);
         }
     }
 }
 
 void
 group_rsc_colocation_lh(pe_resource_t *rsc_lh, pe_resource_t *rsc_rh,
                         pcmk__colocation_t *constraint,
                         pe_working_set_t *data_set)
 {
     GList *gIter = NULL;
     group_variant_data_t *group_data = NULL;
 
     if (rsc_lh == NULL) {
         pe_err("rsc_lh was NULL for %s", constraint->id);
         return;
 
     } else if (rsc_rh == NULL) {
         pe_err("rsc_rh was NULL for %s", constraint->id);
         return;
     }
 
     gIter = rsc_lh->children;
     pe_rsc_trace(rsc_lh, "Processing constraints from %s", rsc_lh->id);
 
     get_group_variant_data(group_data, rsc_lh);
 
     if (group_data->colocated) {
         group_data->first_child->cmds->rsc_colocation_lh(group_data->first_child,
                                                          rsc_rh, constraint,
                                                          data_set);
         return;
 
     } else if (constraint->score >= INFINITY) {
         pcmk__config_err("%s: Cannot perform mandatory colocation "
                          "between non-colocated group and %s",
                          rsc_lh->id, rsc_rh->id);
         return;
     }
 
     for (; gIter != NULL; gIter = gIter->next) {
         pe_resource_t *child_rsc = (pe_resource_t *) gIter->data;
 
         child_rsc->cmds->rsc_colocation_lh(child_rsc, rsc_rh, constraint,
                                            data_set);
     }
 }
 
 void
 group_rsc_colocation_rh(pe_resource_t *rsc_lh, pe_resource_t *rsc_rh,
                         pcmk__colocation_t *constraint,
                         pe_working_set_t *data_set)
 {
     GList *gIter = rsc_rh->children;
     group_variant_data_t *group_data = NULL;
 
     get_group_variant_data(group_data, rsc_rh);
     CRM_CHECK(rsc_lh->variant == pe_native, return);
 
     pe_rsc_trace(rsc_rh, "Processing RH %s of constraint %s (LH is %s)",
                  rsc_rh->id, constraint->id, rsc_lh->id);
 
     if (pcmk_is_set(rsc_rh->flags, pe_rsc_provisional)) {
         return;
 
     } else if (group_data->colocated && group_data->first_child) {
         if (constraint->score >= INFINITY) {
             /* Ensure RHS is _fully_ up before can start LHS */
             group_data->last_child->cmds->rsc_colocation_rh(rsc_lh,
                                                             group_data->last_child,
                                                             constraint,
                                                             data_set);
         } else {
             /* A partially active RHS is fine */
             group_data->first_child->cmds->rsc_colocation_rh(rsc_lh,
                                                              group_data->first_child,
                                                              constraint,
                                                              data_set);
         }
 
         return;
 
     } else if (constraint->score >= INFINITY) {
         pcmk__config_err("%s: Cannot perform mandatory colocation with"
                          " non-colocated group %s", rsc_lh->id, rsc_rh->id);
         return;
     }
 
     for (; gIter != NULL; gIter = gIter->next) {
         pe_resource_t *child_rsc = (pe_resource_t *) gIter->data;
 
         child_rsc->cmds->rsc_colocation_rh(rsc_lh, child_rsc, constraint,
                                            data_set);
     }
 }
 
 enum pe_action_flags
 group_action_flags(pe_action_t * action, pe_node_t * node)
 {
     GList *gIter = NULL;
     enum pe_action_flags flags = (pe_action_optional | pe_action_runnable | pe_action_pseudo);
 
     for (gIter = action->rsc->children; gIter != NULL; gIter = gIter->next) {
         pe_resource_t *child = (pe_resource_t *) gIter->data;
         enum action_tasks task = get_complex_task(child, action->task, TRUE);
         const char *task_s = task2text(task);
         pe_action_t *child_action = find_first_action(child->actions, NULL, task_s, node);
 
         if (child_action) {
             enum pe_action_flags child_flags = child->cmds->action_flags(child_action, node);
 
             if (pcmk_is_set(flags, pe_action_optional)
                 && !pcmk_is_set(child_flags, pe_action_optional)) {
                 pe_rsc_trace(action->rsc, "%s is mandatory because of %s", action->uuid,
                              child_action->uuid);
                 pe__clear_raw_action_flags(flags, "group action",
                                            pe_action_optional);
                 pe__clear_action_flags(action, pe_action_optional);
             }
             if (!pcmk__str_eq(task_s, action->task, pcmk__str_casei)
                 && pcmk_is_set(flags, pe_action_runnable)
                 && !pcmk_is_set(child_flags, pe_action_runnable)) {
 
                 pe_rsc_trace(action->rsc, "%s is not runnable because of %s", action->uuid,
                              child_action->uuid);
                 pe__clear_raw_action_flags(flags, "group action",
                                            pe_action_runnable);
                 pe__clear_action_flags(action, pe_action_runnable);
             }
 
         } else if (task != stop_rsc && task != action_demote) {
             pe_rsc_trace(action->rsc, "%s is not runnable because of %s (not found in %s)",
                          action->uuid, task_s, child->id);
             pe__clear_raw_action_flags(flags, "group action",
                                        pe_action_runnable);
         }
     }
 
     return flags;
 }
 
 enum pe_graph_flags
 group_update_actions(pe_action_t *first, pe_action_t *then, pe_node_t *node,
                      enum pe_action_flags flags, enum pe_action_flags filter,
                      enum pe_ordering type, pe_working_set_t *data_set)
 {
     GList *gIter = then->rsc->children;
     enum pe_graph_flags changed = pe_graph_none;
 
     CRM_ASSERT(then->rsc != NULL);
     changed |= native_update_actions(first, then, node, flags, filter, type,
                                      data_set);
 
     for (; gIter != NULL; gIter = gIter->next) {
         pe_resource_t *child = (pe_resource_t *) gIter->data;
         pe_action_t *child_action = find_first_action(child->actions, NULL, then->task, node);
 
         if (child_action) {
             changed |= child->cmds->update_actions(first, child_action, node,
                                                    flags, filter, type,
                                                    data_set);
         }
     }
 
     return changed;
 }
 
 void
 group_rsc_location(pe_resource_t *rsc, pe__location_t *constraint)
 {
     GList *gIter = rsc->children;
     GList *saved = constraint->node_list_rh;
     GList *zero = pcmk__copy_node_list(constraint->node_list_rh, true);
     gboolean reset_scores = TRUE;
     group_variant_data_t *group_data = NULL;
 
     get_group_variant_data(group_data, rsc);
 
     pe_rsc_debug(rsc, "Processing rsc_location %s for %s", constraint->id, rsc->id);
 
     native_rsc_location(rsc, constraint);
 
     for (; gIter != NULL; gIter = gIter->next) {
         pe_resource_t *child_rsc = (pe_resource_t *) gIter->data;
 
         child_rsc->cmds->rsc_location(child_rsc, constraint);
         if (group_data->colocated && reset_scores) {
             reset_scores = FALSE;
             constraint->node_list_rh = zero;
         }
     }
 
     constraint->node_list_rh = saved;
     g_list_free_full(zero, free);
 }
 
 void
 group_expand(pe_resource_t * rsc, pe_working_set_t * data_set)
 {
     CRM_CHECK(rsc != NULL, return);
 
     pe_rsc_trace(rsc, "Processing actions from %s", rsc->id);
     native_expand(rsc, data_set);
 
     for (GList *gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
         pe_resource_t *child_rsc = (pe_resource_t *) gIter->data;
 
         child_rsc->cmds->expand(child_rsc, data_set);
     }
 }
 
 GHashTable *
 pcmk__group_merge_weights(pe_resource_t *rsc, const char *rhs,
                           GHashTable *nodes, const char *attr, float factor,
                           uint32_t flags)
 {
     GList *gIter = rsc->rsc_cons_lhs;
     group_variant_data_t *group_data = NULL;
 
     get_group_variant_data(group_data, rsc);
 
     if (pcmk_is_set(rsc->flags, pe_rsc_merging)) {
         pe_rsc_info(rsc, "Breaking dependency loop with %s at %s", rsc->id, rhs);
         return nodes;
     }
 
     pe__set_resource_flags(rsc, pe_rsc_merging);
 
     nodes =
         group_data->first_child->cmds->merge_weights(group_data->first_child, rhs, nodes, attr,
                                                      factor, flags);
 
     for (; gIter != NULL; gIter = gIter->next) {
         pcmk__colocation_t *constraint = (pcmk__colocation_t *) gIter->data;
 
         nodes = pcmk__native_merge_weights(constraint->rsc_lh, rsc->id, nodes,
                                            constraint->node_attribute,
                                            constraint->score / (float) INFINITY,
                                            flags);
     }
 
     pe__clear_resource_flags(rsc, pe_rsc_merging);
     return nodes;
 }
 
 void
 group_append_meta(pe_resource_t * rsc, xmlNode * xml)
 {
 }
diff --git a/lib/pacemaker/pcmk_sched_native.c b/lib/pacemaker/pcmk_sched_native.c
index 88a4a0374b..79b20977f1 100644
--- a/lib/pacemaker/pcmk_sched_native.c
+++ b/lib/pacemaker/pcmk_sched_native.c
@@ -1,3223 +1,3215 @@
 /*
  * Copyright 2004-2021 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU General Public License version 2
  * or later (GPLv2+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #include <stdbool.h>
 
 #include <crm/pengine/rules.h>
 #include <crm/msg_xml.h>
 #include <crm/common/xml_internal.h>
 #include <pacemaker-internal.h>
 #include <crm/services.h>
 
 // The controller removes the resource from the CIB, making this redundant
 // #define DELETE_THEN_REFRESH 1
 
 #define INFINITY_HACK   (INFINITY * -100)
 
 #define VARIANT_NATIVE 1
 #include <lib/pengine/variant.h>
 
 extern bool pcmk__is_daemon;
 
 static void Recurring(pe_resource_t *rsc, pe_action_t *start, pe_node_t *node,
                       pe_working_set_t *data_set);
 static void RecurringOp(pe_resource_t *rsc, pe_action_t *start, pe_node_t *node,
                         xmlNode *operation, pe_working_set_t *data_set);
 static void Recurring_Stopped(pe_resource_t *rsc, pe_action_t *start, pe_node_t *node,
                               pe_working_set_t *data_set);
 static void RecurringOp_Stopped(pe_resource_t *rsc, pe_action_t *start, pe_node_t *node,
                                 xmlNode *operation, pe_working_set_t *data_set);
 
 void ReloadRsc(pe_resource_t * rsc, pe_node_t *node, pe_working_set_t * data_set);
 gboolean DeleteRsc(pe_resource_t * rsc, pe_node_t * node, gboolean optional, pe_working_set_t * data_set);
 gboolean StopRsc(pe_resource_t * rsc, pe_node_t * next, gboolean optional, pe_working_set_t * data_set);
 gboolean StartRsc(pe_resource_t * rsc, pe_node_t * next, gboolean optional, pe_working_set_t * data_set);
 gboolean DemoteRsc(pe_resource_t * rsc, pe_node_t * next, gboolean optional, pe_working_set_t * data_set);
 gboolean PromoteRsc(pe_resource_t * rsc, pe_node_t * next, gboolean optional,
                     pe_working_set_t * data_set);
 gboolean RoleError(pe_resource_t * rsc, pe_node_t * next, gboolean optional, pe_working_set_t * data_set);
 gboolean NullOp(pe_resource_t * rsc, pe_node_t * next, gboolean optional, pe_working_set_t * data_set);
 
 /* This array says what the *next* role should be when transitioning from one
  * role to another. For example going from Stopped to Promoted, the next role is
  * RSC_ROLE_UNPROMOTED, because the resource must be started before being promoted.
  * The current state then becomes Started, which is fed into this array again,
  * giving a next role of RSC_ROLE_PROMOTED.
  */
 static enum rsc_role_e rsc_state_matrix[RSC_ROLE_MAX][RSC_ROLE_MAX] = {
 /* Current state  Next state*/
 /*                 Unknown           Stopped           Started           Unpromoted           Promoted */
 /* Unknown */    { RSC_ROLE_UNKNOWN, RSC_ROLE_STOPPED, RSC_ROLE_STOPPED, RSC_ROLE_STOPPED,    RSC_ROLE_STOPPED },
 /* Stopped */    { RSC_ROLE_STOPPED, RSC_ROLE_STOPPED, RSC_ROLE_STARTED, RSC_ROLE_UNPROMOTED, RSC_ROLE_UNPROMOTED },
 /* Started */    { RSC_ROLE_STOPPED, RSC_ROLE_STOPPED, RSC_ROLE_STARTED, RSC_ROLE_UNPROMOTED, RSC_ROLE_PROMOTED },
 /* Unpromoted */ { RSC_ROLE_STOPPED, RSC_ROLE_STOPPED, RSC_ROLE_STOPPED, RSC_ROLE_UNPROMOTED, RSC_ROLE_PROMOTED },
 /* Promoted  */  { RSC_ROLE_STOPPED, RSC_ROLE_UNPROMOTED, RSC_ROLE_UNPROMOTED, RSC_ROLE_UNPROMOTED, RSC_ROLE_PROMOTED },
 };
 
 typedef gboolean (*rsc_transition_fn)(pe_resource_t *rsc, pe_node_t *next,
                                       gboolean optional,
                                       pe_working_set_t *data_set);
 
 // This array picks the function needed to transition from one role to another
 static rsc_transition_fn rsc_action_matrix[RSC_ROLE_MAX][RSC_ROLE_MAX] = {
 /* Current state   Next state                                            */
 /*                 Unknown    Stopped    Started    Unpromoted Promoted  */
 /* Unknown */    { RoleError, StopRsc,   RoleError, RoleError, RoleError,    },
 /* Stopped */    { RoleError, NullOp,    StartRsc,  StartRsc,  RoleError,    },
 /* Started */    { RoleError, StopRsc,   NullOp,    NullOp,    PromoteRsc,   },
 /* Unpromoted */ { RoleError, StopRsc,   StopRsc,   NullOp,    PromoteRsc,   },
 /* Promoted  */  { RoleError, DemoteRsc, DemoteRsc, DemoteRsc, NullOp,       },
 };
 
 #define clear_node_weights_flags(nw_flags, nw_rsc, flags_to_clear) do {     \
         flags = pcmk__clear_flags_as(__func__, __LINE__, LOG_TRACE,         \
                                      "Node weight", (nw_rsc)->id, (flags),  \
                                      (flags_to_clear), #flags_to_clear);    \
     } while (0)
 
 static gboolean
 native_choose_node(pe_resource_t * rsc, pe_node_t * prefer, pe_working_set_t * data_set)
 {
     GList *nodes = NULL;
     pe_node_t *chosen = NULL;
     pe_node_t *best = NULL;
     int multiple = 1;
     int length = 0;
     gboolean result = FALSE;
 
     process_utilization(rsc, &prefer, data_set);
 
     if (!pcmk_is_set(rsc->flags, pe_rsc_provisional)) {
         return rsc->allocated_to ? TRUE : FALSE;
     }
 
     // Sort allowed nodes by weight
     if (rsc->allowed_nodes) {
         length = g_hash_table_size(rsc->allowed_nodes);
     }
     if (length > 0) {
         nodes = g_hash_table_get_values(rsc->allowed_nodes);
         nodes = sort_nodes_by_weight(nodes, pe__current_node(rsc), data_set);
 
         // First node in sorted list has the best score
         best = g_list_nth_data(nodes, 0);
     }
 
     if (prefer && nodes) {
         chosen = g_hash_table_lookup(rsc->allowed_nodes, prefer->details->id);
 
         if (chosen == NULL) {
             pe_rsc_trace(rsc, "Preferred node %s for %s was unknown",
                          prefer->details->uname, rsc->id);
 
         /* Favor the preferred node as long as its weight is at least as good as
          * the best allowed node's.
          *
          * An alternative would be to favor the preferred node even if the best
          * node is better, when the best node's weight is less than INFINITY.
          */
         } else if ((chosen->weight < 0) || (chosen->weight < best->weight)) {
             pe_rsc_trace(rsc, "Preferred node %s for %s was unsuitable",
                          chosen->details->uname, rsc->id);
             chosen = NULL;
 
         } else if (!can_run_resources(chosen)) {
             pe_rsc_trace(rsc, "Preferred node %s for %s was unavailable",
                          chosen->details->uname, rsc->id);
             chosen = NULL;
 
         } else {
             pe_rsc_trace(rsc,
                          "Chose preferred node %s for %s (ignoring %d candidates)",
                          chosen->details->uname, rsc->id, length);
         }
     }
 
     if ((chosen == NULL) && nodes) {
         /* Either there is no preferred node, or the preferred node is not
          * available, but there are other nodes allowed to run the resource.
          */
 
         chosen = best;
         pe_rsc_trace(rsc, "Chose node %s for %s from %d candidates",
                      chosen ? chosen->details->uname : "<none>", rsc->id, length);
 
         if (!pe_rsc_is_unique_clone(rsc->parent)
             && chosen && (chosen->weight > 0) && can_run_resources(chosen)) {
             /* If the resource is already running on a node, prefer that node if
              * it is just as good as the chosen node.
              *
              * We don't do this for unique clone instances, because
              * distribute_children() has already assigned instances to their
              * running nodes when appropriate, and if we get here, we don't want
              * remaining unallocated instances to prefer a node that's already
              * running another instance.
              */
             pe_node_t *running = pe__current_node(rsc);
 
             if (running && (can_run_resources(running) == FALSE)) {
                 pe_rsc_trace(rsc, "Current node for %s (%s) can't run resources",
                              rsc->id, running->details->uname);
             } else if (running) {
                 for (GList *iter = nodes->next; iter; iter = iter->next) {
                     pe_node_t *tmp = (pe_node_t *) iter->data;
 
                     if (tmp->weight != chosen->weight) {
                         // The nodes are sorted by weight, so no more are equal
                         break;
                     }
                     if (tmp->details == running->details) {
                         // Scores are equal, so prefer the current node
                         chosen = tmp;
                     }
                     multiple++;
                 }
             }
         }
     }
 
     if (multiple > 1) {
         static char score[33];
         int log_level = (chosen->weight >= INFINITY)? LOG_WARNING : LOG_INFO;
 
         score2char_stack(chosen->weight, score, sizeof(score));
         do_crm_log(log_level,
                    "Chose node %s for %s from %d nodes with score %s",
                    chosen->details->uname, rsc->id, multiple, score);
     }
 
     result = native_assign_node(rsc, chosen, FALSE);
     g_list_free(nodes);
     return result;
 }
 
 /*!
  * \internal
  * \brief Find score of highest-scored node that matches colocation attribute
  *
  * \param[in] rsc    Resource whose allowed nodes should be searched
  * \param[in] attr   Colocation attribute name (must not be NULL)
  * \param[in] value  Colocation attribute value to require
  */
 static int
 best_node_score_matching_attr(const pe_resource_t *rsc, const char *attr,
                               const char *value)
 {
     GHashTableIter iter;
     pe_node_t *node = NULL;
     int best_score = -INFINITY;
     const char *best_node = NULL;
 
     // Find best allowed node with matching attribute
     g_hash_table_iter_init(&iter, rsc->allowed_nodes);
     while (g_hash_table_iter_next(&iter, NULL, (void **) &node)) {
 
         if ((node->weight > best_score) && can_run_resources(node)
             && pcmk__str_eq(value, pe_node_attribute_raw(node, attr), pcmk__str_casei)) {
 
             best_score = node->weight;
             best_node = node->details->uname;
         }
     }
 
     if (!pcmk__str_eq(attr, CRM_ATTR_UNAME, pcmk__str_casei)) {
         if (best_node == NULL) {
             crm_info("No allowed node for %s matches node attribute %s=%s",
                      rsc->id, attr, value);
         } else {
             crm_info("Allowed node %s for %s had best score (%d) "
                      "of those matching node attribute %s=%s",
                      best_node, rsc->id, best_score, attr, value);
         }
     }
     return best_score;
 }
 
 /*!
  * \internal
  * \brief Add resource's colocation matches to current node allocation scores
  *
  * For each node in a given table, if any of a given resource's allowed nodes
  * have a matching value for the colocation attribute, add the highest of those
  * nodes' scores to the node's score.
  *
  * \param[in,out] nodes  Hash table of nodes with allocation scores so far
  * \param[in]     rsc    Resource whose allowed nodes should be compared
  * \param[in]     attr   Colocation attribute that must match (NULL for default)
  * \param[in]     factor Factor by which to multiply scores being added
  * \param[in]     only_positive  Whether to add only positive scores
  */
 static void
 add_node_scores_matching_attr(GHashTable *nodes, const pe_resource_t *rsc,
                               const char *attr, float factor,
                               bool only_positive)
 {
     GHashTableIter iter;
     pe_node_t *node = NULL;
 
     if (attr == NULL) {
         attr = CRM_ATTR_UNAME;
     }
 
     // Iterate through each node
     g_hash_table_iter_init(&iter, nodes);
     while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
         float weight_f = 0;
         int weight = 0;
         int score = 0;
         int new_score = 0;
 
         score = best_node_score_matching_attr(rsc, attr,
                                               pe_node_attribute_raw(node, attr));
 
         if ((factor < 0) && (score < 0)) {
             /* Negative preference for a node with a negative score
              * should not become a positive preference.
              *
              * @TODO Consider filtering only if weight is -INFINITY
              */
             crm_trace("%s: Filtering %d + %f * %d (double negative disallowed)",
                       node->details->uname, node->weight, factor, score);
             continue;
         }
 
         if (node->weight == INFINITY_HACK) {
             crm_trace("%s: Filtering %d + %f * %d (node was marked unusable)",
                       node->details->uname, node->weight, factor, score);
             continue;
         }
 
         weight_f = factor * score;
 
         // Round the number; see http://c-faq.com/fp/round.html
         weight = (int) ((weight_f < 0)? (weight_f - 0.5) : (weight_f + 0.5));
 
         /* Small factors can obliterate the small scores that are often actually
          * used in configurations. If the score and factor are nonzero, ensure
          * that the result is nonzero as well.
          */
         if ((weight == 0) && (score != 0)) {
             if (factor > 0.0) {
                 weight = 1;
             } else if (factor < 0.0) {
                 weight = -1;
             }
         }
 
         new_score = pe__add_scores(weight, node->weight);
 
         if (only_positive && (new_score < 0) && (node->weight > 0)) {
             crm_trace("%s: Filtering %d + %f * %d = %d "
                       "(negative disallowed, marking node unusable)",
                       node->details->uname, node->weight, factor, score,
                       new_score);
             node->weight = INFINITY_HACK;
             continue;
         }
 
         if (only_positive && (new_score < 0) && (node->weight == 0)) {
             crm_trace("%s: Filtering %d + %f * %d = %d (negative disallowed)",
                       node->details->uname, node->weight, factor, score,
                       new_score);
             continue;
         }
 
         crm_trace("%s: %d + %f * %d = %d", node->details->uname,
                   node->weight, factor, score, new_score);
         node->weight = new_score;
     }
 }
 
 static inline bool
 is_nonempty_group(pe_resource_t *rsc)
 {
     return rsc && (rsc->variant == pe_group) && (rsc->children != NULL);
 }
 
 /*!
  * \internal
  * \brief Incorporate colocation constraint scores into node weights
  *
  * \param[in,out] rsc    Resource being placed
  * \param[in]     rhs    ID of 'with' resource
  * \param[in,out] nodes  Nodes, with scores as of this point
  * \param[in]     attr   Colocation attribute (ID by default)
  * \param[in]     factor Incorporate scores multiplied by this factor
  * \param[in]     flags  Bitmask of enum pe_weights values
  *
  * \return Nodes, with scores modified by this constraint
  * \note This function assumes ownership of the nodes argument. The caller
  *       should free the returned copy rather than the original.
  */
 GHashTable *
 pcmk__native_merge_weights(pe_resource_t *rsc, const char *rhs,
                            GHashTable *nodes, const char *attr, float factor,
                            uint32_t flags)
 {
     GHashTable *work = NULL;
 
     // Avoid infinite recursion
     if (pcmk_is_set(rsc->flags, pe_rsc_merging)) {
         pe_rsc_info(rsc, "%s: Breaking dependency loop at %s", rhs, rsc->id);
         return nodes;
     }
     pe__set_resource_flags(rsc, pe_rsc_merging);
 
     if (pcmk_is_set(flags, pe_weights_init)) {
         if (is_nonempty_group(rsc)) {
             GList *last = g_list_last(rsc->children);
             pe_resource_t *last_rsc = last->data;
 
             pe_rsc_trace(rsc, "%s: Merging scores from group %s "
                          "using last member %s (at %.6f)",
                          rhs, rsc->id, last_rsc->id, factor);
             work = pcmk__native_merge_weights(last_rsc, rhs, NULL, attr, factor,
                                               flags);
         } else {
             work = pcmk__copy_node_table(rsc->allowed_nodes);
         }
         clear_node_weights_flags(flags, rsc, pe_weights_init);
 
     } else if (is_nonempty_group(rsc)) {
         /* The first member of the group will recursively incorporate any
          * constraints involving other members (including the group internal
          * colocation).
          *
          * @TODO The indirect colocations from the dependent group's other
          *       members will be incorporated at full strength rather than by
          *       factor, so the group's combined stickiness will be treated as
          *       (factor + (#members - 1)) * stickiness. It is questionable what
          *       the right approach should be.
          */
         pe_rsc_trace(rsc, "%s: Merging scores from first member of group %s "
                      "(at %.6f)", rhs, rsc->id, factor);
         work = pcmk__copy_node_table(nodes);
         work = pcmk__native_merge_weights(rsc->children->data, rhs, work, attr,
                                           factor, flags);
 
     } else {
         pe_rsc_trace(rsc, "%s: Merging scores from %s (at %.6f)",
                      rhs, rsc->id, factor);
         work = pcmk__copy_node_table(nodes);
         add_node_scores_matching_attr(work, rsc, attr, factor,
                                       pcmk_is_set(flags, pe_weights_positive));
     }
 
     if (can_run_any(work)) {
         GList *gIter = NULL;
         int multiplier = (factor < 0)? -1 : 1;
 
         if (pcmk_is_set(flags, pe_weights_forward)) {
             gIter = rsc->rsc_cons;
             pe_rsc_trace(rsc,
                          "Checking additional %d optional '%s with' constraints",
                          g_list_length(gIter), rsc->id);
 
         } else if (is_nonempty_group(rsc)) {
             pe_resource_t *last_rsc = g_list_last(rsc->children)->data;
 
             gIter = last_rsc->rsc_cons_lhs;
             pe_rsc_trace(rsc, "Checking additional %d optional 'with group %s' "
                          "constraints using last member %s",
                          g_list_length(gIter), rsc->id, last_rsc->id);
 
         } else {
             gIter = rsc->rsc_cons_lhs;
             pe_rsc_trace(rsc,
                          "Checking additional %d optional 'with %s' constraints",
                          g_list_length(gIter), rsc->id);
         }
 
         for (; gIter != NULL; gIter = gIter->next) {
             pe_resource_t *other = NULL;
             pcmk__colocation_t *constraint = (pcmk__colocation_t *) gIter->data;
 
             if (pcmk_is_set(flags, pe_weights_forward)) {
                 other = constraint->rsc_rh;
             } else if (!pcmk__colocation_has_influence(constraint, NULL)) {
                 continue;
             } else {
                 other = constraint->rsc_lh;
             }
 
             pe_rsc_trace(rsc, "Optionally merging score of '%s' constraint (%s with %s)",
                          constraint->id, constraint->rsc_lh->id,
                          constraint->rsc_rh->id);
             work = pcmk__native_merge_weights(other, rhs, work,
                                               constraint->node_attribute,
                                               multiplier * constraint->score / (float) INFINITY,
                                               flags|pe_weights_rollback);
             pe__show_node_weights(true, NULL, rhs, work, rsc->cluster);
         }
 
     } else if (pcmk_is_set(flags, pe_weights_rollback)) {
         pe_rsc_info(rsc, "%s: Rolling back optional scores from %s",
                     rhs, rsc->id);
         g_hash_table_destroy(work);
         pe__clear_resource_flags(rsc, pe_rsc_merging);
         return nodes;
     }
 
 
     if (pcmk_is_set(flags, pe_weights_positive)) {
         pe_node_t *node = NULL;
         GHashTableIter iter;
 
         g_hash_table_iter_init(&iter, work);
         while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
             if (node->weight == INFINITY_HACK) {
                 node->weight = 1;
             }
         }
     }
 
     if (nodes) {
         g_hash_table_destroy(nodes);
     }
 
     pe__clear_resource_flags(rsc, pe_rsc_merging);
     return work;
 }
 
 static inline bool
 node_has_been_unfenced(pe_node_t *node)
 {
     const char *unfenced = pe_node_attribute_raw(node, CRM_ATTR_UNFENCED);
 
     return !pcmk__str_eq(unfenced, "0", pcmk__str_null_matches);
 }
 
 static inline bool
 is_unfence_device(pe_resource_t *rsc, pe_working_set_t *data_set)
 {
     return pcmk_is_set(rsc->flags, pe_rsc_fence_device)
            && pcmk_is_set(data_set->flags, pe_flag_enable_unfencing);
 }
 
 pe_node_t *
 pcmk__native_allocate(pe_resource_t *rsc, pe_node_t *prefer,
                       pe_working_set_t *data_set)
 {
     GList *gIter = NULL;
 
     if (rsc->parent && !pcmk_is_set(rsc->parent->flags, pe_rsc_allocating)) {
         /* never allocate children on their own */
         pe_rsc_debug(rsc, "Escalating allocation of %s to its parent: %s", rsc->id,
                      rsc->parent->id);
         rsc->parent->cmds->allocate(rsc->parent, prefer, data_set);
     }
 
     if (!pcmk_is_set(rsc->flags, pe_rsc_provisional)) {
         return rsc->allocated_to;
     }
 
     if (pcmk_is_set(rsc->flags, pe_rsc_allocating)) {
         pe_rsc_debug(rsc, "Dependency loop detected involving %s", rsc->id);
         return NULL;
     }
 
     pe__set_resource_flags(rsc, pe_rsc_allocating);
     pe__show_node_weights(true, rsc, "Pre-alloc", rsc->allowed_nodes, data_set);
 
     for (gIter = rsc->rsc_cons; gIter != NULL; gIter = gIter->next) {
         pcmk__colocation_t *constraint = (pcmk__colocation_t *) gIter->data;
 
         GHashTable *archive = NULL;
         pe_resource_t *rsc_rh = constraint->rsc_rh;
 
         if ((constraint->role_lh >= RSC_ROLE_PROMOTED)
             || (constraint->score < 0 && constraint->score > -INFINITY)) {
             archive = pcmk__copy_node_table(rsc->allowed_nodes);
         }
 
         pe_rsc_trace(rsc,
                      "%s: Allocating %s first (constraint=%s score=%d role=%s)",
                      rsc->id, rsc_rh->id, constraint->id,
                      constraint->score, role2text(constraint->role_lh));
         rsc_rh->cmds->allocate(rsc_rh, NULL, data_set);
         rsc->cmds->rsc_colocation_lh(rsc, rsc_rh, constraint, data_set);
         if (archive && can_run_any(rsc->allowed_nodes) == FALSE) {
             pe_rsc_info(rsc, "%s: Rolling back scores from %s", rsc->id, rsc_rh->id);
             g_hash_table_destroy(rsc->allowed_nodes);
             rsc->allowed_nodes = archive;
             archive = NULL;
         }
         if (archive) {
             g_hash_table_destroy(archive);
         }
     }
 
     pe__show_node_weights(true, rsc, "Post-coloc", rsc->allowed_nodes, data_set);
 
     for (gIter = rsc->rsc_cons_lhs; gIter != NULL; gIter = gIter->next) {
         pcmk__colocation_t *constraint = (pcmk__colocation_t *) gIter->data;
 
         if (!pcmk__colocation_has_influence(constraint, NULL)) {
             continue;
         }
         pe_rsc_trace(rsc, "Merging score of '%s' constraint (%s with %s)",
                      constraint->id, constraint->rsc_lh->id,
                      constraint->rsc_rh->id);
         rsc->allowed_nodes =
             constraint->rsc_lh->cmds->merge_weights(constraint->rsc_lh, rsc->id, rsc->allowed_nodes,
                                                     constraint->node_attribute,
                                                     (float)constraint->score / INFINITY,
                                                     pe_weights_rollback);
     }
 
     if (rsc->next_role == RSC_ROLE_STOPPED) {
         pe_rsc_trace(rsc, "Making sure %s doesn't get allocated", rsc->id);
         /* make sure it doesn't come up again */
         resource_location(rsc, NULL, -INFINITY, XML_RSC_ATTR_TARGET_ROLE, data_set);
 
     } else if(rsc->next_role > rsc->role
               && !pcmk_is_set(data_set->flags, pe_flag_have_quorum)
               && data_set->no_quorum_policy == no_quorum_freeze) {
         crm_notice("Resource %s cannot be elevated from %s to %s: no-quorum-policy=freeze",
                    rsc->id, role2text(rsc->role), role2text(rsc->next_role));
         pe__set_next_role(rsc, rsc->role, "no-quorum-policy=freeze");
     }
 
     pe__show_node_weights(!pcmk_is_set(data_set->flags, pe_flag_show_scores),
                           rsc, __func__, rsc->allowed_nodes, data_set);
     if (pcmk_is_set(data_set->flags, pe_flag_stonith_enabled)
         && !pcmk_is_set(data_set->flags, pe_flag_have_stonith_resource)) {
         pe__clear_resource_flags(rsc, pe_rsc_managed);
     }
 
     if (!pcmk_is_set(rsc->flags, pe_rsc_managed)) {
         const char *reason = NULL;
         pe_node_t *assign_to = NULL;
 
         pe__set_next_role(rsc, rsc->role, "unmanaged");
         assign_to = pe__current_node(rsc);
         if (assign_to == NULL) {
             reason = "inactive";
         } else if (rsc->role == RSC_ROLE_PROMOTED) {
             reason = "promoted";
         } else if (pcmk_is_set(rsc->flags, pe_rsc_failed)) {
             reason = "failed";
         } else {
             reason = "active";
         }
         pe_rsc_info(rsc, "Unmanaged resource %s allocated to %s: %s", rsc->id,
                     (assign_to? assign_to->details->uname : "no node"), reason);
         native_assign_node(rsc, assign_to, TRUE);
 
     } else if (pcmk_is_set(data_set->flags, pe_flag_stop_everything)) {
         pe_rsc_debug(rsc, "Forcing %s to stop", rsc->id);
         native_assign_node(rsc, NULL, TRUE);
 
     } else if (pcmk_is_set(rsc->flags, pe_rsc_provisional)
                && native_choose_node(rsc, prefer, data_set)) {
         pe_rsc_trace(rsc, "Allocated resource %s to %s", rsc->id,
                      rsc->allocated_to->details->uname);
 
     } else if (rsc->allocated_to == NULL) {
         if (!pcmk_is_set(rsc->flags, pe_rsc_orphan)) {
             pe_rsc_info(rsc, "Resource %s cannot run anywhere", rsc->id);
         } else if (rsc->running_on != NULL) {
             pe_rsc_info(rsc, "Stopping orphan resource %s", rsc->id);
         }
 
     } else {
         pe_rsc_debug(rsc, "Pre-Allocated resource %s to %s", rsc->id,
                      rsc->allocated_to->details->uname);
     }
 
     pe__clear_resource_flags(rsc, pe_rsc_allocating);
 
     if (rsc->is_remote_node) {
         pe_node_t *remote_node = pe_find_node(data_set->nodes, rsc->id);
 
         CRM_ASSERT(remote_node != NULL);
         if (rsc->allocated_to && rsc->next_role != RSC_ROLE_STOPPED) {
             crm_trace("Setting Pacemaker Remote node %s to ONLINE",
                       remote_node->details->id);
             remote_node->details->online = TRUE;
             /* We shouldn't consider an unseen remote-node unclean if we are going
              * to try and connect to it. Otherwise we get an unnecessary fence */
             if (remote_node->details->unseen == TRUE) {
                 remote_node->details->unclean = FALSE;
             }
 
         } else {
             crm_trace("Setting Pacemaker Remote node %s to SHUTDOWN (next role %s, %sallocated)",
                       remote_node->details->id, role2text(rsc->next_role),
                       (rsc->allocated_to? "" : "un"));
             remote_node->details->shutdown = TRUE;
         }
     }
 
     return rsc->allocated_to;
 }
 
 static gboolean
 is_op_dup(pe_resource_t *rsc, const char *name, guint interval_ms)
 {
     gboolean dup = FALSE;
     const char *id = NULL;
     const char *value = NULL;
     xmlNode *operation = NULL;
     guint interval2_ms = 0;
 
     CRM_ASSERT(rsc);
     for (operation = pcmk__xe_first_child(rsc->ops_xml); operation != NULL;
          operation = pcmk__xe_next(operation)) {
 
         if (pcmk__str_eq((const char *)operation->name, "op", pcmk__str_none)) {
             value = crm_element_value(operation, "name");
             if (!pcmk__str_eq(value, name, pcmk__str_casei)) {
                 continue;
             }
 
             value = crm_element_value(operation, XML_LRM_ATTR_INTERVAL);
             interval2_ms = crm_parse_interval_spec(value);
             if (interval_ms != interval2_ms) {
                 continue;
             }
 
             if (id == NULL) {
                 id = ID(operation);
 
             } else {
                 pcmk__config_err("Operation %s is duplicate of %s (do not use "
                                  "same name and interval combination more "
                                  "than once per resource)", ID(operation), id);
                 dup = TRUE;
             }
         }
     }
 
     return dup;
 }
 
 static bool
 op_cannot_recur(const char *name)
 {
     return pcmk__strcase_any_of(name, RSC_STOP, RSC_START, RSC_DEMOTE, RSC_PROMOTE, NULL);
 }
 
 static void
 RecurringOp(pe_resource_t * rsc, pe_action_t * start, pe_node_t * node,
             xmlNode * operation, pe_working_set_t * data_set)
 {
     char *key = NULL;
     const char *name = NULL;
     const char *role = NULL;
     const char *interval_spec = NULL;
     const char *node_uname = node? node->details->uname : "n/a";
 
     guint interval_ms = 0;
     pe_action_t *mon = NULL;
     gboolean is_optional = TRUE;
     GList *possible_matches = NULL;
 
     CRM_ASSERT(rsc);
 
     /* Only process for the operations without role="Stopped" */
     role = crm_element_value(operation, "role");
     if (role && text2role(role) == RSC_ROLE_STOPPED) {
         return;
     }
 
     interval_spec = crm_element_value(operation, XML_LRM_ATTR_INTERVAL);
     interval_ms = crm_parse_interval_spec(interval_spec);
     if (interval_ms == 0) {
         return;
     }
 
     name = crm_element_value(operation, "name");
     if (is_op_dup(rsc, name, interval_ms)) {
         crm_trace("Not creating duplicate recurring action %s for %dms %s",
                   ID(operation), interval_ms, name);
         return;
     }
 
     if (op_cannot_recur(name)) {
         pcmk__config_err("Ignoring %s because action '%s' cannot be recurring",
                          ID(operation), name);
         return;
     }
 
     key = pcmk__op_key(rsc->id, name, interval_ms);
     if (find_rsc_op_entry(rsc, key) == NULL) {
         crm_trace("Not creating recurring action %s for disabled resource %s",
                   ID(operation), rsc->id);
         free(key);
         return;
     }
 
     pe_rsc_trace(rsc, "Creating recurring action %s for %s in role %s on %s",
                  ID(operation), rsc->id, role2text(rsc->next_role), node_uname);
 
     if (start != NULL) {
         pe_rsc_trace(rsc, "Marking %s %s due to %s", key,
                      pcmk_is_set(start->flags, pe_action_optional)? "optional" : "mandatory",
                      start->uuid);
         is_optional = (rsc->cmds->action_flags(start, NULL) & pe_action_optional);
     } else {
         pe_rsc_trace(rsc, "Marking %s optional", key);
         is_optional = TRUE;
     }
 
     /* start a monitor for an already active resource */
     possible_matches = find_actions_exact(rsc->actions, key, node);
     if (possible_matches == NULL) {
         is_optional = FALSE;
         pe_rsc_trace(rsc, "Marking %s mandatory: not active", key);
 
     } else {
         GList *gIter = NULL;
 
         for (gIter = possible_matches; gIter != NULL; gIter = gIter->next) {
             pe_action_t *op = (pe_action_t *) gIter->data;
 
             if (pcmk_is_set(op->flags, pe_action_reschedule)) {
                 is_optional = FALSE;
                 break;
             }
         }
         g_list_free(possible_matches);
     }
 
     if (((rsc->next_role == RSC_ROLE_PROMOTED) && (role == NULL))
         || (role != NULL && text2role(role) != rsc->next_role)) {
         int log_level = LOG_TRACE;
         const char *result = "Ignoring";
 
         if (is_optional) {
             char *after_key = NULL;
             pe_action_t *cancel_op = NULL;
 
             // It's running, so cancel it
             log_level = LOG_INFO;
             result = "Cancelling";
             cancel_op = pe_cancel_op(rsc, name, interval_ms, node, data_set);
 
             switch (rsc->role) {
                 case RSC_ROLE_UNPROMOTED:
                 case RSC_ROLE_STARTED:
                     if (rsc->next_role == RSC_ROLE_PROMOTED) {
                         after_key = promote_key(rsc);
 
                     } else if (rsc->next_role == RSC_ROLE_STOPPED) {
                         after_key = stop_key(rsc);
                     }
 
                     break;
                 case RSC_ROLE_PROMOTED:
                     after_key = demote_key(rsc);
                     break;
                 default:
                     break;
             }
 
             if (after_key) {
                 custom_action_order(rsc, NULL, cancel_op, rsc, after_key, NULL,
                                     pe_order_runnable_left, data_set);
             }
         }
 
         do_crm_log(log_level, "%s action %s (%s vs. %s)",
                    result, key, role ? role : role2text(RSC_ROLE_UNPROMOTED),
                    role2text(rsc->next_role));
 
         free(key);
         return;
     }
 
     mon = custom_action(rsc, key, name, node, is_optional, TRUE, data_set);
     key = mon->uuid;
     if (is_optional) {
         pe_rsc_trace(rsc, "%s\t   %s (optional)", node_uname, mon->uuid);
     }
 
     if ((start == NULL) || !pcmk_is_set(start->flags, pe_action_runnable)) {
         pe_rsc_debug(rsc, "%s\t   %s (cancelled : start un-runnable)",
                      node_uname, mon->uuid);
-        update_action_flags(mon, pe_action_runnable | pe_action_clear,
-                            __func__, __LINE__);
+        pe__clear_action_flags(mon, pe_action_runnable);
 
     } else if (node == NULL || node->details->online == FALSE || node->details->unclean) {
         pe_rsc_debug(rsc, "%s\t   %s (cancelled : no node available)",
                      node_uname, mon->uuid);
-        update_action_flags(mon, pe_action_runnable | pe_action_clear,
-                            __func__, __LINE__);
+        pe__clear_action_flags(mon, pe_action_runnable);
 
     } else if (!pcmk_is_set(mon->flags, pe_action_optional)) {
         pe_rsc_info(rsc, " Start recurring %s (%us) for %s on %s",
                     mon->task, interval_ms / 1000, rsc->id, node_uname);
     }
 
     if (rsc->next_role == RSC_ROLE_PROMOTED) {
         char *running_promoted = pcmk__itoa(PCMK_OCF_RUNNING_PROMOTED);
 
         add_hash_param(mon->meta, XML_ATTR_TE_TARGET_RC, running_promoted);
         free(running_promoted);
     }
 
     if ((node == NULL) || pcmk_is_set(rsc->flags, pe_rsc_managed)) {
         custom_action_order(rsc, start_key(rsc), NULL,
                             NULL, strdup(key), mon,
                             pe_order_implies_then | pe_order_runnable_left, data_set);
 
         custom_action_order(rsc, reload_key(rsc), NULL,
                             NULL, strdup(key), mon,
                             pe_order_implies_then | pe_order_runnable_left, data_set);
 
         if (rsc->next_role == RSC_ROLE_PROMOTED) {
             custom_action_order(rsc, promote_key(rsc), NULL,
                                 rsc, NULL, mon,
                                 pe_order_optional | pe_order_runnable_left, data_set);
 
         } else if (rsc->role == RSC_ROLE_PROMOTED) {
             custom_action_order(rsc, demote_key(rsc), NULL,
                                 rsc, NULL, mon,
                                 pe_order_optional | pe_order_runnable_left, data_set);
         }
     }
 }
 
 static void
 Recurring(pe_resource_t * rsc, pe_action_t * start, pe_node_t * node, pe_working_set_t * data_set)
 {
     if (!pcmk_is_set(rsc->flags, pe_rsc_maintenance) &&
         (node == NULL || node->details->maintenance == FALSE)) {
         xmlNode *operation = NULL;
 
         for (operation = pcmk__xe_first_child(rsc->ops_xml);
              operation != NULL;
              operation = pcmk__xe_next(operation)) {
 
             if (pcmk__str_eq((const char *)operation->name, "op", pcmk__str_none)) {
                 RecurringOp(rsc, start, node, operation, data_set);
             }
         }
     }
 }
 
 static void
 RecurringOp_Stopped(pe_resource_t * rsc, pe_action_t * start, pe_node_t * node,
                     xmlNode * operation, pe_working_set_t * data_set)
 {
     char *key = NULL;
     const char *name = NULL;
     const char *role = NULL;
     const char *interval_spec = NULL;
     const char *node_uname = node? node->details->uname : "n/a";
 
     guint interval_ms = 0;
     GList *possible_matches = NULL;
     GList *gIter = NULL;
 
     /* Only process for the operations with role="Stopped" */
     role = crm_element_value(operation, "role");
     if (role == NULL || text2role(role) != RSC_ROLE_STOPPED) {
         return;
     }
 
     interval_spec = crm_element_value(operation, XML_LRM_ATTR_INTERVAL);
     interval_ms = crm_parse_interval_spec(interval_spec);
     if (interval_ms == 0) {
         return;
     }
 
     name = crm_element_value(operation, "name");
     if (is_op_dup(rsc, name, interval_ms)) {
         crm_trace("Not creating duplicate recurring action %s for %dms %s",
                   ID(operation), interval_ms, name);
         return;
     }
 
     if (op_cannot_recur(name)) {
         pcmk__config_err("Ignoring %s because action '%s' cannot be recurring",
                          ID(operation), name);
         return;
     }
 
     key = pcmk__op_key(rsc->id, name, interval_ms);
     if (find_rsc_op_entry(rsc, key) == NULL) {
         crm_trace("Not creating recurring action %s for disabled resource %s",
                   ID(operation), rsc->id);
         free(key);
         return;
     }
 
     // @TODO add support
     if (!pcmk_is_set(rsc->flags, pe_rsc_unique)) {
         crm_notice("Ignoring %s (recurring monitors for Stopped role are "
                    "not supported for anonymous clones)",
                    ID(operation));
         return;
     }
 
     pe_rsc_trace(rsc,
                  "Creating recurring action %s for %s in role %s on nodes where it should not be running",
                  ID(operation), rsc->id, role2text(rsc->next_role));
 
     /* if the monitor exists on the node where the resource will be running, cancel it */
     if (node != NULL) {
         possible_matches = find_actions_exact(rsc->actions, key, node);
         if (possible_matches) {
             pe_action_t *cancel_op = NULL;
 
             g_list_free(possible_matches);
 
             cancel_op = pe_cancel_op(rsc, name, interval_ms, node, data_set);
 
             if ((rsc->next_role == RSC_ROLE_STARTED)
                 || (rsc->next_role == RSC_ROLE_UNPROMOTED)) {
                 /* rsc->role == RSC_ROLE_STOPPED: cancel the monitor before start */
                 /* rsc->role == RSC_ROLE_STARTED: for a migration, cancel the monitor on the target node before start */
                 custom_action_order(rsc, NULL, cancel_op, rsc, start_key(rsc), NULL,
                                     pe_order_runnable_left, data_set);
             }
 
             pe_rsc_info(rsc, "Cancel action %s (%s vs. %s) on %s",
                         key, role, role2text(rsc->next_role), node_uname);
         }
     }
 
     for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
         pe_node_t *stop_node = (pe_node_t *) gIter->data;
         const char *stop_node_uname = stop_node->details->uname;
         gboolean is_optional = TRUE;
         gboolean probe_is_optional = TRUE;
         gboolean stop_is_optional = TRUE;
         pe_action_t *stopped_mon = NULL;
         char *rc_inactive = NULL;
         GList *probe_complete_ops = NULL;
         GList *stop_ops = NULL;
         GList *local_gIter = NULL;
 
         if (node && pcmk__str_eq(stop_node_uname, node_uname, pcmk__str_casei)) {
             continue;
         }
 
         pe_rsc_trace(rsc, "Creating recurring action %s for %s on %s",
                      ID(operation), rsc->id, crm_str(stop_node_uname));
 
         /* start a monitor for an already stopped resource */
         possible_matches = find_actions_exact(rsc->actions, key, stop_node);
         if (possible_matches == NULL) {
             pe_rsc_trace(rsc, "Marking %s mandatory on %s: not active", key,
                          crm_str(stop_node_uname));
             is_optional = FALSE;
         } else {
             pe_rsc_trace(rsc, "Marking %s optional on %s: already active", key,
                          crm_str(stop_node_uname));
             is_optional = TRUE;
             g_list_free(possible_matches);
         }
 
         stopped_mon = custom_action(rsc, strdup(key), name, stop_node, is_optional, TRUE, data_set);
 
         rc_inactive = pcmk__itoa(PCMK_OCF_NOT_RUNNING);
         add_hash_param(stopped_mon->meta, XML_ATTR_TE_TARGET_RC, rc_inactive);
         free(rc_inactive);
 
         if (pcmk_is_set(rsc->flags, pe_rsc_managed)) {
             GList *probes = pe__resource_actions(rsc, stop_node, RSC_STATUS,
                                                  FALSE);
             GList *pIter = NULL;
 
             for (pIter = probes; pIter != NULL; pIter = pIter->next) {
                 pe_action_t *probe = (pe_action_t *) pIter->data;
 
                 order_actions(probe, stopped_mon, pe_order_runnable_left);
                 crm_trace("%s then %s on %s", probe->uuid, stopped_mon->uuid, stop_node->details->uname);
             }
 
             g_list_free(probes);
         }
 
         if (probe_complete_ops) {
             g_list_free(probe_complete_ops);
         }
 
         stop_ops = pe__resource_actions(rsc, stop_node, RSC_STOP, TRUE);
 
         for (local_gIter = stop_ops; local_gIter != NULL; local_gIter = local_gIter->next) {
             pe_action_t *stop = (pe_action_t *) local_gIter->data;
 
             if (!pcmk_is_set(stop->flags, pe_action_optional)) {
                 stop_is_optional = FALSE;
             }
 
             if (!pcmk_is_set(stop->flags, pe_action_runnable)) {
                 crm_debug("%s\t   %s (cancelled : stop un-runnable)",
                           crm_str(stop_node_uname), stopped_mon->uuid);
-                update_action_flags(stopped_mon, pe_action_runnable | pe_action_clear,
-                                    __func__, __LINE__);
+                pe__clear_action_flags(stopped_mon, pe_action_runnable);
             }
 
             if (pcmk_is_set(rsc->flags, pe_rsc_managed)) {
                 custom_action_order(rsc, stop_key(rsc), stop,
                                     NULL, strdup(key), stopped_mon,
                                     pe_order_implies_then | pe_order_runnable_left, data_set);
             }
 
         }
 
         if (stop_ops) {
             g_list_free(stop_ops);
         }
 
         if (is_optional == FALSE && probe_is_optional && stop_is_optional
             && !pcmk_is_set(rsc->flags, pe_rsc_managed)) {
             pe_rsc_trace(rsc, "Marking %s optional on %s due to unmanaged",
                          key, crm_str(stop_node_uname));
-            update_action_flags(stopped_mon, pe_action_optional, __func__,
-                                __LINE__);
+            pe__set_action_flags(stopped_mon, pe_action_optional);
         }
 
         if (pcmk_is_set(stopped_mon->flags, pe_action_optional)) {
             pe_rsc_trace(rsc, "%s\t   %s (optional)", crm_str(stop_node_uname), stopped_mon->uuid);
         }
 
         if (stop_node->details->online == FALSE || stop_node->details->unclean) {
             pe_rsc_debug(rsc, "%s\t   %s (cancelled : no node available)",
                          crm_str(stop_node_uname), stopped_mon->uuid);
-            update_action_flags(stopped_mon, pe_action_runnable | pe_action_clear,
-                                __func__, __LINE__);
+            pe__clear_action_flags(stopped_mon, pe_action_runnable);
         }
 
         if (pcmk_is_set(stopped_mon->flags, pe_action_runnable)
             && !pcmk_is_set(stopped_mon->flags, pe_action_optional)) {
             crm_notice(" Start recurring %s (%us) for %s on %s", stopped_mon->task,
                        interval_ms / 1000, rsc->id, crm_str(stop_node_uname));
         }
     }
 
     free(key);
 }
 
 static void
 Recurring_Stopped(pe_resource_t * rsc, pe_action_t * start, pe_node_t * node, pe_working_set_t * data_set)
 {
     if (!pcmk_is_set(rsc->flags, pe_rsc_maintenance) &&
         (node == NULL || node->details->maintenance == FALSE)) {
         xmlNode *operation = NULL;
 
         for (operation = pcmk__xe_first_child(rsc->ops_xml);
              operation != NULL;
              operation = pcmk__xe_next(operation)) {
 
             if (pcmk__str_eq((const char *)operation->name, "op", pcmk__str_none)) {
                 RecurringOp_Stopped(rsc, start, node, operation, data_set);
             }
         }
     }
 }
 
 static void
 handle_migration_actions(pe_resource_t * rsc, pe_node_t *current, pe_node_t *chosen, pe_working_set_t * data_set)
 {
     pe_action_t *migrate_to = NULL;
     pe_action_t *migrate_from = NULL;
     pe_action_t *start = NULL;
     pe_action_t *stop = NULL;
     gboolean partial = rsc->partial_migration_target ? TRUE : FALSE;
 
     pe_rsc_trace(rsc, "Processing migration actions %s moving from %s to %s . partial migration = %s",
     rsc->id, current->details->id, chosen->details->id, partial ? "TRUE" : "FALSE");
     start = start_action(rsc, chosen, TRUE);
     stop = stop_action(rsc, current, TRUE);
 
     if (partial == FALSE) {
         migrate_to = custom_action(rsc, pcmk__op_key(rsc->id, RSC_MIGRATE, 0),
                                    RSC_MIGRATE, current, TRUE, TRUE, data_set);
     }
 
     migrate_from = custom_action(rsc, pcmk__op_key(rsc->id, RSC_MIGRATED, 0),
                                  RSC_MIGRATED, chosen, TRUE, TRUE, data_set);
 
     if ((migrate_to && migrate_from) || (migrate_from && partial)) {
 
         pe__set_action_flags(start, pe_action_migrate_runnable);
         pe__set_action_flags(stop, pe_action_migrate_runnable);
 
         // This is easier than trying to delete it from the graph
-        update_action_flags(start, pe_action_pseudo, __func__, __LINE__);
+        pe__set_action_flags(start, pe_action_pseudo);
 
         /* order probes before migrations */
         if (partial) {
             pe__set_action_flags(migrate_from, pe_action_migrate_runnable);
             migrate_from->needs = start->needs;
 
             custom_action_order(rsc, pcmk__op_key(rsc->id, RSC_STATUS, 0), NULL,
                                 rsc, pcmk__op_key(rsc->id, RSC_MIGRATED, 0),
                                 NULL, pe_order_optional, data_set);
 
         } else {
             pe__set_action_flags(migrate_from, pe_action_migrate_runnable);
             pe__set_action_flags(migrate_to, pe_action_migrate_runnable);
             migrate_to->needs = start->needs;
 
             custom_action_order(rsc, pcmk__op_key(rsc->id, RSC_STATUS, 0), NULL,
                                 rsc, pcmk__op_key(rsc->id, RSC_MIGRATE, 0),
                                 NULL, pe_order_optional, data_set);
             custom_action_order(rsc, pcmk__op_key(rsc->id, RSC_MIGRATE, 0),
                                 NULL, rsc,
                                 pcmk__op_key(rsc->id, RSC_MIGRATED, 0), NULL,
                                 pe_order_optional|pe_order_implies_first_migratable,
                                 data_set);
         }
 
         custom_action_order(rsc, pcmk__op_key(rsc->id, RSC_MIGRATED, 0), NULL,
                             rsc, pcmk__op_key(rsc->id, RSC_STOP, 0), NULL,
                             pe_order_optional|pe_order_implies_first_migratable,
                             data_set);
         custom_action_order(rsc, pcmk__op_key(rsc->id, RSC_MIGRATED, 0), NULL,
                             rsc, pcmk__op_key(rsc->id, RSC_START, 0), NULL,
                             pe_order_optional|pe_order_implies_first_migratable|pe_order_pseudo_left,
                             data_set);
 
     }
 
     if (migrate_to) {
         add_hash_param(migrate_to->meta, XML_LRM_ATTR_MIGRATE_SOURCE, current->details->uname);
         add_hash_param(migrate_to->meta, XML_LRM_ATTR_MIGRATE_TARGET, chosen->details->uname);
 
         /* Pacemaker Remote connections don't require pending to be recorded in
          * the CIB. We can reduce CIB writes by not setting PENDING for them.
          */
         if (rsc->is_remote_node == FALSE) {
             /* migrate_to takes place on the source node, but can 
              * have an effect on the target node depending on how
              * the agent is written. Because of this, we have to maintain
              * a record that the migrate_to occurred, in case the source node
              * loses membership while the migrate_to action is still in-flight.
              */
             add_hash_param(migrate_to->meta, XML_OP_ATTR_PENDING, "true");
         }
     }
 
     if (migrate_from) {
         add_hash_param(migrate_from->meta, XML_LRM_ATTR_MIGRATE_SOURCE, current->details->uname);
         add_hash_param(migrate_from->meta, XML_LRM_ATTR_MIGRATE_TARGET, chosen->details->uname);
     }
 }
 
 void
 native_create_actions(pe_resource_t * rsc, pe_working_set_t * data_set)
 {
     pe_action_t *start = NULL;
     pe_node_t *chosen = NULL;
     pe_node_t *current = NULL;
     gboolean need_stop = FALSE;
     bool need_promote = FALSE;
     gboolean is_moving = FALSE;
     gboolean allow_migrate = pcmk_is_set(rsc->flags, pe_rsc_allow_migrate)? TRUE : FALSE;
 
     GList *gIter = NULL;
     unsigned int num_all_active = 0;
     unsigned int num_clean_active = 0;
     bool multiply_active = FALSE;
     enum rsc_role_e role = RSC_ROLE_UNKNOWN;
     enum rsc_role_e next_role = RSC_ROLE_UNKNOWN;
 
     CRM_ASSERT(rsc);
     chosen = rsc->allocated_to;
     next_role = rsc->next_role;
     if (next_role == RSC_ROLE_UNKNOWN) {
         pe__set_next_role(rsc,
                           (chosen == NULL)? RSC_ROLE_STOPPED : RSC_ROLE_STARTED,
                           "allocation");
     }
     pe_rsc_trace(rsc, "Creating all actions for %s transition from %s to %s (%s) on %s",
                  rsc->id, role2text(rsc->role), role2text(rsc->next_role),
                  ((next_role == RSC_ROLE_UNKNOWN)? "implicit" : "explicit"),
                  ((chosen == NULL)? "no node" : chosen->details->uname));
 
     current = pe__find_active_on(rsc, &num_all_active, &num_clean_active);
 
     for (gIter = rsc->dangling_migrations; gIter != NULL; gIter = gIter->next) {
         pe_node_t *dangling_source = (pe_node_t *) gIter->data;
 
         pe_action_t *stop = NULL;
 
         pe_rsc_trace(rsc, "Creating stop action %sfor %s on %s due to dangling migration",
                      pcmk_is_set(data_set->flags, pe_flag_remove_after_stop)? "and cleanup " : "",
                      rsc->id, dangling_source->details->uname);
         stop = stop_action(rsc, dangling_source, FALSE);
         pe__set_action_flags(stop, pe_action_dangle);
         if (pcmk_is_set(data_set->flags, pe_flag_remove_after_stop)) {
             DeleteRsc(rsc, dangling_source, FALSE, data_set);
         }
     }
 
     if ((num_all_active == 2) && (num_clean_active == 2) && chosen
         && rsc->partial_migration_source && rsc->partial_migration_target
         && (current->details == rsc->partial_migration_source->details)
         && (chosen->details == rsc->partial_migration_target->details)) {
 
         /* The chosen node is still the migration target from a partial
          * migration. Attempt to continue the migration instead of recovering
          * by stopping the resource everywhere and starting it on a single node.
          */
         pe_rsc_trace(rsc, "Will attempt to continue with partial migration "
                      "to target %s from %s",
                      rsc->partial_migration_target->details->id,
                      rsc->partial_migration_source->details->id);
 
     } else if (!pcmk_is_set(rsc->flags, pe_rsc_needs_fencing)) {
         /* If a resource has "requires" set to nothing or quorum, don't consider
          * it active on unclean nodes (similar to how all resources behave when
          * stonith-enabled is false). We can start such resources elsewhere
          * before fencing completes, and if we considered the resource active on
          * the failed node, we would attempt recovery for being active on
          * multiple nodes.
          */
         multiply_active = (num_clean_active > 1);
     } else {
         multiply_active = (num_all_active > 1);
     }
 
     if (multiply_active) {
         if (rsc->partial_migration_target && rsc->partial_migration_source) {
             // Migration was in progress, but we've chosen a different target
             crm_notice("Resource %s can no longer migrate from %s to %s "
                        "(will stop on both nodes)",
                        rsc->id, rsc->partial_migration_source->details->uname,
                        rsc->partial_migration_target->details->uname);
 
         } else {
             const char *class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
 
             // Resource was incorrectly multiply active
             pe_proc_err("%s resource %s is active on %u nodes (%s)",
                         crm_str(class), rsc->id, num_all_active,
                         recovery2text(rsc->recovery_type));
             crm_notice("See https://wiki.clusterlabs.org/wiki/FAQ#Resource_is_Too_Active for more information");
         }
 
         if (rsc->recovery_type == recovery_stop_start) {
             need_stop = TRUE;
         }
 
         /* If by chance a partial migration is in process, but the migration
          * target is not chosen still, clear all partial migration data.
          */
         rsc->partial_migration_source = rsc->partial_migration_target = NULL;
         allow_migrate = FALSE;
     }
 
     if (pcmk_is_set(rsc->flags, pe_rsc_start_pending)) {
         pe_rsc_trace(rsc, "Creating start action for %s to represent already pending start",
                      rsc->id);
         start = start_action(rsc, chosen, TRUE);
         pe__set_action_flags(start, pe_action_print_always);
     }
 
     if (current && chosen && current->details != chosen->details) {
         pe_rsc_trace(rsc, "Moving %s from %s to %s",
                      rsc->id, crm_str(current->details->uname),
                      crm_str(chosen->details->uname));
         is_moving = TRUE;
         need_stop = TRUE;
 
     } else if (pcmk_is_set(rsc->flags, pe_rsc_failed)) {
         if (pcmk_is_set(rsc->flags, pe_rsc_stop)) {
             need_stop = TRUE;
             pe_rsc_trace(rsc, "Recovering %s", rsc->id);
         } else {
             pe_rsc_trace(rsc, "Recovering %s by demotion", rsc->id);
             if (rsc->next_role == RSC_ROLE_PROMOTED) {
                 need_promote = TRUE;
             }
         }
 
     } else if (pcmk_is_set(rsc->flags, pe_rsc_block)) {
         pe_rsc_trace(rsc, "Blocking further actions on %s", rsc->id);
         need_stop = TRUE;
 
     } else if (rsc->role > RSC_ROLE_STARTED && current != NULL && chosen != NULL) {
         pe_rsc_trace(rsc, "Creating start action for promoted resource %s",
                      rsc->id);
         start = start_action(rsc, chosen, TRUE);
         if (!pcmk_is_set(start->flags, pe_action_optional)) {
             // Recovery of a promoted resource
             pe_rsc_trace(rsc, "%s restart is required for recovery", rsc->id);
             need_stop = TRUE;
         }
     }
 
     /* Create any additional actions required when bringing resource down and
      * back up to same level.
      */
     role = rsc->role;
     while (role != RSC_ROLE_STOPPED) {
         next_role = rsc_state_matrix[role][RSC_ROLE_STOPPED];
         pe_rsc_trace(rsc, "Creating %s action to take %s down from %s to %s",
                      (need_stop? "required" : "optional"), rsc->id,
                      role2text(role), role2text(next_role));
         if (rsc_action_matrix[role][next_role] (rsc, current, !need_stop, data_set) == FALSE) {
             break;
         }
         role = next_role;
     }
 
 
     while ((rsc->role <= rsc->next_role) && (role != rsc->role)
            && !pcmk_is_set(rsc->flags, pe_rsc_block)) {
         bool required = need_stop;
 
         next_role = rsc_state_matrix[role][rsc->role];
         if ((next_role == RSC_ROLE_PROMOTED) && need_promote) {
             required = true;
         }
         pe_rsc_trace(rsc, "Creating %s action to take %s up from %s to %s",
                      (required? "required" : "optional"), rsc->id,
                      role2text(role), role2text(next_role));
         if (rsc_action_matrix[role][next_role](rsc, chosen, !required,
                                                data_set) == FALSE) {
             break;
         }
         role = next_role;
     }
     role = rsc->role;
 
     /* Required steps from this role to the next */
     while (role != rsc->next_role) {
         next_role = rsc_state_matrix[role][rsc->next_role];
         pe_rsc_trace(rsc, "Creating action to take %s from %s to %s (ending at %s)",
                      rsc->id, role2text(role), role2text(next_role),
                      role2text(rsc->next_role));
         if (rsc_action_matrix[role][next_role] (rsc, chosen, FALSE, data_set) == FALSE) {
             break;
         }
         role = next_role;
     }
 
     if (pcmk_is_set(rsc->flags, pe_rsc_block)) {
         pe_rsc_trace(rsc, "Not creating recurring monitors for blocked resource %s",
                      rsc->id);
 
     } else if ((rsc->next_role != RSC_ROLE_STOPPED)
                || !pcmk_is_set(rsc->flags, pe_rsc_managed)) {
         pe_rsc_trace(rsc, "Creating recurring monitors for %s resource %s",
                      ((rsc->next_role == RSC_ROLE_STOPPED)? "unmanaged" : "active"),
                      rsc->id);
         start = start_action(rsc, chosen, TRUE);
         Recurring(rsc, start, chosen, data_set);
         Recurring_Stopped(rsc, start, chosen, data_set);
 
     } else {
         pe_rsc_trace(rsc, "Creating recurring monitors for inactive resource %s",
                      rsc->id);
         Recurring_Stopped(rsc, NULL, NULL, data_set);
     }
 
     /* if we are stuck in a partial migration, where the target
      * of the partial migration no longer matches the chosen target.
      * A full stop/start is required */
     if (rsc->partial_migration_target && (chosen == NULL || rsc->partial_migration_target->details != chosen->details)) {
         pe_rsc_trace(rsc, "Not allowing partial migration of %s to continue",
                      rsc->id);
         allow_migrate = FALSE;
 
     } else if (!is_moving || !pcmk_is_set(rsc->flags, pe_rsc_managed)
                || pcmk_any_flags_set(rsc->flags,
                                      pe_rsc_failed|pe_rsc_start_pending)
                || (current && current->details->unclean)
                || rsc->next_role < RSC_ROLE_STARTED) {
 
         allow_migrate = FALSE;
     }
 
     if (allow_migrate) {
         handle_migration_actions(rsc, current, chosen, data_set);
     }
 }
 
 static void
 rsc_avoids_remote_nodes(pe_resource_t *rsc)
 {
     GHashTableIter iter;
     pe_node_t *node = NULL;
     g_hash_table_iter_init(&iter, rsc->allowed_nodes);
     while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
         if (node->details->remote_rsc) {
             node->weight = -INFINITY;
         }
     }
 }
 
 /*!
  * \internal
  * \brief Return allowed nodes as (possibly sorted) list
  *
  * Convert a resource's hash table of allowed nodes to a list. If printing to
  * stdout, sort the list, to keep action ID numbers consistent for regression
  * test output (while avoiding the performance hit on a live cluster).
  *
  * \param[in] rsc       Resource to check for allowed nodes
  * \param[in] data_set  Cluster working set
  *
  * \return List of resource's allowed nodes
  * \note Callers should take care not to rely on the list being sorted.
  */
 static GList *
 allowed_nodes_as_list(pe_resource_t *rsc, pe_working_set_t *data_set)
 {
     GList *allowed_nodes = NULL;
 
     if (rsc->allowed_nodes) {
         allowed_nodes = g_hash_table_get_values(rsc->allowed_nodes);
     }
 
     if (!pcmk__is_daemon) {
         allowed_nodes = g_list_sort(allowed_nodes, sort_node_uname);
     }
 
     return allowed_nodes;
 }
 
 void
 native_internal_constraints(pe_resource_t * rsc, pe_working_set_t * data_set)
 {
     /* This function is on the critical path and worth optimizing as much as possible */
 
     pe_resource_t *top = NULL;
     GList *allowed_nodes = NULL;
     bool check_unfencing = FALSE;
     bool check_utilization = FALSE;
 
     if (!pcmk_is_set(rsc->flags, pe_rsc_managed)) {
         pe_rsc_trace(rsc,
                      "Skipping native constraints for unmanaged resource: %s",
                      rsc->id);
         return;
     }
 
     top = uber_parent(rsc);
 
     // Whether resource requires unfencing
     check_unfencing = !pcmk_is_set(rsc->flags, pe_rsc_fence_device)
                       && pcmk_is_set(data_set->flags, pe_flag_enable_unfencing)
                       && pcmk_is_set(rsc->flags, pe_rsc_needs_unfencing);
 
     // Whether a non-default placement strategy is used
     check_utilization = (g_hash_table_size(rsc->utilization) > 0)
                         && !pcmk__str_eq(data_set->placement_strategy,
                                          "default", pcmk__str_casei);
 
     // Order stops before starts (i.e. restart)
     custom_action_order(rsc, pcmk__op_key(rsc->id, RSC_STOP, 0), NULL,
                         rsc, pcmk__op_key(rsc->id, RSC_START, 0), NULL,
                         pe_order_optional|pe_order_implies_then|pe_order_restart,
                         data_set);
 
     // Promotable ordering: demote before stop, start before promote
     if (pcmk_is_set(top->flags, pe_rsc_promotable)
         || (rsc->role > RSC_ROLE_UNPROMOTED)) {
 
         custom_action_order(rsc, pcmk__op_key(rsc->id, RSC_DEMOTE, 0), NULL,
                             rsc, pcmk__op_key(rsc->id, RSC_STOP, 0), NULL,
                             pe_order_promoted_implies_first, data_set);
 
         custom_action_order(rsc, pcmk__op_key(rsc->id, RSC_START, 0), NULL,
                             rsc, pcmk__op_key(rsc->id, RSC_PROMOTE, 0), NULL,
                             pe_order_runnable_left, data_set);
     }
 
     // Don't clear resource history if probing on same node
     custom_action_order(rsc, pcmk__op_key(rsc->id, CRM_OP_LRM_DELETE, 0),
                         NULL, rsc, pcmk__op_key(rsc->id, RSC_STATUS, 0),
                         NULL, pe_order_same_node|pe_order_then_cancels_first,
                         data_set);
 
     // Certain checks need allowed nodes
     if (check_unfencing || check_utilization || rsc->container) {
         allowed_nodes = allowed_nodes_as_list(rsc, data_set);
     }
 
     if (check_unfencing) {
         /* Check if the node needs to be unfenced first */
 
         for (GList *item = allowed_nodes; item; item = item->next) {
             pe_node_t *node = item->data;
             pe_action_t *unfence = pe_fence_op(node, "on", TRUE, NULL, FALSE, data_set);
 
             crm_debug("Ordering any stops of %s before %s, and any starts after",
                       rsc->id, unfence->uuid);
 
             /*
              * It would be more efficient to order clone resources once,
              * rather than order each instance, but ordering the instance
              * allows us to avoid unnecessary dependencies that might conflict
              * with user constraints.
              *
              * @TODO: This constraint can still produce a transition loop if the
              * resource has a stop scheduled on the node being unfenced, and
              * there is a user ordering constraint to start some other resource
              * (which will be ordered after the unfence) before stopping this
              * resource. An example is "start some slow-starting cloned service
              * before stopping an associated virtual IP that may be moving to
              * it":
              *       stop this -> unfencing -> start that -> stop this
              */
             custom_action_order(rsc, stop_key(rsc), NULL,
                                 NULL, strdup(unfence->uuid), unfence,
                                 pe_order_optional|pe_order_same_node, data_set);
 
             custom_action_order(NULL, strdup(unfence->uuid), unfence,
                                 rsc, start_key(rsc), NULL,
                                 pe_order_implies_then_on_node|pe_order_same_node,
                                 data_set);
         }
     }
 
     if (check_utilization) {
         GList *gIter = NULL;
 
         pe_rsc_trace(rsc, "Creating utilization constraints for %s - strategy: %s",
                      rsc->id, data_set->placement_strategy);
 
         for (gIter = rsc->running_on; gIter != NULL; gIter = gIter->next) {
             pe_node_t *current = (pe_node_t *) gIter->data;
 
             char *load_stopped_task = crm_strdup_printf(LOAD_STOPPED "_%s",
                                                         current->details->uname);
             pe_action_t *load_stopped = get_pseudo_op(load_stopped_task, data_set);
 
             if (load_stopped->node == NULL) {
                 load_stopped->node = pe__copy_node(current);
-                update_action_flags(load_stopped, pe_action_optional | pe_action_clear,
-                                    __func__, __LINE__);
+                pe__clear_action_flags(load_stopped, pe_action_optional);
             }
 
             custom_action_order(rsc, stop_key(rsc), NULL,
                                 NULL, load_stopped_task, load_stopped, pe_order_load, data_set);
         }
 
         for (GList *item = allowed_nodes; item; item = item->next) {
             pe_node_t *next = item->data;
             char *load_stopped_task = crm_strdup_printf(LOAD_STOPPED "_%s",
                                                         next->details->uname);
             pe_action_t *load_stopped = get_pseudo_op(load_stopped_task, data_set);
 
             if (load_stopped->node == NULL) {
                 load_stopped->node = pe__copy_node(next);
-                update_action_flags(load_stopped, pe_action_optional | pe_action_clear,
-                                    __func__, __LINE__);
+                pe__clear_action_flags(load_stopped, pe_action_optional);
             }
 
             custom_action_order(NULL, strdup(load_stopped_task), load_stopped,
                                 rsc, start_key(rsc), NULL, pe_order_load, data_set);
 
             custom_action_order(NULL, strdup(load_stopped_task), load_stopped,
                                 rsc, pcmk__op_key(rsc->id, RSC_MIGRATE, 0),
                                 NULL, pe_order_load, data_set);
 
             free(load_stopped_task);
         }
     }
 
     if (rsc->container) {
         pe_resource_t *remote_rsc = NULL;
 
         if (rsc->is_remote_node) {
             // rsc is the implicit remote connection for a guest or bundle node
 
             /* Do not allow a guest resource to live on a Pacemaker Remote node,
              * to avoid nesting remotes. However, allow bundles to run on remote
              * nodes.
              */
             if (!pcmk_is_set(rsc->flags, pe_rsc_allow_remote_remotes)) {
                 rsc_avoids_remote_nodes(rsc->container);
             }
 
             /* If someone cleans up a guest or bundle node's container, we will
              * likely schedule a (re-)probe of the container and recovery of the
              * connection. Order the connection stop after the container probe,
              * so that if we detect the container running, we will trigger a new
              * transition and avoid the unnecessary recovery.
              */
             new_rsc_order(rsc->container, RSC_STATUS, rsc, RSC_STOP,
                           pe_order_optional, data_set);
 
         /* A user can specify that a resource must start on a Pacemaker Remote
          * node by explicitly configuring it with the container=NODENAME
          * meta-attribute. This is of questionable merit, since location
          * constraints can accomplish the same thing. But we support it, so here
          * we check whether a resource (that is not itself a remote connection)
          * has container set to a remote node or guest node resource.
          */
         } else if (rsc->container->is_remote_node) {
             remote_rsc = rsc->container;
         } else  {
             remote_rsc = pe__resource_contains_guest_node(data_set,
                                                           rsc->container);
         }
 
         if (remote_rsc) {
             /* Force the resource on the Pacemaker Remote node instead of
              * colocating the resource with the container resource.
              */
             for (GList *item = allowed_nodes; item; item = item->next) {
                 pe_node_t *node = item->data;
 
                 if (node->details->remote_rsc != remote_rsc) {
                     node->weight = -INFINITY;
                 }
             }
 
         } else {
             /* This resource is either a filler for a container that does NOT
              * represent a Pacemaker Remote node, or a Pacemaker Remote
              * connection resource for a guest node or bundle.
              */
             int score;
 
             crm_trace("Order and colocate %s relative to its container %s",
                       rsc->id, rsc->container->id);
 
             custom_action_order(rsc->container,
                                 pcmk__op_key(rsc->container->id, RSC_START, 0),
                                 NULL, rsc, pcmk__op_key(rsc->id, RSC_START, 0),
                                 NULL,
                                 pe_order_implies_then|pe_order_runnable_left,
                                 data_set);
 
             custom_action_order(rsc, pcmk__op_key(rsc->id, RSC_STOP, 0), NULL,
                                 rsc->container,
                                 pcmk__op_key(rsc->container->id, RSC_STOP, 0),
                                 NULL, pe_order_implies_first, data_set);
 
             if (pcmk_is_set(rsc->flags, pe_rsc_allow_remote_remotes)) {
                 score = 10000;    /* Highly preferred but not essential */
             } else {
                 score = INFINITY; /* Force them to run on the same host */
             }
             pcmk__new_colocation("resource-with-container", NULL, score, rsc,
                                  rsc->container, NULL, NULL, true, data_set);
         }
     }
 
     if (rsc->is_remote_node || pcmk_is_set(rsc->flags, pe_rsc_fence_device)) {
         /* don't allow remote nodes to run stonith devices
          * or remote connection resources.*/
         rsc_avoids_remote_nodes(rsc);
     }
     g_list_free(allowed_nodes);
 }
 
 void
 native_rsc_colocation_lh(pe_resource_t *rsc_lh, pe_resource_t *rsc_rh,
                          pcmk__colocation_t *constraint,
                          pe_working_set_t *data_set)
 {
     if (rsc_lh == NULL) {
         pe_err("rsc_lh was NULL for %s", constraint->id);
         return;
 
     } else if (constraint->rsc_rh == NULL) {
         pe_err("rsc_rh was NULL for %s", constraint->id);
         return;
     }
 
     pe_rsc_trace(rsc_lh, "Processing colocation constraint between %s and %s", rsc_lh->id,
                  rsc_rh->id);
 
     rsc_rh->cmds->rsc_colocation_rh(rsc_lh, rsc_rh, constraint, data_set);
 }
 
 enum filter_colocation_res
 filter_colocation_constraint(pe_resource_t * rsc_lh, pe_resource_t * rsc_rh,
                              pcmk__colocation_t *constraint, gboolean preview)
 {
     /* rh side must be allocated before we can process constraint */
     if (!preview && pcmk_is_set(rsc_rh->flags, pe_rsc_provisional)) {
         return influence_nothing;
     }
 
     if ((constraint->role_lh >= RSC_ROLE_UNPROMOTED) &&
         rsc_lh->parent && pcmk_is_set(rsc_lh->parent->flags, pe_rsc_promotable)
         && !pcmk_is_set(rsc_lh->flags, pe_rsc_provisional)) {
 
         /* LH and RH resources have already been allocated, place the correct
          * priority on LH rsc for the given promotable clone resource role */
         return influence_rsc_priority;
     }
 
     if (!preview && !pcmk_is_set(rsc_lh->flags, pe_rsc_provisional)) {
         // Log an error if we violated a mandatory colocation constraint
         const pe_node_t *rh_node = rsc_rh->allocated_to;
 
         if (rsc_lh->allocated_to == NULL) {
             // Dependent resource isn't allocated, so constraint doesn't matter
             return influence_nothing;
         }
 
         if (constraint->score >= INFINITY) {
             // Dependent resource must colocate with rh_node
 
             if ((rh_node == NULL)
                 || (rh_node->details != rsc_lh->allocated_to->details)) {
                 crm_err("%s must be colocated with %s but is not (%s vs. %s)",
                         rsc_lh->id, rsc_rh->id,
                         rsc_lh->allocated_to->details->uname,
                         (rh_node? rh_node->details->uname : "unallocated"));
             }
 
         } else if (constraint->score <= -INFINITY) {
             // Dependent resource must anti-colocate with rh_node
 
             if ((rh_node != NULL)
                 && (rsc_lh->allocated_to->details == rh_node->details)) {
                 crm_err("%s and %s must be anti-colocated but are allocated "
                         "to the same node (%s)",
                         rsc_lh->id, rsc_rh->id, rh_node->details->uname);
             }
         }
         return influence_nothing;
     }
 
     if (constraint->score > 0
         && constraint->role_lh != RSC_ROLE_UNKNOWN && constraint->role_lh != rsc_lh->next_role) {
         crm_trace("LH: Skipping constraint: \"%s\" state filter nextrole is %s",
                   role2text(constraint->role_lh), role2text(rsc_lh->next_role));
         return influence_nothing;
     }
 
     if (constraint->score > 0
         && constraint->role_rh != RSC_ROLE_UNKNOWN && constraint->role_rh != rsc_rh->next_role) {
         crm_trace("RH: Skipping constraint: \"%s\" state filter", role2text(constraint->role_rh));
         return influence_nothing;
     }
 
     if (constraint->score < 0
         && constraint->role_lh != RSC_ROLE_UNKNOWN && constraint->role_lh == rsc_lh->next_role) {
         crm_trace("LH: Skipping negative constraint: \"%s\" state filter",
                   role2text(constraint->role_lh));
         return influence_nothing;
     }
 
     if (constraint->score < 0
         && constraint->role_rh != RSC_ROLE_UNKNOWN && constraint->role_rh == rsc_rh->next_role) {
         crm_trace("RH: Skipping negative constraint: \"%s\" state filter",
                   role2text(constraint->role_rh));
         return influence_nothing;
     }
 
     return influence_rsc_location;
 }
 
 static void
 influence_priority(pe_resource_t *rsc_lh, pe_resource_t *rsc_rh,
                    pcmk__colocation_t *constraint)
 {
     const char *rh_value = NULL;
     const char *lh_value = NULL;
     const char *attribute = CRM_ATTR_ID;
     int score_multiplier = 1;
 
     if (!rsc_rh->allocated_to || !rsc_lh->allocated_to) {
         return;
     }
 
     if (constraint->node_attribute != NULL) {
         attribute = constraint->node_attribute;
     }
 
     lh_value = pe_node_attribute_raw(rsc_lh->allocated_to, attribute);
     rh_value = pe_node_attribute_raw(rsc_rh->allocated_to, attribute);
 
     if (!pcmk__str_eq(lh_value, rh_value, pcmk__str_casei)) {
         if ((constraint->score == INFINITY)
             && (constraint->role_lh == RSC_ROLE_PROMOTED)) {
             rsc_lh->priority = -INFINITY;
         }
         return;
     }
 
     if (constraint->role_rh && (constraint->role_rh != rsc_rh->next_role)) {
         return;
     }
 
     if (constraint->role_lh == RSC_ROLE_UNPROMOTED) {
         score_multiplier = -1;
     }
 
     rsc_lh->priority = pe__add_scores(score_multiplier * constraint->score,
                                       rsc_lh->priority);
 }
 
 static void
 colocation_match(pe_resource_t *rsc_lh, pe_resource_t *rsc_rh,
                  pcmk__colocation_t *constraint)
 {
     const char *attribute = CRM_ATTR_ID;
     const char *value = NULL;
     GHashTable *work = NULL;
     GHashTableIter iter;
     pe_node_t *node = NULL;
 
     if (constraint->node_attribute != NULL) {
         attribute = constraint->node_attribute;
     }
 
     if (rsc_rh->allocated_to) {
         value = pe_node_attribute_raw(rsc_rh->allocated_to, attribute);
 
     } else if (constraint->score < 0) {
         // Nothing to do (anti-colocation with something that is not running)
         return;
     }
 
     work = pcmk__copy_node_table(rsc_lh->allowed_nodes);
 
     g_hash_table_iter_init(&iter, work);
     while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
         if (rsc_rh->allocated_to == NULL) {
             pe_rsc_trace(rsc_lh, "%s: %s@%s -= %d (%s inactive)",
                          constraint->id, rsc_lh->id, node->details->uname,
                          constraint->score, rsc_rh->id);
             node->weight = pe__add_scores(-constraint->score, node->weight);
 
         } else if (pcmk__str_eq(pe_node_attribute_raw(node, attribute), value, pcmk__str_casei)) {
             if (constraint->score < CRM_SCORE_INFINITY) {
                 pe_rsc_trace(rsc_lh, "%s: %s@%s += %d",
                              constraint->id, rsc_lh->id,
                              node->details->uname, constraint->score);
                 node->weight = pe__add_scores(constraint->score, node->weight);
             }
 
         } else if (constraint->score >= CRM_SCORE_INFINITY) {
             pe_rsc_trace(rsc_lh, "%s: %s@%s -= %d (%s mismatch)",
                          constraint->id, rsc_lh->id, node->details->uname,
                          constraint->score, attribute);
             node->weight = pe__add_scores(-constraint->score, node->weight);
         }
     }
 
     if (can_run_any(work)
         || constraint->score <= -INFINITY || constraint->score >= INFINITY) {
         g_hash_table_destroy(rsc_lh->allowed_nodes);
         rsc_lh->allowed_nodes = work;
         work = NULL;
 
     } else {
         pe_rsc_info(rsc_lh,
                     "%s: Rolling back scores from %s (no available nodes)",
                     rsc_lh->id, rsc_rh->id);
     }
 
     if (work) {
         g_hash_table_destroy(work);
     }
 }
 
 void
 native_rsc_colocation_rh(pe_resource_t *rsc_lh, pe_resource_t *rsc_rh,
                          pcmk__colocation_t *constraint,
                          pe_working_set_t *data_set)
 {
     enum filter_colocation_res filter_results;
 
     CRM_ASSERT(rsc_lh);
     CRM_ASSERT(rsc_rh);
     filter_results = filter_colocation_constraint(rsc_lh, rsc_rh, constraint, FALSE);
     pe_rsc_trace(rsc_lh, "%s %s with %s (%s, score=%d, filter=%d)",
                  ((constraint->score > 0)? "Colocating" : "Anti-colocating"),
                  rsc_lh->id, rsc_rh->id, constraint->id, constraint->score, filter_results);
 
     switch (filter_results) {
         case influence_rsc_priority:
             influence_priority(rsc_lh, rsc_rh, constraint);
             break;
         case influence_rsc_location:
             colocation_match(rsc_lh, rsc_rh, constraint);
             break;
         case influence_nothing:
         default:
             return;
     }
 }
 
 static gboolean
 filter_rsc_ticket(pe_resource_t * rsc_lh, rsc_ticket_t * rsc_ticket)
 {
     if (rsc_ticket->role_lh != RSC_ROLE_UNKNOWN && rsc_ticket->role_lh != rsc_lh->role) {
         pe_rsc_trace(rsc_lh, "LH: Skipping constraint: \"%s\" state filter",
                      role2text(rsc_ticket->role_lh));
         return FALSE;
     }
 
     return TRUE;
 }
 
 void
 rsc_ticket_constraint(pe_resource_t * rsc_lh, rsc_ticket_t * rsc_ticket, pe_working_set_t * data_set)
 {
     if (rsc_ticket == NULL) {
         pe_err("rsc_ticket was NULL");
         return;
     }
 
     if (rsc_lh == NULL) {
         pe_err("rsc_lh was NULL for %s", rsc_ticket->id);
         return;
     }
 
     if (rsc_ticket->ticket->granted && rsc_ticket->ticket->standby == FALSE) {
         return;
     }
 
     if (rsc_lh->children) {
         GList *gIter = rsc_lh->children;
 
         pe_rsc_trace(rsc_lh, "Processing ticket dependencies from %s", rsc_lh->id);
 
         for (; gIter != NULL; gIter = gIter->next) {
             pe_resource_t *child_rsc = (pe_resource_t *) gIter->data;
 
             rsc_ticket_constraint(child_rsc, rsc_ticket, data_set);
         }
         return;
     }
 
     pe_rsc_trace(rsc_lh, "%s: Processing ticket dependency on %s (%s, %s)",
                  rsc_lh->id, rsc_ticket->ticket->id, rsc_ticket->id,
                  role2text(rsc_ticket->role_lh));
 
     if ((rsc_ticket->ticket->granted == FALSE)
         && (rsc_lh->running_on != NULL)) {
 
         GList *gIter = NULL;
 
         switch (rsc_ticket->loss_policy) {
             case loss_ticket_stop:
                 resource_location(rsc_lh, NULL, -INFINITY, "__loss_of_ticket__", data_set);
                 break;
 
             case loss_ticket_demote:
                 // Promotion score will be set to -INFINITY in promotion_order()
                 if (rsc_ticket->role_lh != RSC_ROLE_PROMOTED) {
                     resource_location(rsc_lh, NULL, -INFINITY, "__loss_of_ticket__", data_set);
                 }
                 break;
 
             case loss_ticket_fence:
                 if (filter_rsc_ticket(rsc_lh, rsc_ticket) == FALSE) {
                     return;
                 }
 
                 resource_location(rsc_lh, NULL, -INFINITY, "__loss_of_ticket__", data_set);
 
                 for (gIter = rsc_lh->running_on; gIter != NULL; gIter = gIter->next) {
                     pe_node_t *node = (pe_node_t *) gIter->data;
 
                     pe_fence_node(data_set, node, "deadman ticket was lost", FALSE);
                 }
                 break;
 
             case loss_ticket_freeze:
                 if (filter_rsc_ticket(rsc_lh, rsc_ticket) == FALSE) {
                     return;
                 }
                 if (rsc_lh->running_on != NULL) {
                     pe__clear_resource_flags(rsc_lh, pe_rsc_managed);
                     pe__set_resource_flags(rsc_lh, pe_rsc_block);
                 }
                 break;
         }
 
     } else if (rsc_ticket->ticket->granted == FALSE) {
 
         if ((rsc_ticket->role_lh != RSC_ROLE_PROMOTED)
             || (rsc_ticket->loss_policy == loss_ticket_stop)) {
             resource_location(rsc_lh, NULL, -INFINITY, "__no_ticket__", data_set);
         }
 
     } else if (rsc_ticket->ticket->standby) {
 
         if ((rsc_ticket->role_lh != RSC_ROLE_PROMOTED)
             || (rsc_ticket->loss_policy == loss_ticket_stop)) {
             resource_location(rsc_lh, NULL, -INFINITY, "__ticket_standby__", data_set);
         }
     }
 }
 
 enum pe_action_flags
 native_action_flags(pe_action_t * action, pe_node_t * node)
 {
     return action->flags;
 }
 
 static inline bool
 is_primitive_action(pe_action_t *action)
 {
     return action && action->rsc && (action->rsc->variant == pe_native);
 }
 
 /*!
  * \internal
  * \brief Set action bits appropriately when pe_restart_order is used
  *
  * \param[in] first   'First' action in an ordering with pe_restart_order
  * \param[in] then    'Then' action in an ordering with pe_restart_order
  * \param[in] filter  What ordering flags to care about
  *
  * \note pe_restart_order is set for "stop resource before starting it" and
  *       "stop later group member before stopping earlier group member"
  */
 static void
 handle_restart_ordering(pe_action_t *first, pe_action_t *then,
                         enum pe_action_flags filter)
 {
     const char *reason = NULL;
 
     CRM_ASSERT(is_primitive_action(first));
     CRM_ASSERT(is_primitive_action(then));
 
     // We need to update the action in two cases:
 
     // ... if 'then' is required
     if (pcmk_is_set(filter, pe_action_optional)
         && !pcmk_is_set(then->flags, pe_action_optional)) {
         reason = "restart";
     }
 
     /* ... if 'then' is unrunnable start of managed resource (if a resource
      * should restart but can't start, we still want to stop)
      */
     if (pcmk_is_set(filter, pe_action_runnable)
         && !pcmk_is_set(then->flags, pe_action_runnable)
         && pcmk_is_set(then->rsc->flags, pe_rsc_managed)
         && pcmk__str_eq(then->task, RSC_START, pcmk__str_casei)) {
         reason = "stop";
     }
 
     if (reason == NULL) {
         return;
     }
 
     pe_rsc_trace(first->rsc, "Handling %s -> %s for %s",
                  first->uuid, then->uuid, reason);
 
     // Make 'first' required if it is runnable
     if (pcmk_is_set(first->flags, pe_action_runnable)) {
         pe_action_implies(first, then, pe_action_optional);
     }
 
     // Make 'first' required if 'then' is required
     if (!pcmk_is_set(then->flags, pe_action_optional)) {
         pe_action_implies(first, then, pe_action_optional);
     }
 
     // Make 'first' unmigratable if 'then' is unmigratable
     if (!pcmk_is_set(then->flags, pe_action_migrate_runnable)) {
         pe_action_implies(first, then, pe_action_migrate_runnable);
     }
 
     // Make 'then' unrunnable if 'first' is required but unrunnable
     if (!pcmk_is_set(first->flags, pe_action_optional)
         && !pcmk_is_set(first->flags, pe_action_runnable)) {
         pe_action_implies(then, first, pe_action_runnable);
     }
 }
 
 enum pe_graph_flags
 native_update_actions(pe_action_t *first, pe_action_t *then, pe_node_t *node,
                       enum pe_action_flags flags, enum pe_action_flags filter,
                       enum pe_ordering type, pe_working_set_t *data_set)
 {
     /* flags == get_action_flags(first, then_node) called from update_action() */
     enum pe_graph_flags changed = pe_graph_none;
     enum pe_action_flags then_flags = then->flags;
     enum pe_action_flags first_flags = first->flags;
 
     crm_trace(   "Testing %s on %s (0x%.6x) with %s 0x%.6x",
                  first->uuid, first->node ? first->node->details->uname : "[none]",
                  first->flags, then->uuid, then->flags);
 
     if (type & pe_order_asymmetrical) {
         pe_resource_t *then_rsc = then->rsc;
         enum rsc_role_e then_rsc_role = then_rsc ? then_rsc->fns->state(then_rsc, TRUE) : 0;
 
         if (!then_rsc) {
             /* ignore */
         } else if ((then_rsc_role == RSC_ROLE_STOPPED) && pcmk__str_eq(then->task, RSC_STOP, pcmk__str_casei)) {
             /* ignore... if 'then' is supposed to be stopped after 'first', but
              * then is already stopped, there is nothing to be done when non-symmetrical.  */
         } else if ((then_rsc_role >= RSC_ROLE_STARTED)
                    && pcmk__str_eq(then->task, RSC_START, pcmk__str_casei)
                    && pcmk_is_set(then->flags, pe_action_optional)
                    && then->node
                    && pcmk__list_of_1(then_rsc->running_on)
                    && then->node->details == ((pe_node_t *) then_rsc->running_on->data)->details) {
             /* Ignore. If 'then' is supposed to be started after 'first', but
              * 'then' is already started, there is nothing to be done when
              * asymmetrical -- unless the start is mandatory, which indicates
              * the resource is restarting, and the ordering is still needed.
              */
         } else if (!(first->flags & pe_action_runnable)) {
             /* prevent 'then' action from happening if 'first' is not runnable and
              * 'then' has not yet occurred. */
             pe_action_implies(then, first, pe_action_optional);
             pe_action_implies(then, first, pe_action_runnable);
 
             pe_rsc_trace(then->rsc, "Unset optional and runnable on %s", then->uuid);
         } else {
             /* ignore... then is allowed to start/stop if it wants to. */
         }
     }
 
-    if (type & pe_order_implies_first) {
+    if (pcmk_is_set(type, pe_order_implies_first)
+        && !pcmk_is_set(then_flags, pe_action_optional)) {
+        // Then is required, and implies first should be, too
+
         if (pcmk_is_set(filter, pe_action_optional)
-            && !pcmk_is_set(flags /* Should be then_flags? */, pe_action_optional)) {
-            // Needs pcmk_is_set(first_flags, pe_action_optional) too?
-            pe_rsc_trace(first->rsc, "Unset optional on %s because of %s", first->uuid, then->uuid);
+            && !pcmk_is_set(flags, pe_action_optional)
+            && pcmk_is_set(first_flags, pe_action_optional)) {
+            pe_rsc_trace(first->rsc,
+                         "Unset optional on %s because %s implies first",
+                         first->uuid, then->uuid);
             pe_action_implies(first, then, pe_action_optional);
         }
 
         if (pcmk_is_set(flags, pe_action_migrate_runnable) &&
-            !pcmk_is_set(then->flags, pe_action_migrate_runnable) &&
-            !pcmk_is_set(then->flags, pe_action_optional)) {
+            !pcmk_is_set(then->flags, pe_action_migrate_runnable)) {
 
             pe_rsc_trace(first->rsc, "Unset migrate runnable on %s because of %s",
                          first->uuid, then->uuid);
             pe_action_implies(first, then, pe_action_migrate_runnable);
         }
     }
 
     if (type & pe_order_promoted_implies_first) {
         if ((filter & pe_action_optional) &&
             ((then->flags & pe_action_optional) == FALSE) &&
             (then->rsc != NULL) && (then->rsc->role == RSC_ROLE_PROMOTED)) {
             pe_action_implies(first, then, pe_action_optional);
 
             if (pcmk_is_set(first->flags, pe_action_migrate_runnable) &&
                 !pcmk_is_set(then->flags, pe_action_migrate_runnable)) {
 
                 pe_rsc_trace(first->rsc, "Unset migrate runnable on %s because of %s", first->uuid, then->uuid);
                 pe_action_implies(first, then, pe_action_migrate_runnable);
             }
-            pe_rsc_trace(then->rsc, "Unset optional on %s because of %s", first->uuid, then->uuid);
+            pe_rsc_trace(then->rsc,
+                         "Unset optional on %s because %s (promoted) implies first",
+                         first->uuid, then->uuid);
         }
     }
 
     if ((type & pe_order_implies_first_migratable)
         && pcmk_is_set(filter, pe_action_optional)) {
 
         if (((then->flags & pe_action_migrate_runnable) == FALSE) ||
             ((then->flags & pe_action_runnable) == FALSE)) {
 
             pe_rsc_trace(then->rsc, "Unset runnable on %s because %s is neither runnable or migratable", first->uuid, then->uuid);
             pe_action_implies(first, then, pe_action_runnable);
         }
 
         if ((then->flags & pe_action_optional) == 0) {
             pe_rsc_trace(then->rsc, "Unset optional on %s because %s is not optional", first->uuid, then->uuid);
             pe_action_implies(first, then, pe_action_optional);
         }
     }
 
     if ((type & pe_order_pseudo_left)
         && pcmk_is_set(filter, pe_action_optional)) {
 
         if ((first->flags & pe_action_runnable) == FALSE) {
             pe_action_implies(then, first, pe_action_migrate_runnable);
             pe__clear_action_flags(then, pe_action_pseudo);
             pe_rsc_trace(then->rsc, "Unset pseudo on %s because %s is not runnable", then->uuid, first->uuid);
         }
 
     }
 
     if (pcmk_is_set(type, pe_order_runnable_left)
         && pcmk_is_set(filter, pe_action_runnable)
         && pcmk_is_set(then->flags, pe_action_runnable)
         && !pcmk_is_set(flags, pe_action_runnable)) {
         pe_rsc_trace(then->rsc, "Unset runnable on %s because of %s", then->uuid, first->uuid);
         pe_action_implies(then, first, pe_action_runnable);
         pe_action_implies(then, first, pe_action_migrate_runnable);
     }
 
     if (pcmk_is_set(type, pe_order_implies_then)
         && pcmk_is_set(filter, pe_action_optional)
         && pcmk_is_set(then->flags, pe_action_optional)
-        && !pcmk_is_set(flags, pe_action_optional)) {
+        && !pcmk_is_set(flags, pe_action_optional)
+        && !pcmk_is_set(first->flags, pe_action_migrate_runnable)) {
 
-        /* in this case, treat migrate_runnable as if first is optional */
-        if (!pcmk_is_set(first->flags, pe_action_migrate_runnable)) {
-           pe_rsc_trace(then->rsc, "Unset optional on %s because of %s", then->uuid, first->uuid);
-           pe_action_implies(then, first, pe_action_optional);
-        }
+        pe_rsc_trace(then->rsc,
+                     "Unset optional on %s because %s implies 'then'",
+                     then->uuid, first->uuid);
+        pe_action_implies(then, first, pe_action_optional);
     }
 
     if (pcmk_is_set(type, pe_order_restart)) {
         handle_restart_ordering(first, then, filter);
     }
 
     if (then_flags != then->flags) {
         pe__set_graph_flags(changed, first, pe_graph_updated_then);
         pe_rsc_trace(then->rsc,
                      "Then: Flags for %s on %s are now  0x%.6x (was 0x%.6x) because of %s 0x%.6x",
                      then->uuid, then->node ? then->node->details->uname : "[none]", then->flags,
                      then_flags, first->uuid, first->flags);
 
         if(then->rsc && then->rsc->parent) {
             /* "X_stop then X_start" doesn't get handled for cloned groups unless we do this */
             update_action(then, data_set);
         }
     }
 
     if (first_flags != first->flags) {
         pe__set_graph_flags(changed, first, pe_graph_updated_first);
         pe_rsc_trace(first->rsc,
                      "First: Flags for %s on %s are now  0x%.6x (was 0x%.6x) because of %s 0x%.6x",
                      first->uuid, first->node ? first->node->details->uname : "[none]",
                      first->flags, first_flags, then->uuid, then->flags);
     }
 
     return changed;
 }
 
 void
 native_rsc_location(pe_resource_t *rsc, pe__location_t *constraint)
 {
     GList *gIter = NULL;
     bool need_role = false;
 
     CRM_CHECK((constraint != NULL) && (rsc != NULL), return);
 
     // If a role was specified, ensure constraint is applicable
     need_role = (constraint->role_filter > RSC_ROLE_UNKNOWN);
     if (need_role && (constraint->role_filter != rsc->next_role)) {
         pe_rsc_trace(rsc,
                      "Not applying %s to %s because role will be %s not %s",
                      constraint->id, rsc->id, role2text(rsc->next_role),
                      role2text(constraint->role_filter));
         return;
     }
 
     if (constraint->node_list_rh == NULL) {
         pe_rsc_trace(rsc, "Not applying %s to %s because no nodes match",
                      constraint->id, rsc->id);
         return;
     }
 
     pe_rsc_trace(rsc, "Applying %s%s%s to %s", constraint->id,
                  (need_role? " for role " : ""),
                  (need_role? role2text(constraint->role_filter) : ""), rsc->id);
 
     for (gIter = constraint->node_list_rh; gIter != NULL; gIter = gIter->next) {
         pe_node_t *node = (pe_node_t *) gIter->data;
         pe_node_t *other_node = NULL;
 
         other_node = (pe_node_t *) pe_hash_table_lookup(rsc->allowed_nodes, node->details->id);
 
         if (other_node != NULL) {
             pe_rsc_trace(rsc, "* + %d on %s",
                          node->weight, node->details->uname);
             other_node->weight = pe__add_scores(other_node->weight,
                                                 node->weight);
 
         } else {
             pe_rsc_trace(rsc, "* = %d on %s",
                          node->weight, node->details->uname);
             other_node = pe__copy_node(node);
             g_hash_table_insert(rsc->allowed_nodes, (gpointer) other_node->details->id, other_node);
         }
 
         if (other_node->rsc_discover_mode < constraint->discover_mode) {
             if (constraint->discover_mode == pe_discover_exclusive) {
                 rsc->exclusive_discover = TRUE;
             }
             /* exclusive > never > always... always is default */
             other_node->rsc_discover_mode = constraint->discover_mode;
         }
     }
 }
 
 void
 native_expand(pe_resource_t * rsc, pe_working_set_t * data_set)
 {
     GList *gIter = NULL;
 
     CRM_ASSERT(rsc);
     pe_rsc_trace(rsc, "Processing actions from %s", rsc->id);
 
     for (gIter = rsc->actions; gIter != NULL; gIter = gIter->next) {
         pe_action_t *action = (pe_action_t *) gIter->data;
 
         crm_trace("processing action %d for rsc=%s", action->id, rsc->id);
         graph_element_from_action(action, data_set);
     }
 
     for (gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
         pe_resource_t *child_rsc = (pe_resource_t *) gIter->data;
 
         child_rsc->cmds->expand(child_rsc, data_set);
     }
 }
 
 #define STOP_SANITY_ASSERT(lineno) do {                                 \
         if(current && current->details->unclean) {                      \
             /* It will be a pseudo op */                                \
         } else if(stop == NULL) {                                       \
             crm_err("%s:%d: No stop action exists for %s",              \
                     __func__, lineno, rsc->id);                         \
             CRM_ASSERT(stop != NULL);                                   \
         } else if (pcmk_is_set(stop->flags, pe_action_optional)) {      \
             crm_err("%s:%d: Action %s is still optional",               \
                     __func__, lineno, stop->uuid);                      \
             CRM_ASSERT(!pcmk_is_set(stop->flags, pe_action_optional));  \
         }                                                               \
     } while(0)
 
 void
 LogActions(pe_resource_t * rsc, pe_working_set_t * data_set)
 {
     pcmk__output_t *out = data_set->priv;
 
     pe_node_t *next = NULL;
     pe_node_t *current = NULL;
 
     gboolean moving = FALSE;
 
     if(rsc->variant == pe_container) {
         pcmk__bundle_log_actions(rsc, data_set);
         return;
     }
 
     if (rsc->children) {
         GList *gIter = NULL;
 
         for (gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
             pe_resource_t *child_rsc = (pe_resource_t *) gIter->data;
 
             LogActions(child_rsc, data_set);
         }
         return;
     }
 
     next = rsc->allocated_to;
     if (rsc->running_on) {
         current = pe__current_node(rsc);
         if (rsc->role == RSC_ROLE_STOPPED) {
             /*
              * This can occur when resources are being recovered
              * We fiddle with the current role in native_create_actions()
              */
             rsc->role = RSC_ROLE_STARTED;
         }
     }
 
     if ((current == NULL) && pcmk_is_set(rsc->flags, pe_rsc_orphan)) {
         /* Don't log stopped orphans */
         return;
     }
 
     out->message(out, "rsc-action", rsc, current, next, moving);
 }
 
 gboolean
 StopRsc(pe_resource_t * rsc, pe_node_t * next, gboolean optional, pe_working_set_t * data_set)
 {
     GList *gIter = NULL;
 
     CRM_ASSERT(rsc);
     pe_rsc_trace(rsc, "%s", rsc->id);
 
     for (gIter = rsc->running_on; gIter != NULL; gIter = gIter->next) {
         pe_node_t *current = (pe_node_t *) gIter->data;
         pe_action_t *stop;
 
         if (rsc->partial_migration_target) {
             if (rsc->partial_migration_target->details == current->details) {
                 pe_rsc_trace(rsc, "Filtered %s -> %s %s", current->details->uname,
                              next->details->uname, rsc->id);
                 continue;
             } else {
                 pe_rsc_trace(rsc, "Forced on %s %s", current->details->uname, rsc->id);
                 optional = FALSE;
             }
         }
 
         pe_rsc_trace(rsc, "%s on %s", rsc->id, current->details->uname);
         stop = stop_action(rsc, current, optional);
 
         if(rsc->allocated_to == NULL) {
             pe_action_set_reason(stop, "node availability", TRUE);
         }
 
         if (!pcmk_is_set(rsc->flags, pe_rsc_managed)) {
-            update_action_flags(stop, pe_action_runnable | pe_action_clear,
-                                __func__, __LINE__);
+            pe__clear_action_flags(stop, pe_action_runnable);
         }
 
         if (pcmk_is_set(data_set->flags, pe_flag_remove_after_stop)) {
             DeleteRsc(rsc, current, optional, data_set);
         }
 
         if (pcmk_is_set(rsc->flags, pe_rsc_needs_unfencing)) {
             pe_action_t *unfence = pe_fence_op(current, "on", TRUE, NULL, FALSE, data_set);
 
             order_actions(stop, unfence, pe_order_implies_first);
             if (!node_has_been_unfenced(current)) {
                 pe_proc_err("Stopping %s until %s can be unfenced", rsc->id, current->details->uname);
             }
         }
     }
 
     return TRUE;
 }
 
 static void
 order_after_unfencing(pe_resource_t *rsc, pe_node_t *node, pe_action_t *action,
                       enum pe_ordering order, pe_working_set_t *data_set)
 {
     /* When unfencing is in use, we order unfence actions before any probe or
      * start of resources that require unfencing, and also of fence devices.
      *
      * This might seem to violate the principle that fence devices require
      * only quorum. However, fence agents that unfence often don't have enough
      * information to even probe or start unless the node is first unfenced.
      */
     if (is_unfence_device(rsc, data_set)
         || pcmk_is_set(rsc->flags, pe_rsc_needs_unfencing)) {
 
         /* Start with an optional ordering. Requiring unfencing would result in
          * the node being unfenced, and all its resources being stopped,
          * whenever a new resource is added -- which would be highly suboptimal.
          */
         pe_action_t *unfence = pe_fence_op(node, "on", TRUE, NULL, FALSE, data_set);
 
         order_actions(unfence, action, order);
 
         if (!node_has_been_unfenced(node)) {
             // But unfencing is required if it has never been done
             char *reason = crm_strdup_printf("required by %s %s",
                                              rsc->id, action->task);
 
             trigger_unfencing(NULL, node, reason, NULL, data_set);
             free(reason);
         }
     }
 }
 
 gboolean
 StartRsc(pe_resource_t * rsc, pe_node_t * next, gboolean optional, pe_working_set_t * data_set)
 {
     pe_action_t *start = NULL;
 
     CRM_ASSERT(rsc);
     pe_rsc_trace(rsc, "%s on %s %d %d", rsc->id, next ? next->details->uname : "N/A", optional, next ? next->weight : 0);
     start = start_action(rsc, next, TRUE);
 
     order_after_unfencing(rsc, next, start, pe_order_implies_then, data_set);
 
     if (pcmk_is_set(start->flags, pe_action_runnable) && !optional) {
-        update_action_flags(start, pe_action_optional | pe_action_clear,
-                            __func__, __LINE__);
+        pe__clear_action_flags(start, pe_action_optional);
     }
 
 
     return TRUE;
 }
 
 gboolean
 PromoteRsc(pe_resource_t * rsc, pe_node_t * next, gboolean optional, pe_working_set_t * data_set)
 {
     GList *gIter = NULL;
     gboolean runnable = TRUE;
     GList *action_list = NULL;
 
     CRM_ASSERT(rsc);
     CRM_CHECK(next != NULL, return FALSE);
     pe_rsc_trace(rsc, "%s on %s", rsc->id, next->details->uname);
 
     action_list = pe__resource_actions(rsc, next, RSC_START, TRUE);
 
     for (gIter = action_list; gIter != NULL; gIter = gIter->next) {
         pe_action_t *start = (pe_action_t *) gIter->data;
 
         if (!pcmk_is_set(start->flags, pe_action_runnable)) {
             runnable = FALSE;
         }
     }
     g_list_free(action_list);
 
     if (runnable) {
         promote_action(rsc, next, optional);
         return TRUE;
     }
 
     pe_rsc_debug(rsc, "%s\tPromote %s (canceled)", next->details->uname, rsc->id);
 
     action_list = pe__resource_actions(rsc, next, RSC_PROMOTE, TRUE);
 
     for (gIter = action_list; gIter != NULL; gIter = gIter->next) {
         pe_action_t *promote = (pe_action_t *) gIter->data;
 
-        update_action_flags(promote, pe_action_runnable | pe_action_clear,
-                            __func__, __LINE__);
+        pe__clear_action_flags(promote, pe_action_runnable);
     }
 
     g_list_free(action_list);
     return TRUE;
 }
 
 gboolean
 DemoteRsc(pe_resource_t * rsc, pe_node_t * next, gboolean optional, pe_working_set_t * data_set)
 {
     GList *gIter = NULL;
 
     CRM_ASSERT(rsc);
     pe_rsc_trace(rsc, "%s", rsc->id);
 
     /* CRM_CHECK(rsc->next_role == RSC_ROLE_UNPROMOTED, return FALSE); */
     for (gIter = rsc->running_on; gIter != NULL; gIter = gIter->next) {
         pe_node_t *current = (pe_node_t *) gIter->data;
 
         pe_rsc_trace(rsc, "%s on %s", rsc->id, next ? next->details->uname : "N/A");
         demote_action(rsc, current, optional);
     }
     return TRUE;
 }
 
 gboolean
 RoleError(pe_resource_t * rsc, pe_node_t * next, gboolean optional, pe_working_set_t * data_set)
 {
     CRM_ASSERT(rsc);
     crm_err("%s on %s", rsc->id, next ? next->details->uname : "N/A");
     CRM_CHECK(FALSE, return FALSE);
     return FALSE;
 }
 
 gboolean
 NullOp(pe_resource_t * rsc, pe_node_t * next, gboolean optional, pe_working_set_t * data_set)
 {
     CRM_ASSERT(rsc);
     pe_rsc_trace(rsc, "%s", rsc->id);
     return FALSE;
 }
 
 gboolean
 DeleteRsc(pe_resource_t * rsc, pe_node_t * node, gboolean optional, pe_working_set_t * data_set)
 {
     if (pcmk_is_set(rsc->flags, pe_rsc_failed)) {
         pe_rsc_trace(rsc, "Resource %s not deleted from %s: failed", rsc->id, node->details->uname);
         return FALSE;
 
     } else if (node == NULL) {
         pe_rsc_trace(rsc, "Resource %s not deleted: NULL node", rsc->id);
         return FALSE;
 
     } else if (node->details->unclean || node->details->online == FALSE) {
         pe_rsc_trace(rsc, "Resource %s not deleted from %s: unrunnable", rsc->id,
                      node->details->uname);
         return FALSE;
     }
 
     crm_notice("Removing %s from %s", rsc->id, node->details->uname);
 
     delete_action(rsc, node, optional);
 
     new_rsc_order(rsc, RSC_STOP, rsc, RSC_DELETE,
                   optional ? pe_order_implies_then : pe_order_optional, data_set);
 
     new_rsc_order(rsc, RSC_DELETE, rsc, RSC_START,
                   optional ? pe_order_implies_then : pe_order_optional, data_set);
 
     return TRUE;
 }
 
 gboolean
 native_create_probe(pe_resource_t * rsc, pe_node_t * node, pe_action_t * complete,
                     gboolean force, pe_working_set_t * data_set)
 {
     enum pe_ordering flags = pe_order_optional;
     char *key = NULL;
     pe_action_t *probe = NULL;
     pe_node_t *running = NULL;
     pe_node_t *allowed = NULL;
     pe_resource_t *top = uber_parent(rsc);
 
     static const char *rc_promoted = NULL;
     static const char *rc_inactive = NULL;
 
     if (rc_inactive == NULL) {
         rc_inactive = pcmk__itoa(PCMK_OCF_NOT_RUNNING);
         rc_promoted = pcmk__itoa(PCMK_OCF_RUNNING_PROMOTED);
     }
 
     CRM_CHECK(node != NULL, return FALSE);
     if (!force && !pcmk_is_set(data_set->flags, pe_flag_startup_probes)) {
         pe_rsc_trace(rsc, "Skipping active resource detection for %s", rsc->id);
         return FALSE;
     }
 
     if (pe__is_guest_or_remote_node(node)) {
         const char *class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
 
         if (pcmk__str_eq(class, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_casei)) {
             pe_rsc_trace(rsc,
                          "Skipping probe for %s on %s because Pacemaker Remote nodes cannot run stonith agents",
                          rsc->id, node->details->id);
             return FALSE;
         } else if (pe__is_guest_node(node)
                    && pe__resource_contains_guest_node(data_set, rsc)) {
             pe_rsc_trace(rsc,
                          "Skipping probe for %s on %s because guest nodes cannot run resources containing guest nodes",
                          rsc->id, node->details->id);
             return FALSE;
         } else if (rsc->is_remote_node) {
             pe_rsc_trace(rsc,
                          "Skipping probe for %s on %s because Pacemaker Remote nodes cannot host remote connections",
                          rsc->id, node->details->id);
             return FALSE;
         }
     }
 
     if (rsc->children) {
         GList *gIter = NULL;
         gboolean any_created = FALSE;
 
         for (gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
             pe_resource_t *child_rsc = (pe_resource_t *) gIter->data;
 
             any_created = child_rsc->cmds->create_probe(child_rsc, node, complete, force, data_set)
                 || any_created;
         }
 
         return any_created;
 
     } else if ((rsc->container) && (!rsc->is_remote_node)) {
         pe_rsc_trace(rsc, "Skipping %s: it is within container %s", rsc->id, rsc->container->id);
         return FALSE;
     }
 
     if (pcmk_is_set(rsc->flags, pe_rsc_orphan)) {
         pe_rsc_trace(rsc, "Skipping orphan: %s", rsc->id);
         return FALSE;
     }
 
     // Check whether resource is already known on node
     if (!force && g_hash_table_lookup(rsc->known_on, node->details->id)) {
         pe_rsc_trace(rsc, "Skipping known: %s on %s", rsc->id, node->details->uname);
         return FALSE;
     }
 
     allowed = g_hash_table_lookup(rsc->allowed_nodes, node->details->id);
 
     if (rsc->exclusive_discover || top->exclusive_discover) {
         if (allowed == NULL) {
             /* exclusive discover is enabled and this node is not in the allowed list. */    
             pe_rsc_trace(rsc, "Skipping probe for %s on node %s, A", rsc->id, node->details->id);
             return FALSE;
         } else if (allowed->rsc_discover_mode != pe_discover_exclusive) {
             /* exclusive discover is enabled and this node is not marked
              * as a node this resource should be discovered on */ 
             pe_rsc_trace(rsc, "Skipping probe for %s on node %s, B", rsc->id, node->details->id);
             return FALSE;
         }
     }
 
     if(allowed == NULL && node->rsc_discover_mode == pe_discover_never) {
         /* If this node was allowed to host this resource it would
          * have been explicitly added to the 'allowed_nodes' list.
          * However it wasn't and the node has discovery disabled, so
          * no need to probe for this resource.
          */
         pe_rsc_trace(rsc, "Skipping probe for %s on node %s, C", rsc->id, node->details->id);
         return FALSE;
     }
 
     if (allowed && allowed->rsc_discover_mode == pe_discover_never) {
         /* this resource is marked as not needing to be discovered on this node */
         pe_rsc_trace(rsc, "Skipping probe for %s on node %s, discovery mode", rsc->id, node->details->id);
         return FALSE;
     }
 
     if (pe__is_guest_node(node)) {
         pe_resource_t *remote = node->details->remote_rsc->container;
 
         if(remote->role == RSC_ROLE_STOPPED) {
             /* If the container is stopped, then we know anything that
              * might have been inside it is also stopped and there is
              * no need to probe.
              *
              * If we don't know the container's state on the target
              * either:
              *
              * - the container is running, the transition will abort
              *   and we'll end up in a different case next time, or
              *
              * - the container is stopped
              *
              * Either way there is no need to probe.
              *
              */
             if(remote->allocated_to
                && g_hash_table_lookup(remote->known_on, remote->allocated_to->details->id) == NULL) {
                 /* For safety, we order the 'rsc' start after 'remote'
                  * has been probed.
                  *
                  * Using 'top' helps for groups, but we may need to
                  * follow the start's ordering chain backwards.
                  */
                 custom_action_order(remote,
                                     pcmk__op_key(remote->id, RSC_STATUS, 0),
                                     NULL, top,
                                     pcmk__op_key(top->id, RSC_START, 0), NULL,
                                     pe_order_optional, data_set);
             }
             pe_rsc_trace(rsc, "Skipping probe for %s on node %s, %s is stopped",
                          rsc->id, node->details->id, remote->id);
             return FALSE;
 
             /* Here we really we want to check if remote->stop is required,
              * but that information doesn't exist yet
              */
         } else if(node->details->remote_requires_reset
                   || node->details->unclean
                   || pcmk_is_set(remote->flags, pe_rsc_failed)
                   || remote->next_role == RSC_ROLE_STOPPED
                   || (remote->allocated_to
                       && pe_find_node(remote->running_on, remote->allocated_to->details->uname) == NULL)
             ) {
             /* The container is stopping or restarting, don't start
              * 'rsc' until 'remote' stops as this also implies that
              * 'rsc' is stopped - avoiding the need to probe
              */
             custom_action_order(remote, pcmk__op_key(remote->id, RSC_STOP, 0),
                                 NULL, top, pcmk__op_key(top->id, RSC_START, 0),
                                 NULL, pe_order_optional, data_set);
         pe_rsc_trace(rsc, "Skipping probe for %s on node %s, %s is stopping, restarting or moving",
                      rsc->id, node->details->id, remote->id);
             return FALSE;
 /*      } else {
  *            The container is running so there is no problem probing it
  */
         }
     }
 
     key = pcmk__op_key(rsc->id, RSC_STATUS, 0);
     probe = custom_action(rsc, key, RSC_STATUS, node, FALSE, TRUE, data_set);
-    update_action_flags(probe, pe_action_optional | pe_action_clear, __func__,
-                        __LINE__);
+    pe__clear_action_flags(probe, pe_action_optional);
 
     order_after_unfencing(rsc, node, probe, pe_order_optional, data_set);
 
     /*
      * We need to know if it's running_on (not just known_on) this node
      * to correctly determine the target rc.
      */
     running = pe_find_node_id(rsc->running_on, node->details->id);
     if (running == NULL) {
         add_hash_param(probe->meta, XML_ATTR_TE_TARGET_RC, rc_inactive);
 
     } else if (rsc->role == RSC_ROLE_PROMOTED) {
         add_hash_param(probe->meta, XML_ATTR_TE_TARGET_RC, rc_promoted);
     }
 
     crm_debug("Probing %s on %s (%s) %d %p", rsc->id, node->details->uname, role2text(rsc->role),
               pcmk_is_set(probe->flags, pe_action_runnable), rsc->running_on);
 
     if (is_unfence_device(rsc, data_set) || !pe_rsc_is_clone(top)) {
         top = rsc;
     } else {
         crm_trace("Probing %s on %s (%s) as %s", rsc->id, node->details->uname, role2text(rsc->role), top->id);
     }
 
     if (!pcmk_is_set(probe->flags, pe_action_runnable)
         && (rsc->running_on == NULL)) {
         /* Prevent the start from occurring if rsc isn't active, but
          * don't cause it to stop if it was active already
          */
         pe__set_order_flags(flags, pe_order_runnable_left);
     }
 
     custom_action_order(rsc, NULL, probe,
                         top, pcmk__op_key(top->id, RSC_START, 0), NULL,
                         flags, data_set);
 
     // Order the probe before any agent reload
     custom_action_order(rsc, NULL, probe,
                         top, reload_key(rsc), NULL,
                         pe_order_optional, data_set);
 
 #if 0
     // complete is always null currently
     if (!is_unfence_device(rsc, data_set)) {
         /* Normally rsc.start depends on probe complete which depends
          * on rsc.probe. But this can't be the case for fence devices
          * with unfencing, as it would create graph loops.
          *
          * So instead we explicitly order 'rsc.probe then rsc.start'
          */
         order_actions(probe, complete, pe_order_implies_then);
     }
 #endif
     return TRUE;
 }
 
 /*!
  * \internal
  * \brief Check whether a resource is known on a particular node
  *
  * \param[in] rsc   Resource to check
  * \param[in] node  Node to check
  *
  * \return TRUE if resource (or parent if an anonymous clone) is known
  */
 static bool
 rsc_is_known_on(pe_resource_t *rsc, const pe_node_t *node)
 {
    if (pe_hash_table_lookup(rsc->known_on, node->details->id)) {
        return TRUE;
 
    } else if ((rsc->variant == pe_native)
               && pe_rsc_is_anon_clone(rsc->parent)
               && pe_hash_table_lookup(rsc->parent->known_on, node->details->id)) {
        /* We check only the parent, not the uber-parent, because we cannot
         * assume that the resource is known if it is in an anonymously cloned
         * group (which may be only partially known).
         */
        return TRUE;
    }
    return FALSE;
 }
 
 /*!
  * \internal
  * \brief Order a resource's start and promote actions relative to fencing
  *
  * \param[in] rsc         Resource to be ordered
  * \param[in] stonith_op  Fence action
  * \param[in] data_set    Cluster information
  */
 static void
 native_start_constraints(pe_resource_t * rsc, pe_action_t * stonith_op, pe_working_set_t * data_set)
 {
     pe_node_t *target;
     GList *gIter = NULL;
 
     CRM_CHECK(stonith_op && stonith_op->node, return);
     target = stonith_op->node;
 
     for (gIter = rsc->actions; gIter != NULL; gIter = gIter->next) {
         pe_action_t *action = (pe_action_t *) gIter->data;
 
         switch (action->needs) {
             case rsc_req_nothing:
                 // Anything other than start or promote requires nothing
                 break;
 
             case rsc_req_stonith:
                 order_actions(stonith_op, action, pe_order_optional);
                 break;
 
             case rsc_req_quorum:
                 if (pcmk__str_eq(action->task, RSC_START, pcmk__str_casei)
                     && pe_hash_table_lookup(rsc->allowed_nodes, target->details->id)
                     && !rsc_is_known_on(rsc, target)) {
 
                     /* If we don't know the status of the resource on the node
                      * we're about to shoot, we have to assume it may be active
                      * there. Order the resource start after the fencing. This
                      * is analogous to waiting for all the probes for a resource
                      * to complete before starting it.
                      *
                      * The most likely explanation is that the DC died and took
                      * its status with it.
                      */
                     pe_rsc_debug(rsc, "Ordering %s after %s recovery", action->uuid,
                                  target->details->uname);
                     order_actions(stonith_op, action,
                                   pe_order_optional | pe_order_runnable_left);
                 }
                 break;
         }
     }
 }
 
 static void
 native_stop_constraints(pe_resource_t * rsc, pe_action_t * stonith_op, pe_working_set_t * data_set)
 {
     GList *gIter = NULL;
     GList *action_list = NULL;
     bool order_implicit = false;
 
     pe_resource_t *top = uber_parent(rsc);
     pe_action_t *parent_stop = NULL;
     pe_node_t *target;
 
     CRM_CHECK(stonith_op && stonith_op->node, return);
     target = stonith_op->node;
 
     /* Get a list of stop actions potentially implied by the fencing */
     action_list = pe__resource_actions(rsc, target, RSC_STOP, FALSE);
 
     /* If resource requires fencing, implicit actions must occur after fencing.
      *
      * Implied stops and demotes of resources running on guest nodes are always
      * ordered after fencing, even if the resource does not require fencing,
      * because guest node "fencing" is actually just a resource stop.
      */
     if (pcmk_is_set(rsc->flags, pe_rsc_needs_fencing)
         || pe__is_guest_node(target)) {
 
         order_implicit = true;
     }
 
     if (action_list && order_implicit) {
         parent_stop = find_first_action(top->actions, NULL, RSC_STOP, NULL);
     }
 
     for (gIter = action_list; gIter != NULL; gIter = gIter->next) {
         pe_action_t *action = (pe_action_t *) gIter->data;
 
         // The stop would never complete, so convert it into a pseudo-action.
-        update_action_flags(action, pe_action_pseudo|pe_action_runnable,
-                            __func__, __LINE__);
+        pe__set_action_flags(action, pe_action_pseudo|pe_action_runnable);
 
         if (order_implicit) {
-            update_action_flags(action, pe_action_implied_by_stonith,
-                                __func__, __LINE__);
+            pe__set_action_flags(action, pe_action_implied_by_stonith);
 
             /* Order the stonith before the parent stop (if any).
              *
              * Also order the stonith before the resource stop, unless the
              * resource is inside a bundle -- that would cause a graph loop.
              * We can rely on the parent stop's ordering instead.
              *
              * User constraints must not order a resource in a guest node
              * relative to the guest node container resource. The
              * pe_order_preserve flag marks constraints as generated by the
              * cluster and thus immune to that check (and is irrelevant if
              * target is not a guest).
              */
             if (!pe_rsc_is_bundled(rsc)) {
                 order_actions(stonith_op, action, pe_order_preserve);
             }
             order_actions(stonith_op, parent_stop, pe_order_preserve);
         }
 
         if (pcmk_is_set(rsc->flags, pe_rsc_failed)) {
             crm_notice("Stop of failed resource %s is implicit %s %s is fenced",
                        rsc->id, (order_implicit? "after" : "because"),
                        target->details->uname);
         } else {
             crm_info("%s is implicit %s %s is fenced",
                      action->uuid, (order_implicit? "after" : "because"),
                      target->details->uname);
         }
 
         if (pcmk_is_set(rsc->flags, pe_rsc_notify)) {
             /* Create a second notification that will be delivered
              *   immediately after the node is fenced
              *
              * Basic problem:
              * - C is a clone active on the node to be shot and stopping on another
              * - R is a resource that depends on C
              *
              * + C.stop depends on R.stop
              * + C.stopped depends on STONITH
              * + C.notify depends on C.stopped
              * + C.healthy depends on C.notify
              * + R.stop depends on C.healthy
              *
              * The extra notification here changes
              *  + C.healthy depends on C.notify
              * into:
              *  + C.healthy depends on C.notify'
              *  + C.notify' depends on STONITH'
              * thus breaking the loop
              */
             create_secondary_notification(action, rsc, stonith_op, data_set);
         }
 
 /* From Bug #1601, successful fencing must be an input to a failed resources stop action.
 
    However given group(rA, rB) running on nodeX and B.stop has failed,
    A := stop healthy resource (rA.stop)
    B := stop failed resource (pseudo operation B.stop)
    C := stonith nodeX
    A requires B, B requires C, C requires A
    This loop would prevent the cluster from making progress.
 
    This block creates the "C requires A" dependency and therefore must (at least
    for now) be disabled.
 
    Instead, run the block above and treat all resources on nodeX as B would be
    (marked as a pseudo op depending on the STONITH).
 
    TODO: Break the "A requires B" dependency in update_action() and re-enable this block
 
    } else if(is_stonith == FALSE) {
    crm_info("Moving healthy resource %s"
    " off %s before fencing",
    rsc->id, node->details->uname);
 
    * stop healthy resources before the
    * stonith op
    *
    custom_action_order(
    rsc, stop_key(rsc), NULL,
    NULL,strdup(CRM_OP_FENCE),stonith_op,
    pe_order_optional, data_set);
 */
     }
 
     g_list_free(action_list);
 
     /* Get a list of demote actions potentially implied by the fencing */
     action_list = pe__resource_actions(rsc, target, RSC_DEMOTE, FALSE);
 
     for (gIter = action_list; gIter != NULL; gIter = gIter->next) {
         pe_action_t *action = (pe_action_t *) gIter->data;
 
         if (action->node->details->online == FALSE || action->node->details->unclean == TRUE
             || pcmk_is_set(rsc->flags, pe_rsc_failed)) {
 
             if (pcmk_is_set(rsc->flags, pe_rsc_failed)) {
                 pe_rsc_info(rsc,
                             "Demote of failed resource %s is implicit after %s is fenced",
                             rsc->id, target->details->uname);
             } else {
                 pe_rsc_info(rsc, "%s is implicit after %s is fenced",
                             action->uuid, target->details->uname);
             }
 
             /* The demote would never complete and is now implied by the
              * fencing, so convert it into a pseudo-action.
              */
-            update_action_flags(action, pe_action_pseudo|pe_action_runnable,
-                                __func__, __LINE__);
+            pe__set_action_flags(action, pe_action_pseudo|pe_action_runnable);
 
             if (pe_rsc_is_bundled(rsc)) {
                 /* Do nothing, let the recovery be ordered after the parent's implied stop */
 
             } else if (order_implicit) {
                 order_actions(stonith_op, action, pe_order_preserve|pe_order_optional);
             }
         }
     }
 
     g_list_free(action_list);
 }
 
 void
 rsc_stonith_ordering(pe_resource_t * rsc, pe_action_t * stonith_op, pe_working_set_t * data_set)
 {
     if (rsc->children) {
         GList *gIter = NULL;
 
         for (gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
             pe_resource_t *child_rsc = (pe_resource_t *) gIter->data;
 
             rsc_stonith_ordering(child_rsc, stonith_op, data_set);
         }
 
     } else if (!pcmk_is_set(rsc->flags, pe_rsc_managed)) {
         pe_rsc_trace(rsc, "Skipping fencing constraints for unmanaged resource: %s", rsc->id);
 
     } else {
         native_start_constraints(rsc, stonith_op, data_set);
         native_stop_constraints(rsc, stonith_op, data_set);
     }
 }
 
 void
 ReloadRsc(pe_resource_t * rsc, pe_node_t *node, pe_working_set_t * data_set)
 {
     GList *gIter = NULL;
     pe_action_t *reload = NULL;
 
     if (rsc->children) {
         for (gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
             pe_resource_t *child_rsc = (pe_resource_t *) gIter->data;
 
             ReloadRsc(child_rsc, node, data_set);
         }
         return;
 
     } else if (rsc->variant > pe_native) {
         /* Complex resource with no children */
         return;
 
     } else if (!pcmk_is_set(rsc->flags, pe_rsc_managed)) {
         pe_rsc_trace(rsc, "%s: unmanaged", rsc->id);
         return;
 
     } else if (pcmk_is_set(rsc->flags, pe_rsc_failed)) {
         /* We don't need to specify any particular actions here, normal failure
          * recovery will apply.
          */
         pe_rsc_trace(rsc, "%s: preventing agent reload because failed",
                      rsc->id);
         return;
 
     } else if (pcmk_is_set(rsc->flags, pe_rsc_start_pending)) {
         /* If a resource's configuration changed while a start was pending,
          * force a full restart.
          */
         pe_rsc_trace(rsc, "%s: preventing agent reload because start pending",
                      rsc->id);
         stop_action(rsc, node, FALSE);
         return;
 
     } else if (node == NULL) {
         pe_rsc_trace(rsc, "%s: not active", rsc->id);
         return;
     }
 
     pe_rsc_trace(rsc, "Processing %s", rsc->id);
     pe__set_resource_flags(rsc, pe_rsc_reload);
 
     reload = custom_action(rsc, reload_key(rsc), CRMD_ACTION_RELOAD_AGENT, node,
                            FALSE, TRUE, data_set);
     pe_action_set_reason(reload, "resource definition change", FALSE);
 
     custom_action_order(NULL, NULL, reload, rsc, stop_key(rsc), NULL,
                         pe_order_optional|pe_order_then_cancels_first,
                         data_set);
     custom_action_order(NULL, NULL, reload, rsc, demote_key(rsc), NULL,
                         pe_order_optional|pe_order_then_cancels_first,
                         data_set);
 }
 
 void
 native_append_meta(pe_resource_t * rsc, xmlNode * xml)
 {
     char *value = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_INCARNATION);
     pe_resource_t *parent;
 
     if (value) {
         char *name = NULL;
 
         name = crm_meta_name(XML_RSC_ATTR_INCARNATION);
         crm_xml_add(xml, name, value);
         free(name);
     }
 
     value = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_REMOTE_NODE);
     if (value) {
         char *name = NULL;
 
         name = crm_meta_name(XML_RSC_ATTR_REMOTE_NODE);
         crm_xml_add(xml, name, value);
         free(name);
     }
 
     for (parent = rsc; parent != NULL; parent = parent->parent) {
         if (parent->container) {
             crm_xml_add(xml, CRM_META"_"XML_RSC_ATTR_CONTAINER, parent->container->id);
         }
     }
 }
diff --git a/lib/pacemaker/pcmk_sched_notif.c b/lib/pacemaker/pcmk_sched_notif.c
index c150305927..fbe416edf6 100644
--- a/lib/pacemaker/pcmk_sched_notif.c
+++ b/lib/pacemaker/pcmk_sched_notif.c
@@ -1,818 +1,802 @@
 /*
  * Copyright 2004-2021 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU General Public License version 2
  * or later (GPLv2+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 #include <crm/msg_xml.h>
 #include <pacemaker-internal.h>
 
 extern bool pcmk__is_daemon;
 
 typedef struct notify_entry_s {
     pe_resource_t *rsc;
     pe_node_t *node;
 } notify_entry_t;
 
 static gint
 sort_notify_entries(gconstpointer a, gconstpointer b)
 {
     int tmp;
     const notify_entry_t *entry_a = a;
     const notify_entry_t *entry_b = b;
 
     if (entry_a == NULL && entry_b == NULL) {
         return 0;
     }
     if (entry_a == NULL) {
         return 1;
     }
     if (entry_b == NULL) {
         return -1;
     }
 
     if (entry_a->rsc == NULL && entry_b->rsc == NULL) {
         return 0;
     }
     if (entry_a->rsc == NULL) {
         return 1;
     }
     if (entry_b->rsc == NULL) {
         return -1;
     }
 
     tmp = strcmp(entry_a->rsc->id, entry_b->rsc->id);
     if (tmp != 0) {
         return tmp;
     }
 
     if (entry_a->node == NULL && entry_b->node == NULL) {
         return 0;
     }
     if (entry_a->node == NULL) {
         return 1;
     }
     if (entry_b->node == NULL) {
         return -1;
     }
 
     return strcmp(entry_a->node->details->id, entry_b->node->details->id);
 }
 
 static notify_entry_t *dup_notify_entry(notify_entry_t *entry)
 {
     notify_entry_t *dup = malloc(sizeof(notify_entry_t));
 
     CRM_ASSERT(dup != NULL);
     dup->rsc = entry->rsc;
     dup->node = entry->node;
     return dup;
 }
 
 static void
 expand_node_list(GList *list, char **uname, char **metal)
 {
     GList *gIter = NULL;
     char *node_list = NULL;
     char *metal_list = NULL;
     size_t node_list_len = 0;
     size_t metal_list_len = 0;
 
     CRM_ASSERT(uname != NULL);
     if (list == NULL) {
         *uname = strdup(" ");
         if(metal) {
             *metal = strdup(" ");
         }
         return;
     }
 
     for (gIter = list; gIter != NULL; gIter = gIter->next) {
         pe_node_t *node = (pe_node_t *) gIter->data;
 
         if (node->details->uname == NULL) {
             continue;
         }
         pcmk__add_word(&node_list, &node_list_len, node->details->uname);
         if(metal) {
             if(node->details->remote_rsc
                && node->details->remote_rsc->container
                && node->details->remote_rsc->container->running_on) {
                 node = pe__current_node(node->details->remote_rsc->container);
             }
 
             if (node->details->uname == NULL) {
                 continue;
             }
             pcmk__add_word(&metal_list, &metal_list_len, node->details->uname);
         }
     }
 
     *uname = node_list;
     if(metal) {
         *metal = metal_list;
     }
 }
 
 /*!
  * \internal
  * \brief Separate a list of notification entries into resource and node strings
  *
  * \param[in,out] list       List of notify_entry_t* (will be sorted here)
  * \param[out]    rsc_list   String list of clone instances from \p list
  * \param[out]    node_list  String list of nodes from \p list
  *
  * \return (Possibly new head of) sorted \p list
  */
 static GList *
 expand_list(GList *list, char **rsc_list, char **node_list)
 {
     const char *last_rsc_id = NULL;
     size_t rsc_list_len = 0;
     size_t node_list_len = 0;
 
     CRM_CHECK(rsc_list != NULL, return list);
 
     // If there are no entries, return "empty" lists
     if (list == NULL) {
         *rsc_list = strdup(" ");
         if (node_list) {
             *node_list = strdup(" ");
         }
         return list;
     }
 
     // Initialize output lists to NULL
     *rsc_list = NULL;
     if (node_list) {
         *node_list = NULL;
     }
 
     // Sort input list for user-friendliness (and ease of filtering duplicates)
     list = g_list_sort(list, sort_notify_entries);
 
     for (GList *gIter = list; gIter != NULL; gIter = gIter->next) {
         notify_entry_t *entry = (notify_entry_t *) gIter->data;
 
         // Entry must have a resource (with ID)
         CRM_LOG_ASSERT((entry != NULL) && (entry->rsc != NULL)
                        && (entry->rsc->id != NULL));
         if ((entry == NULL) || (entry->rsc == NULL)
             || (entry->rsc->id == NULL)) {
             continue;
         }
 
         // Entry must have a node unless listing inactive resources
         CRM_LOG_ASSERT((node_list == NULL) || (entry->node != NULL));
         if ((node_list != NULL) && (entry->node == NULL)) {
             continue;
         }
 
         // Don't add duplicates of a particular clone instance
         if (pcmk__str_eq(entry->rsc->id, last_rsc_id, pcmk__str_none)) {
             continue;
         }
         last_rsc_id = entry->rsc->id;
         pcmk__add_word(rsc_list, &rsc_list_len, entry->rsc->id);
         if ((node_list != NULL) && (entry->node->details->uname != NULL)) {
             pcmk__add_word(node_list, &node_list_len,
                            entry->node->details->uname);
         }
     }
     return list;
 }
 
 static void
 dup_attr(gpointer key, gpointer value, gpointer user_data)
 {
     add_hash_param(user_data, key, value);
 }
 
 static void
 add_notify_data_to_action_meta(notify_data_t *n_data, pe_action_t *action)
 {
     for (GSList *item = n_data->keys; item; item = item->next) {
         pcmk_nvpair_t *nvpair = item->data;
 
         add_hash_param(action->meta, nvpair->name, nvpair->value);
     }
 }
 
 static pe_action_t *
 pe_notify(pe_resource_t * rsc, pe_node_t * node, pe_action_t * op, pe_action_t * confirm,
           notify_data_t * n_data, pe_working_set_t * data_set)
 {
     char *key = NULL;
     pe_action_t *trigger = NULL;
     const char *value = NULL;
     const char *task = NULL;
 
     if (op == NULL || confirm == NULL) {
         pe_rsc_trace(rsc, "Op=%p confirm=%p", op, confirm);
         return NULL;
     }
 
     CRM_CHECK(rsc != NULL, return NULL);
     CRM_CHECK(node != NULL, return NULL);
 
     if (node->details->online == FALSE) {
         pe_rsc_trace(rsc, "Skipping notification for %s: node offline", rsc->id);
         return NULL;
     } else if (!pcmk_is_set(op->flags, pe_action_runnable)) {
         pe_rsc_trace(rsc, "Skipping notification for %s: not runnable", op->uuid);
         return NULL;
     }
 
     value = g_hash_table_lookup(op->meta, "notify_type");
     task = g_hash_table_lookup(op->meta, "notify_operation");
 
     pe_rsc_trace(rsc, "Creating notify actions for %s: %s (%s-%s)", op->uuid, rsc->id, value, task);
 
     key = pcmk__notify_key(rsc->id, value, task);
     trigger = custom_action(rsc, key, op->task, node,
                             pcmk_is_set(op->flags, pe_action_optional),
                             TRUE, data_set);
     g_hash_table_foreach(op->meta, dup_attr, trigger->meta);
     add_notify_data_to_action_meta(n_data, trigger);
 
     /* pseudo_notify before notify */
     pe_rsc_trace(rsc, "Ordering %s before %s (%d->%d)", op->uuid, trigger->uuid, trigger->id,
                  op->id);
 
     order_actions(op, trigger, pe_order_optional);
     order_actions(trigger, confirm, pe_order_optional);
     return trigger;
 }
 
 static void
 pe_post_notify(pe_resource_t * rsc, pe_node_t * node, notify_data_t * n_data, pe_working_set_t * data_set)
 {
     pe_action_t *notify = NULL;
 
     CRM_CHECK(rsc != NULL, return);
 
     if (n_data->post == NULL) {
         return;                 /* Nothing to do */
     }
 
     notify = pe_notify(rsc, node, n_data->post, n_data->post_done, n_data, data_set);
 
     if (notify != NULL) {
         notify->priority = INFINITY;
     }
 
     if (n_data->post_done) {
         GList *gIter = rsc->actions;
 
         for (; gIter != NULL; gIter = gIter->next) {
             pe_action_t *mon = (pe_action_t *) gIter->data;
             const char *interval_ms_s = g_hash_table_lookup(mon->meta,
                                                             XML_LRM_ATTR_INTERVAL_MS);
 
             if (pcmk__str_eq(interval_ms_s, "0", pcmk__str_null_matches | pcmk__str_casei)) {
                 pe_rsc_trace(rsc, "Skipping %s: interval", mon->uuid);
                 continue;
             } else if (pcmk__str_eq(mon->task, RSC_CANCEL, pcmk__str_casei)) {
                 pe_rsc_trace(rsc, "Skipping %s: cancel", mon->uuid);
                 continue;
             }
 
             order_actions(n_data->post_done, mon, pe_order_optional);
         }
     }
 }
 
 notify_data_t *
 create_notification_boundaries(pe_resource_t * rsc, const char *action, pe_action_t * start,
                                pe_action_t * end, pe_working_set_t * data_set)
 {
     /* Create the pseudo ops that precede and follow the actual notifications */
 
     /*
      * Creates two sequences (conditional on start and end being supplied):
      *   pre_notify -> pre_notify_complete -> start, and
      *   end -> post_notify -> post_notify_complete
      *
      * 'start' and 'end' may be the same event or ${X} and ${X}ed as per clones
      */
     char *key = NULL;
     notify_data_t *n_data = NULL;
 
     if (!pcmk_is_set(rsc->flags, pe_rsc_notify)) {
         return NULL;
     }
 
     n_data = calloc(1, sizeof(notify_data_t));
     n_data->action = action;
 
     if (start) {
         /* create pre-event notification wrappers */
         key = pcmk__notify_key(rsc->id, "pre", start->task);
         n_data->pre =
             custom_action(rsc, key, RSC_NOTIFY, NULL,
                           pcmk_is_set(start->flags, pe_action_optional),
                           TRUE, data_set);
-
-        update_action_flags(n_data->pre, pe_action_pseudo, __func__, __LINE__);
-        update_action_flags(n_data->pre, pe_action_runnable, __func__,
-                            __LINE__);
+        pe__set_action_flags(n_data->pre, pe_action_pseudo|pe_action_runnable);
 
         add_hash_param(n_data->pre->meta, "notify_type", "pre");
         add_hash_param(n_data->pre->meta, "notify_operation", n_data->action);
 
         add_hash_param(n_data->pre->meta, "notify_key_type", "pre");
         add_hash_param(n_data->pre->meta, "notify_key_operation", start->task);
 
         /* create pre_notify_complete */
         key = pcmk__notify_key(rsc->id, "confirmed-pre", start->task);
         n_data->pre_done = custom_action(rsc, key, RSC_NOTIFIED, NULL,
                                          pcmk_is_set(start->flags, pe_action_optional),
                                          TRUE, data_set);
-
-        update_action_flags(n_data->pre_done, pe_action_pseudo, __func__,
-                            __LINE__);
-        update_action_flags(n_data->pre_done, pe_action_runnable, __func__,
-                            __LINE__);
+        pe__set_action_flags(n_data->pre_done,
+                             pe_action_pseudo|pe_action_runnable);
 
         add_hash_param(n_data->pre_done->meta, "notify_type", "pre");
         add_hash_param(n_data->pre_done->meta, "notify_operation", n_data->action);
 
         add_hash_param(n_data->pre_done->meta, "notify_key_type", "confirmed-pre");
         add_hash_param(n_data->pre_done->meta, "notify_key_operation", start->task);
 
         order_actions(n_data->pre_done, start, pe_order_optional);
         order_actions(n_data->pre, n_data->pre_done, pe_order_optional);
     }
 
     if (end) {
         /* create post-event notification wrappers */
         key = pcmk__notify_key(rsc->id, "post", end->task);
         n_data->post = custom_action(rsc, key, RSC_NOTIFY, NULL,
                                      pcmk_is_set(end->flags, pe_action_optional),
                                      TRUE, data_set);
 
         n_data->post->priority = INFINITY;
-        update_action_flags(n_data->post, pe_action_pseudo, __func__,
-                            __LINE__);
+        pe__set_action_flags(n_data->post, pe_action_pseudo);
         if (pcmk_is_set(end->flags, pe_action_runnable)) {
-            update_action_flags(n_data->post, pe_action_runnable, __func__,
-                                __LINE__);
+            pe__set_action_flags(n_data->post, pe_action_runnable);
         } else {
-            update_action_flags(n_data->post, pe_action_runnable | pe_action_clear,
-                                __func__, __LINE__);
+            pe__clear_action_flags(n_data->post, pe_action_runnable);
         }
 
         add_hash_param(n_data->post->meta, "notify_type", "post");
         add_hash_param(n_data->post->meta, "notify_operation", n_data->action);
 
         add_hash_param(n_data->post->meta, "notify_key_type", "post");
         add_hash_param(n_data->post->meta, "notify_key_operation", end->task);
 
         /* create post_notify_complete */
         key = pcmk__notify_key(rsc->id, "confirmed-post", end->task);
         n_data->post_done = custom_action(rsc, key, RSC_NOTIFIED, NULL,
                                           pcmk_is_set(end->flags, pe_action_optional),
                                           TRUE, data_set);
 
         n_data->post_done->priority = INFINITY;
-        update_action_flags(n_data->post_done, pe_action_pseudo, __func__,
-                            __LINE__);
+        pe__set_action_flags(n_data->post_done, pe_action_pseudo);
         if (pcmk_is_set(end->flags, pe_action_runnable)) {
-            update_action_flags(n_data->post_done, pe_action_runnable,
-                                __func__, __LINE__);
+            pe__set_action_flags(n_data->post_done, pe_action_runnable);
         } else {
-            update_action_flags(n_data->post_done, pe_action_runnable | pe_action_clear,
-                                __func__, __LINE__);
+            pe__clear_action_flags(n_data->post_done, pe_action_runnable);
         }
 
         add_hash_param(n_data->post_done->meta, "notify_type", "post");
         add_hash_param(n_data->post_done->meta, "notify_operation", n_data->action);
 
         add_hash_param(n_data->post_done->meta, "notify_key_type", "confirmed-post");
         add_hash_param(n_data->post_done->meta, "notify_key_operation", end->task);
 
         order_actions(end, n_data->post, pe_order_implies_then);
         order_actions(n_data->post, n_data->post_done, pe_order_implies_then);
     }
 
     if (start && end) {
         order_actions(n_data->pre_done, n_data->post, pe_order_optional);
     }
     return n_data;
 }
 
 void
 collect_notification_data(pe_resource_t * rsc, gboolean state, gboolean activity,
                           notify_data_t * n_data)
 {
 
     if(n_data->allowed_nodes == NULL) {
         n_data->allowed_nodes = rsc->allowed_nodes;
     }
 
     if (rsc->children) {
         GList *gIter = rsc->children;
 
         for (; gIter != NULL; gIter = gIter->next) {
             pe_resource_t *child = (pe_resource_t *) gIter->data;
 
             collect_notification_data(child, state, activity, n_data);
         }
         return;
     }
 
     if (state) {
         notify_entry_t *entry = NULL;
 
         entry = calloc(1, sizeof(notify_entry_t));
         entry->rsc = rsc;
         if (rsc->running_on) {
             /* we only take the first one */
             entry->node = rsc->running_on->data;
         }
 
         pe_rsc_trace(rsc, "%s state: %s", rsc->id, role2text(rsc->role));
 
         switch (rsc->role) {
             case RSC_ROLE_STOPPED:
                 n_data->inactive = g_list_prepend(n_data->inactive, entry);
                 break;
             case RSC_ROLE_STARTED:
                 n_data->active = g_list_prepend(n_data->active, entry);
                 break;
             case RSC_ROLE_UNPROMOTED:
                 n_data->unpromoted = g_list_prepend(n_data->unpromoted, entry);
                 n_data->active = g_list_prepend(n_data->active,
                                                 dup_notify_entry(entry));
                 break;
             case RSC_ROLE_PROMOTED:
                 n_data->promoted = g_list_prepend(n_data->promoted, entry);
                 n_data->active = g_list_prepend(n_data->active,
                                                 dup_notify_entry(entry));
                 break;
             default:
                 crm_err("Unsupported notify role");
                 free(entry);
                 break;
         }
     }
 
     if (activity) {
         notify_entry_t *entry = NULL;
         enum action_tasks task;
 
         GList *gIter = rsc->actions;
 
         for (; gIter != NULL; gIter = gIter->next) {
             pe_action_t *op = (pe_action_t *) gIter->data;
 
             if (!pcmk_is_set(op->flags, pe_action_optional)
                 && (op->node != NULL)) {
 
                 task = text2task(op->task);
 
                 if(task == stop_rsc && op->node->details->unclean) {
                     // Create anyway (additional noise if node can't be fenced)
                 } else if (!pcmk_is_set(op->flags, pe_action_runnable)) {
                     continue;
                 }
 
                 entry = calloc(1, sizeof(notify_entry_t));
                 entry->node = op->node;
                 entry->rsc = rsc;
 
                 switch (task) {
                     case start_rsc:
                         n_data->start = g_list_prepend(n_data->start, entry);
                         break;
                     case stop_rsc:
                         n_data->stop = g_list_prepend(n_data->stop, entry);
                         break;
                     case action_promote:
                         n_data->promote = g_list_prepend(n_data->promote, entry);
                         break;
                     case action_demote:
                         n_data->demote = g_list_prepend(n_data->demote, entry);
                         break;
                     default:
                         free(entry);
                         break;
                 }
             }
         }
     }
 }
 
 #define add_notify_env(n_data, key, value) do {                         \
          n_data->keys = pcmk_prepend_nvpair(n_data->keys, key, value);  \
     } while (0)
 
 #define add_notify_env_free(n_data, key, value) do {                    \
          n_data->keys = pcmk_prepend_nvpair(n_data->keys, key, value);  \
          free(value); value = NULL;                                     \
     } while (0)
 
 /*!
  * \internal
  * \brief Create notification name/value pairs from raw data
  *
  * \param[in]     rsc       Resource that notification is for
  * \param[in,out] n_data    Notification data
  * \param[in]     data_set  Cluster working set
  */
 void
 pcmk__create_notification_keys(pe_resource_t *rsc,
                                notify_data_t *n_data,
                                pe_working_set_t *data_set)
 {
     bool required = false; // Whether to make notify actions required
     char *rsc_list = NULL;
     char *node_list = NULL;
     char *metal_list = NULL;
     const char *source = NULL;
     GList *nodes = NULL;
 
     n_data->stop = expand_list(n_data->stop, &rsc_list, &node_list);
     if (!pcmk__str_eq(" ", rsc_list, pcmk__str_null_matches)
         && pcmk__str_eq(n_data->action, RSC_STOP, pcmk__str_casei)) {
         required = true;
     }
     add_notify_env_free(n_data, "notify_stop_resource", rsc_list);
     add_notify_env_free(n_data, "notify_stop_uname", node_list);
 
     if ((n_data->start != NULL)
         && pcmk__str_eq(n_data->action, RSC_START, pcmk__str_casei)) {
         required = true;
     }
     n_data->start = expand_list(n_data->start, &rsc_list, &node_list);
     add_notify_env_free(n_data, "notify_start_resource", rsc_list);
     add_notify_env_free(n_data, "notify_start_uname", node_list);
 
     if ((n_data->demote != NULL)
         && pcmk__str_eq(n_data->action, RSC_DEMOTE, pcmk__str_casei)) {
         required = true;
     }
     n_data->demote = expand_list(n_data->demote, &rsc_list, &node_list);
     add_notify_env_free(n_data, "notify_demote_resource", rsc_list);
     add_notify_env_free(n_data, "notify_demote_uname", node_list);
 
     if ((n_data->promote != NULL)
         && pcmk__str_eq(n_data->action, RSC_PROMOTE, pcmk__str_casei)) {
         required = true;
     }
     n_data->promote = expand_list(n_data->promote, &rsc_list, &node_list);
     add_notify_env_free(n_data, "notify_promote_resource", rsc_list);
     add_notify_env_free(n_data, "notify_promote_uname", node_list);
 
     n_data->active = expand_list(n_data->active, &rsc_list, &node_list);
     add_notify_env_free(n_data, "notify_active_resource", rsc_list);
     add_notify_env_free(n_data, "notify_active_uname", node_list);
 
     n_data->unpromoted = expand_list(n_data->unpromoted, &rsc_list, &node_list);
     add_notify_env(n_data, "notify_unpromoted_resource", rsc_list);
     add_notify_env(n_data, "notify_unpromoted_uname", node_list);
 
     // Deprecated: kept for backward compatibility with older resource agents
     add_notify_env_free(n_data, "notify_slave_resource", rsc_list);
     add_notify_env_free(n_data, "notify_slave_uname", node_list);
 
     n_data->promoted = expand_list(n_data->promoted, &rsc_list, &node_list);
     add_notify_env(n_data, "notify_promoted_resource", rsc_list);
     add_notify_env(n_data, "notify_promoted_uname", node_list);
 
     // Deprecated: kept for backward compatibility with older resource agents
     add_notify_env_free(n_data, "notify_master_resource", rsc_list);
     add_notify_env_free(n_data, "notify_master_uname", node_list);
 
     n_data->inactive = expand_list(n_data->inactive, &rsc_list, NULL);
     add_notify_env_free(n_data, "notify_inactive_resource", rsc_list);
 
     nodes = g_hash_table_get_values(n_data->allowed_nodes);
     if (!pcmk__is_daemon) {
         /* If printing to stdout, sort the node list, for consistent
          * regression test output (while avoiding the performance hit
          * for the live cluster).
          */
         nodes = g_list_sort(nodes, sort_node_uname);
     }
     expand_node_list(nodes, &node_list, NULL);
     add_notify_env_free(n_data, "notify_available_uname", node_list);
     g_list_free(nodes);
 
     source = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_TARGET);
     if (pcmk__str_eq("host", source, pcmk__str_casei)) {
         expand_node_list(data_set->nodes, &node_list, &metal_list);
         add_notify_env_free(n_data, "notify_all_hosts", metal_list);
     } else {
         expand_node_list(data_set->nodes, &node_list, NULL);
     }
     add_notify_env_free(n_data, "notify_all_uname", node_list);
 
     if (required && n_data->pre) {
-        update_action_flags(n_data->pre, pe_action_optional | pe_action_clear,
-                            __func__, __LINE__);
-        update_action_flags(n_data->pre_done, pe_action_optional | pe_action_clear,
-                            __func__, __LINE__);
+        pe__clear_action_flags(n_data->pre, pe_action_optional);
+        pe__clear_action_flags(n_data->pre_done, pe_action_optional);
     }
 
     if (required && n_data->post) {
-        update_action_flags(n_data->post, pe_action_optional | pe_action_clear,
-                            __func__, __LINE__);
-        update_action_flags(n_data->post_done, pe_action_optional | pe_action_clear,
-                            __func__, __LINE__);
+        pe__clear_action_flags(n_data->post, pe_action_optional);
+        pe__clear_action_flags(n_data->post_done, pe_action_optional);
     }
 }
 
 /*
  * \internal
  * \brief Find any remote connection start relevant to an action
  *
  * \param[in] action  Action to chek
  *
  * \return If action is behind a remote connection, connection's start
  */
 static pe_action_t *
 find_remote_start(pe_action_t *action)
 {
     if (action && action->node) {
         pe_resource_t *remote_rsc = action->node->details->remote_rsc;
 
         if (remote_rsc) {
             return find_first_action(remote_rsc->actions, NULL, RSC_START,
                                      NULL);
         }
     }
     return NULL;
 }
 
 void
 create_notifications(pe_resource_t * rsc, notify_data_t * n_data, pe_working_set_t * data_set)
 {
     GList *gIter = NULL;
     pe_action_t *stop = NULL;
     pe_action_t *start = NULL;
     enum action_tasks task = text2task(n_data->action);
 
     if (rsc->children) {
         gIter = rsc->children;
         for (; gIter != NULL; gIter = gIter->next) {
             pe_resource_t *child = (pe_resource_t *) gIter->data;
 
             create_notifications(child, n_data, data_set);
         }
         return;
     }
 
     /* Copy notification details into standard ops */
 
     for (gIter = rsc->actions; gIter != NULL; gIter = gIter->next) {
         pe_action_t *op = (pe_action_t *) gIter->data;
 
         if (!pcmk_is_set(op->flags, pe_action_optional)
             && (op->node != NULL)) {
 
             enum action_tasks t = text2task(op->task);
 
             switch (t) {
                 case start_rsc:
                 case stop_rsc:
                 case action_promote:
                 case action_demote:
                     add_notify_data_to_action_meta(n_data, op);
                     break;
                 default:
                     break;
             }
         }
     }
 
     switch (task) {
         case start_rsc:
             if (n_data->start == NULL) {
                 pe_rsc_trace(rsc, "Skipping empty notification for: %s.%s (%s->%s)",
                              n_data->action, rsc->id, role2text(rsc->role), role2text(rsc->next_role));
                 return;
             }
             break;
         case action_promote:
             if (n_data->promote == NULL) {
                 pe_rsc_trace(rsc, "Skipping empty notification for: %s.%s (%s->%s)",
                              n_data->action, rsc->id, role2text(rsc->role), role2text(rsc->next_role));
                 return;
             }
             break;
         case action_demote:
             if (n_data->demote == NULL) {
                 pe_rsc_trace(rsc, "Skipping empty notification for: %s.%s (%s->%s)",
                              n_data->action, rsc->id, role2text(rsc->role), role2text(rsc->next_role));
                 return;
             }
             break;
         default:
             /* We cannot do the same for stop_rsc/n_data->stop at it
              * might be implied by fencing
              */
             break;
     }
 
     pe_rsc_trace(rsc, "Creating notifications for: %s.%s (%s->%s)",
                  n_data->action, rsc->id, role2text(rsc->role), role2text(rsc->next_role));
 
     stop = find_first_action(rsc->actions, NULL, RSC_STOP, NULL);
     start = find_first_action(rsc->actions, NULL, RSC_START, NULL);
 
     /* stop / demote */
     if (rsc->role != RSC_ROLE_STOPPED) {
         if (task == stop_rsc || task == action_demote) {
             gIter = rsc->running_on;
             for (; gIter != NULL; gIter = gIter->next) {
                 pe_node_t *current_node = (pe_node_t *) gIter->data;
 
                 /* if this stop action is a pseudo action as a result of the current
                  * node being fenced, this stop action is implied by the fencing 
                  * action. There's no reason to send the fenced node a stop notification */ 
                 if (stop && pcmk_is_set(stop->flags, pe_action_pseudo) &&
                     (current_node->details->unclean || current_node->details->remote_requires_reset) ) {
 
                     continue;
                 }
 
                 pe_notify(rsc, current_node, n_data->pre, n_data->pre_done, n_data, data_set);
                 if (task == action_demote || stop == NULL
                     || pcmk_is_set(stop->flags, pe_action_optional)) {
                     pe_post_notify(rsc, current_node, n_data, data_set);
                 }
             }
         }
     }
 
     /* start / promote */
     if (rsc->next_role != RSC_ROLE_STOPPED) {
         if (rsc->allocated_to == NULL) {
             pe_proc_err("Next role '%s' but %s is not allocated", role2text(rsc->next_role),
                         rsc->id);
 
         } else if (task == start_rsc || task == action_promote) {
 
             if (start) {
                 pe_action_t *remote_start = find_remote_start(start);
 
                 if (remote_start
                     && !pcmk_is_set(remote_start->flags, pe_action_runnable)) {
                     /* Start and promote actions for a clone instance behind
                      * a Pacemaker Remote connection happen after the
                      * connection starts. If the connection start is blocked, do
                      * not schedule notifications for these actions.
                      */
                     return;
                 }
             }
             if ((task != start_rsc) || (start == NULL)
                 || pcmk_is_set(start->flags, pe_action_optional)) {
 
                 pe_notify(rsc, rsc->allocated_to, n_data->pre, n_data->pre_done, n_data, data_set);
             }
             pe_post_notify(rsc, rsc->allocated_to, n_data, data_set);
         }
     }
 }
 
 void
 free_notification_data(notify_data_t * n_data)
 {
     if (n_data == NULL) {
         return;
     }
 
     g_list_free_full(n_data->stop, free);
     g_list_free_full(n_data->start, free);
     g_list_free_full(n_data->demote, free);
     g_list_free_full(n_data->promote, free);
     g_list_free_full(n_data->promoted, free);
     g_list_free_full(n_data->unpromoted, free);
     g_list_free_full(n_data->active, free);
     g_list_free_full(n_data->inactive, free);
     pcmk_free_nvpairs(n_data->keys);
     free(n_data);
 }
 
 void
 create_secondary_notification(pe_action_t *action, pe_resource_t *rsc,
                               pe_action_t *stonith_op,
                               pe_working_set_t *data_set)
 {
     notify_data_t *n_data;
 
     crm_info("Creating secondary notification for %s", action->uuid);
     n_data = create_notification_boundaries(rsc, RSC_STOP, NULL, stonith_op,
                                             data_set);
     collect_notification_data(rsc, TRUE, FALSE, n_data);
     add_notify_env(n_data, "notify_stop_resource", rsc->id);
     add_notify_env(n_data, "notify_stop_uname", action->node->details->uname);
     create_notifications(uber_parent(rsc), n_data, data_set);
     free_notification_data(n_data);
 }
diff --git a/lib/pacemaker/pcmk_sched_utils.c b/lib/pacemaker/pcmk_sched_utils.c
index b97c0304aa..85249f8608 100644
--- a/lib/pacemaker/pcmk_sched_utils.c
+++ b/lib/pacemaker/pcmk_sched_utils.c
@@ -1,792 +1,789 @@
 /*
  * Copyright 2004-2021 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU General Public License version 2
  * or later (GPLv2+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 #include <crm/msg_xml.h>
 #include <crm/lrmd.h>       // lrmd_event_data_t
 #include <crm/common/xml_internal.h>
 #include <pacemaker-internal.h>
 
 pe__location_t *
 rsc2node_new(const char *id, pe_resource_t *rsc,
              int node_weight, const char *discover_mode,
              pe_node_t *foo_node, pe_working_set_t *data_set)
 {
     pe__location_t *new_con = NULL;
 
     if (rsc == NULL || id == NULL) {
         pe_err("Invalid constraint %s for rsc=%p", crm_str(id), rsc);
         return NULL;
 
     } else if (foo_node == NULL) {
         CRM_CHECK(node_weight == 0, return NULL);
     }
 
     new_con = calloc(1, sizeof(pe__location_t));
     if (new_con != NULL) {
         new_con->id = strdup(id);
         new_con->rsc_lh = rsc;
         new_con->node_list_rh = NULL;
         new_con->role_filter = RSC_ROLE_UNKNOWN;
 
 
         if (pcmk__str_eq(discover_mode, "always", pcmk__str_null_matches | pcmk__str_casei)) {
             new_con->discover_mode = pe_discover_always;
         } else if (pcmk__str_eq(discover_mode, "never", pcmk__str_casei)) {
             new_con->discover_mode = pe_discover_never;
         } else if (pcmk__str_eq(discover_mode, "exclusive", pcmk__str_casei)) {
             new_con->discover_mode = pe_discover_exclusive;
             rsc->exclusive_discover = TRUE;
         } else {
             pe_err("Invalid %s value %s in location constraint", XML_LOCATION_ATTR_DISCOVERY, discover_mode);
         }
 
         if (foo_node != NULL) {
             pe_node_t *copy = pe__copy_node(foo_node);
 
             copy->weight = node_weight;
             new_con->node_list_rh = g_list_prepend(NULL, copy);
         }
 
         data_set->placement_constraints = g_list_prepend(data_set->placement_constraints, new_con);
         rsc->rsc_location = g_list_prepend(rsc->rsc_location, new_con);
     }
 
     return new_con;
 }
 
 gboolean
 can_run_resources(const pe_node_t * node)
 {
     if (node == NULL) {
         return FALSE;
     }
 #if 0
     if (node->weight < 0) {
         return FALSE;
     }
 #endif
 
     if (node->details->online == FALSE
         || node->details->shutdown || node->details->unclean
         || node->details->standby || node->details->maintenance) {
         crm_trace("%s: online=%d, unclean=%d, standby=%d, maintenance=%d",
                   node->details->uname, node->details->online,
                   node->details->unclean, node->details->standby, node->details->maintenance);
         return FALSE;
     }
     return TRUE;
 }
 
 /*!
  * \internal
  * \brief Copy a hash table of node objects
  *
  * \param[in] nodes  Hash table to copy
  *
  * \return New copy of nodes (or NULL if nodes is NULL)
  */
 GHashTable *
 pcmk__copy_node_table(GHashTable *nodes)
 {
     GHashTable *new_table = NULL;
     GHashTableIter iter;
     pe_node_t *node = NULL;
 
     if (nodes == NULL) {
         return NULL;
     }
     new_table = pcmk__strkey_table(NULL, free);
     g_hash_table_iter_init(&iter, nodes);
     while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &node)) {
         pe_node_t *new_node = pe__copy_node(node);
 
         g_hash_table_insert(new_table, (gpointer) new_node->details->id,
                             new_node);
     }
     return new_table;
 }
 
 /*!
  * \internal
  * \brief Copy a list of node objects
  *
  * \param[in] list   List to copy
  * \param[in] reset  Set copies' scores to 0
  *
  * \return New list of shallow copies of nodes in original list
  */
 GList *
 pcmk__copy_node_list(const GList *list, bool reset)
 {
     GList *result = NULL;
 
     for (const GList *gIter = list; gIter != NULL; gIter = gIter->next) {
         pe_node_t *new_node = NULL;
         pe_node_t *this_node = (pe_node_t *) gIter->data;
 
         new_node = pe__copy_node(this_node);
         if (reset) {
             new_node->weight = 0;
         }
         result = g_list_prepend(result, new_node);
     }
     return result;
 }
 
 struct node_weight_s {
     pe_node_t *active;
     pe_working_set_t *data_set;
 };
 
 /* return -1 if 'a' is more preferred
  * return  1 if 'b' is more preferred
  */
 
 static gint
 sort_node_weight(gconstpointer a, gconstpointer b, gpointer data)
 {
     const pe_node_t *node1 = (const pe_node_t *)a;
     const pe_node_t *node2 = (const pe_node_t *)b;
     struct node_weight_s *nw = data;
 
     int node1_weight = 0;
     int node2_weight = 0;
 
     int result = 0;
 
     if (a == NULL) {
         return 1;
     }
     if (b == NULL) {
         return -1;
     }
 
     node1_weight = node1->weight;
     node2_weight = node2->weight;
 
     if (can_run_resources(node1) == FALSE) {
         node1_weight = -INFINITY;
     }
     if (can_run_resources(node2) == FALSE) {
         node2_weight = -INFINITY;
     }
 
     if (node1_weight > node2_weight) {
         crm_trace("%s (%d) > %s (%d) : weight",
                   node1->details->uname, node1_weight, node2->details->uname, node2_weight);
         return -1;
     }
 
     if (node1_weight < node2_weight) {
         crm_trace("%s (%d) < %s (%d) : weight",
                   node1->details->uname, node1_weight, node2->details->uname, node2_weight);
         return 1;
     }
 
     crm_trace("%s (%d) == %s (%d) : weight",
               node1->details->uname, node1_weight, node2->details->uname, node2_weight);
 
     if (pcmk__str_eq(nw->data_set->placement_strategy, "minimal", pcmk__str_casei)) {
         goto equal;
     }
 
     if (pcmk__str_eq(nw->data_set->placement_strategy, "balanced", pcmk__str_casei)) {
         result = compare_capacity(node1, node2);
         if (result < 0) {
             crm_trace("%s > %s : capacity (%d)",
                       node1->details->uname, node2->details->uname, result);
             return -1;
         } else if (result > 0) {
             crm_trace("%s < %s : capacity (%d)",
                       node1->details->uname, node2->details->uname, result);
             return 1;
         }
     }
 
     /* now try to balance resources across the cluster */
     if (node1->details->num_resources < node2->details->num_resources) {
         crm_trace("%s (%d) > %s (%d) : resources",
                   node1->details->uname, node1->details->num_resources,
                   node2->details->uname, node2->details->num_resources);
         return -1;
 
     } else if (node1->details->num_resources > node2->details->num_resources) {
         crm_trace("%s (%d) < %s (%d) : resources",
                   node1->details->uname, node1->details->num_resources,
                   node2->details->uname, node2->details->num_resources);
         return 1;
     }
 
     if (nw->active && nw->active->details == node1->details) {
         crm_trace("%s (%d) > %s (%d) : active",
                   node1->details->uname, node1->details->num_resources,
                   node2->details->uname, node2->details->num_resources);
         return -1;
     } else if (nw->active && nw->active->details == node2->details) {
         crm_trace("%s (%d) < %s (%d) : active",
                   node1->details->uname, node1->details->num_resources,
                   node2->details->uname, node2->details->num_resources);
         return 1;
     }
   equal:
     crm_trace("%s = %s", node1->details->uname, node2->details->uname);
     return strcmp(node1->details->uname, node2->details->uname);
 }
 
 GList *
 sort_nodes_by_weight(GList *nodes, pe_node_t *active_node,
                      pe_working_set_t *data_set)
 {
     struct node_weight_s nw = { active_node, data_set };
 
     return g_list_sort_with_data(nodes, sort_node_weight, &nw);
 }
 
 void
 native_deallocate(pe_resource_t * rsc)
 {
     if (rsc->allocated_to) {
         pe_node_t *old = rsc->allocated_to;
 
         crm_info("Deallocating %s from %s", rsc->id, old->details->uname);
         pe__set_resource_flags(rsc, pe_rsc_provisional);
         rsc->allocated_to = NULL;
 
         old->details->allocated_rsc = g_list_remove(old->details->allocated_rsc, rsc);
         old->details->num_resources--;
         /* old->count--; */
         calculate_utilization(old->details->utilization, rsc->utilization, TRUE);
         free(old);
     }
 }
 
 gboolean
 native_assign_node(pe_resource_t *rsc, pe_node_t *chosen, gboolean force)
 {
     pcmk__output_t *out = rsc->cluster->priv;
 
     CRM_ASSERT(rsc->variant == pe_native);
 
     if (force == FALSE && chosen != NULL) {
         bool unset = FALSE;
 
         if(chosen->weight < 0) {
             unset = TRUE;
 
             // Allow the graph to assume that the remote resource will come up
         } else if (!can_run_resources(chosen) && !pe__is_guest_node(chosen)) {
             unset = TRUE;
         }
 
         if(unset) {
             crm_debug("All nodes for resource %s are unavailable"
                       ", unclean or shutting down (%s: %d, %d)",
                       rsc->id, chosen->details->uname, can_run_resources(chosen), chosen->weight);
             pe__set_next_role(rsc, RSC_ROLE_STOPPED, "node availability");
             chosen = NULL;
         }
     }
 
     /* todo: update the old node for each resource to reflect its
      * new resource count
      */
 
     native_deallocate(rsc);
     pe__clear_resource_flags(rsc, pe_rsc_provisional);
 
     if (chosen == NULL) {
         GList *gIter = NULL;
         char *rc_inactive = pcmk__itoa(PCMK_OCF_NOT_RUNNING);
 
         crm_debug("Could not allocate a node for %s", rsc->id);
         pe__set_next_role(rsc, RSC_ROLE_STOPPED, "unable to allocate");
 
         for (gIter = rsc->actions; gIter != NULL; gIter = gIter->next) {
             pe_action_t *op = (pe_action_t *) gIter->data;
             const char *interval_ms_s = g_hash_table_lookup(op->meta, XML_LRM_ATTR_INTERVAL_MS);
 
             crm_debug("Processing %s", op->uuid);
             if(pcmk__str_eq(RSC_STOP, op->task, pcmk__str_casei)) {
-                update_action_flags(op, pe_action_optional | pe_action_clear,
-                                    __func__, __LINE__);
+                pe__clear_action_flags(op, pe_action_optional);
 
             } else if(pcmk__str_eq(RSC_START, op->task, pcmk__str_casei)) {
-                update_action_flags(op, pe_action_runnable | pe_action_clear,
-                                    __func__, __LINE__);
+                pe__clear_action_flags(op, pe_action_runnable);
                 //pe__set_resource_flags(rsc, pe_rsc_block);
 
             } else if (interval_ms_s && !pcmk__str_eq(interval_ms_s, "0", pcmk__str_casei)) {
                 if(pcmk__str_eq(rc_inactive, g_hash_table_lookup(op->meta, XML_ATTR_TE_TARGET_RC), pcmk__str_casei)) {
                     /* This is a recurring monitor for the stopped state, leave it alone */
 
                 } else {
                     /* Normal monitor operation, cancel it */
-                    update_action_flags(op, pe_action_runnable | pe_action_clear,
-                                        __func__, __LINE__);
+                    pe__clear_action_flags(op, pe_action_runnable);
                 }
             }
         }
 
         free(rc_inactive);
         return FALSE;
     }
 
     crm_debug("Assigning %s to %s", chosen->details->uname, rsc->id);
     rsc->allocated_to = pe__copy_node(chosen);
 
     chosen->details->allocated_rsc = g_list_prepend(chosen->details->allocated_rsc, rsc);
     chosen->details->num_resources++;
     chosen->count++;
     calculate_utilization(chosen->details->utilization, rsc->utilization, FALSE);
 
     if (pcmk_is_set(rsc->cluster->flags, pe_flag_show_utilization)) {
         out->message(out, "resource-util", rsc, chosen, __func__);
     }
 
     return TRUE;
 }
 
 void
 log_action(unsigned int log_level, const char *pre_text, pe_action_t * action, gboolean details)
 {
     const char *node_uname = NULL;
     const char *node_uuid = NULL;
     const char *desc = NULL;
 
     if (action == NULL) {
         crm_trace("%s%s: <NULL>", pre_text == NULL ? "" : pre_text, pre_text == NULL ? "" : ": ");
         return;
     }
 
     if (pcmk_is_set(action->flags, pe_action_pseudo)) {
         node_uname = NULL;
         node_uuid = NULL;
 
     } else if (action->node != NULL) {
         node_uname = action->node->details->uname;
         node_uuid = action->node->details->id;
     } else {
         node_uname = "<none>";
         node_uuid = NULL;
     }
 
     switch (text2task(action->task)) {
         case stonith_node:
         case shutdown_crm:
             if (pcmk_is_set(action->flags, pe_action_pseudo)) {
                 desc = "Pseudo ";
             } else if (pcmk_is_set(action->flags, pe_action_optional)) {
                 desc = "Optional ";
             } else if (!pcmk_is_set(action->flags, pe_action_runnable)) {
                 desc = "!!Non-Startable!! ";
             } else if (pcmk_is_set(action->flags, pe_action_processed)) {
                desc = "";
             } else {
                desc = "(Provisional) ";
             }
             crm_trace("%s%s%sAction %d: %s%s%s%s%s%s",
                       ((pre_text == NULL)? "" : pre_text),
                       ((pre_text == NULL)? "" : ": "),
                       desc, action->id, action->uuid,
                       (node_uname? "\ton " : ""), (node_uname? node_uname : ""),
                       (node_uuid? "\t\t(" : ""), (node_uuid? node_uuid : ""),
                       (node_uuid? ")" : ""));
             break;
         default:
             if (pcmk_is_set(action->flags, pe_action_optional)) {
                 desc = "Optional ";
             } else if (pcmk_is_set(action->flags, pe_action_pseudo)) {
                 desc = "Pseudo ";
             } else if (!pcmk_is_set(action->flags, pe_action_runnable)) {
                 desc = "!!Non-Startable!! ";
             } else if (pcmk_is_set(action->flags, pe_action_processed)) {
                desc = "";
             } else {
                desc = "(Provisional) ";
             }
             crm_trace("%s%s%sAction %d: %s %s%s%s%s%s%s",
                       ((pre_text == NULL)? "" : pre_text),
                       ((pre_text == NULL)? "" : ": "),
                       desc, action->id, action->uuid,
                       (action->rsc? action->rsc->id : "<none>"),
                       (node_uname? "\ton " : ""), (node_uname? node_uname : ""),
                       (node_uuid? "\t\t(" : ""), (node_uuid? node_uuid : ""),
                       (node_uuid? ")" : ""));
             break;
     }
 
     if (details) {
         GList *gIter = NULL;
 
         crm_trace("\t\t====== Preceding Actions");
 
         gIter = action->actions_before;
         for (; gIter != NULL; gIter = gIter->next) {
             pe_action_wrapper_t *other = (pe_action_wrapper_t *) gIter->data;
 
             log_action(log_level + 1, "\t\t", other->action, FALSE);
         }
 
         crm_trace("\t\t====== Subsequent Actions");
 
         gIter = action->actions_after;
         for (; gIter != NULL; gIter = gIter->next) {
             pe_action_wrapper_t *other = (pe_action_wrapper_t *) gIter->data;
 
             log_action(log_level + 1, "\t\t", other->action, FALSE);
         }
 
         crm_trace("\t\t====== End");
 
     } else {
         crm_trace("\t\t(before=%d, after=%d)",
                   g_list_length(action->actions_before), g_list_length(action->actions_after));
     }
 }
 
 gboolean
 can_run_any(GHashTable * nodes)
 {
     GHashTableIter iter;
     pe_node_t *node = NULL;
 
     if (nodes == NULL) {
         return FALSE;
     }
 
     g_hash_table_iter_init(&iter, nodes);
     while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
         if (can_run_resources(node) && node->weight >= 0) {
             return TRUE;
         }
     }
 
     return FALSE;
 }
 
 pe_action_t *
 create_pseudo_resource_op(pe_resource_t * rsc, const char *task, bool optional, bool runnable, pe_working_set_t *data_set)
 {
     pe_action_t *action = custom_action(rsc, pcmk__op_key(rsc->id, task, 0),
                                         task, NULL, optional, TRUE, data_set);
-    update_action_flags(action, pe_action_pseudo, __func__, __LINE__);
-    update_action_flags(action, pe_action_runnable, __func__, __LINE__);
+
+    pe__set_action_flags(action, pe_action_pseudo);
     if(runnable) {
-        update_action_flags(action, pe_action_runnable, __func__, __LINE__);
+        pe__set_action_flags(action, pe_action_runnable);
     }
     return action;
 }
 
 /*!
  * \internal
  * \brief Create an executor cancel op
  *
  * \param[in] rsc          Resource of action to cancel
  * \param[in] task         Name of action to cancel
  * \param[in] interval_ms  Interval of action to cancel
  * \param[in] node         Node of action to cancel
  * \param[in] data_set     Working set of cluster
  *
  * \return Created op
  */
 pe_action_t *
 pe_cancel_op(pe_resource_t *rsc, const char *task, guint interval_ms,
              pe_node_t *node, pe_working_set_t *data_set)
 {
     pe_action_t *cancel_op;
     char *interval_ms_s = crm_strdup_printf("%u", interval_ms);
 
     // @TODO dangerous if possible to schedule another action with this key
     char *key = pcmk__op_key(rsc->id, task, interval_ms);
 
     cancel_op = custom_action(rsc, key, RSC_CANCEL, node, FALSE, TRUE,
                               data_set);
 
     free(cancel_op->task);
     cancel_op->task = strdup(RSC_CANCEL);
 
     free(cancel_op->cancel_task);
     cancel_op->cancel_task = strdup(task);
 
     add_hash_param(cancel_op->meta, XML_LRM_ATTR_TASK, task);
     add_hash_param(cancel_op->meta, XML_LRM_ATTR_INTERVAL_MS, interval_ms_s);
     free(interval_ms_s);
 
     return cancel_op;
 }
 
 /*!
  * \internal
  * \brief Create a shutdown op for a scheduler transition
  *
  * \param[in] node         Node being shut down
  * \param[in] data_set     Working set of cluster
  *
  * \return Created op
  */
 pe_action_t *
 sched_shutdown_op(pe_node_t *node, pe_working_set_t *data_set)
 {
     char *shutdown_id = crm_strdup_printf("%s-%s", CRM_OP_SHUTDOWN,
                                           node->details->uname);
 
     pe_action_t *shutdown_op = custom_action(NULL, shutdown_id, CRM_OP_SHUTDOWN,
                                              node, FALSE, TRUE, data_set);
 
     crm_notice("Scheduling shutdown of node %s", node->details->uname);
     shutdown_constraints(node, shutdown_op, data_set);
     add_hash_param(shutdown_op->meta, XML_ATTR_TE_NOWAIT, XML_BOOLEAN_TRUE);
     return shutdown_op;
 }
 
 static char *
 generate_transition_magic(const char *transition_key, int op_status, int op_rc)
 {
     CRM_CHECK(transition_key != NULL, return NULL);
     return crm_strdup_printf("%d:%d;%s", op_status, op_rc, transition_key);
 }
 
 static void
 append_digest(lrmd_event_data_t *op, xmlNode *update, const char *version,
               const char *magic, int level)
 {
     /* this will enable us to later determine that the
      *   resource's parameters have changed and we should force
      *   a restart
      */
     char *digest = NULL;
     xmlNode *args_xml = NULL;
 
     if (op->params == NULL) {
         return;
     }
 
     args_xml = create_xml_node(NULL, XML_TAG_PARAMS);
     g_hash_table_foreach(op->params, hash2field, args_xml);
     pcmk__filter_op_for_digest(args_xml);
     digest = calculate_operation_digest(args_xml, version);
 
 #if 0
     if (level < get_crm_log_level()
         && op->interval_ms == 0 && pcmk__str_eq(op->op_type, CRMD_ACTION_START, pcmk__str_none)) {
         char *digest_source = dump_xml_unformatted(args_xml);
 
         do_crm_log(level, "Calculated digest %s for %s (%s). Source: %s\n",
                    digest, ID(update), magic, digest_source);
         free(digest_source);
     }
 #endif
     crm_xml_add(update, XML_LRM_ATTR_OP_DIGEST, digest);
 
     free_xml(args_xml);
     free(digest);
 }
 
 #define FAKE_TE_ID     "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"
 
 /*!
  * \internal
  * \brief Create XML for resource operation history update
  *
  * \param[in,out] parent          Parent XML node to add to
  * \param[in,out] op              Operation event data
  * \param[in]     caller_version  DC feature set
  * \param[in]     target_rc       Expected result of operation
  * \param[in]     node            Name of node on which operation was performed
  * \param[in]     origin          Arbitrary description of update source
  * \param[in]     level           A log message will be logged at this level
  *
  * \return Newly created XML node for history update
  */
 xmlNode *
 pcmk__create_history_xml(xmlNode *parent, lrmd_event_data_t *op,
                          const char *caller_version, int target_rc,
                          const char *node, const char *origin, int level)
 {
     char *key = NULL;
     char *magic = NULL;
     char *op_id = NULL;
     char *op_id_additional = NULL;
     char *local_user_data = NULL;
     const char *exit_reason = NULL;
 
     xmlNode *xml_op = NULL;
     const char *task = NULL;
 
     CRM_CHECK(op != NULL, return NULL);
     do_crm_log(level, "%s: Updating resource %s after %s op %s (interval=%u)",
                origin, op->rsc_id, op->op_type, services_lrm_status_str(op->op_status),
                op->interval_ms);
 
     crm_trace("DC version: %s", caller_version);
 
     task = op->op_type;
 
     /* Record a successful agent reload as a start, and a failed one as a
      * monitor, to make life easier for the scheduler when determining the
      * current state.
      *
      * @COMPAT We should check "reload" here only if the operation was for a
      * pre-OCF-1.1 resource agent, but we don't know that here, and we should
      * only ever get results for actions scheduled by us, so we can reasonably
      * assume any "reload" is actually a pre-1.1 agent reload.
      */
     if (pcmk__str_any_of(task, CRMD_ACTION_RELOAD, CRMD_ACTION_RELOAD_AGENT,
                          NULL)) {
         if (op->op_status == PCMK_LRM_OP_DONE) {
             task = CRMD_ACTION_START;
         } else {
             task = CRMD_ACTION_STATUS;
         }
     }
 
     key = pcmk__op_key(op->rsc_id, task, op->interval_ms);
     if (pcmk__str_eq(task, CRMD_ACTION_NOTIFY, pcmk__str_none)) {
         const char *n_type = crm_meta_value(op->params, "notify_type");
         const char *n_task = crm_meta_value(op->params, "notify_operation");
 
         CRM_LOG_ASSERT(n_type != NULL);
         CRM_LOG_ASSERT(n_task != NULL);
         op_id = pcmk__notify_key(op->rsc_id, n_type, n_task);
 
         if (op->op_status != PCMK_LRM_OP_PENDING) {
             /* Ignore notify errors.
              *
              * @TODO It might be better to keep the correct result here, and
              * ignore it in process_graph_event().
              */
             op->op_status = PCMK_LRM_OP_DONE;
             op->rc = 0;
         }
 
     } else if (did_rsc_op_fail(op, target_rc)) {
         op_id = pcmk__op_key(op->rsc_id, "last_failure", 0);
         if (op->interval_ms == 0) {
             // Ensure 'last' gets updated, in case record-pending is true
             op_id_additional = pcmk__op_key(op->rsc_id, "last", 0);
         }
         exit_reason = op->exit_reason;
 
     } else if (op->interval_ms > 0) {
         op_id = strdup(key);
 
     } else {
         op_id = pcmk__op_key(op->rsc_id, "last", 0);
     }
 
   again:
     xml_op = pcmk__xe_match(parent, XML_LRM_TAG_RSC_OP, XML_ATTR_ID, op_id);
     if (xml_op == NULL) {
         xml_op = create_xml_node(parent, XML_LRM_TAG_RSC_OP);
     }
 
     if (op->user_data == NULL) {
         crm_debug("Generating fake transition key for: " PCMK__OP_FMT
                   " %d from %s", op->rsc_id, op->op_type, op->interval_ms,
                   op->call_id, origin);
         local_user_data = pcmk__transition_key(-1, op->call_id, target_rc,
                                                FAKE_TE_ID);
         op->user_data = local_user_data;
     }
 
     if(magic == NULL) {
         magic = generate_transition_magic(op->user_data, op->op_status, op->rc);
     }
 
     crm_xml_add(xml_op, XML_ATTR_ID, op_id);
     crm_xml_add(xml_op, XML_LRM_ATTR_TASK_KEY, key);
     crm_xml_add(xml_op, XML_LRM_ATTR_TASK, task);
     crm_xml_add(xml_op, XML_ATTR_ORIGIN, origin);
     crm_xml_add(xml_op, XML_ATTR_CRM_VERSION, caller_version);
     crm_xml_add(xml_op, XML_ATTR_TRANSITION_KEY, op->user_data);
     crm_xml_add(xml_op, XML_ATTR_TRANSITION_MAGIC, magic);
     crm_xml_add(xml_op, XML_LRM_ATTR_EXIT_REASON, exit_reason == NULL ? "" : exit_reason);
     crm_xml_add(xml_op, XML_LRM_ATTR_TARGET, node); /* For context during triage */
 
     crm_xml_add_int(xml_op, XML_LRM_ATTR_CALLID, op->call_id);
     crm_xml_add_int(xml_op, XML_LRM_ATTR_RC, op->rc);
     crm_xml_add_int(xml_op, XML_LRM_ATTR_OPSTATUS, op->op_status);
     crm_xml_add_ms(xml_op, XML_LRM_ATTR_INTERVAL_MS, op->interval_ms);
 
     if (compare_version("2.1", caller_version) <= 0) {
         if (op->t_run || op->t_rcchange || op->exec_time || op->queue_time) {
             crm_trace("Timing data (" PCMK__OP_FMT
                       "): last=%u change=%u exec=%u queue=%u",
                       op->rsc_id, op->op_type, op->interval_ms,
                       op->t_run, op->t_rcchange, op->exec_time, op->queue_time);
 
             if ((op->interval_ms != 0) && (op->t_rcchange != 0)) {
                 // Recurring ops may have changed rc after initial run
                 crm_xml_add_ll(xml_op, XML_RSC_OP_LAST_CHANGE,
                                (long long) op->t_rcchange);
             } else {
                 crm_xml_add_ll(xml_op, XML_RSC_OP_LAST_CHANGE,
                                (long long) op->t_run);
             }
 
             crm_xml_add_int(xml_op, XML_RSC_OP_T_EXEC, op->exec_time);
             crm_xml_add_int(xml_op, XML_RSC_OP_T_QUEUE, op->queue_time);
         }
     }
 
     if (pcmk__str_any_of(op->op_type, CRMD_ACTION_MIGRATE, CRMD_ACTION_MIGRATED, NULL)) {
         /*
          * Record migrate_source and migrate_target always for migrate ops.
          */
         const char *name = XML_LRM_ATTR_MIGRATE_SOURCE;
 
         crm_xml_add(xml_op, name, crm_meta_value(op->params, name));
 
         name = XML_LRM_ATTR_MIGRATE_TARGET;
         crm_xml_add(xml_op, name, crm_meta_value(op->params, name));
     }
 
     append_digest(op, xml_op, caller_version, magic, LOG_DEBUG);
 
     if (op_id_additional) {
         free(op_id);
         op_id = op_id_additional;
         op_id_additional = NULL;
         goto again;
     }
 
     if (local_user_data) {
         free(local_user_data);
         op->user_data = NULL;
     }
     free(magic);
     free(op_id);
     free(key);
     return xml_op;
 }
 
 pcmk__output_t *
 pcmk__new_logger(void)
 {
     int rc = pcmk_rc_ok;
     pcmk__output_t *out = NULL;
     const char* argv[] = { "", NULL };
     pcmk__supported_format_t formats[] = {
         PCMK__SUPPORTED_FORMAT_LOG,
         { NULL, NULL, NULL }
     };
 
     pcmk__register_formats(NULL, formats);
     rc = pcmk__output_new(&out, "log", NULL, (char**)argv);
     if ((rc != pcmk_rc_ok) || (out == NULL)) {
         crm_err("Can't log resource details due to internal error: %s\n",
                 pcmk_rc_str(rc));
         return NULL;
     }
 
     pe__register_messages(out);
     pcmk__register_lib_messages(out);
     return out;
 }