No OneTemporary
Actions

Size

232 KB

Referenced Files

None

Subscribers

None

View Options

	diff --git a/cts/cts-scheduler.in b/cts/cts-scheduler.in
	index 6a12ded8ee..ee0cb7b472 100644
	--- a/cts/cts-scheduler.in
	+++ b/cts/cts-scheduler.in
	@@ -1,1623 +1,1626 @@
	#!@PYTHON@
	""" Regression tests for Pacemaker's scheduler
	"""

	__copyright__ = "Copyright 2004-2023 the Pacemaker project contributors"
	__license__ = "GNU General Public License version 2 or later (GPLv2+) WITHOUT ANY WARRANTY"

	import io
	import os
	import re
	import sys
	import stat
	import shlex
	import shutil
	import argparse
	import subprocess
	import platform
	import tempfile

	# These imports allow running from a source checkout after running `make`.
	# Note that while this doesn't necessarily mean it will successfully run tests,
	# but being able to see --help output can be useful.
	if os.path.exists("@abs_top_srcdir@/python"):
	sys.path.insert(0, "@abs_top_srcdir@/python")

	if os.path.exists("@abs_top_builddir@/python") and "@abs_top_builddir@" != "@abs_top_srcdir@":
	sys.path.insert(0, "@abs_top_builddir@/python")

	from pacemaker.buildoptions import BuildOptions
	from pacemaker.exitstatus import ExitStatus

	DESC = """Regression tests for Pacemaker's scheduler"""

	# Each entry in TESTS is a group of tests, where each test consists of a
	# test base name, test description, and additional test arguments.
	# Test groups will be separated by newlines in output.
	TESTS = [
	[
	[ "simple1", "Offline" ],
	[ "simple2", "Start" ],
	[ "simple3", "Start 2" ],
	[ "simple4", "Start Failed" ],
	[ "simple6", "Stop Start" ],
	[ "simple7", "Shutdown" ],
	#[ "simple8", "Stonith" ],
	#[ "simple9", "Lower version" ],
	#[ "simple10", "Higher version" ],
	[ "simple11", "Priority (ne)" ],
	[ "simple12", "Priority (eq)" ],
	[ "simple8", "Stickiness" ],
	],
	[
	[ "group1", "Group" ],
	[ "group2", "Group + Native" ],
	[ "group3", "Group + Group" ],
	[ "group4", "Group + Native (nothing)" ],
	[ "group5", "Group + Native (move)" ],
	[ "group6", "Group + Group (move)" ],
	[ "group7", "Group colocation" ],
	[ "group13", "Group colocation (cant run)" ],
	[ "group8", "Group anti-colocation" ],
	[ "group9", "Group recovery" ],
	[ "group10", "Group partial recovery" ],
	[ "group11", "Group target_role" ],
	[ "group14", "Group stop (graph terminated)" ],
	[ "group15", "Negative group colocation" ],
	[ "bug-1573", "Partial stop of a group with two children" ],
	[ "bug-1718", "Mandatory group ordering - Stop group_FUN" ],
	[ "failed-sticky-group", "Move group on last member failure despite infinite stickiness" ],
	[ "failed-sticky-anticolocated-group",
	"Move group on last member failure despite infinite stickiness and optional anti-colocation"
	],
	[ "bug-lf-2619", "Move group on clone failure" ],
	[ "group-fail", "Ensure stop order is preserved for partially active groups" ],
	[ "group-unmanaged", "No need to restart r115 because r114 is unmanaged" ],
	[ "group-unmanaged-stopped", "Make sure r115 is stopped when r114 fails" ],
	+ [ "partial-unmanaged-group",
	+ "New member in partially unmanaged group"
	+ ],
	[ "group-dependents", "Account for the location preferences of things colocated with a group" ],
	[ "group-stop-ordering", "Ensure blocked group member stop does not force other member stops" ],
	[ "colocate-unmanaged-group", "Respect mandatory colocations even if earlier group member is unmanaged" ],
	],
	[
	[ "rsc_dep1", "Must not" ],
	[ "rsc_dep3", "Must" ],
	[ "rsc_dep5", "Must not 3" ],
	[ "rsc_dep7", "Must 3" ],
	[ "rsc_dep10", "Must (but cant)" ],
	[ "rsc_dep2", "Must (running)" ],
	[ "rsc_dep8", "Must (running : alt)" ],
	[ "rsc_dep4", "Must (running + move)" ],
	[ "asymmetric", "Asymmetric - require explicit location constraints" ],
	],
	[
	[ "orphan-0", "Orphan ignore" ],
	[ "orphan-1", "Orphan stop" ],
	[ "orphan-2", "Orphan stop, remove failcount" ],
	],
	[
	[ "params-0", "Params: No change" ],
	[ "params-1", "Params: Changed" ],
	[ "params-2", "Params: Resource definition" ],
	[ "params-3", "Params: Restart instead of reload if start pending" ],
	[ "params-4", "Params: Reload" ],
	[ "params-5", "Params: Restart based on probe digest" ],
	[ "novell-251689", "Resource definition change + target_role=stopped" ],
	[ "bug-lf-2106", "Restart all anonymous clone instances after config change" ],
	[ "params-6", "Params: Detect reload in previously migrated resource" ],
	[ "nvpair-id-ref", "Support id-ref in nvpair with optional name" ],
	[ "not-reschedule-unneeded-monitor",
	"Do not reschedule unneeded monitors while resource definitions have changed" ],
	[ "reload-becomes-restart", "Cancel reload if restart becomes required" ],
	[ "restart-with-extra-op-params", "Restart if with extra operation parameters upon changes of any" ],
	],
	[
	[ "target-0", "Target Role : baseline" ],
	[ "target-1", "Target Role : promoted" ],
	[ "target-2", "Target Role : invalid" ],
	],
	[
	[ "base-score", "Set a node's default score for all nodes" ],
	],
	[
	[ "date-1", "Dates", [ "-t", "2005-020" ] ],
	[ "date-2", "Date Spec - Pass", [ "-t", "2005-020T12:30" ] ],
	[ "date-3", "Date Spec - Fail", [ "-t", "2005-020T11:30" ] ],
	[ "origin", "Timing of recurring operations", [ "-t", "2014-05-07 00:28:00" ] ],
	[ "probe-0", "Probe (anon clone)" ],
	[ "probe-1", "Pending Probe" ],
	[ "probe-2", "Correctly re-probe cloned groups" ],
	[ "probe-3", "Probe (pending node)" ],
	[ "probe-4", "Probe (pending node + stopped resource)" ],
	[ "probe-pending-node", "Probe (pending node + unmanaged resource)" ],
	[ "failed-probe-primitive", "Maskable vs. unmaskable probe failures on primitive resources" ],
	[ "failed-probe-clone", "Maskable vs. unmaskable probe failures on cloned resources" ],
	[ "expired-failed-probe-primitive", "Maskable, expired probe failure on primitive resources" ],
	[ "standby", "Standby" ],
	[ "comments", "Comments" ],
	],
	[
	[ "one-or-more-0", "Everything starts" ],
	[ "one-or-more-1", "Nothing starts because of A" ],
	[ "one-or-more-2", "D can start because of C" ],
	[ "one-or-more-3", "D cannot start because of B and C" ],
	[ "one-or-more-4", "D cannot start because of target-role" ],
	[ "one-or-more-5", "Start A and F even though C and D are stopped" ],
	[ "one-or-more-6", "Leave A running even though B is stopped" ],
	[ "one-or-more-7", "Leave A running even though C is stopped" ],
	[ "bug-5140-require-all-false", "Allow basegrp:0 to stop" ],
	[ "clone-require-all-1", "clone B starts node 3 and 4" ],
	[ "clone-require-all-2", "clone B remains stopped everywhere" ],
	[ "clone-require-all-3", "clone B stops everywhere because A stops everywhere" ],
	[ "clone-require-all-4", "clone B remains on node 3 and 4 with only one instance of A remaining" ],
	[ "clone-require-all-5", "clone B starts on node 1 3 and 4" ],
	[ "clone-require-all-6", "clone B remains active after shutting down instances of A" ],
	[ "clone-require-all-7",
	"clone A and B both start at the same time. all instances of A start before B" ],
	[ "clone-require-all-no-interleave-1", "C starts everywhere after A and B" ],
	[ "clone-require-all-no-interleave-2",
	"C starts on nodes 1, 2, and 4 with only one active instance of B" ],
	[ "clone-require-all-no-interleave-3",
	"C remains active when instance of B is stopped on one node and started on another" ],
	[ "one-or-more-unrunnable-instances", "Avoid dependencies on instances that won't ever be started" ],
	],
	[
	[ "location-date-rules-1", "Use location constraints with ineffective date-based rules" ],
	[ "location-date-rules-2", "Use location constraints with effective date-based rules" ],
	[ "nvpair-date-rules-1", "Use nvpair blocks with a variety of date-based rules" ],
	[ "value-source", "Use location constraints with node attribute expressions using value-source" ],
	[ "rule-dbl-as-auto-number-match",
	"Floating-point rule values default to number comparison: match" ],
	[ "rule-dbl-as-auto-number-no-match",
	"Floating-point rule values default to number comparison: no "
	"match" ],
	[ "rule-dbl-as-integer-match",
	"Floating-point rule values set to integer comparison: match" ],
	[ "rule-dbl-as-integer-no-match",
	"Floating-point rule values set to integer comparison: no match" ],
	[ "rule-dbl-as-number-match",
	"Floating-point rule values set to number comparison: match" ],
	[ "rule-dbl-as-number-no-match",
	"Floating-point rule values set to number comparison: no match" ],
	[ "rule-dbl-parse-fail-default-str-match",
	"Floating-point rule values fail to parse, default to string "
	"comparison: match" ],
	[ "rule-dbl-parse-fail-default-str-no-match",
	"Floating-point rule values fail to parse, default to string "
	"comparison: no match" ],
	[ "rule-int-as-auto-integer-match",
	"Integer rule values default to integer comparison: match" ],
	[ "rule-int-as-auto-integer-no-match",
	"Integer rule values default to integer comparison: no match" ],
	[ "rule-int-as-integer-match",
	"Integer rule values set to integer comparison: match" ],
	[ "rule-int-as-integer-no-match",
	"Integer rule values set to integer comparison: no match" ],
	[ "rule-int-as-number-match",
	"Integer rule values set to number comparison: match" ],
	[ "rule-int-as-number-no-match",
	"Integer rule values set to number comparison: no match" ],
	[ "rule-int-parse-fail-default-str-match",
	"Integer rule values fail to parse, default to string "
	"comparison: match" ],
	[ "rule-int-parse-fail-default-str-no-match",
	"Integer rule values fail to parse, default to string "
	"comparison: no match" ],
	],
	[
	[ "order1", "Order start 1" ],
	[ "order2", "Order start 2" ],
	[ "order3", "Order stop" ],
	[ "order4", "Order (multiple)" ],
	[ "order5", "Order (move)" ],
	[ "order6", "Order (move w/ restart)" ],
	[ "order7", "Order (mandatory)" ],
	[ "order-optional", "Order (score=0)" ],
	[ "order-required", "Order (score=INFINITY)" ],
	[ "bug-lf-2171", "Prevent group start when clone is stopped" ],
	[ "order-clone", "Clone ordering should be able to prevent startup of dependent clones" ],
	[ "order-sets", "Ordering for resource sets" ],
	[ "order-serialize", "Serialize resources without inhibiting migration" ],
	[ "order-serialize-set", "Serialize a set of resources without inhibiting migration" ],
	[ "clone-order-primitive", "Order clone start after a primitive" ],
	[ "clone-order-16instances", "Verify ordering of 16 cloned resources" ],
	[ "order-optional-keyword", "Order (optional keyword)" ],
	[ "order-mandatory", "Order (mandatory keyword)" ],
	[ "bug-lf-2493",
	"Don't imply colocation requirements when applying ordering constraints with clones" ],
	[ "ordered-set-basic-startup", "Constraint set with default order settings" ],
	[ "ordered-set-natural", "Allow natural set ordering" ],
	[ "order-wrong-kind", "Order (error)" ],
	],
	[
	[ "coloc-loop", "Colocation - loop" ],
	[ "coloc-many-one", "Colocation - many-to-one" ],
	[ "coloc-list", "Colocation - many-to-one with list" ],
	[ "coloc-group", "Colocation - groups" ],
	[ "coloc-unpromoted-anti", "Anti-colocation with unpromoted shouldn't prevent promoted colocation" ],
	[ "coloc-attr", "Colocation based on node attributes" ],
	[ "coloc-negative-group", "Negative colocation with a group" ],
	[ "coloc-intra-set", "Intra-set colocation" ],
	[ "bug-lf-2435", "Colocation sets with a negative score" ],
	[ "coloc-clone-stays-active",
	"Ensure clones don't get stopped/demoted because a dependent must stop" ],
	[ "coloc_fp_logic", "Verify floating point calculations in colocation are working" ],
	[ "colo_promoted_w_native",
	"cl#5070 - Verify promotion order is affected when colocating promoted with primitive" ],
	[ "colo_unpromoted_w_native",
	"cl#5070 - Verify promotion order is affected when colocating unpromoted with primitive" ],
	[ "anti-colocation-order",
	"cl#5187 - Prevent resources in an anti-colocation from even temporarily running on a same node" ],
	[ "anti-colocation-promoted", "Organize order of actions for promoted resources in anti-colocations" ],
	[ "anti-colocation-unpromoted", "Organize order of actions for unpromoted resources in anti-colocations" ],
	[ "group-anticolocation", "Group with failed last member anti-colocated with another group" ],
	[ "group-colocation-failure",
	"Group with sole member failed, colocated with another group"
	],
	[ "enforce-colo1", "Always enforce B with A INFINITY" ],
	[ "complex_enforce_colo", "Always enforce B with A INFINITY. (make sure heat-engine stops)" ],
	[ "coloc-dependee-should-stay", "Stickiness outweighs group colocation" ],
	[ "coloc-dependee-should-move", "Group colocation outweighs stickiness" ],
	[ "colocation-influence", "Respect colocation influence" ],
	[ "colocation-priority-group", "Apply group colocations in order of primary priority" ],
	[ "colocation-vs-stickiness", "Group stickiness outweighs anti-colocation score" ],
	[ "promoted-with-blocked", "Promoted role colocated with a resource with blocked start" ],
	[ "primitive-with-group-with-clone",
	"Consider group dependent when colocating with clone"
	],
	[ "primitive-with-group-with-promoted",
	"Consider group dependent when colocating with promoted role"
	],
	[ "primitive-with-unrunnable-group",
	"Block primitive colocated with group that can't start",
	],
	],
	[
	[ "rsc-sets-seq-true", "Resource Sets - sequential=false" ],
	[ "rsc-sets-seq-false", "Resource Sets - sequential=true" ],
	[ "rsc-sets-clone", "Resource Sets - Clone" ],
	[ "rsc-sets-promoted", "Resource Sets - Promoted" ],
	[ "rsc-sets-clone-1", "Resource Sets - Clone (lf#2404)" ],
	],
	[
	[ "attrs1", "string: eq (and)" ],
	[ "attrs2", "string: lt / gt (and)" ],
	[ "attrs3", "string: ne (or)" ],
	[ "attrs4", "string: exists" ],
	[ "attrs5", "string: not_exists" ],
	[ "attrs6", "is_dc: true" ],
	[ "attrs7", "is_dc: false" ],
	[ "attrs8", "score_attribute" ],
	[ "per-node-attrs", "Per node resource parameters" ],
	],
	[
	[ "mon-rsc-1", "Schedule Monitor - start" ],
	[ "mon-rsc-2", "Schedule Monitor - move" ],
	[ "mon-rsc-3", "Schedule Monitor - pending start" ],
	[ "mon-rsc-4", "Schedule Monitor - move/pending start" ],
	],
	[
	[ "rec-rsc-0", "Resource Recover - no start" ],
	[ "rec-rsc-1", "Resource Recover - start" ],
	[ "rec-rsc-2", "Resource Recover - monitor" ],
	[ "rec-rsc-3", "Resource Recover - stop - ignore" ],
	[ "rec-rsc-4", "Resource Recover - stop - block" ],
	[ "rec-rsc-5", "Resource Recover - stop - fence" ],
	[ "rec-rsc-6", "Resource Recover - multiple - restart" ],
	[ "rec-rsc-7", "Resource Recover - multiple - stop" ],
	[ "rec-rsc-8", "Resource Recover - multiple - block" ],
	[ "rec-rsc-9", "Resource Recover - group/group" ],
	[ "stop-unexpected", "Recover multiply active group with stop_unexpected" ],
	[ "stop-unexpected-2", "Resource multiply active primitve with stop_unexpected" ],
	[ "monitor-recovery", "on-fail=block + resource recovery detected by recurring monitor" ],
	[ "stop-failure-no-quorum", "Stop failure without quorum" ],
	[ "stop-failure-no-fencing", "Stop failure without fencing available" ],
	[ "stop-failure-with-fencing", "Stop failure with fencing available" ],
	[ "multiple-active-block-group", "Support of multiple-active=block for resource groups" ],
	[ "multiple-monitor-one-failed",
	"Consider resource failed if any of the configured monitor operations failed" ],
	],
	[
	[ "quorum-1", "No quorum - ignore" ],
	[ "quorum-2", "No quorum - freeze" ],
	[ "quorum-3", "No quorum - stop" ],
	[ "quorum-4", "No quorum - start anyway" ],
	[ "quorum-5", "No quorum - start anyway (group)" ],
	[ "quorum-6", "No quorum - start anyway (clone)" ],
	[ "bug-cl-5212", "No promotion with no-quorum-policy=freeze" ],
	[ "suicide-needed-inquorate", "no-quorum-policy=suicide: suicide necessary" ],
	[ "suicide-not-needed-initial-quorum",
	"no-quorum-policy=suicide: suicide not necessary at initial quorum" ],
	[ "suicide-not-needed-never-quorate",
	"no-quorum-policy=suicide: suicide not necessary if never quorate" ],
	[ "suicide-not-needed-quorate", "no-quorum-policy=suicide: suicide necessary if quorate" ],
	],
	[
	[ "rec-node-1", "Node Recover - Startup - no fence" ],
	[ "rec-node-2", "Node Recover - Startup - fence" ],
	[ "rec-node-3", "Node Recover - HA down - no fence" ],
	[ "rec-node-4", "Node Recover - HA down - fence" ],
	[ "rec-node-5", "Node Recover - CRM down - no fence" ],
	[ "rec-node-6", "Node Recover - CRM down - fence" ],
	[ "rec-node-7", "Node Recover - no quorum - ignore" ],
	[ "rec-node-8", "Node Recover - no quorum - freeze" ],
	[ "rec-node-9", "Node Recover - no quorum - stop" ],
	[ "rec-node-10", "Node Recover - no quorum - stop w/fence" ],
	[ "rec-node-11", "Node Recover - CRM down w/ group - fence" ],
	[ "rec-node-12", "Node Recover - nothing active - fence" ],
	[ "rec-node-13", "Node Recover - failed resource + shutdown - fence" ],
	[ "rec-node-15", "Node Recover - unknown lrm section" ],
	[ "rec-node-14", "Serialize all stonith's" ],
	],
	[
	[ "multi1", "Multiple Active (stop/start)" ],
	],
	[
	[ "migrate-begin", "Normal migration" ],
	[ "migrate-success", "Completed migration" ],
	[ "migrate-partial-1", "Completed migration, missing stop on source" ],
	[ "migrate-partial-2", "Successful migrate_to only" ],
	[ "migrate-partial-3", "Successful migrate_to only, target down" ],
	[ "migrate-partial-4", "Migrate from the correct host after migrate_to+migrate_from" ],
	[ "bug-5186-partial-migrate", "Handle partial migration when src node loses membership" ],
	[ "migrate-fail-2", "Failed migrate_from" ],
	[ "migrate-fail-3", "Failed migrate_from + stop on source" ],
	[ "migrate-fail-4",
	"Failed migrate_from + stop on target - ideally we wouldn't need to re-stop on target" ],
	[ "migrate-fail-5", "Failed migrate_from + stop on source and target" ],
	[ "migrate-fail-6", "Failed migrate_to" ],
	[ "migrate-fail-7", "Failed migrate_to + stop on source" ],
	[ "migrate-fail-8",
	"Failed migrate_to + stop on target - ideally we wouldn't need to re-stop on target" ],
	[ "migrate-fail-9", "Failed migrate_to + stop on source and target" ],
	[ "migration-ping-pong", "Old migrate_to failure + successful migrate_from on same node" ],
	[ "migrate-stop", "Migration in a stopping stack" ],
	[ "migrate-start", "Migration in a starting stack" ],
	[ "migrate-stop_start", "Migration in a restarting stack" ],
	[ "migrate-stop-complex", "Migration in a complex stopping stack" ],
	[ "migrate-start-complex", "Migration in a complex starting stack" ],
	[ "migrate-stop-start-complex", "Migration in a complex moving stack" ],
	[ "migrate-shutdown", "Order the post-migration 'stop' before node shutdown" ],
	[ "migrate-1", "Migrate (migrate)" ],
	[ "migrate-2", "Migrate (stable)" ],
	[ "migrate-3", "Migrate (failed migrate_to)" ],
	[ "migrate-4", "Migrate (failed migrate_from)" ],
	[ "novell-252693", "Migration in a stopping stack" ],
	[ "novell-252693-2", "Migration in a starting stack" ],
	[ "novell-252693-3", "Non-Migration in a starting and stopping stack" ],
	[ "bug-1820", "Migration in a group" ],
	[ "bug-1820-1", "Non-migration in a group" ],
	[ "migrate-5", "Primitive migration with a clone" ],
	[ "migrate-fencing", "Migration after Fencing" ],
	[ "migrate-both-vms", "Migrate two VMs that have no colocation" ],
	[ "migration-behind-migrating-remote", "Migrate resource behind migrating remote connection" ],
	[ "1-a-then-bm-move-b", "Advanced migrate logic. A then B. migrate B" ],
	[ "2-am-then-b-move-a", "Advanced migrate logic, A then B, migrate A without stopping B" ],
	[ "3-am-then-bm-both-migrate", "Advanced migrate logic. A then B. migrate both" ],
	[ "4-am-then-bm-b-not-migratable", "Advanced migrate logic, A then B, B not migratable" ],
	[ "5-am-then-bm-a-not-migratable", "Advanced migrate logic. A then B. move both, a not migratable" ],
	[ "6-migrate-group", "Advanced migrate logic, migrate a group" ],
	[ "7-migrate-group-one-unmigratable",
	"Advanced migrate logic, migrate group mixed with allow-migrate true/false" ],
	[ "8-am-then-bm-a-migrating-b-stopping",
	"Advanced migrate logic, A then B, A migrating, B stopping" ],
	[ "9-am-then-bm-b-migrating-a-stopping",
	"Advanced migrate logic, A then B, B migrate, A stopping" ],
	[ "10-a-then-bm-b-move-a-clone",
	"Advanced migrate logic, A clone then B, migrate B while stopping A" ],
	[ "11-a-then-bm-b-move-a-clone-starting",
	"Advanced migrate logic, A clone then B, B moving while A is start/stopping" ],
	[ "a-promote-then-b-migrate", "A promote then B start. migrate B" ],
	[ "a-demote-then-b-migrate", "A demote then B stop. migrate B" ],
	[ "probe-target-of-failed-migrate_to-1", "Failed migrate_to, target rejoins" ],
	[ "probe-target-of-failed-migrate_to-2", "Failed migrate_to, target rejoined and probed" ],
	[ "partial-live-migration-multiple-active", "Prevent running on multiple nodes due to partial live migration" ],
	[ "migration-intermediary-cleaned",
	"Probe live-migration intermediary with no history"
	],
	[ "bug-lf-2422", "Dependency on partially active group - stop ocfs:*" ],
	],
	[
	[ "clone-anon-probe-1", "Probe the correct (anonymous) clone instance for each node" ],
	[ "clone-anon-probe-2", "Avoid needless re-probing of anonymous clones" ],
	[ "clone-anon-failcount", "Merge failcounts for anonymous clones" ],
	[ "force-anon-clone-max", "Update clone-max properly when forcing a clone to be anonymous" ],
	[ "anon-instance-pending", "Assign anonymous clone instance numbers properly when action pending" ],
	[ "inc0", "Incarnation start" ],
	[ "inc1", "Incarnation start order" ],
	[ "inc2", "Incarnation silent restart, stop, move" ],
	[ "inc3", "Inter-incarnation ordering, silent restart, stop, move" ],
	[ "inc4", "Inter-incarnation ordering, silent restart, stop, move (ordered)" ],
	[ "inc5", "Inter-incarnation ordering, silent restart, stop, move (restart 1)" ],
	[ "inc6", "Inter-incarnation ordering, silent restart, stop, move (restart 2)" ],
	[ "inc7", "Clone colocation" ],
	[ "inc8", "Clone anti-colocation" ],
	[ "inc9", "Non-unique clone" ],
	[ "inc10", "Non-unique clone (stop)" ],
	[ "inc11", "Primitive colocation with clones" ],
	[ "inc12", "Clone shutdown" ],
	[ "cloned-group", "Make sure only the correct number of cloned groups are started" ],
	[ "cloned-group-stop", "Ensure stopping qpidd also stops glance and cinder" ],
	[ "clone-no-shuffle", "Don't prioritize allocation of instances that must be moved" ],
	[ "clone-max-zero", "Orphan processing with clone-max=0" ],
	[ "clone-anon-dup",
	"Bug LF#2087 - Correctly parse the state of anonymous clones that are active more than once per node" ],
	[ "bug-lf-2160", "Don't shuffle clones due to colocation" ],
	[ "bug-lf-2213", "clone-node-max enforcement for cloned groups" ],
	[ "bug-lf-2153", "Clone ordering constraints" ],
	[ "bug-lf-2361", "Ensure clones observe mandatory ordering constraints if the LHS is unrunnable" ],
	[ "bug-lf-2317", "Avoid needless restart of primitive depending on a clone" ],
	[ "bug-lf-2453", "Enforce mandatory clone ordering without colocation" ],
	[ "bug-lf-2508", "Correctly reconstruct the status of anonymous cloned groups" ],
	[ "bug-lf-2544", "Balanced clone placement" ],
	[ "bug-lf-2445", "Redistribute clones with node-max > 1 and stickiness = 0" ],
	[ "bug-lf-2574", "Avoid clone shuffle" ],
	[ "bug-lf-2581", "Avoid group restart due to unrelated clone (re)start" ],
	[ "bug-cl-5168", "Don't shuffle clones" ],
	[ "bug-cl-5170", "Prevent clone from starting with on-fail=block" ],
	[ "clone-fail-block-colocation", "Move colocated group when failed clone has on-fail=block" ],
	[ "clone-interleave-1",
	"Clone-3 cannot start on pcmk-1 due to interleaved ordering (no colocation)" ],
	[ "clone-interleave-2", "Clone-3 must stop on pcmk-1 due to interleaved ordering (no colocation)" ],
	[ "clone-interleave-3",
	"Clone-3 must be recovered on pcmk-1 due to interleaved ordering (no colocation)" ],
	[ "rebalance-unique-clones", "Rebalance unique clone instances with no stickiness" ],
	[ "clone-requires-quorum-recovery", "Clone with requires=quorum on failed node needing recovery" ],
	[ "clone-requires-quorum",
	"Clone with requires=quorum with presumed-inactive instance on failed node" ],
	],
	[
	[ "cloned_start_one", "order first clone then clone... first clone_min=2" ],
	[ "cloned_start_two", "order first clone then clone... first clone_min=2" ],
	[ "cloned_stop_one", "order first clone then clone... first clone_min=2" ],
	[ "cloned_stop_two", "order first clone then clone... first clone_min=2" ],
	[ "clone_min_interleave_start_one",
	"order first clone then clone... first clone_min=2 and then has interleave=true" ],
	[ "clone_min_interleave_start_two",
	"order first clone then clone... first clone_min=2 and then has interleave=true" ],
	[ "clone_min_interleave_stop_one",
	"order first clone then clone... first clone_min=2 and then has interleave=true" ],
	[ "clone_min_interleave_stop_two",
	"order first clone then clone... first clone_min=2 and then has interleave=true" ],
	[ "clone_min_start_one", "order first clone then primitive... first clone_min=2" ],
	[ "clone_min_start_two", "order first clone then primitive... first clone_min=2" ],
	[ "clone_min_stop_all", "order first clone then primitive... first clone_min=2" ],
	[ "clone_min_stop_one", "order first clone then primitive... first clone_min=2" ],
	[ "clone_min_stop_two", "order first clone then primitive... first clone_min=2" ],
	],
	[
	[ "unfence-startup", "Clean unfencing" ],
	[ "unfence-definition", "Unfencing when the agent changes" ],
	[ "unfence-parameters", "Unfencing when the agent parameters changes" ],
	[ "unfence-device", "Unfencing when a cluster has only fence devices" ],
	],
	[
	[ "promoted-0", "Stopped -> Unpromoted" ],
	[ "promoted-1", "Stopped -> Promote" ],
	[ "promoted-2", "Stopped -> Promote : notify" ],
	[ "promoted-3", "Stopped -> Promote : promoted location" ],
	[ "promoted-4", "Started -> Promote : promoted location" ],
	[ "promoted-5", "Promoted -> Promoted" ],
	[ "promoted-6", "Promoted -> Promoted (2)" ],
	[ "promoted-7", "Promoted -> Fenced" ],
	[ "promoted-8", "Promoted -> Fenced -> Moved" ],
	[ "promoted-9", "Stopped + Promotable + No quorum" ],
	[ "promoted-10", "Stopped -> Promotable : notify with monitor" ],
	[ "promoted-11", "Stopped -> Promote : colocation" ],
	[ "novell-239082", "Demote/Promote ordering" ],
	[ "novell-239087", "Stable promoted placement" ],
	[ "promoted-12", "Promotion based solely on rsc_location constraints" ],
	[ "promoted-13", "Include preferences of colocated resources when placing promoted" ],
	[ "promoted-demote", "Ordering when actions depends on demoting an unpromoted resource" ],
	[ "promoted-ordering", "Prevent resources from starting that need a promoted" ],
	[ "bug-1765", "Verify promoted-with-promoted colocation does not stop unpromoted instances" ],
	[ "promoted-group", "Promotion of cloned groups" ],
	[ "bug-lf-1852", "Don't shuffle promotable instances unnecessarily" ],
	[ "promoted-failed-demote", "Don't retry failed demote actions" ],
	[ "promoted-failed-demote-2", "Don't retry failed demote actions (notify=false)" ],
	[ "promoted-depend",
	"Ensure resources that depend on promoted instance don't get allocated until that does" ],
	[ "promoted-reattach", "Re-attach to a running promoted" ],
	[ "promoted-allow-start", "Don't include promoted score if it would prevent allocation" ],
	[ "promoted-colocation",
	"Allow promoted instances placemaker to be influenced by colocation constraints" ],
	[ "promoted-pseudo", "Make sure promote/demote pseudo actions are created correctly" ],
	[ "promoted-role", "Prevent target-role from promoting more than promoted-max instances" ],
	[ "bug-lf-2358", "Anti-colocation of promoted instances" ],
	[ "promoted-promotion-constraint", "Mandatory promoted colocation constraints" ],
	[ "unmanaged-promoted", "Ensure role is preserved for unmanaged resources" ],
	[ "promoted-unmanaged-monitor", "Start correct monitor for unmanaged promoted instances" ],
	[ "promoted-demote-2", "Demote does not clear past failure" ],
	[ "promoted-move", "Move promoted based on failure of colocated group" ],
	[ "promoted-probed-score", "Observe the promotion score of probed resources" ],
	[ "colocation_constraint_stops_promoted",
	"cl#5054 - Ensure promoted is demoted when stopped by colocation constraint" ],
	[ "colocation_constraint_stops_unpromoted",
	"cl#5054 - Ensure unpromoted is not demoted when stopped by colocation constraint" ],
	[ "order_constraint_stops_promoted",
	"cl#5054 - Ensure promoted is demoted when stopped by order constraint" ],
	[ "order_constraint_stops_unpromoted",
	"cl#5054 - Ensure unpromoted is not demoted when stopped by order constraint" ],
	[ "promoted_monitor_restart", "cl#5072 - Ensure promoted monitor operation will start after promotion" ],
	[ "bug-rh-880249", "Handle replacement of an m/s resource with a primitive" ],
	[ "bug-5143-ms-shuffle", "Prevent promoted instance shuffling due to promotion score" ],
	[ "promoted-demote-block", "Block promotion if demote fails with on-fail=block" ],
	[ "promoted-dependent-ban",
	"Don't stop instances from being active because a dependent is banned from that host" ],
	[ "promoted-stop", "Stop instances due to location constraint with role=Started" ],
	[ "promoted-partially-demoted-group", "Allow partially demoted group to finish demoting" ],
	[ "bug-cl-5213", "Ensure role colocation with -INFINITY is enforced" ],
	[ "bug-cl-5219", "Allow unrelated resources with a common colocation target to remain promoted" ],
	[ "promoted-asymmetrical-order",
	"Fix the behaviors of multi-state resources with asymmetrical ordering" ],
	[ "promoted-notify", "Promotion with notifications" ],
	[ "promoted-score-startup", "Use permanent promoted scores without LRM history" ],
	[ "failed-demote-recovery", "Recover resource in unpromoted role after demote fails" ],
	[ "failed-demote-recovery-promoted", "Recover resource in promoted role after demote fails" ],
	[ "on_fail_demote1", "Recovery with on-fail=\"demote\" on healthy cluster, remote, guest, and bundle nodes" ],
	[ "on_fail_demote2", "Recovery with on-fail=\"demote\" with promotion on different node" ],
	[ "on_fail_demote3", "Recovery with on-fail=\"demote\" with no promotion" ],
	[ "on_fail_demote4", "Recovery with on-fail=\"demote\" on failed cluster, remote, guest, and bundle nodes" ],
	[ "no_quorum_demote", "Promotable demotion and primitive stop with no-quorum-policy=\"demote\"" ],
	[ "no-promote-on-unrunnable-guest", "Don't select bundle instance for promotion when container can't run" ],
	[ "leftover-pending-monitor", "Prevent a leftover pending monitor from causing unexpected stop of other instances" ],
	],
	[
	[ "history-1", "Correctly parse stateful-1 resource state" ],
	],
	[
	[ "managed-0", "Managed (reference)" ],
	[ "managed-1", "Not managed - down" ],
	[ "managed-2", "Not managed - up" ],
	[ "bug-5028", "Shutdown should block if anything depends on an unmanaged resource" ],
	[ "bug-5028-detach", "Ensure detach still works" ],
	[ "bug-5028-bottom",
	"Ensure shutdown still blocks if the blocked resource is at the bottom of the stack" ],
	[ "unmanaged-stop-1",
	"cl#5155 - Block the stop of resources if any depending resource is unmanaged" ],
	[ "unmanaged-stop-2",
	"cl#5155 - Block the stop of resources if the first resource in a mandatory stop order is unmanaged" ],
	[ "unmanaged-stop-3",
	"cl#5155 - Block the stop of resources if any depending resource in a group is unmanaged" ],
	[ "unmanaged-stop-4",
	"cl#5155 - Block the stop of resources if any depending resource in the middle of a group is unmanaged" ],
	[ "unmanaged-block-restart",
	"Block restart of resources if any dependent resource in a group is unmanaged" ],
	],
	[
	[ "interleave-0", "Interleave (reference)" ],
	[ "interleave-1", "coloc - not interleaved" ],
	[ "interleave-2", "coloc - interleaved" ],
	[ "interleave-3", "coloc - interleaved (2)" ],
	[ "interleave-pseudo-stop", "Interleaved clone during stonith" ],
	[ "interleave-stop", "Interleaved clone during stop" ],
	[ "interleave-restart", "Interleaved clone during dependency restart" ],
	],
	[
	[ "notify-0", "Notify reference" ],
	[ "notify-1", "Notify simple" ],
	[ "notify-2", "Notify simple, confirm" ],
	[ "notify-3", "Notify move, confirm" ],
	[ "novell-239079", "Notification priority" ],
	#[ "notify-2", "Notify - 764" ],
	[ "notifs-for-unrunnable", "Don't schedule notifications for an unrunnable action" ],
	[ "route-remote-notify", "Route remote notify actions through correct cluster node" ],
	[ "notify-behind-stopping-remote", "Don't schedule notifications behind stopped remote" ],
	],
	[
	[ "594", "OSDL #594 - Unrunnable actions scheduled in transition" ],
	[ "662", "OSDL #662 - Two resources start on one node when incarnation_node_max = 1" ],
	[ "696", "OSDL #696 - CRM starts stonith RA without monitor" ],
	[ "726", "OSDL #726 - Attempting to schedule rsc_posic041_monitor_5000 _after_ a stop" ],
	[ "735", "OSDL #735 - Correctly detect that rsc_hadev1 is stopped on hadev3" ],
	[ "764", "OSDL #764 - Missing monitor op for DoFencing:child_DoFencing:1" ],
	[ "797", "OSDL #797 - Assert triggered: task_id_i > max_call_id" ],
	[ "829", "OSDL #829" ],
	[ "994",
	"OSDL #994 - Stopping the last resource in a resource group causes the entire group to be restarted" ],
	[ "994-2", "OSDL #994 - with a dependent resource" ],
	[ "1360", "OSDL #1360 - Clone stickiness" ],
	[ "1484", "OSDL #1484 - on_fail=stop" ],
	[ "1494", "OSDL #1494 - Clone stability" ],
	[ "unrunnable-1", "Unrunnable" ],
	[ "unrunnable-2", "Unrunnable 2" ],
	[ "stonith-0", "Stonith loop - 1" ],
	[ "stonith-1", "Stonith loop - 2" ],
	[ "stonith-2", "Stonith loop - 3" ],
	[ "stonith-3", "Stonith startup" ],
	[ "stonith-4", "Stonith node state" ],
	[ "dc-fence-ordering", "DC needs fencing while other nodes are shutting down" ],
	[ "bug-1572-1", "Recovery of groups depending on promotable role" ],
	[ "bug-1572-2", "Recovery of groups depending on promotable role when promoted is not re-promoted" ],
	[ "bug-1685", "Depends-on-promoted ordering" ],
	[ "bug-1822", "Don't promote partially active groups" ],
	[ "bug-pm-11", "New resource added to a m/s group" ],
	[ "bug-pm-12", "Recover only the failed portion of a cloned group" ],
	[ "bug-n-387749", "Don't shuffle clone instances" ],
	[ "bug-n-385265",
	"Don't ignore the failure stickiness of group children - resource_idvscommon should stay stopped" ],
	[ "bug-n-385265-2",
	"Ensure groups are migrated instead of remaining partially active on the current node" ],
	[ "bug-lf-1920", "Correctly handle probes that find active resources" ],
	[ "bnc-515172", "Location constraint with multiple expressions" ],
	[ "colocate-primitive-with-clone", "Optional colocation with a clone" ],
	[ "use-after-free-merge", "Use-after-free in native_merge_weights" ],
	[ "bug-lf-2551", "STONITH ordering for stop" ],
	[ "bug-lf-2606", "Stonith implies demote" ],
	[ "bug-lf-2474", "Ensure resource op timeout takes precedence over op_defaults" ],
	[ "bug-suse-707150", "Prevent vm-01 from starting due to colocation/ordering" ],
	[ "bug-5014-A-start-B-start", "Verify when A starts B starts using symmetrical=false" ],
	[ "bug-5014-A-stop-B-started",
	"Verify when A stops B does not stop if it has already started using symmetric=false" ],
	[ "bug-5014-A-stopped-B-stopped",
	"Verify when A is stopped and B has not started, B does not start before A using symmetric=false" ],
	[ "bug-5014-CthenAthenB-C-stopped",
	"Verify when C then A is symmetrical=true, A then B is symmetric=false, and C is stopped that nothing starts" ],
	[ "bug-5014-CLONE-A-start-B-start",
	"Verify when A starts B starts using clone resources with symmetric=false" ],
	[ "bug-5014-CLONE-A-stop-B-started",
	"Verify when A stops B does not stop if it has already started using clone resources with symmetric=false" ],
	[ "bug-5014-GROUP-A-start-B-start",
	"Verify when A starts B starts when using group resources with symmetric=false" ],
	[ "bug-5014-GROUP-A-stopped-B-started",
	"Verify when A stops B does not stop if it has already started using group resources with symmetric=false" ],
	[ "bug-5014-GROUP-A-stopped-B-stopped",
	"Verify when A is stopped and B has not started, B does not start before A using group resources with symmetric=false" ],
	[ "bug-5014-ordered-set-symmetrical-false",
	"Verify ordered sets work with symmetrical=false" ],
	[ "bug-5014-ordered-set-symmetrical-true",
	"Verify ordered sets work with symmetrical=true" ],
	[ "clbz5007-promotable-colocation",
	"Verify use of colocation scores other than INFINITY and -INFINITY work on multi-state resources" ],
	[ "bug-5038", "Prevent restart of anonymous clones when clone-max decreases" ],
	[ "bug-5025-1", "Automatically clean up failcount after resource config change with reload" ],
	[ "bug-5025-2", "Make sure clear failcount action isn't set when config does not change" ],
	[ "bug-5025-3", "Automatically clean up failcount after resource config change with restart" ],
	[ "bug-5025-4", "Clear failcount when last failure is a start op and rsc attributes changed" ],
	[ "failcount", "Ensure failcounts are correctly expired" ],
	[ "failcount-block", "Ensure failcounts are not expired when on-fail=block is present" ],
	[ "per-op-failcount", "Ensure per-operation failcount is handled and not passed to fence agent" ],
	[ "on-fail-ignore", "Ensure on-fail=ignore works even beyond migration-threshold" ],
	[ "monitor-onfail-restart", "bug-5058 - Monitor failure with on-fail set to restart" ],
	[ "monitor-onfail-stop", "bug-5058 - Monitor failure wiht on-fail set to stop" ],
	[ "bug-5059", "No need to restart p_stateful1:*" ],
	[ "bug-5069-op-enabled", "Test on-fail=ignore with failure when monitor is enabled" ],
	[ "bug-5069-op-disabled", "Test on-fail-ignore with failure when monitor is disabled" ],
	[ "obsolete-lrm-resource", "cl#5115 - Do not use obsolete lrm_resource sections" ],
	[ "expire-non-blocked-failure",
	"Ignore failure-timeout only if the failed operation has on-fail=block" ],
	[ "asymmetrical-order-move", "Respect asymmetrical ordering when trying to move resources" ],
	[ "asymmetrical-order-restart", "Respect asymmetrical ordering when restarting dependent resource" ],
	[ "start-then-stop-with-unfence", "Avoid graph loop with start-then-stop constraint plus unfencing" ],
	[ "order-expired-failure", "Order failcount cleanup after remote fencing" ],
	[ "expired-stop-1", "Expired stop failure should not block resource" ],

	[ "ignore_stonith_rsc_order1",
	"cl#5056- Ignore order constraint between stonith and non-stonith rsc" ],
	[ "ignore_stonith_rsc_order2",
	"cl#5056- Ignore order constraint with group rsc containing mixed stonith and non-stonith" ],
	[ "ignore_stonith_rsc_order3", "cl#5056- Ignore order constraint, stonith clone and mixed group" ],
	[ "ignore_stonith_rsc_order4",
	"cl#5056- Ignore order constraint, stonith clone and clone with nested mixed group" ],
	[ "honor_stonith_rsc_order1",
	"cl#5056- Honor order constraint, stonith clone and pure stonith group(single rsc)" ],
	[ "honor_stonith_rsc_order2",
	"cl#5056- Honor order constraint, stonith clone and pure stonith group(multiple rsc)" ],
	[ "honor_stonith_rsc_order3",
	"cl#5056- Honor order constraint, stonith clones with nested pure stonith group" ],
	[ "honor_stonith_rsc_order4",
	"cl#5056- Honor order constraint, between two native stonith rscs" ],
	[ "multiply-active-stonith", "Multiply active stonith" ],
	[ "probe-timeout", "cl#5099 - Default probe timeout" ],
	[ "order-first-probes",
	"cl#5301 - respect order constraints when relevant resources are being probed" ],
	[ "concurrent-fencing", "Allow performing fencing operations in parallel" ],
	[ "priority-fencing-delay", "Delay fencing targeting the more significant node" ],
	],
	[
	[ "systemhealth1", "System Health () #1" ],
	[ "systemhealth2", "System Health () #2" ],
	[ "systemhealth3", "System Health () #3" ],
	[ "systemhealthn1", "System Health (None) #1" ],
	[ "systemhealthn2", "System Health (None) #2" ],
	[ "systemhealthn3", "System Health (None) #3" ],
	[ "systemhealthm1", "System Health (Migrate On Red) #1" ],
	[ "systemhealthm2", "System Health (Migrate On Red) #2" ],
	[ "systemhealthm3", "System Health (Migrate On Red) #3" ],
	[ "systemhealtho1", "System Health (Only Green) #1" ],
	[ "systemhealtho2", "System Health (Only Green) #2" ],
	[ "systemhealtho3", "System Health (Only Green) #3" ],
	[ "systemhealthp1", "System Health (Progessive) #1" ],
	[ "systemhealthp2", "System Health (Progessive) #2" ],
	[ "systemhealthp3", "System Health (Progessive) #3" ],
	[ "allow-unhealthy-nodes", "System Health (migrate-on-red + allow-unhealth-nodes)" ],
	],
	[
	[ "utilization", "Placement Strategy - utilization" ],
	[ "minimal", "Placement Strategy - minimal" ],
	[ "balanced", "Placement Strategy - balanced" ],
	],
	[
	[ "placement-stickiness", "Optimized Placement Strategy - stickiness" ],
	[ "placement-priority", "Optimized Placement Strategy - priority" ],
	[ "placement-location", "Optimized Placement Strategy - location" ],
	[ "placement-capacity", "Optimized Placement Strategy - capacity" ],
	],
	[
	[ "utilization-order1", "Utilization Order - Simple" ],
	[ "utilization-order2", "Utilization Order - Complex" ],
	[ "utilization-order3", "Utilization Order - Migrate" ],
	[ "utilization-order4", "Utilization Order - Live Migration (bnc#695440)" ],
	[ "utilization-complex", "Utilization with complex relationships" ],
	[ "utilization-shuffle",
	"Don't displace prmExPostgreSQLDB2 on act2, Start prmExPostgreSQLDB1 on act3" ],
	[ "load-stopped-loop", "Avoid transition loop due to load_stopped (cl#5044)" ],
	[ "load-stopped-loop-2",
	"cl#5235 - Prevent graph loops that can be introduced by load_stopped -> migrate_to ordering" ],
	],
	[
	[ "colocated-utilization-primitive-1", "Colocated Utilization - Primitive" ],
	[ "colocated-utilization-primitive-2", "Colocated Utilization - Choose the most capable node" ],
	[ "colocated-utilization-group", "Colocated Utilization - Group" ],
	[ "colocated-utilization-clone", "Colocated Utilization - Clone" ],
	[ "utilization-check-allowed-nodes",
	"Only check the capacities of the nodes that can run the resource" ],
	],
	[
	[ "reprobe-target_rc", "Ensure correct target_rc for reprobe of inactive resources" ],
	[ "node-maintenance-1", "cl#5128 - Node maintenance" ],
	[ "node-maintenance-2", "cl#5128 - Node maintenance (coming out of maintenance mode)" ],
	[ "shutdown-maintenance-node", "Do not fence a maintenance node if it shuts down cleanly" ],
	[ "rsc-maintenance", "Per-resource maintenance" ],
	],
	[
	[ "not-installed-agent", "The resource agent is missing" ],
	[ "not-installed-tools", "Something the resource agent needs is missing" ],
	],
	[
	[ "stopped-monitor-00", "Stopped Monitor - initial start" ],
	[ "stopped-monitor-01", "Stopped Monitor - failed started" ],
	[ "stopped-monitor-02", "Stopped Monitor - started multi-up" ],
	[ "stopped-monitor-03", "Stopped Monitor - stop started" ],
	[ "stopped-monitor-04", "Stopped Monitor - failed stop" ],
	[ "stopped-monitor-05", "Stopped Monitor - start unmanaged" ],
	[ "stopped-monitor-06", "Stopped Monitor - unmanaged multi-up" ],
	[ "stopped-monitor-07", "Stopped Monitor - start unmanaged multi-up" ],
	[ "stopped-monitor-08", "Stopped Monitor - migrate" ],
	[ "stopped-monitor-09", "Stopped Monitor - unmanage started" ],
	[ "stopped-monitor-10", "Stopped Monitor - unmanaged started multi-up" ],
	[ "stopped-monitor-11", "Stopped Monitor - stop unmanaged started" ],
	[ "stopped-monitor-12", "Stopped Monitor - unmanaged started multi-up (target-role=Stopped)" ],
	[ "stopped-monitor-20", "Stopped Monitor - initial stop" ],
	[ "stopped-monitor-21", "Stopped Monitor - stopped single-up" ],
	[ "stopped-monitor-22", "Stopped Monitor - stopped multi-up" ],
	[ "stopped-monitor-23", "Stopped Monitor - start stopped" ],
	[ "stopped-monitor-24", "Stopped Monitor - unmanage stopped" ],
	[ "stopped-monitor-25", "Stopped Monitor - unmanaged stopped multi-up" ],
	[ "stopped-monitor-26", "Stopped Monitor - start unmanaged stopped" ],
	[ "stopped-monitor-27", "Stopped Monitor - unmanaged stopped multi-up (target-role=Started)" ],
	[ "stopped-monitor-30", "Stopped Monitor - new node started" ],
	[ "stopped-monitor-31", "Stopped Monitor - new node stopped" ],
	],
	[
	# This is a combo test to check:
	# - probe timeout defaults to the minimum-interval monitor's
	# - duplicate recurring operations are ignored
	# - if timeout spec is bad, the default timeout is used
	# - failure is blocked with on-fail=block even if ISO8601 interval is specified
	# - started/stopped role monitors are started/stopped on right nodes
	[ "intervals", "Recurring monitor interval handling" ],
	],
	[
	[ "ticket-primitive-1", "Ticket - Primitive (loss-policy=stop, initial)" ],
	[ "ticket-primitive-2", "Ticket - Primitive (loss-policy=stop, granted)" ],
	[ "ticket-primitive-3", "Ticket - Primitive (loss-policy-stop, revoked)" ],
	[ "ticket-primitive-4", "Ticket - Primitive (loss-policy=demote, initial)" ],
	[ "ticket-primitive-5", "Ticket - Primitive (loss-policy=demote, granted)" ],
	[ "ticket-primitive-6", "Ticket - Primitive (loss-policy=demote, revoked)" ],
	[ "ticket-primitive-7", "Ticket - Primitive (loss-policy=fence, initial)" ],
	[ "ticket-primitive-8", "Ticket - Primitive (loss-policy=fence, granted)" ],
	[ "ticket-primitive-9", "Ticket - Primitive (loss-policy=fence, revoked)" ],
	[ "ticket-primitive-10", "Ticket - Primitive (loss-policy=freeze, initial)" ],
	[ "ticket-primitive-11", "Ticket - Primitive (loss-policy=freeze, granted)" ],
	[ "ticket-primitive-12", "Ticket - Primitive (loss-policy=freeze, revoked)" ],
	[ "ticket-primitive-13", "Ticket - Primitive (loss-policy=stop, standby, granted)" ],
	[ "ticket-primitive-14", "Ticket - Primitive (loss-policy=stop, granted, standby)" ],
	[ "ticket-primitive-15", "Ticket - Primitive (loss-policy=stop, standby, revoked)" ],
	[ "ticket-primitive-16", "Ticket - Primitive (loss-policy=demote, standby, granted)" ],
	[ "ticket-primitive-17", "Ticket - Primitive (loss-policy=demote, granted, standby)" ],
	[ "ticket-primitive-18", "Ticket - Primitive (loss-policy=demote, standby, revoked)" ],
	[ "ticket-primitive-19", "Ticket - Primitive (loss-policy=fence, standby, granted)" ],
	[ "ticket-primitive-20", "Ticket - Primitive (loss-policy=fence, granted, standby)" ],
	[ "ticket-primitive-21", "Ticket - Primitive (loss-policy=fence, standby, revoked)" ],
	[ "ticket-primitive-22", "Ticket - Primitive (loss-policy=freeze, standby, granted)" ],
	[ "ticket-primitive-23", "Ticket - Primitive (loss-policy=freeze, granted, standby)" ],
	[ "ticket-primitive-24", "Ticket - Primitive (loss-policy=freeze, standby, revoked)" ],
	],
	[
	[ "ticket-group-1", "Ticket - Group (loss-policy=stop, initial)" ],
	[ "ticket-group-2", "Ticket - Group (loss-policy=stop, granted)" ],
	[ "ticket-group-3", "Ticket - Group (loss-policy-stop, revoked)" ],
	[ "ticket-group-4", "Ticket - Group (loss-policy=demote, initial)" ],
	[ "ticket-group-5", "Ticket - Group (loss-policy=demote, granted)" ],
	[ "ticket-group-6", "Ticket - Group (loss-policy=demote, revoked)" ],
	[ "ticket-group-7", "Ticket - Group (loss-policy=fence, initial)" ],
	[ "ticket-group-8", "Ticket - Group (loss-policy=fence, granted)" ],
	[ "ticket-group-9", "Ticket - Group (loss-policy=fence, revoked)" ],
	[ "ticket-group-10", "Ticket - Group (loss-policy=freeze, initial)" ],
	[ "ticket-group-11", "Ticket - Group (loss-policy=freeze, granted)" ],
	[ "ticket-group-12", "Ticket - Group (loss-policy=freeze, revoked)" ],
	[ "ticket-group-13", "Ticket - Group (loss-policy=stop, standby, granted)" ],
	[ "ticket-group-14", "Ticket - Group (loss-policy=stop, granted, standby)" ],
	[ "ticket-group-15", "Ticket - Group (loss-policy=stop, standby, revoked)" ],
	[ "ticket-group-16", "Ticket - Group (loss-policy=demote, standby, granted)" ],
	[ "ticket-group-17", "Ticket - Group (loss-policy=demote, granted, standby)" ],
	[ "ticket-group-18", "Ticket - Group (loss-policy=demote, standby, revoked)" ],
	[ "ticket-group-19", "Ticket - Group (loss-policy=fence, standby, granted)" ],
	[ "ticket-group-20", "Ticket - Group (loss-policy=fence, granted, standby)" ],
	[ "ticket-group-21", "Ticket - Group (loss-policy=fence, standby, revoked)" ],
	[ "ticket-group-22", "Ticket - Group (loss-policy=freeze, standby, granted)" ],
	[ "ticket-group-23", "Ticket - Group (loss-policy=freeze, granted, standby)" ],
	[ "ticket-group-24", "Ticket - Group (loss-policy=freeze, standby, revoked)" ],
	],
	[
	[ "ticket-clone-1", "Ticket - Clone (loss-policy=stop, initial)" ],
	[ "ticket-clone-2", "Ticket - Clone (loss-policy=stop, granted)" ],
	[ "ticket-clone-3", "Ticket - Clone (loss-policy-stop, revoked)" ],
	[ "ticket-clone-4", "Ticket - Clone (loss-policy=demote, initial)" ],
	[ "ticket-clone-5", "Ticket - Clone (loss-policy=demote, granted)" ],
	[ "ticket-clone-6", "Ticket - Clone (loss-policy=demote, revoked)" ],
	[ "ticket-clone-7", "Ticket - Clone (loss-policy=fence, initial)" ],
	[ "ticket-clone-8", "Ticket - Clone (loss-policy=fence, granted)" ],
	[ "ticket-clone-9", "Ticket - Clone (loss-policy=fence, revoked)" ],
	[ "ticket-clone-10", "Ticket - Clone (loss-policy=freeze, initial)" ],
	[ "ticket-clone-11", "Ticket - Clone (loss-policy=freeze, granted)" ],
	[ "ticket-clone-12", "Ticket - Clone (loss-policy=freeze, revoked)" ],
	[ "ticket-clone-13", "Ticket - Clone (loss-policy=stop, standby, granted)" ],
	[ "ticket-clone-14", "Ticket - Clone (loss-policy=stop, granted, standby)" ],
	[ "ticket-clone-15", "Ticket - Clone (loss-policy=stop, standby, revoked)" ],
	[ "ticket-clone-16", "Ticket - Clone (loss-policy=demote, standby, granted)" ],
	[ "ticket-clone-17", "Ticket - Clone (loss-policy=demote, granted, standby)" ],
	[ "ticket-clone-18", "Ticket - Clone (loss-policy=demote, standby, revoked)" ],
	[ "ticket-clone-19", "Ticket - Clone (loss-policy=fence, standby, granted)" ],
	[ "ticket-clone-20", "Ticket - Clone (loss-policy=fence, granted, standby)" ],
	[ "ticket-clone-21", "Ticket - Clone (loss-policy=fence, standby, revoked)" ],
	[ "ticket-clone-22", "Ticket - Clone (loss-policy=freeze, standby, granted)" ],
	[ "ticket-clone-23", "Ticket - Clone (loss-policy=freeze, granted, standby)" ],
	[ "ticket-clone-24", "Ticket - Clone (loss-policy=freeze, standby, revoked)" ],
	],
	[
	[ "ticket-promoted-1", "Ticket - Promoted (loss-policy=stop, initial)" ],
	[ "ticket-promoted-2", "Ticket - Promoted (loss-policy=stop, granted)" ],
	[ "ticket-promoted-3", "Ticket - Promoted (loss-policy-stop, revoked)" ],
	[ "ticket-promoted-4", "Ticket - Promoted (loss-policy=demote, initial)" ],
	[ "ticket-promoted-5", "Ticket - Promoted (loss-policy=demote, granted)" ],
	[ "ticket-promoted-6", "Ticket - Promoted (loss-policy=demote, revoked)" ],
	[ "ticket-promoted-7", "Ticket - Promoted (loss-policy=fence, initial)" ],
	[ "ticket-promoted-8", "Ticket - Promoted (loss-policy=fence, granted)" ],
	[ "ticket-promoted-9", "Ticket - Promoted (loss-policy=fence, revoked)" ],
	[ "ticket-promoted-10", "Ticket - Promoted (loss-policy=freeze, initial)" ],
	[ "ticket-promoted-11", "Ticket - Promoted (loss-policy=freeze, granted)" ],
	[ "ticket-promoted-12", "Ticket - Promoted (loss-policy=freeze, revoked)" ],
	[ "ticket-promoted-13", "Ticket - Promoted (loss-policy=stop, standby, granted)" ],
	[ "ticket-promoted-14", "Ticket - Promoted (loss-policy=stop, granted, standby)" ],
	[ "ticket-promoted-15", "Ticket - Promoted (loss-policy=stop, standby, revoked)" ],
	[ "ticket-promoted-16", "Ticket - Promoted (loss-policy=demote, standby, granted)" ],
	[ "ticket-promoted-17", "Ticket - Promoted (loss-policy=demote, granted, standby)" ],
	[ "ticket-promoted-18", "Ticket - Promoted (loss-policy=demote, standby, revoked)" ],
	[ "ticket-promoted-19", "Ticket - Promoted (loss-policy=fence, standby, granted)" ],
	[ "ticket-promoted-20", "Ticket - Promoted (loss-policy=fence, granted, standby)" ],
	[ "ticket-promoted-21", "Ticket - Promoted (loss-policy=fence, standby, revoked)" ],
	[ "ticket-promoted-22", "Ticket - Promoted (loss-policy=freeze, standby, granted)" ],
	[ "ticket-promoted-23", "Ticket - Promoted (loss-policy=freeze, granted, standby)" ],
	[ "ticket-promoted-24", "Ticket - Promoted (loss-policy=freeze, standby, revoked)" ],
	],
	[
	[ "ticket-rsc-sets-1", "Ticket - Resource sets (1 ticket, initial)" ],
	[ "ticket-rsc-sets-2", "Ticket - Resource sets (1 ticket, granted)" ],
	[ "ticket-rsc-sets-3", "Ticket - Resource sets (1 ticket, revoked)" ],
	[ "ticket-rsc-sets-4", "Ticket - Resource sets (2 tickets, initial)" ],
	[ "ticket-rsc-sets-5", "Ticket - Resource sets (2 tickets, granted)" ],
	[ "ticket-rsc-sets-6", "Ticket - Resource sets (2 tickets, granted)" ],
	[ "ticket-rsc-sets-7", "Ticket - Resource sets (2 tickets, revoked)" ],
	[ "ticket-rsc-sets-8", "Ticket - Resource sets (1 ticket, standby, granted)" ],
	[ "ticket-rsc-sets-9", "Ticket - Resource sets (1 ticket, granted, standby)" ],
	[ "ticket-rsc-sets-10", "Ticket - Resource sets (1 ticket, standby, revoked)" ],
	[ "ticket-rsc-sets-11", "Ticket - Resource sets (2 tickets, standby, granted)" ],
	[ "ticket-rsc-sets-12", "Ticket - Resource sets (2 tickets, standby, granted)" ],
	[ "ticket-rsc-sets-13", "Ticket - Resource sets (2 tickets, granted, standby)" ],
	[ "ticket-rsc-sets-14", "Ticket - Resource sets (2 tickets, standby, revoked)" ],
	[ "cluster-specific-params", "Cluster-specific instance attributes based on rules" ],
	[ "site-specific-params", "Site-specific instance attributes based on rules" ],
	],
	[
	[ "template-1", "Template - 1" ],
	[ "template-2", "Template - 2" ],
	[ "template-3", "Template - 3 (merge operations)" ],
	[ "template-coloc-1", "Template - Colocation 1" ],
	[ "template-coloc-2", "Template - Colocation 2" ],
	[ "template-coloc-3", "Template - Colocation 3" ],
	[ "template-order-1", "Template - Order 1" ],
	[ "template-order-2", "Template - Order 2" ],
	[ "template-order-3", "Template - Order 3" ],
	[ "template-ticket", "Template - Ticket" ],
	[ "template-rsc-sets-1", "Template - Resource Sets 1" ],
	[ "template-rsc-sets-2", "Template - Resource Sets 2" ],
	[ "template-rsc-sets-3", "Template - Resource Sets 3" ],
	[ "template-rsc-sets-4", "Template - Resource Sets 4" ],
	[ "template-clone-primitive", "Cloned primitive from template" ],
	[ "template-clone-group", "Cloned group from template" ],
	[ "location-sets-templates", "Resource sets and templates - Location" ],
	[ "tags-coloc-order-1", "Tags - Colocation and Order (Simple)" ],
	[ "tags-coloc-order-2", "Tags - Colocation and Order (Resource Sets with Templates)" ],
	[ "tags-location", "Tags - Location" ],
	[ "tags-ticket", "Tags - Ticket" ],
	],
	[
	[ "container-1", "Container - initial" ],
	[ "container-2", "Container - monitor failed" ],
	[ "container-3", "Container - stop failed" ],
	[ "container-4", "Container - reached migration-threshold" ],
	[ "container-group-1", "Container in group - initial" ],
	[ "container-group-2", "Container in group - monitor failed" ],
	[ "container-group-3", "Container in group - stop failed" ],
	[ "container-group-4", "Container in group - reached migration-threshold" ],
	[ "container-is-remote-node", "Place resource within container when container is remote-node" ],
	[ "bug-rh-1097457", "Kill user defined container/contents ordering" ],
	[ "bug-cl-5247", "Graph loop when recovering m/s resource in a container" ],
	[ "bundle-order-startup", "Bundle startup ordering" ],
	[ "bundle-order-partial-start",
	"Bundle startup ordering when some dependencies are already running" ],
	[ "bundle-order-partial-start-2",
	"Bundle startup ordering when some dependencies and the container are already running" ],
	[ "bundle-order-stop", "Bundle stop ordering" ],
	[ "bundle-order-partial-stop", "Bundle startup ordering when some dependencies are already stopped" ],
	[ "bundle-order-stop-on-remote", "Stop nested resource after bringing up the connection" ],
	[ "bundle-order-startup-clone", "Prevent startup because bundle isn't promoted" ],
	[ "bundle-order-startup-clone-2", "Bundle startup with clones" ],
	[ "bundle-order-stop-clone", "Stop bundle because clone is stopping" ],
	[ "bundle-interleave-start", "Interleave bundle starts" ],
	[ "bundle-interleave-promote", "Interleave bundle promotes" ],
	[ "bundle-nested-colocation", "Colocation of nested connection resources" ],
	[ "bundle-order-fencing",
	"Order pseudo bundle fencing after parent node fencing if both are happening" ],
	[ "bundle-probe-order-1", "order 1" ],
	[ "bundle-probe-order-2", "order 2" ],
	[ "bundle-probe-order-3", "order 3" ],
	[ "bundle-probe-remotes", "Ensure remotes get probed too" ],
	[ "bundle-replicas-change", "Change bundle from 1 replica to multiple" ],
	[ "bundle-connection-with-container", "Don't move a container due to connection preferences" ],
	[ "nested-remote-recovery", "Recover bundle's container hosted on remote node" ],
	],
	[
	[ "whitebox-fail1", "Fail whitebox container rsc" ],
	[ "whitebox-fail2", "Fail cluster connection to guest node" ],
	[ "whitebox-fail3", "Failed containers should not run nested on remote nodes" ],
	[ "whitebox-start", "Start whitebox container with resources assigned to it" ],
	[ "whitebox-stop", "Stop whitebox container with resources assigned to it" ],
	[ "whitebox-move", "Move whitebox container with resources assigned to it" ],
	[ "whitebox-asymmetric", "Verify connection rsc opts-in based on container resource" ],
	[ "whitebox-ms-ordering", "Verify promote/demote can not occur before connection is established" ],
	[ "whitebox-ms-ordering-move", "Stop/Start cycle within a moving container" ],
	[ "whitebox-orphaned", "Properly shutdown orphaned whitebox container" ],
	[ "whitebox-orphan-ms", "Properly tear down orphan ms resources on remote-nodes" ],
	[ "whitebox-unexpectedly-running", "Recover container nodes the cluster did not start" ],
	[ "whitebox-migrate1", "Migrate both container and connection resource" ],
	[ "whitebox-imply-stop-on-fence",
	"imply stop action on container node rsc when host node is fenced" ],
	[ "whitebox-nested-group", "Verify guest remote-node works nested in a group" ],
	[ "guest-node-host-dies", "Verify guest node is recovered if host goes away" ],
	[ "guest-node-cleanup", "Order guest node connection recovery after container probe" ],
	[ "guest-host-not-fenceable", "Actions on guest node are unrunnable if host is unclean and cannot be fenced" ],
	],
	[
	[ "remote-startup-probes", "Baremetal remote-node startup probes" ],
	[ "remote-startup", "Startup a newly discovered remote-nodes with no status" ],
	[ "remote-fence-unclean", "Fence unclean baremetal remote-node" ],
	[ "remote-fence-unclean2",
	"Fence baremetal remote-node after cluster node fails and connection can not be recovered" ],
	[ "remote-fence-unclean-3", "Probe failed remote nodes (triggers fencing)" ],
	[ "remote-move", "Move remote-node connection resource" ],
	[ "remote-disable", "Disable a baremetal remote-node" ],
	[ "remote-probe-disable", "Probe then stop a baremetal remote-node" ],
	[ "remote-orphaned", "Properly shutdown orphaned connection resource" ],
	[ "remote-orphaned2",
	"verify we can handle orphaned remote connections with active resources on the remote" ],
	[ "remote-recover", "Recover connection resource after cluster-node fails" ],
	[ "remote-stale-node-entry",
	"Make sure we properly handle leftover remote-node entries in the node section" ],
	[ "remote-partial-migrate",
	"Make sure partial migrations are handled before ops on the remote node" ],
	[ "remote-partial-migrate2",
	"Make sure partial migration target is prefered for remote connection" ],
	[ "remote-recover-fail", "Make sure start failure causes fencing if rsc are active on remote" ],
	[ "remote-start-fail",
	"Make sure a start failure does not result in fencing if no active resources are on remote" ],
	[ "remote-unclean2",
	"Make monitor failure always results in fencing, even if no rsc are active on remote" ],
	[ "remote-fence-before-reconnect", "Fence before clearing recurring monitor failure" ],
	[ "remote-recovery", "Recover remote connections before attempting demotion" ],
	[ "remote-recover-connection", "Optimistically recovery of only the connection" ],
	[ "remote-recover-all", "Fencing when the connection has no home" ],
	[ "remote-recover-no-resources", "Fencing when the connection has no home and no active resources" ],
	[ "remote-recover-unknown",
	"Fencing when the connection has no home and the remote has no operation history" ],
	[ "remote-reconnect-delay", "Waiting for remote reconnect interval to expire" ],
	[ "remote-connection-unrecoverable",
	"Remote connection host must be fenced, with connection unrecoverable" ],
	[ "remote-connection-shutdown", "Remote connection shutdown" ],
	[ "cancel-behind-moving-remote",
	"Route recurring monitor cancellations through original node of a moving remote connection" ],
	],
	[
	[ "resource-discovery", "Exercises resource-discovery location constraint option" ],
	[ "rsc-discovery-per-node", "Disable resource discovery per node" ],
	[ "shutdown-lock", "Ensure shutdown lock works properly" ],
	[ "shutdown-lock-expiration", "Ensure shutdown lock expiration works properly" ],
	],
	[
	[ "op-defaults", "Test op_defaults conditional expressions" ],
	[ "op-defaults-2", "Test op_defaults AND'ed conditional expressions" ],
	[ "op-defaults-3", "Test op_defaults precedence" ],
	[ "rsc-defaults", "Test rsc_defaults conditional expressions" ],
	[ "rsc-defaults-2", "Test rsc_defaults conditional expressions without type" ],
	],
	[ [ "stop-all-resources", "Test stop-all-resources=true "],
	],
	[ [ "ocf_degraded-remap-ocf_ok", "Test degraded remapped to OK" ],
	[ "ocf_degraded_promoted-remap-ocf_ok", "Test degraded promoted remapped to OK"],
	],
	]

	TESTS_64BIT = [
	[
	[ "year-2038", "Check handling of timestamps beyond 2038-01-19 03:14:08 UTC" ],
	],
	]


	def is_executable(path):
	""" Check whether a file at a given path is executable. """

	try:
	return os.stat(path)[stat.ST_MODE] & stat.S_IXUSR
	except OSError:
	return False


	def diff(file1, file2, **kwargs):
	""" Call diff on two files """

	return subprocess.call([ "diff", "-u", "-N", "--ignore-all-space",
	"--ignore-blank-lines", file1, file2 ], **kwargs)


	def sort_file(filename):
	""" Sort a file alphabetically """

	with io.open(filename, "rt") as f:
	lines = sorted(f)
	with io.open(filename, "wt") as f:
	f.writelines(lines)


	def remove_files(filenames):
	""" Remove a list of files """

	for filename in filenames:
	try:
	os.remove(filename)
	except OSError:
	pass


	def normalize(filename):
	""" Remove text from a file that isn't important for comparison """

	if not hasattr(normalize, "patterns"):
	normalize.patterns = [
	re.compile(r'crm_feature_set="[^"]*"'),
	re.compile(r'batch-limit="[0-9]*"')
	]
	if os.path.isfile(filename):
	with io.open(filename, "rt") as f:
	lines = f.readlines()
	with io.open(filename, "wt") as f:
	for line in lines:
	for pattern in normalize.patterns:
	line = pattern.sub("", line)
	f.write(line)


	def cat(filename, dest=sys.stdout):
	""" Copy a file to a destination file descriptor """

	with io.open(filename, "rt") as f:
	shutil.copyfileobj(f, dest)


	class CtsScheduler(object):
	""" Regression tests for Pacemaker's scheduler """

	def _parse_args(self, argv):
	""" Parse command-line arguments """

	parser = argparse.ArgumentParser(description=DESC)

	parser.add_argument('-V', '--verbose', action='count',
	help='Display any differences from expected output')

	parser.add_argument('--run', metavar='TEST',
	help=('Run only single specified test (any further '
	'arguments will be passed to crm_simulate)'))

	parser.add_argument('--update', action='store_true',
	help='Update expected results with actual results')

	parser.add_argument('-b', '--binary', metavar='PATH',
	help='Specify path to crm_simulate')

	parser.add_argument('-i', '--io-dir', metavar='PATH',
	help='Specify path to regression test data directory')

	parser.add_argument('-o', '--out-dir', metavar='PATH',
	help='Specify where intermediate and output files should go')

	parser.add_argument('-v', '--valgrind', action='store_true',
	help='Run all commands under valgrind')

	parser.add_argument('--valgrind-dhat', action='store_true',
	help='Run all commands under valgrind with heap analyzer')

	parser.add_argument('--valgrind-skip-output', action='store_true',
	help='If running under valgrind, do not display output')

	parser.add_argument('--testcmd-options', metavar='OPTIONS', default='',
	help='Additional options for command under test')

	# argparse can't handle "everything after --run TEST", so grab that
	self.single_test_args = []
	narg = 0
	for arg in argv:
	narg = narg + 1
	if arg == '--run':
	(argv, self.single_test_args) = (argv[:narg+1], argv[narg+1:])
	break

	self.args = parser.parse_args(argv[1:])

	def _error(self, s):
	print(" * ERROR: %s" % s)

	def _failed(self, s):
	print(" * FAILED: %s" % s)

	def _get_valgrind_cmd(self):
	""" Return command arguments needed (or not) to run valgrind """

	if self.args.valgrind:
	os.environ['G_SLICE'] = "always-malloc"
	return [
	"valgrind",
	"-q",
	"--gen-suppressions=all",
	"--time-stamp=yes",
	"--trace-children=no",
	"--show-reachable=no",
	"--leak-check=full",
	"--num-callers=20",
	"--suppressions=%s/valgrind-pcmk.suppressions" % (self.test_home)
	]

	if self.args.valgrind_dhat:
	os.environ['G_SLICE'] = "always-malloc"
	return [
	"valgrind",
	"--tool=exp-dhat",
	"--time-stamp=yes",
	"--trace-children=no",
	"--show-top-n=100",
	"--num-callers=4"
	]

	return []

	def _get_simulator_cmd(self):
	""" Locate the simulation binary """

	if self.args.binary is None:
	self.args.binary = BuildOptions._BUILD_DIR + "/tools/crm_simulate"
	if not is_executable(self.args.binary):
	self.args.binary = BuildOptions.SBIN_DIR + "/crm_simulate"

	if not is_executable(self.args.binary):
	# @TODO it would be more pythonic to raise an exception
	self._error("Test binary " + self.args.binary + " not found")
	sys.exit(ExitStatus.NOT_INSTALLED)

	return [ self.args.binary ] + shlex.split(self.args.testcmd_options)

	def set_schema_env(self):
	""" Ensure schema directory environment variable is set, if possible """

	try:
	return os.environ['PCMK_schema_directory']
	except KeyError:
	for d in [ os.path.join(BuildOptions._BUILD_DIR, "xml"),
	BuildOptions.SCHEMA_DIR ]:
	if os.path.isdir(d):
	os.environ['PCMK_schema_directory'] = d
	return d
	return None

	def __init__(self, argv=sys.argv):

	# Ensure all command output is in portable locale for comparison
	os.environ['LC_ALL'] = "C"

	self._parse_args(argv)

	# Where this executable lives
	self.test_home = os.path.dirname(os.path.realpath(argv[0]))

	# Where test data resides
	if self.args.io_dir is None:
	self.args.io_dir = os.path.join(self.test_home, "scheduler")

	self.xml_input_dir = os.path.join(self.args.io_dir, "xml")
	self.expected_dir = os.path.join(self.args.io_dir, "exp")
	self.dot_expected_dir = os.path.join(self.args.io_dir, "dot")
	self.scores_dir = os.path.join(self.args.io_dir, "scores")
	self.summary_dir = os.path.join(self.args.io_dir, "summary")
	self.stderr_expected_dir = os.path.join(self.args.io_dir, "stderr")

	# Create a temporary directory to store diff file
	self.failed_dir = tempfile.mkdtemp(prefix='cts-scheduler_')

	# Where to store generated files
	if self.args.out_dir is None:
	self.args.out_dir = self.args.io_dir
	self.failed_filename = os.path.join(self.failed_dir, "test-output.diff")
	else:
	self.failed_filename = os.path.join(self.args.out_dir, "test-output.diff")
	os.environ['CIB_shadow_dir'] = self.args.out_dir

	self.failed_file = None

	self.outfile_out_dir = os.path.join(self.args.out_dir, "out")
	self.dot_out_dir = os.path.join(self.args.out_dir, "dot")
	self.scores_out_dir = os.path.join(self.args.out_dir, "scores")
	self.summary_out_dir = os.path.join(self.args.out_dir, "summary")
	self.stderr_out_dir = os.path.join(self.args.out_dir, "stderr")
	self.valgrind_out_dir = os.path.join(self.args.out_dir, "valgrind")

	# Single test mode (if requested)
	try:
	# User can give test base name or file name of a test input
	self.args.run = os.path.splitext(os.path.basename(self.args.run))[0]
	except (AttributeError, TypeError):
	pass # --run was not specified

	self.set_schema_env()

	# Arguments needed (or not) to run commands
	self.valgrind_args = self._get_valgrind_cmd()
	self.simulate_args = self._get_simulator_cmd()

	# Test counters
	self.num_failed = 0
	self.num_tests = 0

	# Ensure that the main output directory exists
	# We don't want to create it with os.makedirs below
	if not os.path.isdir(self.args.out_dir):
	self._error("Output directory missing; can't create output files")
	sys.exit(ExitStatus.CANTCREAT)

	# Create output subdirectories if they don't exist
	try:
	os.makedirs(self.outfile_out_dir, 0o755, True)
	os.makedirs(self.dot_out_dir, 0o755, True)
	os.makedirs(self.scores_out_dir, 0o755, True)
	os.makedirs(self.summary_out_dir, 0o755, True)
	os.makedirs(self.stderr_out_dir, 0o755, True)
	if self.valgrind_args:
	os.makedirs(self.valgrind_out_dir, 0o755, True)
	except OSError as ex:
	self._error("Unable to create output subdirectory: %s" % ex)
	remove_files([
	self.outfile_out_dir,
	self.dot_out_dir,
	self.scores_out_dir,
	self.summary_out_dir,
	self.stderr_out_dir,
	])
	sys.exit(ExitStatus.CANTCREAT)

	def _compare_files(self, filename1, filename2):
	""" Add any file differences to failed results """

	if diff(filename1, filename2, stdout=subprocess.DEVNULL) != 0:
	diff(filename1, filename2, stdout=self.failed_file, stderr=subprocess.DEVNULL)
	self.failed_file.write("\n")
	return True
	return False

	def run_one(self, test_name, test_desc, test_args=[]):
	""" Run one scheduler test """

	print(" Test %-25s %s" % ((test_name + ":"), test_desc))

	did_fail = False
	self.num_tests = self.num_tests + 1

	# Test inputs
	input_filename = os.path.join(
	self.xml_input_dir, "%s.xml" % test_name)
	expected_filename = os.path.join(
	self.expected_dir, "%s.exp" % test_name)
	dot_expected_filename = os.path.join(
	self.dot_expected_dir, "%s.dot" % test_name)
	scores_filename = os.path.join(
	self.scores_dir, "%s.scores" % test_name)
	summary_filename = os.path.join(
	self.summary_dir, "%s.summary" % test_name)
	stderr_expected_filename = os.path.join(
	self.stderr_expected_dir, "%s.stderr" % test_name)

	# (Intermediate) test outputs
	output_filename = os.path.join(
	self.outfile_out_dir, "%s.out" % test_name)
	dot_output_filename = os.path.join(
	self.dot_out_dir, "%s.dot.pe" % test_name)
	score_output_filename = os.path.join(
	self.scores_out_dir, "%s.scores.pe" % test_name)
	summary_output_filename = os.path.join(
	self.summary_out_dir, "%s.summary.pe" % test_name)
	stderr_output_filename = os.path.join(
	self.stderr_out_dir, "%s.stderr.pe" % test_name)
	valgrind_output_filename = os.path.join(
	self.valgrind_out_dir, "%s.valgrind" % test_name)

	# Common arguments for running test
	test_cmd = []
	if self.valgrind_args:
	test_cmd = self.valgrind_args + [ "--log-file=%s" % valgrind_output_filename ]
	test_cmd = test_cmd + self.simulate_args

	# @TODO It would be more pythonic to raise exceptions for errors,
	# then perhaps it would be nice to make a single-test class

	# Ensure necessary test inputs exist
	if not os.path.isfile(input_filename):
	self._error("No input")
	self.num_failed = self.num_failed + 1
	return ExitStatus.NOINPUT
	if not self.args.update and not os.path.isfile(expected_filename):
	self._error("no stored output")
	return ExitStatus.NOINPUT

	# Run simulation to generate summary output
	if self.args.run: # Single test mode
	test_cmd_full = test_cmd + [ '-x', input_filename, '-S' ] + test_args
	print(" ".join(test_cmd_full))
	else:
	# @TODO Why isn't test_args added here?
	test_cmd_full = test_cmd + [ '-x', input_filename, '-S' ]
	with io.open(summary_output_filename, "wt") as f:
	simulation = subprocess.Popen(test_cmd_full, stdout=subprocess.PIPE,
	stderr=subprocess.STDOUT,
	env=os.environ)
	# This makes diff happy regardless of --enable-compat-2.0.
	# Use sed -E to make Linux and BSD special characters more compatible.
	sed = subprocess.Popen(["sed", "-E",
	"-e", "s/ocf::/ocf:/g",
	"-e", r"s/Masters:/Promoted:/",
	"-e", r"s/Slaves:/Unpromoted:/",
	"-e", r"s/ Master( \|\[\|$)/ Promoted\1/",
	"-e", r"s/ Slave / Unpromoted /",
	], stdin=simulation.stdout, stdout=f,
	stderr=subprocess.STDOUT)
	simulation.stdout.close()
	sed.communicate()
	if self.args.run:
	cat(summary_output_filename)

	# Re-run simulation to generate dot, graph, and scores
	test_cmd_full = test_cmd + [
	'-x', input_filename,
	'-D', dot_output_filename,
	'-G', output_filename,
	'-sSQ' ] + test_args
	with io.open(stderr_output_filename, "wt") as f_stderr, \
	io.open(score_output_filename, "wt") as f_score:
	rc = subprocess.call(test_cmd_full, stdout=f_score, stderr=f_stderr, env=os.environ)

	# Check for test command failure
	if rc != ExitStatus.OK:
	self._failed("Test returned: %d" % rc)
	did_fail = True
	print(" ".join(test_cmd_full))

	# Check for valgrind errors
	if self.valgrind_args and not self.args.valgrind_skip_output:
	if os.stat(valgrind_output_filename).st_size > 0:
	self._failed("Valgrind reported errors")
	did_fail = True
	cat(valgrind_output_filename)
	remove_files([ valgrind_output_filename ])

	# Check for core dump
	if os.path.isfile("core"):
	self._failed("Core-file detected: core." + test_name)
	did_fail = True
	os.rename("core", "%s/core.%s" % (self.test_home, test_name))

	# Check any stderr output
	if os.path.isfile(stderr_expected_filename):
	if self._compare_files(stderr_expected_filename, stderr_output_filename):
	self._failed("stderr changed")
	did_fail = True
	elif os.stat(stderr_output_filename).st_size > 0:
	self._failed("Output was written to stderr")
	did_fail = True
	cat(stderr_output_filename)
	remove_files([ stderr_output_filename ])

	# Check whether output graph exists, and normalize it
	if (not os.path.isfile(output_filename)
	or os.stat(output_filename).st_size == 0):
	self._error("No graph produced")
	did_fail = True
	self.num_failed = self.num_failed + 1
	remove_files([ output_filename ])
	return ExitStatus.ERROR
	normalize(output_filename)

	# Check whether dot output exists, and sort it
	if (not os.path.isfile(dot_output_filename) or
	os.stat(dot_output_filename).st_size == 0):
	self._error("No dot-file summary produced")
	did_fail = True
	self.num_failed = self.num_failed + 1
	remove_files([ dot_output_filename, output_filename ])
	return ExitStatus.ERROR
	with io.open(dot_output_filename, "rt") as f:
	first_line = f.readline() # "digraph" line with opening brace
	lines = f.readlines()
	last_line = lines[-1] # closing brace
	del lines[-1]
	lines = sorted(set(lines)) # unique sort
	with io.open(dot_output_filename, "wt") as f:
	f.write(first_line)
	f.writelines(lines)
	f.write(last_line)

	# Check whether score output exists, and sort it
	if (not os.path.isfile(score_output_filename)
	or os.stat(score_output_filename).st_size == 0):
	self._error("No allocation scores produced")
	did_fail = True
	self.num_failed = self.num_failed + 1
	remove_files([ score_output_filename, output_filename ])
	return ExitStatus.ERROR
	else:
	sort_file(score_output_filename)

	if self.args.update:
	shutil.copyfile(output_filename, expected_filename)
	shutil.copyfile(dot_output_filename, dot_expected_filename)
	shutil.copyfile(score_output_filename, scores_filename)
	shutil.copyfile(summary_output_filename, summary_filename)
	print(" Updated expected outputs")

	if self._compare_files(summary_filename, summary_output_filename):
	self._failed("summary changed")
	did_fail = True

	if self._compare_files(dot_expected_filename, dot_output_filename):
	self._failed("dot-file summary changed")
	did_fail = True
	else:
	remove_files([ dot_output_filename ])

	if self._compare_files(expected_filename, output_filename):
	self._failed("xml-file changed")
	did_fail = True

	if self._compare_files(scores_filename, score_output_filename):
	self._failed("scores-file changed")
	did_fail = True

	remove_files([ output_filename,
	dot_output_filename,
	score_output_filename,
	summary_output_filename])

	if did_fail:
	self.num_failed = self.num_failed + 1
	return ExitStatus.ERROR

	return ExitStatus.OK

	def run_all(self):
	""" Run all defined tests """

	if platform.architecture()[0] == "64bit":
	TESTS.extend(TESTS_64BIT)

	for group in TESTS:
	for test in group:
	try:
	args = test[2]
	except IndexError:
	args = []
	self.run_one(test[0], test[1], args)
	print()

	def _print_summary(self):
	""" Print a summary of parameters for this test run """

	print("Test home is:\t" + self.test_home)
	print("Test binary is:\t" + self.args.binary)
	if 'PCMK_schema_directory' in os.environ:
	print("Schema home is:\t" + os.environ['PCMK_schema_directory'])
	if self.valgrind_args != []:
	print("Activating memory testing with valgrind")
	print()

	def _test_results(self):
	if self.num_failed == 0:
	shutil.rmtree(self.failed_dir)
	return ExitStatus.OK

	if os.path.isfile(self.failed_filename) and os.stat(self.failed_filename).st_size != 0:
	if self.args.verbose:
	self._error("Results of %d failed tests (out of %d):" %
	(self.num_failed, self.num_tests))
	cat(self.failed_filename)
	else:
	self._error("Results of %d failed tests (out of %d) are in %s" %
	(self.num_failed, self.num_tests, self.failed_filename))
	self._error("Use -V to display them after running the tests")
	else:
	self._error("%d (of %d) tests failed (no diff results)" %
	(self.num_failed, self.num_tests))
	if os.path.isfile(self.failed_filename):
	shutil.rmtree(self.failed_dir)
	return ExitStatus.ERROR

	def run(self):
	""" Run test(s) as specified """

	# Check for pre-existing core so we don't think it's from us
	if os.path.exists("core"):
	self._failed("Can't run with core already present in " + self.test_home)
	return ExitStatus.OSFILE

	self._print_summary()

	# Zero out the error log
	self.failed_file = io.open(self.failed_filename, "wt")

	if self.args.run is None:
	print("Performing the following tests from " + self.args.io_dir)
	print()
	self.run_all()
	print()
	self.failed_file.close()
	rc = self._test_results()
	else:
	rc = self.run_one(self.args.run, "Single shot", self.single_test_args)
	self.failed_file.close()
	if self.num_failed > 0:
	print("\nFailures:\nThese have also been written to: " + self.failed_filename + "\n")
	cat(self.failed_filename)
	shutil.rmtree(self.failed_dir)

	return rc


	if __name__ == "__main__":
	sys.exit(CtsScheduler().run())

	# vim: set filetype=python expandtab tabstop=4 softtabstop=4 shiftwidth=4 textwidth=120:
	diff --git a/cts/scheduler/dot/group-fail.dot b/cts/scheduler/dot/group-fail.dot
	index efddf77011..48e73bee68 100644
	--- a/cts/scheduler/dot/group-fail.dot
	+++ b/cts/scheduler/dot/group-fail.dot
	@@ -1,33 +1,35 @@
	digraph "g" {
	"group1_running_0" [ style=bold color="green" fontcolor="orange"]
	"group1_start_0" -> "group1_running_0" [ style = bold]
	"group1_start_0" -> "rsc1_start_0 node1" [ style = bold]
	"group1_start_0" -> "rsc2_start_0 node1" [ style = bold]
	"group1_start_0" -> "rsc3_start_0 node1" [ style = bold]
	"group1_start_0" -> "rsc4_start_0 node1" [ style = bold]
	"group1_start_0" [ style=bold color="green" fontcolor="orange"]
	"group1_stop_0" -> "group1_stopped_0" [ style = bold]
	"group1_stop_0" -> "rsc2_stop_0 node1" [ style = bold]
	"group1_stop_0" -> "rsc4_stop_0 node1" [ style = bold]
	"group1_stop_0" [ style=bold color="green" fontcolor="orange"]
	"group1_stopped_0" -> "group1_start_0" [ style = bold]
	"group1_stopped_0" [ style=bold color="green" fontcolor="orange"]
	"rsc1_start_0 node1" -> "group1_running_0" [ style = bold]
	"rsc1_start_0 node1" -> "rsc2_start_0 node1" [ style = bold]
	"rsc1_start_0 node1" [ style=bold color="green" fontcolor="black"]
	"rsc2_start_0 node1" -> "group1_running_0" [ style = bold]
	"rsc2_start_0 node1" -> "rsc3_start_0 node1" [ style = bold]
	"rsc2_start_0 node1" [ style=bold color="green" fontcolor="black"]
	"rsc2_stop_0 node1" -> "group1_stopped_0" [ style = bold]
	+"rsc2_stop_0 node1" -> "rsc1_start_0 node1" [ style = bold]
	"rsc2_stop_0 node1" -> "rsc2_start_0 node1" [ style = bold]
	"rsc2_stop_0 node1" [ style=bold color="green" fontcolor="black"]
	"rsc3_start_0 node1" -> "group1_running_0" [ style = bold]
	"rsc3_start_0 node1" -> "rsc4_start_0 node1" [ style = bold]
	"rsc3_start_0 node1" [ style=bold color="green" fontcolor="black"]
	"rsc4_start_0 node1" -> "group1_running_0" [ style = bold]
	"rsc4_start_0 node1" [ style=bold color="green" fontcolor="black"]
	"rsc4_stop_0 node1" -> "group1_stopped_0" [ style = bold]
	"rsc4_stop_0 node1" -> "rsc2_stop_0 node1" [ style = bold]
	+"rsc4_stop_0 node1" -> "rsc3_start_0 node1" [ style = bold]
	"rsc4_stop_0 node1" -> "rsc4_start_0 node1" [ style = bold]
	"rsc4_stop_0 node1" [ style=bold color="green" fontcolor="black"]
	}
	diff --git a/cts/scheduler/dot/partial-unmanaged-group.dot b/cts/scheduler/dot/partial-unmanaged-group.dot
	new file mode 100644
	index 0000000000..ea1e8cf33e
	--- /dev/null
	+++ b/cts/scheduler/dot/partial-unmanaged-group.dot
	@@ -0,0 +1,18 @@
	+ digraph "g" {
	+"grp1_running_0" [ style=dashed color="red" fontcolor="orange"]
	+"grp1_start_0" -> "grp1_running_0" [ style = dashed]
	+"grp1_start_0" -> "interloper_start_0 rhel8-2" [ style = dashed]
	+"grp1_start_0" [ style=bold color="green" fontcolor="orange"]
	+"interloper_monitor_0 rhel8-1" -> "interloper_start_0 rhel8-2" [ style = dashed]
	+"interloper_monitor_0 rhel8-1" [ style=bold color="green" fontcolor="black"]
	+"interloper_monitor_0 rhel8-2" -> "interloper_start_0 rhel8-2" [ style = dashed]
	+"interloper_monitor_0 rhel8-2" [ style=bold color="green" fontcolor="black"]
	+"interloper_monitor_0 rhel8-4" -> "interloper_start_0 rhel8-2" [ style = dashed]
	+"interloper_monitor_0 rhel8-4" [ style=bold color="green" fontcolor="black"]
	+"interloper_monitor_0 rhel8-5" -> "interloper_start_0 rhel8-2" [ style = dashed]
	+"interloper_monitor_0 rhel8-5" [ style=bold color="green" fontcolor="black"]
	+"interloper_monitor_10000 rhel8-2" [ style=dashed color="red" fontcolor="black"]
	+"interloper_start_0 rhel8-2" -> "grp1_running_0" [ style = dashed]
	+"interloper_start_0 rhel8-2" -> "interloper_monitor_10000 rhel8-2" [ style = dashed]
	+"interloper_start_0 rhel8-2" [ style=dashed color="red" fontcolor="black"]
	+}
	diff --git a/cts/scheduler/dot/unmanaged-block-restart.dot b/cts/scheduler/dot/unmanaged-block-restart.dot
	index bb32ea13b5..68cd22bfb0 100644
	--- a/cts/scheduler/dot/unmanaged-block-restart.dot
	+++ b/cts/scheduler/dot/unmanaged-block-restart.dot
	@@ -1,35 +1,36 @@
	digraph "g" {
	"group1_running_0" [ style=dashed color="red" fontcolor="orange"]
	"group1_start_0" -> "group1_running_0" [ style = dashed]
	-"group1_start_0" -> "rsc1_start_0 yingying.site" [ style = bold]
	+"group1_start_0" -> "rsc1_start_0 yingying.site" [ style = dashed]
	"group1_start_0" -> "rsc2_start_0 yingying.site" [ style = dashed]
	"group1_start_0" -> "rsc3_start_0 yingying.site" [ style = dashed]
	"group1_start_0" [ style=bold color="green" fontcolor="orange"]
	"group1_stop_0" -> "group1_stopped_0" [ style = dashed]
	"group1_stop_0" -> "rsc2_stop_0 yingying.site" [ style = dashed]
	"group1_stop_0" -> "rsc3_stop_0 yingying.site" [ style = dashed]
	"group1_stop_0" [ style=bold color="green" fontcolor="orange"]
	"group1_stopped_0" -> "group1_start_0" [ style = dashed]
	"group1_stopped_0" [ style=dashed color="red" fontcolor="orange"]
	-"rsc1_monitor_10000 yingying.site" [ style=bold color="green" fontcolor="black"]
	+"rsc1_monitor_10000 yingying.site" [ style=dashed color="red" fontcolor="black"]
	"rsc1_start_0 yingying.site" -> "group1_running_0" [ style = dashed]
	-"rsc1_start_0 yingying.site" -> "rsc1_monitor_10000 yingying.site" [ style = bold]
	+"rsc1_start_0 yingying.site" -> "rsc1_monitor_10000 yingying.site" [ style = dashed]
	"rsc1_start_0 yingying.site" -> "rsc2_start_0 yingying.site" [ style = dashed]
	-"rsc1_start_0 yingying.site" [ style=bold color="green" fontcolor="black"]
	+"rsc1_start_0 yingying.site" [ style=dashed color="red" fontcolor="black"]
	"rsc2_monitor_10000 yingying.site" [ style=dashed color="red" fontcolor="black"]
	"rsc2_start_0 yingying.site" -> "group1_running_0" [ style = dashed]
	"rsc2_start_0 yingying.site" -> "rsc2_monitor_10000 yingying.site" [ style = dashed]
	"rsc2_start_0 yingying.site" -> "rsc3_start_0 yingying.site" [ style = dashed]
	"rsc2_start_0 yingying.site" [ style=dashed color="red" fontcolor="black"]
	"rsc2_stop_0 yingying.site" -> "group1_stopped_0" [ style = dashed]
	+"rsc2_stop_0 yingying.site" -> "rsc1_start_0 yingying.site" [ style = dashed]
	"rsc2_stop_0 yingying.site" -> "rsc2_start_0 yingying.site" [ style = dashed]
	"rsc2_stop_0 yingying.site" [ style=dashed color="red" fontcolor="black"]
	"rsc3_monitor_10000 yingying.site" [ style=dashed color="red" fontcolor="black"]
	"rsc3_start_0 yingying.site" -> "group1_running_0" [ style = dashed]
	"rsc3_start_0 yingying.site" -> "rsc3_monitor_10000 yingying.site" [ style = dashed]
	"rsc3_start_0 yingying.site" [ style=dashed color="red" fontcolor="black"]
	"rsc3_stop_0 yingying.site" -> "group1_stopped_0" [ style = dashed]
	"rsc3_stop_0 yingying.site" -> "rsc2_stop_0 yingying.site" [ style = dashed]
	"rsc3_stop_0 yingying.site" -> "rsc3_start_0 yingying.site" [ style = dashed]
	"rsc3_stop_0 yingying.site" [ style=dashed color="red" fontcolor="black"]
	}
	diff --git a/cts/scheduler/exp/group-fail.exp b/cts/scheduler/exp/group-fail.exp
	index 1fe939f03d..80bf087155 100644
	--- a/cts/scheduler/exp/group-fail.exp
	+++ b/cts/scheduler/exp/group-fail.exp
	@@ -1,160 +1,166 @@
	<transition_graph cluster-delay="60s" stonith-timeout="60s" failed-stop-offset="INFINITY" failed-start-offset="INFINITY" transition_id="0">
	<synapse id="0">
	<action_set>
	<pseudo_event id="10" operation="stopped" operation_key="group1_stopped_0">
	<attributes CRM_meta_timeout="20000" />
	</pseudo_event>
	</action_set>
	<inputs>
	<trigger>
	<rsc_op id="2" operation="stop" operation_key="rsc2_stop_0" on_node="node1" on_node_uuid="uuid1"/>
	</trigger>
	<trigger>
	<rsc_op id="5" operation="stop" operation_key="rsc4_stop_0" on_node="node1" on_node_uuid="uuid1"/>
	</trigger>
	<trigger>
	<pseudo_event id="9" operation="stop" operation_key="group1_stop_0"/>
	</trigger>
	</inputs>
	</synapse>
	<synapse id="1">
	<action_set>
	<pseudo_event id="9" operation="stop" operation_key="group1_stop_0">
	<attributes CRM_meta_timeout="20000" />
	</pseudo_event>
	</action_set>
	<inputs/>
	</synapse>
	<synapse id="2">
	<action_set>
	<pseudo_event id="8" operation="running" operation_key="group1_running_0">
	<attributes CRM_meta_timeout="20000" />
	</pseudo_event>
	</action_set>
	<inputs>
	<trigger>
	<rsc_op id="1" operation="start" operation_key="rsc1_start_0" on_node="node1" on_node_uuid="uuid1"/>
	</trigger>
	<trigger>
	<rsc_op id="3" operation="start" operation_key="rsc2_start_0" on_node="node1" on_node_uuid="uuid1"/>
	</trigger>
	<trigger>
	<rsc_op id="4" operation="start" operation_key="rsc3_start_0" on_node="node1" on_node_uuid="uuid1"/>
	</trigger>
	<trigger>
	<rsc_op id="6" operation="start" operation_key="rsc4_start_0" on_node="node1" on_node_uuid="uuid1"/>
	</trigger>
	<trigger>
	<pseudo_event id="7" operation="start" operation_key="group1_start_0"/>
	</trigger>
	</inputs>
	</synapse>
	<synapse id="3">
	<action_set>
	<pseudo_event id="7" operation="start" operation_key="group1_start_0">
	<attributes CRM_meta_timeout="20000" />
	</pseudo_event>
	</action_set>
	<inputs>
	<trigger>
	<pseudo_event id="10" operation="stopped" operation_key="group1_stopped_0"/>
	</trigger>
	</inputs>
	</synapse>
	<synapse id="4">
	<action_set>
	<rsc_op id="1" operation="start" operation_key="rsc1_start_0" on_node="node1" on_node_uuid="uuid1">
	<primitive id="rsc1" class="ocf" provider="heartbeat" type="apache"/>
	<attributes CRM_meta_on_node="node1" CRM_meta_on_node_uuid="uuid1" CRM_meta_timeout="20000" />
	</rsc_op>
	</action_set>
	<inputs>
	+ <trigger>
	+ <rsc_op id="2" operation="stop" operation_key="rsc2_stop_0" on_node="node1" on_node_uuid="uuid1"/>
	+ </trigger>
	<trigger>
	<pseudo_event id="7" operation="start" operation_key="group1_start_0"/>
	</trigger>
	</inputs>
	</synapse>
	<synapse id="5">
	<action_set>
	<rsc_op id="3" operation="start" operation_key="rsc2_start_0" on_node="node1" on_node_uuid="uuid1">
	<primitive id="rsc2" class="ocf" provider="heartbeat" type="apache"/>
	<attributes CRM_meta_on_node="node1" CRM_meta_on_node_uuid="uuid1" CRM_meta_timeout="20000" />
	</rsc_op>
	</action_set>
	<inputs>
	<trigger>
	<rsc_op id="1" operation="start" operation_key="rsc1_start_0" on_node="node1" on_node_uuid="uuid1"/>
	</trigger>
	<trigger>
	<rsc_op id="2" operation="stop" operation_key="rsc2_stop_0" on_node="node1" on_node_uuid="uuid1"/>
	</trigger>
	<trigger>
	<pseudo_event id="7" operation="start" operation_key="group1_start_0"/>
	</trigger>
	</inputs>
	</synapse>
	<synapse id="6">
	<action_set>
	<rsc_op id="2" operation="stop" operation_key="rsc2_stop_0" on_node="node1" on_node_uuid="uuid1">
	<primitive id="rsc2" class="ocf" provider="heartbeat" type="apache"/>
	<attributes CRM_meta_on_node="node1" CRM_meta_on_node_uuid="uuid1" CRM_meta_timeout="20000" />
	</rsc_op>
	</action_set>
	<inputs>
	<trigger>
	<rsc_op id="5" operation="stop" operation_key="rsc4_stop_0" on_node="node1" on_node_uuid="uuid1"/>
	</trigger>
	<trigger>
	<pseudo_event id="9" operation="stop" operation_key="group1_stop_0"/>
	</trigger>
	</inputs>
	</synapse>
	<synapse id="7">
	<action_set>
	<rsc_op id="4" operation="start" operation_key="rsc3_start_0" on_node="node1" on_node_uuid="uuid1">
	<primitive id="rsc3" class="ocf" provider="heartbeat" type="apache"/>
	<attributes CRM_meta_on_node="node1" CRM_meta_on_node_uuid="uuid1" CRM_meta_timeout="20000" />
	</rsc_op>
	</action_set>
	<inputs>
	<trigger>
	<rsc_op id="3" operation="start" operation_key="rsc2_start_0" on_node="node1" on_node_uuid="uuid1"/>
	</trigger>
	+ <trigger>
	+ <rsc_op id="5" operation="stop" operation_key="rsc4_stop_0" on_node="node1" on_node_uuid="uuid1"/>
	+ </trigger>
	<trigger>
	<pseudo_event id="7" operation="start" operation_key="group1_start_0"/>
	</trigger>
	</inputs>
	</synapse>
	<synapse id="8">
	<action_set>
	<rsc_op id="6" operation="start" operation_key="rsc4_start_0" on_node="node1" on_node_uuid="uuid1">
	<primitive id="rsc4" class="ocf" provider="heartbeat" type="apache"/>
	<attributes CRM_meta_on_node="node1" CRM_meta_on_node_uuid="uuid1" CRM_meta_timeout="20000" />
	</rsc_op>
	</action_set>
	<inputs>
	<trigger>
	<rsc_op id="4" operation="start" operation_key="rsc3_start_0" on_node="node1" on_node_uuid="uuid1"/>
	</trigger>
	<trigger>
	<rsc_op id="5" operation="stop" operation_key="rsc4_stop_0" on_node="node1" on_node_uuid="uuid1"/>
	</trigger>
	<trigger>
	<pseudo_event id="7" operation="start" operation_key="group1_start_0"/>
	</trigger>
	</inputs>
	</synapse>
	<synapse id="9">
	<action_set>
	<rsc_op id="5" operation="stop" operation_key="rsc4_stop_0" on_node="node1" on_node_uuid="uuid1">
	<primitive id="rsc4" class="ocf" provider="heartbeat" type="apache"/>
	<attributes CRM_meta_on_node="node1" CRM_meta_on_node_uuid="uuid1" CRM_meta_timeout="20000" />
	</rsc_op>
	</action_set>
	<inputs>
	<trigger>
	<pseudo_event id="9" operation="stop" operation_key="group1_stop_0"/>
	</trigger>
	</inputs>
	</synapse>
	</transition_graph>
	diff --git a/cts/scheduler/exp/partial-unmanaged-group.exp b/cts/scheduler/exp/partial-unmanaged-group.exp
	new file mode 100644
	index 0000000000..62c849dcdf
	--- /dev/null
	+++ b/cts/scheduler/exp/partial-unmanaged-group.exp
	@@ -0,0 +1,46 @@
	+<transition_graph cluster-delay="60s" stonith-timeout="60s" failed-stop-offset="INFINITY" failed-start-offset="1" transition_id="0">
	+ <synapse id="0">
	+ <action_set>
	+ <pseudo_event id="25" operation="start" operation_key="grp1_start_0">
	+ <attributes CRM_meta_timeout="90000" />
	+ </pseudo_event>
	+ </action_set>
	+ <inputs/>
	+ </synapse>
	+ <synapse id="1">
	+ <action_set>
	+ <rsc_op id="10" operation="monitor" operation_key="interloper_monitor_0" on_node="rhel8-5" on_node_uuid="5">
	+ <primitive id="interloper" class="ocf" provider="pacemaker" type="Dummy"/>
	+ <attributes CRM_meta_on_node="rhel8-5" CRM_meta_on_node_uuid="5" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" />
	+ </rsc_op>
	+ </action_set>
	+ <inputs/>
	+ </synapse>
	+ <synapse id="2">
	+ <action_set>
	+ <rsc_op id="9" operation="monitor" operation_key="interloper_monitor_0" on_node="rhel8-4" on_node_uuid="4">
	+ <primitive id="interloper" class="ocf" provider="pacemaker" type="Dummy"/>
	+ <attributes CRM_meta_on_node="rhel8-4" CRM_meta_on_node_uuid="4" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" />
	+ </rsc_op>
	+ </action_set>
	+ <inputs/>
	+ </synapse>
	+ <synapse id="3">
	+ <action_set>
	+ <rsc_op id="8" operation="monitor" operation_key="interloper_monitor_0" on_node="rhel8-2" on_node_uuid="2">
	+ <primitive id="interloper" class="ocf" provider="pacemaker" type="Dummy"/>
	+ <attributes CRM_meta_on_node="rhel8-2" CRM_meta_on_node_uuid="2" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" />
	+ </rsc_op>
	+ </action_set>
	+ <inputs/>
	+ </synapse>
	+ <synapse id="4">
	+ <action_set>
	+ <rsc_op id="7" operation="monitor" operation_key="interloper_monitor_0" on_node="rhel8-1" on_node_uuid="1">
	+ <primitive id="interloper" class="ocf" provider="pacemaker" type="Dummy"/>
	+ <attributes CRM_meta_on_node="rhel8-1" CRM_meta_on_node_uuid="1" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" />
	+ </rsc_op>
	+ </action_set>
	+ <inputs/>
	+ </synapse>
	+</transition_graph>
	diff --git a/cts/scheduler/exp/unmanaged-block-restart.exp b/cts/scheduler/exp/unmanaged-block-restart.exp
	index c96b4bc3d0..c23188803c 100644
	--- a/cts/scheduler/exp/unmanaged-block-restart.exp
	+++ b/cts/scheduler/exp/unmanaged-block-restart.exp
	@@ -1,44 +1,18 @@
	<transition_graph cluster-delay="60s" stonith-timeout="60s" failed-stop-offset="INFINITY" failed-start-offset="INFINITY" transition_id="0">
	<synapse id="0">
	<action_set>
	<pseudo_event id="12" operation="stop" operation_key="group1_stop_0">
	<attributes CRM_meta_timeout="20000" />
	</pseudo_event>
	</action_set>
	<inputs/>
	</synapse>
	<synapse id="1">
	<action_set>
	<pseudo_event id="10" operation="start" operation_key="group1_start_0">
	<attributes CRM_meta_timeout="20000" />
	</pseudo_event>
	</action_set>
	<inputs/>
	</synapse>
	- <synapse id="2">
	- <action_set>
	- <rsc_op id="4" operation="monitor" operation_key="rsc1_monitor_10000" on_node="yingying.site" on_node_uuid="yingying.site">
	- <primitive id="rsc1" class="ocf" provider="pacemaker" type="Dummy"/>
	- <attributes CRM_meta_interval="10000" CRM_meta_name="monitor" CRM_meta_on_node="yingying.site" CRM_meta_on_node_uuid="yingying.site" CRM_meta_timeout="20000" />
	- </rsc_op>
	- </action_set>
	- <inputs>
	- <trigger>
	- <rsc_op id="3" operation="start" operation_key="rsc1_start_0" on_node="yingying.site" on_node_uuid="yingying.site"/>
	- </trigger>
	- </inputs>
	- </synapse>
	- <synapse id="3">
	- <action_set>
	- <rsc_op id="3" operation="start" operation_key="rsc1_start_0" on_node="yingying.site" on_node_uuid="yingying.site">
	- <primitive id="rsc1" class="ocf" provider="pacemaker" type="Dummy"/>
	- <attributes CRM_meta_on_node="yingying.site" CRM_meta_on_node_uuid="yingying.site" CRM_meta_timeout="20000" />
	- </rsc_op>
	- </action_set>
	- <inputs>
	- <trigger>
	- <pseudo_event id="10" operation="start" operation_key="group1_start_0"/>
	- </trigger>
	- </inputs>
	- </synapse>
	</transition_graph>
	diff --git a/cts/scheduler/scores/partial-unmanaged-group.scores b/cts/scheduler/scores/partial-unmanaged-group.scores
	new file mode 100644
	index 0000000000..a029f332d7
	--- /dev/null
	+++ b/cts/scheduler/scores/partial-unmanaged-group.scores
	@@ -0,0 +1,61 @@
	+
	+pcmk__group_assign: grp1 allocation score on rhel8-1: 0
	+pcmk__group_assign: grp1 allocation score on rhel8-2: 0
	+pcmk__group_assign: grp1 allocation score on rhel8-3: 0
	+pcmk__group_assign: grp1 allocation score on rhel8-4: 0
	+pcmk__group_assign: grp1 allocation score on rhel8-5: 0
	+pcmk__group_assign: grp1a allocation score on rhel8-1: 0
	+pcmk__group_assign: grp1a allocation score on rhel8-2: 1
	+pcmk__group_assign: grp1a allocation score on rhel8-3: 0
	+pcmk__group_assign: grp1a allocation score on rhel8-4: 0
	+pcmk__group_assign: grp1a allocation score on rhel8-5: 0
	+pcmk__group_assign: grp1b allocation score on rhel8-1: 0
	+pcmk__group_assign: grp1b allocation score on rhel8-2: INFINITY
	+pcmk__group_assign: grp1b allocation score on rhel8-3: 0
	+pcmk__group_assign: grp1b allocation score on rhel8-4: 0
	+pcmk__group_assign: grp1b allocation score on rhel8-5: 0
	+pcmk__group_assign: grp1c allocation score on rhel8-1: 0
	+pcmk__group_assign: grp1c allocation score on rhel8-2: INFINITY
	+pcmk__group_assign: grp1c allocation score on rhel8-3: 0
	+pcmk__group_assign: grp1c allocation score on rhel8-4: 0
	+pcmk__group_assign: grp1c allocation score on rhel8-5: 0
	+pcmk__group_assign: interloper allocation score on rhel8-1: 0
	+pcmk__group_assign: interloper allocation score on rhel8-2: 0
	+pcmk__group_assign: interloper allocation score on rhel8-3: 0
	+pcmk__group_assign: interloper allocation score on rhel8-4: 0
	+pcmk__group_assign: interloper allocation score on rhel8-5: 0
	+pcmk__primitive_assign: Fencing allocation score on rhel8-1: 1
	+pcmk__primitive_assign: Fencing allocation score on rhel8-2: 0
	+pcmk__primitive_assign: Fencing allocation score on rhel8-3: 0
	+pcmk__primitive_assign: Fencing allocation score on rhel8-4: 0
	+pcmk__primitive_assign: Fencing allocation score on rhel8-5: 0
	+pcmk__primitive_assign: grp1a allocation score on rhel8-1: 0
	+pcmk__primitive_assign: grp1a allocation score on rhel8-2: INFINITY
	+pcmk__primitive_assign: grp1a allocation score on rhel8-3: -INFINITY
	+pcmk__primitive_assign: grp1a allocation score on rhel8-4: 0
	+pcmk__primitive_assign: grp1a allocation score on rhel8-5: 0
	+pcmk__primitive_assign: grp1b allocation score on rhel8-1: -INFINITY
	+pcmk__primitive_assign: grp1b allocation score on rhel8-2: INFINITY
	+pcmk__primitive_assign: grp1b allocation score on rhel8-3: -INFINITY
	+pcmk__primitive_assign: grp1b allocation score on rhel8-4: -INFINITY
	+pcmk__primitive_assign: grp1b allocation score on rhel8-5: -INFINITY
	+pcmk__primitive_assign: grp1c allocation score on rhel8-1: -INFINITY
	+pcmk__primitive_assign: grp1c allocation score on rhel8-2: INFINITY
	+pcmk__primitive_assign: grp1c allocation score on rhel8-3: -INFINITY
	+pcmk__primitive_assign: grp1c allocation score on rhel8-4: -INFINITY
	+pcmk__primitive_assign: grp1c allocation score on rhel8-5: -INFINITY
	+pcmk__primitive_assign: interloper allocation score on rhel8-1: -INFINITY
	+pcmk__primitive_assign: interloper allocation score on rhel8-2: INFINITY
	+pcmk__primitive_assign: interloper allocation score on rhel8-3: -INFINITY
	+pcmk__primitive_assign: interloper allocation score on rhel8-4: -INFINITY
	+pcmk__primitive_assign: interloper allocation score on rhel8-5: -INFINITY
	+pcmk__primitive_assign: rsc1 allocation score on rhel8-1: 0
	+pcmk__primitive_assign: rsc1 allocation score on rhel8-2: 0
	+pcmk__primitive_assign: rsc1 allocation score on rhel8-3: 0
	+pcmk__primitive_assign: rsc1 allocation score on rhel8-4: 1
	+pcmk__primitive_assign: rsc1 allocation score on rhel8-5: 0
	+pcmk__primitive_assign: rsc2 allocation score on rhel8-1: 0
	+pcmk__primitive_assign: rsc2 allocation score on rhel8-2: 0
	+pcmk__primitive_assign: rsc2 allocation score on rhel8-3: 0
	+pcmk__primitive_assign: rsc2 allocation score on rhel8-4: 0
	+pcmk__primitive_assign: rsc2 allocation score on rhel8-5: 1
	diff --git a/cts/scheduler/summary/partial-unmanaged-group.summary b/cts/scheduler/summary/partial-unmanaged-group.summary
	new file mode 100644
	index 0000000000..9cb68bc71a
	--- /dev/null
	+++ b/cts/scheduler/summary/partial-unmanaged-group.summary
	@@ -0,0 +1,41 @@
	+Using the original execution date of: 2020-01-20 21:19:17Z
	+Current cluster status:
	+ * Node List:
	+ * Online: [ rhel8-1 rhel8-2 rhel8-4 rhel8-5 ]
	+ * OFFLINE: [ rhel8-3 ]
	+
	+ * Full List of Resources:
	+ * Fencing (stonith:fence_xvm): Started rhel8-1
	+ * rsc1 (ocf:pacemaker:Dummy): Started rhel8-4
	+ * rsc2 (ocf:pacemaker:Dummy): Started rhel8-5
	+ * Resource Group: grp1:
	+ * grp1a (ocf:pacemaker:Dummy): Started rhel8-2
	+ * interloper (ocf:pacemaker:Dummy): Stopped
	+ * grp1b (ocf:pacemaker:Dummy): Started rhel8-2 (unmanaged)
	+ * grp1c (ocf:pacemaker:Dummy): Started rhel8-2 (unmanaged)
	+
	+Transition Summary:
	+ * Start interloper ( rhel8-2 ) due to unrunnable grp1b stop (blocked)
	+
	+Executing Cluster Transition:
	+ * Pseudo action: grp1_start_0
	+ * Resource action: interloper monitor on rhel8-5
	+ * Resource action: interloper monitor on rhel8-4
	+ * Resource action: interloper monitor on rhel8-2
	+ * Resource action: interloper monitor on rhel8-1
	+Using the original execution date of: 2020-01-20 21:19:17Z
	+
	+Revised Cluster Status:
	+ * Node List:
	+ * Online: [ rhel8-1 rhel8-2 rhel8-4 rhel8-5 ]
	+ * OFFLINE: [ rhel8-3 ]
	+
	+ * Full List of Resources:
	+ * Fencing (stonith:fence_xvm): Started rhel8-1
	+ * rsc1 (ocf:pacemaker:Dummy): Started rhel8-4
	+ * rsc2 (ocf:pacemaker:Dummy): Started rhel8-5
	+ * Resource Group: grp1:
	+ * grp1a (ocf:pacemaker:Dummy): Started rhel8-2
	+ * interloper (ocf:pacemaker:Dummy): Stopped
	+ * grp1b (ocf:pacemaker:Dummy): Started rhel8-2 (unmanaged)
	+ * grp1c (ocf:pacemaker:Dummy): Started rhel8-2 (unmanaged)
	diff --git a/cts/scheduler/summary/unmanaged-block-restart.summary b/cts/scheduler/summary/unmanaged-block-restart.summary
	index 9d3def5285..c771449f89 100644
	--- a/cts/scheduler/summary/unmanaged-block-restart.summary
	+++ b/cts/scheduler/summary/unmanaged-block-restart.summary
	@@ -1,34 +1,32 @@
	0 of 4 resource instances DISABLED and 1 BLOCKED from further action due to failure

	Current cluster status:
	* Node List:
	* Online: [ yingying.site ]

	* Full List of Resources:
	* Resource Group: group1:
	* rsc1 (ocf:pacemaker:Dummy): Stopped
	* rsc2 (ocf:pacemaker:Dummy): Started yingying.site
	* rsc3 (ocf:pacemaker:Dummy): Started yingying.site
	* rsc4 (ocf:pacemaker:Dummy): FAILED yingying.site (blocked)

	Transition Summary:
	- * Start rsc1 ( yingying.site )
	+ * Start rsc1 ( yingying.site ) due to unrunnable rsc2 stop (blocked)
	* Stop rsc2 ( yingying.site ) due to unrunnable rsc3 stop (blocked)
	* Stop rsc3 ( yingying.site ) due to required rsc2 stop (blocked)

	Executing Cluster Transition:
	* Pseudo action: group1_stop_0
	* Pseudo action: group1_start_0
	- * Resource action: rsc1 start on yingying.site
	- * Resource action: rsc1 monitor=10000 on yingying.site

	Revised Cluster Status:
	* Node List:
	* Online: [ yingying.site ]

	* Full List of Resources:
	* Resource Group: group1:
	- * rsc1 (ocf:pacemaker:Dummy): Started yingying.site
	+ * rsc1 (ocf:pacemaker:Dummy): Stopped
	* rsc2 (ocf:pacemaker:Dummy): Started yingying.site
	* rsc3 (ocf:pacemaker:Dummy): Started yingying.site
	* rsc4 (ocf:pacemaker:Dummy): FAILED yingying.site (blocked)
	diff --git a/cts/scheduler/xml/partial-unmanaged-group.xml b/cts/scheduler/xml/partial-unmanaged-group.xml
	new file mode 100644
	index 0000000000..7083d7ceb7
	--- /dev/null
	+++ b/cts/scheduler/xml/partial-unmanaged-group.xml
	@@ -0,0 +1,262 @@
	+<cib crm_feature_set="3.3.0" validate-with="pacemaker-3.2" epoch="19" num_updates="0" admin_epoch="1" cib-last-written="Mon Jan 20 15:19:17 2020" update-origin="rhel8-2" update-client="cibadmin" update-user="root" have-quorum="1" dc-uuid="4" execution-date="1579555157">
	+ <!-- The key elements of this test are:
	+ A new member (interloper) has been added to an active group (grp1)
	+ whose last two members (grp1b and grp1c) are unmanaged. Since the last
	+ members can't be stopped, the new member should be left stopped.
	+ -->
	+ <configuration>
	+ <crm_config>
	+ <cluster_property_set id="cib-bootstrap-options">
	+ <nvpair id="cts-stonith-enabled" name="stonith-enabled" value="1"/>
	+ <nvpair id="cts-start-failure-is-fatal" name="start-failure-is-fatal" value="false"/>
	+ <nvpair id="cts-pe-input-series-max" name="pe-input-series-max" value="5000"/>
	+ <nvpair id="cts-shutdown-escalation" name="shutdown-escalation" value="5min"/>
	+ <nvpair id="cts-batch-limit" name="batch-limit" value="10"/>
	+ <nvpair id="cts-dc-deadtime" name="dc-deadtime" value="5s"/>
	+ <nvpair id="cts-no-quorum-policy" name="no-quorum-policy" value="stop"/>
	+ <nvpair id="cib-bootstrap-options-have-watchdog" name="have-watchdog" value="false"/>
	+ <nvpair id="cib-bootstrap-options-dc-version" name="dc-version" value="2.0.3-4.el8-4b1f869f0f"/>
	+ <nvpair id="cib-bootstrap-options-cluster-infrastructure" name="cluster-infrastructure" value="corosync"/>
	+ <nvpair id="cib-bootstrap-options-cluster-name" name="cluster-name" value="test"/>
	+ <nvpair id="cib-bootstrap-options-shutdown-lock" name="shutdown-lock" value="true"/>
	+ <nvpair id="cib-bootstrap-options-shutdown-lock-limit" name="shutdown-lock-limit" value="5min"/>
	+ <nvpair id="cib-bootstrap-options-last-lrm-refresh" name="last-lrm-refresh" value="1579547164"/>
	+ </cluster_property_set>
	+ </crm_config>
	+ <nodes>
	+ <node id="1" uname="rhel8-1">
	+ <instance_attributes id="nodes-1">
	+ <nvpair id="nodes-1-standby" name="standby" value="off"/>
	+ </instance_attributes>
	+ </node>
	+ <node id="2" uname="rhel8-2"/>
	+ <node id="3" uname="rhel8-3">
	+ <instance_attributes id="nodes-3">
	+ <nvpair id="nodes-3-standby" name="standby" value="off"/>
	+ </instance_attributes>
	+ </node>
	+ <node id="4" uname="rhel8-4"/>
	+ <node id="5" uname="rhel8-5">
	+ <instance_attributes id="nodes-5">
	+ <nvpair id="nodes-5-standby" name="standby" value="off"/>
	+ </instance_attributes>
	+ </node>
	+ </nodes>
	+ <resources>
	+ <primitive class="stonith" id="Fencing" type="fence_xvm">
	+ <meta_attributes id="Fencing-meta">
	+ <nvpair id="Fencing-migration-threshold" name="migration-threshold" value="5"/>
	+ </meta_attributes>
	+ <instance_attributes id="Fencing-params">
	+ <nvpair id="Fencing-pcmk_host_map" name="pcmk_host_map" value="remote-rhel8-1:rhel8-1;remote-rhel8-2:rhel8-2;remote-rhel8-3:rhel8-3;remote-rhel8-4:rhel8-4;remote-rhel8-5:rhel8-5;"/>
	+ <nvpair id="Fencing-key_file" name="key_file" value="/etc/pacemaker/fence_xvm.key"/>
	+ <nvpair id="Fencing-multicast_address" name="multicast_address" value="239.255.100.100"/>
	+ <nvpair id="Fencing-pcmk_host_list" name="pcmk_host_list" value="rhel8-1 remote-rhel8-1 rhel8-2 remote-rhel8-2 rhel8-3 remote-rhel8-3 rhel8-4 remote-rhel8-4 rhel8-5 remote-rhel8-5"/>
	+ </instance_attributes>
	+ <operations>
	+ <op id="Fencing-monitor-120s" interval="120s" name="monitor" timeout="120s"/>
	+ <op id="Fencing-stop-0" interval="0" name="stop" timeout="60s"/>
	+ <op id="Fencing-start-0" interval="0" name="start" timeout="60s"/>
	+ </operations>
	+ </primitive>
	+ <primitive class="ocf" id="rsc1" provider="pacemaker" type="Dummy">
	+ <operations>
	+ <op id="rsc1-migrate_from-interval-0s" interval="0s" name="migrate_from" timeout="20s"/>
	+ <op id="rsc1-migrate_to-interval-0s" interval="0s" name="migrate_to" timeout="20s"/>
	+ <op id="rsc1-monitor-interval-10s" interval="10s" name="monitor" timeout="20s"/>
	+ <op id="rsc1-reload-interval-0s" interval="0s" name="reload" timeout="20s"/>
	+ <op id="rsc1-start-interval-0s" interval="0s" name="start" timeout="20s"/>
	+ <op id="rsc1-stop-interval-0s" interval="0s" name="stop" timeout="20s"/>
	+ </operations>
	+ </primitive>
	+ <primitive class="ocf" id="rsc2" provider="pacemaker" type="Dummy">
	+ <operations>
	+ <op id="rsc2-migrate_from-interval-0s" interval="0s" name="migrate_from" timeout="20s"/>
	+ <op id="rsc2-migrate_to-interval-0s" interval="0s" name="migrate_to" timeout="20s"/>
	+ <op id="rsc2-monitor-interval-10s" interval="10s" name="monitor" timeout="20s"/>
	+ <op id="rsc2-reload-interval-0s" interval="0s" name="reload" timeout="20s"/>
	+ <op id="rsc2-start-interval-0s" interval="0s" name="start" timeout="20s"/>
	+ <op id="rsc2-stop-interval-0s" interval="0s" name="stop" timeout="20s"/>
	+ </operations>
	+ </primitive>
	+ <group id="grp1">
	+ <primitive class="ocf" id="grp1a" provider="pacemaker" type="Dummy">
	+ <operations>
	+ <op id="grp1a-migrate_from-interval-0s" interval="0s" name="migrate_from" timeout="20s"/>
	+ <op id="grp1a-migrate_to-interval-0s" interval="0s" name="migrate_to" timeout="20s"/>
	+ <op id="grp1a-monitor-interval-10s" interval="10s" name="monitor" timeout="20s"/>
	+ <op id="grp1a-reload-interval-0s" interval="0s" name="reload" timeout="20s"/>
	+ <op id="grp1a-start-interval-0s" interval="0s" name="start" timeout="20s"/>
	+ <op id="grp1a-stop-interval-0s" interval="0s" name="stop" timeout="20s"/>
	+ </operations>
	+ </primitive>
	+ <primitive class="ocf" id="interloper" provider="pacemaker" type="Dummy">
	+ <operations>
	+ <op id="interloper-migrate_from-interval-0s" interval="0s" name="migrate_from" timeout="20s"/>
	+ <op id="interloper-migrate_to-interval-0s" interval="0s" name="migrate_to" timeout="20s"/>
	+ <op id="interloper-monitor-interval-10s" interval="10s" name="monitor" timeout="20s"/>
	+ <op id="interloper-reload-interval-0s" interval="0s" name="reload" timeout="20s"/>
	+ <op id="interloper-start-interval-0s" interval="0s" name="start" timeout="20s"/>
	+ <op id="interloper-stop-interval-0s" interval="0s" name="stop" timeout="20s"/>
	+ </operations>
	+ </primitive>
	+ <primitive class="ocf" id="grp1b" provider="pacemaker" type="Dummy">
	+ <meta_attributes id="id1">
	+ <nvpair id="id2" name="is-managed" value="false"/>
	+ </meta_attributes>
	+ <operations>
	+ <op id="grp1b-migrate_from-interval-0s" interval="0s" name="migrate_from" timeout="20s"/>
	+ <op id="grp1b-migrate_to-interval-0s" interval="0s" name="migrate_to" timeout="20s"/>
	+ <op id="grp1b-monitor-interval-10s" interval="10s" name="monitor" timeout="20s"/>
	+ <op id="grp1b-reload-interval-0s" interval="0s" name="reload" timeout="20s"/>
	+ <op id="grp1b-start-interval-0s" interval="0s" name="start" timeout="20s"/>
	+ <op id="grp1b-stop-interval-0s" interval="0s" name="stop" timeout="20s"/>
	+ </operations>
	+ </primitive>
	+ <primitive class="ocf" id="grp1c" provider="pacemaker" type="Dummy">
	+ <meta_attributes id="id3">
	+ <nvpair id="id4" name="is-managed" value="false"/>
	+ </meta_attributes>
	+ <operations>
	+ <op id="grp1c-migrate_from-interval-0s" interval="0s" name="migrate_from" timeout="20s"/>
	+ <op id="grp1c-migrate_to-interval-0s" interval="0s" name="migrate_to" timeout="20s"/>
	+ <op id="grp1c-monitor-interval-10s" interval="10s" name="monitor" timeout="20s"/>
	+ <op id="grp1c-reload-interval-0s" interval="0s" name="reload" timeout="20s"/>
	+ <op id="grp1c-start-interval-0s" interval="0s" name="start" timeout="20s"/>
	+ <op id="grp1c-stop-interval-0s" interval="0s" name="stop" timeout="20s"/>
	+ </operations>
	+ </primitive>
	+ </group>
	+ </resources>
	+ <constraints/>
	+ <fencing-topology/>
	+ <op_defaults>
	+ <meta_attributes id="cts-op_defaults-meta">
	+ <nvpair id="cts-op_defaults-timeout" name="timeout" value="90s"/>
	+ </meta_attributes>
	+ </op_defaults>
	+ <rsc_defaults>
	+ <meta_attributes id="rsc_defaults-options">
	+ <nvpair id="rsc_defaults-options-resource-stickiness" name="resource-stickiness" value="1"/>
	+ </meta_attributes>
	+ </rsc_defaults>
	+ </configuration>
	+ <status>
	+ <node_state id="2" uname="rhel8-2" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
	+ <lrm id="2">
	+ <lrm_resources>
	+ <lrm_resource id="Fencing" type="fence_xvm" class="stonith">
	+ <lrm_rsc_op id="Fencing_last_0" operation_key="Fencing_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.3.0" transition-key="6:0:7:c20e2e63-e011-463b-b23a-2efafd6450ab" transition-magic="0:7;6:0:7:c20e2e63-e011-463b-b23a-2efafd6450ab" exit-reason="" on_node="rhel8-2" call-id="5" rc-code="7" op-status="0" interval="0" last-rc-change="1579543459" last-run="1579543459" exec-time="6" queue-time="0" op-digest="bf974d77f2d4d33e434be1f89e362a52"/>
	+ </lrm_resource>
	+ <lrm_resource id="rsc1" type="Dummy" class="ocf" provider="pacemaker">
	+ <lrm_rsc_op id="rsc1_last_0" operation_key="rsc1_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.3.0" transition-key="9:0:7:c20e2e63-e011-463b-b23a-2efafd6450ab" transition-magic="0:7;9:0:7:c20e2e63-e011-463b-b23a-2efafd6450ab" exit-reason="" on_node="rhel8-2" call-id="17" rc-code="7" op-status="0" interval="0" last-rc-change="1579543459" last-run="1579543459" exec-time="38" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" envfile op_sleep passwd state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-secure-params=" passwd " op-secure-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ </lrm_resource>
	+ <lrm_resource id="rsc2" type="Dummy" class="ocf" provider="pacemaker">
	+ <lrm_rsc_op id="rsc2_last_0" operation_key="rsc2_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.3.0" transition-key="10:0:7:c20e2e63-e011-463b-b23a-2efafd6450ab" transition-magic="0:7;10:0:7:c20e2e63-e011-463b-b23a-2efafd6450ab" exit-reason="" on_node="rhel8-2" call-id="21" rc-code="7" op-status="0" interval="0" last-rc-change="1579543459" last-run="1579543459" exec-time="24" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" envfile op_sleep passwd state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-secure-params=" passwd " op-secure-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ </lrm_resource>
	+ <lrm_resource id="grp1a" type="Dummy" class="ocf" provider="pacemaker">
	+ <lrm_rsc_op id="grp1a_last_0" operation_key="grp1a_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.3.0" transition-key="14:23:0:c20e2e63-e011-463b-b23a-2efafd6450ab" transition-magic="0:0;14:23:0:c20e2e63-e011-463b-b23a-2efafd6450ab" exit-reason="" on_node="rhel8-2" call-id="30" rc-code="0" op-status="0" interval="0" last-rc-change="1579555103" last-run="1579555103" exec-time="31" queue-time="1" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" envfile op_sleep passwd state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-secure-params=" passwd " op-secure-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ <lrm_rsc_op id="grp1a_monitor_10000" operation_key="grp1a_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.3.0" transition-key="15:23:0:c20e2e63-e011-463b-b23a-2efafd6450ab" transition-magic="0:0;15:23:0:c20e2e63-e011-463b-b23a-2efafd6450ab" exit-reason="" on_node="rhel8-2" call-id="31" rc-code="0" op-status="0" interval="10000" last-rc-change="1579555103" exec-time="21" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd" op-secure-params=" passwd " op-secure-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ </lrm_resource>
	+ <lrm_resource id="grp1b" type="Dummy" class="ocf" provider="pacemaker">
	+ <lrm_rsc_op id="grp1b_last_0" operation_key="grp1b_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.3.0" transition-key="17:24:0:c20e2e63-e011-463b-b23a-2efafd6450ab" transition-magic="0:0;17:24:0:c20e2e63-e011-463b-b23a-2efafd6450ab" exit-reason="" on_node="rhel8-2" call-id="36" rc-code="0" op-status="0" interval="0" last-rc-change="1579555107" last-run="1579555107" exec-time="26" queue-time="2" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" envfile op_sleep passwd state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-secure-params=" passwd " op-secure-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ <lrm_rsc_op id="grp1b_monitor_10000" operation_key="grp1b_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.3.0" transition-key="18:24:0:c20e2e63-e011-463b-b23a-2efafd6450ab" transition-magic="0:0;18:24:0:c20e2e63-e011-463b-b23a-2efafd6450ab" exit-reason="" on_node="rhel8-2" call-id="37" rc-code="0" op-status="0" interval="10000" last-rc-change="1579555107" exec-time="19" queue-time="1" op-digest="4811cef7f7f94e3a35a70be7916cb2fd" op-secure-params=" passwd " op-secure-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ </lrm_resource>
	+ <lrm_resource id="grp1c" type="Dummy" class="ocf" provider="pacemaker">
	+ <lrm_rsc_op id="grp1c_last_0" operation_key="grp1c_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.3.0" transition-key="20:25:0:c20e2e63-e011-463b-b23a-2efafd6450ab" transition-magic="0:0;20:25:0:c20e2e63-e011-463b-b23a-2efafd6450ab" exit-reason="" on_node="rhel8-2" call-id="42" rc-code="0" op-status="0" interval="0" last-rc-change="1579555110" last-run="1579555110" exec-time="35" queue-time="2" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" envfile op_sleep passwd state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-secure-params=" passwd " op-secure-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ <lrm_rsc_op id="grp1c_monitor_10000" operation_key="grp1c_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.3.0" transition-key="21:25:0:c20e2e63-e011-463b-b23a-2efafd6450ab" transition-magic="0:0;21:25:0:c20e2e63-e011-463b-b23a-2efafd6450ab" exit-reason="" on_node="rhel8-2" call-id="43" rc-code="0" op-status="0" interval="10000" last-rc-change="1579555110" exec-time="20" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd" op-secure-params=" passwd " op-secure-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ </lrm_resource>
	+ </lrm_resources>
	+ </lrm>
	+ <transient_attributes id="2">
	+ <instance_attributes id="status-2"/>
	+ </transient_attributes>
	+ </node_state>
	+ <node_state id="3" uname="rhel8-3" in_ccm="false" crmd="offline" crm-debug-origin="send_stonith_update" join="down" expected="down"/>
	+ <node_state id="4" uname="rhel8-4" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
	+ <lrm id="4">
	+ <lrm_resources>
	+ <lrm_resource id="rsc1" type="Dummy" class="ocf" provider="pacemaker">
	+ <lrm_rsc_op id="rsc1_last_failure_0" operation_key="rsc1_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.3.0" transition-key="3:11:7:c20e2e63-e011-463b-b23a-2efafd6450ab" transition-magic="0:0;3:11:7:c20e2e63-e011-463b-b23a-2efafd6450ab" exit-reason="" on_node="rhel8-4" call-id="35" rc-code="0" op-status="0" interval="0" last-rc-change="1579547166" last-run="1579547166" exec-time="25" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ <lrm_rsc_op id="rsc1_last_0" operation_key="rsc1_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.3.0" transition-key="3:11:7:c20e2e63-e011-463b-b23a-2efafd6450ab" transition-magic="0:0;3:11:7:c20e2e63-e011-463b-b23a-2efafd6450ab" exit-reason="" on_node="rhel8-4" call-id="35" rc-code="0" op-status="0" interval="0" last-rc-change="1579547166" last-run="1579547166" exec-time="25" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" envfile op_sleep passwd state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-secure-params=" passwd " op-secure-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ <lrm_rsc_op id="rsc1_monitor_10000" operation_key="rsc1_monitor_10000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.3.0" transition-key="7:12:0:c20e2e63-e011-463b-b23a-2efafd6450ab" transition-magic="0:0;7:12:0:c20e2e63-e011-463b-b23a-2efafd6450ab" exit-reason="" on_node="rhel8-4" call-id="36" rc-code="0" op-status="0" interval="10000" last-rc-change="1579547166" exec-time="22" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd" op-secure-params=" passwd " op-secure-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ </lrm_resource>
	+ <lrm_resource id="Fencing" type="fence_xvm" class="stonith">
	+ <lrm_rsc_op id="Fencing_last_0" operation_key="Fencing_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.3.0" transition-key="16:0:7:c20e2e63-e011-463b-b23a-2efafd6450ab" transition-magic="0:7;16:0:7:c20e2e63-e011-463b-b23a-2efafd6450ab" exit-reason="" on_node="rhel8-4" call-id="5" rc-code="7" op-status="0" interval="0" last-rc-change="1579543459" last-run="1579543459" exec-time="1" queue-time="0" op-digest="bf974d77f2d4d33e434be1f89e362a52"/>
	+ </lrm_resource>
	+ <lrm_resource id="rsc2" type="Dummy" class="ocf" provider="pacemaker">
	+ <lrm_rsc_op id="rsc2_last_0" operation_key="rsc2_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.3.0" transition-key="20:0:7:c20e2e63-e011-463b-b23a-2efafd6450ab" transition-magic="0:7;20:0:7:c20e2e63-e011-463b-b23a-2efafd6450ab" exit-reason="" on_node="rhel8-4" call-id="21" rc-code="7" op-status="0" interval="0" last-rc-change="1579543459" last-run="1579543459" exec-time="43" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" envfile op_sleep passwd state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-secure-params=" passwd " op-secure-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ </lrm_resource>
	+ <lrm_resource id="grp1a" type="Dummy" class="ocf" provider="pacemaker">
	+ <lrm_rsc_op id="grp1a_last_0" operation_key="grp1a_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.3.0" transition-key="6:23:7:c20e2e63-e011-463b-b23a-2efafd6450ab" transition-magic="0:7;6:23:7:c20e2e63-e011-463b-b23a-2efafd6450ab" exit-reason="" on_node="rhel8-4" call-id="40" rc-code="7" op-status="0" interval="0" last-rc-change="1579555103" last-run="1579555103" exec-time="54" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" envfile op_sleep passwd state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-secure-params=" passwd " op-secure-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ </lrm_resource>
	+ <lrm_resource id="grp1b" type="Dummy" class="ocf" provider="pacemaker">
	+ <lrm_rsc_op id="grp1b_last_0" operation_key="grp1b_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.3.0" transition-key="7:24:7:c20e2e63-e011-463b-b23a-2efafd6450ab" transition-magic="0:7;7:24:7:c20e2e63-e011-463b-b23a-2efafd6450ab" exit-reason="" on_node="rhel8-4" call-id="44" rc-code="7" op-status="0" interval="0" last-rc-change="1579555107" last-run="1579555107" exec-time="45" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" envfile op_sleep passwd state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-secure-params=" passwd " op-secure-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ </lrm_resource>
	+ <lrm_resource id="grp1c" type="Dummy" class="ocf" provider="pacemaker">
	+ <lrm_rsc_op id="grp1c_last_0" operation_key="grp1c_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.3.0" transition-key="8:25:7:c20e2e63-e011-463b-b23a-2efafd6450ab" transition-magic="0:7;8:25:7:c20e2e63-e011-463b-b23a-2efafd6450ab" exit-reason="" on_node="rhel8-4" call-id="48" rc-code="7" op-status="0" interval="0" last-rc-change="1579555110" last-run="1579555110" exec-time="46" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" envfile op_sleep passwd state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-secure-params=" passwd " op-secure-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ </lrm_resource>
	+ </lrm_resources>
	+ </lrm>
	+ <transient_attributes id="4">
	+ <instance_attributes id="status-4"/>
	+ </transient_attributes>
	+ </node_state>
	+ <node_state id="5" uname="rhel8-5" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
	+ <lrm id="5">
	+ <lrm_resources>
	+ <lrm_resource id="Fencing" type="fence_xvm" class="stonith">
	+ <lrm_rsc_op id="Fencing_last_0" operation_key="Fencing_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.3.0" transition-key="21:0:7:c20e2e63-e011-463b-b23a-2efafd6450ab" transition-magic="0:7;21:0:7:c20e2e63-e011-463b-b23a-2efafd6450ab" exit-reason="" on_node="rhel8-5" call-id="5" rc-code="7" op-status="0" interval="0" last-rc-change="1579543459" last-run="1579543459" exec-time="4" queue-time="0" op-digest="bf974d77f2d4d33e434be1f89e362a52"/>
	+ </lrm_resource>
	+ <lrm_resource id="rsc1" type="Dummy" class="ocf" provider="pacemaker">
	+ <lrm_rsc_op id="rsc1_last_0" operation_key="rsc1_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.3.0" transition-key="24:0:7:c20e2e63-e011-463b-b23a-2efafd6450ab" transition-magic="0:7;24:0:7:c20e2e63-e011-463b-b23a-2efafd6450ab" exit-reason="" on_node="rhel8-5" call-id="17" rc-code="7" op-status="0" interval="0" last-rc-change="1579543459" last-run="1579543459" exec-time="63" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" envfile op_sleep passwd state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-secure-params=" passwd " op-secure-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ </lrm_resource>
	+ <lrm_resource id="rsc2" type="Dummy" class="ocf" provider="pacemaker">
	+ <lrm_rsc_op id="rsc2_last_0" operation_key="rsc2_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.3.0" transition-key="32:0:0:c20e2e63-e011-463b-b23a-2efafd6450ab" transition-magic="0:0;32:0:0:c20e2e63-e011-463b-b23a-2efafd6450ab" exit-reason="" on_node="rhel8-5" call-id="22" rc-code="0" op-status="0" interval="0" last-rc-change="1579543460" last-run="1579543460" exec-time="36" queue-time="1" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" envfile op_sleep passwd state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-secure-params=" passwd " op-secure-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ <lrm_rsc_op id="rsc2_monitor_10000" operation_key="rsc2_monitor_10000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.3.0" transition-key="33:0:0:c20e2e63-e011-463b-b23a-2efafd6450ab" transition-magic="0:0;33:0:0:c20e2e63-e011-463b-b23a-2efafd6450ab" exit-reason="" on_node="rhel8-5" call-id="23" rc-code="0" op-status="0" interval="10000" last-rc-change="1579543460" exec-time="25" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd" op-secure-params=" passwd " op-secure-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ </lrm_resource>
	+ <lrm_resource id="grp1a" type="Dummy" class="ocf" provider="pacemaker">
	+ <lrm_rsc_op id="grp1a_last_0" operation_key="grp1a_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.3.0" transition-key="7:23:7:c20e2e63-e011-463b-b23a-2efafd6450ab" transition-magic="0:7;7:23:7:c20e2e63-e011-463b-b23a-2efafd6450ab" exit-reason="" on_node="rhel8-5" call-id="29" rc-code="7" op-status="0" interval="0" last-rc-change="1579555103" last-run="1579555103" exec-time="37" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" envfile op_sleep passwd state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-secure-params=" passwd " op-secure-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ </lrm_resource>
	+ <lrm_resource id="grp1b" type="Dummy" class="ocf" provider="pacemaker">
	+ <lrm_rsc_op id="grp1b_last_0" operation_key="grp1b_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.3.0" transition-key="8:24:7:c20e2e63-e011-463b-b23a-2efafd6450ab" transition-magic="0:7;8:24:7:c20e2e63-e011-463b-b23a-2efafd6450ab" exit-reason="" on_node="rhel8-5" call-id="33" rc-code="7" op-status="0" interval="0" last-rc-change="1579555107" last-run="1579555107" exec-time="34" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" envfile op_sleep passwd state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-secure-params=" passwd " op-secure-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ </lrm_resource>
	+ <lrm_resource id="grp1c" type="Dummy" class="ocf" provider="pacemaker">
	+ <lrm_rsc_op id="grp1c_last_0" operation_key="grp1c_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.3.0" transition-key="9:25:7:c20e2e63-e011-463b-b23a-2efafd6450ab" transition-magic="0:7;9:25:7:c20e2e63-e011-463b-b23a-2efafd6450ab" exit-reason="" on_node="rhel8-5" call-id="37" rc-code="7" op-status="0" interval="0" last-rc-change="1579555110" last-run="1579555110" exec-time="39" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" envfile op_sleep passwd state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-secure-params=" passwd " op-secure-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ </lrm_resource>
	+ </lrm_resources>
	+ </lrm>
	+ <transient_attributes id="5">
	+ <instance_attributes id="status-5"/>
	+ </transient_attributes>
	+ </node_state>
	+ <node_state id="1" uname="rhel8-1" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
	+ <lrm id="1">
	+ <lrm_resources>
	+ <lrm_resource id="Fencing" type="fence_xvm" class="stonith">
	+ <lrm_rsc_op id="Fencing_last_0" operation_key="Fencing_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.3.0" transition-key="26:0:0:c20e2e63-e011-463b-b23a-2efafd6450ab" transition-magic="0:0;26:0:0:c20e2e63-e011-463b-b23a-2efafd6450ab" exit-reason="" on_node="rhel8-1" call-id="6" rc-code="0" op-status="0" interval="0" last-rc-change="1579543459" last-run="1579543459" exec-time="88" queue-time="0" op-digest="bf974d77f2d4d33e434be1f89e362a52"/>
	+ <lrm_rsc_op id="Fencing_monitor_120000" operation_key="Fencing_monitor_120000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.3.0" transition-key="27:0:0:c20e2e63-e011-463b-b23a-2efafd6450ab" transition-magic="0:0;27:0:0:c20e2e63-e011-463b-b23a-2efafd6450ab" exit-reason="" on_node="rhel8-1" call-id="7" rc-code="0" op-status="0" interval="120000" last-rc-change="1579543459" exec-time="54" queue-time="0" op-digest="24c9c9364f847dcb857d6fb4e1b4d3c8"/>
	+ </lrm_resource>
	+ <lrm_resource id="rsc1" type="Dummy" class="ocf" provider="pacemaker">
	+ <lrm_rsc_op id="rsc1_last_0" operation_key="rsc1_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.3.0" transition-key="4:0:7:c20e2e63-e011-463b-b23a-2efafd6450ab" transition-magic="0:7;4:0:7:c20e2e63-e011-463b-b23a-2efafd6450ab" exit-reason="" on_node="rhel8-1" call-id="19" rc-code="7" op-status="0" interval="0" last-rc-change="1579543459" last-run="1579543459" exec-time="24" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" envfile op_sleep passwd state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-secure-params=" passwd " op-secure-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ </lrm_resource>
	+ <lrm_resource id="rsc2" type="Dummy" class="ocf" provider="pacemaker">
	+ <lrm_rsc_op id="rsc2_last_0" operation_key="rsc2_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.3.0" transition-key="5:0:7:c20e2e63-e011-463b-b23a-2efafd6450ab" transition-magic="0:7;5:0:7:c20e2e63-e011-463b-b23a-2efafd6450ab" exit-reason="" on_node="rhel8-1" call-id="23" rc-code="7" op-status="0" interval="0" last-rc-change="1579543459" last-run="1579543459" exec-time="27" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" envfile op_sleep passwd state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-secure-params=" passwd " op-secure-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ </lrm_resource>
	+ <lrm_resource id="grp1a" type="Dummy" class="ocf" provider="pacemaker">
	+ <lrm_rsc_op id="grp1a_last_0" operation_key="grp1a_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.3.0" transition-key="4:23:7:c20e2e63-e011-463b-b23a-2efafd6450ab" transition-magic="0:7;4:23:7:c20e2e63-e011-463b-b23a-2efafd6450ab" exit-reason="" on_node="rhel8-1" call-id="29" rc-code="7" op-status="0" interval="0" last-rc-change="1579555103" last-run="1579555103" exec-time="33" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" envfile op_sleep passwd state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-secure-params=" passwd " op-secure-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ </lrm_resource>
	+ <lrm_resource id="grp1b" type="Dummy" class="ocf" provider="pacemaker">
	+ <lrm_rsc_op id="grp1b_last_0" operation_key="grp1b_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.3.0" transition-key="5:24:7:c20e2e63-e011-463b-b23a-2efafd6450ab" transition-magic="0:7;5:24:7:c20e2e63-e011-463b-b23a-2efafd6450ab" exit-reason="" on_node="rhel8-1" call-id="33" rc-code="7" op-status="0" interval="0" last-rc-change="1579555107" last-run="1579555107" exec-time="42" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" envfile op_sleep passwd state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-secure-params=" passwd " op-secure-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ </lrm_resource>
	+ <lrm_resource id="grp1c" type="Dummy" class="ocf" provider="pacemaker">
	+ <lrm_rsc_op id="grp1c_last_0" operation_key="grp1c_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.3.0" transition-key="6:25:7:c20e2e63-e011-463b-b23a-2efafd6450ab" transition-magic="0:7;6:25:7:c20e2e63-e011-463b-b23a-2efafd6450ab" exit-reason="" on_node="rhel8-1" call-id="37" rc-code="7" op-status="0" interval="0" last-rc-change="1579555110" last-run="1579555110" exec-time="43" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" envfile op_sleep passwd state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-secure-params=" passwd " op-secure-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ </lrm_resource>
	+ </lrm_resources>
	+ </lrm>
	+ <transient_attributes id="1">
	+ <instance_attributes id="status-1"/>
	+ </transient_attributes>
	+ </node_state>
	+ </status>
	+</cib>
	diff --git a/lib/pacemaker/pcmk_sched_group.c b/lib/pacemaker/pcmk_sched_group.c
	index 946c9c0d81..cb139f7ddf 100644
	--- a/lib/pacemaker/pcmk_sched_group.c
	+++ b/lib/pacemaker/pcmk_sched_group.c
	@@ -1,849 +1,865 @@
	/*
	* Copyright 2004-2023 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdbool.h>

	#include <crm/msg_xml.h>

	#include <pacemaker-internal.h>
	#include "libpacemaker_private.h"

	/*!
	* \internal
	* \brief Assign a group resource to a node
	*
	* \param[in,out] rsc Group resource to assign to a node
	* \param[in] prefer Node to prefer, if all else is equal
	*
	* \return Node that \p rsc is assigned to, if assigned entirely to one node
	*/
	pe_node_t *
	pcmk__group_assign(pe_resource_t rsc, const pe_node_t prefer)
	{
	pe_node_t *first_assigned_node = NULL;
	pe_resource_t *first_member = NULL;

	CRM_ASSERT(rsc != NULL);

	if (!pcmk_is_set(rsc->flags, pe_rsc_provisional)) {
	return rsc->allocated_to; // Assignment already done
	}
	if (pcmk_is_set(rsc->flags, pe_rsc_allocating)) {
	pe_rsc_debug(rsc, "Assignment dependency loop detected involving %s",
	rsc->id);
	return NULL;
	}

	if (rsc->children == NULL) {
	// No members to assign
	pe__clear_resource_flags(rsc, pe_rsc_provisional);
	return NULL;
	}

	pe__set_resource_flags(rsc, pe_rsc_allocating);
	first_member = (pe_resource_t *) rsc->children->data;
	rsc->role = first_member->role;

	pe__show_node_weights(!pcmk_is_set(rsc->cluster->flags, pe_flag_show_scores),
	rsc, __func__, rsc->allowed_nodes, rsc->cluster);

	for (GList *iter = rsc->children; iter != NULL; iter = iter->next) {
	pe_resource_t member = (pe_resource_t ) iter->data;
	pe_node_t *node = NULL;

	pe_rsc_trace(rsc, "Assigning group %s member %s",
	rsc->id, member->id);
	node = member->cmds->assign(member, prefer);
	if (first_assigned_node == NULL) {
	first_assigned_node = node;
	}
	}

	pe__set_next_role(rsc, first_member->next_role, "first group member");
	pe__clear_resource_flags(rsc, pe_rsc_allocating\|pe_rsc_provisional);

	if (!pe__group_flag_is_set(rsc, pe__group_colocated)) {
	return NULL;
	}
	return first_assigned_node;
	}

	/*!
	* \internal
	* \brief Create a pseudo-operation for a group as an ordering point
	*
	* \param[in,out] group Group resource to create action for
	* \param[in] action Action name
	*
	* \return Newly created pseudo-operation
	*/
	static pe_action_t *
	create_group_pseudo_op(pe_resource_t group, const char action)
	{
	pe_action_t *op = custom_action(group, pcmk__op_key(group->id, action, 0),
	action, NULL, TRUE, TRUE, group->cluster);
	pe__set_action_flags(op, pe_action_pseudo\|pe_action_runnable);
	return op;
	}

	/*!
	* \internal
	* \brief Create all actions needed for a given group resource
	*
	* \param[in,out] rsc Group resource to create actions for
	*/
	void
	pcmk__group_create_actions(pe_resource_t *rsc)
	{
	CRM_ASSERT(rsc != NULL);

	pe_rsc_trace(rsc, "Creating actions for group %s", rsc->id);

	// Create actions for individual group members
	for (GList *iter = rsc->children; iter != NULL; iter = iter->next) {
	pe_resource_t member = (pe_resource_t ) iter->data;

	member->cmds->create_actions(member);
	}

	// Create pseudo-actions for group itself to serve as ordering points
	create_group_pseudo_op(rsc, RSC_START);
	create_group_pseudo_op(rsc, RSC_STARTED);
	create_group_pseudo_op(rsc, RSC_STOP);
	create_group_pseudo_op(rsc, RSC_STOPPED);
	if (crm_is_true(g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_PROMOTABLE))) {
	create_group_pseudo_op(rsc, RSC_DEMOTE);
	create_group_pseudo_op(rsc, RSC_DEMOTED);
	create_group_pseudo_op(rsc, RSC_PROMOTE);
	create_group_pseudo_op(rsc, RSC_PROMOTED);
	}
	}

	// User data for member_internal_constraints()
	struct member_data {
	// These could be derived from member but this avoids some function calls
	bool ordered;
	bool colocated;
	bool promotable;

	pe_resource_t *last_active;
	pe_resource_t *previous_member;
	};

	/*!
	* \internal
	* \brief Create implicit constraints needed for a group member
	*
	* \param[in,out] data Group member to create implicit constraints for
	- * \param[in,out] user_data Group member to create implicit constraints for
	+ * \param[in,out] user_data Member data (struct member_data *)
	*/
	static void
	member_internal_constraints(gpointer data, gpointer user_data)
	{
	pe_resource_t member = (pe_resource_t ) data;
	struct member_data member_data = (struct member_data ) user_data;

	// For ordering demote vs demote or stop vs stop
	uint32_t down_flags = pe_order_implies_first_printed;

	// For ordering demote vs demoted or stop vs stopped
	uint32_t post_down_flags = pe_order_implies_then_printed;

	// Create the individual member's implicit constraints
	member->cmds->internal_constraints(member);

	if (member_data->previous_member == NULL) {
	// This is first member
	if (member_data->ordered) {
	pe__set_order_flags(down_flags, pe_order_optional);
	post_down_flags = pe_order_implies_then;
	}

	} else if (member_data->colocated) {
	// Colocate this member with the previous one
	pcmk__new_colocation("group:internal_colocation", NULL, INFINITY,
	member, member_data->previous_member, NULL, NULL,
	pcmk_is_set(member->flags, pe_rsc_critical),
	member->cluster);
	}

	if (member_data->promotable) {
	// Demote group -> demote member -> group is demoted
	pcmk__order_resource_actions(member->parent, RSC_DEMOTE,
	member, RSC_DEMOTE, down_flags);
	pcmk__order_resource_actions(member, RSC_DEMOTE,
	member->parent, RSC_DEMOTED,
	post_down_flags);

	// Promote group -> promote member -> group is promoted
	pcmk__order_resource_actions(member, RSC_PROMOTE,
	member->parent, RSC_PROMOTED,
	pe_order_runnable_left
	\|pe_order_implies_then
	\|pe_order_implies_then_printed);
	pcmk__order_resource_actions(member->parent, RSC_PROMOTE,
	member, RSC_PROMOTE,
	pe_order_implies_first_printed);
	}

	// Stop group -> stop member -> group is stopped
	pcmk__order_stops(member->parent, member, down_flags);
	pcmk__order_resource_actions(member, RSC_STOP, member->parent, RSC_STOPPED,
	post_down_flags);

	// Start group -> start member -> group is started
	pcmk__order_starts(member->parent, member, pe_order_implies_first_printed);
	pcmk__order_resource_actions(member, RSC_START, member->parent, RSC_STARTED,
	pe_order_runnable_left
	\|pe_order_implies_then
	\|pe_order_implies_then_printed);

	if (!member_data->ordered) {
	pcmk__order_starts(member->parent, member,
	pe_order_implies_then
	\|pe_order_runnable_left
	\|pe_order_implies_first_printed);
	if (member_data->promotable) {
	pcmk__order_resource_actions(member->parent, RSC_PROMOTE, member,
	RSC_PROMOTE,
	pe_order_implies_then
	\|pe_order_runnable_left
	\|pe_order_implies_first_printed);
	}

	} else if (member_data->previous_member == NULL) {
	pcmk__order_starts(member->parent, member, pe_order_none);
	if (member_data->promotable) {
	pcmk__order_resource_actions(member->parent, RSC_PROMOTE, member,
	RSC_PROMOTE, pe_order_none);
	}

	} else {
	// Order this member relative to the previous one
	+
	pcmk__order_starts(member_data->previous_member, member,
	pe_order_implies_then\|pe_order_runnable_left);
	pcmk__order_stops(member, member_data->previous_member,
	pe_order_optional\|pe_order_restart);
	+
	+ /* In unusual circumstances (such as adding a new member to the middle
	+ * of a group with unmanaged later members), this member may be active
	+ * while the previous (new) member is inactive. In this situation, the
	+ * usual restart orderings will be irrelevant, so we need to order this
	+ * member's stop before the previous member's start.
	+ */
	+ if ((member->running_on != NULL)
	+ && (member_data->previous_member->running_on == NULL)) {
	+ pcmk__order_resource_actions(member, RSC_STOP,
	+ member_data->previous_member, RSC_START,
	+ pe_order_implies_first
	+ \|pe_order_runnable_left);
	+ }
	+
	if (member_data->promotable) {
	pcmk__order_resource_actions(member_data->previous_member,
	RSC_PROMOTE, member, RSC_PROMOTE,
	pe_order_implies_then
	\|pe_order_runnable_left);
	pcmk__order_resource_actions(member, RSC_DEMOTE,
	member_data->previous_member,
	RSC_DEMOTE, pe_order_optional);
	}
	}

	// Make sure partially active groups shut down in sequence
	if (member->running_on != NULL) {
	if (member_data->ordered && (member_data->previous_member != NULL)
	&& (member_data->previous_member->running_on == NULL)
	&& (member_data->last_active != NULL)
	&& (member_data->last_active->running_on != NULL)) {
	pcmk__order_stops(member, member_data->last_active, pe_order_optional);
	}
	member_data->last_active = member;
	}

	member_data->previous_member = member;
	}

	/*!
	* \internal
	* \brief Create implicit constraints needed for a group resource
	*
	* \param[in,out] rsc Group resource to create implicit constraints for
	*/
	void
	pcmk__group_internal_constraints(pe_resource_t *rsc)
	{
	struct member_data member_data = { false, };

	CRM_ASSERT(rsc != NULL);

	/* Order group pseudo-actions relative to each other for restarting:
	* stop group -> group is stopped -> start group -> group is started
	*/
	pcmk__order_resource_actions(rsc, RSC_STOP, rsc, RSC_STOPPED,
	pe_order_runnable_left);
	pcmk__order_resource_actions(rsc, RSC_STOPPED, rsc, RSC_START,
	pe_order_optional);
	pcmk__order_resource_actions(rsc, RSC_START, rsc, RSC_STARTED,
	pe_order_runnable_left);

	member_data.ordered = pe__group_flag_is_set(rsc, pe__group_ordered);
	member_data.colocated = pe__group_flag_is_set(rsc, pe__group_colocated);
	member_data.promotable = pcmk_is_set(pe__const_top_resource(rsc, false)->flags,
	pe_rsc_promotable);
	g_list_foreach(rsc->children, member_internal_constraints, &member_data);
	}

	/*!
	* \internal
	* \brief Apply a colocation's score to node weights or resource priority
	*
	* Given a colocation constraint for a group with some other resource, apply the
	* score to the dependent's allowed node weights (if we are still placing
	* resources) or priority (if we are choosing promotable clone instance roles).
	*
	* \param[in,out] dependent Dependent group resource in colocation
	* \param[in] primary Primary resource in colocation
	* \param[in] colocation Colocation constraint to apply
	*/
	static void
	colocate_group_with(pe_resource_t dependent, const pe_resource_t primary,
	const pcmk__colocation_t *colocation)
	{
	pe_resource_t *member = NULL;

	if (dependent->children == NULL) {
	return;
	}

	pe_rsc_trace(primary, "Processing %s (group %s with %s) for dependent",
	colocation->id, dependent->id, primary->id);

	if (pe__group_flag_is_set(dependent, pe__group_colocated)) {
	// Colocate first member (internal colocations will handle the rest)
	member = (pe_resource_t *) dependent->children->data;
	member->cmds->apply_coloc_score(member, primary, colocation, true);
	return;
	}

	if (colocation->score >= INFINITY) {
	pcmk__config_err("%s: Cannot perform mandatory colocation between "
	"non-colocated group and %s",
	dependent->id, primary->id);
	return;
	}

	// Colocate each member individually
	for (GList *iter = dependent->children; iter != NULL; iter = iter->next) {
	member = (pe_resource_t *) iter->data;
	member->cmds->apply_coloc_score(member, primary, colocation, true);
	}
	}

	/*!
	* \internal
	* \brief Apply a colocation's score to node weights or resource priority
	*
	* Given a colocation constraint for some other resource with a group, apply the
	* score to the dependent's allowed node weights (if we are still placing
	* resources) or priority (if we are choosing promotable clone instance roles).
	*
	* \param[in,out] dependent Dependent resource in colocation
	* \param[in] primary Primary group resource in colocation
	* \param[in] colocation Colocation constraint to apply
	*/
	static void
	colocate_with_group(pe_resource_t dependent, const pe_resource_t primary,
	const pcmk__colocation_t *colocation)
	{
	pe_resource_t *member = NULL;

	pe_rsc_trace(primary,
	"Processing colocation %s (%s with group %s) for primary",
	colocation->id, dependent->id, primary->id);

	if (pcmk_is_set(primary->flags, pe_rsc_provisional)) {
	return;
	}

	if (pe__group_flag_is_set(primary, pe__group_colocated)) {

	if (colocation->score >= INFINITY) {
	/* For mandatory colocations, the entire group must be assignable
	* (and in the specified role if any), so apply the colocation based
	* on the last member.
	*/
	member = pe__last_group_member(primary);
	} else if (primary->children != NULL) {
	/* For optional colocations, whether the group is partially or fully
	* up doesn't matter, so apply the colocation based on the first
	* member.
	*/
	member = (pe_resource_t *) primary->children->data;
	}
	if (member == NULL) {
	return; // Nothing to colocate with
	}

	member->cmds->apply_coloc_score(dependent, member, colocation, false);
	return;
	}

	if (colocation->score >= INFINITY) {
	pcmk__config_err("%s: Cannot perform mandatory colocation with"
	" non-colocated group %s",
	dependent->id, primary->id);
	return;
	}

	// Colocate dependent with each member individually
	for (GList *iter = primary->children; iter != NULL; iter = iter->next) {
	member = (pe_resource_t *) iter->data;
	member->cmds->apply_coloc_score(dependent, member, colocation, false);
	}
	}

	/*!
	* \internal
	* \brief Apply a colocation's score to node weights or resource priority
	*
	* Given a colocation constraint, apply its score to the dependent's
	* allowed node weights (if we are still placing resources) or priority (if
	* we are choosing promotable clone instance roles).
	*
	* \param[in,out] dependent Dependent resource in colocation
	* \param[in] primary Primary resource in colocation
	* \param[in] colocation Colocation constraint to apply
	* \param[in] for_dependent true if called on behalf of dependent
	*/
	void
	pcmk__group_apply_coloc_score(pe_resource_t *dependent,
	const pe_resource_t *primary,
	const pcmk__colocation_t *colocation,
	bool for_dependent)
	{
	CRM_ASSERT((dependent != NULL) && (primary != NULL)
	&& (colocation != NULL));

	if (for_dependent) {
	colocate_group_with(dependent, primary, colocation);

	} else {
	// Method should only be called for primitive dependents
	CRM_ASSERT(dependent->variant == pe_native);

	colocate_with_group(dependent, primary, colocation);
	}
	}

	/*!
	* \internal
	* \brief Return action flags for a given group resource action
	*
	* \param[in,out] action Group action to get flags for
	* \param[in] node If not NULL, limit effects to this node
	*
	* \return Flags appropriate to \p action on \p node
	*/
	enum pe_action_flags
	pcmk__group_action_flags(pe_action_t action, const pe_node_t node)
	{
	// Default flags for a group action
	enum pe_action_flags flags = pe_action_optional
	\|pe_action_runnable
	\|pe_action_pseudo;

	CRM_ASSERT(action != NULL);

	// Update flags considering each member's own flags for same action
	for (GList *iter = action->rsc->children; iter != NULL; iter = iter->next) {
	pe_resource_t member = (pe_resource_t ) iter->data;

	// Check whether member has the same action
	enum action_tasks task = get_complex_task(member, action->task);
	const char *task_s = task2text(task);
	pe_action_t *member_action = find_first_action(member->actions, NULL,
	task_s, node);

	if (member_action != NULL) {
	enum pe_action_flags member_flags;

	member_flags = member->cmds->action_flags(member_action, node);

	// Group action is mandatory if any member action is
	if (pcmk_is_set(flags, pe_action_optional)
	&& !pcmk_is_set(member_flags, pe_action_optional)) {
	pe_rsc_trace(action->rsc, "%s is mandatory because %s is",
	action->uuid, member_action->uuid);
	pe__clear_raw_action_flags(flags, "group action",
	pe_action_optional);
	pe__clear_action_flags(action, pe_action_optional);
	}

	// Group action is unrunnable if any member action is
	if (!pcmk__str_eq(task_s, action->task, pcmk__str_none)
	&& pcmk_is_set(flags, pe_action_runnable)
	&& !pcmk_is_set(member_flags, pe_action_runnable)) {

	pe_rsc_trace(action->rsc, "%s is unrunnable because %s is",
	action->uuid, member_action->uuid);
	pe__clear_raw_action_flags(flags, "group action",
	pe_action_runnable);
	pe__clear_action_flags(action, pe_action_runnable);
	}

	/* Group (pseudo-)actions other than stop or demote are unrunnable
	* unless every member will do it.
	*/
	} else if ((task != stop_rsc) && (task != action_demote)) {
	pe_rsc_trace(action->rsc,
	"%s is not runnable because %s will not %s",
	action->uuid, member->id, task_s);
	pe__clear_raw_action_flags(flags, "group action",
	pe_action_runnable);
	}
	}

	return flags;
	}

	/*!
	* \internal
	* \brief Update two actions according to an ordering between them
	*
	* Given information about an ordering of two actions, update the actions' flags
	* (and runnable_before members if appropriate) as appropriate for the ordering.
	* Effects may cascade to other orderings involving the actions as well.
	*
	* \param[in,out] first 'First' action in an ordering
	* \param[in,out] then 'Then' action in an ordering
	* \param[in] node If not NULL, limit scope of ordering to this node
	* (only used when interleaving instances)
	* \param[in] flags Action flags for \p first for ordering purposes
	* \param[in] filter Action flags to limit scope of certain updates (may
	* include pe_action_optional to affect only mandatory
	* actions, and pe_action_runnable to affect only
	* runnable actions)
	* \param[in] type Group of enum pe_ordering flags to apply
	* \param[in,out] data_set Cluster working set
	*
	* \return Group of enum pcmk__updated flags indicating what was updated
	*/
	uint32_t
	pcmk__group_update_ordered_actions(pe_action_t first, pe_action_t then,
	const pe_node_t *node, uint32_t flags,
	uint32_t filter, uint32_t type,
	pe_working_set_t *data_set)
	{
	uint32_t changed = pcmk__updated_none;

	CRM_ASSERT((first != NULL) && (then != NULL) && (data_set != NULL));

	// Group method can be called only for group action as "then" action
	CRM_ASSERT(then->rsc != NULL);

	// Update the actions for the group itself
	changed \|= pcmk__update_ordered_actions(first, then, node, flags, filter,
	type, data_set);

	// Update the actions for each group member
	for (GList *iter = then->rsc->children; iter != NULL; iter = iter->next) {
	pe_resource_t member = (pe_resource_t ) iter->data;

	pe_action_t *member_action = find_first_action(member->actions, NULL,
	then->task, node);

	if (member_action != NULL) {
	changed \|= member->cmds->update_ordered_actions(first,
	member_action, node,
	flags, filter, type,
	data_set);
	}
	}
	return changed;
	}

	/*!
	* \internal
	* \brief Apply a location constraint to a group's allowed node scores
	*
	* \param[in,out] rsc Group resource to apply constraint to
	* \param[in,out] location Location constraint to apply
	*/
	void
	pcmk__group_apply_location(pe_resource_t rsc, pe__location_t location)
	{
	GList *node_list_orig = NULL;
	GList *node_list_copy = NULL;
	bool reset_scores = true;

	CRM_ASSERT((rsc != NULL) && (location != NULL));

	node_list_orig = location->node_list_rh;
	node_list_copy = pcmk__copy_node_list(node_list_orig, true);
	reset_scores = pe__group_flag_is_set(rsc, pe__group_colocated);

	// Apply the constraint for the group itself (updates node scores)
	pcmk__apply_location(rsc, location);

	// Apply the constraint for each member
	for (GList *iter = rsc->children; iter != NULL; iter = iter->next) {
	pe_resource_t member = (pe_resource_t ) iter->data;

	member->cmds->apply_location(member, location);

	if (reset_scores) {
	/* The first member of colocated groups needs to use the original
	* node scores, but subsequent members should work on a copy, since
	* the first member's scores already incorporate theirs.
	*/
	reset_scores = false;
	location->node_list_rh = node_list_copy;
	}
	}

	location->node_list_rh = node_list_orig;
	g_list_free_full(node_list_copy, free);
	}

	// Group implementation of resource_alloc_functions_t:colocated_resources()
	GList *
	pcmk__group_colocated_resources(const pe_resource_t *rsc,
	const pe_resource_t *orig_rsc,
	GList *colocated_rscs)
	{
	const pe_resource_t *member = NULL;

	CRM_ASSERT(rsc != NULL);

	if (orig_rsc == NULL) {
	orig_rsc = rsc;
	}

	if (pe__group_flag_is_set(rsc, pe__group_colocated)
	\|\| pe_rsc_is_clone(rsc->parent)) {
	/* This group has colocated members and/or is cloned -- either way,
	* add every child's colocated resources to the list. The first and last
	* members will include the group's own colocations.
	*/
	colocated_rscs = g_list_prepend(colocated_rscs, (gpointer) rsc);
	for (const GList *iter = rsc->children;
	iter != NULL; iter = iter->next) {

	member = (const pe_resource_t *) iter->data;
	colocated_rscs = member->cmds->colocated_resources(member, orig_rsc,
	colocated_rscs);
	}

	} else if (rsc->children != NULL) {
	/* This group's members are not colocated, and the group is not cloned,
	* so just add the group's own colocations to the list.
	*/
	colocated_rscs = pcmk__colocated_resources(rsc, orig_rsc, colocated_rscs);
	}

	return colocated_rscs;
	}

	// Group implementation of resource_alloc_functions_t:with_this_colocations()
	void
	pcmk__with_group_colocations(const pe_resource_t *rsc,
	const pe_resource_t orig_rsc, GList *list)

	{
	CRM_CHECK((rsc != NULL) && (rsc->variant == pe_group)
	&& (orig_rsc != NULL) && (list != NULL),
	return);

	// Ignore empty groups
	if (rsc->children == NULL) {
	return;
	}

	/* "With this" colocations are needed only for the group itself and for its
	* last member. Add the group's colocations plus any relevant
	* parent colocations if cloned.
	*/
	if ((rsc == orig_rsc) \|\| (orig_rsc == pe__last_group_member(rsc))) {
	crm_trace("Adding 'with %s' colocations to list for %s",
	rsc->id, orig_rsc->id);
	pcmk__add_with_this_list(list, rsc->rsc_cons_lhs);
	if (rsc->parent != NULL) { // Cloned group
	rsc->parent->cmds->with_this_colocations(rsc->parent, orig_rsc,
	list);
	}
	}
	}

	// Group implementation of resource_alloc_functions_t:this_with_colocations()
	void
	pcmk__group_with_colocations(const pe_resource_t *rsc,
	const pe_resource_t orig_rsc, GList *list)
	{
	CRM_CHECK((rsc != NULL) && (rsc->variant == pe_group)
	&& (orig_rsc != NULL) && (list != NULL),
	return);

	// Ignore empty groups
	if (rsc->children == NULL) {
	return;
	}

	/* Colocations for the group itself, or for its first member, consist of the
	* group's colocations plus any relevant parent colocations if cloned.
	*/
	if ((rsc == orig_rsc)
	\|\| (orig_rsc == (const pe_resource_t *) rsc->children->data)) {
	crm_trace("Adding '%s with' colocations to list for %s",
	rsc->id, orig_rsc->id);
	pcmk__add_this_with_list(list, rsc->rsc_cons);
	if (rsc->parent != NULL) { // Cloned group
	rsc->parent->cmds->this_with_colocations(rsc->parent, orig_rsc,
	list);
	}
	return;
	}

	/* Later group members honor the group's colocations indirectly, due to the
	* internal group colocations that chain everything from the first member.
	* However, if an earlier group member is unmanaged, this chaining will not
	* happen, so the group's mandatory colocations must be explicitly added.
	*/
	for (GList *iter = rsc->children; iter != NULL; iter = iter->next) {
	const pe_resource_t member = (const pe_resource_t ) iter->data;

	if (orig_rsc == member) {
	break; // We've seen all earlier members, and none are unmanaged
	}

	if (!pcmk_is_set(member->flags, pe_rsc_managed)) {
	crm_trace("Adding mandatory '%s with' colocations to list for "
	"member %s because earlier member %s is unmanaged",
	rsc->id, orig_rsc->id, member->id);
	for (const GList *cons_iter = rsc->rsc_cons; cons_iter != NULL;
	cons_iter = cons_iter->next) {
	const pcmk__colocation_t *colocation = NULL;

	colocation = (const pcmk__colocation_t *) cons_iter->data;
	if (colocation->score == INFINITY) {
	pcmk__add_this_with(list, colocation);
	}
	}
	// @TODO Add mandatory (or all?) clone constraints if cloned
	break;
	}
	}
	}

	/*!
	* \internal
	* \brief Update nodes with scores of colocated resources' nodes
	*
	* Given a table of nodes and a resource, update the nodes' scores with the
	* scores of the best nodes matching the attribute used for each of the
	* resource's relevant colocations.
	*
	* \param[in,out] rsc Resource to check colocations for
	* \param[in] log_id Resource ID to use in logs (if NULL, use \p rsc ID)
	* \param[in,out] nodes Nodes to update
	* \param[in] attr Colocation attribute (NULL to use default)
	* \param[in] factor Incorporate scores multiplied by this factor
	* \param[in] flags Bitmask of enum pcmk__coloc_select values
	*
	* \note The caller remains responsible for freeing \p *nodes.
	*/
	void
	pcmk__group_add_colocated_node_scores(pe_resource_t rsc, const char log_id,
	GHashTable *nodes, const char attr,
	float factor, uint32_t flags)
	{
	pe_resource_t *member = NULL;

	CRM_CHECK((rsc != NULL) && (nodes != NULL), return);

	if (log_id == NULL) {
	log_id = rsc->id;
	}

	// Avoid infinite recursion
	if (pcmk_is_set(rsc->flags, pe_rsc_merging)) {
	pe_rsc_info(rsc, "%s: Breaking dependency loop at %s",
	log_id, rsc->id);
	return;
	}
	pe__set_resource_flags(rsc, pe_rsc_merging);

	// Ignore empty groups (only possible with schema validation disabled)
	if (rsc->children == NULL) {
	return;
	}

	/* Refer the operation to the first or last member as appropriate.
	*
	* cmp_resources() is the only caller that passes a NULL nodes table,
	* and is also the only caller using pcmk__coloc_select_this_with.
	* For "this with" colocations, the last member will recursively incorporate
	* all the other members' "this with" colocations via the internal group
	* colocations (and via the first member, the group's own colocations).
	*
	* For "with this" colocations, the first member works similarly.
	*/
	if (*nodes == NULL) {
	member = pe__last_group_member(rsc);
	} else {
	member = rsc->children->data;
	}
	pe_rsc_trace(rsc, "%s: Merging scores from group %s using member %s "
	"(at %.6f)", log_id, rsc->id, member->id, factor);
	member->cmds->add_colocated_node_scores(member, log_id, nodes, attr, factor,
	flags);
	pe__clear_resource_flags(rsc, pe_rsc_merging);
	}

	// Group implementation of resource_alloc_functions_t:add_utilization()
	void
	pcmk__group_add_utilization(const pe_resource_t *rsc,
	const pe_resource_t orig_rsc, GList all_rscs,
	GHashTable *utilization)
	{
	pe_resource_t *member = NULL;

	CRM_ASSERT((rsc != NULL) && (orig_rsc != NULL) && (utilization != NULL));

	if (!pcmk_is_set(rsc->flags, pe_rsc_provisional)) {
	return;
	}

	pe_rsc_trace(orig_rsc, "%s: Adding group %s as colocated utilization",
	orig_rsc->id, rsc->id);
	if (pe__group_flag_is_set(rsc, pe__group_colocated)
	\|\| pe_rsc_is_clone(rsc->parent)) {
	// Every group member will be on same node, so sum all members
	for (GList *iter = rsc->children; iter != NULL; iter = iter->next) {
	member = (pe_resource_t *) iter->data;

	if (pcmk_is_set(member->flags, pe_rsc_provisional)
	&& (g_list_find(all_rscs, member) == NULL)) {
	member->cmds->add_utilization(member, orig_rsc, all_rscs,
	utilization);
	}
	}

	} else if (rsc->children != NULL) {
	// Just add first member's utilization
	member = (pe_resource_t *) rsc->children->data;
	if ((member != NULL)
	&& pcmk_is_set(member->flags, pe_rsc_provisional)
	&& (g_list_find(all_rscs, member) == NULL)) {

	member->cmds->add_utilization(member, orig_rsc, all_rscs,
	utilization);
	}
	}
	}

	// Group implementation of resource_alloc_functions_t:shutdown_lock()
	void
	pcmk__group_shutdown_lock(pe_resource_t *rsc)
	{
	CRM_ASSERT(rsc != NULL);

	for (GList *iter = rsc->children; iter != NULL; iter = iter->next) {
	pe_resource_t member = (pe_resource_t ) iter->data;

	member->cmds->shutdown_lock(member);
	}
	}
	diff --git a/lib/pacemaker/pcmk_sched_ordering.c b/lib/pacemaker/pcmk_sched_ordering.c
	index 3fd29e8c30..6629999014 100644
	--- a/lib/pacemaker/pcmk_sched_ordering.c
	+++ b/lib/pacemaker/pcmk_sched_ordering.c
	@@ -1,1463 +1,1463 @@
	/*
	* Copyright 2004-2023 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <inttypes.h> // PRIx32
	#include <stdbool.h>
	#include <glib.h>

	#include <crm/crm.h>
	#include <pacemaker-internal.h>
	#include "libpacemaker_private.h"

	enum pe_order_kind {
	pe_order_kind_optional,
	pe_order_kind_mandatory,
	pe_order_kind_serialize,
	};

	enum ordering_symmetry {
	ordering_asymmetric, // the only relation in an asymmetric ordering
	ordering_symmetric, // the normal relation in a symmetric ordering
	ordering_symmetric_inverse, // the inverse relation in a symmetric ordering
	};

	#define EXPAND_CONSTRAINT_IDREF(__set, __rsc, __name) do { \
	__rsc = pcmk__find_constraint_resource(data_set->resources, __name); \
	if (__rsc == NULL) { \
	pcmk__config_err("%s: No resource found for %s", __set, __name); \
	return pcmk_rc_unpack_error; \
	} \
	} while (0)

	static const char *
	invert_action(const char *action)
	{
	if (pcmk__str_eq(action, RSC_START, pcmk__str_casei)) {
	return RSC_STOP;

	} else if (pcmk__str_eq(action, RSC_STOP, pcmk__str_casei)) {
	return RSC_START;

	} else if (pcmk__str_eq(action, RSC_PROMOTE, pcmk__str_casei)) {
	return RSC_DEMOTE;

	} else if (pcmk__str_eq(action, RSC_DEMOTE, pcmk__str_casei)) {
	return RSC_PROMOTE;

	} else if (pcmk__str_eq(action, RSC_PROMOTED, pcmk__str_casei)) {
	return RSC_DEMOTED;

	} else if (pcmk__str_eq(action, RSC_DEMOTED, pcmk__str_casei)) {
	return RSC_PROMOTED;

	} else if (pcmk__str_eq(action, RSC_STARTED, pcmk__str_casei)) {
	return RSC_STOPPED;

	} else if (pcmk__str_eq(action, RSC_STOPPED, pcmk__str_casei)) {
	return RSC_STARTED;
	}
	crm_warn("Unknown action '%s' specified in order constraint", action);
	return NULL;
	}

	static enum pe_order_kind
	get_ordering_type(const xmlNode *xml_obj)
	{
	enum pe_order_kind kind_e = pe_order_kind_mandatory;
	const char *kind = crm_element_value(xml_obj, XML_ORDER_ATTR_KIND);

	if (kind == NULL) {
	const char *score = crm_element_value(xml_obj, XML_RULE_ATTR_SCORE);

	kind_e = pe_order_kind_mandatory;

	if (score) {
	// @COMPAT deprecated informally since 1.0.7, formally since 2.0.1
	int score_i = char2score(score);

	if (score_i == 0) {
	kind_e = pe_order_kind_optional;
	}
	pe_warn_once(pe_wo_order_score,
	"Support for 'score' in rsc_order is deprecated "
	"and will be removed in a future release "
	"(use 'kind' instead)");
	}

	} else if (pcmk__str_eq(kind, "Mandatory", pcmk__str_casei)) {
	kind_e = pe_order_kind_mandatory;

	} else if (pcmk__str_eq(kind, "Optional", pcmk__str_casei)) {
	kind_e = pe_order_kind_optional;

	} else if (pcmk__str_eq(kind, "Serialize", pcmk__str_casei)) {
	kind_e = pe_order_kind_serialize;

	} else {
	pcmk__config_err("Resetting '" XML_ORDER_ATTR_KIND "' for constraint "
	"%s to 'Mandatory' because '%s' is not valid",
	pcmk__s(ID(xml_obj), "missing ID"), kind);
	}
	return kind_e;
	}

	/*!
	* \internal
	* \brief Get ordering symmetry from XML
	*
	* \param[in] xml_obj Ordering XML
	* \param[in] parent_kind Default ordering kind
	* \param[in] parent_symmetrical_s Parent element's symmetrical setting, if any
	*
	* \retval ordering_symmetric Ordering is symmetric
	* \retval ordering_asymmetric Ordering is asymmetric
	*/
	static enum ordering_symmetry
	get_ordering_symmetry(const xmlNode *xml_obj, enum pe_order_kind parent_kind,
	const char *parent_symmetrical_s)
	{
	int rc = pcmk_rc_ok;
	bool symmetric = false;
	enum pe_order_kind kind = parent_kind; // Default to parent's kind

	// Check ordering XML for explicit kind
	if ((crm_element_value(xml_obj, XML_ORDER_ATTR_KIND) != NULL)
	\|\| (crm_element_value(xml_obj, XML_RULE_ATTR_SCORE) != NULL)) {
	kind = get_ordering_type(xml_obj);
	}

	// Check ordering XML (and parent) for explicit symmetrical setting
	rc = pcmk__xe_get_bool_attr(xml_obj, XML_CONS_ATTR_SYMMETRICAL, &symmetric);

	if (rc != pcmk_rc_ok && parent_symmetrical_s != NULL) {
	symmetric = crm_is_true(parent_symmetrical_s);
	rc = pcmk_rc_ok;
	}

	if (rc == pcmk_rc_ok) {
	if (symmetric) {
	if (kind == pe_order_kind_serialize) {
	pcmk__config_warn("Ignoring " XML_CONS_ATTR_SYMMETRICAL
	" for '%s' because not valid with "
	XML_ORDER_ATTR_KIND " of 'Serialize'",
	ID(xml_obj));
	} else {
	return ordering_symmetric;
	}
	}
	return ordering_asymmetric;
	}

	// Use default symmetry
	if (kind == pe_order_kind_serialize) {
	return ordering_asymmetric;
	}
	return ordering_symmetric;
	}

	/*!
	* \internal
	* \brief Get ordering flags appropriate to ordering kind
	*
	* \param[in] kind Ordering kind
	* \param[in] first Action name for 'first' action
	* \param[in] symmetry This ordering's symmetry role
	*
	* \return Minimal ordering flags appropriate to \p kind
	*/
	static uint32_t
	ordering_flags_for_kind(enum pe_order_kind kind, const char *first,
	enum ordering_symmetry symmetry)
	{
	uint32_t flags = pe_order_none; // so we trace-log all flags set

	pe__set_order_flags(flags, pe_order_optional);

	switch (kind) {
	case pe_order_kind_optional:
	break;

	case pe_order_kind_serialize:
	pe__set_order_flags(flags, pe_order_serialize_only);
	break;

	case pe_order_kind_mandatory:
	switch (symmetry) {
	case ordering_asymmetric:
	pe__set_order_flags(flags, pe_order_asymmetrical);
	break;

	case ordering_symmetric:
	pe__set_order_flags(flags, pe_order_implies_then);
	if (pcmk__strcase_any_of(first, RSC_START, RSC_PROMOTE,
	NULL)) {
	pe__set_order_flags(flags, pe_order_runnable_left);
	}
	break;

	case ordering_symmetric_inverse:
	pe__set_order_flags(flags, pe_order_implies_first);
	break;
	}
	break;
	}
	return flags;
	}

	/*!
	* \internal
	* \brief Find resource corresponding to ID specified in ordering
	*
	* \param[in] xml Ordering XML
	* \param[in] resource_attr XML attribute name for resource ID
	* \param[in] instance_attr XML attribute name for instance number.
	* This option is deprecated and will be removed in a
	* future release.
	* \param[in] data_set Cluster working set
	*
	* \return Resource corresponding to \p id, or NULL if none
	*/
	static pe_resource_t *
	get_ordering_resource(const xmlNode xml, const char resource_attr,
	const char *instance_attr,
	const pe_working_set_t *data_set)
	{
	// @COMPAT: instance_attr and instance_id variables deprecated since 2.1.5
	pe_resource_t *rsc = NULL;
	const char *rsc_id = crm_element_value(xml, resource_attr);
	const char *instance_id = crm_element_value(xml, instance_attr);

	if (rsc_id == NULL) {
	pcmk__config_err("Ignoring constraint '%s' without %s",
	ID(xml), resource_attr);
	return NULL;
	}

	rsc = pcmk__find_constraint_resource(data_set->resources, rsc_id);
	if (rsc == NULL) {
	pcmk__config_err("Ignoring constraint '%s' because resource '%s' "
	"does not exist", ID(xml), rsc_id);
	return NULL;
	}

	if (instance_id != NULL) {
	pe_warn_once(pe_wo_order_inst,
	"Support for " XML_ORDER_ATTR_FIRST_INSTANCE " and "
	XML_ORDER_ATTR_THEN_INSTANCE " is deprecated and will be "
	"removed in a future release.");

	if (!pe_rsc_is_clone(rsc)) {
	pcmk__config_err("Ignoring constraint '%s' because resource '%s' "
	"is not a clone but instance '%s' was requested",
	ID(xml), rsc_id, instance_id);
	return NULL;
	}
	rsc = find_clone_instance(rsc, instance_id);
	if (rsc == NULL) {
	pcmk__config_err("Ignoring constraint '%s' because resource '%s' "
	"does not have an instance '%s'",
	"'%s'", ID(xml), rsc_id, instance_id);
	return NULL;
	}
	}
	return rsc;
	}

	/*!
	* \internal
	* \brief Determine minimum number of 'first' instances required in ordering
	*
	* \param[in] rsc 'First' resource in ordering
	* \param[in] xml Ordering XML
	*
	* \return Minimum 'first' instances required (or 0 if not applicable)
	*/
	static int
	get_minimum_first_instances(const pe_resource_t rsc, const xmlNode xml)
	{
	const char *clone_min = NULL;
	bool require_all = false;

	if (!pe_rsc_is_clone(rsc)) {
	return 0;
	}

	clone_min = g_hash_table_lookup(rsc->meta,
	XML_RSC_ATTR_INCARNATION_MIN);
	if (clone_min != NULL) {
	int clone_min_int = 0;

	pcmk__scan_min_int(clone_min, &clone_min_int, 0);
	return clone_min_int;
	}

	/* @COMPAT 1.1.13:
	* require-all=false is deprecated equivalent of clone-min=1
	*/
	if (pcmk__xe_get_bool_attr(xml, "require-all", &require_all) != ENODATA) {
	pe_warn_once(pe_wo_require_all,
	"Support for require-all in ordering constraints "
	"is deprecated and will be removed in a future release"
	" (use clone-min clone meta-attribute instead)");
	if (!require_all) {
	return 1;
	}
	}

	return 0;
	}

	/*!
	* \internal
	* \brief Create orderings for a constraint with clone-min > 0
	*
	* \param[in] id Ordering ID
	* \param[in,out] rsc_first 'First' resource in ordering (a clone)
	* \param[in] action_first 'First' action in ordering
	* \param[in] rsc_then 'Then' resource in ordering
	* \param[in] action_then 'Then' action in ordering
	* \param[in] flags Ordering flags
	* \param[in] clone_min Minimum required instances of 'first'
	* \param[in,out] data_set Cluster working set
	*/
	static void
	clone_min_ordering(const char *id,
	pe_resource_t rsc_first, const char action_first,
	pe_resource_t rsc_then, const char action_then,
	uint32_t flags, int clone_min, pe_working_set_t *data_set)
	{
	// Create a pseudo-action for when the minimum instances are active
	char *task = crm_strdup_printf(CRM_OP_RELAXED_CLONE ":%s", id);
	pe_action_t *clone_min_met = get_pseudo_op(task, data_set);

	free(task);

	/* Require the pseudo-action to have the required number of actions to be
	* considered runnable before allowing the pseudo-action to be runnable.
	*/
	clone_min_met->required_runnable_before = clone_min;
	pe__set_action_flags(clone_min_met, pe_action_requires_any);

	// Order the actions for each clone instance before the pseudo-action
	for (GList *rIter = rsc_first->children; rIter != NULL;
	rIter = rIter->next) {

	pe_resource_t *child = rIter->data;

	pcmk__new_ordering(child, pcmk__op_key(child->id, action_first, 0),
	NULL, NULL, NULL, clone_min_met,
	pe_order_one_or_more\|pe_order_implies_then_printed,
	data_set);
	}

	// Order "then" action after the pseudo-action (if runnable)
	pcmk__new_ordering(NULL, NULL, clone_min_met, rsc_then,
	pcmk__op_key(rsc_then->id, action_then, 0),
	NULL, flags\|pe_order_runnable_left, data_set);
	}

	/*!
	* \internal
	* \brief Update ordering flags for restart-type=restart
	*
	* \param[in] rsc 'Then' resource in ordering
	* \param[in] kind Ordering kind
	* \param[in] flag Ordering flag to set (when applicable)
	* \param[in,out] flags Ordering flag set to update
	*
	* \compat The restart-type resource meta-attribute is deprecated. Eventually,
	* it will be removed, and pe_restart_ignore will be the only behavior,
	* at which time this can just be removed entirely.
	*/
	#define handle_restart_type(rsc, kind, flag, flags) do { \
	if (((kind) == pe_order_kind_optional) \
	&& ((rsc)->restart_type == pe_restart_restart)) { \
	pe__set_order_flags((flags), (flag)); \
	} \
	} while (0)

	/*!
	* \internal
	* \brief Create new ordering for inverse of symmetric constraint
	*
	* \param[in] id Ordering ID (for logging only)
	* \param[in] kind Ordering kind
	* \param[in] rsc_first 'First' resource in ordering (a clone)
	* \param[in] action_first 'First' action in ordering
	* \param[in,out] rsc_then 'Then' resource in ordering
	* \param[in] action_then 'Then' action in ordering
	*/
	static void
	inverse_ordering(const char *id, enum pe_order_kind kind,
	pe_resource_t rsc_first, const char action_first,
	pe_resource_t rsc_then, const char action_then)
	{
	action_then = invert_action(action_then);
	action_first = invert_action(action_first);
	if ((action_then == NULL) \|\| (action_first == NULL)) {
	pcmk__config_warn("Cannot invert constraint '%s' "
	"(please specify inverse manually)", id);
	} else {
	uint32_t flags = ordering_flags_for_kind(kind, action_first,
	ordering_symmetric_inverse);

	handle_restart_type(rsc_then, kind, pe_order_implies_first, flags);
	pcmk__order_resource_actions(rsc_then, action_then, rsc_first,
	action_first, flags);
	}
	}

	static void
	unpack_simple_rsc_order(xmlNode xml_obj, pe_working_set_t data_set)
	{
	pe_resource_t *rsc_then = NULL;
	pe_resource_t *rsc_first = NULL;
	int min_required_before = 0;
	enum pe_order_kind kind = pe_order_kind_mandatory;
	uint32_t cons_weight = pe_order_none;
	enum ordering_symmetry symmetry;

	const char *action_then = NULL;
	const char *action_first = NULL;
	const char *id = NULL;

	CRM_CHECK(xml_obj != NULL, return);

	id = crm_element_value(xml_obj, XML_ATTR_ID);
	if (id == NULL) {
	pcmk__config_err("Ignoring <%s> constraint without " XML_ATTR_ID,
	crm_element_name(xml_obj));
	return;
	}

	rsc_first = get_ordering_resource(xml_obj, XML_ORDER_ATTR_FIRST,
	XML_ORDER_ATTR_FIRST_INSTANCE,
	data_set);
	if (rsc_first == NULL) {
	return;
	}

	rsc_then = get_ordering_resource(xml_obj, XML_ORDER_ATTR_THEN,
	XML_ORDER_ATTR_THEN_INSTANCE,
	data_set);
	if (rsc_then == NULL) {
	return;
	}

	action_first = crm_element_value(xml_obj, XML_ORDER_ATTR_FIRST_ACTION);
	if (action_first == NULL) {
	action_first = RSC_START;
	}

	action_then = crm_element_value(xml_obj, XML_ORDER_ATTR_THEN_ACTION);
	if (action_then == NULL) {
	action_then = action_first;
	}

	kind = get_ordering_type(xml_obj);

	symmetry = get_ordering_symmetry(xml_obj, kind, NULL);
	cons_weight = ordering_flags_for_kind(kind, action_first, symmetry);

	handle_restart_type(rsc_then, kind, pe_order_implies_then, cons_weight);

	/* If there is a minimum number of instances that must be runnable before
	* the 'then' action is runnable, we use a pseudo-action for convenience:
	* minimum number of clone instances have runnable actions ->
	* pseudo-action is runnable -> dependency is runnable.
	*/
	min_required_before = get_minimum_first_instances(rsc_first, xml_obj);
	if (min_required_before > 0) {
	clone_min_ordering(id, rsc_first, action_first, rsc_then, action_then,
	cons_weight, min_required_before, data_set);
	} else {
	pcmk__order_resource_actions(rsc_first, action_first, rsc_then,
	action_then, cons_weight);
	}

	if (symmetry == ordering_symmetric) {
	inverse_ordering(id, kind, rsc_first, action_first,
	rsc_then, action_then);
	}
	}

	/*!
	* \internal
	* \brief Create a new ordering between two actions
	*
	* \param[in,out] first_rsc Resource for 'first' action (if NULL and
	* \p first_action is a resource action, that
	* resource will be used)
	* \param[in,out] first_action_task Action key for 'first' action (if NULL and
	* \p first_action is not NULL, its UUID will
	* be used)
	* \param[in,out] first_action 'first' action (if NULL, \p first_rsc and
	* \p first_action_task must be set)
	*
	* \param[in] then_rsc Resource for 'then' action (if NULL and
	* \p then_action is a resource action, that
	* resource will be used)
	* \param[in,out] then_action_task Action key for 'then' action (if NULL and
	* \p then_action is not NULL, its UUID will
	* be used)
	* \param[in] then_action 'then' action (if NULL, \p then_rsc and
	* \p then_action_task must be set)
	*
	* \param[in] flags Flag set of enum pe_ordering
	* \param[in,out] data_set Cluster working set to add ordering to
	*
	* \note This function takes ownership of first_action_task and
	* then_action_task, which do not need to be freed by the caller.
	*/
	void
	pcmk__new_ordering(pe_resource_t first_rsc, char first_action_task,
	pe_action_t first_action, pe_resource_t then_rsc,
	char then_action_task, pe_action_t then_action,
	uint32_t flags, pe_working_set_t *data_set)
	{
	pe__ordering_t *order = NULL;

	// One of action or resource must be specified for each side
	CRM_CHECK(((first_action != NULL) \|\| (first_rsc != NULL))
	&& ((then_action != NULL) \|\| (then_rsc != NULL)),
	free(first_action_task); free(then_action_task); return);

	if ((first_rsc == NULL) && (first_action != NULL)) {
	first_rsc = first_action->rsc;
	}
	if ((then_rsc == NULL) && (then_action != NULL)) {
	then_rsc = then_action->rsc;
	}

	order = calloc(1, sizeof(pe__ordering_t));
	CRM_ASSERT(order != NULL);

	order->id = data_set->order_id++;
	order->flags = flags;
	order->lh_rsc = first_rsc;
	order->rh_rsc = then_rsc;
	order->lh_action = first_action;
	order->rh_action = then_action;
	order->lh_action_task = first_action_task;
	order->rh_action_task = then_action_task;

	if ((order->lh_action_task == NULL) && (first_action != NULL)) {
	order->lh_action_task = strdup(first_action->uuid);
	}

	if ((order->rh_action_task == NULL) && (then_action != NULL)) {
	order->rh_action_task = strdup(then_action->uuid);
	}

	if ((order->lh_rsc == NULL) && (first_action != NULL)) {
	order->lh_rsc = first_action->rsc;
	}

	if ((order->rh_rsc == NULL) && (then_action != NULL)) {
	order->rh_rsc = then_action->rsc;
	}

	pe_rsc_trace(first_rsc, "Created ordering %d for %s then %s",
	(data_set->order_id - 1),
	- ((first_action_task == NULL)? "?" : first_action_task),
	- ((then_action_task == NULL)? "?" : then_action_task));
	+ pcmk__s(order->lh_action_task, "an underspecified action"),
	+ pcmk__s(order->rh_action_task, "an underspecified action"));

	data_set->ordering_constraints = g_list_prepend(data_set->ordering_constraints,
	order);
	pcmk__order_migration_equivalents(order);
	}

	/*!
	* \brief Unpack a set in an ordering constraint
	*
	* \param[in] set Set XML to unpack
	* \param[in] parent_kind rsc_order XML "kind" attribute
	* \param[in] parent_symmetrical_s rsc_order XML "symmetrical" attribute
	* \param[in,out] data_set Cluster working set
	*
	* \return Standard Pacemaker return code
	*/
	static int
	unpack_order_set(const xmlNode *set, enum pe_order_kind parent_kind,
	const char parent_symmetrical_s, pe_working_set_t data_set)
	{
	GList *set_iter = NULL;
	GList *resources = NULL;

	pe_resource_t *last = NULL;
	pe_resource_t *resource = NULL;

	int local_kind = parent_kind;
	bool sequential = false;
	uint32_t flags = pe_order_optional;
	enum ordering_symmetry symmetry;

	char *key = NULL;
	const char *id = ID(set);
	const char *action = crm_element_value(set, "action");
	const char *sequential_s = crm_element_value(set, "sequential");
	const char *kind_s = crm_element_value(set, XML_ORDER_ATTR_KIND);

	if (action == NULL) {
	action = RSC_START;
	}

	if (kind_s) {
	local_kind = get_ordering_type(set);
	}
	if (sequential_s == NULL) {
	sequential_s = "1";
	}

	sequential = crm_is_true(sequential_s);

	symmetry = get_ordering_symmetry(set, parent_kind, parent_symmetrical_s);
	flags = ordering_flags_for_kind(local_kind, action, symmetry);

	for (const xmlNode *xml_rsc = first_named_child(set, XML_TAG_RESOURCE_REF);
	xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {

	EXPAND_CONSTRAINT_IDREF(id, resource, ID(xml_rsc));
	resources = g_list_append(resources, resource);
	}

	if (pcmk__list_of_1(resources)) {
	crm_trace("Single set: %s", id);
	goto done;
	}

	set_iter = resources;
	while (set_iter != NULL) {
	resource = (pe_resource_t *) set_iter->data;
	set_iter = set_iter->next;

	key = pcmk__op_key(resource->id, action, 0);

	if (local_kind == pe_order_kind_serialize) {
	/* Serialize before everything that comes after */

	for (GList *gIter = set_iter; gIter != NULL; gIter = gIter->next) {
	pe_resource_t then_rsc = (pe_resource_t ) gIter->data;
	char *then_key = pcmk__op_key(then_rsc->id, action, 0);

	pcmk__new_ordering(resource, strdup(key), NULL, then_rsc,
	then_key, NULL, flags, data_set);
	}

	} else if (sequential) {
	if (last != NULL) {
	pcmk__order_resource_actions(last, action, resource, action,
	flags);
	}
	last = resource;
	}
	free(key);
	}

	if (symmetry == ordering_asymmetric) {
	goto done;
	}

	last = NULL;
	action = invert_action(action);

	flags = ordering_flags_for_kind(local_kind, action,
	ordering_symmetric_inverse);

	set_iter = resources;
	while (set_iter != NULL) {
	resource = (pe_resource_t *) set_iter->data;
	set_iter = set_iter->next;

	if (sequential) {
	if (last != NULL) {
	pcmk__order_resource_actions(resource, action, last, action,
	flags);
	}
	last = resource;
	}
	}

	done:
	g_list_free(resources);
	return pcmk_rc_ok;
	}

	/*!
	* \brief Order two resource sets relative to each other
	*
	* \param[in] id Ordering ID (for logging)
	* \param[in] set1 First listed set
	* \param[in] set2 Second listed set
	* \param[in] kind Ordering kind
	* \param[in,out] data_set Cluster working set
	* \param[in] symmetry Which ordering symmetry applies to this relation
	*
	* \return Standard Pacemaker return code
	*/
	static int
	order_rsc_sets(const char id, const xmlNode set1, const xmlNode *set2,
	enum pe_order_kind kind, pe_working_set_t *data_set,
	enum ordering_symmetry symmetry)
	{

	const xmlNode *xml_rsc = NULL;
	const xmlNode *xml_rsc_2 = NULL;

	pe_resource_t *rsc_1 = NULL;
	pe_resource_t *rsc_2 = NULL;

	const char *action_1 = crm_element_value(set1, "action");
	const char *action_2 = crm_element_value(set2, "action");

	uint32_t flags = pe_order_none;

	bool require_all = true;

	(void) pcmk__xe_get_bool_attr(set1, "require-all", &require_all);

	if (action_1 == NULL) {
	action_1 = RSC_START;
	}

	if (action_2 == NULL) {
	action_2 = RSC_START;
	}

	if (symmetry == ordering_symmetric_inverse) {
	action_1 = invert_action(action_1);
	action_2 = invert_action(action_2);
	}

	if (pcmk__str_eq(RSC_STOP, action_1, pcmk__str_casei)
	\|\| pcmk__str_eq(RSC_DEMOTE, action_1, pcmk__str_casei)) {
	/* Assuming: A -> ( B \|\| C) -> D
	* The one-or-more logic only applies during the start/promote phase.
	* During shutdown neither B nor can shutdown until D is down, so simply
	* turn require_all back on.
	*/
	require_all = true;
	}

	flags = ordering_flags_for_kind(kind, action_1, symmetry);

	/* If we have an unordered set1, whether it is sequential or not is
	* irrelevant in regards to set2.
	*/
	if (!require_all) {
	char *task = crm_strdup_printf(CRM_OP_RELAXED_SET ":%s", ID(set1));
	pe_action_t *unordered_action = get_pseudo_op(task, data_set);

	free(task);
	pe__set_action_flags(unordered_action, pe_action_requires_any);

	for (xml_rsc = first_named_child(set1, XML_TAG_RESOURCE_REF);
	xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {

	EXPAND_CONSTRAINT_IDREF(id, rsc_1, ID(xml_rsc));

	/* Add an ordering constraint between every element in set1 and the
	* pseudo action. If any action in set1 is runnable the pseudo
	* action will be runnable.
	*/
	pcmk__new_ordering(rsc_1, pcmk__op_key(rsc_1->id, action_1, 0),
	NULL, NULL, NULL, unordered_action,
	pe_order_one_or_more\|pe_order_implies_then_printed,
	data_set);
	}
	for (xml_rsc_2 = first_named_child(set2, XML_TAG_RESOURCE_REF);
	xml_rsc_2 != NULL; xml_rsc_2 = crm_next_same_xml(xml_rsc_2)) {

	EXPAND_CONSTRAINT_IDREF(id, rsc_2, ID(xml_rsc_2));

	/* Add an ordering constraint between the pseudo-action and every
	* element in set2. If the pseudo-action is runnable, every action
	* in set2 will be runnable.
	*/
	pcmk__new_ordering(NULL, NULL, unordered_action,
	rsc_2, pcmk__op_key(rsc_2->id, action_2, 0),
	NULL, flags\|pe_order_runnable_left, data_set);
	}

	return pcmk_rc_ok;
	}

	if (pcmk__xe_attr_is_true(set1, "sequential")) {
	if (symmetry == ordering_symmetric_inverse) {
	// Get the first one
	xml_rsc = first_named_child(set1, XML_TAG_RESOURCE_REF);
	if (xml_rsc != NULL) {
	EXPAND_CONSTRAINT_IDREF(id, rsc_1, ID(xml_rsc));
	}

	} else {
	// Get the last one
	const char *rid = NULL;

	for (xml_rsc = first_named_child(set1, XML_TAG_RESOURCE_REF);
	xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {

	rid = ID(xml_rsc);
	}
	EXPAND_CONSTRAINT_IDREF(id, rsc_1, rid);
	}
	}

	if (pcmk__xe_attr_is_true(set2, "sequential")) {
	if (symmetry == ordering_symmetric_inverse) {
	// Get the last one
	const char *rid = NULL;

	for (xml_rsc = first_named_child(set2, XML_TAG_RESOURCE_REF);
	xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {

	rid = ID(xml_rsc);
	}
	EXPAND_CONSTRAINT_IDREF(id, rsc_2, rid);

	} else {
	// Get the first one
	xml_rsc = first_named_child(set2, XML_TAG_RESOURCE_REF);
	if (xml_rsc != NULL) {
	EXPAND_CONSTRAINT_IDREF(id, rsc_2, ID(xml_rsc));
	}
	}
	}

	if ((rsc_1 != NULL) && (rsc_2 != NULL)) {
	pcmk__order_resource_actions(rsc_1, action_1, rsc_2, action_2, flags);

	} else if (rsc_1 != NULL) {
	for (xml_rsc = first_named_child(set2, XML_TAG_RESOURCE_REF);
	xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {

	EXPAND_CONSTRAINT_IDREF(id, rsc_2, ID(xml_rsc));
	pcmk__order_resource_actions(rsc_1, action_1, rsc_2, action_2,
	flags);
	}

	} else if (rsc_2 != NULL) {
	for (xml_rsc = first_named_child(set1, XML_TAG_RESOURCE_REF);
	xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {

	EXPAND_CONSTRAINT_IDREF(id, rsc_1, ID(xml_rsc));
	pcmk__order_resource_actions(rsc_1, action_1, rsc_2, action_2,
	flags);
	}

	} else {
	for (xml_rsc = first_named_child(set1, XML_TAG_RESOURCE_REF);
	xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {

	EXPAND_CONSTRAINT_IDREF(id, rsc_1, ID(xml_rsc));

	for (xmlNode *xml_rsc_2 = first_named_child(set2, XML_TAG_RESOURCE_REF);
	xml_rsc_2 != NULL; xml_rsc_2 = crm_next_same_xml(xml_rsc_2)) {

	EXPAND_CONSTRAINT_IDREF(id, rsc_2, ID(xml_rsc_2));
	pcmk__order_resource_actions(rsc_1, action_1, rsc_2,
	action_2, flags);
	}
	}
	}

	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief If an ordering constraint uses resource tags, expand them
	*
	* \param[in,out] xml_obj Ordering constraint XML
	* \param[out] expanded_xml Equivalent XML with tags expanded
	* \param[in] data_set Cluster working set
	*
	* \return Standard Pacemaker return code (specifically, pcmk_rc_ok on success,
	* and pcmk_rc_unpack_error on invalid configuration)
	*/
	static int
	unpack_order_tags(xmlNode xml_obj, xmlNode *expanded_xml,
	const pe_working_set_t *data_set)
	{
	const char *id_first = NULL;
	const char *id_then = NULL;
	const char *action_first = NULL;
	const char *action_then = NULL;

	pe_resource_t *rsc_first = NULL;
	pe_resource_t *rsc_then = NULL;
	pe_tag_t *tag_first = NULL;
	pe_tag_t *tag_then = NULL;

	xmlNode *rsc_set_first = NULL;
	xmlNode *rsc_set_then = NULL;
	bool any_sets = false;

	// Check whether there are any resource sets with template or tag references
	*expanded_xml = pcmk__expand_tags_in_sets(xml_obj, data_set);
	if (*expanded_xml != NULL) {
	crm_log_xml_trace(*expanded_xml, "Expanded rsc_order");
	return pcmk_rc_ok;
	}

	id_first = crm_element_value(xml_obj, XML_ORDER_ATTR_FIRST);
	id_then = crm_element_value(xml_obj, XML_ORDER_ATTR_THEN);
	if ((id_first == NULL) \|\| (id_then == NULL)) {
	return pcmk_rc_ok;
	}

	if (!pcmk__valid_resource_or_tag(data_set, id_first, &rsc_first,
	&tag_first)) {
	pcmk__config_err("Ignoring constraint '%s' because '%s' is not a "
	"valid resource or tag", ID(xml_obj), id_first);
	return pcmk_rc_unpack_error;
	}

	if (!pcmk__valid_resource_or_tag(data_set, id_then, &rsc_then, &tag_then)) {
	pcmk__config_err("Ignoring constraint '%s' because '%s' is not a "
	"valid resource or tag", ID(xml_obj), id_then);
	return pcmk_rc_unpack_error;
	}

	if ((rsc_first != NULL) && (rsc_then != NULL)) {
	// Neither side references a template or tag
	return pcmk_rc_ok;
	}

	action_first = crm_element_value(xml_obj, XML_ORDER_ATTR_FIRST_ACTION);
	action_then = crm_element_value(xml_obj, XML_ORDER_ATTR_THEN_ACTION);

	*expanded_xml = copy_xml(xml_obj);

	// Convert template/tag reference in "first" into resource_set under constraint
	if (!pcmk__tag_to_set(*expanded_xml, &rsc_set_first, XML_ORDER_ATTR_FIRST,
	true, data_set)) {
	free_xml(*expanded_xml);
	*expanded_xml = NULL;
	return pcmk_rc_unpack_error;
	}

	if (rsc_set_first != NULL) {
	if (action_first != NULL) {
	// Move "first-action" into converted resource_set as "action"
	crm_xml_add(rsc_set_first, "action", action_first);
	xml_remove_prop(*expanded_xml, XML_ORDER_ATTR_FIRST_ACTION);
	}
	any_sets = true;
	}

	// Convert template/tag reference in "then" into resource_set under constraint
	if (!pcmk__tag_to_set(*expanded_xml, &rsc_set_then, XML_ORDER_ATTR_THEN,
	true, data_set)) {
	free_xml(*expanded_xml);
	*expanded_xml = NULL;
	return pcmk_rc_unpack_error;
	}

	if (rsc_set_then != NULL) {
	if (action_then != NULL) {
	// Move "then-action" into converted resource_set as "action"
	crm_xml_add(rsc_set_then, "action", action_then);
	xml_remove_prop(*expanded_xml, XML_ORDER_ATTR_THEN_ACTION);
	}
	any_sets = true;
	}

	if (any_sets) {
	crm_log_xml_trace(*expanded_xml, "Expanded rsc_order");
	} else {
	free_xml(*expanded_xml);
	*expanded_xml = NULL;
	}

	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief Unpack ordering constraint XML
	*
	* \param[in,out] xml_obj Ordering constraint XML to unpack
	* \param[in,out] data_set Cluster working set
	*/
	void
	pcmk__unpack_ordering(xmlNode xml_obj, pe_working_set_t data_set)
	{
	xmlNode *set = NULL;
	xmlNode *last = NULL;

	xmlNode *orig_xml = NULL;
	xmlNode *expanded_xml = NULL;

	const char *id = crm_element_value(xml_obj, XML_ATTR_ID);
	const char *invert = crm_element_value(xml_obj, XML_CONS_ATTR_SYMMETRICAL);
	enum pe_order_kind kind = get_ordering_type(xml_obj);

	enum ordering_symmetry symmetry = get_ordering_symmetry(xml_obj, kind,
	NULL);

	// Expand any resource tags in the constraint XML
	if (unpack_order_tags(xml_obj, &expanded_xml, data_set) != pcmk_rc_ok) {
	return;
	}
	if (expanded_xml != NULL) {
	orig_xml = xml_obj;
	xml_obj = expanded_xml;
	}

	// If the constraint has resource sets, unpack them
	for (set = first_named_child(xml_obj, XML_CONS_TAG_RSC_SET);
	set != NULL; set = crm_next_same_xml(set)) {

	set = expand_idref(set, data_set->input);
	if ((set == NULL) // Configuration error, message already logged
	\|\| (unpack_order_set(set, kind, invert, data_set) != pcmk_rc_ok)) {

	if (expanded_xml != NULL) {
	free_xml(expanded_xml);
	}
	return;
	}

	if (last != NULL) {

	if (order_rsc_sets(id, last, set, kind, data_set,
	symmetry) != pcmk_rc_ok) {
	if (expanded_xml != NULL) {
	free_xml(expanded_xml);
	}
	return;
	}

	if ((symmetry == ordering_symmetric)
	&& (order_rsc_sets(id, set, last, kind, data_set,
	ordering_symmetric_inverse) != pcmk_rc_ok)) {
	if (expanded_xml != NULL) {
	free_xml(expanded_xml);
	}
	return;
	}

	}
	last = set;
	}

	if (expanded_xml) {
	free_xml(expanded_xml);
	xml_obj = orig_xml;
	}

	// If the constraint has no resource sets, unpack it as a simple ordering
	if (last == NULL) {
	return unpack_simple_rsc_order(xml_obj, data_set);
	}
	}

	static bool
	ordering_is_invalid(pe_action_t action, pe_action_wrapper_t input)
	{
	/* Prevent user-defined ordering constraints between resources
	* running in a guest node and the resource that defines that node.
	*/
	if (!pcmk_is_set(input->type, pe_order_preserve)
	&& (input->action->rsc != NULL)
	&& pcmk__rsc_corresponds_to_guest(action->rsc, input->action->node)) {

	crm_warn("Invalid ordering constraint between %s and %s",
	input->action->rsc->id, action->rsc->id);
	return true;
	}

	/* If there's an order like
	* "rscB_stop node2"-> "load_stopped_node2" -> "rscA_migrate_to node1"
	*
	* then rscA is being migrated from node1 to node2, while rscB is being
	* migrated from node2 to node1. If there would be a graph loop,
	* break the order "load_stopped_node2" -> "rscA_migrate_to node1".
	*/
	if ((input->type == pe_order_load) && action->rsc
	&& pcmk__str_eq(action->task, RSC_MIGRATE, pcmk__str_casei)
	&& pcmk__graph_has_loop(action, action, input)) {
	return true;
	}

	return false;
	}

	void
	pcmk__disable_invalid_orderings(pe_working_set_t *data_set)
	{
	for (GList *iter = data_set->actions; iter != NULL; iter = iter->next) {
	pe_action_t action = (pe_action_t ) iter->data;
	pe_action_wrapper_t *input = NULL;

	for (GList *input_iter = action->actions_before;
	input_iter != NULL; input_iter = input_iter->next) {

	input = (pe_action_wrapper_t *) input_iter->data;
	if (ordering_is_invalid(action, input)) {
	input->type = pe_order_none;
	}
	}
	}
	}

	/*!
	* \internal
	* \brief Order stops on a node before the node's shutdown
	*
	* \param[in,out] node Node being shut down
	* \param[in] shutdown_op Shutdown action for node
	*/
	void
	pcmk__order_stops_before_shutdown(pe_node_t node, pe_action_t shutdown_op)
	{
	for (GList *iter = node->details->data_set->actions;
	iter != NULL; iter = iter->next) {

	pe_action_t action = (pe_action_t ) iter->data;

	// Only stops on the node shutting down are relevant
	if ((action->rsc == NULL) \|\| (action->node == NULL)
	\|\| (action->node->details != node->details)
	\|\| !pcmk__str_eq(action->task, RSC_STOP, pcmk__str_casei)) {
	continue;
	}

	// Resources and nodes in maintenance mode won't be touched

	if (pcmk_is_set(action->rsc->flags, pe_rsc_maintenance)) {
	pe_rsc_trace(action->rsc,
	"Not ordering %s before shutdown of %s because "
	"resource in maintenance mode",
	action->uuid, pe__node_name(node));
	continue;

	} else if (node->details->maintenance) {
	pe_rsc_trace(action->rsc,
	"Not ordering %s before shutdown of %s because "
	"node in maintenance mode",
	action->uuid, pe__node_name(node));
	continue;
	}

	/* Don't touch a resource that is unmanaged or blocked, to avoid
	* blocking the shutdown (though if another action depends on this one,
	* we may still end up blocking)
	*/
	if (!pcmk_any_flags_set(action->rsc->flags,
	pe_rsc_managed\|pe_rsc_block)) {
	pe_rsc_trace(action->rsc,
	"Not ordering %s before shutdown of %s because "
	"resource is unmanaged or blocked",
	action->uuid, pe__node_name(node));
	continue;
	}

	pe_rsc_trace(action->rsc, "Ordering %s before shutdown of %s",
	action->uuid, pe__node_name(node));
	pe__clear_action_flags(action, pe_action_optional);
	pcmk__new_ordering(action->rsc, NULL, action, NULL,
	strdup(CRM_OP_SHUTDOWN), shutdown_op,
	pe_order_optional\|pe_order_runnable_left,
	node->details->data_set);
	}
	}

	/*!
	* \brief Find resource actions matching directly or as child
	*
	* \param[in] rsc Resource to check
	* \param[in] original_key Action key to search for (possibly referencing
	* parent of \rsc)
	*
	* \return Newly allocated list of matching actions
	* \note It is the caller's responsibility to free the result with g_list_free()
	*/
	static GList *
	find_actions_by_task(const pe_resource_t rsc, const char original_key)
	{
	// Search under given task key directly
	GList *list = find_actions(rsc->actions, original_key, NULL);

	if (list == NULL) {
	// Search again using this resource's ID
	char *key = NULL;
	char *task = NULL;
	guint interval_ms = 0;

	if (parse_op_key(original_key, NULL, &task, &interval_ms)) {
	key = pcmk__op_key(rsc->id, task, interval_ms);
	list = find_actions(rsc->actions, key, NULL);
	free(key);
	free(task);
	} else {
	crm_err("Invalid operation key (bug?): %s", original_key);
	}
	}
	return list;
	}

	/*!
	* \internal
	* \brief Order relevant resource actions after a given action
	*
	* \param[in,out] first_action Action to order after (or NULL if none runnable)
	* \param[in] rsc Resource whose actions should be ordered
	* \param[in,out] order Ordering constraint being applied
	*/
	static void
	order_resource_actions_after(pe_action_t *first_action,
	const pe_resource_t rsc, pe__ordering_t order)
	{
	GList *then_actions = NULL;
	uint32_t flags = pe_order_none;

	CRM_CHECK((rsc != NULL) && (order != NULL), return);

	flags = order->flags;
	pe_rsc_trace(rsc, "Applying ordering %d for 'then' resource %s",
	order->id, rsc->id);

	if (order->rh_action != NULL) {
	then_actions = g_list_prepend(NULL, order->rh_action);

	} else {
	then_actions = find_actions_by_task(rsc, order->rh_action_task);
	}

	if (then_actions == NULL) {
	pe_rsc_trace(rsc, "Ignoring ordering %d: no %s actions found for %s",
	order->id, order->rh_action_task, rsc->id);
	return;
	}

	if ((first_action != NULL) && (first_action->rsc == rsc)
	&& pcmk_is_set(first_action->flags, pe_action_dangle)) {

	pe_rsc_trace(rsc,
	"Detected dangling migration ordering (%s then %s %s)",
	first_action->uuid, order->rh_action_task, rsc->id);
	pe__clear_order_flags(flags, pe_order_implies_then);
	}

	if ((first_action == NULL) && !pcmk_is_set(flags, pe_order_implies_then)) {
	pe_rsc_debug(rsc,
	"Ignoring ordering %d for %s: No first action found",
	order->id, rsc->id);
	g_list_free(then_actions);
	return;
	}

	for (GList *iter = then_actions; iter != NULL; iter = iter->next) {
	pe_action_t then_action_iter = (pe_action_t ) iter->data;

	if (first_action != NULL) {
	order_actions(first_action, then_action_iter, flags);
	} else {
	pe__clear_action_flags(then_action_iter, pe_action_runnable);
	crm_warn("%s of %s is unrunnable because there is no %s of %s "
	"to order it after", then_action_iter->task, rsc->id,
	order->lh_action_task, order->lh_rsc->id);
	}
	}

	g_list_free(then_actions);
	}

	static void
	rsc_order_first(pe_resource_t first_rsc, pe__ordering_t order,
	pe_working_set_t *data_set)
	{
	GList *first_actions = NULL;
	pe_action_t *first_action = order->lh_action;
	pe_resource_t *then_rsc = order->rh_rsc;

	CRM_ASSERT(first_rsc != NULL);
	pe_rsc_trace(first_rsc, "Applying ordering constraint %d (first: %s)",
	order->id, first_rsc->id);

	if (first_action != NULL) {
	first_actions = g_list_prepend(NULL, first_action);

	} else {
	first_actions = find_actions_by_task(first_rsc, order->lh_action_task);
	}

	if ((first_actions == NULL) && (first_rsc == then_rsc)) {
	pe_rsc_trace(first_rsc,
	"Ignoring constraint %d: first (%s for %s) not found",
	order->id, order->lh_action_task, first_rsc->id);

	} else if (first_actions == NULL) {
	char *key = NULL;
	char *op_type = NULL;
	guint interval_ms = 0;

	parse_op_key(order->lh_action_task, NULL, &op_type, &interval_ms);
	key = pcmk__op_key(first_rsc->id, op_type, interval_ms);

	if ((first_rsc->fns->state(first_rsc, TRUE) == RSC_ROLE_STOPPED)
	&& pcmk__str_eq(op_type, RSC_STOP, pcmk__str_casei)) {
	free(key);
	pe_rsc_trace(first_rsc,
	"Ignoring constraint %d: first (%s for %s) not found",
	order->id, order->lh_action_task, first_rsc->id);

	} else if ((first_rsc->fns->state(first_rsc, TRUE) == RSC_ROLE_UNPROMOTED)
	&& pcmk__str_eq(op_type, RSC_DEMOTE, pcmk__str_casei)) {
	free(key);
	pe_rsc_trace(first_rsc,
	"Ignoring constraint %d: first (%s for %s) not found",
	order->id, order->lh_action_task, first_rsc->id);

	} else {
	pe_rsc_trace(first_rsc,
	"Creating first (%s for %s) for constraint %d ",
	order->lh_action_task, first_rsc->id, order->id);
	first_action = custom_action(first_rsc, key, op_type, NULL, TRUE,
	TRUE, data_set);
	first_actions = g_list_prepend(NULL, first_action);
	}

	free(op_type);
	}

	if (then_rsc == NULL) {
	if (order->rh_action == NULL) {
	pe_rsc_trace(first_rsc, "Ignoring constraint %d: then not found",
	order->id);
	return;
	}
	then_rsc = order->rh_action->rsc;
	}
	for (GList *gIter = first_actions; gIter != NULL; gIter = gIter->next) {
	first_action = (pe_action_t *) gIter->data;

	if (then_rsc == NULL) {
	order_actions(first_action, order->rh_action, order->flags);

	} else {
	order_resource_actions_after(first_action, then_rsc, order);
	}
	}

	g_list_free(first_actions);
	}

	void
	pcmk__apply_orderings(pe_working_set_t *data_set)
	{
	crm_trace("Applying ordering constraints");

	/* Ordering constraints need to be processed in the order they were created.
	* rsc_order_first() and order_resource_actions_after() require the relevant
	* actions to already exist in some cases, but rsc_order_first() will create
	* the 'first' action in certain cases. Thus calling rsc_order_first() can
	* change the behavior of later-created orderings.
	*
	* Also, g_list_append() should be avoided for performance reasons, so we
	* prepend orderings when creating them and reverse the list here.
	*
	* @TODO This is brittle and should be carefully redesigned so that the
	* order of creation doesn't matter, and the reverse becomes unneeded.
	*/
	data_set->ordering_constraints = g_list_reverse(data_set->ordering_constraints);

	for (GList *gIter = data_set->ordering_constraints;
	gIter != NULL; gIter = gIter->next) {

	pe__ordering_t *order = gIter->data;
	pe_resource_t *rsc = order->lh_rsc;

	if (rsc != NULL) {
	rsc_order_first(rsc, order, data_set);
	continue;
	}

	rsc = order->rh_rsc;
	if (rsc != NULL) {
	order_resource_actions_after(order->lh_action, rsc, order);

	} else {
	crm_trace("Applying ordering constraint %d (non-resource actions)",
	order->id);
	order_actions(order->lh_action, order->rh_action, order->flags);
	}
	}

	g_list_foreach(data_set->actions, (GFunc) pcmk__block_colocation_dependents,
	data_set);

	crm_trace("Ordering probes");
	pcmk__order_probes(data_set);

	crm_trace("Updating %d actions", g_list_length(data_set->actions));
	g_list_foreach(data_set->actions,
	(GFunc) pcmk__update_action_for_orderings, data_set);

	pcmk__disable_invalid_orderings(data_set);
	}

	/*!
	* \internal
	* \brief Order a given action after each action in a given list
	*
	* \param[in,out] after "After" action
	* \param[in,out] list List of "before" actions
	*/
	void
	pcmk__order_after_each(pe_action_t after, GList list)
	{
	const char *after_desc = (after->task == NULL)? after->uuid : after->task;

	for (GList *iter = list; iter != NULL; iter = iter->next) {
	pe_action_t before = (pe_action_t ) iter->data;
	const char *before_desc = before->task? before->task : before->uuid;

	crm_debug("Ordering %s on %s before %s on %s",
	before_desc, pe__node_name(before->node),
	after_desc, pe__node_name(after->node));
	order_actions(before, after, pe_order_optional);
	}
	}

	/*!
	* \internal
	* \brief Order promotions and demotions for restarts of a clone or bundle
	*
	* \param[in,out] rsc Clone or bundle to order
	*/
	void
	pcmk__promotable_restart_ordering(pe_resource_t *rsc)
	{
	// Order start and promote after all instances are stopped
	pcmk__order_resource_actions(rsc, RSC_STOPPED, rsc, RSC_START,
	pe_order_optional);
	pcmk__order_resource_actions(rsc, RSC_STOPPED, rsc, RSC_PROMOTE,
	pe_order_optional);

	// Order stop, start, and promote after all instances are demoted
	pcmk__order_resource_actions(rsc, RSC_DEMOTED, rsc, RSC_STOP,
	pe_order_optional);
	pcmk__order_resource_actions(rsc, RSC_DEMOTED, rsc, RSC_START,
	pe_order_optional);
	pcmk__order_resource_actions(rsc, RSC_DEMOTED, rsc, RSC_PROMOTE,
	pe_order_optional);

	// Order promote after all instances are started
	pcmk__order_resource_actions(rsc, RSC_STARTED, rsc, RSC_PROMOTE,
	pe_order_optional);

	// Order demote after all instances are demoted
	pcmk__order_resource_actions(rsc, RSC_DEMOTE, rsc, RSC_DEMOTED,
	pe_order_optional);
	}

File Metadata

Mime Type: text/x-diff
Expires: Mon, Apr 21, 7:25 PM (4 h, 21 m)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 1665533
Default Alt Text: (232 KB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions