No OneTemporary
Actions

Size

578 KB

Referenced Files

None

Subscribers

None

View Options

This file is larger than 256 KB, so syntax highlighting was skipped.

	diff --git a/cts/cli/regression.tools.exp b/cts/cli/regression.tools.exp
	index fee49a2625..1bd6eca5db 100644
	--- a/cts/cli/regression.tools.exp
	+++ b/cts/cli/regression.tools.exp
	@@ -1,4883 +1,4885 @@
	Created new pacemaker configuration
	Setting up shadow instance
	A new shadow instance was created. To begin using it paste the following into your shell:
	CIB_shadow=cts-cli ; export CIB_shadow
	=#=#=#= Begin test: Validate CIB =#=#=#=
	<cib epoch="1" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config/>
	<nodes/>
	<resources/>
	<constraints/>
	</configuration>
	<status/>
	</cib>
	=#=#=#= Current cib after: Validate CIB =#=#=#=
	<cib epoch="1" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config/>
	<nodes/>
	<resources/>
	<constraints/>
	</configuration>
	<status/>
	</cib>
	=#=#=#= End test: Validate CIB - OK (0) =#=#=#=
	* Passed: cibadmin - Validate CIB
	=#=#=#= Begin test: Configure something before erasing =#=#=#=
	=#=#=#= Current cib after: Configure something before erasing =#=#=#=
	<cib epoch="2" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-cluster-delay" name="cluster-delay" value="60s"/>
	</cluster_property_set>
	</crm_config>
	<nodes/>
	<resources/>
	<constraints/>
	</configuration>
	<status/>
	</cib>
	=#=#=#= End test: Configure something before erasing - OK (0) =#=#=#=
	* Passed: crm_attribute - Configure something before erasing
	=#=#=#= Begin test: Require --force for CIB erasure =#=#=#=
	The supplied command is considered dangerous. To prevent accidental destruction of the cluster, the --force flag is required in order to proceed.
	=#=#=#= Current cib after: Require --force for CIB erasure =#=#=#=
	<cib epoch="2" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-cluster-delay" name="cluster-delay" value="60s"/>
	</cluster_property_set>
	</crm_config>
	<nodes/>
	<resources/>
	<constraints/>
	</configuration>
	<status/>
	</cib>
	=#=#=#= End test: Require --force for CIB erasure - Operation not safe (107) =#=#=#=
	* Passed: cibadmin - Require --force for CIB erasure
	=#=#=#= Begin test: Allow CIB erasure with --force =#=#=#=
	=#=#=#= End test: Allow CIB erasure with --force - OK (0) =#=#=#=
	* Passed: cibadmin - Allow CIB erasure with --force
	=#=#=#= Begin test: Query CIB =#=#=#=
	<cib epoch="1" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config/>
	<nodes/>
	<resources/>
	<constraints/>
	</configuration>
	<status/>
	</cib>
	=#=#=#= Current cib after: Query CIB =#=#=#=
	<cib epoch="1" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config/>
	<nodes/>
	<resources/>
	<constraints/>
	</configuration>
	<status/>
	</cib>
	=#=#=#= End test: Query CIB - OK (0) =#=#=#=
	* Passed: cibadmin - Query CIB
	=#=#=#= Begin test: Set cluster option =#=#=#=
	=#=#=#= Current cib after: Set cluster option =#=#=#=
	<cib epoch="2" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-cluster-delay" name="cluster-delay" value="60s"/>
	</cluster_property_set>
	</crm_config>
	<nodes/>
	<resources/>
	<constraints/>
	</configuration>
	<status/>
	</cib>
	=#=#=#= End test: Set cluster option - OK (0) =#=#=#=
	* Passed: crm_attribute - Set cluster option
	=#=#=#= Begin test: Query new cluster option =#=#=#=
	<nvpair id="cib-bootstrap-options-cluster-delay" name="cluster-delay" value="60s"/>
	=#=#=#= Current cib after: Query new cluster option =#=#=#=
	<cib epoch="2" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-cluster-delay" name="cluster-delay" value="60s"/>
	</cluster_property_set>
	</crm_config>
	<nodes/>
	<resources/>
	<constraints/>
	</configuration>
	<status/>
	</cib>
	=#=#=#= End test: Query new cluster option - OK (0) =#=#=#=
	* Passed: cibadmin - Query new cluster option
	=#=#=#= Begin test: Query cluster options =#=#=#=
	=#=#=#= Current cib after: Query cluster options =#=#=#=
	<cib epoch="2" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-cluster-delay" name="cluster-delay" value="60s"/>
	</cluster_property_set>
	</crm_config>
	<nodes/>
	<resources/>
	<constraints/>
	</configuration>
	<status/>
	</cib>
	=#=#=#= End test: Query cluster options - OK (0) =#=#=#=
	* Passed: cibadmin - Query cluster options
	=#=#=#= Begin test: Set no-quorum policy =#=#=#=
	=#=#=#= Current cib after: Set no-quorum policy =#=#=#=
	<cib epoch="3" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-cluster-delay" name="cluster-delay" value="60s"/>
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	</crm_config>
	<nodes/>
	<resources/>
	<constraints/>
	</configuration>
	<status/>
	</cib>
	=#=#=#= End test: Set no-quorum policy - OK (0) =#=#=#=
	* Passed: crm_attribute - Set no-quorum policy
	=#=#=#= Begin test: Delete nvpair =#=#=#=
	=#=#=#= Current cib after: Delete nvpair =#=#=#=
	<cib epoch="4" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	</crm_config>
	<nodes/>
	<resources/>
	<constraints/>
	</configuration>
	<status/>
	</cib>
	=#=#=#= End test: Delete nvpair - OK (0) =#=#=#=
	* Passed: cibadmin - Delete nvpair
	=#=#=#= Begin test: Create operation should fail =#=#=#=
	Call failed: File exists
	<failed>
	<failed_update id="cib-bootstrap-options" object_type="cluster_property_set" operation="cib_create" reason="File exists">
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-cluster-delay" name="cluster-delay" value="60s"/>
	</cluster_property_set>
	</failed_update>
	</failed>
	=#=#=#= Current cib after: Create operation should fail =#=#=#=
	<cib epoch="4" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	</crm_config>
	<nodes/>
	<resources/>
	<constraints/>
	</configuration>
	<status/>
	</cib>
	=#=#=#= End test: Create operation should fail - Requested item already exists (108) =#=#=#=
	* Passed: cibadmin - Create operation should fail
	=#=#=#= Begin test: Modify cluster options section =#=#=#=
	=#=#=#= Current cib after: Modify cluster options section =#=#=#=
	<cib epoch="5" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	<nvpair id="cib-bootstrap-options-cluster-delay" name="cluster-delay" value="60s"/>
	</cluster_property_set>
	</crm_config>
	<nodes/>
	<resources/>
	<constraints/>
	</configuration>
	<status/>
	</cib>
	=#=#=#= End test: Modify cluster options section - OK (0) =#=#=#=
	* Passed: cibadmin - Modify cluster options section
	=#=#=#= Begin test: Query updated cluster option =#=#=#=
	<nvpair id="cib-bootstrap-options-cluster-delay" name="cluster-delay" value="60s"/>
	=#=#=#= Current cib after: Query updated cluster option =#=#=#=
	<cib epoch="5" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	<nvpair id="cib-bootstrap-options-cluster-delay" name="cluster-delay" value="60s"/>
	</cluster_property_set>
	</crm_config>
	<nodes/>
	<resources/>
	<constraints/>
	</configuration>
	<status/>
	</cib>
	=#=#=#= End test: Query updated cluster option - OK (0) =#=#=#=
	* Passed: cibadmin - Query updated cluster option
	=#=#=#= Begin test: Set duplicate cluster option =#=#=#=
	=#=#=#= Current cib after: Set duplicate cluster option =#=#=#=
	<cib epoch="6" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	<nvpair id="cib-bootstrap-options-cluster-delay" name="cluster-delay" value="60s"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="40s"/>
	</cluster_property_set>
	</crm_config>
	<nodes/>
	<resources/>
	<constraints/>
	</configuration>
	<status/>
	</cib>
	=#=#=#= End test: Set duplicate cluster option - OK (0) =#=#=#=
	* Passed: crm_attribute - Set duplicate cluster option
	=#=#=#= Begin test: Setting multiply defined cluster option should fail =#=#=#=
	crm_attribute: Please choose from one of the matches below and supply the 'id' with --attr-id
	Multiple attributes match name=cluster-delay
	Value: 60s (id=cib-bootstrap-options-cluster-delay)
	Value: 40s (id=duplicate-cluster-delay)
	=#=#=#= Current cib after: Setting multiply defined cluster option should fail =#=#=#=
	<cib epoch="6" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	<nvpair id="cib-bootstrap-options-cluster-delay" name="cluster-delay" value="60s"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="40s"/>
	</cluster_property_set>
	</crm_config>
	<nodes/>
	<resources/>
	<constraints/>
	</configuration>
	<status/>
	</cib>
	=#=#=#= End test: Setting multiply defined cluster option should fail - Multiple items match request (109) =#=#=#=
	* Passed: crm_attribute - Setting multiply defined cluster option should fail
	=#=#=#= Begin test: Set cluster option with -s =#=#=#=
	=#=#=#= Current cib after: Set cluster option with -s =#=#=#=
	<cib epoch="7" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	<nvpair id="cib-bootstrap-options-cluster-delay" name="cluster-delay" value="60s"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes/>
	<resources/>
	<constraints/>
	</configuration>
	<status/>
	</cib>
	=#=#=#= End test: Set cluster option with -s - OK (0) =#=#=#=
	* Passed: crm_attribute - Set cluster option with -s
	=#=#=#= Begin test: Delete cluster option with -i =#=#=#=
	Deleted crm_config option: id=(null) name=cluster-delay
	=#=#=#= Current cib after: Delete cluster option with -i =#=#=#=
	<cib epoch="8" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes/>
	<resources/>
	<constraints/>
	</configuration>
	<status/>
	</cib>
	=#=#=#= End test: Delete cluster option with -i - OK (0) =#=#=#=
	* Passed: crm_attribute - Delete cluster option with -i
	=#=#=#= Begin test: Create node1 and bring it online =#=#=#=
	Current cluster status:
	* Full List of Resources:
	* No resources

	Performing Requested Modifications:
	* Bringing node node1 online

	Transition Summary:

	Executing Cluster Transition:

	Revised Cluster Status:
	* Node List:
	* Online: [ node1 ]

	* Full List of Resources:
	* No resources
	=#=#=#= Current cib after: Create node1 and bring it online =#=#=#=
	<cib epoch="9" num_updates="2" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1"/>
	</nodes>
	<resources/>
	<constraints/>
	</configuration>
	<status>
	<node_state id="node1" uname="node1" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate"/>
	</status>
	</cib>
	=#=#=#= End test: Create node1 and bring it online - OK (0) =#=#=#=
	* Passed: crm_simulate - Create node1 and bring it online
	=#=#=#= Begin test: Create node attribute =#=#=#=
	=#=#=#= Current cib after: Create node attribute =#=#=#=
	<cib epoch="10" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	</nodes>
	<resources/>
	<constraints/>
	</configuration>
	<status>
	<node_state id="node1" uname="node1" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate"/>
	</status>
	</cib>
	=#=#=#= End test: Create node attribute - OK (0) =#=#=#=
	* Passed: crm_attribute - Create node attribute
	=#=#=#= Begin test: Query new node attribute =#=#=#=
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	=#=#=#= Current cib after: Query new node attribute =#=#=#=
	<cib epoch="10" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	</nodes>
	<resources/>
	<constraints/>
	</configuration>
	<status>
	<node_state id="node1" uname="node1" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate"/>
	</status>
	</cib>
	=#=#=#= End test: Query new node attribute - OK (0) =#=#=#=
	* Passed: cibadmin - Query new node attribute
	=#=#=#= Begin test: Set a transient (fail-count) node attribute =#=#=#=
	=#=#=#= Current cib after: Set a transient (fail-count) node attribute =#=#=#=
	<cib epoch="10" num_updates="1" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	</nodes>
	<resources/>
	<constraints/>
	</configuration>
	<status>
	<node_state id="node1" uname="node1" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<transient_attributes id="node1">
	<instance_attributes id="status-node1">
	<nvpair id="status-node1-fail-count-foo" name="fail-count-foo" value="3"/>
	</instance_attributes>
	</transient_attributes>
	</node_state>
	</status>
	</cib>
	=#=#=#= End test: Set a transient (fail-count) node attribute - OK (0) =#=#=#=
	* Passed: crm_attribute - Set a transient (fail-count) node attribute
	=#=#=#= Begin test: Query a fail count =#=#=#=
	scope=status name=fail-count-foo value=3
	=#=#=#= Current cib after: Query a fail count =#=#=#=
	<cib epoch="10" num_updates="1" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	</nodes>
	<resources/>
	<constraints/>
	</configuration>
	<status>
	<node_state id="node1" uname="node1" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<transient_attributes id="node1">
	<instance_attributes id="status-node1">
	<nvpair id="status-node1-fail-count-foo" name="fail-count-foo" value="3"/>
	</instance_attributes>
	</transient_attributes>
	</node_state>
	</status>
	</cib>
	=#=#=#= End test: Query a fail count - OK (0) =#=#=#=
	* Passed: crm_failcount - Query a fail count
	=#=#=#= Begin test: Show node attributes with crm_simulate =#=#=#=
	Current cluster status:
	* Node List:
	* Online: [ node1 ]

	* Full List of Resources:
	* No resources

	* Node Attributes:
	* Node: node1:
	* ram : 1024M
	=#=#=#= End test: Show node attributes with crm_simulate - OK (0) =#=#=#=
	* Passed: crm_simulate - Show node attributes with crm_simulate
	=#=#=#= Begin test: Delete a transient (fail-count) node attribute =#=#=#=
	Deleted status attribute: id=status-node1-fail-count-foo name=fail-count-foo
	=#=#=#= Current cib after: Delete a transient (fail-count) node attribute =#=#=#=
	<cib epoch="10" num_updates="2" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	</nodes>
	<resources/>
	<constraints/>
	</configuration>
	<status>
	<node_state id="node1" uname="node1" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<transient_attributes id="node1">
	<instance_attributes id="status-node1"/>
	</transient_attributes>
	</node_state>
	</status>
	</cib>
	=#=#=#= End test: Delete a transient (fail-count) node attribute - OK (0) =#=#=#=
	* Passed: crm_attribute - Delete a transient (fail-count) node attribute
	=#=#=#= Begin test: Digest calculation =#=#=#=
	Digest: =#=#=#= Current cib after: Digest calculation =#=#=#=
	<cib epoch="10" num_updates="2" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	</nodes>
	<resources/>
	<constraints/>
	</configuration>
	<status>
	<node_state id="node1" uname="node1" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<transient_attributes id="node1">
	<instance_attributes id="status-node1"/>
	</transient_attributes>
	</node_state>
	</status>
	</cib>
	=#=#=#= End test: Digest calculation - OK (0) =#=#=#=
	* Passed: cibadmin - Digest calculation
	=#=#=#= Begin test: Replace operation should fail =#=#=#=
	Call failed: Update was older than existing configuration
	=#=#=#= Current cib after: Replace operation should fail =#=#=#=
	<cib epoch="10" num_updates="2" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	</nodes>
	<resources/>
	<constraints/>
	</configuration>
	<status>
	<node_state id="node1" uname="node1" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<transient_attributes id="node1">
	<instance_attributes id="status-node1"/>
	</transient_attributes>
	</node_state>
	</status>
	</cib>
	=#=#=#= End test: Replace operation should fail - Update was older than existing configuration (103) =#=#=#=
	* Passed: cibadmin - Replace operation should fail
	=#=#=#= Begin test: Default standby value =#=#=#=
	scope=status name=standby value=off
	=#=#=#= Current cib after: Default standby value =#=#=#=
	<cib epoch="10" num_updates="2" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	</nodes>
	<resources/>
	<constraints/>
	</configuration>
	<status>
	<node_state id="node1" uname="node1" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<transient_attributes id="node1">
	<instance_attributes id="status-node1"/>
	</transient_attributes>
	</node_state>
	</status>
	</cib>
	=#=#=#= End test: Default standby value - OK (0) =#=#=#=
	* Passed: crm_standby - Default standby value
	=#=#=#= Begin test: Set standby status =#=#=#=
	=#=#=#= Current cib after: Set standby status =#=#=#=
	<cib epoch="11" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	<nvpair id="nodes-node1-standby" name="standby" value="true"/>
	</instance_attributes>
	</node>
	</nodes>
	<resources/>
	<constraints/>
	</configuration>
	<status>
	<node_state id="node1" uname="node1" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<transient_attributes id="node1">
	<instance_attributes id="status-node1"/>
	</transient_attributes>
	</node_state>
	</status>
	</cib>
	=#=#=#= End test: Set standby status - OK (0) =#=#=#=
	* Passed: crm_standby - Set standby status
	=#=#=#= Begin test: Query standby value =#=#=#=
	scope=nodes name=standby value=true
	=#=#=#= Current cib after: Query standby value =#=#=#=
	<cib epoch="11" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	<nvpair id="nodes-node1-standby" name="standby" value="true"/>
	</instance_attributes>
	</node>
	</nodes>
	<resources/>
	<constraints/>
	</configuration>
	<status>
	<node_state id="node1" uname="node1" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<transient_attributes id="node1">
	<instance_attributes id="status-node1"/>
	</transient_attributes>
	</node_state>
	</status>
	</cib>
	=#=#=#= End test: Query standby value - OK (0) =#=#=#=
	* Passed: crm_standby - Query standby value
	=#=#=#= Begin test: Delete standby value =#=#=#=
	Deleted nodes attribute: id=nodes-node1-standby name=standby
	=#=#=#= Current cib after: Delete standby value =#=#=#=
	<cib epoch="12" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	</nodes>
	<resources/>
	<constraints/>
	</configuration>
	<status>
	<node_state id="node1" uname="node1" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<transient_attributes id="node1">
	<instance_attributes id="status-node1"/>
	</transient_attributes>
	</node_state>
	</status>
	</cib>
	=#=#=#= End test: Delete standby value - OK (0) =#=#=#=
	* Passed: crm_standby - Delete standby value
	=#=#=#= Begin test: Create a resource =#=#=#=
	=#=#=#= Current cib after: Create a resource =#=#=#=
	<cib epoch="13" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	</nodes>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy"/>
	</resources>
	<constraints/>
	</configuration>
	<status>
	<node_state id="node1" uname="node1" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<transient_attributes id="node1">
	<instance_attributes id="status-node1"/>
	</transient_attributes>
	</node_state>
	</status>
	</cib>
	=#=#=#= End test: Create a resource - OK (0) =#=#=#=
	* Passed: cibadmin - Create a resource
	=#=#=#= Begin test: Create a resource meta attribute =#=#=#=
	Set 'dummy' option: id=dummy-meta_attributes-is-managed set=dummy-meta_attributes name=is-managed value=false
	=#=#=#= Current cib after: Create a resource meta attribute =#=#=#=
	<cib epoch="14" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	</nodes>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes">
	<nvpair id="dummy-meta_attributes-is-managed" name="is-managed" value="false"/>
	</meta_attributes>
	</primitive>
	</resources>
	<constraints/>
	</configuration>
	<status>
	<node_state id="node1" uname="node1" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<transient_attributes id="node1">
	<instance_attributes id="status-node1"/>
	</transient_attributes>
	</node_state>
	</status>
	</cib>
	=#=#=#= End test: Create a resource meta attribute - OK (0) =#=#=#=
	* Passed: crm_resource - Create a resource meta attribute
	=#=#=#= Begin test: Query a resource meta attribute =#=#=#=
	false
	=#=#=#= Current cib after: Query a resource meta attribute =#=#=#=
	<cib epoch="14" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	</nodes>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes">
	<nvpair id="dummy-meta_attributes-is-managed" name="is-managed" value="false"/>
	</meta_attributes>
	</primitive>
	</resources>
	<constraints/>
	</configuration>
	<status>
	<node_state id="node1" uname="node1" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<transient_attributes id="node1">
	<instance_attributes id="status-node1"/>
	</transient_attributes>
	</node_state>
	</status>
	</cib>
	=#=#=#= End test: Query a resource meta attribute - OK (0) =#=#=#=
	* Passed: crm_resource - Query a resource meta attribute
	=#=#=#= Begin test: Remove a resource meta attribute =#=#=#=
	Deleted 'dummy' option: id=dummy-meta_attributes-is-managed name=is-managed
	=#=#=#= Current cib after: Remove a resource meta attribute =#=#=#=
	<cib epoch="15" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	</nodes>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes"/>
	</primitive>
	</resources>
	<constraints/>
	</configuration>
	<status>
	<node_state id="node1" uname="node1" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<transient_attributes id="node1">
	<instance_attributes id="status-node1"/>
	</transient_attributes>
	</node_state>
	</status>
	</cib>
	=#=#=#= End test: Remove a resource meta attribute - OK (0) =#=#=#=
	* Passed: crm_resource - Remove a resource meta attribute
	=#=#=#= Begin test: Create another resource meta attribute =#=#=#=
	<pacemaker-result api-version="X" request="crm_resource -r dummy --meta -p target-role -v Stopped --output-as=xml">
	<status code="0" message="OK"/>
	</pacemaker-result>
	=#=#=#= End test: Create another resource meta attribute - OK (0) =#=#=#=
	* Passed: crm_resource - Create another resource meta attribute
	=#=#=#= Begin test: Show why a resource is not running =#=#=#=
	Resource dummy is not running
	Configuration specifies 'dummy' should remain stopped
	=#=#=#= End test: Show why a resource is not running - OK (0) =#=#=#=
	* Passed: crm_resource - Show why a resource is not running
	=#=#=#= Begin test: Remove another resource meta attribute =#=#=#=
	<pacemaker-result api-version="X" request="crm_resource -r dummy --meta -d target-role --output-as=xml">
	<status code="0" message="OK"/>
	</pacemaker-result>
	=#=#=#= End test: Remove another resource meta attribute - OK (0) =#=#=#=
	* Passed: crm_resource - Remove another resource meta attribute
	=#=#=#= Begin test: Create a resource attribute =#=#=#=
	Set 'dummy' option: id=dummy-instance_attributes-delay set=dummy-instance_attributes name=delay value=10s
	=#=#=#= Current cib after: Create a resource attribute =#=#=#=
	<cib epoch="18" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	</nodes>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes"/>
	<instance_attributes id="dummy-instance_attributes">
	<nvpair id="dummy-instance_attributes-delay" name="delay" value="10s"/>
	</instance_attributes>
	</primitive>
	</resources>
	<constraints/>
	</configuration>
	<status>
	<node_state id="node1" uname="node1" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<transient_attributes id="node1">
	<instance_attributes id="status-node1"/>
	</transient_attributes>
	</node_state>
	</status>
	</cib>
	=#=#=#= End test: Create a resource attribute - OK (0) =#=#=#=
	* Passed: crm_resource - Create a resource attribute
	=#=#=#= Begin test: List the configured resources =#=#=#=
	Full List of Resources:
	* dummy (ocf:pacemaker:Dummy): Stopped
	=#=#=#= Current cib after: List the configured resources =#=#=#=
	<cib epoch="18" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	</nodes>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes"/>
	<instance_attributes id="dummy-instance_attributes">
	<nvpair id="dummy-instance_attributes-delay" name="delay" value="10s"/>
	</instance_attributes>
	</primitive>
	</resources>
	<constraints/>
	</configuration>
	<status>
	<node_state id="node1" uname="node1" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<transient_attributes id="node1">
	<instance_attributes id="status-node1"/>
	</transient_attributes>
	</node_state>
	</status>
	</cib>
	=#=#=#= End test: List the configured resources - OK (0) =#=#=#=
	* Passed: crm_resource - List the configured resources
	=#=#=#= Begin test: List the configured resources in XML =#=#=#=
	<pacemaker-result api-version="X" request="crm_resource -L --output-as=xml">
	<resources>
	<resource id="dummy" resource_agent="ocf:pacemaker:Dummy" role="Stopped" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
	</resources>
	<status code="0" message="OK"/>
	</pacemaker-result>
	=#=#=#= End test: List the configured resources in XML - OK (0) =#=#=#=
	* Passed: crm_resource - List the configured resources in XML
	=#=#=#= Begin test: List IDs of instantiated resources =#=#=#=
	dummy
	=#=#=#= End test: List IDs of instantiated resources - OK (0) =#=#=#=
	* Passed: crm_resource - List IDs of instantiated resources
	=#=#=#= Begin test: Show XML configuration of resource =#=#=#=
	dummy (ocf:pacemaker:Dummy): Stopped
	Resource XML:
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes"/>
	<instance_attributes id="dummy-instance_attributes">
	<nvpair id="dummy-instance_attributes-delay" name="delay" value="10s"/>
	</instance_attributes>
	</primitive>
	=#=#=#= End test: Show XML configuration of resource - OK (0) =#=#=#=
	* Passed: crm_resource - Show XML configuration of resource
	=#=#=#= Begin test: Require a destination when migrating a resource that is stopped =#=#=#=
	crm_resource: Resource 'dummy' not moved: active in 0 locations.
	To prevent 'dummy' from running on a specific location, specify a node.
	=#=#=#= Current cib after: Require a destination when migrating a resource that is stopped =#=#=#=
	<cib epoch="18" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	</nodes>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes"/>
	<instance_attributes id="dummy-instance_attributes">
	<nvpair id="dummy-instance_attributes-delay" name="delay" value="10s"/>
	</instance_attributes>
	</primitive>
	</resources>
	<constraints/>
	</configuration>
	<status>
	<node_state id="node1" uname="node1" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<transient_attributes id="node1">
	<instance_attributes id="status-node1"/>
	</transient_attributes>
	</node_state>
	</status>
	</cib>
	=#=#=#= End test: Require a destination when migrating a resource that is stopped - Incorrect usage (64) =#=#=#=
	* Passed: crm_resource - Require a destination when migrating a resource that is stopped
	=#=#=#= Begin test: Don't support migration to non-existent locations =#=#=#=
	-crm_resource: Error performing operation: No such object
	+crm_resource: Node 'i.do.not.exist' not found
	+Error performing operation: No such object
	=#=#=#= Current cib after: Don't support migration to non-existent locations =#=#=#=
	<cib epoch="18" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	</nodes>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes"/>
	<instance_attributes id="dummy-instance_attributes">
	<nvpair id="dummy-instance_attributes-delay" name="delay" value="10s"/>
	</instance_attributes>
	</primitive>
	</resources>
	<constraints/>
	</configuration>
	<status>
	<node_state id="node1" uname="node1" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<transient_attributes id="node1">
	<instance_attributes id="status-node1"/>
	</transient_attributes>
	</node_state>
	</status>
	</cib>
	=#=#=#= End test: Don't support migration to non-existent locations - No such object (105) =#=#=#=
	* Passed: crm_resource - Don't support migration to non-existent locations
	=#=#=#= Begin test: Create a fencing resource =#=#=#=
	=#=#=#= Current cib after: Create a fencing resource =#=#=#=
	<cib epoch="19" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	</nodes>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes"/>
	<instance_attributes id="dummy-instance_attributes">
	<nvpair id="dummy-instance_attributes-delay" name="delay" value="10s"/>
	</instance_attributes>
	</primitive>
	<primitive id="Fence" class="stonith" type="fence_true"/>
	</resources>
	<constraints/>
	</configuration>
	<status>
	<node_state id="node1" uname="node1" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<transient_attributes id="node1">
	<instance_attributes id="status-node1"/>
	</transient_attributes>
	</node_state>
	</status>
	</cib>
	=#=#=#= End test: Create a fencing resource - OK (0) =#=#=#=
	* Passed: cibadmin - Create a fencing resource
	=#=#=#= Begin test: Bring resources online =#=#=#=
	Current cluster status:
	* Node List:
	* Online: [ node1 ]

	* Full List of Resources:
	* dummy (ocf:pacemaker:Dummy): Stopped
	* Fence (stonith:fence_true): Stopped

	Transition Summary:
	* Start dummy ( node1 )
	* Start Fence ( node1 )

	Executing Cluster Transition:
	* Resource action: dummy monitor on node1
	* Resource action: Fence monitor on node1
	* Resource action: dummy start on node1
	* Resource action: Fence start on node1

	Revised Cluster Status:
	* Node List:
	* Online: [ node1 ]

	* Full List of Resources:
	* dummy (ocf:pacemaker:Dummy): Started node1
	* Fence (stonith:fence_true): Started node1
	=#=#=#= Current cib after: Bring resources online =#=#=#=
	<cib epoch="19" num_updates="4" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	</nodes>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes"/>
	<instance_attributes id="dummy-instance_attributes">
	<nvpair id="dummy-instance_attributes-delay" name="delay" value="10s"/>
	</instance_attributes>
	</primitive>
	<primitive id="Fence" class="stonith" type="fence_true"/>
	</resources>
	<constraints/>
	</configuration>
	<status>
	<node_state id="node1" uname="node1" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<transient_attributes id="node1">
	<instance_attributes id="status-node1"/>
	</transient_attributes>
	<lrm id="node1">
	<lrm_resources>
	<lrm_resource id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<lrm_rsc_op id="dummy_last_0" operation_key="dummy_start_0" operation="start" crm-debug-origin="crm_simulate" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="2" rc-code="0" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="0830891652dabe627ca72b8e879199b1"/>
	</lrm_resource>
	<lrm_resource id="Fence" class="stonith" type="fence_true">
	<lrm_rsc_op id="Fence_last_0" operation_key="Fence_start_0" operation="start" crm-debug-origin="crm_simulate" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="2" rc-code="0" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	</lrm_resource>
	</lrm_resources>
	</lrm>
	</node_state>
	</status>
	</cib>
	=#=#=#= End test: Bring resources online - OK (0) =#=#=#=
	* Passed: crm_simulate - Bring resources online
	=#=#=#= Begin test: Try to move a resource to its existing location =#=#=#=
	crm_resource: Error performing operation: Requested item already exists
	=#=#=#= Current cib after: Try to move a resource to its existing location =#=#=#=
	<cib epoch="19" num_updates="4" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	</nodes>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes"/>
	<instance_attributes id="dummy-instance_attributes">
	<nvpair id="dummy-instance_attributes-delay" name="delay" value="10s"/>
	</instance_attributes>
	</primitive>
	<primitive id="Fence" class="stonith" type="fence_true"/>
	</resources>
	<constraints/>
	</configuration>
	<status>
	<node_state id="node1" uname="node1" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<transient_attributes id="node1">
	<instance_attributes id="status-node1"/>
	</transient_attributes>
	<lrm id="node1">
	<lrm_resources>
	<lrm_resource id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<lrm_rsc_op id="dummy_last_0" operation_key="dummy_start_0" operation="start" crm-debug-origin="crm_simulate" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="2" rc-code="0" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="0830891652dabe627ca72b8e879199b1"/>
	</lrm_resource>
	<lrm_resource id="Fence" class="stonith" type="fence_true">
	<lrm_rsc_op id="Fence_last_0" operation_key="Fence_start_0" operation="start" crm-debug-origin="crm_simulate" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="2" rc-code="0" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	</lrm_resource>
	</lrm_resources>
	</lrm>
	</node_state>
	</status>
	</cib>
	=#=#=#= End test: Try to move a resource to its existing location - Requested item already exists (108) =#=#=#=
	* Passed: crm_resource - Try to move a resource to its existing location
	=#=#=#= Begin test: Move a resource from its existing location =#=#=#=
	WARNING: Creating rsc_location constraint 'cli-ban-dummy-on-node1' with a score of -INFINITY for resource dummy on node1.
	This will prevent dummy from running on node1 until the constraint is removed using the clear option or by editing the CIB with an appropriate tool
	This will be the case even if node1 is the last node in the cluster
	=#=#=#= Current cib after: Move a resource from its existing location =#=#=#=
	<cib epoch="20" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	</nodes>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes"/>
	<instance_attributes id="dummy-instance_attributes">
	<nvpair id="dummy-instance_attributes-delay" name="delay" value="10s"/>
	</instance_attributes>
	</primitive>
	<primitive id="Fence" class="stonith" type="fence_true"/>
	</resources>
	<constraints>
	<rsc_location id="cli-ban-dummy-on-node1" rsc="dummy" role="Started" node="node1" score="-INFINITY"/>
	</constraints>
	</configuration>
	<status>
	<node_state id="node1" uname="node1" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<transient_attributes id="node1">
	<instance_attributes id="status-node1"/>
	</transient_attributes>
	<lrm id="node1">
	<lrm_resources>
	<lrm_resource id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<lrm_rsc_op id="dummy_last_0" operation_key="dummy_start_0" operation="start" crm-debug-origin="crm_simulate" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="2" rc-code="0" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="0830891652dabe627ca72b8e879199b1"/>
	</lrm_resource>
	<lrm_resource id="Fence" class="stonith" type="fence_true">
	<lrm_rsc_op id="Fence_last_0" operation_key="Fence_start_0" operation="start" crm-debug-origin="crm_simulate" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="2" rc-code="0" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	</lrm_resource>
	</lrm_resources>
	</lrm>
	</node_state>
	</status>
	</cib>
	=#=#=#= End test: Move a resource from its existing location - OK (0) =#=#=#=
	* Passed: crm_resource - Move a resource from its existing location
	=#=#=#= Begin test: Clear out constraints generated by --move =#=#=#=
	Removing constraint: cli-ban-dummy-on-node1
	=#=#=#= Current cib after: Clear out constraints generated by --move =#=#=#=
	<cib epoch="21" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	</nodes>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes"/>
	<instance_attributes id="dummy-instance_attributes">
	<nvpair id="dummy-instance_attributes-delay" name="delay" value="10s"/>
	</instance_attributes>
	</primitive>
	<primitive id="Fence" class="stonith" type="fence_true"/>
	</resources>
	<constraints/>
	</configuration>
	<status>
	<node_state id="node1" uname="node1" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<transient_attributes id="node1">
	<instance_attributes id="status-node1"/>
	</transient_attributes>
	<lrm id="node1">
	<lrm_resources>
	<lrm_resource id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<lrm_rsc_op id="dummy_last_0" operation_key="dummy_start_0" operation="start" crm-debug-origin="crm_simulate" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="2" rc-code="0" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="0830891652dabe627ca72b8e879199b1"/>
	</lrm_resource>
	<lrm_resource id="Fence" class="stonith" type="fence_true">
	<lrm_rsc_op id="Fence_last_0" operation_key="Fence_start_0" operation="start" crm-debug-origin="crm_simulate" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="2" rc-code="0" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	</lrm_resource>
	</lrm_resources>
	</lrm>
	</node_state>
	</status>
	</cib>
	=#=#=#= End test: Clear out constraints generated by --move - OK (0) =#=#=#=
	* Passed: crm_resource - Clear out constraints generated by --move
	=#=#=#= Begin test: Default ticket granted state =#=#=#=
	false
	=#=#=#= Current cib after: Default ticket granted state =#=#=#=
	<cib epoch="21" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	</nodes>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes"/>
	<instance_attributes id="dummy-instance_attributes">
	<nvpair id="dummy-instance_attributes-delay" name="delay" value="10s"/>
	</instance_attributes>
	</primitive>
	<primitive id="Fence" class="stonith" type="fence_true"/>
	</resources>
	<constraints/>
	</configuration>
	<status>
	<node_state id="node1" uname="node1" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<transient_attributes id="node1">
	<instance_attributes id="status-node1"/>
	</transient_attributes>
	<lrm id="node1">
	<lrm_resources>
	<lrm_resource id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<lrm_rsc_op id="dummy_last_0" operation_key="dummy_start_0" operation="start" crm-debug-origin="crm_simulate" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="2" rc-code="0" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="0830891652dabe627ca72b8e879199b1"/>
	</lrm_resource>
	<lrm_resource id="Fence" class="stonith" type="fence_true">
	<lrm_rsc_op id="Fence_last_0" operation_key="Fence_start_0" operation="start" crm-debug-origin="crm_simulate" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="2" rc-code="0" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	</lrm_resource>
	</lrm_resources>
	</lrm>
	</node_state>
	</status>
	</cib>
	=#=#=#= End test: Default ticket granted state - OK (0) =#=#=#=
	* Passed: crm_ticket - Default ticket granted state
	=#=#=#= Begin test: Set ticket granted state =#=#=#=
	=#=#=#= Current cib after: Set ticket granted state =#=#=#=
	<cib epoch="21" num_updates="1" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	</nodes>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes"/>
	<instance_attributes id="dummy-instance_attributes">
	<nvpair id="dummy-instance_attributes-delay" name="delay" value="10s"/>
	</instance_attributes>
	</primitive>
	<primitive id="Fence" class="stonith" type="fence_true"/>
	</resources>
	<constraints/>
	</configuration>
	<status>
	<node_state id="node1" uname="node1" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<transient_attributes id="node1">
	<instance_attributes id="status-node1"/>
	</transient_attributes>
	<lrm id="node1">
	<lrm_resources>
	<lrm_resource id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<lrm_rsc_op id="dummy_last_0" operation_key="dummy_start_0" operation="start" crm-debug-origin="crm_simulate" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="2" rc-code="0" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="0830891652dabe627ca72b8e879199b1"/>
	</lrm_resource>
	<lrm_resource id="Fence" class="stonith" type="fence_true">
	<lrm_rsc_op id="Fence_last_0" operation_key="Fence_start_0" operation="start" crm-debug-origin="crm_simulate" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="2" rc-code="0" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	</lrm_resource>
	</lrm_resources>
	</lrm>
	</node_state>
	<tickets>
	<ticket_state id="ticketA" granted="false"/>
	</tickets>
	</status>
	</cib>
	=#=#=#= End test: Set ticket granted state - OK (0) =#=#=#=
	* Passed: crm_ticket - Set ticket granted state
	=#=#=#= Begin test: Query ticket granted state =#=#=#=
	false
	=#=#=#= Current cib after: Query ticket granted state =#=#=#=
	<cib epoch="21" num_updates="1" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	</nodes>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes"/>
	<instance_attributes id="dummy-instance_attributes">
	<nvpair id="dummy-instance_attributes-delay" name="delay" value="10s"/>
	</instance_attributes>
	</primitive>
	<primitive id="Fence" class="stonith" type="fence_true"/>
	</resources>
	<constraints/>
	</configuration>
	<status>
	<node_state id="node1" uname="node1" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<transient_attributes id="node1">
	<instance_attributes id="status-node1"/>
	</transient_attributes>
	<lrm id="node1">
	<lrm_resources>
	<lrm_resource id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<lrm_rsc_op id="dummy_last_0" operation_key="dummy_start_0" operation="start" crm-debug-origin="crm_simulate" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="2" rc-code="0" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="0830891652dabe627ca72b8e879199b1"/>
	</lrm_resource>
	<lrm_resource id="Fence" class="stonith" type="fence_true">
	<lrm_rsc_op id="Fence_last_0" operation_key="Fence_start_0" operation="start" crm-debug-origin="crm_simulate" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="2" rc-code="0" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	</lrm_resource>
	</lrm_resources>
	</lrm>
	</node_state>
	<tickets>
	<ticket_state id="ticketA" granted="false"/>
	</tickets>
	</status>
	</cib>
	=#=#=#= End test: Query ticket granted state - OK (0) =#=#=#=
	* Passed: crm_ticket - Query ticket granted state
	=#=#=#= Begin test: Delete ticket granted state =#=#=#=
	=#=#=#= Current cib after: Delete ticket granted state =#=#=#=
	<cib epoch="21" num_updates="2" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	</nodes>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes"/>
	<instance_attributes id="dummy-instance_attributes">
	<nvpair id="dummy-instance_attributes-delay" name="delay" value="10s"/>
	</instance_attributes>
	</primitive>
	<primitive id="Fence" class="stonith" type="fence_true"/>
	</resources>
	<constraints/>
	</configuration>
	<status>
	<node_state id="node1" uname="node1" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<transient_attributes id="node1">
	<instance_attributes id="status-node1"/>
	</transient_attributes>
	<lrm id="node1">
	<lrm_resources>
	<lrm_resource id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<lrm_rsc_op id="dummy_last_0" operation_key="dummy_start_0" operation="start" crm-debug-origin="crm_simulate" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="2" rc-code="0" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="0830891652dabe627ca72b8e879199b1"/>
	</lrm_resource>
	<lrm_resource id="Fence" class="stonith" type="fence_true">
	<lrm_rsc_op id="Fence_last_0" operation_key="Fence_start_0" operation="start" crm-debug-origin="crm_simulate" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="2" rc-code="0" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	</lrm_resource>
	</lrm_resources>
	</lrm>
	</node_state>
	<tickets>
	<ticket_state id="ticketA"/>
	</tickets>
	</status>
	</cib>
	=#=#=#= End test: Delete ticket granted state - OK (0) =#=#=#=
	* Passed: crm_ticket - Delete ticket granted state
	=#=#=#= Begin test: Make a ticket standby =#=#=#=
	=#=#=#= Current cib after: Make a ticket standby =#=#=#=
	<cib epoch="21" num_updates="3" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	</nodes>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes"/>
	<instance_attributes id="dummy-instance_attributes">
	<nvpair id="dummy-instance_attributes-delay" name="delay" value="10s"/>
	</instance_attributes>
	</primitive>
	<primitive id="Fence" class="stonith" type="fence_true"/>
	</resources>
	<constraints/>
	</configuration>
	<status>
	<node_state id="node1" uname="node1" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<transient_attributes id="node1">
	<instance_attributes id="status-node1"/>
	</transient_attributes>
	<lrm id="node1">
	<lrm_resources>
	<lrm_resource id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<lrm_rsc_op id="dummy_last_0" operation_key="dummy_start_0" operation="start" crm-debug-origin="crm_simulate" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="2" rc-code="0" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="0830891652dabe627ca72b8e879199b1"/>
	</lrm_resource>
	<lrm_resource id="Fence" class="stonith" type="fence_true">
	<lrm_rsc_op id="Fence_last_0" operation_key="Fence_start_0" operation="start" crm-debug-origin="crm_simulate" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="2" rc-code="0" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	</lrm_resource>
	</lrm_resources>
	</lrm>
	</node_state>
	<tickets>
	<ticket_state id="ticketA" standby="true"/>
	</tickets>
	</status>
	</cib>
	=#=#=#= End test: Make a ticket standby - OK (0) =#=#=#=
	* Passed: crm_ticket - Make a ticket standby
	=#=#=#= Begin test: Query ticket standby state =#=#=#=
	true
	=#=#=#= Current cib after: Query ticket standby state =#=#=#=
	<cib epoch="21" num_updates="3" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	</nodes>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes"/>
	<instance_attributes id="dummy-instance_attributes">
	<nvpair id="dummy-instance_attributes-delay" name="delay" value="10s"/>
	</instance_attributes>
	</primitive>
	<primitive id="Fence" class="stonith" type="fence_true"/>
	</resources>
	<constraints/>
	</configuration>
	<status>
	<node_state id="node1" uname="node1" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<transient_attributes id="node1">
	<instance_attributes id="status-node1"/>
	</transient_attributes>
	<lrm id="node1">
	<lrm_resources>
	<lrm_resource id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<lrm_rsc_op id="dummy_last_0" operation_key="dummy_start_0" operation="start" crm-debug-origin="crm_simulate" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="2" rc-code="0" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="0830891652dabe627ca72b8e879199b1"/>
	</lrm_resource>
	<lrm_resource id="Fence" class="stonith" type="fence_true">
	<lrm_rsc_op id="Fence_last_0" operation_key="Fence_start_0" operation="start" crm-debug-origin="crm_simulate" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="2" rc-code="0" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	</lrm_resource>
	</lrm_resources>
	</lrm>
	</node_state>
	<tickets>
	<ticket_state id="ticketA" standby="true"/>
	</tickets>
	</status>
	</cib>
	=#=#=#= End test: Query ticket standby state - OK (0) =#=#=#=
	* Passed: crm_ticket - Query ticket standby state
	=#=#=#= Begin test: Activate a ticket =#=#=#=
	=#=#=#= Current cib after: Activate a ticket =#=#=#=
	<cib epoch="21" num_updates="4" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	</nodes>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes"/>
	<instance_attributes id="dummy-instance_attributes">
	<nvpair id="dummy-instance_attributes-delay" name="delay" value="10s"/>
	</instance_attributes>
	</primitive>
	<primitive id="Fence" class="stonith" type="fence_true"/>
	</resources>
	<constraints/>
	</configuration>
	<status>
	<node_state id="node1" uname="node1" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<transient_attributes id="node1">
	<instance_attributes id="status-node1"/>
	</transient_attributes>
	<lrm id="node1">
	<lrm_resources>
	<lrm_resource id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<lrm_rsc_op id="dummy_last_0" operation_key="dummy_start_0" operation="start" crm-debug-origin="crm_simulate" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="2" rc-code="0" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="0830891652dabe627ca72b8e879199b1"/>
	</lrm_resource>
	<lrm_resource id="Fence" class="stonith" type="fence_true">
	<lrm_rsc_op id="Fence_last_0" operation_key="Fence_start_0" operation="start" crm-debug-origin="crm_simulate" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="2" rc-code="0" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	</lrm_resource>
	</lrm_resources>
	</lrm>
	</node_state>
	<tickets>
	<ticket_state id="ticketA" standby="false"/>
	</tickets>
	</status>
	</cib>
	=#=#=#= End test: Activate a ticket - OK (0) =#=#=#=
	* Passed: crm_ticket - Activate a ticket
	=#=#=#= Begin test: Delete ticket standby state =#=#=#=
	=#=#=#= Current cib after: Delete ticket standby state =#=#=#=
	<cib epoch="21" num_updates="5" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	</nodes>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes"/>
	<instance_attributes id="dummy-instance_attributes">
	<nvpair id="dummy-instance_attributes-delay" name="delay" value="10s"/>
	</instance_attributes>
	</primitive>
	<primitive id="Fence" class="stonith" type="fence_true"/>
	</resources>
	<constraints/>
	</configuration>
	<status>
	<node_state id="node1" uname="node1" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<transient_attributes id="node1">
	<instance_attributes id="status-node1"/>
	</transient_attributes>
	<lrm id="node1">
	<lrm_resources>
	<lrm_resource id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<lrm_rsc_op id="dummy_last_0" operation_key="dummy_start_0" operation="start" crm-debug-origin="crm_simulate" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="2" rc-code="0" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="0830891652dabe627ca72b8e879199b1"/>
	</lrm_resource>
	<lrm_resource id="Fence" class="stonith" type="fence_true">
	<lrm_rsc_op id="Fence_last_0" operation_key="Fence_start_0" operation="start" crm-debug-origin="crm_simulate" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="2" rc-code="0" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	</lrm_resource>
	</lrm_resources>
	</lrm>
	</node_state>
	<tickets>
	<ticket_state id="ticketA"/>
	</tickets>
	</status>
	</cib>
	=#=#=#= End test: Delete ticket standby state - OK (0) =#=#=#=
	* Passed: crm_ticket - Delete ticket standby state
	=#=#=#= Begin test: Ban a resource on unknown node =#=#=#=
	-crm_resource: Error performing operation: No such object
	+crm_resource: Node 'host1' not found
	+Error performing operation: No such object
	=#=#=#= Current cib after: Ban a resource on unknown node =#=#=#=
	<cib epoch="21" num_updates="5" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	</nodes>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes"/>
	<instance_attributes id="dummy-instance_attributes">
	<nvpair id="dummy-instance_attributes-delay" name="delay" value="10s"/>
	</instance_attributes>
	</primitive>
	<primitive id="Fence" class="stonith" type="fence_true"/>
	</resources>
	<constraints/>
	</configuration>
	<status>
	<node_state id="node1" uname="node1" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<transient_attributes id="node1">
	<instance_attributes id="status-node1"/>
	</transient_attributes>
	<lrm id="node1">
	<lrm_resources>
	<lrm_resource id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<lrm_rsc_op id="dummy_last_0" operation_key="dummy_start_0" operation="start" crm-debug-origin="crm_simulate" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="2" rc-code="0" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="0830891652dabe627ca72b8e879199b1"/>
	</lrm_resource>
	<lrm_resource id="Fence" class="stonith" type="fence_true">
	<lrm_rsc_op id="Fence_last_0" operation_key="Fence_start_0" operation="start" crm-debug-origin="crm_simulate" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="2" rc-code="0" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	</lrm_resource>
	</lrm_resources>
	</lrm>
	</node_state>
	<tickets>
	<ticket_state id="ticketA"/>
	</tickets>
	</status>
	</cib>
	=#=#=#= End test: Ban a resource on unknown node - No such object (105) =#=#=#=
	* Passed: crm_resource - Ban a resource on unknown node
	=#=#=#= Begin test: Create two more nodes and bring them online =#=#=#=
	Current cluster status:
	* Node List:
	* Online: [ node1 ]

	* Full List of Resources:
	* dummy (ocf:pacemaker:Dummy): Started node1
	* Fence (stonith:fence_true): Started node1

	Performing Requested Modifications:
	* Bringing node node2 online
	* Bringing node node3 online

	Transition Summary:
	* Move Fence ( node1 -> node2 )

	Executing Cluster Transition:
	* Resource action: dummy monitor on node3
	* Resource action: dummy monitor on node2
	* Resource action: Fence stop on node1
	* Resource action: Fence monitor on node3
	* Resource action: Fence monitor on node2
	* Resource action: Fence start on node2

	Revised Cluster Status:
	* Node List:
	* Online: [ node1 node2 node3 ]

	* Full List of Resources:
	* dummy (ocf:pacemaker:Dummy): Started node1
	* Fence (stonith:fence_true): Started node2
	=#=#=#= Current cib after: Create two more nodes and bring them online =#=#=#=
	<cib epoch="23" num_updates="8" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	<node id="node2" uname="node2"/>
	<node id="node3" uname="node3"/>
	</nodes>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes"/>
	<instance_attributes id="dummy-instance_attributes">
	<nvpair id="dummy-instance_attributes-delay" name="delay" value="10s"/>
	</instance_attributes>
	</primitive>
	<primitive id="Fence" class="stonith" type="fence_true"/>
	</resources>
	<constraints/>
	</configuration>
	<status>
	<node_state id="node1" uname="node1" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<transient_attributes id="node1">
	<instance_attributes id="status-node1"/>
	</transient_attributes>
	<lrm id="node1">
	<lrm_resources>
	<lrm_resource id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<lrm_rsc_op id="dummy_last_0" operation_key="dummy_start_0" operation="start" crm-debug-origin="crm_simulate" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="2" rc-code="0" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="0830891652dabe627ca72b8e879199b1"/>
	</lrm_resource>
	<lrm_resource id="Fence" class="stonith" type="fence_true">
	<lrm_rsc_op id="Fence_last_0" operation_key="Fence_stop_0" operation="stop" crm-debug-origin="crm_simulate" transition-key="3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="3" rc-code="0" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	</lrm_resource>
	</lrm_resources>
	</lrm>
	</node_state>
	<tickets>
	<ticket_state id="ticketA"/>
	</tickets>
	<node_state id="node2" uname="node2" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<lrm id="node2">
	<lrm_resources>
	<lrm_resource id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<lrm_rsc_op id="dummy_last_0" operation_key="dummy_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" transition-key="1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:7;1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="1" rc-code="7" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="0830891652dabe627ca72b8e879199b1"/>
	</lrm_resource>
	<lrm_resource id="Fence" class="stonith" type="fence_true">
	<lrm_rsc_op id="Fence_last_0" operation_key="Fence_start_0" operation="start" crm-debug-origin="crm_simulate" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="2" rc-code="0" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	</lrm_resource>
	</lrm_resources>
	</lrm>
	</node_state>
	<node_state id="node3" uname="node3" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<lrm id="node3">
	<lrm_resources>
	<lrm_resource id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<lrm_rsc_op id="dummy_last_0" operation_key="dummy_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" transition-key="1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:7;1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="1" rc-code="7" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="0830891652dabe627ca72b8e879199b1"/>
	</lrm_resource>
	<lrm_resource id="Fence" class="stonith" type="fence_true">
	<lrm_rsc_op id="Fence_last_0" operation_key="Fence_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" transition-key="1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:7;1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="1" rc-code="7" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	</lrm_resource>
	</lrm_resources>
	</lrm>
	</node_state>
	</status>
	</cib>
	=#=#=#= End test: Create two more nodes and bring them online - OK (0) =#=#=#=
	* Passed: crm_simulate - Create two more nodes and bring them online
	=#=#=#= Begin test: Ban dummy from node1 =#=#=#=
	WARNING: Creating rsc_location constraint 'cli-ban-dummy-on-node1' with a score of -INFINITY for resource dummy on node1.
	This will prevent dummy from running on node1 until the constraint is removed using the clear option or by editing the CIB with an appropriate tool
	This will be the case even if node1 is the last node in the cluster
	=#=#=#= Current cib after: Ban dummy from node1 =#=#=#=
	<cib epoch="24" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	<node id="node2" uname="node2"/>
	<node id="node3" uname="node3"/>
	</nodes>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes"/>
	<instance_attributes id="dummy-instance_attributes">
	<nvpair id="dummy-instance_attributes-delay" name="delay" value="10s"/>
	</instance_attributes>
	</primitive>
	<primitive id="Fence" class="stonith" type="fence_true"/>
	</resources>
	<constraints>
	<rsc_location id="cli-ban-dummy-on-node1" rsc="dummy" role="Started" node="node1" score="-INFINITY"/>
	</constraints>
	</configuration>
	<status>
	<node_state id="node1" uname="node1" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<transient_attributes id="node1">
	<instance_attributes id="status-node1"/>
	</transient_attributes>
	<lrm id="node1">
	<lrm_resources>
	<lrm_resource id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<lrm_rsc_op id="dummy_last_0" operation_key="dummy_start_0" operation="start" crm-debug-origin="crm_simulate" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="2" rc-code="0" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="0830891652dabe627ca72b8e879199b1"/>
	</lrm_resource>
	<lrm_resource id="Fence" class="stonith" type="fence_true">
	<lrm_rsc_op id="Fence_last_0" operation_key="Fence_stop_0" operation="stop" crm-debug-origin="crm_simulate" transition-key="3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="3" rc-code="0" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	</lrm_resource>
	</lrm_resources>
	</lrm>
	</node_state>
	<tickets>
	<ticket_state id="ticketA"/>
	</tickets>
	<node_state id="node2" uname="node2" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<lrm id="node2">
	<lrm_resources>
	<lrm_resource id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<lrm_rsc_op id="dummy_last_0" operation_key="dummy_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" transition-key="1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:7;1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="1" rc-code="7" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="0830891652dabe627ca72b8e879199b1"/>
	</lrm_resource>
	<lrm_resource id="Fence" class="stonith" type="fence_true">
	<lrm_rsc_op id="Fence_last_0" operation_key="Fence_start_0" operation="start" crm-debug-origin="crm_simulate" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="2" rc-code="0" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	</lrm_resource>
	</lrm_resources>
	</lrm>
	</node_state>
	<node_state id="node3" uname="node3" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<lrm id="node3">
	<lrm_resources>
	<lrm_resource id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<lrm_rsc_op id="dummy_last_0" operation_key="dummy_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" transition-key="1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:7;1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="1" rc-code="7" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="0830891652dabe627ca72b8e879199b1"/>
	</lrm_resource>
	<lrm_resource id="Fence" class="stonith" type="fence_true">
	<lrm_rsc_op id="Fence_last_0" operation_key="Fence_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" transition-key="1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:7;1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="1" rc-code="7" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	</lrm_resource>
	</lrm_resources>
	</lrm>
	</node_state>
	</status>
	</cib>
	=#=#=#= End test: Ban dummy from node1 - OK (0) =#=#=#=
	* Passed: crm_resource - Ban dummy from node1
	=#=#=#= Begin test: Show where a resource is running =#=#=#=
	resource dummy is running on: node1
	=#=#=#= End test: Show where a resource is running - OK (0) =#=#=#=
	* Passed: crm_resource - Show where a resource is running
	=#=#=#= Begin test: Show constraints on a resource =#=#=#=
	Locations:
	* Node node1 (score=-INFINITY, id=cli-ban-dummy-on-node1, rsc=dummy)
	=#=#=#= End test: Show constraints on a resource - OK (0) =#=#=#=
	* Passed: crm_resource - Show constraints on a resource
	=#=#=#= Begin test: Ban dummy from node2 =#=#=#=
	<pacemaker-result api-version="X" request="crm_resource -r dummy -B -N node2 --output-as=xml">
	<status code="0" message="OK"/>
	</pacemaker-result>
	=#=#=#= Current cib after: Ban dummy from node2 =#=#=#=
	<cib epoch="25" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	<node id="node2" uname="node2"/>
	<node id="node3" uname="node3"/>
	</nodes>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes"/>
	<instance_attributes id="dummy-instance_attributes">
	<nvpair id="dummy-instance_attributes-delay" name="delay" value="10s"/>
	</instance_attributes>
	</primitive>
	<primitive id="Fence" class="stonith" type="fence_true"/>
	</resources>
	<constraints>
	<rsc_location id="cli-ban-dummy-on-node1" rsc="dummy" role="Started" node="node1" score="-INFINITY"/>
	<rsc_location id="cli-ban-dummy-on-node2" rsc="dummy" role="Started" node="node2" score="-INFINITY"/>
	</constraints>
	</configuration>
	<status>
	<node_state id="node1" uname="node1" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<transient_attributes id="node1">
	<instance_attributes id="status-node1"/>
	</transient_attributes>
	<lrm id="node1">
	<lrm_resources>
	<lrm_resource id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<lrm_rsc_op id="dummy_last_0" operation_key="dummy_start_0" operation="start" crm-debug-origin="crm_simulate" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="2" rc-code="0" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="0830891652dabe627ca72b8e879199b1"/>
	</lrm_resource>
	<lrm_resource id="Fence" class="stonith" type="fence_true">
	<lrm_rsc_op id="Fence_last_0" operation_key="Fence_stop_0" operation="stop" crm-debug-origin="crm_simulate" transition-key="3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="3" rc-code="0" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	</lrm_resource>
	</lrm_resources>
	</lrm>
	</node_state>
	<tickets>
	<ticket_state id="ticketA"/>
	</tickets>
	<node_state id="node2" uname="node2" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<lrm id="node2">
	<lrm_resources>
	<lrm_resource id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<lrm_rsc_op id="dummy_last_0" operation_key="dummy_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" transition-key="1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:7;1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="1" rc-code="7" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="0830891652dabe627ca72b8e879199b1"/>
	</lrm_resource>
	<lrm_resource id="Fence" class="stonith" type="fence_true">
	<lrm_rsc_op id="Fence_last_0" operation_key="Fence_start_0" operation="start" crm-debug-origin="crm_simulate" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="2" rc-code="0" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	</lrm_resource>
	</lrm_resources>
	</lrm>
	</node_state>
	<node_state id="node3" uname="node3" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<lrm id="node3">
	<lrm_resources>
	<lrm_resource id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<lrm_rsc_op id="dummy_last_0" operation_key="dummy_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" transition-key="1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:7;1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="1" rc-code="7" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="0830891652dabe627ca72b8e879199b1"/>
	</lrm_resource>
	<lrm_resource id="Fence" class="stonith" type="fence_true">
	<lrm_rsc_op id="Fence_last_0" operation_key="Fence_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" transition-key="1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:7;1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="1" rc-code="7" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	</lrm_resource>
	</lrm_resources>
	</lrm>
	</node_state>
	</status>
	</cib>
	=#=#=#= End test: Ban dummy from node2 - OK (0) =#=#=#=
	* Passed: crm_resource - Ban dummy from node2
	=#=#=#= Begin test: Relocate resources due to ban =#=#=#=
	Current cluster status:
	* Node List:
	* Online: [ node1 node2 node3 ]

	* Full List of Resources:
	* dummy (ocf:pacemaker:Dummy): Started node1
	* Fence (stonith:fence_true): Started node2

	Transition Summary:
	* Move dummy ( node1 -> node3 )

	Executing Cluster Transition:
	* Resource action: dummy stop on node1
	* Resource action: dummy start on node3

	Revised Cluster Status:
	* Node List:
	* Online: [ node1 node2 node3 ]

	* Full List of Resources:
	* dummy (ocf:pacemaker:Dummy): Started node3
	* Fence (stonith:fence_true): Started node2
	=#=#=#= Current cib after: Relocate resources due to ban =#=#=#=
	<cib epoch="25" num_updates="2" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	<node id="node2" uname="node2"/>
	<node id="node3" uname="node3"/>
	</nodes>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes"/>
	<instance_attributes id="dummy-instance_attributes">
	<nvpair id="dummy-instance_attributes-delay" name="delay" value="10s"/>
	</instance_attributes>
	</primitive>
	<primitive id="Fence" class="stonith" type="fence_true"/>
	</resources>
	<constraints>
	<rsc_location id="cli-ban-dummy-on-node1" rsc="dummy" role="Started" node="node1" score="-INFINITY"/>
	<rsc_location id="cli-ban-dummy-on-node2" rsc="dummy" role="Started" node="node2" score="-INFINITY"/>
	</constraints>
	</configuration>
	<status>
	<node_state id="node1" uname="node1" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<transient_attributes id="node1">
	<instance_attributes id="status-node1"/>
	</transient_attributes>
	<lrm id="node1">
	<lrm_resources>
	<lrm_resource id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<lrm_rsc_op id="dummy_last_0" operation_key="dummy_stop_0" operation="stop" crm-debug-origin="crm_simulate" transition-key="3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="3" rc-code="0" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="0830891652dabe627ca72b8e879199b1"/>
	</lrm_resource>
	<lrm_resource id="Fence" class="stonith" type="fence_true">
	<lrm_rsc_op id="Fence_last_0" operation_key="Fence_stop_0" operation="stop" crm-debug-origin="crm_simulate" transition-key="3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="3" rc-code="0" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	</lrm_resource>
	</lrm_resources>
	</lrm>
	</node_state>
	<tickets>
	<ticket_state id="ticketA"/>
	</tickets>
	<node_state id="node2" uname="node2" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<lrm id="node2">
	<lrm_resources>
	<lrm_resource id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<lrm_rsc_op id="dummy_last_0" operation_key="dummy_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" transition-key="1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:7;1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="1" rc-code="7" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="0830891652dabe627ca72b8e879199b1"/>
	</lrm_resource>
	<lrm_resource id="Fence" class="stonith" type="fence_true">
	<lrm_rsc_op id="Fence_last_0" operation_key="Fence_start_0" operation="start" crm-debug-origin="crm_simulate" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="2" rc-code="0" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	</lrm_resource>
	</lrm_resources>
	</lrm>
	</node_state>
	<node_state id="node3" uname="node3" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<lrm id="node3">
	<lrm_resources>
	<lrm_resource id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<lrm_rsc_op id="dummy_last_0" operation_key="dummy_start_0" operation="start" crm-debug-origin="crm_simulate" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="2" rc-code="0" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="0830891652dabe627ca72b8e879199b1"/>
	</lrm_resource>
	<lrm_resource id="Fence" class="stonith" type="fence_true">
	<lrm_rsc_op id="Fence_last_0" operation_key="Fence_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" transition-key="1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:7;1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="1" rc-code="7" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	</lrm_resource>
	</lrm_resources>
	</lrm>
	</node_state>
	</status>
	</cib>
	=#=#=#= End test: Relocate resources due to ban - OK (0) =#=#=#=
	* Passed: crm_simulate - Relocate resources due to ban
	=#=#=#= Begin test: Move dummy to node1 =#=#=#=
	<pacemaker-result api-version="X" request="crm_resource -r dummy -M -N node1 --output-as=xml">
	<status code="0" message="OK"/>
	</pacemaker-result>
	=#=#=#= Current cib after: Move dummy to node1 =#=#=#=
	<cib epoch="27" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	<node id="node2" uname="node2"/>
	<node id="node3" uname="node3"/>
	</nodes>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes"/>
	<instance_attributes id="dummy-instance_attributes">
	<nvpair id="dummy-instance_attributes-delay" name="delay" value="10s"/>
	</instance_attributes>
	</primitive>
	<primitive id="Fence" class="stonith" type="fence_true"/>
	</resources>
	<constraints>
	<rsc_location id="cli-ban-dummy-on-node2" rsc="dummy" role="Started" node="node2" score="-INFINITY"/>
	<rsc_location id="cli-prefer-dummy" rsc="dummy" role="Started" node="node1" score="INFINITY"/>
	</constraints>
	</configuration>
	<status>
	<node_state id="node1" uname="node1" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<transient_attributes id="node1">
	<instance_attributes id="status-node1"/>
	</transient_attributes>
	<lrm id="node1">
	<lrm_resources>
	<lrm_resource id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<lrm_rsc_op id="dummy_last_0" operation_key="dummy_stop_0" operation="stop" crm-debug-origin="crm_simulate" transition-key="3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="3" rc-code="0" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="0830891652dabe627ca72b8e879199b1"/>
	</lrm_resource>
	<lrm_resource id="Fence" class="stonith" type="fence_true">
	<lrm_rsc_op id="Fence_last_0" operation_key="Fence_stop_0" operation="stop" crm-debug-origin="crm_simulate" transition-key="3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="3" rc-code="0" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	</lrm_resource>
	</lrm_resources>
	</lrm>
	</node_state>
	<tickets>
	<ticket_state id="ticketA"/>
	</tickets>
	<node_state id="node2" uname="node2" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<lrm id="node2">
	<lrm_resources>
	<lrm_resource id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<lrm_rsc_op id="dummy_last_0" operation_key="dummy_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" transition-key="1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:7;1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="1" rc-code="7" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="0830891652dabe627ca72b8e879199b1"/>
	</lrm_resource>
	<lrm_resource id="Fence" class="stonith" type="fence_true">
	<lrm_rsc_op id="Fence_last_0" operation_key="Fence_start_0" operation="start" crm-debug-origin="crm_simulate" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="2" rc-code="0" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	</lrm_resource>
	</lrm_resources>
	</lrm>
	</node_state>
	<node_state id="node3" uname="node3" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<lrm id="node3">
	<lrm_resources>
	<lrm_resource id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<lrm_rsc_op id="dummy_last_0" operation_key="dummy_start_0" operation="start" crm-debug-origin="crm_simulate" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="2" rc-code="0" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="0830891652dabe627ca72b8e879199b1"/>
	</lrm_resource>
	<lrm_resource id="Fence" class="stonith" type="fence_true">
	<lrm_rsc_op id="Fence_last_0" operation_key="Fence_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" transition-key="1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:7;1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="1" rc-code="7" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	</lrm_resource>
	</lrm_resources>
	</lrm>
	</node_state>
	</status>
	</cib>
	=#=#=#= End test: Move dummy to node1 - OK (0) =#=#=#=
	* Passed: crm_resource - Move dummy to node1
	=#=#=#= Begin test: Clear implicit constraints for dummy on node2 =#=#=#=
	Removing constraint: cli-ban-dummy-on-node2
	=#=#=#= Current cib after: Clear implicit constraints for dummy on node2 =#=#=#=
	<cib epoch="28" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	<node id="node2" uname="node2"/>
	<node id="node3" uname="node3"/>
	</nodes>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes"/>
	<instance_attributes id="dummy-instance_attributes">
	<nvpair id="dummy-instance_attributes-delay" name="delay" value="10s"/>
	</instance_attributes>
	</primitive>
	<primitive id="Fence" class="stonith" type="fence_true"/>
	</resources>
	<constraints>
	<rsc_location id="cli-prefer-dummy" rsc="dummy" role="Started" node="node1" score="INFINITY"/>
	</constraints>
	</configuration>
	<status>
	<node_state id="node1" uname="node1" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<transient_attributes id="node1">
	<instance_attributes id="status-node1"/>
	</transient_attributes>
	<lrm id="node1">
	<lrm_resources>
	<lrm_resource id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<lrm_rsc_op id="dummy_last_0" operation_key="dummy_stop_0" operation="stop" crm-debug-origin="crm_simulate" transition-key="3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="3" rc-code="0" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="0830891652dabe627ca72b8e879199b1"/>
	</lrm_resource>
	<lrm_resource id="Fence" class="stonith" type="fence_true">
	<lrm_rsc_op id="Fence_last_0" operation_key="Fence_stop_0" operation="stop" crm-debug-origin="crm_simulate" transition-key="3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="3" rc-code="0" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	</lrm_resource>
	</lrm_resources>
	</lrm>
	</node_state>
	<tickets>
	<ticket_state id="ticketA"/>
	</tickets>
	<node_state id="node2" uname="node2" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<lrm id="node2">
	<lrm_resources>
	<lrm_resource id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<lrm_rsc_op id="dummy_last_0" operation_key="dummy_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" transition-key="1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:7;1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="1" rc-code="7" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="0830891652dabe627ca72b8e879199b1"/>
	</lrm_resource>
	<lrm_resource id="Fence" class="stonith" type="fence_true">
	<lrm_rsc_op id="Fence_last_0" operation_key="Fence_start_0" operation="start" crm-debug-origin="crm_simulate" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="2" rc-code="0" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	</lrm_resource>
	</lrm_resources>
	</lrm>
	</node_state>
	<node_state id="node3" uname="node3" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="crm_simulate">
	<lrm id="node3">
	<lrm_resources>
	<lrm_resource id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<lrm_rsc_op id="dummy_last_0" operation_key="dummy_start_0" operation="start" crm-debug-origin="crm_simulate" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="2" rc-code="0" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="0830891652dabe627ca72b8e879199b1"/>
	</lrm_resource>
	<lrm_resource id="Fence" class="stonith" type="fence_true">
	<lrm_rsc_op id="Fence_last_0" operation_key="Fence_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" transition-key="1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:7;1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="1" rc-code="7" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	</lrm_resource>
	</lrm_resources>
	</lrm>
	</node_state>
	</status>
	</cib>
	=#=#=#= End test: Clear implicit constraints for dummy on node2 - OK (0) =#=#=#=
	* Passed: crm_resource - Clear implicit constraints for dummy on node2
	=#=#=#= Begin test: Drop the status section =#=#=#=
	=#=#=#= End test: Drop the status section - OK (0) =#=#=#=
	* Passed: cibadmin - Drop the status section
	=#=#=#= Begin test: Create a clone =#=#=#=
	=#=#=#= End test: Create a clone - OK (0) =#=#=#=
	* Passed: cibadmin - Create a clone
	=#=#=#= Begin test: Create a resource meta attribute =#=#=#=
	Performing update of 'is-managed' on 'test-clone', the parent of 'test-primitive'
	Set 'test-clone' option: id=test-clone-meta_attributes-is-managed set=test-clone-meta_attributes name=is-managed value=false
	=#=#=#= Current cib after: Create a resource meta attribute =#=#=#=
	<cib epoch="30" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	<node id="node2" uname="node2"/>
	<node id="node3" uname="node3"/>
	</nodes>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes"/>
	<instance_attributes id="dummy-instance_attributes">
	<nvpair id="dummy-instance_attributes-delay" name="delay" value="10s"/>
	</instance_attributes>
	</primitive>
	<primitive id="Fence" class="stonith" type="fence_true"/>
	<clone id="test-clone">
	<primitive id="test-primitive" class="ocf" provider="pacemaker" type="Dummy"/>
	<meta_attributes id="test-clone-meta_attributes">
	<nvpair id="test-clone-meta_attributes-is-managed" name="is-managed" value="false"/>
	</meta_attributes>
	</clone>
	</resources>
	<constraints>
	<rsc_location id="cli-prefer-dummy" rsc="dummy" role="Started" node="node1" score="INFINITY"/>
	</constraints>
	</configuration>
	<status/>
	</cib>
	=#=#=#= End test: Create a resource meta attribute - OK (0) =#=#=#=
	* Passed: crm_resource - Create a resource meta attribute
	=#=#=#= Begin test: Create a resource meta attribute in the primitive =#=#=#=
	Set 'test-primitive' option: id=test-primitive-meta_attributes-is-managed set=test-primitive-meta_attributes name=is-managed value=false
	=#=#=#= Current cib after: Create a resource meta attribute in the primitive =#=#=#=
	<cib epoch="31" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	<node id="node2" uname="node2"/>
	<node id="node3" uname="node3"/>
	</nodes>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes"/>
	<instance_attributes id="dummy-instance_attributes">
	<nvpair id="dummy-instance_attributes-delay" name="delay" value="10s"/>
	</instance_attributes>
	</primitive>
	<primitive id="Fence" class="stonith" type="fence_true"/>
	<clone id="test-clone">
	<primitive id="test-primitive" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="test-primitive-meta_attributes">
	<nvpair id="test-primitive-meta_attributes-is-managed" name="is-managed" value="false"/>
	</meta_attributes>
	</primitive>
	<meta_attributes id="test-clone-meta_attributes">
	<nvpair id="test-clone-meta_attributes-is-managed" name="is-managed" value="false"/>
	</meta_attributes>
	</clone>
	</resources>
	<constraints>
	<rsc_location id="cli-prefer-dummy" rsc="dummy" role="Started" node="node1" score="INFINITY"/>
	</constraints>
	</configuration>
	<status/>
	</cib>
	=#=#=#= End test: Create a resource meta attribute in the primitive - OK (0) =#=#=#=
	* Passed: crm_resource - Create a resource meta attribute in the primitive
	=#=#=#= Begin test: Update resource meta attribute with duplicates =#=#=#=
	Multiple attributes match name=is-managed
	Value: false (id=test-primitive-meta_attributes-is-managed)
	Value: false (id=test-clone-meta_attributes-is-managed)

	A value for 'is-managed' already exists in child 'test-primitive', performing update on that instead of 'test-clone'
	Set 'test-primitive' option: id=test-primitive-meta_attributes-is-managed name=is-managed value=true
	=#=#=#= Current cib after: Update resource meta attribute with duplicates =#=#=#=
	<cib epoch="32" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	<node id="node2" uname="node2"/>
	<node id="node3" uname="node3"/>
	</nodes>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes"/>
	<instance_attributes id="dummy-instance_attributes">
	<nvpair id="dummy-instance_attributes-delay" name="delay" value="10s"/>
	</instance_attributes>
	</primitive>
	<primitive id="Fence" class="stonith" type="fence_true"/>
	<clone id="test-clone">
	<primitive id="test-primitive" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="test-primitive-meta_attributes">
	<nvpair id="test-primitive-meta_attributes-is-managed" name="is-managed" value="true"/>
	</meta_attributes>
	</primitive>
	<meta_attributes id="test-clone-meta_attributes">
	<nvpair id="test-clone-meta_attributes-is-managed" name="is-managed" value="false"/>
	</meta_attributes>
	</clone>
	</resources>
	<constraints>
	<rsc_location id="cli-prefer-dummy" rsc="dummy" role="Started" node="node1" score="INFINITY"/>
	</constraints>
	</configuration>
	<status/>
	</cib>
	=#=#=#= End test: Update resource meta attribute with duplicates - OK (0) =#=#=#=
	* Passed: crm_resource - Update resource meta attribute with duplicates
	=#=#=#= Begin test: Update resource meta attribute with duplicates (force clone) =#=#=#=
	Set 'test-clone' option: id=test-clone-meta_attributes-is-managed name=is-managed value=true
	=#=#=#= Current cib after: Update resource meta attribute with duplicates (force clone) =#=#=#=
	<cib epoch="33" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	<node id="node2" uname="node2"/>
	<node id="node3" uname="node3"/>
	</nodes>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes"/>
	<instance_attributes id="dummy-instance_attributes">
	<nvpair id="dummy-instance_attributes-delay" name="delay" value="10s"/>
	</instance_attributes>
	</primitive>
	<primitive id="Fence" class="stonith" type="fence_true"/>
	<clone id="test-clone">
	<primitive id="test-primitive" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="test-primitive-meta_attributes">
	<nvpair id="test-primitive-meta_attributes-is-managed" name="is-managed" value="true"/>
	</meta_attributes>
	</primitive>
	<meta_attributes id="test-clone-meta_attributes">
	<nvpair id="test-clone-meta_attributes-is-managed" name="is-managed" value="true"/>
	</meta_attributes>
	</clone>
	</resources>
	<constraints>
	<rsc_location id="cli-prefer-dummy" rsc="dummy" role="Started" node="node1" score="INFINITY"/>
	</constraints>
	</configuration>
	<status/>
	</cib>
	=#=#=#= End test: Update resource meta attribute with duplicates (force clone) - OK (0) =#=#=#=
	* Passed: crm_resource - Update resource meta attribute with duplicates (force clone)
	=#=#=#= Begin test: Update child resource meta attribute with duplicates =#=#=#=
	Multiple attributes match name=is-managed
	Value: true (id=test-primitive-meta_attributes-is-managed)
	Value: true (id=test-clone-meta_attributes-is-managed)

	Set 'test-primitive' option: id=test-primitive-meta_attributes-is-managed name=is-managed value=false
	=#=#=#= Current cib after: Update child resource meta attribute with duplicates =#=#=#=
	<cib epoch="34" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	<node id="node2" uname="node2"/>
	<node id="node3" uname="node3"/>
	</nodes>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes"/>
	<instance_attributes id="dummy-instance_attributes">
	<nvpair id="dummy-instance_attributes-delay" name="delay" value="10s"/>
	</instance_attributes>
	</primitive>
	<primitive id="Fence" class="stonith" type="fence_true"/>
	<clone id="test-clone">
	<primitive id="test-primitive" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="test-primitive-meta_attributes">
	<nvpair id="test-primitive-meta_attributes-is-managed" name="is-managed" value="false"/>
	</meta_attributes>
	</primitive>
	<meta_attributes id="test-clone-meta_attributes">
	<nvpair id="test-clone-meta_attributes-is-managed" name="is-managed" value="true"/>
	</meta_attributes>
	</clone>
	</resources>
	<constraints>
	<rsc_location id="cli-prefer-dummy" rsc="dummy" role="Started" node="node1" score="INFINITY"/>
	</constraints>
	</configuration>
	<status/>
	</cib>
	=#=#=#= End test: Update child resource meta attribute with duplicates - OK (0) =#=#=#=
	* Passed: crm_resource - Update child resource meta attribute with duplicates
	=#=#=#= Begin test: Delete resource meta attribute with duplicates =#=#=#=
	Multiple attributes match name=is-managed
	Value: false (id=test-primitive-meta_attributes-is-managed)
	Value: true (id=test-clone-meta_attributes-is-managed)

	A value for 'is-managed' already exists in child 'test-primitive', performing delete on that instead of 'test-clone'
	Deleted 'test-primitive' option: id=test-primitive-meta_attributes-is-managed name=is-managed
	=#=#=#= Current cib after: Delete resource meta attribute with duplicates =#=#=#=
	<cib epoch="35" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	<node id="node2" uname="node2"/>
	<node id="node3" uname="node3"/>
	</nodes>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes"/>
	<instance_attributes id="dummy-instance_attributes">
	<nvpair id="dummy-instance_attributes-delay" name="delay" value="10s"/>
	</instance_attributes>
	</primitive>
	<primitive id="Fence" class="stonith" type="fence_true"/>
	<clone id="test-clone">
	<primitive id="test-primitive" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="test-primitive-meta_attributes"/>
	</primitive>
	<meta_attributes id="test-clone-meta_attributes">
	<nvpair id="test-clone-meta_attributes-is-managed" name="is-managed" value="true"/>
	</meta_attributes>
	</clone>
	</resources>
	<constraints>
	<rsc_location id="cli-prefer-dummy" rsc="dummy" role="Started" node="node1" score="INFINITY"/>
	</constraints>
	</configuration>
	<status/>
	</cib>
	=#=#=#= End test: Delete resource meta attribute with duplicates - OK (0) =#=#=#=
	* Passed: crm_resource - Delete resource meta attribute with duplicates
	=#=#=#= Begin test: Delete resource meta attribute in parent =#=#=#=
	Performing delete of 'is-managed' on 'test-clone', the parent of 'test-primitive'
	Deleted 'test-clone' option: id=test-clone-meta_attributes-is-managed name=is-managed
	=#=#=#= Current cib after: Delete resource meta attribute in parent =#=#=#=
	<cib epoch="36" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	<node id="node2" uname="node2"/>
	<node id="node3" uname="node3"/>
	</nodes>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes"/>
	<instance_attributes id="dummy-instance_attributes">
	<nvpair id="dummy-instance_attributes-delay" name="delay" value="10s"/>
	</instance_attributes>
	</primitive>
	<primitive id="Fence" class="stonith" type="fence_true"/>
	<clone id="test-clone">
	<primitive id="test-primitive" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="test-primitive-meta_attributes"/>
	</primitive>
	<meta_attributes id="test-clone-meta_attributes"/>
	</clone>
	</resources>
	<constraints>
	<rsc_location id="cli-prefer-dummy" rsc="dummy" role="Started" node="node1" score="INFINITY"/>
	</constraints>
	</configuration>
	<status/>
	</cib>
	=#=#=#= End test: Delete resource meta attribute in parent - OK (0) =#=#=#=
	* Passed: crm_resource - Delete resource meta attribute in parent
	=#=#=#= Begin test: Create a resource meta attribute in the primitive =#=#=#=
	Set 'test-primitive' option: id=test-primitive-meta_attributes-is-managed set=test-primitive-meta_attributes name=is-managed value=false
	=#=#=#= Current cib after: Create a resource meta attribute in the primitive =#=#=#=
	<cib epoch="37" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	<node id="node2" uname="node2"/>
	<node id="node3" uname="node3"/>
	</nodes>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes"/>
	<instance_attributes id="dummy-instance_attributes">
	<nvpair id="dummy-instance_attributes-delay" name="delay" value="10s"/>
	</instance_attributes>
	</primitive>
	<primitive id="Fence" class="stonith" type="fence_true"/>
	<clone id="test-clone">
	<primitive id="test-primitive" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="test-primitive-meta_attributes">
	<nvpair id="test-primitive-meta_attributes-is-managed" name="is-managed" value="false"/>
	</meta_attributes>
	</primitive>
	<meta_attributes id="test-clone-meta_attributes"/>
	</clone>
	</resources>
	<constraints>
	<rsc_location id="cli-prefer-dummy" rsc="dummy" role="Started" node="node1" score="INFINITY"/>
	</constraints>
	</configuration>
	<status/>
	</cib>
	=#=#=#= End test: Create a resource meta attribute in the primitive - OK (0) =#=#=#=
	* Passed: crm_resource - Create a resource meta attribute in the primitive
	=#=#=#= Begin test: Update existing resource meta attribute =#=#=#=
	A value for 'is-managed' already exists in child 'test-primitive', performing update on that instead of 'test-clone'
	Set 'test-primitive' option: id=test-primitive-meta_attributes-is-managed name=is-managed value=true
	=#=#=#= Current cib after: Update existing resource meta attribute =#=#=#=
	<cib epoch="38" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	<node id="node2" uname="node2"/>
	<node id="node3" uname="node3"/>
	</nodes>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes"/>
	<instance_attributes id="dummy-instance_attributes">
	<nvpair id="dummy-instance_attributes-delay" name="delay" value="10s"/>
	</instance_attributes>
	</primitive>
	<primitive id="Fence" class="stonith" type="fence_true"/>
	<clone id="test-clone">
	<primitive id="test-primitive" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="test-primitive-meta_attributes">
	<nvpair id="test-primitive-meta_attributes-is-managed" name="is-managed" value="true"/>
	</meta_attributes>
	</primitive>
	<meta_attributes id="test-clone-meta_attributes"/>
	</clone>
	</resources>
	<constraints>
	<rsc_location id="cli-prefer-dummy" rsc="dummy" role="Started" node="node1" score="INFINITY"/>
	</constraints>
	</configuration>
	<status/>
	</cib>
	=#=#=#= End test: Update existing resource meta attribute - OK (0) =#=#=#=
	* Passed: crm_resource - Update existing resource meta attribute
	=#=#=#= Begin test: Create a resource meta attribute in the parent =#=#=#=
	Set 'test-clone' option: id=test-clone-meta_attributes-is-managed set=test-clone-meta_attributes name=is-managed value=true
	=#=#=#= Current cib after: Create a resource meta attribute in the parent =#=#=#=
	<cib epoch="39" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	<node id="node2" uname="node2"/>
	<node id="node3" uname="node3"/>
	</nodes>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes"/>
	<instance_attributes id="dummy-instance_attributes">
	<nvpair id="dummy-instance_attributes-delay" name="delay" value="10s"/>
	</instance_attributes>
	</primitive>
	<primitive id="Fence" class="stonith" type="fence_true"/>
	<clone id="test-clone">
	<primitive id="test-primitive" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="test-primitive-meta_attributes">
	<nvpair id="test-primitive-meta_attributes-is-managed" name="is-managed" value="true"/>
	</meta_attributes>
	</primitive>
	<meta_attributes id="test-clone-meta_attributes">
	<nvpair id="test-clone-meta_attributes-is-managed" name="is-managed" value="true"/>
	</meta_attributes>
	</clone>
	</resources>
	<constraints>
	<rsc_location id="cli-prefer-dummy" rsc="dummy" role="Started" node="node1" score="INFINITY"/>
	</constraints>
	</configuration>
	<status/>
	</cib>
	=#=#=#= End test: Create a resource meta attribute in the parent - OK (0) =#=#=#=
	* Passed: crm_resource - Create a resource meta attribute in the parent
	=#=#=#= Begin test: Copy resources =#=#=#=
	=#=#=#= End test: Copy resources - OK (0) =#=#=#=
	* Passed: cibadmin - Copy resources
	=#=#=#= Begin test: Delete resource parent meta attribute (force) =#=#=#=
	Deleted 'test-clone' option: id=test-clone-meta_attributes-is-managed name=is-managed
	=#=#=#= Current cib after: Delete resource parent meta attribute (force) =#=#=#=
	<cib epoch="40" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	<node id="node2" uname="node2"/>
	<node id="node3" uname="node3"/>
	</nodes>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes"/>
	<instance_attributes id="dummy-instance_attributes">
	<nvpair id="dummy-instance_attributes-delay" name="delay" value="10s"/>
	</instance_attributes>
	</primitive>
	<primitive id="Fence" class="stonith" type="fence_true"/>
	<clone id="test-clone">
	<primitive id="test-primitive" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="test-primitive-meta_attributes">
	<nvpair id="test-primitive-meta_attributes-is-managed" name="is-managed" value="true"/>
	</meta_attributes>
	</primitive>
	<meta_attributes id="test-clone-meta_attributes"/>
	</clone>
	</resources>
	<constraints>
	<rsc_location id="cli-prefer-dummy" rsc="dummy" role="Started" node="node1" score="INFINITY"/>
	</constraints>
	</configuration>
	<status/>
	</cib>
	=#=#=#= End test: Delete resource parent meta attribute (force) - OK (0) =#=#=#=
	* Passed: crm_resource - Delete resource parent meta attribute (force)
	=#=#=#= Begin test: Restore duplicates =#=#=#=
	=#=#=#= Current cib after: Restore duplicates =#=#=#=
	<cib epoch="41" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	<node id="node2" uname="node2"/>
	<node id="node3" uname="node3"/>
	</nodes>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes"/>
	<instance_attributes id="dummy-instance_attributes">
	<nvpair id="dummy-instance_attributes-delay" name="delay" value="10s"/>
	</instance_attributes>
	</primitive>
	<primitive id="Fence" class="stonith" type="fence_true"/>
	<clone id="test-clone">
	<primitive id="test-primitive" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="test-primitive-meta_attributes">
	<nvpair id="test-primitive-meta_attributes-is-managed" name="is-managed" value="true"/>
	</meta_attributes>
	</primitive>
	<meta_attributes id="test-clone-meta_attributes">
	<nvpair id="test-clone-meta_attributes-is-managed" name="is-managed" value="true"/>
	</meta_attributes>
	</clone>
	</resources>
	<constraints>
	<rsc_location id="cli-prefer-dummy" rsc="dummy" role="Started" node="node1" score="INFINITY"/>
	</constraints>
	</configuration>
	<status/>
	</cib>
	=#=#=#= End test: Restore duplicates - OK (0) =#=#=#=
	* Passed: cibadmin - Restore duplicates
	=#=#=#= Begin test: Delete resource child meta attribute =#=#=#=
	Multiple attributes match name=is-managed
	Value: true (id=test-primitive-meta_attributes-is-managed)
	Value: true (id=test-clone-meta_attributes-is-managed)

	Deleted 'test-primitive' option: id=test-primitive-meta_attributes-is-managed name=is-managed
	=#=#=#= Current cib after: Delete resource child meta attribute =#=#=#=
	<cib epoch="42" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	<node id="node2" uname="node2"/>
	<node id="node3" uname="node3"/>
	</nodes>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes"/>
	<instance_attributes id="dummy-instance_attributes">
	<nvpair id="dummy-instance_attributes-delay" name="delay" value="10s"/>
	</instance_attributes>
	</primitive>
	<primitive id="Fence" class="stonith" type="fence_true"/>
	<clone id="test-clone">
	<primitive id="test-primitive" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="test-primitive-meta_attributes"/>
	</primitive>
	<meta_attributes id="test-clone-meta_attributes">
	<nvpair id="test-clone-meta_attributes-is-managed" name="is-managed" value="true"/>
	</meta_attributes>
	</clone>
	</resources>
	<constraints>
	<rsc_location id="cli-prefer-dummy" rsc="dummy" role="Started" node="node1" score="INFINITY"/>
	</constraints>
	</configuration>
	<status/>
	</cib>
	=#=#=#= End test: Delete resource child meta attribute - OK (0) =#=#=#=
	* Passed: crm_resource - Delete resource child meta attribute
	=#=#=#= Begin test: Create a resource meta attribute in dummy1 =#=#=#=
	Set 'dummy1' option: id=dummy1-meta_attributes-is-managed set=dummy1-meta_attributes name=is-managed value=true
	=#=#=#= Current cib after: Create a resource meta attribute in dummy1 =#=#=#=
	<cib epoch="44" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	<node id="node2" uname="node2"/>
	<node id="node3" uname="node3"/>
	</nodes>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes"/>
	<instance_attributes id="dummy-instance_attributes">
	<nvpair id="dummy-instance_attributes-delay" name="delay" value="10s"/>
	</instance_attributes>
	</primitive>
	<primitive id="Fence" class="stonith" type="fence_true"/>
	<clone id="test-clone">
	<primitive id="test-primitive" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="test-primitive-meta_attributes"/>
	</primitive>
	<meta_attributes id="test-clone-meta_attributes">
	<nvpair id="test-clone-meta_attributes-is-managed" name="is-managed" value="true"/>
	</meta_attributes>
	</clone>
	<group id="dummy-group">
	<primitive id="dummy1" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy1-meta_attributes">
	<nvpair id="dummy1-meta_attributes-is-managed" name="is-managed" value="true"/>
	</meta_attributes>
	</primitive>
	<primitive id="dummy2" class="ocf" provider="pacemaker" type="Dummy"/>
	</group>
	</resources>
	<constraints>
	<rsc_location id="cli-prefer-dummy" rsc="dummy" role="Started" node="node1" score="INFINITY"/>
	</constraints>
	</configuration>
	<status/>
	</cib>
	=#=#=#= End test: Create a resource meta attribute in dummy1 - OK (0) =#=#=#=
	* Passed: crm_resource - Create a resource meta attribute in dummy1
	=#=#=#= Begin test: Create a resource meta attribute in dummy-group =#=#=#=
	Set 'dummy1' option: id=dummy1-meta_attributes-is-managed name=is-managed value=false
	Set 'dummy-group' option: id=dummy-group-meta_attributes-is-managed set=dummy-group-meta_attributes name=is-managed value=false
	=#=#=#= Current cib after: Create a resource meta attribute in dummy-group =#=#=#=
	<cib epoch="46" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	<node id="node2" uname="node2"/>
	<node id="node3" uname="node3"/>
	</nodes>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes"/>
	<instance_attributes id="dummy-instance_attributes">
	<nvpair id="dummy-instance_attributes-delay" name="delay" value="10s"/>
	</instance_attributes>
	</primitive>
	<primitive id="Fence" class="stonith" type="fence_true"/>
	<clone id="test-clone">
	<primitive id="test-primitive" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="test-primitive-meta_attributes"/>
	</primitive>
	<meta_attributes id="test-clone-meta_attributes">
	<nvpair id="test-clone-meta_attributes-is-managed" name="is-managed" value="true"/>
	</meta_attributes>
	</clone>
	<group id="dummy-group">
	<primitive id="dummy1" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy1-meta_attributes">
	<nvpair id="dummy1-meta_attributes-is-managed" name="is-managed" value="false"/>
	</meta_attributes>
	</primitive>
	<primitive id="dummy2" class="ocf" provider="pacemaker" type="Dummy"/>
	<meta_attributes id="dummy-group-meta_attributes">
	<nvpair id="dummy-group-meta_attributes-is-managed" name="is-managed" value="false"/>
	</meta_attributes>
	</group>
	</resources>
	<constraints>
	<rsc_location id="cli-prefer-dummy" rsc="dummy" role="Started" node="node1" score="INFINITY"/>
	</constraints>
	</configuration>
	<status/>
	</cib>
	=#=#=#= End test: Create a resource meta attribute in dummy-group - OK (0) =#=#=#=
	* Passed: crm_resource - Create a resource meta attribute in dummy-group
	=#=#=#= Begin test: Specify a lifetime when moving a resource =#=#=#=
	Migration will take effect until:
	=#=#=#= Current cib after: Specify a lifetime when moving a resource =#=#=#=
	<cib epoch="49" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	<node id="node2" uname="node2"/>
	<node id="node3" uname="node3"/>
	</nodes>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes"/>
	<instance_attributes id="dummy-instance_attributes">
	<nvpair id="dummy-instance_attributes-delay" name="delay" value="10s"/>
	</instance_attributes>
	</primitive>
	<primitive id="Fence" class="stonith" type="fence_true"/>
	<clone id="test-clone">
	<primitive id="test-primitive" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="test-primitive-meta_attributes"/>
	</primitive>
	<meta_attributes id="test-clone-meta_attributes">
	<nvpair id="test-clone-meta_attributes-is-managed" name="is-managed" value="true"/>
	</meta_attributes>
	</clone>
	</resources>
	<constraints>
	<rsc_location id="cli-prefer-dummy" rsc="dummy" role="Started">
	<rule id="cli-prefer-rule-dummy" score="INFINITY" boolean-op="and">
	<expression id="cli-prefer-expr-dummy" attribute="#uname" operation="eq" value="node2" type="string"/>
	<date_expression id="cli-prefer-lifetime-end-dummy" operation="lt" end=""/>
	</rule>
	</rsc_location>
	</constraints>
	</configuration>
	<status/>
	</cib>
	=#=#=#= End test: Specify a lifetime when moving a resource - OK (0) =#=#=#=
	* Passed: crm_resource - Specify a lifetime when moving a resource
	=#=#=#= Begin test: Try to move a resource previously moved with a lifetime =#=#=#=
	=#=#=#= Current cib after: Try to move a resource previously moved with a lifetime =#=#=#=
	<cib epoch="51" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	<node id="node2" uname="node2"/>
	<node id="node3" uname="node3"/>
	</nodes>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes"/>
	<instance_attributes id="dummy-instance_attributes">
	<nvpair id="dummy-instance_attributes-delay" name="delay" value="10s"/>
	</instance_attributes>
	</primitive>
	<primitive id="Fence" class="stonith" type="fence_true"/>
	<clone id="test-clone">
	<primitive id="test-primitive" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="test-primitive-meta_attributes"/>
	</primitive>
	<meta_attributes id="test-clone-meta_attributes">
	<nvpair id="test-clone-meta_attributes-is-managed" name="is-managed" value="true"/>
	</meta_attributes>
	</clone>
	</resources>
	<constraints>
	<rsc_location id="cli-prefer-dummy" rsc="dummy" role="Started" node="node1" score="INFINITY"/>
	</constraints>
	</configuration>
	<status/>
	</cib>
	=#=#=#= End test: Try to move a resource previously moved with a lifetime - OK (0) =#=#=#=
	* Passed: crm_resource - Try to move a resource previously moved with a lifetime
	=#=#=#= Begin test: Ban dummy from node1 for a short time =#=#=#=
	Migration will take effect until:
	WARNING: Creating rsc_location constraint 'cli-ban-dummy-on-node1' with a score of -INFINITY for resource dummy on node1.
	This will prevent dummy from running on node1 until the constraint is removed using the clear option or by editing the CIB with an appropriate tool
	This will be the case even if node1 is the last node in the cluster
	=#=#=#= Current cib after: Ban dummy from node1 for a short time =#=#=#=
	<cib epoch="52" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	<node id="node2" uname="node2"/>
	<node id="node3" uname="node3"/>
	</nodes>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes"/>
	<instance_attributes id="dummy-instance_attributes">
	<nvpair id="dummy-instance_attributes-delay" name="delay" value="10s"/>
	</instance_attributes>
	</primitive>
	<primitive id="Fence" class="stonith" type="fence_true"/>
	<clone id="test-clone">
	<primitive id="test-primitive" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="test-primitive-meta_attributes"/>
	</primitive>
	<meta_attributes id="test-clone-meta_attributes">
	<nvpair id="test-clone-meta_attributes-is-managed" name="is-managed" value="true"/>
	</meta_attributes>
	</clone>
	</resources>
	<constraints>
	<rsc_location id="cli-prefer-dummy" rsc="dummy" role="Started" node="node1" score="INFINITY"/>
	<rsc_location id="cli-ban-dummy-on-node1" rsc="dummy" role="Started">
	<rule id="cli-ban-dummy-on-node1-rule" score="-INFINITY" boolean-op="and">
	<expression id="cli-ban-dummy-on-node1-expr" attribute="#uname" operation="eq" value="node1" type="string"/>
	<date_expression id="cli-ban-dummy-on-node1-lifetime" operation="lt" end=""/>
	</rule>
	</rsc_location>
	</constraints>
	</configuration>
	<status/>
	</cib>
	=#=#=#= End test: Ban dummy from node1 for a short time - OK (0) =#=#=#=
	* Passed: crm_resource - Ban dummy from node1 for a short time
	=#=#=#= Begin test: Remove expired constraints =#=#=#=
	Removing constraint: cli-ban-dummy-on-node1
	=#=#=#= Current cib after: Remove expired constraints =#=#=#=
	<cib epoch="53" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	<node id="node2" uname="node2"/>
	<node id="node3" uname="node3"/>
	</nodes>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes"/>
	<instance_attributes id="dummy-instance_attributes">
	<nvpair id="dummy-instance_attributes-delay" name="delay" value="10s"/>
	</instance_attributes>
	</primitive>
	<primitive id="Fence" class="stonith" type="fence_true"/>
	<clone id="test-clone">
	<primitive id="test-primitive" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="test-primitive-meta_attributes"/>
	</primitive>
	<meta_attributes id="test-clone-meta_attributes">
	<nvpair id="test-clone-meta_attributes-is-managed" name="is-managed" value="true"/>
	</meta_attributes>
	</clone>
	</resources>
	<constraints>
	<rsc_location id="cli-prefer-dummy" rsc="dummy" role="Started" node="node1" score="INFINITY"/>
	</constraints>
	</configuration>
	<status/>
	</cib>
	=#=#=#= End test: Remove expired constraints - OK (0) =#=#=#=
	* Passed: crm_resource - Remove expired constraints
	=#=#=#= Begin test: Clear all implicit constraints for dummy =#=#=#=
	Removing constraint: cli-prefer-dummy
	=#=#=#= Current cib after: Clear all implicit constraints for dummy =#=#=#=
	<cib epoch="54" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	<node id="node2" uname="node2"/>
	<node id="node3" uname="node3"/>
	</nodes>
	<resources>
	<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="dummy-meta_attributes"/>
	<instance_attributes id="dummy-instance_attributes">
	<nvpair id="dummy-instance_attributes-delay" name="delay" value="10s"/>
	</instance_attributes>
	</primitive>
	<primitive id="Fence" class="stonith" type="fence_true"/>
	<clone id="test-clone">
	<primitive id="test-primitive" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="test-primitive-meta_attributes"/>
	</primitive>
	<meta_attributes id="test-clone-meta_attributes">
	<nvpair id="test-clone-meta_attributes-is-managed" name="is-managed" value="true"/>
	</meta_attributes>
	</clone>
	</resources>
	<constraints/>
	</configuration>
	<status/>
	</cib>
	=#=#=#= End test: Clear all implicit constraints for dummy - OK (0) =#=#=#=
	* Passed: crm_resource - Clear all implicit constraints for dummy
	=#=#=#= Begin test: Delete a resource =#=#=#=
	=#=#=#= Current cib after: Delete a resource =#=#=#=
	<cib epoch="55" num_updates="0" admin_epoch="0">
	<configuration>
	<crm_config>
	<cluster_property_set id="cib-bootstrap-options">
	<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
	</cluster_property_set>
	<cluster_property_set id="duplicate">
	<nvpair id="duplicate-cluster-delay" name="cluster-delay" value="30s"/>
	</cluster_property_set>
	</crm_config>
	<nodes>
	<node id="node1" uname="node1">
	<instance_attributes id="nodes-node1">
	<nvpair id="nodes-node1-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</node>
	<node id="node2" uname="node2"/>
	<node id="node3" uname="node3"/>
	</nodes>
	<resources>
	<primitive id="Fence" class="stonith" type="fence_true"/>
	<clone id="test-clone">
	<primitive id="test-primitive" class="ocf" provider="pacemaker" type="Dummy">
	<meta_attributes id="test-primitive-meta_attributes"/>
	</primitive>
	<meta_attributes id="test-clone-meta_attributes">
	<nvpair id="test-clone-meta_attributes-is-managed" name="is-managed" value="true"/>
	</meta_attributes>
	</clone>
	</resources>
	<constraints/>
	</configuration>
	<status/>
	</cib>
	=#=#=#= End test: Delete a resource - OK (0) =#=#=#=
	* Passed: crm_resource - Delete a resource
	=#=#=#= Begin test: Create an XML patchset =#=#=#=
	<diff format="2">
	<version>
	<source admin_epoch="0" epoch="1" num_updates="0"/>
	<target admin_epoch="0" epoch="1" num_updates="0"/>
	</version>
	<change operation="delete" path="/cib/configuration/comment" position="0"/>
	<change operation="delete" path="/cib/configuration/comment" position="1"/>
	<change operation="delete" path="/cib/configuration/resources/comment" position="0"/>
	<change operation="delete" path="/cib/configuration/resources/primitive[@id='Fencing']/operations/op[@id='Fencing-start-0']"/>
	<change operation="modify" path="/cib/configuration/crm_config/cluster_property_set[@id='cib-bootstrap-options']/nvpair[@id='cib-bootstrap-options-cluster-name']">
	<change-list>
	<change-attr name="value" operation="set" value="mycluster"/>
	<change-attr name="name" operation="set" value="cluster-name"/>
	</change-list>
	<change-result>
	<nvpair id="cib-bootstrap-options-cluster-name" value="mycluster" name="cluster-name"/>
	</change-result>
	</change>
	<change operation="create" path="/cib/configuration/nodes" position="4">
	<node id="4" uname="node4"/>
	</change>
	<change operation="create" path="/cib/configuration" position="3">
	<!-- hello world -->
	</change>
	<change operation="create" path="/cib/configuration/resources" position="0">
	<!-- test: modify this comment to say something different -->
	</change>
	<change operation="modify" path="/cib/configuration/resources/primitive[@id='Fencing']/instance_attributes[@id='Fencing-params']/nvpair[@id='Fencing-pcmk_host_list']">
	<change-list>
	<change-attr name="value" operation="set" value="node1 node2 node3 node4"/>
	</change-list>
	<change-result>
	<nvpair id="Fencing-pcmk_host_list" name="pcmk_host_list" value="node1 node2 node3 node4"/>
	</change-result>
	</change>
	<change operation="modify" path="/cib/configuration/resources/primitive[@id='Fencing']/operations/op[@id='Fencing-monitor-120s']">
	<change-list>
	<change-attr name="timeout" operation="set" value="120s"/>
	<change-attr name="name" operation="set" value="monitor"/>
	</change-list>
	<change-result>
	<op id="Fencing-monitor-120s" interval="120s" timeout="120s" name="monitor"/>
	</change-result>
	</change>
	<change operation="move" path="/cib/configuration/resources/primitive[@id='dummy']/instance_attributes[@id='dummy-params']/nvpair[@id='dummy-op_sleep']" position="1"/>
	<change operation="move" path="/cib/configuration/resources/primitive[@id='dummy']/instance_attributes[@id='dummy-params']/nvpair[@id='dummy-fake']" position="2"/>
	<change operation="modify" path="/cib/configuration/resources/primitive[@id='dummy']/operations/op[@id='dummy-monitor-5s']">
	<change-list>
	<change-attr name="name" operation="set" value="monitor"/>
	<change-attr name="timeout" operation="unset"/>
	</change-list>
	<change-result>
	<op id="dummy-monitor-5s" interval="5s" name="monitor"/>
	</change-result>
	</change>
	<change operation="create" path="/cib/configuration" position="6">
	<!-- test: move this comment to end of configuration -->
	</change>
	</diff>

	=#=#=#= End test: Create an XML patchset - Error occurred (1) =#=#=#=
	* Passed: crm_diff - Create an XML patchset
	=#=#=#= Begin test: Check locations and constraints for prim1 =#=#=#=
	=#=#=#= End test: Check locations and constraints for prim1 - OK (0) =#=#=#=
	* Passed: crm_resource - Check locations and constraints for prim1
	=#=#=#= Begin test: Recursively check locations and constraints for prim1 =#=#=#=
	=#=#=#= End test: Recursively check locations and constraints for prim1 - OK (0) =#=#=#=
	* Passed: crm_resource - Recursively check locations and constraints for prim1
	=#=#=#= Begin test: Check locations and constraints for prim1 in XML =#=#=#=
	<pacemaker-result api-version="X" request="crm_resource -a -r prim1 --output-as=xml">
	<constraints/>
	<status code="0" message="OK"/>
	</pacemaker-result>
	=#=#=#= End test: Check locations and constraints for prim1 in XML - OK (0) =#=#=#=
	* Passed: crm_resource - Check locations and constraints for prim1 in XML
	=#=#=#= Begin test: Recursively check locations and constraints for prim1 in XML =#=#=#=
	<pacemaker-result api-version="X" request="crm_resource -A -r prim1 --output-as=xml">
	<constraints/>
	<status code="0" message="OK"/>
	</pacemaker-result>
	=#=#=#= End test: Recursively check locations and constraints for prim1 in XML - OK (0) =#=#=#=
	* Passed: crm_resource - Recursively check locations and constraints for prim1 in XML
	=#=#=#= Begin test: Check locations and constraints for prim2 =#=#=#=
	Locations:
	* Node cluster01 (score=INFINITY, id=prim2-on-cluster1, rsc=prim2)
	Resources prim2 is colocated with:
	* prim3 (score=INFINITY, id=colocation-prim2-prim3-INFINITY)
	=#=#=#= End test: Check locations and constraints for prim2 - OK (0) =#=#=#=
	* Passed: crm_resource - Check locations and constraints for prim2
	=#=#=#= Begin test: Recursively check locations and constraints for prim2 =#=#=#=
	Locations:
	* Node cluster01 (score=INFINITY, id=prim2-on-cluster1, rsc=prim2)
	Resources prim2 is colocated with:
	* prim3 (score=INFINITY, id=colocation-prim2-prim3-INFINITY)
	* Resources prim3 is colocated with:
	* prim4 (score=INFINITY, id=colocation-prim3-prim4-INFINITY)
	* Locations:
	* Node cluster02 (score=INFINITY, id=prim4-on-cluster2, rsc=prim4)
	* Resources prim4 is colocated with:
	* prim5 (score=INFINITY, id=colocation-prim4-prim5-INFINITY)
	=#=#=#= End test: Recursively check locations and constraints for prim2 - OK (0) =#=#=#=
	* Passed: crm_resource - Recursively check locations and constraints for prim2
	=#=#=#= Begin test: Check locations and constraints for prim2 in XML =#=#=#=
	<pacemaker-result api-version="X" request="crm_resource -a -r prim2 --output-as=xml">
	<constraints>
	<rsc_location node="cluster01" rsc="prim2" id="prim2-on-cluster1" score="INFINITY"/>
	<rsc_colocation id="colocation-prim2-prim3-INFINITY" rsc="prim2" with-rsc="prim3" score="INFINITY"/>
	</constraints>
	<status code="0" message="OK"/>
	</pacemaker-result>
	=#=#=#= End test: Check locations and constraints for prim2 in XML - OK (0) =#=#=#=
	* Passed: crm_resource - Check locations and constraints for prim2 in XML
	=#=#=#= Begin test: Recursively check locations and constraints for prim2 in XML =#=#=#=
	<pacemaker-result api-version="X" request="crm_resource -A -r prim2 --output-as=xml">
	<constraints>
	<rsc_location node="cluster01" rsc="prim2" id="prim2-on-cluster1" score="INFINITY"/>
	<rsc_colocation id="colocation-prim2-prim3-INFINITY" rsc="prim2" with-rsc="prim3" score="INFINITY"/>
	<rsc_colocation id="colocation-prim3-prim4-INFINITY" rsc="prim3" with-rsc="prim4" score="INFINITY"/>
	<rsc_location node="cluster02" rsc="prim4" id="prim4-on-cluster2" score="INFINITY"/>
	<rsc_colocation id="colocation-prim4-prim5-INFINITY" rsc="prim4" with-rsc="prim5" score="INFINITY"/>
	</constraints>
	<status code="0" message="OK"/>
	</pacemaker-result>
	=#=#=#= End test: Recursively check locations and constraints for prim2 in XML - OK (0) =#=#=#=
	* Passed: crm_resource - Recursively check locations and constraints for prim2 in XML
	=#=#=#= Begin test: Check locations and constraints for prim3 =#=#=#=
	Resources colocated with prim3:
	* prim2 (score=INFINITY, id=colocation-prim2-prim3-INFINITY)
	* Locations:
	* Node cluster01 (score=INFINITY, id=prim2-on-cluster1, rsc=prim2)
	Resources prim3 is colocated with:
	* prim4 (score=INFINITY, id=colocation-prim3-prim4-INFINITY)
	* Locations:
	* Node cluster02 (score=INFINITY, id=prim4-on-cluster2, rsc=prim4)
	=#=#=#= End test: Check locations and constraints for prim3 - OK (0) =#=#=#=
	* Passed: crm_resource - Check locations and constraints for prim3
	=#=#=#= Begin test: Recursively check locations and constraints for prim3 =#=#=#=
	Resources colocated with prim3:
	* prim2 (score=INFINITY, id=colocation-prim2-prim3-INFINITY)
	* Locations:
	* Node cluster01 (score=INFINITY, id=prim2-on-cluster1, rsc=prim2)
	Resources prim3 is colocated with:
	* prim4 (score=INFINITY, id=colocation-prim3-prim4-INFINITY)
	* Locations:
	* Node cluster02 (score=INFINITY, id=prim4-on-cluster2, rsc=prim4)
	* Resources prim4 is colocated with:
	* prim5 (score=INFINITY, id=colocation-prim4-prim5-INFINITY)
	=#=#=#= End test: Recursively check locations and constraints for prim3 - OK (0) =#=#=#=
	* Passed: crm_resource - Recursively check locations and constraints for prim3
	=#=#=#= Begin test: Check locations and constraints for prim3 in XML =#=#=#=
	<pacemaker-result api-version="X" request="crm_resource -a -r prim3 --output-as=xml">
	<constraints>
	<rsc_colocation id="colocation-prim2-prim3-INFINITY" rsc="prim2" with-rsc="prim3" score="INFINITY"/>
	<rsc_location node="cluster01" rsc="prim2" id="prim2-on-cluster1" score="INFINITY"/>
	<rsc_colocation id="colocation-prim3-prim4-INFINITY" rsc="prim3" with-rsc="prim4" score="INFINITY"/>
	<rsc_location node="cluster02" rsc="prim4" id="prim4-on-cluster2" score="INFINITY"/>
	</constraints>
	<status code="0" message="OK"/>
	</pacemaker-result>
	=#=#=#= End test: Check locations and constraints for prim3 in XML - OK (0) =#=#=#=
	* Passed: crm_resource - Check locations and constraints for prim3 in XML
	=#=#=#= Begin test: Recursively check locations and constraints for prim3 in XML =#=#=#=
	<pacemaker-result api-version="X" request="crm_resource -A -r prim3 --output-as=xml">
	<constraints>
	<rsc_colocation id="colocation-prim2-prim3-INFINITY" rsc="prim2" with-rsc="prim3" score="INFINITY"/>
	<rsc_location node="cluster01" rsc="prim2" id="prim2-on-cluster1" score="INFINITY"/>
	<rsc_colocation id="colocation-prim3-prim4-INFINITY" rsc="prim3" with-rsc="prim4" score="INFINITY"/>
	<rsc_location node="cluster02" rsc="prim4" id="prim4-on-cluster2" score="INFINITY"/>
	<rsc_colocation id="colocation-prim4-prim5-INFINITY" rsc="prim4" with-rsc="prim5" score="INFINITY"/>
	</constraints>
	<status code="0" message="OK"/>
	</pacemaker-result>
	=#=#=#= End test: Recursively check locations and constraints for prim3 in XML - OK (0) =#=#=#=
	* Passed: crm_resource - Recursively check locations and constraints for prim3 in XML
	=#=#=#= Begin test: Check locations and constraints for prim4 =#=#=#=
	Locations:
	* Node cluster02 (score=INFINITY, id=prim4-on-cluster2, rsc=prim4)
	Resources colocated with prim4:
	* prim10 (score=INFINITY, id=colocation-prim10-prim4-INFINITY)
	* prim3 (score=INFINITY, id=colocation-prim3-prim4-INFINITY)
	Resources prim4 is colocated with:
	* prim5 (score=INFINITY, id=colocation-prim4-prim5-INFINITY)
	=#=#=#= End test: Check locations and constraints for prim4 - OK (0) =#=#=#=
	* Passed: crm_resource - Check locations and constraints for prim4
	=#=#=#= Begin test: Recursively check locations and constraints for prim4 =#=#=#=
	Locations:
	* Node cluster02 (score=INFINITY, id=prim4-on-cluster2, rsc=prim4)
	Resources colocated with prim4:
	* prim10 (score=INFINITY, id=colocation-prim10-prim4-INFINITY)
	* prim3 (score=INFINITY, id=colocation-prim3-prim4-INFINITY)
	* Resources colocated with prim3:
	* prim2 (score=INFINITY, id=colocation-prim2-prim3-INFINITY)
	* Locations:
	* Node cluster01 (score=INFINITY, id=prim2-on-cluster1, rsc=prim2)
	Resources prim4 is colocated with:
	* prim5 (score=INFINITY, id=colocation-prim4-prim5-INFINITY)
	=#=#=#= End test: Recursively check locations and constraints for prim4 - OK (0) =#=#=#=
	* Passed: crm_resource - Recursively check locations and constraints for prim4
	=#=#=#= Begin test: Check locations and constraints for prim4 in XML =#=#=#=
	<pacemaker-result api-version="X" request="crm_resource -a -r prim4 --output-as=xml">
	<constraints>
	<rsc_location node="cluster02" rsc="prim4" id="prim4-on-cluster2" score="INFINITY"/>
	<rsc_colocation id="colocation-prim10-prim4-INFINITY" rsc="prim10" with-rsc="prim4" score="INFINITY"/>
	<rsc_colocation id="colocation-prim3-prim4-INFINITY" rsc="prim3" with-rsc="prim4" score="INFINITY"/>
	<rsc_colocation id="colocation-prim4-prim5-INFINITY" rsc="prim4" with-rsc="prim5" score="INFINITY"/>
	</constraints>
	<status code="0" message="OK"/>
	</pacemaker-result>
	=#=#=#= End test: Check locations and constraints for prim4 in XML - OK (0) =#=#=#=
	* Passed: crm_resource - Check locations and constraints for prim4 in XML
	=#=#=#= Begin test: Recursively check locations and constraints for prim4 in XML =#=#=#=
	<pacemaker-result api-version="X" request="crm_resource -A -r prim4 --output-as=xml">
	<constraints>
	<rsc_location node="cluster02" rsc="prim4" id="prim4-on-cluster2" score="INFINITY"/>
	<rsc_colocation id="colocation-prim10-prim4-INFINITY" rsc="prim10" with-rsc="prim4" score="INFINITY"/>
	<rsc_colocation id="colocation-prim3-prim4-INFINITY" rsc="prim3" with-rsc="prim4" score="INFINITY"/>
	<rsc_colocation id="colocation-prim2-prim3-INFINITY" rsc="prim2" with-rsc="prim3" score="INFINITY"/>
	<rsc_location node="cluster01" rsc="prim2" id="prim2-on-cluster1" score="INFINITY"/>
	<rsc_colocation id="colocation-prim4-prim5-INFINITY" rsc="prim4" with-rsc="prim5" score="INFINITY"/>
	</constraints>
	<status code="0" message="OK"/>
	</pacemaker-result>
	=#=#=#= End test: Recursively check locations and constraints for prim4 in XML - OK (0) =#=#=#=
	* Passed: crm_resource - Recursively check locations and constraints for prim4 in XML
	=#=#=#= Begin test: Check locations and constraints for prim5 =#=#=#=
	Resources colocated with prim5:
	* prim4 (score=INFINITY, id=colocation-prim4-prim5-INFINITY)
	* Locations:
	* Node cluster02 (score=INFINITY, id=prim4-on-cluster2, rsc=prim4)
	=#=#=#= End test: Check locations and constraints for prim5 - OK (0) =#=#=#=
	* Passed: crm_resource - Check locations and constraints for prim5
	=#=#=#= Begin test: Recursively check locations and constraints for prim5 =#=#=#=
	Resources colocated with prim5:
	* prim4 (score=INFINITY, id=colocation-prim4-prim5-INFINITY)
	* Locations:
	* Node cluster02 (score=INFINITY, id=prim4-on-cluster2, rsc=prim4)
	* Resources colocated with prim4:
	* prim10 (score=INFINITY, id=colocation-prim10-prim4-INFINITY)
	* prim3 (score=INFINITY, id=colocation-prim3-prim4-INFINITY)
	* Resources colocated with prim3:
	* prim2 (score=INFINITY, id=colocation-prim2-prim3-INFINITY)
	* Locations:
	* Node cluster01 (score=INFINITY, id=prim2-on-cluster1, rsc=prim2)
	=#=#=#= End test: Recursively check locations and constraints for prim5 - OK (0) =#=#=#=
	* Passed: crm_resource - Recursively check locations and constraints for prim5
	=#=#=#= Begin test: Check locations and constraints for prim5 in XML =#=#=#=
	<pacemaker-result api-version="X" request="crm_resource -a -r prim5 --output-as=xml">
	<constraints>
	<rsc_colocation id="colocation-prim4-prim5-INFINITY" rsc="prim4" with-rsc="prim5" score="INFINITY"/>
	<rsc_location node="cluster02" rsc="prim4" id="prim4-on-cluster2" score="INFINITY"/>
	</constraints>
	<status code="0" message="OK"/>
	</pacemaker-result>
	=#=#=#= End test: Check locations and constraints for prim5 in XML - OK (0) =#=#=#=
	* Passed: crm_resource - Check locations and constraints for prim5 in XML
	=#=#=#= Begin test: Recursively check locations and constraints for prim5 in XML =#=#=#=
	<pacemaker-result api-version="X" request="crm_resource -A -r prim5 --output-as=xml">
	<constraints>
	<rsc_colocation id="colocation-prim4-prim5-INFINITY" rsc="prim4" with-rsc="prim5" score="INFINITY"/>
	<rsc_location node="cluster02" rsc="prim4" id="prim4-on-cluster2" score="INFINITY"/>
	<rsc_colocation id="colocation-prim10-prim4-INFINITY" rsc="prim10" with-rsc="prim4" score="INFINITY"/>
	<rsc_colocation id="colocation-prim3-prim4-INFINITY" rsc="prim3" with-rsc="prim4" score="INFINITY"/>
	<rsc_colocation id="colocation-prim2-prim3-INFINITY" rsc="prim2" with-rsc="prim3" score="INFINITY"/>
	<rsc_location node="cluster01" rsc="prim2" id="prim2-on-cluster1" score="INFINITY"/>
	</constraints>
	<status code="0" message="OK"/>
	</pacemaker-result>
	=#=#=#= End test: Recursively check locations and constraints for prim5 in XML - OK (0) =#=#=#=
	* Passed: crm_resource - Recursively check locations and constraints for prim5 in XML
	=#=#=#= Begin test: Check locations and constraints for prim6 =#=#=#=
	Locations:
	* Node cluster02 (score=-INFINITY, id=prim6-not-on-cluster2, rsc=prim6)
	=#=#=#= End test: Check locations and constraints for prim6 - OK (0) =#=#=#=
	* Passed: crm_resource - Check locations and constraints for prim6
	=#=#=#= Begin test: Recursively check locations and constraints for prim6 =#=#=#=
	Locations:
	* Node cluster02 (score=-INFINITY, id=prim6-not-on-cluster2, rsc=prim6)
	=#=#=#= End test: Recursively check locations and constraints for prim6 - OK (0) =#=#=#=
	* Passed: crm_resource - Recursively check locations and constraints for prim6
	=#=#=#= Begin test: Check locations and constraints for prim6 in XML =#=#=#=
	<pacemaker-result api-version="X" request="crm_resource -a -r prim6 --output-as=xml">
	<constraints>
	<rsc_location node="cluster02" rsc="prim6" id="prim6-not-on-cluster2" score="-INFINITY"/>
	</constraints>
	<status code="0" message="OK"/>
	</pacemaker-result>
	=#=#=#= End test: Check locations and constraints for prim6 in XML - OK (0) =#=#=#=
	* Passed: crm_resource - Check locations and constraints for prim6 in XML
	=#=#=#= Begin test: Recursively check locations and constraints for prim6 in XML =#=#=#=
	<pacemaker-result api-version="X" request="crm_resource -A -r prim6 --output-as=xml">
	<constraints>
	<rsc_location node="cluster02" rsc="prim6" id="prim6-not-on-cluster2" score="-INFINITY"/>
	</constraints>
	<status code="0" message="OK"/>
	</pacemaker-result>
	=#=#=#= End test: Recursively check locations and constraints for prim6 in XML - OK (0) =#=#=#=
	* Passed: crm_resource - Recursively check locations and constraints for prim6 in XML
	=#=#=#= Begin test: Check locations and constraints for prim7 =#=#=#=
	Resources prim7 is colocated with:
	* group (score=INFINITY, id=colocation-prim7-group-INFINITY)
	=#=#=#= End test: Check locations and constraints for prim7 - OK (0) =#=#=#=
	* Passed: crm_resource - Check locations and constraints for prim7
	=#=#=#= Begin test: Recursively check locations and constraints for prim7 =#=#=#=
	Resources prim7 is colocated with:
	* group (score=INFINITY, id=colocation-prim7-group-INFINITY)
	=#=#=#= End test: Recursively check locations and constraints for prim7 - OK (0) =#=#=#=
	* Passed: crm_resource - Recursively check locations and constraints for prim7
	=#=#=#= Begin test: Check locations and constraints for prim7 in XML =#=#=#=
	<pacemaker-result api-version="X" request="crm_resource -a -r prim7 --output-as=xml">
	<constraints>
	<rsc_colocation id="colocation-prim7-group-INFINITY" rsc="prim7" with-rsc="group" score="INFINITY"/>
	</constraints>
	<status code="0" message="OK"/>
	</pacemaker-result>
	=#=#=#= End test: Check locations and constraints for prim7 in XML - OK (0) =#=#=#=
	* Passed: crm_resource - Check locations and constraints for prim7 in XML
	=#=#=#= Begin test: Recursively check locations and constraints for prim7 in XML =#=#=#=
	<pacemaker-result api-version="X" request="crm_resource -A -r prim7 --output-as=xml">
	<constraints>
	<rsc_colocation id="colocation-prim7-group-INFINITY" rsc="prim7" with-rsc="group" score="INFINITY"/>
	</constraints>
	<status code="0" message="OK"/>
	</pacemaker-result>
	=#=#=#= End test: Recursively check locations and constraints for prim7 in XML - OK (0) =#=#=#=
	* Passed: crm_resource - Recursively check locations and constraints for prim7 in XML
	=#=#=#= Begin test: Check locations and constraints for prim8 =#=#=#=
	Resources prim8 is colocated with:
	* gr2 (score=INFINITY, id=colocation-prim8-gr2-INFINITY)
	=#=#=#= End test: Check locations and constraints for prim8 - OK (0) =#=#=#=
	* Passed: crm_resource - Check locations and constraints for prim8
	=#=#=#= Begin test: Recursively check locations and constraints for prim8 =#=#=#=
	Resources prim8 is colocated with:
	* gr2 (score=INFINITY, id=colocation-prim8-gr2-INFINITY)
	=#=#=#= End test: Recursively check locations and constraints for prim8 - OK (0) =#=#=#=
	* Passed: crm_resource - Recursively check locations and constraints for prim8
	=#=#=#= Begin test: Check locations and constraints for prim8 in XML =#=#=#=
	<pacemaker-result api-version="X" request="crm_resource -a -r prim8 --output-as=xml">
	<constraints>
	<rsc_colocation id="colocation-prim8-gr2-INFINITY" rsc="prim8" with-rsc="gr2" score="INFINITY"/>
	</constraints>
	<status code="0" message="OK"/>
	</pacemaker-result>
	=#=#=#= End test: Check locations and constraints for prim8 in XML - OK (0) =#=#=#=
	* Passed: crm_resource - Check locations and constraints for prim8 in XML
	=#=#=#= Begin test: Recursively check locations and constraints for prim8 in XML =#=#=#=
	<pacemaker-result api-version="X" request="crm_resource -A -r prim8 --output-as=xml">
	<constraints>
	<rsc_colocation id="colocation-prim8-gr2-INFINITY" rsc="prim8" with-rsc="gr2" score="INFINITY"/>
	</constraints>
	<status code="0" message="OK"/>
	</pacemaker-result>
	=#=#=#= End test: Recursively check locations and constraints for prim8 in XML - OK (0) =#=#=#=
	* Passed: crm_resource - Recursively check locations and constraints for prim8 in XML
	=#=#=#= Begin test: Check locations and constraints for prim9 =#=#=#=
	Resources prim9 is colocated with:
	* clone (score=INFINITY, id=colocation-prim9-clone-INFINITY)
	=#=#=#= End test: Check locations and constraints for prim9 - OK (0) =#=#=#=
	* Passed: crm_resource - Check locations and constraints for prim9
	=#=#=#= Begin test: Recursively check locations and constraints for prim9 =#=#=#=
	Resources prim9 is colocated with:
	* clone (score=INFINITY, id=colocation-prim9-clone-INFINITY)
	=#=#=#= End test: Recursively check locations and constraints for prim9 - OK (0) =#=#=#=
	* Passed: crm_resource - Recursively check locations and constraints for prim9
	=#=#=#= Begin test: Check locations and constraints for prim9 in XML =#=#=#=
	<pacemaker-result api-version="X" request="crm_resource -a -r prim9 --output-as=xml">
	<constraints>
	<rsc_colocation id="colocation-prim9-clone-INFINITY" rsc="prim9" with-rsc="clone" score="INFINITY"/>
	</constraints>
	<status code="0" message="OK"/>
	</pacemaker-result>
	=#=#=#= End test: Check locations and constraints for prim9 in XML - OK (0) =#=#=#=
	* Passed: crm_resource - Check locations and constraints for prim9 in XML
	=#=#=#= Begin test: Recursively check locations and constraints for prim9 in XML =#=#=#=
	<pacemaker-result api-version="X" request="crm_resource -A -r prim9 --output-as=xml">
	<constraints>
	<rsc_colocation id="colocation-prim9-clone-INFINITY" rsc="prim9" with-rsc="clone" score="INFINITY"/>
	</constraints>
	<status code="0" message="OK"/>
	</pacemaker-result>
	=#=#=#= End test: Recursively check locations and constraints for prim9 in XML - OK (0) =#=#=#=
	* Passed: crm_resource - Recursively check locations and constraints for prim9 in XML
	=#=#=#= Begin test: Check locations and constraints for prim10 =#=#=#=
	Resources prim10 is colocated with:
	* prim4 (score=INFINITY, id=colocation-prim10-prim4-INFINITY)
	* Locations:
	* Node cluster02 (score=INFINITY, id=prim4-on-cluster2, rsc=prim4)
	=#=#=#= End test: Check locations and constraints for prim10 - OK (0) =#=#=#=
	* Passed: crm_resource - Check locations and constraints for prim10
	=#=#=#= Begin test: Recursively check locations and constraints for prim10 =#=#=#=
	Resources prim10 is colocated with:
	* prim4 (score=INFINITY, id=colocation-prim10-prim4-INFINITY)
	* Locations:
	* Node cluster02 (score=INFINITY, id=prim4-on-cluster2, rsc=prim4)
	* Resources prim4 is colocated with:
	* prim5 (score=INFINITY, id=colocation-prim4-prim5-INFINITY)
	=#=#=#= End test: Recursively check locations and constraints for prim10 - OK (0) =#=#=#=
	* Passed: crm_resource - Recursively check locations and constraints for prim10
	=#=#=#= Begin test: Check locations and constraints for prim10 in XML =#=#=#=
	<pacemaker-result api-version="X" request="crm_resource -a -r prim10 --output-as=xml">
	<constraints>
	<rsc_colocation id="colocation-prim10-prim4-INFINITY" rsc="prim10" with-rsc="prim4" score="INFINITY"/>
	<rsc_location node="cluster02" rsc="prim4" id="prim4-on-cluster2" score="INFINITY"/>
	</constraints>
	<status code="0" message="OK"/>
	</pacemaker-result>
	=#=#=#= End test: Check locations and constraints for prim10 in XML - OK (0) =#=#=#=
	* Passed: crm_resource - Check locations and constraints for prim10 in XML
	=#=#=#= Begin test: Recursively check locations and constraints for prim10 in XML =#=#=#=
	<pacemaker-result api-version="X" request="crm_resource -A -r prim10 --output-as=xml">
	<constraints>
	<rsc_colocation id="colocation-prim10-prim4-INFINITY" rsc="prim10" with-rsc="prim4" score="INFINITY"/>
	<rsc_location node="cluster02" rsc="prim4" id="prim4-on-cluster2" score="INFINITY"/>
	<rsc_colocation id="colocation-prim4-prim5-INFINITY" rsc="prim4" with-rsc="prim5" score="INFINITY"/>
	</constraints>
	<status code="0" message="OK"/>
	</pacemaker-result>
	=#=#=#= End test: Recursively check locations and constraints for prim10 in XML - OK (0) =#=#=#=
	* Passed: crm_resource - Recursively check locations and constraints for prim10 in XML
	=#=#=#= Begin test: Check locations and constraints for prim11 =#=#=#=
	Resources colocated with prim11:
	* prim13 (score=INFINITY, id=colocation-prim13-prim11-INFINITY)
	Resources prim11 is colocated with:
	* prim12 (score=INFINITY, id=colocation-prim11-prim12-INFINITY)
	=#=#=#= End test: Check locations and constraints for prim11 - OK (0) =#=#=#=
	* Passed: crm_resource - Check locations and constraints for prim11
	=#=#=#= Begin test: Recursively check locations and constraints for prim11 =#=#=#=
	Resources colocated with prim11:
	* prim13 (score=INFINITY, id=colocation-prim13-prim11-INFINITY)
	* Resources colocated with prim13:
	* prim12 (score=INFINITY, id=colocation-prim12-prim13-INFINITY)
	* Resources colocated with prim12:
	* prim11 (id=colocation-prim11-prim12-INFINITY - loop)
	Resources prim11 is colocated with:
	* prim12 (score=INFINITY, id=colocation-prim11-prim12-INFINITY)
	* Resources prim12 is colocated with:
	* prim13 (score=INFINITY, id=colocation-prim12-prim13-INFINITY)
	* Resources prim13 is colocated with:
	* prim11 (id=colocation-prim13-prim11-INFINITY - loop)
	=#=#=#= End test: Recursively check locations and constraints for prim11 - OK (0) =#=#=#=
	* Passed: crm_resource - Recursively check locations and constraints for prim11
	=#=#=#= Begin test: Check locations and constraints for prim11 in XML =#=#=#=
	<pacemaker-result api-version="X" request="crm_resource -a -r prim11 --output-as=xml">
	<constraints>
	<rsc_colocation id="colocation-prim13-prim11-INFINITY" rsc="prim13" with-rsc="prim11" score="INFINITY"/>
	<rsc_colocation id="colocation-prim11-prim12-INFINITY" rsc="prim11" with-rsc="prim12" score="INFINITY"/>
	</constraints>
	<status code="0" message="OK"/>
	</pacemaker-result>
	=#=#=#= End test: Check locations and constraints for prim11 in XML - OK (0) =#=#=#=
	* Passed: crm_resource - Check locations and constraints for prim11 in XML
	=#=#=#= Begin test: Recursively check locations and constraints for prim11 in XML =#=#=#=
	<pacemaker-result api-version="X" request="crm_resource -A -r prim11 --output-as=xml">
	<constraints>
	<rsc_colocation id="colocation-prim13-prim11-INFINITY" rsc="prim13" with-rsc="prim11" score="INFINITY"/>
	<rsc_colocation id="colocation-prim12-prim13-INFINITY" rsc="prim12" with-rsc="prim13" score="INFINITY"/>
	<rsc_colocation id="colocation-prim11-prim12-INFINITY" rsc="prim11" with-rsc="prim12" score="INFINITY"/>
	<rsc_colocation id="colocation-prim11-prim12-INFINITY" rsc="prim11" with-rsc="prim12" score="INFINITY"/>
	<rsc_colocation id="colocation-prim12-prim13-INFINITY" rsc="prim12" with-rsc="prim13" score="INFINITY"/>
	<rsc_colocation id="colocation-prim13-prim11-INFINITY" rsc="prim13" with-rsc="prim11" score="INFINITY"/>
	</constraints>
	<status code="0" message="OK"/>
	</pacemaker-result>
	=#=#=#= End test: Recursively check locations and constraints for prim11 in XML - OK (0) =#=#=#=
	* Passed: crm_resource - Recursively check locations and constraints for prim11 in XML
	=#=#=#= Begin test: Check locations and constraints for prim12 =#=#=#=
	Resources colocated with prim12:
	* prim11 (score=INFINITY, id=colocation-prim11-prim12-INFINITY)
	Resources prim12 is colocated with:
	* prim13 (score=INFINITY, id=colocation-prim12-prim13-INFINITY)
	=#=#=#= End test: Check locations and constraints for prim12 - OK (0) =#=#=#=
	* Passed: crm_resource - Check locations and constraints for prim12
	=#=#=#= Begin test: Recursively check locations and constraints for prim12 =#=#=#=
	Resources colocated with prim12:
	* prim11 (score=INFINITY, id=colocation-prim11-prim12-INFINITY)
	* Resources colocated with prim11:
	* prim13 (score=INFINITY, id=colocation-prim13-prim11-INFINITY)
	* Resources colocated with prim13:
	* prim12 (id=colocation-prim12-prim13-INFINITY - loop)
	Resources prim12 is colocated with:
	* prim13 (score=INFINITY, id=colocation-prim12-prim13-INFINITY)
	* Resources prim13 is colocated with:
	* prim11 (score=INFINITY, id=colocation-prim13-prim11-INFINITY)
	* Resources prim11 is colocated with:
	* prim12 (id=colocation-prim11-prim12-INFINITY - loop)
	=#=#=#= End test: Recursively check locations and constraints for prim12 - OK (0) =#=#=#=
	* Passed: crm_resource - Recursively check locations and constraints for prim12
	=#=#=#= Begin test: Check locations and constraints for prim12 in XML =#=#=#=
	<pacemaker-result api-version="X" request="crm_resource -a -r prim12 --output-as=xml">
	<constraints>
	<rsc_colocation id="colocation-prim11-prim12-INFINITY" rsc="prim11" with-rsc="prim12" score="INFINITY"/>
	<rsc_colocation id="colocation-prim12-prim13-INFINITY" rsc="prim12" with-rsc="prim13" score="INFINITY"/>
	</constraints>
	<status code="0" message="OK"/>
	</pacemaker-result>
	=#=#=#= End test: Check locations and constraints for prim12 in XML - OK (0) =#=#=#=
	* Passed: crm_resource - Check locations and constraints for prim12 in XML
	=#=#=#= Begin test: Recursively check locations and constraints for prim12 in XML =#=#=#=
	<pacemaker-result api-version="X" request="crm_resource -A -r prim12 --output-as=xml">
	<constraints>
	<rsc_colocation id="colocation-prim11-prim12-INFINITY" rsc="prim11" with-rsc="prim12" score="INFINITY"/>
	<rsc_colocation id="colocation-prim13-prim11-INFINITY" rsc="prim13" with-rsc="prim11" score="INFINITY"/>
	<rsc_colocation id="colocation-prim12-prim13-INFINITY" rsc="prim12" with-rsc="prim13" score="INFINITY"/>
	<rsc_colocation id="colocation-prim12-prim13-INFINITY" rsc="prim12" with-rsc="prim13" score="INFINITY"/>
	<rsc_colocation id="colocation-prim13-prim11-INFINITY" rsc="prim13" with-rsc="prim11" score="INFINITY"/>
	<rsc_colocation id="colocation-prim11-prim12-INFINITY" rsc="prim11" with-rsc="prim12" score="INFINITY"/>
	</constraints>
	<status code="0" message="OK"/>
	</pacemaker-result>
	=#=#=#= End test: Recursively check locations and constraints for prim12 in XML - OK (0) =#=#=#=
	* Passed: crm_resource - Recursively check locations and constraints for prim12 in XML
	=#=#=#= Begin test: Check locations and constraints for prim13 =#=#=#=
	Resources colocated with prim13:
	* prim12 (score=INFINITY, id=colocation-prim12-prim13-INFINITY)
	Resources prim13 is colocated with:
	* prim11 (score=INFINITY, id=colocation-prim13-prim11-INFINITY)
	=#=#=#= End test: Check locations and constraints for prim13 - OK (0) =#=#=#=
	* Passed: crm_resource - Check locations and constraints for prim13
	=#=#=#= Begin test: Recursively check locations and constraints for prim13 =#=#=#=
	Resources colocated with prim13:
	* prim12 (score=INFINITY, id=colocation-prim12-prim13-INFINITY)
	* Resources colocated with prim12:
	* prim11 (score=INFINITY, id=colocation-prim11-prim12-INFINITY)
	* Resources colocated with prim11:
	* prim13 (id=colocation-prim13-prim11-INFINITY - loop)
	Resources prim13 is colocated with:
	* prim11 (score=INFINITY, id=colocation-prim13-prim11-INFINITY)
	* Resources prim11 is colocated with:
	* prim12 (score=INFINITY, id=colocation-prim11-prim12-INFINITY)
	* Resources prim12 is colocated with:
	* prim13 (id=colocation-prim12-prim13-INFINITY - loop)
	=#=#=#= End test: Recursively check locations and constraints for prim13 - OK (0) =#=#=#=
	* Passed: crm_resource - Recursively check locations and constraints for prim13
	=#=#=#= Begin test: Check locations and constraints for prim13 in XML =#=#=#=
	<pacemaker-result api-version="X" request="crm_resource -a -r prim13 --output-as=xml">
	<constraints>
	<rsc_colocation id="colocation-prim12-prim13-INFINITY" rsc="prim12" with-rsc="prim13" score="INFINITY"/>
	<rsc_colocation id="colocation-prim13-prim11-INFINITY" rsc="prim13" with-rsc="prim11" score="INFINITY"/>
	</constraints>
	<status code="0" message="OK"/>
	</pacemaker-result>
	=#=#=#= End test: Check locations and constraints for prim13 in XML - OK (0) =#=#=#=
	* Passed: crm_resource - Check locations and constraints for prim13 in XML
	=#=#=#= Begin test: Recursively check locations and constraints for prim13 in XML =#=#=#=
	<pacemaker-result api-version="X" request="crm_resource -A -r prim13 --output-as=xml">
	<constraints>
	<rsc_colocation id="colocation-prim12-prim13-INFINITY" rsc="prim12" with-rsc="prim13" score="INFINITY"/>
	<rsc_colocation id="colocation-prim11-prim12-INFINITY" rsc="prim11" with-rsc="prim12" score="INFINITY"/>
	<rsc_colocation id="colocation-prim13-prim11-INFINITY" rsc="prim13" with-rsc="prim11" score="INFINITY"/>
	<rsc_colocation id="colocation-prim13-prim11-INFINITY" rsc="prim13" with-rsc="prim11" score="INFINITY"/>
	<rsc_colocation id="colocation-prim11-prim12-INFINITY" rsc="prim11" with-rsc="prim12" score="INFINITY"/>
	<rsc_colocation id="colocation-prim12-prim13-INFINITY" rsc="prim12" with-rsc="prim13" score="INFINITY"/>
	</constraints>
	<status code="0" message="OK"/>
	</pacemaker-result>
	=#=#=#= End test: Recursively check locations and constraints for prim13 in XML - OK (0) =#=#=#=
	* Passed: crm_resource - Recursively check locations and constraints for prim13 in XML
	=#=#=#= Begin test: Check locations and constraints for group =#=#=#=
	Resources colocated with group:
	* prim7 (score=INFINITY, id=colocation-prim7-group-INFINITY)
	=#=#=#= End test: Check locations and constraints for group - OK (0) =#=#=#=
	* Passed: crm_resource - Check locations and constraints for group
	=#=#=#= Begin test: Recursively check locations and constraints for group =#=#=#=
	Resources colocated with group:
	* prim7 (score=INFINITY, id=colocation-prim7-group-INFINITY)
	=#=#=#= End test: Recursively check locations and constraints for group - OK (0) =#=#=#=
	* Passed: crm_resource - Recursively check locations and constraints for group
	=#=#=#= Begin test: Check locations and constraints for group in XML =#=#=#=
	<pacemaker-result api-version="X" request="crm_resource -a -r group --output-as=xml">
	<constraints>
	<rsc_colocation id="colocation-prim7-group-INFINITY" rsc="prim7" with-rsc="group" score="INFINITY"/>
	</constraints>
	<status code="0" message="OK"/>
	</pacemaker-result>
	=#=#=#= End test: Check locations and constraints for group in XML - OK (0) =#=#=#=
	* Passed: crm_resource - Check locations and constraints for group in XML
	=#=#=#= Begin test: Recursively check locations and constraints for group in XML =#=#=#=
	<pacemaker-result api-version="X" request="crm_resource -A -r group --output-as=xml">
	<constraints>
	<rsc_colocation id="colocation-prim7-group-INFINITY" rsc="prim7" with-rsc="group" score="INFINITY"/>
	</constraints>
	<status code="0" message="OK"/>
	</pacemaker-result>
	=#=#=#= End test: Recursively check locations and constraints for group in XML - OK (0) =#=#=#=
	* Passed: crm_resource - Recursively check locations and constraints for group in XML
	=#=#=#= Begin test: Check locations and constraints for clone =#=#=#=
	Resources colocated with clone:
	* prim9 (score=INFINITY, id=colocation-prim9-clone-INFINITY)
	=#=#=#= End test: Check locations and constraints for clone - OK (0) =#=#=#=
	* Passed: crm_resource - Check locations and constraints for clone
	=#=#=#= Begin test: Recursively check locations and constraints for clone =#=#=#=
	Resources colocated with clone:
	* prim9 (score=INFINITY, id=colocation-prim9-clone-INFINITY)
	=#=#=#= End test: Recursively check locations and constraints for clone - OK (0) =#=#=#=
	* Passed: crm_resource - Recursively check locations and constraints for clone
	=#=#=#= Begin test: Check locations and constraints for clone in XML =#=#=#=
	<pacemaker-result api-version="X" request="crm_resource -a -r clone --output-as=xml">
	<constraints>
	<rsc_colocation id="colocation-prim9-clone-INFINITY" rsc="prim9" with-rsc="clone" score="INFINITY"/>
	</constraints>
	<status code="0" message="OK"/>
	</pacemaker-result>
	=#=#=#= End test: Check locations and constraints for clone in XML - OK (0) =#=#=#=
	* Passed: crm_resource - Check locations and constraints for clone in XML
	=#=#=#= Begin test: Recursively check locations and constraints for clone in XML =#=#=#=
	<pacemaker-result api-version="X" request="crm_resource -A -r clone --output-as=xml">
	<constraints>
	<rsc_colocation id="colocation-prim9-clone-INFINITY" rsc="prim9" with-rsc="clone" score="INFINITY"/>
	</constraints>
	<status code="0" message="OK"/>
	</pacemaker-result>
	=#=#=#= End test: Recursively check locations and constraints for clone in XML - OK (0) =#=#=#=
	* Passed: crm_resource - Recursively check locations and constraints for clone in XML
	=#=#=#= Begin test: Show resource digests =#=#=#=
	<pacemaker-result api-version="X" request="crm_resource --digests -r rsc1 -N node1 --output-as=xml">
	<digests resource="rsc1" node="node1" task="start" interval="0ms">
	<digest type="all" hash="3acdbe4c12734ebeb1251a59545af936">
	<parameters passwd="secret" fake="0"/>
	</digest>
	<digest type="nonprivate" hash="279c477dbc38c621904a00ab9e599b2f">
	<parameters fake="0"/>
	</digest>
	<digest type="nonreloadable" hash="5de1fd72a2e7762ed41543231034f6d7">
	<parameters passwd="secret"/>
	</digest>
	</digests>
	<status code="0" message="OK"/>
	</pacemaker-result>
	=#=#=#= End test: Show resource digests - OK (0) =#=#=#=
	* Passed: crm_resource - Show resource digests
	=#=#=#= Begin test: Show resource digests with overrides =#=#=#=
	<pacemaker-result api-version="X" request="crm_resource --digests -r rsc1 -N node1 --output-as=xml CRM_meta_interval=10000 CRM_meta_timeout=20000">
	<digests resource="rsc1" node="node1" task="start" interval="10000ms">
	<digest type="all" hash="720718e8d715d5d3be1403cbbcb953bc">
	<parameters passwd="secret" fake="0" CRM_meta_timeout="20000"/>
	</digest>
	<digest type="nonprivate" hash="279c477dbc38c621904a00ab9e599b2f">
	<parameters fake="0"/>
	</digest>
	<digest type="nonreloadable" hash="5de1fd72a2e7762ed41543231034f6d7">
	<parameters passwd="secret"/>
	</digest>
	</digests>
	<status code="0" message="OK"/>
	</pacemaker-result>
	=#=#=#= End test: Show resource digests with overrides - OK (0) =#=#=#=
	* Passed: crm_resource - Show resource digests with overrides
	=#=#=#= Begin test: List all nodes =#=#=#=
	11
	=#=#=#= End test: List all nodes - OK (0) =#=#=#=
	* Passed: crmadmin - List all nodes
	=#=#=#= Begin test: List cluster nodes =#=#=#=
	6
	=#=#=#= End test: List cluster nodes - OK (0) =#=#=#=
	* Passed: crmadmin - List cluster nodes
	=#=#=#= Begin test: List guest nodes =#=#=#=
	2
	=#=#=#= End test: List guest nodes - OK (0) =#=#=#=
	* Passed: crmadmin - List guest nodes
	=#=#=#= Begin test: List remote nodes =#=#=#=
	3
	=#=#=#= End test: List remote nodes - OK (0) =#=#=#=
	* Passed: crmadmin - List remote nodes
	=#=#=#= Begin test: List cluster,remote nodes =#=#=#=
	9
	=#=#=#= End test: List cluster,remote nodes - OK (0) =#=#=#=
	* Passed: crmadmin - List cluster,remote nodes
	=#=#=#= Begin test: List guest,remote nodes =#=#=#=
	5
	=#=#=#= End test: List guest,remote nodes - OK (0) =#=#=#=
	* Passed: crmadmin - List guest,remote nodes
	=#=#=#= Begin test: Show allocation scores with crm_simulate =#=#=#=
	<pacemaker-result api-version="X" request="crm_mon.xml --show-scores --output-as=xml">
	<cluster_status>
	<nodes>
	<node name="cluster01" id="1" online="true" standby="false" standby_onfail="false" maintenance="false" pending="false" unclean="false" shutdown="false" expected_up="true" is_dc="false" resources_running="7" type="member"/>
	<node name="cluster02" id="2" online="true" standby="false" standby_onfail="false" maintenance="false" pending="false" unclean="false" shutdown="false" expected_up="true" is_dc="true" resources_running="9" type="member"/>
	<node name="httpd-bundle-0" id="httpd-bundle-0" online="true" standby="false" standby_onfail="false" maintenance="false" pending="false" unclean="false" shutdown="false" expected_up="false" is_dc="false" resources_running="1" type="remote" id_as_resource="httpd-bundle-docker-0"/>
	<node name="httpd-bundle-1" id="httpd-bundle-1" online="true" standby="false" standby_onfail="false" maintenance="false" pending="false" unclean="false" shutdown="false" expected_up="false" is_dc="false" resources_running="1" type="remote" id_as_resource="httpd-bundle-docker-1"/>
	<node name="httpd-bundle-2" id="httpd-bundle-2" online="false" standby="false" standby_onfail="false" maintenance="false" pending="false" unclean="false" shutdown="false" expected_up="false" is_dc="false" resources_running="0" type="remote" id_as_resource="httpd-bundle-docker-2"/>
	</nodes>
	<resources>
	<clone id="ping-clone" multi_state="false" unique="false" managed="true" disabled="false" failed="false" failure_ignored="false">
	<resource id="ping" resource_agent="ocf:pacemaker:ping" role="Started" active="true" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="1">
	<node name="cluster02" id="2" cached="true"/>
	</resource>
	<resource id="ping" resource_agent="ocf:pacemaker:ping" role="Started" active="true" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="1">
	<node name="cluster01" id="1" cached="true"/>
	</resource>
	</clone>
	<resource id="Fencing" resource_agent="stonith:fence_xvm" role="Started" active="true" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="1">
	<node name="cluster01" id="1" cached="true"/>
	</resource>
	<resource id="dummy" resource_agent="ocf:pacemaker:Dummy" role="Started" active="true" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="1">
	<node name="cluster02" id="2" cached="true"/>
	</resource>
	<clone id="inactive-clone" multi_state="false" unique="false" managed="true" disabled="true" failed="false" failure_ignored="false" target_role="stopped">
	<resource id="inactive-dhcpd" resource_agent="lsb:dhcpd" role="Stopped" target_role="stopped" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
	<resource id="inactive-dhcpd" resource_agent="lsb:dhcpd" role="Stopped" target_role="stopped" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
	</clone>
	<group id="inactive-group" number_resources="2" managed="true" disabled="true">
	<resource id="inactive-dummy-1" resource_agent="ocf:pacemaker:Dummy" role="Stopped" target_role="stopped" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
	<resource id="inactive-dummy-2" resource_agent="ocf:pacemaker:Dummy" role="Stopped" target_role="stopped" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
	</group>
	<bundle id="httpd-bundle" type="docker" image="pcmk:http" unique="false" managed="true" failed="false">
	<replica id="0">
	<resource id="httpd-bundle-ip-192.168.122.131" resource_agent="ocf:heartbeat:IPaddr2" role="Started" target_role="Started" active="true" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="1">
	<node name="cluster01" id="1" cached="true"/>
	</resource>
	<resource id="httpd" resource_agent="ocf:heartbeat:apache" role="Started" target_role="Started" active="true" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="1">
	<node name="httpd-bundle-0" id="httpd-bundle-0" cached="true"/>
	</resource>
	<resource id="httpd-bundle-docker-0" resource_agent="ocf:heartbeat:docker" role="Started" target_role="Started" active="true" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="1">
	<node name="cluster01" id="1" cached="true"/>
	</resource>
	<resource id="httpd-bundle-0" resource_agent="ocf:pacemaker:remote" role="Started" target_role="Started" active="true" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="1">
	<node name="cluster01" id="1" cached="true"/>
	</resource>
	</replica>
	<replica id="1">
	<resource id="httpd-bundle-ip-192.168.122.132" resource_agent="ocf:heartbeat:IPaddr2" role="Started" target_role="Started" active="true" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="1">
	<node name="cluster02" id="2" cached="true"/>
	</resource>
	<resource id="httpd" resource_agent="ocf:heartbeat:apache" role="Started" target_role="Started" active="true" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="1">
	<node name="httpd-bundle-1" id="httpd-bundle-1" cached="true"/>
	</resource>
	<resource id="httpd-bundle-docker-1" resource_agent="ocf:heartbeat:docker" role="Started" target_role="Started" active="true" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="1">
	<node name="cluster02" id="2" cached="true"/>
	</resource>
	<resource id="httpd-bundle-1" resource_agent="ocf:pacemaker:remote" role="Started" target_role="Started" active="true" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="1">
	<node name="cluster02" id="2" cached="true"/>
	</resource>
	</replica>
	<replica id="2">
	<resource id="httpd-bundle-ip-192.168.122.133" resource_agent="ocf:heartbeat:IPaddr2" role="Stopped" target_role="Started" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
	<resource id="httpd" resource_agent="ocf:heartbeat:apache" role="Stopped" target_role="Started" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
	<resource id="httpd-bundle-docker-2" resource_agent="ocf:heartbeat:docker" role="Stopped" target_role="Started" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
	<resource id="httpd-bundle-2" resource_agent="ocf:pacemaker:remote" role="Stopped" target_role="Started" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
	</replica>
	</bundle>
	<group id="exim-group" number_resources="2" managed="true" disabled="false">
	<resource id="Public-IP" resource_agent="ocf:heartbeat:IPaddr" role="Started" active="true" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="1">
	<node name="cluster02" id="2" cached="true"/>
	</resource>
	<resource id="Email" resource_agent="lsb:exim" role="Started" active="true" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="1">
	<node name="cluster02" id="2" cached="true"/>
	</resource>
	</group>
	<clone id="mysql-clone-group" multi_state="false" unique="false" managed="true" disabled="false" failed="false" failure_ignored="false">
	<group id="mysql-group:0" number_resources="1" managed="true" disabled="false">
	<resource id="mysql-proxy" resource_agent="lsb:mysql-proxy" role="Started" active="true" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="1">
	<node name="cluster02" id="2" cached="true"/>
	</resource>
	</group>
	<group id="mysql-group:1" number_resources="1" managed="true" disabled="false">
	<resource id="mysql-proxy" resource_agent="lsb:mysql-proxy" role="Started" active="true" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="1">
	<node name="cluster01" id="1" cached="true"/>
	</resource>
	</group>
	<group id="mysql-group:2" number_resources="1" managed="true" disabled="false">
	<resource id="mysql-proxy" resource_agent="lsb:mysql-proxy" role="Stopped" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
	</group>
	<group id="mysql-group:3" number_resources="1" managed="true" disabled="false">
	<resource id="mysql-proxy" resource_agent="lsb:mysql-proxy" role="Stopped" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
	</group>
	<group id="mysql-group:4" number_resources="1" managed="true" disabled="false">
	<resource id="mysql-proxy" resource_agent="lsb:mysql-proxy" role="Stopped" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
	</group>
	</clone>
	<clone id="promotable-clone" multi_state="true" unique="false" managed="true" disabled="false" failed="false" failure_ignored="false">
	<resource id="promotable-rsc" resource_agent="ocf:pacemaker:Stateful" role="Promoted" active="true" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="1">
	<node name="cluster02" id="2" cached="true"/>
	</resource>
	<resource id="promotable-rsc" resource_agent="ocf:pacemaker:Stateful" role="Unpromoted" active="true" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="1">
	<node name="cluster01" id="1" cached="true"/>
	</resource>
	<resource id="promotable-rsc" resource_agent="ocf:pacemaker:Stateful" role="Stopped" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
	<resource id="promotable-rsc" resource_agent="ocf:pacemaker:Stateful" role="Stopped" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
	<resource id="promotable-rsc" resource_agent="ocf:pacemaker:Stateful" role="Stopped" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
	</clone>
	</resources>
	</cluster_status>
	<allocations>
	<node_weight function="pcmk__clone_allocate" node="cluster01" score="0" id="ping-clone"/>
	<node_weight function="pcmk__clone_allocate" node="cluster02" score="0" id="ping-clone"/>
	<node_weight function="pcmk__clone_allocate" node="cluster01" score="0" id="ping:0"/>
	<node_weight function="pcmk__clone_allocate" node="cluster02" score="1" id="ping:0"/>
	<node_weight function="pcmk__clone_allocate" node="cluster01" score="1" id="ping:1"/>
	<node_weight function="pcmk__clone_allocate" node="cluster02" score="0" id="ping:1"/>
	<node_weight function="pcmk__native_allocate" node="cluster01" score="0" id="ping:0"/>
	<node_weight function="pcmk__native_allocate" node="cluster02" score="1" id="ping:0"/>
	<node_weight function="pcmk__native_allocate" node="cluster01" score="1" id="ping:1"/>
	<node_weight function="pcmk__native_allocate" node="cluster02" score="-INFINITY" id="ping:1"/>
	<node_weight function="pcmk__native_allocate" node="cluster01" score="0" id="Fencing"/>
	<node_weight function="pcmk__native_allocate" node="cluster02" score="0" id="Fencing"/>
	<node_weight function="pcmk__native_allocate" node="cluster01" score="-INFINITY" id="dummy"/>
	<node_weight function="pcmk__native_allocate" node="cluster02" score="0" id="dummy"/>
	<node_weight function="pcmk__clone_allocate" node="cluster01" score="0" id="inactive-clone"/>
	<node_weight function="pcmk__clone_allocate" node="cluster02" score="0" id="inactive-clone"/>
	<node_weight function="pcmk__clone_allocate" node="cluster01" score="0" id="inactive-dhcpd:0"/>
	<node_weight function="pcmk__clone_allocate" node="cluster02" score="0" id="inactive-dhcpd:0"/>
	<node_weight function="pcmk__clone_allocate" node="cluster01" score="0" id="inactive-dhcpd:1"/>
	<node_weight function="pcmk__clone_allocate" node="cluster02" score="0" id="inactive-dhcpd:1"/>
	<node_weight function="pcmk__native_allocate" node="cluster01" score="-INFINITY" id="inactive-dhcpd:0"/>
	<node_weight function="pcmk__native_allocate" node="cluster02" score="-INFINITY" id="inactive-dhcpd:0"/>
	<node_weight function="pcmk__native_allocate" node="httpd-bundle-0" score="-INFINITY" id="inactive-dhcpd:0"/>
	<node_weight function="pcmk__native_allocate" node="httpd-bundle-1" score="-INFINITY" id="inactive-dhcpd:0"/>
	<node_weight function="pcmk__native_allocate" node="httpd-bundle-2" score="-INFINITY" id="inactive-dhcpd:0"/>
	<node_weight function="pcmk__native_allocate" node="cluster01" score="-INFINITY" id="inactive-dhcpd:1"/>
	<node_weight function="pcmk__native_allocate" node="cluster02" score="-INFINITY" id="inactive-dhcpd:1"/>
	<node_weight function="pcmk__native_allocate" node="httpd-bundle-0" score="-INFINITY" id="inactive-dhcpd:1"/>
	<node_weight function="pcmk__native_allocate" node="httpd-bundle-1" score="-INFINITY" id="inactive-dhcpd:1"/>
	<node_weight function="pcmk__native_allocate" node="httpd-bundle-2" score="-INFINITY" id="inactive-dhcpd:1"/>
	<node_weight function="pcmk__group_allocate" node="cluster01" score="0" id="inactive-group"/>
	<node_weight function="pcmk__group_allocate" node="cluster02" score="0" id="inactive-group"/>
	<node_weight function="pcmk__group_allocate" node="cluster01" score="0" id="inactive-dummy-1"/>
	<node_weight function="pcmk__group_allocate" node="cluster02" score="0" id="inactive-dummy-1"/>
	<node_weight function="pcmk__group_allocate" node="cluster01" score="0" id="inactive-dummy-2"/>
	<node_weight function="pcmk__group_allocate" node="cluster02" score="0" id="inactive-dummy-2"/>
	<node_weight function="pcmk__native_allocate" node="cluster01" score="-INFINITY" id="inactive-dummy-1"/>
	<node_weight function="pcmk__native_allocate" node="cluster02" score="-INFINITY" id="inactive-dummy-1"/>
	<node_weight function="pcmk__native_allocate" node="httpd-bundle-0" score="-INFINITY" id="inactive-dummy-1"/>
	<node_weight function="pcmk__native_allocate" node="httpd-bundle-1" score="-INFINITY" id="inactive-dummy-1"/>
	<node_weight function="pcmk__native_allocate" node="httpd-bundle-2" score="-INFINITY" id="inactive-dummy-1"/>
	<node_weight function="pcmk__native_allocate" node="cluster01" score="-INFINITY" id="inactive-dummy-2"/>
	<node_weight function="pcmk__native_allocate" node="cluster02" score="-INFINITY" id="inactive-dummy-2"/>
	<node_weight function="pcmk__native_allocate" node="httpd-bundle-0" score="-INFINITY" id="inactive-dummy-2"/>
	<node_weight function="pcmk__native_allocate" node="httpd-bundle-1" score="-INFINITY" id="inactive-dummy-2"/>
	<node_weight function="pcmk__native_allocate" node="httpd-bundle-2" score="-INFINITY" id="inactive-dummy-2"/>
	<node_weight function="pcmk__bundle_allocate" node="cluster01" score="0" id="httpd-bundle"/>
	<node_weight function="pcmk__bundle_allocate" node="cluster02" score="0" id="httpd-bundle"/>
	<node_weight function="pcmk__bundle_allocate" node="cluster01" score="0" id="httpd-bundle-docker-0"/>
	<node_weight function="pcmk__bundle_allocate" node="cluster02" score="0" id="httpd-bundle-docker-0"/>
	<node_weight function="pcmk__bundle_allocate" node="cluster01" score="0" id="httpd-bundle-ip-192.168.122.131"/>
	<node_weight function="pcmk__bundle_allocate" node="cluster02" score="0" id="httpd-bundle-ip-192.168.122.131"/>
	<node_weight function="pcmk__bundle_allocate" node="cluster01" score="0" id="httpd-bundle-0"/>
	<node_weight function="pcmk__bundle_allocate" node="cluster02" score="0" id="httpd-bundle-0"/>
	<node_weight function="pcmk__bundle_allocate" node="cluster01" score="0" id="httpd-bundle-docker-1"/>
	<node_weight function="pcmk__bundle_allocate" node="cluster02" score="0" id="httpd-bundle-docker-1"/>
	<node_weight function="pcmk__bundle_allocate" node="cluster01" score="0" id="httpd-bundle-ip-192.168.122.132"/>
	<node_weight function="pcmk__bundle_allocate" node="cluster02" score="0" id="httpd-bundle-ip-192.168.122.132"/>
	<node_weight function="pcmk__bundle_allocate" node="cluster01" score="0" id="httpd-bundle-1"/>
	<node_weight function="pcmk__bundle_allocate" node="cluster02" score="0" id="httpd-bundle-1"/>
	<node_weight function="pcmk__bundle_allocate" node="cluster01" score="0" id="httpd-bundle-docker-2"/>
	<node_weight function="pcmk__bundle_allocate" node="cluster02" score="0" id="httpd-bundle-docker-2"/>
	<node_weight function="pcmk__bundle_allocate" node="cluster01" score="0" id="httpd-bundle-ip-192.168.122.133"/>
	<node_weight function="pcmk__bundle_allocate" node="cluster02" score="0" id="httpd-bundle-ip-192.168.122.133"/>
	<node_weight function="pcmk__bundle_allocate" node="cluster01" score="0" id="httpd-bundle-2"/>
	<node_weight function="pcmk__bundle_allocate" node="cluster02" score="0" id="httpd-bundle-2"/>
	<node_weight function="pcmk__bundle_allocate" node="cluster01" score="0" id="httpd-bundle-clone"/>
	<node_weight function="pcmk__bundle_allocate" node="cluster02" score="0" id="httpd-bundle-clone"/>
	<node_weight function="pcmk__bundle_allocate" node="httpd-bundle-0" score="-INFINITY" id="httpd-bundle-clone"/>
	<node_weight function="pcmk__bundle_allocate" node="httpd-bundle-1" score="-INFINITY" id="httpd-bundle-clone"/>
	<node_weight function="pcmk__bundle_allocate" node="httpd-bundle-2" score="-INFINITY" id="httpd-bundle-clone"/>
	<node_weight function="pcmk__bundle_allocate" node="httpd-bundle-0" score="501" id="httpd:0"/>
	<node_weight function="pcmk__bundle_allocate" node="httpd-bundle-1" score="501" id="httpd:1"/>
	<node_weight function="pcmk__bundle_allocate" node="httpd-bundle-2" score="500" id="httpd:2"/>
	<node_weight function="pcmk__native_allocate" node="cluster01" score="0" id="httpd-bundle-docker-0"/>
	<node_weight function="pcmk__native_allocate" node="cluster02" score="0" id="httpd-bundle-docker-0"/>
	<node_weight function="pcmk__native_allocate" node="cluster01" score="-INFINITY" id="httpd-bundle-docker-1"/>
	<node_weight function="pcmk__native_allocate" node="cluster02" score="0" id="httpd-bundle-docker-1"/>
	<node_weight function="pcmk__native_allocate" node="cluster01" score="-INFINITY" id="httpd-bundle-docker-2"/>
	<node_weight function="pcmk__native_allocate" node="cluster02" score="-INFINITY" id="httpd-bundle-docker-2"/>
	<node_weight function="pcmk__native_allocate" node="cluster01" score="0" id="httpd-bundle-ip-192.168.122.131"/>
	<node_weight function="pcmk__native_allocate" node="cluster02" score="-INFINITY" id="httpd-bundle-ip-192.168.122.131"/>
	<node_weight function="pcmk__native_allocate" node="cluster01" score="10000" id="httpd-bundle-0"/>
	<node_weight function="pcmk__native_allocate" node="cluster02" score="0" id="httpd-bundle-0"/>
	<node_weight function="pcmk__native_allocate" node="httpd-bundle-0" score="INFINITY" id="httpd:0"/>
	<node_weight function="pcmk__native_allocate" node="cluster01" score="-INFINITY" id="httpd-bundle-ip-192.168.122.132"/>
	<node_weight function="pcmk__native_allocate" node="cluster02" score="0" id="httpd-bundle-ip-192.168.122.132"/>
	<node_weight function="pcmk__native_allocate" node="cluster01" score="0" id="httpd-bundle-1"/>
	<node_weight function="pcmk__native_allocate" node="cluster02" score="10000" id="httpd-bundle-1"/>
	<node_weight function="pcmk__native_allocate" node="httpd-bundle-1" score="INFINITY" id="httpd:1"/>
	<node_weight function="pcmk__native_allocate" node="cluster01" score="-INFINITY" id="httpd-bundle-ip-192.168.122.133"/>
	<node_weight function="pcmk__native_allocate" node="cluster02" score="-INFINITY" id="httpd-bundle-ip-192.168.122.133"/>
	<node_weight function="pcmk__native_allocate" node="cluster01" score="0" id="httpd-bundle-2"/>
	<node_weight function="pcmk__native_allocate" node="cluster02" score="0" id="httpd-bundle-2"/>
	<node_weight function="pcmk__native_allocate" node="httpd-bundle-2" score="INFINITY" id="httpd:2"/>
	<node_weight function="pcmk__clone_allocate" node="cluster01" score="-INFINITY" id="httpd-bundle-clone"/>
	<node_weight function="pcmk__clone_allocate" node="cluster02" score="-INFINITY" id="httpd-bundle-clone"/>
	<node_weight function="pcmk__clone_allocate" node="httpd-bundle-0" score="0" id="httpd-bundle-clone"/>
	<node_weight function="pcmk__clone_allocate" node="httpd-bundle-1" score="0" id="httpd-bundle-clone"/>
	<node_weight function="pcmk__clone_allocate" node="httpd-bundle-2" score="0" id="httpd-bundle-clone"/>
	<node_weight function="pcmk__clone_allocate" node="httpd-bundle-0" score="INFINITY" id="httpd:0"/>
	<node_weight function="pcmk__clone_allocate" node="httpd-bundle-1" score="INFINITY" id="httpd:1"/>
	<node_weight function="pcmk__clone_allocate" node="httpd-bundle-2" score="INFINITY" id="httpd:2"/>
	<node_weight function="pcmk__group_allocate" node="cluster01" score="0" id="exim-group"/>
	<node_weight function="pcmk__group_allocate" node="cluster02" score="0" id="exim-group"/>
	<node_weight function="pcmk__group_allocate" node="cluster01" score="0" id="Public-IP"/>
	<node_weight function="pcmk__group_allocate" node="cluster02" score="0" id="Public-IP"/>
	<node_weight function="pcmk__group_allocate" node="cluster01" score="0" id="Email"/>
	<node_weight function="pcmk__group_allocate" node="cluster02" score="0" id="Email"/>
	<node_weight function="pcmk__native_allocate" node="cluster01" score="0" id="Public-IP"/>
	<node_weight function="pcmk__native_allocate" node="cluster02" score="0" id="Public-IP"/>
	<node_weight function="pcmk__native_allocate" node="cluster01" score="-INFINITY" id="Email"/>
	<node_weight function="pcmk__native_allocate" node="cluster02" score="0" id="Email"/>
	<node_weight function="pcmk__clone_allocate" node="cluster01" score="0" id="mysql-clone-group"/>
	<node_weight function="pcmk__clone_allocate" node="cluster02" score="0" id="mysql-clone-group"/>
	<node_weight function="pcmk__clone_allocate" node="cluster01" score="0" id="mysql-group:0"/>
	<node_weight function="pcmk__clone_allocate" node="cluster02" score="0" id="mysql-group:0"/>
	<node_weight function="pcmk__clone_allocate" node="cluster01" score="0" id="mysql-proxy:0"/>
	<node_weight function="pcmk__clone_allocate" node="cluster02" score="1" id="mysql-proxy:0"/>
	<node_weight function="pcmk__clone_allocate" node="cluster01" score="0" id="mysql-group:1"/>
	<node_weight function="pcmk__clone_allocate" node="cluster02" score="0" id="mysql-group:1"/>
	<node_weight function="pcmk__clone_allocate" node="cluster01" score="1" id="mysql-proxy:1"/>
	<node_weight function="pcmk__clone_allocate" node="cluster02" score="0" id="mysql-proxy:1"/>
	<node_weight function="pcmk__clone_allocate" node="cluster01" score="0" id="mysql-group:2"/>
	<node_weight function="pcmk__clone_allocate" node="cluster02" score="0" id="mysql-group:2"/>
	<node_weight function="pcmk__clone_allocate" node="cluster01" score="0" id="mysql-proxy:2"/>
	<node_weight function="pcmk__clone_allocate" node="cluster02" score="0" id="mysql-proxy:2"/>
	<node_weight function="pcmk__clone_allocate" node="cluster01" score="0" id="mysql-group:3"/>
	<node_weight function="pcmk__clone_allocate" node="cluster02" score="0" id="mysql-group:3"/>
	<node_weight function="pcmk__clone_allocate" node="cluster01" score="0" id="mysql-proxy:3"/>
	<node_weight function="pcmk__clone_allocate" node="cluster02" score="0" id="mysql-proxy:3"/>
	<node_weight function="pcmk__clone_allocate" node="cluster01" score="0" id="mysql-group:4"/>
	<node_weight function="pcmk__clone_allocate" node="cluster02" score="0" id="mysql-group:4"/>
	<node_weight function="pcmk__clone_allocate" node="cluster01" score="0" id="mysql-proxy:4"/>
	<node_weight function="pcmk__clone_allocate" node="cluster02" score="0" id="mysql-proxy:4"/>
	<node_weight function="pcmk__group_allocate" node="cluster01" score="0" id="mysql-group:0"/>
	<node_weight function="pcmk__group_allocate" node="cluster02" score="0" id="mysql-group:0"/>
	<node_weight function="pcmk__group_allocate" node="cluster01" score="0" id="mysql-proxy:0"/>
	<node_weight function="pcmk__group_allocate" node="cluster02" score="1" id="mysql-proxy:0"/>
	<node_weight function="pcmk__native_allocate" node="cluster01" score="0" id="mysql-proxy:0"/>
	<node_weight function="pcmk__native_allocate" node="cluster02" score="1" id="mysql-proxy:0"/>
	<node_weight function="pcmk__group_allocate" node="cluster01" score="0" id="mysql-group:1"/>
	<node_weight function="pcmk__group_allocate" node="cluster02" score="-INFINITY" id="mysql-group:1"/>
	<node_weight function="pcmk__group_allocate" node="cluster01" score="1" id="mysql-proxy:1"/>
	<node_weight function="pcmk__group_allocate" node="cluster02" score="-INFINITY" id="mysql-proxy:1"/>
	<node_weight function="pcmk__native_allocate" node="cluster01" score="1" id="mysql-proxy:1"/>
	<node_weight function="pcmk__native_allocate" node="cluster02" score="-INFINITY" id="mysql-proxy:1"/>
	<node_weight function="pcmk__group_allocate" node="cluster01" score="-INFINITY" id="mysql-group:2"/>
	<node_weight function="pcmk__group_allocate" node="cluster02" score="-INFINITY" id="mysql-group:2"/>
	<node_weight function="pcmk__group_allocate" node="cluster01" score="-INFINITY" id="mysql-proxy:2"/>
	<node_weight function="pcmk__group_allocate" node="cluster02" score="-INFINITY" id="mysql-proxy:2"/>
	<node_weight function="pcmk__native_allocate" node="cluster01" score="-INFINITY" id="mysql-proxy:2"/>
	<node_weight function="pcmk__native_allocate" node="cluster02" score="-INFINITY" id="mysql-proxy:2"/>
	<node_weight function="pcmk__group_allocate" node="cluster01" score="-INFINITY" id="mysql-group:3"/>
	<node_weight function="pcmk__group_allocate" node="cluster02" score="-INFINITY" id="mysql-group:3"/>
	<node_weight function="pcmk__group_allocate" node="cluster01" score="-INFINITY" id="mysql-proxy:3"/>
	<node_weight function="pcmk__group_allocate" node="cluster02" score="-INFINITY" id="mysql-proxy:3"/>
	<node_weight function="pcmk__native_allocate" node="cluster01" score="-INFINITY" id="mysql-proxy:3"/>
	<node_weight function="pcmk__native_allocate" node="cluster02" score="-INFINITY" id="mysql-proxy:3"/>
	<node_weight function="pcmk__group_allocate" node="cluster01" score="-INFINITY" id="mysql-group:4"/>
	<node_weight function="pcmk__group_allocate" node="cluster02" score="-INFINITY" id="mysql-group:4"/>
	<node_weight function="pcmk__group_allocate" node="cluster01" score="-INFINITY" id="mysql-proxy:4"/>
	<node_weight function="pcmk__group_allocate" node="cluster02" score="-INFINITY" id="mysql-proxy:4"/>
	<node_weight function="pcmk__native_allocate" node="cluster01" score="-INFINITY" id="mysql-proxy:4"/>
	<node_weight function="pcmk__native_allocate" node="cluster02" score="-INFINITY" id="mysql-proxy:4"/>
	<node_weight function="pcmk__clone_allocate" node="cluster01" score="0" id="promotable-clone"/>
	<node_weight function="pcmk__clone_allocate" node="cluster02" score="0" id="promotable-clone"/>
	<node_weight function="pcmk__clone_allocate" node="cluster01" score="0" id="promotable-rsc:0"/>
	<node_weight function="pcmk__clone_allocate" node="cluster02" score="1" id="promotable-rsc:0"/>
	<node_weight function="pcmk__clone_allocate" node="cluster01" score="1" id="promotable-rsc:1"/>
	<node_weight function="pcmk__clone_allocate" node="cluster02" score="0" id="promotable-rsc:1"/>
	<node_weight function="pcmk__clone_allocate" node="cluster01" score="0" id="promotable-rsc:2"/>
	<node_weight function="pcmk__clone_allocate" node="cluster02" score="0" id="promotable-rsc:2"/>
	<node_weight function="pcmk__clone_allocate" node="cluster01" score="0" id="promotable-rsc:3"/>
	<node_weight function="pcmk__clone_allocate" node="cluster02" score="0" id="promotable-rsc:3"/>
	<node_weight function="pcmk__clone_allocate" node="cluster01" score="0" id="promotable-rsc:4"/>
	<node_weight function="pcmk__clone_allocate" node="cluster02" score="0" id="promotable-rsc:4"/>
	<node_weight function="pcmk__native_allocate" node="cluster01" score="0" id="promotable-rsc:0"/>
	<node_weight function="pcmk__native_allocate" node="cluster02" score="1" id="promotable-rsc:0"/>
	<node_weight function="pcmk__native_allocate" node="cluster01" score="1" id="promotable-rsc:1"/>
	<node_weight function="pcmk__native_allocate" node="cluster02" score="-INFINITY" id="promotable-rsc:1"/>
	<node_weight function="pcmk__native_allocate" node="cluster01" score="-INFINITY" id="promotable-rsc:2"/>
	<node_weight function="pcmk__native_allocate" node="cluster02" score="-INFINITY" id="promotable-rsc:2"/>
	<node_weight function="pcmk__native_allocate" node="cluster01" score="-INFINITY" id="promotable-rsc:3"/>
	<node_weight function="pcmk__native_allocate" node="cluster02" score="-INFINITY" id="promotable-rsc:3"/>
	<node_weight function="pcmk__native_allocate" node="cluster01" score="-INFINITY" id="promotable-rsc:4"/>
	<node_weight function="pcmk__native_allocate" node="cluster02" score="-INFINITY" id="promotable-rsc:4"/>
	<promotion_score id="promotable-rsc:0" score="9" node="cluster02"/>
	<promotion_score id="promotable-rsc:2" score="0"/>
	<promotion_score id="promotable-rsc:3" score="0"/>
	<promotion_score id="promotable-rsc:4" score="0"/>
	<promotion_score id="promotable-rsc:1" score="-1" node="cluster01"/>
	</allocations>
	<actions>
	<rsc_action action="start" resource="httpd-bundle-2" node="cluster01" reason="unrunnable httpd-bundle-docker-2 start" blocked="true"/>
	<rsc_action action="start" resource="httpd:2" node="httpd-bundle-2" reason="unrunnable httpd-bundle-docker-2 start" blocked="true"/>
	</actions>
	<status code="0" message="OK"/>
	</pacemaker-result>
	=#=#=#= End test: Show allocation scores with crm_simulate - OK (0) =#=#=#=
	* Passed: crm_simulate - Show allocation scores with crm_simulate
	=#=#=#= Begin test: Show utilization with crm_simulate =#=#=#=
	4 of 32 resource instances DISABLED and 0 BLOCKED from further action due to failure

	[ cluster01 cluster02 ]
	[ httpd-bundle-0@cluster01 httpd-bundle-1@cluster02 ]

	Started: [ cluster01 cluster02 ]
	Fencing (stonith:fence_xvm): Started cluster01
	dummy (ocf:pacemaker:Dummy): Started cluster02
	Stopped (disabled): [ cluster01 cluster02 ]
	inactive-dummy-1 (ocf:pacemaker:Dummy): Stopped (disabled)
	inactive-dummy-2 (ocf:pacemaker:Dummy): Stopped (disabled)
	httpd-bundle-0 (192.168.122.131) (ocf:heartbeat:apache): Started cluster01
	httpd-bundle-1 (192.168.122.132) (ocf:heartbeat:apache): Started cluster02
	httpd-bundle-2 (192.168.122.133) (ocf:heartbeat:apache): Stopped
	Public-IP (ocf:heartbeat:IPaddr): Started cluster02
	Email (lsb:exim): Started cluster02
	Started: [ cluster01 cluster02 ]
	Promoted: [ cluster02 ]
	Unpromoted: [ cluster01 ]

	Only 'private' parameters to 60s-interval monitor for dummy on cluster02 changed: 0:0;16:2:0:4a9e64d6-e1dd-4395-917c-1596312eafe4
	Original: cluster01 capacity:
	Original: cluster02 capacity:
	Original: httpd-bundle-0 capacity:
	Original: httpd-bundle-1 capacity:
	Original: httpd-bundle-2 capacity:
	pcmk__assign_primitive: ping:0 utilization on cluster02:
	pcmk__assign_primitive: ping:1 utilization on cluster01:
	pcmk__assign_primitive: Fencing utilization on cluster01:
	pcmk__assign_primitive: dummy utilization on cluster02:
	pcmk__assign_primitive: httpd-bundle-docker-0 utilization on cluster01:
	pcmk__assign_primitive: httpd-bundle-docker-1 utilization on cluster02:
	pcmk__assign_primitive: httpd-bundle-ip-192.168.122.131 utilization on cluster01:
	pcmk__assign_primitive: httpd-bundle-0 utilization on cluster01:
	pcmk__assign_primitive: httpd:0 utilization on httpd-bundle-0:
	pcmk__assign_primitive: httpd-bundle-ip-192.168.122.132 utilization on cluster02:
	pcmk__assign_primitive: httpd-bundle-1 utilization on cluster02:
	pcmk__assign_primitive: httpd:1 utilization on httpd-bundle-1:
	pcmk__assign_primitive: httpd-bundle-2 utilization on cluster01:
	pcmk__assign_primitive: httpd:2 utilization on httpd-bundle-2:
	pcmk__assign_primitive: Public-IP utilization on cluster02:
	pcmk__assign_primitive: Email utilization on cluster02:
	pcmk__assign_primitive: mysql-proxy:0 utilization on cluster02:
	pcmk__assign_primitive: mysql-proxy:1 utilization on cluster01:
	pcmk__assign_primitive: promotable-rsc:0 utilization on cluster02:
	pcmk__assign_primitive: promotable-rsc:1 utilization on cluster01:
	Remaining: cluster01 capacity:
	Remaining: cluster02 capacity:
	Remaining: httpd-bundle-0 capacity:
	Remaining: httpd-bundle-1 capacity:
	Remaining: httpd-bundle-2 capacity:

	Start httpd-bundle-2 ( cluster01 ) due to unrunnable httpd-bundle-docker-2 start (blocked)
	Start httpd:2 ( httpd-bundle-2 ) due to unrunnable httpd-bundle-docker-2 start (blocked)
	=#=#=#= End test: Show utilization with crm_simulate - OK (0) =#=#=#=
	* Passed: crm_simulate - Show utilization with crm_simulate
	=#=#=#= Begin test: Simulate injecting a failure =#=#=#=
	4 of 32 resource instances DISABLED and 0 BLOCKED from further action due to failure

	Current cluster status:
	* Node List:
	* Online: [ cluster01 cluster02 ]
	* GuestOnline: [ httpd-bundle-0@cluster01 httpd-bundle-1@cluster02 ]

	* Full List of Resources:
	* Clone Set: ping-clone [ping]:
	* Started: [ cluster01 cluster02 ]
	* Fencing (stonith:fence_xvm): Started cluster01
	* dummy (ocf:pacemaker:Dummy): Started cluster02
	* Clone Set: inactive-clone [inactive-dhcpd] (disabled):
	* Stopped (disabled): [ cluster01 cluster02 ]
	* Resource Group: inactive-group (disabled):
	* inactive-dummy-1 (ocf:pacemaker:Dummy): Stopped (disabled)
	* inactive-dummy-2 (ocf:pacemaker:Dummy): Stopped (disabled)
	* Container bundle set: httpd-bundle [pcmk:http]:
	* httpd-bundle-0 (192.168.122.131) (ocf:heartbeat:apache): Started cluster01
	* httpd-bundle-1 (192.168.122.132) (ocf:heartbeat:apache): Started cluster02
	* httpd-bundle-2 (192.168.122.133) (ocf:heartbeat:apache): Stopped
	* Resource Group: exim-group:
	* Public-IP (ocf:heartbeat:IPaddr): Started cluster02
	* Email (lsb:exim): Started cluster02
	* Clone Set: mysql-clone-group [mysql-group]:
	* Started: [ cluster01 cluster02 ]
	* Clone Set: promotable-clone [promotable-rsc] (promotable):
	* Promoted: [ cluster02 ]
	* Unpromoted: [ cluster01 ]

	Performing Requested Modifications:
	* Injecting ping_monitor_10000@cluster02=1 into the configuration
	* Injecting attribute fail-count-ping#monitor_10000=value++ into /node_state '2'
	* Injecting attribute last-failure-ping#monitor_10000= into /node_state '2'

	Transition Summary:
	* Recover ping:0 ( cluster02 )
	* Start httpd-bundle-2 ( cluster01 ) due to unrunnable httpd-bundle-docker-2 start (blocked)
	* Start httpd:2 ( httpd-bundle-2 ) due to unrunnable httpd-bundle-docker-2 start (blocked)

	Executing Cluster Transition:
	* Cluster action: clear_failcount for ping on cluster02
	* Pseudo action: ping-clone_stop_0
	* Pseudo action: httpd-bundle_start_0
	* Resource action: ping stop on cluster02
	* Pseudo action: ping-clone_stopped_0
	* Pseudo action: ping-clone_start_0
	* Pseudo action: httpd-bundle-clone_start_0
	* Resource action: ping start on cluster02
	* Resource action: ping monitor=10000 on cluster02
	* Pseudo action: ping-clone_running_0
	* Pseudo action: httpd-bundle-clone_running_0
	* Pseudo action: httpd-bundle_running_0

	Revised Cluster Status:
	* Node List:
	* Online: [ cluster01 cluster02 ]
	* GuestOnline: [ httpd-bundle-0@cluster01 httpd-bundle-1@cluster02 ]

	* Full List of Resources:
	* Clone Set: ping-clone [ping]:
	* Started: [ cluster01 cluster02 ]
	* Fencing (stonith:fence_xvm): Started cluster01
	* dummy (ocf:pacemaker:Dummy): Started cluster02
	* Clone Set: inactive-clone [inactive-dhcpd] (disabled):
	* Stopped (disabled): [ cluster01 cluster02 ]
	* Resource Group: inactive-group (disabled):
	* inactive-dummy-1 (ocf:pacemaker:Dummy): Stopped (disabled)
	* inactive-dummy-2 (ocf:pacemaker:Dummy): Stopped (disabled)
	* Container bundle set: httpd-bundle [pcmk:http]:
	* httpd-bundle-0 (192.168.122.131) (ocf:heartbeat:apache): Started cluster01
	* httpd-bundle-1 (192.168.122.132) (ocf:heartbeat:apache): Started cluster02
	* httpd-bundle-2 (192.168.122.133) (ocf:heartbeat:apache): Stopped
	* Resource Group: exim-group:
	* Public-IP (ocf:heartbeat:IPaddr): Started cluster02
	* Email (lsb:exim): Started cluster02
	* Clone Set: mysql-clone-group [mysql-group]:
	* Started: [ cluster01 cluster02 ]
	* Clone Set: promotable-clone [promotable-rsc] (promotable):
	* Promoted: [ cluster02 ]
	* Unpromoted: [ cluster01 ]
	=#=#=#= End test: Simulate injecting a failure - OK (0) =#=#=#=
	* Passed: crm_simulate - Simulate injecting a failure
	=#=#=#= Begin test: Simulate bringing a node down =#=#=#=
	4 of 32 resource instances DISABLED and 0 BLOCKED from further action due to failure

	Current cluster status:
	* Node List:
	* Online: [ cluster01 cluster02 ]
	* GuestOnline: [ httpd-bundle-0@cluster01 httpd-bundle-1@cluster02 ]

	* Full List of Resources:
	* Clone Set: ping-clone [ping]:
	* Started: [ cluster01 cluster02 ]
	* Fencing (stonith:fence_xvm): Started cluster01
	* dummy (ocf:pacemaker:Dummy): Started cluster02
	* Clone Set: inactive-clone [inactive-dhcpd] (disabled):
	* Stopped (disabled): [ cluster01 cluster02 ]
	* Resource Group: inactive-group (disabled):
	* inactive-dummy-1 (ocf:pacemaker:Dummy): Stopped (disabled)
	* inactive-dummy-2 (ocf:pacemaker:Dummy): Stopped (disabled)
	* Container bundle set: httpd-bundle [pcmk:http]:
	* httpd-bundle-0 (192.168.122.131) (ocf:heartbeat:apache): Started cluster01
	* httpd-bundle-1 (192.168.122.132) (ocf:heartbeat:apache): Started cluster02
	* httpd-bundle-2 (192.168.122.133) (ocf:heartbeat:apache): Stopped
	* Resource Group: exim-group:
	* Public-IP (ocf:heartbeat:IPaddr): Started cluster02
	* Email (lsb:exim): Started cluster02
	* Clone Set: mysql-clone-group [mysql-group]:
	* Started: [ cluster01 cluster02 ]
	* Clone Set: promotable-clone [promotable-rsc] (promotable):
	* Promoted: [ cluster02 ]
	* Unpromoted: [ cluster01 ]

	Performing Requested Modifications:
	* Taking node cluster01 offline

	Transition Summary:
	* Fence (off) httpd-bundle-0 (resource: httpd-bundle-docker-0) 'guest is unclean'
	* Start Fencing ( cluster02 )
	* Start httpd-bundle-0 ( cluster02 ) due to unrunnable httpd-bundle-docker-0 start (blocked)
	* Stop httpd:0 ( httpd-bundle-0 ) due to unrunnable httpd-bundle-docker-0 start
	* Start httpd-bundle-2 ( cluster02 ) due to unrunnable httpd-bundle-docker-2 start (blocked)
	* Start httpd:2 ( httpd-bundle-2 ) due to unrunnable httpd-bundle-docker-2 start (blocked)

	Executing Cluster Transition:
	* Resource action: Fencing start on cluster02
	* Pseudo action: stonith-httpd-bundle-0-off on httpd-bundle-0
	* Pseudo action: httpd-bundle_stop_0
	* Pseudo action: httpd-bundle_start_0
	* Resource action: Fencing monitor=60000 on cluster02
	* Pseudo action: httpd-bundle-clone_stop_0
	* Pseudo action: httpd_stop_0
	* Pseudo action: httpd-bundle-clone_stopped_0
	* Pseudo action: httpd-bundle-clone_start_0
	* Pseudo action: httpd-bundle_stopped_0
	* Pseudo action: httpd-bundle-clone_running_0
	* Pseudo action: httpd-bundle_running_0

	Revised Cluster Status:
	* Node List:
	* Online: [ cluster02 ]
	* OFFLINE: [ cluster01 ]
	* GuestOnline: [ httpd-bundle-1@cluster02 ]

	* Full List of Resources:
	* Clone Set: ping-clone [ping]:
	* Started: [ cluster02 ]
	* Stopped: [ cluster01 ]
	* Fencing (stonith:fence_xvm): Started cluster02
	* dummy (ocf:pacemaker:Dummy): Started cluster02
	* Clone Set: inactive-clone [inactive-dhcpd] (disabled):
	* Stopped (disabled): [ cluster01 cluster02 ]
	* Resource Group: inactive-group (disabled):
	* inactive-dummy-1 (ocf:pacemaker:Dummy): Stopped (disabled)
	* inactive-dummy-2 (ocf:pacemaker:Dummy): Stopped (disabled)
	* Container bundle set: httpd-bundle [pcmk:http]:
	* httpd-bundle-0 (192.168.122.131) (ocf:heartbeat:apache): FAILED
	* httpd-bundle-1 (192.168.122.132) (ocf:heartbeat:apache): Started cluster02
	* httpd-bundle-2 (192.168.122.133) (ocf:heartbeat:apache): Stopped
	* Resource Group: exim-group:
	* Public-IP (ocf:heartbeat:IPaddr): Started cluster02
	* Email (lsb:exim): Started cluster02
	* Clone Set: mysql-clone-group [mysql-group]:
	* Started: [ cluster02 ]
	* Stopped: [ cluster01 ]
	* Clone Set: promotable-clone [promotable-rsc] (promotable):
	* Promoted: [ cluster02 ]
	* Stopped: [ cluster01 ]
	=#=#=#= End test: Simulate bringing a node down - OK (0) =#=#=#=
	* Passed: crm_simulate - Simulate bringing a node down
	=#=#=#= Begin test: Simulate a node failing =#=#=#=
	4 of 32 resource instances DISABLED and 0 BLOCKED from further action due to failure

	Current cluster status:
	* Node List:
	* Online: [ cluster01 cluster02 ]
	* GuestOnline: [ httpd-bundle-0@cluster01 httpd-bundle-1@cluster02 ]

	* Full List of Resources:
	* Clone Set: ping-clone [ping]:
	* Started: [ cluster01 cluster02 ]
	* Fencing (stonith:fence_xvm): Started cluster01
	* dummy (ocf:pacemaker:Dummy): Started cluster02
	* Clone Set: inactive-clone [inactive-dhcpd] (disabled):
	* Stopped (disabled): [ cluster01 cluster02 ]
	* Resource Group: inactive-group (disabled):
	* inactive-dummy-1 (ocf:pacemaker:Dummy): Stopped (disabled)
	* inactive-dummy-2 (ocf:pacemaker:Dummy): Stopped (disabled)
	* Container bundle set: httpd-bundle [pcmk:http]:
	* httpd-bundle-0 (192.168.122.131) (ocf:heartbeat:apache): Started cluster01
	* httpd-bundle-1 (192.168.122.132) (ocf:heartbeat:apache): Started cluster02
	* httpd-bundle-2 (192.168.122.133) (ocf:heartbeat:apache): Stopped
	* Resource Group: exim-group:
	* Public-IP (ocf:heartbeat:IPaddr): Started cluster02
	* Email (lsb:exim): Started cluster02
	* Clone Set: mysql-clone-group [mysql-group]:
	* Started: [ cluster01 cluster02 ]
	* Clone Set: promotable-clone [promotable-rsc] (promotable):
	* Promoted: [ cluster02 ]
	* Unpromoted: [ cluster01 ]

	Performing Requested Modifications:
	* Failing node cluster02

	Transition Summary:
	* Fence (off) httpd-bundle-1 (resource: httpd-bundle-docker-1) 'guest is unclean'
	* Fence (reboot) cluster02 'peer is no longer part of the cluster'
	* Stop ping:0 ( cluster02 ) due to node availability
	* Stop dummy ( cluster02 ) due to node availability
	* Stop httpd-bundle-ip-192.168.122.132 ( cluster02 ) due to node availability
	* Stop httpd-bundle-docker-1 ( cluster02 ) due to node availability
	* Stop httpd-bundle-1 ( cluster02 ) due to unrunnable httpd-bundle-docker-1 start
	* Stop httpd:1 ( httpd-bundle-1 ) due to unrunnable httpd-bundle-docker-1 start
	* Start httpd-bundle-2 ( cluster01 ) due to unrunnable httpd-bundle-docker-2 start (blocked)
	* Start httpd:2 ( httpd-bundle-2 ) due to unrunnable httpd-bundle-docker-2 start (blocked)
	* Move Public-IP ( cluster02 -> cluster01 )
	* Move Email ( cluster02 -> cluster01 )
	* Stop mysql-proxy:0 ( cluster02 ) due to node availability
	* Stop promotable-rsc:0 ( Promoted cluster02 ) due to node availability

	Executing Cluster Transition:
	* Pseudo action: httpd-bundle-1_stop_0
	* Pseudo action: promotable-clone_demote_0
	* Pseudo action: httpd-bundle_stop_0
	* Pseudo action: httpd-bundle_start_0
	* Fencing cluster02 (reboot)
	* Pseudo action: ping-clone_stop_0
	* Pseudo action: dummy_stop_0
	* Pseudo action: httpd-bundle-docker-1_stop_0
	* Pseudo action: exim-group_stop_0
	* Pseudo action: Email_stop_0
	* Pseudo action: mysql-clone-group_stop_0
	* Pseudo action: promotable-rsc_demote_0
	* Pseudo action: promotable-clone_demoted_0
	* Pseudo action: promotable-clone_stop_0
	* Pseudo action: stonith-httpd-bundle-1-off on httpd-bundle-1
	* Pseudo action: ping_stop_0
	* Pseudo action: ping-clone_stopped_0
	* Pseudo action: httpd-bundle-clone_stop_0
	* Pseudo action: httpd-bundle-ip-192.168.122.132_stop_0
	* Pseudo action: Public-IP_stop_0
	* Pseudo action: mysql-group:0_stop_0
	* Pseudo action: mysql-proxy_stop_0
	* Pseudo action: promotable-rsc_stop_0
	* Pseudo action: promotable-clone_stopped_0
	* Pseudo action: httpd_stop_0
	* Pseudo action: httpd-bundle-clone_stopped_0
	* Pseudo action: httpd-bundle-clone_start_0
	* Pseudo action: exim-group_stopped_0
	* Pseudo action: exim-group_start_0
	* Resource action: Public-IP start on cluster01
	* Resource action: Email start on cluster01
	* Pseudo action: mysql-group:0_stopped_0
	* Pseudo action: mysql-clone-group_stopped_0
	* Pseudo action: httpd-bundle_stopped_0
	* Pseudo action: httpd-bundle-clone_running_0
	* Pseudo action: exim-group_running_0
	* Pseudo action: httpd-bundle_running_0

	Revised Cluster Status:
	* Node List:
	* Online: [ cluster01 ]
	* OFFLINE: [ cluster02 ]
	* GuestOnline: [ httpd-bundle-0@cluster01 ]

	* Full List of Resources:
	* Clone Set: ping-clone [ping]:
	* Started: [ cluster01 ]
	* Stopped: [ cluster02 ]
	* Fencing (stonith:fence_xvm): Started cluster01
	* dummy (ocf:pacemaker:Dummy): Stopped
	* Clone Set: inactive-clone [inactive-dhcpd] (disabled):
	* Stopped (disabled): [ cluster01 cluster02 ]
	* Resource Group: inactive-group (disabled):
	* inactive-dummy-1 (ocf:pacemaker:Dummy): Stopped (disabled)
	* inactive-dummy-2 (ocf:pacemaker:Dummy): Stopped (disabled)
	* Container bundle set: httpd-bundle [pcmk:http]:
	* httpd-bundle-0 (192.168.122.131) (ocf:heartbeat:apache): Started cluster01
	* httpd-bundle-1 (192.168.122.132) (ocf:heartbeat:apache): FAILED
	* httpd-bundle-2 (192.168.122.133) (ocf:heartbeat:apache): Stopped
	* Resource Group: exim-group:
	* Public-IP (ocf:heartbeat:IPaddr): Started cluster01
	* Email (lsb:exim): Started cluster01
	* Clone Set: mysql-clone-group [mysql-group]:
	* Started: [ cluster01 ]
	* Stopped: [ cluster02 ]
	* Clone Set: promotable-clone [promotable-rsc] (promotable):
	* Unpromoted: [ cluster01 ]
	* Stopped: [ cluster02 ]
	=#=#=#= End test: Simulate a node failing - OK (0) =#=#=#=
	* Passed: crm_simulate - Simulate a node failing
	=#=#=#= Begin test: List a promotable clone resource =#=#=#=
	resource promotable-clone is running on: cluster01
	resource promotable-clone is running on: cluster02 Promoted
	=#=#=#= End test: List a promotable clone resource - OK (0) =#=#=#=
	* Passed: crm_resource - List a promotable clone resource
	=#=#=#= Begin test: List the primitive of a promotable clone resource =#=#=#=
	resource promotable-rsc is running on: cluster01
	resource promotable-rsc is running on: cluster02 Promoted
	=#=#=#= End test: List the primitive of a promotable clone resource - OK (0) =#=#=#=
	* Passed: crm_resource - List the primitive of a promotable clone resource
	=#=#=#= Begin test: List a single instance of a promotable clone resource =#=#=#=
	resource promotable-rsc:0 is running on: cluster02 Promoted
	=#=#=#= End test: List a single instance of a promotable clone resource - OK (0) =#=#=#=
	* Passed: crm_resource - List a single instance of a promotable clone resource
	=#=#=#= Begin test: List another instance of a promotable clone resource =#=#=#=
	resource promotable-rsc:1 is running on: cluster01
	=#=#=#= End test: List another instance of a promotable clone resource - OK (0) =#=#=#=
	* Passed: crm_resource - List another instance of a promotable clone resource
	=#=#=#= Begin test: List a promotable clone resource in XML =#=#=#=
	<pacemaker-result api-version="X" request="crm_resource --locate -r promotable-clone --output-as=xml">
	<nodes resource="promotable-clone">
	<node>cluster01</node>
	<node state="promoted">cluster02</node>
	</nodes>
	<status code="0" message="OK"/>
	</pacemaker-result>
	=#=#=#= End test: List a promotable clone resource in XML - OK (0) =#=#=#=
	* Passed: crm_resource - List a promotable clone resource in XML
	=#=#=#= Begin test: List the primitive of a promotable clone resource in XML =#=#=#=
	<pacemaker-result api-version="X" request="crm_resource --locate -r promotable-rsc --output-as=xml">
	<nodes resource="promotable-rsc">
	<node>cluster01</node>
	<node state="promoted">cluster02</node>
	</nodes>
	<status code="0" message="OK"/>
	</pacemaker-result>
	=#=#=#= End test: List the primitive of a promotable clone resource in XML - OK (0) =#=#=#=
	* Passed: crm_resource - List the primitive of a promotable clone resource in XML
	=#=#=#= Begin test: List a single instance of a promotable clone resource in XML =#=#=#=
	<pacemaker-result api-version="X" request="crm_resource --locate -r promotable-rsc:0 --output-as=xml">
	<nodes resource="promotable-rsc:0">
	<node state="promoted">cluster02</node>
	</nodes>
	<status code="0" message="OK"/>
	</pacemaker-result>
	=#=#=#= End test: List a single instance of a promotable clone resource in XML - OK (0) =#=#=#=
	* Passed: crm_resource - List a single instance of a promotable clone resource in XML
	=#=#=#= Begin test: List another instance of a promotable clone resource in XML =#=#=#=
	<pacemaker-result api-version="X" request="crm_resource --locate -r promotable-rsc:1 --output-as=xml">
	<nodes resource="promotable-rsc:1">
	<node>cluster01</node>
	</nodes>
	<status code="0" message="OK"/>
	</pacemaker-result>
	=#=#=#= End test: List another instance of a promotable clone resource in XML - OK (0) =#=#=#=
	* Passed: crm_resource - List another instance of a promotable clone resource in XML
	=#=#=#= Begin test: Check that CIB_file="-" works - crm_mon =#=#=#=
	Cluster Summary:
	* Stack: corosync
	* Current DC: cluster02 (version) - partition with quorum
	* Last updated:
	* Last change:
	* 5 nodes configured
	* 32 resource instances configured (4 DISABLED)

	Node List:
	* Online: [ cluster01 cluster02 ]
	* GuestOnline: [ httpd-bundle-0@cluster01 httpd-bundle-1@cluster02 ]

	Active Resources:
	* Clone Set: ping-clone [ping]:
	* Started: [ cluster01 cluster02 ]
	* Fencing (stonith:fence_xvm): Started cluster01
	* dummy (ocf:pacemaker:Dummy): Started cluster02
	* Container bundle set: httpd-bundle [pcmk:http]:
	* httpd-bundle-0 (192.168.122.131) (ocf:heartbeat:apache): Started cluster01
	* httpd-bundle-1 (192.168.122.132) (ocf:heartbeat:apache): Started cluster02
	* httpd-bundle-2 (192.168.122.133) (ocf:heartbeat:apache): Stopped
	* Resource Group: exim-group:
	* Public-IP (ocf:heartbeat:IPaddr): Started cluster02
	* Email (lsb:exim): Started cluster02
	* Clone Set: mysql-clone-group [mysql-group]:
	* Started: [ cluster01 cluster02 ]
	* Clone Set: promotable-clone [promotable-rsc] (promotable):
	* Promoted: [ cluster02 ]
	* Unpromoted: [ cluster01 ]
	=#=#=#= End test: Check that CIB_file="-" works - crm_mon - OK (0) =#=#=#=
	* Passed: cat - Check that CIB_file="-" works - crm_mon
	=#=#=#= Begin test: Check that CIB_file="-" works - crm_resource =#=#=#=
	<pacemaker-result api-version="X" request="crm_resource --digests -r rsc1 -N node1 --output-as=xml">
	<digests resource="rsc1" node="node1" task="start" interval="0ms">
	<digest type="all" hash="3acdbe4c12734ebeb1251a59545af936">
	<parameters passwd="secret" fake="0"/>
	</digest>
	<digest type="nonprivate" hash="279c477dbc38c621904a00ab9e599b2f">
	<parameters fake="0"/>
	</digest>
	<digest type="nonreloadable" hash="5de1fd72a2e7762ed41543231034f6d7">
	<parameters passwd="secret"/>
	</digest>
	</digests>
	<status code="0" message="OK"/>
	</pacemaker-result>
	=#=#=#= End test: Check that CIB_file="-" works - crm_resource - OK (0) =#=#=#=
	* Passed: cat - Check that CIB_file="-" works - crm_resource
	=#=#=#= Begin test: Check that CIB_file="-" works - crmadmin =#=#=#=
	11
	=#=#=#= End test: Check that CIB_file="-" works - crmadmin - OK (0) =#=#=#=
	* Passed: cat - Check that CIB_file="-" works - crmadmin
	diff --git a/daemons/based/based_callbacks.c b/daemons/based/based_callbacks.c
	index 623ebc1389..0a8f475516 100644
	--- a/daemons/based/based_callbacks.c
	+++ b/daemons/based/based_callbacks.c
	@@ -1,1629 +1,1629 @@
	/*
	* Copyright 2004-2021 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <sys/param.h>
	#include <stdio.h>
	#include <sys/types.h>
	#include <unistd.h>

	#include <stdlib.h>
	#include <stdint.h> // uint32_t, uint64_t, UINT64_C()
	#include <errno.h>
	#include <fcntl.h>
	#include <inttypes.h> /* U64T ~ PRIu64 */

	#include <crm/crm.h>
	#include <crm/cib.h>
	#include <crm/msg_xml.h>
	#include <crm/cluster/internal.h>

	#include <crm/common/xml.h>
	#include <crm/common/remote_internal.h>

	#include <pacemaker-based.h>

	#define EXIT_ESCALATION_MS 10000

	static unsigned long cib_local_bcast_num = 0;

	typedef struct cib_local_notify_s {
	xmlNode *notify_src;
	char *client_id;
	gboolean from_peer;
	gboolean sync_reply;
	} cib_local_notify_t;

	int next_client_id = 0;

	gboolean legacy_mode = FALSE;

	qb_ipcs_service_t *ipcs_ro = NULL;
	qb_ipcs_service_t *ipcs_rw = NULL;
	qb_ipcs_service_t *ipcs_shm = NULL;

	void send_cib_replace(const xmlNode * sync_request, const char *host);
	static void cib_process_request(xmlNode *request, gboolean privileged,
	pcmk__client_t *cib_client);


	static int cib_process_command(xmlNode request, xmlNode *reply,
	xmlNode **cib_diff, gboolean privileged);

	gboolean cib_common_callback(qb_ipcs_connection_t * c, void *data, size_t size,
	gboolean privileged);

	gboolean cib_legacy_mode(void)
	{
	return legacy_mode;
	}


	static int32_t
	cib_ipc_accept(qb_ipcs_connection_t * c, uid_t uid, gid_t gid)
	{
	if (cib_shutdown_flag) {
	crm_info("Ignoring new IPC client [%d] during shutdown",
	pcmk__client_pid(c));
	return -EPERM;
	}

	if (pcmk__new_client(c, uid, gid) == NULL) {
	return -EIO;
	}
	return 0;
	}

	static int32_t
	cib_ipc_dispatch_rw(qb_ipcs_connection_t * c, void *data, size_t size)
	{
	pcmk__client_t *client = pcmk__find_client(c);

	crm_trace("%p message from %s", c, client->id);
	return cib_common_callback(c, data, size, TRUE);
	}

	static int32_t
	cib_ipc_dispatch_ro(qb_ipcs_connection_t * c, void *data, size_t size)
	{
	pcmk__client_t *client = pcmk__find_client(c);

	crm_trace("%p message from %s", c, client->id);
	return cib_common_callback(c, data, size, FALSE);
	}

	/* Error code means? */
	static int32_t
	cib_ipc_closed(qb_ipcs_connection_t * c)
	{
	pcmk__client_t *client = pcmk__find_client(c);

	if (client == NULL) {
	return 0;
	}
	crm_trace("Connection %p", c);
	pcmk__free_client(client);
	return 0;
	}

	static void
	cib_ipc_destroy(qb_ipcs_connection_t * c)
	{
	crm_trace("Connection %p", c);
	cib_ipc_closed(c);
	if (cib_shutdown_flag) {
	cib_shutdown(0);
	}
	}

	struct qb_ipcs_service_handlers ipc_ro_callbacks = {
	.connection_accept = cib_ipc_accept,
	.connection_created = NULL,
	.msg_process = cib_ipc_dispatch_ro,
	.connection_closed = cib_ipc_closed,
	.connection_destroyed = cib_ipc_destroy
	};

	struct qb_ipcs_service_handlers ipc_rw_callbacks = {
	.connection_accept = cib_ipc_accept,
	.connection_created = NULL,
	.msg_process = cib_ipc_dispatch_rw,
	.connection_closed = cib_ipc_closed,
	.connection_destroyed = cib_ipc_destroy
	};

	void
	cib_common_callback_worker(uint32_t id, uint32_t flags, xmlNode * op_request,
	pcmk__client_t *cib_client, gboolean privileged)
	{
	const char *op = crm_element_value(op_request, F_CIB_OPERATION);

	if (pcmk__str_eq(op, CRM_OP_REGISTER, pcmk__str_none)) {
	if (flags & crm_ipc_client_response) {
	xmlNode *ack = create_xml_node(NULL, __func__);

	crm_xml_add(ack, F_CIB_OPERATION, CRM_OP_REGISTER);
	crm_xml_add(ack, F_CIB_CLIENTID, cib_client->id);
	pcmk__ipc_send_xml(cib_client, id, ack, flags);
	cib_client->request_id = 0;
	free_xml(ack);
	}
	return;

	} else if (pcmk__str_eq(op, T_CIB_NOTIFY, pcmk__str_none)) {
	/* Update the notify filters for this client */
	int on_off = 0;
	crm_exit_t status = CRM_EX_OK;
	uint64_t bit = UINT64_C(0);
	const char *type = crm_element_value(op_request, F_CIB_NOTIFY_TYPE);

	crm_element_value_int(op_request, F_CIB_NOTIFY_ACTIVATE, &on_off);

	crm_debug("Setting %s callbacks %s for client %s",
	type, (on_off? "on" : "off"), pcmk__client_name(cib_client));

	if (pcmk__str_eq(type, T_CIB_POST_NOTIFY, pcmk__str_casei)) {
	bit = cib_notify_post;

	} else if (pcmk__str_eq(type, T_CIB_PRE_NOTIFY, pcmk__str_casei)) {
	bit = cib_notify_pre;

	} else if (pcmk__str_eq(type, T_CIB_UPDATE_CONFIRM, pcmk__str_casei)) {
	bit = cib_notify_confirm;

	} else if (pcmk__str_eq(type, T_CIB_DIFF_NOTIFY, pcmk__str_casei)) {
	bit = cib_notify_diff;

	} else if (pcmk__str_eq(type, T_CIB_REPLACE_NOTIFY, pcmk__str_casei)) {
	bit = cib_notify_replace;

	} else {
	status = CRM_EX_INVALID_PARAM;
	}

	if (bit != 0) {
	if (on_off) {
	pcmk__set_client_flags(cib_client, bit);
	} else {
	pcmk__clear_client_flags(cib_client, bit);
	}
	}

	pcmk__ipc_send_ack(cib_client, id, flags, "ack", status);
	return;
	}

	cib_process_request(op_request, privileged, cib_client);
	}

	int32_t
	cib_common_callback(qb_ipcs_connection_t * c, void *data, size_t size, gboolean privileged)
	{
	uint32_t id = 0;
	uint32_t flags = 0;
	int call_options = 0;
	pcmk__client_t *cib_client = pcmk__find_client(c);
	xmlNode *op_request = pcmk__client_data2xml(cib_client, data, &id, &flags);

	if (op_request) {
	crm_element_value_int(op_request, F_CIB_CALLOPTS, &call_options);
	}

	if (op_request == NULL) {
	crm_trace("Invalid message from %p", c);
	pcmk__ipc_send_ack(cib_client, id, flags, "nack", CRM_EX_PROTOCOL);
	return 0;

	} else if(cib_client == NULL) {
	crm_trace("Invalid client %p", c);
	return 0;
	}

	if (pcmk_is_set(call_options, cib_sync_call)) {
	CRM_LOG_ASSERT(flags & crm_ipc_client_response);
	CRM_LOG_ASSERT(cib_client->request_id == 0); /* This means the client has two synchronous events in-flight */
	cib_client->request_id = id; /* Reply only to the last one */
	}

	if (cib_client->name == NULL) {
	const char *value = crm_element_value(op_request, F_CIB_CLIENTNAME);

	if (value == NULL) {
	cib_client->name = pcmk__itoa(cib_client->pid);
	} else {
	cib_client->name = strdup(value);
	if (crm_is_daemon_name(value)) {
	pcmk__set_client_flags(cib_client, cib_is_daemon);
	}
	}
	}

	/* Allow cluster daemons more leeway before being evicted */
	if (pcmk_is_set(cib_client->flags, cib_is_daemon)) {
	const char *qmax = cib_config_lookup("cluster-ipc-limit");

	if (pcmk__set_client_queue_max(cib_client, qmax)) {
	crm_trace("IPC threshold for client %s[%u] is now %u",
	pcmk__client_name(cib_client), cib_client->pid,
	cib_client->queue_max);
	}
	}

	crm_xml_add(op_request, F_CIB_CLIENTID, cib_client->id);
	crm_xml_add(op_request, F_CIB_CLIENTNAME, cib_client->name);

	CRM_LOG_ASSERT(cib_client->user != NULL);
	pcmk__update_acl_user(op_request, F_CIB_USER, cib_client->user);

	cib_common_callback_worker(id, flags, op_request, cib_client, privileged);
	free_xml(op_request);

	return 0;
	}

	static uint64_t ping_seq = 0;
	static char *ping_digest = NULL;
	static bool ping_modified_since = FALSE;
	int sync_our_cib(xmlNode * request, gboolean all);

	static gboolean
	cib_digester_cb(gpointer data)
	{
	if (cib_is_master) {
	char buffer[32];
	xmlNode *ping = create_xml_node(NULL, "ping");

	ping_seq++;
	free(ping_digest);
	ping_digest = NULL;
	ping_modified_since = FALSE;
	snprintf(buffer, 32, "%" U64T, ping_seq);
	crm_trace("Requesting peer digests (%s)", buffer);

	crm_xml_add(ping, F_TYPE, "cib");
	crm_xml_add(ping, F_CIB_OPERATION, CRM_OP_PING);
	crm_xml_add(ping, F_CIB_PING_ID, buffer);

	crm_xml_add(ping, XML_ATTR_CRM_VERSION, CRM_FEATURE_SET);
	send_cluster_message(NULL, crm_msg_cib, ping, TRUE);

	free_xml(ping);
	}
	return FALSE;
	}

	static void
	process_ping_reply(xmlNode *reply)
	{
	uint64_t seq = 0;
	const char *host = crm_element_value(reply, F_ORIG);

	xmlNode *pong = get_message_xml(reply, F_CIB_CALLDATA);
	const char *seq_s = crm_element_value(pong, F_CIB_PING_ID);
	const char *digest = crm_element_value(pong, XML_ATTR_DIGEST);

	if (seq_s == NULL) {
	crm_debug("Ignoring ping reply with no " F_CIB_PING_ID);
	return;

	} else {
	long long seq_ll;

	if (pcmk__scan_ll(seq_s, &seq_ll, 0LL) != pcmk_rc_ok) {
	return;
	}
	seq = (uint64_t) seq_ll;
	}

	if(digest == NULL) {
	crm_trace("Ignoring ping reply %s from %s with no digest", seq_s, host);

	} else if(seq != ping_seq) {
	crm_trace("Ignoring out of sequence ping reply %s from %s", seq_s, host);

	} else if(ping_modified_since) {
	crm_trace("Ignoring ping reply %s from %s: cib updated since", seq_s, host);

	} else {
	const char *version = crm_element_value(pong, XML_ATTR_CRM_VERSION);

	if(ping_digest == NULL) {
	crm_trace("Calculating new digest");
	ping_digest = calculate_xml_versioned_digest(the_cib, FALSE, TRUE, version);
	}

	crm_trace("Processing ping reply %s from %s (%s)", seq_s, host, digest);
	if (!pcmk__str_eq(ping_digest, digest, pcmk__str_casei)) {
	xmlNode *remote_cib = get_message_xml(pong, F_CIB_CALLDATA);

	crm_notice("Local CIB %s.%s.%s.%s differs from %s: %s.%s.%s.%s %p",
	crm_element_value(the_cib, XML_ATTR_GENERATION_ADMIN),
	crm_element_value(the_cib, XML_ATTR_GENERATION),
	crm_element_value(the_cib, XML_ATTR_NUMUPDATES),
	ping_digest, host,
	remote_cib?crm_element_value(remote_cib, XML_ATTR_GENERATION_ADMIN):"_",
	remote_cib?crm_element_value(remote_cib, XML_ATTR_GENERATION):"_",
	remote_cib?crm_element_value(remote_cib, XML_ATTR_NUMUPDATES):"_",
	digest, remote_cib);

	if(remote_cib && remote_cib->children) {
	/* Additional debug */
	xml_calculate_changes(the_cib, remote_cib);
	xml_log_changes(LOG_INFO, __func__, remote_cib);
	crm_trace("End of differences");
	}

	free_xml(remote_cib);
	sync_our_cib(reply, FALSE);
	}
	}
	}

	static void
	do_local_notify(xmlNode * notify_src, const char *client_id,
	gboolean sync_reply, gboolean from_peer)
	{
	int rid = 0;
	int call_id = 0;
	pcmk__client_t *client_obj = NULL;

	CRM_ASSERT(notify_src && client_id);

	crm_element_value_int(notify_src, F_CIB_CALLID, &call_id);

	client_obj = pcmk__find_client_by_id(client_id);
	if (client_obj == NULL) {
	crm_debug("Could not send response %d: client %s not found",
	call_id, client_id);
	return;
	}

	if (sync_reply) {
	if (client_obj->ipcs) {
	CRM_LOG_ASSERT(client_obj->request_id);

	rid = client_obj->request_id;
	client_obj->request_id = 0;

	crm_trace("Sending response %d to client %s%s",
	rid, pcmk__client_name(client_obj),
	(from_peer? " (originator of delegated request)" : ""));
	} else {
	crm_trace("Sending response (call %d) to client %s%s",
	call_id, pcmk__client_name(client_obj),
	(from_peer? " (originator of delegated request)" : ""));
	}

	} else {
	crm_trace("Sending event %d to client %s%s",
	call_id, pcmk__client_name(client_obj),
	(from_peer? " (originator of delegated request)" : ""));
	}

	switch (PCMK__CLIENT_TYPE(client_obj)) {
	case pcmk__client_ipc:
	{
	int rc = pcmk__ipc_send_xml(client_obj, rid, notify_src,
	(sync_reply? crm_ipc_flags_none
	: crm_ipc_server_event));

	if (rc != pcmk_rc_ok) {
	crm_warn("%s reply to client %s failed: %s " CRM_XS " rc=%d",
	(sync_reply? "Synchronous" : "Asynchronous"),
	pcmk__client_name(client_obj), pcmk_rc_str(rc),
	rc);
	}
	}
	break;
	#ifdef HAVE_GNUTLS_GNUTLS_H
	case pcmk__client_tls:
	#endif
	case pcmk__client_tcp:
	pcmk__remote_send_xml(client_obj->remote, notify_src);
	break;
	default:
	crm_err("Unknown transport for client %s "
	CRM_XS " flags=%#016" PRIx64,
	pcmk__client_name(client_obj), client_obj->flags);
	}
	}

	static void
	local_notify_destroy_callback(gpointer data)
	{
	cib_local_notify_t *notify = data;

	free_xml(notify->notify_src);
	free(notify->client_id);
	free(notify);
	}

	static void
	check_local_notify(int bcast_id)
	{
	cib_local_notify_t *notify = NULL;

	if (!local_notify_queue) {
	return;
	}

	notify = pcmk__intkey_table_lookup(local_notify_queue, bcast_id);

	if (notify) {
	do_local_notify(notify->notify_src, notify->client_id, notify->sync_reply,
	notify->from_peer);
	pcmk__intkey_table_remove(local_notify_queue, bcast_id);
	}
	}

	static void
	queue_local_notify(xmlNode * notify_src, const char *client_id, gboolean sync_reply,
	gboolean from_peer)
	{
	cib_local_notify_t *notify = calloc(1, sizeof(cib_local_notify_t));

	notify->notify_src = notify_src;
	notify->client_id = strdup(client_id);
	notify->sync_reply = sync_reply;
	notify->from_peer = from_peer;

	if (!local_notify_queue) {
	local_notify_queue = pcmk__intkey_table(local_notify_destroy_callback);
	}
	pcmk__intkey_table_insert(local_notify_queue, cib_local_bcast_num, notify);
	// cppcheck doesn't know notify will get freed when hash table is destroyed
	// cppcheck-suppress memleak
	}

	static void
	parse_local_options_v1(pcmk__client_t *cib_client, int call_type,
	int call_options, const char host, const char op,
	gboolean local_notify, gboolean needs_reply,
	gboolean process, gboolean needs_forward)
	{
	if (cib_op_modifies(call_type)
	&& !(call_options & cib_inhibit_bcast)) {
	/* we need to send an update anyway */
	*needs_reply = TRUE;
	} else {
	*needs_reply = FALSE;
	}

	if (host == NULL && (call_options & cib_scope_local)) {
	crm_trace("Processing locally scoped %s op from client %s",
	op, pcmk__client_name(cib_client));
	*local_notify = TRUE;

	} else if (host == NULL && cib_is_master) {
	crm_trace("Processing master %s op locally from client %s",
	op, pcmk__client_name(cib_client));
	*local_notify = TRUE;

	} else if (pcmk__str_eq(host, cib_our_uname, pcmk__str_casei)) {
	crm_trace("Processing locally addressed %s op from client %s",
	op, pcmk__client_name(cib_client));
	*local_notify = TRUE;

	} else if (stand_alone) {
	*needs_forward = FALSE;
	*local_notify = TRUE;
	*process = TRUE;

	} else {
	crm_trace("%s op from %s needs to be forwarded to client %s",
	op, pcmk__client_name(cib_client),
	(host? host : "the master instance"));
	*needs_forward = TRUE;
	*process = FALSE;
	}
	}

	static void
	parse_local_options_v2(pcmk__client_t *cib_client, int call_type,
	int call_options, const char host, const char op,
	gboolean local_notify, gboolean needs_reply,
	gboolean process, gboolean needs_forward)
	{
	if (cib_op_modifies(call_type)) {
	if (pcmk__strcase_any_of(op, CIB_OP_MASTER, CIB_OP_SLAVE, NULL)) {
	/* Always handle these locally */
	*process = TRUE;
	*needs_reply = FALSE;
	*local_notify = TRUE;
	*needs_forward = FALSE;
	return;

	} else {
	/* Redirect all other updates via CPG */
	*needs_reply = TRUE;
	*needs_forward = TRUE;
	*process = FALSE;
	crm_trace("%s op from %s needs to be forwarded to client %s",
	op, pcmk__client_name(cib_client),
	(host? host : "the master instance"));
	return;
	}
	}


	*process = TRUE;
	*needs_reply = FALSE;
	*local_notify = TRUE;
	*needs_forward = FALSE;

	if (stand_alone) {
	crm_trace("Processing %s op from client %s (stand-alone)",
	op, pcmk__client_name(cib_client));

	} else if (host == NULL) {
	crm_trace("Processing unaddressed %s op from client %s",
	op, pcmk__client_name(cib_client));

	} else if (pcmk__str_eq(host, cib_our_uname, pcmk__str_casei)) {
	crm_trace("Processing locally addressed %s op from client %s",
	op, pcmk__client_name(cib_client));

	} else {
	crm_trace("%s op from %s needs to be forwarded to client %s",
	op, pcmk__client_name(cib_client), host);
	*needs_forward = TRUE;
	*process = FALSE;
	}
	}

	static void
	parse_local_options(pcmk__client_t *cib_client, int call_type,
	int call_options, const char host, const char op,
	gboolean local_notify, gboolean needs_reply,
	gboolean process, gboolean needs_forward)
	{
	if(cib_legacy_mode()) {
	parse_local_options_v1(cib_client, call_type, call_options, host,
	op, local_notify, needs_reply, process, needs_forward);
	} else {
	parse_local_options_v2(cib_client, call_type, call_options, host,
	op, local_notify, needs_reply, process, needs_forward);
	}
	}

	static gboolean
	parse_peer_options_v1(int call_type, xmlNode * request,
	gboolean * local_notify, gboolean * needs_reply, gboolean * process,
	gboolean * needs_forward)
	{
	const char *op = NULL;
	const char *host = NULL;
	const char *delegated = NULL;
	const char *originator = crm_element_value(request, F_ORIG);
	const char *reply_to = crm_element_value(request, F_CIB_ISREPLY);

	gboolean is_reply = pcmk__str_eq(reply_to, cib_our_uname, pcmk__str_casei);

	if (pcmk__xe_attr_is_true(request, F_CIB_GLOBAL_UPDATE)) {
	*needs_reply = FALSE;
	if (is_reply) {
	*local_notify = TRUE;
	crm_trace("Processing global/peer update from %s"
	" that originated from us", originator);
	} else {
	crm_trace("Processing global/peer update from %s", originator);
	}
	return TRUE;
	}

	op = crm_element_value(request, F_CIB_OPERATION);
	crm_trace("Processing %s request sent by %s", op, originator);
	if (pcmk__str_eq(op, "cib_shutdown_req", pcmk__str_casei)) {
	/* Always process these */
	*local_notify = FALSE;
	if (reply_to == NULL \|\| is_reply) {
	*process = TRUE;
	}
	if (is_reply) {
	*needs_reply = FALSE;
	}
	return *process;
	}

	if (is_reply && pcmk__str_eq(op, CRM_OP_PING, pcmk__str_casei)) {
	process_ping_reply(request);
	return FALSE;
	}

	if (is_reply) {
	crm_trace("Forward reply sent from %s to local clients", originator);
	*process = FALSE;
	*needs_reply = FALSE;
	*local_notify = TRUE;
	return TRUE;
	}

	host = crm_element_value(request, F_CIB_HOST);
	if (host != NULL && pcmk__str_eq(host, cib_our_uname, pcmk__str_casei)) {
	crm_trace("Processing %s request sent to us from %s", op, originator);
	return TRUE;

	} else if(is_reply == FALSE && pcmk__str_eq(op, CRM_OP_PING, pcmk__str_casei)) {
	crm_trace("Processing %s request sent to %s by %s", op, host?host:"everyone", originator);
	*needs_reply = TRUE;
	return TRUE;

	} else if (host == NULL && cib_is_master == TRUE) {
	crm_trace("Processing %s request sent to master instance from %s", op, originator);
	return TRUE;
	}

	delegated = crm_element_value(request, F_CIB_DELEGATED);
	if (delegated != NULL) {
	crm_trace("Ignoring msg for master instance");

	} else if (host != NULL) {
	/* this is for a specific instance and we're not it */
	crm_trace("Ignoring msg for instance on %s", crm_str(host));

	} else if (reply_to == NULL && cib_is_master == FALSE) {
	/* this is for the master instance and we're not it */
	crm_trace("Ignoring reply to %s", crm_str(reply_to));

	} else if (pcmk__str_eq(op, "cib_shutdown_req", pcmk__str_casei)) {
	if (reply_to != NULL) {
	crm_debug("Processing %s from %s", op, originator);
	*needs_reply = FALSE;

	} else {
	crm_debug("Processing %s reply from %s", op, originator);
	}
	return TRUE;

	} else {
	crm_err("Nothing for us to do?");
	crm_log_xml_err(request, "Peer[inbound]");
	}

	return FALSE;
	}

	static gboolean
	parse_peer_options_v2(int call_type, xmlNode * request,
	gboolean * local_notify, gboolean * needs_reply, gboolean * process,
	gboolean * needs_forward)
	{
	const char *host = NULL;
	const char *delegated = crm_element_value(request, F_CIB_DELEGATED);
	const char *op = crm_element_value(request, F_CIB_OPERATION);
	const char *originator = crm_element_value(request, F_ORIG);
	const char *reply_to = crm_element_value(request, F_CIB_ISREPLY);

	gboolean is_reply = pcmk__str_eq(reply_to, cib_our_uname, pcmk__str_casei);

	if(pcmk__str_eq(op, CIB_OP_REPLACE, pcmk__str_casei)) {
	/* sync_our_cib() sets F_CIB_ISREPLY */
	if (reply_to) {
	delegated = reply_to;
	}
	goto skip_is_reply;

	} else if(pcmk__str_eq(op, CIB_OP_SYNC, pcmk__str_casei)) {

	} else if (is_reply && pcmk__str_eq(op, CRM_OP_PING, pcmk__str_casei)) {
	process_ping_reply(request);
	return FALSE;

	} else if (pcmk__str_eq(op, CIB_OP_UPGRADE, pcmk__str_casei)) {
	/* Only the DC (node with the oldest software) should process
	* this operation if F_CIB_SCHEMA_MAX is unset
	*
	* If the DC is happy it will then send out another
	* CIB_OP_UPGRADE which will tell all nodes to do the actual
	* upgrade.
	*
	* Except this time F_CIB_SCHEMA_MAX will be set which puts a
	* limit on how far newer nodes will go
	*/
	const char *max = crm_element_value(request, F_CIB_SCHEMA_MAX);
	const char *upgrade_rc = crm_element_value(request, F_CIB_UPGRADE_RC);

	crm_trace("Parsing %s operation%s for %s with max=%s and upgrade_rc=%s",
	op, (is_reply? " reply" : ""),
	(cib_is_master? "master" : "slave"),
	(max? max : "none"), (upgrade_rc? upgrade_rc : "none"));

	if (upgrade_rc != NULL) {
	// Our upgrade request was rejected by DC, notify clients of result
	crm_xml_add(request, F_CIB_RC, upgrade_rc);

	} else if ((max == NULL) && cib_is_master) {
	/* We are the DC, check if this upgrade is allowed */
	goto skip_is_reply;

	} else if(max) {
	/* Ok, go ahead and upgrade to 'max' */
	goto skip_is_reply;

	} else {
	// Ignore broadcast client requests when we're not DC
	return FALSE;
	}

	} else if (pcmk__xe_attr_is_true(request, F_CIB_GLOBAL_UPDATE)) {
	crm_info("Detected legacy %s global update from %s", op, originator);
	send_sync_request(NULL);
	legacy_mode = TRUE;
	return FALSE;

	} else if (is_reply && cib_op_modifies(call_type)) {
	crm_trace("Ignoring legacy %s reply sent from %s to local clients", op, originator);
	return FALSE;

	} else if (pcmk__str_eq(op, "cib_shutdown_req", pcmk__str_casei)) {
	/* Legacy handling */
	crm_debug("Legacy handling of %s message from %s", op, originator);
	*local_notify = FALSE;
	if (reply_to == NULL) {
	*process = TRUE;
	}
	return *process;
	}

	if(is_reply) {
	crm_trace("Handling %s reply sent from %s to local clients", op, originator);
	*process = FALSE;
	*needs_reply = FALSE;
	*local_notify = TRUE;
	return TRUE;
	}

	skip_is_reply:
	*process = TRUE;
	*needs_reply = FALSE;

	if(pcmk__str_eq(delegated, cib_our_uname, pcmk__str_casei)) {
	*local_notify = TRUE;
	} else {
	*local_notify = FALSE;
	}

	host = crm_element_value(request, F_CIB_HOST);
	if (host != NULL && pcmk__str_eq(host, cib_our_uname, pcmk__str_casei)) {
	crm_trace("Processing %s request sent to us from %s", op, originator);
	*needs_reply = TRUE;
	return TRUE;

	} else if (host != NULL) {
	/* this is for a specific instance and we're not it */
	crm_trace("Ignoring %s operation for instance on %s", op, crm_str(host));
	return FALSE;

	} else if(is_reply == FALSE && pcmk__str_eq(op, CRM_OP_PING, pcmk__str_casei)) {
	*needs_reply = TRUE;
	}

	crm_trace("Processing %s request sent to everyone by %s/%s on %s %s", op,
	crm_element_value(request, F_CIB_CLIENTNAME),
	crm_element_value(request, F_CIB_CALLID),
	originator, (*local_notify)?"(notify)":"");
	return TRUE;
	}

	static gboolean
	parse_peer_options(int call_type, xmlNode * request,
	gboolean * local_notify, gboolean * needs_reply, gboolean * process,
	gboolean * needs_forward)
	{
	/* TODO: What happens when an update comes in after node A
	* requests the CIB from node B, but before it gets the reply (and
	* sends out the replace operation)
	*/
	if(cib_legacy_mode()) {
	return parse_peer_options_v1(
	call_type, request, local_notify, needs_reply, process, needs_forward);
	} else {
	return parse_peer_options_v2(
	call_type, request, local_notify, needs_reply, process, needs_forward);
	}
	}

	static void
	forward_request(xmlNode * request, pcmk__client_t *cib_client, int call_options)
	{
	const char *op = crm_element_value(request, F_CIB_OPERATION);
	const char *host = crm_element_value(request, F_CIB_HOST);

	crm_xml_add(request, F_CIB_DELEGATED, cib_our_uname);

	if (host != NULL) {
	crm_trace("Forwarding %s op to %s", op, host);
	send_cluster_message(crm_get_peer(0, host), crm_msg_cib, request, FALSE);

	} else {
	crm_trace("Forwarding %s op to master instance", op);
	send_cluster_message(NULL, crm_msg_cib, request, FALSE);
	}

	/* Return the request to its original state */
	xml_remove_prop(request, F_CIB_DELEGATED);

	if (call_options & cib_discard_reply) {
	crm_trace("Client not interested in reply");
	}
	}

	static gboolean
	send_peer_reply(xmlNode * msg, xmlNode * result_diff, const char *originator, gboolean broadcast)
	{
	CRM_ASSERT(msg != NULL);

	if (broadcast) {
	/* this (successful) call modified the CIB _and_ the
	* change needs to be broadcast...
	* send via HA to other nodes
	*/
	int diff_add_updates = 0;
	int diff_add_epoch = 0;
	int diff_add_admin_epoch = 0;

	int diff_del_updates = 0;
	int diff_del_epoch = 0;
	int diff_del_admin_epoch = 0;

	const char *digest = NULL;
	int format = 1;

	CRM_LOG_ASSERT(result_diff != NULL);
	digest = crm_element_value(result_diff, XML_ATTR_DIGEST);
	crm_element_value_int(result_diff, "format", &format);

	cib_diff_version_details(result_diff,
	&diff_add_admin_epoch, &diff_add_epoch, &diff_add_updates,
	&diff_del_admin_epoch, &diff_del_epoch, &diff_del_updates);

	crm_trace("Sending update diff %d.%d.%d -> %d.%d.%d %s",
	diff_del_admin_epoch, diff_del_epoch, diff_del_updates,
	diff_add_admin_epoch, diff_add_epoch, diff_add_updates, digest);

	crm_xml_add(msg, F_CIB_ISREPLY, originator);
	pcmk__xe_set_bool_attr(msg, F_CIB_GLOBAL_UPDATE, true);
	crm_xml_add(msg, F_CIB_OPERATION, CIB_OP_APPLY_DIFF);
	crm_xml_add(msg, F_CIB_USER, CRM_DAEMON_USER);

	if (format == 1) {
	CRM_ASSERT(digest != NULL);
	}

	add_message_xml(msg, F_CIB_UPDATE_DIFF, result_diff);
	crm_log_xml_explicit(msg, "copy");
	return send_cluster_message(NULL, crm_msg_cib, msg, TRUE);

	} else if (originator != NULL) {
	/* send reply via HA to originating node */
	crm_trace("Sending request result to %s only", originator);
	crm_xml_add(msg, F_CIB_ISREPLY, originator);
	return send_cluster_message(crm_get_peer(0, originator), crm_msg_cib, msg, FALSE);
	}

	return FALSE;
	}

	/*!
	* \internal
	* \brief Handle an IPC or CPG message containing a request
	*
	* \param[in] request Request XML
	* \param[in] privileged Whether privileged commands may be run
	* (see cib_server_ops[] definition)
	* \param[in] cib_client IPC client that sent request (or NULL if CPG)
	*/
	static void
	cib_process_request(xmlNode *request, gboolean privileged,
	pcmk__client_t *cib_client)
	{
	int call_type = 0;
	int call_options = 0;

	gboolean process = TRUE; // Whether to process request locally now
	gboolean is_update = TRUE; // Whether request would modify CIB
	gboolean needs_reply = TRUE; // Whether to build a reply
	gboolean local_notify = FALSE; // Whether to notify (local) requester
	gboolean needs_forward = FALSE; // Whether to forward request somewhere else

	xmlNode *op_reply = NULL;
	xmlNode *result_diff = NULL;

	int rc = pcmk_ok;
	const char *op = crm_element_value(request, F_CIB_OPERATION);
	const char *originator = crm_element_value(request, F_ORIG);
	const char *host = crm_element_value(request, F_CIB_HOST);
	const char *target = NULL;
	const char *call_id = crm_element_value(request, F_CIB_CALLID);
	const char *client_id = crm_element_value(request, F_CIB_CLIENTID);
	const char *client_name = crm_element_value(request, F_CIB_CLIENTNAME);
	const char *reply_to = crm_element_value(request, F_CIB_ISREPLY);

	crm_element_value_int(request, F_CIB_CALLOPTS, &call_options);

	if ((host != NULL) && (*host == '\0')) {
	host = NULL;
	}

	if (host) {
	target = host;

	} else if (call_options & cib_scope_local) {
	target = "local host";

	} else {
	target = "master";
	}

	if (cib_client == NULL) {
	crm_trace("Processing peer %s operation from %s/%s on %s intended for %s (reply=%s)",
	op, client_name, call_id, originator, target, reply_to);
	} else {
	crm_xml_add(request, F_ORIG, cib_our_uname);
	crm_trace("Processing local %s operation from %s/%s intended for %s", op, client_name, call_id, target);
	}

	rc = cib_get_operation_id(op, &call_type);
	if (rc != pcmk_ok) {
	/* TODO: construct error reply? */
	crm_err("Pre-processing of command failed: %s", pcmk_strerror(rc));
	return;
	}

	if (cib_client != NULL) {
	parse_local_options(cib_client, call_type, call_options, host, op,
	&local_notify, &needs_reply, &process, &needs_forward);

	} else if (parse_peer_options(call_type, request, &local_notify,
	&needs_reply, &process, &needs_forward) == FALSE) {
	return;
	}

	is_update = cib_op_modifies(call_type);

	if (call_options & cib_discard_reply) {
	/* If the request will modify the CIB, and we are in legacy mode, we
	* need to build a reply so we can broadcast a diff, even if the
	* requester doesn't want one.
	*/
	needs_reply = is_update && cib_legacy_mode();
	local_notify = FALSE;
	}

	if (needs_forward) {
	const char *section = crm_element_value(request, F_CIB_SECTION);
	int log_level = LOG_INFO;

	if (pcmk__str_eq(op, CRM_OP_NOOP, pcmk__str_casei)) {
	log_level = LOG_DEBUG;
	}

	do_crm_log(log_level,
	"Forwarding %s operation for section %s to %s (origin=%s/%s/%s)",
	op,
	section ? section : "'all'",
	host ? host : cib_legacy_mode() ? "master" : "all",
	originator ? originator : "local",
	client_name, call_id);

	forward_request(request, cib_client, call_options);
	return;
	}

	if (cib_status != pcmk_ok) {
	const char *call = crm_element_value(request, F_CIB_CALLID);

	rc = cib_status;
	crm_err("Operation ignored, cluster configuration is invalid."
	" Please repair and restart: %s", pcmk_strerror(cib_status));

	op_reply = create_xml_node(NULL, "cib-reply");
	crm_xml_add(op_reply, F_TYPE, T_CIB);
	crm_xml_add(op_reply, F_CIB_OPERATION, op);
	crm_xml_add(op_reply, F_CIB_CALLID, call);
	crm_xml_add(op_reply, F_CIB_CLIENTID, client_id);
	crm_xml_add_int(op_reply, F_CIB_CALLOPTS, call_options);
	crm_xml_add_int(op_reply, F_CIB_RC, rc);

	crm_trace("Attaching reply output");
	add_message_xml(op_reply, F_CIB_CALLDATA, the_cib);

	crm_log_xml_explicit(op_reply, "cib:reply");

	} else if (process) {
	time_t finished = 0;
	time_t now = time(NULL);
	int level = LOG_INFO;
	const char *section = crm_element_value(request, F_CIB_SECTION);

	rc = cib_process_command(request, &op_reply, &result_diff, privileged);

	if (!is_update) {
	level = LOG_TRACE;

	} else if (pcmk__xe_attr_is_true(request, F_CIB_GLOBAL_UPDATE)) {
	switch (rc) {
	case pcmk_ok:
	level = LOG_INFO;
	break;
	case -pcmk_err_old_data:
	case -pcmk_err_diff_resync:
	case -pcmk_err_diff_failed:
	level = LOG_TRACE;
	break;
	default:
	level = LOG_ERR;
	}

	} else if (rc != pcmk_ok) {
	level = LOG_WARNING;
	}

	do_crm_log(level,
	"Completed %s operation for section %s: %s (rc=%d, origin=%s/%s/%s, version=%s.%s.%s)",
	op, section ? section : "'all'", pcmk_strerror(rc), rc,
	originator ? originator : "local", client_name, call_id,
	the_cib ? crm_element_value(the_cib, XML_ATTR_GENERATION_ADMIN) : "0",
	the_cib ? crm_element_value(the_cib, XML_ATTR_GENERATION) : "0",
	the_cib ? crm_element_value(the_cib, XML_ATTR_NUMUPDATES) : "0");

	finished = time(NULL);
	if ((finished - now) > 3) {
	crm_trace("%s operation took %lds to complete", op, (long)(finished - now));
	crm_write_blackbox(0, NULL);
	}

	if (op_reply == NULL && (needs_reply \|\| local_notify)) {
	crm_err("Unexpected NULL reply to message");
	crm_log_xml_err(request, "null reply");
	needs_reply = FALSE;
	local_notify = FALSE;
	}
	}

	if (is_update && !cib_legacy_mode()) {
	crm_trace("Completed pre-sync update from %s/%s/%s%s",
	originator ? originator : "local", client_name, call_id,
	local_notify?" with local notification":"");

	} else if (!needs_reply \|\| stand_alone) {
	// This was a non-originating slave update
	crm_trace("Completed slave update");

	} else if (cib_legacy_mode() &&
	rc == pcmk_ok && result_diff != NULL && !(call_options & cib_inhibit_bcast)) {
	gboolean broadcast = FALSE;

	cib_local_bcast_num++;
	crm_xml_add_int(request, F_CIB_LOCAL_NOTIFY_ID, cib_local_bcast_num);
	broadcast = send_peer_reply(request, result_diff, originator, TRUE);

	if (broadcast && client_id && local_notify && op_reply) {

	/* If we have been asked to sync the reply,
	* and a bcast msg has gone out, we queue the local notify
	* until we know the bcast message has been received */
	local_notify = FALSE;
	crm_trace("Queuing local %ssync notification for %s",
	(call_options & cib_sync_call) ? "" : "a-", client_id);

	queue_local_notify(op_reply, client_id,
	pcmk_is_set(call_options, cib_sync_call),
	(cib_client == NULL));
	op_reply = NULL; /* the reply is queued, so don't free here */
	}

	} else if (call_options & cib_discard_reply) {
	crm_trace("Caller isn't interested in reply");

	} else if (cib_client == NULL) {
	if (is_update == FALSE \|\| result_diff == NULL) {
	crm_trace("Request not broadcast: R/O call");

	} else if (call_options & cib_inhibit_bcast) {
	crm_trace("Request not broadcast: inhibited");

	} else if (rc != pcmk_ok) {
	crm_trace("Request not broadcast: call failed: %s", pcmk_strerror(rc));

	} else {
	crm_trace("Directing reply to %s", originator);
	}

	send_peer_reply(op_reply, result_diff, originator, FALSE);
	}

	if (local_notify && client_id) {
	crm_trace("Performing local %ssync notification for %s",
	(pcmk_is_set(call_options, cib_sync_call)? "" : "a"),
	client_id);
	if (process == FALSE) {
	do_local_notify(request, client_id,
	pcmk_is_set(call_options, cib_sync_call),
	(cib_client == NULL));
	} else {
	do_local_notify(op_reply, client_id,
	pcmk_is_set(call_options, cib_sync_call),
	(cib_client == NULL));
	}
	}

	free_xml(op_reply);
	free_xml(result_diff);

	return;
	}

	static char *
	calculate_section_digest(const char xpath, xmlNode xml_obj)
	{
	xmlNode *xml_section = NULL;

	if (xml_obj == NULL) {
	return NULL;
	}

	xml_section = get_xpath_object(xpath, xml_obj, LOG_TRACE);
	if (xml_section == NULL) {
	return NULL;
	}
	return calculate_xml_versioned_digest(xml_section, FALSE, TRUE, CRM_FEATURE_SET);

	}

	static int
	cib_process_command(xmlNode * request, xmlNode reply, xmlNode cib_diff, gboolean privileged)
	{
	xmlNode *input = NULL;
	xmlNode *output = NULL;
	xmlNode *result_cib = NULL;
	xmlNode *current_cib = NULL;

	int call_type = 0;
	int call_options = 0;

	const char *op = NULL;
	const char *section = NULL;
	const char *call_id = crm_element_value(request, F_CIB_CALLID);

	int rc = pcmk_ok;
	int rc2 = pcmk_ok;

	gboolean send_r_notify = FALSE;
	gboolean global_update = FALSE;
	gboolean config_changed = FALSE;
	gboolean manage_counters = TRUE;

	static mainloop_timer_t *digest_timer = NULL;

	char *current_nodes_digest = NULL;
	char *current_alerts_digest = NULL;
	char *current_status_digest = NULL;
	int change_section = cib_change_section_nodes \| cib_change_section_alerts \| cib_change_section_status;

	CRM_ASSERT(cib_status == pcmk_ok);

	if(digest_timer == NULL) {
	digest_timer = mainloop_timer_add("digester", 5000, FALSE, cib_digester_cb, NULL);
	}

	*reply = NULL;
	*cib_diff = NULL;
	current_cib = the_cib;

	/* Start processing the request... */
	op = crm_element_value(request, F_CIB_OPERATION);
	crm_element_value_int(request, F_CIB_CALLOPTS, &call_options);
	rc = cib_get_operation_id(op, &call_type);

	if (rc == pcmk_ok && privileged == FALSE) {
	rc = cib_op_can_run(call_type, call_options, privileged, global_update);
	}

	rc2 = cib_op_prepare(call_type, request, &input, &section);
	if (rc == pcmk_ok) {
	rc = rc2;
	}

	if (rc != pcmk_ok) {
	crm_trace("Call setup failed: %s", pcmk_strerror(rc));
	goto done;

	} else if (cib_op_modifies(call_type) == FALSE) {
	rc = cib_perform_op(op, call_options, cib_op_func(call_type), TRUE,
	section, request, input, FALSE, &config_changed,
	current_cib, &result_cib, NULL, &output);

	CRM_CHECK(result_cib == NULL, free_xml(result_cib));
	goto done;
	}

	/* Handle a valid write action */
	if (pcmk__xe_attr_is_true(request, F_CIB_GLOBAL_UPDATE)) {
	/* legacy code */
	manage_counters = FALSE;
	cib__set_call_options(call_options, "call", cib_force_diff);
	crm_trace("Global update detected");

	CRM_CHECK(call_type == 3 \|\| call_type == 4, crm_err("Call type: %d", call_type);
	crm_log_xml_err(request, "bad op"));
	}

	if (rc == pcmk_ok) {
	ping_modified_since = TRUE;
	if (call_options & cib_inhibit_bcast) {
	/* skip */
	crm_trace("Skipping update: inhibit broadcast");
	manage_counters = FALSE;
	}

	if (!pcmk_is_set(call_options, cib_dryrun)
	&& pcmk__str_eq(section, XML_CIB_TAG_STATUS, pcmk__str_casei)) {
	/* Copying large CIBs accounts for a huge percentage of our CIB usage */
	cib__set_call_options(call_options, "call", cib_zero_copy);
	} else {
	cib__clear_call_options(call_options, "call", cib_zero_copy);
	}

	/* Calculate the hash value of the section before the change. */
	if (pcmk__str_eq(CIB_OP_REPLACE, op, pcmk__str_none)) {
	current_nodes_digest = calculate_section_digest("//" XML_TAG_CIB "/" XML_CIB_TAG_CONFIGURATION "/" XML_CIB_TAG_NODES, current_cib);
	current_alerts_digest = calculate_section_digest("//" XML_TAG_CIB "/" XML_CIB_TAG_CONFIGURATION "/" XML_CIB_TAG_ALERTS, current_cib);
	current_status_digest = calculate_section_digest("//" XML_TAG_CIB "/" XML_CIB_TAG_STATUS, current_cib);
	crm_trace("current-digest %s:%s:%s", current_nodes_digest, current_alerts_digest, current_status_digest);
	}

	/* result_cib must not be modified after cib_perform_op() returns */
	rc = cib_perform_op(op, call_options, cib_op_func(call_type), FALSE,
	section, request, input, manage_counters, &config_changed,
	current_cib, &result_cib, cib_diff, &output);

	if (manage_counters == FALSE) {
	int format = 1;
	/* Legacy code
	* If the diff is NULL at this point, it's because nothing changed
	*/
	if (*cib_diff) {
	crm_element_value_int(*cib_diff, "format", &format);
	}

	if (format == 1) {
	config_changed = cib_config_changed(NULL, NULL, cib_diff);
	}
	}

	/* Always write to disk for replace ops,
	* this also negates the need to detect ordering changes
	*/
	if (pcmk__str_eq(CIB_OP_REPLACE, op, pcmk__str_none)) {
	config_changed = TRUE;
	}
	}

	if (rc == pcmk_ok && !pcmk_is_set(call_options, cib_dryrun)) {
	crm_trace("Activating %s->%s%s%s",
	crm_element_value(current_cib, XML_ATTR_NUMUPDATES),
	crm_element_value(result_cib, XML_ATTR_NUMUPDATES),
	(pcmk_is_set(call_options, cib_zero_copy)? " zero-copy" : ""),
	(config_changed? " changed" : ""));
	if (!pcmk_is_set(call_options, cib_zero_copy)) {
	rc = activateCibXml(result_cib, config_changed, op);
	crm_trace("Activated %s (%d)",
	crm_element_value(current_cib, XML_ATTR_NUMUPDATES), rc);
	}

	if (rc == pcmk_ok && cib_internal_config_changed(*cib_diff)) {
	cib_read_config(config_hash, result_cib);
	}

	if (pcmk__str_eq(CIB_OP_REPLACE, op, pcmk__str_none)) {
	char *result_nodes_digest = NULL;
	char *result_alerts_digest = NULL;
	char *result_status_digest = NULL;

	/* Calculate the hash value of the changed section. */
	result_nodes_digest = calculate_section_digest("//" XML_TAG_CIB "/" XML_CIB_TAG_CONFIGURATION "/" XML_CIB_TAG_NODES, result_cib);
	result_alerts_digest = calculate_section_digest("//" XML_TAG_CIB "/" XML_CIB_TAG_CONFIGURATION "/" XML_CIB_TAG_ALERTS, result_cib);
	result_status_digest = calculate_section_digest("//" XML_TAG_CIB "/" XML_CIB_TAG_STATUS, result_cib);
	crm_trace("result-digest %s:%s:%s", result_nodes_digest, result_alerts_digest, result_status_digest);

	if (pcmk__str_eq(current_nodes_digest, result_nodes_digest, pcmk__str_none)) {
	change_section = change_section ^ cib_change_section_nodes;
	}
	if (pcmk__str_eq(current_alerts_digest, result_alerts_digest, pcmk__str_none)) {
	change_section = change_section ^ cib_change_section_alerts;
	}
	if (pcmk__str_eq(current_status_digest, result_status_digest, pcmk__str_none)) {
	change_section = change_section ^ cib_change_section_status;
	}

	if (change_section) {
	send_r_notify = TRUE;
	}

	free(result_nodes_digest);
	free(result_alerts_digest);
	free(result_status_digest);

	} else if (pcmk__str_eq(CIB_OP_ERASE, op, pcmk__str_none)) {
	send_r_notify = TRUE;
	}

	mainloop_timer_stop(digest_timer);
	mainloop_timer_start(digest_timer);

	} else if (rc == -pcmk_err_schema_validation) {
	CRM_ASSERT(!pcmk_is_set(call_options, cib_zero_copy));

	if (output != NULL) {
	crm_log_xml_info(output, "cib:output");
	free_xml(output);
	}

	output = result_cib;

	} else {
	crm_trace("Not activating %d %d %s", rc,
	pcmk_is_set(call_options, cib_dryrun),
	crm_element_value(result_cib, XML_ATTR_NUMUPDATES));
	if (!pcmk_is_set(call_options, cib_zero_copy)) {
	free_xml(result_cib);
	}
	}

	if ((call_options & (cib_inhibit_notify\|cib_dryrun)) == 0) {
	const char *client = crm_element_value(request, F_CIB_CLIENTNAME);

	crm_trace("Sending notifications %d",
	pcmk_is_set(call_options, cib_dryrun));
	cib_diff_notify(call_options, client, call_id, op, input, rc, *cib_diff);
	}

	if (send_r_notify) {
	const char *origin = crm_element_value(request, F_ORIG);

	cib_replace_notify(origin, the_cib, rc, *cib_diff, change_section);
	}

	xml_log_patchset(LOG_TRACE, "cib:diff", *cib_diff);
	done:
	if (!pcmk_is_set(call_options, cib_discard_reply) \|\| cib_legacy_mode()) {
	const char *caller = crm_element_value(request, F_CIB_CLIENTID);

	*reply = create_xml_node(NULL, "cib-reply");
	crm_xml_add(*reply, F_TYPE, T_CIB);
	crm_xml_add(*reply, F_CIB_OPERATION, op);
	crm_xml_add(*reply, F_CIB_CALLID, call_id);
	crm_xml_add(*reply, F_CIB_CLIENTID, caller);
	crm_xml_add_int(*reply, F_CIB_CALLOPTS, call_options);
	crm_xml_add_int(*reply, F_CIB_RC, rc);

	if (output != NULL) {
	crm_trace("Attaching reply output");
	add_message_xml(*reply, F_CIB_CALLDATA, output);
	}

	crm_log_xml_explicit(*reply, "cib:reply");
	}

	crm_trace("cleanup");

	if (cib_op_modifies(call_type) == FALSE && output != current_cib) {
	free_xml(output);
	output = NULL;
	}

	if (call_type >= 0) {
	cib_op_cleanup(call_type, call_options, &input, &output);
	}

	free(current_nodes_digest);
	free(current_alerts_digest);
	free(current_status_digest);

	crm_trace("done");
	return rc;
	}

	void
	cib_peer_callback(xmlNode * msg, void *private_data)
	{
	const char *reason = NULL;
	const char *originator = crm_element_value(msg, F_ORIG);

	if (cib_legacy_mode() && pcmk__str_eq(originator, cib_our_uname, pcmk__str_null_matches)) {
	/* message is from ourselves */
	int bcast_id = 0;

	if (!(crm_element_value_int(msg, F_CIB_LOCAL_NOTIFY_ID, &bcast_id))) {
	check_local_notify(bcast_id);
	}
	return;

	} else if (crm_peer_cache == NULL) {
	reason = "membership not established";
	goto bail;
	}

	if (crm_element_value(msg, F_CIB_CLIENTNAME) == NULL) {
	crm_xml_add(msg, F_CIB_CLIENTNAME, originator);
	}

	- /* crm_log_xml_trace("Peer[inbound]", msg); */
	+ /* crm_log_xml_trace(msg, "Peer[inbound]"); */
	cib_process_request(msg, TRUE, NULL);
	return;

	bail:
	if (reason) {
	const char *seq = crm_element_value(msg, F_SEQ);
	const char *op = crm_element_value(msg, F_CIB_OPERATION);

	crm_warn("Discarding %s message (%s) from %s: %s", op, seq, originator, reason);
	}
	}

	static gboolean
	cib_force_exit(gpointer data)
	{
	crm_notice("Forcing exit!");
	terminate_cib(__func__, CRM_EX_ERROR);
	return FALSE;
	}

	static void
	disconnect_remote_client(gpointer key, gpointer value, gpointer user_data)
	{
	pcmk__client_t *a_client = value;

	crm_err("Can't disconnect client %s: Not implemented",
	pcmk__client_name(a_client));
	}

	void
	cib_shutdown(int nsig)
	{
	struct qb_ipcs_stats srv_stats;

	if (cib_shutdown_flag == FALSE) {
	int disconnects = 0;
	qb_ipcs_connection_t *c = NULL;

	cib_shutdown_flag = TRUE;

	c = qb_ipcs_connection_first_get(ipcs_rw);
	while (c != NULL) {
	qb_ipcs_connection_t *last = c;

	c = qb_ipcs_connection_next_get(ipcs_rw, last);

	crm_debug("Disconnecting r/w client %p...", last);
	qb_ipcs_disconnect(last);
	qb_ipcs_connection_unref(last);
	disconnects++;
	}

	c = qb_ipcs_connection_first_get(ipcs_ro);
	while (c != NULL) {
	qb_ipcs_connection_t *last = c;

	c = qb_ipcs_connection_next_get(ipcs_ro, last);

	crm_debug("Disconnecting r/o client %p...", last);
	qb_ipcs_disconnect(last);
	qb_ipcs_connection_unref(last);
	disconnects++;
	}

	c = qb_ipcs_connection_first_get(ipcs_shm);
	while (c != NULL) {
	qb_ipcs_connection_t *last = c;

	c = qb_ipcs_connection_next_get(ipcs_shm, last);

	crm_debug("Disconnecting non-blocking r/w client %p...", last);
	qb_ipcs_disconnect(last);
	qb_ipcs_connection_unref(last);
	disconnects++;
	}

	disconnects += pcmk__ipc_client_count();

	crm_debug("Disconnecting %d remote clients", pcmk__ipc_client_count());
	pcmk__foreach_ipc_client(disconnect_remote_client, NULL);
	crm_info("Disconnected %d clients", disconnects);
	}

	qb_ipcs_stats_get(ipcs_rw, &srv_stats, QB_FALSE);

	if (pcmk__ipc_client_count() == 0) {
	crm_info("All clients disconnected (%d)", srv_stats.active_connections);
	initiate_exit();

	} else {
	crm_info("Waiting on %d clients to disconnect (%d)",
	pcmk__ipc_client_count(), srv_stats.active_connections);
	}
	}

	void
	initiate_exit(void)
	{
	int active = 0;
	xmlNode *leaving = NULL;

	active = crm_active_peers();
	if (active < 2) {
	terminate_cib(__func__, 0);
	return;
	}

	crm_info("Sending disconnect notification to %d peers...", active);

	leaving = create_xml_node(NULL, "exit-notification");
	crm_xml_add(leaving, F_TYPE, "cib");
	crm_xml_add(leaving, F_CIB_OPERATION, "cib_shutdown_req");

	send_cluster_message(NULL, crm_msg_cib, leaving, TRUE);
	free_xml(leaving);

	g_timeout_add(EXIT_ESCALATION_MS, cib_force_exit, NULL);
	}

	extern int remote_fd;
	extern int remote_tls_fd;

	/*!
	* \internal
	* \brief Close remote sockets, free the global CIB and quit
	*
	* \param[in] caller Name of calling function (for log message)
	* \param[in] fast If -1, skip disconnect; if positive, exit that
	*/
	void
	terminate_cib(const char *caller, int fast)
	{
	crm_info("%s: Exiting%s...", caller,
	(fast > 0)? " fast" : mainloop ? " from mainloop" : "");

	if (remote_fd > 0) {
	close(remote_fd);
	remote_fd = 0;
	}
	if (remote_tls_fd > 0) {
	close(remote_tls_fd);
	remote_tls_fd = 0;
	}

	uninitializeCib();

	if (fast > 0) {
	/* Quit fast on error */
	pcmk__stop_based_ipc(ipcs_ro, ipcs_rw, ipcs_shm);
	crm_exit(fast);

	} else if ((mainloop != NULL) && g_main_loop_is_running(mainloop)) {
	/* Quit via returning from the main loop. If fast == -1, we skip the
	* disconnect here, and it will be done when the main loop returns
	* (this allows the peer status callback to avoid messing with the
	* peer caches).
	*/
	if (fast == 0) {
	crm_cluster_disconnect(&crm_cluster);
	}
	g_main_loop_quit(mainloop);

	} else {
	/* Quit via clean exit. Even the peer status callback can disconnect
	* here, because we're not returning control to the caller. */
	crm_cluster_disconnect(&crm_cluster);
	pcmk__stop_based_ipc(ipcs_ro, ipcs_rw, ipcs_shm);
	crm_exit(CRM_EX_OK);
	}
	}
	diff --git a/daemons/controld/controld_callbacks.c b/daemons/controld/controld_callbacks.c
	index 42006e47b4..a76180aa01 100644
	--- a/daemons/controld/controld_callbacks.c
	+++ b/daemons/controld/controld_callbacks.c
	@@ -1,361 +1,361 @@
	/*
	* Copyright 2004-2022 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <sys/param.h>
	#include <string.h>

	#include <crm/crm.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>
	#include <crm/cluster.h>
	#include <crm/cib.h>

	#include <pacemaker-controld.h>

	/* From join_dc... */
	extern gboolean check_join_state(enum crmd_fsa_state cur_state, const char *source);

	void
	crmd_ha_msg_filter(xmlNode * msg)
	{
	if (AM_I_DC) {
	const char *sys_from = crm_element_value(msg, F_CRM_SYS_FROM);

	if (pcmk__str_eq(sys_from, CRM_SYSTEM_DC, pcmk__str_casei)) {
	const char *from = crm_element_value(msg, F_ORIG);

	if (!pcmk__str_eq(from, fsa_our_uname, pcmk__str_casei)) {
	int level = LOG_INFO;
	const char *op = crm_element_value(msg, F_CRM_TASK);

	/* make sure the election happens NOW */
	if (fsa_state != S_ELECTION) {
	ha_msg_input_t new_input;

	level = LOG_WARNING;
	new_input.msg = msg;
	register_fsa_error_adv(C_FSA_INTERNAL, I_ELECTION, NULL, &new_input,
	__func__);
	}

	do_crm_log(level, "Another DC detected: %s (op=%s)", from, op);
	goto done;
	}
	}

	} else {
	const char *sys_to = crm_element_value(msg, F_CRM_SYS_TO);

	if (pcmk__str_eq(sys_to, CRM_SYSTEM_DC, pcmk__str_casei)) {
	return;
	}
	}

	- /* crm_log_xml_trace("HA[inbound]", msg); */
	+ /* crm_log_xml_trace(msg, "HA[inbound]"); */
	route_message(C_HA_MESSAGE, msg);

	done:
	trigger_fsa();
	}

	/*!
	* \internal
	* \brief Check whether a node is online
	*
	* \param[in] node Node to check
	*
	* \retval -1 if completely dead
	* \retval 0 if partially alive
	* \retval 1 if completely alive
	*/
	static int
	node_alive(const crm_node_t *node)
	{
	if (pcmk_is_set(node->flags, crm_remote_node)) {
	// Pacemaker Remote nodes can't be partially alive
	return pcmk__str_eq(node->state, CRM_NODE_MEMBER, pcmk__str_casei) ? 1: -1;

	} else if (crm_is_peer_active(node)) {
	// Completely up cluster node: both cluster member and peer
	return 1;

	} else if (!pcmk_is_set(node->processes, crm_get_cluster_proc())
	&& !pcmk__str_eq(node->state, CRM_NODE_MEMBER, pcmk__str_casei)) {
	// Completely down cluster node: neither cluster member nor peer
	return -1;
	}

	// Partially up cluster node: only cluster member or only peer
	return 0;
	}

	#define state_text(state) ((state)? (const char *)(state) : "in unknown state")

	bool controld_dc_left = false;

	void
	peer_update_callback(enum crm_status_type type, crm_node_t * node, const void *data)
	{
	uint32_t old = 0;
	bool appeared = FALSE;
	bool is_remote = pcmk_is_set(node->flags, crm_remote_node);

	/* The controller waits to receive some information from the membership
	* layer before declaring itself operational. If this is being called for a
	* cluster node, indicate that we have it.
	*/
	if (!is_remote) {
	controld_set_fsa_input_flags(R_PEER_DATA);
	}

	if (type == crm_status_processes
	&& pcmk_is_set(node->processes, crm_get_cluster_proc())
	&& !AM_I_DC
	&& !is_remote) {
	/*
	* This is a hack until we can send to a nodeid and/or we fix node name lookups
	* These messages are ignored in crmd_ha_msg_filter()
	*/
	xmlNode *query = create_request(CRM_OP_HELLO, NULL, NULL, CRM_SYSTEM_CRMD, CRM_SYSTEM_CRMD, NULL);

	crm_debug("Sending hello to node %u so that it learns our node name", node->id);
	send_cluster_message(node, crm_msg_crmd, query, FALSE);

	free_xml(query);
	}

	if (node->uname == NULL) {
	return;
	}

	switch (type) {
	case crm_status_uname:
	/* If we've never seen the node, then it also won't be in the status section */
	crm_info("%s node %s is now %s",
	(is_remote? "Remote" : "Cluster"),
	node->uname, state_text(node->state));
	return;

	case crm_status_nstate:
	/* This callback should not be called unless the state actually
	* changed, but here's a failsafe just in case.
	*/
	CRM_CHECK(!pcmk__str_eq(data, node->state, pcmk__str_casei),
	return);

	crm_info("%s node %s is now %s (was %s)",
	(is_remote? "Remote" : "Cluster"),
	node->uname, state_text(node->state), state_text(data));

	if (pcmk__str_eq(CRM_NODE_MEMBER, node->state, pcmk__str_casei)) {
	appeared = TRUE;
	if (!is_remote) {
	remove_stonith_cleanup(node->uname);
	}
	} else {
	controld_remove_voter(node->uname);
	}

	crmd_alert_node_event(node);
	break;

	case crm_status_processes:
	CRM_CHECK(data != NULL, return);
	old = (const uint32_t )data;
	appeared = pcmk_is_set(node->processes, crm_get_cluster_proc());

	crm_info("Node %s is %s a peer " CRM_XS " DC=%s old=%#07x new=%#07x",
	node->uname, (appeared? "now" : "no longer"),
	(AM_I_DC? "true" : (fsa_our_dc? fsa_our_dc : "<none>")),
	old, node->processes);

	if (!pcmk_is_set((node->processes ^ old), crm_get_cluster_proc())) {
	/* Peer status did not change. This should not be possible,
	* since we don't track process flags other than peer status.
	*/
	crm_trace("Process flag %#7x did not change from %#7x to %#7x",
	crm_get_cluster_proc(), old, node->processes);
	return;

	}

	if (!appeared) {
	controld_remove_voter(node->uname);
	}

	if (!pcmk_is_set(fsa_input_register, R_CIB_CONNECTED)) {
	crm_trace("Ignoring peer status change because not connected to CIB");
	return;

	} else if (fsa_state == S_STOPPING) {
	crm_trace("Ignoring peer status change because stopping");
	return;
	}

	if (pcmk__str_eq(node->uname, fsa_our_uname, pcmk__str_casei) && !appeared) {
	/* Did we get evicted? */
	crm_notice("Our peer connection failed");
	register_fsa_input(C_CRMD_STATUS_CALLBACK, I_ERROR, NULL);

	} else if (pcmk__str_eq(node->uname, fsa_our_dc, pcmk__str_casei) && crm_is_peer_active(node) == FALSE) {
	/* Did the DC leave us? */
	crm_notice("Our peer on the DC (%s) is dead", fsa_our_dc);
	register_fsa_input(C_CRMD_STATUS_CALLBACK, I_ELECTION, NULL);

	/* @COMPAT DC < 1.1.13: If a DC shuts down normally, we don't
	* want to fence it. Newer DCs will send their shutdown request
	* to all peers, who will update the DC's expected state to
	* down, thus avoiding fencing. We can safely erase the DC's
	* transient attributes when it leaves in that case. However,
	* the only way to avoid fencing older DCs is to leave the
	* transient attributes intact until it rejoins.
	*/
	if (compare_version(fsa_our_dc_version, "3.0.9") > 0) {
	controld_delete_node_state(node->uname,
	controld_section_attrs,
	cib_scope_local);
	}

	} else if (AM_I_DC \|\| controld_dc_left \|\| (fsa_our_dc == NULL)) {
	/* This only needs to be done once, so normally the DC should do
	* it. However if there is no DC, every node must do it, since
	* there is no other way to ensure some one node does it.
	*/
	if (appeared) {
	te_trigger_stonith_history_sync(FALSE);
	} else {
	controld_delete_node_state(node->uname,
	controld_section_attrs,
	cib_scope_local);
	}
	}
	break;
	}

	if (AM_I_DC) {
	xmlNode *update = NULL;
	int flags = node_update_peer;
	int alive = node_alive(node);
	crm_action_t *down = match_down_event(node->uuid);

	crm_trace("Alive=%d, appeared=%d, down=%d",
	alive, appeared, (down? down->id : -1));

	if (appeared && (alive > 0) && !is_remote) {
	register_fsa_input_before(C_FSA_INTERNAL, I_NODE_JOIN, NULL);
	}

	if (down) {
	const char *task = crm_element_value(down->xml, XML_LRM_ATTR_TASK);

	if (pcmk__str_eq(task, CRM_OP_FENCE, pcmk__str_casei)) {

	/* tengine_stonith_callback() confirms fence actions */
	crm_trace("Updating CIB %s fencer reported fencing of %s complete",
	(pcmk_is_set(down->flags, pcmk__graph_action_confirmed)? "after" : "before"), node->uname);

	} else if (!appeared && pcmk__str_eq(task, CRM_OP_SHUTDOWN, pcmk__str_casei)) {

	// Shutdown actions are immediately confirmed (i.e. no_wait)
	if (!is_remote) {
	flags \|= node_update_join \| node_update_expected;
	crmd_peer_down(node, FALSE);
	check_join_state(fsa_state, __func__);
	}
	if (alive >= 0) {
	crm_info("%s of peer %s is in progress " CRM_XS " action=%d",
	task, node->uname, down->id);
	} else {
	crm_notice("%s of peer %s is complete " CRM_XS " action=%d",
	task, node->uname, down->id);
	pcmk__update_graph(transition_graph, down);
	trigger_graph();
	}

	} else {
	crm_trace("Node %s is %s, was expected to %s (op %d)",
	node->uname,
	((alive > 0)? "alive" :
	((alive < 0)? "dead" : "partially alive")),
	task, down->id);
	}

	} else if (appeared == FALSE) {
	crm_warn("Stonith/shutdown of node %s was not expected",
	node->uname);
	if (!is_remote) {
	crm_update_peer_join(__func__, node, crm_join_none);
	check_join_state(fsa_state, __func__);
	}
	abort_transition(INFINITY, tg_restart, "Node failure", NULL);
	fail_incompletable_actions(transition_graph, node->uuid);

	} else {
	crm_trace("Node %s came up, was not expected to be down",
	node->uname);
	}

	if (is_remote) {
	/* A pacemaker_remote node won't have its cluster status updated
	* in the CIB by membership-layer callbacks, so do it here.
	*/
	flags \|= node_update_cluster;

	/* Trigger resource placement on newly integrated nodes */
	if (appeared) {
	abort_transition(INFINITY, tg_restart,
	"Pacemaker Remote node integrated", NULL);
	}
	}

	/* Update the CIB node state */
	update = create_node_state_update(node, flags, NULL, __func__);
	if (update == NULL) {
	crm_debug("Node state update not yet possible for %s", node->uname);
	} else {
	fsa_cib_anon_update(XML_CIB_TAG_STATUS, update);
	}
	free_xml(update);
	}

	trigger_fsa();
	}

	void
	crmd_cib_connection_destroy(gpointer user_data)
	{
	CRM_LOG_ASSERT(user_data == fsa_cib_conn);

	crm_trace("Invoked");
	trigger_fsa();
	fsa_cib_conn->state = cib_disconnected;

	if (!pcmk_is_set(fsa_input_register, R_CIB_CONNECTED)) {
	crm_info("Connection to the CIB manager terminated");
	return;
	}

	// @TODO This should trigger a reconnect, not a shutdown
	crm_crit("Lost connection to the CIB manager, shutting down");
	register_fsa_input(C_FSA_INTERNAL, I_ERROR, NULL);
	controld_clear_fsa_input_flags(R_CIB_CONNECTED);

	return;
	}

	gboolean
	crm_fsa_trigger(gpointer user_data)
	{
	crm_trace("Invoked (queue len: %d)", g_list_length(fsa_message_queue));
	s_crmd_fsa(C_FSA_INTERNAL);
	crm_trace("Exited (queue len: %d)", g_list_length(fsa_message_queue));
	return TRUE;
	}
	diff --git a/lib/common/xml.c b/lib/common/xml.c
	index a0b8965244..0a97a09f67 100644
	--- a/lib/common/xml.c
	+++ b/lib/common/xml.c
	@@ -1,2991 +1,2991 @@
	/*
	* Copyright 2004-2022 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdio.h>
	#include <sys/types.h>
	#include <unistd.h>
	#include <time.h>
	#include <string.h>
	#include <stdlib.h>
	#include <stdarg.h>
	#include <bzlib.h>

	#include <libxml/parser.h>
	#include <libxml/tree.h>
	#include <libxml/xmlIO.h> /* xmlAllocOutputBuffer */

	#include <crm/crm.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>
	#include <crm/common/xml_internal.h> // PCMK__XML_LOG_BASE, etc.
	#include "crmcommon_private.h"

	// Define this as 1 in development to get insanely verbose trace messages
	#ifndef XML_PARSER_DEBUG
	#define XML_PARSER_DEBUG 0
	#endif


	/* @TODO XML_PARSE_RECOVER allows some XML errors to be silently worked around
	* by libxml2, which is potentially ambiguous and dangerous. We should drop it
	* when we can break backward compatibility with configurations that might be
	* relying on it (i.e. pacemaker 3.0.0).
	*
	* It might be a good idea to have a transitional period where we first try
	* parsing without XML_PARSE_RECOVER, and if that fails, try parsing again with
	* it, logging a warning if it succeeds.
	*/
	#define PCMK__XML_PARSE_OPTS (XML_PARSE_NOBLANKS \| XML_PARSE_RECOVER)

	#define CHUNK_SIZE 1024

	bool
	pcmk__tracking_xml_changes(xmlNode *xml, bool lazy)
	{
	if(xml == NULL \|\| xml->doc == NULL \|\| xml->doc->_private == NULL) {
	return FALSE;
	} else if (!pcmk_is_set(((xml_private_t *)xml->doc->_private)->flags,
	pcmk__xf_tracking)) {
	return FALSE;
	} else if (lazy && !pcmk_is_set(((xml_private_t *)xml->doc->_private)->flags,
	pcmk__xf_lazy)) {
	return FALSE;
	}
	return TRUE;
	}

	#define buffer_print(buffer, max, offset, fmt, args...) do { \
	int rc = (max); \
	if(buffer) { \
	rc = snprintf((buffer) + (offset), (max) - (offset), fmt, ##args); \
	} \
	if(buffer && rc < 0) { \
	crm_perror(LOG_ERR, "snprintf failed at offset %d", offset); \
	(buffer)[(offset)] = 0; \
	break; \
	} else if(rc >= ((max) - (offset))) { \
	char *tmp = NULL; \
	(max) = QB_MAX(CHUNK_SIZE, (max) * 2); \
	tmp = pcmk__realloc((buffer), (max)); \
	CRM_ASSERT(tmp); \
	(buffer) = tmp; \
	} else { \
	offset += rc; \
	break; \
	} \
	} while(1);

	static void
	insert_prefix(int options, char *buffer, int offset, int *max, int depth)
	{
	if (options & xml_log_option_formatted) {
	size_t spaces = 2 * depth;

	if ((buffer) == NULL \|\| spaces >= ((max) - (*offset))) {
	(max) = QB_MAX(CHUNK_SIZE, (max) * 2);
	(buffer) = pcmk__realloc((buffer), (*max));
	}
	memset((buffer) + (offset), ' ', spaces);
	(*offset) += spaces;
	}
	}

	static void
	set_parent_flag(xmlNode *xml, long flag)
	{

	for(; xml; xml = xml->parent) {
	xml_private_t *p = xml->_private;

	if(p == NULL) {
	/* During calls to xmlDocCopyNode(), _private will be unset for parent nodes */
	} else {
	pcmk__set_xml_flags(p, flag);
	}
	}
	}

	void
	pcmk__set_xml_doc_flag(xmlNode *xml, enum xml_private_flags flag)
	{

	if(xml && xml->doc && xml->doc->_private){
	/* During calls to xmlDocCopyNode(), xml->doc may be unset */
	xml_private_t *p = xml->doc->_private;

	pcmk__set_xml_flags(p, flag);
	}
	}

	// Mark document, element, and all element's parents as changed
	static void
	mark_xml_node_dirty(xmlNode *xml)
	{
	pcmk__set_xml_doc_flag(xml, pcmk__xf_dirty);
	set_parent_flag(xml, pcmk__xf_dirty);
	}

	// Clear flags on XML node and its children
	static void
	reset_xml_node_flags(xmlNode *xml)
	{
	xmlNode *cIter = NULL;
	xml_private_t *p = xml->_private;

	if (p) {
	p->flags = 0;
	}

	for (cIter = pcmk__xml_first_child(xml); cIter != NULL;
	cIter = pcmk__xml_next(cIter)) {
	reset_xml_node_flags(cIter);
	}
	}

	// Set xpf_created flag on XML node and any children
	void
	pcmk__mark_xml_created(xmlNode *xml)
	{
	xmlNode *cIter = NULL;
	xml_private_t *p = xml->_private;

	if (p && pcmk__tracking_xml_changes(xml, FALSE)) {
	if (!pcmk_is_set(p->flags, pcmk__xf_created)) {
	pcmk__set_xml_flags(p, pcmk__xf_created);
	mark_xml_node_dirty(xml);
	}
	for (cIter = pcmk__xml_first_child(xml); cIter != NULL;
	cIter = pcmk__xml_next(cIter)) {
	pcmk__mark_xml_created(cIter);
	}
	}
	}

	void
	pcmk__mark_xml_attr_dirty(xmlAttr *a)
	{
	xmlNode *parent = a->parent;
	xml_private_t *p = NULL;

	p = a->_private;
	pcmk__set_xml_flags(p, pcmk__xf_dirty\|pcmk__xf_modified);
	pcmk__clear_xml_flags(p, pcmk__xf_deleted);
	mark_xml_node_dirty(parent);
	}

	#define XML_PRIVATE_MAGIC (long) 0x81726354

	// Free an XML object previously marked as deleted
	static void
	free_deleted_object(void *data)
	{
	if(data) {
	pcmk__deleted_xml_t *deleted_obj = data;

	free(deleted_obj->path);
	free(deleted_obj);
	}
	}

	// Free and NULL user, ACLs, and deleted objects in an XML node's private data
	static void
	reset_xml_private_data(xml_private_t *p)
	{
	if(p) {
	CRM_ASSERT(p->check == XML_PRIVATE_MAGIC);

	free(p->user);
	p->user = NULL;

	if(p->acls) {
	pcmk__free_acls(p->acls);
	p->acls = NULL;
	}

	if(p->deleted_objs) {
	g_list_free_full(p->deleted_objs, free_deleted_object);
	p->deleted_objs = NULL;
	}
	}
	}

	// Free all private data associated with an XML node
	static void
	free_private_data(xmlNode *node)
	{
	/* need to explicitly avoid our custom _private field cleanup when
	called from internal XSLT cleanup (xsltApplyStylesheetInternal
	-> xsltFreeTransformContext -> xsltFreeRVTs -> xmlFreeDoc)
	onto result tree fragments, represented as standalone documents
	with otherwise infeasible space-prefixed name (xsltInternals.h:
	XSLT_MARK_RES_TREE_FRAG) and carrying it's own load at _private
	field -- later assert on the XML_PRIVATE_MAGIC would explode */
	if (node->type != XML_DOCUMENT_NODE \|\| node->name == NULL
	\|\| node->name[0] != ' ') {
	reset_xml_private_data(node->_private);
	free(node->_private);
	}
	}

	// Allocate and initialize private data for an XML node
	static void
	new_private_data(xmlNode *node)
	{
	xml_private_t *p = NULL;

	switch(node->type) {
	case XML_ELEMENT_NODE:
	case XML_DOCUMENT_NODE:
	case XML_ATTRIBUTE_NODE:
	case XML_COMMENT_NODE:
	p = calloc(1, sizeof(xml_private_t));
	p->check = XML_PRIVATE_MAGIC;
	/* Flags will be reset if necessary when tracking is enabled */
	pcmk__set_xml_flags(p, pcmk__xf_dirty\|pcmk__xf_created);
	node->_private = p;
	break;
	case XML_TEXT_NODE:
	case XML_DTD_NODE:
	case XML_CDATA_SECTION_NODE:
	break;
	default:
	/* Ignore */
	crm_trace("Ignoring %p %d", node, node->type);
	CRM_LOG_ASSERT(node->type == XML_ELEMENT_NODE);
	break;
	}

	if(p && pcmk__tracking_xml_changes(node, FALSE)) {
	/* XML_ELEMENT_NODE doesn't get picked up here, node->doc is
	* not hooked up at the point we are called
	*/
	mark_xml_node_dirty(node);
	}
	}

	void
	xml_track_changes(xmlNode * xml, const char user, xmlNode acl_source, bool enforce_acls)
	{
	xml_accept_changes(xml);
	crm_trace("Tracking changes%s to %p", enforce_acls?" with ACLs":"", xml);
	pcmk__set_xml_doc_flag(xml, pcmk__xf_tracking);
	if(enforce_acls) {
	if(acl_source == NULL) {
	acl_source = xml;
	}
	pcmk__set_xml_doc_flag(xml, pcmk__xf_acl_enabled);
	pcmk__unpack_acl(acl_source, xml, user);
	pcmk__apply_acl(xml);
	}
	}

	bool xml_tracking_changes(xmlNode * xml)
	{
	return (xml != NULL) && (xml->doc != NULL) && (xml->doc->_private != NULL)
	&& pcmk_is_set(((xml_private_t *)(xml->doc->_private))->flags,
	pcmk__xf_tracking);
	}

	bool xml_document_dirty(xmlNode *xml)
	{
	return (xml != NULL) && (xml->doc != NULL) && (xml->doc->_private != NULL)
	&& pcmk_is_set(((xml_private_t *)(xml->doc->_private))->flags,
	pcmk__xf_dirty);
	}

	/*!
	* \internal
	* \brief Return ordinal position of an XML node among its siblings
	*
	* \param[in] xml XML node to check
	* \param[in] ignore_if_set Don't count siblings with this flag set
	*
	* \return Ordinal position of \p xml (starting with 0)
	*/
	int
	pcmk__xml_position(xmlNode *xml, enum xml_private_flags ignore_if_set)
	{
	int position = 0;
	xmlNode *cIter = NULL;

	for(cIter = xml; cIter->prev; cIter = cIter->prev) {
	xml_private_t p = ((xmlNode)cIter->prev)->_private;

	if (!pcmk_is_set(p->flags, ignore_if_set)) {
	position++;
	}
	}

	return position;
	}

	// This also clears attribute's flags if not marked as deleted
	static bool
	marked_as_deleted(xmlAttrPtr a, void *user_data)
	{
	xml_private_t *p = a->_private;

	if (pcmk_is_set(p->flags, pcmk__xf_deleted)) {
	return true;
	}
	p->flags = pcmk__xf_none;
	return false;
	}

	// Remove all attributes marked as deleted from an XML node
	static void
	accept_attr_deletions(xmlNode *xml)
	{
	// Clear XML node's flags
	((xml_private_t *) xml->_private)->flags = pcmk__xf_none;

	// Remove this XML node's attributes that were marked as deleted
	pcmk__xe_remove_matching_attrs(xml, marked_as_deleted, NULL);

	// Recursively do the same for this XML node's children
	for (xmlNodePtr cIter = pcmk__xml_first_child(xml); cIter != NULL;
	cIter = pcmk__xml_next(cIter)) {
	accept_attr_deletions(cIter);
	}
	}

	/*!
	* \internal
	* \brief Find first child XML node matching another given XML node
	*
	* \param[in] haystack XML whose children should be checked
	* \param[in] needle XML to match (comment content or element name and ID)
	* \param[in] exact If true and needle is a comment, position must match
	*/
	xmlNode *
	pcmk__xml_match(xmlNode haystack, xmlNode needle, bool exact)
	{
	CRM_CHECK(needle != NULL, return NULL);

	if (needle->type == XML_COMMENT_NODE) {
	return pcmk__xc_match(haystack, needle, exact);

	} else {
	const char *id = ID(needle);
	const char *attr = (id == NULL)? NULL : XML_ATTR_ID;

	return pcmk__xe_match(haystack, crm_element_name(needle), attr, id);
	}
	}

	void
	xml_log_changes(uint8_t log_level, const char function, xmlNode xml)
	{
	GList *gIter = NULL;
	xml_private_t *doc = NULL;

	if (log_level == LOG_NEVER) {
	return;
	}

	CRM_ASSERT(xml);
	CRM_ASSERT(xml->doc);

	doc = xml->doc->_private;
	if (!pcmk_is_set(doc->flags, pcmk__xf_dirty)) {
	return;
	}

	for(gIter = doc->deleted_objs; gIter; gIter = gIter->next) {
	pcmk__deleted_xml_t *deleted_obj = gIter->data;

	if (deleted_obj->position >= 0) {
	do_crm_log_alias(log_level, __FILE__, function, __LINE__, "-- %s (%d)",
	deleted_obj->path, deleted_obj->position);

	} else {
	do_crm_log_alias(log_level, __FILE__, function, __LINE__, "-- %s",
	deleted_obj->path);
	}
	}

	log_data_element(log_level, __FILE__, function, __LINE__, "+ ", xml, 0,
	xml_log_option_formatted\|xml_log_option_dirty_add);
	}

	void
	xml_accept_changes(xmlNode * xml)
	{
	xmlNode *top = NULL;
	xml_private_t *doc = NULL;

	if(xml == NULL) {
	return;
	}

	crm_trace("Accepting changes to %p", xml);
	doc = xml->doc->_private;
	top = xmlDocGetRootElement(xml->doc);

	reset_xml_private_data(xml->doc->_private);

	if (!pcmk_is_set(doc->flags, pcmk__xf_dirty)) {
	doc->flags = pcmk__xf_none;
	return;
	}

	doc->flags = pcmk__xf_none;
	accept_attr_deletions(top);
	}

	xmlNode *
	find_xml_node(xmlNode * root, const char *search_path, gboolean must_find)
	{
	xmlNode *a_child = NULL;
	const char *name = "NULL";

	if (root != NULL) {
	name = crm_element_name(root);
	}

	if (search_path == NULL) {
	crm_warn("Will never find <NULL>");
	return NULL;
	}

	for (a_child = pcmk__xml_first_child(root); a_child != NULL;
	a_child = pcmk__xml_next(a_child)) {
	if (strcmp((const char *)a_child->name, search_path) == 0) {
	/* crm_trace("returning node (%s).", crm_element_name(a_child)); */
	return a_child;
	}
	}

	if (must_find) {
	crm_warn("Could not find %s in %s.", search_path, name);
	} else if (root != NULL) {
	crm_trace("Could not find %s in %s.", search_path, name);
	} else {
	crm_trace("Could not find %s in <NULL>.", search_path);
	}

	return NULL;
	}

	#define attr_matches(c, n, v) pcmk__str_eq(crm_element_value((c), (n)), \
	(v), pcmk__str_none)

	/*!
	* \internal
	* \brief Find first XML child element matching given criteria
	*
	* \param[in] parent XML element to search
	* \param[in] node_name If not NULL, only match children of this type
	* \param[in] attr_n If not NULL, only match children with an attribute
	* of this name and a value of \p attr_v
	* \param[in] attr_v If \p attr_n and this are not NULL, only match children
	* with an attribute named \p attr_n and this value
	*
	* \return Matching XML child element, or NULL if none found
	*/
	xmlNode *
	pcmk__xe_match(xmlNode parent, const char node_name,
	const char attr_n, const char attr_v)
	{
	/* ensure attr_v specified when attr_n is */
	CRM_CHECK(attr_n == NULL \|\| attr_v != NULL, return NULL);

	for (xmlNode *child = pcmk__xml_first_child(parent); child != NULL;
	child = pcmk__xml_next(child)) {
	if (pcmk__str_eq(node_name, (const char *) (child->name),
	pcmk__str_null_matches)
	&& ((attr_n == NULL) \|\| attr_matches(child, attr_n, attr_v))) {
	return child;
	}
	}
	crm_trace("XML child node <%s%s%s%s%s> not found in %s",
	(node_name? node_name : "(any)"),
	(attr_n? " " : ""),
	(attr_n? attr_n : ""),
	(attr_n? "=" : ""),
	(attr_n? attr_v : ""),
	crm_element_name(parent));
	return NULL;
	}

	void
	copy_in_properties(xmlNode * target, xmlNode * src)
	{
	if (src == NULL) {
	crm_warn("No node to copy properties from");

	} else if (target == NULL) {
	crm_err("No node to copy properties into");

	} else {
	for (xmlAttrPtr a = pcmk__xe_first_attr(src); a != NULL; a = a->next) {
	const char p_name = (const char ) a->name;
	const char *p_value = pcmk__xml_attr_value(a);

	expand_plus_plus(target, p_name, p_value);
	}
	}

	return;
	}

	void
	fix_plus_plus_recursive(xmlNode * target)
	{
	/* TODO: Remove recursion and use xpath searches for value++ */
	xmlNode *child = NULL;

	for (xmlAttrPtr a = pcmk__xe_first_attr(target); a != NULL; a = a->next) {
	const char p_name = (const char ) a->name;
	const char *p_value = pcmk__xml_attr_value(a);

	expand_plus_plus(target, p_name, p_value);
	}
	for (child = pcmk__xml_first_child(target); child != NULL;
	child = pcmk__xml_next(child)) {
	fix_plus_plus_recursive(child);
	}
	}

	void
	expand_plus_plus(xmlNode * target, const char name, const char value)
	{
	int offset = 1;
	int name_len = 0;
	int int_value = 0;
	int value_len = 0;

	const char *old_value = NULL;

	if (value == NULL \|\| name == NULL) {
	return;
	}

	old_value = crm_element_value(target, name);

	if (old_value == NULL) {
	/* if no previous value, set unexpanded */
	goto set_unexpanded;

	} else if (strstr(value, name) != value) {
	goto set_unexpanded;
	}

	name_len = strlen(name);
	value_len = strlen(value);
	if (value_len < (name_len + 2)
	\|\| value[name_len] != '+' \|\| (value[name_len + 1] != '+' && value[name_len + 1] != '=')) {
	goto set_unexpanded;
	}

	/* if we are expanding ourselves,
	* then no previous value was set and leave int_value as 0
	*/
	if (old_value != value) {
	int_value = char2score(old_value);
	}

	if (value[name_len + 1] != '+') {
	const char *offset_s = value + (name_len + 2);

	offset = char2score(offset_s);
	}
	int_value += offset;

	if (int_value > INFINITY) {
	int_value = (int)INFINITY;
	}

	crm_xml_add_int(target, name, int_value);
	return;

	set_unexpanded:
	if (old_value == value) {
	/* the old value is already set, nothing to do */
	return;
	}
	crm_xml_add(target, name, value);
	return;
	}

	/*!
	* \internal
	* \brief Remove an XML element's attributes that match some criteria
	*
	* \param[in,out] element XML element to modify
	* \param[in] match If not NULL, only remove attributes for which
	* this function returns true
	* \param[in] user_data Data to pass to \p match
	*/
	void
	pcmk__xe_remove_matching_attrs(xmlNode *element,
	bool (match)(xmlAttrPtr, void ),
	void *user_data)
	{
	xmlAttrPtr next = NULL;

	for (xmlAttrPtr a = pcmk__xe_first_attr(element); a != NULL; a = next) {
	next = a->next; // Grab now because attribute might get removed
	if ((match == NULL) \|\| match(a, user_data)) {
	if (!pcmk__check_acl(element, NULL, pcmk__xf_acl_write)) {
	crm_trace("ACLs prevent removal of attributes (%s and "
	"possibly others) from %s element",
	(const char ) a->name, (const char ) element->name);
	return; // ACLs apply to element, not particular attributes
	}

	if (pcmk__tracking_xml_changes(element, false)) {
	// Leave (marked for removal) until after diff is calculated
	set_parent_flag(element, pcmk__xf_dirty);
	pcmk__set_xml_flags((xml_private_t *) a->_private,
	pcmk__xf_deleted);
	} else {
	xmlRemoveProp(a);
	}
	}
	}
	}

	xmlDoc *
	getDocPtr(xmlNode * node)
	{
	xmlDoc *doc = NULL;

	CRM_CHECK(node != NULL, return NULL);

	doc = node->doc;
	if (doc == NULL) {
	doc = xmlNewDoc((pcmkXmlStr) "1.0");
	xmlDocSetRootElement(doc, node);
	xmlSetTreeDoc(node, doc);
	}
	return doc;
	}

	xmlNode *
	add_node_copy(xmlNode * parent, xmlNode * src_node)
	{
	xmlNode *child = NULL;
	xmlDoc *doc = getDocPtr(parent);

	CRM_CHECK(src_node != NULL, return NULL);

	child = xmlDocCopyNode(src_node, doc, 1);
	xmlAddChild(parent, child);
	pcmk__mark_xml_created(child);
	return child;
	}

	int
	add_node_nocopy(xmlNode * parent, const char name, xmlNode child)
	{
	add_node_copy(parent, child);
	free_xml(child);
	return 1;
	}

	xmlNode *
	create_xml_node(xmlNode * parent, const char *name)
	{
	xmlDoc *doc = NULL;
	xmlNode *node = NULL;

	if (pcmk__str_empty(name)) {
	CRM_CHECK(name != NULL && name[0] == 0, return NULL);
	return NULL;
	}

	if (parent == NULL) {
	doc = xmlNewDoc((pcmkXmlStr) "1.0");
	node = xmlNewDocRawNode(doc, NULL, (pcmkXmlStr) name, NULL);
	xmlDocSetRootElement(doc, node);

	} else {
	doc = getDocPtr(parent);
	node = xmlNewDocRawNode(doc, NULL, (pcmkXmlStr) name, NULL);
	xmlAddChild(parent, node);
	}
	pcmk__mark_xml_created(node);
	return node;
	}

	xmlNode *
	pcmk_create_xml_text_node(xmlNode * parent, const char name, const char content)
	{
	xmlNode *node = create_xml_node(parent, name);

	if (node != NULL) {
	xmlNodeSetContent(node, (pcmkXmlStr) content);
	}

	return node;
	}

	xmlNode *
	pcmk_create_html_node(xmlNode * parent, const char element_name, const char id,
	const char class_name, const char text)
	{
	xmlNode *node = pcmk_create_xml_text_node(parent, element_name, text);

	if (class_name != NULL) {
	crm_xml_add(node, "class", class_name);
	}

	if (id != NULL) {
	crm_xml_add(node, "id", id);
	}

	return node;
	}

	/*!
	* Free an XML element and all of its children, removing it from its parent
	*
	* \param[in] xml XML element to free
	*/
	void
	pcmk_free_xml_subtree(xmlNode *xml)
	{
	xmlUnlinkNode(xml); // Detaches from parent and siblings
	xmlFreeNode(xml); // Frees
	}

	static void
	free_xml_with_position(xmlNode * child, int position)
	{
	if (child != NULL) {
	xmlNode *top = NULL;
	xmlDoc *doc = child->doc;
	xml_private_t *p = child->_private;

	if (doc != NULL) {
	top = xmlDocGetRootElement(doc);
	}

	if (doc != NULL && top == child) {
	/* Free everything */
	xmlFreeDoc(doc);

	} else if (pcmk__check_acl(child, NULL, pcmk__xf_acl_write) == FALSE) {
	int offset = 0;
	char buffer[PCMK__BUFFER_SIZE];

	pcmk__element_xpath(NULL, child, buffer, offset, sizeof(buffer));
	crm_trace("Cannot remove %s %x", buffer, p->flags);
	return;

	} else {
	if (doc && pcmk__tracking_xml_changes(child, FALSE)
	&& !pcmk_is_set(p->flags, pcmk__xf_created)) {
	int offset = 0;
	char buffer[PCMK__BUFFER_SIZE];

	if (pcmk__element_xpath(NULL, child, buffer, offset,
	sizeof(buffer)) > 0) {
	pcmk__deleted_xml_t *deleted_obj = NULL;

	crm_trace("Deleting %s %p from %p", buffer, child, doc);

	deleted_obj = calloc(1, sizeof(pcmk__deleted_xml_t));
	deleted_obj->path = strdup(buffer);

	deleted_obj->position = -1;
	/* Record the "position" only for XML comments for now */
	if (child->type == XML_COMMENT_NODE) {
	if (position >= 0) {
	deleted_obj->position = position;

	} else {
	deleted_obj->position = pcmk__xml_position(child,
	pcmk__xf_skip);
	}
	}

	p = doc->_private;
	p->deleted_objs = g_list_append(p->deleted_objs, deleted_obj);
	pcmk__set_xml_doc_flag(child, pcmk__xf_dirty);
	}
	}
	pcmk_free_xml_subtree(child);
	}
	}
	}


	void
	free_xml(xmlNode * child)
	{
	free_xml_with_position(child, -1);
	}

	xmlNode *
	copy_xml(xmlNode * src)
	{
	xmlDoc *doc = xmlNewDoc((pcmkXmlStr) "1.0");
	xmlNode *copy = xmlDocCopyNode(src, doc, 1);

	xmlDocSetRootElement(doc, copy);
	xmlSetTreeDoc(copy, doc);
	return copy;
	}

	static void
	log_xmllib_err(void ctx, const char fmt, ...)
	G_GNUC_PRINTF(2, 3);

	// Log an XML library error
	static void
	log_xmllib_err(void ctx, const char fmt, ...)
	{
	va_list ap;
	static struct qb_log_callsite *xml_error_cs = NULL;

	if (xml_error_cs == NULL) {
	xml_error_cs = qb_log_callsite_get(
	__func__, __FILE__, "xml library error", LOG_TRACE, __LINE__, crm_trace_nonlog);
	}

	va_start(ap, fmt);
	if (xml_error_cs && xml_error_cs->targets) {
	PCMK__XML_LOG_BASE(LOG_ERR, TRUE,
	crm_abort(__FILE__, __PRETTY_FUNCTION__, __LINE__, "xml library error",
	TRUE, TRUE),
	"XML Error: ", fmt, ap);
	} else {
	PCMK__XML_LOG_BASE(LOG_ERR, TRUE, 0, "XML Error: ", fmt, ap);
	}
	va_end(ap);
	}

	xmlNode *
	string2xml(const char *input)
	{
	xmlNode *xml = NULL;
	xmlDocPtr output = NULL;
	xmlParserCtxtPtr ctxt = NULL;
	xmlErrorPtr last_error = NULL;

	if (input == NULL) {
	crm_err("Can't parse NULL input");
	return NULL;
	}

	/* create a parser context */
	ctxt = xmlNewParserCtxt();
	CRM_CHECK(ctxt != NULL, return NULL);

	xmlCtxtResetLastError(ctxt);
	xmlSetGenericErrorFunc(ctxt, log_xmllib_err);
	output = xmlCtxtReadDoc(ctxt, (pcmkXmlStr) input, NULL, NULL,
	PCMK__XML_PARSE_OPTS);
	if (output) {
	xml = xmlDocGetRootElement(output);
	}
	last_error = xmlCtxtGetLastError(ctxt);
	if (last_error && last_error->code != XML_ERR_OK) {
	/* crm_abort(__FILE__,__func__,__LINE__, "last_error->code != XML_ERR_OK", TRUE, TRUE); */
	/*
	* http://xmlsoft.org/html/libxml-xmlerror.html#xmlErrorLevel
	* http://xmlsoft.org/html/libxml-xmlerror.html#xmlParserErrors
	*/
	crm_warn("Parsing failed (domain=%d, level=%d, code=%d): %s",
	last_error->domain, last_error->level, last_error->code, last_error->message);

	if (last_error->code == XML_ERR_DOCUMENT_EMPTY) {
	CRM_LOG_ASSERT("Cannot parse an empty string");

	} else if (last_error->code != XML_ERR_DOCUMENT_END) {
	crm_err("Couldn't%s parse %d chars: %s", xml ? " fully" : "", (int)strlen(input),
	input);
	if (xml != NULL) {
	crm_log_xml_err(xml, "Partial");
	}

	} else {
	int len = strlen(input);
	int lpc = 0;

	while(lpc < len) {
	crm_warn("Parse error[+%.3d]: %.80s", lpc, input+lpc);
	lpc += 80;
	}

	CRM_LOG_ASSERT("String parsing error");
	}
	}

	xmlFreeParserCtxt(ctxt);
	return xml;
	}

	xmlNode *
	stdin2xml(void)
	{
	size_t data_length = 0;
	size_t read_chars = 0;

	char *xml_buffer = NULL;
	xmlNode *xml_obj = NULL;

	do {
	xml_buffer = pcmk__realloc(xml_buffer, data_length + PCMK__BUFFER_SIZE);
	read_chars = fread(xml_buffer + data_length, 1, PCMK__BUFFER_SIZE,
	stdin);
	data_length += read_chars;
	} while (read_chars == PCMK__BUFFER_SIZE);

	if (data_length == 0) {
	crm_warn("No XML supplied on stdin");
	free(xml_buffer);
	return NULL;
	}

	xml_buffer[data_length] = '\0';
	xml_obj = string2xml(xml_buffer);
	free(xml_buffer);

	crm_log_xml_trace(xml_obj, "Created fragment");
	return xml_obj;
	}

	static char *
	decompress_file(const char *filename)
	{
	char *buffer = NULL;
	int rc = 0;
	size_t length = 0, read_len = 0;
	BZFILE *bz_file = NULL;
	FILE *input = fopen(filename, "r");

	if (input == NULL) {
	crm_perror(LOG_ERR, "Could not open %s for reading", filename);
	return NULL;
	}

	bz_file = BZ2_bzReadOpen(&rc, input, 0, 0, NULL, 0);
	if (rc != BZ_OK) {
	crm_err("Could not prepare to read compressed %s: %s "
	CRM_XS " bzerror=%d", filename, bz2_strerror(rc), rc);
	BZ2_bzReadClose(&rc, bz_file);
	fclose(input);
	return NULL;
	}

	rc = BZ_OK;
	// cppcheck seems not to understand the abort-logic in pcmk__realloc
	// cppcheck-suppress memleak
	while (rc == BZ_OK) {
	buffer = pcmk__realloc(buffer, PCMK__BUFFER_SIZE + length + 1);
	read_len = BZ2_bzRead(&rc, bz_file, buffer + length, PCMK__BUFFER_SIZE);

	crm_trace("Read %ld bytes from file: %d", (long)read_len, rc);

	if (rc == BZ_OK \|\| rc == BZ_STREAM_END) {
	length += read_len;
	}
	}

	buffer[length] = '\0';

	if (rc != BZ_STREAM_END) {
	crm_err("Could not read compressed %s: %s "
	CRM_XS " bzerror=%d", filename, bz2_strerror(rc), rc);
	free(buffer);
	buffer = NULL;
	}

	BZ2_bzReadClose(&rc, bz_file);
	fclose(input);
	return buffer;
	}

	/*!
	* \internal
	* \brief Remove XML text nodes from specified XML and all its children
	*
	* \param[in,out] xml XML to strip text from
	*/
	void
	pcmk__strip_xml_text(xmlNode *xml)
	{
	xmlNode *iter = xml->children;

	while (iter) {
	xmlNode *next = iter->next;

	switch (iter->type) {
	case XML_TEXT_NODE:
	/* Remove it */
	pcmk_free_xml_subtree(iter);
	break;

	case XML_ELEMENT_NODE:
	/* Search it */
	pcmk__strip_xml_text(iter);
	break;

	default:
	/* Leave it */
	break;
	}

	iter = next;
	}
	}

	xmlNode *
	filename2xml(const char *filename)
	{
	xmlNode *xml = NULL;
	xmlDocPtr output = NULL;
	bool uncompressed = true;
	xmlParserCtxtPtr ctxt = NULL;
	xmlErrorPtr last_error = NULL;

	/* create a parser context */
	ctxt = xmlNewParserCtxt();
	CRM_CHECK(ctxt != NULL, return NULL);

	xmlCtxtResetLastError(ctxt);
	xmlSetGenericErrorFunc(ctxt, log_xmllib_err);

	if (filename) {
	uncompressed = !pcmk__ends_with_ext(filename, ".bz2");
	}

	if (pcmk__str_eq(filename, "-", pcmk__str_null_matches)) {
	/* STDIN_FILENO == fileno(stdin) */
	output = xmlCtxtReadFd(ctxt, STDIN_FILENO, "unknown.xml", NULL,
	PCMK__XML_PARSE_OPTS);

	} else if (uncompressed) {
	output = xmlCtxtReadFile(ctxt, filename, NULL, PCMK__XML_PARSE_OPTS);

	} else {
	char *input = decompress_file(filename);

	output = xmlCtxtReadDoc(ctxt, (pcmkXmlStr) input, NULL, NULL,
	PCMK__XML_PARSE_OPTS);
	free(input);
	}

	if (output && (xml = xmlDocGetRootElement(output))) {
	pcmk__strip_xml_text(xml);
	}

	last_error = xmlCtxtGetLastError(ctxt);
	if (last_error && last_error->code != XML_ERR_OK) {
	/* crm_abort(__FILE__,__func__,__LINE__, "last_error->code != XML_ERR_OK", TRUE, TRUE); */
	/*
	* http://xmlsoft.org/html/libxml-xmlerror.html#xmlErrorLevel
	* http://xmlsoft.org/html/libxml-xmlerror.html#xmlParserErrors
	*/
	crm_err("Parsing failed (domain=%d, level=%d, code=%d): %s",
	last_error->domain, last_error->level, last_error->code, last_error->message);

	if (last_error && last_error->code != XML_ERR_OK) {
	crm_err("Couldn't%s parse %s", xml ? " fully" : "", filename);
	if (xml != NULL) {
	crm_log_xml_err(xml, "Partial");
	}
	}
	}

	xmlFreeParserCtxt(ctxt);
	return xml;
	}

	/*!
	* \internal
	* \brief Add a "last written" attribute to an XML element, set to current time
	*
	* \param[in] xe XML element to add attribute to
	*
	* \return Value that was set, or NULL on error
	*/
	const char *
	pcmk__xe_add_last_written(xmlNode *xe)
	{
	const char *now_str = pcmk__epoch2str(NULL);

	return crm_xml_add(xe, XML_CIB_ATTR_WRITTEN,
	now_str ? now_str : "Could not determine current time");
	}

	/*!
	* \brief Sanitize a string so it is usable as an XML ID
	*
	* \param[in,out] id String to sanitize
	*/
	void
	crm_xml_sanitize_id(char *id)
	{
	char *c;

	for (c = id; *c; ++c) {
	/* @TODO Sanitize more comprehensively */
	switch (*c) {
	case ':':
	case '#':
	*c = '.';
	}
	}
	}

	/*!
	* \brief Set the ID of an XML element using a format
	*
	* \param[in,out] xml XML element
	* \param[in] fmt printf-style format
	* \param[in] ... any arguments required by format
	*/
	void
	crm_xml_set_id(xmlNode xml, const char format, ...)
	{
	va_list ap;
	int len = 0;
	char *id = NULL;

	/* equivalent to crm_strdup_printf() */
	va_start(ap, format);
	len = vasprintf(&id, format, ap);
	va_end(ap);
	CRM_ASSERT(len > 0);

	crm_xml_sanitize_id(id);
	crm_xml_add(xml, XML_ATTR_ID, id);
	free(id);
	}

	/*!
	* \internal
	* \brief Write XML to a file stream
	*
	* \param[in] xml_node XML to write
	* \param[in] filename Name of file being written (for logging only)
	* \param[in] stream Open file stream corresponding to filename
	* \param[in] compress Whether to compress XML before writing
	* \param[out] nbytes Number of bytes written
	*
	* \return Standard Pacemaker return code
	*/
	static int
	write_xml_stream(xmlNode xml_node, const char filename, FILE *stream,
	bool compress, unsigned int *nbytes)
	{
	int rc = pcmk_rc_ok;
	char *buffer = NULL;

	*nbytes = 0;
	crm_log_xml_trace(xml_node, "writing");

	buffer = dump_xml_formatted(xml_node);
	CRM_CHECK(buffer && strlen(buffer),
	crm_log_xml_warn(xml_node, "formatting failed");
	rc = pcmk_rc_error;
	goto bail);

	if (compress) {
	unsigned int in = 0;
	BZFILE *bz_file = NULL;

	rc = BZ_OK;
	bz_file = BZ2_bzWriteOpen(&rc, stream, 5, 0, 30);
	if (rc != BZ_OK) {
	crm_warn("Not compressing %s: could not prepare file stream: %s "
	CRM_XS " bzerror=%d", filename, bz2_strerror(rc), rc);
	} else {
	BZ2_bzWrite(&rc, bz_file, buffer, strlen(buffer));
	if (rc != BZ_OK) {
	crm_warn("Not compressing %s: could not compress data: %s "
	CRM_XS " bzerror=%d errno=%d",
	filename, bz2_strerror(rc), rc, errno);
	}
	}

	if (rc == BZ_OK) {
	BZ2_bzWriteClose(&rc, bz_file, 0, &in, nbytes);
	if (rc != BZ_OK) {
	crm_warn("Not compressing %s: could not write compressed data: %s "
	CRM_XS " bzerror=%d errno=%d",
	filename, bz2_strerror(rc), rc, errno);
	*nbytes = 0; // retry without compression
	} else {
	crm_trace("Compressed XML for %s from %u bytes to %u",
	filename, in, *nbytes);
	}
	}
	rc = pcmk_rc_ok; // Either true, or we'll retry without compression
	}

	if (*nbytes == 0) {
	rc = fprintf(stream, "%s", buffer);
	if (rc < 0) {
	rc = errno;
	crm_perror(LOG_ERR, "writing %s", filename);
	} else {
	*nbytes = (unsigned int) rc;
	rc = pcmk_rc_ok;
	}
	}

	bail:

	if (fflush(stream) != 0) {
	rc = errno;
	crm_perror(LOG_ERR, "flushing %s", filename);
	}

	/* Don't report error if the file does not support synchronization */
	if (fsync(fileno(stream)) < 0 && errno != EROFS && errno != EINVAL) {
	rc = errno;
	crm_perror(LOG_ERR, "synchronizing %s", filename);
	}

	fclose(stream);

	crm_trace("Saved %d bytes to %s as XML", *nbytes, filename);
	free(buffer);

	return rc;
	}

	/*!
	* \brief Write XML to a file descriptor
	*
	* \param[in] xml_node XML to write
	* \param[in] filename Name of file being written (for logging only)
	* \param[in] fd Open file descriptor corresponding to filename
	* \param[in] compress Whether to compress XML before writing
	*
	* \return Number of bytes written on success, -errno otherwise
	*/
	int
	write_xml_fd(xmlNode * xml_node, const char *filename, int fd, gboolean compress)
	{
	FILE *stream = NULL;
	unsigned int nbytes = 0;
	int rc = pcmk_rc_ok;

	CRM_CHECK(xml_node && (fd > 0), return -EINVAL);
	stream = fdopen(fd, "w");
	if (stream == NULL) {
	return -errno;
	}
	rc = write_xml_stream(xml_node, filename, stream, compress, &nbytes);
	if (rc != pcmk_rc_ok) {
	return pcmk_rc2legacy(rc);
	}
	return (int) nbytes;
	}

	/*!
	* \brief Write XML to a file
	*
	* \param[in] xml_node XML to write
	* \param[in] filename Name of file to write
	* \param[in] compress Whether to compress XML before writing
	*
	* \return Number of bytes written on success, -errno otherwise
	*/
	int
	write_xml_file(xmlNode * xml_node, const char *filename, gboolean compress)
	{
	FILE *stream = NULL;
	unsigned int nbytes = 0;
	int rc = pcmk_rc_ok;

	CRM_CHECK(xml_node && filename, return -EINVAL);
	stream = fopen(filename, "w");
	if (stream == NULL) {
	return -errno;
	}
	rc = write_xml_stream(xml_node, filename, stream, compress, &nbytes);
	if (rc != pcmk_rc_ok) {
	return pcmk_rc2legacy(rc);
	}
	return (int) nbytes;
	}

	// Replace a portion of a dynamically allocated string (reallocating memory)
	static char *
	replace_text(char text, int start, size_t length, const char *replace)
	{
	size_t offset = strlen(replace) - 1; // We have space for 1 char already

	*length += offset;
	text = pcmk__realloc(text, *length);

	for (size_t lpc = (*length) - 1; lpc > (start + offset); lpc--) {
	text[lpc] = text[lpc - offset];
	}

	memcpy(text + start, replace, offset + 1);
	return text;
	}

	/*!
	* \brief Replace special characters with their XML escape sequences
	*
	* \param[in] text Text to escape
	*
	* \return Newly allocated string equivalent to \p text but with special
	* characters replaced with XML escape sequences (or NULL if \p text
	* is NULL)
	*/
	char *
	crm_xml_escape(const char *text)
	{
	size_t length;
	char *copy;

	/*
	* When xmlCtxtReadDoc() parses < and friends in a
	* value, it converts them to their human readable
	* form.
	*
	* If one uses xmlNodeDump() to convert it back to a
	* string, all is well, because special characters are
	* converted back to their escape sequences.
	*
	* However xmlNodeDump() is randomly dog slow, even with the same
	* input. So we need to replicate the escaping in our custom
	* version so that the result can be re-parsed by xmlCtxtReadDoc()
	* when necessary.
	*/

	if (text == NULL) {
	return NULL;
	}

	length = 1 + strlen(text);
	copy = strdup(text);
	CRM_ASSERT(copy != NULL);
	for (size_t index = 0; index < length; index++) {
	switch (copy[index]) {
	case 0:
	break;
	case '<':
	copy = replace_text(copy, index, &length, "<");
	break;
	case '>':
	copy = replace_text(copy, index, &length, ">");
	break;
	case '"':
	copy = replace_text(copy, index, &length, """);
	break;
	case '\'':
	copy = replace_text(copy, index, &length, "'");
	break;
	case '&':
	copy = replace_text(copy, index, &length, "&");
	break;
	case '\t':
	/* Might as well just expand to a few spaces... */
	copy = replace_text(copy, index, &length, " ");
	break;
	case '\n':
	copy = replace_text(copy, index, &length, "\\n");
	break;
	case '\r':
	copy = replace_text(copy, index, &length, "\\r");
	break;
	default:
	/* Check for and replace non-printing characters with their octal equivalent */
	if(copy[index] < ' ' \|\| copy[index] > '~') {
	char *replace = crm_strdup_printf("\\%.3o", copy[index]);

	copy = replace_text(copy, index, &length, replace);
	free(replace);
	}
	}
	}
	return copy;
	}

	static inline void
	dump_xml_attr(xmlAttrPtr attr, int options, char *buffer, int offset, int *max)
	{
	char *p_value = NULL;
	const char *p_name = NULL;
	xml_private_t *p = NULL;

	CRM_ASSERT(buffer != NULL);
	if (attr == NULL \|\| attr->children == NULL) {
	return;
	}

	p = attr->_private;
	if (p && pcmk_is_set(p->flags, pcmk__xf_deleted)) {
	return;
	}

	p_name = (const char *)attr->name;
	p_value = crm_xml_escape((const char *)attr->children->content);
	buffer_print(buffer, max, *offset, " %s=\"%s\"",
	p_name, crm_str(p_value));
	free(p_value);
	}

	// Log an XML element (and any children) in a formatted way
	void
	pcmk__xe_log(int log_level, const char file, const char function, int line,
	const char prefix, xmlNode data, int depth, int options)
	{
	int max = 0;
	int offset = 0;
	const char *name = NULL;
	const char *hidden = NULL;

	xmlNode *child = NULL;

	if ((data == NULL) \|\| (log_level == LOG_NEVER)) {
	return;
	}

	name = crm_element_name(data);

	if (pcmk_is_set(options, xml_log_option_open)) {
	char *buffer = NULL;

	insert_prefix(options, &buffer, &offset, &max, depth);

	if (data->type == XML_COMMENT_NODE) {
	buffer_print(buffer, max, offset, "<!--%s-->", data->content);

	} else {
	buffer_print(buffer, max, offset, "<%s", name);

	hidden = crm_element_value(data, "hidden");
	for (xmlAttrPtr a = pcmk__xe_first_attr(data); a != NULL;
	a = a->next) {

	xml_private_t *p = a->_private;
	const char p_name = (const char ) a->name;
	const char *p_value = pcmk__xml_attr_value(a);
	char *p_copy = NULL;

	if (pcmk_is_set(p->flags, pcmk__xf_deleted)) {
	continue;
	} else if (pcmk_any_flags_set(options,
	xml_log_option_diff_plus
	\|xml_log_option_diff_minus)
	&& (strcmp(XML_DIFF_MARKER, p_name) == 0)) {
	continue;

	} else if (hidden != NULL && p_name[0] != 0 && strstr(hidden, p_name) != NULL) {
	p_copy = strdup("*****");

	} else {
	p_copy = crm_xml_escape(p_value);
	}

	buffer_print(buffer, max, offset, " %s=\"%s\"",
	p_name, crm_str(p_copy));
	free(p_copy);
	}

	if(xml_has_children(data) == FALSE) {
	buffer_print(buffer, max, offset, "/>");

	} else if (pcmk_is_set(options, xml_log_option_children)) {
	buffer_print(buffer, max, offset, ">");

	} else {
	buffer_print(buffer, max, offset, "/>");
	}
	}

	do_crm_log_alias(log_level, file, function, line, "%s %s", prefix, buffer);
	free(buffer);
	}

	if(data->type == XML_COMMENT_NODE) {
	return;

	} else if(xml_has_children(data) == FALSE) {
	return;

	} else if (pcmk_is_set(options, xml_log_option_children)) {
	offset = 0;
	max = 0;

	for (child = pcmk__xml_first_child(data); child != NULL;
	child = pcmk__xml_next(child)) {
	pcmk__xe_log(log_level, file, function, line, prefix, child,
	depth + 1,
	options\|xml_log_option_open\|xml_log_option_close);
	}
	}

	if (pcmk_is_set(options, xml_log_option_close)) {
	char *buffer = NULL;

	insert_prefix(options, &buffer, &offset, &max, depth);
	buffer_print(buffer, max, offset, "</%s>", name);

	do_crm_log_alias(log_level, file, function, line, "%s %s", prefix, buffer);
	free(buffer);
	}
	}

	// Log XML portions that have been marked as changed
	static void
	log_xml_changes(int log_level, const char file, const char function, int line,
	const char prefix, xmlNode data, int depth, int options)
	{
	xml_private_t *p;
	char *prefix_m = NULL;
	xmlNode *child = NULL;

	if ((data == NULL) \|\| (log_level == LOG_NEVER)) {
	return;
	}

	p = data->_private;

	prefix_m = strdup(prefix);
	prefix_m[1] = '+';

	if (pcmk_all_flags_set(p->flags, pcmk__xf_dirty\|pcmk__xf_created)) {
	/* Continue and log full subtree */
	pcmk__xe_log(log_level, file, function, line, prefix_m, data, depth,
	options\|xml_log_option_open\|xml_log_option_close
	\|xml_log_option_children);

	} else if (pcmk_is_set(p->flags, pcmk__xf_dirty)) {
	char *spaces = calloc(80, 1);
	int s_count = 0, s_max = 80;
	char *prefix_del = NULL;
	char *prefix_moved = NULL;
	const char *flags = prefix;

	insert_prefix(options, &spaces, &s_count, &s_max, depth);
	prefix_del = strdup(prefix);
	prefix_del[0] = '-';
	prefix_del[1] = '-';
	prefix_moved = strdup(prefix);
	prefix_moved[1] = '~';

	if (pcmk_is_set(p->flags, pcmk__xf_moved)) {
	flags = prefix_moved;
	} else {
	flags = prefix;
	}

	pcmk__xe_log(log_level, file, function, line, flags, data, depth,
	options\|xml_log_option_open);

	for (xmlAttrPtr a = pcmk__xe_first_attr(data); a != NULL; a = a->next) {
	const char aname = (const char) a->name;

	p = a->_private;
	if (pcmk_is_set(p->flags, pcmk__xf_deleted)) {
	const char *value = crm_element_value(data, aname);
	flags = prefix_del;
	do_crm_log_alias(log_level, file, function, line,
	"%s %s @%s=%s", flags, spaces, aname, value);

	} else if (pcmk_is_set(p->flags, pcmk__xf_dirty)) {
	const char *value = crm_element_value(data, aname);

	if (pcmk_is_set(p->flags, pcmk__xf_created)) {
	flags = prefix_m;

	} else if (pcmk_is_set(p->flags, pcmk__xf_modified)) {
	flags = prefix;

	} else if (pcmk_is_set(p->flags, pcmk__xf_moved)) {
	flags = prefix_moved;

	} else {
	flags = prefix;
	}
	do_crm_log_alias(log_level, file, function, line,
	"%s %s @%s=%s", flags, spaces, aname, value);
	}
	}
	free(prefix_moved);
	free(prefix_del);
	free(spaces);

	for (child = pcmk__xml_first_child(data); child != NULL;
	child = pcmk__xml_next(child)) {
	log_xml_changes(log_level, file, function, line, prefix, child,
	depth + 1, options);
	}

	pcmk__xe_log(log_level, file, function, line, prefix, data, depth,
	options\|xml_log_option_close);

	} else {
	for (child = pcmk__xml_first_child(data); child != NULL;
	child = pcmk__xml_next(child)) {
	log_xml_changes(log_level, file, function, line, prefix, child,
	depth + 1, options);
	}
	}

	free(prefix_m);

	}

	void
	log_data_element(int log_level, const char file, const char function, int line,
	const char prefix, xmlNode data, int depth, int options)
	{
	xmlNode *a_child = NULL;

	char *prefix_m = NULL;

	if (log_level == LOG_NEVER) {
	return;
	}

	if (prefix == NULL) {
	prefix = "";
	}

	/* Since we use the same file and line, to avoid confusing libqb, we need to use the same format strings */
	if (data == NULL) {
	do_crm_log_alias(log_level, file, function, line, "%s: %s", prefix,
	"No data to dump as XML");
	return;
	}

	if (pcmk_is_set(options, xml_log_option_dirty_add)) {
	log_xml_changes(log_level, file, function, line, prefix, data, depth,
	options);
	return;
	}

	if (pcmk_is_set(options, xml_log_option_formatted)) {
	if (pcmk_is_set(options, xml_log_option_diff_plus)
	&& (data->children == NULL \|\| crm_element_value(data, XML_DIFF_MARKER))) {
	options \|= xml_log_option_diff_all;
	prefix_m = strdup(prefix);
	prefix_m[1] = '+';
	prefix = prefix_m;

	} else if (pcmk_is_set(options, xml_log_option_diff_minus)
	&& (data->children == NULL \|\| crm_element_value(data, XML_DIFF_MARKER))) {
	options \|= xml_log_option_diff_all;
	prefix_m = strdup(prefix);
	prefix_m[1] = '-';
	prefix = prefix_m;
	}
	}

	if (pcmk_is_set(options, xml_log_option_diff_short)
	&& !pcmk_is_set(options, xml_log_option_diff_all)) {
	/* Still searching for the actual change */
	for (a_child = pcmk__xml_first_child(data); a_child != NULL;
	a_child = pcmk__xml_next(a_child)) {
	log_data_element(log_level, file, function, line, prefix, a_child, depth + 1, options);
	}
	} else {
	pcmk__xe_log(log_level, file, function, line, prefix, data, depth,
	options\|xml_log_option_open\|xml_log_option_close
	\|xml_log_option_children);
	}
	free(prefix_m);
	}

	static void
	dump_filtered_xml(xmlNode * data, int options, char *buffer, int offset, int *max)
	{
	for (xmlAttrPtr a = pcmk__xe_first_attr(data); a != NULL; a = a->next) {
	if (!pcmk__xa_filterable((const char *) (a->name))) {
	dump_xml_attr(a, options, buffer, offset, max);
	}
	}
	}

	static void
	dump_xml_element(xmlNode * data, int options, char *buffer, int offset, int *max, int depth)
	{
	const char *name = NULL;

	CRM_ASSERT(max != NULL);
	CRM_ASSERT(offset != NULL);
	CRM_ASSERT(buffer != NULL);

	if (data == NULL) {
	crm_trace("Nothing to dump");
	return;
	}

	if (*buffer == NULL) {
	*offset = 0;
	*max = 0;
	}

	name = crm_element_name(data);
	CRM_ASSERT(name != NULL);

	insert_prefix(options, buffer, offset, max, depth);
	buffer_print(buffer, max, *offset, "<%s", name);

	if (options & xml_log_option_filtered) {
	dump_filtered_xml(data, options, buffer, offset, max);

	} else {
	for (xmlAttrPtr a = pcmk__xe_first_attr(data); a != NULL; a = a->next) {
	dump_xml_attr(a, options, buffer, offset, max);
	}
	}

	if (data->children == NULL) {
	buffer_print(buffer, max, *offset, "/>");

	} else {
	buffer_print(buffer, max, *offset, ">");
	}

	if (options & xml_log_option_formatted) {
	buffer_print(buffer, max, *offset, "\n");
	}

	if (data->children) {
	xmlNode *xChild = NULL;
	for(xChild = data->children; xChild != NULL; xChild = xChild->next) {
	pcmk__xml2text(xChild, options, buffer, offset, max, depth + 1);
	}

	insert_prefix(options, buffer, offset, max, depth);
	buffer_print(buffer, max, *offset, "</%s>", name);

	if (options & xml_log_option_formatted) {
	buffer_print(buffer, max, *offset, "\n");
	}
	}
	}

	static void
	dump_xml_text(xmlNode * data, int options, char *buffer, int offset, int *max, int depth)
	{
	CRM_ASSERT(max != NULL);
	CRM_ASSERT(offset != NULL);
	CRM_ASSERT(buffer != NULL);

	if (data == NULL) {
	crm_trace("Nothing to dump");
	return;
	}

	if (*buffer == NULL) {
	*offset = 0;
	*max = 0;
	}

	insert_prefix(options, buffer, offset, max, depth);

	buffer_print(buffer, max, *offset, "%s", data->content);

	if (options & xml_log_option_formatted) {
	buffer_print(buffer, max, *offset, "\n");
	}
	}

	static void
	dump_xml_cdata(xmlNode * data, int options, char *buffer, int offset, int *max, int depth)
	{
	CRM_ASSERT(max != NULL);
	CRM_ASSERT(offset != NULL);
	CRM_ASSERT(buffer != NULL);

	if (data == NULL) {
	crm_trace("Nothing to dump");
	return;
	}

	if (*buffer == NULL) {
	*offset = 0;
	*max = 0;
	}

	insert_prefix(options, buffer, offset, max, depth);

	buffer_print(buffer, max, *offset, "<![CDATA[");
	buffer_print(buffer, max, *offset, "%s", data->content);
	buffer_print(buffer, max, *offset, "]]>");

	if (options & xml_log_option_formatted) {
	buffer_print(buffer, max, *offset, "\n");
	}
	}

	static void
	dump_xml_comment(xmlNode * data, int options, char *buffer, int offset, int *max, int depth)
	{
	CRM_ASSERT(max != NULL);
	CRM_ASSERT(offset != NULL);
	CRM_ASSERT(buffer != NULL);

	if (data == NULL) {
	crm_trace("Nothing to dump");
	return;
	}

	if (*buffer == NULL) {
	*offset = 0;
	*max = 0;
	}

	insert_prefix(options, buffer, offset, max, depth);

	buffer_print(buffer, max, *offset, "<!--");
	buffer_print(buffer, max, *offset, "%s", data->content);
	buffer_print(buffer, max, *offset, "-->");

	if (options & xml_log_option_formatted) {
	buffer_print(buffer, max, *offset, "\n");
	}
	}

	#define PCMK__XMLDUMP_STATS 0

	/*!
	* \internal
	* \brief Create a text representation of an XML object
	*
	* \param[in] data XML to convert
	* \param[in] options Group of enum xml_log_options flags
	* \param[in,out] buffer Buffer to store text in (may be reallocated)
	* \param[in,out] offset Current position of null terminator within \p buffer
	* \param[in,out] max Current size of \p buffer in bytes
	* \param[in] depth Current indentation level
	*/
	void
	pcmk__xml2text(xmlNode data, int options, char buffer, int offset,
	int *max, int depth)
	{
	if(data == NULL) {
	*offset = 0;
	*max = 0;
	return;
	}

	if (!pcmk_is_set(options, xml_log_option_filtered)
	&& pcmk_is_set(options, xml_log_option_full_fledged)) {
	/* libxml's serialization reuse is a good idea, sadly we cannot
	apply it for the filtered cases (preceding filtering pass
	would preclude further reuse of such in-situ modified XML
	in generic context and is likely not a win performance-wise),
	and there's also a historically unstable throughput argument
	(likely stemming from memory allocation overhead, eventhough
	that shall be minimized with defaults preset in crm_xml_init) */
	#if (PCMK__XMLDUMP_STATS - 0)
	time_t next, new = time(NULL);
	#endif
	xmlDoc *doc;
	xmlOutputBuffer *xml_buffer;

	doc = getDocPtr(data);
	/* doc will only be NULL if data is */
	CRM_CHECK(doc != NULL, return);

	xml_buffer = xmlAllocOutputBuffer(NULL);
	CRM_ASSERT(xml_buffer != NULL);

	/* XXX we could setup custom allocation scheme for the particular
	buffer, but it's subsumed with crm_xml_init that needs to
	be invoked prior to entering this function as such, since
	its other branch vitally depends on it -- what can be done
	about this all is to have a facade parsing functions that
	would 100% mark entering libxml code for us, since we don't
	do anything as crazy as swapping out the binary form of the
	parsed tree (but those would need to be strictly used as
	opposed to libxml's raw functions) */

	xmlNodeDumpOutput(xml_buffer, doc, data, 0,
	(options & xml_log_option_formatted), NULL);
	/* attempt adding final NL - failing shouldn't be fatal here */
	(void) xmlOutputBufferWrite(xml_buffer, sizeof("\n") - 1, "\n");
	if (xml_buffer->buffer != NULL) {
	buffer_print(buffer, max, *offset, "%s",
	(char *) xmlBufContent(xml_buffer->buffer));
	}

	#if (PCMK__XMLDUMP_STATS - 0)
	next = time(NULL);
	if ((now + 1) < next) {
	crm_log_xml_trace(data, "Long time");
	crm_err("xmlNodeDump() -> %dbytes took %ds", *max, next - now);
	}
	#endif

	/* asserted allocation before so there should be something to remove */
	(void) xmlOutputBufferClose(xml_buffer);
	return;
	}

	switch(data->type) {
	case XML_ELEMENT_NODE:
	/* Handle below */
	dump_xml_element(data, options, buffer, offset, max, depth);
	break;
	case XML_TEXT_NODE:
	/* if option xml_log_option_text is enabled, then dump XML_TEXT_NODE */
	if (options & xml_log_option_text) {
	dump_xml_text(data, options, buffer, offset, max, depth);
	}
	return;
	case XML_COMMENT_NODE:
	dump_xml_comment(data, options, buffer, offset, max, depth);
	break;
	case XML_CDATA_SECTION_NODE:
	dump_xml_cdata(data, options, buffer, offset, max, depth);
	break;
	default:
	crm_warn("Unhandled type: %d", data->type);
	return;

	/*
	XML_ATTRIBUTE_NODE = 2
	XML_ENTITY_REF_NODE = 5
	XML_ENTITY_NODE = 6
	XML_PI_NODE = 7
	XML_DOCUMENT_NODE = 9
	XML_DOCUMENT_TYPE_NODE = 10
	XML_DOCUMENT_FRAG_NODE = 11
	XML_NOTATION_NODE = 12
	XML_HTML_DOCUMENT_NODE = 13
	XML_DTD_NODE = 14
	XML_ELEMENT_DECL = 15
	XML_ATTRIBUTE_DECL = 16
	XML_ENTITY_DECL = 17
	XML_NAMESPACE_DECL = 18
	XML_XINCLUDE_START = 19
	XML_XINCLUDE_END = 20
	XML_DOCB_DOCUMENT_NODE = 21
	*/
	}

	}

	/*!
	* \internal
	* \brief Add a single character to a dynamically allocated buffer
	*
	* \param[in,out] buffer Buffer to store text in (may be reallocated)
	* \param[in,out] offset Current position of null terminator within \p buffer
	* \param[in,out] max Current size of \p buffer in bytes
	* \param[in] c Character to add to \p buffer
	*/
	void
	pcmk__buffer_add_char(char *buffer, int offset, int *max, char c)
	{
	buffer_print(buffer, max, *offset, "%c", c);
	}

	char *
	dump_xml_formatted_with_text(xmlNode * an_xml_node)
	{
	char *buffer = NULL;
	int offset = 0, max = 0;

	pcmk__xml2text(an_xml_node,
	xml_log_option_formatted\|xml_log_option_full_fledged,
	&buffer, &offset, &max, 0);
	return buffer;
	}

	char *
	dump_xml_formatted(xmlNode * an_xml_node)
	{
	char *buffer = NULL;
	int offset = 0, max = 0;

	pcmk__xml2text(an_xml_node, xml_log_option_formatted, &buffer, &offset,
	&max, 0);
	return buffer;
	}

	char *
	dump_xml_unformatted(xmlNode * an_xml_node)
	{
	char *buffer = NULL;
	int offset = 0, max = 0;

	pcmk__xml2text(an_xml_node, 0, &buffer, &offset, &max, 0);
	return buffer;
	}

	gboolean
	xml_has_children(const xmlNode * xml_root)
	{
	if (xml_root != NULL && xml_root->children != NULL) {
	return TRUE;
	}
	return FALSE;
	}

	void
	xml_remove_prop(xmlNode * obj, const char *name)
	{
	if (pcmk__check_acl(obj, NULL, pcmk__xf_acl_write) == FALSE) {
	crm_trace("Cannot remove %s from %s", name, obj->name);

	} else if (pcmk__tracking_xml_changes(obj, FALSE)) {
	/* Leave in place (marked for removal) until after the diff is calculated */
	xml_private_t *p = NULL;
	xmlAttr *attr = xmlHasProp(obj, (pcmkXmlStr) name);

	p = attr->_private;
	set_parent_flag(obj, pcmk__xf_dirty);
	pcmk__set_xml_flags(p, pcmk__xf_deleted);
	} else {
	xmlUnsetProp(obj, (pcmkXmlStr) name);
	}
	}

	void
	save_xml_to_file(xmlNode * xml, const char desc, const char filename)
	{
	char *f = NULL;

	if (filename == NULL) {
	char *uuid = crm_generate_uuid();

	f = crm_strdup_printf("%s/%s", pcmk__get_tmpdir(), uuid);
	filename = f;
	free(uuid);
	}

	crm_info("Saving %s to %s", desc, filename);
	write_xml_file(xml, filename, FALSE);
	free(f);
	}

	/*!
	* \internal
	* \brief Set a flag on all attributes of an XML element
	*
	* \param[in,out] xml XML node to set flags on
	* \param[in] flag XML private flag to set
	*/
	static void
	set_attrs_flag(xmlNode *xml, enum xml_private_flags flag)
	{
	for (xmlAttr *attr = pcmk__xe_first_attr(xml); attr; attr = attr->next) {
	pcmk__set_xml_flags((xml_private_t *) (attr->_private), flag);
	}
	}

	/*!
	* \internal
	* \brief Add an XML attribute to a node, marked as deleted
	*
	* When calculating XML changes, we need to know when an attribute has been
	* deleted. Add the attribute back to the new XML, so that we can check the
	* removal against ACLs, and mark it as deleted for later removal after
	* differences have been calculated.
	*/
	static void
	mark_attr_deleted(xmlNode new_xml, const char element, const char *attr_name,
	const char *old_value)
	{
	xml_private_t *p = new_xml->doc->_private;
	xmlAttr *attr = NULL;

	// Prevent the dirty flag being set recursively upwards
	pcmk__clear_xml_flags(p, pcmk__xf_tracking);

	// Restore the old value (and the tracking flag)
	attr = xmlSetProp(new_xml, (pcmkXmlStr) attr_name, (pcmkXmlStr) old_value);
	pcmk__set_xml_flags(p, pcmk__xf_tracking);

	// Reset flags (so the attribute doesn't appear as newly created)
	p = attr->_private;
	p->flags = 0;

	// Check ACLs and mark restored value for later removal
	xml_remove_prop(new_xml, attr_name);

	crm_trace("XML attribute %s=%s was removed from %s",
	attr_name, old_value, element);
	}

	/*
	* \internal
	* \brief Check ACLs for a changed XML attribute
	*/
	static void
	mark_attr_changed(xmlNode new_xml, const char element, const char *attr_name,
	const char *old_value)
	{
	char *vcopy = crm_element_value_copy(new_xml, attr_name);

	crm_trace("XML attribute %s was changed from '%s' to '%s' in %s",
	attr_name, old_value, vcopy, element);

	// Restore the original value
	xmlSetProp(new_xml, (pcmkXmlStr) attr_name, (pcmkXmlStr) old_value);

	// Change it back to the new value, to check ACLs
	crm_xml_add(new_xml, attr_name, vcopy);
	free(vcopy);
	}

	/*!
	* \internal
	* \brief Mark an XML attribute as having changed position
	*/
	static void
	mark_attr_moved(xmlNode new_xml, const char element, xmlAttr *old_attr,
	xmlAttr *new_attr, int p_old, int p_new)
	{
	xml_private_t *p = new_attr->_private;

	crm_trace("XML attribute %s moved from position %d to %d in %s",
	old_attr->name, p_old, p_new, element);

	// Mark document, element, and all element's parents as changed
	mark_xml_node_dirty(new_xml);

	// Mark attribute as changed
	pcmk__set_xml_flags(p, pcmk__xf_dirty\|pcmk__xf_moved);

	p = (p_old > p_new)? old_attr->_private : new_attr->_private;
	pcmk__set_xml_flags(p, pcmk__xf_skip);
	}

	/*!
	* \internal
	* \brief Calculate differences in all previously existing XML attributes
	*/
	static void
	xml_diff_old_attrs(xmlNode old_xml, xmlNode new_xml)
	{
	xmlAttr *attr_iter = pcmk__xe_first_attr(old_xml);

	while (attr_iter != NULL) {
	xmlAttr *old_attr = attr_iter;
	xmlAttr *new_attr = xmlHasProp(new_xml, attr_iter->name);
	const char name = (const char ) attr_iter->name;
	const char *old_value = crm_element_value(old_xml, name);

	attr_iter = attr_iter->next;
	if (new_attr == NULL) {
	mark_attr_deleted(new_xml, (const char *) old_xml->name, name,
	old_value);

	} else {
	xml_private_t *p = new_attr->_private;
	int new_pos = pcmk__xml_position((xmlNode*) new_attr,
	pcmk__xf_skip);
	int old_pos = pcmk__xml_position((xmlNode*) old_attr,
	pcmk__xf_skip);
	const char *new_value = crm_element_value(new_xml, name);

	// This attribute isn't new
	pcmk__clear_xml_flags(p, pcmk__xf_created);

	if (strcmp(new_value, old_value) != 0) {
	mark_attr_changed(new_xml, (const char *) old_xml->name, name,
	old_value);

	} else if ((old_pos != new_pos)
	&& !pcmk__tracking_xml_changes(new_xml, TRUE)) {
	mark_attr_moved(new_xml, (const char *) old_xml->name,
	old_attr, new_attr, old_pos, new_pos);
	}
	}
	}
	}

	/*!
	* \internal
	* \brief Check all attributes in new XML for creation
	*/
	static void
	mark_created_attrs(xmlNode *new_xml)
	{
	xmlAttr *attr_iter = pcmk__xe_first_attr(new_xml);

	while (attr_iter != NULL) {
	xmlAttr *new_attr = attr_iter;
	xml_private_t *p = attr_iter->_private;

	attr_iter = attr_iter->next;
	if (pcmk_is_set(p->flags, pcmk__xf_created)) {
	const char attr_name = (const char ) new_attr->name;

	crm_trace("Created new attribute %s=%s in %s",
	attr_name, crm_element_value(new_xml, attr_name),
	new_xml->name);

	/* Check ACLs (we can't use the remove-then-create trick because it
	* would modify the attribute position).
	*/
	if (pcmk__check_acl(new_xml, attr_name, pcmk__xf_acl_write)) {
	pcmk__mark_xml_attr_dirty(new_attr);
	} else {
	// Creation was not allowed, so remove the attribute
	xmlUnsetProp(new_xml, new_attr->name);
	}
	}
	}
	}

	/*!
	* \internal
	* \brief Calculate differences in attributes between two XML nodes
	*/
	static void
	xml_diff_attrs(xmlNode old_xml, xmlNode new_xml)
	{
	set_attrs_flag(new_xml, pcmk__xf_created); // cleared later if not really new
	xml_diff_old_attrs(old_xml, new_xml);
	mark_created_attrs(new_xml);
	}

	/*!
	* \internal
	* \brief Add an XML child element to a node, marked as deleted
	*
	* When calculating XML changes, we need to know when a child element has been
	* deleted. Add the child back to the new XML, so that we can check the removal
	* against ACLs, and mark it as deleted for later removal after differences have
	* been calculated.
	*/
	static void
	mark_child_deleted(xmlNode old_child, xmlNode new_parent)
	{
	// Re-create the child element so we can check ACLs
	xmlNode *candidate = add_node_copy(new_parent, old_child);

	// Clear flags on new child and its children
	reset_xml_node_flags(candidate);

	// Check whether ACLs allow the deletion
	pcmk__apply_acl(xmlDocGetRootElement(candidate->doc));

	// Remove the child again (which will track it in document's deleted_objs)
	free_xml_with_position(candidate,
	pcmk__xml_position(old_child, pcmk__xf_skip));

	if (pcmk__xml_match(new_parent, old_child, true) == NULL) {
	pcmk__set_xml_flags((xml_private_t *) (old_child->_private),
	pcmk__xf_skip);
	}
	}

	static void
	mark_child_moved(xmlNode old_child, xmlNode new_parent, xmlNode *new_child,
	int p_old, int p_new)
	{
	xml_private_t *p = new_child->_private;

	crm_trace("Child element %s with id='%s' moved from position %d to %d under %s",
	new_child->name, (ID(new_child)? ID(new_child) : "<no id>"),
	p_old, p_new, new_parent->name);
	mark_xml_node_dirty(new_parent);
	pcmk__set_xml_flags(p, pcmk__xf_moved);

	if (p_old > p_new) {
	p = old_child->_private;
	} else {
	p = new_child->_private;
	}
	pcmk__set_xml_flags(p, pcmk__xf_skip);
	}

	// Given original and new XML, mark new XML portions that have changed
	static void
	mark_xml_changes(xmlNode old_xml, xmlNode new_xml, bool check_top)
	{
	xmlNode *cIter = NULL;
	xml_private_t *p = NULL;

	CRM_CHECK(new_xml != NULL, return);
	if (old_xml == NULL) {
	pcmk__mark_xml_created(new_xml);
	pcmk__apply_creation_acl(new_xml, check_top);
	return;
	}

	p = new_xml->_private;
	CRM_CHECK(p != NULL, return);

	if(p->flags & pcmk__xf_processed) {
	/* Avoid re-comparing nodes */
	return;
	}
	pcmk__set_xml_flags(p, pcmk__xf_processed);

	xml_diff_attrs(old_xml, new_xml);

	// Check for differences in the original children
	for (cIter = pcmk__xml_first_child(old_xml); cIter != NULL; ) {
	xmlNode *old_child = cIter;
	xmlNode *new_child = pcmk__xml_match(new_xml, cIter, true);

	cIter = pcmk__xml_next(cIter);
	if(new_child) {
	mark_xml_changes(old_child, new_child, TRUE);

	} else {
	mark_child_deleted(old_child, new_xml);
	}
	}

	// Check for moved or created children
	for (cIter = pcmk__xml_first_child(new_xml); cIter != NULL; ) {
	xmlNode *new_child = cIter;
	xmlNode *old_child = pcmk__xml_match(old_xml, cIter, true);

	cIter = pcmk__xml_next(cIter);
	if(old_child == NULL) {
	// This is a newly created child
	p = new_child->_private;
	pcmk__set_xml_flags(p, pcmk__xf_skip);
	mark_xml_changes(old_child, new_child, TRUE);

	} else {
	/* Check for movement, we already checked for differences */
	int p_new = pcmk__xml_position(new_child, pcmk__xf_skip);
	int p_old = pcmk__xml_position(old_child, pcmk__xf_skip);

	if(p_old != p_new) {
	mark_child_moved(old_child, new_xml, new_child, p_old, p_new);
	}
	}
	}
	}

	void
	xml_calculate_significant_changes(xmlNode old_xml, xmlNode new_xml)
	{
	pcmk__set_xml_doc_flag(new_xml, pcmk__xf_lazy);
	xml_calculate_changes(old_xml, new_xml);
	}

	void
	xml_calculate_changes(xmlNode old_xml, xmlNode new_xml)
	{
	CRM_CHECK(pcmk__str_eq(crm_element_name(old_xml), crm_element_name(new_xml), pcmk__str_casei),
	return);
	CRM_CHECK(pcmk__str_eq(ID(old_xml), ID(new_xml), pcmk__str_casei), return);

	if(xml_tracking_changes(new_xml) == FALSE) {
	xml_track_changes(new_xml, NULL, NULL, FALSE);
	}

	mark_xml_changes(old_xml, new_xml, FALSE);
	}

	gboolean
	can_prune_leaf(xmlNode * xml_node)
	{
	xmlNode *cIter = NULL;
	gboolean can_prune = TRUE;
	const char *name = crm_element_name(xml_node);

	if (pcmk__strcase_any_of(name, XML_TAG_RESOURCE_REF, XML_CIB_TAG_OBJ_REF,
	XML_ACL_TAG_ROLE_REF, XML_ACL_TAG_ROLE_REFv1, NULL)) {
	return FALSE;
	}

	for (xmlAttrPtr a = pcmk__xe_first_attr(xml_node); a != NULL; a = a->next) {
	const char p_name = (const char ) a->name;

	if (strcmp(p_name, XML_ATTR_ID) == 0) {
	continue;
	}
	can_prune = FALSE;
	}

	cIter = pcmk__xml_first_child(xml_node);
	while (cIter) {
	xmlNode *child = cIter;

	cIter = pcmk__xml_next(cIter);
	if (can_prune_leaf(child)) {
	free_xml(child);
	} else {
	can_prune = FALSE;
	}
	}
	return can_prune;
	}

	/*!
	* \internal
	* \brief Find a comment with matching content in specified XML
	*
	* \param[in] root XML to search
	* \param[in] search_comment Comment whose content should be searched for
	* \param[in] exact If true, comment must also be at same position
	*/
	xmlNode *
	pcmk__xc_match(xmlNode root, xmlNode search_comment, bool exact)
	{
	xmlNode *a_child = NULL;
	int search_offset = pcmk__xml_position(search_comment, pcmk__xf_skip);

	CRM_CHECK(search_comment->type == XML_COMMENT_NODE, return NULL);

	for (a_child = pcmk__xml_first_child(root); a_child != NULL;
	a_child = pcmk__xml_next(a_child)) {
	if (exact) {
	int offset = pcmk__xml_position(a_child, pcmk__xf_skip);
	xml_private_t *p = a_child->_private;

	if (offset < search_offset) {
	continue;

	} else if (offset > search_offset) {
	return NULL;
	}

	if (pcmk_is_set(p->flags, pcmk__xf_skip)) {
	continue;
	}
	}

	if (a_child->type == XML_COMMENT_NODE
	&& pcmk__str_eq((const char )a_child->content, (const char )search_comment->content, pcmk__str_casei)) {
	return a_child;

	} else if (exact) {
	return NULL;
	}
	}

	return NULL;
	}

	/*!
	* \internal
	* \brief Make one XML comment match another (in content)
	*
	* \param[in,out] parent If \p target is NULL and this is not, add or update
	* comment child of this XML node that matches \p update
	* \param[in,out] target If not NULL, update this XML comment node
	* \param[in] update Make comment content match this (must not be NULL)
	*
	* \note At least one of \parent and \target must be non-NULL
	*/
	void
	pcmk__xc_update(xmlNode parent, xmlNode target, xmlNode *update)
	{
	CRM_CHECK(update != NULL, return);
	CRM_CHECK(update->type == XML_COMMENT_NODE, return);

	if (target == NULL) {
	target = pcmk__xc_match(parent, update, false);
	}

	if (target == NULL) {
	add_node_copy(parent, update);

	} else if (!pcmk__str_eq((const char )target->content, (const char )update->content, pcmk__str_casei)) {
	xmlFree(target->content);
	target->content = xmlStrdup(update->content);
	}
	}

	/*!
	* \internal
	* \brief Make one XML tree match another (in children and attributes)
	*
	* \param[in,out] parent If \p target is NULL and this is not, add or update
	* child of this XML node that matches \p update
	* \param[in,out] target If not NULL, update this XML
	* \param[in] update Make the desired XML match this (must not be NULL)
	* \param[in] as_diff If true, expand "++" when making attributes match
	*
	* \note At least one of \parent and \target must be non-NULL
	*/
	void
	pcmk__xml_update(xmlNode parent, xmlNode target, xmlNode *update,
	bool as_diff)
	{
	xmlNode *a_child = NULL;
	const char *object_name = NULL,
	*object_href = NULL,
	*object_href_val = NULL;

	#if XML_PARSER_DEBUG
	- crm_log_xml_trace("update:", update);
	- crm_log_xml_trace("target:", target);
	+ crm_log_xml_trace(update, "update:");
	+ crm_log_xml_trace(target, "target:");
	#endif

	CRM_CHECK(update != NULL, return);

	if (update->type == XML_COMMENT_NODE) {
	pcmk__xc_update(parent, target, update);
	return;
	}

	object_name = crm_element_name(update);
	object_href_val = ID(update);
	if (object_href_val != NULL) {
	object_href = XML_ATTR_ID;
	} else {
	object_href_val = crm_element_value(update, XML_ATTR_IDREF);
	object_href = (object_href_val == NULL) ? NULL : XML_ATTR_IDREF;
	}

	CRM_CHECK(object_name != NULL, return);
	CRM_CHECK(target != NULL \|\| parent != NULL, return);

	if (target == NULL) {
	target = pcmk__xe_match(parent, object_name,
	object_href, object_href_val);
	}

	if (target == NULL) {
	target = create_xml_node(parent, object_name);
	CRM_CHECK(target != NULL, return);
	#if XML_PARSER_DEBUG
	crm_trace("Added <%s%s%s%s%s/>", crm_str(object_name),
	object_href ? " " : "",
	object_href ? object_href : "",
	object_href ? "=" : "",
	object_href ? object_href_val : "");

	} else {
	crm_trace("Found node <%s%s%s%s%s/> to update", crm_str(object_name),
	object_href ? " " : "",
	object_href ? object_href : "",
	object_href ? "=" : "",
	object_href ? object_href_val : "");
	#endif
	}

	CRM_CHECK(pcmk__str_eq(crm_element_name(target), crm_element_name(update),
	pcmk__str_casei),
	return);

	if (as_diff == FALSE) {
	/* So that expand_plus_plus() gets called */
	copy_in_properties(target, update);

	} else {
	/* No need for expand_plus_plus(), just raw speed */
	for (xmlAttrPtr a = pcmk__xe_first_attr(update); a != NULL;
	a = a->next) {
	const char *p_value = pcmk__xml_attr_value(a);

	/* Remove it first so the ordering of the update is preserved */
	xmlUnsetProp(target, a->name);
	xmlSetProp(target, a->name, (pcmkXmlStr) p_value);
	}
	}

	for (a_child = pcmk__xml_first_child(update); a_child != NULL;
	a_child = pcmk__xml_next(a_child)) {
	#if XML_PARSER_DEBUG
	crm_trace("Updating child <%s%s%s%s%s/>", crm_str(object_name),
	object_href ? " " : "",
	object_href ? object_href : "",
	object_href ? "=" : "",
	object_href ? object_href_val : "");
	#endif
	pcmk__xml_update(target, NULL, a_child, as_diff);
	}

	#if XML_PARSER_DEBUG
	crm_trace("Finished with <%s%s%s%s%s/>", crm_str(object_name),
	object_href ? " " : "",
	object_href ? object_href : "",
	object_href ? "=" : "",
	object_href ? object_href_val : "");
	#endif
	}

	gboolean
	update_xml_child(xmlNode * child, xmlNode * to_update)
	{
	gboolean can_update = TRUE;
	xmlNode *child_of_child = NULL;

	CRM_CHECK(child != NULL, return FALSE);
	CRM_CHECK(to_update != NULL, return FALSE);

	if (!pcmk__str_eq(crm_element_name(to_update), crm_element_name(child), pcmk__str_none)) {
	can_update = FALSE;

	} else if (!pcmk__str_eq(ID(to_update), ID(child), pcmk__str_none)) {
	can_update = FALSE;

	} else if (can_update) {
	#if XML_PARSER_DEBUG
	crm_log_xml_trace(child, "Update match found...");
	#endif
	pcmk__xml_update(NULL, child, to_update, false);
	}

	for (child_of_child = pcmk__xml_first_child(child); child_of_child != NULL;
	child_of_child = pcmk__xml_next(child_of_child)) {
	/* only update the first one */
	if (can_update) {
	break;
	}
	can_update = update_xml_child(child_of_child, to_update);
	}

	return can_update;
	}

	int
	find_xml_children(xmlNode ** children, xmlNode * root,
	const char tag, const char field, const char *value, gboolean search_matches)
	{
	int match_found = 0;

	CRM_CHECK(root != NULL, return FALSE);
	CRM_CHECK(children != NULL, return FALSE);

	if (tag != NULL && !pcmk__str_eq(tag, crm_element_name(root), pcmk__str_casei)) {

	} else if (value != NULL && !pcmk__str_eq(value, crm_element_value(root, field), pcmk__str_casei)) {

	} else {
	if (*children == NULL) {
	*children = create_xml_node(NULL, __func__);
	}
	add_node_copy(*children, root);
	match_found = 1;
	}

	if (search_matches \|\| match_found == 0) {
	xmlNode *child = NULL;

	for (child = pcmk__xml_first_child(root); child != NULL;
	child = pcmk__xml_next(child)) {
	match_found += find_xml_children(children, child, tag, field, value, search_matches);
	}
	}

	return match_found;
	}

	gboolean
	replace_xml_child(xmlNode * parent, xmlNode * child, xmlNode * update, gboolean delete_only)
	{
	gboolean can_delete = FALSE;
	xmlNode *child_of_child = NULL;

	const char *up_id = NULL;
	const char *child_id = NULL;
	const char *right_val = NULL;

	CRM_CHECK(child != NULL, return FALSE);
	CRM_CHECK(update != NULL, return FALSE);

	up_id = ID(update);
	child_id = ID(child);

	if (up_id == NULL \|\| (child_id && strcmp(child_id, up_id) == 0)) {
	can_delete = TRUE;
	}
	if (!pcmk__str_eq(crm_element_name(update), crm_element_name(child), pcmk__str_casei)) {
	can_delete = FALSE;
	}
	if (can_delete && delete_only) {
	for (xmlAttrPtr a = pcmk__xe_first_attr(update); a != NULL;
	a = a->next) {
	const char p_name = (const char ) a->name;
	const char *p_value = pcmk__xml_attr_value(a);

	right_val = crm_element_value(child, p_name);
	if (!pcmk__str_eq(p_value, right_val, pcmk__str_casei)) {
	can_delete = FALSE;
	}
	}
	}

	if (can_delete && parent != NULL) {
	crm_log_xml_trace(child, "Delete match found...");
	if (delete_only \|\| update == NULL) {
	free_xml(child);

	} else {
	xmlNode *tmp = copy_xml(update);
	xmlDoc *doc = tmp->doc;
	xmlNode *old = NULL;

	xml_accept_changes(tmp);
	old = xmlReplaceNode(child, tmp);

	if(xml_tracking_changes(tmp)) {
	/* Replaced sections may have included relevant ACLs */
	pcmk__apply_acl(tmp);
	}

	xml_calculate_changes(old, tmp);
	xmlDocSetRootElement(doc, old);
	free_xml(old);
	}
	child = NULL;
	return TRUE;

	} else if (can_delete) {
	crm_log_xml_debug(child, "Cannot delete the search root");
	can_delete = FALSE;
	}

	child_of_child = pcmk__xml_first_child(child);
	while (child_of_child) {
	xmlNode *next = pcmk__xml_next(child_of_child);

	can_delete = replace_xml_child(child, child_of_child, update, delete_only);

	/* only delete the first one */
	if (can_delete) {
	child_of_child = NULL;
	} else {
	child_of_child = next;
	}
	}

	return can_delete;
	}

	xmlNode *
	sorted_xml(xmlNode input, xmlNode parent, gboolean recursive)
	{
	xmlNode *child = NULL;
	GSList *nvpairs = NULL;
	xmlNode *result = NULL;
	const char *name = NULL;

	CRM_CHECK(input != NULL, return NULL);

	name = crm_element_name(input);
	CRM_CHECK(name != NULL, return NULL);

	result = create_xml_node(parent, name);
	nvpairs = pcmk_xml_attrs2nvpairs(input);
	nvpairs = pcmk_sort_nvpairs(nvpairs);
	pcmk_nvpairs2xml_attrs(nvpairs, result);
	pcmk_free_nvpairs(nvpairs);

	for (child = pcmk__xml_first_child(input); child != NULL;
	child = pcmk__xml_next(child)) {

	if (recursive) {
	sorted_xml(child, result, recursive);
	} else {
	add_node_copy(result, child);
	}
	}

	return result;
	}

	xmlNode *
	first_named_child(const xmlNode parent, const char name)
	{
	xmlNode *match = NULL;

	for (match = pcmk__xe_first_child(parent); match != NULL;
	match = pcmk__xe_next(match)) {
	/*
	* name == NULL gives first child regardless of name; this is
	* semantically incorrect in this function, but may be necessary
	* due to prior use of xml_child_iter_filter
	*/
	if (pcmk__str_eq(name, (const char *)match->name, pcmk__str_null_matches)) {
	return match;
	}
	}
	return NULL;
	}

	/*!
	* \brief Get next instance of same XML tag
	*
	* \param[in] sibling XML tag to start from
	*
	* \return Next sibling XML tag with same name
	*/
	xmlNode *
	crm_next_same_xml(const xmlNode *sibling)
	{
	xmlNode *match = pcmk__xe_next(sibling);
	const char *name = crm_element_name(sibling);

	while (match != NULL) {
	if (!strcmp(crm_element_name(match), name)) {
	return match;
	}
	match = pcmk__xe_next(match);
	}
	return NULL;
	}

	void
	crm_xml_init(void)
	{
	static bool init = true;

	if(init) {
	init = false;
	/* The default allocator XML_BUFFER_ALLOC_EXACT does far too many
	* pcmk__realloc()s and it can take upwards of 18 seconds (yes, seconds)
	* to dump a 28kb tree which XML_BUFFER_ALLOC_DOUBLEIT can do in
	* less than 1 second.
	*/
	xmlSetBufferAllocationScheme(XML_BUFFER_ALLOC_DOUBLEIT);

	/* Populate and free the _private field when nodes are created and destroyed */
	xmlDeregisterNodeDefault(free_private_data);
	xmlRegisterNodeDefault(new_private_data);

	crm_schema_init();
	}
	}

	void
	crm_xml_cleanup(void)
	{
	crm_info("Cleaning up memory from libxml2");
	crm_schema_cleanup();
	xmlCleanupParser();
	}

	#define XPATH_MAX 512

	xmlNode *
	expand_idref(xmlNode * input, xmlNode * top)
	{
	const char *tag = NULL;
	const char *ref = NULL;
	xmlNode *result = input;

	if (result == NULL) {
	return NULL;

	} else if (top == NULL) {
	top = input;
	}

	tag = crm_element_name(result);
	ref = crm_element_value(result, XML_ATTR_IDREF);

	if (ref != NULL) {
	char *xpath_string = crm_strdup_printf("//%s[@id='%s']", tag, ref);

	result = get_xpath_object(xpath_string, top, LOG_ERR);
	if (result == NULL) {
	char nodePath = (char )xmlGetNodePath(top);

	crm_err("No match for %s found in %s: Invalid configuration", xpath_string,
	crm_str(nodePath));
	free(nodePath);
	}
	free(xpath_string);
	}
	return result;
	}

	void
	crm_destroy_xml(gpointer data)
	{
	free_xml(data);
	}

	char *
	pcmk__xml_artefact_root(enum pcmk__xml_artefact_ns ns)
	{
	static const char *base = NULL;
	char *ret = NULL;

	if (base == NULL) {
	base = getenv("PCMK_schema_directory");
	}
	if (pcmk__str_empty(base)) {
	base = CRM_SCHEMA_DIRECTORY;
	}

	switch (ns) {
	case pcmk__xml_artefact_ns_legacy_rng:
	case pcmk__xml_artefact_ns_legacy_xslt:
	ret = strdup(base);
	break;
	case pcmk__xml_artefact_ns_base_rng:
	case pcmk__xml_artefact_ns_base_xslt:
	ret = crm_strdup_printf("%s/base", base);
	break;
	default:
	crm_err("XML artefact family specified as %u not recognized", ns);
	}
	return ret;
	}

	char *
	pcmk__xml_artefact_path(enum pcmk__xml_artefact_ns ns, const char *filespec)
	{
	char base = pcmk__xml_artefact_root(ns), ret = NULL;

	switch (ns) {
	case pcmk__xml_artefact_ns_legacy_rng:
	case pcmk__xml_artefact_ns_base_rng:
	ret = crm_strdup_printf("%s/%s.rng", base, filespec);
	break;
	case pcmk__xml_artefact_ns_legacy_xslt:
	case pcmk__xml_artefact_ns_base_xslt:
	ret = crm_strdup_printf("%s/%s.xsl", base, filespec);
	break;
	default:
	crm_err("XML artefact family specified as %u not recognized", ns);
	}
	free(base);

	return ret;
	}

	void
	pcmk__xe_set_propv(xmlNodePtr node, va_list pairs)
	{
	while (true) {
	const char name, value;

	name = va_arg(pairs, const char *);
	if (name == NULL) {
	return;
	}

	value = va_arg(pairs, const char *);
	if (value != NULL) {
	crm_xml_add(node, name, value);
	}
	}
	}

	void
	pcmk__xe_set_props(xmlNodePtr node, ...)
	{
	va_list pairs;
	va_start(pairs, node);
	pcmk__xe_set_propv(node, pairs);
	va_end(pairs);
	}

	// Deprecated functions kept only for backward API compatibility
	// LCOV_EXCL_START

	#include <crm/common/xml_compat.h>

	xmlNode *
	find_entity(xmlNode parent, const char node_name, const char *id)
	{
	return pcmk__xe_match(parent, node_name,
	((id == NULL)? id : XML_ATTR_ID), id);
	}

	// LCOV_EXCL_STOP
	// End deprecated API
	diff --git a/tools/crm_resource.c b/tools/crm_resource.c
	index db8bab3df2..46e165aa72 100644
	--- a/tools/crm_resource.c
	+++ b/tools/crm_resource.c
	@@ -1,2125 +1,2143 @@
	/*
	* Copyright 2004-2022 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <crm_resource.h>
	#include <crm/lrmd_internal.h>
	#include <crm/common/cmdline_internal.h>
	#include <crm/common/lists_internal.h>
	#include <crm/common/output.h>
	#include <pacemaker-internal.h>

	#include <sys/param.h>
	#include <stdio.h>
	#include <sys/types.h>
	#include <unistd.h>
	#include <stdlib.h>
	#include <errno.h>
	#include <fcntl.h>
	#include <libgen.h>
	#include <time.h>

	#include <crm/crm.h>
	#include <crm/stonith-ng.h>
	#include <crm/common/ipc_controld.h>
	#include <crm/cib/internal.h>

	#define SUMMARY "crm_resource - perform tasks related to Pacemaker cluster resources"

	enum rsc_command {
	cmd_none = 0, // No command option given (yet)
	cmd_ban,
	cmd_cleanup,
	cmd_clear,
	cmd_colocations,
	cmd_colocations_deep,
	cmd_cts,
	cmd_delete,
	cmd_delete_param,
	cmd_digests,
	cmd_execute_agent,
	cmd_fail,
	cmd_get_param,
	cmd_get_property,
	cmd_list_active_ops,
	cmd_list_agents,
	cmd_list_all_ops,
	cmd_list_alternatives,
	cmd_list_instances,
	cmd_list_providers,
	cmd_list_resources,
	cmd_list_standards,
	cmd_locate,
	cmd_metadata,
	cmd_move,
	cmd_query_raw_xml,
	cmd_query_xml,
	cmd_refresh,
	cmd_restart,
	cmd_set_param,
	cmd_set_property,
	cmd_wait,
	cmd_why,
	};

	struct {
	enum rsc_command rsc_cmd; // crm_resource command to perform

	// Infrastructure that given command needs to work
	gboolean require_cib; // Whether command requires CIB IPC
	int cib_options; // Options to use with CIB IPC calls
	gboolean require_crmd; // Whether command requires controller IPC
	gboolean require_dataset; // Whether command requires populated data set
	gboolean require_resource; // Whether command requires resource specified
	gboolean require_node; // Whether command requires node specified
	int find_flags; // Flags to use when searching for resource

	// Command-line option values
	gchar *rsc_id; // Value of --resource
	gchar *rsc_type; // Value of --resource-type
	gboolean force; // --force was given
	gboolean clear_expired; // --expired was given
	gboolean recursive; // --recursive was given
	gboolean promoted_role_only; // --promoted was given
	gchar *host_uname; // Value of --node
	gchar *interval_spec; // Value of --interval
	gchar *move_lifetime; // Value of --lifetime
	gchar *operation; // Value of --operation
	const char *attr_set_type; // Instance, meta, or utilization attribute
	gchar *prop_id; // --nvpair (attribute XML ID)
	char *prop_name; // Attribute name
	gchar *prop_set; // --set-name (attribute block XML ID)
	gchar *prop_value; // --parameter-value (attribute value)
	int timeout_ms; // Parsed from --timeout value
	char *agent_spec; // Standard and/or provider and/or agent
	gchar *xml_file; // Value of (deprecated) --xml-file
	int check_level; // Optional value of --validate or --force-check

	// Resource configuration specified via command-line arguments
	gboolean cmdline_config; // Resource configuration was via arguments
	char *v_agent; // Value of --agent
	char *v_class; // Value of --class
	char *v_provider; // Value of --provider
	GHashTable *cmdline_params; // Resource parameters specified

	// Positional command-line arguments
	gchar **remainder; // Positional arguments as given
	GHashTable *override_params; // Resource parameter values that override config
	} options = {
	.attr_set_type = XML_TAG_ATTR_SETS,
	.check_level = -1,
	.cib_options = cib_sync_call,
	.require_cib = TRUE,
	.require_dataset = TRUE,
	.require_resource = TRUE,
	};

	#if 0
	// @COMPAT @TODO enable this at next backward compatibility break
	#define SET_COMMAND(cmd) do { \
	if (options.rsc_cmd != cmd_none) { \
	g_set_error(error, PCMK__EXITC_ERROR, CRM_EX_USAGE, \
	"Only one command option may be specified"); \
	return FALSE; \
	} \
	options.rsc_cmd = (cmd); \
	} while (0)
	#else
	#define SET_COMMAND(cmd) do { options.rsc_cmd = (cmd); } while (0)
	#endif

	gboolean agent_provider_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error);
	gboolean attr_set_type_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error);
	gboolean class_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error);
	gboolean cleanup_refresh_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error);
	gboolean delete_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error);
	gboolean expired_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error);
	gboolean list_agents_cb(const gchar option_name, const gchar optarg,
	gpointer data, GError **error);
	gboolean list_providers_cb(const gchar option_name, const gchar optarg,
	gpointer data, GError **error);
	gboolean list_standards_cb(const gchar option_name, const gchar optarg,
	gpointer data, GError **error);
	gboolean list_alternatives_cb(const gchar option_name, const gchar optarg,
	gpointer data, GError **error);
	gboolean metadata_cb(const gchar option_name, const gchar optarg,
	gpointer data, GError **error);
	gboolean option_cb(const gchar option_name, const gchar optarg,
	gpointer data, GError **error);
	gboolean fail_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error);
	gboolean flag_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error);
	gboolean get_param_prop_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error);
	gboolean list_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error);
	gboolean set_delete_param_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error);
	gboolean set_prop_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error);
	gboolean timeout_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error);
	gboolean validate_or_force_cb(const gchar option_name, const gchar optarg,
	gpointer data, GError **error);
	gboolean restart_cb(const gchar option_name, const gchar optarg,
	gpointer data, GError **error);
	gboolean digests_cb(const gchar option_name, const gchar optarg,
	gpointer data, GError **error);
	gboolean wait_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error);
	gboolean why_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error);

	static crm_exit_t exit_code = CRM_EX_OK;
	static pcmk__output_t *out = NULL;
	static pcmk__common_args_t *args = NULL;

	// Things that should be cleaned up on exit
	static GError *error = NULL;
	static GMainLoop *mainloop = NULL;
	static cib_t *cib_conn = NULL;
	static pcmk_ipc_api_t *controld_api = NULL;
	static pe_working_set_t *data_set = NULL;

	#define MESSAGE_TIMEOUT_S 60

	#define INDENT " "

	static pcmk__supported_format_t formats[] = {
	PCMK__SUPPORTED_FORMAT_NONE,
	PCMK__SUPPORTED_FORMAT_TEXT,
	PCMK__SUPPORTED_FORMAT_XML,
	{ NULL, NULL, NULL }
	};

	// Clean up and exit
	static crm_exit_t
	bye(crm_exit_t ec)
	{
	pcmk__output_and_clear_error(error, out);

	if (out != NULL) {
	out->finish(out, ec, true, NULL);
	pcmk__output_free(out);
	}

	if (cib_conn != NULL) {
	cib_t *save_cib_conn = cib_conn;

	cib_conn = NULL; // Ensure we can't free this twice
	cib__clean_up_connection(&save_cib_conn);
	}
	if (controld_api != NULL) {
	pcmk_ipc_api_t *save_controld_api = controld_api;

	controld_api = NULL; // Ensure we can't free this twice
	pcmk_free_ipc_api(save_controld_api);
	}
	if (mainloop != NULL) {
	g_main_loop_unref(mainloop);
	mainloop = NULL;
	}
	pe_free_working_set(data_set);
	data_set = NULL;
	crm_exit(ec);
	return ec;
	}

	static void
	quit_main_loop(crm_exit_t ec)
	{
	exit_code = ec;
	if (mainloop != NULL) {
	GMainLoop *mloop = mainloop;

	mainloop = NULL; // Don't re-enter this block
	pcmk_quit_main_loop(mloop, 10);
	g_main_loop_unref(mloop);
	}
	}

	static gboolean
	resource_ipc_timeout(gpointer data)
	{
	// Start with newline because "Waiting for ..." message doesn't have one
	if (error != NULL) {
	g_clear_error(&error);
	}

	g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_TIMEOUT,
	"Aborting because no messages received in %d seconds", MESSAGE_TIMEOUT_S);

	quit_main_loop(CRM_EX_TIMEOUT);
	return FALSE;
	}

	static void
	controller_event_callback(pcmk_ipc_api_t *api, enum pcmk_ipc_event event_type,
	crm_exit_t status, void event_data, void user_data)
	{
	switch (event_type) {
	case pcmk_ipc_event_disconnect:
	if (exit_code == CRM_EX_DISCONNECT) { // Unexpected
	crm_info("Connection to controller was terminated");
	}
	quit_main_loop(exit_code);
	break;

	case pcmk_ipc_event_reply:
	if (status != CRM_EX_OK) {
	out->err(out, "Error: bad reply from controller: %s",
	crm_exit_str(status));
	pcmk_disconnect_ipc(api);
	quit_main_loop(status);
	} else {
	if ((pcmk_controld_api_replies_expected(api) == 0)
	&& mainloop && g_main_loop_is_running(mainloop)) {
	out->info(out, "... got reply (done)");
	crm_debug("Got all the replies we expected");
	pcmk_disconnect_ipc(api);
	quit_main_loop(CRM_EX_OK);
	} else {
	out->info(out, "... got reply");
	}
	}
	break;

	default:
	break;
	}
	}

	static void
	start_mainloop(pcmk_ipc_api_t *capi)
	{
	unsigned int count = pcmk_controld_api_replies_expected(capi);

	if (count > 0) {
	out->info(out, "Waiting for %d %s from the controller",
	count, pcmk__plural_alt(count, "reply", "replies"));
	exit_code = CRM_EX_DISCONNECT; // For unexpected disconnects
	mainloop = g_main_loop_new(NULL, FALSE);
	g_timeout_add(MESSAGE_TIMEOUT_S * 1000, resource_ipc_timeout, NULL);
	g_main_loop_run(mainloop);
	}
	}

	static int
	compare_id(gconstpointer a, gconstpointer b)
	{
	return strcmp((const char )a, (const char )b);
	}

	static GList *
	build_constraint_list(xmlNode *root)
	{
	GList *retval = NULL;
	xmlNode *cib_constraints = NULL;
	xmlXPathObjectPtr xpathObj = NULL;
	int ndx = 0;

	cib_constraints = pcmk_find_cib_element(root, XML_CIB_TAG_CONSTRAINTS);
	xpathObj = xpath_search(cib_constraints, "//" XML_CONS_TAG_RSC_LOCATION);

	for (ndx = 0; ndx < numXpathResults(xpathObj); ndx++) {
	xmlNode *match = getXpathResult(xpathObj, ndx);
	retval = g_list_insert_sorted(retval, (gpointer) ID(match), compare_id);
	}

	freeXpathObject(xpathObj);
	return retval;
	}

	/* short option letters still available: eEJkKXyYZ */

	static GOptionEntry query_entries[] = {
	{ "list", 'L', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, list_cb,
	"List all cluster resources with status",
	NULL },
	{ "list-raw", 'l', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, list_cb,
	"List IDs of all instantiated resources (individual members\n"
	INDENT "rather than groups etc.)",
	NULL },
	{ "list-cts", 'c', G_OPTION_FLAG_HIDDEN\|G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, list_cb,
	NULL,
	NULL },
	{ "list-operations", 'O', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, list_cb,
	"List active resource operations, optionally filtered by\n"
	INDENT "--resource and/or --node",
	NULL },
	{ "list-all-operations", 'o', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, list_cb,
	"List all resource operations, optionally filtered by\n"
	INDENT "--resource and/or --node",
	NULL },
	{ "list-standards", 0, G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK,
	list_standards_cb,
	"List supported standards",
	NULL },
	{ "list-ocf-providers", 0, G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK,
	list_providers_cb,
	"List all available OCF providers",
	NULL },
	{ "list-agents", 0, G_OPTION_FLAG_NONE, G_OPTION_ARG_CALLBACK,
	list_agents_cb,
	"List all agents available for the named standard and/or provider",
	"STD:PROV" },
	{ "list-ocf-alternatives", 0, G_OPTION_FLAG_NONE, G_OPTION_ARG_CALLBACK,
	list_alternatives_cb,
	"List all available providers for the named OCF agent",
	"AGENT" },
	{ "show-metadata", 0, G_OPTION_FLAG_NONE, G_OPTION_ARG_CALLBACK,
	metadata_cb,
	"Show the metadata for the named class:provider:agent",
	"SPEC" },
	{ "query-xml", 'q', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, flag_cb,
	"Show XML configuration of resource (after any template expansion)",
	NULL },
	{ "query-xml-raw", 'w', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, flag_cb,
	"Show XML configuration of resource (before any template expansion)",
	NULL },
	{ "get-parameter", 'g', G_OPTION_FLAG_NONE, G_OPTION_ARG_CALLBACK, get_param_prop_cb,
	"Display named parameter for resource (use instance attribute\n"
	INDENT "unless --meta or --utilization is specified)",
	"PARAM" },
	{ "get-property", 'G', G_OPTION_FLAG_HIDDEN, G_OPTION_ARG_CALLBACK, get_param_prop_cb,
	"Display named property of resource ('class', 'type', or 'provider') "
	"(requires --resource)",
	"PROPERTY" },
	{ "locate", 'W', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, flag_cb,
	"Show node(s) currently running resource",
	NULL },
	{ "stack", 'A', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, flag_cb,
	"Display the (co)location constraints that apply to a resource\n"
	INDENT "and the resources is it colocated with",
	NULL },
	{ "constraints", 'a', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, flag_cb,
	"Display the (co)location constraints that apply to a resource",
	NULL },
	{ "why", 'Y', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, why_cb,
	"Show why resources are not running, optionally filtered by\n"
	INDENT "--resource and/or --node",
	NULL },

	{ NULL }
	};

	static GOptionEntry command_entries[] = {
	{ "validate", 0, G_OPTION_FLAG_OPTIONAL_ARG, G_OPTION_ARG_CALLBACK,
	validate_or_force_cb,
	"Validate resource configuration by calling agent's validate-all\n"
	INDENT "action. The configuration may be specified either by giving an\n"
	INDENT "existing resource name with -r, or by specifying --class,\n"
	INDENT "--agent, and --provider arguments, along with any number of\n"
	INDENT "--option arguments. An optional LEVEL argument can be given\n"
	INDENT "to control the level of checking performed.",
	"LEVEL" },
	{ "cleanup", 'C', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, cleanup_refresh_cb,
	"If resource has any past failures, clear its history and fail\n"
	INDENT "count. Optionally filtered by --resource, --node, --operation\n"
	INDENT "and --interval (otherwise all). --operation and --interval\n"
	INDENT "apply to fail counts, but entire history is always clear, to\n"
	INDENT "allow current state to be rechecked. If the named resource is\n"
	INDENT "part of a group, or one numbered instance of a clone or bundled\n"
	INDENT "resource, the clean-up applies to the whole collective resource\n"
	INDENT "unless --force is given.",
	NULL },
	{ "refresh", 'R', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, cleanup_refresh_cb,
	"Delete resource's history (including failures) so its current state\n"
	INDENT "is rechecked. Optionally filtered by --resource and --node\n"
	INDENT "(otherwise all). If the named resource is part of a group, or one\n"
	INDENT "numbered instance of a clone or bundled resource, the refresh\n"
	INDENT "applies to the whole collective resource unless --force is given.",
	NULL },
	{ "set-parameter", 'p', G_OPTION_FLAG_NONE, G_OPTION_ARG_CALLBACK, set_delete_param_cb,
	"Set named parameter for resource (requires -v). Use instance\n"
	INDENT "attribute unless --meta or --utilization is specified.",
	"PARAM" },
	{ "delete-parameter", 'd', G_OPTION_FLAG_NONE, G_OPTION_ARG_CALLBACK, set_delete_param_cb,
	"Delete named parameter for resource. Use instance attribute\n"
	INDENT "unless --meta or --utilization is specified.",
	"PARAM" },
	{ "set-property", 'S', G_OPTION_FLAG_HIDDEN, G_OPTION_ARG_CALLBACK, set_prop_cb,
	"Set named property of resource ('class', 'type', or 'provider') "
	"(requires -r, -t, -v)",
	"PROPERTY" },

	{ NULL }
	};

	static GOptionEntry location_entries[] = {
	{ "move", 'M', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, flag_cb,
	"Create a constraint to move resource. If --node is specified,\n"
	INDENT "the constraint will be to move to that node, otherwise it\n"
	INDENT "will be to ban the current node. Unless --force is specified\n"
	INDENT "this will return an error if the resource is already running\n"
	INDENT "on the specified node. If --force is specified, this will\n"
	INDENT "always ban the current node.\n"
	INDENT "Optional: --lifetime, --promoted. NOTE: This may prevent the\n"
	INDENT "resource from running on its previous location until the\n"
	INDENT "implicit constraint expires or is removed with --clear.",
	NULL },
	{ "ban", 'B', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, flag_cb,
	"Create a constraint to keep resource off a node.\n"
	INDENT "Optional: --node, --lifetime, --promoted.\n"
	INDENT "NOTE: This will prevent the resource from running on the\n"
	INDENT "affected node until the implicit constraint expires or is\n"
	INDENT "removed with --clear. If --node is not specified, it defaults\n"
	INDENT "to the node currently running the resource for primitives\n"
	INDENT "and groups, or the promoted instance of promotable clones with\n"
	INDENT "promoted-max=1 (all other situations result in an error as\n"
	INDENT "there is no sane default).",
	NULL },
	{ "clear", 'U', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, flag_cb,
	"Remove all constraints created by the --ban and/or --move\n"
	INDENT "commands. Requires: --resource. Optional: --node, --promoted,\n"
	INDENT "--expired. If --node is not specified, all constraints created\n"
	INDENT "by --ban and --move will be removed for the named resource. If\n"
	INDENT "--node and --force are specified, any constraint created by\n"
	INDENT "--move will be cleared, even if it is not for the specified\n"
	INDENT "node. If --expired is specified, only those constraints whose\n"
	INDENT "lifetimes have expired will be removed.",
	NULL },
	{ "expired", 'e', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, expired_cb,
	"Modifies the --clear argument to remove constraints with\n"
	INDENT "expired lifetimes.",
	NULL },
	{ "lifetime", 'u', G_OPTION_FLAG_NONE, G_OPTION_ARG_STRING, &options.move_lifetime,
	"Lifespan (as ISO 8601 duration) of created constraints (with\n"
	INDENT "-B, -M) see https://en.wikipedia.org/wiki/ISO_8601#Durations)",
	"TIMESPEC" },
	{ "promoted", 0, G_OPTION_FLAG_NONE, G_OPTION_ARG_NONE,
	&options.promoted_role_only,
	"Limit scope of command to promoted role (with -B, -M, -U). For\n"
	INDENT "-B and -M, previously promoted instances may remain\n"
	INDENT "active in the unpromoted role.",
	NULL },

	// Deprecated since 2.1.0
	{ "master", 0, G_OPTION_FLAG_NONE, G_OPTION_ARG_NONE,
	&options.promoted_role_only,
	"Deprecated: Use --promoted instead", NULL },

	{ NULL }
	};

	static GOptionEntry advanced_entries[] = {
	{ "delete", 'D', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, delete_cb,
	"(Advanced) Delete a resource from the CIB. Required: -t",
	NULL },
	{ "fail", 'F', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, fail_cb,
	"(Advanced) Tell the cluster this resource has failed",
	NULL },
	{ "restart", 0, G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, restart_cb,
	"(Advanced) Tell the cluster to restart this resource and\n"
	INDENT "anything that depends on it",
	NULL },
	{ "wait", 0, G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, wait_cb,
	"(Advanced) Wait until the cluster settles into a stable state",
	NULL },
	{ "digests", 0, G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, digests_cb,
	"(Advanced) Show parameter hashes that Pacemaker uses to detect\n"
	INDENT "configuration changes (only accurate if there is resource\n"
	INDENT "history on the specified node). Required: --resource, --node.\n"
	INDENT "Optional: any NAME=VALUE parameters will be used to override\n"
	INDENT "the configuration (to see what the hash would be with those\n"
	INDENT "changes).",
	NULL },
	{ "force-demote", 0, G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK,
	validate_or_force_cb,
	"(Advanced) Bypass the cluster and demote a resource on the local\n"
	INDENT "node. Unless --force is specified, this will refuse to do so if\n"
	INDENT "the cluster believes the resource is a clone instance already\n"
	INDENT "running on the local node.",
	NULL },
	{ "force-stop", 0, G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK,
	validate_or_force_cb,
	"(Advanced) Bypass the cluster and stop a resource on the local node",
	NULL },
	{ "force-start", 0, G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK,
	validate_or_force_cb,
	"(Advanced) Bypass the cluster and start a resource on the local\n"
	INDENT "node. Unless --force is specified, this will refuse to do so if\n"
	INDENT "the cluster believes the resource is a clone instance already\n"
	INDENT "running on the local node.",
	NULL },
	{ "force-promote", 0, G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK,
	validate_or_force_cb,
	"(Advanced) Bypass the cluster and promote a resource on the local\n"
	INDENT "node. Unless --force is specified, this will refuse to do so if\n"
	INDENT "the cluster believes the resource is a clone instance already\n"
	INDENT "running on the local node.",
	NULL },
	{ "force-check", 0, G_OPTION_FLAG_OPTIONAL_ARG, G_OPTION_ARG_CALLBACK,
	validate_or_force_cb,
	"(Advanced) Bypass the cluster and check the state of a resource on\n"
	INDENT "the local node. An optional LEVEL argument can be given\n"
	INDENT "to control the level of checking performed.",
	"LEVEL" },

	{ NULL }
	};

	static GOptionEntry addl_entries[] = {
	{ "node", 'N', G_OPTION_FLAG_NONE, G_OPTION_ARG_STRING, &options.host_uname,
	"Node name",
	"NAME" },
	{ "recursive", 0, G_OPTION_FLAG_NONE, G_OPTION_ARG_NONE, &options.recursive,
	"Follow colocation chains when using --set-parameter",
	NULL },
	{ "resource-type", 't', G_OPTION_FLAG_NONE, G_OPTION_ARG_STRING, &options.rsc_type,
	"Resource XML element (primitive, group, etc.) (with -D)",
	"ELEMENT" },
	{ "parameter-value", 'v', G_OPTION_FLAG_NONE, G_OPTION_ARG_STRING, &options.prop_value,
	"Value to use with -p",
	"PARAM" },
	{ "meta", 'm', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, attr_set_type_cb,
	"Use resource meta-attribute instead of instance attribute\n"
	INDENT "(with -p, -g, -d)",
	NULL },
	{ "utilization", 'z', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, attr_set_type_cb,
	"Use resource utilization attribute instead of instance attribute\n"
	INDENT "(with -p, -g, -d)",
	NULL },
	{ "operation", 'n', G_OPTION_FLAG_NONE, G_OPTION_ARG_STRING, &options.operation,
	"Operation to clear instead of all (with -C -r)",
	"OPERATION" },
	{ "interval", 'I', G_OPTION_FLAG_NONE, G_OPTION_ARG_STRING, &options.interval_spec,
	"Interval of operation to clear (default 0) (with -C -r -n)",
	"N" },
	{ "class", 0, G_OPTION_FLAG_NONE, G_OPTION_ARG_CALLBACK, class_cb,
	"The standard the resource agent conforms to (for example, ocf).\n"
	INDENT "Use with --agent, --provider, --option, and --validate.",
	"CLASS" },
	{ "agent", 0, G_OPTION_FLAG_NONE, G_OPTION_ARG_CALLBACK, agent_provider_cb,
	"The agent to use (for example, IPaddr). Use with --class,\n"
	INDENT "--provider, --option, and --validate.",
	"AGENT" },
	{ "provider", 0, G_OPTION_FLAG_NONE, G_OPTION_ARG_CALLBACK, agent_provider_cb,
	"The vendor that supplies the resource agent (for example,\n"
	INDENT "heartbeat). Use with --class, --agent, --option, and --validate.",
	"PROVIDER" },
	{ "option", 0, G_OPTION_FLAG_NONE, G_OPTION_ARG_CALLBACK, option_cb,
	"Specify a device configuration parameter as NAME=VALUE (may be\n"
	INDENT "specified multiple times). Use with --validate and without the\n"
	INDENT "-r option.",
	"PARAM" },
	{ "set-name", 's', G_OPTION_FLAG_NONE, G_OPTION_ARG_STRING, &options.prop_set,
	"(Advanced) XML ID of attributes element to use (with -p, -d)",
	"ID" },
	{ "nvpair", 'i', G_OPTION_FLAG_NONE, G_OPTION_ARG_STRING, &options.prop_id,
	"(Advanced) XML ID of nvpair element to use (with -p, -d)",
	"ID" },
	{ "timeout", 'T', G_OPTION_FLAG_NONE, G_OPTION_ARG_CALLBACK, timeout_cb,
	"(Advanced) Abort if command does not finish in this time (with\n"
	INDENT "--restart, --wait, --force-*)",
	"N" },
	{ "force", 'f', G_OPTION_FLAG_NONE, G_OPTION_ARG_NONE, &options.force,
	"If making CIB changes, do so regardless of quorum. See help for\n"
	INDENT "individual commands for additional behavior.",
	NULL },
	{ "xml-file", 'x', G_OPTION_FLAG_HIDDEN, G_OPTION_ARG_FILENAME, &options.xml_file,
	NULL,
	"FILE" },
	{ "host-uname", 'H', G_OPTION_FLAG_HIDDEN, G_OPTION_ARG_STRING, &options.host_uname,
	NULL,
	"HOST" },

	{ NULL }
	};

	gboolean
	agent_provider_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	options.cmdline_config = TRUE;
	options.require_resource = FALSE;

	if (pcmk__str_eq(option_name, "--provider", pcmk__str_casei)) {
	pcmk__str_update(&options.v_provider, optarg);
	} else {
	pcmk__str_update(&options.v_agent, optarg);
	}

	return TRUE;
	}

	gboolean
	attr_set_type_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	if (pcmk__str_any_of(option_name, "-m", "--meta", NULL)) {
	options.attr_set_type = XML_TAG_META_SETS;
	} else if (pcmk__str_any_of(option_name, "-z", "--utilization", NULL)) {
	options.attr_set_type = XML_TAG_UTILIZATION;
	}

	return TRUE;
	}

	gboolean
	class_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	pcmk__str_update(&options.v_class, optarg);
	options.cmdline_config = TRUE;
	options.require_resource = FALSE;
	return TRUE;
	}

	gboolean
	cleanup_refresh_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	if (pcmk__str_any_of(option_name, "-C", "--cleanup", NULL)) {
	SET_COMMAND(cmd_cleanup);
	} else {
	SET_COMMAND(cmd_refresh);
	}

	options.require_resource = FALSE;
	if (getenv("CIB_file") == NULL) {
	options.require_crmd = TRUE;
	}
	options.find_flags = pe_find_renamed\|pe_find_anon;
	return TRUE;
	}

	gboolean
	delete_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	options.require_dataset = FALSE;
	SET_COMMAND(cmd_delete);
	options.find_flags = pe_find_renamed\|pe_find_any;
	return TRUE;
	}

	gboolean
	expired_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	options.clear_expired = TRUE;
	options.require_resource = FALSE;
	return TRUE;
	}

	static void
	get_agent_spec(const gchar *optarg)
	{
	options.require_cib = FALSE;
	options.require_dataset = FALSE;
	options.require_resource = FALSE;
	pcmk__str_update(&options.agent_spec, optarg);
	}

	gboolean
	list_agents_cb(const gchar option_name, const gchar optarg, gpointer data,
	GError **error)
	{
	SET_COMMAND(cmd_list_agents);
	get_agent_spec(optarg);
	return TRUE;
	}

	gboolean
	list_providers_cb(const gchar option_name, const gchar optarg, gpointer data,
	GError **error)
	{
	SET_COMMAND(cmd_list_providers);
	get_agent_spec(optarg);
	return TRUE;
	}

	gboolean
	list_standards_cb(const gchar option_name, const gchar optarg, gpointer data,
	GError **error)
	{
	SET_COMMAND(cmd_list_standards);
	options.require_cib = FALSE;
	options.require_dataset = FALSE;
	options.require_resource = FALSE;
	return TRUE;
	}

	gboolean
	list_alternatives_cb(const gchar option_name, const gchar optarg,
	gpointer data, GError **error)
	{
	SET_COMMAND(cmd_list_alternatives);
	get_agent_spec(optarg);
	return TRUE;
	}

	gboolean
	metadata_cb(const gchar option_name, const gchar optarg, gpointer data,
	GError **error)
	{
	SET_COMMAND(cmd_metadata);
	get_agent_spec(optarg);
	return TRUE;
	}

	gboolean
	option_cb(const gchar option_name, const gchar optarg, gpointer data,
	GError **error)
	{
	char *name = NULL;
	char *value = NULL;

	if (pcmk__scan_nvpair(optarg, &name, &value) != 2) {
	return FALSE;
	}
	if (options.cmdline_params == NULL) {
	options.cmdline_params = pcmk__strkey_table(free, free);
	}
	g_hash_table_replace(options.cmdline_params, name, value);
	return TRUE;
	}

	gboolean
	fail_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	options.require_crmd = TRUE;
	options.require_node = TRUE;
	SET_COMMAND(cmd_fail);
	return TRUE;
	}

	gboolean
	flag_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	if (pcmk__str_any_of(option_name, "-U", "--clear", NULL)) {
	options.find_flags = pe_find_renamed\|pe_find_anon;
	SET_COMMAND(cmd_clear);
	} else if (pcmk__str_any_of(option_name, "-B", "--ban", NULL)) {
	options.find_flags = pe_find_renamed\|pe_find_anon;
	SET_COMMAND(cmd_ban);
	} else if (pcmk__str_any_of(option_name, "-M", "--move", NULL)) {
	options.find_flags = pe_find_renamed\|pe_find_anon;
	SET_COMMAND(cmd_move);
	} else if (pcmk__str_any_of(option_name, "-q", "--query-xml", NULL)) {
	options.find_flags = pe_find_renamed\|pe_find_any;
	SET_COMMAND(cmd_query_xml);
	} else if (pcmk__str_any_of(option_name, "-w", "--query-xml-raw", NULL)) {
	options.find_flags = pe_find_renamed\|pe_find_any;
	SET_COMMAND(cmd_query_raw_xml);
	} else if (pcmk__str_any_of(option_name, "-W", "--locate", NULL)) {
	options.find_flags = pe_find_renamed\|pe_find_anon;
	SET_COMMAND(cmd_locate);
	} else if (pcmk__str_any_of(option_name, "-A", "--stack", NULL)) {
	options.find_flags = pe_find_renamed\|pe_find_anon;
	SET_COMMAND(cmd_colocations_deep);
	} else {
	options.find_flags = pe_find_renamed\|pe_find_anon;
	SET_COMMAND(cmd_colocations);
	}

	return TRUE;
	}

	gboolean
	get_param_prop_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	if (pcmk__str_any_of(option_name, "-g", "--get-parameter", NULL)) {
	SET_COMMAND(cmd_get_param);
	} else {
	SET_COMMAND(cmd_get_property);
	}

	pcmk__str_update(&options.prop_name, optarg);
	options.find_flags = pe_find_renamed\|pe_find_any;
	return TRUE;
	}

	gboolean
	list_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	if (pcmk__str_any_of(option_name, "-c", "--list-cts", NULL)) {
	SET_COMMAND(cmd_cts);
	} else if (pcmk__str_any_of(option_name, "-L", "--list", NULL)) {
	SET_COMMAND(cmd_list_resources);
	} else if (pcmk__str_any_of(option_name, "-l", "--list-raw", NULL)) {
	SET_COMMAND(cmd_list_instances);
	} else if (pcmk__str_any_of(option_name, "-O", "--list-operations", NULL)) {
	SET_COMMAND(cmd_list_active_ops);
	} else {
	SET_COMMAND(cmd_list_all_ops);
	}

	options.require_resource = FALSE;
	return TRUE;
	}

	gboolean
	set_delete_param_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	if (pcmk__str_any_of(option_name, "-p", "--set-parameter", NULL)) {
	SET_COMMAND(cmd_set_param);
	} else {
	SET_COMMAND(cmd_delete_param);
	}

	pcmk__str_update(&options.prop_name, optarg);
	options.find_flags = pe_find_renamed\|pe_find_any;
	return TRUE;
	}

	gboolean
	set_prop_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	options.require_dataset = FALSE;
	pcmk__str_update(&options.prop_name, optarg);
	SET_COMMAND(cmd_set_property);
	options.find_flags = pe_find_renamed\|pe_find_any;
	return TRUE;
	}

	gboolean
	timeout_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	options.timeout_ms = crm_get_msec(optarg);
	return TRUE;
	}

	gboolean
	validate_or_force_cb(const gchar option_name, const gchar optarg,
	gpointer data, GError **error)
	{
	SET_COMMAND(cmd_execute_agent);
	if (options.operation) {
	g_free(options.operation);
	}
	options.operation = g_strdup(option_name + 2); // skip "--"
	options.find_flags = pe_find_renamed\|pe_find_anon;
	if (options.override_params == NULL) {
	options.override_params = pcmk__strkey_table(free, free);
	}

	if (optarg != NULL) {
	if (pcmk__scan_min_int(optarg, &options.check_level, 0) != pcmk_rc_ok) {
	g_set_error(error, G_OPTION_ERROR, CRM_EX_INVALID_PARAM,
	"Invalid check level setting: %s", optarg);
	return FALSE;
	}
	}

	return TRUE;
	}

	gboolean
	restart_cb(const gchar option_name, const gchar optarg, gpointer data,
	GError **error)
	{
	SET_COMMAND(cmd_restart);
	options.find_flags = pe_find_renamed\|pe_find_anon;
	return TRUE;
	}

	gboolean
	digests_cb(const gchar option_name, const gchar optarg, gpointer data,
	GError **error)
	{
	SET_COMMAND(cmd_digests);
	options.find_flags = pe_find_renamed\|pe_find_anon;
	if (options.override_params == NULL) {
	options.override_params = pcmk__strkey_table(free, free);
	}
	options.require_node = TRUE;
	options.require_dataset = TRUE;
	return TRUE;
	}

	gboolean
	wait_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	SET_COMMAND(cmd_wait);
	options.require_resource = FALSE;
	options.require_dataset = FALSE;
	return TRUE;
	}

	gboolean
	why_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	options.require_resource = FALSE;
	SET_COMMAND(cmd_why);
	options.find_flags = pe_find_renamed\|pe_find_anon;
	return TRUE;
	}

	static int
	ban_or_move(pcmk__output_t out, pe_resource_t rsc, const char *move_lifetime)
	{
	int rc = pcmk_rc_ok;
	pe_node_t *current = NULL;
	unsigned int nactive = 0;

	CRM_CHECK(rsc != NULL, return EINVAL);

	current = pe__find_active_requires(rsc, &nactive);

	if (nactive == 1) {
	rc = cli_resource_ban(out, options.rsc_id, current->details->uname, move_lifetime, NULL,
	cib_conn, options.cib_options, options.promoted_role_only);

	} else if (pcmk_is_set(rsc->flags, pe_rsc_promotable)) {
	int count = 0;
	GList *iter = NULL;

	current = NULL;
	for(iter = rsc->children; iter; iter = iter->next) {
	pe_resource_t child = (pe_resource_t )iter->data;
	enum rsc_role_e child_role = child->fns->state(child, TRUE);

	if (child_role == RSC_ROLE_PROMOTED) {
	count++;
	current = pe__current_node(child);
	}
	}

	if(count == 1 && current) {
	rc = cli_resource_ban(out, options.rsc_id, current->details->uname, move_lifetime, NULL,
	cib_conn, options.cib_options, options.promoted_role_only);

	} else {
	rc = EINVAL;
	g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_USAGE,
	"Resource '%s' not moved: active in %d locations (promoted in %d).\n"
	"To prevent '%s' from running on a specific location, "
	"specify a node."
	"To prevent '%s' from being promoted at a specific "
	"location, specify a node and the --promoted option.",
	options.rsc_id, nactive, count, options.rsc_id, options.rsc_id);
	}

	} else {
	rc = EINVAL;
	g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_USAGE,
	"Resource '%s' not moved: active in %d locations.\n"
	"To prevent '%s' from running on a specific location, "
	"specify a node.",
	options.rsc_id, nactive, options.rsc_id);
	}

	return rc;
	}

	static void
	cleanup(pcmk__output_t out, pe_resource_t rsc)
	{
	int rc = pcmk_rc_ok;

	if (options.force == FALSE) {
	rsc = uber_parent(rsc);
	}

	crm_debug("Erasing failures of %s (%s requested) on %s",
	rsc->id, options.rsc_id, (options.host_uname? options.host_uname: "all nodes"));
	rc = cli_resource_delete(controld_api, options.host_uname, rsc, options.operation,
	options.interval_spec, TRUE, data_set, options.force);

	if ((rc == pcmk_rc_ok) && !out->is_quiet(out)) {
	// Show any reasons why resource might stay stopped
	cli_resource_check(out, cib_conn, rsc);
	}

	if (rc == pcmk_rc_ok) {
	start_mainloop(controld_api);
	}
	}

	static int
	clear_constraints(pcmk__output_t out, xmlNodePtr cib_xml_copy)
	{
	GList *before = NULL;
	GList *after = NULL;
	GList *remaining = NULL;
	GList *ele = NULL;
	pe_node_t *dest = NULL;
	int rc = pcmk_rc_ok;

	if (!out->is_quiet(out)) {
	before = build_constraint_list(data_set->input);
	}

	if (options.clear_expired) {
	rc = cli_resource_clear_all_expired(data_set->input, cib_conn, options.cib_options,
	options.rsc_id, options.host_uname,
	options.promoted_role_only);

	} else if (options.host_uname) {
	dest = pe_find_node(data_set->nodes, options.host_uname);
	if (dest == NULL) {
	rc = pcmk_rc_node_unknown;
	if (!out->is_quiet(out)) {
	g_list_free(before);
	}
	return rc;
	}
	rc = cli_resource_clear(options.rsc_id, dest->details->uname, NULL,
	cib_conn, options.cib_options, TRUE, options.force);

	} else {
	rc = cli_resource_clear(options.rsc_id, NULL, data_set->nodes,
	cib_conn, options.cib_options, TRUE, options.force);
	}

	if (!out->is_quiet(out)) {
	rc = cib_conn->cmds->query(cib_conn, NULL, cib_xml_copy, cib_scope_local \| cib_sync_call);
	rc = pcmk_legacy2rc(rc);

	if (rc != pcmk_rc_ok) {
	g_set_error(&error, PCMK__RC_ERROR, rc,
	"Could not get modified CIB: %s\n", pcmk_strerror(rc));
	g_list_free(before);
	free_xml(*cib_xml_copy);
	*cib_xml_copy = NULL;
	return rc;
	}

	data_set->input = *cib_xml_copy;
	cluster_status(data_set);

	after = build_constraint_list(data_set->input);
	remaining = pcmk__subtract_lists(before, after, (GCompareFunc) strcmp);

	for (ele = remaining; ele != NULL; ele = ele->next) {
	out->info(out, "Removing constraint: %s", (char *) ele->data);
	}

	g_list_free(before);
	g_list_free(after);
	g_list_free(remaining);
	}

	return rc;
	}

	static int
	delete(void)
	{
	int rc = pcmk_rc_ok;
	xmlNode *msg_data = NULL;

	if (options.rsc_type == NULL) {
	rc = ENXIO;
	g_set_error(&error, PCMK__RC_ERROR, rc,
	"You need to specify a resource type with -t");
	return rc;
	}

	msg_data = create_xml_node(NULL, options.rsc_type);
	crm_xml_add(msg_data, XML_ATTR_ID, options.rsc_id);

	rc = cib_conn->cmds->remove(cib_conn, XML_CIB_TAG_RESOURCES, msg_data,
	options.cib_options);
	rc = pcmk_legacy2rc(rc);
	free_xml(msg_data);
	return rc;
	}

	static int
	list_agents(pcmk__output_t out, const char agent_spec)
	{
	int rc = pcmk_rc_ok;
	char *provider = strchr(agent_spec, ':');
	lrmd_t *lrmd_conn = NULL;
	lrmd_list_t *list = NULL;

	rc = lrmd__new(&lrmd_conn, NULL, NULL, 0);
	if (rc != pcmk_rc_ok) {
	goto error;
	}

	if (provider) {
	*provider++ = 0;
	}

	rc = lrmd_conn->cmds->list_agents(lrmd_conn, &list, agent_spec, provider);

	if (rc > 0) {
	rc = out->message(out, "agents-list", list, agent_spec, provider);
	} else {
	rc = pcmk_rc_error;
	}

	error:
	if (rc != pcmk_rc_ok) {
	if (provider == NULL) {
	g_set_error(&error, PCMK__RC_ERROR, rc,
	"No agents found for standard '%s'", agent_spec);
	} else {
	g_set_error(&error, PCMK__RC_ERROR, rc,
	"No agents found for standard '%s' and provider '%s'",
	agent_spec, provider);
	}
	}

	lrmd_api_delete(lrmd_conn);
	return rc;
	}

	static int
	list_providers(pcmk__output_t out, const char agent_spec)
	{
	int rc;
	const char *text = NULL;
	lrmd_t *lrmd_conn = NULL;
	lrmd_list_t *list = NULL;

	rc = lrmd__new(&lrmd_conn, NULL, NULL, 0);
	if (rc != pcmk_rc_ok) {
	goto error;
	}

	switch (options.rsc_cmd) {
	case cmd_list_alternatives:
	rc = lrmd_conn->cmds->list_ocf_providers(lrmd_conn, agent_spec, &list);

	if (rc > 0) {
	rc = out->message(out, "alternatives-list", list, agent_spec);
	} else {
	rc = pcmk_rc_error;
	}

	text = "OCF providers";
	break;
	case cmd_list_standards:
	rc = lrmd_conn->cmds->list_standards(lrmd_conn, &list);

	if (rc > 0) {
	rc = out->message(out, "standards-list", list);
	} else {
	rc = pcmk_rc_error;
	}

	text = "standards";
	break;
	case cmd_list_providers:
	rc = lrmd_conn->cmds->list_ocf_providers(lrmd_conn, agent_spec, &list);

	if (rc > 0) {
	rc = out->message(out, "providers-list", list, agent_spec);
	} else {
	rc = pcmk_rc_error;
	}

	text = "OCF providers";
	break;
	default:
	g_set_error(&error, PCMK__RC_ERROR, pcmk_rc_error, "Bug");
	lrmd_api_delete(lrmd_conn);
	return pcmk_rc_error;
	}

	error:
	if (rc != pcmk_rc_ok) {
	if (agent_spec != NULL) {
	rc = ENXIO;
	g_set_error(&error, PCMK__RC_ERROR, rc,
	"No %s found for %s", text, agent_spec);

	} else {
	rc = ENXIO;
	g_set_error(&error, PCMK__RC_ERROR, rc,
	"No %s found", text);
	}
	}

	lrmd_api_delete(lrmd_conn);
	return rc;
	}

	static int
	populate_working_set(xmlNodePtr *cib_xml_copy)
	{
	int rc = pcmk_rc_ok;

	if (options.xml_file != NULL) {
	*cib_xml_copy = filename2xml(options.xml_file);
	if (*cib_xml_copy == NULL) {
	rc = pcmk_rc_cib_corrupt;
	}
	} else {
	rc = cib_conn->cmds->query(cib_conn, NULL, cib_xml_copy, cib_scope_local \| cib_sync_call);
	rc = pcmk_legacy2rc(rc);
	}

	if (rc == pcmk_rc_ok) {
	data_set = pe_new_working_set();
	if (data_set == NULL) {
	rc = ENOMEM;
	} else {
	pe__set_working_set_flags(data_set,
	pe_flag_no_counts\|pe_flag_no_compat);
	data_set->priv = out;
	rc = update_working_set_xml(data_set, cib_xml_copy);
	}
	}

	if (rc != pcmk_rc_ok) {
	free_xml(*cib_xml_copy);
	*cib_xml_copy = NULL;
	return rc;
	}

	cluster_status(data_set);
	return pcmk_rc_ok;
	}

	static int
	refresh(pcmk__output_t *out)
	{
	int rc = pcmk_rc_ok;
	const char *router_node = options.host_uname;
	int attr_options = pcmk__node_attr_none;

	if (options.host_uname) {
	pe_node_t *node = pe_find_node(data_set->nodes, options.host_uname);

	if (pe__is_guest_or_remote_node(node)) {
	node = pe__current_node(node->details->remote_rsc);
	if (node == NULL) {
	rc = ENXIO;
	g_set_error(&error, PCMK__RC_ERROR, rc,
	"No cluster connection to Pacemaker Remote node %s detected",
	options.host_uname);
	return rc;
	}
	router_node = node->details->uname;
	attr_options \|= pcmk__node_attr_remote;
	}
	}

	if (controld_api == NULL) {
	out->info(out, "Dry run: skipping clean-up of %s due to CIB_file",
	options.host_uname? options.host_uname : "all nodes");
	rc = pcmk_rc_ok;
	return rc;
	}

	crm_debug("Re-checking the state of all resources on %s", options.host_uname?options.host_uname:"all nodes");

	rc = pcmk__node_attr_request_clear(NULL, options.host_uname,
	NULL, NULL, NULL,
	NULL, attr_options);

	if (pcmk_controld_api_reprobe(controld_api, options.host_uname,
	router_node) == pcmk_rc_ok) {
	start_mainloop(controld_api);
	}

	return rc;
	}

	static void
	refresh_resource(pcmk__output_t out, pe_resource_t rsc)
	{
	int rc = pcmk_rc_ok;

	if (options.force == FALSE) {
	rsc = uber_parent(rsc);
	}

	crm_debug("Re-checking the state of %s (%s requested) on %s",
	rsc->id, options.rsc_id, (options.host_uname? options.host_uname: "all nodes"));
	rc = cli_resource_delete(controld_api, options.host_uname, rsc, NULL,
	0, FALSE, data_set, options.force);

	if ((rc == pcmk_rc_ok) && !out->is_quiet(out)) {
	// Show any reasons why resource might stay stopped
	cli_resource_check(out, cib_conn, rsc);
	}

	if (rc == pcmk_rc_ok) {
	start_mainloop(controld_api);
	}
	}

	static int
	set_property(void)
	{
	int rc = pcmk_rc_ok;
	xmlNode *msg_data = NULL;

	if (pcmk__str_empty(options.rsc_type)) {
	g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_USAGE,
	"Must specify -t with resource type");
	rc = ENXIO;
	return rc;

	} else if (pcmk__str_empty(options.prop_value)) {
	g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_USAGE,
	"Must supply -v with new value");
	rc = ENXIO;
	return rc;
	}

	CRM_LOG_ASSERT(options.prop_name != NULL);

	msg_data = create_xml_node(NULL, options.rsc_type);
	crm_xml_add(msg_data, XML_ATTR_ID, options.rsc_id);
	crm_xml_add(msg_data, options.prop_name, options.prop_value);

	rc = cib_conn->cmds->modify(cib_conn, XML_CIB_TAG_RESOURCES, msg_data,
	options.cib_options);
	rc = pcmk_legacy2rc(rc);
	free_xml(msg_data);

	return rc;
	}

	static int
	show_metadata(pcmk__output_t out, const char agent_spec)
	{
	int rc = pcmk_rc_ok;
	char *standard = NULL;
	char *provider = NULL;
	char *type = NULL;
	char *metadata = NULL;
	lrmd_t *lrmd_conn = NULL;

	rc = lrmd__new(&lrmd_conn, NULL, NULL, 0);
	if (rc != pcmk_rc_ok) {
	g_set_error(&error, PCMK__RC_ERROR, rc,
	"Could not create executor connection");
	lrmd_api_delete(lrmd_conn);
	return rc;
	}

	rc = crm_parse_agent_spec(agent_spec, &standard, &provider, &type);
	rc = pcmk_legacy2rc(rc);

	if (rc == pcmk_rc_ok) {
	rc = lrmd_conn->cmds->get_metadata(lrmd_conn, standard,
	provider, type,
	&metadata, 0);
	rc = pcmk_legacy2rc(rc);

	if (metadata) {
	out->output_xml(out, "metadata", metadata);
	free(metadata);
	} else {
	/* We were given a validly formatted spec, but it doesn't necessarily
	* match up with anything that exists. Use ENXIO as the return code
	* here because that maps to an exit code of CRM_EX_NOSUCH, which
	* probably is the most common reason to get here.
	*/
	rc = ENXIO;
	g_set_error(&error, PCMK__RC_ERROR, rc,
	_("Metadata query for %s failed: %s"),
	agent_spec, pcmk_rc_str(rc));
	}
	} else {
	rc = ENXIO;
	g_set_error(&error, PCMK__RC_ERROR, rc,
	_("'%s' is not a valid agent specification"), agent_spec);
	}

	lrmd_api_delete(lrmd_conn);
	return rc;
	}

	static void
	validate_cmdline_config(void)
	{
	// Cannot use both --resource and command-line resource configuration
	if (options.rsc_id != NULL) {
	g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_USAGE,
	"--resource cannot be used with --class, --agent, and --provider");

	// Not all commands support command-line resource configuration
	} else if (options.rsc_cmd != cmd_execute_agent) {
	g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_USAGE,
	"--class, --agent, and --provider can only be used with "
	"--validate and --force-*");

	// Not all of --class, --agent, and --provider need to be given. Not all
	// classes support the concept of a provider. Check that what we were given
	// is valid.
	} else if (pcmk__str_eq(options.v_class, "stonith", pcmk__str_none)) {
	if (options.v_provider != NULL) {
	g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_USAGE,
	"stonith does not support providers");

	} else if (stonith_agent_exists(options.v_agent, 0) == FALSE) {
	g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_USAGE,
	"%s is not a known stonith agent", options.v_agent ? options.v_agent : "");
	}

	} else if (resources_agent_exists(options.v_class, options.v_provider, options.v_agent) == FALSE) {
	g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_USAGE,
	"%s:%s:%s is not a known resource",
	options.v_class ? options.v_class : "",
	options.v_provider ? options.v_provider : "",
	options.v_agent ? options.v_agent : "");
	}

	if (error != NULL) {
	return;
	}

	if (options.cmdline_params == NULL) {
	options.cmdline_params = pcmk__strkey_table(free, free);
	}
	options.require_resource = FALSE;
	options.require_dataset = FALSE;
	options.require_cib = FALSE;
	}

	static GOptionContext *
	build_arg_context(pcmk__common_args_t args, GOptionGroup *group) {
	GOptionContext *context = NULL;

	GOptionEntry extra_prog_entries[] = {
	{ "quiet", 'Q', G_OPTION_FLAG_NONE, G_OPTION_ARG_NONE, &(args->quiet),
	"Be less descriptive in output.",
	NULL },
	{ "resource", 'r', G_OPTION_FLAG_NONE, G_OPTION_ARG_STRING, &options.rsc_id,
	"Resource ID",
	"ID" },
	{ G_OPTION_REMAINING, 0, G_OPTION_FLAG_NONE, G_OPTION_ARG_STRING_ARRAY, &options.remainder,
	NULL,
	NULL },

	{ NULL }
	};

	const char *description = "Examples:\n\n"
	"List the available OCF agents:\n\n"
	"\t# crm_resource --list-agents ocf\n\n"
	"List the available OCF agents from the linux-ha project:\n\n"
	"\t# crm_resource --list-agents ocf:heartbeat\n\n"
	"Move 'myResource' to a specific node:\n\n"
	"\t# crm_resource --resource myResource --move --node altNode\n\n"
	"Allow (but not force) 'myResource' to move back to its original "
	"location:\n\n"
	"\t# crm_resource --resource myResource --clear\n\n"
	"Stop 'myResource' (and anything that depends on it):\n\n"
	"\t# crm_resource --resource myResource --set-parameter target-role "
	"--meta --parameter-value Stopped\n\n"
	"Tell the cluster not to manage 'myResource' (the cluster will not "
	"attempt to start or stop the\n"
	"resource under any circumstances; useful when performing maintenance "
	"tasks on a resource):\n\n"
	"\t# crm_resource --resource myResource --set-parameter is-managed "
	"--meta --parameter-value false\n\n"
	"Erase the operation history of 'myResource' on 'aNode' (the cluster "
	"will 'forget' the existing\n"
	"resource state, including any errors, and attempt to recover the"
	"resource; useful when a resource\n"
	"had failed permanently and has been repaired by an administrator):\n\n"
	"\t# crm_resource --resource myResource --cleanup --node aNode\n\n";

	context = pcmk__build_arg_context(args, "text (default), xml", group, NULL);
	g_option_context_set_description(context, description);

	/* Add the -Q option, which cannot be part of the globally supported options
	* because some tools use that flag for something else.
	*/
	pcmk__add_main_args(context, extra_prog_entries);

	pcmk__add_arg_group(context, "queries", "Queries:",
	"Show query help", query_entries);
	pcmk__add_arg_group(context, "commands", "Commands:",
	"Show command help", command_entries);
	pcmk__add_arg_group(context, "locations", "Locations:",
	"Show location help", location_entries);
	pcmk__add_arg_group(context, "advanced", "Advanced:",
	"Show advanced option help", advanced_entries);
	pcmk__add_arg_group(context, "additional", "Additional Options:",
	"Show additional options", addl_entries);
	return context;
	}

	int
	main(int argc, char **argv)
	{
	xmlNode *cib_xml_copy = NULL;
	pe_resource_t *rsc = NULL;
	pe_node_t *node = NULL;
	int rc = pcmk_rc_ok;

	GOptionGroup *output_group = NULL;
	gchar **processed_args = NULL;
	GOptionContext *context = NULL;

	/*
	* Parse command line arguments
	*/

	args = pcmk__new_common_args(SUMMARY);
	processed_args = pcmk__cmdline_preproc(argv, "GHINSTdginpstuvx");
	context = build_arg_context(args, &output_group);

	pcmk__register_formats(output_group, formats);
	if (!g_option_context_parse_strv(context, &processed_args, &error)) {
	exit_code = CRM_EX_USAGE;
	goto done;
	}

	pcmk__cli_init_logging("crm_resource", args->verbosity);

	rc = pcmk__output_new(&out, args->output_ty, args->output_dest, argv);
	if (rc != pcmk_rc_ok) {
	exit_code = CRM_EX_ERROR;
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code, "Error creating output format %s: %s",
	args->output_ty, pcmk_rc_str(rc));
	goto done;
	}

	pe__register_messages(out);
	crm_resource_register_messages(out);
	lrmd__register_messages(out);
	pcmk__register_lib_messages(out);

	out->quiet = args->quiet;

	crm_log_args(argc, argv);

	/*
	* Validate option combinations
	*/

	// If the user didn't explicitly specify a command, list resources
	if (options.rsc_cmd == cmd_none) {
	options.rsc_cmd = cmd_list_resources;
	options.require_resource = FALSE;
	}

	// --expired without --clear/-U doesn't make sense
	if (options.clear_expired && (options.rsc_cmd != cmd_clear)) {
	exit_code = CRM_EX_USAGE;
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code, "--expired requires --clear or -U");
	goto done;
	}

	if ((options.remainder != NULL) && (options.override_params != NULL)) {
	// Commands that use positional arguments will create override_params
	for (gchar *s = options.remainder; s; s++) {
	char name = calloc(1, strlen(s));
	char value = calloc(1, strlen(s));
	int rc = sscanf(*s, "%[^=]=%s", name, value);

	if (rc == 2) {
	g_hash_table_replace(options.override_params, name, value);

	} else {
	exit_code = CRM_EX_USAGE;
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	"Error parsing '%s' as a name=value pair",
	argv[optind]);
	free(value);
	free(name);
	goto done;
	}
	}

	} else if (options.remainder != NULL) {
	gchar **strv = NULL;
	gchar *msg = NULL;
	int i = 1;
	int len = 0;

	for (gchar *s = options.remainder; s; s++) {
	len++;
	}

	CRM_ASSERT(len > 0);

	strv = calloc(len, sizeof(char *));
	strv[0] = strdup("non-option ARGV-elements:");

	for (gchar *s = options.remainder; s; s++) {
	strv[i] = crm_strdup_printf("[%d of %d] %s\n", i, len, *s);
	i++;
	}

	exit_code = CRM_EX_USAGE;
	msg = g_strjoinv("", strv);
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code, "%s", msg);
	g_free(msg);

	for(i = 0; i < len; i++) {
	free(strv[i]);
	}
	free(strv);

	goto done;
	}

	if (pcmk__str_eq(args->output_ty, "xml", pcmk__str_none)) {
	/* Kind of a hack to display XML lists using a real tag instead of <list>. This just
	* saves from having to write custom messages to build the lists around all these things
	*/
	switch (options.rsc_cmd) {
	case cmd_execute_agent:
	case cmd_list_resources:
	case cmd_query_xml:
	case cmd_query_raw_xml:
	case cmd_list_active_ops:
	case cmd_list_all_ops:
	case cmd_colocations:
	case cmd_colocations_deep:
	pcmk__force_args(context, &error, "%s --xml-simple-list --xml-substitute", g_get_prgname());
	break;

	default:
	pcmk__force_args(context, &error, "%s --xml-substitute", g_get_prgname());
	break;
	}
	} else if (pcmk__str_eq(args->output_ty, "text", pcmk__str_null_matches)) {
	if (options.rsc_cmd == cmd_colocations \|\| options.rsc_cmd == cmd_colocations_deep \|\|
	options.rsc_cmd == cmd_list_resources) {
	pcmk__force_args(context, &error, "%s --text-fancy", g_get_prgname());
	}
	}

	if (args->version) {
	out->version(out, false);
	goto done;
	}

	if (options.cmdline_config) {
	/* A resource configuration was given on the command line. Sanity-check
	* the values and set error if they don't make sense.
	*/
	validate_cmdline_config();
	if (error != NULL) {
	exit_code = CRM_EX_USAGE;
	goto done;
	}

	} else if (options.cmdline_params != NULL) {
	// @COMPAT @TODO error out here when we can break backward compatibility
	g_hash_table_destroy(options.cmdline_params);
	options.cmdline_params = NULL;
	}

	if (options.require_resource && (options.rsc_id == NULL)) {
	exit_code = CRM_EX_USAGE;
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	"Must supply a resource id with -r");
	goto done;
	}
	if (options.require_node && (options.host_uname == NULL)) {
	exit_code = CRM_EX_USAGE;
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	"Must supply a node name with -N");
	goto done;
	}

	/*
	* Set up necessary connections
	*/

	if (options.force) {
	crm_debug("Forcing...");
	cib__set_call_options(options.cib_options, crm_system_name,
	cib_quorum_override);
	}

	if (options.find_flags && options.rsc_id) {
	options.require_dataset = TRUE;
	}

	// Establish a connection to the CIB if needed
	if (options.require_cib) {
	cib_conn = cib_new();
	if ((cib_conn == NULL) \|\| (cib_conn->cmds == NULL)) {
	exit_code = CRM_EX_DISCONNECT;
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	"Could not create CIB connection");
	goto done;
	}
	rc = cib_conn->cmds->signon(cib_conn, crm_system_name, cib_command);
	rc = pcmk_legacy2rc(rc);
	if (rc != pcmk_rc_ok) {
	exit_code = pcmk_rc2exitc(rc);
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	"Could not connect to the CIB: %s", pcmk_rc_str(rc));
	goto done;
	}
	}

	/* Populate working set from XML file if specified or CIB query otherwise */
	if (options.require_dataset) {
	rc = populate_working_set(&cib_xml_copy);
	if (rc != pcmk_rc_ok) {
	exit_code = pcmk_rc2exitc(rc);
	goto done;
	}
	}

	// If command requires that resource exist if specified, find it
	if (options.find_flags && options.rsc_id) {
	rsc = pe_find_resource_with_flags(data_set->resources, options.rsc_id,
	options.find_flags);
	if (rsc == NULL) {
	exit_code = CRM_EX_NOSUCH;
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	"Resource '%s' not found", options.rsc_id);
	goto done;
	}
	+
	+ /* The --ban, --clear, --move, and --restart commands do not work with
	+ * instances of clone resourcs.
	+ */
	+ if (strchr(options.rsc_id, ':') != NULL && pe_rsc_is_clone(rsc->parent) &&
	+ (options.rsc_cmd == cmd_ban \|\| options.rsc_cmd == cmd_clear \|\|
	+ options.rsc_cmd == cmd_move \|\| options.rsc_cmd == cmd_restart)) {
	+ exit_code = CRM_EX_INVALID_PARAM;
	+ g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	+ "Cannot operate on clone resource instance '%s'", options.rsc_id);
	+ goto done;
	+ }
	}

	// If user supplied a node name, check whether it exists
	if ((options.host_uname != NULL) && (data_set != NULL)) {
	node = pe_find_node(data_set->nodes, options.host_uname);
	+
	+ if (node == NULL) {
	+ exit_code = CRM_EX_NOSUCH;
	+ g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	+ "Node '%s' not found", options.host_uname);
	+ goto done;
	+ }
	}

	// Establish a connection to the controller if needed
	if (options.require_crmd) {
	rc = pcmk_new_ipc_api(&controld_api, pcmk_ipc_controld);
	if (rc != pcmk_rc_ok) {
	exit_code = pcmk_rc2exitc(rc);
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	"Error connecting to the controller: %s", pcmk_rc_str(rc));
	goto done;
	}
	pcmk_register_ipc_callback(controld_api, controller_event_callback,
	NULL);
	rc = pcmk_connect_ipc(controld_api, pcmk_ipc_dispatch_main);
	if (rc != pcmk_rc_ok) {
	exit_code = pcmk_rc2exitc(rc);
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	"Error connecting to the controller: %s", pcmk_rc_str(rc));
	goto done;
	}
	}

	/*
	* Handle requested command
	*/

	switch (options.rsc_cmd) {
	case cmd_list_resources: {
	GList *all = NULL;
	all = g_list_prepend(all, (gpointer) "*");
	rc = out->message(out, "resource-list", data_set,
	pcmk_show_inactive_rscs \| pcmk_show_rsc_only \| pcmk_show_pending,
	TRUE, all, all, FALSE);
	g_list_free(all);

	if (rc == pcmk_rc_no_output) {
	rc = ENXIO;
	}
	break;
	}

	case cmd_list_instances:
	rc = out->message(out, "resource-names-list", data_set->resources);

	if (rc != pcmk_rc_ok) {
	rc = ENXIO;
	}

	break;

	case cmd_list_standards:
	case cmd_list_providers:
	case cmd_list_alternatives:
	rc = list_providers(out, options.agent_spec);
	break;

	case cmd_list_agents:
	rc = list_agents(out, options.agent_spec);
	break;

	case cmd_metadata:
	rc = show_metadata(out, options.agent_spec);
	break;

	case cmd_restart:
	/* We don't pass data_set because rsc needs to stay valid for the
	* entire lifetime of cli_resource_restart(), but it will reset and
	* update the working set multiple times, so it needs to use its own
	* copy.
	*/
	- rc = cli_resource_restart(out, rsc, options.host_uname,
	- options.move_lifetime, options.timeout_ms,
	- cib_conn, options.cib_options,
	- options.promoted_role_only,
	+ rc = cli_resource_restart(out, rsc, node, options.move_lifetime,
	+ options.timeout_ms, cib_conn,
	+ options.cib_options, options.promoted_role_only,
	options.force);
	break;

	case cmd_wait:
	rc = wait_till_stable(out, options.timeout_ms, cib_conn);
	break;

	case cmd_execute_agent:
	if (options.cmdline_config) {
	exit_code = cli_resource_execute_from_params(out, NULL,
	options.v_class, options.v_provider, options.v_agent,
	options.operation, options.cmdline_params,
	options.override_params, options.timeout_ms,
	args->verbosity, options.force, options.check_level);
	} else {
	exit_code = cli_resource_execute(rsc, options.rsc_id,
	options.operation, options.override_params,
	options.timeout_ms, cib_conn, data_set,
	args->verbosity, options.force, options.check_level);
	}
	goto done;

	case cmd_digests:
	node = pe_find_node(data_set->nodes, options.host_uname);
	if (node == NULL) {
	rc = pcmk_rc_node_unknown;
	} else {
	rc = pcmk__resource_digests(out, rsc, node,
	options.override_params, data_set);
	}
	break;

	case cmd_colocations:
	rc = out->message(out, "stacks-constraints", rsc, data_set, false);
	break;

	case cmd_colocations_deep:
	rc = out->message(out, "stacks-constraints", rsc, data_set, true);
	break;

	case cmd_cts:
	rc = pcmk_rc_ok;
	g_list_foreach(data_set->resources, (GFunc) cli_resource_print_cts, out);
	cli_resource_print_cts_constraints(data_set);
	break;

	case cmd_fail:
	rc = cli_resource_fail(controld_api, options.host_uname,
	options.rsc_id, data_set);
	if (rc == pcmk_rc_ok) {
	start_mainloop(controld_api);
	}
	break;

	case cmd_list_active_ops:
	rc = cli_resource_print_operations(options.rsc_id,
	options.host_uname, TRUE,
	data_set);
	break;

	case cmd_list_all_ops:
	rc = cli_resource_print_operations(options.rsc_id,
	options.host_uname, FALSE,
	data_set);
	break;

	case cmd_locate: {
	GList *nodes = cli_resource_search(rsc, options.rsc_id, data_set);
	rc = out->message(out, "resource-search-list", nodes, options.rsc_id);
	g_list_free_full(nodes, free);
	break;
	}

	case cmd_query_xml:
	rc = cli_resource_print(rsc, data_set, TRUE);
	break;

	case cmd_query_raw_xml:
	rc = cli_resource_print(rsc, data_set, FALSE);
	break;

	case cmd_why:
	if ((options.host_uname != NULL) && (node == NULL)) {
	rc = pcmk_rc_node_unknown;
	} else {
	rc = out->message(out, "resource-reasons-list", cib_conn,
	data_set->resources, rsc, node);
	}
	break;

	case cmd_clear:
	rc = clear_constraints(out, &cib_xml_copy);
	break;

	case cmd_move:
	if (options.host_uname == NULL) {
	rc = ban_or_move(out, rsc, options.move_lifetime);
	} else {
	rc = cli_resource_move(rsc, options.rsc_id, options.host_uname,
	options.move_lifetime, cib_conn,
	options.cib_options, data_set,
	options.promoted_role_only,
	options.force);
	}

	if (rc == EINVAL) {
	exit_code = CRM_EX_USAGE;
	goto done;
	}

	break;

	case cmd_ban:
	if (options.host_uname == NULL) {
	rc = ban_or_move(out, rsc, options.move_lifetime);
	} else if (node == NULL) {
	rc = pcmk_rc_node_unknown;
	} else {
	rc = cli_resource_ban(out, options.rsc_id, node->details->uname,
	options.move_lifetime, NULL, cib_conn,
	options.cib_options,
	options.promoted_role_only);
	}

	if (rc == EINVAL) {
	exit_code = CRM_EX_USAGE;
	goto done;
	}

	break;

	case cmd_get_property:
	rc = out->message(out, "property-list", rsc, options.prop_name);
	if (rc == pcmk_rc_no_output) {
	rc = ENXIO;
	}

	break;

	case cmd_set_property:
	rc = set_property();
	break;

	case cmd_get_param: {
	unsigned int count = 0;
	GHashTable *params = NULL;
	pe_node_t *current = pe__find_active_on(rsc, &count, NULL);
	bool free_params = true;

	if (count > 1) {
	out->err(out, "%s is active on more than one node,"
	" returning the default value for %s", rsc->id, crm_str(options.prop_name));
	current = NULL;
	}

	crm_debug("Looking up %s in %s", options.prop_name, rsc->id);

	if (pcmk__str_eq(options.attr_set_type, XML_TAG_ATTR_SETS, pcmk__str_casei)) {
	params = pe_rsc_params(rsc, current, data_set);
	free_params = false;

	} else if (pcmk__str_eq(options.attr_set_type, XML_TAG_META_SETS, pcmk__str_casei)) {
	params = pcmk__strkey_table(free, free);
	get_meta_attributes(params, rsc, current, data_set);

	} else {
	params = pcmk__strkey_table(free, free);
	pe__unpack_dataset_nvpairs(rsc->xml, XML_TAG_UTILIZATION, NULL, params,
	NULL, FALSE, data_set);
	}

	rc = out->message(out, "attribute-list", rsc, options.prop_name, params);
	if (free_params) {
	g_hash_table_destroy(params);
	}
	break;
	}

	case cmd_set_param:
	if (pcmk__str_empty(options.prop_value)) {
	exit_code = CRM_EX_USAGE;
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	"You need to supply a value with the -v option");
	goto done;
	}

	/* coverity[var_deref_model] False positive */
	rc = cli_resource_update_attribute(rsc, options.rsc_id,
	options.prop_set,
	options.attr_set_type,
	options.prop_id,
	options.prop_name,
	options.prop_value,
	options.recursive, cib_conn,
	options.cib_options, data_set,
	options.force);
	break;

	case cmd_delete_param:
	/* coverity[var_deref_model] False positive */
	rc = cli_resource_delete_attribute(rsc, options.rsc_id,
	options.prop_set,
	options.attr_set_type,
	options.prop_id,
	options.prop_name, cib_conn,
	options.cib_options, data_set,
	options.force);
	break;

	case cmd_cleanup:
	if (rsc == NULL) {
	rc = cli_cleanup_all(controld_api, options.host_uname,
	options.operation, options.interval_spec,
	data_set);
	if (rc == pcmk_rc_ok) {
	start_mainloop(controld_api);
	}
	} else {
	cleanup(out, rsc);
	}
	break;

	case cmd_refresh:
	if (rsc == NULL) {
	rc = refresh(out);
	} else {
	refresh_resource(out, rsc);
	}
	break;

	case cmd_delete:
	rc = delete();
	break;

	default:
	exit_code = CRM_EX_USAGE;
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	"Unimplemented command: %d", (int) options.rsc_cmd);
	goto done;
	}

	/* Convert rc into an exit code. */
	if (rc != pcmk_rc_ok && rc != pcmk_rc_no_output) {
	if (rc == pcmk_rc_no_quorum) {
	g_prefix_error(&error, "To ignore quorum, use the force option.\n");
	}

	exit_code = pcmk_rc2exitc(rc);
	}

	/*
	* Clean up and exit
	*/

	done:
	/* When we get here, exit_code has been set one of two ways - either at one of
	* the spots where there's a "goto done" (which itself could have happened either
	* directly or by calling pcmk_rc2exitc), or just up above after any of the break
	* statements.
	*
	* Thus, we can use just exit_code here to decide what to do.
	*/
	if (exit_code != CRM_EX_OK && exit_code != CRM_EX_USAGE) {
	if (error != NULL) {
	char *msg = crm_strdup_printf("%s\nError performing operation: %s",
	error->message, crm_exit_str(exit_code));
	g_clear_error(&error);
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code, "%s", msg);
	free(msg);
	} else {
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	"Error performing operation: %s", crm_exit_str(exit_code));
	}
	}

	g_free(options.host_uname);
	g_free(options.interval_spec);
	g_free(options.move_lifetime);
	g_free(options.operation);
	g_free(options.prop_id);
	free(options.prop_name);
	g_free(options.prop_set);
	g_free(options.prop_value);
	g_free(options.rsc_id);
	g_free(options.rsc_type);
	free(options.agent_spec);
	free(options.v_agent);
	free(options.v_class);
	free(options.v_provider);
	g_free(options.xml_file);
	g_strfreev(options.remainder);

	if (options.override_params != NULL) {
	g_hash_table_destroy(options.override_params);
	}

	/* options.cmdline_params does not need to be destroyed here. See the
	* comments in cli_resource_execute_from_params.
	*/

	g_strfreev(processed_args);
	g_option_context_free(context);

	return bye(exit_code);
	}
	diff --git a/tools/crm_resource.h b/tools/crm_resource.h
	index 039d2c9651..71a978893b 100644
	--- a/tools/crm_resource.h
	+++ b/tools/crm_resource.h
	@@ -1,113 +1,113 @@
	/*
	- * Copyright 2004-2021 the Pacemaker project contributors
	+ * Copyright 2004-2022 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>
	#include <crm/crm.h>

	#include <crm/msg_xml.h>
	#include <crm/services.h>
	#include <crm/common/xml.h>
	#include <crm/common/mainloop.h>
	#include <crm/common/output_internal.h>

	#include <crm/cib.h>
	#include <crm/common/attrd_internal.h>
	#include <crm/pengine/rules.h>
	#include <crm/pengine/status.h>
	#include <crm/pengine/internal.h>
	#include <pacemaker-internal.h>

	typedef struct node_info_s {
	const char *node_name;
	bool promoted;
	} node_info_t;

	enum resource_check_flags {
	rsc_remain_stopped = (1 << 0),
	rsc_unpromotable = (1 << 1),
	rsc_unmanaged = (1 << 2)
	};

	typedef struct resource_checks_s {
	pe_resource_t *rsc;
	unsigned int flags;
	const char *lock_node;
	} resource_checks_t;

	resource_checks_t cli_check_resource(pe_resource_t rsc, char role_s, char managed);

	/* ban */
	int cli_resource_prefer(pcmk__output_t out, const char rsc_id, const char *host,
	const char move_lifetime, cib_t cib_conn, int cib_options,
	gboolean promoted_role_only);
	int cli_resource_ban(pcmk__output_t out, const char rsc_id, const char *host,
	const char move_lifetime, GList allnodes, cib_t * cib_conn,
	int cib_options, gboolean promoted_role_only);
	int cli_resource_clear(const char rsc_id, const char host, GList *allnodes,
	cib_t * cib_conn, int cib_options, bool clear_ban_constraints, gboolean force);
	int cli_resource_clear_all_expired(xmlNode root, cib_t cib_conn, int cib_options,
	const char rsc, const char node, gboolean promoted_role_only);

	/* print */
	void cli_resource_print_cts(pe_resource_t * rsc, pcmk__output_t *out);
	void cli_resource_print_cts_constraints(pe_working_set_t * data_set);

	int cli_resource_print(pe_resource_t rsc, pe_working_set_t data_set, bool expanded);
	int cli_resource_print_operations(const char rsc_id, const char host_uname,
	bool active, pe_working_set_t * data_set);

	/* runtime */
	int cli_resource_check(pcmk__output_t out, cib_t cib, pe_resource_t *rsc);
	int cli_resource_fail(pcmk_ipc_api_t controld_api, const char host_uname,
	const char rsc_id, pe_working_set_t data_set);
	GList cli_resource_search(pe_resource_t rsc, const char *requested_name,
	pe_working_set_t *data_set);
	int cli_resource_delete(pcmk_ipc_api_t controld_api, const char host_uname,
	pe_resource_t rsc, const char operation,
	const char *interval_spec, bool just_failures,
	pe_working_set_t *data_set, gboolean force);
	int cli_cleanup_all(pcmk_ipc_api_t controld_api, const char node_name,
	const char operation, const char interval_spec,
	pe_working_set_t *data_set);
	-int cli_resource_restart(pcmk__output_t out, pe_resource_t rsc, const char *host,
	+int cli_resource_restart(pcmk__output_t out, pe_resource_t rsc, pe_node_t *node,
	const char move_lifetime, int timeout_ms, cib_t cib,
	int cib_options, gboolean promoted_role_only, gboolean force);
	int cli_resource_move(pe_resource_t rsc, const char rsc_id, const char *host_name,
	const char move_lifetime, cib_t cib, int cib_options,
	pe_working_set_t *data_set, gboolean promoted_role_only,
	gboolean force);
	crm_exit_t cli_resource_execute_from_params(pcmk__output_t out, const char rsc_name,
	const char rsc_class, const char rsc_prov,
	const char rsc_type, const char rsc_action,
	GHashTable params, GHashTable override_hash,
	int timeout_ms, int resource_verbose,
	gboolean force, int check_level);
	crm_exit_t cli_resource_execute(pe_resource_t rsc, const char requested_name,
	const char rsc_action, GHashTable override_hash,
	int timeout_ms, cib_t cib, pe_working_set_t data_set,
	int resource_verbose, gboolean force, int check_level);

	int cli_resource_update_attribute(pe_resource_t rsc, const char requested_name,
	const char attr_set, const char attr_set_type,
	const char attr_id, const char attr_name,
	const char *attr_value, gboolean recursive,
	cib_t *cib, int cib_options,
	pe_working_set_t *data_set, gboolean force);
	int cli_resource_delete_attribute(pe_resource_t rsc, const char requested_name,
	const char attr_set, const char attr_set_type,
	const char attr_id, const char attr_name,
	cib_t *cib, int cib_options,
	pe_working_set_t *data_set, gboolean force);

	int update_working_set_xml(pe_working_set_t data_set, xmlNode *xml);
	int wait_till_stable(pcmk__output_t out, int timeout_ms, cib_t cib);

	bool resource_is_running_on(pe_resource_t rsc, const char host);

	void crm_resource_register_messages(pcmk__output_t *out);
	diff --git a/tools/crm_resource_runtime.c b/tools/crm_resource_runtime.c
	index 01c4aa0f84..8e120f3342 100644
	--- a/tools/crm_resource_runtime.c
	+++ b/tools/crm_resource_runtime.c
	@@ -1,2009 +1,2050 @@
	/*
	* Copyright 2004-2022 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <crm_resource.h>
	#include <crm/common/ipc_controld.h>
	#include <crm/common/lists_internal.h>
	#include <crm/services_internal.h>

	resource_checks_t *
	cli_check_resource(pe_resource_t rsc, char role_s, char *managed)
	{
	pe_resource_t *parent = uber_parent(rsc);
	resource_checks_t *rc = calloc(1, sizeof(resource_checks_t));

	if (role_s) {
	enum rsc_role_e role = text2role(role_s);

	if (role == RSC_ROLE_STOPPED) {
	rc->flags \|= rsc_remain_stopped;
	} else if (pcmk_is_set(parent->flags, pe_rsc_promotable) &&
	(role == RSC_ROLE_UNPROMOTED)) {
	rc->flags \|= rsc_unpromotable;
	}
	}

	if (managed && !crm_is_true(managed)) {
	rc->flags \|= rsc_unmanaged;
	}

	if (rsc->lock_node) {
	rc->lock_node = rsc->lock_node->details->uname;
	}

	rc->rsc = rsc;
	return rc;
	}

	static GList *
	build_node_info_list(pe_resource_t *rsc)
	{
	GList *retval = NULL;

	for (GList *iter = rsc->children; iter != NULL; iter = iter->next) {
	pe_resource_t child = (pe_resource_t ) iter->data;

	for (GList *iter2 = child->running_on; iter2 != NULL; iter2 = iter2->next) {
	pe_node_t node = (pe_node_t ) iter2->data;
	node_info_t *ni = calloc(1, sizeof(node_info_t));
	ni->node_name = node->details->uname;
	ni->promoted = pcmk_is_set(rsc->flags, pe_rsc_promotable) &&
	child->fns->state(child, TRUE) == RSC_ROLE_PROMOTED;

	retval = g_list_prepend(retval, ni);
	}
	}

	return retval;
	}

	GList *
	cli_resource_search(pe_resource_t rsc, const char requested_name,
	pe_working_set_t *data_set)
	{
	GList *retval = NULL;
	pe_resource_t *parent = uber_parent(rsc);

	if (pe_rsc_is_clone(rsc)) {
	retval = build_node_info_list(rsc);

	/* The anonymous clone children's common ID is supplied */
	} else if (pe_rsc_is_clone(parent)
	&& !pcmk_is_set(rsc->flags, pe_rsc_unique)
	&& rsc->clone_name
	&& pcmk__str_eq(requested_name, rsc->clone_name, pcmk__str_casei)
	&& !pcmk__str_eq(requested_name, rsc->id, pcmk__str_casei)) {

	retval = build_node_info_list(parent);

	} else if (rsc->running_on != NULL) {
	for (GList *iter = rsc->running_on; iter != NULL; iter = iter->next) {
	pe_node_t node = (pe_node_t ) iter->data;
	node_info_t *ni = calloc(1, sizeof(node_info_t));
	ni->node_name = node->details->uname;
	ni->promoted = (rsc->fns->state(rsc, TRUE) == RSC_ROLE_PROMOTED);

	retval = g_list_prepend(retval, ni);
	}
	}

	return retval;
	}

	#define XPATH_MAX 1024

	// \return Standard Pacemaker return code
	static int
	find_resource_attr(pcmk__output_t out, cib_t the_cib, const char *attr,
	const char rsc, const char attr_set_type, const char *set_name,
	const char attr_id, const char attr_name, char **value)
	{
	int offset = 0;
	int rc = pcmk_rc_ok;
	xmlNode *xml_search = NULL;
	char *xpath_string = NULL;
	const char *xpath_base = NULL;

	if(value) {
	*value = NULL;
	}

	if(the_cib == NULL) {
	return ENOTCONN;
	}

	xpath_base = pcmk_cib_xpath_for(XML_CIB_TAG_RESOURCES);
	if (xpath_base == NULL) {
	crm_err(XML_CIB_TAG_RESOURCES " CIB element not known (bug?)");
	return ENOMSG;
	}

	xpath_string = calloc(1, XPATH_MAX);
	offset += snprintf(xpath_string + offset, XPATH_MAX - offset, "%s",
	xpath_base);

	offset += snprintf(xpath_string + offset, XPATH_MAX - offset, "//*[@id=\"%s\"]", rsc);

	if (attr_set_type) {
	offset += snprintf(xpath_string + offset, XPATH_MAX - offset, "/%s", attr_set_type);
	if (set_name) {
	offset += snprintf(xpath_string + offset, XPATH_MAX - offset, "[@id=\"%s\"]", set_name);
	}
	}

	offset += snprintf(xpath_string + offset, XPATH_MAX - offset, "//nvpair[");
	if (attr_id) {
	offset += snprintf(xpath_string + offset, XPATH_MAX - offset, "@id=\"%s\"", attr_id);
	}

	if (attr_name) {
	if (attr_id) {
	offset += snprintf(xpath_string + offset, XPATH_MAX - offset, " and ");
	}
	offset += snprintf(xpath_string + offset, XPATH_MAX - offset, "@name=\"%s\"", attr_name);
	}
	offset += snprintf(xpath_string + offset, XPATH_MAX - offset, "]");
	CRM_LOG_ASSERT(offset > 0);

	rc = the_cib->cmds->query(the_cib, xpath_string, &xml_search,
	cib_sync_call \| cib_scope_local \| cib_xpath);
	rc = pcmk_legacy2rc(rc);

	if (rc != pcmk_rc_ok) {
	goto done;
	}

	crm_log_xml_debug(xml_search, "Match");
	if (xml_has_children(xml_search)) {
	xmlNode *child = NULL;

	rc = ENOTUNIQ;
	out->info(out, "Multiple attributes match name=%s", attr_name);

	for (child = pcmk__xml_first_child(xml_search); child != NULL;
	child = pcmk__xml_next(child)) {
	out->info(out, " Value: %s \t(id=%s)",
	crm_element_value(child, XML_NVPAIR_ATTR_VALUE), ID(child));
	}

	out->spacer(out);

	} else if(value) {
	pcmk__str_update(value, crm_element_value(xml_search, attr));
	}

	done:
	free(xpath_string);
	free_xml(xml_search);
	return rc;
	}

	/* PRIVATE. Use the find_matching_attr_resources instead. */
	static void
	find_matching_attr_resources_recursive(pcmk__output_t out, GList/ <pe_resource_t> / ** result,
	pe_resource_t * rsc, const char * rsc_id,
	const char * attr_set, const char * attr_set_type,
	const char * attr_id, const char * attr_name,
	cib_t * cib, const char * cmd, int depth)
	{
	int rc = pcmk_rc_ok;
	char *lookup_id = clone_strip(rsc->id);
	char *local_attr_id = NULL;

	/* visit the children */
	for(GList *gIter = rsc->children; gIter; gIter = gIter->next) {
	find_matching_attr_resources_recursive(out, result, (pe_resource_t*)gIter->data,
	rsc_id, attr_set, attr_set_type,
	attr_id, attr_name, cib, cmd, depth+1);
	/* do it only once for clones */
	if(pe_clone == rsc->variant) {
	break;
	}
	}

	rc = find_resource_attr(out, cib, XML_ATTR_ID, lookup_id, attr_set_type,
	attr_set, attr_id, attr_name, &local_attr_id);
	/* Post-order traversal.
	* The root is always on the list and it is the last item. */
	if((0 == depth) \|\| (pcmk_rc_ok == rc)) {
	/* push the head */
	result = g_list_append(result, rsc);
	}

	free(local_attr_id);
	free(lookup_id);
	}


	/* The result is a linearized pre-ordered tree of resources. */
	static GList/<pe_resource_t>/
	find_matching_attr_resources(pcmk__output_t out, pe_resource_t rsc,
	const char * rsc_id, const char * attr_set,
	const char * attr_set_type, const char * attr_id,
	const char * attr_name, cib_t * cib, const char * cmd,
	gboolean force)
	{
	int rc = pcmk_rc_ok;
	char *lookup_id = NULL;
	char *local_attr_id = NULL;
	GList * result = NULL;
	/* If --force is used, update only the requested resource (clone or primitive).
	* Otherwise, if the primitive has the attribute, use that.
	* Otherwise use the clone. */
	if(force == TRUE) {
	return g_list_append(result, rsc);
	}
	if(rsc->parent && pe_clone == rsc->parent->variant) {
	int rc = pcmk_rc_ok;
	char *local_attr_id = NULL;
	rc = find_resource_attr(out, cib, XML_ATTR_ID, rsc_id, attr_set_type,
	attr_set, attr_id, attr_name, &local_attr_id);
	free(local_attr_id);

	if(rc != pcmk_rc_ok) {
	rsc = rsc->parent;
	out->info(out, "Performing %s of '%s' on '%s', the parent of '%s'",
	cmd, attr_name, rsc->id, rsc_id);
	}
	return g_list_append(result, rsc);
	} else if(rsc->parent == NULL && rsc->children && pe_clone == rsc->variant) {
	pe_resource_t *child = rsc->children->data;

	if(child->variant == pe_native) {
	lookup_id = clone_strip(child->id); /* Could be a cloned group! */
	rc = find_resource_attr(out, cib, XML_ATTR_ID, lookup_id, attr_set_type,
	attr_set, attr_id, attr_name, &local_attr_id);

	if(rc == pcmk_rc_ok) {
	rsc = child;
	out->info(out, "A value for '%s' already exists in child '%s', performing %s on that instead of '%s'",
	attr_name, lookup_id, cmd, rsc_id);
	}

	free(local_attr_id);
	free(lookup_id);
	}
	return g_list_append(result, rsc);
	}
	/* If the resource is a group ==> children inherit the attribute if defined. */
	find_matching_attr_resources_recursive(out, &result, rsc, rsc_id, attr_set,
	attr_set_type, attr_id, attr_name,
	cib, cmd, 0);
	return result;
	}

	// \return Standard Pacemaker return code
	int
	cli_resource_update_attribute(pe_resource_t rsc, const char requested_name,
	const char attr_set, const char attr_set_type,
	const char attr_id, const char attr_name,
	const char *attr_value, gboolean recursive,
	cib_t *cib, int cib_options,
	pe_working_set_t *data_set, gboolean force)
	{
	pcmk__output_t *out = data_set->priv;
	int rc = pcmk_rc_ok;
	- static bool need_init = TRUE;
	+ static bool need_init = true;

	char *local_attr_id = NULL;
	char *local_attr_set = NULL;

	GList/<pe_resource_t>/ resources = NULL;
	const char *common_attr_id = attr_id;

	if (attr_id == NULL && force == FALSE) {
	find_resource_attr (out, cib, XML_ATTR_ID, uber_parent(rsc)->id, NULL,
	NULL, NULL, attr_name, NULL);
	}

	if (pcmk__str_eq(attr_set_type, XML_TAG_ATTR_SETS, pcmk__str_casei)) {
	if (force == FALSE) {
	rc = find_resource_attr(out, cib, XML_ATTR_ID, uber_parent(rsc)->id,
	XML_TAG_META_SETS, attr_set, attr_id,
	attr_name, &local_attr_id);
	if (rc == pcmk_rc_ok && !out->is_quiet(out)) {
	out->err(out, "WARNING: There is already a meta attribute for '%s' called '%s' (id=%s)",
	uber_parent(rsc)->id, attr_name, local_attr_id);
	out->err(out, " Delete '%s' first or use the force option to override",
	local_attr_id);
	}
	free(local_attr_id);
	if (rc == pcmk_rc_ok) {
	return ENOTUNIQ;
	}
	}
	resources = g_list_append(resources, rsc);

	} else {
	resources = find_matching_attr_resources(out, rsc, requested_name, attr_set, attr_set_type,
	attr_id, attr_name, cib, "update", force);
	}

	/* If either attr_set or attr_id is specified,
	* one clearly intends to modify a single resource.
	* It is the last item on the resource list.*/
	for(GList *gIter = (attr_set\|\|attr_id) ? g_list_last(resources) : resources
	; gIter; gIter = gIter->next) {
	char *lookup_id = NULL;

	xmlNode *xml_top = NULL;
	xmlNode *xml_obj = NULL;
	local_attr_id = NULL;
	local_attr_set = NULL;

	rsc = (pe_resource_t*)gIter->data;
	attr_id = common_attr_id;

	lookup_id = clone_strip(rsc->id); /* Could be a cloned group! */
	rc = find_resource_attr(out, cib, XML_ATTR_ID, lookup_id, attr_set_type,
	attr_set, attr_id, attr_name, &local_attr_id);

	if (rc == pcmk_rc_ok) {
	crm_debug("Found a match for name=%s: id=%s", attr_name, local_attr_id);
	attr_id = local_attr_id;

	} else if (rc != ENXIO) {
	free(lookup_id);
	free(local_attr_id);
	g_list_free(resources);
	return rc;

	} else {
	const char *tag = crm_element_name(rsc->xml);

	if (attr_set == NULL) {
	local_attr_set = crm_strdup_printf("%s-%s", lookup_id,
	attr_set_type);
	attr_set = local_attr_set;
	}
	if (attr_id == NULL) {
	local_attr_id = crm_strdup_printf("%s-%s", attr_set, attr_name);
	attr_id = local_attr_id;
	}

	xml_top = create_xml_node(NULL, tag);
	crm_xml_add(xml_top, XML_ATTR_ID, lookup_id);

	xml_obj = create_xml_node(xml_top, attr_set_type);
	crm_xml_add(xml_obj, XML_ATTR_ID, attr_set);
	}

	xml_obj = crm_create_nvpair_xml(xml_obj, attr_id, attr_name, attr_value);
	if (xml_top == NULL) {
	xml_top = xml_obj;
	}

	crm_log_xml_debug(xml_top, "Update");

	rc = cib->cmds->modify(cib, XML_CIB_TAG_RESOURCES, xml_top, cib_options);
	rc = pcmk_legacy2rc(rc);

	if (rc == pcmk_rc_ok) {
	out->info(out, "Set '%s' option: id=%s%s%s%s%s value=%s", lookup_id, local_attr_id,
	attr_set ? " set=" : "", attr_set ? attr_set : "",
	attr_name ? " name=" : "", attr_name ? attr_name : "", attr_value);
	}

	free_xml(xml_top);

	free(lookup_id);
	free(local_attr_id);
	free(local_attr_set);

	if(recursive && pcmk__str_eq(attr_set_type, XML_TAG_META_SETS, pcmk__str_casei)) {
	GList *lpc = NULL;

	if(need_init) {
	- need_init = FALSE;
	+ need_init = false;
	pcmk__unpack_constraints(data_set);
	pe__clear_resource_flags_on_all(data_set, pe_rsc_allocating);
	}

	crm_debug("Looking for dependencies %p", rsc->rsc_cons_lhs);
	pe__set_resource_flags(rsc, pe_rsc_allocating);
	for (lpc = rsc->rsc_cons_lhs; lpc != NULL; lpc = lpc->next) {
	pcmk__colocation_t cons = (pcmk__colocation_t ) lpc->data;

	crm_debug("Checking %s %d", cons->id, cons->score);
	if ((cons->score > 0)
	&& !pcmk_is_set(cons->dependent->flags, pe_rsc_allocating)) {
	/* Don't get into colocation loops */
	crm_debug("Setting %s=%s for dependent resource %s",
	attr_name, attr_value, cons->dependent->id);
	cli_resource_update_attribute(cons->dependent,
	cons->dependent->id, NULL,
	attr_set_type, NULL,
	attr_name, attr_value,
	recursive, cib, cib_options,
	data_set, force);
	}
	}
	}
	}
	g_list_free(resources);
	return rc;
	}

	// \return Standard Pacemaker return code
	int
	cli_resource_delete_attribute(pe_resource_t rsc, const char requested_name,
	const char attr_set, const char attr_set_type,
	const char attr_id, const char attr_name,
	cib_t *cib, int cib_options,
	pe_working_set_t *data_set, gboolean force)
	{
	pcmk__output_t *out = data_set->priv;
	int rc = pcmk_rc_ok;
	GList/<pe_resource_t>/ resources = NULL;

	if (attr_id == NULL && force == FALSE) {
	find_resource_attr(out, cib, XML_ATTR_ID, uber_parent(rsc)->id, NULL,
	NULL, NULL, attr_name, NULL);
	}

	if(pcmk__str_eq(attr_set_type, XML_TAG_META_SETS, pcmk__str_casei)) {
	resources = find_matching_attr_resources(out, rsc, requested_name, attr_set, attr_set_type,
	attr_id, attr_name, cib, "delete", force);
	} else {
	resources = g_list_append(resources, rsc);
	}

	for(GList *gIter = resources; gIter; gIter = gIter->next) {
	char *lookup_id = NULL;
	xmlNode *xml_obj = NULL;
	char *local_attr_id = NULL;

	rsc = (pe_resource_t*)gIter->data;

	lookup_id = clone_strip(rsc->id);
	rc = find_resource_attr(out, cib, XML_ATTR_ID, lookup_id, attr_set_type,
	attr_set, attr_id, attr_name, &local_attr_id);

	if (rc == ENXIO) {
	free(lookup_id);
	rc = pcmk_rc_ok;
	continue;

	} else if (rc != pcmk_rc_ok) {
	free(lookup_id);
	g_list_free(resources);
	return rc;
	}

	if (attr_id == NULL) {
	attr_id = local_attr_id;
	}

	xml_obj = crm_create_nvpair_xml(NULL, attr_id, attr_name, NULL);
	crm_log_xml_debug(xml_obj, "Delete");

	CRM_ASSERT(cib);
	rc = cib->cmds->remove(cib, XML_CIB_TAG_RESOURCES, xml_obj, cib_options);
	rc = pcmk_legacy2rc(rc);

	if (rc == pcmk_rc_ok) {
	out->info(out, "Deleted '%s' option: id=%s%s%s%s%s", lookup_id, local_attr_id,
	attr_set ? " set=" : "", attr_set ? attr_set : "",
	attr_name ? " name=" : "", attr_name ? attr_name : "");
	}

	free(lookup_id);
	free_xml(xml_obj);
	free(local_attr_id);
	}
	g_list_free(resources);
	return rc;
	}

	// \return Standard Pacemaker return code
	static int
	send_lrm_rsc_op(pcmk_ipc_api_t *controld_api, bool do_fail_resource,
	const char host_uname, const char rsc_id, pe_working_set_t *data_set)
	{
	pcmk__output_t *out = data_set->priv;
	const char *router_node = host_uname;
	const char *rsc_api_id = NULL;
	const char *rsc_long_id = NULL;
	const char *rsc_class = NULL;
	const char *rsc_provider = NULL;
	const char *rsc_type = NULL;
	bool cib_only = false;
	pe_resource_t *rsc = pe_find_resource(data_set->resources, rsc_id);

	if (rsc == NULL) {
	out->err(out, "Resource %s not found", rsc_id);
	return ENXIO;

	} else if (rsc->variant != pe_native) {
	out->err(out, "We can only process primitive resources, not %s", rsc_id);
	return EINVAL;
	}

	rsc_class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
	rsc_provider = crm_element_value(rsc->xml, XML_AGENT_ATTR_PROVIDER),
	rsc_type = crm_element_value(rsc->xml, XML_ATTR_TYPE);
	if ((rsc_class == NULL) \|\| (rsc_type == NULL)) {
	out->err(out, "Resource %s does not have a class and type", rsc_id);
	return EINVAL;
	}

	{
	pe_node_t *node = pe_find_node(data_set->nodes, host_uname);

	if (node == NULL) {
	out->err(out, "Node %s not found", host_uname);
	return pcmk_rc_node_unknown;
	}

	if (!(node->details->online)) {
	if (do_fail_resource) {
	out->err(out, "Node %s is not online", host_uname);
	return ENOTCONN;
	} else {
	cib_only = true;
	}
	}
	if (!cib_only && pe__is_guest_or_remote_node(node)) {
	node = pe__current_node(node->details->remote_rsc);
	if (node == NULL) {
	out->err(out, "No cluster connection to Pacemaker Remote node %s detected",
	host_uname);
	return ENOTCONN;
	}
	router_node = node->details->uname;
	}
	}

	if (rsc->clone_name) {
	rsc_api_id = rsc->clone_name;
	rsc_long_id = rsc->id;
	} else {
	rsc_api_id = rsc->id;
	}
	if (do_fail_resource) {
	return pcmk_controld_api_fail(controld_api, host_uname, router_node,
	rsc_api_id, rsc_long_id,
	rsc_class, rsc_provider, rsc_type);
	} else {
	return pcmk_controld_api_refresh(controld_api, host_uname, router_node,
	rsc_api_id, rsc_long_id, rsc_class,
	rsc_provider, rsc_type, cib_only);
	}
	}

	/*!
	* \internal
	* \brief Get resource name as used in failure-related node attributes
	*
	* \param[in] rsc Resource to check
	*
	* \return Newly allocated string containing resource's fail name
	* \note The caller is responsible for freeing the result.
	*/
	static inline char *
	rsc_fail_name(pe_resource_t *rsc)
	{
	const char *name = (rsc->clone_name? rsc->clone_name : rsc->id);

	return pcmk_is_set(rsc->flags, pe_rsc_unique)? strdup(name) : clone_strip(name);
	}

	// \return Standard Pacemaker return code
	static int
	clear_rsc_history(pcmk_ipc_api_t controld_api, const char host_uname,
	const char rsc_id, pe_working_set_t data_set)
	{
	int rc = pcmk_rc_ok;

	/* Erase the resource's entire LRM history in the CIB, even if we're only
	* clearing a single operation's fail count. If we erased only entries for a
	* single operation, we might wind up with a wrong idea of the current
	* resource state, and we might not re-probe the resource.
	*/
	rc = send_lrm_rsc_op(controld_api, false, host_uname, rsc_id, data_set);
	if (rc != pcmk_rc_ok) {
	return rc;
	}

	crm_trace("Processing %d mainloop inputs",
	pcmk_controld_api_replies_expected(controld_api));
	while (g_main_context_iteration(NULL, FALSE)) {
	crm_trace("Processed mainloop input, %d still remaining",
	pcmk_controld_api_replies_expected(controld_api));
	}
	return rc;
	}

	// \return Standard Pacemaker return code
	static int
	clear_rsc_failures(pcmk__output_t out, pcmk_ipc_api_t controld_api,
	const char node_name, const char rsc_id, const char *operation,
	const char interval_spec, pe_working_set_t data_set)
	{
	int rc = pcmk_rc_ok;
	const char *failed_value = NULL;
	const char *failed_id = NULL;
	const char *interval_ms_s = NULL;
	GHashTable *rscs = NULL;
	GHashTableIter iter;

	/* Create a hash table to use as a set of resources to clean. This lets us
	* clean each resource only once (per node) regardless of how many failed
	* operations it has.
	*/
	rscs = pcmk__strkey_table(NULL, NULL);

	// Normalize interval to milliseconds for comparison to history entry
	if (operation) {
	interval_ms_s = crm_strdup_printf("%u",
	crm_parse_interval_spec(interval_spec));
	}

	for (xmlNode *xml_op = pcmk__xml_first_child(data_set->failed);
	xml_op != NULL;
	xml_op = pcmk__xml_next(xml_op)) {

	failed_id = crm_element_value(xml_op, XML_LRM_ATTR_RSCID);
	if (failed_id == NULL) {
	// Malformed history entry, should never happen
	continue;
	}

	// No resource specified means all resources match
	if (rsc_id) {
	pe_resource_t *fail_rsc = pe_find_resource_with_flags(data_set->resources,
	failed_id,
	pe_find_renamed\|pe_find_anon);

	if (!fail_rsc \|\| !pcmk__str_eq(rsc_id, fail_rsc->id, pcmk__str_casei)) {
	continue;
	}
	}

	// Host name should always have been provided by this point
	failed_value = crm_element_value(xml_op, XML_ATTR_UNAME);
	if (!pcmk__str_eq(node_name, failed_value, pcmk__str_casei)) {
	continue;
	}

	// No operation specified means all operations match
	if (operation) {
	failed_value = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
	if (!pcmk__str_eq(operation, failed_value, pcmk__str_casei)) {
	continue;
	}

	// Interval (if operation was specified) defaults to 0 (not all)
	failed_value = crm_element_value(xml_op, XML_LRM_ATTR_INTERVAL_MS);
	if (!pcmk__str_eq(interval_ms_s, failed_value, pcmk__str_casei)) {
	continue;
	}
	}

	g_hash_table_add(rscs, (gpointer) failed_id);
	}

	g_hash_table_iter_init(&iter, rscs);
	while (g_hash_table_iter_next(&iter, (gpointer *) &failed_id, NULL)) {
	crm_debug("Erasing failures of %s on %s", failed_id, node_name);
	rc = clear_rsc_history(controld_api, node_name, failed_id, data_set);
	if (rc != pcmk_rc_ok) {
	return rc;
	}
	}
	g_hash_table_destroy(rscs);
	return rc;
	}

	// \return Standard Pacemaker return code
	static int
	clear_rsc_fail_attrs(pe_resource_t rsc, const char operation,
	const char interval_spec, pe_node_t node)
	{
	int rc = pcmk_rc_ok;
	int attr_options = pcmk__node_attr_none;
	char *rsc_name = rsc_fail_name(rsc);

	if (pe__is_guest_or_remote_node(node)) {
	attr_options \|= pcmk__node_attr_remote;
	}
	rc = pcmk__node_attr_request_clear(NULL, node->details->uname, rsc_name,
	operation, interval_spec, NULL,
	attr_options);
	free(rsc_name);
	return rc;
	}

	// \return Standard Pacemaker return code
	int
	cli_resource_delete(pcmk_ipc_api_t controld_api, const char host_uname,
	pe_resource_t rsc, const char operation,
	const char *interval_spec, bool just_failures,
	pe_working_set_t *data_set, gboolean force)
	{
	pcmk__output_t *out = data_set->priv;
	int rc = pcmk_rc_ok;
	pe_node_t *node = NULL;

	if (rsc == NULL) {
	return ENXIO;

	} else if (rsc->children) {
	GList *lpc = NULL;

	for (lpc = rsc->children; lpc != NULL; lpc = lpc->next) {
	pe_resource_t child = (pe_resource_t ) lpc->data;

	rc = cli_resource_delete(controld_api, host_uname, child, operation,
	interval_spec, just_failures, data_set,
	force);
	if (rc != pcmk_rc_ok) {
	return rc;
	}
	}
	return pcmk_rc_ok;

	} else if (host_uname == NULL) {
	GList *lpc = NULL;
	GList *nodes = g_hash_table_get_values(rsc->known_on);

	if(nodes == NULL && force) {
	nodes = pcmk__copy_node_list(data_set->nodes, false);

	} else if(nodes == NULL && rsc->exclusive_discover) {
	GHashTableIter iter;
	pe_node_t *node = NULL;

	g_hash_table_iter_init(&iter, rsc->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void**)&node)) {
	if(node->weight >= 0) {
	nodes = g_list_prepend(nodes, node);
	}
	}

	} else if(nodes == NULL) {
	nodes = g_hash_table_get_values(rsc->allowed_nodes);
	}

	for (lpc = nodes; lpc != NULL; lpc = lpc->next) {
	node = (pe_node_t *) lpc->data;

	if (node->details->online) {
	rc = cli_resource_delete(controld_api, node->details->uname,
	rsc, operation, interval_spec,
	just_failures, data_set, force);
	}
	if (rc != pcmk_rc_ok) {
	g_list_free(nodes);
	return rc;
	}
	}

	g_list_free(nodes);
	return pcmk_rc_ok;
	}

	node = pe_find_node(data_set->nodes, host_uname);

	if (node == NULL) {
	out->err(out, "Unable to clean up %s because node %s not found",
	rsc->id, host_uname);
	return ENODEV;
	}

	if (!node->details->rsc_discovery_enabled) {
	out->err(out, "Unable to clean up %s because resource discovery disabled on %s",
	rsc->id, host_uname);
	return EOPNOTSUPP;
	}

	if (controld_api == NULL) {
	out->err(out, "Dry run: skipping clean-up of %s on %s due to CIB_file",
	rsc->id, host_uname);
	return pcmk_rc_ok;
	}

	rc = clear_rsc_fail_attrs(rsc, operation, interval_spec, node);
	if (rc != pcmk_rc_ok) {
	out->err(out, "Unable to clean up %s failures on %s: %s",
	rsc->id, host_uname, pcmk_rc_str(rc));
	return rc;
	}

	if (just_failures) {
	rc = clear_rsc_failures(out, controld_api, host_uname, rsc->id, operation,
	interval_spec, data_set);
	} else {
	rc = clear_rsc_history(controld_api, host_uname, rsc->id, data_set);
	}
	if (rc != pcmk_rc_ok) {
	out->err(out, "Cleaned %s failures on %s, but unable to clean history: %s",
	rsc->id, host_uname, pcmk_strerror(rc));
	} else {
	out->info(out, "Cleaned up %s on %s", rsc->id, host_uname);
	}
	return rc;
	}

	// \return Standard Pacemaker return code
	int
	cli_cleanup_all(pcmk_ipc_api_t controld_api, const char node_name,
	const char operation, const char interval_spec,
	pe_working_set_t *data_set)
	{
	pcmk__output_t *out = data_set->priv;
	int rc = pcmk_rc_ok;
	int attr_options = pcmk__node_attr_none;
	const char *display_name = node_name? node_name : "all nodes";

	if (controld_api == NULL) {
	out->info(out, "Dry run: skipping clean-up of %s due to CIB_file",
	display_name);
	return rc;
	}

	if (node_name) {
	pe_node_t *node = pe_find_node(data_set->nodes, node_name);

	if (node == NULL) {
	out->err(out, "Unknown node: %s", node_name);
	return ENXIO;
	}
	if (pe__is_guest_or_remote_node(node)) {
	attr_options \|= pcmk__node_attr_remote;
	}
	}

	rc = pcmk__node_attr_request_clear(NULL, node_name, NULL, operation,
	interval_spec, NULL, attr_options);
	if (rc != pcmk_rc_ok) {
	out->err(out, "Unable to clean up all failures on %s: %s",
	display_name, pcmk_rc_str(rc));
	return rc;
	}

	if (node_name) {
	rc = clear_rsc_failures(out, controld_api, node_name, NULL,
	operation, interval_spec, data_set);
	if (rc != pcmk_rc_ok) {
	out->err(out, "Cleaned all resource failures on %s, but unable to clean history: %s",
	node_name, pcmk_strerror(rc));
	return rc;
	}
	} else {
	for (GList *iter = data_set->nodes; iter; iter = iter->next) {
	pe_node_t node = (pe_node_t ) iter->data;

	rc = clear_rsc_failures(out, controld_api, node->details->uname, NULL,
	operation, interval_spec, data_set);
	if (rc != pcmk_rc_ok) {
	out->err(out, "Cleaned all resource failures on all nodes, but unable to clean history: %s",
	pcmk_strerror(rc));
	return rc;
	}
	}
	}

	out->info(out, "Cleaned up all resources on %s", display_name);
	return rc;
	}

	int
	cli_resource_check(pcmk__output_t out, cib_t cib_conn, pe_resource_t *rsc)
	{
	char *role_s = NULL;
	char *managed = NULL;
	pe_resource_t *parent = uber_parent(rsc);
	int rc = pcmk_rc_no_output;
	resource_checks_t *checks = NULL;

	find_resource_attr(out, cib_conn, XML_NVPAIR_ATTR_VALUE, parent->id,
	NULL, NULL, NULL, XML_RSC_ATTR_MANAGED, &managed);

	find_resource_attr(out, cib_conn, XML_NVPAIR_ATTR_VALUE, parent->id,
	NULL, NULL, NULL, XML_RSC_ATTR_TARGET_ROLE, &role_s);

	checks = cli_check_resource(rsc, role_s, managed);

	if (checks->flags != 0 \|\| checks->lock_node != NULL) {
	rc = out->message(out, "resource-check-list", checks);
	}

	free(role_s);
	free(managed);
	free(checks);
	return rc;
	}

	// \return Standard Pacemaker return code
	int
	cli_resource_fail(pcmk_ipc_api_t controld_api, const char host_uname,
	const char rsc_id, pe_working_set_t data_set)
	{
	crm_notice("Failing %s on %s", rsc_id, host_uname);
	return send_lrm_rsc_op(controld_api, true, host_uname, rsc_id, data_set);
	}

	static GHashTable *
	generate_resource_params(pe_resource_t rsc, pe_node_t node,
	pe_working_set_t *data_set)
	{
	GHashTable *params = NULL;
	GHashTable *meta = NULL;
	GHashTable *combined = NULL;
	GHashTableIter iter;
	char *key = NULL;
	char *value = NULL;

	combined = pcmk__strkey_table(free, free);

	params = pe_rsc_params(rsc, node, data_set);
	if (params != NULL) {
	g_hash_table_iter_init(&iter, params);
	while (g_hash_table_iter_next(&iter, (gpointer ) & key, (gpointer ) & value)) {
	g_hash_table_insert(combined, strdup(key), strdup(value));
	}
	}

	meta = pcmk__strkey_table(free, free);
	get_meta_attributes(meta, rsc, node, data_set);
	if (meta != NULL) {
	g_hash_table_iter_init(&iter, meta);
	while (g_hash_table_iter_next(&iter, (gpointer ) & key, (gpointer ) & value)) {
	char *crm_name = crm_meta_name(key);

	g_hash_table_insert(combined, crm_name, strdup(value));
	}
	g_hash_table_destroy(meta);
	}

	return combined;
	}

	bool resource_is_running_on(pe_resource_t rsc, const char host)
	{
	- bool found = TRUE;
	+ bool found = true;
	GList *hIter = NULL;
	GList *hosts = NULL;

	- if(rsc == NULL) {
	- return FALSE;
	+ if (rsc == NULL) {
	+ return false;
	}

	rsc->fns->location(rsc, &hosts, TRUE);
	for (hIter = hosts; host != NULL && hIter != NULL; hIter = hIter->next) {
	pe_node_t node = (pe_node_t ) hIter->data;

	- if(strcmp(host, node->details->uname) == 0) {
	- crm_trace("Resource %s is running on %s\n", rsc->id, host);
	- goto done;
	- } else if(strcmp(host, node->details->id) == 0) {
	+ if (pcmk__strcase_any_of(host, node->details->uname, node->details->id, NULL)) {
	crm_trace("Resource %s is running on %s\n", rsc->id, host);
	goto done;
	}
	}

	- if(host != NULL) {
	+ if (host != NULL) {
	crm_trace("Resource %s is not running on: %s\n", rsc->id, host);
	- found = FALSE;
	+ found = false;

	} else if(host == NULL && hosts == NULL) {
	crm_trace("Resource %s is not running\n", rsc->id);
	- found = FALSE;
	+ found = false;
	}

	done:
	g_list_free(hosts);
	return found;
	}

	/*!
	* \internal
	* \brief Create a list of all resources active on host from a given list
	*
	* \param[in] host Name of host to check whether resources are active
	* \param[in] rsc_list List of resources to check
	*
	* \return New list of resources from list that are active on host
	*/
	static GList *
	get_active_resources(const char host, GList rsc_list)
	{
	GList *rIter = NULL;
	GList *active = NULL;

	for (rIter = rsc_list; rIter != NULL; rIter = rIter->next) {
	pe_resource_t rsc = (pe_resource_t ) rIter->data;

	/* Expand groups to their members, because if we're restarting a member
	* other than the first, we can't otherwise tell which resources are
	* stopping and starting.
	*/
	if (rsc->variant == pe_group) {
	active = g_list_concat(active,
	get_active_resources(host, rsc->children));
	} else if (resource_is_running_on(rsc, host)) {
	active = g_list_append(active, strdup(rsc->id));
	}
	}
	return active;
	}

	static void dump_list(GList items, const char tag)
	{
	int lpc = 0;
	GList *item = NULL;

	for (item = items; item != NULL; item = item->next) {
	crm_trace("%s[%d]: %s", tag, lpc, (char*)item->data);
	lpc++;
	}
	}

	static void display_list(pcmk__output_t out, GList items, const char *tag)
	{
	GList *item = NULL;

	for (item = items; item != NULL; item = item->next) {
	out->info(out, "%s%s", tag, (const char *)item->data);
	}
	}

	/*!
	* \internal
	* \brief Upgrade XML to latest schema version and use it as working set input
	*
	* This also updates the working set timestamp to the current time.
	*
	* \param[in] data_set Working set instance to update
	* \param[in] xml XML to use as input
	*
	* \return Standard Pacemaker return code
	* \note On success, caller is responsible for freeing memory allocated for
	* data_set->now.
	* \todo This follows the example of other callers of cli_config_update()
	* and returns ENOKEY ("Required key not available") if that fails,
	* but perhaps pcmk_rc_schema_validation would be better in that case.
	*/
	int
	update_working_set_xml(pe_working_set_t data_set, xmlNode *xml)
	{
	if (cli_config_update(xml, NULL, FALSE) == FALSE) {
	return ENOKEY;
	}
	data_set->input = *xml;
	data_set->now = crm_time_new(NULL);
	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief Update a working set's XML input based on a CIB query
	*
	* \param[in] data_set Data set instance to initialize
	* \param[in] cib Connection to the CIB manager
	*
	* \return Standard Pacemaker return code
	* \note On success, caller is responsible for freeing memory allocated for
	* data_set->input and data_set->now.
	*/
	static int
	update_working_set_from_cib(pcmk__output_t out, pe_working_set_t data_set,
	cib_t *cib)
	{
	xmlNode *cib_xml_copy = NULL;
	int rc = pcmk_rc_ok;

	rc = cib->cmds->query(cib, NULL, &cib_xml_copy, cib_scope_local \| cib_sync_call);
	rc = pcmk_legacy2rc(rc);

	if (rc != pcmk_rc_ok) {
	out->err(out, "Could not obtain the current CIB: %s (%d)", pcmk_strerror(rc), rc);
	return rc;
	}
	rc = update_working_set_xml(data_set, &cib_xml_copy);
	if (rc != pcmk_rc_ok) {
	out->err(out, "Could not upgrade the current CIB XML");
	free_xml(cib_xml_copy);
	return rc;
	}

	return rc;
	}

	// \return Standard Pacemaker return code
	static int
	update_dataset(cib_t cib, pe_working_set_t data_set, bool simulate)
	{
	char *pid = NULL;
	char *shadow_file = NULL;
	cib_t *shadow_cib = NULL;
	int rc = pcmk_rc_ok;

	pcmk__output_t *out = data_set->priv;

	pe_reset_working_set(data_set);
	pe__set_working_set_flags(data_set, pe_flag_no_counts\|pe_flag_no_compat);
	rc = update_working_set_from_cib(out, data_set, cib);
	if (rc != pcmk_rc_ok) {
	return rc;
	}

	if(simulate) {
	bool prev_quiet = false;

	pid = pcmk__getpid_s();
	shadow_cib = cib_shadow_new(pid);
	shadow_file = get_shadow_file(pid);

	if (shadow_cib == NULL) {
	out->err(out, "Could not create shadow cib: '%s'", pid);
	rc = ENXIO;
	goto done;
	}

	rc = write_xml_file(data_set->input, shadow_file, FALSE);

	if (rc < 0) {
	out->err(out, "Could not populate shadow cib: %s (%d)", pcmk_strerror(rc), rc);
	goto done;
	}

	rc = shadow_cib->cmds->signon(shadow_cib, crm_system_name, cib_command);
	rc = pcmk_legacy2rc(rc);

	if (rc != pcmk_rc_ok) {
	out->err(out, "Could not connect to shadow cib: %s (%d)", pcmk_strerror(rc), rc);
	goto done;
	}

	pcmk__schedule_actions(data_set->input,
	pe_flag_no_counts\|pe_flag_no_compat, data_set);

	prev_quiet = out->is_quiet(out);
	out->quiet = true;
	pcmk__simulate_transition(data_set, shadow_cib, NULL);
	out->quiet = prev_quiet;

	- rc = update_dataset(shadow_cib, data_set, FALSE);
	+ rc = update_dataset(shadow_cib, data_set, false);

	} else {
	cluster_status(data_set);
	}

	done:
	/* Do not free data_set->input here, we need rsc->xml to be valid later on */
	cib_delete(shadow_cib);
	free(pid);

	if(shadow_file) {
	unlink(shadow_file);
	free(shadow_file);
	}

	return rc;
	}

	static int
	max_delay_for_resource(pe_working_set_t * data_set, pe_resource_t *rsc)
	{
	int delay = 0;
	int max_delay = 0;

	if(rsc && rsc->children) {
	GList *iter = NULL;

	for(iter = rsc->children; iter; iter = iter->next) {
	pe_resource_t child = (pe_resource_t )iter->data;

	delay = max_delay_for_resource(data_set, child);
	if(delay > max_delay) {
	double seconds = delay / 1000.0;
	crm_trace("Calculated new delay of %.1fs due to %s", seconds, child->id);
	max_delay = delay;
	}
	}

	} else if(rsc) {
	char *key = crm_strdup_printf("%s_%s_0", rsc->id, RSC_STOP);
	pe_action_t *stop = custom_action(rsc, key, RSC_STOP, NULL, TRUE, FALSE, data_set);
	const char *value = g_hash_table_lookup(stop->meta, XML_ATTR_TIMEOUT);
	long long result_ll;

	if ((pcmk__scan_ll(value, &result_ll, -1LL) == pcmk_rc_ok)
	&& (result_ll >= 0) && (result_ll <= INT_MAX)) {
	max_delay = (int) result_ll;
	} else {
	max_delay = -1;
	}
	pe_free_action(stop);
	}

	return max_delay;
	}

	static int
	max_delay_in(pe_working_set_t * data_set, GList *resources)
	{
	int max_delay = 0;
	GList *item = NULL;

	for (item = resources; item != NULL; item = item->next) {
	int delay = 0;
	pe_resource_t rsc = pe_find_resource(data_set->resources, (const char )item->data);

	delay = max_delay_for_resource(data_set, rsc);

	if(delay > max_delay) {
	double seconds = delay / 1000.0;
	crm_trace("Calculated new delay of %.1fs due to %s", seconds, rsc->id);
	max_delay = delay;
	}
	}

	return 5 + (max_delay / 1000);
	}

	#define waiting_for_starts(d, r, h) ((d != NULL) \|\| \
	(!resource_is_running_on((r), (h))))

	/*!
	* \internal
	* \brief Restart a resource (on a particular host if requested).
	*
	* \param[in] rsc The resource to restart
	* \param[in] host The host to restart the resource on (or NULL for all)
	* \param[in] timeout_ms Consider failed if actions do not complete in this time
	* (specified in milliseconds, but a two-second
	* granularity is actually used; if 0, a timeout will be
	* calculated based on the resource timeout)
	* \param[in] cib Connection to the CIB manager
	*
	* \return Standard Pacemaker return code (exits on certain failures)
	*/
	int
	-cli_resource_restart(pcmk__output_t out, pe_resource_t rsc, const char *host,
	+cli_resource_restart(pcmk__output_t out, pe_resource_t rsc, pe_node_t *node,
	const char move_lifetime, int timeout_ms, cib_t cib,
	int cib_options, gboolean promoted_role_only, gboolean force)
	{
	int rc = pcmk_rc_ok;
	int lpc = 0;
	int before = 0;
	int step_timeout_s = 0;
	int sleep_interval = 2;
	int timeout = timeout_ms / 1000;

	- bool stop_via_ban = FALSE;
	+ bool stop_via_ban = false;
	char *rsc_id = NULL;
	+ char *lookup_id = NULL;
	char *orig_target_role = NULL;

	GList *list_delta = NULL;
	GList *target_active = NULL;
	GList *current_active = NULL;
	GList *restart_target_active = NULL;

	pe_working_set_t *data_set = NULL;
	+ pe_resource_t *parent = uber_parent(rsc);
	+
	+ bool running = false;
	+ const char *id = rsc->clone_name ? rsc->clone_name : rsc->id;
	+ const char *host = node ? node->details->uname : NULL;
	+
	+ /* If the implicit resource or primitive resource of a bundle is given, operate on the
	+ * bundle itself instead.
	+ */
	+ if (pe_rsc_is_bundled(rsc)) {
	+ rsc = parent->parent;
	+ }
	+
	+ running = resource_is_running_on(rsc, host);
	+
	+ if (pe_rsc_is_clone(parent) && !running) {
	+ if (pe_rsc_is_unique_clone(parent)) {
	+ lookup_id = strdup(rsc->id);
	+ } else {
	+ lookup_id = clone_strip(rsc->id);
	+ }

	- if (!resource_is_running_on(rsc, host)) {
	- const char *id = rsc->clone_name?rsc->clone_name:rsc->id;
	- if(host) {
	+ rsc = parent->fns->find_rsc(parent, lookup_id, node, pe_find_any\|pe_find_current);
	+ free(lookup_id);
	+ running = resource_is_running_on(rsc, host);
	+ }
	+
	+ if (!running) {
	+ if (host) {
	out->err(out, "%s is not running on %s and so cannot be restarted", id, host);
	} else {
	out->err(out, "%s is not running anywhere and so cannot be restarted", id);
	}
	return ENXIO;
	}

	rsc_id = strdup(rsc->id);
	- if ((pe_rsc_is_clone(rsc) \|\| pe_bundle_replicas(rsc)) && host) {
	- stop_via_ban = TRUE;
	+
	+ if (pe_rsc_is_unique_clone(parent)) {
	+ lookup_id = strdup(rsc->id);
	+ } else {
	+ lookup_id = clone_strip(rsc->id);
	+ }
	+
	+ if (host) {
	+ if (pe_rsc_is_clone(rsc) \|\| pe_bundle_replicas(rsc)) {
	+ stop_via_ban = true;
	+ } else if (pe_rsc_is_clone(parent)) {
	+ stop_via_ban = true;
	+ free(lookup_id);
	+ lookup_id = strdup(parent->id);
	+ }
	}

	/*
	grab full cib
	determine originally active resources
	disable or ban
	poll cib and watch for affected resources to get stopped
	without --timeout, calculate the stop timeout for each step and wait for that
	if we hit --timeout or the service timeout, re-enable or un-ban, report failure and indicate which resources we couldn't take down
	if everything stopped, re-enable or un-ban
	poll cib and watch for affected resources to get started
	without --timeout, calculate the start timeout for each step and wait for that
	if we hit --timeout or the service timeout, report (different) failure and indicate which resources we couldn't bring back up
	report success

	Optimizations:
	- use constraints to determine ordered list of affected resources
	- Allow a --no-deps option (aka. --force-restart)
	*/

	data_set = pe_new_working_set();
	if (data_set == NULL) {
	crm_perror(LOG_ERR, "Could not allocate working set");
	rc = ENOMEM;
	goto done;
	}

	data_set->priv = out;
	- rc = update_dataset(cib, data_set, FALSE);
	+ rc = update_dataset(cib, data_set, false);

	if(rc != pcmk_rc_ok) {
	out->err(out, "Could not get new resource list: %s (%d)", pcmk_strerror(rc), rc);
	goto done;
	}

	restart_target_active = get_active_resources(host, data_set->resources);
	current_active = get_active_resources(host, data_set->resources);

	dump_list(current_active, "Origin");

	if (stop_via_ban) {
	/* Stop the clone or bundle instance by banning it from the host */
	out->quiet = true;
	- rc = cli_resource_ban(out, rsc_id, host, move_lifetime, NULL, cib,
	+ rc = cli_resource_ban(out, lookup_id, host, move_lifetime, NULL, cib,
	cib_options, promoted_role_only);

	} else {
	/* Stop the resource by setting target-role to Stopped.
	* Remember any existing target-role so we can restore it later
	* (though it only makes any difference if it's Unpromoted).
	*/
	- char *lookup_id = clone_strip(rsc->id);

	find_resource_attr(out, cib, XML_NVPAIR_ATTR_VALUE, lookup_id, NULL, NULL,
	NULL, XML_RSC_ATTR_TARGET_ROLE, &orig_target_role);
	- free(lookup_id);
	rc = cli_resource_update_attribute(rsc, rsc_id, NULL, XML_TAG_META_SETS,
	NULL, XML_RSC_ATTR_TARGET_ROLE,
	RSC_STOPPED, FALSE, cib, cib_options,
	data_set, force);
	}
	if(rc != pcmk_rc_ok) {
	out->err(out, "Could not set target-role for %s: %s (%d)", rsc_id, pcmk_strerror(rc), rc);
	- if (current_active) {
	+ if (current_active != NULL) {
	g_list_free_full(current_active, free);
	+ current_active = NULL;
	}
	- if (restart_target_active) {
	+ if (restart_target_active != NULL) {
	g_list_free_full(restart_target_active, free);
	+ restart_target_active = NULL;
	}
	goto done;
	}

	- rc = update_dataset(cib, data_set, TRUE);
	+ rc = update_dataset(cib, data_set, true);
	if(rc != pcmk_rc_ok) {
	out->err(out, "Could not determine which resources would be stopped");
	goto failure;
	}

	target_active = get_active_resources(host, data_set->resources);
	dump_list(target_active, "Target");

	list_delta = pcmk__subtract_lists(current_active, target_active, (GCompareFunc) strcmp);
	out->info(out, "Waiting for %d resources to stop:", g_list_length(list_delta));
	display_list(out, list_delta, " * ");

	step_timeout_s = timeout / sleep_interval;
	while (list_delta != NULL) {
	before = g_list_length(list_delta);
	if(timeout_ms == 0) {
	step_timeout_s = max_delay_in(data_set, list_delta) / sleep_interval;
	}

	/* We probably don't need the entire step timeout */
	for(lpc = 0; (lpc < step_timeout_s) && (list_delta != NULL); lpc++) {
	sleep(sleep_interval);
	if(timeout) {
	timeout -= sleep_interval;
	crm_trace("%ds remaining", timeout);
	}
	rc = update_dataset(cib, data_set, FALSE);
	if(rc != pcmk_rc_ok) {
	out->err(out, "Could not determine which resources were stopped");
	goto failure;
	}

	- if (current_active) {
	+ if (current_active != NULL) {
	g_list_free_full(current_active, free);
	+ current_active = NULL;
	}
	current_active = get_active_resources(host, data_set->resources);
	g_list_free(list_delta);
	+ list_delta = NULL;
	list_delta = pcmk__subtract_lists(current_active, target_active, (GCompareFunc) strcmp);
	dump_list(current_active, "Current");
	dump_list(list_delta, "Delta");
	}

	crm_trace("%d (was %d) resources remaining", g_list_length(list_delta), before);
	if(before == g_list_length(list_delta)) {
	/* aborted during stop phase, print the contents of list_delta */
	out->err(out, "Could not complete shutdown of %s, %d resources remaining", rsc_id, g_list_length(list_delta));
	display_list(out, list_delta, " * ");
	rc = ETIME;
	goto failure;
	}

	}

	if (stop_via_ban) {
	- rc = cli_resource_clear(rsc_id, host, NULL, cib, cib_options, TRUE, force);
	+ rc = cli_resource_clear(lookup_id, host, NULL, cib, cib_options, true, force);

	} else if (orig_target_role) {
	rc = cli_resource_update_attribute(rsc, rsc_id, NULL, XML_TAG_META_SETS,
	NULL, XML_RSC_ATTR_TARGET_ROLE,
	orig_target_role, FALSE, cib,
	cib_options, data_set, force);
	free(orig_target_role);
	orig_target_role = NULL;
	} else {
	rc = cli_resource_delete_attribute(rsc, rsc_id, NULL, XML_TAG_META_SETS,
	NULL, XML_RSC_ATTR_TARGET_ROLE, cib,
	cib_options, data_set, force);
	}

	if(rc != pcmk_rc_ok) {
	out->err(out, "Could not unset target-role for %s: %s (%d)", rsc_id, pcmk_strerror(rc), rc);
	goto done;
	}

	- if (target_active) {
	+ if (target_active != NULL) {
	g_list_free_full(target_active, free);
	+ target_active = NULL;
	}
	target_active = restart_target_active;
	list_delta = pcmk__subtract_lists(target_active, current_active, (GCompareFunc) strcmp);
	out->info(out, "Waiting for %d resources to start again:", g_list_length(list_delta));
	display_list(out, list_delta, " * ");

	step_timeout_s = timeout / sleep_interval;
	while (waiting_for_starts(list_delta, rsc, host)) {
	before = g_list_length(list_delta);
	if(timeout_ms == 0) {
	step_timeout_s = max_delay_in(data_set, list_delta) / sleep_interval;
	}

	/* We probably don't need the entire step timeout */
	for (lpc = 0; (lpc < step_timeout_s) && waiting_for_starts(list_delta, rsc, host); lpc++) {

	sleep(sleep_interval);
	if(timeout) {
	timeout -= sleep_interval;
	crm_trace("%ds remaining", timeout);
	}

	- rc = update_dataset(cib, data_set, FALSE);
	+ rc = update_dataset(cib, data_set, false);
	if(rc != pcmk_rc_ok) {
	out->err(out, "Could not determine which resources were started");
	goto failure;
	}

	- if (current_active) {
	+ if (current_active != NULL) {
	g_list_free_full(current_active, free);
	+ current_active = NULL;
	}

	/* It's OK if dependent resources moved to a different node,
	* so we check active resources on all nodes.
	*/
	current_active = get_active_resources(NULL, data_set->resources);
	g_list_free(list_delta);
	list_delta = pcmk__subtract_lists(target_active, current_active, (GCompareFunc) strcmp);
	dump_list(current_active, "Current");
	dump_list(list_delta, "Delta");
	}

	if(before == g_list_length(list_delta)) {
	/* aborted during start phase, print the contents of list_delta */
	out->err(out, "Could not complete restart of %s, %d resources remaining", rsc_id, g_list_length(list_delta));
	display_list(out, list_delta, " * ");
	rc = ETIME;
	goto failure;
	}

	}

	rc = pcmk_rc_ok;
	goto done;

	failure:
	if (stop_via_ban) {
	- cli_resource_clear(rsc_id, host, NULL, cib, cib_options, TRUE, force);
	+ cli_resource_clear(lookup_id, host, NULL, cib, cib_options, true, force);
	} else if (orig_target_role) {
	cli_resource_update_attribute(rsc, rsc_id, NULL, XML_TAG_META_SETS, NULL,
	XML_RSC_ATTR_TARGET_ROLE, orig_target_role,
	FALSE, cib, cib_options, data_set, force);
	free(orig_target_role);
	} else {
	cli_resource_delete_attribute(rsc, rsc_id, NULL, XML_TAG_META_SETS, NULL,
	XML_RSC_ATTR_TARGET_ROLE, cib, cib_options,
	data_set, force);
	}

	done:
	- if (list_delta) {
	+ if (list_delta != NULL) {
	g_list_free(list_delta);
	}
	- if (current_active) {
	+ if (current_active != NULL) {
	g_list_free_full(current_active, free);
	}
	- if (target_active && (target_active != restart_target_active)) {
	+ if (target_active != NULL && (target_active != restart_target_active)) {
	g_list_free_full(target_active, free);
	}
	- if (restart_target_active) {
	+ if (restart_target_active != NULL) {
	g_list_free_full(restart_target_active, free);
	}
	free(rsc_id);
	+ free(lookup_id);
	pe_free_working_set(data_set);
	return rc;
	}

	static inline bool action_is_pending(pe_action_t *action)
	{
	if (pcmk_any_flags_set(action->flags, pe_action_optional\|pe_action_pseudo)
	\|\| !pcmk_is_set(action->flags, pe_action_runnable)
	\|\| pcmk__str_eq("notify", action->task, pcmk__str_casei)) {
	return false;
	}
	return true;
	}

	/*!
	* \internal
	* \brief Return TRUE if any actions in a list are pending
	*
	* \param[in] actions List of actions to check
	*
	* \return TRUE if any actions in the list are pending, FALSE otherwise
	*/
	static bool
	actions_are_pending(GList *actions)
	{
	GList *action;

	for (action = actions; action != NULL; action = action->next) {
	pe_action_t a = (pe_action_t )action->data;
	if (action_is_pending(a)) {
	crm_notice("Waiting for %s (flags=%#.8x)", a->uuid, a->flags);
	- return TRUE;
	+ return true;
	}
	}
	- return FALSE;
	+ return false;
	}

	static void
	print_pending_actions(pcmk__output_t out, GList actions)
	{
	GList *action;

	out->info(out, "Pending actions:");
	for (action = actions; action != NULL; action = action->next) {
	pe_action_t a = (pe_action_t ) action->data;

	if (!action_is_pending(a)) {
	continue;
	}

	if (a->node) {
	out->info(out, "\tAction %d: %s\ton %s", a->id, a->uuid, a->node->details->uname);
	} else {
	out->info(out, "\tAction %d: %s", a->id, a->uuid);
	}
	}
	}

	/* For --wait, timeout (in seconds) to use if caller doesn't specify one */
	#define WAIT_DEFAULT_TIMEOUT_S (60 * 60)

	/* For --wait, how long to sleep between cluster state checks */
	#define WAIT_SLEEP_S (2)

	/*!
	* \internal
	* \brief Wait until all pending cluster actions are complete
	*
	* This waits until either the CIB's transition graph is idle or a timeout is
	* reached.
	*
	* \param[in] timeout_ms Consider failed if actions do not complete in this time
	* (specified in milliseconds, but one-second granularity
	* is actually used; if 0, a default will be used)
	* \param[in] cib Connection to the CIB manager
	*
	* \return Standard Pacemaker return code
	*/
	int
	wait_till_stable(pcmk__output_t out, int timeout_ms, cib_t cib)
	{
	pe_working_set_t *data_set = NULL;
	int rc = pcmk_rc_ok;
	int timeout_s = timeout_ms? ((timeout_ms + 999) / 1000) : WAIT_DEFAULT_TIMEOUT_S;
	time_t expire_time = time(NULL) + timeout_s;
	time_t time_diff;
	bool printed_version_warning = out->is_quiet(out); // i.e. don't print if quiet

	data_set = pe_new_working_set();
	if (data_set == NULL) {
	return ENOMEM;
	}

	do {

	/* Abort if timeout is reached */
	time_diff = expire_time - time(NULL);
	if (time_diff > 0) {
	crm_info("Waiting up to %ld seconds for cluster actions to complete", time_diff);
	} else {
	print_pending_actions(out, data_set->actions);
	pe_free_working_set(data_set);
	return ETIME;
	}
	if (rc == pcmk_rc_ok) { /* this avoids sleep on first loop iteration */
	sleep(WAIT_SLEEP_S);
	}

	/* Get latest transition graph */
	pe_reset_working_set(data_set);
	rc = update_working_set_from_cib(out, data_set, cib);
	if (rc != pcmk_rc_ok) {
	pe_free_working_set(data_set);
	return rc;
	}
	pcmk__schedule_actions(data_set->input,
	pe_flag_no_counts\|pe_flag_no_compat, data_set);

	if (!printed_version_warning) {
	/* If the DC has a different version than the local node, the two
	* could come to different conclusions about what actions need to be
	* done. Warn the user in this case.
	*
	* @TODO A possible long-term solution would be to reimplement the
	* wait as a new controller operation that would be forwarded to the
	* DC. However, that would have potential problems of its own.
	*/
	const char *dc_version = g_hash_table_lookup(data_set->config_hash,
	"dc-version");

	if (!pcmk__str_eq(dc_version, PACEMAKER_VERSION "-" BUILD_VERSION, pcmk__str_casei)) {
	out->info(out, "warning: wait option may not work properly in "
	"mixed-version cluster");
	- printed_version_warning = TRUE;
	+ printed_version_warning = true;
	}
	}

	} while (actions_are_pending(data_set->actions));

	pe_free_working_set(data_set);
	return rc;
	}

	static const char *
	get_action(const char *rsc_action) {
	const char *action = NULL;

	if (pcmk__str_eq(rsc_action, "validate", pcmk__str_casei)) {
	action = "validate-all";

	} else if (pcmk__str_eq(rsc_action, "force-check", pcmk__str_casei)) {
	action = "monitor";

	} else if (pcmk__strcase_any_of(rsc_action, "force-start", "force-stop",
	"force-demote", "force-promote", NULL)) {
	action = rsc_action+6;
	} else {
	action = rsc_action;
	}

	return action;
	}

	/*!
	* \brief Set up environment variables as expected by resource agents
	*
	* When the cluster executes resource agents, it adds certain environment
	* variables (directly or via resource meta-attributes) expected by some
	* resource agents. Add the essential ones that many resource agents expect, so
	* the behavior is the same for command-line execution.
	*
	* \param[in] params Resource parameters that will be passed to agent
	* \param[in] timeout_ms Action timeout (in milliseconds)
	* \param[in] check_level OCF check level
	* \param[in] verbosity Verbosity level
	*/
	static void
	set_agent_environment(GHashTable *params, int timeout_ms, int check_level,
	int verbosity)
	{
	g_hash_table_insert(params, strdup("CRM_meta_timeout"),
	crm_strdup_printf("%d", timeout_ms));

	g_hash_table_insert(params, strdup(XML_ATTR_CRM_VERSION),
	strdup(CRM_FEATURE_SET));

	if (check_level >= 0) {
	char *level = crm_strdup_printf("%d", check_level);

	setenv("OCF_CHECK_LEVEL", level, 1);
	free(level);
	}

	setenv("HA_debug", (verbosity > 0)? "1" : "0", 1);
	if (verbosity > 1) {
	setenv("OCF_TRACE_RA", "1", 1);
	}

	/* A resource agent using the standard ocf-shellfuncs library will not print
	* messages to stderr if it doesn't have a controlling terminal (e.g. if
	* crm_resource is called via script or ssh). This forces it to do so.
	*/
	setenv("OCF_TRACE_FILE", "/dev/stderr", 0);
	}

	/*!
	* \internal
	* \brief Apply command-line overrides to resource parameters
	*
	* \param[in] params Parameters to be passed to agent
	* \param[in] overrides Parameters to override (or NULL if none)
	*/
	static void
	apply_overrides(GHashTable params, GHashTable overrides)
	{
	if (overrides != NULL) {
	GHashTableIter iter;
	char *name = NULL;
	char *value = NULL;

	g_hash_table_iter_init(&iter, overrides);
	while (g_hash_table_iter_next(&iter, (gpointer *) &name,
	(gpointer *) &value)) {
	g_hash_table_replace(params, strdup(name), strdup(value));
	}
	}
	}

	crm_exit_t
	cli_resource_execute_from_params(pcmk__output_t out, const char rsc_name,
	const char rsc_class, const char rsc_prov,
	const char rsc_type, const char rsc_action,
	GHashTable params, GHashTable override_hash,
	int timeout_ms, int resource_verbose, gboolean force,
	int check_level)
	{
	const char *class = rsc_class;
	const char *action = get_action(rsc_action);
	crm_exit_t exit_code = CRM_EX_OK;
	svc_action_t *op = NULL;

	// If no timeout was provided, use the same default as the cluster
	if (timeout_ms == 0) {
	timeout_ms = crm_get_msec(CRM_DEFAULT_OP_TIMEOUT_S);
	}

	set_agent_environment(params, timeout_ms, check_level, resource_verbose);
	apply_overrides(params, override_hash);

	op = services__create_resource_action(rsc_name? rsc_name : "test",
	rsc_class, rsc_prov, rsc_type, action,
	0, timeout_ms, params, 0);
	if (op == NULL) {
	out->err(out, "Could not execute %s using %s%s%s:%s: %s",
	action, rsc_class, (rsc_prov? ":" : ""),
	(rsc_prov? rsc_prov : ""), rsc_type, strerror(ENOMEM));
	g_hash_table_destroy(params);
	return CRM_EX_OSERR;
	}

	if (pcmk__str_eq(rsc_class, PCMK_RESOURCE_CLASS_SERVICE, pcmk__str_casei)) {
	class = resources_find_service_class(rsc_type);
	}
	if (!pcmk__strcase_any_of(class, PCMK_RESOURCE_CLASS_OCF,
	PCMK_RESOURCE_CLASS_LSB, NULL)) {
	services__format_result(op, CRM_EX_UNIMPLEMENT_FEATURE, PCMK_EXEC_ERROR,
	"Manual execution of the %s standard "
	"is unsupported", crm_str(class));
	}

	if (op->rc != PCMK_OCF_UNKNOWN) {
	exit_code = op->rc;
	goto done;
	}

	services_action_sync(op);

	// Map results to OCF codes for consistent reporting to user
	{
	enum ocf_exitcode ocf_code = services_result2ocf(class, action, op->rc);

	// Cast variable instead of function return to keep compilers happy
	exit_code = (crm_exit_t) ocf_code;
	}

	done:
	out->message(out, "resource-agent-action", resource_verbose, rsc_class,
	rsc_prov, rsc_type, rsc_name, rsc_action, override_hash,
	exit_code, op->status, services__exit_reason(op),
	op->stdout_data, op->stderr_data);
	services_action_free(op);
	return exit_code;
	}

	crm_exit_t
	cli_resource_execute(pe_resource_t rsc, const char requested_name,
	const char rsc_action, GHashTable override_hash,
	int timeout_ms, cib_t * cib, pe_working_set_t *data_set,
	int resource_verbose, gboolean force, int check_level)
	{
	pcmk__output_t *out = data_set->priv;
	crm_exit_t exit_code = CRM_EX_OK;
	const char *rid = NULL;
	const char *rtype = NULL;
	const char *rprov = NULL;
	const char *rclass = NULL;
	GHashTable *params = NULL;

	if (pcmk__strcase_any_of(rsc_action, "force-start", "force-demote",
	"force-promote", NULL)) {
	if(pe_rsc_is_clone(rsc)) {
	GList *nodes = cli_resource_search(rsc, requested_name, data_set);
	if(nodes != NULL && force == FALSE) {
	out->err(out, "It is not safe to %s %s here: the cluster claims it is already active",
	rsc_action, rsc->id);
	out->err(out, "Try setting target-role=Stopped first or specifying "
	"the force option");
	return CRM_EX_UNSAFE;
	}

	g_list_free_full(nodes, free);
	}
	}

	if(pe_rsc_is_clone(rsc)) {
	/* Grab the first child resource in the hope it's not a group */
	rsc = rsc->children->data;
	}

	if(rsc->variant == pe_group) {
	out->err(out, "Sorry, the %s option doesn't support group resources", rsc_action);
	return CRM_EX_UNIMPLEMENT_FEATURE;
	} else if (rsc->variant == pe_container \|\| pe_rsc_is_bundled(rsc)) {
	out->err(out, "Sorry, the %s option doesn't support bundled resources", rsc_action);
	return CRM_EX_UNIMPLEMENT_FEATURE;
	}

	rclass = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
	rprov = crm_element_value(rsc->xml, XML_AGENT_ATTR_PROVIDER);
	rtype = crm_element_value(rsc->xml, XML_ATTR_TYPE);

	params = generate_resource_params(rsc, NULL /* @TODO use local node */,
	data_set);

	if (timeout_ms == 0) {
	timeout_ms = pe_get_configured_timeout(rsc, get_action(rsc_action), data_set);
	}

	rid = pe_rsc_is_anon_clone(rsc->parent)? requested_name : rsc->id;

	exit_code = cli_resource_execute_from_params(out, rid, rclass, rprov, rtype, rsc_action,
	params, override_hash, timeout_ms,
	resource_verbose, force, check_level);
	return exit_code;
	}

	// \return Standard Pacemaker return code
	int
	cli_resource_move(pe_resource_t rsc, const char rsc_id, const char *host_name,
	const char move_lifetime, cib_t cib, int cib_options,
	pe_working_set_t *data_set, gboolean promoted_role_only,
	gboolean force)
	{
	pcmk__output_t *out = data_set->priv;
	int rc = pcmk_rc_ok;
	unsigned int count = 0;
	pe_node_t *current = NULL;
	pe_node_t *dest = pe_find_node(data_set->nodes, host_name);
	- bool cur_is_dest = FALSE;
	+ bool cur_is_dest = false;

	if (dest == NULL) {
	return pcmk_rc_node_unknown;
	}

	if (promoted_role_only && !pcmk_is_set(rsc->flags, pe_rsc_promotable)) {
	pe_resource_t *p = uber_parent(rsc);

	if (pcmk_is_set(p->flags, pe_rsc_promotable)) {
	out->info(out, "Using parent '%s' for move instead of '%s'.", rsc->id, rsc_id);
	rsc_id = p->id;
	rsc = p;

	} else {
	out->info(out, "Ignoring master option: %s is not promotable", rsc_id);
	promoted_role_only = FALSE;
	}
	}

	current = pe__find_active_requires(rsc, &count);

	if (pcmk_is_set(rsc->flags, pe_rsc_promotable)) {
	GList *iter = NULL;
	unsigned int promoted_count = 0;
	pe_node_t *promoted_node = NULL;

	for(iter = rsc->children; iter; iter = iter->next) {
	pe_resource_t child = (pe_resource_t )iter->data;
	enum rsc_role_e child_role = child->fns->state(child, TRUE);

	if (child_role == RSC_ROLE_PROMOTED) {
	rsc = child;
	promoted_node = pe__current_node(child);
	promoted_count++;
	}
	}
	if (promoted_role_only \|\| (promoted_count != 0)) {
	count = promoted_count;
	current = promoted_node;
	}

	}

	if (count > 1) {
	if (pe_rsc_is_clone(rsc)) {
	current = NULL;
	} else {
	return pcmk_rc_multiple;
	}
	}

	if (current && (current->details == dest->details)) {
	- cur_is_dest = TRUE;
	+ cur_is_dest = true;
	if (force) {
	crm_info("%s is already %s on %s, reinforcing placement with location constraint.",
	rsc_id, promoted_role_only?"promoted":"active", dest->details->uname);
	} else {
	return pcmk_rc_already;
	}
	}

	/* Clear any previous prefer constraints across all nodes. */
	- cli_resource_clear(rsc_id, NULL, data_set->nodes, cib, cib_options, FALSE, force);
	+ cli_resource_clear(rsc_id, NULL, data_set->nodes, cib, cib_options, false, force);

	/* Clear any previous ban constraints on 'dest'. */
	cli_resource_clear(rsc_id, dest->details->uname, data_set->nodes, cib,
	cib_options, TRUE, force);

	/* Record an explicit preference for 'dest' */
	rc = cli_resource_prefer(out, rsc_id, dest->details->uname, move_lifetime,
	cib, cib_options, promoted_role_only);

	crm_trace("%s%s now prefers node %s%s",
	rsc->id, (promoted_role_only? " (promoted)" : ""),
	dest->details->uname, force?"(forced)":"");

	/* only ban the previous location if current location != destination location.
	* it is possible to use -M to enforce a location without regard of where the
	* resource is currently located */
	- if(force && (cur_is_dest == FALSE)) {
	+ if (force && !cur_is_dest) {
	/* Ban the original location if possible */
	if(current) {
	(void)cli_resource_ban(out, rsc_id, current->details->uname, move_lifetime,
	NULL, cib, cib_options, promoted_role_only);

	} else if(count > 1) {
	out->info(out, "Resource '%s' is currently %s in %d locations. "
	"One may now move to %s",
	rsc_id, (promoted_role_only? "promoted" : "active"),
	count, dest->details->uname);
	out->info(out, "To prevent '%s' from being %s at a specific location, "
	"specify a node.",
	rsc_id, (promoted_role_only? "promoted" : "active"));

	} else {
	crm_trace("Not banning %s from its current location: not active", rsc_id);
	}
	}

	return rc;
	}

File Metadata

Mime Type: text/x-diff
Expires: Sat, Jan 25, 10:48 AM (20 h, 10 m)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 1322188
Default Alt Text: (578 KB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions