diff --git a/cts/cli/regression.acls.exp b/cts/cli/regression.acls.exp
index ac58d838d6..be91b93455 100644
--- a/cts/cli/regression.acls.exp
+++ b/cts/cli/regression.acls.exp
@@ -1,2872 +1,2875 @@
 =#=#=#= Begin test: Configure some ACLs =#=#=#=
 =#=#=#= Current cib after: Configure some ACLs =#=#=#=
 <cib epoch="2" num_updates="0" admin_epoch="0">
   <configuration>
     <crm_config/>
     <nodes/>
     <resources/>
     <constraints/>
     <acls>
       <acl_target id="l33t-haxor">
         <role id="nothing"/>
       </acl_target>
       <acl_target id="niceguy">
         <role id="observer"/>
       </acl_target>
       <acl_target id="bob">
         <role id="admin"/>
       </acl_target>
       <acl_target id="joe">
         <role id="super_user"/>
       </acl_target>
       <acl_target id="mike">
         <role id="rsc_writer"/>
       </acl_target>
       <acl_target id="chris">
         <role id="rsc_denied"/>
       </acl_target>
       <acl_role id="nothing">
         <acl_permission id="nothing-deny" kind="deny" xpath="/cib"/>
       </acl_role>
       <acl_role id="observer">
         <acl_permission id="observer-read-1" kind="read" xpath="/cib"/>
         <acl_permission id="observer-write-1" kind="write" xpath="//nvpair[@name='stonith-enabled']"/>
         <acl_permission id="observer-write-2" kind="write" xpath="//nvpair[@name='target-role']"/>
       </acl_role>
       <acl_role id="admin">
         <acl_permission id="admin-read-1" kind="read" xpath="/cib"/>
         <acl_permission id="admin-write-1" kind="write" xpath="//resources"/>
       </acl_role>
       <acl_role id="super_user">
         <acl_permission id="super_user-write-1" kind="write" xpath="/cib"/>
       </acl_role>
       <acl_role id="rsc_writer">
         <acl_permission id="rsc-writer-deny-1" kind="deny" xpath="/cib"/>
         <acl_permission id="rsc-writer-write-1" kind="write" xpath="//resources"/>
       </acl_role>
       <acl_role id="rsc_denied">
         <acl_permission id="rsc-denied-write-1" kind="write" xpath="/cib"/>
         <acl_permission id="rsc-denied-deny-1" kind="deny" xpath="//resources"/>
       </acl_role>
     </acls>
   </configuration>
   <status/>
 </cib>
 =#=#=#= End test: Configure some ACLs - OK (0) =#=#=#=
 * Passed: cibadmin              - Configure some ACLs
 =#=#=#= Begin test: Enable ACLs =#=#=#=
 =#=#=#= Current cib after: Enable ACLs =#=#=#=
 <cib epoch="3" num_updates="0" admin_epoch="0">
   <configuration>
     <crm_config>
       <cluster_property_set id="cib-bootstrap-options">
         <nvpair id="cib-bootstrap-options-enable-acl" name="enable-acl" value="true"/>
       </cluster_property_set>
     </crm_config>
     <nodes/>
     <resources/>
     <constraints/>
     <acls>
       <acl_target id="l33t-haxor">
         <role id="nothing"/>
       </acl_target>
       <acl_target id="niceguy">
         <role id="observer"/>
       </acl_target>
       <acl_target id="bob">
         <role id="admin"/>
       </acl_target>
       <acl_target id="joe">
         <role id="super_user"/>
       </acl_target>
       <acl_target id="mike">
         <role id="rsc_writer"/>
       </acl_target>
       <acl_target id="chris">
         <role id="rsc_denied"/>
       </acl_target>
       <acl_role id="nothing">
         <acl_permission id="nothing-deny" kind="deny" xpath="/cib"/>
       </acl_role>
       <acl_role id="observer">
         <acl_permission id="observer-read-1" kind="read" xpath="/cib"/>
         <acl_permission id="observer-write-1" kind="write" xpath="//nvpair[@name='stonith-enabled']"/>
         <acl_permission id="observer-write-2" kind="write" xpath="//nvpair[@name='target-role']"/>
       </acl_role>
       <acl_role id="admin">
         <acl_permission id="admin-read-1" kind="read" xpath="/cib"/>
         <acl_permission id="admin-write-1" kind="write" xpath="//resources"/>
       </acl_role>
       <acl_role id="super_user">
         <acl_permission id="super_user-write-1" kind="write" xpath="/cib"/>
       </acl_role>
       <acl_role id="rsc_writer">
         <acl_permission id="rsc-writer-deny-1" kind="deny" xpath="/cib"/>
         <acl_permission id="rsc-writer-write-1" kind="write" xpath="//resources"/>
       </acl_role>
       <acl_role id="rsc_denied">
         <acl_permission id="rsc-denied-write-1" kind="write" xpath="/cib"/>
         <acl_permission id="rsc-denied-deny-1" kind="deny" xpath="//resources"/>
       </acl_role>
     </acls>
   </configuration>
   <status/>
 </cib>
 =#=#=#= End test: Enable ACLs - OK (0) =#=#=#=
 * Passed: crm_attribute         - Enable ACLs
 =#=#=#= Begin test: Set cluster option =#=#=#=
 =#=#=#= Current cib after: Set cluster option =#=#=#=
 <cib epoch="4" num_updates="0" admin_epoch="0">
   <configuration>
     <crm_config>
       <cluster_property_set id="cib-bootstrap-options">
         <nvpair id="cib-bootstrap-options-enable-acl" name="enable-acl" value="true"/>
         <nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
       </cluster_property_set>
     </crm_config>
     <nodes/>
     <resources/>
     <constraints/>
     <acls>
       <acl_target id="l33t-haxor">
         <role id="nothing"/>
       </acl_target>
       <acl_target id="niceguy">
         <role id="observer"/>
       </acl_target>
       <acl_target id="bob">
         <role id="admin"/>
       </acl_target>
       <acl_target id="joe">
         <role id="super_user"/>
       </acl_target>
       <acl_target id="mike">
         <role id="rsc_writer"/>
       </acl_target>
       <acl_target id="chris">
         <role id="rsc_denied"/>
       </acl_target>
       <acl_role id="nothing">
         <acl_permission id="nothing-deny" kind="deny" xpath="/cib"/>
       </acl_role>
       <acl_role id="observer">
         <acl_permission id="observer-read-1" kind="read" xpath="/cib"/>
         <acl_permission id="observer-write-1" kind="write" xpath="//nvpair[@name='stonith-enabled']"/>
         <acl_permission id="observer-write-2" kind="write" xpath="//nvpair[@name='target-role']"/>
       </acl_role>
       <acl_role id="admin">
         <acl_permission id="admin-read-1" kind="read" xpath="/cib"/>
         <acl_permission id="admin-write-1" kind="write" xpath="//resources"/>
       </acl_role>
       <acl_role id="super_user">
         <acl_permission id="super_user-write-1" kind="write" xpath="/cib"/>
       </acl_role>
       <acl_role id="rsc_writer">
         <acl_permission id="rsc-writer-deny-1" kind="deny" xpath="/cib"/>
         <acl_permission id="rsc-writer-write-1" kind="write" xpath="//resources"/>
       </acl_role>
       <acl_role id="rsc_denied">
         <acl_permission id="rsc-denied-write-1" kind="write" xpath="/cib"/>
         <acl_permission id="rsc-denied-deny-1" kind="deny" xpath="//resources"/>
       </acl_role>
     </acls>
   </configuration>
   <status/>
 </cib>
 =#=#=#= End test: Set cluster option - OK (0) =#=#=#=
 * Passed: crm_attribute         - Set cluster option
 =#=#=#= Begin test: New ACL role =#=#=#=
 =#=#=#= Current cib after: New ACL role =#=#=#=
 <cib epoch="5" num_updates="0" admin_epoch="0">
   <configuration>
     <crm_config>
       <cluster_property_set id="cib-bootstrap-options">
         <nvpair id="cib-bootstrap-options-enable-acl" name="enable-acl" value="true"/>
         <nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
       </cluster_property_set>
     </crm_config>
     <nodes/>
     <resources/>
     <constraints/>
     <acls>
       <acl_target id="l33t-haxor">
         <role id="nothing"/>
       </acl_target>
       <acl_target id="niceguy">
         <role id="observer"/>
       </acl_target>
       <acl_target id="bob">
         <role id="admin"/>
       </acl_target>
       <acl_target id="joe">
         <role id="super_user"/>
       </acl_target>
       <acl_target id="mike">
         <role id="rsc_writer"/>
       </acl_target>
       <acl_target id="chris">
         <role id="rsc_denied"/>
       </acl_target>
       <acl_role id="nothing">
         <acl_permission id="nothing-deny" kind="deny" xpath="/cib"/>
       </acl_role>
       <acl_role id="observer">
         <acl_permission id="observer-read-1" kind="read" xpath="/cib"/>
         <acl_permission id="observer-write-1" kind="write" xpath="//nvpair[@name='stonith-enabled']"/>
         <acl_permission id="observer-write-2" kind="write" xpath="//nvpair[@name='target-role']"/>
       </acl_role>
       <acl_role id="admin">
         <acl_permission id="admin-read-1" kind="read" xpath="/cib"/>
         <acl_permission id="admin-write-1" kind="write" xpath="//resources"/>
       </acl_role>
       <acl_role id="super_user">
         <acl_permission id="super_user-write-1" kind="write" xpath="/cib"/>
       </acl_role>
       <acl_role id="rsc_writer">
         <acl_permission id="rsc-writer-deny-1" kind="deny" xpath="/cib"/>
         <acl_permission id="rsc-writer-write-1" kind="write" xpath="//resources"/>
       </acl_role>
       <acl_role id="rsc_denied">
         <acl_permission id="rsc-denied-write-1" kind="write" xpath="/cib"/>
         <acl_permission id="rsc-denied-deny-1" kind="deny" xpath="//resources"/>
       </acl_role>
       <acl_role id="badidea-role">
         <acl_permission id="badidea-resources" kind="read" xpath="//meta_attributes"/>
       </acl_role>
     </acls>
   </configuration>
   <status/>
 </cib>
 =#=#=#= End test: New ACL role - OK (0) =#=#=#=
 * Passed: cibadmin              - New ACL role
 =#=#=#= Begin test: New ACL target =#=#=#=
 =#=#=#= Current cib after: New ACL target =#=#=#=
 <cib epoch="6" num_updates="0" admin_epoch="0">
   <configuration>
     <crm_config>
       <cluster_property_set id="cib-bootstrap-options">
         <nvpair id="cib-bootstrap-options-enable-acl" name="enable-acl" value="true"/>
         <nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
       </cluster_property_set>
     </crm_config>
     <nodes/>
     <resources/>
     <constraints/>
     <acls>
       <acl_target id="l33t-haxor">
         <role id="nothing"/>
       </acl_target>
       <acl_target id="niceguy">
         <role id="observer"/>
       </acl_target>
       <acl_target id="bob">
         <role id="admin"/>
       </acl_target>
       <acl_target id="joe">
         <role id="super_user"/>
       </acl_target>
       <acl_target id="mike">
         <role id="rsc_writer"/>
       </acl_target>
       <acl_target id="chris">
         <role id="rsc_denied"/>
       </acl_target>
       <acl_role id="nothing">
         <acl_permission id="nothing-deny" kind="deny" xpath="/cib"/>
       </acl_role>
       <acl_role id="observer">
         <acl_permission id="observer-read-1" kind="read" xpath="/cib"/>
         <acl_permission id="observer-write-1" kind="write" xpath="//nvpair[@name='stonith-enabled']"/>
         <acl_permission id="observer-write-2" kind="write" xpath="//nvpair[@name='target-role']"/>
       </acl_role>
       <acl_role id="admin">
         <acl_permission id="admin-read-1" kind="read" xpath="/cib"/>
         <acl_permission id="admin-write-1" kind="write" xpath="//resources"/>
       </acl_role>
       <acl_role id="super_user">
         <acl_permission id="super_user-write-1" kind="write" xpath="/cib"/>
       </acl_role>
       <acl_role id="rsc_writer">
         <acl_permission id="rsc-writer-deny-1" kind="deny" xpath="/cib"/>
         <acl_permission id="rsc-writer-write-1" kind="write" xpath="//resources"/>
       </acl_role>
       <acl_role id="rsc_denied">
         <acl_permission id="rsc-denied-write-1" kind="write" xpath="/cib"/>
         <acl_permission id="rsc-denied-deny-1" kind="deny" xpath="//resources"/>
       </acl_role>
       <acl_role id="badidea-role">
         <acl_permission id="badidea-resources" kind="read" xpath="//meta_attributes"/>
       </acl_role>
       <acl_target id="badidea">
         <role id="badidea-role"/>
       </acl_target>
     </acls>
   </configuration>
   <status/>
 </cib>
 =#=#=#= End test: New ACL target - OK (0) =#=#=#=
 * Passed: cibadmin              - New ACL target
 =#=#=#= Begin test: Another ACL role =#=#=#=
 =#=#=#= Current cib after: Another ACL role =#=#=#=
 <cib epoch="7" num_updates="0" admin_epoch="0">
   <configuration>
     <crm_config>
       <cluster_property_set id="cib-bootstrap-options">
         <nvpair id="cib-bootstrap-options-enable-acl" name="enable-acl" value="true"/>
         <nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
       </cluster_property_set>
     </crm_config>
     <nodes/>
     <resources/>
     <constraints/>
     <acls>
       <acl_target id="l33t-haxor">
         <role id="nothing"/>
       </acl_target>
       <acl_target id="niceguy">
         <role id="observer"/>
       </acl_target>
       <acl_target id="bob">
         <role id="admin"/>
       </acl_target>
       <acl_target id="joe">
         <role id="super_user"/>
       </acl_target>
       <acl_target id="mike">
         <role id="rsc_writer"/>
       </acl_target>
       <acl_target id="chris">
         <role id="rsc_denied"/>
       </acl_target>
       <acl_role id="nothing">
         <acl_permission id="nothing-deny" kind="deny" xpath="/cib"/>
       </acl_role>
       <acl_role id="observer">
         <acl_permission id="observer-read-1" kind="read" xpath="/cib"/>
         <acl_permission id="observer-write-1" kind="write" xpath="//nvpair[@name='stonith-enabled']"/>
         <acl_permission id="observer-write-2" kind="write" xpath="//nvpair[@name='target-role']"/>
       </acl_role>
       <acl_role id="admin">
         <acl_permission id="admin-read-1" kind="read" xpath="/cib"/>
         <acl_permission id="admin-write-1" kind="write" xpath="//resources"/>
       </acl_role>
       <acl_role id="super_user">
         <acl_permission id="super_user-write-1" kind="write" xpath="/cib"/>
       </acl_role>
       <acl_role id="rsc_writer">
         <acl_permission id="rsc-writer-deny-1" kind="deny" xpath="/cib"/>
         <acl_permission id="rsc-writer-write-1" kind="write" xpath="//resources"/>
       </acl_role>
       <acl_role id="rsc_denied">
         <acl_permission id="rsc-denied-write-1" kind="write" xpath="/cib"/>
         <acl_permission id="rsc-denied-deny-1" kind="deny" xpath="//resources"/>
       </acl_role>
       <acl_role id="badidea-role">
         <acl_permission id="badidea-resources" kind="read" xpath="//meta_attributes"/>
       </acl_role>
       <acl_target id="badidea">
         <role id="badidea-role"/>
       </acl_target>
       <acl_role id="betteridea-role">
         <acl_permission id="betteridea-resources" kind="read" xpath="//meta_attributes"/>
       </acl_role>
     </acls>
   </configuration>
   <status/>
 </cib>
 =#=#=#= End test: Another ACL role - OK (0) =#=#=#=
 * Passed: cibadmin              - Another ACL role
 =#=#=#= Begin test: Another ACL target =#=#=#=
 =#=#=#= Current cib after: Another ACL target =#=#=#=
 <cib epoch="8" num_updates="0" admin_epoch="0">
   <configuration>
     <crm_config>
       <cluster_property_set id="cib-bootstrap-options">
         <nvpair id="cib-bootstrap-options-enable-acl" name="enable-acl" value="true"/>
         <nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
       </cluster_property_set>
     </crm_config>
     <nodes/>
     <resources/>
     <constraints/>
     <acls>
       <acl_target id="l33t-haxor">
         <role id="nothing"/>
       </acl_target>
       <acl_target id="niceguy">
         <role id="observer"/>
       </acl_target>
       <acl_target id="bob">
         <role id="admin"/>
       </acl_target>
       <acl_target id="joe">
         <role id="super_user"/>
       </acl_target>
       <acl_target id="mike">
         <role id="rsc_writer"/>
       </acl_target>
       <acl_target id="chris">
         <role id="rsc_denied"/>
       </acl_target>
       <acl_role id="nothing">
         <acl_permission id="nothing-deny" kind="deny" xpath="/cib"/>
       </acl_role>
       <acl_role id="observer">
         <acl_permission id="observer-read-1" kind="read" xpath="/cib"/>
         <acl_permission id="observer-write-1" kind="write" xpath="//nvpair[@name='stonith-enabled']"/>
         <acl_permission id="observer-write-2" kind="write" xpath="//nvpair[@name='target-role']"/>
       </acl_role>
       <acl_role id="admin">
         <acl_permission id="admin-read-1" kind="read" xpath="/cib"/>
         <acl_permission id="admin-write-1" kind="write" xpath="//resources"/>
       </acl_role>
       <acl_role id="super_user">
         <acl_permission id="super_user-write-1" kind="write" xpath="/cib"/>
       </acl_role>
       <acl_role id="rsc_writer">
         <acl_permission id="rsc-writer-deny-1" kind="deny" xpath="/cib"/>
         <acl_permission id="rsc-writer-write-1" kind="write" xpath="//resources"/>
       </acl_role>
       <acl_role id="rsc_denied">
         <acl_permission id="rsc-denied-write-1" kind="write" xpath="/cib"/>
         <acl_permission id="rsc-denied-deny-1" kind="deny" xpath="//resources"/>
       </acl_role>
       <acl_role id="badidea-role">
         <acl_permission id="badidea-resources" kind="read" xpath="//meta_attributes"/>
       </acl_role>
       <acl_target id="badidea">
         <role id="badidea-role"/>
       </acl_target>
       <acl_role id="betteridea-role">
         <acl_permission id="betteridea-resources" kind="read" xpath="//meta_attributes"/>
       </acl_role>
       <acl_target id="betteridea">
         <role id="betteridea-role"/>
       </acl_target>
     </acls>
   </configuration>
   <status/>
 </cib>
 =#=#=#= End test: Another ACL target - OK (0) =#=#=#=
 * Passed: cibadmin              - Another ACL target
 =#=#=#= Begin test: Updated ACL =#=#=#=
 =#=#=#= Current cib after: Updated ACL =#=#=#=
 <cib epoch="9" num_updates="0" admin_epoch="0">
   <configuration>
     <crm_config>
       <cluster_property_set id="cib-bootstrap-options">
         <nvpair id="cib-bootstrap-options-enable-acl" name="enable-acl" value="true"/>
         <nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
       </cluster_property_set>
     </crm_config>
     <nodes/>
     <resources/>
     <constraints/>
     <acls>
       <acl_target id="l33t-haxor">
         <role id="nothing"/>
       </acl_target>
       <acl_target id="niceguy">
         <role id="observer"/>
       </acl_target>
       <acl_target id="bob">
         <role id="admin"/>
       </acl_target>
       <acl_target id="joe">
         <role id="super_user"/>
       </acl_target>
       <acl_target id="mike">
         <role id="rsc_writer"/>
       </acl_target>
       <acl_target id="chris">
         <role id="rsc_denied"/>
       </acl_target>
       <acl_role id="nothing">
         <acl_permission id="nothing-deny" kind="deny" xpath="/cib"/>
       </acl_role>
       <acl_role id="observer">
         <acl_permission id="observer-read-1" kind="read" xpath="/cib"/>
         <acl_permission id="observer-write-1" kind="write" xpath="//nvpair[@name='stonith-enabled']"/>
         <acl_permission id="observer-write-2" kind="write" xpath="//nvpair[@name='target-role']"/>
       </acl_role>
       <acl_role id="admin">
         <acl_permission id="admin-read-1" kind="read" xpath="/cib"/>
         <acl_permission id="admin-write-1" kind="write" xpath="//resources"/>
       </acl_role>
       <acl_role id="super_user">
         <acl_permission id="super_user-write-1" kind="write" xpath="/cib"/>
       </acl_role>
       <acl_role id="rsc_writer">
         <acl_permission id="rsc-writer-deny-1" kind="deny" xpath="/cib"/>
         <acl_permission id="rsc-writer-write-1" kind="write" xpath="//resources"/>
       </acl_role>
       <acl_role id="rsc_denied">
         <acl_permission id="rsc-denied-write-1" kind="write" xpath="/cib"/>
         <acl_permission id="rsc-denied-deny-1" kind="deny" xpath="//resources"/>
       </acl_role>
       <acl_role id="badidea-role">
         <acl_permission id="badidea-resources" kind="read" xpath="//meta_attributes"/>
       </acl_role>
       <acl_target id="badidea">
         <role id="badidea-role"/>
       </acl_target>
       <acl_role id="betteridea-role">
         <acl_permission id="betteridea-nothing" kind="deny" xpath="/cib"/>
         <acl_permission id="betteridea-resources" kind="read" xpath="//meta_attributes"/>
       </acl_role>
       <acl_target id="betteridea">
         <role id="betteridea-role"/>
       </acl_target>
     </acls>
   </configuration>
   <status/>
 </cib>
 =#=#=#= End test: Updated ACL - OK (0) =#=#=#=
 * Passed: cibadmin              - Updated ACL
 =#=#=#= Begin test: unknownguy: Query configuration =#=#=#=
 Call failed: Permission denied
 =#=#=#= End test: unknownguy: Query configuration - Insufficient privileges (4) =#=#=#=
 * Passed: cibadmin              - unknownguy: Query configuration
 =#=#=#= Begin test: unknownguy: Set enable-acl =#=#=#=
 crm_attribute: Error performing operation: Permission denied
 =#=#=#= End test: unknownguy: Set enable-acl - Insufficient privileges (4) =#=#=#=
 * Passed: crm_attribute         - unknownguy: Set enable-acl
 =#=#=#= Begin test: unknownguy: Set stonith-enabled =#=#=#=
 crm_attribute: Error performing operation: Permission denied
 =#=#=#= End test: unknownguy: Set stonith-enabled - Insufficient privileges (4) =#=#=#=
 * Passed: crm_attribute         - unknownguy: Set stonith-enabled
 =#=#=#= Begin test: unknownguy: Create a resource =#=#=#=
 pcmk__check_acl 	trace: User 'unknownguy' without ACLs denied read/write access to /cib/configuration/resources/primitive[@id='dummy']
 pcmk__apply_creation_acl 	trace: ACLs disallow creation of <primitive> with id="dummy"
 Call failed: Permission denied
 =#=#=#= End test: unknownguy: Create a resource - Insufficient privileges (4) =#=#=#=
 * Passed: cibadmin              - unknownguy: Create a resource
 =#=#=#= Begin test: l33t-haxor: Query configuration =#=#=#=
 Call failed: Permission denied
 =#=#=#= End test: l33t-haxor: Query configuration - Insufficient privileges (4) =#=#=#=
 * Passed: cibadmin              - l33t-haxor: Query configuration
 =#=#=#= Begin test: l33t-haxor: Set enable-acl =#=#=#=
 crm_attribute: Error performing operation: Permission denied
 =#=#=#= End test: l33t-haxor: Set enable-acl - Insufficient privileges (4) =#=#=#=
 * Passed: crm_attribute         - l33t-haxor: Set enable-acl
 =#=#=#= Begin test: l33t-haxor: Set stonith-enabled =#=#=#=
 crm_attribute: Error performing operation: Permission denied
 =#=#=#= End test: l33t-haxor: Set stonith-enabled - Insufficient privileges (4) =#=#=#=
 * Passed: crm_attribute         - l33t-haxor: Set stonith-enabled
 =#=#=#= Begin test: l33t-haxor: Create a resource =#=#=#=
 pcmk__check_acl 	trace: Parent ACL denies user 'l33t-haxor' read/write access to /cib/configuration/resources/primitive[@id='dummy']
 pcmk__apply_creation_acl 	trace: ACLs disallow creation of <primitive> with id="dummy"
 Call failed: Permission denied
 =#=#=#= End test: l33t-haxor: Create a resource - Insufficient privileges (4) =#=#=#=
 * Passed: cibadmin              - l33t-haxor: Create a resource
 =#=#=#= Begin test: niceguy: Query configuration =#=#=#=
 <cib epoch="9" num_updates="0" admin_epoch="0">
   <configuration>
     <crm_config>
       <cluster_property_set id="cib-bootstrap-options">
         <nvpair id="cib-bootstrap-options-enable-acl" name="enable-acl" value="true"/>
         <nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
       </cluster_property_set>
     </crm_config>
     <nodes/>
     <resources/>
     <constraints/>
     <acls>
       <acl_target id="l33t-haxor">
         <role id="nothing"/>
       </acl_target>
       <acl_target id="niceguy">
         <role id="observer"/>
       </acl_target>
       <acl_target id="bob">
         <role id="admin"/>
       </acl_target>
       <acl_target id="joe">
         <role id="super_user"/>
       </acl_target>
       <acl_target id="mike">
         <role id="rsc_writer"/>
       </acl_target>
       <acl_target id="chris">
         <role id="rsc_denied"/>
       </acl_target>
       <acl_role id="nothing">
         <acl_permission id="nothing-deny" kind="deny" xpath="/cib"/>
       </acl_role>
       <acl_role id="observer">
         <acl_permission id="observer-read-1" kind="read" xpath="/cib"/>
         <acl_permission id="observer-write-1" kind="write" xpath="//nvpair[@name='stonith-enabled']"/>
         <acl_permission id="observer-write-2" kind="write" xpath="//nvpair[@name='target-role']"/>
       </acl_role>
       <acl_role id="admin">
         <acl_permission id="admin-read-1" kind="read" xpath="/cib"/>
         <acl_permission id="admin-write-1" kind="write" xpath="//resources"/>
       </acl_role>
       <acl_role id="super_user">
         <acl_permission id="super_user-write-1" kind="write" xpath="/cib"/>
       </acl_role>
       <acl_role id="rsc_writer">
         <acl_permission id="rsc-writer-deny-1" kind="deny" xpath="/cib"/>
         <acl_permission id="rsc-writer-write-1" kind="write" xpath="//resources"/>
       </acl_role>
       <acl_role id="rsc_denied">
         <acl_permission id="rsc-denied-write-1" kind="write" xpath="/cib"/>
         <acl_permission id="rsc-denied-deny-1" kind="deny" xpath="//resources"/>
       </acl_role>
       <acl_role id="badidea-role">
         <acl_permission id="badidea-resources" kind="read" xpath="//meta_attributes"/>
       </acl_role>
       <acl_target id="badidea">
         <role id="badidea-role"/>
       </acl_target>
       <acl_role id="betteridea-role">
         <acl_permission id="betteridea-nothing" kind="deny" xpath="/cib"/>
         <acl_permission id="betteridea-resources" kind="read" xpath="//meta_attributes"/>
       </acl_role>
       <acl_target id="betteridea">
         <role id="betteridea-role"/>
       </acl_target>
     </acls>
   </configuration>
   <status/>
 </cib>
 =#=#=#= End test: niceguy: Query configuration - OK (0) =#=#=#=
 * Passed: cibadmin              - niceguy: Query configuration
 =#=#=#= Begin test: niceguy: Set enable-acl =#=#=#=
 pcmk__check_acl 	trace: Default ACL denies user 'niceguy' read/write access to /cib/configuration/crm_config/cluster_property_set[@id='cib-bootstrap-options']/nvpair[@id='cib-bootstrap-options-enable-acl'][@value]
 Error setting enable-acl=false (section=crm_config, set=<null>): Permission denied
 crm_attribute: Error performing operation: Permission denied
 =#=#=#= End test: niceguy: Set enable-acl - Insufficient privileges (4) =#=#=#=
 * Passed: crm_attribute         - niceguy: Set enable-acl
 =#=#=#= Begin test: niceguy: Set stonith-enabled =#=#=#=
 pcmk__apply_creation_acl 	trace: ACLs allow creation of <nvpair> with id="cib-bootstrap-options-stonith-enabled"
 =#=#=#= Current cib after: niceguy: Set stonith-enabled =#=#=#=
 <cib epoch="10" num_updates="0" admin_epoch="0">
   <configuration>
     <crm_config>
       <cluster_property_set id="cib-bootstrap-options">
         <nvpair id="cib-bootstrap-options-enable-acl" name="enable-acl" value="true"/>
         <nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
         <nvpair id="cib-bootstrap-options-stonith-enabled" name="stonith-enabled" value="false"/>
       </cluster_property_set>
     </crm_config>
     <nodes/>
     <resources/>
     <constraints/>
     <acls>
       <acl_target id="l33t-haxor">
         <role id="nothing"/>
       </acl_target>
       <acl_target id="niceguy">
         <role id="observer"/>
       </acl_target>
       <acl_target id="bob">
         <role id="admin"/>
       </acl_target>
       <acl_target id="joe">
         <role id="super_user"/>
       </acl_target>
       <acl_target id="mike">
         <role id="rsc_writer"/>
       </acl_target>
       <acl_target id="chris">
         <role id="rsc_denied"/>
       </acl_target>
       <acl_role id="nothing">
         <acl_permission id="nothing-deny" kind="deny" xpath="/cib"/>
       </acl_role>
       <acl_role id="observer">
         <acl_permission id="observer-read-1" kind="read" xpath="/cib"/>
         <acl_permission id="observer-write-1" kind="write" xpath="//nvpair[@name='stonith-enabled']"/>
         <acl_permission id="observer-write-2" kind="write" xpath="//nvpair[@name='target-role']"/>
       </acl_role>
       <acl_role id="admin">
         <acl_permission id="admin-read-1" kind="read" xpath="/cib"/>
         <acl_permission id="admin-write-1" kind="write" xpath="//resources"/>
       </acl_role>
       <acl_role id="super_user">
         <acl_permission id="super_user-write-1" kind="write" xpath="/cib"/>
       </acl_role>
       <acl_role id="rsc_writer">
         <acl_permission id="rsc-writer-deny-1" kind="deny" xpath="/cib"/>
         <acl_permission id="rsc-writer-write-1" kind="write" xpath="//resources"/>
       </acl_role>
       <acl_role id="rsc_denied">
         <acl_permission id="rsc-denied-write-1" kind="write" xpath="/cib"/>
         <acl_permission id="rsc-denied-deny-1" kind="deny" xpath="//resources"/>
       </acl_role>
       <acl_role id="badidea-role">
         <acl_permission id="badidea-resources" kind="read" xpath="//meta_attributes"/>
       </acl_role>
       <acl_target id="badidea">
         <role id="badidea-role"/>
       </acl_target>
       <acl_role id="betteridea-role">
         <acl_permission id="betteridea-nothing" kind="deny" xpath="/cib"/>
         <acl_permission id="betteridea-resources" kind="read" xpath="//meta_attributes"/>
       </acl_role>
       <acl_target id="betteridea">
         <role id="betteridea-role"/>
       </acl_target>
     </acls>
   </configuration>
   <status/>
 </cib>
 =#=#=#= End test: niceguy: Set stonith-enabled - OK (0) =#=#=#=
 * Passed: crm_attribute         - niceguy: Set stonith-enabled
 =#=#=#= Begin test: niceguy: Create a resource =#=#=#=
 pcmk__check_acl 	trace: Default ACL denies user 'niceguy' read/write access to /cib/configuration/resources/primitive[@id='dummy']
 pcmk__apply_creation_acl 	trace: ACLs disallow creation of <primitive> with id="dummy"
 Call failed: Permission denied
 =#=#=#= End test: niceguy: Create a resource - Insufficient privileges (4) =#=#=#=
 * Passed: cibadmin              - niceguy: Create a resource
 =#=#=#= Begin test: root: Query configuration =#=#=#=
 <cib epoch="10" num_updates="0" admin_epoch="0">
   <configuration>
     <crm_config>
       <cluster_property_set id="cib-bootstrap-options">
         <nvpair id="cib-bootstrap-options-enable-acl" name="enable-acl" value="true"/>
         <nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
         <nvpair id="cib-bootstrap-options-stonith-enabled" name="stonith-enabled" value="false"/>
       </cluster_property_set>
     </crm_config>
     <nodes/>
     <resources/>
     <constraints/>
     <acls>
       <acl_target id="l33t-haxor">
         <role id="nothing"/>
       </acl_target>
       <acl_target id="niceguy">
         <role id="observer"/>
       </acl_target>
       <acl_target id="bob">
         <role id="admin"/>
       </acl_target>
       <acl_target id="joe">
         <role id="super_user"/>
       </acl_target>
       <acl_target id="mike">
         <role id="rsc_writer"/>
       </acl_target>
       <acl_target id="chris">
         <role id="rsc_denied"/>
       </acl_target>
       <acl_role id="nothing">
         <acl_permission id="nothing-deny" kind="deny" xpath="/cib"/>
       </acl_role>
       <acl_role id="observer">
         <acl_permission id="observer-read-1" kind="read" xpath="/cib"/>
         <acl_permission id="observer-write-1" kind="write" xpath="//nvpair[@name='stonith-enabled']"/>
         <acl_permission id="observer-write-2" kind="write" xpath="//nvpair[@name='target-role']"/>
       </acl_role>
       <acl_role id="admin">
         <acl_permission id="admin-read-1" kind="read" xpath="/cib"/>
         <acl_permission id="admin-write-1" kind="write" xpath="//resources"/>
       </acl_role>
       <acl_role id="super_user">
         <acl_permission id="super_user-write-1" kind="write" xpath="/cib"/>
       </acl_role>
       <acl_role id="rsc_writer">
         <acl_permission id="rsc-writer-deny-1" kind="deny" xpath="/cib"/>
         <acl_permission id="rsc-writer-write-1" kind="write" xpath="//resources"/>
       </acl_role>
       <acl_role id="rsc_denied">
         <acl_permission id="rsc-denied-write-1" kind="write" xpath="/cib"/>
         <acl_permission id="rsc-denied-deny-1" kind="deny" xpath="//resources"/>
       </acl_role>
       <acl_role id="badidea-role">
         <acl_permission id="badidea-resources" kind="read" xpath="//meta_attributes"/>
       </acl_role>
       <acl_target id="badidea">
         <role id="badidea-role"/>
       </acl_target>
       <acl_role id="betteridea-role">
         <acl_permission id="betteridea-nothing" kind="deny" xpath="/cib"/>
         <acl_permission id="betteridea-resources" kind="read" xpath="//meta_attributes"/>
       </acl_role>
       <acl_target id="betteridea">
         <role id="betteridea-role"/>
       </acl_target>
     </acls>
   </configuration>
   <status/>
 </cib>
 =#=#=#= End test: root: Query configuration - OK (0) =#=#=#=
 * Passed: cibadmin              - root: Query configuration
 =#=#=#= Begin test: root: Set stonith-enabled =#=#=#=
 =#=#=#= Current cib after: root: Set stonith-enabled =#=#=#=
 <cib epoch="11" num_updates="0" admin_epoch="0">
   <configuration>
     <crm_config>
       <cluster_property_set id="cib-bootstrap-options">
         <nvpair id="cib-bootstrap-options-enable-acl" name="enable-acl" value="true"/>
         <nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
         <nvpair id="cib-bootstrap-options-stonith-enabled" name="stonith-enabled" value="true"/>
       </cluster_property_set>
     </crm_config>
     <nodes/>
     <resources/>
     <constraints/>
     <acls>
       <acl_target id="l33t-haxor">
         <role id="nothing"/>
       </acl_target>
       <acl_target id="niceguy">
         <role id="observer"/>
       </acl_target>
       <acl_target id="bob">
         <role id="admin"/>
       </acl_target>
       <acl_target id="joe">
         <role id="super_user"/>
       </acl_target>
       <acl_target id="mike">
         <role id="rsc_writer"/>
       </acl_target>
       <acl_target id="chris">
         <role id="rsc_denied"/>
       </acl_target>
       <acl_role id="nothing">
         <acl_permission id="nothing-deny" kind="deny" xpath="/cib"/>
       </acl_role>
       <acl_role id="observer">
         <acl_permission id="observer-read-1" kind="read" xpath="/cib"/>
         <acl_permission id="observer-write-1" kind="write" xpath="//nvpair[@name='stonith-enabled']"/>
         <acl_permission id="observer-write-2" kind="write" xpath="//nvpair[@name='target-role']"/>
       </acl_role>
       <acl_role id="admin">
         <acl_permission id="admin-read-1" kind="read" xpath="/cib"/>
         <acl_permission id="admin-write-1" kind="write" xpath="//resources"/>
       </acl_role>
       <acl_role id="super_user">
         <acl_permission id="super_user-write-1" kind="write" xpath="/cib"/>
       </acl_role>
       <acl_role id="rsc_writer">
         <acl_permission id="rsc-writer-deny-1" kind="deny" xpath="/cib"/>
         <acl_permission id="rsc-writer-write-1" kind="write" xpath="//resources"/>
       </acl_role>
       <acl_role id="rsc_denied">
         <acl_permission id="rsc-denied-write-1" kind="write" xpath="/cib"/>
         <acl_permission id="rsc-denied-deny-1" kind="deny" xpath="//resources"/>
       </acl_role>
       <acl_role id="badidea-role">
         <acl_permission id="badidea-resources" kind="read" xpath="//meta_attributes"/>
       </acl_role>
       <acl_target id="badidea">
         <role id="badidea-role"/>
       </acl_target>
       <acl_role id="betteridea-role">
         <acl_permission id="betteridea-nothing" kind="deny" xpath="/cib"/>
         <acl_permission id="betteridea-resources" kind="read" xpath="//meta_attributes"/>
       </acl_role>
       <acl_target id="betteridea">
         <role id="betteridea-role"/>
       </acl_target>
     </acls>
   </configuration>
   <status/>
 </cib>
 =#=#=#= End test: root: Set stonith-enabled - OK (0) =#=#=#=
 * Passed: crm_attribute         - root: Set stonith-enabled
 =#=#=#= Begin test: root: Create a resource =#=#=#=
 =#=#=#= Current cib after: root: Create a resource =#=#=#=
 <cib epoch="12" num_updates="0" admin_epoch="0">
   <configuration>
     <crm_config>
       <cluster_property_set id="cib-bootstrap-options">
         <nvpair id="cib-bootstrap-options-enable-acl" name="enable-acl" value="true"/>
         <nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
         <nvpair id="cib-bootstrap-options-stonith-enabled" name="stonith-enabled" value="true"/>
       </cluster_property_set>
     </crm_config>
     <nodes/>
     <resources>
       <primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy"/>
     </resources>
     <constraints/>
     <acls>
       <acl_target id="l33t-haxor">
         <role id="nothing"/>
       </acl_target>
       <acl_target id="niceguy">
         <role id="observer"/>
       </acl_target>
       <acl_target id="bob">
         <role id="admin"/>
       </acl_target>
       <acl_target id="joe">
         <role id="super_user"/>
       </acl_target>
       <acl_target id="mike">
         <role id="rsc_writer"/>
       </acl_target>
       <acl_target id="chris">
         <role id="rsc_denied"/>
       </acl_target>
       <acl_role id="nothing">
         <acl_permission id="nothing-deny" kind="deny" xpath="/cib"/>
       </acl_role>
       <acl_role id="observer">
         <acl_permission id="observer-read-1" kind="read" xpath="/cib"/>
         <acl_permission id="observer-write-1" kind="write" xpath="//nvpair[@name='stonith-enabled']"/>
         <acl_permission id="observer-write-2" kind="write" xpath="//nvpair[@name='target-role']"/>
       </acl_role>
       <acl_role id="admin">
         <acl_permission id="admin-read-1" kind="read" xpath="/cib"/>
         <acl_permission id="admin-write-1" kind="write" xpath="//resources"/>
       </acl_role>
       <acl_role id="super_user">
         <acl_permission id="super_user-write-1" kind="write" xpath="/cib"/>
       </acl_role>
       <acl_role id="rsc_writer">
         <acl_permission id="rsc-writer-deny-1" kind="deny" xpath="/cib"/>
         <acl_permission id="rsc-writer-write-1" kind="write" xpath="//resources"/>
       </acl_role>
       <acl_role id="rsc_denied">
         <acl_permission id="rsc-denied-write-1" kind="write" xpath="/cib"/>
         <acl_permission id="rsc-denied-deny-1" kind="deny" xpath="//resources"/>
       </acl_role>
       <acl_role id="badidea-role">
         <acl_permission id="badidea-resources" kind="read" xpath="//meta_attributes"/>
       </acl_role>
       <acl_target id="badidea">
         <role id="badidea-role"/>
       </acl_target>
       <acl_role id="betteridea-role">
         <acl_permission id="betteridea-nothing" kind="deny" xpath="/cib"/>
         <acl_permission id="betteridea-resources" kind="read" xpath="//meta_attributes"/>
       </acl_role>
       <acl_target id="betteridea">
         <role id="betteridea-role"/>
       </acl_target>
     </acls>
   </configuration>
   <status/>
 </cib>
 =#=#=#= End test: root: Create a resource - OK (0) =#=#=#=
 * Passed: cibadmin              - root: Create a resource
 =#=#=#= Begin test: root: Create another resource (with description) =#=#=#=
 =#=#=#= Current cib after: root: Create another resource (with description) =#=#=#=
 <cib epoch="13" num_updates="0" admin_epoch="0">
   <configuration>
     <crm_config>
       <cluster_property_set id="cib-bootstrap-options">
         <nvpair id="cib-bootstrap-options-enable-acl" name="enable-acl" value="true"/>
         <nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
         <nvpair id="cib-bootstrap-options-stonith-enabled" name="stonith-enabled" value="true"/>
       </cluster_property_set>
     </crm_config>
     <nodes/>
     <resources>
       <primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy"/>
       <primitive id="dummy_desc" class="ocf" provider="pacemaker" type="Dummy" description="resource with description"/>
     </resources>
     <constraints/>
     <acls>
       <acl_target id="l33t-haxor">
         <role id="nothing"/>
       </acl_target>
       <acl_target id="niceguy">
         <role id="observer"/>
       </acl_target>
       <acl_target id="bob">
         <role id="admin"/>
       </acl_target>
       <acl_target id="joe">
         <role id="super_user"/>
       </acl_target>
       <acl_target id="mike">
         <role id="rsc_writer"/>
       </acl_target>
       <acl_target id="chris">
         <role id="rsc_denied"/>
       </acl_target>
       <acl_role id="nothing">
         <acl_permission id="nothing-deny" kind="deny" xpath="/cib"/>
       </acl_role>
       <acl_role id="observer">
         <acl_permission id="observer-read-1" kind="read" xpath="/cib"/>
         <acl_permission id="observer-write-1" kind="write" xpath="//nvpair[@name='stonith-enabled']"/>
         <acl_permission id="observer-write-2" kind="write" xpath="//nvpair[@name='target-role']"/>
       </acl_role>
       <acl_role id="admin">
         <acl_permission id="admin-read-1" kind="read" xpath="/cib"/>
         <acl_permission id="admin-write-1" kind="write" xpath="//resources"/>
       </acl_role>
       <acl_role id="super_user">
         <acl_permission id="super_user-write-1" kind="write" xpath="/cib"/>
       </acl_role>
       <acl_role id="rsc_writer">
         <acl_permission id="rsc-writer-deny-1" kind="deny" xpath="/cib"/>
         <acl_permission id="rsc-writer-write-1" kind="write" xpath="//resources"/>
       </acl_role>
       <acl_role id="rsc_denied">
         <acl_permission id="rsc-denied-write-1" kind="write" xpath="/cib"/>
         <acl_permission id="rsc-denied-deny-1" kind="deny" xpath="//resources"/>
       </acl_role>
       <acl_role id="badidea-role">
         <acl_permission id="badidea-resources" kind="read" xpath="//meta_attributes"/>
       </acl_role>
       <acl_target id="badidea">
         <role id="badidea-role"/>
       </acl_target>
       <acl_role id="betteridea-role">
         <acl_permission id="betteridea-nothing" kind="deny" xpath="/cib"/>
         <acl_permission id="betteridea-resources" kind="read" xpath="//meta_attributes"/>
       </acl_role>
       <acl_target id="betteridea">
         <role id="betteridea-role"/>
       </acl_target>
     </acls>
   </configuration>
   <status/>
 </cib>
 =#=#=#= End test: root: Create another resource (with description) - OK (0) =#=#=#=
 * Passed: cibadmin              - root: Create another resource (with description)
 =#=#=#= Begin test: l33t-haxor: Create a resource meta attribute =#=#=#=
+Could not obtain the current CIB: Permission denied
 crm_resource: Error performing operation: Insufficient privileges
 =#=#=#= End test: l33t-haxor: Create a resource meta attribute - Insufficient privileges (4) =#=#=#=
 * Passed: crm_resource          - l33t-haxor: Create a resource meta attribute
 =#=#=#= Begin test: l33t-haxor: Query a resource meta attribute =#=#=#=
+Could not obtain the current CIB: Permission denied
 crm_resource: Error performing operation: Insufficient privileges
 =#=#=#= End test: l33t-haxor: Query a resource meta attribute - Insufficient privileges (4) =#=#=#=
 * Passed: crm_resource          - l33t-haxor: Query a resource meta attribute
 =#=#=#= Begin test: l33t-haxor: Remove a resource meta attribute =#=#=#=
+Could not obtain the current CIB: Permission denied
 crm_resource: Error performing operation: Insufficient privileges
 =#=#=#= End test: l33t-haxor: Remove a resource meta attribute - Insufficient privileges (4) =#=#=#=
 * Passed: crm_resource          - l33t-haxor: Remove a resource meta attribute
 =#=#=#= Begin test: niceguy: Create a resource meta attribute =#=#=#=
 unpack_resources 	error: Resource start-up disabled since no STONITH resources have been defined
 unpack_resources 	error: Either configure some or disable STONITH with the stonith-enabled option
 unpack_resources 	error: NOTE: Clusters with shared data need STONITH to ensure data integrity
 pcmk__apply_creation_acl 	trace: Creation of <meta_attributes> scaffolding with id="dummy-meta_attributes" is implicitly allowed
 pcmk__apply_creation_acl 	trace: ACLs allow creation of <nvpair> with id="dummy-meta_attributes-target-role"
 Set 'dummy' option: id=dummy-meta_attributes-target-role set=dummy-meta_attributes name=target-role value=Stopped
 =#=#=#= Current cib after: niceguy: Create a resource meta attribute =#=#=#=
 <cib epoch="14" num_updates="0" admin_epoch="0">
   <configuration>
     <crm_config>
       <cluster_property_set id="cib-bootstrap-options">
         <nvpair id="cib-bootstrap-options-enable-acl" name="enable-acl" value="true"/>
         <nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
         <nvpair id="cib-bootstrap-options-stonith-enabled" name="stonith-enabled" value="true"/>
       </cluster_property_set>
     </crm_config>
     <nodes/>
     <resources>
       <primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
         <meta_attributes id="dummy-meta_attributes">
           <nvpair id="dummy-meta_attributes-target-role" name="target-role" value="Stopped"/>
         </meta_attributes>
       </primitive>
       <primitive id="dummy_desc" class="ocf" provider="pacemaker" type="Dummy" description="resource with description"/>
     </resources>
     <constraints/>
     <acls>
       <acl_target id="l33t-haxor">
         <role id="nothing"/>
       </acl_target>
       <acl_target id="niceguy">
         <role id="observer"/>
       </acl_target>
       <acl_target id="bob">
         <role id="admin"/>
       </acl_target>
       <acl_target id="joe">
         <role id="super_user"/>
       </acl_target>
       <acl_target id="mike">
         <role id="rsc_writer"/>
       </acl_target>
       <acl_target id="chris">
         <role id="rsc_denied"/>
       </acl_target>
       <acl_role id="nothing">
         <acl_permission id="nothing-deny" kind="deny" xpath="/cib"/>
       </acl_role>
       <acl_role id="observer">
         <acl_permission id="observer-read-1" kind="read" xpath="/cib"/>
         <acl_permission id="observer-write-1" kind="write" xpath="//nvpair[@name='stonith-enabled']"/>
         <acl_permission id="observer-write-2" kind="write" xpath="//nvpair[@name='target-role']"/>
       </acl_role>
       <acl_role id="admin">
         <acl_permission id="admin-read-1" kind="read" xpath="/cib"/>
         <acl_permission id="admin-write-1" kind="write" xpath="//resources"/>
       </acl_role>
       <acl_role id="super_user">
         <acl_permission id="super_user-write-1" kind="write" xpath="/cib"/>
       </acl_role>
       <acl_role id="rsc_writer">
         <acl_permission id="rsc-writer-deny-1" kind="deny" xpath="/cib"/>
         <acl_permission id="rsc-writer-write-1" kind="write" xpath="//resources"/>
       </acl_role>
       <acl_role id="rsc_denied">
         <acl_permission id="rsc-denied-write-1" kind="write" xpath="/cib"/>
         <acl_permission id="rsc-denied-deny-1" kind="deny" xpath="//resources"/>
       </acl_role>
       <acl_role id="badidea-role">
         <acl_permission id="badidea-resources" kind="read" xpath="//meta_attributes"/>
       </acl_role>
       <acl_target id="badidea">
         <role id="badidea-role"/>
       </acl_target>
       <acl_role id="betteridea-role">
         <acl_permission id="betteridea-nothing" kind="deny" xpath="/cib"/>
         <acl_permission id="betteridea-resources" kind="read" xpath="//meta_attributes"/>
       </acl_role>
       <acl_target id="betteridea">
         <role id="betteridea-role"/>
       </acl_target>
     </acls>
   </configuration>
   <status/>
 </cib>
 =#=#=#= End test: niceguy: Create a resource meta attribute - OK (0) =#=#=#=
 * Passed: crm_resource          - niceguy: Create a resource meta attribute
 =#=#=#= Begin test: niceguy: Query a resource meta attribute =#=#=#=
 unpack_resources 	error: Resource start-up disabled since no STONITH resources have been defined
 unpack_resources 	error: Either configure some or disable STONITH with the stonith-enabled option
 unpack_resources 	error: NOTE: Clusters with shared data need STONITH to ensure data integrity
 Stopped
 =#=#=#= Current cib after: niceguy: Query a resource meta attribute =#=#=#=
 <cib epoch="14" num_updates="0" admin_epoch="0">
   <configuration>
     <crm_config>
       <cluster_property_set id="cib-bootstrap-options">
         <nvpair id="cib-bootstrap-options-enable-acl" name="enable-acl" value="true"/>
         <nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
         <nvpair id="cib-bootstrap-options-stonith-enabled" name="stonith-enabled" value="true"/>
       </cluster_property_set>
     </crm_config>
     <nodes/>
     <resources>
       <primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
         <meta_attributes id="dummy-meta_attributes">
           <nvpair id="dummy-meta_attributes-target-role" name="target-role" value="Stopped"/>
         </meta_attributes>
       </primitive>
       <primitive id="dummy_desc" class="ocf" provider="pacemaker" type="Dummy" description="resource with description"/>
     </resources>
     <constraints/>
     <acls>
       <acl_target id="l33t-haxor">
         <role id="nothing"/>
       </acl_target>
       <acl_target id="niceguy">
         <role id="observer"/>
       </acl_target>
       <acl_target id="bob">
         <role id="admin"/>
       </acl_target>
       <acl_target id="joe">
         <role id="super_user"/>
       </acl_target>
       <acl_target id="mike">
         <role id="rsc_writer"/>
       </acl_target>
       <acl_target id="chris">
         <role id="rsc_denied"/>
       </acl_target>
       <acl_role id="nothing">
         <acl_permission id="nothing-deny" kind="deny" xpath="/cib"/>
       </acl_role>
       <acl_role id="observer">
         <acl_permission id="observer-read-1" kind="read" xpath="/cib"/>
         <acl_permission id="observer-write-1" kind="write" xpath="//nvpair[@name='stonith-enabled']"/>
         <acl_permission id="observer-write-2" kind="write" xpath="//nvpair[@name='target-role']"/>
       </acl_role>
       <acl_role id="admin">
         <acl_permission id="admin-read-1" kind="read" xpath="/cib"/>
         <acl_permission id="admin-write-1" kind="write" xpath="//resources"/>
       </acl_role>
       <acl_role id="super_user">
         <acl_permission id="super_user-write-1" kind="write" xpath="/cib"/>
       </acl_role>
       <acl_role id="rsc_writer">
         <acl_permission id="rsc-writer-deny-1" kind="deny" xpath="/cib"/>
         <acl_permission id="rsc-writer-write-1" kind="write" xpath="//resources"/>
       </acl_role>
       <acl_role id="rsc_denied">
         <acl_permission id="rsc-denied-write-1" kind="write" xpath="/cib"/>
         <acl_permission id="rsc-denied-deny-1" kind="deny" xpath="//resources"/>
       </acl_role>
       <acl_role id="badidea-role">
         <acl_permission id="badidea-resources" kind="read" xpath="//meta_attributes"/>
       </acl_role>
       <acl_target id="badidea">
         <role id="badidea-role"/>
       </acl_target>
       <acl_role id="betteridea-role">
         <acl_permission id="betteridea-nothing" kind="deny" xpath="/cib"/>
         <acl_permission id="betteridea-resources" kind="read" xpath="//meta_attributes"/>
       </acl_role>
       <acl_target id="betteridea">
         <role id="betteridea-role"/>
       </acl_target>
     </acls>
   </configuration>
   <status/>
 </cib>
 =#=#=#= End test: niceguy: Query a resource meta attribute - OK (0) =#=#=#=
 * Passed: crm_resource          - niceguy: Query a resource meta attribute
 =#=#=#= Begin test: niceguy: Remove a resource meta attribute =#=#=#=
 unpack_resources 	error: Resource start-up disabled since no STONITH resources have been defined
 unpack_resources 	error: Either configure some or disable STONITH with the stonith-enabled option
 unpack_resources 	error: NOTE: Clusters with shared data need STONITH to ensure data integrity
 Deleted 'dummy' option: id=dummy-meta_attributes-target-role name=target-role
 =#=#=#= Current cib after: niceguy: Remove a resource meta attribute =#=#=#=
 <cib epoch="15" num_updates="0" admin_epoch="0">
   <configuration>
     <crm_config>
       <cluster_property_set id="cib-bootstrap-options">
         <nvpair id="cib-bootstrap-options-enable-acl" name="enable-acl" value="true"/>
         <nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
         <nvpair id="cib-bootstrap-options-stonith-enabled" name="stonith-enabled" value="true"/>
       </cluster_property_set>
     </crm_config>
     <nodes/>
     <resources>
       <primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
         <meta_attributes id="dummy-meta_attributes"/>
       </primitive>
       <primitive id="dummy_desc" class="ocf" provider="pacemaker" type="Dummy" description="resource with description"/>
     </resources>
     <constraints/>
     <acls>
       <acl_target id="l33t-haxor">
         <role id="nothing"/>
       </acl_target>
       <acl_target id="niceguy">
         <role id="observer"/>
       </acl_target>
       <acl_target id="bob">
         <role id="admin"/>
       </acl_target>
       <acl_target id="joe">
         <role id="super_user"/>
       </acl_target>
       <acl_target id="mike">
         <role id="rsc_writer"/>
       </acl_target>
       <acl_target id="chris">
         <role id="rsc_denied"/>
       </acl_target>
       <acl_role id="nothing">
         <acl_permission id="nothing-deny" kind="deny" xpath="/cib"/>
       </acl_role>
       <acl_role id="observer">
         <acl_permission id="observer-read-1" kind="read" xpath="/cib"/>
         <acl_permission id="observer-write-1" kind="write" xpath="//nvpair[@name='stonith-enabled']"/>
         <acl_permission id="observer-write-2" kind="write" xpath="//nvpair[@name='target-role']"/>
       </acl_role>
       <acl_role id="admin">
         <acl_permission id="admin-read-1" kind="read" xpath="/cib"/>
         <acl_permission id="admin-write-1" kind="write" xpath="//resources"/>
       </acl_role>
       <acl_role id="super_user">
         <acl_permission id="super_user-write-1" kind="write" xpath="/cib"/>
       </acl_role>
       <acl_role id="rsc_writer">
         <acl_permission id="rsc-writer-deny-1" kind="deny" xpath="/cib"/>
         <acl_permission id="rsc-writer-write-1" kind="write" xpath="//resources"/>
       </acl_role>
       <acl_role id="rsc_denied">
         <acl_permission id="rsc-denied-write-1" kind="write" xpath="/cib"/>
         <acl_permission id="rsc-denied-deny-1" kind="deny" xpath="//resources"/>
       </acl_role>
       <acl_role id="badidea-role">
         <acl_permission id="badidea-resources" kind="read" xpath="//meta_attributes"/>
       </acl_role>
       <acl_target id="badidea">
         <role id="badidea-role"/>
       </acl_target>
       <acl_role id="betteridea-role">
         <acl_permission id="betteridea-nothing" kind="deny" xpath="/cib"/>
         <acl_permission id="betteridea-resources" kind="read" xpath="//meta_attributes"/>
       </acl_role>
       <acl_target id="betteridea">
         <role id="betteridea-role"/>
       </acl_target>
     </acls>
   </configuration>
   <status/>
 </cib>
 =#=#=#= End test: niceguy: Remove a resource meta attribute - OK (0) =#=#=#=
 * Passed: crm_resource          - niceguy: Remove a resource meta attribute
 =#=#=#= Begin test: niceguy: Create a resource meta attribute =#=#=#=
 unpack_resources 	error: Resource start-up disabled since no STONITH resources have been defined
 unpack_resources 	error: Either configure some or disable STONITH with the stonith-enabled option
 unpack_resources 	error: NOTE: Clusters with shared data need STONITH to ensure data integrity
 pcmk__apply_creation_acl 	trace: ACLs allow creation of <nvpair> with id="dummy-meta_attributes-target-role"
 Set 'dummy' option: id=dummy-meta_attributes-target-role set=dummy-meta_attributes name=target-role value=Started
 =#=#=#= Current cib after: niceguy: Create a resource meta attribute =#=#=#=
 <cib epoch="16" num_updates="0" admin_epoch="0">
   <configuration>
     <crm_config>
       <cluster_property_set id="cib-bootstrap-options">
         <nvpair id="cib-bootstrap-options-enable-acl" name="enable-acl" value="true"/>
         <nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
         <nvpair id="cib-bootstrap-options-stonith-enabled" name="stonith-enabled" value="true"/>
       </cluster_property_set>
     </crm_config>
     <nodes/>
     <resources>
       <primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
         <meta_attributes id="dummy-meta_attributes">
           <nvpair id="dummy-meta_attributes-target-role" name="target-role" value="Started"/>
         </meta_attributes>
       </primitive>
       <primitive id="dummy_desc" class="ocf" provider="pacemaker" type="Dummy" description="resource with description"/>
     </resources>
     <constraints/>
     <acls>
       <acl_target id="l33t-haxor">
         <role id="nothing"/>
       </acl_target>
       <acl_target id="niceguy">
         <role id="observer"/>
       </acl_target>
       <acl_target id="bob">
         <role id="admin"/>
       </acl_target>
       <acl_target id="joe">
         <role id="super_user"/>
       </acl_target>
       <acl_target id="mike">
         <role id="rsc_writer"/>
       </acl_target>
       <acl_target id="chris">
         <role id="rsc_denied"/>
       </acl_target>
       <acl_role id="nothing">
         <acl_permission id="nothing-deny" kind="deny" xpath="/cib"/>
       </acl_role>
       <acl_role id="observer">
         <acl_permission id="observer-read-1" kind="read" xpath="/cib"/>
         <acl_permission id="observer-write-1" kind="write" xpath="//nvpair[@name='stonith-enabled']"/>
         <acl_permission id="observer-write-2" kind="write" xpath="//nvpair[@name='target-role']"/>
       </acl_role>
       <acl_role id="admin">
         <acl_permission id="admin-read-1" kind="read" xpath="/cib"/>
         <acl_permission id="admin-write-1" kind="write" xpath="//resources"/>
       </acl_role>
       <acl_role id="super_user">
         <acl_permission id="super_user-write-1" kind="write" xpath="/cib"/>
       </acl_role>
       <acl_role id="rsc_writer">
         <acl_permission id="rsc-writer-deny-1" kind="deny" xpath="/cib"/>
         <acl_permission id="rsc-writer-write-1" kind="write" xpath="//resources"/>
       </acl_role>
       <acl_role id="rsc_denied">
         <acl_permission id="rsc-denied-write-1" kind="write" xpath="/cib"/>
         <acl_permission id="rsc-denied-deny-1" kind="deny" xpath="//resources"/>
       </acl_role>
       <acl_role id="badidea-role">
         <acl_permission id="badidea-resources" kind="read" xpath="//meta_attributes"/>
       </acl_role>
       <acl_target id="badidea">
         <role id="badidea-role"/>
       </acl_target>
       <acl_role id="betteridea-role">
         <acl_permission id="betteridea-nothing" kind="deny" xpath="/cib"/>
         <acl_permission id="betteridea-resources" kind="read" xpath="//meta_attributes"/>
       </acl_role>
       <acl_target id="betteridea">
         <role id="betteridea-role"/>
       </acl_target>
     </acls>
   </configuration>
   <status/>
 </cib>
 =#=#=#= End test: niceguy: Create a resource meta attribute - OK (0) =#=#=#=
 * Passed: crm_resource          - niceguy: Create a resource meta attribute
 =#=#=#= Begin test: badidea: Query configuration - implied deny =#=#=#=
 <cib>
   <configuration>
     <resources>
       <primitive id="dummy">
         <meta_attributes id="dummy-meta_attributes">
           <nvpair id="dummy-meta_attributes-target-role" name="target-role" value="Started"/>
         </meta_attributes>
       </primitive>
     </resources>
   </configuration>
 </cib>
 =#=#=#= End test: badidea: Query configuration - implied deny - OK (0) =#=#=#=
 * Passed: cibadmin              - badidea: Query configuration - implied deny
 =#=#=#= Begin test: betteridea: Query configuration - explicit deny =#=#=#=
 <cib>
   <configuration>
     <resources>
       <primitive id="dummy">
         <meta_attributes id="dummy-meta_attributes">
           <nvpair id="dummy-meta_attributes-target-role" name="target-role" value="Started"/>
         </meta_attributes>
       </primitive>
     </resources>
   </configuration>
 </cib>
 =#=#=#= End test: betteridea: Query configuration - explicit deny - OK (0) =#=#=#=
 * Passed: cibadmin              - betteridea: Query configuration - explicit deny
 <cib epoch="17" num_updates="0" admin_epoch="0">
   <configuration>
     <crm_config>
       <cluster_property_set id="cib-bootstrap-options">
         <nvpair id="cib-bootstrap-options-enable-acl" name="enable-acl" value="true"/>
         <nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
         <nvpair id="cib-bootstrap-options-stonith-enabled" name="stonith-enabled" value="true"/>
       </cluster_property_set>
     </crm_config>
     <nodes/>
     <resources>
       <primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
         <meta_attributes id="dummy-meta_attributes">
           <nvpair id="dummy-meta_attributes-target-role" name="target-role" value="Started"/>
         </meta_attributes>
       </primitive>
       <primitive id="dummy_desc" class="ocf" provider="pacemaker" type="Dummy" description="resource with description"/>
     </resources>
     <constraints/>
   </configuration>
   <status/>
 </cib>
 =#=#=#= Begin test: niceguy: Replace - remove acls =#=#=#=
 pcmk__check_acl 	trace: Default ACL denies user 'niceguy' read/write access to /cib[@epoch]
 pcmk__check_acl 	trace: Default ACL denies user 'niceguy' read/write access to /cib/configuration/acls
 Call failed: Permission denied
 =#=#=#= End test: niceguy: Replace - remove acls - Insufficient privileges (4) =#=#=#=
 * Passed: cibadmin              - niceguy: Replace - remove acls
 <cib epoch="17" num_updates="0" admin_epoch="0">
   <configuration>
     <crm_config>
       <cluster_property_set id="cib-bootstrap-options">
         <nvpair id="cib-bootstrap-options-enable-acl" name="enable-acl" value="true"/>
         <nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
         <nvpair id="cib-bootstrap-options-stonith-enabled" name="stonith-enabled" value="true"/>
       </cluster_property_set>
     </crm_config>
     <nodes/>
     <resources>
       <primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
         <meta_attributes id="dummy-meta_attributes">
           <nvpair id="dummy-meta_attributes-target-role" name="target-role" value="Started"/>
         </meta_attributes>
       </primitive>
       <primitive id="dummy_desc" class="ocf" provider="pacemaker" type="Dummy" description="resource with description"/>
       <primitive id="dummy2" class="ocf" provider="pacemaker" type="Dummy"/>
     </resources>
     <constraints/>
     <acls>
       <acl_target id="l33t-haxor">
         <role id="nothing"/>
       </acl_target>
       <acl_target id="niceguy">
         <role id="observer"/>
       </acl_target>
       <acl_target id="bob">
         <role id="admin"/>
       </acl_target>
       <acl_target id="joe">
         <role id="super_user"/>
       </acl_target>
       <acl_target id="mike">
         <role id="rsc_writer"/>
       </acl_target>
       <acl_target id="chris">
         <role id="rsc_denied"/>
       </acl_target>
       <acl_role id="nothing">
         <acl_permission id="nothing-deny" kind="deny" xpath="/cib"/>
       </acl_role>
       <acl_role id="observer">
         <acl_permission id="observer-read-1" kind="read" xpath="/cib"/>
         <acl_permission id="observer-write-1" kind="write" xpath="//nvpair[@name='stonith-enabled']"/>
         <acl_permission id="observer-write-2" kind="write" xpath="//nvpair[@name='target-role']"/>
       </acl_role>
       <acl_role id="admin">
         <acl_permission id="admin-read-1" kind="read" xpath="/cib"/>
         <acl_permission id="admin-write-1" kind="write" xpath="//resources"/>
       </acl_role>
       <acl_role id="super_user">
         <acl_permission id="super_user-write-1" kind="write" xpath="/cib"/>
       </acl_role>
       <acl_role id="rsc_writer">
         <acl_permission id="rsc-writer-deny-1" kind="deny" xpath="/cib"/>
         <acl_permission id="rsc-writer-write-1" kind="write" xpath="//resources"/>
       </acl_role>
       <acl_role id="rsc_denied">
         <acl_permission id="rsc-denied-write-1" kind="write" xpath="/cib"/>
         <acl_permission id="rsc-denied-deny-1" kind="deny" xpath="//resources"/>
       </acl_role>
       <acl_role id="badidea-role">
         <acl_permission id="badidea-resources" kind="read" xpath="//meta_attributes"/>
       </acl_role>
       <acl_target id="badidea">
         <role id="badidea-role"/>
       </acl_target>
       <acl_role id="betteridea-role">
         <acl_permission id="betteridea-nothing" kind="deny" xpath="/cib"/>
         <acl_permission id="betteridea-resources" kind="read" xpath="//meta_attributes"/>
       </acl_role>
       <acl_target id="betteridea">
         <role id="betteridea-role"/>
       </acl_target>
     </acls>
   </configuration>
   <status/>
 </cib>
 =#=#=#= Begin test: niceguy: Replace - create resource =#=#=#=
 pcmk__check_acl 	trace: Default ACL denies user 'niceguy' read/write access to /cib[@epoch]
 pcmk__check_acl 	trace: Default ACL denies user 'niceguy' read/write access to /cib/configuration/resources/primitive[@id='dummy2']
 pcmk__apply_creation_acl 	trace: ACLs disallow creation of <primitive> with id="dummy2"
 Call failed: Permission denied
 =#=#=#= End test: niceguy: Replace - create resource - Insufficient privileges (4) =#=#=#=
 * Passed: cibadmin              - niceguy: Replace - create resource
 <cib epoch="17" num_updates="0" admin_epoch="0">
   <configuration>
     <crm_config>
       <cluster_property_set id="cib-bootstrap-options">
         <nvpair id="cib-bootstrap-options-enable-acl" name="enable-acl" value="false"/>
         <nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
         <nvpair id="cib-bootstrap-options-stonith-enabled" name="stonith-enabled" value="true"/>
       </cluster_property_set>
     </crm_config>
     <nodes/>
     <resources>
       <primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
         <meta_attributes id="dummy-meta_attributes">
           <nvpair id="dummy-meta_attributes-target-role" name="target-role" value="Started"/>
         </meta_attributes>
       </primitive>
       <primitive id="dummy_desc" class="ocf" provider="pacemaker" type="Dummy" description="resource with description"/>
     </resources>
     <constraints/>
     <acls>
       <acl_target id="l33t-haxor">
         <role id="nothing"/>
       </acl_target>
       <acl_target id="niceguy">
         <role id="observer"/>
       </acl_target>
       <acl_target id="bob">
         <role id="admin"/>
       </acl_target>
       <acl_target id="joe">
         <role id="super_user"/>
       </acl_target>
       <acl_target id="mike">
         <role id="rsc_writer"/>
       </acl_target>
       <acl_target id="chris">
         <role id="rsc_denied"/>
       </acl_target>
       <acl_role id="nothing">
         <acl_permission id="nothing-deny" kind="deny" xpath="/cib"/>
       </acl_role>
       <acl_role id="observer">
         <acl_permission id="observer-read-1" kind="read" xpath="/cib"/>
         <acl_permission id="observer-write-1" kind="write" xpath="//nvpair[@name='stonith-enabled']"/>
         <acl_permission id="observer-write-2" kind="write" xpath="//nvpair[@name='target-role']"/>
       </acl_role>
       <acl_role id="admin">
         <acl_permission id="admin-read-1" kind="read" xpath="/cib"/>
         <acl_permission id="admin-write-1" kind="write" xpath="//resources"/>
       </acl_role>
       <acl_role id="super_user">
         <acl_permission id="super_user-write-1" kind="write" xpath="/cib"/>
       </acl_role>
       <acl_role id="rsc_writer">
         <acl_permission id="rsc-writer-deny-1" kind="deny" xpath="/cib"/>
         <acl_permission id="rsc-writer-write-1" kind="write" xpath="//resources"/>
       </acl_role>
       <acl_role id="rsc_denied">
         <acl_permission id="rsc-denied-write-1" kind="write" xpath="/cib"/>
         <acl_permission id="rsc-denied-deny-1" kind="deny" xpath="//resources"/>
       </acl_role>
       <acl_role id="badidea-role">
         <acl_permission id="badidea-resources" kind="read" xpath="//meta_attributes"/>
       </acl_role>
       <acl_target id="badidea">
         <role id="badidea-role"/>
       </acl_target>
       <acl_role id="betteridea-role">
         <acl_permission id="betteridea-nothing" kind="deny" xpath="/cib"/>
         <acl_permission id="betteridea-resources" kind="read" xpath="//meta_attributes"/>
       </acl_role>
       <acl_target id="betteridea">
         <role id="betteridea-role"/>
       </acl_target>
     </acls>
   </configuration>
   <status/>
 </cib>
 =#=#=#= Begin test: niceguy: Replace - modify attribute (deny) =#=#=#=
 pcmk__check_acl 	trace: Default ACL denies user 'niceguy' read/write access to /cib[@epoch]
 pcmk__check_acl 	trace: Default ACL denies user 'niceguy' read/write access to /cib/configuration/crm_config/cluster_property_set[@id='cib-bootstrap-options']/nvpair[@id='cib-bootstrap-options-enable-acl'][@value]
 Call failed: Permission denied
 =#=#=#= End test: niceguy: Replace - modify attribute (deny) - Insufficient privileges (4) =#=#=#=
 * Passed: cibadmin              - niceguy: Replace - modify attribute (deny)
 <cib epoch="17" num_updates="0" admin_epoch="0">
   <configuration>
     <crm_config>
       <cluster_property_set id="cib-bootstrap-options">
         <nvpair id="cib-bootstrap-options-enable-acl" name="enable-acl" value="true"/>
         <nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
         <nvpair id="cib-bootstrap-options-stonith-enabled" name="stonith-enabled" value="true"/>
       </cluster_property_set>
     </crm_config>
     <nodes/>
     <resources>
       <primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy">
         <meta_attributes id="dummy-meta_attributes">
           <nvpair id="dummy-meta_attributes-target-role" name="target-role" value="Started"/>
         </meta_attributes>
       </primitive>
       <primitive id="dummy_desc" class="ocf" provider="pacemaker" type="Dummy"/>
     </resources>
     <constraints/>
     <acls>
       <acl_target id="l33t-haxor">
         <role id="nothing"/>
       </acl_target>
       <acl_target id="niceguy">
         <role id="observer"/>
       </acl_target>
       <acl_target id="bob">
         <role id="admin"/>
       </acl_target>
       <acl_target id="joe">
         <role id="super_user"/>
       </acl_target>
       <acl_target id="mike">
         <role id="rsc_writer"/>
       </acl_target>
       <acl_target id="chris">
         <role id="rsc_denied"/>
       </acl_target>
       <acl_role id="nothing">
         <acl_permission id="nothing-deny" kind="deny" xpath="/cib"/>
       </acl_role>
       <acl_role id="observer">
         <acl_permission id="observer-read-1" kind="read" xpath="/cib"/>
         <acl_permission id="observer-write-1" kind="write" xpath="//nvpair[@name='stonith-enabled']"/>
         <acl_permission id="observer-write-2" kind="write" xpath="//nvpair[@name='target-role']"/>
       </acl_role>
       <acl_role id="admin">
         <acl_permission id="admin-read-1" kind="read" xpath="/cib"/>
         <acl_permission id="admin-write-1" kind="write" xpath="//resources"/>
       </acl_role>
       <acl_role id="super_user">
         <acl_permission id="super_user-write-1" kind="write" xpath="/cib"/>
       </acl_role>
       <acl_role id="rsc_writer">
         <acl_permission id="rsc-writer-deny-1" kind="deny" xpath="/cib"/>
         <acl_permission id="rsc-writer-write-1" kind="write" xpath="//resources"/>
       </acl_role>
       <acl_role id="rsc_denied">
         <acl_permission id="rsc-denied-write-1" kind="write" xpath="/cib"/>
         <acl_permission id="rsc-denied-deny-1" kind="deny" xpath="//resources"/>
       </acl_role>
       <acl_role id="badidea-role">
         <acl_permission id="badidea-resources" kind="read" xpath="//meta_attributes"/>
       </acl_role>
       <acl_target id="badidea">
         <role id="badidea-role"/>
       </acl_target>
       <acl_role id="betteridea-role">
         <acl_permission id="betteridea-nothing" kind="deny" xpath="/cib"/>
         <acl_permission id="betteridea-resources" kind="read" xpath="//meta_attributes"/>
       </acl_role>
       <acl_target id="betteridea">
         <role id="betteridea-role"/>
       </acl_target>
     </acls>
   </configuration>
   <status/>
 </cib>
 =#=#=#= Begin test: niceguy: Replace - delete attribute (deny) =#=#=#=
 pcmk__check_acl 	trace: Default ACL denies user 'niceguy' read/write access to /cib[@epoch]
 pcmk__check_acl 	trace: Default ACL denies user 'niceguy' read/write access to /cib/configuration/resources/primitive[@id='dummy_desc']
 Call failed: Permission denied
 =#=#=#= End test: niceguy: Replace - delete attribute (deny) - Insufficient privileges (4) =#=#=#=
 * Passed: cibadmin              - niceguy: Replace - delete attribute (deny)
 <cib epoch="17" num_updates="0" admin_epoch="0">
   <configuration>
     <crm_config>
       <cluster_property_set id="cib-bootstrap-options">
         <nvpair id="cib-bootstrap-options-enable-acl" name="enable-acl" value="true"/>
         <nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
         <nvpair id="cib-bootstrap-options-stonith-enabled" name="stonith-enabled" value="true"/>
       </cluster_property_set>
     </crm_config>
     <nodes/>
     <resources>
       <primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy" description="nothing interesting">
         <meta_attributes id="dummy-meta_attributes">
           <nvpair id="dummy-meta_attributes-target-role" name="target-role" value="Started"/>
         </meta_attributes>
       </primitive>
       <primitive id="dummy_desc" class="ocf" provider="pacemaker" type="Dummy" description="resource with description"/>
     </resources>
     <constraints/>
     <acls>
       <acl_target id="l33t-haxor">
         <role id="nothing"/>
       </acl_target>
       <acl_target id="niceguy">
         <role id="observer"/>
       </acl_target>
       <acl_target id="bob">
         <role id="admin"/>
       </acl_target>
       <acl_target id="joe">
         <role id="super_user"/>
       </acl_target>
       <acl_target id="mike">
         <role id="rsc_writer"/>
       </acl_target>
       <acl_target id="chris">
         <role id="rsc_denied"/>
       </acl_target>
       <acl_role id="nothing">
         <acl_permission id="nothing-deny" kind="deny" xpath="/cib"/>
       </acl_role>
       <acl_role id="observer">
         <acl_permission id="observer-read-1" kind="read" xpath="/cib"/>
         <acl_permission id="observer-write-1" kind="write" xpath="//nvpair[@name='stonith-enabled']"/>
         <acl_permission id="observer-write-2" kind="write" xpath="//nvpair[@name='target-role']"/>
       </acl_role>
       <acl_role id="admin">
         <acl_permission id="admin-read-1" kind="read" xpath="/cib"/>
         <acl_permission id="admin-write-1" kind="write" xpath="//resources"/>
       </acl_role>
       <acl_role id="super_user">
         <acl_permission id="super_user-write-1" kind="write" xpath="/cib"/>
       </acl_role>
       <acl_role id="rsc_writer">
         <acl_permission id="rsc-writer-deny-1" kind="deny" xpath="/cib"/>
         <acl_permission id="rsc-writer-write-1" kind="write" xpath="//resources"/>
       </acl_role>
       <acl_role id="rsc_denied">
         <acl_permission id="rsc-denied-write-1" kind="write" xpath="/cib"/>
         <acl_permission id="rsc-denied-deny-1" kind="deny" xpath="//resources"/>
       </acl_role>
       <acl_role id="badidea-role">
         <acl_permission id="badidea-resources" kind="read" xpath="//meta_attributes"/>
       </acl_role>
       <acl_target id="badidea">
         <role id="badidea-role"/>
       </acl_target>
       <acl_role id="betteridea-role">
         <acl_permission id="betteridea-nothing" kind="deny" xpath="/cib"/>
         <acl_permission id="betteridea-resources" kind="read" xpath="//meta_attributes"/>
       </acl_role>
       <acl_target id="betteridea">
         <role id="betteridea-role"/>
       </acl_target>
     </acls>
   </configuration>
   <status/>
 </cib>
 =#=#=#= Begin test: niceguy: Replace - create attribute (deny) =#=#=#=
 pcmk__check_acl 	trace: Default ACL denies user 'niceguy' read/write access to /cib[@epoch]
 pcmk__check_acl 	trace: Default ACL denies user 'niceguy' read/write access to /cib/configuration/resources/primitive[@id='dummy'][@description]
 Call failed: Permission denied
 =#=#=#= End test: niceguy: Replace - create attribute (deny) - Insufficient privileges (4) =#=#=#=
 * Passed: cibadmin              - niceguy: Replace - create attribute (deny)
 <cib epoch="17" num_updates="0" admin_epoch="0">
   <configuration>
     <crm_config>
       <cluster_property_set id="cib-bootstrap-options">
         <nvpair id="cib-bootstrap-options-enable-acl" name="enable-acl" value="true"/>
         <nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
         <nvpair id="cib-bootstrap-options-stonith-enabled" name="stonith-enabled" value="true"/>
       </cluster_property_set>
     </crm_config>
     <nodes/>
     <resources>
       <primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy" description="nothing interesting">
         <meta_attributes id="dummy-meta_attributes">
           <nvpair id="dummy-meta_attributes-target-role" name="target-role" value="Started"/>
         </meta_attributes>
       </primitive>
       <primitive id="dummy_desc" class="ocf" provider="pacemaker" type="Dummy" description="resource with description"/>
     </resources>
     <constraints/>
     <acls>
       <acl_target id="l33t-haxor">
         <role id="nothing"/>
       </acl_target>
       <acl_target id="niceguy">
         <role id="observer"/>
       </acl_target>
       <acl_target id="bob">
         <role id="admin"/>
       </acl_target>
       <acl_target id="joe">
         <role id="super_user"/>
       </acl_target>
       <acl_target id="mike">
         <role id="rsc_writer"/>
       </acl_target>
       <acl_target id="chris">
         <role id="rsc_denied"/>
       </acl_target>
       <acl_role id="nothing">
         <acl_permission id="nothing-deny" kind="deny" xpath="/cib"/>
       </acl_role>
       <acl_role id="observer">
         <acl_permission id="observer-read-1" kind="read" xpath="/cib"/>
         <acl_permission id="observer-write-1" kind="write" xpath="//nvpair[@name='stonith-enabled']"/>
         <acl_permission id="observer-write-2" kind="write" xpath="//nvpair[@name='target-role']"/>
       </acl_role>
       <acl_role id="admin">
         <acl_permission id="admin-read-1" kind="read" xpath="/cib"/>
         <acl_permission id="admin-write-1" kind="write" xpath="//resources"/>
       </acl_role>
       <acl_role id="super_user">
         <acl_permission id="super_user-write-1" kind="write" xpath="/cib"/>
       </acl_role>
       <acl_role id="rsc_writer">
         <acl_permission id="rsc-writer-deny-1" kind="deny" xpath="/cib"/>
         <acl_permission id="rsc-writer-write-1" kind="write" xpath="//resources"/>
       </acl_role>
       <acl_role id="rsc_denied">
         <acl_permission id="rsc-denied-write-1" kind="write" xpath="/cib"/>
         <acl_permission id="rsc-denied-deny-1" kind="deny" xpath="//resources"/>
       </acl_role>
       <acl_role id="badidea-role">
         <acl_permission id="badidea-resources" kind="read" xpath="//meta_attributes"/>
       </acl_role>
       <acl_target id="badidea">
         <role id="badidea-role"/>
       </acl_target>
       <acl_role id="betteridea-role">
         <acl_permission id="betteridea-nothing" kind="deny" xpath="/cib"/>
         <acl_permission id="betteridea-resources" kind="read" xpath="//meta_attributes"/>
       </acl_role>
       <acl_target id="betteridea">
         <role id="betteridea-role"/>
       </acl_target>
     </acls>
   </configuration>
   <status/>
 </cib>
 =#=#=#= Begin test: bob: Replace - create attribute (direct allow) =#=#=#=
 =#=#=#= End test: bob: Replace - create attribute (direct allow) - OK (0) =#=#=#=
 * Passed: cibadmin              - bob: Replace - create attribute (direct allow)
 <cib epoch="18" num_updates="0" admin_epoch="0">
   <configuration>
     <crm_config>
       <cluster_property_set id="cib-bootstrap-options">
         <nvpair id="cib-bootstrap-options-enable-acl" name="enable-acl" value="true"/>
         <nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
         <nvpair id="cib-bootstrap-options-stonith-enabled" name="stonith-enabled" value="true"/>
       </cluster_property_set>
     </crm_config>
     <nodes/>
     <resources>
       <primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy" description="something interesting">
         <meta_attributes id="dummy-meta_attributes">
           <nvpair id="dummy-meta_attributes-target-role" name="target-role" value="Started"/>
         </meta_attributes>
       </primitive>
       <primitive id="dummy_desc" class="ocf" provider="pacemaker" type="Dummy" description="resource with description"/>
     </resources>
     <constraints/>
     <acls>
       <acl_target id="l33t-haxor">
         <role id="nothing"/>
       </acl_target>
       <acl_target id="niceguy">
         <role id="observer"/>
       </acl_target>
       <acl_target id="bob">
         <role id="admin"/>
       </acl_target>
       <acl_target id="joe">
         <role id="super_user"/>
       </acl_target>
       <acl_target id="mike">
         <role id="rsc_writer"/>
       </acl_target>
       <acl_target id="chris">
         <role id="rsc_denied"/>
       </acl_target>
       <acl_role id="nothing">
         <acl_permission id="nothing-deny" kind="deny" xpath="/cib"/>
       </acl_role>
       <acl_role id="observer">
         <acl_permission id="observer-read-1" kind="read" xpath="/cib"/>
         <acl_permission id="observer-write-1" kind="write" xpath="//nvpair[@name='stonith-enabled']"/>
         <acl_permission id="observer-write-2" kind="write" xpath="//nvpair[@name='target-role']"/>
       </acl_role>
       <acl_role id="admin">
         <acl_permission id="admin-read-1" kind="read" xpath="/cib"/>
         <acl_permission id="admin-write-1" kind="write" xpath="//resources"/>
       </acl_role>
       <acl_role id="super_user">
         <acl_permission id="super_user-write-1" kind="write" xpath="/cib"/>
       </acl_role>
       <acl_role id="rsc_writer">
         <acl_permission id="rsc-writer-deny-1" kind="deny" xpath="/cib"/>
         <acl_permission id="rsc-writer-write-1" kind="write" xpath="//resources"/>
       </acl_role>
       <acl_role id="rsc_denied">
         <acl_permission id="rsc-denied-write-1" kind="write" xpath="/cib"/>
         <acl_permission id="rsc-denied-deny-1" kind="deny" xpath="//resources"/>
       </acl_role>
       <acl_role id="badidea-role">
         <acl_permission id="badidea-resources" kind="read" xpath="//meta_attributes"/>
       </acl_role>
       <acl_target id="badidea">
         <role id="badidea-role"/>
       </acl_target>
       <acl_role id="betteridea-role">
         <acl_permission id="betteridea-nothing" kind="deny" xpath="/cib"/>
         <acl_permission id="betteridea-resources" kind="read" xpath="//meta_attributes"/>
       </acl_role>
       <acl_target id="betteridea">
         <role id="betteridea-role"/>
       </acl_target>
     </acls>
   </configuration>
   <status/>
 </cib>
 =#=#=#= Begin test: bob: Replace - modify attribute (direct allow) =#=#=#=
 =#=#=#= End test: bob: Replace - modify attribute (direct allow) - OK (0) =#=#=#=
 * Passed: cibadmin              - bob: Replace - modify attribute (direct allow)
 <cib epoch="19" num_updates="0" admin_epoch="0">
   <configuration>
     <crm_config>
       <cluster_property_set id="cib-bootstrap-options">
         <nvpair id="cib-bootstrap-options-enable-acl" name="enable-acl" value="true"/>
         <nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
         <nvpair id="cib-bootstrap-options-stonith-enabled" name="stonith-enabled" value="true"/>
       </cluster_property_set>
     </crm_config>
     <nodes/>
     <resources>
       <primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy"/>
       <primitive id="dummy_desc" class="ocf" provider="pacemaker" type="Dummy" description="resource with description"/>
     </resources>
     <constraints/>
     <acls>
       <acl_target id="l33t-haxor">
         <role id="nothing"/>
       </acl_target>
       <acl_target id="niceguy">
         <role id="observer"/>
       </acl_target>
       <acl_target id="bob">
         <role id="admin"/>
       </acl_target>
       <acl_target id="joe">
         <role id="super_user"/>
       </acl_target>
       <acl_target id="mike">
         <role id="rsc_writer"/>
       </acl_target>
       <acl_target id="chris">
         <role id="rsc_denied"/>
       </acl_target>
       <acl_role id="nothing">
         <acl_permission id="nothing-deny" kind="deny" xpath="/cib"/>
       </acl_role>
       <acl_role id="observer">
         <acl_permission id="observer-read-1" kind="read" xpath="/cib"/>
         <acl_permission id="observer-write-1" kind="write" xpath="//nvpair[@name='stonith-enabled']"/>
         <acl_permission id="observer-write-2" kind="write" xpath="//nvpair[@name='target-role']"/>
       </acl_role>
       <acl_role id="admin">
         <acl_permission id="admin-read-1" kind="read" xpath="/cib"/>
         <acl_permission id="admin-write-1" kind="write" xpath="//resources"/>
       </acl_role>
       <acl_role id="super_user">
         <acl_permission id="super_user-write-1" kind="write" xpath="/cib"/>
       </acl_role>
       <acl_role id="rsc_writer">
         <acl_permission id="rsc-writer-deny-1" kind="deny" xpath="/cib"/>
         <acl_permission id="rsc-writer-write-1" kind="write" xpath="//resources"/>
       </acl_role>
       <acl_role id="rsc_denied">
         <acl_permission id="rsc-denied-write-1" kind="write" xpath="/cib"/>
         <acl_permission id="rsc-denied-deny-1" kind="deny" xpath="//resources"/>
       </acl_role>
       <acl_role id="badidea-role">
         <acl_permission id="badidea-resources" kind="read" xpath="//meta_attributes"/>
       </acl_role>
       <acl_target id="badidea">
         <role id="badidea-role"/>
       </acl_target>
       <acl_role id="betteridea-role">
         <acl_permission id="betteridea-nothing" kind="deny" xpath="/cib"/>
         <acl_permission id="betteridea-resources" kind="read" xpath="//meta_attributes"/>
       </acl_role>
       <acl_target id="betteridea">
         <role id="betteridea-role"/>
       </acl_target>
     </acls>
   </configuration>
   <status/>
 </cib>
 =#=#=#= Begin test: bob: Replace - delete attribute (direct allow) =#=#=#=
 =#=#=#= End test: bob: Replace - delete attribute (direct allow) - OK (0) =#=#=#=
 * Passed: cibadmin              - bob: Replace - delete attribute (direct allow)
 <cib epoch="20" num_updates="0" admin_epoch="0">
   <configuration>
     <crm_config>
       <cluster_property_set id="cib-bootstrap-options">
         <nvpair id="cib-bootstrap-options-enable-acl" name="enable-acl" value="true"/>
         <nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
         <nvpair id="cib-bootstrap-options-stonith-enabled" name="stonith-enabled" value="true"/>
       </cluster_property_set>
     </crm_config>
     <nodes/>
     <resources>
       <primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy" description="nothing interesting"/>
       <primitive id="dummy_desc" class="ocf" provider="pacemaker" type="Dummy" description="resource with description"/>
     </resources>
     <constraints/>
     <acls>
       <acl_target id="l33t-haxor">
         <role id="nothing"/>
       </acl_target>
       <acl_target id="niceguy">
         <role id="observer"/>
       </acl_target>
       <acl_target id="bob">
         <role id="admin"/>
       </acl_target>
       <acl_target id="joe">
         <role id="super_user"/>
       </acl_target>
       <acl_target id="mike">
         <role id="rsc_writer"/>
       </acl_target>
       <acl_target id="chris">
         <role id="rsc_denied"/>
       </acl_target>
       <acl_role id="nothing">
         <acl_permission id="nothing-deny" kind="deny" xpath="/cib"/>
       </acl_role>
       <acl_role id="observer">
         <acl_permission id="observer-read-1" kind="read" xpath="/cib"/>
         <acl_permission id="observer-write-1" kind="write" xpath="//nvpair[@name='stonith-enabled']"/>
         <acl_permission id="observer-write-2" kind="write" xpath="//nvpair[@name='target-role']"/>
       </acl_role>
       <acl_role id="admin">
         <acl_permission id="admin-read-1" kind="read" xpath="/cib"/>
         <acl_permission id="admin-write-1" kind="write" xpath="//resources"/>
       </acl_role>
       <acl_role id="super_user">
         <acl_permission id="super_user-write-1" kind="write" xpath="/cib"/>
       </acl_role>
       <acl_role id="rsc_writer">
         <acl_permission id="rsc-writer-deny-1" kind="deny" xpath="/cib"/>
         <acl_permission id="rsc-writer-write-1" kind="write" xpath="//resources"/>
       </acl_role>
       <acl_role id="rsc_denied">
         <acl_permission id="rsc-denied-write-1" kind="write" xpath="/cib"/>
         <acl_permission id="rsc-denied-deny-1" kind="deny" xpath="//resources"/>
       </acl_role>
       <acl_role id="badidea-role">
         <acl_permission id="badidea-resources" kind="read" xpath="//meta_attributes"/>
       </acl_role>
       <acl_target id="badidea">
         <role id="badidea-role"/>
       </acl_target>
       <acl_role id="betteridea-role">
         <acl_permission id="betteridea-nothing" kind="deny" xpath="/cib"/>
         <acl_permission id="betteridea-resources" kind="read" xpath="//meta_attributes"/>
       </acl_role>
       <acl_target id="betteridea">
         <role id="betteridea-role"/>
       </acl_target>
     </acls>
   </configuration>
   <status/>
 </cib>
 =#=#=#= Begin test: joe: Replace - create attribute (inherited allow) =#=#=#=
 =#=#=#= End test: joe: Replace - create attribute (inherited allow) - OK (0) =#=#=#=
 * Passed: cibadmin              - joe: Replace - create attribute (inherited allow)
 <cib epoch="21" num_updates="0" admin_epoch="0">
   <configuration>
     <crm_config>
       <cluster_property_set id="cib-bootstrap-options">
         <nvpair id="cib-bootstrap-options-enable-acl" name="enable-acl" value="true"/>
         <nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
         <nvpair id="cib-bootstrap-options-stonith-enabled" name="stonith-enabled" value="true"/>
       </cluster_property_set>
     </crm_config>
     <nodes/>
     <resources>
       <primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy" description="something interesting"/>
       <primitive id="dummy_desc" class="ocf" provider="pacemaker" type="Dummy" description="resource with description"/>
     </resources>
     <constraints/>
     <acls>
       <acl_target id="l33t-haxor">
         <role id="nothing"/>
       </acl_target>
       <acl_target id="niceguy">
         <role id="observer"/>
       </acl_target>
       <acl_target id="bob">
         <role id="admin"/>
       </acl_target>
       <acl_target id="joe">
         <role id="super_user"/>
       </acl_target>
       <acl_target id="mike">
         <role id="rsc_writer"/>
       </acl_target>
       <acl_target id="chris">
         <role id="rsc_denied"/>
       </acl_target>
       <acl_role id="nothing">
         <acl_permission id="nothing-deny" kind="deny" xpath="/cib"/>
       </acl_role>
       <acl_role id="observer">
         <acl_permission id="observer-read-1" kind="read" xpath="/cib"/>
         <acl_permission id="observer-write-1" kind="write" xpath="//nvpair[@name='stonith-enabled']"/>
         <acl_permission id="observer-write-2" kind="write" xpath="//nvpair[@name='target-role']"/>
       </acl_role>
       <acl_role id="admin">
         <acl_permission id="admin-read-1" kind="read" xpath="/cib"/>
         <acl_permission id="admin-write-1" kind="write" xpath="//resources"/>
       </acl_role>
       <acl_role id="super_user">
         <acl_permission id="super_user-write-1" kind="write" xpath="/cib"/>
       </acl_role>
       <acl_role id="rsc_writer">
         <acl_permission id="rsc-writer-deny-1" kind="deny" xpath="/cib"/>
         <acl_permission id="rsc-writer-write-1" kind="write" xpath="//resources"/>
       </acl_role>
       <acl_role id="rsc_denied">
         <acl_permission id="rsc-denied-write-1" kind="write" xpath="/cib"/>
         <acl_permission id="rsc-denied-deny-1" kind="deny" xpath="//resources"/>
       </acl_role>
       <acl_role id="badidea-role">
         <acl_permission id="badidea-resources" kind="read" xpath="//meta_attributes"/>
       </acl_role>
       <acl_target id="badidea">
         <role id="badidea-role"/>
       </acl_target>
       <acl_role id="betteridea-role">
         <acl_permission id="betteridea-nothing" kind="deny" xpath="/cib"/>
         <acl_permission id="betteridea-resources" kind="read" xpath="//meta_attributes"/>
       </acl_role>
       <acl_target id="betteridea">
         <role id="betteridea-role"/>
       </acl_target>
     </acls>
   </configuration>
   <status/>
 </cib>
 =#=#=#= Begin test: joe: Replace - modify attribute (inherited allow) =#=#=#=
 =#=#=#= End test: joe: Replace - modify attribute (inherited allow) - OK (0) =#=#=#=
 * Passed: cibadmin              - joe: Replace - modify attribute (inherited allow)
 <cib epoch="22" num_updates="0" admin_epoch="0">
   <configuration>
     <crm_config>
       <cluster_property_set id="cib-bootstrap-options">
         <nvpair id="cib-bootstrap-options-enable-acl" name="enable-acl" value="true"/>
         <nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
         <nvpair id="cib-bootstrap-options-stonith-enabled" name="stonith-enabled" value="true"/>
       </cluster_property_set>
     </crm_config>
     <nodes/>
     <resources>
       <primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy"/>
       <primitive id="dummy_desc" class="ocf" provider="pacemaker" type="Dummy" description="resource with description"/>
     </resources>
     <constraints/>
     <acls>
       <acl_target id="l33t-haxor">
         <role id="nothing"/>
       </acl_target>
       <acl_target id="niceguy">
         <role id="observer"/>
       </acl_target>
       <acl_target id="bob">
         <role id="admin"/>
       </acl_target>
       <acl_target id="joe">
         <role id="super_user"/>
       </acl_target>
       <acl_target id="mike">
         <role id="rsc_writer"/>
       </acl_target>
       <acl_target id="chris">
         <role id="rsc_denied"/>
       </acl_target>
       <acl_role id="nothing">
         <acl_permission id="nothing-deny" kind="deny" xpath="/cib"/>
       </acl_role>
       <acl_role id="observer">
         <acl_permission id="observer-read-1" kind="read" xpath="/cib"/>
         <acl_permission id="observer-write-1" kind="write" xpath="//nvpair[@name='stonith-enabled']"/>
         <acl_permission id="observer-write-2" kind="write" xpath="//nvpair[@name='target-role']"/>
       </acl_role>
       <acl_role id="admin">
         <acl_permission id="admin-read-1" kind="read" xpath="/cib"/>
         <acl_permission id="admin-write-1" kind="write" xpath="//resources"/>
       </acl_role>
       <acl_role id="super_user">
         <acl_permission id="super_user-write-1" kind="write" xpath="/cib"/>
       </acl_role>
       <acl_role id="rsc_writer">
         <acl_permission id="rsc-writer-deny-1" kind="deny" xpath="/cib"/>
         <acl_permission id="rsc-writer-write-1" kind="write" xpath="//resources"/>
       </acl_role>
       <acl_role id="rsc_denied">
         <acl_permission id="rsc-denied-write-1" kind="write" xpath="/cib"/>
         <acl_permission id="rsc-denied-deny-1" kind="deny" xpath="//resources"/>
       </acl_role>
       <acl_role id="badidea-role">
         <acl_permission id="badidea-resources" kind="read" xpath="//meta_attributes"/>
       </acl_role>
       <acl_target id="badidea">
         <role id="badidea-role"/>
       </acl_target>
       <acl_role id="betteridea-role">
         <acl_permission id="betteridea-nothing" kind="deny" xpath="/cib"/>
         <acl_permission id="betteridea-resources" kind="read" xpath="//meta_attributes"/>
       </acl_role>
       <acl_target id="betteridea">
         <role id="betteridea-role"/>
       </acl_target>
     </acls>
   </configuration>
   <status/>
 </cib>
 =#=#=#= Begin test: joe: Replace - delete attribute (inherited allow) =#=#=#=
 =#=#=#= End test: joe: Replace - delete attribute (inherited allow) - OK (0) =#=#=#=
 * Passed: cibadmin              - joe: Replace - delete attribute (inherited allow)
 <cib epoch="23" num_updates="0" admin_epoch="0">
   <configuration>
     <crm_config>
       <cluster_property_set id="cib-bootstrap-options">
         <nvpair id="cib-bootstrap-options-enable-acl" name="enable-acl" value="true"/>
         <nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
         <nvpair id="cib-bootstrap-options-stonith-enabled" name="stonith-enabled" value="true"/>
       </cluster_property_set>
     </crm_config>
     <nodes/>
     <resources>
       <primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy" description="nothing interesting"/>
       <primitive id="dummy_desc" class="ocf" provider="pacemaker" type="Dummy" description="resource with description"/>
     </resources>
     <constraints/>
     <acls>
       <acl_target id="l33t-haxor">
         <role id="nothing"/>
       </acl_target>
       <acl_target id="niceguy">
         <role id="observer"/>
       </acl_target>
       <acl_target id="bob">
         <role id="admin"/>
       </acl_target>
       <acl_target id="joe">
         <role id="super_user"/>
       </acl_target>
       <acl_target id="mike">
         <role id="rsc_writer"/>
       </acl_target>
       <acl_target id="chris">
         <role id="rsc_denied"/>
       </acl_target>
       <acl_role id="nothing">
         <acl_permission id="nothing-deny" kind="deny" xpath="/cib"/>
       </acl_role>
       <acl_role id="observer">
         <acl_permission id="observer-read-1" kind="read" xpath="/cib"/>
         <acl_permission id="observer-write-1" kind="write" xpath="//nvpair[@name='stonith-enabled']"/>
         <acl_permission id="observer-write-2" kind="write" xpath="//nvpair[@name='target-role']"/>
       </acl_role>
       <acl_role id="admin">
         <acl_permission id="admin-read-1" kind="read" xpath="/cib"/>
         <acl_permission id="admin-write-1" kind="write" xpath="//resources"/>
       </acl_role>
       <acl_role id="super_user">
         <acl_permission id="super_user-write-1" kind="write" xpath="/cib"/>
       </acl_role>
       <acl_role id="rsc_writer">
         <acl_permission id="rsc-writer-deny-1" kind="deny" xpath="/cib"/>
         <acl_permission id="rsc-writer-write-1" kind="write" xpath="//resources"/>
       </acl_role>
       <acl_role id="rsc_denied">
         <acl_permission id="rsc-denied-write-1" kind="write" xpath="/cib"/>
         <acl_permission id="rsc-denied-deny-1" kind="deny" xpath="//resources"/>
       </acl_role>
       <acl_role id="badidea-role">
         <acl_permission id="badidea-resources" kind="read" xpath="//meta_attributes"/>
       </acl_role>
       <acl_target id="badidea">
         <role id="badidea-role"/>
       </acl_target>
       <acl_role id="betteridea-role">
         <acl_permission id="betteridea-nothing" kind="deny" xpath="/cib"/>
         <acl_permission id="betteridea-resources" kind="read" xpath="//meta_attributes"/>
       </acl_role>
       <acl_target id="betteridea">
         <role id="betteridea-role"/>
       </acl_target>
     </acls>
   </configuration>
   <status/>
 </cib>
 =#=#=#= Begin test: mike: Replace - create attribute (allow overrides deny) =#=#=#=
 =#=#=#= End test: mike: Replace - create attribute (allow overrides deny) - OK (0) =#=#=#=
 * Passed: cibadmin              - mike: Replace - create attribute (allow overrides deny)
 <cib epoch="24" num_updates="0" admin_epoch="0">
   <configuration>
     <crm_config>
       <cluster_property_set id="cib-bootstrap-options">
         <nvpair id="cib-bootstrap-options-enable-acl" name="enable-acl" value="true"/>
         <nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
         <nvpair id="cib-bootstrap-options-stonith-enabled" name="stonith-enabled" value="true"/>
       </cluster_property_set>
     </crm_config>
     <nodes/>
     <resources>
       <primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy" description="something interesting"/>
       <primitive id="dummy_desc" class="ocf" provider="pacemaker" type="Dummy" description="resource with description"/>
     </resources>
     <constraints/>
     <acls>
       <acl_target id="l33t-haxor">
         <role id="nothing"/>
       </acl_target>
       <acl_target id="niceguy">
         <role id="observer"/>
       </acl_target>
       <acl_target id="bob">
         <role id="admin"/>
       </acl_target>
       <acl_target id="joe">
         <role id="super_user"/>
       </acl_target>
       <acl_target id="mike">
         <role id="rsc_writer"/>
       </acl_target>
       <acl_target id="chris">
         <role id="rsc_denied"/>
       </acl_target>
       <acl_role id="nothing">
         <acl_permission id="nothing-deny" kind="deny" xpath="/cib"/>
       </acl_role>
       <acl_role id="observer">
         <acl_permission id="observer-read-1" kind="read" xpath="/cib"/>
         <acl_permission id="observer-write-1" kind="write" xpath="//nvpair[@name='stonith-enabled']"/>
         <acl_permission id="observer-write-2" kind="write" xpath="//nvpair[@name='target-role']"/>
       </acl_role>
       <acl_role id="admin">
         <acl_permission id="admin-read-1" kind="read" xpath="/cib"/>
         <acl_permission id="admin-write-1" kind="write" xpath="//resources"/>
       </acl_role>
       <acl_role id="super_user">
         <acl_permission id="super_user-write-1" kind="write" xpath="/cib"/>
       </acl_role>
       <acl_role id="rsc_writer">
         <acl_permission id="rsc-writer-deny-1" kind="deny" xpath="/cib"/>
         <acl_permission id="rsc-writer-write-1" kind="write" xpath="//resources"/>
       </acl_role>
       <acl_role id="rsc_denied">
         <acl_permission id="rsc-denied-write-1" kind="write" xpath="/cib"/>
         <acl_permission id="rsc-denied-deny-1" kind="deny" xpath="//resources"/>
       </acl_role>
       <acl_role id="badidea-role">
         <acl_permission id="badidea-resources" kind="read" xpath="//meta_attributes"/>
       </acl_role>
       <acl_target id="badidea">
         <role id="badidea-role"/>
       </acl_target>
       <acl_role id="betteridea-role">
         <acl_permission id="betteridea-nothing" kind="deny" xpath="/cib"/>
         <acl_permission id="betteridea-resources" kind="read" xpath="//meta_attributes"/>
       </acl_role>
       <acl_target id="betteridea">
         <role id="betteridea-role"/>
       </acl_target>
     </acls>
   </configuration>
   <status/>
 </cib>
 =#=#=#= Begin test: mike: Replace - modify attribute (allow overrides deny) =#=#=#=
 =#=#=#= End test: mike: Replace - modify attribute (allow overrides deny) - OK (0) =#=#=#=
 * Passed: cibadmin              - mike: Replace - modify attribute (allow overrides deny)
 <cib epoch="25" num_updates="0" admin_epoch="0">
   <configuration>
     <crm_config>
       <cluster_property_set id="cib-bootstrap-options">
         <nvpair id="cib-bootstrap-options-enable-acl" name="enable-acl" value="true"/>
         <nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
         <nvpair id="cib-bootstrap-options-stonith-enabled" name="stonith-enabled" value="true"/>
       </cluster_property_set>
     </crm_config>
     <nodes/>
     <resources>
       <primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy"/>
       <primitive id="dummy_desc" class="ocf" provider="pacemaker" type="Dummy" description="resource with description"/>
     </resources>
     <constraints/>
     <acls>
       <acl_target id="l33t-haxor">
         <role id="nothing"/>
       </acl_target>
       <acl_target id="niceguy">
         <role id="observer"/>
       </acl_target>
       <acl_target id="bob">
         <role id="admin"/>
       </acl_target>
       <acl_target id="joe">
         <role id="super_user"/>
       </acl_target>
       <acl_target id="mike">
         <role id="rsc_writer"/>
       </acl_target>
       <acl_target id="chris">
         <role id="rsc_denied"/>
       </acl_target>
       <acl_role id="nothing">
         <acl_permission id="nothing-deny" kind="deny" xpath="/cib"/>
       </acl_role>
       <acl_role id="observer">
         <acl_permission id="observer-read-1" kind="read" xpath="/cib"/>
         <acl_permission id="observer-write-1" kind="write" xpath="//nvpair[@name='stonith-enabled']"/>
         <acl_permission id="observer-write-2" kind="write" xpath="//nvpair[@name='target-role']"/>
       </acl_role>
       <acl_role id="admin">
         <acl_permission id="admin-read-1" kind="read" xpath="/cib"/>
         <acl_permission id="admin-write-1" kind="write" xpath="//resources"/>
       </acl_role>
       <acl_role id="super_user">
         <acl_permission id="super_user-write-1" kind="write" xpath="/cib"/>
       </acl_role>
       <acl_role id="rsc_writer">
         <acl_permission id="rsc-writer-deny-1" kind="deny" xpath="/cib"/>
         <acl_permission id="rsc-writer-write-1" kind="write" xpath="//resources"/>
       </acl_role>
       <acl_role id="rsc_denied">
         <acl_permission id="rsc-denied-write-1" kind="write" xpath="/cib"/>
         <acl_permission id="rsc-denied-deny-1" kind="deny" xpath="//resources"/>
       </acl_role>
       <acl_role id="badidea-role">
         <acl_permission id="badidea-resources" kind="read" xpath="//meta_attributes"/>
       </acl_role>
       <acl_target id="badidea">
         <role id="badidea-role"/>
       </acl_target>
       <acl_role id="betteridea-role">
         <acl_permission id="betteridea-nothing" kind="deny" xpath="/cib"/>
         <acl_permission id="betteridea-resources" kind="read" xpath="//meta_attributes"/>
       </acl_role>
       <acl_target id="betteridea">
         <role id="betteridea-role"/>
       </acl_target>
     </acls>
   </configuration>
   <status/>
 </cib>
 =#=#=#= Begin test: mike: Replace - delete attribute (allow overrides deny) =#=#=#=
 =#=#=#= End test: mike: Replace - delete attribute (allow overrides deny) - OK (0) =#=#=#=
 * Passed: cibadmin              - mike: Replace - delete attribute (allow overrides deny)
 <cib epoch="25" num_updates="0" admin_epoch="0">
   <configuration>
     <crm_config>
       <cluster_property_set id="cib-bootstrap-options">
         <nvpair id="cib-bootstrap-options-enable-acl" name="enable-acl" value="true"/>
         <nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
         <nvpair id="cib-bootstrap-options-stonith-enabled" name="stonith-enabled" value="true"/>
       </cluster_property_set>
     </crm_config>
     <nodes/>
     <resources>
       <primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy"/>
       <primitive id="dummy_desc" class="ocf" provider="pacemaker" type="Dummy" description="resource with description"/>
     </resources>
     <constraints/>
     <acls>
       <acl_target id="l33t-haxor">
         <role id="nothing"/>
       </acl_target>
       <acl_target id="niceguy">
         <role id="observer"/>
       </acl_target>
       <acl_target id="bob">
         <role id="admin"/>
       </acl_target>
       <acl_target id="joe">
         <role id="super_user"/>
       </acl_target>
       <acl_target id="mike">
         <role id="rsc_writer"/>
       </acl_target>
       <acl_target id="chris">
         <role id="rsc_denied"/>
       </acl_target>
       <acl_role id="nothing">
         <acl_permission id="nothing-deny" kind="deny" xpath="/cib"/>
       </acl_role>
       <acl_role id="observer">
         <acl_permission id="observer-read-1" kind="read" xpath="/cib"/>
         <acl_permission id="observer-write-1" kind="write" xpath="//nvpair[@name='stonith-enabled']"/>
         <acl_permission id="observer-write-2" kind="write" xpath="//nvpair[@name='target-role']"/>
       </acl_role>
       <acl_role id="admin">
         <acl_permission id="admin-read-1" kind="read" xpath="/cib"/>
         <acl_permission id="admin-write-1" kind="write" xpath="//resources"/>
       </acl_role>
       <acl_role id="super_user">
         <acl_permission id="super_user-write-1" kind="write" xpath="/cib"/>
       </acl_role>
       <acl_role id="rsc_writer">
         <acl_permission id="rsc-writer-deny-1" kind="deny" xpath="/cib"/>
         <acl_permission id="rsc-writer-write-1" kind="write" xpath="//resources"/>
       </acl_role>
       <acl_role id="rsc_denied">
         <acl_permission id="rsc-denied-write-1" kind="write" xpath="/cib"/>
         <acl_permission id="rsc-denied-deny-1" kind="deny" xpath="//resources"/>
       </acl_role>
       <acl_role id="badidea-role">
         <acl_permission id="badidea-resources" kind="read" xpath="//meta_attributes"/>
       </acl_role>
       <acl_target id="badidea">
         <role id="badidea-role"/>
       </acl_target>
       <acl_role id="betteridea-role">
         <acl_permission id="betteridea-nothing" kind="deny" xpath="/cib"/>
         <acl_permission id="betteridea-resources" kind="read" xpath="//meta_attributes"/>
       </acl_role>
       <acl_target id="betteridea">
         <role id="betteridea-role"/>
       </acl_target>
     </acls>
   </configuration>
   <status/>
 </cib>
 =#=#=#= Begin test: mike: Create another resource =#=#=#=
 pcmk__apply_creation_acl 	trace: ACLs allow creation of <primitive> with id="dummy2"
 =#=#=#= Current cib after: mike: Create another resource =#=#=#=
 <cib epoch="26" num_updates="0" admin_epoch="0">
   <configuration>
     <crm_config>
       <cluster_property_set id="cib-bootstrap-options">
         <nvpair id="cib-bootstrap-options-enable-acl" name="enable-acl" value="true"/>
         <nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
         <nvpair id="cib-bootstrap-options-stonith-enabled" name="stonith-enabled" value="true"/>
       </cluster_property_set>
     </crm_config>
     <nodes/>
     <resources>
       <primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy"/>
       <primitive id="dummy_desc" class="ocf" provider="pacemaker" type="Dummy" description="resource with description"/>
       <primitive id="dummy2" class="ocf" provider="pacemaker" type="Dummy" description="nothing interesting"/>
     </resources>
     <constraints/>
     <acls>
       <acl_target id="l33t-haxor">
         <role id="nothing"/>
       </acl_target>
       <acl_target id="niceguy">
         <role id="observer"/>
       </acl_target>
       <acl_target id="bob">
         <role id="admin"/>
       </acl_target>
       <acl_target id="joe">
         <role id="super_user"/>
       </acl_target>
       <acl_target id="mike">
         <role id="rsc_writer"/>
       </acl_target>
       <acl_target id="chris">
         <role id="rsc_denied"/>
       </acl_target>
       <acl_role id="nothing">
         <acl_permission id="nothing-deny" kind="deny" xpath="/cib"/>
       </acl_role>
       <acl_role id="observer">
         <acl_permission id="observer-read-1" kind="read" xpath="/cib"/>
         <acl_permission id="observer-write-1" kind="write" xpath="//nvpair[@name='stonith-enabled']"/>
         <acl_permission id="observer-write-2" kind="write" xpath="//nvpair[@name='target-role']"/>
       </acl_role>
       <acl_role id="admin">
         <acl_permission id="admin-read-1" kind="read" xpath="/cib"/>
         <acl_permission id="admin-write-1" kind="write" xpath="//resources"/>
       </acl_role>
       <acl_role id="super_user">
         <acl_permission id="super_user-write-1" kind="write" xpath="/cib"/>
       </acl_role>
       <acl_role id="rsc_writer">
         <acl_permission id="rsc-writer-deny-1" kind="deny" xpath="/cib"/>
         <acl_permission id="rsc-writer-write-1" kind="write" xpath="//resources"/>
       </acl_role>
       <acl_role id="rsc_denied">
         <acl_permission id="rsc-denied-write-1" kind="write" xpath="/cib"/>
         <acl_permission id="rsc-denied-deny-1" kind="deny" xpath="//resources"/>
       </acl_role>
       <acl_role id="badidea-role">
         <acl_permission id="badidea-resources" kind="read" xpath="//meta_attributes"/>
       </acl_role>
       <acl_target id="badidea">
         <role id="badidea-role"/>
       </acl_target>
       <acl_role id="betteridea-role">
         <acl_permission id="betteridea-nothing" kind="deny" xpath="/cib"/>
         <acl_permission id="betteridea-resources" kind="read" xpath="//meta_attributes"/>
       </acl_role>
       <acl_target id="betteridea">
         <role id="betteridea-role"/>
       </acl_target>
     </acls>
   </configuration>
   <status/>
 </cib>
 =#=#=#= End test: mike: Create another resource - OK (0) =#=#=#=
 * Passed: cibadmin              - mike: Create another resource
 <cib epoch="27" num_updates="0" admin_epoch="0">
   <configuration>
     <crm_config>
       <cluster_property_set id="cib-bootstrap-options">
         <nvpair id="cib-bootstrap-options-enable-acl" name="enable-acl" value="true"/>
         <nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
         <nvpair id="cib-bootstrap-options-stonith-enabled" name="stonith-enabled" value="true"/>
       </cluster_property_set>
     </crm_config>
     <nodes/>
     <resources>
       <primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy" description="nothing interesting"/>
       <primitive id="dummy_desc" class="ocf" provider="pacemaker" type="Dummy" description="resource with description"/>
       <primitive id="dummy2" class="ocf" provider="pacemaker" type="Dummy" description="nothing interesting"/>
     </resources>
     <constraints/>
     <acls>
       <acl_target id="l33t-haxor">
         <role id="nothing"/>
       </acl_target>
       <acl_target id="niceguy">
         <role id="observer"/>
       </acl_target>
       <acl_target id="bob">
         <role id="admin"/>
       </acl_target>
       <acl_target id="joe">
         <role id="super_user"/>
       </acl_target>
       <acl_target id="mike">
         <role id="rsc_writer"/>
       </acl_target>
       <acl_target id="chris">
         <role id="rsc_denied"/>
       </acl_target>
       <acl_role id="nothing">
         <acl_permission id="nothing-deny" kind="deny" xpath="/cib"/>
       </acl_role>
       <acl_role id="observer">
         <acl_permission id="observer-read-1" kind="read" xpath="/cib"/>
         <acl_permission id="observer-write-1" kind="write" xpath="//nvpair[@name='stonith-enabled']"/>
         <acl_permission id="observer-write-2" kind="write" xpath="//nvpair[@name='target-role']"/>
       </acl_role>
       <acl_role id="admin">
         <acl_permission id="admin-read-1" kind="read" xpath="/cib"/>
         <acl_permission id="admin-write-1" kind="write" xpath="//resources"/>
       </acl_role>
       <acl_role id="super_user">
         <acl_permission id="super_user-write-1" kind="write" xpath="/cib"/>
       </acl_role>
       <acl_role id="rsc_writer">
         <acl_permission id="rsc-writer-deny-1" kind="deny" xpath="/cib"/>
         <acl_permission id="rsc-writer-write-1" kind="write" xpath="//resources"/>
       </acl_role>
       <acl_role id="rsc_denied">
         <acl_permission id="rsc-denied-write-1" kind="write" xpath="/cib"/>
         <acl_permission id="rsc-denied-deny-1" kind="deny" xpath="//resources"/>
       </acl_role>
       <acl_role id="badidea-role">
         <acl_permission id="badidea-resources" kind="read" xpath="//meta_attributes"/>
       </acl_role>
       <acl_target id="badidea">
         <role id="badidea-role"/>
       </acl_target>
       <acl_role id="betteridea-role">
         <acl_permission id="betteridea-nothing" kind="deny" xpath="/cib"/>
         <acl_permission id="betteridea-resources" kind="read" xpath="//meta_attributes"/>
       </acl_role>
       <acl_target id="betteridea">
         <role id="betteridea-role"/>
       </acl_target>
     </acls>
   </configuration>
   <status/>
 </cib>
 =#=#=#= Begin test: chris: Replace - create attribute (deny overrides allow) =#=#=#=
 pcmk__check_acl 	trace: Parent ACL denies user 'chris' read/write access to /cib/configuration/resources/primitive[@id='dummy'][@description]
 Call failed: Permission denied
 =#=#=#= End test: chris: Replace - create attribute (deny overrides allow) - Insufficient privileges (4) =#=#=#=
 * Passed: cibadmin              - chris: Replace - create attribute (deny overrides allow)
 <cib epoch="27" num_updates="0" admin_epoch="0">
   <configuration>
     <crm_config>
       <cluster_property_set id="cib-bootstrap-options">
         <nvpair id="cib-bootstrap-options-enable-acl" name="enable-acl" value="true"/>
         <nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
         <nvpair id="cib-bootstrap-options-stonith-enabled" name="stonith-enabled" value="true"/>
       </cluster_property_set>
     </crm_config>
     <nodes/>
     <resources>
       <primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy" description="something interesting"/>
       <primitive id="dummy_desc" class="ocf" provider="pacemaker" type="Dummy" description="resource with description"/>
       <primitive id="dummy2" class="ocf" provider="pacemaker" type="Dummy" description="nothing interesting"/>
     </resources>
     <constraints/>
     <acls>
       <acl_target id="l33t-haxor">
         <role id="nothing"/>
       </acl_target>
       <acl_target id="niceguy">
         <role id="observer"/>
       </acl_target>
       <acl_target id="bob">
         <role id="admin"/>
       </acl_target>
       <acl_target id="joe">
         <role id="super_user"/>
       </acl_target>
       <acl_target id="mike">
         <role id="rsc_writer"/>
       </acl_target>
       <acl_target id="chris">
         <role id="rsc_denied"/>
       </acl_target>
       <acl_role id="nothing">
         <acl_permission id="nothing-deny" kind="deny" xpath="/cib"/>
       </acl_role>
       <acl_role id="observer">
         <acl_permission id="observer-read-1" kind="read" xpath="/cib"/>
         <acl_permission id="observer-write-1" kind="write" xpath="//nvpair[@name='stonith-enabled']"/>
         <acl_permission id="observer-write-2" kind="write" xpath="//nvpair[@name='target-role']"/>
       </acl_role>
       <acl_role id="admin">
         <acl_permission id="admin-read-1" kind="read" xpath="/cib"/>
         <acl_permission id="admin-write-1" kind="write" xpath="//resources"/>
       </acl_role>
       <acl_role id="super_user">
         <acl_permission id="super_user-write-1" kind="write" xpath="/cib"/>
       </acl_role>
       <acl_role id="rsc_writer">
         <acl_permission id="rsc-writer-deny-1" kind="deny" xpath="/cib"/>
         <acl_permission id="rsc-writer-write-1" kind="write" xpath="//resources"/>
       </acl_role>
       <acl_role id="rsc_denied">
         <acl_permission id="rsc-denied-write-1" kind="write" xpath="/cib"/>
         <acl_permission id="rsc-denied-deny-1" kind="deny" xpath="//resources"/>
       </acl_role>
       <acl_role id="badidea-role">
         <acl_permission id="badidea-resources" kind="read" xpath="//meta_attributes"/>
       </acl_role>
       <acl_target id="badidea">
         <role id="badidea-role"/>
       </acl_target>
       <acl_role id="betteridea-role">
         <acl_permission id="betteridea-nothing" kind="deny" xpath="/cib"/>
         <acl_permission id="betteridea-resources" kind="read" xpath="//meta_attributes"/>
       </acl_role>
       <acl_target id="betteridea">
         <role id="betteridea-role"/>
       </acl_target>
     </acls>
   </configuration>
   <status/>
 </cib>
 =#=#=#= Begin test: chris: Replace - modify attribute (deny overrides allow) =#=#=#=
 pcmk__check_acl 	trace: Parent ACL denies user 'chris' read/write access to /cib/configuration/resources/primitive[@id='dummy'][@description]
 Call failed: Permission denied
 =#=#=#= End test: chris: Replace - modify attribute (deny overrides allow) - Insufficient privileges (4) =#=#=#=
 * Passed: cibadmin              - chris: Replace - modify attribute (deny overrides allow)
 <cib epoch="27" num_updates="0" admin_epoch="0">
   <configuration>
     <crm_config>
       <cluster_property_set id="cib-bootstrap-options">
         <nvpair id="cib-bootstrap-options-enable-acl" name="enable-acl" value="true"/>
         <nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="ignore"/>
         <nvpair id="cib-bootstrap-options-stonith-enabled" name="stonith-enabled" value="true"/>
       </cluster_property_set>
     </crm_config>
     <nodes/>
     <resources>
       <primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy"/>
       <primitive id="dummy_desc" class="ocf" provider="pacemaker" type="Dummy" description="resource with description"/>
       <primitive id="dummy2" class="ocf" provider="pacemaker" type="Dummy"/>
     </resources>
     <constraints/>
     <acls>
       <acl_target id="l33t-haxor">
         <role id="nothing"/>
       </acl_target>
       <acl_target id="niceguy">
         <role id="observer"/>
       </acl_target>
       <acl_target id="bob">
         <role id="admin"/>
       </acl_target>
       <acl_target id="joe">
         <role id="super_user"/>
       </acl_target>
       <acl_target id="mike">
         <role id="rsc_writer"/>
       </acl_target>
       <acl_target id="chris">
         <role id="rsc_denied"/>
       </acl_target>
       <acl_role id="nothing">
         <acl_permission id="nothing-deny" kind="deny" xpath="/cib"/>
       </acl_role>
       <acl_role id="observer">
         <acl_permission id="observer-read-1" kind="read" xpath="/cib"/>
         <acl_permission id="observer-write-1" kind="write" xpath="//nvpair[@name='stonith-enabled']"/>
         <acl_permission id="observer-write-2" kind="write" xpath="//nvpair[@name='target-role']"/>
       </acl_role>
       <acl_role id="admin">
         <acl_permission id="admin-read-1" kind="read" xpath="/cib"/>
         <acl_permission id="admin-write-1" kind="write" xpath="//resources"/>
       </acl_role>
       <acl_role id="super_user">
         <acl_permission id="super_user-write-1" kind="write" xpath="/cib"/>
       </acl_role>
       <acl_role id="rsc_writer">
         <acl_permission id="rsc-writer-deny-1" kind="deny" xpath="/cib"/>
         <acl_permission id="rsc-writer-write-1" kind="write" xpath="//resources"/>
       </acl_role>
       <acl_role id="rsc_denied">
         <acl_permission id="rsc-denied-write-1" kind="write" xpath="/cib"/>
         <acl_permission id="rsc-denied-deny-1" kind="deny" xpath="//resources"/>
       </acl_role>
       <acl_role id="badidea-role">
         <acl_permission id="badidea-resources" kind="read" xpath="//meta_attributes"/>
       </acl_role>
       <acl_target id="badidea">
         <role id="badidea-role"/>
       </acl_target>
       <acl_role id="betteridea-role">
         <acl_permission id="betteridea-nothing" kind="deny" xpath="/cib"/>
         <acl_permission id="betteridea-resources" kind="read" xpath="//meta_attributes"/>
       </acl_role>
       <acl_target id="betteridea">
         <role id="betteridea-role"/>
       </acl_target>
     </acls>
   </configuration>
   <status/>
 </cib>
 =#=#=#= Begin test: chris: Replace - delete attribute (deny overrides allow) =#=#=#=
 pcmk__check_acl 	trace: Parent ACL denies user 'chris' read/write access to /cib/configuration/resources/primitive[@id='dummy2']
 Call failed: Permission denied
 =#=#=#= End test: chris: Replace - delete attribute (deny overrides allow) - Insufficient privileges (4) =#=#=#=
 * Passed: cibadmin              - chris: Replace - delete attribute (deny overrides allow)
diff --git a/tools/crm_resource.c b/tools/crm_resource.c
index 4536291469..86c2b21c53 100644
--- a/tools/crm_resource.c
+++ b/tools/crm_resource.c
@@ -1,2098 +1,2093 @@
 /*
  * Copyright 2004-2024 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU General Public License version 2
  * or later (GPLv2+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #include <crm_resource.h>
 #include <crm/lrmd_internal.h>
 #include <crm/common/cmdline_internal.h>
 #include <crm/common/ipc_attrd_internal.h>
 #include <crm/common/lists_internal.h>
 #include <crm/common/output.h>
 #include <pacemaker-internal.h>
 
 #include <sys/param.h>
 #include <stdint.h>         // uint32_t
 #include <stdio.h>
 #include <sys/types.h>
 #include <unistd.h>
 #include <stdlib.h>
 #include <errno.h>
 #include <fcntl.h>
 #include <libgen.h>
 #include <time.h>
 
 #include <crm/crm.h>
 #include <crm/stonith-ng.h>
 #include <crm/common/ipc_controld.h>
 #include <crm/cib/internal.h>
 
 #define SUMMARY "crm_resource - perform tasks related to Pacemaker cluster resources"
 
 enum rsc_command {
     cmd_none = 0,           // No command option given (yet)
     cmd_ban,
     cmd_cleanup,
     cmd_clear,
     cmd_colocations,
     cmd_cts,
     cmd_delete,
     cmd_delete_param,
     cmd_digests,
     cmd_execute_agent,
     cmd_fail,
     cmd_get_param,
     cmd_list_active_ops,
     cmd_list_agents,
     cmd_list_all_ops,
     cmd_list_alternatives,
     cmd_list_instances,
     cmd_list_options,
     cmd_list_providers,
     cmd_list_resources,
     cmd_list_standards,
     cmd_locate,
     cmd_metadata,
     cmd_move,
     cmd_query_xml,
     cmd_query_xml_raw,
     cmd_refresh,
     cmd_restart,
     cmd_set_param,
     cmd_wait,
     cmd_why,
 };
 
 struct {
     enum rsc_command rsc_cmd;     // crm_resource command to perform
 
     // Command-line option values
     gchar *rsc_id;                // Value of --resource
     gchar *rsc_type;              // Value of --resource-type
     gboolean all;                 // --all was given
     gboolean force;               // --force was given
     gboolean clear_expired;       // --expired was given
     gboolean recursive;           // --recursive was given
     gboolean promoted_role_only;  // --promoted was given
     gchar *host_uname;            // Value of --node
     gchar *interval_spec;         // Value of --interval
     gchar *move_lifetime;         // Value of --lifetime
     gchar *operation;             // Value of --operation
     enum pcmk__opt_flags opt_list;  // Parsed from --list-options
     const char *attr_set_type;    // Instance, meta, utilization, or element attribute
     gchar *prop_id;               // --nvpair (attribute XML ID)
     char *prop_name;              // Attribute name
     gchar *prop_set;              // --set-name (attribute block XML ID)
     gchar *prop_value;            // --parameter-value (attribute value)
     guint timeout_ms;             // Parsed from --timeout value
     char *agent_spec;             // Standard and/or provider and/or agent
     int check_level;              // Optional value of --validate or --force-check
 
     // Resource configuration specified via command-line arguments
     bool cmdline_config;          // Resource configuration was via arguments
     char *v_agent;                // Value of --agent
     char *v_class;                // Value of --class
     char *v_provider;             // Value of --provider
     GHashTable *cmdline_params;   // Resource parameters specified
 
     // Positional command-line arguments
     gchar **remainder;            // Positional arguments as given
     GHashTable *override_params;  // Resource parameter values that override config
 } options = {
     .attr_set_type = PCMK_XE_INSTANCE_ATTRIBUTES,
     .check_level = -1,
     .rsc_cmd = cmd_list_resources,  // List all resources if no command given
 };
 
 gboolean attr_set_type_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error);
 gboolean cmdline_config_cb(const gchar *option_name, const gchar *optarg,
                            gpointer data, GError **error);
 gboolean option_cb(const gchar *option_name, const gchar *optarg,
                    gpointer data, GError **error);
 gboolean timeout_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error);
 
 static crm_exit_t exit_code = CRM_EX_OK;
 static pcmk__output_t *out = NULL;
 static pcmk__common_args_t *args = NULL;
 
 // Things that should be cleaned up on exit
 static GError *error = NULL;
 static GMainLoop *mainloop = NULL;
 static cib_t *cib_conn = NULL;
 static pcmk_ipc_api_t *controld_api = NULL;
 static pcmk_scheduler_t *scheduler = NULL;
 
 #define MESSAGE_TIMEOUT_S 60
 
 #define INDENT "                                    "
 
 static pcmk__supported_format_t formats[] = {
     PCMK__SUPPORTED_FORMAT_NONE,
     PCMK__SUPPORTED_FORMAT_TEXT,
     PCMK__SUPPORTED_FORMAT_XML,
     { NULL, NULL, NULL }
 };
 
 // Clean up and exit
 static crm_exit_t
 bye(crm_exit_t ec)
 {
     pcmk__output_and_clear_error(&error, out);
 
     if (out != NULL) {
         out->finish(out, ec, true, NULL);
         pcmk__output_free(out);
     }
     pcmk__unregister_formats();
 
     if (cib_conn != NULL) {
         cib_t *save_cib_conn = cib_conn;
 
         cib_conn = NULL; // Ensure we can't free this twice
         cib__clean_up_connection(&save_cib_conn);
     }
 
     if (controld_api != NULL) {
         pcmk_ipc_api_t *save_controld_api = controld_api;
 
         controld_api = NULL; // Ensure we can't free this twice
         pcmk_free_ipc_api(save_controld_api);
     }
 
     if (mainloop != NULL) {
         g_main_loop_unref(mainloop);
         mainloop = NULL;
     }
 
     pe_free_working_set(scheduler);
     scheduler = NULL;
     crm_exit(ec);
     return ec;
 }
 
 static void
 quit_main_loop(crm_exit_t ec)
 {
     exit_code = ec;
     if (mainloop != NULL) {
         GMainLoop *mloop = mainloop;
 
         mainloop = NULL; // Don't re-enter this block
         pcmk_quit_main_loop(mloop, 10);
         g_main_loop_unref(mloop);
     }
 }
 
 static gboolean
 resource_ipc_timeout(gpointer data)
 {
     // Start with newline because "Waiting for ..." message doesn't have one
     if (error != NULL) {
         g_clear_error(&error);
     }
 
     g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_TIMEOUT,
                 _("Aborting because no messages received in %d seconds"), MESSAGE_TIMEOUT_S);
 
     quit_main_loop(CRM_EX_TIMEOUT);
     return FALSE;
 }
 
 static void
 controller_event_callback(pcmk_ipc_api_t *api, enum pcmk_ipc_event event_type,
                           crm_exit_t status, void *event_data, void *user_data)
 {
     switch (event_type) {
         case pcmk_ipc_event_disconnect:
             if (exit_code == CRM_EX_DISCONNECT) { // Unexpected
                 crm_info("Connection to controller was terminated");
             }
             quit_main_loop(exit_code);
             break;
 
         case pcmk_ipc_event_reply:
             if (status != CRM_EX_OK) {
                 out->err(out, "Error: bad reply from controller: %s",
                          crm_exit_str(status));
                 pcmk_disconnect_ipc(api);
                 quit_main_loop(status);
             } else {
                 if ((pcmk_controld_api_replies_expected(api) == 0)
                     && mainloop && g_main_loop_is_running(mainloop)) {
                     out->info(out, "... got reply (done)");
                     crm_debug("Got all the replies we expected");
                     pcmk_disconnect_ipc(api);
                     quit_main_loop(CRM_EX_OK);
                 } else {
                     out->info(out, "... got reply");
                 }
             }
             break;
 
         default:
             break;
     }
 }
 
 static void
 start_mainloop(pcmk_ipc_api_t *capi)
 {
     unsigned int count = pcmk_controld_api_replies_expected(capi);
 
     if (count > 0) {
         out->info(out, "Waiting for %u %s from the controller",
                   count, pcmk__plural_alt(count, "reply", "replies"));
         exit_code = CRM_EX_DISCONNECT; // For unexpected disconnects
         mainloop = g_main_loop_new(NULL, FALSE);
         g_timeout_add(MESSAGE_TIMEOUT_S * 1000, resource_ipc_timeout, NULL);
         g_main_loop_run(mainloop);
     }
 }
 
 static int
 compare_id(gconstpointer a, gconstpointer b)
 {
     return strcmp((const char *)a, (const char *)b);
 }
 
 static GList *
 build_constraint_list(xmlNode *root)
 {
     GList *retval = NULL;
     xmlNode *cib_constraints = NULL;
     xmlXPathObjectPtr xpathObj = NULL;
     int ndx = 0;
 
     cib_constraints = pcmk_find_cib_element(root, PCMK_XE_CONSTRAINTS);
     xpathObj = xpath_search(cib_constraints, "//" PCMK_XE_RSC_LOCATION);
 
     for (ndx = 0; ndx < numXpathResults(xpathObj); ndx++) {
         xmlNode *match = getXpathResult(xpathObj, ndx);
         retval = g_list_insert_sorted(retval, (gpointer) pcmk__xe_id(match),
                                       compare_id);
     }
 
     freeXpathObject(xpathObj);
     return retval;
 }
 
 static gboolean
 validate_opt_list(const gchar *optarg)
 {
     if (pcmk__str_eq(optarg, PCMK_VALUE_FENCING, pcmk__str_none)) {
         options.opt_list = pcmk__opt_fencing;
 
     } else if (pcmk__str_eq(optarg, PCMK__VALUE_PRIMITIVE, pcmk__str_none)) {
         options.opt_list = pcmk__opt_primitive;
 
     } else {
         return FALSE;
     }
 
     return TRUE;
 }
 
 /*!
  * \internal
  * \brief Process options that set the command
  *
  * Nothing else should set \c options.rsc_cmd.
  *
  * \param[in]  option_name  Name of the option being parsed
  * \param[in]  optarg       Value to be parsed
  * \param[in]  data         Ignored
  * \param[out] error        Where to store recoverable error, if any
  *
  * \return \c TRUE if the option was successfully parsed, or \c FALSE if an
  *         error occurred, in which case \p *error is set
  */
 static gboolean
 command_cb(const gchar *option_name, const gchar *optarg, gpointer data,
            GError **error)
 {
     // Sorted by enum rsc_command name
     if (pcmk__str_any_of(option_name, "-B", "--ban", NULL)) {
         options.rsc_cmd = cmd_ban;
 
     } else if (pcmk__str_any_of(option_name, "-C", "--cleanup", NULL)) {
         options.rsc_cmd = cmd_cleanup;
 
     } else if (pcmk__str_any_of(option_name, "-U", "--clear", NULL)) {
         options.rsc_cmd = cmd_clear;
 
     } else if (pcmk__str_any_of(option_name, "-a", "--constraints", NULL)) {
         options.rsc_cmd = cmd_colocations;
 
     } else if (pcmk__str_any_of(option_name, "-A", "--stack", NULL)) {
         options.rsc_cmd = cmd_colocations;
         options.recursive = TRUE;
 
     } else if (pcmk__str_any_of(option_name, "-c", "--list-cts", NULL)) {
         options.rsc_cmd = cmd_cts;
 
     } else if (pcmk__str_any_of(option_name, "-D", "--delete", NULL)) {
         options.rsc_cmd = cmd_delete;
 
     } else if (pcmk__str_any_of(option_name, "-d", "--delete-parameter",
                                 NULL)) {
         options.rsc_cmd = cmd_delete_param;
         pcmk__str_update(&options.prop_name, optarg);
 
     } else if (pcmk__str_eq(option_name, "--digests", pcmk__str_none)) {
         options.rsc_cmd = cmd_digests;
 
         if (options.override_params == NULL) {
             options.override_params = pcmk__strkey_table(free, free);
         }
 
     } else if (pcmk__str_any_of(option_name,
                                 "--force-demote", "--force-promote",
                                 "--force-start", "--force-stop",
                                 "--force-check", "--validate", NULL)) {
         options.rsc_cmd = cmd_execute_agent;
 
         g_free(options.operation);
         options.operation = g_strdup(option_name + 2);  // skip "--"
 
         if (options.override_params == NULL) {
             options.override_params = pcmk__strkey_table(free, free);
         }
 
         if (optarg != NULL) {
             if (pcmk__scan_min_int(optarg, &options.check_level,
                                    0) != pcmk_rc_ok) {
                 g_set_error(error, G_OPTION_ERROR, CRM_EX_INVALID_PARAM,
                             _("Invalid check level setting: %s"), optarg);
                 return FALSE;
             }
         }
 
     } else if (pcmk__str_any_of(option_name, "-F", "--fail", NULL)) {
         options.rsc_cmd = cmd_fail;
 
     } else if (pcmk__str_any_of(option_name, "-g", "--get-parameter", NULL)) {
         options.rsc_cmd = cmd_get_param;
         pcmk__str_update(&options.prop_name, optarg);
 
     } else if (pcmk__str_any_of(option_name, "-O", "--list-operations", NULL)) {
         options.rsc_cmd = cmd_list_active_ops;
 
     } else if (pcmk__str_eq(option_name, "--list-agents", pcmk__str_none)) {
         options.rsc_cmd = cmd_list_agents;
         pcmk__str_update(&options.agent_spec, optarg);
 
     } else if (pcmk__str_any_of(option_name, "-o", "--list-all-operations",
                                 NULL)) {
         options.rsc_cmd = cmd_list_all_ops;
 
     } else if (pcmk__str_eq(option_name, "--list-ocf-alternatives",
                             pcmk__str_none)) {
         options.rsc_cmd = cmd_list_alternatives;
         pcmk__str_update(&options.agent_spec, optarg);
 
     } else if (pcmk__str_eq(option_name, "--list-options", pcmk__str_none)) {
         options.rsc_cmd = cmd_list_options;
         return validate_opt_list(optarg);
 
     } else if (pcmk__str_any_of(option_name, "-l", "--list-raw", NULL)) {
         options.rsc_cmd = cmd_list_instances;
 
     } else if (pcmk__str_eq(option_name, "--list-ocf-providers",
                             pcmk__str_none)) {
         options.rsc_cmd = cmd_list_providers;
         pcmk__str_update(&options.agent_spec, optarg);
 
     } else if (pcmk__str_any_of(option_name, "-L", "--list", NULL)) {
         options.rsc_cmd = cmd_list_resources;
 
     } else if (pcmk__str_eq(option_name, "--list-standards", pcmk__str_none)) {
         options.rsc_cmd = cmd_list_standards;
 
     } else if (pcmk__str_any_of(option_name, "-W", "--locate", NULL)) {
         options.rsc_cmd = cmd_locate;
 
     } else if (pcmk__str_eq(option_name, "--show-metadata", pcmk__str_none)) {
         options.rsc_cmd = cmd_metadata;
         pcmk__str_update(&options.agent_spec, optarg);
 
     } else if (pcmk__str_any_of(option_name, "-M", "--move", NULL)) {
         options.rsc_cmd = cmd_move;
 
     } else if (pcmk__str_any_of(option_name, "-q", "--query-xml", NULL)) {
         options.rsc_cmd = cmd_query_xml;
 
     } else if (pcmk__str_any_of(option_name, "-w", "--query-xml-raw", NULL)) {
         options.rsc_cmd = cmd_query_xml_raw;
 
     } else if (pcmk__str_any_of(option_name, "-R", "--refresh", NULL)) {
         options.rsc_cmd = cmd_refresh;
 
     } else if (pcmk__str_eq(option_name, "--restart", pcmk__str_none)) {
         options.rsc_cmd = cmd_restart;
 
     } else if (pcmk__str_any_of(option_name, "-p", "--set-parameter", NULL)) {
         options.rsc_cmd = cmd_set_param;
         pcmk__str_update(&options.prop_name, optarg);
 
     } else if (pcmk__str_eq(option_name, "--wait", pcmk__str_none)) {
         options.rsc_cmd = cmd_wait;
 
     } else if (pcmk__str_any_of(option_name, "-Y", "--why", NULL)) {
         options.rsc_cmd = cmd_why;
     }
 
     return TRUE;
 }
 
 /* short option letters still available: eEJkKXyYZ */
 
 static GOptionEntry query_entries[] = {
     { "list", 'L', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, command_cb,
       "List all cluster resources with status",
       NULL },
     { "list-raw", 'l', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, command_cb,
       "List IDs of all instantiated resources (individual members\n"
       INDENT "rather than groups etc.)",
       NULL },
     { "list-cts", 'c', G_OPTION_FLAG_HIDDEN|G_OPTION_FLAG_NO_ARG,
           G_OPTION_ARG_CALLBACK, command_cb,
       NULL,
       NULL },
     { "list-operations", 'O', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK,
           command_cb,
       "List active resource operations, optionally filtered by\n"
       INDENT "--resource and/or --node",
       NULL },
     { "list-all-operations", 'o', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK,
           command_cb,
       "List all resource operations, optionally filtered by\n"
       INDENT "--resource and/or --node",
       NULL },
     { "list-options", 0, G_OPTION_FLAG_NONE, G_OPTION_ARG_CALLBACK, command_cb,
       "List all available options of the given type.\n"
       INDENT "Allowed values:\n"
       INDENT PCMK__VALUE_PRIMITIVE " (primitive resource meta-attributes),\n"
       INDENT PCMK_VALUE_FENCING " (parameters common to all fencing resources)",
       "TYPE" },
     { "list-standards", 0, G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK,
           command_cb,
       "List supported standards",
       NULL },
     { "list-ocf-providers", 0, G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK,
           command_cb,
       "List all available OCF providers",
       NULL },
     { "list-agents", 0, G_OPTION_FLAG_NONE, G_OPTION_ARG_CALLBACK,
           command_cb,
       "List all agents available for the named standard and/or provider",
       "STD:PROV" },
     { "list-ocf-alternatives", 0, G_OPTION_FLAG_NONE, G_OPTION_ARG_CALLBACK,
           command_cb,
       "List all available providers for the named OCF agent",
       "AGENT" },
     { "show-metadata", 0, G_OPTION_FLAG_NONE, G_OPTION_ARG_CALLBACK, command_cb,
       "Show the metadata for the named class:provider:agent",
       "SPEC" },
     { "query-xml", 'q', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, command_cb,
       "Show XML configuration of resource (after any template expansion)",
       NULL },
     { "query-xml-raw", 'w', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK,
           command_cb,
       "Show XML configuration of resource (before any template expansion)",
       NULL },
     { "get-parameter", 'g', G_OPTION_FLAG_NONE, G_OPTION_ARG_CALLBACK,
           command_cb,
       "Display named parameter for resource (use instance attribute\n"
       INDENT "unless --element, --meta, or --utilization is specified)",
       "PARAM" },
     { "locate", 'W', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, command_cb,
       "Show node(s) currently running resource",
       NULL },
     { "constraints", 'a', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK,
           command_cb,
       "Display the location and colocation constraints that apply to a\n"
       INDENT "resource, and if --recursive is specified, to the resources\n"
       INDENT "directly or indirectly involved in those colocations.\n"
       INDENT "If the named resource is part of a group, or a clone or\n"
       INDENT "bundle instance, constraints for the collective resource\n"
       INDENT "will be shown unless --force is given.",
       NULL },
     { "stack", 'A', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, command_cb,
       "Equivalent to --constraints --recursive",
       NULL },
     { "why", 'Y', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, command_cb,
       "Show why resources are not running, optionally filtered by\n"
       INDENT "--resource and/or --node",
       NULL },
 
     { NULL }
 };
 
 static GOptionEntry command_entries[] = {
     { "validate", 0, G_OPTION_FLAG_OPTIONAL_ARG, G_OPTION_ARG_CALLBACK,
           command_cb,
       "Validate resource configuration by calling agent's validate-all\n"
       INDENT "action. The configuration may be specified either by giving an\n"
       INDENT "existing resource name with -r, or by specifying --class,\n"
       INDENT "--agent, and --provider arguments, along with any number of\n"
       INDENT "--option arguments. An optional LEVEL argument can be given\n"
       INDENT "to control the level of checking performed.",
       "LEVEL" },
     { "cleanup", 'C', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, command_cb,
       "If resource has any past failures, clear its history and fail\n"
       INDENT "count. Optionally filtered by --resource, --node, --operation\n"
       INDENT "and --interval (otherwise all). --operation and --interval\n"
       INDENT "apply to fail counts, but entire history is always clear, to\n"
       INDENT "allow current state to be rechecked. If the named resource is\n"
       INDENT "part of a group, or one numbered instance of a clone or bundled\n"
       INDENT "resource, the clean-up applies to the whole collective resource\n"
       INDENT "unless --force is given.",
       NULL },
     { "refresh", 'R', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, command_cb,
       "Delete resource's history (including failures) so its current state\n"
       INDENT "is rechecked. Optionally filtered by --resource and --node\n"
       INDENT "(otherwise all). If the named resource is part of a group, or one\n"
       INDENT "numbered instance of a clone or bundled resource, the refresh\n"
       INDENT "applies to the whole collective resource unless --force is given.",
       NULL },
     { "set-parameter", 'p', G_OPTION_FLAG_NONE, G_OPTION_ARG_CALLBACK,
           command_cb,
       "Set named parameter for resource (requires -v). Use instance\n"
       INDENT "attribute unless --element, --meta, or --utilization is "
       "specified.",
       "PARAM" },
     { "delete-parameter", 'd', G_OPTION_FLAG_NONE, G_OPTION_ARG_CALLBACK,
           command_cb,
       "Delete named parameter for resource. Use instance attribute\n"
       INDENT "unless --element, --meta or, --utilization is specified.",
       "PARAM" },
 
     { NULL }
 };
 
 static GOptionEntry location_entries[] = {
     { "move", 'M', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, command_cb,
       "Create a constraint to move resource. If --node is specified,\n"
       INDENT "the constraint will be to move to that node, otherwise it\n"
       INDENT "will be to ban the current node. Unless --force is specified\n"
       INDENT "this will return an error if the resource is already running\n"
       INDENT "on the specified node. If --force is specified, this will\n"
       INDENT "always ban the current node.\n"
       INDENT "Optional: --lifetime, --promoted. NOTE: This may prevent the\n"
       INDENT "resource from running on its previous location until the\n"
       INDENT "implicit constraint expires or is removed with --clear.",
       NULL },
     { "ban", 'B', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, command_cb,
       "Create a constraint to keep resource off a node.\n"
       INDENT "Optional: --node, --lifetime, --promoted.\n"
       INDENT "NOTE: This will prevent the resource from running on the\n"
       INDENT "affected node until the implicit constraint expires or is\n"
       INDENT "removed with --clear. If --node is not specified, it defaults\n"
       INDENT "to the node currently running the resource for primitives\n"
       INDENT "and groups, or the promoted instance of promotable clones with\n"
       INDENT PCMK_META_PROMOTED_MAX "=1 (all other situations result in an\n"
       INDENT "error as there is no sane default).",
       NULL },
     { "clear", 'U', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, command_cb,
       "Remove all constraints created by the --ban and/or --move\n"
       INDENT "commands. Requires: --resource. Optional: --node, --promoted,\n"
       INDENT "--expired. If --node is not specified, all constraints created\n"
       INDENT "by --ban and --move will be removed for the named resource. If\n"
       INDENT "--node and --force are specified, any constraint created by\n"
       INDENT "--move will be cleared, even if it is not for the specified\n"
       INDENT "node. If --expired is specified, only those constraints whose\n"
       INDENT "lifetimes have expired will be removed.",
       NULL },
     { "expired", 'e', G_OPTION_FLAG_NONE, G_OPTION_ARG_NONE,
           &options.clear_expired,
       "Modifies the --clear argument to remove constraints with\n"
       INDENT "expired lifetimes.",
       NULL },
     { "lifetime", 'u', G_OPTION_FLAG_NONE, G_OPTION_ARG_STRING, &options.move_lifetime,
       "Lifespan (as ISO 8601 duration) of created constraints (with\n"
       INDENT "-B, -M) see https://en.wikipedia.org/wiki/ISO_8601#Durations)",
       "TIMESPEC" },
     { "promoted", 0, G_OPTION_FLAG_NONE, G_OPTION_ARG_NONE,
       &options.promoted_role_only,
       "Limit scope of command to promoted role (with -B, -M, -U). For\n"
       INDENT "-B and -M, previously promoted instances may remain\n"
       INDENT "active in the unpromoted role.",
       NULL },
 
     // Deprecated since 2.1.0
     { "master", 0, G_OPTION_FLAG_NONE, G_OPTION_ARG_NONE,
       &options.promoted_role_only,
       "Deprecated: Use --promoted instead", NULL },
 
     { NULL }
 };
 
 static GOptionEntry advanced_entries[] = {
     { "delete", 'D', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, command_cb,
       "(Advanced) Delete a resource from the CIB. Required: -t",
       NULL },
     { "fail", 'F', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, command_cb,
       "(Advanced) Tell the cluster this resource has failed",
       NULL },
     { "restart", 0, G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, command_cb,
       "(Advanced) Tell the cluster to restart this resource and\n"
       INDENT "anything that depends on it",
       NULL },
     { "wait", 0, G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, command_cb,
       "(Advanced) Wait until the cluster settles into a stable state",
       NULL },
     { "digests", 0, G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, command_cb,
       "(Advanced) Show parameter hashes that Pacemaker uses to detect\n"
       INDENT "configuration changes (only accurate if there is resource\n"
       INDENT "history on the specified node). Required: --resource, --node.\n"
       INDENT "Optional: any NAME=VALUE parameters will be used to override\n"
       INDENT "the configuration (to see what the hash would be with those\n"
       INDENT "changes).",
       NULL },
     { "force-demote", 0, G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK,
           command_cb,
       "(Advanced) Bypass the cluster and demote a resource on the local\n"
       INDENT "node. Unless --force is specified, this will refuse to do so if\n"
       INDENT "the cluster believes the resource is a clone instance already\n"
       INDENT "running on the local node.",
       NULL },
     { "force-stop", 0, G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, command_cb,
       "(Advanced) Bypass the cluster and stop a resource on the local node",
       NULL },
     { "force-start", 0, G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, command_cb,
       "(Advanced) Bypass the cluster and start a resource on the local\n"
       INDENT "node. Unless --force is specified, this will refuse to do so if\n"
       INDENT "the cluster believes the resource is a clone instance already\n"
       INDENT "running on the local node.",
       NULL },
     { "force-promote", 0, G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK,
           command_cb,
       "(Advanced) Bypass the cluster and promote a resource on the local\n"
       INDENT "node. Unless --force is specified, this will refuse to do so if\n"
       INDENT "the cluster believes the resource is a clone instance already\n"
       INDENT "running on the local node.",
       NULL },
     { "force-check", 0, G_OPTION_FLAG_OPTIONAL_ARG, G_OPTION_ARG_CALLBACK,
           command_cb,
       "(Advanced) Bypass the cluster and check the state of a resource on\n"
       INDENT "the local node. An optional LEVEL argument can be given\n"
       INDENT "to control the level of checking performed.",
       "LEVEL" },
 
     { NULL }
 };
 
 static GOptionEntry addl_entries[] = {
     { "node", 'N', G_OPTION_FLAG_NONE, G_OPTION_ARG_STRING, &options.host_uname,
       "Node name",
       "NAME" },
     { "recursive", 0, G_OPTION_FLAG_NONE, G_OPTION_ARG_NONE, &options.recursive,
       "Follow colocation chains when using --set-parameter or --constraints",
       NULL },
     { "resource-type", 't', G_OPTION_FLAG_NONE, G_OPTION_ARG_STRING, &options.rsc_type,
       "Resource XML element (primitive, group, etc.) (with -D)",
       "ELEMENT" },
     { "parameter-value", 'v', G_OPTION_FLAG_NONE, G_OPTION_ARG_STRING, &options.prop_value,
       "Value to use with -p",
       "PARAM" },
     { "meta", 'm', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, attr_set_type_cb,
       "Use resource meta-attribute instead of instance attribute\n"
       INDENT "(with -p, -g, -d)",
       NULL },
     { "utilization", 'z', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, attr_set_type_cb,
       "Use resource utilization attribute instead of instance attribute\n"
       INDENT "(with -p, -g, -d)",
       NULL },
     { "element", 0, G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, attr_set_type_cb,
       "Use resource element attribute instead of instance attribute\n"
       INDENT "(with -p, -g, -d)",
       NULL },
     { "operation", 'n', G_OPTION_FLAG_NONE, G_OPTION_ARG_STRING, &options.operation,
       "Operation to clear instead of all (with -C -r)",
       "OPERATION" },
     { "interval", 'I', G_OPTION_FLAG_NONE, G_OPTION_ARG_STRING, &options.interval_spec,
       "Interval of operation to clear (default 0) (with -C -r -n)",
       "N" },
     { "class", 0, G_OPTION_FLAG_NONE, G_OPTION_ARG_CALLBACK, cmdline_config_cb,
       "The standard the resource agent conforms to (for example, ocf).\n"
       INDENT "Use with --agent, --provider, --option, and --validate.",
       "CLASS" },
     { "agent", 0, G_OPTION_FLAG_NONE, G_OPTION_ARG_CALLBACK, cmdline_config_cb,
       "The agent to use (for example, IPaddr). Use with --class,\n"
       INDENT "--provider, --option, and --validate.",
       "AGENT" },
     { "provider", 0, G_OPTION_FLAG_NONE, G_OPTION_ARG_CALLBACK,
           cmdline_config_cb,
       "The vendor that supplies the resource agent (for example,\n"
       INDENT "heartbeat). Use with --class, --agent, --option, and --validate.",
       "PROVIDER" },
     { "option", 0, G_OPTION_FLAG_NONE, G_OPTION_ARG_CALLBACK, option_cb,
       "Specify a device configuration parameter as NAME=VALUE (may be\n"
       INDENT "specified multiple times). Use with --validate and without the\n"
       INDENT "-r option.",
       "PARAM" },
     { "set-name", 's', G_OPTION_FLAG_NONE, G_OPTION_ARG_STRING, &options.prop_set,
       "(Advanced) XML ID of attributes element to use (with -p, -d)",
       "ID" },
     { "nvpair", 'i', G_OPTION_FLAG_NONE, G_OPTION_ARG_STRING, &options.prop_id,
       "(Advanced) XML ID of nvpair element to use (with -p, -d)",
       "ID" },
     { "timeout", 'T', G_OPTION_FLAG_NONE, G_OPTION_ARG_CALLBACK, timeout_cb,
       "(Advanced) Abort if command does not finish in this time (with\n"
       INDENT "--restart, --wait, --force-*)",
       "N" },
     { "all", 0, G_OPTION_FLAG_NONE, G_OPTION_ARG_NONE, &options.all,
       "List all options, including advanced and deprecated (with\n"
       INDENT "--list-options)",
       NULL },
     { "force", 'f', G_OPTION_FLAG_NONE, G_OPTION_ARG_NONE, &options.force,
       "Force the action to be performed. See help for individual commands for\n"
       INDENT "additional behavior.",
       NULL },
 
     // @COMPAT Used in resource-agents prior to v4.2.0
     { "host-uname", 'H', G_OPTION_FLAG_HIDDEN, G_OPTION_ARG_STRING, &options.host_uname,
       NULL,
       "HOST" },
 
     { NULL }
 };
 
 gboolean
 attr_set_type_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
     if (pcmk__str_any_of(option_name, "-m", "--meta", NULL)) {
         options.attr_set_type = PCMK_XE_META_ATTRIBUTES;
     } else if (pcmk__str_any_of(option_name, "-z", "--utilization", NULL)) {
         options.attr_set_type = PCMK_XE_UTILIZATION;
     } else if (pcmk__str_eq(option_name, "--element", pcmk__str_none)) {
         options.attr_set_type = ATTR_SET_ELEMENT;
     }
     return TRUE;
 }
 
 gboolean
 cmdline_config_cb(const gchar *option_name, const gchar *optarg, gpointer data,
                   GError **error)
 {
     options.cmdline_config = true;
 
     if (pcmk__str_eq(option_name, "--class", pcmk__str_none)) {
         pcmk__str_update(&options.v_class, optarg);
 
     } else if (pcmk__str_eq(option_name, "--provider", pcmk__str_none)) {
         pcmk__str_update(&options.v_provider, optarg);
 
     } else {    // --agent
         pcmk__str_update(&options.v_agent, optarg);
     }
     return TRUE;
 }
 
 gboolean
 option_cb(const gchar *option_name, const gchar *optarg, gpointer data,
           GError **error)
 {
     char *name = NULL;
     char *value = NULL;
 
     if (pcmk__scan_nvpair(optarg, &name, &value) != 2) {
         return FALSE;
     }
     if (options.cmdline_params == NULL) {
         options.cmdline_params = pcmk__strkey_table(free, free);
     }
     g_hash_table_replace(options.cmdline_params, name, value);
     return TRUE;
 }
 
 gboolean
 timeout_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
     long long timeout_ms = crm_get_msec(optarg);
 
     if (timeout_ms < 0) {
         return FALSE;
     }
     options.timeout_ms = (guint) QB_MIN(timeout_ms, UINT_MAX);
     return TRUE;
 }
 
 static int
 ban_or_move(pcmk__output_t *out, pcmk_resource_t *rsc,
             const char *move_lifetime)
 {
     int rc = pcmk_rc_ok;
     pcmk_node_t *current = NULL;
     unsigned int nactive = 0;
 
     CRM_CHECK(rsc != NULL, return EINVAL);
 
     current = pe__find_active_requires(rsc, &nactive);
 
     if (nactive == 1) {
         rc = cli_resource_ban(out, options.rsc_id, current->priv->name,
                               move_lifetime, cib_conn,
                               options.promoted_role_only, PCMK_ROLE_PROMOTED);
 
     } else if (pcmk_is_set(rsc->flags, pcmk__rsc_promotable)) {
         int count = 0;
         GList *iter = NULL;
 
         current = NULL;
         for (iter = rsc->priv->children; iter != NULL; iter = iter->next) {
             pcmk_resource_t *child = (pcmk_resource_t *)iter->data;
             enum rsc_role_e child_role = child->priv->fns->state(child, TRUE);
 
             if (child_role == pcmk_role_promoted) {
                 count++;
                 current = pcmk__current_node(child);
             }
         }
 
         if(count == 1 && current) {
             rc = cli_resource_ban(out, options.rsc_id, current->priv->name,
                                   move_lifetime, cib_conn,
                                   options.promoted_role_only,
                                   PCMK_ROLE_PROMOTED);
 
         } else {
             rc = EINVAL;
             g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_USAGE,
                         _("Resource '%s' not moved: active in %d locations (promoted in %d).\n"
                         "To prevent '%s' from running on a specific location, "
                         "specify a node."
                         "To prevent '%s' from being promoted at a specific "
                         "location, specify a node and the --promoted option."),
                         options.rsc_id, nactive, count, options.rsc_id, options.rsc_id);
         }
 
     } else {
         rc = EINVAL;
         g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_USAGE,
                     _("Resource '%s' not moved: active in %d locations.\n"
                     "To prevent '%s' from running on a specific location, "
                     "specify a node."),
                     options.rsc_id, nactive, options.rsc_id);
     }
 
     return rc;
 }
 
 static void
 cleanup(pcmk__output_t *out, pcmk_resource_t *rsc, pcmk_node_t *node)
 {
     int rc = pcmk_rc_ok;
 
     if (options.force == FALSE) {
         rsc = uber_parent(rsc);
     }
 
     crm_debug("Erasing failures of %s (%s requested) on %s",
               rsc->id, options.rsc_id, (options.host_uname? options.host_uname: "all nodes"));
     rc = cli_resource_delete(controld_api, options.host_uname, rsc,
                              options.operation, options.interval_spec, TRUE,
                              scheduler, options.force);
 
     if ((rc == pcmk_rc_ok) && !out->is_quiet(out)) {
         // Show any reasons why resource might stay stopped
         cli_resource_check(out, rsc, node);
     }
 
     if (rc == pcmk_rc_ok) {
         start_mainloop(controld_api);
     }
 }
 
 static int
 clear_constraints(pcmk__output_t *out)
 {
     GList *before = NULL;
     GList *after = NULL;
     GList *remaining = NULL;
     GList *ele = NULL;
     pcmk_node_t *dest = NULL;
     int rc = pcmk_rc_ok;
 
     if (!out->is_quiet(out)) {
         before = build_constraint_list(scheduler->input);
     }
 
     if (options.clear_expired) {
         rc = cli_resource_clear_all_expired(scheduler->input, cib_conn,
                                             options.rsc_id, options.host_uname,
                                             options.promoted_role_only);
 
     } else if (options.host_uname) {
         dest = pcmk_find_node(scheduler, options.host_uname);
         if (dest == NULL) {
             rc = pcmk_rc_node_unknown;
             if (!out->is_quiet(out)) {
                 g_list_free(before);
             }
             return rc;
         }
         rc = cli_resource_clear(options.rsc_id, dest->priv->name, NULL,
                                 cib_conn, true, options.force);
 
     } else {
         rc = cli_resource_clear(options.rsc_id, NULL, scheduler->nodes,
                                 cib_conn, true, options.force);
     }
 
     if (!out->is_quiet(out)) {
         xmlNode *cib_xml = NULL;
 
         rc = cib_conn->cmds->query(cib_conn, NULL, &cib_xml, cib_sync_call);
         rc = pcmk_legacy2rc(rc);
 
         if (rc != pcmk_rc_ok) {
             g_set_error(&error, PCMK__RC_ERROR, rc,
                         _("Could not get modified CIB: %s\n"), pcmk_rc_str(rc));
             g_list_free(before);
             pcmk__xml_free(cib_xml);
             return rc;
         }
 
         scheduler->input = cib_xml;
         cluster_status(scheduler);
 
         after = build_constraint_list(scheduler->input);
         remaining = pcmk__subtract_lists(before, after, (GCompareFunc) strcmp);
 
         for (ele = remaining; ele != NULL; ele = ele->next) {
             out->info(out, "Removing constraint: %s", (char *) ele->data);
         }
 
         g_list_free(before);
         g_list_free(after);
         g_list_free(remaining);
     }
 
     return rc;
 }
 
 static int
-initialize_scheduler_data(void)
+initialize_scheduler_data(xmlNode **cib_xml_orig)
 {
-    xmlNode *cib_xml = NULL;
     int rc = pcmk_rc_ok;
 
-    rc = cib_conn->cmds->query(cib_conn, NULL, &cib_xml, cib_sync_call);
-    rc = pcmk_legacy2rc(rc);
-
-    if (rc == pcmk_rc_ok) {
-        scheduler = pe_new_working_set();
-        if (scheduler == NULL) {
-            rc = ENOMEM;
-        } else {
-            pcmk__set_scheduler_flags(scheduler, pcmk__sched_no_counts);
-            scheduler->priv->out = out;
-            rc = update_scheduler_input(scheduler, &cib_xml);
-        }
+    scheduler = pe_new_working_set();
+    if (scheduler == NULL) {
+        return ENOMEM;
     }
 
+    pcmk__set_scheduler_flags(scheduler, pcmk__sched_no_counts);
+    scheduler->priv->out = out;
+    rc = update_scheduler_input(out, scheduler, cib_conn, cib_xml_orig);
     if (rc != pcmk_rc_ok) {
-        pcmk__xml_free(cib_xml);
         return rc;
     }
 
     cluster_status(scheduler);
     return pcmk_rc_ok;
 }
 
 static void
 list_options(void)
 {
     switch (options.opt_list) {
         case pcmk__opt_fencing:
             exit_code = pcmk_rc2exitc(pcmk__list_fencing_params(out,
                                                                 options.all));
             break;
         case pcmk__opt_primitive:
             exit_code = pcmk_rc2exitc(pcmk__list_primitive_meta(out,
                                                                 options.all));
             break;
         default:
             exit_code = CRM_EX_SOFTWARE;
             g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
                         "BUG: Invalid option list type");
             break;
     }
 }
 
 static int
 refresh(pcmk__output_t *out)
 {
     int rc = pcmk_rc_ok;
     const char *router_node = options.host_uname;
     int attr_options = pcmk__node_attr_none;
 
     if (options.host_uname) {
         pcmk_node_t *node = pcmk_find_node(scheduler, options.host_uname);
 
         if (pcmk__is_pacemaker_remote_node(node)) {
             node = pcmk__current_node(node->priv->remote);
             if (node == NULL) {
                 rc = ENXIO;
                 g_set_error(&error, PCMK__RC_ERROR, rc,
                             _("No cluster connection to Pacemaker Remote node %s detected"),
                             options.host_uname);
                 return rc;
             }
             router_node = node->priv->name;
             attr_options |= pcmk__node_attr_remote;
         }
     }
 
     if (controld_api == NULL) {
         out->info(out, "Dry run: skipping clean-up of %s due to CIB_file",
                   options.host_uname? options.host_uname : "all nodes");
         rc = pcmk_rc_ok;
         return rc;
     }
 
     crm_debug("Re-checking the state of all resources on %s", options.host_uname?options.host_uname:"all nodes");
 
     rc = pcmk__attrd_api_clear_failures(NULL, options.host_uname, NULL,
                                         NULL, NULL, NULL, attr_options);
 
     if (pcmk_controld_api_reprobe(controld_api, options.host_uname,
                                   router_node) == pcmk_rc_ok) {
         start_mainloop(controld_api);
     }
 
     return rc;
 }
 
 static void
 refresh_resource(pcmk__output_t *out, pcmk_resource_t *rsc, pcmk_node_t *node)
 {
     int rc = pcmk_rc_ok;
 
     if (options.force == FALSE) {
         rsc = uber_parent(rsc);
     }
 
     crm_debug("Re-checking the state of %s (%s requested) on %s",
               rsc->id, options.rsc_id, (options.host_uname? options.host_uname: "all nodes"));
     rc = cli_resource_delete(controld_api, options.host_uname, rsc, NULL, 0,
                              FALSE, scheduler, options.force);
 
     if ((rc == pcmk_rc_ok) && !out->is_quiet(out)) {
         // Show any reasons why resource might stay stopped
         cli_resource_check(out, rsc, node);
     }
 
     if (rc == pcmk_rc_ok) {
         start_mainloop(controld_api);
     }
 }
 
 static int
 show_metadata(pcmk__output_t *out, const char *agent_spec)
 {
     int rc = pcmk_rc_ok;
     char *standard = NULL;
     char *provider = NULL;
     char *type = NULL;
     char *metadata = NULL;
     lrmd_t *lrmd_conn = NULL;
 
     rc = lrmd__new(&lrmd_conn, NULL, NULL, 0);
     if (rc != pcmk_rc_ok) {
         g_set_error(&error, PCMK__RC_ERROR, rc,
                     _("Could not create executor connection"));
         lrmd_api_delete(lrmd_conn);
         return rc;
     }
 
     rc = crm_parse_agent_spec(agent_spec, &standard, &provider, &type);
     rc = pcmk_legacy2rc(rc);
 
     if (rc == pcmk_rc_ok) {
         rc = lrmd_conn->cmds->get_metadata(lrmd_conn, standard,
                                            provider, type,
                                            &metadata, 0);
         rc = pcmk_legacy2rc(rc);
 
         if (metadata) {
             out->output_xml(out, PCMK_XE_METADATA, metadata);
             free(metadata);
         } else {
             /* We were given a validly formatted spec, but it doesn't necessarily
              * match up with anything that exists.  Use ENXIO as the return code
              * here because that maps to an exit code of CRM_EX_NOSUCH, which
              * probably is the most common reason to get here.
              */
             rc = ENXIO;
             g_set_error(&error, PCMK__RC_ERROR, rc,
                         _("Metadata query for %s failed: %s"),
                         agent_spec, pcmk_rc_str(rc));
         }
     } else {
         rc = ENXIO;
         g_set_error(&error, PCMK__RC_ERROR, rc,
                     _("'%s' is not a valid agent specification"), agent_spec);
     }
 
     lrmd_api_delete(lrmd_conn);
     return rc;
 }
 
 static void
 validate_cmdline_config(void)
 {
     // Cannot use both --resource and command-line resource configuration
     if (options.rsc_id != NULL) {
         g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_USAGE,
                     _("--resource cannot be used with --class, --agent, and --provider"));
 
     // Not all commands support command-line resource configuration
     } else if (options.rsc_cmd != cmd_execute_agent) {
         g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_USAGE,
                     _("--class, --agent, and --provider can only be used with "
                     "--validate and --force-*"));
 
     // Not all of --class, --agent, and --provider need to be given.  Not all
     // classes support the concept of a provider.  Check that what we were given
     // is valid.
     } else if (pcmk__str_eq(options.v_class, "stonith", pcmk__str_none)) {
         if (options.v_provider != NULL) {
             g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_USAGE,
                         _("stonith does not support providers"));
 
         } else if (stonith_agent_exists(options.v_agent, 0) == FALSE) {
             g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_USAGE,
                         _("%s is not a known stonith agent"), options.v_agent ? options.v_agent : "");
         }
 
     } else if (resources_agent_exists(options.v_class, options.v_provider, options.v_agent) == FALSE) {
         g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_USAGE,
                     _("%s:%s:%s is not a known resource"),
                     options.v_class ? options.v_class : "",
                     options.v_provider ? options.v_provider : "",
                     options.v_agent ? options.v_agent : "");
     }
 
     if ((error == NULL) && (options.cmdline_params == NULL)) {
         options.cmdline_params = pcmk__strkey_table(free, free);
     }
 }
 
 /*!
  * \internal
  * \brief Get the <tt>enum pe_find</tt> flags for a given command
  *
  * \return <tt>enum pe_find</tt> flag group appropriate for \c options.rsc_cmd.
  */
 static uint32_t
 get_find_flags(void)
 {
     switch (options.rsc_cmd) {
         case cmd_ban:
         case cmd_cleanup:
         case cmd_clear:
         case cmd_colocations:
         case cmd_digests:
         case cmd_execute_agent:
         case cmd_locate:
         case cmd_move:
         case cmd_refresh:
         case cmd_restart:
         case cmd_why:
             return pcmk_rsc_match_history|pcmk_rsc_match_anon_basename;
 
         case cmd_delete_param:
         case cmd_get_param:
         case cmd_query_xml_raw:
         case cmd_query_xml:
         case cmd_set_param:
             return pcmk_rsc_match_history|pcmk_rsc_match_basename;
 
         default:
             return 0;
     }
 }
 
 /*!
  * \internal
  * \brief Check whether a node argument is required
  *
  * \return \c true if a \c --node argument is required, or \c false otherwise
  */
 static bool
 is_node_required(void)
 {
     switch (options.rsc_cmd) {
         case cmd_digests:
         case cmd_fail:
             return true;
         default:
             return false;
     }
 }
 
 /*!
  * \internal
  * \brief Check whether a resource argument is required
  *
  * \return \c true if a \c --resource argument is required, or \c false
  *         otherwise
  */
 static bool
 is_resource_required(void)
 {
     if (options.cmdline_config) {
         return false;
     }
 
     switch (options.rsc_cmd) {
         case cmd_clear:
             return !options.clear_expired;
 
         case cmd_cleanup:
         case cmd_cts:
         case cmd_list_active_ops:
         case cmd_list_agents:
         case cmd_list_all_ops:
         case cmd_list_alternatives:
         case cmd_list_instances:
         case cmd_list_options:
         case cmd_list_providers:
         case cmd_list_resources:
         case cmd_list_standards:
         case cmd_metadata:
         case cmd_refresh:
         case cmd_wait:
         case cmd_why:
             return false;
 
         default:
             return true;
     }
 }
 
 /*!
  * \internal
  * \brief Check whether a CIB connection is required
  *
  * \return \c true if a CIB connection is required, or \c false otherwise
  */
 static bool
 is_cib_required(void)
 {
     if (options.cmdline_config) {
         return false;
     }
 
     switch (options.rsc_cmd) {
         case cmd_list_agents:
         case cmd_list_alternatives:
         case cmd_list_options:
         case cmd_list_providers:
         case cmd_list_standards:
         case cmd_metadata:
             return false;
         default:
             return true;
     }
 }
 
 /*!
  * \internal
  * \brief Check whether a controller IPC connection is required
  *
  * \return \c true if a controller connection is required, or \c false otherwise
  */
 static bool
 is_controller_required(void)
 {
     switch (options.rsc_cmd) {
         case cmd_cleanup:
         case cmd_refresh:
             return getenv("CIB_file") == NULL;
 
         case cmd_fail:
             return true;
 
         default:
             return false;
     }
 }
 
 /*!
  * \internal
  * \brief Check whether a scheduler IPC connection is required
  *
  * \return \c true if a scheduler connection is required, or \c false otherwise
  */
 static bool
 is_scheduler_required(void)
 {
     if (options.cmdline_config) {
         return false;
     }
 
     switch (options.rsc_cmd) {
         case cmd_delete:
         case cmd_list_agents:
         case cmd_list_alternatives:
         case cmd_list_options:
         case cmd_list_providers:
         case cmd_list_standards:
         case cmd_metadata:
         case cmd_wait:
             return false;
         default:
             return true;
     }
 }
 
 /*!
  * \internal
  * \brief Check whether the chosen command accepts clone instances
  *
  * \return \c true if \p options.rsc_cmd accepts or ignores clone instances, or
  *         \c false otherwise
  */
 static bool
 accept_clone_instance(void)
 {
     switch (options.rsc_cmd) {
         case cmd_ban:
         case cmd_clear:
         case cmd_delete:
         case cmd_move:
         case cmd_restart:
             return false;
         default:
             return true;
     }
 }
 
 static GOptionContext *
 build_arg_context(pcmk__common_args_t *args, GOptionGroup **group) {
     GOptionContext *context = NULL;
 
     GOptionEntry extra_prog_entries[] = {
         { "quiet", 'Q', G_OPTION_FLAG_NONE, G_OPTION_ARG_NONE, &(args->quiet),
           "Be less descriptive in output.",
           NULL },
         { "resource", 'r', G_OPTION_FLAG_NONE, G_OPTION_ARG_STRING, &options.rsc_id,
           "Resource ID",
           "ID" },
         { G_OPTION_REMAINING, 0, G_OPTION_FLAG_NONE, G_OPTION_ARG_STRING_ARRAY, &options.remainder,
           NULL,
           NULL },
 
         { NULL }
     };
 
     const char *description = "Examples:\n\n"
                               "List the available OCF agents:\n\n"
                               "\t# crm_resource --list-agents ocf\n\n"
                               "List the available OCF agents from the linux-ha project:\n\n"
                               "\t# crm_resource --list-agents ocf:heartbeat\n\n"
                               "Move 'myResource' to a specific node:\n\n"
                               "\t# crm_resource --resource myResource --move --node altNode\n\n"
                               "Allow (but not force) 'myResource' to move back to its original "
                               "location:\n\n"
                               "\t# crm_resource --resource myResource --clear\n\n"
                               "Stop 'myResource' (and anything that depends on it):\n\n"
                               "\t# crm_resource --resource myResource --set-parameter "
                               PCMK_META_TARGET_ROLE "--meta --parameter-value Stopped\n\n"
                               "Tell the cluster not to manage 'myResource' (the cluster will not "
                               "attempt to start or stop the\n"
                               "resource under any circumstances; useful when performing maintenance "
                               "tasks on a resource):\n\n"
                               "\t# crm_resource --resource myResource --set-parameter "
                               PCMK_META_IS_MANAGED "--meta --parameter-value false\n\n"
                               "Erase the operation history of 'myResource' on 'aNode' (the cluster "
                               "will 'forget' the existing\n"
                               "resource state, including any errors, and attempt to recover the"
                               "resource; useful when a resource\n"
                               "had failed permanently and has been repaired by an administrator):\n\n"
                               "\t# crm_resource --resource myResource --cleanup --node aNode\n\n";
 
     context = pcmk__build_arg_context(args, "text (default), xml", group, NULL);
     g_option_context_set_description(context, description);
 
     /* Add the -Q option, which cannot be part of the globally supported options
      * because some tools use that flag for something else.
      */
     pcmk__add_main_args(context, extra_prog_entries);
 
     pcmk__add_arg_group(context, "queries", "Queries:",
                         "Show query help", query_entries);
     pcmk__add_arg_group(context, "commands", "Commands:",
                         "Show command help", command_entries);
     pcmk__add_arg_group(context, "locations", "Locations:",
                         "Show location help", location_entries);
     pcmk__add_arg_group(context, "advanced", "Advanced:",
                         "Show advanced option help", advanced_entries);
     pcmk__add_arg_group(context, "additional", "Additional Options:",
                         "Show additional options", addl_entries);
     return context;
 }
 
 int
 main(int argc, char **argv)
 {
+    xmlNode *cib_xml_orig = NULL;
     pcmk_resource_t *rsc = NULL;
     pcmk_node_t *node = NULL;
     uint32_t find_flags = 0;
     int rc = pcmk_rc_ok;
 
     GOptionGroup *output_group = NULL;
     gchar **processed_args = NULL;
     GOptionContext *context = NULL;
 
     /*
      * Parse command line arguments
      */
 
     args = pcmk__new_common_args(SUMMARY);
     processed_args = pcmk__cmdline_preproc(argv, "GHINSTdginpstuvx");
     context = build_arg_context(args, &output_group);
 
     pcmk__register_formats(output_group, formats);
     if (!g_option_context_parse_strv(context, &processed_args, &error)) {
         exit_code = CRM_EX_USAGE;
         goto done;
     }
 
     pcmk__cli_init_logging("crm_resource", args->verbosity);
 
     rc = pcmk__output_new(&out, args->output_ty, args->output_dest, argv);
     if (rc != pcmk_rc_ok) {
         exit_code = CRM_EX_ERROR;
         g_set_error(&error, PCMK__EXITC_ERROR, exit_code, _("Error creating output format %s: %s"),
                     args->output_ty, pcmk_rc_str(rc));
         goto done;
     }
 
     pe__register_messages(out);
     crm_resource_register_messages(out);
     lrmd__register_messages(out);
     pcmk__register_lib_messages(out);
 
     out->quiet = args->quiet;
 
     crm_log_args(argc, argv);
 
     /*
      * Validate option combinations
      */
 
     // --expired without --clear/-U doesn't make sense
     if (options.clear_expired && (options.rsc_cmd != cmd_clear)) {
         exit_code = CRM_EX_USAGE;
         g_set_error(&error, PCMK__EXITC_ERROR, exit_code, _("--expired requires --clear or -U"));
         goto done;
     }
 
     if ((options.remainder != NULL) && (options.override_params != NULL)) {
         // Commands that use positional arguments will create override_params
         for (gchar **s = options.remainder; *s; s++) {
             char *name = pcmk__assert_alloc(1, strlen(*s));
             char *value = pcmk__assert_alloc(1, strlen(*s));
             int rc = sscanf(*s, "%[^=]=%s", name, value);
 
             if (rc == 2) {
                 g_hash_table_replace(options.override_params, name, value);
 
             } else {
                 exit_code = CRM_EX_USAGE;
                 g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
                             _("Error parsing '%s' as a name=value pair"),
                             argv[optind]);
                 free(value);
                 free(name);
                 goto done;
             }
         }
 
     } else if (options.remainder != NULL) {
         gchar **strv = NULL;
         gchar *msg = NULL;
         int i = 1;
         int len = 0;
 
         for (gchar **s = options.remainder; *s; s++) {
             len++;
         }
 
         pcmk__assert(len > 0);
 
         /* Add 1 for the strv[0] string below, and add another 1 for the NULL
          * at the end of the array so g_strjoinv knows when to stop.
          */
         strv = pcmk__assert_alloc(len+2, sizeof(char *));
         strv[0] = strdup("non-option ARGV-elements:\n");
 
         for (gchar **s = options.remainder; *s; s++) {
             strv[i] = crm_strdup_printf("[%d of %d] %s\n", i, len, *s);
             i++;
         }
 
         strv[i] = NULL;
 
         exit_code = CRM_EX_USAGE;
         msg = g_strjoinv("", strv);
         g_set_error(&error, PCMK__EXITC_ERROR, exit_code, "%s", msg);
         g_free(msg);
 
         /* Don't try to free the last element, which is just NULL. */
         for(i = 0; i < len+1; i++) {
             free(strv[i]);
         }
         free(strv);
 
         goto done;
     }
 
     if (pcmk__str_eq(args->output_ty, "xml", pcmk__str_none)) {
         switch (options.rsc_cmd) {
             /* These are the only commands that have historically used the <list>
              * elements in their XML schema.  For all others, use the simple list
              * argument.
              */
             case cmd_get_param:
             case cmd_list_instances:
             case cmd_list_standards:
                 pcmk__output_enable_list_element(out);
                 break;
 
             default:
                 break;
         }
 
     } else if (pcmk__str_eq(args->output_ty, "text", pcmk__str_null_matches)) {
         switch (options.rsc_cmd) {
             case cmd_colocations:
             case cmd_list_resources:
                 pcmk__output_text_set_fancy(out, true);
                 break;
             default:
                 break;
         }
     }
 
     if (args->version) {
         out->version(out, false);
         goto done;
     }
 
     if (options.cmdline_config) {
         /* A resource configuration was given on the command line. Sanity-check
          * the values and set error if they don't make sense.
          */
         validate_cmdline_config();
         if (error != NULL) {
             exit_code = CRM_EX_USAGE;
             goto done;
         }
 
     } else if (options.cmdline_params != NULL) {
         exit_code = CRM_EX_USAGE;
         g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
                     _("--option must be used with --validate and without -r"));
         g_hash_table_destroy(options.cmdline_params);
         options.cmdline_params = NULL;
         goto done;
     }
 
     if (is_resource_required() && (options.rsc_id == NULL)) {
         exit_code = CRM_EX_USAGE;
         g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
                     _("Must supply a resource id with -r"));
         goto done;
     }
     if (is_node_required() && (options.host_uname == NULL)) {
         exit_code = CRM_EX_USAGE;
         g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
                     _("Must supply a node name with -N"));
         goto done;
     }
 
     /*
      * Set up necessary connections
      */
 
     // Establish a connection to the CIB if needed
     if (is_cib_required()) {
         cib_conn = cib_new();
         if ((cib_conn == NULL) || (cib_conn->cmds == NULL)) {
             exit_code = CRM_EX_DISCONNECT;
             g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
                         _("Could not create CIB connection"));
             goto done;
         }
         rc = cib__signon_attempts(cib_conn, cib_command, 5);
         rc = pcmk_legacy2rc(rc);
         if (rc != pcmk_rc_ok) {
             exit_code = pcmk_rc2exitc(rc);
             g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
                         _("Could not connect to the CIB: %s"), pcmk_rc_str(rc));
             goto done;
         }
     }
 
     // Populate scheduler data from XML file if specified or CIB query otherwise
     if (is_scheduler_required()) {
-        rc = initialize_scheduler_data();
+        rc = initialize_scheduler_data(&cib_xml_orig);
         if (rc != pcmk_rc_ok) {
             exit_code = pcmk_rc2exitc(rc);
             goto done;
         }
     }
 
     find_flags = get_find_flags();
 
     // If command requires that resource exist if specified, find it
     if ((find_flags != 0) && (options.rsc_id != NULL)) {
         rsc = pe_find_resource_with_flags(scheduler->priv->resources,
                                           options.rsc_id, find_flags);
         if (rsc == NULL) {
             exit_code = CRM_EX_NOSUCH;
             g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
                         _("Resource '%s' not found"), options.rsc_id);
             goto done;
         }
 
         /* The --ban, --clear, --move, and --restart commands do not work with
          * instances of clone resourcs.
          */
         if (pcmk__is_clone(rsc->priv->parent)
             && (strchr(options.rsc_id, ':') != NULL)
             && !accept_clone_instance()) {
 
             exit_code = CRM_EX_INVALID_PARAM;
             g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
                         _("Cannot operate on clone resource instance '%s'"), options.rsc_id);
             goto done;
         }
     }
 
     // If user supplied a node name, check whether it exists
     if ((options.host_uname != NULL) && (scheduler != NULL)) {
         node = pcmk_find_node(scheduler, options.host_uname);
 
         if (node == NULL) {
             exit_code = CRM_EX_NOSUCH;
             g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
                         _("Node '%s' not found"), options.host_uname);
             goto done;
         }
     }
 
     // Establish a connection to the controller if needed
     if (is_controller_required()) {
         rc = pcmk_new_ipc_api(&controld_api, pcmk_ipc_controld);
         if (rc != pcmk_rc_ok) {
             exit_code = pcmk_rc2exitc(rc);
             g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
                         _("Error connecting to the controller: %s"), pcmk_rc_str(rc));
             goto done;
         }
         pcmk_register_ipc_callback(controld_api, controller_event_callback,
                                    NULL);
         rc = pcmk__connect_ipc(controld_api, pcmk_ipc_dispatch_main, 5);
         if (rc != pcmk_rc_ok) {
             exit_code = pcmk_rc2exitc(rc);
             g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
                         _("Error connecting to %s: %s"),
                         pcmk_ipc_name(controld_api, true), pcmk_rc_str(rc));
             goto done;
         }
     }
 
     /*
      * Handle requested command
      */
 
     switch (options.rsc_cmd) {
         case cmd_list_resources: {
             GList *all = NULL;
             uint32_t show_opts = pcmk_show_inactive_rscs | pcmk_show_rsc_only | pcmk_show_pending;
 
             all = g_list_prepend(all, (gpointer) "*");
             rc = out->message(out, "resource-list", scheduler,
                               show_opts, true, all, all, false);
             g_list_free(all);
 
             if (rc == pcmk_rc_no_output) {
                 rc = ENXIO;
             }
             break;
         }
 
         case cmd_list_instances:
             rc = out->message(out, "resource-names-list",
                               scheduler->priv->resources);
 
             if (rc != pcmk_rc_ok) {
                 rc = ENXIO;
             }
 
             break;
 
         case cmd_list_options:
             list_options();
             break;
 
         case cmd_list_alternatives:
             rc = pcmk__list_alternatives(out, options.agent_spec);
             break;
 
         case cmd_list_agents:
             rc = pcmk__list_agents(out, options.agent_spec);
             break;
 
         case cmd_list_standards:
             rc = pcmk__list_standards(out);
             break;
 
         case cmd_list_providers:
             rc = pcmk__list_providers(out, options.agent_spec);
             break;
 
         case cmd_metadata:
             rc = show_metadata(out, options.agent_spec);
             break;
 
         case cmd_restart:
             /* We don't pass scheduler because rsc needs to stay valid for the
              * entire lifetime of cli_resource_restart(), but it will reset and
              * update the scheduler data multiple times, so it needs to use its
              * own copy.
              */
             rc = cli_resource_restart(out, rsc, node, options.move_lifetime,
                                       options.timeout_ms, cib_conn,
                                       options.promoted_role_only,
                                       options.force);
             break;
 
         case cmd_wait:
             rc = wait_till_stable(out, options.timeout_ms, cib_conn);
             break;
 
         case cmd_execute_agent:
             if (options.cmdline_config) {
                 exit_code = cli_resource_execute_from_params(out, NULL,
                     options.v_class, options.v_provider, options.v_agent,
                     options.operation, options.cmdline_params,
                     options.override_params, options.timeout_ms,
                     args->verbosity, options.force, options.check_level);
             } else {
                 exit_code = cli_resource_execute(rsc, options.rsc_id,
                     options.operation, options.override_params,
                     options.timeout_ms, cib_conn, scheduler,
                     args->verbosity, options.force, options.check_level);
             }
             goto done;
 
         case cmd_digests:
             node = pcmk_find_node(scheduler, options.host_uname);
             if (node == NULL) {
                 rc = pcmk_rc_node_unknown;
             } else {
                 rc = pcmk__resource_digests(out, rsc, node,
                                             options.override_params);
             }
             break;
 
         case cmd_colocations:
             rc = out->message(out, "locations-and-colocations", rsc,
                               options.recursive, (bool) options.force);
             break;
 
         case cmd_cts:
             rc = pcmk_rc_ok;
             g_list_foreach(scheduler->priv->resources,
                            (GFunc) cli_resource_print_cts, out);
             cli_resource_print_cts_constraints(scheduler);
             break;
 
         case cmd_fail:
             rc = cli_resource_fail(controld_api, options.host_uname,
                                    options.rsc_id, scheduler);
             if (rc == pcmk_rc_ok) {
                 start_mainloop(controld_api);
             }
             break;
 
         case cmd_list_active_ops:
             rc = cli_resource_print_operations(options.rsc_id,
                                                options.host_uname, TRUE,
                                                scheduler);
             break;
 
         case cmd_list_all_ops:
             rc = cli_resource_print_operations(options.rsc_id,
                                                options.host_uname, FALSE,
                                                scheduler);
             break;
 
         case cmd_locate: {
             GList *nodes = cli_resource_search(rsc, options.rsc_id, scheduler);
             rc = out->message(out, "resource-search-list", nodes, options.rsc_id);
             g_list_free_full(nodes, free);
             break;
         }
 
         case cmd_query_xml:
             rc = cli_resource_print(rsc, scheduler, true);
             break;
 
         case cmd_query_xml_raw:
             rc = cli_resource_print(rsc, scheduler, false);
             break;
 
         case cmd_why:
             if ((options.host_uname != NULL) && (node == NULL)) {
                 rc = pcmk_rc_node_unknown;
             } else {
                 rc = out->message(out, "resource-reasons-list",
                                   scheduler->priv->resources, rsc, node);
             }
             break;
 
         case cmd_clear:
             rc = clear_constraints(out);
             break;
 
         case cmd_move:
             if (options.host_uname == NULL) {
                 rc = ban_or_move(out, rsc, options.move_lifetime);
             } else {
                 rc = cli_resource_move(rsc, options.rsc_id, options.host_uname,
                                        options.move_lifetime, cib_conn,
                                        scheduler, options.promoted_role_only,
                                        options.force);
             }
 
             if (rc == EINVAL) {
                 exit_code = CRM_EX_USAGE;
                 goto done;
             }
 
             break;
 
         case cmd_ban:
             if (options.host_uname == NULL) {
                 rc = ban_or_move(out, rsc, options.move_lifetime);
             } else if (node == NULL) {
                 rc = pcmk_rc_node_unknown;
             } else {
                 rc = cli_resource_ban(out, options.rsc_id, node->priv->name,
                                       options.move_lifetime, cib_conn,
                                       options.promoted_role_only,
                                       PCMK_ROLE_PROMOTED);
             }
 
             if (rc == EINVAL) {
                 exit_code = CRM_EX_USAGE;
                 goto done;
             }
 
             break;
 
         case cmd_get_param: {
             unsigned int count = 0;
             GHashTable *params = NULL;
             pcmk_node_t *current = rsc->priv->fns->active_node(rsc, &count,
                                                                NULL);
             bool free_params = true;
             const char* value = NULL;
 
             if (count > 1) {
                 out->err(out, "%s is active on more than one node,"
                          " returning the default value for %s", rsc->id,
                          pcmk__s(options.prop_name, "unspecified property"));
                 current = NULL;
             }
 
             crm_debug("Looking up %s in %s", options.prop_name, rsc->id);
 
             if (pcmk__str_eq(options.attr_set_type, PCMK_XE_INSTANCE_ATTRIBUTES,
                              pcmk__str_none)) {
                 params = pe_rsc_params(rsc, current, scheduler);
                 free_params = false;
 
                 value = g_hash_table_lookup(params, options.prop_name);
 
             } else if (pcmk__str_eq(options.attr_set_type,
                                     PCMK_XE_META_ATTRIBUTES, pcmk__str_none)) {
                 params = pcmk__strkey_table(free, free);
                 get_meta_attributes(params, rsc, NULL, scheduler);
 
                 value = g_hash_table_lookup(params, options.prop_name);
 
             } else if (pcmk__str_eq(options.attr_set_type, ATTR_SET_ELEMENT, pcmk__str_none)) {
 
                 value = crm_element_value(rsc->priv->xml, options.prop_name);
                 free_params = false;
 
             } else {
                 pe_rule_eval_data_t rule_data = {
                     .now = scheduler->priv->now,
                 };
 
                 params = pcmk__strkey_table(free, free);
                 pe__unpack_dataset_nvpairs(rsc->priv->xml, PCMK_XE_UTILIZATION,
                                            &rule_data, params, NULL, scheduler);
 
                 value = g_hash_table_lookup(params, options.prop_name);
             }
 
             rc = out->message(out, "attribute-list", rsc, options.prop_name, value);
             if (free_params) {
                 g_hash_table_destroy(params);
             }
 
             break;
         }
 
         case cmd_set_param:
             if (pcmk__str_empty(options.prop_value)) {
                 exit_code = CRM_EX_USAGE;
                 g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
                             _("You need to supply a value with the -v option"));
                 goto done;
             }
 
             /* coverity[var_deref_model] False positive */
             rc = cli_resource_update_attribute(rsc, options.rsc_id,
                                                options.prop_set,
                                                options.attr_set_type,
                                                options.prop_id,
                                                options.prop_name,
                                                options.prop_value,
                                                options.recursive, cib_conn,
-                                               options.force);
+                                               cib_xml_orig, options.force);
             break;
 
         case cmd_delete_param:
             /* coverity[var_deref_model] False positive */
             rc = cli_resource_delete_attribute(rsc, options.rsc_id,
                                                options.prop_set,
                                                options.attr_set_type,
                                                options.prop_id,
                                                options.prop_name, cib_conn,
-                                               options.force);
+                                               cib_xml_orig, options.force);
             break;
 
         case cmd_cleanup:
             if (rsc == NULL) {
                 rc = cli_cleanup_all(controld_api, options.host_uname,
                                      options.operation, options.interval_spec,
                                      scheduler);
                 if (rc == pcmk_rc_ok) {
                     start_mainloop(controld_api);
                 }
             } else {
                 cleanup(out, rsc, node);
             }
             break;
 
         case cmd_refresh:
             if (rsc == NULL) {
                 rc = refresh(out);
             } else {
                 refresh_resource(out, rsc, node);
             }
             break;
 
         case cmd_delete:
             /* rsc_id was already checked for NULL much earlier when validating
              * command line arguments.
              */
             if (options.rsc_type == NULL) {
                 exit_code = CRM_EX_USAGE;
                 g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_USAGE,
                             _("You need to specify a resource type with -t"));
             } else {
                 rc = pcmk__resource_delete(cib_conn, cib_sync_call,
                                            options.rsc_id, options.rsc_type);
 
                 if (rc != pcmk_rc_ok) {
                     g_set_error(&error, PCMK__RC_ERROR, rc,
                                 _("Could not delete resource %s: %s"),
                                 options.rsc_id, pcmk_rc_str(rc));
                 }
             }
 
             break;
 
         default:
             exit_code = CRM_EX_USAGE;
             g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
                         _("Unimplemented command: %d"), (int) options.rsc_cmd);
             goto done;
     }
 
     /* Convert rc into an exit code. */
     if (rc != pcmk_rc_ok && rc != pcmk_rc_no_output) {
         exit_code = pcmk_rc2exitc(rc);
     }
 
     /*
      * Clean up and exit
      */
 
 done:
     /* When we get here, exit_code has been set one of two ways - either at one of
      * the spots where there's a "goto done" (which itself could have happened either
      * directly or by calling pcmk_rc2exitc), or just up above after any of the break
      * statements.
      *
      * Thus, we can use just exit_code here to decide what to do.
      */
     if (exit_code != CRM_EX_OK && exit_code != CRM_EX_USAGE) {
         if (error != NULL) {
             char *msg = crm_strdup_printf("%s\nError performing operation: %s",
                                           error->message, crm_exit_str(exit_code));
             g_clear_error(&error);
             g_set_error(&error, PCMK__EXITC_ERROR, exit_code, "%s", msg);
             free(msg);
         } else {
             g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
                         _("Error performing operation: %s"), crm_exit_str(exit_code));
         }
     }
 
+    pcmk__xml_free(cib_xml_orig);
+
     g_free(options.host_uname);
     g_free(options.interval_spec);
     g_free(options.move_lifetime);
     g_free(options.operation);
     g_free(options.prop_id);
     free(options.prop_name);
     g_free(options.prop_set);
     g_free(options.prop_value);
     g_free(options.rsc_id);
     g_free(options.rsc_type);
     free(options.agent_spec);
     free(options.v_agent);
     free(options.v_class);
     free(options.v_provider);
     g_strfreev(options.remainder);
 
     if (options.override_params != NULL) {
         g_hash_table_destroy(options.override_params);
     }
 
     /* options.cmdline_params does not need to be destroyed here.  See the
      * comments in cli_resource_execute_from_params.
      */
 
     g_strfreev(processed_args);
     g_option_context_free(context);
 
     return bye(exit_code);
 }
diff --git a/tools/crm_resource.h b/tools/crm_resource.h
index 5855011d5a..56d11a55b8 100644
--- a/tools/crm_resource.h
+++ b/tools/crm_resource.h
@@ -1,139 +1,142 @@
 /*
  * Copyright 2004-2024 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU Lesser General Public License
  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #include <stdint.h>
 #include <stdbool.h>
 
 #include <crm/crm.h>
 
 #include <crm/services.h>
 #include <crm/common/xml.h>
 #include <crm/common/mainloop.h>
 #include <crm/common/output_internal.h>
 #include <crm/common/scheduler_internal.h>
 
 #include <crm/cib.h>
 #include <crm/common/attrs_internal.h>
 #include <crm/pengine/rules.h>
 #include <crm/pengine/status.h>
 #include <crm/pengine/internal.h>
 #include <pacemaker-internal.h>
 
 #define ATTR_SET_ELEMENT "attr_set_element"
 
 typedef struct node_info_s {
     const char *node_name;
     bool promoted;
 } node_info_t;
 
 typedef struct {
     char *attr_set_type;
     char *attr_set_id;
     char *attr_name;
     char *attr_value;
     char *given_rsc_id;
     char *found_attr_id;
     pcmk_resource_t *rsc;
 } attr_update_data_t;
 
 enum resource_check_flags {
     rsc_remain_stopped  = (1 << 0),
     rsc_unpromotable    = (1 << 1),
     rsc_unmanaged       = (1 << 2),
     rsc_locked          = (1 << 3),
     rsc_node_health     = (1 << 4),
 };
 
 typedef struct resource_checks_s {
     pcmk_resource_t *rsc;   // Resource being checked
     uint32_t flags;         // Group of enum resource_check_flags
     const char *lock_node;  // Node that resource is shutdown-locked to, if any
 } resource_checks_t;
 
 resource_checks_t *cli_check_resource(pcmk_resource_t *rsc, char *role_s,
                                       char *managed);
 
 /* ban */
 int cli_resource_prefer(pcmk__output_t *out, const char *rsc_id, const char *host,
                         const char *move_lifetime, cib_t *cib_conn,
                         gboolean promoted_role_only, const char *promoted_role);
 int cli_resource_ban(pcmk__output_t *out, const char *rsc_id, const char *host,
                      const char *move_lifetime, cib_t *cib_conn,
                      gboolean promoted_role_only, const char *promoted_role);
 int cli_resource_clear(const char *rsc_id, const char *host, GList *allnodes,
                        cib_t *cib_conn, bool clear_ban_constraints,
                        gboolean force);
 int cli_resource_clear_all_expired(xmlNode *root, cib_t *cib_conn,
                                    const char *rsc, const char *node, gboolean promoted_role_only);
 
 /* print */
 void cli_resource_print_cts(pcmk_resource_t *rsc, pcmk__output_t *out);
 void cli_resource_print_cts_constraints(pcmk_scheduler_t *scheduler);
 
 int cli_resource_print(pcmk_resource_t *rsc, pcmk_scheduler_t *scheduler,
                        bool expanded);
 int cli_resource_print_operations(const char *rsc_id, const char *host_uname,
                                   bool active, pcmk_scheduler_t *scheduler);
 
 /* runtime */
 int cli_resource_check(pcmk__output_t *out, pcmk_resource_t *rsc,
                        pcmk_node_t *node);
 int cli_resource_fail(pcmk_ipc_api_t *controld_api, const char *host_uname,
                       const char *rsc_id, pcmk_scheduler_t *scheduler);
 GList *cli_resource_search(pcmk_resource_t *rsc, const char *requested_name,
                              pcmk_scheduler_t *scheduler);
 int cli_resource_delete(pcmk_ipc_api_t *controld_api, const char *host_uname,
                         const pcmk_resource_t *rsc, const char *operation,
                         const char *interval_spec, bool just_failures,
                         pcmk_scheduler_t *scheduler, gboolean force);
 int cli_cleanup_all(pcmk_ipc_api_t *controld_api, const char *node_name,
                     const char *operation, const char *interval_spec,
                     pcmk_scheduler_t *scheduler);
 int cli_resource_restart(pcmk__output_t *out, pcmk_resource_t *rsc,
                          const pcmk_node_t *node, const char *move_lifetime,
                          guint timeout_ms, cib_t *cib,
                          gboolean promoted_role_only, gboolean force);
 int cli_resource_move(const pcmk_resource_t *rsc, const char *rsc_id,
                       const char *host_name, const char *move_lifetime,
                       cib_t *cib, pcmk_scheduler_t *scheduler,
                       gboolean promoted_role_only, gboolean force);
 crm_exit_t cli_resource_execute_from_params(pcmk__output_t *out, const char *rsc_name,
                                             const char *rsc_class, const char *rsc_prov,
                                             const char *rsc_type, const char *rsc_action,
                                             GHashTable *params, GHashTable *override_hash,
                                             guint timeout_ms,
                                             int resource_verbose,
                                             gboolean force, int check_level);
 crm_exit_t cli_resource_execute(pcmk_resource_t *rsc,
                                 const char *requested_name,
                                 const char *rsc_action, GHashTable *override_hash,
                                 guint timeout_ms, cib_t *cib,
                                 pcmk_scheduler_t *scheduler,
                                 int resource_verbose, gboolean force, int check_level);
 
 int cli_resource_update_attribute(pcmk_resource_t *rsc,
                                   const char *requested_name,
                                   const char *attr_set, const char *attr_set_type,
                                   const char *attr_id, const char *attr_name,
                                   const char *attr_value, gboolean recursive,
-                                  cib_t *cib, gboolean force);
+                                  cib_t *cib, xmlNode *cib_xml_orig,
+                                  gboolean force);
 int cli_resource_delete_attribute(pcmk_resource_t *rsc,
                                   const char *requested_name,
                                   const char *attr_set, const char *attr_set_type,
                                   const char *attr_id, const char *attr_name,
-                                  cib_t *cib, gboolean force);
+                                  cib_t *cib, xmlNode *cib_xml_orig,
+                                  gboolean force);
 
-int update_scheduler_input(pcmk_scheduler_t *scheduler, xmlNode **xml);
+int update_scheduler_input(pcmk__output_t *out, pcmk_scheduler_t *scheduler,
+                           cib_t *cib, xmlNode **cib_xml_orig);
 int wait_till_stable(pcmk__output_t *out, guint timeout_ms, cib_t * cib);
 
 bool resource_is_running_on(pcmk_resource_t *rsc, const char *host);
 
 void crm_resource_register_messages(pcmk__output_t *out);
diff --git a/tools/crm_resource_runtime.c b/tools/crm_resource_runtime.c
index 530ae1d866..7574ece6de 100644
--- a/tools/crm_resource_runtime.c
+++ b/tools/crm_resource_runtime.c
@@ -1,2429 +1,2497 @@
 /*
  * Copyright 2004-2024 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU General Public License version 2
  * or later (GPLv2+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #include <stdio.h>
 #include <limits.h>
 #include <glib.h>
 #include <libxml/tree.h>
 
 #include <crm/common/ipc_attrd_internal.h>
 #include <crm/common/ipc_controld.h>
 #include <crm/common/lists_internal.h>
 #include <crm/services_internal.h>
 
 #include <crm_resource.h>
 
 static GList *
 build_node_info_list(const pcmk_resource_t *rsc)
 {
     GList *retval = NULL;
 
     for (const GList *iter = rsc->priv->children;
          iter != NULL; iter = iter->next) {
 
         const pcmk_resource_t *child = (const pcmk_resource_t *) iter->data;
 
         for (const GList *iter2 = child->priv->active_nodes;
              iter2 != NULL; iter2 = iter2->next) {
 
             const pcmk_node_t *node = (const pcmk_node_t *) iter2->data;
             node_info_t *ni = pcmk__assert_alloc(1, sizeof(node_info_t));
 
             ni->node_name = node->priv->name;
             if (pcmk_is_set(rsc->flags, pcmk__rsc_promotable)
                 && (child->priv->fns->state(child,
                                             TRUE) == pcmk_role_promoted)) {
                 ni->promoted = true;
             }
 
             retval = g_list_prepend(retval, ni);
         }
     }
 
     return retval;
 }
 
 GList *
 cli_resource_search(pcmk_resource_t *rsc, const char *requested_name,
                     pcmk_scheduler_t *scheduler)
 {
     GList *retval = NULL;
     const pcmk_resource_t *parent = pe__const_top_resource(rsc, false);
 
     if (pcmk__is_clone(rsc)) {
         retval = build_node_info_list(rsc);
 
     /* The anonymous clone children's common ID is supplied */
     } else if (pcmk__is_clone(parent)
                && !pcmk_is_set(rsc->flags, pcmk__rsc_unique)
                && (rsc->priv->history_id != NULL)
                && pcmk__str_eq(requested_name, rsc->priv->history_id,
                                pcmk__str_none)
                && !pcmk__str_eq(requested_name, rsc->id, pcmk__str_none)) {
 
         retval = build_node_info_list(parent);
 
     } else {
         for (GList *iter = rsc->priv->active_nodes;
              iter != NULL; iter = iter->next) {
 
             pcmk_node_t *node = (pcmk_node_t *) iter->data;
             node_info_t *ni = pcmk__assert_alloc(1, sizeof(node_info_t));
 
             ni->node_name = node->priv->name;
             if (rsc->priv->fns->state(rsc, TRUE) == pcmk_role_promoted) {
                 ni->promoted = true;
             }
 
             retval = g_list_prepend(retval, ni);
         }
     }
 
     return retval;
 }
 
 // \return Standard Pacemaker return code
 static int
 find_resource_attr(pcmk__output_t *out, cib_t * the_cib, const char *attr,
                    const char *rsc, const char *attr_set_type, const char *set_name,
                    const char *attr_id, const char *attr_name, xmlNode **result)
 {
     xmlNode *xml_search;
     int rc = pcmk_rc_ok;
     GString *xpath = NULL;
     const char *xpath_base = NULL;
 
     if (result) {
         *result = NULL;
     }
 
     if(the_cib == NULL) {
         return ENOTCONN;
     }
 
     xpath_base = pcmk_cib_xpath_for(PCMK_XE_RESOURCES);
     if (xpath_base == NULL) {
         crm_err(PCMK_XE_RESOURCES " CIB element not known (bug?)");
         return ENOMSG;
     }
 
     xpath = g_string_sized_new(1024);
     pcmk__g_strcat(xpath,
                    xpath_base, "//*[@" PCMK_XA_ID "=\"", rsc, "\"]", NULL);
 
     if (attr_set_type != NULL) {
         pcmk__g_strcat(xpath, "/", attr_set_type, NULL);
         if (set_name != NULL) {
             pcmk__g_strcat(xpath, "[@" PCMK_XA_ID "=\"", set_name, "\"]",
                            NULL);
         }
     }
 
     g_string_append(xpath, "//" PCMK_XE_NVPAIR);
 
     if (attr_id != NULL && attr_name!= NULL) {
         pcmk__g_strcat(xpath,
                        "[@" PCMK_XA_ID "='", attr_id, "' "
                        "and @" PCMK_XA_NAME "='", attr_name, "']", NULL);
 
     } else if (attr_id != NULL) {
         pcmk__g_strcat(xpath, "[@" PCMK_XA_ID "='", attr_id, "']", NULL);
 
     } else if (attr_name != NULL) {
         pcmk__g_strcat(xpath, "[@" PCMK_XA_NAME "='", attr_name, "']", NULL);
     }
 
     rc = the_cib->cmds->query(the_cib, xpath->str, &xml_search,
                               cib_sync_call|cib_xpath);
     rc = pcmk_legacy2rc(rc);
 
     if (rc == pcmk_rc_ok) {
         crm_log_xml_debug(xml_search, "Match");
         if (xml_search->children != NULL) {
             rc = ENOTUNIQ;
             pcmk__warn_multiple_name_matches(out, xml_search, attr_name);
             out->spacer(out);
         }
     }
 
     if (result) {
         *result = xml_search;
     } else {
         pcmk__xml_free(xml_search);
     }
 
     g_string_free(xpath, TRUE);
     return rc;
 }
 
 /* PRIVATE. Use the find_matching_attr_resources instead. */
 static void
 find_matching_attr_resources_recursive(pcmk__output_t *out,
                                        GList /* <pcmk_resource_t*> */ **result,
                                        pcmk_resource_t *rsc, const char * attr_set,
                                        const char * attr_set_type, const char * attr_id,
                                        const char * attr_name, cib_t * cib, int depth)
 {
     int rc = pcmk_rc_ok;
     char *lookup_id = clone_strip(rsc->id);
 
     for (GList *gIter = rsc->priv->children;
          gIter != NULL; gIter = gIter->next) {
 
         find_matching_attr_resources_recursive(out, result,
                                                (pcmk_resource_t *) gIter->data,
                                                attr_set, attr_set_type, attr_id,
                                                attr_name, cib, depth+1);
         /* do it only once for clones */
         if (pcmk__is_clone(rsc)) {
             break;
         }
     }
 
     rc = find_resource_attr(out, cib, PCMK_XA_ID, lookup_id, attr_set_type,
                             attr_set, attr_id, attr_name, NULL);
     /* Post-order traversal.
      * The root is always on the list and it is the last item. */
     if((0 == depth) || (pcmk_rc_ok == rc)) {
         /* push the head */
         *result = g_list_append(*result, rsc);
     }
 
     free(lookup_id);
 }
 
 
 /* The result is a linearized pre-ordered tree of resources. */
 static GList/*<pcmk_resource_t*>*/ *
 find_matching_attr_resources(pcmk__output_t *out, pcmk_resource_t *rsc,
                              const char * rsc_id, const char * attr_set,
                              const char * attr_set_type, const char * attr_id,
                              const char * attr_name, cib_t * cib, const char * cmd,
                              gboolean force)
 {
     int rc = pcmk_rc_ok;
     char *lookup_id = NULL;
     GList * result = NULL;
 
     /* If --force is used, update only the requested resource (clone or primitive).
      * Otherwise, if the primitive has the attribute, use that.
      * Otherwise use the clone. */
     if(force == TRUE) {
         return g_list_append(result, rsc);
     }
     if (pcmk__is_clone(rsc->priv->parent)) {
         int rc = find_resource_attr(out, cib, PCMK_XA_ID, rsc_id, attr_set_type,
                                     attr_set, attr_id, attr_name, NULL);
 
         if(rc != pcmk_rc_ok) {
             rsc = rsc->priv->parent;
             out->info(out, "Performing %s of '%s' on '%s', the parent of '%s'",
                       cmd, attr_name, rsc->id, rsc_id);
         }
         return g_list_append(result, rsc);
 
     } else if ((rsc->priv->parent == NULL)
                && (rsc->priv->children != NULL) && pcmk__is_clone(rsc)) {
 
         pcmk_resource_t *child = rsc->priv->children->data;
 
         if (pcmk__is_primitive(child)) {
             lookup_id = clone_strip(child->id); /* Could be a cloned group! */
             rc = find_resource_attr(out, cib, PCMK_XA_ID, lookup_id,
                                     attr_set_type, attr_set, attr_id, attr_name, NULL);
 
             if(rc == pcmk_rc_ok) {
                 rsc = child;
                 out->info(out, "A value for '%s' already exists in child '%s', performing %s on that instead of '%s'",
                           attr_name, lookup_id, cmd, rsc_id);
             }
 
             free(lookup_id);
         }
         return g_list_append(result, rsc);
     }
     /* If the resource is a group ==> children inherit the attribute if defined. */
     find_matching_attr_resources_recursive(out, &result, rsc, attr_set,
                                            attr_set_type, attr_id, attr_name,
                                            cib, 0);
     return result;
 }
 
+/*!
+ * \internal
+ * \brief Get a resource's XML by resource ID from a given CIB XML tree
+ *
+ * \param[in] cib_xml  CIB XML to search
+ * \param[in] rsc      Resource whose XML to get
+ *
+ * \return Subtree of \p cib_xml belonging to \p rsc, or \c NULL if not found
+ */
+static xmlNode *
+get_cib_rsc(xmlNode *cib_xml, const pcmk_resource_t *rsc)
+{
+    char *xpath = crm_strdup_printf("%s//*[@" PCMK_XA_ID "='%s']",
+                                    pcmk_cib_xpath_for(PCMK_XE_RESOURCES),
+                                    pcmk__xe_id(rsc->priv->xml));
+    xmlNode *rsc_xml = get_xpath_object(xpath, cib_xml, LOG_ERR);
+
+    free(xpath);
+    return rsc_xml;
+}
+
 static int
 update_element_attribute(pcmk__output_t *out, pcmk_resource_t *rsc,
-                         cib_t *cib, const char *attr_name, const char *attr_value)
+                         cib_t *cib, xmlNode *cib_xml_orig,
+                         const char *attr_name, const char *attr_value)
 {
     int rc = pcmk_rc_ok;
+    xmlNode *rsc_xml = rsc->priv->xml;
 
-    if (cib == NULL) {
-        return ENOTCONN;
+    rsc_xml = get_cib_rsc(cib_xml_orig, rsc);
+    if (rsc_xml == NULL) {
+        return ENXIO;
     }
 
-    crm_xml_add(rsc->priv->xml, attr_name, attr_value);
+    crm_xml_add(rsc_xml, attr_name, attr_value);
 
-    rc = cib->cmds->replace(cib, PCMK_XE_RESOURCES, rsc->priv->xml,
-                            cib_sync_call);
+    rc = cib->cmds->replace(cib, PCMK_XE_RESOURCES, rsc_xml, cib_sync_call);
     rc = pcmk_legacy2rc(rc);
     if (rc == pcmk_rc_ok) {
         out->info(out, "Set attribute: " PCMK_XA_NAME "=%s value=%s",
                   attr_name, attr_value);
     }
 
     return rc;
 }
 
 static int
 resources_with_attr(pcmk__output_t *out, cib_t *cib, pcmk_resource_t *rsc,
                     const char *requested_name, const char *attr_set,
                     const char *attr_set_type, const char *attr_id,
                     const char *attr_name, const char *top_id, gboolean force,
                     GList **resources)
 {
     if (pcmk__str_eq(attr_set_type, PCMK_XE_INSTANCE_ATTRIBUTES,
                      pcmk__str_casei)) {
         if (!force) {
             xmlNode *xml_search = NULL;
             int rc = pcmk_rc_ok;
 
             rc = find_resource_attr(out, cib, PCMK_XA_ID, top_id,
                                     PCMK_XE_META_ATTRIBUTES, attr_set, attr_id,
                                     attr_name, &xml_search);
 
             if (rc == pcmk_rc_ok || rc == ENOTUNIQ) {
                 char *found_attr_id = NULL;
 
                 found_attr_id = crm_element_value_copy(xml_search, PCMK_XA_ID);
 
                 if (!out->is_quiet(out)) {
                     out->err(out,
                              "WARNING: There is already a meta attribute "
                              "for '%s' called '%s' (id=%s)",
                              top_id, attr_name, found_attr_id);
                     out->err(out,
                              "         Delete '%s' first or use the force option "
                              "to override", found_attr_id);
                 }
 
                 free(found_attr_id);
                 pcmk__xml_free(xml_search);
                 return ENOTUNIQ;
             }
 
             pcmk__xml_free(xml_search);
         }
 
         *resources = g_list_append(*resources, rsc);
 
     } else {
         *resources = find_matching_attr_resources(out, rsc, requested_name,
                                                   attr_set, attr_set_type,
                                                   attr_id, attr_name, cib,
                                                   "update", force);
     }
 
     /* If the user specified attr_set or attr_id, the intent is to modify a
      * single resource, which will be the last item in the list.
      */
     if ((attr_set != NULL) || (attr_id != NULL)) {
         GList *last = g_list_last(*resources);
 
         *resources = g_list_remove_link(*resources, last);
         g_list_free(*resources);
         *resources = last;
     }
 
     return pcmk_rc_ok;
 }
 
 static void
 free_attr_update_data(gpointer data)
 {
     attr_update_data_t *ud = data;
 
     if (ud == NULL) {
         return;
     }
 
     free(ud->attr_set_type);
     free(ud->attr_set_id);
     free(ud->attr_name);
     free(ud->attr_value);
     free(ud->given_rsc_id);
     free(ud->found_attr_id);
     free(ud);
 }
 
 static int
 update_attribute(pcmk_resource_t *rsc, const char *requested_name,
                  const char *attr_set, const char *attr_set_type,
                  const char *attr_id, const char *attr_name,
                  const char *attr_value, gboolean recursive, cib_t *cib,
-                 gboolean force, GList **results)
+                 xmlNode *cib_xml_orig, gboolean force, GList **results)
 {
     pcmk__output_t *out = rsc->priv->scheduler->priv->out;
     int rc = pcmk_rc_ok;
 
     GList/*<pcmk_resource_t*>*/ *resources = NULL;
     const char *top_id = pe__const_top_resource(rsc, false)->id;
 
     if ((attr_id == NULL) && !force) {
         find_resource_attr(out, cib, PCMK_XA_ID, top_id, NULL, NULL, NULL,
                            attr_name, NULL);
     }
 
     rc = resources_with_attr(out, cib, rsc, requested_name, attr_set, attr_set_type,
                              attr_id, attr_name, top_id, force, &resources);
 
     if (rc != pcmk_rc_ok) {
         return rc;
     }
 
     for (GList *iter = resources; iter != NULL; iter = iter->next) {
+        // @TODO Functionize loop body to simplify freeing allocated memory
         char *lookup_id = NULL;
         char *local_attr_set = NULL;
         char *found_attr_id = NULL;
         const char *rsc_attr_id = attr_id;
         const char *rsc_attr_set = attr_set;
 
+        xmlNode *rsc_xml = rsc->priv->xml;
         xmlNode *xml_top = NULL;
         xmlNode *xml_obj = NULL;
         xmlNode *xml_search = NULL;
 
         rsc = (pcmk_resource_t *) iter->data;
 
         lookup_id = clone_strip(rsc->id); /* Could be a cloned group! */
         rc = find_resource_attr(out, cib, PCMK_XA_ID, lookup_id, attr_set_type,
                                 attr_set, attr_id, attr_name, &xml_search);
 
         switch (rc) {
             case pcmk_rc_ok:
                 found_attr_id = crm_element_value_copy(xml_search, PCMK_XA_ID);
                 crm_debug("Found a match for " PCMK_XA_NAME "='%s': "
                           PCMK_XA_ID "='%s'", attr_name, found_attr_id);
                 rsc_attr_id = found_attr_id;
                 break;
 
             case ENXIO:
                 if (rsc_attr_set == NULL) {
                     local_attr_set = crm_strdup_printf("%s-%s", lookup_id,
                                                        attr_set_type);
                     rsc_attr_set = local_attr_set;
                 }
                 if (rsc_attr_id == NULL) {
                     found_attr_id = crm_strdup_printf("%s-%s",
                                                       rsc_attr_set, attr_name);
                     rsc_attr_id = found_attr_id;
                 }
 
-                xml_top = pcmk__xe_create(NULL,
-                                          (const char *)
-                                          rsc->priv->xml->name);
+                rsc_xml = get_cib_rsc(cib_xml_orig, rsc);
+                if (rsc_xml == NULL) {
+                    /* @TODO Warn and continue through the rest of the resources
+                     * and return the error at the end? This should never
+                     * happen, but if it does, then we could have a partial
+                     * update.
+                     */
+                    free(lookup_id);
+                    free(found_attr_id);
+                    pcmk__xml_free(xml_search);
+                    g_list_free(resources);
+                    return ENXIO;
+                }
+
+                xml_top = pcmk__xe_create(NULL, (const char *) rsc_xml->name);
                 crm_xml_add(xml_top, PCMK_XA_ID, lookup_id);
 
                 xml_obj = pcmk__xe_create(xml_top, attr_set_type);
                 crm_xml_add(xml_obj, PCMK_XA_ID, rsc_attr_set);
                 break;
 
             default:
                 free(lookup_id);
                 free(found_attr_id);
                 pcmk__xml_free(xml_search);
                 g_list_free(resources);
                 return rc;
         }
 
         xml_obj = crm_create_nvpair_xml(xml_obj, rsc_attr_id, attr_name,
                                         attr_value);
         if (xml_top == NULL) {
             xml_top = xml_obj;
         }
 
         crm_log_xml_debug(xml_top, "Update");
 
         rc = cib->cmds->modify(cib, PCMK_XE_RESOURCES, xml_top, cib_sync_call);
         rc = pcmk_legacy2rc(rc);
         if (rc == pcmk_rc_ok) {
             attr_update_data_t *ud = pcmk__assert_alloc(1, sizeof(attr_update_data_t));
 
             if (attr_set_type == NULL) {
                 attr_set_type = (const char *) xml_search->parent->name;
             }
 
             if (rsc_attr_set == NULL) {
                 rsc_attr_set = crm_element_value(xml_search->parent, PCMK_XA_ID);
             }
 
             ud->attr_set_type = pcmk__str_copy(attr_set_type);
             ud->attr_set_id = pcmk__str_copy(rsc_attr_set);
             ud->attr_name = pcmk__str_copy(attr_name);
             ud->attr_value = pcmk__str_copy(attr_value);
             ud->given_rsc_id = pcmk__str_copy(lookup_id);
             ud->found_attr_id = pcmk__str_copy(found_attr_id);
             ud->rsc = rsc;
 
             *results = g_list_append(*results, ud);
         }
 
         pcmk__xml_free(xml_top);
         pcmk__xml_free(xml_search);
 
         free(lookup_id);
         free(found_attr_id);
         free(local_attr_set);
 
         if (recursive
             && pcmk__str_eq(attr_set_type, PCMK_XE_META_ATTRIBUTES,
                             pcmk__str_casei)) {
             /* We want to set the attribute only on resources explicitly
              * colocated with this one, so we use
              * rsc->priv->with_this_colocations directly rather than the
              * with_this_colocations() method.
              */
             pcmk__set_rsc_flags(rsc, pcmk__rsc_detect_loop);
             for (GList *lpc = rsc->priv->with_this_colocations;
                  lpc != NULL; lpc = lpc->next) {
                 pcmk__colocation_t *cons = (pcmk__colocation_t *) lpc->data;
 
                 crm_debug("Checking %s %d", cons->id, cons->score);
 
                 if (pcmk_is_set(cons->dependent->flags, pcmk__rsc_detect_loop)
                     || (cons->score <= 0)) {
                     continue;
                 }
 
                 crm_debug("Setting %s=%s for dependent resource %s",
                           attr_name, attr_value, cons->dependent->id);
                 update_attribute(cons->dependent, cons->dependent->id, NULL,
                                  attr_set_type, NULL, attr_name, attr_value,
-                                 recursive, cib, force, results);
+                                 recursive, cib, cib_xml_orig, force, results);
             }
         }
     }
 
     g_list_free(resources);
     return rc;
 }
 
 // \return Standard Pacemaker return code
 int
 cli_resource_update_attribute(pcmk_resource_t *rsc, const char *requested_name,
                               const char *attr_set, const char *attr_set_type,
                               const char *attr_id, const char *attr_name,
                               const char *attr_value, gboolean recursive,
-                              cib_t *cib, gboolean force)
+                              cib_t *cib, xmlNode *cib_xml_orig, gboolean force)
 {
     static bool need_init = true;
     int rc = pcmk_rc_ok;
 
     GList *results = NULL;
     pcmk__output_t *out = rsc->priv->scheduler->priv->out;
 
+    pcmk__assert(cib_xml_orig != NULL);
+
     /* If we were asked to update the attribute in a resource element (for
      * instance, <primitive class="ocf">) there's really not much we need to do.
      */
     if (pcmk__str_eq(attr_set_type, ATTR_SET_ELEMENT, pcmk__str_none)) {
-        return update_element_attribute(out, rsc, cib, attr_name, attr_value);
+        return update_element_attribute(out, rsc, cib, cib_xml_orig, attr_name,
+                                        attr_value);
     }
 
     /* One time initialization - clear flags so we can detect loops */
     if (need_init) {
         need_init = false;
         pcmk__unpack_constraints(rsc->priv->scheduler);
         pe__clear_resource_flags_on_all(rsc->priv->scheduler,
                                         pcmk__rsc_detect_loop);
     }
 
     rc = update_attribute(rsc, requested_name, attr_set, attr_set_type,
                           attr_id, attr_name, attr_value, recursive, cib,
-                          force, &results);
+                          cib_xml_orig, force, &results);
 
     if (rc == pcmk_rc_ok) {
         if (results == NULL) {
             return rc;
         }
 
         out->message(out, "attribute-changed-list", results);
         g_list_free_full(results, free_attr_update_data);
     }
 
     return rc;
 }
 
 // \return Standard Pacemaker return code
 int
 cli_resource_delete_attribute(pcmk_resource_t *rsc, const char *requested_name,
                               const char *attr_set, const char *attr_set_type,
                               const char *attr_id, const char *attr_name,
-                              cib_t *cib, gboolean force)
+                              cib_t *cib, xmlNode *cib_xml_orig, gboolean force)
 {
     pcmk__output_t *out = rsc->priv->scheduler->priv->out;
     int rc = pcmk_rc_ok;
     GList/*<pcmk_resource_t*>*/ *resources = NULL;
 
+    pcmk__assert((cib != NULL) && (cib_xml_orig != NULL));
+
     if ((attr_id == NULL) && !force) {
         find_resource_attr(out, cib, PCMK_XA_ID,
                            pe__const_top_resource(rsc, false)->id, NULL,
                            NULL, NULL, attr_name, NULL);
     }
 
-    if (pcmk__str_eq(attr_set_type, PCMK_XE_META_ATTRIBUTES, pcmk__str_casei)) {
-        resources = find_matching_attr_resources(out, rsc, requested_name,
-                                                 attr_set, attr_set_type,
-                                                 attr_id, attr_name, cib,
-                                                 "delete", force);
+    if (pcmk__str_eq(attr_set_type, ATTR_SET_ELEMENT, pcmk__str_none)) {
+        xmlNode *rsc_xml = rsc->priv->xml;
 
-    } else if (pcmk__str_eq(attr_set_type, ATTR_SET_ELEMENT, pcmk__str_none)) {
-        pcmk__xe_remove_attr(rsc->priv->xml, attr_name);
-        pcmk__assert(cib != NULL);
-        rc = cib->cmds->replace(cib, PCMK_XE_RESOURCES, rsc->priv->xml,
-                                cib_sync_call);
+        rsc_xml = get_cib_rsc(cib_xml_orig, rsc);
+        if (rsc_xml == NULL) {
+            return ENXIO;
+        }
+
+        pcmk__xe_remove_attr(rsc_xml, attr_name);
+        rc = cib->cmds->replace(cib, PCMK_XE_RESOURCES, rsc_xml, cib_sync_call);
         rc = pcmk_legacy2rc(rc);
         if (rc == pcmk_rc_ok) {
             out->info(out, "Deleted attribute: %s", attr_name);
         }
         return rc;
+    }
 
+    if (pcmk__str_eq(attr_set_type, PCMK_XE_META_ATTRIBUTES, pcmk__str_none)) {
+        resources = find_matching_attr_resources(out, rsc, requested_name,
+                                                 attr_set, attr_set_type,
+                                                 attr_id, attr_name, cib,
+                                                 "delete", force);
     } else {
         resources = g_list_append(resources, rsc);
     }
 
     for (GList *iter = resources; iter != NULL; iter = iter->next) {
         char *lookup_id = NULL;
         xmlNode *xml_obj = NULL;
         xmlNode *xml_search = NULL;
         char *found_attr_id = NULL;
         const char *rsc_attr_id = attr_id;
 
         rsc = (pcmk_resource_t *) iter->data;
 
+        /* @TODO Search the original CIB in find_resource_attr() for
+         * future-proofing, to ensure that we're getting IDs of nvpairs that
+         * exist in the CIB.
+         */
         lookup_id = clone_strip(rsc->id);
         rc = find_resource_attr(out, cib, PCMK_XA_ID, lookup_id, attr_set_type,
                                 attr_set, attr_id, attr_name, &xml_search);
         switch (rc) {
             case pcmk_rc_ok:
                 found_attr_id = crm_element_value_copy(xml_search, PCMK_XA_ID);
                 pcmk__xml_free(xml_search);
                 break;
 
             case ENXIO:
                 free(lookup_id);
                 pcmk__xml_free(xml_search);
                 continue;
 
             default:
                 free(lookup_id);
                 pcmk__xml_free(xml_search);
                 g_list_free(resources);
                 return rc;
         }
 
         if (rsc_attr_id == NULL) {
             rsc_attr_id = found_attr_id;
         }
 
         xml_obj = crm_create_nvpair_xml(NULL, rsc_attr_id, attr_name, NULL);
         crm_log_xml_debug(xml_obj, "Delete");
 
-        pcmk__assert(cib != NULL);
         rc = cib->cmds->remove(cib, PCMK_XE_RESOURCES, xml_obj, cib_sync_call);
         rc = pcmk_legacy2rc(rc);
 
         if (rc == pcmk_rc_ok) {
             out->info(out, "Deleted '%s' option: " PCMK_XA_ID "=%s%s%s%s%s",
                       lookup_id, found_attr_id,
                       ((attr_set == NULL)? "" : " set="),
                       pcmk__s(attr_set, ""),
                       ((attr_name == NULL)? "" : " " PCMK_XA_NAME "="),
                       pcmk__s(attr_name, ""));
         }
 
         free(lookup_id);
         pcmk__xml_free(xml_obj);
         free(found_attr_id);
     }
+
     g_list_free(resources);
     return rc;
 }
 
 // \return Standard Pacemaker return code
 static int
 send_lrm_rsc_op(pcmk_ipc_api_t *controld_api, bool do_fail_resource,
                 const char *host_uname, const char *rsc_id,
                 pcmk_scheduler_t *scheduler)
 {
     pcmk__output_t *out = scheduler->priv->out;
     const char *router_node = host_uname;
     const char *rsc_api_id = NULL;
     const char *rsc_long_id = NULL;
     const char *rsc_class = NULL;
     const char *rsc_provider = NULL;
     const char *rsc_type = NULL;
     bool cib_only = false;
     pcmk_resource_t *rsc = pe_find_resource(scheduler->priv->resources, rsc_id);
 
     if (rsc == NULL) {
         out->err(out, "Resource %s not found", rsc_id);
         return ENXIO;
 
     } else if (!pcmk__is_primitive(rsc)) {
         out->err(out, "We can only process primitive resources, not %s", rsc_id);
         return EINVAL;
     }
 
     rsc_class = crm_element_value(rsc->priv->xml, PCMK_XA_CLASS);
     rsc_provider = crm_element_value(rsc->priv->xml, PCMK_XA_PROVIDER);
     rsc_type = crm_element_value(rsc->priv->xml, PCMK_XA_TYPE);
     if ((rsc_class == NULL) || (rsc_type == NULL)) {
         out->err(out, "Resource %s does not have a class and type", rsc_id);
         return EINVAL;
     }
 
     {
         pcmk_node_t *node = pcmk_find_node(scheduler, host_uname);
 
         if (node == NULL) {
             out->err(out, "Node %s not found", host_uname);
             return pcmk_rc_node_unknown;
         }
 
         if (!(node->details->online)) {
             if (do_fail_resource) {
                 out->err(out, "Node %s is not online", host_uname);
                 return ENOTCONN;
             } else {
                 cib_only = true;
             }
         }
         if (!cib_only && pcmk__is_pacemaker_remote_node(node)) {
             node = pcmk__current_node(node->priv->remote);
             if (node == NULL) {
                 out->err(out, "No cluster connection to Pacemaker Remote node %s detected",
                          host_uname);
                 return ENOTCONN;
             }
             router_node = node->priv->name;
         }
     }
 
     if (rsc->priv->history_id != NULL) {
         rsc_api_id = rsc->priv->history_id;
         rsc_long_id = rsc->id;
     } else {
         rsc_api_id = rsc->id;
     }
     if (do_fail_resource) {
         return pcmk_controld_api_fail(controld_api, host_uname, router_node,
                                       rsc_api_id, rsc_long_id,
                                       rsc_class, rsc_provider, rsc_type);
     } else {
         return pcmk_controld_api_refresh(controld_api, host_uname, router_node,
                                          rsc_api_id, rsc_long_id, rsc_class,
                                          rsc_provider, rsc_type, cib_only);
     }
 }
 
 /*!
  * \internal
  * \brief Get resource name as used in failure-related node attributes
  *
  * \param[in] rsc  Resource to check
  *
  * \return Newly allocated string containing resource's fail name
  * \note The caller is responsible for freeing the result.
  */
 static inline char *
 rsc_fail_name(const pcmk_resource_t *rsc)
 {
     const char *name = pcmk__s(rsc->priv->history_id, rsc->id);
 
     if (pcmk_is_set(rsc->flags, pcmk__rsc_unique)) {
         return strdup(name);
     }
     return clone_strip(name);
 }
 
 // \return Standard Pacemaker return code
 static int
 clear_rsc_history(pcmk_ipc_api_t *controld_api, const char *host_uname,
                   const char *rsc_id, pcmk_scheduler_t *scheduler)
 {
     int rc = pcmk_rc_ok;
 
     /* Erase the resource's entire LRM history in the CIB, even if we're only
      * clearing a single operation's fail count. If we erased only entries for a
      * single operation, we might wind up with a wrong idea of the current
      * resource state, and we might not re-probe the resource.
      */
     rc = send_lrm_rsc_op(controld_api, false, host_uname, rsc_id, scheduler);
     if (rc != pcmk_rc_ok) {
         return rc;
     }
 
     crm_trace("Processing %d mainloop inputs",
               pcmk_controld_api_replies_expected(controld_api));
     while (g_main_context_iteration(NULL, FALSE)) {
         crm_trace("Processed mainloop input, %d still remaining",
                   pcmk_controld_api_replies_expected(controld_api));
     }
     return rc;
 }
 
 // \return Standard Pacemaker return code
 static int
 clear_rsc_failures(pcmk__output_t *out, pcmk_ipc_api_t *controld_api,
                    const char *node_name, const char *rsc_id, const char *operation,
                    const char *interval_spec, pcmk_scheduler_t *scheduler)
 {
     int rc = pcmk_rc_ok;
     const char *failed_value = NULL;
     const char *failed_id = NULL;
     char *interval_ms_s = NULL;
     GHashTable *rscs = NULL;
     GHashTableIter iter;
 
     /* Create a hash table to use as a set of resources to clean. This lets us
      * clean each resource only once (per node) regardless of how many failed
      * operations it has.
      */
     rscs = pcmk__strkey_table(NULL, NULL);
 
     // Normalize interval to milliseconds for comparison to history entry
     if (operation) {
         guint interval_ms = 0U;
 
         pcmk_parse_interval_spec(interval_spec, &interval_ms);
         interval_ms_s = crm_strdup_printf("%u", interval_ms);
     }
 
     for (xmlNode *xml_op = pcmk__xe_first_child(scheduler->priv->failed, NULL,
                                                 NULL, NULL);
          xml_op != NULL; xml_op = pcmk__xe_next(xml_op)) {
 
         failed_id = crm_element_value(xml_op, PCMK__XA_RSC_ID);
         if (failed_id == NULL) {
             // Malformed history entry, should never happen
             continue;
         }
 
         // No resource specified means all resources match
         if (rsc_id) {
             pcmk_resource_t *fail_rsc = NULL;
 
             fail_rsc = pe_find_resource_with_flags(scheduler->priv->resources,
                                                    failed_id,
                                                    pcmk_rsc_match_history
                                                    |pcmk_rsc_match_anon_basename);
             if (!fail_rsc || !pcmk__str_eq(rsc_id, fail_rsc->id, pcmk__str_casei)) {
                 continue;
             }
         }
 
         // Host name should always have been provided by this point
         failed_value = crm_element_value(xml_op, PCMK_XA_UNAME);
         if (!pcmk__str_eq(node_name, failed_value, pcmk__str_casei)) {
             continue;
         }
 
         // No operation specified means all operations match
         if (operation) {
             failed_value = crm_element_value(xml_op, PCMK_XA_OPERATION);
             if (!pcmk__str_eq(operation, failed_value, pcmk__str_casei)) {
                 continue;
             }
 
             // Interval (if operation was specified) defaults to 0 (not all)
             failed_value = crm_element_value(xml_op, PCMK_META_INTERVAL);
             if (!pcmk__str_eq(interval_ms_s, failed_value, pcmk__str_casei)) {
                 continue;
             }
         }
 
         g_hash_table_add(rscs, (gpointer) failed_id);
     }
 
     free(interval_ms_s);
 
     g_hash_table_iter_init(&iter, rscs);
     while (g_hash_table_iter_next(&iter, (gpointer *) &failed_id, NULL)) {
         crm_debug("Erasing failures of %s on %s", failed_id, node_name);
         rc = clear_rsc_history(controld_api, node_name, failed_id, scheduler);
         if (rc != pcmk_rc_ok) {
             return rc;
         }
     }
     g_hash_table_destroy(rscs);
     return rc;
 }
 
 // \return Standard Pacemaker return code
 static int
 clear_rsc_fail_attrs(const pcmk_resource_t *rsc, const char *operation,
                      const char *interval_spec, const pcmk_node_t *node)
 {
     int rc = pcmk_rc_ok;
     int attr_options = pcmk__node_attr_none;
     char *rsc_name = rsc_fail_name(rsc);
 
     if (pcmk__is_pacemaker_remote_node(node)) {
         attr_options |= pcmk__node_attr_remote;
     }
 
     rc = pcmk__attrd_api_clear_failures(NULL, node->priv->name, rsc_name,
                                         operation, interval_spec, NULL,
                                         attr_options);
     free(rsc_name);
     return rc;
 }
 
 // \return Standard Pacemaker return code
 int
 cli_resource_delete(pcmk_ipc_api_t *controld_api, const char *host_uname,
                     const pcmk_resource_t *rsc, const char *operation,
                     const char *interval_spec, bool just_failures,
                     pcmk_scheduler_t *scheduler, gboolean force)
 {
     pcmk__output_t *out = scheduler->priv->out;
     int rc = pcmk_rc_ok;
     pcmk_node_t *node = NULL;
 
     if (rsc == NULL) {
         return ENXIO;
 
     } else if (rsc->priv->children != NULL) {
 
         for (const GList *lpc = rsc->priv->children;
              lpc != NULL; lpc = lpc->next) {
 
             const pcmk_resource_t *child = (const pcmk_resource_t *) lpc->data;
 
             rc = cli_resource_delete(controld_api, host_uname, child, operation,
                                      interval_spec, just_failures, scheduler,
                                      force);
             if (rc != pcmk_rc_ok) {
                 return rc;
             }
         }
         return pcmk_rc_ok;
 
     } else if (host_uname == NULL) {
         GList *lpc = NULL;
         GList *nodes = g_hash_table_get_values(rsc->priv->probed_nodes);
 
         if(nodes == NULL && force) {
             nodes = pcmk__copy_node_list(scheduler->nodes, false);
 
         } else if ((nodes == NULL)
                    && pcmk_is_set(rsc->flags, pcmk__rsc_exclusive_probes)) {
             GHashTableIter iter;
             pcmk_node_t *node = NULL;
 
             g_hash_table_iter_init(&iter, rsc->priv->allowed_nodes);
             while (g_hash_table_iter_next(&iter, NULL, (void**)&node)) {
                 if (node->assign->score >= 0) {
                     nodes = g_list_prepend(nodes, node);
                 }
             }
 
         } else if(nodes == NULL) {
             nodes = g_hash_table_get_values(rsc->priv->allowed_nodes);
         }
 
         for (lpc = nodes; lpc != NULL; lpc = lpc->next) {
             node = (pcmk_node_t *) lpc->data;
 
             if (node->details->online) {
                 rc = cli_resource_delete(controld_api, node->priv->name, rsc,
                                          operation, interval_spec, just_failures,
                                          scheduler, force);
             }
             if (rc != pcmk_rc_ok) {
                 g_list_free(nodes);
                 return rc;
             }
         }
 
         g_list_free(nodes);
         return pcmk_rc_ok;
     }
 
     node = pcmk_find_node(scheduler, host_uname);
 
     if (node == NULL) {
         out->err(out, "Unable to clean up %s because node %s not found",
                  rsc->id, host_uname);
         return ENODEV;
     }
 
     if (!pcmk_is_set(node->priv->flags, pcmk__node_probes_allowed)) {
         out->err(out, "Unable to clean up %s because resource discovery disabled on %s",
                  rsc->id, host_uname);
         return EOPNOTSUPP;
     }
 
     if (controld_api == NULL) {
         out->err(out, "Dry run: skipping clean-up of %s on %s due to CIB_file",
                  rsc->id, host_uname);
         return pcmk_rc_ok;
     }
 
     rc = clear_rsc_fail_attrs(rsc, operation, interval_spec, node);
     if (rc != pcmk_rc_ok) {
         out->err(out, "Unable to clean up %s failures on %s: %s",
                  rsc->id, host_uname, pcmk_rc_str(rc));
         return rc;
     }
 
     if (just_failures) {
         rc = clear_rsc_failures(out, controld_api, host_uname, rsc->id, operation,
                                 interval_spec, scheduler);
     } else {
         rc = clear_rsc_history(controld_api, host_uname, rsc->id, scheduler);
     }
     if (rc != pcmk_rc_ok) {
         out->err(out, "Cleaned %s failures on %s, but unable to clean history: %s",
                  rsc->id, host_uname, pcmk_rc_str(rc));
     } else {
         out->info(out, "Cleaned up %s on %s", rsc->id, host_uname);
     }
     return rc;
 }
 
 // \return Standard Pacemaker return code
 int
 cli_cleanup_all(pcmk_ipc_api_t *controld_api, const char *node_name,
                 const char *operation, const char *interval_spec,
                 pcmk_scheduler_t *scheduler)
 {
     pcmk__output_t *out = scheduler->priv->out;
     int rc = pcmk_rc_ok;
     int attr_options = pcmk__node_attr_none;
     const char *display_name = node_name? node_name : "all nodes";
 
     if (controld_api == NULL) {
         out->info(out, "Dry run: skipping clean-up of %s due to CIB_file",
                   display_name);
         return rc;
     }
 
     if (node_name) {
         pcmk_node_t *node = pcmk_find_node(scheduler, node_name);
 
         if (node == NULL) {
             out->err(out, "Unknown node: %s", node_name);
             return ENXIO;
         }
         if (pcmk__is_pacemaker_remote_node(node)) {
             attr_options |= pcmk__node_attr_remote;
         }
     }
 
     rc = pcmk__attrd_api_clear_failures(NULL, node_name, NULL, operation,
                                         interval_spec, NULL, attr_options);
     if (rc != pcmk_rc_ok) {
         out->err(out, "Unable to clean up all failures on %s: %s",
                  display_name, pcmk_rc_str(rc));
         return rc;
     }
 
     if (node_name) {
         rc = clear_rsc_failures(out, controld_api, node_name, NULL,
                                 operation, interval_spec, scheduler);
         if (rc != pcmk_rc_ok) {
             out->err(out, "Cleaned all resource failures on %s, but unable to clean history: %s",
                      node_name, pcmk_rc_str(rc));
             return rc;
         }
     } else {
         for (GList *iter = scheduler->nodes; iter; iter = iter->next) {
             pcmk_node_t *node = (pcmk_node_t *) iter->data;
 
             rc = clear_rsc_failures(out, controld_api, node->priv->name,
                                     NULL, operation, interval_spec, scheduler);
             if (rc != pcmk_rc_ok) {
                 out->err(out, "Cleaned all resource failures on all nodes, but unable to clean history: %s",
                          pcmk_rc_str(rc));
                 return rc;
             }
         }
     }
 
     out->info(out, "Cleaned up all resources on %s", display_name);
     return rc;
 }
 
 static void
 check_role(resource_checks_t *checks)
 {
     const char *role_s = g_hash_table_lookup(checks->rsc->priv->meta,
                                              PCMK_META_TARGET_ROLE);
 
     if (role_s == NULL) {
         return;
     }
     switch (pcmk_parse_role(role_s)) {
         case pcmk_role_stopped:
             checks->flags |= rsc_remain_stopped;
             break;
 
         case pcmk_role_unpromoted:
             if (pcmk_is_set(pe__const_top_resource(checks->rsc, false)->flags,
                             pcmk__rsc_promotable)) {
                 checks->flags |= rsc_unpromotable;
             }
             break;
 
         default:
             break;
     }
 }
 
 static void
 check_managed(resource_checks_t *checks)
 {
     const char *managed_s = g_hash_table_lookup(checks->rsc->priv->meta,
                                                 PCMK_META_IS_MANAGED);
 
     if ((managed_s != NULL) && !crm_is_true(managed_s)) {
         checks->flags |= rsc_unmanaged;
     }
 }
 
 static void
 check_locked(resource_checks_t *checks)
 {
     const pcmk_node_t *lock_node = checks->rsc->priv->lock_node;
 
     if (lock_node != NULL) {
         checks->flags |= rsc_locked;
         checks->lock_node = lock_node->priv->name;
     }
 }
 
 static bool
 node_is_unhealthy(pcmk_node_t *node)
 {
     switch (pe__health_strategy(node->priv->scheduler)) {
         case pcmk__health_strategy_none:
             break;
 
         case pcmk__health_strategy_no_red:
             if (pe__node_health(node) < 0) {
                 return true;
             }
             break;
 
         case pcmk__health_strategy_only_green:
             if (pe__node_health(node) <= 0) {
                 return true;
             }
             break;
 
         case pcmk__health_strategy_progressive:
         case pcmk__health_strategy_custom:
             /* @TODO These are finite scores, possibly with rules, and possibly
              * combining with other scores, so attributing these as a cause is
              * nontrivial.
              */
             break;
     }
     return false;
 }
 
 static void
 check_node_health(resource_checks_t *checks, pcmk_node_t *node)
 {
     if (node == NULL) {
         GHashTableIter iter;
         bool allowed = false;
         bool all_nodes_unhealthy = true;
 
         g_hash_table_iter_init(&iter, checks->rsc->priv->allowed_nodes);
         while (g_hash_table_iter_next(&iter, NULL, (void **) &node)) {
             allowed = true;
             if (!node_is_unhealthy(node)) {
                 all_nodes_unhealthy = false;
                 break;
             }
         }
         if (allowed && all_nodes_unhealthy) {
             checks->flags |= rsc_node_health;
         }
 
     } else if (node_is_unhealthy(node)) {
         checks->flags |= rsc_node_health;
     }
 }
 
 int
 cli_resource_check(pcmk__output_t *out, pcmk_resource_t *rsc, pcmk_node_t *node)
 {
     resource_checks_t checks = { .rsc = rsc };
 
     check_role(&checks);
     check_managed(&checks);
     check_locked(&checks);
     check_node_health(&checks, node);
 
     return out->message(out, "resource-check-list", &checks);
 }
 
 // \return Standard Pacemaker return code
 int
 cli_resource_fail(pcmk_ipc_api_t *controld_api, const char *host_uname,
                   const char *rsc_id, pcmk_scheduler_t *scheduler)
 {
     crm_notice("Failing %s on %s", rsc_id, host_uname);
     return send_lrm_rsc_op(controld_api, true, host_uname, rsc_id, scheduler);
 }
 
 static GHashTable *
 generate_resource_params(pcmk_resource_t *rsc, pcmk_node_t *node,
                          pcmk_scheduler_t *scheduler)
 {
     GHashTable *params = NULL;
     GHashTable *meta = NULL;
     GHashTable *combined = NULL;
     GHashTableIter iter;
     char *key = NULL;
     char *value = NULL;
 
     combined = pcmk__strkey_table(free, free);
 
     params = pe_rsc_params(rsc, node, scheduler);
     if (params != NULL) {
         g_hash_table_iter_init(&iter, params);
         while (g_hash_table_iter_next(&iter, (gpointer *) & key, (gpointer *) & value)) {
             pcmk__insert_dup(combined, key, value);
         }
     }
 
     meta = pcmk__strkey_table(free, free);
     get_meta_attributes(meta, rsc, NULL, scheduler);
     if (meta != NULL) {
         g_hash_table_iter_init(&iter, meta);
         while (g_hash_table_iter_next(&iter, (gpointer *) & key, (gpointer *) & value)) {
             char *crm_name = crm_meta_name(key);
 
             g_hash_table_insert(combined, crm_name, strdup(value));
         }
         g_hash_table_destroy(meta);
     }
 
     return combined;
 }
 
 bool resource_is_running_on(pcmk_resource_t *rsc, const char *host)
 {
     bool found = true;
     GList *hIter = NULL;
     GList *hosts = NULL;
 
     if (rsc == NULL) {
         return false;
     }
 
     rsc->priv->fns->location(rsc, &hosts, pcmk__rsc_node_current);
     for (hIter = hosts; host != NULL && hIter != NULL; hIter = hIter->next) {
         pcmk_node_t *node = (pcmk_node_t *) hIter->data;
 
         if (pcmk__strcase_any_of(host, node->priv->name, node->priv->id,
                                  NULL)) {
             crm_trace("Resource %s is running on %s\n", rsc->id, host);
             goto done;
         }
     }
 
     if (host != NULL) {
         crm_trace("Resource %s is not running on: %s\n", rsc->id, host);
         found = false;
 
     } else if(host == NULL && hosts == NULL) {
         crm_trace("Resource %s is not running\n", rsc->id);
         found = false;
     }
 
   done:
     g_list_free(hosts);
     return found;
 }
 
 /*!
  * \internal
  * \brief Create a list of all resources active on host from a given list
  *
  * \param[in] host      Name of host to check whether resources are active
  * \param[in] rsc_list  List of resources to check
  *
  * \return New list of resources from list that are active on host
  */
 static GList *
 get_active_resources(const char *host, GList *rsc_list)
 {
     GList *rIter = NULL;
     GList *active = NULL;
 
     for (rIter = rsc_list; rIter != NULL; rIter = rIter->next) {
         pcmk_resource_t *rsc = (pcmk_resource_t *) rIter->data;
 
         /* Expand groups to their members, because if we're restarting a member
          * other than the first, we can't otherwise tell which resources are
          * stopping and starting.
          */
         if (pcmk__is_group(rsc)) {
             GList *member_active = NULL;
 
             member_active = get_active_resources(host, rsc->priv->children);
             active = g_list_concat(active, member_active);
         } else if (resource_is_running_on(rsc, host)) {
             active = g_list_append(active, strdup(rsc->id));
         }
     }
     return active;
 }
 
 static void dump_list(GList *items, const char *tag)
 {
     int lpc = 0;
     GList *item = NULL;
 
     for (item = items; item != NULL; item = item->next) {
         crm_trace("%s[%d]: %s", tag, lpc, (char*)item->data);
         lpc++;
     }
 }
 
 static void display_list(pcmk__output_t *out, GList *items, const char *tag)
 {
     GList *item = NULL;
 
     for (item = items; item != NULL; item = item->next) {
         out->info(out, "%s%s", tag, (const char *)item->data);
     }
 }
 
 /*!
  * \internal
- * \brief Upgrade XML to latest schema version and use it as scheduler input
+ * \brief Update scheduler XML input based on a CIB query and the current time
  *
- * This also updates the scheduler timestamp to the current time.
+ * The CIB XML is upgraded to the latest schema version.
  *
- * \param[in,out] scheduler  Scheduler data to update
- * \param[in,out] xml        XML to use as input
+ * \param[in,out] out           Output object
+ * \param[in,out] scheduler     Scheduler data to update
+ * \param[in]     cib           Connection to the CIB manager
+ * \param[out]    cib_xml_orig  Where to store CIB XML before any schema
+ *                              upgrades (can be \c NULL)
  *
  * \return Standard Pacemaker return code
- * \note On success, \p scheduler takes ownership of \p xml, and the caller is
- *       responsible for freeing memory allocated for \c scheduler->priv->now.
  */
 int
-update_scheduler_input(pcmk_scheduler_t *scheduler, xmlNode **xml)
-{
-    int rc = pcmk__update_configured_schema(xml, false);
-
-    if (rc == pcmk_rc_ok) {
-        scheduler->input = *xml;
-        scheduler->priv->now = crm_time_new(NULL);
-    }
-    return rc;
-}
-
-/*!
- * \internal
- * \brief Update scheduler XML input based on a CIB query
- *
- * \param[in] scheduler  Scheduler data to initialize
- * \param[in] cib        Connection to the CIB manager
- *
- * \return Standard Pacemaker return code
- * \note On success, caller is responsible for freeing memory allocated for
- *       scheduler->input and scheduler->priv->now.
- */
-static int
-update_scheduler_input_to_cib(pcmk__output_t *out, pcmk_scheduler_t *scheduler,
-                              cib_t *cib)
+update_scheduler_input(pcmk__output_t *out, pcmk_scheduler_t *scheduler,
+                       cib_t *cib, xmlNode **cib_xml_orig)
 {
-    xmlNode *cib_xml_copy = NULL;
+    xmlNode *queried_xml = NULL;
+    xmlNode *updated_xml = NULL;
     int rc = pcmk_rc_ok;
 
-    rc = cib->cmds->query(cib, NULL, &cib_xml_copy, cib_sync_call);
-    rc = pcmk_legacy2rc(rc);
+    pcmk__assert((out != NULL) && (scheduler != NULL)
+                 && (scheduler->input == NULL) && (scheduler->priv->now == NULL)
+                 && (cib != NULL)
+                 && ((cib_xml_orig == NULL) || (*cib_xml_orig == NULL)));
 
+    rc = cib->cmds->query(cib, NULL, &queried_xml, cib_sync_call);
+    rc = pcmk_legacy2rc(rc);
     if (rc != pcmk_rc_ok) {
-        out->err(out, "Could not obtain the current CIB: %s (%d)", pcmk_rc_str(rc), rc);
-        return rc;
+        out->err(out, "Could not obtain the current CIB: %s", pcmk_rc_str(rc));
+        goto done;
     }
-    rc = update_scheduler_input(scheduler, &cib_xml_copy);
+
+    if (cib_xml_orig != NULL) {
+        updated_xml = pcmk__xml_copy(NULL, queried_xml);
+    } else {
+        // No need to preserve the pre-upgrade CIB, so don't make a copy
+        updated_xml = queried_xml;
+        queried_xml = NULL;
+    }
+
+    rc = pcmk__update_configured_schema(&updated_xml, false);
     if (rc != pcmk_rc_ok) {
-        out->err(out, "Could not upgrade the current CIB XML");
-        pcmk__xml_free(cib_xml_copy);
-        return rc;
+        out->err(out, "Could not upgrade the current CIB XML: %s",
+                 pcmk_rc_str(rc));
+        pcmk__xml_free(updated_xml);
+        goto done;
     }
 
+    scheduler->input = updated_xml;
+    scheduler->priv->now = crm_time_new(NULL);
+
+done:
+    if ((rc == pcmk_rc_ok) && (cib_xml_orig != NULL)) {
+        *cib_xml_orig = queried_xml;
+    } else {
+        pcmk__xml_free(queried_xml);
+    }
     return rc;
 }
 
 // \return Standard Pacemaker return code
 static int
-update_dataset(cib_t *cib, pcmk_scheduler_t *scheduler, bool simulate)
+update_dataset(cib_t *cib, pcmk_scheduler_t *scheduler, xmlNode **cib_xml_orig,
+               bool simulate)
 {
     char *pid = NULL;
     char *shadow_file = NULL;
     cib_t *shadow_cib = NULL;
     int rc = pcmk_rc_ok;
 
     pcmk__output_t *out = scheduler->priv->out;
 
     pe_reset_working_set(scheduler);
     pcmk__set_scheduler_flags(scheduler, pcmk__sched_no_counts);
-    rc = update_scheduler_input_to_cib(out, scheduler, cib);
-    if (rc != pcmk_rc_ok) {
-        return rc;
-    }
 
     if(simulate) {
         bool prev_quiet = false;
 
+        rc = update_scheduler_input(out, scheduler, cib, NULL);
+        if (rc != pcmk_rc_ok) {
+            goto done;
+        }
+
         pid = pcmk__getpid_s();
         shadow_cib = cib_shadow_new(pid);
         shadow_file = get_shadow_file(pid);
 
         if (shadow_cib == NULL) {
             out->err(out, "Could not create shadow cib: '%s'", pid);
             rc = ENXIO;
             goto done;
         }
 
         rc = pcmk__xml_write_file(scheduler->input, shadow_file, false);
         if (rc != pcmk_rc_ok) {
             out->err(out, "Could not populate shadow cib: %s", pcmk_rc_str(rc));
             goto done;
         }
 
         rc = shadow_cib->cmds->signon(shadow_cib, crm_system_name, cib_command);
         rc = pcmk_legacy2rc(rc);
 
         if (rc != pcmk_rc_ok) {
             out->err(out, "Could not connect to shadow cib: %s",
                      pcmk_rc_str(rc));
             goto done;
         }
 
         pcmk__schedule_actions(scheduler->input, pcmk__sched_no_counts,
                                scheduler);
 
         prev_quiet = out->is_quiet(out);
         out->quiet = true;
         pcmk__simulate_transition(scheduler, shadow_cib, NULL);
         out->quiet = prev_quiet;
 
-        rc = update_dataset(shadow_cib, scheduler, false);
+        rc = update_dataset(shadow_cib, scheduler, cib_xml_orig, false);
 
     } else {
+        xmlNode *xml = NULL;
+
+        rc = update_scheduler_input(out, scheduler, cib, &xml);
+        if (rc != pcmk_rc_ok) {
+            goto done;
+        }
+
+        pcmk__xml_free(*cib_xml_orig);
+        *cib_xml_orig = xml;
         cluster_status(scheduler);
     }
 
   done:
     // Do not free scheduler->input because rsc->priv->xml must remain valid
     cib_delete(shadow_cib);
     free(pid);
 
     if(shadow_file) {
         unlink(shadow_file);
         free(shadow_file);
     }
 
     return rc;
 }
 
 /*!
  * \internal
  * \brief Find the maximum stop timeout of a resource and its children (if any)
  *
  * \param[in,out] rsc  Resource to get timeout for
  *
  * \return Maximum stop timeout for \p rsc (in milliseconds)
  */
 static guint
 max_rsc_stop_timeout(pcmk_resource_t *rsc)
 {
     long long result_ll;
     guint max_delay = 0;
     xmlNode *config = NULL;
     GHashTable *meta = NULL;
 
     if (rsc == NULL) {
         return 0;
     }
 
     // If resource is collective, use maximum of its children's stop timeouts
     if (rsc->priv->children != NULL) {
 
         for (GList *iter = rsc->priv->children;
              iter != NULL; iter = iter->next) {
 
             pcmk_resource_t *child = iter->data;
             guint delay = max_rsc_stop_timeout(child);
 
             if (delay > max_delay) {
                 pcmk__rsc_trace(rsc,
                                 "Maximum stop timeout for %s is now %s "
                                 "due to %s", rsc->id,
                                 pcmk__readable_interval(delay), child->id);
                 max_delay = delay;
             }
         }
         return max_delay;
     }
 
     // Get resource's stop action configuration from CIB
     config = pcmk__find_action_config(rsc, PCMK_ACTION_STOP, 0, true);
 
     /* Get configured timeout for stop action (fully evaluated for rules,
      * defaults, etc.).
      *
      * @TODO This currently ignores node (which might matter for rules)
      */
     meta = pcmk__unpack_action_meta(rsc, NULL, PCMK_ACTION_STOP, 0, config);
     if ((pcmk__scan_ll(g_hash_table_lookup(meta, PCMK_META_TIMEOUT),
                        &result_ll, -1LL) == pcmk_rc_ok) && (result_ll >= 0)) {
         max_delay = (guint) QB_MIN(result_ll, UINT_MAX);
     }
     g_hash_table_destroy(meta);
 
     return max_delay;
 }
 
 /*!
  * \internal
  * \brief Find a reasonable waiting time for stopping any one resource in a list
  *
  * \param[in,out] scheduler  Scheduler data
  * \param[in]     resources  List of names of resources that will be stopped
  *
  * \return Rough estimate of a reasonable time to wait (in seconds) to stop any
  *         one resource in \p resources
  * \note This estimate is very rough, simply the maximum stop timeout of all
  *       given resources and their children, plus a small fudge factor. It does
  *       not account for children that must be stopped in sequence, action
  *       throttling, or any demotions needed. It checks the stop timeout, even
  *       if the resources in question are actually being started.
  */
 static guint
 wait_time_estimate(pcmk_scheduler_t *scheduler, const GList *resources)
 {
     guint max_delay = 0U;
 
     // Find maximum stop timeout in milliseconds
     for (const GList *item = resources; item != NULL; item = item->next) {
         pcmk_resource_t *rsc = pe_find_resource(scheduler->priv->resources,
                                                 (const char *) item->data);
         guint delay = max_rsc_stop_timeout(rsc);
 
         if (delay > max_delay) {
             pcmk__rsc_trace(rsc,
                             "Wait time is now %s due to %s",
                             pcmk__readable_interval(delay), rsc->id);
             max_delay = delay;
         }
     }
 
     return (max_delay / 1000U) + 5U;
 }
 
 #define waiting_for_starts(d, r, h) ((d != NULL) || \
                                     (!resource_is_running_on((r), (h))))
 
 /*!
  * \internal
  * \brief Restart a resource (on a particular host if requested).
  *
  * \param[in,out] out                 Output object
  * \param[in,out] rsc                 The resource to restart
  * \param[in]     node                Node to restart resource on (NULL for all)
  * \param[in]     move_lifetime       If not NULL, how long constraint should
  *                                    remain in effect (as ISO 8601 string)
  * \param[in]     timeout_ms          Consider failed if actions do not complete
  *                                    in this time (specified in milliseconds,
  *                                    but a two-second granularity is actually
  *                                    used; if 0, it will be calculated based on
  *                                    the resource timeout)
  * \param[in,out] cib                 Connection to the CIB manager
  * \param[in]     promoted_role_only  If true, limit to promoted instances
  * \param[in]     force               If true, apply only to requested instance
  *                                    if part of a collective resource
  *
  * \return Standard Pacemaker return code (exits on certain failures)
  */
 int
 cli_resource_restart(pcmk__output_t *out, pcmk_resource_t *rsc,
                      const pcmk_node_t *node, const char *move_lifetime,
                      guint timeout_ms, cib_t *cib, gboolean promoted_role_only,
                      gboolean force)
 {
     int rc = pcmk_rc_ok;
     int lpc = 0;
     int before = 0;
     guint step_timeout_s = 0;
     guint sleep_interval = 2U;
     guint timeout = timeout_ms / 1000U;
 
     bool stop_via_ban = false;
     char *rsc_id = NULL;
     char *lookup_id = NULL;
     char *orig_target_role = NULL;
+    xmlNode *cib_xml_orig = NULL;
 
     GList *list_delta = NULL;
     GList *target_active = NULL;
     GList *current_active = NULL;
     GList *restart_target_active = NULL;
 
     pcmk_scheduler_t *scheduler = NULL;
     pcmk_resource_t *parent = uber_parent(rsc);
 
     bool running = false;
     const char *id = pcmk__s(rsc->priv->history_id, rsc->id);
     const char *host = node ? node->priv->name : NULL;
 
     /* If the implicit resource or primitive resource of a bundle is given, operate on the
      * bundle itself instead.
      */
     if (pcmk__is_bundled(rsc)) {
         rsc = parent->priv->parent;
     }
 
     running = resource_is_running_on(rsc, host);
 
     if (pcmk__is_clone(parent) && !running) {
         if (pcmk__is_unique_clone(parent)) {
             lookup_id = strdup(rsc->id);
         } else {
             lookup_id = clone_strip(rsc->id);
         }
 
         rsc = parent->priv->fns->find_rsc(parent, lookup_id, node,
                                           pcmk_rsc_match_basename
                                           |pcmk_rsc_match_current_node);
         free(lookup_id);
         running = resource_is_running_on(rsc, host);
     }
 
     if (!running) {
         if (host) {
             out->err(out, "%s is not running on %s and so cannot be restarted", id, host);
         } else {
             out->err(out, "%s is not running anywhere and so cannot be restarted", id);
         }
         return ENXIO;
     }
 
     if (!pcmk_is_set(rsc->flags, pcmk__rsc_managed)) {
         out->err(out, "Unmanaged resources cannot be restarted.");
         return EAGAIN;
     }
 
     rsc_id = strdup(rsc->id);
 
     if (pcmk__is_unique_clone(parent)) {
         lookup_id = strdup(rsc->id);
     } else {
         lookup_id = clone_strip(rsc->id);
     }
 
     if (host) {
         if (pcmk__is_clone(rsc) || pe_bundle_replicas(rsc)) {
             stop_via_ban = true;
         } else if (pcmk__is_clone(parent)) {
             stop_via_ban = true;
             free(lookup_id);
             lookup_id = strdup(parent->id);
         }
     }
 
     /*
       grab full cib
       determine originally active resources
       disable or ban
       poll cib and watch for affected resources to get stopped
       without --timeout, calculate the stop timeout for each step and wait for that
       if we hit --timeout or the service timeout, re-enable or un-ban, report failure and indicate which resources we couldn't take down
       if everything stopped, re-enable or un-ban
       poll cib and watch for affected resources to get started
       without --timeout, calculate the start timeout for each step and wait for that
       if we hit --timeout or the service timeout, report (different) failure and indicate which resources we couldn't bring back up
       report success
 
       Optimizations:
       - use constraints to determine ordered list of affected resources
       - Allow a --no-deps option (aka. --force-restart)
     */
 
     scheduler = pe_new_working_set();
     if (scheduler == NULL) {
         rc = errno;
         out->err(out, "Could not allocate scheduler data: %s", pcmk_rc_str(rc));
         goto done;
     }
 
     scheduler->priv->out = out;
-    rc = update_dataset(cib, scheduler, false);
+    rc = update_dataset(cib, scheduler, &cib_xml_orig, false);
 
     if(rc != pcmk_rc_ok) {
         out->err(out, "Could not get new resource list: %s (%d)", pcmk_rc_str(rc), rc);
         goto done;
     }
 
     restart_target_active = get_active_resources(host,
                                                  scheduler->priv->resources);
     current_active = get_active_resources(host, scheduler->priv->resources);
 
     dump_list(current_active, "Origin");
 
     if (stop_via_ban) {
         /* Stop the clone or bundle instance by banning it from the host */
         out->quiet = true;
         rc = cli_resource_ban(out, lookup_id, host, move_lifetime, cib,
                               promoted_role_only, PCMK_ROLE_PROMOTED);
     } else {
         xmlNode *xml_search = NULL;
 
         /* Stop the resource by setting PCMK_META_TARGET_ROLE to Stopped.
          * Remember any existing PCMK_META_TARGET_ROLE so we can restore it
          * later (though it only makes any difference if it's Unpromoted).
          */
 
         rc = find_resource_attr(out, cib, PCMK_XA_VALUE, lookup_id, NULL, NULL, NULL,
                                 PCMK_META_TARGET_ROLE, &xml_search);
 
         if (rc == pcmk_rc_ok) {
             orig_target_role = crm_element_value_copy(xml_search, PCMK_XA_VALUE);
         }
 
         pcmk__xml_free(xml_search);
 
         rc = cli_resource_update_attribute(rsc, rsc_id, NULL,
                                            PCMK_XE_META_ATTRIBUTES, NULL,
                                            PCMK_META_TARGET_ROLE,
                                            PCMK_ACTION_STOPPED, FALSE, cib,
-                                           force);
+                                           cib_xml_orig, force);
     }
     if(rc != pcmk_rc_ok) {
         out->err(out, "Could not set " PCMK_META_TARGET_ROLE " for %s: %s (%d)",
                  rsc_id, pcmk_rc_str(rc), rc);
         if (current_active != NULL) {
             g_list_free_full(current_active, free);
             current_active = NULL;
         }
         if (restart_target_active != NULL) {
             g_list_free_full(restart_target_active, free);
             restart_target_active = NULL;
         }
         goto done;
     }
 
-    rc = update_dataset(cib, scheduler, true);
+    rc = update_dataset(cib, scheduler, &cib_xml_orig, true);
     if(rc != pcmk_rc_ok) {
         out->err(out, "Could not determine which resources would be stopped");
         goto failure;
     }
 
     target_active = get_active_resources(host, scheduler->priv->resources);
     dump_list(target_active, "Target");
 
     list_delta = pcmk__subtract_lists(current_active, target_active, (GCompareFunc) strcmp);
     out->info(out, "Waiting for %d resources to stop:", g_list_length(list_delta));
     display_list(out, list_delta, " * ");
 
     step_timeout_s = timeout / sleep_interval;
     while (list_delta != NULL) {
         before = g_list_length(list_delta);
         if(timeout_ms == 0) {
             step_timeout_s = wait_time_estimate(scheduler, list_delta)
                              / sleep_interval;
         }
 
         /* We probably don't need the entire step timeout */
         for(lpc = 0; (lpc < step_timeout_s) && (list_delta != NULL); lpc++) {
             sleep(sleep_interval);
             if(timeout) {
                 timeout -= sleep_interval;
                 crm_trace("%us remaining", timeout);
             }
-            rc = update_dataset(cib, scheduler, FALSE);
+            rc = update_dataset(cib, scheduler, &cib_xml_orig, false);
             if(rc != pcmk_rc_ok) {
                 out->err(out, "Could not determine which resources were stopped");
                 goto failure;
             }
 
             if (current_active != NULL) {
                 g_list_free_full(current_active, free);
             }
             current_active = get_active_resources(host,
                                                   scheduler->priv->resources);
 
             g_list_free(list_delta);
             list_delta = pcmk__subtract_lists(current_active, target_active, (GCompareFunc) strcmp);
 
             dump_list(current_active, "Current");
             dump_list(list_delta, "Delta");
         }
 
         crm_trace("%d (was %d) resources remaining", g_list_length(list_delta), before);
         if(before == g_list_length(list_delta)) {
             /* aborted during stop phase, print the contents of list_delta */
             out->err(out, "Could not complete shutdown of %s, %d resources remaining", rsc_id, g_list_length(list_delta));
             display_list(out, list_delta, " * ");
             rc = ETIME;
             goto failure;
         }
 
     }
 
     if (stop_via_ban) {
         rc = cli_resource_clear(lookup_id, host, NULL, cib, true, force);
 
     } else if (orig_target_role) {
         rc = cli_resource_update_attribute(rsc, rsc_id, NULL,
                                            PCMK_XE_META_ATTRIBUTES, NULL,
                                            PCMK_META_TARGET_ROLE,
-                                           orig_target_role, FALSE, cib, force);
+                                           orig_target_role, FALSE, cib,
+                                           cib_xml_orig, force);
         free(orig_target_role);
         orig_target_role = NULL;
     } else {
         rc = cli_resource_delete_attribute(rsc, rsc_id, NULL,
                                            PCMK_XE_META_ATTRIBUTES, NULL,
-                                           PCMK_META_TARGET_ROLE, cib, force);
+                                           PCMK_META_TARGET_ROLE, cib,
+                                           cib_xml_orig, force);
     }
 
     if(rc != pcmk_rc_ok) {
         out->err(out,
                  "Could not unset " PCMK_META_TARGET_ROLE " for %s: %s (%d)",
                  rsc_id, pcmk_rc_str(rc), rc);
         goto done;
     }
 
     if (target_active != NULL) {
         g_list_free_full(target_active, free);
     }
     target_active = restart_target_active;
 
     list_delta = pcmk__subtract_lists(target_active, current_active, (GCompareFunc) strcmp);
     out->info(out, "Waiting for %d resources to start again:", g_list_length(list_delta));
     display_list(out, list_delta, " * ");
 
     step_timeout_s = timeout / sleep_interval;
     while (waiting_for_starts(list_delta, rsc, host)) {
         before = g_list_length(list_delta);
         if(timeout_ms == 0) {
             step_timeout_s = wait_time_estimate(scheduler, list_delta)
                              / sleep_interval;
         }
 
         /* We probably don't need the entire step timeout */
         for (lpc = 0; (lpc < step_timeout_s) && waiting_for_starts(list_delta, rsc, host); lpc++) {
 
             sleep(sleep_interval);
             if(timeout) {
                 timeout -= sleep_interval;
                 crm_trace("%ds remaining", timeout);
             }
 
-            rc = update_dataset(cib, scheduler, false);
+            rc = update_dataset(cib, scheduler, &cib_xml_orig, false);
             if(rc != pcmk_rc_ok) {
                 out->err(out, "Could not determine which resources were started");
                 goto failure;
             }
 
             /* It's OK if dependent resources moved to a different node,
              * so we check active resources on all nodes.
              */
             if (current_active != NULL) {
                 g_list_free_full(current_active, free);
             }
             current_active = get_active_resources(NULL,
                                                   scheduler->priv->resources);
 
             g_list_free(list_delta);
             list_delta = pcmk__subtract_lists(target_active, current_active, (GCompareFunc) strcmp);
             dump_list(current_active, "Current");
             dump_list(list_delta, "Delta");
         }
 
         if(before == g_list_length(list_delta)) {
             /* aborted during start phase, print the contents of list_delta */
             out->err(out, "Could not complete restart of %s, %d resources remaining", rsc_id, g_list_length(list_delta));
             display_list(out, list_delta, " * ");
             rc = ETIME;
             goto failure;
         }
 
     }
 
     rc = pcmk_rc_ok;
     goto done;
 
   failure:
     if (stop_via_ban) {
         cli_resource_clear(lookup_id, host, NULL, cib, true, force);
     } else if (orig_target_role) {
         cli_resource_update_attribute(rsc, rsc_id, NULL,
                                       PCMK_XE_META_ATTRIBUTES, NULL,
                                       PCMK_META_TARGET_ROLE, orig_target_role,
-                                      FALSE, cib, force);
+                                      FALSE, cib, cib_xml_orig, force);
         free(orig_target_role);
     } else {
         cli_resource_delete_attribute(rsc, rsc_id, NULL,
                                       PCMK_XE_META_ATTRIBUTES, NULL,
-                                      PCMK_META_TARGET_ROLE, cib, force);
+                                      PCMK_META_TARGET_ROLE, cib, cib_xml_orig,
+                                      force);
     }
 
 done:
     if (list_delta != NULL) {
         g_list_free(list_delta);
     }
     if (current_active != NULL) {
         g_list_free_full(current_active, free);
     }
     if (target_active != NULL && (target_active != restart_target_active)) {
         g_list_free_full(target_active, free);
     }
     if (restart_target_active != NULL) {
         g_list_free_full(restart_target_active, free);
     }
     free(rsc_id);
     free(lookup_id);
     pe_free_working_set(scheduler);
     return rc;
 }
 
 static inline bool
 action_is_pending(const pcmk_action_t *action)
 {
     if (pcmk_any_flags_set(action->flags,
                            pcmk__action_optional|pcmk__action_pseudo)
         || !pcmk_is_set(action->flags, pcmk__action_runnable)
         || pcmk__str_eq(PCMK_ACTION_NOTIFY, action->task, pcmk__str_casei)) {
         return false;
     }
     return true;
 }
 
 /*!
  * \internal
  * \brief Check whether any actions in a list are pending
  *
  * \param[in] actions   List of actions to check
  *
  * \return true if any actions in the list are pending, otherwise false
  */
 static bool
 actions_are_pending(const GList *actions)
 {
     for (const GList *action = actions; action != NULL; action = action->next) {
         const pcmk_action_t *a = (const pcmk_action_t *) action->data;
 
         if (action_is_pending(a)) {
             crm_notice("Waiting for %s (flags=%#.8x)", a->uuid, a->flags);
             return true;
         }
     }
     return false;
 }
 
 static void
 print_pending_actions(pcmk__output_t *out, GList *actions)
 {
     GList *action;
 
     out->info(out, "Pending actions:");
     for (action = actions; action != NULL; action = action->next) {
         pcmk_action_t *a = (pcmk_action_t *) action->data;
 
         if (!action_is_pending(a)) {
             continue;
         }
 
         if (a->node) {
             out->info(out, "\tAction %d: %s\ton %s",
                       a->id, a->uuid, pcmk__node_name(a->node));
         } else {
             out->info(out, "\tAction %d: %s", a->id, a->uuid);
         }
     }
 }
 
 /* For --wait, timeout (in seconds) to use if caller doesn't specify one */
 #define WAIT_DEFAULT_TIMEOUT_S (60 * 60)
 
 /* For --wait, how long to sleep between cluster state checks */
 #define WAIT_SLEEP_S (2)
 
 /*!
  * \internal
  * \brief Wait until all pending cluster actions are complete
  *
  * This waits until either the CIB's transition graph is idle or a timeout is
  * reached.
  *
  * \param[in,out] out          Output object
  * \param[in]     timeout_ms   Consider failed if actions do not complete in
  *                             this time (specified in milliseconds, but
  *                             one-second granularity is actually used; if 0, a
  *                             default will be used)
  * \param[in,out] cib          Connection to the CIB manager
  *
  * \return Standard Pacemaker return code
  */
 int
 wait_till_stable(pcmk__output_t *out, guint timeout_ms, cib_t * cib)
 {
     pcmk_scheduler_t *scheduler = NULL;
     xmlXPathObjectPtr search;
     int rc = pcmk_rc_ok;
     bool pending_unknown_state_resources;
     time_t expire_time = time(NULL);
     time_t time_diff;
     bool printed_version_warning = out->is_quiet(out); // i.e. don't print if quiet
     char *xpath = NULL;
 
     if (timeout_ms == 0) {
         expire_time += WAIT_DEFAULT_TIMEOUT_S;
     } else {
         expire_time += (timeout_ms + 999) / 1000;
     }
 
     scheduler = pe_new_working_set();
     if (scheduler == NULL) {
         return ENOMEM;
     }
 
     xpath = crm_strdup_printf("/" PCMK_XE_CIB "/" PCMK_XE_STATUS
                               "/" PCMK__XE_NODE_STATE "/" PCMK__XE_LRM
                               "/" PCMK__XE_LRM_RESOURCES
                               "/" PCMK__XE_LRM_RESOURCE
                               "/" PCMK__XE_LRM_RSC_OP
                               "[@" PCMK__XA_RC_CODE "='%d']",
                               PCMK_OCF_UNKNOWN);
     do {
         /* Abort if timeout is reached */
         time_diff = expire_time - time(NULL);
         if (time_diff <= 0) {
             print_pending_actions(out, scheduler->priv->actions);
             rc = ETIME;
             break;
         }
 
         crm_info("Waiting up to %lld seconds for cluster actions to complete",
                  (long long) time_diff);
 
         if (rc == pcmk_rc_ok) { /* this avoids sleep on first loop iteration */
             sleep(WAIT_SLEEP_S);
         }
 
         /* Get latest transition graph */
         pe_reset_working_set(scheduler);
-        rc = update_scheduler_input_to_cib(out, scheduler, cib);
+        rc = update_scheduler_input(out, scheduler, cib, NULL);
         if (rc != pcmk_rc_ok) {
             break;
         }
         pcmk__schedule_actions(scheduler->input, pcmk__sched_no_counts,
                                scheduler);
 
         if (!printed_version_warning) {
             /* If the DC has a different version than the local node, the two
              * could come to different conclusions about what actions need to be
              * done. Warn the user in this case.
              *
              * @TODO A possible long-term solution would be to reimplement the
              * wait as a new controller operation that would be forwarded to the
              * DC. However, that would have potential problems of its own.
              */
             const char *dc_version = NULL;
 
             dc_version = g_hash_table_lookup(scheduler->priv->options,
                                              PCMK_OPT_DC_VERSION);
             if (!pcmk__str_eq(dc_version, PACEMAKER_VERSION "-" BUILD_VERSION, pcmk__str_casei)) {
                 out->info(out, "warning: wait option may not work properly in "
                           "mixed-version cluster");
                 printed_version_warning = true;
             }
         }
 
         search = xpath_search(scheduler->input, xpath);
         pending_unknown_state_resources = (numXpathResults(search) > 0);
         freeXpathObject(search);
     } while (actions_are_pending(scheduler->priv->actions)
              || pending_unknown_state_resources);
 
     pe_free_working_set(scheduler);
     free(xpath);
     return rc;
 }
 
 static const char *
 get_action(const char *rsc_action) {
     const char *action = NULL;
 
     if (pcmk__str_eq(rsc_action, "validate", pcmk__str_casei)) {
         action = PCMK_ACTION_VALIDATE_ALL;
 
     } else if (pcmk__str_eq(rsc_action, "force-check", pcmk__str_casei)) {
         action = PCMK_ACTION_MONITOR;
 
     } else if (pcmk__strcase_any_of(rsc_action, "force-start", "force-stop",
                                     "force-demote", "force-promote", NULL)) {
         action = rsc_action+6;
     } else {
         action = rsc_action;
     }
 
     return action;
 }
 
 /*!
  * \brief Set up environment variables as expected by resource agents
  *
  * When the cluster executes resource agents, it adds certain environment
  * variables (directly or via resource meta-attributes) expected by some
  * resource agents. Add the essential ones that many resource agents expect, so
  * the behavior is the same for command-line execution.
  *
  * \param[in,out] params       Resource parameters that will be passed to agent
  * \param[in]     timeout_ms   Action timeout (in milliseconds)
  * \param[in]     check_level  OCF check level
  * \param[in]     verbosity    Verbosity level
  */
 static void
 set_agent_environment(GHashTable *params, guint timeout_ms, int check_level,
                       int verbosity)
 {
     g_hash_table_insert(params, crm_meta_name(PCMK_META_TIMEOUT),
                         crm_strdup_printf("%u", timeout_ms));
 
     pcmk__insert_dup(params, PCMK_XA_CRM_FEATURE_SET, CRM_FEATURE_SET);
 
     if (check_level >= 0) {
         char *level = crm_strdup_printf("%d", check_level);
 
         setenv("OCF_CHECK_LEVEL", level, 1);
         free(level);
     }
 
     pcmk__set_env_option(PCMK__ENV_DEBUG, ((verbosity > 0)? "1" : "0"), true);
     if (verbosity > 1) {
         setenv("OCF_TRACE_RA", "1", 1);
     }
 
     /* A resource agent using the standard ocf-shellfuncs library will not print
      * messages to stderr if it doesn't have a controlling terminal (e.g. if
      * crm_resource is called via script or ssh). This forces it to do so.
      */
     setenv("OCF_TRACE_FILE", "/dev/stderr", 0);
 }
 
 /*!
  * \internal
  * \brief Apply command-line overrides to resource parameters
  *
  * \param[in,out] params     Parameters to be passed to agent
  * \param[in]     overrides  Parameters to override (or NULL if none)
  */
 static void
 apply_overrides(GHashTable *params, GHashTable *overrides)
 {
     if (overrides != NULL) {
         GHashTableIter iter;
         char *name = NULL;
         char *value = NULL;
 
         g_hash_table_iter_init(&iter, overrides);
         while (g_hash_table_iter_next(&iter, (gpointer *) &name,
                                       (gpointer *) &value)) {
             pcmk__insert_dup(params, name, value);
         }
     }
 }
 
 crm_exit_t
 cli_resource_execute_from_params(pcmk__output_t *out, const char *rsc_name,
                                  const char *rsc_class, const char *rsc_prov,
                                  const char *rsc_type, const char *rsc_action,
                                  GHashTable *params, GHashTable *override_hash,
                                  guint timeout_ms, int resource_verbose,
                                  gboolean force, int check_level)
 {
     const char *class = rsc_class;
     const char *action = get_action(rsc_action);
     crm_exit_t exit_code = CRM_EX_OK;
     svc_action_t *op = NULL;
 
     // If no timeout was provided, use the same default as the cluster
     if (timeout_ms == 0U) {
         timeout_ms = PCMK_DEFAULT_ACTION_TIMEOUT_MS;
     }
 
     set_agent_environment(params, timeout_ms, check_level, resource_verbose);
     apply_overrides(params, override_hash);
 
     op = services__create_resource_action(rsc_name? rsc_name : "test",
                                           rsc_class, rsc_prov, rsc_type, action,
                                           0, QB_MIN(timeout_ms, INT_MAX),
                                           params, 0);
     if (op == NULL) {
         out->err(out, "Could not execute %s using %s%s%s:%s: %s",
                  action, rsc_class, (rsc_prov? ":" : ""),
                  (rsc_prov? rsc_prov : ""), rsc_type, strerror(ENOMEM));
         g_hash_table_destroy(params);
         return CRM_EX_OSERR;
     }
 
 #if PCMK__ENABLE_SERVICE
     if (pcmk__str_eq(rsc_class, PCMK_RESOURCE_CLASS_SERVICE, pcmk__str_casei)) {
         class = resources_find_service_class(rsc_type);
     }
 #endif
 
     if (!pcmk_is_set(pcmk_get_ra_caps(class), pcmk_ra_cap_cli_exec)) {
         services__format_result(op, CRM_EX_UNIMPLEMENT_FEATURE, PCMK_EXEC_ERROR,
                                 "Manual execution of the %s standard is "
                                 "unsupported", pcmk__s(class, "unspecified"));
     }
 
     if (op->rc != PCMK_OCF_UNKNOWN) {
         exit_code = op->rc;
         goto done;
     }
 
     services_action_sync(op);
 
     // Map results to OCF codes for consistent reporting to user
     {
         enum ocf_exitcode ocf_code = services_result2ocf(class, action, op->rc);
 
         // Cast variable instead of function return to keep compilers happy
         exit_code = (crm_exit_t) ocf_code;
     }
 
 done:
     out->message(out, "resource-agent-action", resource_verbose, rsc_class,
                  rsc_prov, rsc_type, rsc_name, rsc_action, override_hash,
                  exit_code, op->status, services__exit_reason(op),
                  op->stdout_data, op->stderr_data);
     services_action_free(op);
     return exit_code;
 }
 
 /*!
  * \internal
  * \brief Get the timeout the cluster would use for an action
  *
  * \param[in] rsc     Resource that action is for
  * \param[in] action  Name of action
  */
 static guint
 get_action_timeout(pcmk_resource_t *rsc, const char *action)
 {
     long long timeout_ms = -1LL;
     xmlNode *op = pcmk__find_action_config(rsc, action, 0, true);
     GHashTable *meta = pcmk__unpack_action_meta(rsc, NULL, action, 0, op);
 
     if ((pcmk__scan_ll(g_hash_table_lookup(meta, PCMK_META_TIMEOUT),
                        &timeout_ms, -1LL) != pcmk_rc_ok)
         || (timeout_ms <= 0LL)) {
         timeout_ms = PCMK_DEFAULT_ACTION_TIMEOUT_MS;
     }
     g_hash_table_destroy(meta);
     return (guint) QB_MIN(timeout_ms, UINT_MAX);
 }
 
 crm_exit_t
 cli_resource_execute(pcmk_resource_t *rsc, const char *requested_name,
                      const char *rsc_action, GHashTable *override_hash,
                      guint timeout_ms, cib_t *cib, pcmk_scheduler_t *scheduler,
                      int resource_verbose, gboolean force, int check_level)
 {
     pcmk__output_t *out = scheduler->priv->out;
     crm_exit_t exit_code = CRM_EX_OK;
     const char *rid = requested_name;
     const char *rtype = NULL;
     const char *rprov = NULL;
     const char *rclass = NULL;
     GHashTable *params = NULL;
 
     if (pcmk__strcase_any_of(rsc_action, "force-start", "force-demote",
                                     "force-promote", NULL)) {
         if (pcmk__is_clone(rsc)) {
             GList *nodes = cli_resource_search(rsc, requested_name, scheduler);
             if(nodes != NULL && force == FALSE) {
                 out->err(out, "It is not safe to %s %s here: the cluster claims it is already active",
                          rsc_action, rsc->id);
                 out->err(out,
                          "Try setting "
                          PCMK_META_TARGET_ROLE "=" PCMK_ROLE_STOPPED
                          " first or specifying the force option");
                 return CRM_EX_UNSAFE;
             }
 
             g_list_free_full(nodes, free);
         }
     }
 
     if (pcmk__is_clone(rsc)) {
         /* Grab the first child resource in the hope it's not a group */
         rsc = rsc->priv->children->data;
     }
 
     if (pcmk__is_group(rsc)) {
         out->err(out, "Sorry, the %s option doesn't support group resources", rsc_action);
         return CRM_EX_UNIMPLEMENT_FEATURE;
     } else if (pcmk__is_bundled(rsc)) {
         out->err(out, "Sorry, the %s option doesn't support bundled resources", rsc_action);
         return CRM_EX_UNIMPLEMENT_FEATURE;
     }
 
     rclass = crm_element_value(rsc->priv->xml, PCMK_XA_CLASS);
     rprov = crm_element_value(rsc->priv->xml, PCMK_XA_PROVIDER);
     rtype = crm_element_value(rsc->priv->xml, PCMK_XA_TYPE);
 
     params = generate_resource_params(rsc, NULL /* @TODO use local node */,
                                       scheduler);
 
     if (timeout_ms == 0U) {
         timeout_ms = get_action_timeout(rsc, get_action(rsc_action));
     }
 
     if (!pcmk__is_anonymous_clone(rsc->priv->parent)) {
         rid = rsc->id;
     }
 
     exit_code = cli_resource_execute_from_params(out, rid, rclass, rprov, rtype, rsc_action,
                                                  params, override_hash, timeout_ms,
                                                  resource_verbose, force, check_level);
     return exit_code;
 }
 
 // \return Standard Pacemaker return code
 int
 cli_resource_move(const pcmk_resource_t *rsc, const char *rsc_id,
                   const char *host_name, const char *move_lifetime, cib_t *cib,
                   pcmk_scheduler_t *scheduler, gboolean promoted_role_only,
                   gboolean force)
 {
     pcmk__output_t *out = scheduler->priv->out;
     int rc = pcmk_rc_ok;
     unsigned int count = 0;
     pcmk_node_t *current = NULL;
     pcmk_node_t *dest = pcmk_find_node(scheduler, host_name);
     bool cur_is_dest = false;
 
     if (dest == NULL) {
         return pcmk_rc_node_unknown;
     }
 
     if (promoted_role_only
         && !pcmk_is_set(rsc->flags, pcmk__rsc_promotable)) {
 
         const pcmk_resource_t *p = pe__const_top_resource(rsc, false);
 
         if (pcmk_is_set(p->flags, pcmk__rsc_promotable)) {
             out->info(out, "Using parent '%s' for move instead of '%s'.", rsc->id, rsc_id);
             rsc_id = p->id;
             rsc = p;
 
         } else {
             out->info(out, "Ignoring --promoted option: %s is not promotable",
                       rsc_id);
             promoted_role_only = FALSE;
         }
     }
 
     current = pe__find_active_requires(rsc, &count);
 
     if (pcmk_is_set(rsc->flags, pcmk__rsc_promotable)) {
         unsigned int promoted_count = 0;
         pcmk_node_t *promoted_node = NULL;
 
         for (const GList *iter = rsc->priv->children;
              iter != NULL; iter = iter->next) {
 
             const pcmk_resource_t *child = (const pcmk_resource_t *) iter->data;
             enum rsc_role_e child_role = child->priv->fns->state(child, TRUE);
 
             if (child_role == pcmk_role_promoted) {
                 rsc = child;
                 promoted_node = pcmk__current_node(child);
                 promoted_count++;
             }
         }
         if (promoted_role_only || (promoted_count != 0)) {
             count = promoted_count;
             current = promoted_node;
         }
 
     }
 
     if (count > 1) {
         if (pcmk__is_clone(rsc)) {
             current = NULL;
         } else {
             return pcmk_rc_multiple;
         }
     }
 
     if (pcmk__same_node(current, dest)) {
         cur_is_dest = true;
         if (force) {
             crm_info("%s is already %s on %s, reinforcing placement with location constraint.",
                      rsc_id, promoted_role_only?"promoted":"active",
                      pcmk__node_name(dest));
         } else {
             return pcmk_rc_already;
         }
     }
 
     /* Clear any previous prefer constraints across all nodes. */
     cli_resource_clear(rsc_id, NULL, scheduler->nodes, cib, false, force);
 
     /* Clear any previous ban constraints on 'dest'. */
     cli_resource_clear(rsc_id, dest->priv->name, scheduler->nodes, cib, true,
                        force);
 
     /* Record an explicit preference for 'dest' */
     rc = cli_resource_prefer(out, rsc_id, dest->priv->name, move_lifetime,
                              cib, promoted_role_only, PCMK_ROLE_PROMOTED);
 
     crm_trace("%s%s now prefers %s%s",
               rsc->id, (promoted_role_only? " (promoted)" : ""),
               pcmk__node_name(dest), force?"(forced)":"");
 
     /* only ban the previous location if current location != destination location.
      * it is possible to use -M to enforce a location without regard of where the
      * resource is currently located */
     if (force && !cur_is_dest) {
         /* Ban the original location if possible */
         if(current) {
             (void)cli_resource_ban(out, rsc_id, current->priv->name,
                                    move_lifetime, cib, promoted_role_only,
                                    PCMK_ROLE_PROMOTED);
         } else if(count > 1) {
             out->info(out, "Resource '%s' is currently %s in %d locations. "
                       "One may now move to %s",
                       rsc_id, (promoted_role_only? "promoted" : "active"),
                       count, pcmk__node_name(dest));
             out->info(out, "To prevent '%s' from being %s at a specific location, "
                       "specify a node.",
                       rsc_id, (promoted_role_only? "promoted" : "active"));
 
         } else {
             crm_trace("Not banning %s from its current location: not active", rsc_id);
         }
     }
 
     return rc;
 }