diff --git a/.gitignore b/.gitignore
index e4fdd13598..16dd0d118c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,238 +1,239 @@
 # Common
 \#*
 .\#*
 GPATH
 GRTAGS
 GTAGS
 TAGS
 Makefile
 Makefile.in
 .deps
 .dirstamp
 .libs
 *.pc
 *.pyc
 *.bz2
 *.tar.gz
 *.tgz
 *.la
 *.lo
 *.o
 *~
 *.gcda
 *.gcno
 
 
 # Autobuild
 aclocal.m4
 autoconf
 autoheader
 autom4te.cache/
 automake
 build.counter
 compile
 /confdefs.h
 config.guess
 config.log
 config.status
 config.sub
 configure
 /conftest*
 depcomp
 install-sh
 include/stamp-*
 libtool
 libtool.m4
 ltdl.m4
 libltdl
 ltmain.sh
 missing
 py-compile
 /m4/argz.m4
 /m4/ltargz.m4
 /m4/ltoptions.m4
 /m4/ltsugar.m4
 /m4/ltversion.m4
 /m4/lt~obsolete.m4
 test-driver
 ylwrap
 
 # Configure targets
 /cts/CTS.py
 /cts/CTSlab.py
 /cts/CTSvars.py
 /cts/LSBDummy
 /cts/OCFIPraTest.py
 /cts/benchmark/clubench
 /cts/cluster_test
 /cts/cts
 /cts/cts-cli
 /cts/cts-coverage
 /cts/cts-exec
 /cts/cts-fencing
 /cts/cts-log-watcher
 /cts/cts-regression
 /cts/cts-scheduler
 /cts/cts-support
 /cts/fence_dummy
 /cts/lxc_autogen.sh
 /cts/pacemaker-cts-dummyd
 /cts/pacemaker-cts-dummyd@.service
 /daemons/execd/pacemaker_remote
 /daemons/execd/pacemaker_remote.service
 /daemons/fenced/fence_legacy
 /daemons/pacemakerd/pacemaker
 /daemons/pacemakerd/pacemaker.combined.upstart
 /daemons/pacemakerd/pacemaker.service
 /daemons/pacemakerd/pacemaker.upstart
 /doc/Doxyfile
 /extra/logrotate/pacemaker
 /extra/resources/ClusterMon
 /extra/resources/HealthSMART
 /extra/resources/SysInfo
 /extra/resources/ifspeed
 /extra/resources/o2cb
 include/config.h
 include/config.h.in
 include/crm_config.h
 publican.cfg
 /tools/cibsecret
 /tools/crm_error
 /tools/crm_failcount
 /tools/crm_master
 /tools/crm_mon.service
 /tools/crm_mon.upstart
 /tools/crm_report
 /tools/crm_rule
 /tools/crm_standby
 /tools/pcmk_simtimes
 /tools/report.collector
 /tools/report.common
 
 # Build targets
 *.7
 *.7.xml
 *.7.html
 *.8
 *.8.xml
 *.8.html
 /daemons/attrd/pacemaker-attrd
 /daemons/based/pacemaker-based
 /daemons/based/cibmon
 /daemons/controld/pacemaker-controld
 /daemons/execd/cts-exec-helper
 /daemons/execd/pacemaker-execd
 /daemons/execd/pacemaker-remoted
 /daemons/fenced/cts-fence-helper
 /daemons/fenced/pacemaker-fenced
 /daemons/fenced/pacemaker-fenced.xml
 /daemons/pacemakerd/pacemakerd
 /daemons/schedulerd/pacemaker-schedulerd
 /daemons/schedulerd/pacemaker-schedulerd.xml
 /doc/*/tmp/**
 /doc/*/publish
 /doc/*.build
 /doc/*/en-US/Ap-*.xml
 /doc/*/en-US/Ch-*.xml
 /doc/.ABI-build
 /doc/HTML
 /doc/abi_dumps
 /doc/abi-check
 /doc/api/*
 /doc/compat_reports
 /doc/crm_fencing.html
 /doc/publican-catalog*
 /doc/shared/en-US/*.xml
 /doc/shared/en-US/images/pcmk-*.png
 /doc/shared/en-US/images/Policy-Engine-*.png
 /doc/sphinx/*/_build
 /doc/sphinx/*/conf.py
 /lib/common/md5.c
+/maint/testcc_helper.cc
 /maint/testcc_*_h
 /maint/mocked/based
 scratch
 /tools/attrd_updater
 /tools/cibadmin
 /tools/crmadmin
 /tools/crm_attribute
 /tools/crm_diff
 /tools/crm_mon
 /tools/crm_node
 /tools/crm_resource
 /tools/crm_shadow
 /tools/crm_simulate
 /tools/crm_ticket
 /tools/crm_verify
 /tools/iso8601
 /tools/stonith_admin
 xml/crm.dtd
 xml/pacemaker*.rng
 xml/versions.rng
 xml/api/api-result*.rng
 lib/gnu/libgnu.a
 lib/gnu/stdalign.h
 *.coverity
 
 # Packager artifacts
 *.rpm
 /mock
 /pacemaker.spec
 /rpm/[A-Z]*
 
 # make dist/export working directory
 pacemaker-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]
 
 # Test detritus
 /cts/.regression.failed.diff
 /cts/scheduler/*.ref
 /cts/scheduler/*.up
 /cts/scheduler/*.up.err
 /cts/scheduler/bug-rh-1097457.log
 /cts/scheduler/bug-rh-1097457.trs
 /cts/scheduler/shadow.*
 /cts/test-suite.log
 /xml/test-*/*.up
 /xml/test-*/*.up.err
 /xml/assets/*.rng
 /xml/assets/diffview.js
 /xml/assets/xmlcatalog
 
 # Test results
 *.log
 *.trs
 /lib/common/tests/strings/pcmk__parse_ll_range
 /lib/pengine/tests/rules/pe_cron_range_satisfied
 
 # Release maintenance detritus
 /maint/gnulib
 
 # Formerly built files (helps when jumping back and forth in checkout)
 /.ABI-build
 /Doxyfile
 /HTML
 /abi_dumps
 /abi-check
 /compat_reports
 /attrd
 /cib
 /coverage.sh
 /crmd
 /cts/HBDummy
 /doc/Clusters_from_Scratch.txt
 /doc/Pacemaker_Explained.txt
 /doc/acls.html
 /fencing
 /lrmd
 /mcp
 /pacemaker-*.spec
 /pengine
 
 #Other 
 coverity-*
 logs
 *.patch
 *.diff
 *.sed
 *.orig
 *.rej
 *.swp
diff --git a/ChangeLog b/ChangeLog
index 8843af18db..271f7a2660 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,2891 +1,2902 @@
+* Wed Jun 03 2020 Klaus Wenninger <klaus.wenninger@aon.at> Pacemaker-2.0.4-rc3
+- Changesets: 14
+- Diff: 10 files changed, 21 insertions(+), 26 deletions(-)
+
+- Fixes for regressions / new buggy features introduced in Pacemaker-2.0.4-rc1
+  + libpe_status: Modify filtering of inactive resources.
+
+- Changes since Pacemaker-2.0.4-rc2
+  + based: use crm_exit to free qb-logging on daemon exit.
+  + cts: handle last-run= and last-rc-change= as full date strings.
+
 * Sat May 16 2020 Klaus Wenninger <klaus.wenninger@aon.at> Pacemaker-2.0.4-rc2
 - Changesets: 34
 - Diff: 34 files changed, 461 insertions(+), 142 deletions(-)
 
 - Fixes for regressions introduced in Pacemaker-2.0.0
   + controller: properly detect remote node info requests
 
 - Fixes for regressions / new buggy features introduced in Pacemaker-2.0.4-rc1
   + libpe_status: Apply node filtering to clone resource lists.
   + libpe_status: Don't filter in pe__bundle_replica_output_text.
   + libpe_status: Don't output unnecessary resource headers.
   + libpe_status: Handle filtering of inactive resources.
   + tools: get stonith_admin --list-targets working again
   + tools: Print inactive resources by default with crm_mon xml output.
 
 - Changes since Pacemaker-2.0.4-rc1
   + fencer: avoid infinite loop if device is removed during operation
   + fencer: avoid possible use-of-NULL when parsing metadata
   + libfencing, fencer: Improve parameter handling for RHCS-style fence-agents
                 - add `port` or `plug` parameter according to metadata
                 - `plug` parameter shown as non-required in the metadata
   + controld: Avoid delay when recovering resources after shutdown-lock
               expiration. (CLBZ#5424)
   + fencing: Remove dangling 'pending' for completed DC fencing. (CLBZ#5401)
 
 * Tue Apr 28 2020 Klaus Wenninger <klaus.wenninger@aon.at> Pacemaker-2.0.4-rc1
 - Changesets: 442
 - Diff: 269 files changed, 21845 insertions(+), 13363 deletions(-)
 
 - Features added since Pacemaker-2.0.3
   + build: Add support for glib-based unit tests.
   + controller: add new feature 'shutdown-locks'
         Optionally allow a gracefully shutdown node to have the resources
         locked to it for a configurable time.
         So that it can be restarted with exactly the same resources
         running as before.
   + controller/fencer/scheduler: add new feature 'priority-fencing-delay'
         Optionally derive the priority of a node from the resource-priorities
         of the resources it is running.
         In a fencing-race the node with the highest priority has a certain
         advantage over the others as fencing requests for that node are
         executed with an additional delay.
         controlled via cluster option priority-fencing-delay (default = 0)
   + stonith_admin: add --delay option (default = 0) to support enforced
                    fencing delay
   + tools: Add --include= and --exclude= to crm_mon.
   + tools: Add --node= to filter crm_mon output for a node (or tag).
   + tools: Allow more fine grained filtering of fence-history in crm_mon.
   + tools: Allow crm_rule to check some date_specs.
 
 - Changes since Pacemaker-2.0.3
   + build: improve checking headers for C++ fitness
   + build: restore build on latest OpenSUSE
   + libstonithd: Report an error getting history in output instead of
                  empty history
   + libstonithd: Improve parameter handling for RHCS-style fence-agents
                  - make parameter `action` shown as not required
   + controller: clear leaving node's transient attributes even if there is no DC
   + controller: don't ack same request twice
   + iso8601: Fix crm_time_parse_offset() to parse offset with plus sign.
   + libcrmcommon, libpe: validate interval specs correctly
   + libcrmcommon: Add pcmk_str_is_infinity, pcmk_str_is_minus_infinity and
                   pcmk__unregister_formats.
   + libcrmcommon: Extend what pcmk__split_range can do.
   + libfencing: Export formatted output functions.
   + libpe_status: Add node list arg to output messages preventing weird
                   behavior + potential segfaults
   + libpe_status: Update the maint mode message for HTML output.
   + libstonithd, tools: Fix arguments to stonith-event.
   + scheduler: don't incorporate dependencies' colocations multiple times
   + scheduler: ensure attenuated scores still have some effect
   + scheduler: ignore colocation constraints with 0 score
   + scheduler: make sure cluster-wide maintenance-mode=true overrides
                per-resource settings
   + scheduler: properly detect whether guest node is fenceable
   + scheduler: do not differentiate case where all the nodes have equal priority
   + tests: Add tests for pe_cron_range_satisfied.
   + tests: Add tests for the current behavior of pcmk__split_range.
   + tools, lib: Use standard pacemaker return codes in crm_rule.
   + tools: Correct stonith-event arguments in crm_mon.
   + tools: Fix man pages for crm_mon & crm_diff.
   + tools: Make crm_mon --interval understand more formats.
   + tools: Fix --html-title= in crm_mon.
   + tools: Print errors to stderr for crm_mon's legacy xml.
   + tools: use return codes consistently in stonith_admin
   + tools: Use glib for cmdline handling in crm_diff, crm_simulate & crm_error
   + xml: Add a new version of the tags schema.
   + based: populate cib_nodes when 'cibadmin -R -scope=configuration'
 
 * Mon Nov 25 2019 Ken Gaillot <kgaillot@redhat.com> Pacemaker-2.0.3
 - Changesets: 601
 - Diff: 227 files changed, 17862 insertions(+), 10471 deletions(-)
 
 - Features added since Pacemaker-2.0.2
   + controller: new 'fence-reaction' cluster option specifies whether local node
                 should 'stop' or 'panic' if notified of own fencing
   + controller: more cluster properties support ISO 8601 time specifications
   + controller: calculate cluster recheck interval dynamically when possible
   + Pacemaker Remote: allow file for environment variables when used in bundle
   + Pacemaker Remote: allow configurable listen address and TLS priorities
   + tools: crm_mon now supports standard --output-as/--output-to options
   + tools: crm_mon HTML output supports user-defined CSS stylesheet
   + tools: stonith_admin supports HTML output in addition to text and XML
   + tools: crm_simulate supports --repeat option to repeat profiling tests
   + tools: new pcmk_simtimes tool compares crm_simulate profiling output
   + agents: SysInfo supports K, T, and P units in addition to Kb and G
 
 - Changes since Pacemaker-2.0.2
   + fencer: do not block concurrent fencing actions on a device
             (regression since 2.0.2)
   + all: avoid Year 2038 issues
   + all: allow ISO 8601 strings of form "<date>T<time> <offset>"
   + rpm: pacemaker-cts package now explicitly requires pacemaker-cli
   + controller: set timeout on scheduler responses to avoid infinite wait
   + controller: confirm cancel of failed monitors, to avoid transition timeout
   + executor: let controller cancel monitors, to avoid transition timeout
   + executor: return error for stonith probes if stonith connection was lost
   + fencer: ensure concurrent fencing commands always get triggered to execute
   + fencer: fail pending actions and re-sync history after crash and restart
   + fencer: don't let command with long delay block other pending commands
   + fencer: allow functioning even if CIB updates arrive unceasingly
   + scheduler: wait for probe actions to complete to prevent unnecessary
                restart/re-promote of dependent resources
   + scheduler: avoid invalid transition when guest node host is not fenceable
   + scheduler: properly detect dangling migrations, to avoid restart loop
   + scheduler: avoid scheduling actions on remote node that is shutting down
   + scheduler: avoid delay in recovery of failed remote connections
   + scheduler: clarify action failure log messages by including failure time
   + scheduler: calculate secure digests for unfencing, for replaying saved CIBs
   + libcrmcommon: avoid possible use-of-NULL when applying XML diffs
   + libcrmcommon: correctly apply XML diffs with multiple move/create changes
   + libcrmcommon: return error when applying XML diffs with unknown operations
   + tools: avoid duplicate lines between nodes in crm_simulate dot graph
   + tools: count disabled/blocked resources correctly in crm_mon/crm_simulate
   + tools: crm_mon --interval now accepts ISO 8601 and has correct help
   + tools: organize crm_mon text output with list headings, indents, bullets
   + tools: crm_report: fail if tar is not available
   + tools: crm_report: correct argument parsing
   + tools: crm_report: don't ignore log if unrelated file is too large
   + tools: stonith_admin --list-targets should show what fencer would use
   + agents: calculate #health_disk correctly in SysInfo
   + agents: handle run-as-user properly in ClusterMon
 
 * Tue Jun 04 2019 Ken Gaillot <kgaillot@redhat.com> Pacemaker-2.0.2
 - Changesets: 288
 - Diff: 225 files changed, 28494 insertions(+), 24465 deletions(-)
 
 - Features added since Pacemaker-2.0.1
   + tools: crm_resource --validate can get resource parameters from command line
   + tools: crm_resource --clear prints out any cleared constraints
   + tools: new crm_rule tool for checking rule expiration (experimental)
   + tools: stonith_admin supports XML output for machine parsing (experimental)
   + resources: new HealthIOWait resource agent for node health tracking
 
 - Changes since Pacemaker-2.0.1
   + Important security fixes for CVE-2018-16878, CVE-2018-16877, CVE-2019-3885
   + build: crm_report bug report URL is now configurable at build time
   + build: private libpengine/libtransitioner libraries combined as libpacemaker
   + controller: avoid memory leak when duplicate monitor is scheduled
   + scheduler: respect order constraints when resources are being probed
   + scheduler: one group stop shouldn't make another required
   + libcrmcommon: handle out-of-range integers in configuration better
   + libcrmcommon: export logfile environment variable if using default
   + libcrmcommon: avoid segmentation fault when beginning formatted text list
   + libcrmservice: fix use-after-free memory error in alert handling
   + libstonithd: handle more than 64KB output from fence agents
 
 
 * Mon Mar 4 2019 Ken Gaillot <kgaillot@redhat.com> Pacemaker-2.0.1
 - Changesets: 592
 - Diff: 173 files changed, 9268 insertions(+), 5344 deletions(-)
 
 - Features added since Pacemaker-2.0.0
   + Pacemaker bundles support podman for container management
   + fencing: SBD may be used in a cluster that has guest nodes or bundles
   + fencing: fencing history is synchronized among all nodes
   + fencing: stonith_admin has option to clear fence history
   + tools: crm_mon can show fencing action failures and history
   + tools: crm_resource --clear supports new --expired option
   + Pacemaker Remote: new options to restrict TLS Diffie-Hellman prime length
 
 - Changes since Pacemaker-2.0.0
   + scheduler: clone notifications could be scheduled for a stopped
     Pacemaker Remote node and block all further cluster actions
     (regression since 2.0.0)
   + libcrmcommon: correct behavior for completing interrupted live migrations
     (regression since 2.0.0)
   + tools: crm_resource -C could fail to clean up all failures in one run
     (regression since 2.0.0)
   + Pacemaker Remote: avoid unnecessary downtime when moving resource to
     Pacemaker Remote node that fails to come up (regression since 1.1.18)
   + tools: restore stonith_admin ability to confirm unseen nodes are down
     (regression since 1.1.12)
   + build: minor logging fixes to allow compatibility with GCC 9 -Werror
   + build: spec file now puts XML schemas in new pacemaker-schemas package
   + build: spec file now provides virtual pcmk-cluster-manager package
   + pacemaker-attrd: wait a short time before re-attempting failed writes
   + pacemaker-attrd: ignore attribute delays when writing after node (re-)join
   + pacemaker-attrd: start new election immediately if writer is lost
   + pacemaker-attrd: clear election dampening when the writer leaves
   + pacemaker-attrd: detect alert configuration changes when CIB is replaced
   + CIB: inform originator of CIB upgrade failure
   + controller: support resource agents that require node name even for meta-data
   + controller: don't record pending clone notifications in CIB
   + controller: DC detects completion of another node's shutdown more accurately
   + controller: shut down DC if unable to update node attributes
   + controller: handle corosync peer/join notifications for new node in any order
   + controller: clear election dampening when DC is lost
   + executor: cancel recurring monitors if fence device registration is lost
   + fencing: check for fence device update when resource defaults change
   + fencing: avoid pacemaker-fenced crash possible with stonith_admin misuse
   + fencing: limit fencing history to 500 entries
   + fencing: stonith_admin now complains if no action option is specified
   + pacemakerd: do not modify kernel.sysrq on Linux
   + scheduler: regression test compatibility with glib 2.59.0
   + scheduler: avoid unnecessary recovery of cleaned guest nodes and bundles
   + scheduler: ensure failures causing fencing not expired until fencing done
   + scheduler: start unique clone instances in numerical order
   + scheduler: convert unique clones to anonymous clones when not supported
   + scheduler: associate pending tasks with correct clone instance
   + scheduler: ensure bundle clone notifications are directed to correct host
   + scheduler: avoid improper bundle monitor rescheduling or fail count clearing
   + scheduler: honor asymmetric orderings even when restarting
   + scheduler: don't order non-DC shutdowns before DC fencing
   + ACLs: assume unprivileged ACL user if can't get user info
   + Pacemaker Remote: get Diffie-Hellman prime bit length from GnuTLS API
   + libcrmservice: cancel DBus call when cancelling systemd/upstart actions
   + libcrmservice: order systemd resources relative to pacemaker_remote
   + libpe_status: add public API constructor/destructor for pe_working_set_t
   + tools: fix crm_resource --clear when lifetime was used with ban/move
   + tools: fix crm_resource --move when lifetime was used with previous move
   + tools: make crm_mon CIB connection errors non-fatal if previously successful
   + tools: improve crm_mon messages when generating HTML output
   + tools: crm_mon cluster connection failure is now "critical" in nagios mode
   + tools: crm_mon listing of standby nodes shows if they have active resources
   + tools: crm_diff now ignores attribute ordering when comparing in CIB mode
   + tools: improve crm_report detection of logs, CIB directory, and processes
   + tools: crm_verify returns reliable exit codes
   + tools: crm_simulate resource history uses same name as live cluster would
 
 
 * Fri Jul 6 2018 Ken Gaillot <kgaillot@redhat.com> Pacemaker-2.0.0
 - Changesets: 885
 - Diff:       549 files changed, 89865 insertions(+), 95100 deletions(-)
 
 - Deprecated features removed since Pacemaker-1.1.18
   + All of these have newer forms, and the cluster will automatically convert
     most older syntax usage in saved configurations to newer syntax as needed
   + Drop support for heartbeat and corosync 1 (whether using CMAN or plugin)
   + Drop support for rolling upgrades from Pacemaker versions older than 1.1.11
   + Drop support for built-in SMTP and SNMP in crm_mon
   + Drop support for legacy option aliases including default-action-timeout,
     default-resource-stickiness, resource-failure-stickiness,
     default-resource-failure-stickiness, is-managed-default,
     and all names using underbar instead of dash
   + Drop support for "requires" operation meta-attribute
   + Drop support for the pcmk_*_cmd, pcmk_arg_map, and pcmk_poweroff_action
     fence resource parameters
   + Drop support for deprecated command-line options to crmadmin,
     crm_attribute, crm_resource, crm_verify, crm_mon, and stonith_admin
   + Drop support for operation meta-attributes in instance_attributes
   + Drop support for PCMK_legacy and LRMD_MAX_CHILDREN environment variables
   + Drop support for undocumented resource isolation feature
   + Drop support for processing very old saved CIB files (including
     pre-0.6.0 start failure entries, pre-0.6.5 operation history entries,
     pre-0.7 transition keys, pre-1.1.4 migration history entries,
     pre-1.0 XML configuration schemas, pre-1.1.6 ticket state entries, and
     pre-1.1.7 failed recurring operation history entries)
 
 - Features added since Pacemaker-1.1.18
   + The pacemaker daemons have been renamed to make logs more intuitive
     and easier to search
   + The default location of the Pacemaker detail log is now
     /var/log/pacemaker/pacemaker.log (instead of being directly in /var/log),
     and Pacemaker will no longer use Corosync's logging preferences;
     configure script options are available to change default log locations
   + The detail log's message format has been improved
   + The master XML tag is deprecated in favor of using a standard clone tag
     with a new "promotable" meta-attribute set to true, and the "master-max"
     and "master-node-max" master meta-attributes are deprecated in favor of
     new "promoted-max" and "promoted-node-max" clone meta-attributes;
     documentation now refers to these as promotable clones rather than
     master/slave, stateful, or multistate clones, and refers to
     promotion scores instead of master scores
   + Administration-related documentation has been moved from the
     "Pacemaker Explained" document to a new "Pacemaker Administration" document
   + record-pending now defaults to TRUE (pending actions are shown in status)
   + All Python code in Pacemaker now supports both Python 2.7 and Python 3
   + The command-line tools now return consistent, well-defined exit codes;
     crm_error has an --exit option to list these
   + Pacemaker's systemd unit files now remove systemd's spawned process limit
   + mount, path, and timer systemd unit types are now supported as resources
   + A negative stonith-watchdog-timeout now tells the cluster to automatically
     calculate the value based on SBD_WATCHDOG_TIMEOUT (which was the behavior
     of 0 before 1.1.15; 0 retains its post-1.1.15 behavior of disabling use of
     the watchdog as a fencing device)
   + The undocumented restart-type and role_after_failure
     resource meta-attributes are now deprecated
   + Regression testing code has been consolidated and overhauled
     (the most obvious change is new command names)
   + build: create /etc/pacemaker directory when installing
   + build: improved portability to BSD-based platforms
   + tools: crm_resource --cleanup now cleans only failed operation history;
     crm_resource --reprobe retains the previous behavior of cleaning all
     operation history
   + tools: add stonith_admin --validate option to check device configuration
   + tools: crm_node is now in the pacemaker-cli package (instead of pacemaker)
   + alerts: add epoch and usec alert variables for improved SNMP alerts
   + controller: deprecate "crmd-*" cluster options in favor of new names
   + scheduler: deprecate stonith-action value "poweroff" (use "off" instead)
   + scheduler: deprecate require-all in rsc_order
   + libcrmcluster: prefer corosync name over ring0_addr
   + xml: allow local "kind" in resource_set within rsc_order
 
 - Changes since Pacemaker-1.1.18
   + Restore systemd unit dependency on DBus (regression in 1.1.17)
   + CIB: handle mixed-case node names when modifying attributes (regression in 1.1.17)
   + scheduler: avoid crash when logging ignored failure timeout (regression in 1.1.17)
   + attrd: ensure node name is broadcast at start-up (regression in 1.1.18)
   + scheduler: unfence before probing or starting fence devices (regression in 1.1.18)
   + tools: treat INFINITY correctly in crm_failcount (regression in 1.1.17)
   + tools: show master scores with crm_simulate -sL (regression in 1.1.18)
   + tools: crm_master did not work without explicit --lifetime (regression in 1.1.18)
   + Numerous changes to public C API of libraries
   + Choose current node correctly when a resource is multiply active
   + controller,executor,tools: avoid minor memory leaks
   + CIB: don't use empty CIB if real CIB has bad permissions
   + controller: avoid double free after ACL rejection of resource deletion
   + controller: don't record pending clone notifications in CIB
   + controller: always write faked failures to CIB whenever possible
   + controller: quorum gain without a node join should cause new transition
   + executor: handle systemd actions correctly when used with "service:"
   + executor: find absolute LSB paths when used with "service:"
   + scheduler: handle "requires" of "quorum" or "nothing" properly
   + scheduler: ensure orphaned recurring monitors have interval set
   + scheduler: handle pending migrations correctly when record-pending is true
   + scheduler: don't time out failures that cause fencing until fencing completes
   + scheduler: handle globally-unique bundle children correctly
   + scheduler: use correct default timeout for monitors
   + scheduler: "symmetrical" defaults to "false" for serialize orders
   + scheduler: avoid potential use-of-NULL when unpacking ordering constraint
   + scheduler: properly cancel recurring monitors
   + scheduler: do not schedule notifications for unrunnable actions
   + scheduler: ensure stops occur after stopped remote connections come back up
   + scheduler: consider only allowed nodes when ordering start after all recovery
   + scheduler: avoid graph loop from ordering bundle child stops/demotes after container fencing
   + scheduler: remote connection resources are safe to require only quorum
   + scheduler: correctly observe colocation with bundles in Master role
   + scheduler: restart resource after failed demote when appropriate
   + Pacemaker Remote: always use most recent remote proxy
   + tools: crm_node now gets correct node name and ID on Pacemaker Remote nodes
   + tools: correctly check crm_resource --move for master role
   + tools: cibsecret --help/--version doesn't require cluster to be running
   + tools: ignore attribute placement when crm_diff compares in cib mode
   + tools: prevent notify actions from causing crm_resource --wait to hang
   + resources: drop broken configdir parameter from ocf:pacemaker:controld
 
 - For further details, see:
   https://wiki.clusterlabs.org/wiki/Pacemaker_2.0_Changes
 
 
 * Tue Nov 14 2017 Ken Gaillot <kgaillot@redhat.com> Pacemaker-1.1.18
 - Update source tarball to revision: a9fbd15
 - Changesets: 644
 - Diff:       167 files changed, 9753 insertions(+), 5596 deletions(-)
 
 - Features added since Pacemaker-1.1.17
   + warnings are now logged when using legacy syntax to be removed in 2.0
   + agents: ifspeed agent is now installed when building
   + agents: ifspeed agent can optionally detect interface name from IP address
   + alerts: support alert filters
   + alerts: experimental support for alerts for node attribute changes
   + crmd,pengine: support unfencing of remote nodes
   + pengine: bundles now support all constraint types
   + pengine: bundles now support rkt containers
   + pengine: bundles support new container-attribute-target parameter
   + pengine,tools: logs and crm_mon show why resources changed state
   + stonith-ng: support new fencing resource parameter pcmk_delay_base
   + tools: new crm_resource option --why explains why resources are stopped
 
 - Changes since Pacemaker-1.1.17
   + many documentation improvements
   + agents: ifspeed properly calculates speed of hfi1 interfaces
   + agents: ClusterMon now interprets "update" less than 1000 as seconds
   + attrd: don't lose attributes set between attrd start-up and cluster join
   + attrd: fix multiple minor memory leaks
   + crmd: correctly record that unfencing is complete
   + crmd: error more quickly if remote start fails due to missing key
   + lrmd: remote resource operations return immediate error if key setup fails
   + lrmd: allow pre-1.1.15 cluster nodes to connect to current Pacemaker Remote
   + pengine: guest nodes are now probed like other nodes
   + pengine: probe remote nodes for guest node resources
   + pengine: do not probe guest/bundle connections until guest/bundle is active
   + pengine: allow resources to stop prior to probes completing
   + pengine: bundles wait only for other containers on same node to be probed
   + pengine: have bundles log to stderr so 'docker logs'/'journalctl -M' works
   + pengine: only pass requests for promote/demote flags onto the bundle's child
   + pengine: do not map ports into Docker container when net=host is specified
   + pengine: allow resources inside bundles to receive clone notifications
   + pengine: default to non-interleaved bundle ordering for safety
   + pengine: ensure bundle nodes and child resources are correctly cleaned up
   + pengine: prevent graph loops when fencing the host underneath a bundle
   + pengine: fix multiple memory issues (use-after-free, use-of-NULL) with bundles
   + pengine: resources in bundles respect failcounts
   + pengine: ensure nested container connections run on the same host
   + pengine: ensure unrecoverable remote nodes are fenced even with no resources
   + pengine: handle resource migrating behind a migrating remote connection
   + pengine: don't prefer to keep unique instances on same node
   + pengine: exclude exclusive resources and nodes from symmetric default score
   + pengine: if ignoring failure, also ignore migration-threshold
   + pengine: restore the ability to send the transition graph via the disk if it gets too big
   + pengine: validate no-quorum-policy=suicide correctly
   + pengine: avoid crash when alerts section has comments
   + pengine: detect permanent master scores at start-up
   + pengine: do not re-add a node's default score for each location constraint
   + pengine: make sure calculated resource scores are consistent on different architectures
   + pengine: retrigger unfencing for changed device parameters only when necessary
   + pengine: don't schedule reload and restart in same transition (CLBZ#5309, regression introduced in 1.1.15)
   + stonith-ng: make fencing-device reappear properly after reenabling
   + stonith-ng: include pcmk_on_action in meta-data so 'on' can be overridden
   + tools: allow crm_report to work with no log files specified
   + tools: fix use-after-free in crm_diff introduced in 1.1.17
   + tools: allow crm_resource to operate on anonymous clones in unknown states
   + tools: crm_resource --cleanup on appropriate nodes if we don't know state of resource
   + tools: prevent disconnection from crmd during crm_resource --cleanup
   + tools: improve messages for crm_resource --force-* options
   + tools: crm_mon: avoid infinite process spawning if -E script can't be run
   + tools: crm_mon: don't show previous exit-reason for failed action with none
   + libcrmservice: list systemd unit files, not only active units (CLBZ#5299)
   + libcrmservice: parse long description correctly for LSB meta-data
 
 
 * Thu Jul 06 2017 Ken Gaillot <kgaillot@redhat.com> Pacemaker-1.1.17
 - Update source tarball to revision: 301bc44
 - Changesets: 539
 - Diff:       177 files changed, 11525 insertions(+), 5036 deletions(-)
 
 - Features added since Pacemaker-1.1.16
   + New "bundle" resource type for Docker container use cases (experimental)
   + New "PCMK_node_start_state" environment variable to start node in standby
   + New "value-source" rule expression attribute in location constraints to
     compare a node attribute against a resource parameter
   + New "stonith-max-attempts" cluster option to specify how many times
     fencing can fail for a target before the cluster will no longer
     immediately re-attempt it (previously hard-coded at 10) 
   + New "cluster-ipc-limit" cluster option to avoid IPC client eviction in
     large clusters
   + Failures are now tracked per operation type, as well as per node and
     resource (the "fail-count" and "last-failure" node attribute names now end
     in "#OPERATION_INTERVAL")
   + attrd: Pacemaker Remote node attributes and regular expressions are now
     supported on legacy cluster stacks (heartbeat, CMAN, and corosync plugin)
   + tools: New "crm_resource --validate" option
   + tools: New "stonith_admin --list-targets" option
   + tools: New "crm_attribute --pattern" option to match a regular expression
   + tools: "crm_resource --cleanup" and "crm_failcount" can now take
     --operation and --interval options to operate on a single operation type
 
 - Changes since Pacemaker-1.1.16
   + Fix multiple memory issues (leaks, use-after-free) in libraries
   + pengine: unmanaging a guest node resource puts guest in maintenance mode
   + cib: broadcasts of cib changes should always pass ACL checks
   + crmd,libcrmcommon: update throttling when CPUs are hot-plugged
   + crmd: abort transition whenever we lose quorum
   + crmd: avoid attribute write-out on join when atomic attrd is used
   + crmd: check for too many stonith failures only when aborting for that reason
   + crmd: correctly clear failure counts only for a specified node
   + crmd: don't fence old DC if it's shutting down as soon-to-be DC joins
   + crmd: forget stonith failures when forgetting node
   + crmd: all nodes should track stonith failure counts in case they become DC
   + crmd: update cache status for guest node whose host is fenced
   + dbus: prevent lrmd from hanging on dbus calls
   + fencing: detect newly added constraints for stonith devices
   + pengine: order remote actions after connection recovery
     (regression introduced in 1.1.15)
   + pengine: quicker recovery from failed demote
   + libcib: determine remote nodes correctly from node status entries
   + libcrmcommon: avoid evicting IPC client if messages spike briefly
   + libcrmcommon: better XML comment handling prevents infinite election loop
   + libcrmcommon: set month correctly in date/time string sent to alert agents
   + libfencing,fencing: intelligently remap "action" wrongly specified in config
   + libservices: ensure completed ops aren't on blocked ops list
   + libservices: properly detect and cancel in-flight systemd/upstart ops
   + libservices: properly watch writable DBus handles
   + libservices: systemd service that is reloading doesn't cause monitor failure
   + pacemaker_remoted: allow graceful shutdown while unmanaged
   + pengine,libpe_status: don't clear same fail-count twice
   + pengine: consider guest node unclean if its host is unclean
   + pengine: do not re-add a node's default score for each location constraint
   + pengine: avoid restarting services when recovering remote connection
   + pengine: better guest node recovery when host fails
   + pengine: guest node fencing doesn't require stonith enabled
   + pengine: allow probes of guest node connection resources
   + pengine: properly handle allow-migrate explicitly set for remote connection
   + pengine: fence failed remote nodes even if no resources can run on them
   + tools: resource agents will now get the correct node name on
     Pacemaker Remote nodes when using crm_node and crm_attribute
   + tools: avoid grep crashes in crm_report when looking for system logs
   + tools: crm_resource -C now clears last-failure as well as fail-count
   + tools: implement crm_failcount command-line options correctly
   + tools: properly ignore version with crm_diff --no-version
 
 
 * Wed Nov 30 2016 Ken Gaillot <kgaillot@redhat.com> Pacemaker-1.1.16
 - Update source tarball to revision: 76876b3
 - Changesets: 382
 - Diff:       145 files changed, 7200 insertions(+), 5621 deletions(-)
 
 - Features added since Pacemaker-1.1.15
   + Location constraints may use rsc-pattern, with submatches expanded
   + node-health-base available with node-health-strategy=progressive
   + new Pacemaker Development document for working on pacemaker code base
   + new PCMK_panic_action variable allows crash instead of reboot on panic
   + resources: add resource agent for managing a node attribute
   + systemd: include socket units when listing all systemd agents
 
 - Changes since Pacemaker-1.1.15
   + Important security fix for CVE-2016-7035
   + Logging is now synchronous when blackboxes are enabled
   + All python code except CTS is now compatible with python 2.6+ and 3.2+
   + build: take advantage of compiler features for security and performance
   + build: update SuSE spec modifications for recent spec changes
   + build: avoid watchdog reboot when upgrading pacemaker_remote with sbd
   + build: numerous other improvements in environment detection, etc.
   + cib: fix infinite loop when no schema validates
   + crmd: cl#5185 - record pending operations in CIB before they are performed
   + crmd: don't abort transitions for CIB comment changes
   + crmd: resend shutdown request if DC loses original request
   + documentation: install improved README in doc instead of now-removed AUTHORS
   + documentation: clarify licensing and provide copy of all licenses
   + documentation: document various features and upgrades better
   + fence_legacy: use "list" action when searching cluster-glue agents
   + libcib: don't stop sending alerts after releasing DC role
   + libcrmcommon: properly handle XML comments when comparing v2 patchset diffs
   + libcrmcommon: report errors consistently when waiting for data on connection
   + libpengine: avoid potential use-of-NULL
   + libservices: use DBusError API properly
   + pacemaker_remote: init script stop should always return 0
   + pacemaker_remote: allow remote clients to timeout/reconnect
   + pacemaker_remote: correctly calculate remaining timeout when receiving messages
   + pengine: avoid transition loop for start-then-stop + unfencing
   + pengine: correctly update dependent actions of un-runnable clones
   + pengine: do not fence a node in maintenance mode if it shuts down cleanly
   + pengine: set OCF_RESKEY_CRM_meta_notify_active_* for multistate resources
   + resources: ping - avoid temporary files for fping check, support FreeBSD
   + resources: SysInfo - better support for FreeBSD
   + resources: variable name typo in docker-wrapper
   + systemd: order pacemaker after time-sync target
   + tools: correct attrd_updater help and error messages when using CMAN
   + tools: crm_standby --version/--help should work without cluster running
   + tools: make crm_report sanitize CIB before generating readable version
   + tools: display pending resource state by default when available
   + tools: avoid matching other process with same PID in ClusterMon
 
 
 * Tue Jun 21 2016 Ken Gaillot <kgaillot@redhat.com> Pacemaker-1.1.15
 - Update source tarball to revision: 32fa6a5
 - Changesets: 533
 - Diff:       219 files changed, 6659 insertions(+), 3989 deletions(-)
 
 - Features added since Pacemaker-1.1.14
   + Event-driven alerts allow scripts to be called after significant events
   + build: Some files moved from pacemaker package to pacemaker-cli for cleaner pacemaker-remote dependencies
   + build: ./configure --with-configdir argument for /etc/sysconfig, /etc/default, etc.
   + fencing: Simplify watchdog integration
   + fencing: Support concurrent fencing actions via new pcmk_action_limit option
   + remote: pacemaker_remote may be stopped without disabling resource first
   + remote: Report integration status of Pacemaker Remote nodes in CIB node_state
   + tools: crm_mon now reports why resources are not starting
   + tools: crm_report now obscures passwords in logfiles
   + tools: attrd_updater --update-both/--update-delay options allow changing dampening value
   + tools: allow stonith_admin -H '*' to show history for all nodes
 
 - Changes since Pacemaker-1.1.14
   + Fix multiple memory issues (leaks, use-after-free) in daemons, libraries and tools
   + Make various log messages more user-friendly
   + Improve FreeBSD and Hurd support
   + attrd: Prevent possible segfault on exit
   + cib: Fix regression to restore support for compressed CIB larger than 1MB
   + common: fix regression in 1.1.14 that made have-watchdog always true
   + controld: handle DLM "wait fencing" state better
   + crmd: Fix regression so that fenced unseen nodes do not remain unclean
   + crmd: Take start-delay into account when calculation action timeouts
   + crmd: Avoid timeout on older peers when cancelling a resource operation
   + fencing: Allow fencing by node ID (e.g. by DLM) even if node left cluster
   + lrmd: Fix potential issues when cluster is stopped via systemd shutdown
   + pacemakerd: Properly respawn stonithd if it fails
   + pengine: Fix regression with multiple monitor levels that could ignore failure
   + pengine: Correctly set OCF_RESKEY_CRM_meta_timeout when start-delay is configured
   + pengine: Properly order actions for master/slave resources in anti-colocations
   + pengine: Respect asymmetrical ordering when trying to move resources
   + pengine: Properly order stop actions on guest node relative to host stonith
   + pengine: Correctly block actions dependent on unrunnable clones
   + remote: Allow remote nodes to have node attributes even with legacy attrd
   + remote: Recover from remote node fencing more quickly
   + remote: Place resources on newly rejoined remote nodes more quickly
   + resources: ping agent can now use fping6 for IPv6 hosts
   + resources: SysInfo now resets #health_disk to green when there's sufficient free disk
   + tools: crm_report is now more efficient and handles Pacemaker Remote nodes better
   + tools: Prevent crm_resource segfault when --resource is not supplied with --restart
   + tools: crm_shadow --display option now works
   + tools: crm_resource --restart handles groups, target-roles and moving resources better
 
 
 * Thu Jan 14 2016 Ken Gaillot <kgaillot@redhat.com> Pacemaker-1.1.14
 - Update source tarball to revision: f0b585a
 - Changesets: 724
 - Diff:        179 files changed, 13142 insertions(+), 7695 deletions(-)
 
 - Features added since Pacemaker-1.1.13
   + crm_resource: Indicate common reasons why a resource may not start after a cleanup
   + crm_resource: New --force-promote and --force-demote options for debugging
   + fencing: Support targeting fencing topologies by node name pattern or node attribute
   + fencing: Remap sequential topology reboots to all-off-then-all-on
   + pengine: Allow resources to start and stop as soon as their state is known on all nodes
   + pengine: Include a list of all and available nodes with clone notifications
   + pengine: Addition of the clone resource clone-min metadata option
   + pengine: Support of multiple-active=block for resource groups
   + remote: Resources that create guest nodes can be included in a group resource
   + remote: reconnect_interval option for remote nodes to delay reconnect after fence
 
 - Changes since Pacemaker-1.1.13
   + improve support for building on FreeBSD and Debian
   + fix multiple memory issues (leaks, use-after-free, double free, use-of-NULL) in components and tools
   + cib: Do not terminate due to badly behaving clients
   + cman: handle corosync-invented node names of the form Node{id} for peers not in its node list
   + controld: replace bashism
   + crm_node: Display node state with -l and quorum status with -q, if available
   + crmd: resources would sometimes be restarted when only non-unique parameters changed
   + crmd: fence remote node after connection failure only once
   + crmd: handle resources named the same as cluster nodes
   + crmd: Pre-emptively fail in-flight actions when lrmd connections fail
   + crmd: Record actions in the CIB as failed if we cannot execute them
   + crm_report: Enable password sanitizing by default
   + crm_report: Allow log file discovery to be disabled
   + crm_resource: Allow the resource configuration to be modified for --force-{check,start,..} calls
   + crm_resource: Compensate for -C and -p being called with the child resource for clones
   + crm_resource: Correctly clean up all children for anonymous cloned groups
   + crm_resource: Correctly clean up failcounts for inactive anonymous clones
   + crm_resource: Correctly observe --force when deleting and updating attributes
   + crm_shadow: Fix "crm_shadow --diff"
   + crm_simulate: Prevent segfault on arches with 64bit time_t
   + fencing: ensure "required"/"automatic" only apply to "on" actions
   + fencing: Return a provider for the internal fencing agent "#watchdog" instead of logging an error
   + fencing: ignore stderr output of fence agents (often used for debug messages)
   + fencing: fix issue where deleting a fence device attribute can delete the device
   + libcib: potential user input overflow
   + libcluster: overhaul peer cache management
   + log: make syslog less noisy
   + log: fix various misspellings in log messages
   + lrmd: cancel currently pending STONITH op if stonithd connection is lost
   + lrmd: Finalize all pending and recurring operations when cleaning up a resource
   + pengine: Bug cl#5247 - Imply resources running on a container are stopped when the container is stopped
   + pengine: cl#5235 - Prevent graph loops that can be introduced by "load_stopped -> migrate_to" ordering
   + pengine: Correctly bypass fencing for resources that do not require it
   + pengine: do not timeout remote node recurring monitor op failure until after fencing
   + pengine: Ensure recurring monitor operations are cancelled when clone instances are de-allocated
   + pengine: fixes segfault in pengine when fencing remote node
   + pengine: properly handle blocked clone actions
   + pengine: ensure failed actions that occurred in node shutdown are displayed
   + remote: Correctly display the usage of the ocf:pacemaker:remote resource agent
   + remote: do not fail operations because of a migration
   + remote: enable reloads for select remote connection options
   + resources: allow for top output with or without percent sign in HealthCPU
   + resources: Prevent an error message on stopping "Dummy" resource
   + systemd: Prevent segfault when logging failed operations
   + systemd: Reconnect to System DBus if the connection is closed
   + systemd: set systemd resources' timeout values higher than systemd's own default
   + tools: Do not send command lines to syslog
   + tools: update SNMP MIB
   + upstart: Ensure pending structs are correctly unreferenced
 
 
 * Wed Jun 24 2015 Andrew Beekhof <andrew@beekhof.net> Pacemaker-1.1.13
 - Update source tarball to revision: 2a1847e
 - Changesets: 750
 - Diff:       156 files changed, 11323 insertions(+), 3725 deletions(-)
 
 - Features added since Pacemaker-1.1.12
   + Allow fail-counts to be removed en-mass when the new attrd is in operation
   + attrd supports private attributes (not written to CIB)
   + crmd: Ensure a watchdog device is in use if stonith-watchdog-timeout is configured
   + crmd: If configured, trigger the watchdog immediately if we lose quorum and no-quorum-policy=suicide
   + crm_diff: Support generating a difference without versions details if --no-version/-u is supplied
   + crm_resource: Implement an intelligent restart capability
   + Fencing: Advertise the watchdog device for fencing operations
   + Fencing: Allow the cluster to recover resources if the watchdog is in use
   + fencing: cl#5134 - Support random fencing delay to avoid double fencing
   + mcp: Allow orphan children to initiate node panic via SIGQUIT
   + mcp: Turn on sbd integration if pacemakerd finds it running
   + mcp: Two new error codes that result in machine reset or power off
   + Officially support the resource-discovery attribute for location constraints
   + PE: Allow natural ordering of colocation sets
   + PE: Support non-actionable degraded mode for OCF
   + pengine: cl#5207 - Display "UNCLEAN" for resources running on unclean offline nodes
   + remote: pcmk remote client tool for use with container wrapper script
   + Support machine panics for some kinds of errors (via sbd if available)
   + tools: add crm_resource --wait option
   + tools: attrd_updater supports --query and --all options
   + tools: attrd_updater: Allow attributes to be set for other nodes
 
 - Changes since Pacemaker-1.1.12
   + pengine: exclusive discovery implies rsc is only allowed on exclusive subset of nodes
   + acl: Correctly implement the 'reference' acl directive
   + acl: Do not delay evaluation of added nodes in some situations
   + attrd: b22b1fe did uuid test too early
   + attrd: Clean out the node cache when requested by the admin
   + attrd: fixes double free in attrd legacy
   + attrd: properly write attributes for peers once uuid is discovered
   + attrd: refresh should force an immediate write-out of all attributes
   + attrd: Simplify how node deletions happen
   + Bug rhbz#1067544 - Tools: Correctly handle --ban, --move and --locate for master/slave groups
   + Bug rhbz#1181824 - Ensure the DC can be reliably fenced
   + cib: Ability to upgrade cib validation schema in legacy mode
   + cib: Always generate digests for cib diffs in legacy mode
   + cib: assignment where comparison intended
   + cib: Avoid nodeid conflicts we don't care about
   + cib: Correctly add "update-origin", "update-client" and "update-user" attributes for cib
   + cib: Correctly set up signal handlers
   + cib: Correctly track node state
   + cib: Do not update on disk backups if we're just querying them
   + cib: Enable cib legacy mode for plugin-based clusters
   + cib: Ensure file-based backends treat '-o section' consistently with the native backend
   + cib: Ensure upgrade operations from a non-DC get an acknowledgement
   + cib: No need to enforce cib digests for v2 diffs in legacy mode
   + cib: Revert d153b86 to instantly get cib synchronized in legacy mode
   + cib: tls sock cleanup for remote cib connections
   + cli: Ensure subsequent unknown long options are correctly detected
   + cluster: Invoke crm_remove_conflicting_peer() only when the new node's uname is being assigned in the node cache
   + common: Increment current and age for lib common as a result of APIs being added
   + corosync:  Bug cl#5232 - Somewhat gracefully handle nodes with invalid UUIDs
   + corosync: Avoid unnecessary repeated CMAP API calls
   + crmd/pengine: handle on-fail=ignore properly
   + crmd: Add "on_node" attribute for *_last_failure_0 lrm resource operations
   + crmd: All peers need to track node shutdown requests
   + crmd: Cached copies of transient attributes cease to be valid once a node leaves the membership
   + crmd: Correctly add the local option that validates against schema for pengine to calculate
   + crmd: Disable debug logging that results in significant overhead
   + crmd: do not remove connection resources during re-probe
   + crmd: don't update fail count twice for same failure
   + crmd: Ensure remote connection resources timeout properly during 'migrate_from' action
   + crmd: Ensure throttle_mode() does something on Linux
   + crmd: Fixes crash when remote connection migration fails
   + crmd: gracefully handle remote node disconnects during op execution
   + crmd: Handle remote connection failures while executing ops on remote connection
   + crmd: include remote nodes when forcing cluster wide resource reprobe
   + crmd: never stop recurring monitor ops for pcmk remote during incomplete migration
   + crmd: Prevent the old version of DC from being fenced when it shuts down for rolling-upgrade
   + crmd: Prevent use-of-NULL during reprobe
   + crmd: properly update job limit for baremetal remote-nodes
   + crmd: Remote-node throttle jobs count towards cluster-node hosting conneciton rsc
   + crmd: Reset stonith failcount to recover transitioner when the node rejoins
   + crmd: resolves memory leak in crmd.
   + crmd: respect start-failure-is-fatal even for artifically injected events
   + crmd: Wait for all pending operations to complete before poking the policy engine
   + crmd: When container's host is fenced, cancel in-flight operations
   + crm_attribute: Correctly update config options when -o crm_config is specified
   + crm_failcount: Better error reporting when no resource is specified
   + crm_mon: add exit reason to resource failure output
   + crm_mon: Fill CRM_notify_node in traps with node's uname rather than node's id if possible
   + crm_mon: Repair notification delivery when the v2 patch format is in use
   + crm_node: Correctly remove nodes from the CIB by nodeid
   + crm_report: More patterns for finding logs on non-DC nodes
   + crm_resource: Allow resource restart operations to be node specific
   + crm_resource: avoid deletion of lrm cache on node with resource discovery disabled.
   + crm_resource: Calculate how long to wait for a restart based on the resource timeouts
   + crm_resource: Clean up memory in --restart error paths
   + crm_resource: Display the locations of all anonymous clone children when supplying the children's common ID
   + crm_resource: Ensure --restart sets/clears meta attributes
   + crm_resource: Ensure fail-counts are purged when we redetect the state of all resources
   + crm_resource: Implement --timeout for resource restart operations
   + crm_resource: Include group members when calculating the next timeout
   + crm_resource: Memory leak in error paths
   + crm_resource: Prevent use-after-free
   + crm_resource: Repair regression test outputs
   + crm_resource: Use-after-free when restarting a resource
   + dbus: ref count leaks
   + dbus: Ensure both the read and write queues get dispatched
   + dbus: Fail gracefully if malloc fails
   + dbus: handle dispatch queue when multiple replies need to be processed
   + dbus: Notice when dbus connections get disabled
   + dbus: Remove double-free introduced while trying to make coverity shut up
   + ensure if B is colocated with A, B can never run without A
   + fence_legacy: Avoid passing 'port' to cluster-glue agents
   + fencing: Allow nodes to be purged from the member cache
   + fencing: Correctly make args for fencing agents
   + fencing: Correctly wait for self-fencing to occur when the watchdog is in use
   + fencing: Ensure the hostlist parameter is set for watchdog agents
   + fencing: Force 'stonith-ng' as the system name
   + fencing: Gracefully handle invalid metadata from agents
   + fencing: If configured, wait stonith-watchdog-timer seconds for self-fencing to complete
   + fencing: Reject actions for devices that haven't been explicitly registered yet
   + ipc: properly allocate server enforced buffer size on client
   + ipc: use server enforced buffer during ipc client send
   + lrmd, services: interpret LSB status codes properly
   + lrmd: add back support for class heartbeat agents
   + lrmd: cancel pending async connection during disconnect
   + lrmd: enable ipc proxy for docker-wrapper privileged mode
   + lrmd: fix rescheduling of systemd monitor op during start
   + lrmd: Handle systemd reporting 'done' before a resource is actually stopped
   + lrmd: Hint to child processes that using sd_notify is not required
   + lrmd: Log with the correct personality
   + lrmd: Prevent glib assert triggered by timers being removed from mainloop more than once
   + lrmd: report original timeout when systemd operation completes
   + lrmd: store failed operation exit reason in cib
   + mainloop: resolves race condition mainloop poll involving modification of ipc connections
   + make targetted reprobe for remote node work, crm_resource -C -N <remote node>
   + mcp: Allow a configurable delay when debugging shutdown issues
   + mcp: Avoid requiring 'export' for SYS-V sysconfig options
   + Membership: Detect and resolve nodes that change their ID
   + pacemakerd: resolves memory leak of xml structure in pacemakerd
   + pengine: ability to launch resources in isolated containers
   + pengine: add #kind=remote for baremetal remote-nodes
   + pengine: allow baremetal remote-nodes to recover without requiring fencing when cluster-node fails
   + pengine: allow remote-nodes to be placed in maintenance mode
   + pengine: Avoid trailing whitespaces when printing resource state
   + pengine: cl#5130 - Choose nodes capable of running all the colocated utilization resources
   + pengine: cl#5130 - Only check the capacities of the nodes that are allowed to run the resource
   + pengine: Correctly compare feature set to determine how to unpack meta attributes
   + pengine: disable migrations for resources with isolation containers
   + pengine: disable reloading of resources within isolated container wrappers
   + pengine: Do not aggregate children in a pending state into the started/stopped/etc lists
   + pengine: Do not record duplicate copies of the failed actions
   + pengine: Do not reschedule monitors that are no longer needed while resource definitions have changed
   + pengine: Fence baremetal remote when recurring monitor op fails
   + pengine: Fix colocation with unmanaged resources
   + pengine: Fix the behaviors of multi-state resources with asymmetrical ordering
   + pengine: fixes pengine crash with orphaned remote node connection resource
   + pengine: fixes segfault caused by malformed log warning
   + pengine: handle cloned isolated resources in a sane way
   + pengine: handle isolated resource scenario, cloned group of isolated resources
   + pengine: Handle ordering between stateful and migratable resources
   + pengine: imply stop in container node resources when host node is fenced
   + pengine: only fence baremetal remote when connection can fails or can not be recovered
   + pengine: only kill process group on timeout when on-fail does not equal block.
   + pengine: per-node control over resource discovery
   + pengine: prefer migration target for remote node connections
   + pengine: prevent disabling rsc discovery per node in certain situations
   + pengine: Prevent use-after-free in sort_rsc_process_order()
   + pengine: properly handle ordering during remote connection partial migration
   + pengine: properly recover remote-nodes when cluster-node proxy goes offline
   + pengine: remove unnecessary whitespace from notify environment variables
   + pengine: require-all feature for ordered clones
   + pengine: Resolve memory leaks
   + pengine: resource discovery mode for location constraints
   + pengine: restart master instances on instance attribute changes
   + pengine: Turn off legacy unpacking of resource options into the meta hashtable
   + pengine: Watchdog integration is sufficient for fencing
   + Perform systemd reloads asynchronously
   + ping: Correctly advertise multiplier default
   + Prefer to inherit the  watchdog timeout from SBD
   + properly record stop args after reload
   + provide fake meta data for ra class heartbeat
   + remote: report timestamps for remote connection resource operations
   + remote: Treat recv msg timeout as a disconnect
   + service: Prevent potential use-of-NULL in metadata lookups
   + solaris: Allow compilation when dirent.d_type is not available
   + solaris: Correctly replace the linux swab functions
   + solaris: Disable throttling since /proc doesn't exist
   + stonith-ng: Correctly observe the watchdog completion timeout
   + stonith-ng: Correctly track node state
   + stonith-ng: Reset mainloop source IDs after removing them
   + systemd: Correctly handle long running stop actions
   + systemd: Ensure failed monitor operations always return
   + systemd: Ensure we don't call dbus_message_unref() with NULL
   + systemd: fix crash caused when canceling in-flight operation
   + systemd: Kindly ask dbus NOT to kill the process if the dbus connection fails
   + systemd: Perform actions asynchronously
   + systemd: Perform monitor operations without blocking
   + systemd: Tell systemd not to take DBus down from underneath us
   + systemd: Trick systemd into not stopping our services before us during shutdown
   + tools: Improve crm_mon output with certain option combinations
   + upstart: Monitor actions always return 'ok' or 'not running'
   + upstart: Perform more parts of monitor operations without blocking
   + xml: add 'require-all' to xml schema for constraints
   + xml: cl#5231 - Unset the deleted attributes in the resulting diffs
   + xml: Clone the latest constraint schema in preparation for changes"
   + xml: Correctly create v1 patchsets when deleting attributes
   + xml: Do not change the ordering of properties when applying v1 cib diffs
   + xml: Do not dump deleted attributes
   + xml: Do not prune leaves from v1 cib diffs that are being created with digests
   + xml: Ensure ACLs are reapplied before calculating what a replace operation changed
   + xml: Fix upgrade-1.3.xsl to correctly transform ACL rules with "attribute"
   + xml: Prevent assert errors in crm_element_value() on applying a patch without version information
   + xml: Prevent potential use-of-NULL
 
 
 * Tue Jul 22 2014 Andrew Beekhof <andrew@beekhof.net> Pacemaker-1.1.12
 - Update source tarball to revision: 93a037d
 - Changesets: 795
 - Diff:       195 files changed, 13772 insertions(+), 6176 deletions(-)
 
 - Features added since Pacemaker-1.1.11
   + Changes to the ACL schema to support nodes and unix groups
   + cib: Check ACLs prior to making the update instead of parsing the diff afterwards
   + cib: Default ACL support to on
   + cib: Enable the more efficient xml patchset format
   + cib: Implement zero-copy status update
   + cib: Send all r/w operations via the cluster connection and have all nodes process them
   + crmd: Set "cluster-name" property to corosync's "cluster_name" by default for corosync-2
   + crm_mon: Display brief output if "-b/--brief" is supplied or 'b' is toggled
   + crm_report: Allow ssh alternatives to be used
   + crm_ticket: Support multiple modifications for a ticket in an atomic operation
   + extra: Add logrotate configuration file for /var/log/pacemaker.log
   + Fencing: Add the ability to call stonith_api_time() from stonith_admin
   + logging: daemons always get a log file, unless explicitly set to configured 'none'
   + logging: allows the user to specify a log level that is output to syslog
   + PE: Automatically re-unfence a node if the fencing device definition changes
   + pengine: cl#5174 - Allow resource sets and templates for location constraints
   + pengine: Support cib object tags
   + pengine: Support cluster-specific instance attributes based on rules
   + pengine: Support id-ref in nvpair with optional "name"
   + pengine: Support per-resource maintenance mode
   + pengine: Support site-specific instance attributes based on rules
   + tools: Allow crm_shadow to create older configuration versions
   + tools: Display pending state in crm_mon/crm_resource/crm_simulate if --pending/-j is supplied (cl#5178)
   + xml: Add the ability to have lightweight schema revisions
   + xml: Enable resource sets in location constraints for 1.2 schema
   + xml: Support resources that require unfencing
 
 - Changes since Pacemaker-1.1.11
   + acl: Authenticate pacemaker-remote requests with the node name as the client
   + acl: Read access must be explicitly granted
   + attrd: Ensure attribute dampening is always observed
   + attrd: Remove offline nodes from node cache for "peer-remove" requests
   + Bug cl#5055 - Improved migration support.
   + Bug cl#5184 - Ensure pending probes that ultimately fail are correctly updated
   + Bug cl#5196 - pengine: Check values after expanding templates
   + Bug cl#5212 - Do not promote instances when quorum is lots and no-quorum-policy=freeze
   + Bug cl#5213 - Ensure role colocation with -INFINITY is enforced
   + Bug cl#5213 - Limit the scope of the previous commit to the masters role
   + Bug cl#5219 - pengine: Allow unrelated resources with a common colocation target to remain promoted
   + Bug cl#5222 - cib: Repair rolling update capability
   + Bug cl#5222 - Enable legacy mode whenever a broadcast update is detected
   + Bug rhbz#1036631 - Stop members of cloned groups when dependencies are stopped
   + Bug rhbz#1054307 - cname pattern match should be more restrictive in init script
   + Bug rhbz#1057697 - Use native DBus library for systemd/upstart support to avoid problematic use of threads
   + Bug rhbz#1097457 - Limit the scope of the previous fix and include a helpful comment
   + Bug rhbz#1097457 - Prevent invalid transition when resource are ordered to start after the container they're started in
   + cib: allow setting permanent remote-node attributes
   + cib: Auto-detect which patchset format to use
   + cib: Determine the best value of validate-with if one is not supplied
   + cib: Do not disable cib disk writes if on-disk cib is corrupt
   + cib: Ensure 'cibadmin -R/--replace' commands get replies
   + cib: Erasing the cib is an admin action, bump the admin_epoch instead
   + cib: Fix remote cib based on TLS
   + cib: Ignore patch failures if we already have their contents
   + cib: Validate that everyone still sees the same configuration once all updates have completed
   + cibadmin: Allow priviliged clients to perform tasks as unpriviliged users
   + cibadmin: Remove dangerous commands that exposed unnecessary implementation internal details
   + cluster: Fix segfault on removing a node
   + cluster: Prevent search of unames from attempting to create node entries for unknown nodes
   + cluster: Remove unknown offline nodes with conflicting unames from node cache
   + controld: Do not consider the dlm up until the address list is present
   + controld: handling startup fencing within the controld agent, not the dlm
   + controld: Return OCF_ERR_INSTALLED instead of OCF_NOT_INSTALLED
   + crmd: Ack pending operations that were cancelled due to rsc deletion
   + crmd: Actions can only be executed if their pre-requisits completed successfully
   + crmd: avoid double free caused by nested hash table removal
   + crmd: Avoid spamming the cib by triggering a transition only once per non-status change
   + crmd: Correctly react to successful unfencing operations
   + crmd: Correctly recognise operation cancellations we initiated
   + crmd: Do not erase the status section for unfenced nodes
   + crmd: Do not overwrite existing node state when fencing completes
   + crmd: Do not start timers for already completed operations
   + crmd: Ensure crm_config options are re-read on updates
   + crmd: Fenced nodes that return prior to an election do not need to have their status section reset
   + crmd: make lrm_state hash table not case sensitive
   + crmd: make node_state erase correctly
   + crmd: Only write fence_averride if open() returns a positive file descriptor
   + crmd: Prevent manual fencing confirmations from attempting to create node entries for unknown nodes
   + crmd: Prevent SIGPIPE when notifying CMAN about fencing operations
   + crmd: Remove state of unknown nodes with conflicting unames from CIB
   + crmd: Remove unknown nodes with conflicting unames from CIB
   + crmd: Report unsuccessful unfencing operations
   + crm_diff: Allow the generation of xml patchsets without digests
   + crm_mon: Allow the file created by --as-html to be world readable
   + crm_mon: Ensure resource attributes have been unpacked before displaying connectivity data
   + crm_node: Only remove the named resource from the cib
   + crm_report: Gracefully handle rediculously large logfiles
   + crm_report: Only gather dlm data if dlm_controld is running
   + crm_resource: Gracefully handle -EACCESS when querying the cib
   + crm_verify: Perform a full set of calculations whenever the status section is present
   + fencing: Advertise support for reboot/on/off in the metadata for legacy agents
   + fencing: Automatically switch from 'list' to 'status' to 'static-list' if those actions are not advertised in the metadata
   + fencing: Cache metadata lookups to avoid repeated blocking during device registration
   + fencing: Correctly record which peer performed the fencing operation
   + fencing: default to 'off' when agent does not advertise 'reboot' in metadata
   + fencing: Do not unregister/register all stonith devices on every resource agent change
   + fencing: Execute all required fencing devices regardless of what topology level they are at
   + fencing: Fence using all required devices
   + fencing: Pass the correct options when looking up the history by node name
   + fencing: Update stonith device list only if stonith is enabled
   + get_cluster_type: failing concurrent tool invocations on heartbeat
   + ignore SIGPIPE when gnutls is in use
   + iso8601: Different logic is needed when logging and calculating durations
   + iso8601: Fix memory leak in duration calculation
   + Logging: Bootstrap daemon logging before processing arguments but configure it afterwards
   + lrmd: Cancel recurring operations before stop action is executed
   + lrmd: Expose logging variables expected by OCF agents
   + lrmd: Handle systemd reporting 'done' before a resource is actually stopped/started
   + lrmd: Merge duplicate recurring monitor operations
   + lrmd: Prevent OCF agents from logging to random files due to "value" of setenv() being NULL
   + lrmd: Provide stderr output from agents if available, otherwise fall back to stdout
   + mainloop: Better handle the killing of processes in the act of exiting
   + mainloop: Canceling in-flight operations should not fail if child process has already exited.
   + mainloop: Fixes use after free in process monitor code
   + mcp: Tell systemd not to respawn us if we exit with rc=100
   + membership: Avoid duplicate peer entries in the peer cache
   + pengine: Allow container nodes to migrate with connection resource
   + pengine: avoid assert by searching for stop action on correct node during LogActions
   + pengine: Block restart of resources if any dependent resource in a group is unmanaged
   + pengine: cl#5186 - Avoid running rsc on two nodes when node is fenced during migration
   + pengine: cl#5187 - Prevent resources in an anti-colocation from even temporarily running on a same node
   + pengine: cl#5200 - Before migrating utilization-using resources to a node, take off the load that will no longer run there if it's not introducing transition loop
   + pengine: Correctly handle origin offsets in the future
   + pengine: Correctly observe requires=nothing
   + pengine: Default sequential to TRUE for resource sets for consistency with colocation sets
   + pengine: Delay unfencing until after we know the state of all resources that require unfencing
   + pengine: Do not initiate fencing for unclean nodes when fencing is disabled
   + pengine: Ensure instance numbers are preserved for cloned templates
   + pengine: Ensure unfencing only happens once, even if the transition is interrupted
   + pengine: Fencing devices default to only requiring quorum in order to start
   + pengine: fixes invalid transition caused by clones with more than 10 instances
   + pengine: Force record pending for migrate_to actions
   + pengine: handles edge case where container order constraints are not honored during migration
   + pengine: Ignore failure-timeout only if the failed operation has on-fail="block"
   + pengine: Mark unrunnable stop actions as "blocked" and show the correct current locations
   + pengine: Memory leaks
   + pengine: properly handle fencing of container remote-nodes when the container is orphaned
   + pengine: properly place resource within a container when container is a remote-node.
   + pengine: Unfencing is based on device probes, there is no need to unfence when normal resources are found active
   + pengine: Use "#cluster-name" in rules for setting cluster-specific instance attributes
   + pengine: Use "#site-name" in rules for setting site-specific instance attributes
   + remote: Allow baremetal remote-node connection resources to migrate
   + remote: clear remote-node status correctly
   + remote: Enable migration support for baremetal connection resources by default
   + remote: Handle request/response ipc proxy correctly
   + services: Correctly reset the nice value for lrmd's children
   + services: Do not allow duplicate recurring op entries
   + services: Do not block synced service executions
   + services: Fixes segfault associated with cancelling in-flight recurring operations.
   + services: Remove cancelled recurring ops from internal lists as early as possible
   + services: Remove file descriptors from mainloop as soon as we have drained them
   + services: Reset the scheduling policy and priority for lrmd's children without replying on SCHED_RESET_ON_FORK
   + services_action_cancel: Interpret return code from mainloop_child_kill() correctly
   + stonith_admin: Ensure pointers passed to sscanf() are properly initialized
   + stonith_api_time_helper now returns when the most recent fencing operation completed
   + systemd: Prevent use-of-NULL when determining if an agent exists
   + systemd: Try to handle dbus actions that complete prior to configuring a callback
   + Tools: Non-daemons shouldn't abort just because xml parsing failed
   + Upstart: Allow comilation with glib versions older than 2.28
   + Upstart: Do not attempt upstart jobs if we cannot connect to dbus
   + When data was old, it fixed so that the newest cib might not be acquired.
   + xml: Check all available schemas when doing upgrades
   + xml: Correctly determine the lowest allowed schema version
   + xml: Correctly enforce ACLs after a replace operation
   + xml: Correctly infer attribute changes after a replace operation
   + xml: Create the correct diff when only part of a document is changed
   + xml: Detect attribute ordering changes
   + xml: Detect content that is added and removed in the same update
   + xml: Do not prune meaningful leaves from v1 patchsets
   + xml: Empty patchsets are considered to have applied cleanly
   + xml: Ensure patches always have version details set
   + xml: Find the minimal set of changes when part of a document is replaced
   + xml: If validate-with is missing, we find the most recent schema that accepts it and go from there
   + xml: Introduce a 'move' primitive for v2 patch sets
   + xml: Preserve the attribute order in the patch for subsequent digest validation
   + xml: Resolve memory leak when logging xml blobs
   + xml: Update xml validation to allow '<node type=remote />'
 
 
 * Thu Feb 13 2014 David Vossel <davidvossel@gmail.com> Pacemaker-1.1.11
 - Update source tarball to revision: 33f9d09
 - Changesets: 462
 - Diff:       147 files changed, 6810 insertions(+), 4057 deletions(-)
 
 - Features added since Pacemaker-1.1.10
 
   + attrd: A truly atomic version of attrd for use where CPG is used for cluster communication
   + cib: Allow values to be added/updated and removed in a single update
   + cib: Support XML comments in diffs
   + Core: Allow blackbox logging to be disabled with SIGUSR2
   + crmd: Do not block on proxied calls from pacemaker_remoted
   + crmd: Enable cluster-wide throttling when the cib heavily exceeds its target load
   + crmd: Make the per-node action limit directly configurable in the CIB
   + crmd: Slow down recovery on nodes with IO load
   + crmd: Track CPU usage on cluster nodes and slow down recovery on nodes with high CPU/IO load
   + crm_mon: add --hide-headers option to hide all headers
   + crm_node: Display partition output in sorted order
   + crm_report: Collect logs directly from journald if available
   + Fencing: On timeout, clean up the agent's entire process group
   + Fencing: Support agents that need the host to be unfenced at startup
   + ipc: Raise the default buffer size to 128k
   + PE: Add a special attribute for distinguishing between real nodes and containers in constraint rules
   + PE: Allow location constraints to take a regex pattern to match against resource IDs
   + pengine: Distinguish between the agent being missing and something the agent needs being missing
   + remote: Properly version the remote connection protocol
 
 - Changes since Pacemaker-1.1.10
 
   + Bug rhbz#1011618 - Consistently use 'Slave' as the role for unpromoted master/slave resources
   + Bug rhbz#1057697 - Use native DBus library for systemd and upstart support to avoid problematic use of threads
   + attrd: Any variable called 'cluster' makes the daemon crash before reaching main()
   + attrd: Avoid infinite write loop for unknown peers
   + attrd: Drop all attributes for peers that left the cluster
   + attrd: Give remote-nodes ability to set attributes with attrd
   + attrd: Prevent inflation of attribute dampen intervals
   + attrd: Support SI units for attribute dampening
   + Bug cl#5171 - pengine: Don't prevent clones from running due to dependent resources
   + Bug cl#5179 - Corosync: Attempt to retrieve a peer's node name if it is not already known
   + Bug cl#5181 - corosync: Ensure node IDs are written to the CIB as unsigned integers
   + Bug rhbz#902407 - crm_resource: Handle --ban for master/slave resources as advertised
   + cib: Correctly check for archived configuration files
   + cib: Correctly log short-form xml diffs
   + cib: Fix remote cib based on TLS
   + cibadmin: Report errors during sign-off
   + cli: Do not enabled blackbox for cli tools
   + cluster: Fix segfault on removing a node
   + cman: Do not start pacemaker if cman startup fails
   + cman: Start clvmd and friends from the init script if enabled
   + Command-line tools should stop after an assertion failure
   + controld: Use the correct variant of dlm_controld for corosync-2 clusters
   + cpg: Correctly set the group name length
   + cpg: Ensure the CPG group is always null-terminated
   + cpg: Only process one message at a time to allow other priority jobs to be performed
   + crmd: Correctly observe the configured batch-limit
   + crmd: Correctly update expected state when the previous DC shuts down
   + crmd: Correcty update the history cache when recurring ops change their return code
   + crmd: Don't add node_state to cib, if we have not seen or fenced this node yet
   + crmd: don't segfault on shutdown when using heartbeat
   + crmd: Prevent recurring monitors being cancelled due to notify operations
   + crmd: Reliably detect and act on reprobe operations from the policy engine
   + crmd: When a peer expectedly shuts down, record the new join and expected states into the cib
   + crmd: When the DC gracefully shuts down, record the new expected state into the cib
   + crm_attribute: Do not swallow hostname lookup failures
   + crm_mon: Do not display duplicates of failed actions
   + crm_mon: Reduce flickering in interactive mode
   + crm_resource: Observe --master modifier for --move
   + crm_resource: Provide a meaningful error if --master is used for primitives and groups
   + fencing: Allow fencing for node after topology entries are deleted
   + fencing: Apply correct score to the resource of group
   + fencing: Ignore changes to non-fencing resources
   + fencing: Observe pcmk_host_list during automatic unfencing
   + fencing: Put all fencing agent processes into their own process group
   + fencing: Wait until all possible replies are recieved before continuing with unverified devices
   + ipc: Compress msgs based on client's actual max send size
   + ipc: Have the ipc server enforce a minimum buffer size all clients must use.
   + iso8601: Prevent dates from jumping backwards a day in some timezones
   + lrmd: Correctly calculate metadata for the 'service' class
   + lrmd: Correctly cancel monitor actions for lsb/systemd/service resources on cleaning up
   + mcp: Remove LSB hints that instruct chkconfig to start pacemaker at boot time
   + mcp: Some distros complain when LSB scripts do not include Default-Start/Stop directives
   + pengine: Allow fencing of baremetal remote nodes
   + pengine: cl#5186 - Avoid running rsc on two nodes when node is fenced during migration
   + pengine: Correctly account for the location preferences of things colocated with a group
   + pengine: Correctly handle demotion of grouped masters that are partially demoted
   + pengine: Disable container node probes due to constraint conflicts
   + pengine: Do not allow colocation with blocked clone instances
   + pengine: Do not re-allocate clone instances that are blocked in the Stopped state
   + pengine: Do not restart resources that depend on unmanaged resources
   + pengine: Force record pending for migrate_to actions
   + pengine: Location constraints with role=Started should prevent masters from running at all
   + pengine: Order demote/promote of resources on remote nodes to happen only once the connection is up
   + pengine: Properly handle orphaned multistate resources living on remote-nodes
   + pengine: Properly shutdown orphaned remote connection resources
   + pengine: Recover unexpectedly running container nodes.
   + remote: Add support for ipv6 into pacemaker_remote daemon
   + remote: Handle endian changes between client and server and improve forward compatibility
   + services: Fixes segfault associated with cancelling in-flight recurring operations.
   + services: Reset the scheduling policy and priority for lrmd's children without replying on SCHED_RESET_ON_FORK
 
 * Fri Jul 26 2013 Andrew Beekhof <andrew@beekhof.net> Pacemaker-1.1.10
 - Update source tarball to revision: ab2e209
 - Changesets: 602
 - Diff:       143 files changed, 8162 insertions(+), 5159 deletions(-)
 
 - Features added since Pacemaker-1.1.9
   + Core: Convert all exit codes to positive errno values
   + crm_error: Add the ability to list and print error symbols
   + crm_resource: Allow individual resources to be reprobed
   + crm_resource: Allow options to be set recursively
   + crm_resource: Implement --ban for moving resources away from nodes and --clear (replaces --unmove)
   + crm_resource: Support OCF tracing when using --force-(check|start|stop)
   + PE: Allow active nodes in our current membership to be fenced without quorum
   + PE: Suppress meaningless IDs when displaying anonymous clone status
   + Turn off auto-respawning of systemd services when the cluster starts them
   + Bug cl#5128 - pengine: Support maintenance mode for a single node
 
 - Changes since Pacemaker-1.1.9
   + crmd: cib: stonithd: Memory leaks resolved and improved use of glib reference counting
   + attrd: Fixes deleted attributes during dc election
   + Bug cf#5153 - Correctly display clone failcounts in crm_mon
   + Bug cl#5133 - pengine: Correctly observe on-fail=block for failed demote operation
   + Bug cl#5148 - legacy: Correctly remove a node that used to have a different nodeid
   + Bug cl#5151 - Ensure node names are consistently compared without case
   + Bug cl#5152 - crmd: Correctly clean up fenced nodes during membership changes
   + Bug cl#5154 - Do not expire failures when on-fail=block is present
   + Bug cl#5155 - pengine: Block the stop of resources if any depending resource is unmanaged
   + Bug cl#5157 - Allow migration in the absence of some colocation constraints
   + Bug cl#5161 - crmd: Prevent memory leak in operation cache
   + Bug cl#5164 - crmd: Fixes crash when using pacemaker-remote
   + Bug cl#5164 - pengine: Fixes segfault when calculating transition with remote-nodes.
   + Bug cl#5167 - crm_mon: Only print "stopped" node list for incomplete clone sets
   + Bug cl#5168 - Prevent clones from being bounced around the cluster due to location constraints
   + Bug cl#5170 - Correctly support on-fail=block for clones
   + cib: Correctly read back archived configurations if the primary is corrupted
   + cib: The result is not valid when diffs fail to apply cleanly for CLI tools
   + cib: Restore the ability to embed comments in the configuration
   + cluster: Detect and warn about node names with capitals
   + cman: Do not pretend we know the state of nodes we've never seen
   + cman: Do not unconditionally start cman if it is already running
   + cman: Support non-blocking CPG calls
   + Core: Ensure the blackbox is saved on abnormal program termination
   + corosync: Detect the loss of members for which we only know the nodeid
   + corosync: Do not pretend we know the state of nodes we've never seen
   + corosync: Ensure removed peers are erased from all caches
   + corosync: Nodes that can persist in sending CPG messages must be alive afterall
   + crmd: Do not get stuck in S_POLICY_ENGINE if a node we couldn't fence returns
   + crmd: Do not update fail-count and last-failure for old failures
   + crmd: Ensure all membership operations can complete while trying to cancel a transition
   + crmd: Ensure operations for cleaned up resources don't block recovery
   + crmd: Ensure we return to a stable state if there have been too many fencing failures
   + crmd: Initiate node shutdown if another node claims to have successfully fenced us
   + crmd: Prevent messages for remote crmd clients from being relayed to wrong daemons
   + crmd: Properly handle recurring monitor operations for remote-node agent
   + crmd: Store last-run and last-rc-change for all operations
   + crm_mon: Ensure stale pid files are updated when a new process is started
   + crm_report: Correctly collect logs when 'uname -n' reports fully qualified names
   + fencing: Fail the operation once all peers have been exhausted
   + fencing: Restore the ability to manually confirm that fencing completed
   + ipc: Allow unpriviliged clients to clean up after server failures
   + ipc: Restore the ability for members of the haclient group to connect to the cluster
   + legacy: Support "crm_node --remove" with a node name for corosync plugin (bnc#805278)
   + lrmd: Default to the upstream location for resource agent scratch directory
   + lrmd: Pass errors from lsb metadata generation back to the caller
   + pengine: Correctly handle resources that recover before we operate on them
   + pengine: Delete the old resource state on every node whenever the resource type is changed
   + pengine: Detect constraints with inappropriate actions (ie. promote for a clone)
   + pengine: Ensure per-node resource parameters are used during probes
   + pengine: If fencing is unavailable or disabled, block further recovery for resources that fail to stop
   + pengine: Implement the rest of get_timet_now() and rename to get_effective_time
   + pengine: Re-initiate _active_ recurring monitors that previously failed but have timed out
   + remote: Workaround for inconsistent tls handshake behavior between gnutls versions
   + systemd: Ensure we get shut down correctly by systemd
   + systemd: Reload systemd after adding/removing override files for cluster services
   + xml: Check for and replace non-printing characters with their octal equivalent while exporting xml text
   + xml: Prevent lockups by setting a more reliable buffer allocation strategy
 
 * Fri Mar 08 2013 Andrew Beekhof <andrew@beekhof.net> Pacemaker-1.1.9
 - Update source tarball to revision: 7e42d77
 - Statistics:
   Changesets: 731
   Diff:       1301 files changed, 92909 insertions(+), 57455 deletions(-)
 
 - Features added in Pacemaker-1.1.9
   + corosync: Allow cman and corosync 2.0 nodes to use a name other than uname()
   + corosync: Use queues to avoid blocking when sending CPG messages
   + ipc: Compress messages that exceed the configured IPC message limit
   + ipc: Use queues to prevent slow clients from blocking the server
   + ipc: Use shared memory by default
   + lrmd: Support nagios remote monitoring
   + lrmd: Pacemaker Remote Daemon for extending pacemaker functionality outside corosync cluster.
   + pengine: Check for master/slave resources that are not OCF agents
   + pengine: Support a 'requires' resource meta-attribute for controlling whether it needs quorum, fencing or nothing
   + pengine: Support for resource container
   + pengine: Support resources that require unfencing before start
 
 - Changes since Pacemaker-1.1.8
   + attrd: Correctly handle deletion of non-existant attributes
   + Bug cl#5135 - Improved detection of the active cluster type
   + Bug rhbz#913093 - Use crm_node instead of uname
   + cib: Avoid use-after-free by correctly support cib_no_children for non-xpath queries
   + cib: Correctly process XML diff's involving element removal
   + cib: Performance improvements for non-DC nodes
   + cib: Prevent error message by correctly handling peer replies
   + cib: Prevent ordering changes when applying xml diffs
   + cib: Remove text nodes from cib replace operations
   + cluster: Detect node name collisions in corosync
   + cluster: Preserve corosync membership state when matching node name/id entries
   + cman: Force fenced to terminate on shutdown
   + cman: Ignore qdisk 'nodes'
   + core: Drop per-user core directories
   + corosync: Avoid errors when closing failed connections
   + corosync: Ensure peer state is preserved when matching names to nodeids
   + corosync: Clean up CMAP connections after querying node name
   + corosync: Correctly detect corosync 2.0 clusters even if we don't have permission to access it
   + crmd: Bug cl#5144 - Do not updated the expected status of failed nodes
   + crmd: Correctly determin if cluster disconnection was abnormal
   + crmd: Correctly relay messages for remote clients (bnc#805626, bnc#804704)
   + crmd: Correctly stall the FSA when waiting for additional inputs
   + crmd: Detect and recover when we are evicted from CPG
   + crmd: Differentiate between a node that is up and coming up in peer_update_callback()
   + crmd: Have cib operation timeouts scale with node count
   + crmd: Improved continue/wait logic in do_dc_join_finalize()
   + crmd: Prevent election storms caused by getrusage() values being too close
   + crmd: Prevent timeouts when performing pacemaker level membership negotiation
   + crmd: Prevent use-after-free of fsa_message_queue during exit
   + crmd: Store all current actions when stalling the FSA
   + crm_mon: Do not try to render a blank cib and indicate the previous output is now stale
   + crm_mon: Fixes crm_mon crash when using snmp traps.
   + crm_mon: Look for the correct error codes when applying configuration updates
   + crm_report: Ensure policy engine logs are found
   + crm_report: Fix node list detection
   + crm_resource: Have crm_resource generate a valid transition key when sending resource commands to the crmd
   + date/time: Bug cl#5118 - Correctly convert seconds-since-epoch to the current time
   + fencing: Attempt to provide more information that just 'generic error' for failed actions
   + fencing: Correctly record completed but previously unknown fencing operations
   + fencing: Correctly terminate when all device options have been exhausted
   + fencing: cov#739453 - String not null terminated
   + fencing: Do not merge new fencing requests with stale ones from dead nodes
   + fencing: Do not start fencing until entire device topology is found or query results timeout.
   + fencing: Do not wait for the query timeout if all replies have arrived
   + fencing: Fix passing of parameters from CMAN containing '='
   + fencing: Fix non-comparison when sorting devices by priority
   + fencing: On failure, only try a topology device once from the remote level.
   + fencing: Only try peers for non-topology based operations once
   + fencing: Retry stonith device for duration of action's timeout period.
   + heartbeat: Remove incorrect assert during cluster connect
   + ipc: Bug cl#5110 - Prevent 100% CPU usage when looking for synchronous replies
   + ipc: Use 50k as the default compression threshold
   + legacy: Prevent assertion failure on routing ais messages (bnc#805626)
   + legacy: Re-enable logging from the pacemaker plugin
   + legacy: Relax the 'active' check for plugin based clusters to avoid false negatives
   + legacy: Skip peer process check if the process list is empty in crm_is_corosync_peer_active()
   + mcp: Only define HA_DEBUGLOG to avoid agent calls to ocf_log printing everything twice
   + mcp: Re-attach to existing pacemaker components when mcp fails
   + pengine: Any location constraint for the slave role applies to all roles
   + pengine: Avoid leaking memory when cleaning up failcounts and using containers
   + pengine: Bug cl#5101 - Ensure stop order is preserved for partially active groups
   + pengine: Bug cl#5140 - Allow set members to be stopped when the subseqent set has require-all=false
   + pengine: Bug cl#5143 - Prevent shuffling of anonymous master/slave instances
   + pengine: Bug rhbz#880249 - Ensure orphan masters are demoted before being stopped
   + pengine: Bug rhbz#880249 - Teach the PE how to recover masters into primitives
   + pengine: cl#5025 - Automatically clear failcount for start/monitor failures after resource parameters change
   + pengine: cl#5099 - Probe operation uses the timeout value from the minimum interval monitor by default (#bnc776386)
   + pengine: cl#5111 - When clone/master child rsc has on-fail=stop, insure all children stop on failure.
   + pengine: cl#5142 - Do not delete orphaned children of an anonymous clone
   + pengine: Correctly unpack active anonymous clones
   + pengine: Ensure previous migrations are closed out before attempting another one
   + pengine: Introducing the whitebox container resources feature
   + pengine: Prevent double-free for cloned primitive from template
   + pengine: Process rsc_ticket dependencies earlier for correctly allocating resources (bnc#802307)
   + pengine: Remove special cases for fencing resources
   + pengine: rhbz#902459 - Remove rsc node status for orphan resources
   + systemd: Gracefully handle unexpected DBus return types
   + Replace the use of the insecure mktemp(3) with mkstemp(3)
 
 * Thu Sep 20 2012 Andrew Beekhof <andrew@beekhof.net> Pacemaker-1.1.8
 
 - Update source tarball to revision: 1a5341f
 - Statistics:
   Changesets: 1019
   Diff:       2107 files changed, 117258 insertions(+), 73606 deletions(-)
 
 - All APIs have been cleaned up and reduced to essentials
 - Pacemaker now includes a replacement lrmd that supports systemd and upstart agents
 - Config and state files (cib.xml, PE inputs and core files) have moved to new locations
 - The crm shell has become a separate project and no longer included with Pacemaker
 - All daemons/tools now have a unified set of error codes based on errno.h (see crm_error)
 
 - Changes since Pacemaker-1.1.7
   + Core: Bug cl#5032 - Rewrite the iso8601 date handling code
   + Core: Correctly extract the version details from a diff
   + Core: Log blackbox contents, if enabled, when an error occurs
   + Core: Only LOG_NOTICE and higher are sent to syslog
   + Core: Replace use of IPC from clplumbing with IPC from libqb
   + Core: SIGUSR1 now enables blackbox logging, SIGTRAP to write out
   + Core: Support a blackbox for additional logging detail after crashes/errors
   + Promote support for advanced fencing logic to the stable schema
   + Promote support for node starting scores to the stable schema
   + Promote support for service and systemd to the stable schema
 
   + attrd: Differentiate between updating all our attributes and everybody updating all theirs too
   + attrd: Have single-shot clients wait for an ack before disconnecting
   + cib: cl#5026 - Synced cib updates should not return until the cpg broadcast is complete.
   + corosync: Detect when the first corosync has not yet formed and handle it gracefully
   + corosync: Obtain a full list of configured nodes, including their names, when we connect to the quorum API
   + corosync: Obtain a node name from DNS if one was not already known
   + corosync: Populate the cib nodelist from corosync if available
   + corosync: Use the CFG API and DNS to determine node names if not configured in corosync.conf
   + crmd: Block after 10 failed fencing attempts for a node
   + crmd: cl#5051 - Fixes file leak in PE ipc connection initialization.
   + crmd: cl#5053 - Fixes fail-count not being updated properly.
   + crmd: cl#5057 - Restart sub-systems correctly (bnc#755671)
   + crmd: cl#5068 - Fixes crm_node -R option so it works with corosync 2.0
   + crmd: Correctly re-establish failed attrd connections
   + crmd: Detect when the quorum API isn't configured for corosync 2.0
   + crmd: Do not overwrite any configured node type (eg. quorum node)
   + crmd: Enable use of new lrmd daemon and client library in crmd.
   + crmd: Overhaul the way node state is recorded and updated in the CIB
   + fencing: Bug rhbz#853537 - Prevent use-of-NULL when the cib libraries are not available
   + fencing: cl#5073 - Add 'off' as an valid value for stonith-action option.
   + fencing: cl#5092 - Always timeout stonith operations if timeout period expires.
   + fencing: cl#5093 - Stonith per device timeout option
   + fencing: Clean up if we detect a failed connection
   + fencing: Delegate complex self fencing requests - we wont be around to see it to completion
   + fencing: Ensure all peers are notified of complex fencing op completion
   + fencing: Fix passing of fence_legacy parameters containing '='
   + fencing: Gracefully handle metadata requests for unknown agents
   + fencing: Return cached dynamic target list for busy devices.
   + fencing: rhbz#801355 - Abort transition on DC when external fencing operation is detected
   + fencing: rhbz#801355 - Merge fence requests for identical operations already in progress.
   + fencing: rhbz#801355 - Report fencing operations external of pacemaker to cib
   + fencing: Specify the action to perform using action= instead of the older option=
   + fencing: Stop building fake metadata for broken agents
   + fencing: Tolerate agents that report empty metadata in the admin tool
   + mcp: Correctly retry the connection to corosync on failure
   + mcp: Do not shut down IPC until the last client exits
   + mcp: Prevent use-after-free when running against corosync 1.x
   + pengine: Bug cl#5059 - Use the correct action's status when calculating required actions for interleaved clones
   + pengine: Bypass online/offline checking resource detection for ping/quorum nodes
   + pengine: cl#5044 - migrate_to no longer requires load_stopped for avoiding possible transition loop
   + pengine: cl#5069 - Honor 'on-fail=ignore' even when operation is disabled.
   + pengine: cl#5070 - Allow influence of promotion score when multistate rsc is left hand of colocation
   + pengine: cl#5072 - Fixes monitor op stopping after rsc promotion.
   + pengine: cl#5072 - Fixes pengine regression test failures
   + pengine: Correctly set the status for nodes not intended to run Pacemaker
   + pengine: Do not append instance numbers to anonymous clones
   + pengine: Fix failcount expiration
   + pengine: Fix memory leaks found by valgrind
   + pengine: Fix use-after-free and use-of-NULL errors detected by coverity
   + pengine: Fixes use of colocation scores other than +/- INFINITY
   + pengine: Improve detection of rejoining nodes
   + pengine: Prevent use-of-NULL when tracing is enabled
   + pengine: Stonith resources are allowed to start even if their probes haven't completed on partially active nodes
   + services: New class called 'service' which expands to the correct (LSB/systemd/upstart) standard
   + services: Support Asynchronous systemd/upstart actions
   + Tools: crm_shadow - Bug cl#5062 - Correctly set argv[0] when forking a shell process
   + Tools: crm_report: Always include system logs (if we can find them)
 
 * Wed Mar 28 2012 Andrew Beekhof <andrew@beekhof.net> Pacemaker-1.1.7
 - Update source tarball to revision: bc7ff2c
 - Statistics:
   Changesets: 513
   Diff:       1171 files changed, 90472 insertions(+), 19368 deletions(-)
 
 - Changes since Pacemaker-1.1.6.1
   + ais: Prepare for corosync versions using IPC from libqb
   + cib: Correctly shutdown in the presence of peers without relying on timers
   + cib: Don't halt disk writes if the previous digest is missing
   + cib: Determine when there are no peers to respond to our shutdown request and exit
   + cib: Ensure no additional messages are processed after we begin terminating
   + Cluster: Hook up the callbacks to the corosync quorum notifications
   + Core: basename() may modify its input, do not pass in a constant
   + Core: Bug cl#5016 - Prevent failures in recurring ops from being lost
   + Core: Bug rhbz#800054 - Correctly retrieve heartbeat uuids
   + Core: Correctly determine when an XML file should be decompressed
   + Core: Correctly track the length of a string without reading from uninitialzied memory (valgrind)
   + Core: Ensure signals are handled eventually in the absense of timer sources or IPC messages
   + Core: Prevent use-of-NULL in crm_update_peer()
   + Core: Strip text nodes from on disk xml files
   + Core: Support libqb for logging
   + corosync: Consistently set the correct uuid with get_node_uuid()
   + Corosync: Correctly disconnect from corosync variants
   + Corosync: Correctly extract the node id from membership udpates
   + corosync: Correctly infer lost members from the quorum API
   + Corosync: Default to using the nodeid as the node's uuid (instead of uname)
   + corosync: Ensure we catch nodes that leave the membership, even if the ringid doesn't change
   + corosync: Hook up CPG membership
   + corosync: Relax a development assert and gracefully handle the error condition
   + corosync: Remove deprecated member of the CFG API
   + corosync: Treat CS_ERR_QUEUE_FULL the same as CS_ERR_TRY_AGAIN
   + corosync: Unset the process list when nodes dissappear on us
   + crmd: Also purge fencing results when we enter S_NOT_DC
   + crmd: Bug cl#5015 - Remove the failed operation as well as the resulting fail-count and last-failure attributes
   + crmd: Correctly determine when a node can suicide with fencing
   + crmd: Election - perform the age comparison only once
   + crmd: Fast-track shutdown if we couldn't request it via attrd
   + crmd: Leave it up to the PE to decide which ops can/cannot be reload
   + crmd: Prevent use-after-free when calling delete_resource due to CRM_OP_REPROBE
   + crmd: Supply format arguments in the correct order
   + fencing: Add missing format parameter
   + fencing: Add the fencing topology section to the 1.1 configuration schema
   + fencing: fence_legacy - Drop spurilous host argument from status query
   + fencing: fence_legacy - Ensure port is available as an environment variable when calling monitor
   + fencing: fence_pcmk - don't block if nothing is specified on stdin
   + fencing: Fix log format error
   + fencing: Fix segfault caused by passing garbage to dlsym()
   + fencing: Fix use-of-NULL in process_remote_stonith_query()
   + fencing: Fix use-of-NULL when listing installed devices
   + fencing: Implement support for advanced fencing topologies: eg. kdump || (network && disk) || power
   + fencing: More gracefully handle failed 'list' operations for devices that only support a single connection
   + fencing: Prevent duplicate free when listing devices
   + fencing: Prevent uninitialized pointers being passed to free
   + fencing: Prevent use-after-free, we may need the query result for subsequent operations
   + fencing: Provide enough data to construct an entry in the node's fencing history
   + fencing: Standardize on /one/ method for clients to request members be fenced
   + fencing: Supress errors when listing all registered devices
   + mcp: corosync_cfg_state_track was removed from the corosync API, luckily we didnt use it for anything
   + mcp: Do not specify a WorkingDirectory in the systemd unit file - startup fails if its not available
   + mcp: Set the HA_quorum_type env variable consistently with our corosync plugin
   + mcp: Shut down if one of our child processes can/should not be respawned
   + pengine: Bug cl#5000 - Ensure ordering is preserved when depending on partial sets
   + pengine: Bug cl#5028 - Unmanaged services should block shutdown unless in maintenance mode
   + pengine: Bug cl#5038 - Prevent restart of anonymous clones when clone-max decreases
   + pengine: Bug cl#5007 - Fixes use of colocation constraints with multi-state resources
   + pengine: Bug cl#5014 - Prevent asymmetrical order constraints from causing resource stops
   + pengine: Bug cl#5000 - Implements ability to create rsc_order constraint sets such that A can start after B or C has started.
   + pengine: Correctly migrate a resource that has just migrated
   + pengine: Correct return from error path
   + pengine: Detect reloads of previously migrated resources
   + pengine: Ensure post-migration stop actions occur before node shutdown
   + pengine: Log as loudly as possible when we cannot shut down a cluster node
   + pengine: Reload of a resource no longer causes a restart of dependent resources
   + pengine: Support limiting the number of concurrent live migrations
   + pengine: Support referencing templates in constraints
   + pengine: Support of referencing resource templates in resource sets
   + pengine: Support to make tickets standby for relinquishing tickets gracefully
   + stonith: A "start" operation of a stonith resource does a "monitor" on the device beyond registering it
   + stonith: Bug rhbz#745526 - Ensure stonith_admin actually gets called by fence_pcmk
   + Stonith: Ensure all nodes receive and deliver notifications of the manual override
   + stonith: Fix the stonith timeout issue (cl#5009, bnc#727498)
   + Stonith: Implement a manual override for when nodes are known to be safely off
   + Tools: Bug cl#5003 - Prevent use-after-free in crm_simlate
   + Tools: crm_mon - Support to display tickets (based on Yuusuke Iida's work)
   + Tools: crm_simulate - Support to grant/revoke/standby/activate tickets from the new ticket state section
   + Tools: Implement crm_node functionality for native corosync
   + Fix a number of potential problems reported by coverity
 
 * Wed Aug 31 2011 Andrew Beekhof <andrew@beekhof.net> 1.1.6
 - Update source tarball to revision: 676e5f25aa46 tip
 - Statistics:
   Changesets: 376
   Diff:       1761 files changed, 36259 insertions(+), 140578 deletions(-)
 
 - Changes since Pacemaker-1.1.5
   + ais: check for retryable errors when dispatching AIS messages
   + ais: Correctly disconnect from Corosync and Cman based clusters
   + ais: Followup to previous patch - Ensure we drain the corosync queue of messages when Glib tells us there is input
   + ais: Handle IPC error before checking for NULL data (bnc#702907)
   + cib: Check the validation version before adding the originator details of a CIB change
   + cib: Remove disconnected remote connections from mainloop
   + cman: Correctly override existing fenced operations
   + cman: Dequeue all the cman emitted events and not only the first one leaving the others in the event's queue.
   + cman: Don't call fenced_join and fenced_leave when notifying cman of a fencing event.
   + cman: We need to run the crmd as root for CMAN so that we can ACK fencing operations
   + Core: Cancelled and pending operations do not count as failed
   + Core: Ensure there is sufficient space for EOS when building short-form option strings
   + Core: Fix variable expansion in pkg-config files
   + Core: Partial revert of accidental commit in previous patch
   + Core: Use dlopen to load heartbeat libraries on-demand
   + crmd: Bug lf#2509 - Watch for config option changes from the CIB even if we're not the DC
   + crmd: Bug lf#2528 - Introduce a slight delay when creating a transition to allow attrd time to perform its updates
   + crmd: Bug lf#2559 - Fail actions that were scheduled for a failed/fenced node
   + crmd: Bug lf#2584 - Allow nodes to fence themselves if they're the last one standing
   + crmd: Bug lf#2632 - Correctly handle nodes that return faster than stonith
   + crmd: Cancel timers for actions that were pending on dead nodes
   + crmd: Catch fence operations that claim to succeed but did not really
   + crmd: Do not wait for actions that were pending on dead nodes
   + crmd: Ensure we do not attempt to perform action on failed nodes
   + crmd: Prevent use-of-NULL by g_hash_table_iter_next()
   + crmd: Recurring actions shouldn't cause the last non-recurring action to be forgotten
   + crmd: Store only the last and last failed operation in the CIB
   + mcp: dirname() modifies the input path - pass in a copy of the logfile path
   + mcp: Enable stack detection logic instead of forcing 'corosync'
   + mcp: Fix spelling mistake in systemd service script that prevents shutdown
   + mcp: Shut down if corosync becomes unavailable
   + mcp: systemd control file is now functional
   + pengine: Before migrating an utilization-using resource to a node, take off the load which will no longer run there (lf#2599, bnc#695440)
   + pengine: Before migrating an utilization-using resource to a node, take off the load which will no longer run there (regression tests) (lf#2599, bnc#695440)
   + pengine: Bug lf#2574 - Prevent shuffling by choosing the correct clone instance to stop
   + pengine: Bug lf#2575 - Use uname for migration variables, id is a UUID on heartbeat
   + pengine: Bug lf#2581 - Avoid group restart when clone (re)starts on an unrelated node
   + pengine: Bug lf#2613, lf#2619 - Group migration after failures and non-default utilization policies
   + pengine: Bug suse#707150 - Prevent services being active if dependencies on clones are not satisfied
   + pengine: Correctly recognise which recurring operations are currently active
   + pengine: Demote from Master does not clear previous errors
   + pengine: Ensure restarts due to definition changes cause the start action to be re-issued not probes
   + pengine: Ensure role is preserved for unmanaged resources
   + pengine: Ensure unmanaged resources have the correct role set so the correct monitor operation is chosen
   + pengine: Fix memory leak for re-allocated resources reported by valgrind
   + pengine: Implement cluster ticket and deadman
   + pengine: Implement resource template
   + pengine: Correctly determine the state of multi-state resources with a partial operation history
   + pengine: Only allocate master/slave resources once
   + pengine: Partial revert of 'Minor code cleanup CS: cf6bca32376c On: 2011-08-15'
   + pengine: Resolve memory leak reported by valgrind
   + pengine: Restore the ability to save inputs to disk
   + Shell: implement -w,--wait option to wait for the transition to finish
   + Shell: repair template list command
   + Shell: set of commands to examine logs, reports, etc
   + Stonith: Consolidate pcmk_host_map into run_stonith_agent so that it is applied consistently
   + Stonith: Deprecate pcmk_arg_map for the saner pcmk_host_argument
   + Stonith: Fix use-of-NULL by g_hash_table_lookup
   + Stonith: Improved pcmk_host_map parsing
   + Stonith: Prevent use-of-NULL by g_hash_table_lookup
   + Stonith: Prevent use-of-NULL when no Linux-HA stonith agents are present
   + stonith: Add missing entries to stonith_error2string()
   + Stonith: Correctly finish sending agent options if the initial write is interrupted
   + stonith: Correctly handle synchronous calls
   + stonith: Coverity - Correctly construct result list for the query API call
   + stonith: Coverity - Remove badly constructed memory allocation from the query API call
   + stonith: Ensure completed operations are recorded as such in the history
   + Stonith: Ensure device parameters are passed to the daemon during registration
   + stonith: Fix use-of-NULL in stonith_api_device_list()
   + stonith: stonith_admin - Prevent use of uninitialized pointer by --history command
   + Tools: Bug lf#2528 - Make progress when attrd_updater is called repeatedly within the dampen interval but with the same value
   + Tools: crm_report - Correctly extract data from the local node
   + Tools: crm_report - Remove newlines when detecting the node list
   + Tools: crm_report - Repair the ability to extract data from the local machine
   + Tools: crm_report - Report on all detected backtraces
 
 * Fri Feb 11 2011 Andrew Beekhof <andrew@beekhof.net> 1.1.5
 - Update source tarball to revision: baad6636a053
 - Statistics:
   Changesets: 184
   Diff:       605 files changed, 46103 insertions(+), 26417 deletions(-)
 
 - Changes since Pacemaker-1.1.4
   + Add the ability to delegate sub-sections of the cluster to non-root users via ACLs
 	  Needs to be enabled at compile time, not enabled by default.
   + ais: Bug lf#2550 - Report failed processes immediately
   + Core: Prevent recently introduced use-after-free in replace_xml_child()
   + Core: Reinstate the logic that skips past non-XML_ELEMENT_NODE children
   + Core: Remove extra calls to xmlCleanupParser resulting in use-after-free
   + Core: Repair reference to child-of-child after removal of xml_child_iter_filter from get_message_xml()
   + crmd: Bug lf#2545 - Ensure notify variables are accurate for stop operations
   + crmd: Cancel recurring operations while we're still connected to the lrmd
   + crmd: Reschedule the PE_START action if its not already running when we try to use it
   + crmd: Update failcount for failed promote and demote operations
   + pengine: Bug lf#2445 - Avoid relying on stickness for stable clone placement
   + pengine: Bug lf#2445 - Do not override configured clone stickiness values
   + pengine: Bug lf#2493 - Don't imply colocation requirements when applying ordering constraints with clones
   + pengine: Bug lf#2495 - Prevent segfault by validating the contents of ordering sets
   + pengine: Bug lf#2508 - Correctly reconstruct the status of anonymous cloned groups
   + pengine: Bug lf#2518 - Avoid spamming the logs with errors for orphan resources
   + pengine: Bug lf#2544 - Prevent unstable clone placement by factoring in the current node's score before all others
   + pengine: Bug lf#2554 - target-role alone is not sufficient to promote resources
   + pengine: Correct target_rc for probes of inactive resources (fix regression introduced by cs:ac3f03006e95)
   + pengine: Ensure that fencing has completed for stop actions on stonith-dependent resources (lf#2551)
   + pengine: Only update the node's promotion score if the resource is active there
   + pengine: Only use the promotion score from the current clone instance
   + pengine: Prevent use-of-NULL resulting from variable shadowing spotted by Coverity
   + pengine: Prevent use-of-NULL when there is status for an undefined node
   + pengine: Prevet use-after-free resulting from unintended recursion when chosing a node to promote master/slave resources
   + Shell: don't create empty optional sections (bnc#665131)
   + Stonith: Teach stonith_admin to automagically obtain the current node attributes for the target from the CIB
   + tools: Bug lf#2527 - Prevent use-of-NULL in crm_simulate
   + Tools: Prevent crm_resource commands from being lost due to the use of cib_scope_local
 
 * Wed Oct 20 2010 Andrew Beekhof <andrew@beekhof.net> 1.1.4
 - Update source tarball to revision: 75406c3eb2c1 tip
 - Statistics:
   Changesets: 169
   Diff:       772 files changed, 56172 insertions(+), 39309 deletions(-)
 
 - Changes since Pacemaker-1.1.3
   + Italian translation of Clusters from Scratch
   + Significant performance enhancements to the Policy Engine and CIB
   + cib: Bug lf#2506 - Don't remove client's when notifications fail, they might just be too big
   + cib: Drop invalid/failed connections from the client hashtable
   + cib: Ensure all diffs sent to peers have sufficient ordering information
   + cib: Ensure non-change diffs can preserve the ordering on the other side
   + cib: Fix the feature set check
   + cib: Include version information on our synthesised diffs when nothing changed
   + cib: Optimize the way we detect group/set ordering changes - 15% speedup
   + cib: Prevent false detection of config updates with the new diff format
   + cib: Reduce unnecessary copying when comparing xml objects
   + cib: Repair the processing of updates sent from peer nodes
   + cib: Revert part of a recent commit that purged still valid connections
   + cib: The feature set version check is only valid if the current value is non-NULL
   + Core: Actually removing diff markers is necessary
   + Core: Bug lf#2506 - Drop the compression limit because Heartbeat's IPC code sucks
   + Core: Cache Relax-NG schemas - profiling indicates many cycles are wasted needlessly re-parsing them
   + Core: Correctly compare against crm_log_level in the logging macros
   + Core: Correctly extract the version details from a diff
   + Core: Correctly hook up the RNG schema cache
   + Core: Correctly use lazy_xml_sort() for v2 digests
   + Core: Don't compress large payload elements unless we're approaching message limits
   + Core: Don't insert empty ID tags when applying diffs
   + Core: Enable the improve v2 digests
   + Core: Ensure ordering is preserved when applying diffs
   + Core: Fix the CRM_CHECK macro
   + Core: Modify the v2 digest algorithm so that some fields are sorted
   + Core: Prevent use-after-free when creating a CIB update for a timed out action
   + Core: Prevent use-of-NULL when cleaning up RelaxNG data structures
   + Core: Provide significant performance improvements by implementing versioned diffs and digests
   + crmd: All pending operations should be recorded, even recurring ones with high start delays
   + crmd: Don't abort transitions when probes are completed on a node
   + crmd: Don't hide stop events that time out - allowing faster recovery in the presence of overloaded hosts
   + crmd: Ensure the CIB is always writable on the DC by removing a timing hole
   + crmd: Include the correct transition details for timed out operations
   + crmd: Prevent use of NULL by making copies of the operation's hash table
   + crmd: There's no need to check the cib version from the 'added' part of diff updates
   + crmd: Use the supplied timeout for stop actions
   + mcp: Ensure valgrind is able to log its output somewhere
   + mcp: Use 99/01 for the start/stop sequence to avoid problems with services (such as libvirtd) started by init - Patch from Vladislav Bogdanov
   + pengine: Ensure fencing of the DC preceeds the STONITH_DONE operation
   + pengine: Fix memory leak introduced as part of the conversion to GHashTables
   + pengine: Fix memory leak when processing completed migration actions
   + pengine: Fix typo leading to use-of-NULL in the new ordering code
   + pengine: Free memory in recently introduced helper function
   + pengine: lf#2478 - Implement improved handling and recovery of atomic resource migrations
   + pengine: Obtain massive speedup by prepending to the list of ordering constraints (which can grow quite large)
   + pengine: Optimize the logic for deciding which non-grouped anonymous clone instances to probe for
   + pengine: Prevent clones from being stopped because resources colocated with them cannot be active
   + pengine: Try to ensure atomic migration ops occur within a single transition
   + pengine: Use hashtables instead of linked lists for performance sensitive datastructures
   + pengine: Use the original digest algorithm for parameter lists
   + stonith: cleanup children on timeout in fence_legacy
   + Stonith: Fix two memory leaks
   + Tools: crm_shadow - Avoid replacing the entire configuration (including status)
 
 * Tue Sep 21 2010 Andrew Beekhof <andrew@beekhof.net> 1.1.3
 - Update source tarball to revision: e3bb31c56244 tip
 - Statistics:
   Changesets: 352
   Diff:       481 files changed, 14130 insertions(+), 11156 deletions(-)
 
 - Changes since Pacemaker-1.1.2.1
   + ais: Bug lf#2401 - Improved processing when the peer crmd processes join/leave
   + ais: Correct the logic for conecting to plugin based clusters
   + ais: Do not supply a process list in mcp-mode
   + ais: Drop support for whitetank in the 1.1 release series
   + ais: Get an initial dump of the node membership when connecting to quorum-based clusters
   + ais: Guard against saturated cpg connections
   + ais: Handle CS_ERR_TRY_AGAIN in more cases
   + ais: Move the code for finding uid before the fork so that the child does no logging
   + ais: Never allow quorum plugins to affect connection to the pacemaker plugin
   + ais: Sign everyone up for peer process updates, not just the crmd
   + ais: The cluster type needs to be set before initializing classic openais connections
   + cib: Also free query result for xpath operations that return more than one hit
   + cib: Attempt to resolve memory corruption when forking a child to write the cib to disk
   + cib: Correctly free memory when writing out the cib to disk
   + cib: Fix the application of unversioned diffs
   + cib: Remove old developmental error logging
   + cib: Restructure the 'valid peer' check for deciding which instructions to ignore
   + cman: Correctly process membership/quorum changes from the pcmk plugin. Allow other message types through untouched
   + cman: Filter directed messages not intended for us
   + cman: Grab the initial membership when we connect
   + cman: Keep the list of peer processes up-to-date
   + cman: Make sure our common hooks are called after a cman membership update
   + cman: Make sure we can compile without cman present
   + cman: Populate sender details for cpg messages
   + cman: Update the ringid for cman based clusters
   + Core: Correctly unpack HA_Messages containing multiple entries with the same name
   + Core: crm_count_member() should only track nodes that have the full stack up
   + Core: New developmental logging system inspired by the kernel and a PoC from Lars Ellenberg
   + crmd: All nodes should see status updates, not just he DC
   + crmd: Allow non-DC nodes to clear failcounts too
   + crmd: Base DC election on process relative uptime
   + crmd: Bug lf#2439 - cancel_op() can also return HA_RSCBUSY
   + crmd: Bug lf#2439 - Handle asynchronous notification of resource deletion events
   + crmd: Bug lf#2458 - Ensure stop actions always have the relevant resource attributes
   + crmd: Disable age as a criteria for cman based clusters, its not reliable enough
   + crmd: Ensure we activate the DC timer if we detect an alternate DC
   + crmd: Factor the nanosecond component of process uptime in elections
   + crmd: Fix assertion failure when performing async resource failures
   + crmd: Fix handling of async resource deletion results
   + crmd: Include the action for crm graph operations
   + crmd: Make sure the membership cache is accurate after a sucessful fencing operation
   + crmd: Make sure we always poke the FSA after a transition to clear any TE_HALT actions
   + crmd: Offer crm-level membership once the peer starts the crmd process
   + crmd: Only need to request quorum update for plugin based clusters
   + crmd: Prevent assertion failure for stop actions resulting from cs: 3c0bc17c6daf
   + crmd: Prevent everyone from loosing DC elections by correctly initializing all relevant variables
   + crmd: Prevent segmentation fault
   + crmd: several fixes for async resource delete (thanks to beekhof)
   + crmd: Use the correct define/size for lrm resource IDs
   + Introduce two new cluster types 'cman' and 'corosync', replaces 'quorum_provider' concept
   + mcp: Add missing headers when built without heartbeat support
   + mcp: Correctly initialize the string containing the list of active daemons
   + mcp: Fix macro expansion in init script
   + mcp: Fix the expansion of the pid file in the init script
   + mcp: Handle CS_ERR_TRY_AGAIN when connecting to libcfg
   + mcp: Make sure we can compile the mcp without cman present
   + mcp: New master control process for (re)spawning pacemaker daemons
   + mcp: Read config early so we can re-initialize logging asap if daemonizing
   + mcp: Rename the mcp binary to pacemakerd and create a 'pacemaker' init script
   + mcp: Resend our process list after every CPG change
   + mcp: Tell chkconfig we need to shut down early on
   + pengine: Avoid creating invalid ordering constraints for probes that are not needed
   + pengine: Bug lf#1959 - Fail unmanaged resources should not prevent other services from shutting down
   + pengine: Bug lf#2422 - Ordering dependencies on partially active groups not observed properly
   + pengine: Bug lf#2424 - Use notify oepration definition if it exists in the configuration
   + pengine: Bug lf#2433 - No services should be stopped until probes finish
   + pengine: Bug lf#2453 - Enforce clone ordering in the absense of colocation constraints
   + pengine: Bug lf#2476 - Repair on-fail=block for groups and primitive resources
   + pengine: Correctly detect when there is a real failcount that expired and needs to be cleared
   + pengine: Correctly handle pseudo action creation
   + pengine: Correctly order clone startup after group/clone start
   + pengine: Correct use-after-free introduced in the prior patch
   + pengine: Do not demote resources because something that requires it can not run
   + pengine: Fix colocation for interleaved clones
   + pengine: Fix colocation with partially active groups
   + pengine: Fix potential use-after-free defect from coverity
   + pengine: Fix previous merge
   + pengine: Fix use-after-free in order_actions() reported by valgrind
   + pengine: Make the current data set a global variable so it does not need to be passed around everywhere
   + pengine: Prevent endless loop when looking for operation definitions in the configuration
   + pengine: Prevent segfault by ensuring the arguments to do_calculations() are initialized
   + pengine: Rewrite the ordering constraint logic to be simplicity, clarity and maintainability
   + pengine: Wait until stonith is available, do not fall back to shutdown for nodes requesting termination
   + Resolve coverity RESOURCE_LEAK defects
   + Shell: Complete the transition to using crm_attribute instead of crm_failcount and crm_standby
   + stonith: Advertise stonith-ng options in the metadata
   + stonith: Bug lf#2461 - Prevent segfault by not looking up operations if the hashtable has not been initialized yet
   + stonith: Bug lf#2473 - Add the timeout at the top level where the daemon is looking for it
   + Stonith: Bug lf#2473 - Ensure stonith operations complete within the timeout and are terminated if they run too long
   + stonith: Bug lf#2473 - Ensure timeouts are included for fencing operations
   + stonith: Bug lf#2473 - Gracefully handle remote operations that arrive late (after we have done notifications)
   + stonith: Correctly parse pcmk_host_list parameters that appear on a single line
   + stonith: Map poweron/poweroff back to on/off expected by the stonith tool from cluster-glue
   + stonith: pass the configuration to the stonith program via environment variables (bnc#620781)
   + Stonith: Use the timeout specified by the user
   + Support starting plugin-based Pacemaker clusters with the MCP as well
   + Tools: Bug lf#2456 - Fix assertion failure in crm_resource
   + tools: crm_node - Repair the ability to connect to openais based clusters
   + tools: crm_node - Use the correct short option for --cman
   + tools: crm_report - corosync.conf wont necessarily contain the text 'pacemaker' anymore
   + Tools: crm_simulate - Fix use-after-free in when terminating
   + tools: crm_simulate - Resolve coverity USE_AFTER_FREE defect
   + Tools: Drop the 'pingd' daemon and resource agent in favor of ocf:pacemaker:ping
   + Tools: Fix recently introduced use-of-NULL
   + Tools: Fix use-after-free defects from coverity
 
 * Wed May 12 2010 Andrew Beekhof <andrew@beekhof.net> 1.1.2
 - Update source tarball to revision: c25c972a25cc tip
 - Statistics:
   Changesets: 339
   Diff:       708 files changed, 37918 insertions(+), 10584 deletions(-)
 - Changes since Pacemaker-1.1.1
   + ais: Do not count votes from offline nodes and calculate current votes before sending quorum data
   + ais: Ensure the list of active processes sent to clients is always up-to-date
   + ais: Look for the correct conf variable for turning on file logging
   + ais: Need to find a better and thread-safe way to set core_uses_pid. Disable for now.
   + ais: Use the threadsafe version of getpwnam
   + Core: Bump the feature set due to the new failcount expiry feature
   + Core: fix memory leaks exposed by valgrind
   + Core: Bug lf#2414 - Prevent use-after-free reported by valgrind when doing xpath based deletions
   + crmd: Bug lf#2414 - Prevent use-after-free of the PE connection after it dies
   + crmd: Bug lf#2414 - Prevent use-after-free of the stonith-ng connection
   + crmd: Bug lf#2401 - Improved detection of partially active peers
   + crmd: Bug lf#2379 - Ensure the cluster terminates when the PE is not available
   + crmd: Do not allow the target_rc to be misused by resource agents
   + crmd: Do not ignore action timeouts based on FSA state
   + crmd: Ensure we don't get stuck in S_PENDING if we lose an election to someone that never talks to us again
   + crmd: Fix memory leaks exposed by valgrind
   + crmd: Remove race condition that could lead to multiple instances of a clone being active on a machine
   + crmd: Send erase_status_tag() calls to the local CIB when the DC is fenced, since there is no DC to accept them
   + crmd: Use global fencing notifications to prevent secondary fencing operations of the DC
   + pengine: Bug lf#2317 - Avoid needless restart of primitive depending on a clone
   + pengine: Bug lf#2361 - Ensure clones observe mandatory ordering constraints if the LHS is unrunnable
   + pengine: Bug lf#2383 - Combine failcounts for all instances of an anonymous clone on a host
   + pengine: Bug lf#2384 - Fix intra-set colocation and ordering
   + pengine: Bug lf#2403 - Enforce mandatory promotion (colocation) constraints
   + pengine: Bug lf#2412 - Correctly find clone instances by their prefix
   + pengine: Do not be so quick to pull the trigger on nodes that are coming up
   + pengine: Fix memory leaks exposed by valgrind
   + pengine: Rewrite native_merge_weights() to avoid Fix use-after-free
   + Shell: Bug bnc#590035 - always reload status if working with the cluster
   + Shell: Bug bnc#592762 - Default to using the status section from the live CIB
   + Shell: Bug lf#2315 - edit multiple meta_attributes sets in resource management
   + Shell: Bug lf#2221 - enable comments
   + Shell: Bug bnc#580492 - implement new cibstatus interface and commands
   + Shell: Bug bnc#585471 - new cibstatus import command
   + Shell: check timeouts also against the default-action-timeout property
   + Shell: new configure filter command
   + Tools: crm_mon - fix memory leaks exposed by valgrind
 
 * Tue Feb 16 2010 Andrew Beekhof <andrew@beekhof.net> - 1.1.1
 - First public release of Pacemaker 1.1
 - Package reference documentation in a doc subpackage
 - Move cts into a subpackage so that it can be easily consumed by others
 - Update source tarball to revision: 17d9cd4ee29f
   + New stonith daemon that supports global notifications
   + Service placement influenced by the physical resources
   + A new tool for simulating failures and the cluster’s reaction to them
   + Ability to serialize an otherwise unrelated a set of resource actions (eg. Xen migrations)
 
 * Mon Jan 18 2010 Andrew Beekhof <andrew@beekhof.net> - 1.0.7
 - Update source tarball to revision: 2eed906f43e9 (stable-1.0) tip
 - Statistics:
       Changesets:      193
       Diff:            220 files changed, 15933 insertions(+), 8782 deletions(-)
 - Changes since 1.0.5-4
   + pengine: Bug 2213 - Ensure groups process location constraints so that clone-node-max works for cloned groups
   + pengine: Bug lf#2153 - non-clones should not restart when clones stop/start on other nodes
   + pengine: Bug lf#2209 - Clone ordering should be able to prevent startup of dependent clones
   + pengine: Bug lf#2216 - Correctly identify the state of anonymous clones when deciding when to probe
   + pengine: Bug lf#2225 - Operations that require fencing should wait for 'stonith_complete' not 'all_stopped'.
   + pengine: Bug lf#2225 - Prevent clone peers from stopping while another is instance is (potentially) being fenced
   + pengine: Correctly anti-colocate with a group
   + pengine: Correctly unpack ordering constraints for resource sets to avoid graph loops
   + Tools: crm: load help from crm_cli.txt
   + Tools: crm: resource sets (bnc#550923)
   + Tools: crm: support for comments (LF 2221)
   + Tools: crm: support for description attribute in resources/operations (bnc#548690)
   + Tools: hb2openais: add EVMS2 CSM processing (and other changes) (bnc#548093)
   + Tools: hb2openais: do not allow empty rules, clones, or groups (LF 2215)
   + Tools: hb2openais: refuse to convert pure EVMS volumes
   + cib: Ensure the loop for login message terminates
   + cib: Finally fix reliability of receiving large messages over remote plaintext connections
   + cib: Fix remote notifications
   + cib: For remote connections, default to CRM_DAEMON_USER since thats the only one that the cib can validate the password for using PAM
   + cib: Remote plaintext - Retry sending parts of the message that did not fit the first time
   + crmd: Ensure batch-limit is correctly enforced
   + crmd: Ensure we have the latest status after a transition abort
   + (bnc#547579,547582): Tools: crm: status section editing support
   + shell: Add allow-migrate as allowed meta-attribute (bnc#539968)
   + Medium: Build: Do not automatically add -L/lib, it could cause 64-bit arches to break
   + Medium: pengine: Bug lf#2206 - rsc_order constraints always use score at the top level
   + Medium: pengine: Only complain about target-role=master for non m/s resources
   + Medium: pengine: Prevent non-multistate resources from being promoted through target-role
   + Medium: pengine: Provide a default action for resource-set ordering
   + Medium: pengine: Silently fix requires=fencing for stonith resources so that it can be set in op_defaults
   + Medium: Tools: Bug lf#2286 - Allow the shell to accept template parameters on the command line
   + Medium: Tools: Bug lf#2307 - Provide a way to determin the nodeid of past cluster members
   + Medium: Tools: crm: add update method to template apply (LF 2289)
   + Medium: Tools: crm: direct RA interface for ocf class resource agents (LF 2270)
   + Medium: Tools: crm: direct RA interface for stonith class resource agents (LF 2270)
   + Medium: Tools: crm: do not add score which does not exist
   + Medium: Tools: crm: do not consider warnings as errors (LF 2274)
   + Medium: Tools: crm: do not remove sets which contain id-ref attribute (LF 2304)
   + Medium: Tools: crm: drop empty attributes elements
   + Medium: Tools: crm: exclude locations when testing for pathological constraints (LF 2300)
   + Medium: Tools: crm: fix exit code on single shot commands
   + Medium: Tools: crm: fix node delete (LF 2305)
   + Medium: Tools: crm: implement -F (--force) option
   + Medium: Tools: crm: rename status to cibstatus (LF 2236)
   + Medium: Tools: crm: revisit configure commit
   + Medium: Tools: crm: stay in crm if user specified level only (LF 2286)
   + Medium: Tools: crm: verify changes on exit from the configure level
   + Medium: ais: Some clients such as gfs_controld want a cluster name, allow one to be specified in corosync.conf
   + Medium: cib: Clean up logic for receiving remote messages
   + Medium: cib: Create valid notification control messages
   + Medium: cib: Indicate where the remote connection came from
   + Medium: cib: Send password prompt to stderr so that stdout can be redirected
   + Medium: cts: Fix rsh handling when stdout is not required
   + Medium: doc: Fill in the section on removing a node from an AIS-based cluster
   + Medium: doc: Update the docs to reflect the 0.6/1.0 rolling upgrade problem
   + Medium: doc: Use Publican for docbook based documentation
   + Medium: fencing: stonithd: add metadata for stonithd instance attributes (and support in the shell)
   + Medium: fencing: stonithd: ignore case when comparing host names (LF 2292)
   + Medium: tools: Make crm_mon functional with remote connections
   + Medium: xml: Add stopped as a supported role for operations
   + Medium: xml: Bug bnc#552713 - Treat node unames as text fields not IDs
   + Medium: xml: Bug lf#2215 - Create an always-true expression for empty rules when upgrading from 0.6
 
 * Thu Oct 29 2009 Andrew Beekhof <andrew@beekhof.net> - 1.0.5-4
 - Include the fixes from CoroSync integration testing
 - Move the resource templates - they are not documentation
 - Ensure documentation is placed in a standard location
 - Exclude documentation that is included elsewhere in the package
 
 - Update the tarball from upstream to version ee19d8e83c2a
   + cib: Correctly clean up when both plaintext and tls remote ports are requested
   + pengine: Bug bnc#515172 - Provide better defaults for lt(e) and gt(e) comparisions
   + pengine: Bug lf#2197 - Allow master instances placemaker to be influenced by colocation constraints
   + pengine: Make sure promote/demote pseudo actions are created correctly
   + pengine: Prevent target-role from promoting more than master-max instances
   + ais: Bug lf#2199 - Prevent expected-quorum-votes from being populated with garbage
   + ais: Prevent deadlock - don't try to release IPC message if the connection failed
   + cib: For validation errors, send back the full CIB so the client can display the errors
   + cib: Prevent use-after-free for remote plaintext connections
   + crmd: Bug lf#2201 - Prevent use-of-NULL when running heartbeat
 
 * Wed Oct 13 2009 Andrew Beekhof <andrew@beekhof.net> - 1.0.5-3
 - Update the tarball from upstream to version 38cd629e5c3c
   + Core: Bug lf#2169 - Allow dtd/schema validation to be disabled
   + pengine: Bug lf#2106 - Not all anonymous clone children are restarted after configuration change
   + pengine: Bug lf#2170 - stop-all-resources option had no effect
   + pengine: Bug lf#2171 - Prevent groups from starting if they depend on a complex resource which can not
   + pengine: Disable resource management if stonith-enabled=true and no stonith resources are defined
   + pengine: do not include master score if it would prevent allocation
   + ais: Avoid excessive load by checking for dead children every 1s (instead of 100ms)
   + ais: Bug rh#525589 - Prevent shutdown deadlocks when running on CoroSync
   + ais: Gracefully handle changes to the AIS nodeid
   + crmd: Bug bnc#527530 - Wait for the transition to complete before leaving S_TRANSITION_ENGINE
   + crmd: Prevent use-after-free with LOG_DEBUG_3
   + Medium: xml: Mask the "symmetrical" attribute on rsc_colocation constraints (bnc#540672)
   + Medium (bnc#520707): Tools: crm: new templates ocfs2 and clvm
   + Medium: Build: Invert the disable ais/heartbeat logic so that --without (ais|heartbeat) is available to rpmbuild
   + Medium: pengine: Bug lf#2178 - Indicate unmanaged clones
   + Medium: pengine: Bug lf#2180 - Include node information for all failed ops
   + Medium: pengine: Bug lf#2189 - Incorrect error message when unpacking simple ordering constraint
   + Medium: pengine: Correctly log resources that would like to start but can not
   + Medium: pengine: Stop ptest from logging to syslog
   + Medium: ais: Include version details in plugin name
   + Medium: crmd: Requery the resource metadata after every start operation
 
 * Fri Aug 21 2009 Tomas Mraz <tmraz@redhat.com> - 1.0.5-2.1
 - rebuilt with new openssl
 
 * Wed Aug 19 2009 Andrew Beekhof <andrew@beekhof.net> - 1.0.5-2
 - Add versioned perl dependency as specified by
     https://fedoraproject.org/wiki/Packaging/Perl#Packages_that_link_to_libperl
 - No longer remove RPATH data, it prevents us finding libperl.so and no other
   libraries were being hardcoded
 - Compile in support for heartbeat
 - Conditionally add heartbeat-devel and corosynclib-devel to the -devel requirements
   depending on which stacks are supported
 
 * Mon Aug 17 2009 Andrew Beekhof <andrew@beekhof.net> - 1.0.5
 - Add dependency on resource-agents
 - Use the version of the configure macro that supplies --prefix, --libdir, etc
 - Update the tarball from upstream to version 462f1569a437 (Pacemaker 1.0.5 final)
   + Tools: crm_resource - Advertise --move instead of --migrate
   + Medium: Extra: New node connectivity RA that uses system ping and attrd_updater
   + Medium: crmd: Note that dc-deadtime can be used to mask the brokeness of some switches
 
 * Tue Aug 11 2009 Ville Skyttä <ville.skytta@iki.fi> - 1.0.5-0.7.c9120a53a6ae.hg
 - Use bzipped upstream tarball.
 
 * Wed Jul  29 2009 Andrew Beekhof <andrew@beekhof.net> - 1.0.5-0.6.c9120a53a6ae.hg
 - Add back missing build auto* dependencies
 - Minor cleanups to the install directive
 
 * Tue Jul  28 2009 Andrew Beekhof <andrew@beekhof.net> - 1.0.5-0.5.c9120a53a6ae.hg
 - Add a leading zero to the revision when alphatag is used
 
 * Tue Jul  28 2009 Andrew Beekhof <andrew@beekhof.net> - 1.0.5-0.4.c9120a53a6ae.hg
 - Incorporate the feedback from the cluster-glue review
 - Realistically, the version is a 1.0.5 pre-release
 - Use the global directive instead of define for variables
 - Use the haclient/hacluster group/user instead of daemon
 - Use the _configure macro
 - Fix install dependencies
 
 * Fri Jul  24 2009 Andrew Beekhof <andrew@beekhof.net> - 1.0.4-3
 - Initial Fedora checkin
 - Include an AUTHORS and license file in each package
 - Change the library package name to pacemaker-libs to be more
   Fedora compliant
 - Remove execute permissions from xml related files
 - Reference the new cluster-glue devel package name
 - Update the tarball from upstream to version c9120a53a6ae
   + pengine: Only prevent migration if the clone dependency is stopping/starting on the target node
   + pengine: Bug 2160 - Don't shuffle clones due to colocation
   + pengine: New implementation of the resource migration (not stop/start) logic
   + Medium: Tools: crm_resource - Prevent use-of-NULL by requiring a resource name for the -A and -a options
   + Medium: pengine: Prevent use-of-NULL in find_first_action()
 
 * Tue Jul 14 2009 Andrew Beekhof <andrew@beekhof.net> - 1.0.4-2
 - Reference authors from the project AUTHORS file instead of listing in description
 - Change Source0 to reference the Mercurial repo
 - Cleaned up the summaries and descriptions
 - Incorporate the results of Fedora package self-review
 
 * Thu Jun 04 2009 Andrew Beekhof <abeekhof@suse.de> - 1.0.4
 - Update source tarball to revision: 1d87d3e0fc7f (stable-1.0)
 - Statistics:
     Changesets:      209
     Diff:            266 files changed, 12010 insertions(+), 8276 deletions(-)
 - Changes since Pacemaker-1.0.3
   + (bnc#488291): ais: do not rely on byte endianness on ptr cast
   + (bnc#507255): Tools: crm: delete rsc/op_defaults (these meta_attributes are killing me)
   + (bnc#507255): Tools: crm: import properly rsc/op_defaults
   + (LF 2114): Tools: crm: add support for operation instance attributes
   + ais: Bug lf#2126 - Messages replies cannot be routed to transient clients
   + ais: Fix compilation for the latest Corosync API (v1719)
   + attrd: Do not perform all updates as complete refreshes
   + cib: Fix huge memory leak affecting heartbeat-based clusters
   + Core: Allow xpath queries to match attributes
   + Core: Generate the help text directly from a tool options struct
   + Core: Handle differences in 0.6 messaging format
   + crmd: Bug lf#2120 - All transient node attribute updates need to go via attrd
   + crmd: Correctly calculate how long an FSA action took to avoid spamming the logs with errors
   + crmd: Fix another large memory leak affecting Heartbeat based clusters
   + lha: Restore compatibility with older versions
   + pengine: Bug bnc#495687 - Filesystem is not notified of successful STONITH under some conditions
   + pengine: Make running a cluster with STONITH enabled but no STONITH resources an error and provide details on resolutions
   + pengine: Prevent use-ofNULL when using resource ordering sets
   + pengine: Provide inter-notification ordering guarantees
   + pengine: Rewrite the notification code to be understanable and extendable
   + Tools: attrd - Prevent race condition resulting in the cluster forgetting the node wishes to shut down
   + Tools: crm: regression tests
   + Tools: crm_mon - Fix smtp notifications
   + Tools: crm_resource - Repair the ability to query meta attributes
   + Low Build: Bug lf#2105 - Debian package should contain pacemaker doc and crm templates
   + Medium (bnc#507255): Tools: crm: handle empty rsc/op_defaults properly
   + Medium (bnc#507255): Tools: crm: use the right obj_type when creating objects from xml nodes
   + Medium (LF 2107): Tools: crm: revisit exit codes in configure
   + Medium: cib: Do not bother validating updates that only affect the status section
   + Medium: Core: Include supported stacks in version information
   + Medium: crmd: Record in the CIB, the cluster infrastructure being used
   + Medium: cts: Do not combine crm_standby arguments - the wrapper can not process them
   + Medium: cts: Fix the CIBAusdit class
   + Medium: Extra: Refresh showscores script from Dominik
   + Medium: pengine: Build a statically linked version of ptest
   + Medium: pengine: Correctly log the actions for resources that are being recovered
   + Medium: pengine: Correctly log the occurrence of promotion events
   + Medium: pengine: Implememt node health based on a patch from Mark Hamzy
   + Medium: Tools: Add examples to help text outputs
   + Medium: Tools: crm: catch syntax errors for configure load
   + Medium: Tools: crm: implement erasing nodes in configure erase
   + Medium: Tools: crm: work with parents only when managing xml objects
   + Medium: Tools: crm_mon - Add option to run custom notification program on resource operations (Patch by Dominik Klein)
   + Medium: Tools: crm_resource - Allow --cleanup to function on complex resources and cluster-wide
   + Medium: Tools: haresource2cib.py - Patch from horms to fix conversion error
   + Medium: Tools: Include stack information in crm_mon output
   + Medium: Tools: Two new options (--stack,--constraints) to crm_resource for querying how a resource is configured
 
 * Wed Apr 08 2009 Andrew Beekhof <abeekhof@suse.de> - 1.0.3
 - Update source tarball to revision: b133b3f19797 (stable-1.0) tip
 - Statistics:
     Changesets:      383
     Diff:            329 files changed, 15471 insertions(+), 15119 deletions(-)
 - Changes since Pacemaker-1.0.2
   + Added tag SLE11-HAE-GMC for changeset 9196be9830c2
   + ais plugin: Fix quorum calculation (bnc#487003)
   + ais: Another memory fix leak in error path
   + ais: Bug bnc#482847, bnc#482905 - Force a clean exit of OpenAIS once Pacemaker has finished unloading
   + ais: Bug bnc#486858 - Fix update_member() to prevent spamming clients with membership events containing no changes
   + ais: Centralize all quorum calculations in the ais plugin and allow expected votes to be configured int he cib
   + ais: Correctly handle a return value of zero from openais_dispatch_recv()
   + ais: Disable logging to a file
   + ais: Fix memory leak in error path
   + ais: IPC messages are only in scope until a response is sent
   + All signal handlers used with CL_SIGNAL() need to be as minimal as possible
   + cib: Bug bnc#482885 - Simplify CIB disk-writes to prevent data loss.  Required a change to the backup filename format
   + cib: crmd: Revert part of 9782ab035003.  Complex shutdown routines need G_main_add_SignalHandler to avoid race coditions
   + crm: Avoid infinite loop during crm configure edit (bnc#480327)
   + crmd: Avoid a race condition by waiting for the attrd update to trigger a transition automatically
   + crmd: Bug bnc#480977 - Prevent extra, partial, shutdown when a node restarts too quickly
   + crmd: Bug bnc#480977 - Prevent extra, partial, shutdown when a node restarts too quickly (verified)
   + crmd: Bug bnc#489063 - Ensure the DC is always unset after we 'lose' an election
   + crmd: Bug BSC#479543 - Correctly find the migration source for timed out migrate_from actions
   + crmd: Call crm_peer_init() before we start the FSA - prevents a race condition when used with Heartbeat
   + crmd: Erasing the status section should not be forced to the local node
   + crmd: Fix memory leak in cib notication processing code
   + crmd: Fix memory leak in transition graph processing
   + crmd: Fix memory leaks found by valgrind
   + crmd: More memory leaks fixes found by valgrind
   + fencing: stonithd: is_heartbeat_cluster is a no-no if there is no heartbeat support
   + pengine: Bug bnc#466788 - Exclude nodes that can not run resources
   + pengine: Bug bnc#466788 - Make colocation based on node attributes work
   + pengine: Bug BNC#478687 - Do not crash when clone-max is 0
   + pengine: Bug bnc#488721 - Fix id-ref expansion for clones, the doc-root for clone children is not the cib root
   + pengine: Bug bnc#490418 - Correctly determine node state for nodes wishing to be terminated
   + pengine: Bug LF#2087 - Correctly parse the state of anonymous clones that have multiple instances on a given node
   + pengine: Bug lf#2089 - Meta attributes are not inherited by clone children
   + pengine: Bug lf#2091 - Correctly restart modified resources that were found active by a probe
   + pengine: Bug lf#2094 - Fix probe ordering for cloned groups
   + pengine: Bug LF:2075 - Fix large pingd memory leaks
   + pengine: Correctly attach orphaned clone children to their parent
   + pengine: Correctly handle terminate node attributes that are set to the output from time()
   + pengine: Ensure orphaned clone members are hooked up to the parent when clone-max=0
   + pengine: Fix memory leak in LogActions
   + pengine: Fix the determination of whether a group is active
   + pengine: Look up the correct promotion preference for anonymous masters
   + pengine: Simplify handling of start failures by changing the default migration-threshold to INFINITY
   + pengine: The ordered option for clones no longer causes extra start/stop operations
   + RA: Bug bnc#490641 - Shut down dlm_controld with -TERM instead of -KILL
   + RA: pingd: Set default ping interval to 1 instead of 0 seconds
   + Resources: pingd - Correctly tell the ping daemon to shut down
   + Tools: Bug bnc#483365 - Ensure the command from cluster_test includes a value for --log-facility
   + Tools: cli: fix and improve delete command
   + Tools: crm: add and implement templates
   + Tools: crm: add support for command aliases and some common commands (i.e. cd,exit)
   + Tools: crm: create top configuration nodes if they are missing
   + Tools: crm: fix parsing attributes for rules (broken by the previous changeset)
   + Tools: crm: new ra set of commands
   + Tools: crm: resource agents information management
   + Tools: crm: rsc/op_defaults
   + Tools: crm: support for no value attribute in nvpairs
   + Tools: crm: the new configure monitor command
   + Tools: crm: the new configure node command
   + Tools: crm_mon - Prevent use-of-NULL when summarizing an orphan
   + Tools: hb2openais: create clvmd clone for respawn evmsd in ha.cf
   + Tools: hb2openais: fix a serious recursion bug in xml node processing
   + Tools: hb2openais: fix ocfs2 processing
   + Tools: pingd - prevent double free of getaddrinfo() output in error path
   + Tools: The default re-ping interval for pingd should be 1s not 1ms
   + Medium (bnc#479049): Tools: crm: add validation of resource type for the configure primitive command
   + Medium (bnc#479050): Tools: crm: add help for RA parameters in tab completion
   + Medium (bnc#479050): Tools: crm: add tab completion for primitive params/meta/op
   + Medium (bnc#479050): Tools: crm: reimplement cluster properties completion
   + Medium (bnc#486968): Tools: crm: listnodes function requires no parameters (do not mix completion with other stuff)
   + Medium: ais: Remove the ugly hack for dampening AIS membership changes
   + Medium: cib: Fix memory leaks by using mainloop_add_signal
   + Medium: cib: Move more logging to the debug level (was info)
   + Medium: cib: Overhaul the processing of synchronous replies
   + Medium: Core: Add library functions for instructing the cluster to terminate nodes
   + Medium: crmd: Add new expected-quorum-votes option
   + Medium: crmd: Allow up to 5 retires when an attrd update fails
   + Medium: crmd: Automatically detect and use new values for crm_config options
   + Medium: crmd: Bug bnc#490426 - Escalated shutdowns stall when there are pending resource operations
   + Medium: crmd: Clean up and optimize the DC election algorithm
   + Medium: crmd: Fix memory leak in shutdown
   + Medium: crmd: Fix memory leaks spotted by Valgrind
   + Medium: crmd: Ignore join messages from hosts other than our DC
   + Medium: crmd: Limit the scope of resource updates to the status section
   + Medium: crmd: Prevent the crmd from being respawned if its told to shut down when it did not ask to be
   + Medium: crmd: Re-check the election status after membership events
   + Medium: crmd: Send resource updates via the local CIB during elections
   + Medium: pengine: Bug bnc#491441 - crm_mon does not display operations returning 'uninstalled' correctly
   + Medium: pengine: Bug lf#2101 - For location constraints, role=Slave is equivalent to role=Started
   + Medium: pengine: Clean up the API - removed ->children() and renamed ->find_child() to fine_rsc()
   + Medium: pengine: Compress the display of healthy anonymous clones
   + Medium: pengine: Correctly log the actions for resources that are being recovered
   + Medium: pengine: Determin a promotion score for complex resources
   + Medium: pengine: Ensure clones always have a value for globally-unique
   + Medium: pengine: Prevent orphan clones from being allocated
   + Medium: RA: controld: Return proper exit code for stop op.
   + Medium: Tools: Bug bnc#482558 - Fix logging test in cluster_test
   + Medium: Tools: Bug bnc#482828 - Fix quoting in cluster_test logging setup
   + Medium: Tools: Bug bnc#482840 - Include directory path to CTSlab.py
   + Medium: Tools: crm: add more user input checks
   + Medium: Tools: crm: do not check resource status of we are working with a shadow
   + Medium: Tools: crm: fix id-refs and allow reference to top objects (i.e. primitive)
   + Medium: Tools: crm: ignore comments in the CIB
   + Medium: Tools: crm: multiple column output would not work with small lists
   + Medium: Tools: crm: refuse to delete running resources
   + Medium: Tools: crm: rudimentary if-else for templates
   + Medium: Tools: crm: Start/stop clones via target-role.
   + Medium: Tools: crm_mon - Compress the node status for healthy and offline nodes
   + Medium: Tools: crm_shadow - Return 0/cib_ok when --create-empty succeeds
   + Medium: Tools: crm_shadow - Support -e, the short form of --create-empty
   + Medium: Tools: Make attrd quieter
   + Medium: Tools: pingd - Avoid using various clplumbing functions as they seem to leak
   + Medium: Tools: Reduce pingd logging
 
 * Mon Feb 16 2009 Andrew Beekhof <abeekhof@suse.de> - 1.0.2
 - Update source tarball to revision: d232d19daeb9 (stable-1.0) tip
 - Statistics:
     Changesets:      441
     Diff:            639 files changed, 20871 insertions(+), 21594 deletions(-)
 - Changes since Pacemaker-1.0.1
   + (bnc#450815): Tools: crm cli: do not generate id for the operations tag
   + ais: Add support for the new AIS IPC layer
   + ais: Always set header.error to the correct default: SA_AIS_OK
   + ais: Bug BNC#456243 - Ensure the membership cache always contains an entry for the local node
   + ais: Bug BNC:456208 - Prevent deadlocks by not logging in the child process before exec()
   + ais: By default, disable supprt for the WIP openais IPC patch
   + ais: Detect and handle situations where ais and the crm disagree on the node name
   + ais: Ensure crm_peer_seq is updated after a membership update
   + ais: Make sure all IPC header fields are set to sane defaults
   + ais: Repair and streamline service load now that whitetank startup functions correctly
   + build: create and install doc files
   + cib: Allow clients without mainloop to connect to the cib
   + cib: CID:18 - Fix use-of-NULL in cib_perform_op
   + cib: CID:18 - Repair errors introduced in b5a18704477b - Fix use-of-NULL in cib_perform_op
   + cib: Ensure diffs contain the correct values of admin_epoch
   + cib: Fix four moderately sized memory leaks detected by Valgrind
   + Core: CID:10 - Prevent indexing into an array of schemas with a negative value
   + Core: CID:13 - Fix memory leak in log_data_element
   + Core: CID:15 - Fix memory leak in crm_get_peer
   + Core: CID:6 - Fix use-of-NULL in copy_ha_msg_input
   + Core: Fix crash in the membership code preventing node shutdown
   + Core: Fix more memory leaks foudn by valgrind
   + Core: Prevent unterminated strings after decompression
   + crmd: Bug BNC:467995 - Delay marking STONITH operations complete until STONITH tells us so
   + crmd: Bug LF:1962 - Do not NACK peers because they are not (yet) in our membership.  Just ignore them.
   + crmd: Bug LF:2010 - Ensure fencing cib updates create the node_state entry if needed to preent re-fencing during cluster startup
   + crmd: Correctly handle reconnections to attrd
   + crmd: Ensure updates for lost migrate operations indicate which node it tried to migrating to
   + crmd: If there are no nodes to finalize, start an election.
   + crmd: If there are no nodes to welcome, start an election.
   + crmd: Prevent node attribute loss by detecting attrd disconnections immediately
   + crmd: Prevent node re-probe loops by ensuring mandatory actions always complete
   + pengine: Bug 2005 - Fix startup ordering of cloned stonith groups
   + pengine: Bug 2006 - Correctly reprobe cloned groups
   + pengine: Bug BNC:465484 - Fix the no-quorum-policy=suicide option
   + pengine: Bug LF:1996 - Correctly process disabled monitor operations
   + pengine: CID:19 - Fix use-of-NULL in determine_online_status
   + pengine: Clones now default to globally-unique=false
   + pengine: Correctly calculate the number of available nodes for the clone to use
   + pengine: Only shoot online nodes with no-quorum-policy=suicide
   + pengine: Prevent on-fail settings being ignored after a resource is successfully stopped
   + pengine: Prevent use-of-NULL for failed migrate actions in process_rsc_state()
   + pengine: Remove an optimization for the terminate node attribute that caused the cluster to block indefinitly
   + pengine: Repar the ability to colocate based on node attributes other than uname
   + pengine: Start the correct monitor operation for unmanaged masters
   + stonith: CID:3 - Fix another case of exceptionally poor error handling by the original stonith developers
   + stonith: CID:5 - Checking for NULL and then dereferencing it anyway is an interesting approach to error handling
   + stonithd: Sending IPC to the cluster is a privileged operation
   + stonithd: wrong checks for shmid (0 is a valid id)
   + Tools: attrd - Correctly determine when an attribute has stopped changing and should be committed to the CIB
   + Tools: Bug 2003 - pingd does not correctly detect failures when the interface is down
   + Tools: Bug 2003 - pingd does not correctly handle node-down events on multi-NIC systems
   + Tools: Bug 2021 - pingd does not detect sequence wrapping correctly, incorrectly reports nodes offline
   + Tools: Bug BNC:468066 - Do not use the result of uname() when its no longer in scope
   + Tools: Bug BNC:473265 - crm_resource -L dumps core
   + Tools: Bug LF:2001 - Transient node attributes should be set via attrd
   + Tools: Bug LF:2036 - crm_resource cannot set/get parameters for cloned resources
   + Tools: Bug LF:2046 - Node attribute updates are lost because attrd can take too long to start
   + Tools: Cause the correct clone instance to be failed with crm_resource -F
   + Tools: cluster_test - Allow the user to select a stack and fix CTS invocation
   + Tools: crm cli: allow rename only if the resource is stopped
   + Tools: crm cli: catch system errors on file operations
   + Tools: crm cli: completion for ids in configure
   + Tools: crm cli: drop '-rsc' from attributes for order constraint
   + Tools: crm cli: exit with an appropriate exit code
   + Tools: crm cli: fix wrong order of action and resource in order constraint
   + Tools: crm cli: fox wrong exit code
   + Tools: crm cli: improve handling of cib attributes
   + Tools: crm cli: new command: configure rename
   + Tools: crm cli: new command: configure upgrade
   + Tools: crm cli: new command: node delete
   + Tools: crm cli: prevent key errors on missing cib attributes
   + Tools: crm cli: print long help for help topics
   + Tools: crm cli: return on syntax error when parsing score
   + Tools: crm cli: rsc_location can be without nvpairs
   + Tools: crm cli: short node preference location constraint
   + Tools: crm cli: sometimes, on errors, level would change on single shot use
   + Tools: crm cli: syntax: drop a bunch of commas (remains of help tables conversion)
   + Tools: crm cli: verify user input for sanity
   + Tools: crm: find expressions within rules (do not always skip xml nodes due to used id)
   + Tools: crm_master should not define a set id now that attrd is used.  Defining one can break lookups
   + Tools: crm_mon Use the OID assigned to the project by IANA for SNMP traps
   + Medium (bnc#445622): Tools: crm cli: improve the node show command and drop node status
   + Medium (LF 2009): stonithd: improve timeouts for remote fencing
   + Medium: ais: Allow dead peers to be removed from membership calculations
   + Medium: ais: Pass node deletion events on to clients
   + Medium: ais: Sanitize ipc usage
   + Medium: ais: Supply the node uname in addtion to the id
   + Medium: Build: Clean up configure to ensure NON_FATAL_CFLAGS is consistent with CFLAGS (ie. includes -g)
   + Medium: Build: Install cluster_test
   + Medium: Build: Use more restrictive CFLAGS and fix the resulting errors
   + Medium: cib: CID:20 - Fix potential use-after-free in cib_native_signon
   + Medium: Core: Bug BNC:474727 - Set a maximum time to wait for IPC messages
   + Medium: Core: CID:12 - Fix memory leak in decode_transition_magic error path
   + Medium: Core: CID:14 - Fix memory leak in calculate_xml_digest error path
   + Medium: Core: CID:16 - Fix memory leak in date_to_string error path
   + Medium: Core: Try to track down the cause of XML parsing errors
   + Medium: crmd: Bug BNC:472473 - Do not wait excessive amounts of time for lost actions
   + Medium: crmd: Bug BNC:472473 - Reduce the transition timeout to action_timeout+network_delay
   + Medium: crmd: Do not fast-track the processing of LRM refreshes when there are pending actions.
   + Medium: crmd: do_dc_join_filter_offer - Check the 'join' message is for the current instance before deciding to NACK peers
   + Medium: crmd: Find option values without having to do a config upgrade
   + Medium: crmd: Implement shutdown using a transient node attribute
   + Medium: crmd: Update the crmd options to use dashes instead of underscores
   + Medium: cts: Add 'cluster reattach' to the suite of automated regression tests
   + Medium: cts: cluster_test - Make some usability enhancements
   + Medium: CTS: cluster_test - suggest a valid port number
   + Medium: CTS: Fix python import order
   + Medium: cts: Implement an automated SplitBrain test
   + Medium: CTS: Remove references to deleted classes
   + Medium: Extra: Resources - Use HA_VARRUN instead of HA_RSCTMP for state files as Heartbeat removes HA_RSCTMP at startup
   + Medium: HB: Bug 1933 - Fake crmd_client_status_callback() calls because HB does not provide them for already running processes
   + Medium: pengine: CID:17 - Fix memory leak in find_actions_by_task error path
   + Medium: pengine: CID:7,8 - Prevent hypothetical use-of-NULL in LogActions
   + Medium: pengine: Defer logging the actions performed on a resource until we have processed ordering constraints
   + Medium: pengine: Remove the symmetrical attribute of colocation constraints
   + Medium: Resources: pingd - fix the meta defaults
   + Medium: Resources: Stateful - Add missing meta defaults
   + Medium: stonithd: exit if we the pid file cannot be locked
   + Medium: Tools: Allow attrd clients to specify the ID the attribute should be created with
   + Medium: Tools: attrd - Allow attribute updates to be performed from a hosts peer
   + Medium: Tools: Bug LF:1994 - Clean up crm_verify return codes
   + Medium: Tools: Change the pingd defaults to ping hosts once every second (instead of 5 times every 10 seconds)
   + Medium: Tools: cibmin - Detect resource operations with a view to providing email/snmp/cim notification
   + Medium: Tools: crm cli: add back symmetrical for order constraints
   + Medium: Tools: crm cli: generate role in location when converting from xml
   + Medium: Tools: crm cli: handle shlex exceptions
   + Medium: Tools: crm cli: keep order of help topics
   + Medium: Tools: crm cli: refine completion for ids in configure
   + Medium: Tools: crm cli: replace inf with INFINITY
   + Medium: Tools: crm cli: streamline cib load and parsing
   + Medium: Tools: crm cli: supply provider only for ocf class primitives
   + Medium: Tools: crm_mon - Add support for sending mail notifications of resource events
   + Medium: Tools: crm_mon - Include the DC version in status summary
   + Medium: Tools: crm_mon - Sanitize startup and option processing
   + Medium: Tools: crm_mon - switch to event-driven updates and add support for sending snmp traps
   + Medium: Tools: crm_shadow - Replace the --locate option with the saner --edit
   + Medium: Tools: hb2openais: do not remove Evmsd resources, but replace them with clvmd
   + Medium: Tools: hb2openais: replace crmadmin with crm_mon
   + Medium: Tools: hb2openais: replace the lsb class with ocf for o2cb
   + Medium: Tools: hb2openais: reuse code
   + Medium: Tools: LF:2029 - Display an error if crm_resource is used to reset the operation history of non-primitive resources
   + Medium: Tools: Make pingd resilient to attrd failures
   + Medium: Tools: pingd - fix the command line switches
   + Medium: Tools: Rename ccm_tool to crm_node
 
 * Tue Nov 18 2008 Andrew Beekhof <abeekhof@suse.de> - 1.0.1
 - Update source tarball to revision: 6fc5ce8302ab (stable-1.0) tip
 - Statistics:
     Changesets:      170
     Diff:            816 files changed, 7633 insertions(+), 6286 deletions(-)
 - Changes since Pacemaker-1.0.1
   + ais: Allow the crmd to get callbacks whenever a node state changes
   + ais: Create an option for starting the mgmtd daemon automatically
   + ais: Ensure HA_RSCTMP exists for use by resource agents
   + ais: Hook up the openais.conf config logging options
   + ais: Zero out the PID of disconnecting clients
   + cib: Ensure global updates cause a disk write when appropriate
   + Core: Add an extra snaity check to getXpathResults() to prevent segfaults
   + Core: Do not redefine __FUNCTION__ unnecessarily
   + Core: Repair the ability to have comments in the configuration
   + crmd: Bug:1975 - crmd should wait indefinitely for stonith operations to complete
   + crmd: Ensure PE processing does not occur for all error cases in do_pe_invoke_callback
   + crmd: Requests to the CIB should cause any prior PE calculations to be ignored
   + heartbeat: Wait for membership 'up' events before removing stale node status data
   + pengine: Bug LF:1988 - Ensure recurring operations always have the correct target-rc set
   + pengine: Bug LF:1988 - For unmanaged resources we need to skip the usual can_run_resources() checks
   + pengine: Ensure the terminate node attribute is handled correctly
   + pengine: Fix optional colocation
   + pengine: Improve up the detection of 'new' nodes joining the cluster
   + pengine: Prevent assert failures in master_color() by ensuring unmanaged masters are always reallocated to their current location
   + Tools: crm cli: parser: return False on syntax error and None for comments
   + Tools: crm cli: unify template and edit commands
   + Tools: crm_shadow - Show more line number information after validation failures
   + Tools: hb2openais: add option to upgrade the CIB to v3.0
   + Tools: hb2openais: add U option to getopts and update usage
   + Tools: hb2openais: backup improved and multiple fixes
   + Tools: hb2openais: fix class/provider reversal
   + Tools: hb2openais: fix testing
   + Tools: hb2openais: move the CIB update to the end
   + Tools: hb2openais: update logging and set logfile appropriately
   + Tools: LF:1969 - Attrd never sets any properties in the cib
   + Tools: Make attrd functional on OpenAIS
   + Medium: ais: Hook up the options for specifying the expected number of nodes and total quorum votes
   + Medium: ais: Look for pacemaker options inside the service block with 'name: pacemaker' instead of creating an addtional configuration block
   + Medium: ais: Provide better feedback when nodes change nodeids (in openais.conf)
   + Medium: cib: Always store cib contents on disk with num_updates=0
   + Medium: cib: Ensure remote access ports are cleaned up on shutdown
   + Medium: crmd: Detect deleted resource operations automatically
   + Medium: crmd: Erase a nodes resource operations and transient attributes after a successful STONITH
   + Medium: crmd: Find a more appropriate place to update quorum and refresh attrd attributes
   + Medium: crmd: Fix the handling of unexpected PE exits to ensure the current CIB is stored
   + Medium: crmd: Fix the recording of pending operations in the CIB
   + Medium: crmd: Initiate an attrd refresh _after_ the status section has been fully repopulated
   + Medium: crmd: Only the DC should update quorum in an openais cluster
   + Medium: Ensure meta attributes are used consistantly
   + Medium: pengine: Allow group and clone level resource attributes
   + Medium: pengine: Bug N:437719 - Ensure scores from colocated resources count when allocating groups
   + Medium: pengine: Prevent lsb scripts from being used in globally unique clones
   + Medium: pengine: Make a best-effort guess at a migration threshold for people with 0.6 configs
   + Medium: Resources: controld - ensure we are part of a clone with globally_unique=false
   + Medium: Tools: attrd - Automatically refresh all attributes after a CIB replace operation
   + Medium: Tools: Bug LF:1985 - crm_mon - Correctly process failed cib queries to allow reconnection after cluster restarts
   + Medium: Tools: Bug LF:1987 - crm_verify incorrectly warns of configuration upgrades for the most recent version
   + Medium: Tools: crm (bnc#441028): check for key error in attributes management
   + Medium: Tools: crm_mon - display the meaning of the operation rc code instead of the status
   + Medium: Tools: crm_mon - Fix the display of timing data
   + Medium: Tools: crm_verify - check that we are being asked to validate a complete config
   + Medium: xml: Relax the restriction on the contents of rsc_locaiton.node
 
 * Thu Oct 16 2008 Andrew Beekhof <abeekhof@suse.de> - 1.0.0
 - Update source tarball to revision: 388654dfef8f tip
 - Statistics:
     Changesets:      261
     Diff:            3021 files changed, 244985 insertions(+), 111596 deletions(-)
 - Changes since f805e1b30103
   + add the crm cli program
   + ais: Move the service id definition to a common location and make sure it is always used
   + build: rename hb2openais.sh to .in and replace paths with vars
   + cib: Implement --create for crm_shadow
   + cib: Remove dead files
   + Core: Allow the expected number of quorum votes to be configrable
   + Core: cl_malloc and friends were removed from Heartbeat
   + Core: Only call xmlCleanupParser() if we parsed anything.  Doing so unconditionally seems to cause a segfault
   + hb2openais.sh: improve pingd handling; several bugs fixed
   + hb2openais: fix clone creation; replace EVMS strings
   + new hb2openais.sh conversion script
   + pengine: Bug LF:1950 - Ensure the current values for all notification variables are always set (even if empty)
   + pengine: Bug LF:1955 - Ensure unmanaged masters are unconditionally repromoted to ensure they are monitored correctly.
   + pengine: Bug LF:1955 - Fix another case of filtering causing unmanaged master failures
   + pengine: Bug LF:1955 - Umanaged mode prevents master resources from being allocated correctly
   + pengine: Bug N:420538 - Anit-colocation caused a positive node preference
   + pengine: Correctly handle unmanaged resources to prevent them from being started elsewhere
   + pengine: crm_resource - Fix the --migrate command
   + pengine: MAke stonith-enabled default to true and warn if no STONITH resources are found
   + pengine: Make sure orphaned clone children are created correctly
   + pengine: Monitors for unmanaged resources do not need to wait for start/promote/demote actions to complete
   + stonithd (LF 1951): fix remote stonith operations
   + stonithd: fix handling of timeouts
   + stonithd: fix logic for stonith resource priorities
   + stonithd: implement the fence-timeout instance attribute
   + stonithd: initialize value before reading fence-timeout
   + stonithd: set timeouts for fencing ops to the timeout of the start op
   + stonithd: stonith rsc priorities (new feature)
   + Tools: Add hb2openais - a tool for upgrading a Heartbeat cluster to use OpenAIS instead
   + Tools: crm_verify - clean up the upgrade logic to prevent crash on invalid configurations
   + Tools: Make pingd functional on Linux
   + Update version numbers for 1.0 candidates
   + Medium: ais: Add support for a synchronous call to retrieve the nodes nodeid
   + Medium: ais: Use the agreed service number
   + Medium: Build: Reliably detect heartbeat libraries during configure
   + Medium: Build: Supply prototypes for libreplace functions when needed
   + Medium: Build: Teach configure how to find corosync
   + Medium: Core: Provide better feedback if Pacemaker is started by a stack it does not support
   + Medium: crmd: Avoid calling GHashTable functions with NULL
   + Medium: crmd: Delay raising I_ERROR when the PE exits until we have had a chance to save the current CIB
   + Medium: crmd: Hook up the stonith-timeout option to stonithd
   + Medium: crmd: Prevent potential use-of-NULL in global_timer_callback
   + Medium: crmd: Rationalize the logging of graph aborts
   + Medium: pengine: Add a stonith_timeout option and remove new options that are better set in rsc_defaults
   + Medium: pengine: Allow external entities to ask for a node to be shot by creating a terminate=true transient node attribute
   + Medium: pengine: Bug LF:1950 - Notifications do not contain all documented resource state fields
   + Medium: pengine: Bug N:417585 - Do not restart group children whos individual score drops below zero
   + Medium: pengine: Detect clients that disconnect before receiving their reply
   + Medium: pengine: Implement a true maintenance mode
   + Medium: pengine: Implement on-fail=standby for NTT.  Derived from a patch by Satomi TANIGUCHI
   + Medium: pengine: Print the correct message when stonith is disabled
   + Medium: pengine: ptest - check the input is valid before proceeding
   + Medium: pengine: Revert group stickiness to the 'old way'
   + Medium: pengine: Use the correct attribute for action 'requires' (was prereq)
   + Medium: stonithd: Fix compilation without full heartbeat install
   + Medium: stonithd: exit with better code on empty host list
   + Medium: tools: Add a new regression test for CLI tools
   + Medium: tools: crm_resource - return with non-zero when a resource migration command is invalid
   + Medium: tools: crm_shadow - Allow the admin to start with an empty CIB (and no cluster connection)
   + Medium: xml: pacemaker-0.7 is now an alias for the 1.0 schema
 
 * Mon Sep 22 2008 Andrew Beekhof <abeekhof@suse.de> - 0.7.3
 - Update source tarball to revision: 33e677ab7764+ tip
 - Statistics:
     Changesets:      133
     Diff:            89 files changed, 7492 insertions(+), 1125 deletions(-)
 - Changes since f805e1b30103
   + Tools: add the crm cli program
   + Core: cl_malloc and friends were removed from Heartbeat
   + Core: Only call xmlCleanupParser() if we parsed anything.  Doing so unconditionally seems to cause a segfault
   + new hb2openais.sh conversion script
   + pengine: Bug LF:1950 - Ensure the current values for all notification variables are always set (even if empty)
   + pengine: Bug LF:1955 - Ensure unmanaged masters are unconditionally repromoted to ensure they are monitored correctly.
   + pengine: Bug LF:1955 - Fix another case of filtering causing unmanaged master failures
   + pengine: Bug LF:1955 - Umanaged mode prevents master resources from being allocated correctly
   + pengine: Bug N:420538 - Anit-colocation caused a positive node preference
   + pengine: Correctly handle unmanaged resources to prevent them from being started elsewhere
   + pengine: crm_resource - Fix the --migrate command
   + pengine: MAke stonith-enabled default to true and warn if no STONITH resources are found
   + pengine: Make sure orphaned clone children are created correctly
   + pengine: Monitors for unmanaged resources do not need to wait for start/promote/demote actions to complete
   + stonithd (LF 1951): fix remote stonith operations
   + Tools: crm_verify - clean up the upgrade logic to prevent crash on invalid configurations
   + Medium: ais: Add support for a synchronous call to retrieve the nodes nodeid
   + Medium: ais: Use the agreed service number
   + Medium: pengine: Allow external entities to ask for a node to be shot by creating a terminate=true transient node attribute
   + Medium: pengine: Bug LF:1950 - Notifications do not contain all documented resource state fields
   + Medium: pengine: Bug N:417585 - Do not restart group children whos individual score drops below zero
   + Medium: pengine: Implement a true maintenance mode
   + Medium: pengine: Print the correct message when stonith is disabled
   + Medium: stonithd: exit with better code on empty host list
   + Medium: xml: pacemaker-0.7 is now an alias for the 1.0 schema
 
 * Wed Aug 20 2008 Andrew Beekhof <abeekhof@suse.de> - 0.7.1
 - Update source tarball to revision: f805e1b30103+ tip
 - Statistics:
     Changesets:      184
     Diff:            513 files changed, 43408 insertions(+), 43783 deletions(-)
 - Changes since 0.7.0-19
   + Fix compilation when GNUTLS isn't found
   + admin: Fix use-after-free in crm_mon
   + Build: Remove testing code that prevented heartbeat-only builds
   + cib: Use single quotes so that the xpath queries for nvpairs will succeed
   + crmd: Always connect to stonithd when the TE starts and ensure we notice if it dies
   + crmd: Correctly handle a dead PE process
   + crmd: Make sure async-failures cause the failcount to be incremented
   + pengine: Bug LF:1941 - Handle failed clone instance probes when clone-max < #nodes
   + pengine: Parse resource ordering sets correctly
   + pengine: Prevent use-of-NULL - order->rsc_rh will not always be non-NULL
   + pengine: Unpack colocation sets correctly
   + Tools: crm_mon - Prevent use-of-NULL for orphaned resources
   + Medium: ais: Add support for a synchronous call to retrieve the nodes nodeid
   + Medium: ais: Allow transient clients to receive membership updates
   + Medium: ais: Avoid double-free in error path
   + Medium: ais: Include in the mebership nodes for which we have not determined their hostname
   + Medium: ais: Spawn the PE from the ais plugin instead of the crmd
   + Medium: cib: By default, new configurations use the latest schema
   + Medium: cib: Clean up the CIB if it was already disconnected
   + Medium: cib: Only increment num_updates if something actually changed
   + Medium: cib: Prevent use-after-free in client after abnormal termination of the CIB
   + Medium: Core: Fix memory leak in xpath searches
   + Medium: Core: Get more details regarding parser errors
   + Medium: Core: Repair expand_plus_plus - do not call char2score on unexpanded values
   + Medium: Core: Switch to the libxml2 parser - its significantly faster
   + Medium: Core: Use a libxml2 library function for xml -> text conversion
   + Medium: crmd: Asynchronous failure actions have no parameters
   + Medium: crmd: Avoid calling glib functions with NULL
   + Medium: crmd: Do not allow an election to promote a node from S_STARTING
   + Medium: crmd: Do not vote if we have not completed the local startup
   + Medium: crmd: Fix te_update_diff() now that get_object_root() functions differently
   + Medium: crmd: Fix the lrmd xpath expressions to not contain quotes
   + Medium: crmd: If we get a join offer during an election, better restart the election
   + Medium: crmd: No further processing is needed when using the LRMs API call for failing resources
   + Medium: crmd: Only update have-quorum if the value changed
   + Medium: crmd: Repair the input validation logic in do_te_invoke
   + Medium: cts: CIBs can no longer contain comments
   + Medium: cts: Enable a bunch of tests that were incorrectly disabled
   + Medium: cts: The libxml2 parser wont allow v1 resources to use integers as parameter names
   + Medium: Do not use the cluster UID and GID directly.  Look them up based on the configured value of HA_CCMUSER
   + Medium: Fix compilation when heartbeat is not supported
   + Medium: pengine: Allow groups to be involved in optional ordering constraints
   + Medium: pengine: Allow sets of operations to be reused by multiple resources
   + Medium: pengine: Bug LF:1941 - Mark extra clone instances as orphans and do not show inactive ones
   + Medium: pengine: Determin the correct migration-threshold during resource expansion
   + Medium: pengine: Implement no-quorum-policy=suicide (FATE #303619)
   + Medium: pengine: Clean up resources after stopping old copies of the PE
   + Medium: pengine: Teach the PE how to stop old copies of itself
   + Medium: Tools: Backport hb_report updates
   + Medium: Tools: cib_shadow - On create, spawn a new shell with CIB_shadow and PS1 set accordingly
   + Medium: Tools: Rename cib_shadow to crm_shadow
 
 * Fri Jul 18 2008 Andrew Beekhof <abeekhof@suse.de> - 0.7.0-19
 - Update source tarball to revision: 007c3a1c50f5 (unstable) tip
 - Statistics:
     Changesets:      108
     Diff:            216 files changed, 4632 insertions(+), 4173 deletions(-)
 - Changes added since unstable-0.7
   + admin: Fix use-after-free in crm_mon
   + ais: Change the tag for the ais plugin to "pacemaker" (used in openais.conf)
   + ais: Log terminated processes as an error
   + cib: Performance - Reorganize things to avoid calculating the XML diff twice
   + pengine: Bug LF:1941 - Handle failed clone instance probes when clone-max < #nodes
   + pengine: Fix memory leak in action2xml
   + pengine: Make OCF_ERR_ARGS a node-level error rather than a cluster-level one
   + pengine: Properly handle clones that are not installed on all nodes
   + Medium: admin: cibadmin - Show any validation errors if the upgrade failed
   + Medium: admin: cib_shadow - Implement --locate to display the underlying filename
   + Medium: admin: cib_shadow - Implement a --diff option
   + Medium: admin: cib_shadow - Implement a --switch option
   + Medium: admin: crm_resource - create more compact constraints that do not use lifetime (which is deprecated)
   + Medium: ais: Approximate born_on for OpenAIS based clusters
   + Medium: cib: Remove do_id_check, it is a poor substitute for ID validation by a schema
   + Medium: cib: Skip construction of pre-notify messages if no-one wants one
   + Medium: Core: Attempt to streamline some key functions to increase performance
   + Medium: Core: Clean up XML parser after validation
   + Medium: crmd: Detect and optimize the CRMs behavior when processing diffs of an LRM refresh
   + Medium: Fix memory leaks when resetting the name of an XML object
   + Medium: pengine: Prefer the current location if it is one of a group of nodes with the same (highest) score
 
 * Wed Jun 25 2008 Andrew Beekhof <abeekhof@suse.de> - 0.7.0
 - Update source tarball to revision: bde0c7db74fb tip
 - Statistics:
     Changesets:      439
     Diff:            676 files changed, 41310 insertions(+), 52071 deletions(-)
 - Changes added since stable-0.6
   + A new tool for setting up and invoking CTS
   + Admin: All tools now use --node (-N) for specifying node unames
   + Admin: All tools now use --xml-file (-x) and --xml-text (-X) for specifying where to find XML blobs
   + cib: Cleanup the API - remove redundant input fields
   + cib: Implement CIB_shadow - a facility for making and testing changes before uploading them to the cluster
   + cib: Make registering per-op callbacks an API call and renamed (for clarity) the API call for requesting notifications
   + Core: Add a facility for automatically upgrading old configurations
   + Core: Adopt libxml2 as the XML processing library - all external clients need to be recompiled
   + Core: Allow sending TLS messages larger than the MTU
   + Core: Fix parsing of time-only ISO dates
   + Core: Smarter handling of XML values containing quotes
   + Core: XML memory corruption - catch, and handle, cases where we are overwriting an attribute value with itself
   + Core: The xml ID type does not allow UUIDs that start with a number
   + Core: Implement XPath based versions of query/delete/replace/modify
   + Core: Remove some HA2.0.(3,4) compatibility code
   + crmd: Overhaul the detection of nodes that are starting vs. failed
   + pengine: Bug LF:1459 - Allow failures to expire
   + pengine: Have the PE do non-persistent configuration upgrades before performing calculations
   + pengine: Replace failure-stickiness with a simple 'migration-threshold'
   + tengine: Simplify the design by folding the tengine process into the crmd
   + Medium: Admin: Bug LF:1438 - Allow the list of all/active resource operations to be queried by crm_resource
   + Medium: Admin: Bug LF:1708 - crm_resource should print a warning if an attribute is already set as a meta attribute
   + Medium: Admin: Bug LF:1883 - crm_mon should display fail-count and operation history
   + Medium: Admin: Bug LF:1883 - crm_mon should display operation timing data
   + Medium: Admin: Bug N:371785 - crm_resource -C does not also clean up fail-count attributes
   + Medium: Admin: crm_mon - include timing data for failed actions
   + Medium: ais: Read options from the environment since objdb is not completely usable yet
   + Medium: cib: Add sections for op_defaults and rsc_defaults
   + Medium: cib: Better matching notification callbacks (for detecting duplicates and removal)
   + Medium: cib: Bug LF:1348 - Allow rules and attribute sets to be referenced for use in other objects
   + Medium: cib: BUG LF:1918 - By default, all cib calls now timeout after 30s
   + Medium: cib: Detect updates that decrease the version tuple
   + Medium: cib: Implement a client-side operation timeout - Requires LHA update
   + Medium: cib: Implement callbacks and async notifications for remote connections
   + Medium: cib: Make cib->cmds->update() an alias for modify at the API level (also implemented in cibadmin)
   + Medium: cib: Mark the CIB as disconnected if the IPC connection is terminated
   + Medium: cib: New call option 'cib_can_create' which can be passed to modify actions - allows the object to be created if it does not exist yet
   + Medium: cib: Reimplement get|set|delete attributes using XPath
   + Medium: cib: Remove some useless parts of the API
   + Medium: cib: Remove the 'attributes' scaffolding from the new format
   + Medium: cib: Implement the ability for clients to connect to remote servers
   + Medium: Core: Add support for validating xml against RelaxNG schemas
   + Medium: Core: Allow more than one item to be modified/deleted in XPath based operations
   + Medium: Core: Fix the sort_pairs function for creating sorted xml objects
   + Medium: Core: iso8601 - Implement subtract_duration and fix subtract_time
   + Medium: Core: Reduce the amount of xml copying
   + Medium: Core: Support value='value+=N' XML updates (in addtion to value='value++')
   + Medium: crmd: Add support for lrm_ops->fail_rsc if its available
   + Medium: crmd: HB - watch link status for node leaving events
   + Medium: crmd: Bug LF:1924 - Improved handling of lrmd disconnects and shutdowns
   + Medium: crmd: Do not wait for actions with a start_delay over 5 minutes. Confirm them immediately
   + Medium: pengine: Bug LF:1328 - Do not fencing nodes in clusters without managed resources
   + Medium: pengine: Bug LF:1461 - Give transient node attributes (in <status/>) preference over persistent ones (in <nodes/>)
   + Medium: pengine: Bug LF:1884, Bug LF:1885 - Implement N:M ordering and colocation constraints
   + Medium: pengine: Bug LF:1886 - Create a resource and operation 'defaults' config section
   + Medium: pengine: Bug LF:1892 - Allow recurring actions to be triggered at known times
   + Medium: pengine: Bug LF:1926 - Probes should complete before stop actions are invoked
   + Medium: pengine: Fix the standby when its set as a transient attribute
   + Medium: pengine: Implement a global 'stop-all-resources' option
   + Medium: pengine: Implement cibpipe, a tool for performing/simulating config changes "offline"
   + Medium: pengine: We do not allow colocation with specific clone instances
   + Medium: Tools: pingd - Implement a stack-independent version of pingd
   + Medium: xml: Ship an xslt for upgrading from 0.6 to 0.7
 
 * Thu Jun 19 2008 Andrew Beekhof <abeekhof@suse.de> - 0.6.5
 - Update source tarball to revision: b9fe723d1ac5 tip
 - Statistics:
     Changesets:      48
     Diff:            37 files changed, 1204 insertions(+), 234 deletions(-)
 - Changes since Pacemaker-0.6.4
   + Admin: Repair the ability to delete failcounts
   + ais: Audit IPC handling between the AIS plugin and CRM processes
   + ais: Have the plugin create needed /var/lib directories
   + ais: Make sure the sync and async connections are assigned correctly (not swapped)
   + cib: Correctly detect configuration changes - num_updates does not count
   + pengine: Apply stickiness values to the whole group, not the individual resources
   + pengine: Bug N:385265 - Ensure groups are migrated instead of remaining partially active on the current node
   + pengine: Bug N:396293 - Enforce mandatory group restarts due to ordering constraints
   + pengine: Correctly recover master instances found active on more than one node
   + pengine: Fix memory leaks reported by Valgrind
   + Medium: Admin: crm_mon - Misc improvements from Satomi Taniguchi
   + Medium: Bug LF:1900 - Resource stickiness should not allow placement in asynchronous clusters
   + Medium: crmd: Ensure joins are completed promptly when a node taking part dies
   + Medium: pengine: Avoid clone instance shuffling in more cases
   + Medium: pengine: Bug LF:1906 - Remove an optimization in native_merge_weights() causing group scores to behave eratically
   + Medium: pengine: Make use of target_rc data to correctly process resource operations
   + Medium: pengine: Prevent a possible use of NULL in sort_clone_instance()
   + Medium: tengine: Include target rc in the transition key - used to correctly determin operation failure
 
 * Thu May 22 2008 Andrew Beekhof <abeekhof@suse.de> - 0.6.4
 - Update source tarball to revision: 226d8e356924 tip
 - Statistics:
     Changesets:       55
     Diff:             199 files changed, 7103 insertions(+), 12378 deletions(-)
 - Changes since Pacemaker-0.6.3
   + crmd: Bug LF:1881 LF:1882 - Overhaul the logic for operation cancelation and deletion
   + crmd: Bug LF:1894 - Make sure cancelled recurring operations are cleaned out from the CIB
   + pengine: Bug N:387749 - Colocation with clones causes unnecessary clone instance shuffling
   + pengine: Ensure 'master' monitor actions are cancelled _before_ we demote the resource
   + pengine: Fix assert failure leading to core dump - make sure variable is properly initialized
   + pengine: Make sure 'slave' monitoring happens after the resource has been demoted
   + pengine: Prevent failure stickiness underflows (where too many failures become a _positive_ preference)
   + Medium: Admin: crm_mon - Only complain if the output file could not be opened
   + Medium: Common: filter_action_parameters - enable legacy handling only for older versions
   + Medium: pengine: Bug N:385265 - The failure stickiness of group children is ignored until it reaches -INFINITY
   + Medium: pengine: Implement master and clone colocation by exlcuding nodes rather than setting ones score to INFINITY (similar to cs: 756afc42dc51)
   + Medium: tengine: Bug LF:1875 - Correctly find actions to cancel when their node leaves the cluster
 
 * Wed Apr 23 2008 Andrew Beekhof <abeekhof@suse.de> - 0.6.3
 - Update source tarball to revision: fd8904c9bc67 tip
 - Statistics:
     Changesets:      117
     Diff:            354 files changed, 19094 insertions(+), 11338 deletions(-)
 - Changes since Pacemaker-0.6.2
   + Admin: Bug LF:1848 - crm_resource - Pass set name and id to delete_resource_attr() in the correct order
   + Build: SNMP has been moved to the management/pygui project
   + crmd: Bug LF1837 - Unmanaged resources prevent crmd from shutting down
   + crmd: Prevent use-after-free in lrm interface code (Patch based on work by Keisuke MORI)
   + pengine: Allow the cluster to make progress by not retrying failed demote actions
   + pengine: Anti-colocation with slave should not prevent master colocation
   + pengine: Bug LF 1768 - Wait more often for STONITH ops to complete before starting resources
   + pengine: Bug LF1836 - Allow is-managed-default=false to be overridden by individual resources
   + pengine: Bug LF185 - Prevent pointless master/slave instance shuffling by ignoring the master-pref of stopped instances
   + pengine: Bug N-191176 - Implement interleaved ordering for clone-to-clone scenarios
   + pengine: Bug N-347004 - Ensure clone notifications are always sent when an instance is stopped/started
   + pengine: Bug N-347004 - Include notification ordering is correct for interleaved clones
   + pengine: Bug PM-11 - Directly link probe_complete to starting clone instances
   + pengine: Bug PM1 - Fix setting failcounts when applied to complex resources
   + pengine: Bug PM12, LF1648 - Extensive revision of group ordering
   + pengine: Bug PM7 - Ensure masters are always demoted before they are stopped
   + pengine: Create probes after allocation to allow smarter handling of anonymous clones
   + pengine: Do not prioritize clone instances that must be moved
   + pengine: Fix error in previous commit that allowed more than the required number of masters to be promoted
   + pengine: Group start ordering fixes
   + pengine: Implement promote/demote ordering for cloned groups
   + tengine: Repair failcount updates
   + tengine: Use the correct offset when updating failcount
   + Medium: Admin: Add a summary output that can be easily parsed by CTS for audit purposes
   + Medium: Build: Make configure fail if bz2 or libxml2 are not present
   + Medium: Build: Re-instate a better default for LCRSODIR
   + Medium: CIB: Bug LF-1861 - Filter irrelvant error status from synchronous CIB clients
   + Medium: Core: Bug 1849 - Invalid conversion of ordinal leap year to gregorian date
   + Medium: Core: Drop compatibility code for 2.0.4 and 2.0.5 clusters
   + Medium: crmd: Bug LF-1860 - Automatically cancel recurring ops before demote and promote operations (not only stops)
   + Medium: crmd: Save the current CIB contents if we detect the PE crashed
   + Medium: pengine: Bug LF:1866 - Fix version check when applying compatibility handling for failed start operations
   + Medium: pengine: Bug LF:1866 - Restore the ability to have start failures not be fatal
   + Medium: pengine: Bug PM1 - Failcount applies to all instances of non-unique clone
   + Medium: pengine: Correctly set the state of partially active master/slave groups
   + Medium: pengine: Do not claim to be stopping an already stopped orphan
   + Medium: pengine: Ensure implies_left ordering constraints are always effective
   + Medium: pengine: Indicate each resources 'promotion' score
   + Medium: pengine: Prevent a possible use-of-NULL
   + Medium: pengine: Reprocess the current action if it changed (so that any prior dependencies are updated)
   + Medium: tengine: Bug LF-1859 - Wait for fail-count updates to complete before terminating the transition
   + Medium: tengine: Bug LF:1859 - Do not abort graphs due to our own failcount updates
   + Medium: tengine: Bug LF:1859 - Prevent the TE from interupting itself
 
 * Thu Feb 14 2008 Andrew Beekhof <abeekhof@suse.de> - 0.6.2
 - Update source tarball to revision: 28b1a8c1868b tip
 - Statistics:
     Changesets:    11
     Diff:          7 files changed, 58 insertions(+), 18 deletions(-)
 - Changes since Pacemaker-0.6.1
   + haresources2cib.py: set default-action-timeout to the default (20s)
   + haresources2cib.py: update ra parameters lists
   + Medium: SNMP: Allow the snmp subagent to be built (patch from MATSUDA, Daiki)
   + Medium: Tools: Make sure the autoconf variables in haresources2cib are expanded
 
 * Tue Feb 12 2008 Andrew Beekhof <abeekhof@suse.de> - 0.6.1
 - Update source tarball to revision: e7152d1be933 tip
 - Statistics:
     Changesets:    25
     Diff:          37 files changed, 1323 insertions(+), 227 deletions(-)
 - Changes since Pacemaker-0.6.0
   + CIB: Ensure changes to top-level attributes (like admin_epoch) cause a disk write
   + CIB: Ensure the archived file hits the disk before returning
   + CIB: Repair the ability to do 'atomic increment' updates (value="value++")
   + crmd: Bug #7 - Connecting to the crmd immediately after startup causes use-of-NULL
   + Medium: CIB: Mask cib_diff_resync results from the caller - they do not need to know
   + Medium: crmd: Delay starting the IPC server until we are fully functional
   + Medium: CTS: Fix the startup patterns
   + Medium: pengine: Bug 1820 - Allow the first resource in a group to be migrated
   + Medium: pengine: Bug 1820 - Check the colocation dependencies of resources to be migrated
 
 * Mon Jan 14 2008 Andrew Beekhof <abeekhof@suse.de> - 0.6.0
 - This is the first release of the Pacemaker Cluster Resource Manager formerly part of Heartbeat.
 - For those looking for the GUI, mgmtd, CIM or TSA components, they are now found in
   the new pacemaker-pygui project.  Build dependencies prevent them from being
   included in Heartbeat (since the built-in CRM is no longer supported) and,
   being non-core components, are not included with Pacemaker.
 - Update source tarball to revision: c94b92d550cf
 - Statistics:
     Changesets:      347
     Diff:            2272 files changed, 132508 insertions(+), 305991 deletions(-)
 - Test hardware:
     + 6-node vmware cluster (sles10-sp1/256MB/vmware stonith) on a single host (opensuse10.3/2GB/2.66GHz Quad Core2)
     + 7-node EMC Centera cluster (sles10/512MB/2GHz Xeon/ssh stonith)
 - Notes: Heartbeat Stack
     + All testing was performed with STONITH enabled
     + The CRM was enabled using the "crm respawn" directive
 - Notes: OpenAIS Stack
     + This release contains a preview of support for the OpenAIS cluster stack
     + The current release of the OpenAIS project is missing two important
     patches that we require.  OpenAIS packages containing these patches are
     available for most major distributions at:
     http://download.opensuse.org/repositories/server:/ha-clustering
     + The OpenAIS stack is not currently recommended for use in clusters that
     have shared data as STONITH support is not yet implimented
     + pingd is not yet available for use with the OpenAIS stack
     + 3 significant OpenAIS issues were found during testing of 4 and 6 node
     clusters.  We are activly working together with the OpenAIS project to
     get these resolved.
 - Pending bugs encountered during testing:
     + OpenAIS   #1736 - Openais membership took 20s to stabilize
     + Heartbeat #1750 - ipc_bufpool_update: magic number in head does not match
     + OpenAIS   #1793 - Assertion failure in memb_state_gather_enter()
     + OpenAIS   #1796 - Cluster message corruption
 - Changes since Heartbeat-2.1.2-24
   + Add OpenAIS support
   + Admin: crm_uuid - Look in the right place for Heartbeat UUID files
   + admin: Exit and indicate a problem if the crmd exits while crmadmin is performing a query
   + cib: Fix CIB_OP_UPDATE calls that modify the whole CIB
   + cib: Fix compilation when supporting the heartbeat stack
   + cib: Fix memory leaks caused by the switch to get_message_xml()
   + cib: HA_VALGRIND_ENABLED needs to be set _and_ set to 1|yes|true
   + cib: Use get_message_xml() in preference to cl_get_struct()
   + cib: Use the return value from call to write() in cib_send_plaintext()
   + Core: ccm nodes can legitimately have a node id of 0
   + Core: Fix peer-process tracking for the Heartbeat stack
   + Core: Heartbeat does not send status notifications for nodes that were already part of the cluster.  Fake them instead
   + CRM: Add children to HA_Messages such that the field name matches F_XML_TAGNAME
   + crm: Adopt a more flexible appraoch to enabling Valgrind
   + crm: Fix compilation when bzip2 is not installed
   + CRM: Future-proof get_message_xml()
   + crmd: Filter election responses based on time not FSA state
   + crmd: Handle all possible peer states in crmd_ha_status_callback()
   + crmd: Make sure the current date/time is set - prevents use-of-NULL when evaluating rules
   + crmd: Relax an assertion regrading  ccm membership instances
   + crmd: Use (node->processes&crm_proc_ais) to accurately update the CIB after replace operations
   + crmd: Heartbeat: Accurately record peer client status
   + pengine: Bug 1777 - Allow colocation with a resource in the Stopped state
   + pengine: Bug 1822 - Prevent use-of-NULL in PromoteRsc()
   + pengine: Implement three recovery policies based on op_status and op_rc
   + pengine: Parse fail-count correctly (it may be set to ININFITY)
   + pengine: Prevent graph-loop when stonith agents need to be moved around before a STONITH op
   + pengine: Prevent graph-loops when two operations have the same name+interval
   + tengine: Cancel active timers when destroying graphs
   + tengine: Ensure failcount is set correctly for failed stops/starts
   + tengine: Update failcount for oeprations that time out
   + Medium: admin: Prevent hang in crm_mon -1 when there is no cib connection - Patch from Junko IKEDA
   + Medium: cib: Require --force|-f when performing potentially dangerous commands with cibadmin
   + Medium: cib: Tweak the shutdown code
   + Medium: Common: Only count peer processes of active nodes
   + Medium: Core: Create generic cluster sign-in method
   + Medium: core: Fix compilation when Heartbeat support is disabled
   + Medium: Core: General cleanup for supporting two stacks
   + Medium: Core: iso6601 - Support parsing of time-only strings
   + Medium: core: Isolate more code that is only needed when SUPPORT_HEARTBEAT is enabled
   + Medium: crm: Improved logging of errors in the XML parser
   + Medium: crmd: Fix potential use-of-NULL in string comparison
   + Medium: crmd: Reimpliment syncronizing of CIB queries and updates when invoking the PE
   + Medium: crm_mon: Indicate when a node is both in standby mode and offline
   + Medium: pengine: Bug 1822 - Do not try an promote groups if not all of it is active
   + Medium: pengine: on_fail=nothing is an alias for 'ignore' not 'restart'
   + Medium: pengine: Prevent a potential use-of-NULL in cron_range_satisfied()
   + snmp subagent: fix a problem on displaying an unmanaged group
   + snmp subagent: use the syslog setting
   + snmp: v2 support (thanks to Keisuke MORI)
   + snmp_subagent - made it not complain about some things if shutting down
diff --git a/cts/cli/regression.crm_mon.exp b/cts/cli/regression.crm_mon.exp
new file mode 100644
index 0000000000..65a9a21995
--- /dev/null
+++ b/cts/cli/regression.crm_mon.exp
@@ -0,0 +1,855 @@
+=#=#=#= Begin test: Basic text output =#=#=#=
+Cluster Summary:
+  * Stack: corosync
+  * Current DC: cluster02 (version) - partition with quorum
+  * Last updated:
+  * Last change:
+  * 5 nodes configured
+  * 19 resource instances configured (1 DISABLED)
+
+Node List:
+  * Online: [ cluster01 cluster02 ]
+
+Active Resources:
+  * Clone Set: ping-clone [ping]:
+    * Started: [ cluster01 cluster02 ]
+  * Fencing	(stonith:fence_xvm):	 Started cluster01
+  * dummy	(ocf::pacemaker:Dummy):	 Started cluster02
+=#=#=#= End test: Basic text output - OK (0) =#=#=#=
+* Passed: crm_mon        - Basic text output
+=#=#=#= Begin test: XML output =#=#=#=
+<pacemaker-result api-version="2.2" request="crm_mon --output-as=xml">
+  <summary>
+    <stack type="corosync"/>
+    <current_dc present="true" version="" with_quorum="true"/>
+    <last_update time=""/>
+    <last_change time=""/>
+    <nodes_configured number="5"/>
+    <resources_configured number="19" disabled="1" blocked="0"/>
+    <cluster_options stonith-enabled="true" symmetric-cluster="true" no-quorum-policy="stop" maintenance-mode="false"/>
+  </summary>
+  <nodes>
+    <node name="cluster01" id="1" online="true" standby="false" standby_onfail="false" maintenance="false" pending="false" unclean="false" shutdown="false" expected_up="true" is_dc="false" resources_running="2" type="member"/>
+    <node name="cluster02" id="2" online="true" standby="false" standby_onfail="false" maintenance="false" pending="false" unclean="false" shutdown="false" expected_up="true" is_dc="true" resources_running="2" type="member"/>
+    <node name="httpd-bundle-0" id="httpd-bundle-0" online="false" standby="false" standby_onfail="false" maintenance="false" pending="false" unclean="false" shutdown="false" expected_up="false" is_dc="false" resources_running="0" type="remote" id_as_resource="httpd-bundle-docker-0"/>
+    <node name="httpd-bundle-1" id="httpd-bundle-1" online="false" standby="false" standby_onfail="false" maintenance="false" pending="false" unclean="false" shutdown="false" expected_up="false" is_dc="false" resources_running="0" type="remote" id_as_resource="httpd-bundle-docker-1"/>
+    <node name="httpd-bundle-2" id="httpd-bundle-2" online="false" standby="false" standby_onfail="false" maintenance="false" pending="false" unclean="false" shutdown="false" expected_up="false" is_dc="false" resources_running="0" type="remote" id_as_resource="httpd-bundle-docker-2"/>
+  </nodes>
+  <resources>
+    <clone id="ping-clone" multi_state="false" unique="false" managed="true" failed="false" failure_ignored="false">
+      <resource id="ping" resource_agent="ocf::pacemaker:ping" role="Started" active="true" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="1">
+        <node name="cluster02" id="2" cached="true"/>
+      </resource>
+      <resource id="ping" resource_agent="ocf::pacemaker:ping" role="Started" active="true" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="1">
+        <node name="cluster01" id="1" cached="true"/>
+      </resource>
+    </clone>
+    <resource id="Fencing" resource_agent="stonith:fence_xvm" role="Started" active="true" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="1">
+      <node name="cluster01" id="1" cached="true"/>
+    </resource>
+    <resource id="dummy" resource_agent="ocf::pacemaker:Dummy" role="Started" active="true" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="1">
+      <node name="cluster02" id="2" cached="true"/>
+    </resource>
+    <resource id="inactive-dummy" resource_agent="ocf::pacemaker:Dummy" role="Stopped" target_role="stopped" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
+    <clone id="inactive-clone-master" multi_state="true" unique="false" managed="true" failed="false" failure_ignored="false">
+      <resource id="inactive-clone" resource_agent="ocf::pacemaker:Stateful" role="Stopped" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
+      <resource id="inactive-clone" resource_agent="ocf::pacemaker:Stateful" role="Stopped" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
+    </clone>
+    <bundle id="httpd-bundle" type="docker" image="pcmk:http" unique="false" managed="true" failed="false">
+      <replica id="0">
+        <resource id="httpd-bundle-ip-192.168.122.131" resource_agent="ocf::heartbeat:IPaddr2" role="Stopped" target_role="Started" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
+        <resource id="httpd" resource_agent="ocf::heartbeat:apache" role="Stopped" target_role="Started" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
+        <resource id="httpd-bundle-docker-0" resource_agent="ocf::heartbeat:docker" role="Stopped" target_role="Started" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
+        <resource id="httpd-bundle-0" resource_agent="ocf::pacemaker:remote" role="Stopped" target_role="Started" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
+      </replica>
+      <replica id="1">
+        <resource id="httpd-bundle-ip-192.168.122.132" resource_agent="ocf::heartbeat:IPaddr2" role="Stopped" target_role="Started" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
+        <resource id="httpd" resource_agent="ocf::heartbeat:apache" role="Stopped" target_role="Started" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
+        <resource id="httpd-bundle-docker-1" resource_agent="ocf::heartbeat:docker" role="Stopped" target_role="Started" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
+        <resource id="httpd-bundle-1" resource_agent="ocf::pacemaker:remote" role="Stopped" target_role="Started" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
+      </replica>
+      <replica id="2">
+        <resource id="httpd-bundle-ip-192.168.122.133" resource_agent="ocf::heartbeat:IPaddr2" role="Stopped" target_role="Started" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
+        <resource id="httpd" resource_agent="ocf::heartbeat:apache" role="Stopped" target_role="Started" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
+        <resource id="httpd-bundle-docker-2" resource_agent="ocf::heartbeat:docker" role="Stopped" target_role="Started" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
+        <resource id="httpd-bundle-2" resource_agent="ocf::pacemaker:remote" role="Stopped" target_role="Started" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
+      </replica>
+    </bundle>
+  </resources>
+  <node_attributes>
+    <node name="cluster01">
+      <attribute name="location" value="office"/>
+      <attribute name="pingd" value="1000" expected="1000"/>
+    </node>
+    <node name="cluster02">
+      <attribute name="pingd" value="1000" expected="1000"/>
+    </node>
+  </node_attributes>
+  <node_history>
+    <node name="cluster02">
+      <resource_history id="ping" orphan="false" migration-threshold="1000000">
+        <operation_history call="11" task="start" exec-time="2044ms" queue-time="0ms" rc="0" rc_text="ok"/>
+        <operation_history call="12" task="monitor" interval="10000ms" exec-time="2031ms" queue-time="0ms" rc="0" rc_text="ok"/>
+      </resource_history>
+      <resource_history id="dummy" orphan="false" migration-threshold="1000000">
+        <operation_history call="18" task="start" exec-time="6020ms" queue-time="0ms" rc="0" rc_text="ok"/>
+        <operation_history call="19" task="monitor" interval="60000ms" exec-time="6015ms" queue-time="0ms" rc="0" rc_text="ok"/>
+      </resource_history>
+    </node>
+    <node name="cluster01">
+      <resource_history id="ping" orphan="false" migration-threshold="1000000">
+        <operation_history call="17" task="start" exec-time="2038ms" queue-time="0ms" rc="0" rc_text="ok"/>
+        <operation_history call="18" task="monitor" interval="10000ms" exec-time="2034ms" queue-time="0ms" rc="0" rc_text="ok"/>
+      </resource_history>
+      <resource_history id="Fencing" orphan="false" migration-threshold="1000000">
+        <operation_history call="15" task="start" exec-time="36ms" queue-time="0ms" rc="0" rc_text="ok"/>
+        <operation_history call="19" task="monitor" interval="60000ms" exec-time="24ms" queue-time="0ms" rc="0" rc_text="ok"/>
+      </resource_history>
+      <resource_history id="dummy" orphan="false" migration-threshold="1000000">
+        <operation_history call="16" task="stop" exec-time="6048ms" queue-time="0ms" rc="0" rc_text="ok"/>
+      </resource_history>
+    </node>
+  </node_history>
+  <status code="0" message="OK"/>
+</pacemaker-result>
+=#=#=#= End test: XML output - OK (0) =#=#=#=
+* Passed: crm_mon        - XML output
+=#=#=#= Begin test: Basic text output without node section =#=#=#=
+Cluster Summary:
+  * Stack: corosync
+  * Current DC: cluster02 (version) - partition with quorum
+  * Last updated:
+  * Last change:
+  * 5 nodes configured
+  * 19 resource instances configured (1 DISABLED)
+
+Active Resources:
+  * Clone Set: ping-clone [ping]:
+    * Started: [ cluster01 cluster02 ]
+  * Fencing	(stonith:fence_xvm):	 Started cluster01
+  * dummy	(ocf::pacemaker:Dummy):	 Started cluster02
+=#=#=#= End test: Basic text output without node section - OK (0) =#=#=#=
+* Passed: crm_mon        - Basic text output without node section
+=#=#=#= Begin test: XML output without the node section =#=#=#=
+<pacemaker-result api-version="2.2" request="crm_mon --output-as=xml --exclude=nodes">
+  <summary>
+    <stack type="corosync"/>
+    <current_dc present="true" version="" with_quorum="true"/>
+    <last_update time=""/>
+    <last_change time=""/>
+    <nodes_configured number="5"/>
+    <resources_configured number="19" disabled="1" blocked="0"/>
+    <cluster_options stonith-enabled="true" symmetric-cluster="true" no-quorum-policy="stop" maintenance-mode="false"/>
+  </summary>
+  <resources>
+    <clone id="ping-clone" multi_state="false" unique="false" managed="true" failed="false" failure_ignored="false">
+      <resource id="ping" resource_agent="ocf::pacemaker:ping" role="Started" active="true" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="1">
+        <node name="cluster02" id="2" cached="true"/>
+      </resource>
+      <resource id="ping" resource_agent="ocf::pacemaker:ping" role="Started" active="true" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="1">
+        <node name="cluster01" id="1" cached="true"/>
+      </resource>
+    </clone>
+    <resource id="Fencing" resource_agent="stonith:fence_xvm" role="Started" active="true" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="1">
+      <node name="cluster01" id="1" cached="true"/>
+    </resource>
+    <resource id="dummy" resource_agent="ocf::pacemaker:Dummy" role="Started" active="true" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="1">
+      <node name="cluster02" id="2" cached="true"/>
+    </resource>
+    <resource id="inactive-dummy" resource_agent="ocf::pacemaker:Dummy" role="Stopped" target_role="stopped" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
+    <clone id="inactive-clone-master" multi_state="true" unique="false" managed="true" failed="false" failure_ignored="false">
+      <resource id="inactive-clone" resource_agent="ocf::pacemaker:Stateful" role="Stopped" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
+      <resource id="inactive-clone" resource_agent="ocf::pacemaker:Stateful" role="Stopped" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
+    </clone>
+    <bundle id="httpd-bundle" type="docker" image="pcmk:http" unique="false" managed="true" failed="false">
+      <replica id="0">
+        <resource id="httpd-bundle-ip-192.168.122.131" resource_agent="ocf::heartbeat:IPaddr2" role="Stopped" target_role="Started" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
+        <resource id="httpd" resource_agent="ocf::heartbeat:apache" role="Stopped" target_role="Started" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
+        <resource id="httpd-bundle-docker-0" resource_agent="ocf::heartbeat:docker" role="Stopped" target_role="Started" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
+        <resource id="httpd-bundle-0" resource_agent="ocf::pacemaker:remote" role="Stopped" target_role="Started" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
+      </replica>
+      <replica id="1">
+        <resource id="httpd-bundle-ip-192.168.122.132" resource_agent="ocf::heartbeat:IPaddr2" role="Stopped" target_role="Started" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
+        <resource id="httpd" resource_agent="ocf::heartbeat:apache" role="Stopped" target_role="Started" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
+        <resource id="httpd-bundle-docker-1" resource_agent="ocf::heartbeat:docker" role="Stopped" target_role="Started" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
+        <resource id="httpd-bundle-1" resource_agent="ocf::pacemaker:remote" role="Stopped" target_role="Started" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
+      </replica>
+      <replica id="2">
+        <resource id="httpd-bundle-ip-192.168.122.133" resource_agent="ocf::heartbeat:IPaddr2" role="Stopped" target_role="Started" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
+        <resource id="httpd" resource_agent="ocf::heartbeat:apache" role="Stopped" target_role="Started" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
+        <resource id="httpd-bundle-docker-2" resource_agent="ocf::heartbeat:docker" role="Stopped" target_role="Started" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
+        <resource id="httpd-bundle-2" resource_agent="ocf::pacemaker:remote" role="Stopped" target_role="Started" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
+      </replica>
+    </bundle>
+  </resources>
+  <node_attributes>
+    <node name="cluster01">
+      <attribute name="location" value="office"/>
+      <attribute name="pingd" value="1000" expected="1000"/>
+    </node>
+    <node name="cluster02">
+      <attribute name="pingd" value="1000" expected="1000"/>
+    </node>
+  </node_attributes>
+  <node_history>
+    <node name="cluster02">
+      <resource_history id="ping" orphan="false" migration-threshold="1000000">
+        <operation_history call="11" task="start" exec-time="2044ms" queue-time="0ms" rc="0" rc_text="ok"/>
+        <operation_history call="12" task="monitor" interval="10000ms" exec-time="2031ms" queue-time="0ms" rc="0" rc_text="ok"/>
+      </resource_history>
+      <resource_history id="dummy" orphan="false" migration-threshold="1000000">
+        <operation_history call="18" task="start" exec-time="6020ms" queue-time="0ms" rc="0" rc_text="ok"/>
+        <operation_history call="19" task="monitor" interval="60000ms" exec-time="6015ms" queue-time="0ms" rc="0" rc_text="ok"/>
+      </resource_history>
+    </node>
+    <node name="cluster01">
+      <resource_history id="ping" orphan="false" migration-threshold="1000000">
+        <operation_history call="17" task="start" exec-time="2038ms" queue-time="0ms" rc="0" rc_text="ok"/>
+        <operation_history call="18" task="monitor" interval="10000ms" exec-time="2034ms" queue-time="0ms" rc="0" rc_text="ok"/>
+      </resource_history>
+      <resource_history id="Fencing" orphan="false" migration-threshold="1000000">
+        <operation_history call="15" task="start" exec-time="36ms" queue-time="0ms" rc="0" rc_text="ok"/>
+        <operation_history call="19" task="monitor" interval="60000ms" exec-time="24ms" queue-time="0ms" rc="0" rc_text="ok"/>
+      </resource_history>
+      <resource_history id="dummy" orphan="false" migration-threshold="1000000">
+        <operation_history call="16" task="stop" exec-time="6048ms" queue-time="0ms" rc="0" rc_text="ok"/>
+      </resource_history>
+    </node>
+  </node_history>
+  <status code="0" message="OK"/>
+</pacemaker-result>
+=#=#=#= End test: XML output without the node section - OK (0) =#=#=#=
+* Passed: crm_mon        - XML output without the node section
+=#=#=#= Begin test: Text output with only the node section =#=#=#=
+Node List:
+  * Online: [ cluster01 cluster02 ]
+=#=#=#= End test: Text output with only the node section - OK (0) =#=#=#=
+* Passed: crm_mon        - Text output with only the node section
+=#=#=#= Begin test: Complete text output =#=#=#=
+Cluster Summary:
+  * Stack: corosync
+  * Current DC: cluster02 (version) - partition with quorum
+  * Last updated:
+  * Last change:
+  * 5 nodes configured
+  * 19 resource instances configured (1 DISABLED)
+
+Node List:
+  * Online: [ cluster01 cluster02 ]
+
+Active Resources:
+  * Clone Set: ping-clone [ping]:
+    * Started: [ cluster01 cluster02 ]
+  * Fencing	(stonith:fence_xvm):	 Started cluster01
+  * dummy	(ocf::pacemaker:Dummy):	 Started cluster02
+
+Node Attributes:
+  * Node: cluster01:
+    * location                        	: office    
+    * pingd                           	: 1000      
+  * Node: cluster02:
+    * pingd                           	: 1000      
+
+Operations:
+  * Node: cluster02:
+    * ping: migration-threshold=1000000:
+      * (11) start
+      * (12) monitor: interval="10000ms"
+    * dummy: migration-threshold=1000000:
+      * (18) start
+      * (19) monitor: interval="60000ms"
+  * Node: cluster01:
+    * ping: migration-threshold=1000000:
+      * (17) start
+      * (18) monitor: interval="10000ms"
+    * Fencing: migration-threshold=1000000:
+      * (15) start
+      * (19) monitor: interval="60000ms"
+    * dummy: migration-threshold=1000000:
+      * (16) stop
+=#=#=#= End test: Complete text output - OK (0) =#=#=#=
+* Passed: crm_mon        - Complete text output
+=#=#=#= Begin test: Complete text output filtered by node =#=#=#=
+Cluster Summary:
+  * Stack: corosync
+  * Current DC: cluster02 (version) - partition with quorum
+  * Last updated:
+  * Last change:
+  * 5 nodes configured
+  * 19 resource instances configured (1 DISABLED)
+
+Node List:
+  * Online: [ cluster01 ]
+
+Active Resources:
+  * Clone Set: ping-clone [ping]:
+    * Started: [ cluster01 ]
+  * Fencing	(stonith:fence_xvm):	 Started cluster01
+
+Node Attributes:
+  * Node: cluster01:
+    * location                        	: office    
+    * pingd                           	: 1000      
+
+Operations:
+  * Node: cluster01:
+    * ping: migration-threshold=1000000:
+      * (17) start
+      * (18) monitor: interval="10000ms"
+    * Fencing: migration-threshold=1000000:
+      * (15) start
+      * (19) monitor: interval="60000ms"
+    * dummy: migration-threshold=1000000:
+      * (16) stop
+=#=#=#= End test: Complete text output filtered by node - OK (0) =#=#=#=
+* Passed: crm_mon        - Complete text output filtered by node
+=#=#=#= Begin test: XML output filtered by node =#=#=#=
+<pacemaker-result api-version="2.2" request="crm_mon --output-as xml --include=all --node=cluster01">
+  <summary>
+    <stack type="corosync"/>
+    <current_dc present="true" version="" with_quorum="true"/>
+    <last_update time=""/>
+    <last_change time=""/>
+    <nodes_configured number="5"/>
+    <resources_configured number="19" disabled="1" blocked="0"/>
+    <cluster_options stonith-enabled="true" symmetric-cluster="true" no-quorum-policy="stop" maintenance-mode="false"/>
+  </summary>
+  <nodes>
+    <node name="cluster01" id="1" online="true" standby="false" standby_onfail="false" maintenance="false" pending="false" unclean="false" shutdown="false" expected_up="true" is_dc="false" resources_running="2" type="member"/>
+  </nodes>
+  <resources>
+    <clone id="ping-clone" multi_state="false" unique="false" managed="true" failed="false" failure_ignored="false">
+      <resource id="ping" resource_agent="ocf::pacemaker:ping" role="Started" active="true" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="1">
+        <node name="cluster01" id="1" cached="true"/>
+      </resource>
+    </clone>
+    <resource id="Fencing" resource_agent="stonith:fence_xvm" role="Started" active="true" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="1">
+      <node name="cluster01" id="1" cached="true"/>
+    </resource>
+    <resource id="inactive-dummy" resource_agent="ocf::pacemaker:Dummy" role="Stopped" target_role="stopped" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
+    <clone id="inactive-clone-master" multi_state="true" unique="false" managed="true" failed="false" failure_ignored="false">
+      <resource id="inactive-clone" resource_agent="ocf::pacemaker:Stateful" role="Stopped" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
+      <resource id="inactive-clone" resource_agent="ocf::pacemaker:Stateful" role="Stopped" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
+    </clone>
+    <bundle id="httpd-bundle" type="docker" image="pcmk:http" unique="false" managed="true" failed="false">
+      <replica id="0">
+        <resource id="httpd-bundle-ip-192.168.122.131" resource_agent="ocf::heartbeat:IPaddr2" role="Stopped" target_role="Started" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
+        <resource id="httpd" resource_agent="ocf::heartbeat:apache" role="Stopped" target_role="Started" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
+        <resource id="httpd-bundle-docker-0" resource_agent="ocf::heartbeat:docker" role="Stopped" target_role="Started" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
+        <resource id="httpd-bundle-0" resource_agent="ocf::pacemaker:remote" role="Stopped" target_role="Started" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
+      </replica>
+      <replica id="1">
+        <resource id="httpd-bundle-ip-192.168.122.132" resource_agent="ocf::heartbeat:IPaddr2" role="Stopped" target_role="Started" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
+        <resource id="httpd" resource_agent="ocf::heartbeat:apache" role="Stopped" target_role="Started" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
+        <resource id="httpd-bundle-docker-1" resource_agent="ocf::heartbeat:docker" role="Stopped" target_role="Started" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
+        <resource id="httpd-bundle-1" resource_agent="ocf::pacemaker:remote" role="Stopped" target_role="Started" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
+      </replica>
+      <replica id="2">
+        <resource id="httpd-bundle-ip-192.168.122.133" resource_agent="ocf::heartbeat:IPaddr2" role="Stopped" target_role="Started" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
+        <resource id="httpd" resource_agent="ocf::heartbeat:apache" role="Stopped" target_role="Started" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
+        <resource id="httpd-bundle-docker-2" resource_agent="ocf::heartbeat:docker" role="Stopped" target_role="Started" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
+        <resource id="httpd-bundle-2" resource_agent="ocf::pacemaker:remote" role="Stopped" target_role="Started" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
+      </replica>
+    </bundle>
+  </resources>
+  <node_attributes>
+    <node name="cluster01">
+      <attribute name="location" value="office"/>
+      <attribute name="pingd" value="1000" expected="1000"/>
+    </node>
+  </node_attributes>
+  <node_history>
+    <node name="cluster01">
+      <resource_history id="ping" orphan="false" migration-threshold="1000000">
+        <operation_history call="17" task="start" exec-time="2038ms" queue-time="0ms" rc="0" rc_text="ok"/>
+        <operation_history call="18" task="monitor" interval="10000ms" exec-time="2034ms" queue-time="0ms" rc="0" rc_text="ok"/>
+      </resource_history>
+      <resource_history id="Fencing" orphan="false" migration-threshold="1000000">
+        <operation_history call="15" task="start" exec-time="36ms" queue-time="0ms" rc="0" rc_text="ok"/>
+        <operation_history call="19" task="monitor" interval="60000ms" exec-time="24ms" queue-time="0ms" rc="0" rc_text="ok"/>
+      </resource_history>
+      <resource_history id="dummy" orphan="false" migration-threshold="1000000">
+        <operation_history call="16" task="stop" exec-time="6048ms" queue-time="0ms" rc="0" rc_text="ok"/>
+      </resource_history>
+    </node>
+  </node_history>
+  <status code="0" message="OK"/>
+</pacemaker-result>
+=#=#=#= End test: XML output filtered by node - OK (0) =#=#=#=
+* Passed: crm_mon        - XML output filtered by node
+=#=#=#= Begin test: Complete text output filtered by tag =#=#=#=
+Cluster Summary:
+  * Stack: corosync
+  * Current DC: cluster02 (version) - partition with quorum
+  * Last updated:
+  * Last change:
+  * 5 nodes configured
+  * 19 resource instances configured (1 DISABLED)
+
+Node List:
+  * Online: [ cluster02 ]
+
+Active Resources:
+  * Clone Set: ping-clone [ping]:
+    * Started: [ cluster02 ]
+  * dummy	(ocf::pacemaker:Dummy):	 Started cluster02
+
+Node Attributes:
+  * Node: cluster02:
+    * pingd                           	: 1000      
+
+Operations:
+  * Node: cluster02:
+    * ping: migration-threshold=1000000:
+      * (11) start
+      * (12) monitor: interval="10000ms"
+    * dummy: migration-threshold=1000000:
+      * (18) start
+      * (19) monitor: interval="60000ms"
+=#=#=#= End test: Complete text output filtered by tag - OK (0) =#=#=#=
+* Passed: crm_mon        - Complete text output filtered by tag
+=#=#=#= Begin test: XML output filtered by tag =#=#=#=
+<pacemaker-result api-version="2.2" request="crm_mon --output-as=xml --include=all --node=even-nodes">
+  <summary>
+    <stack type="corosync"/>
+    <current_dc present="true" version="" with_quorum="true"/>
+    <last_update time=""/>
+    <last_change time=""/>
+    <nodes_configured number="5"/>
+    <resources_configured number="19" disabled="1" blocked="0"/>
+    <cluster_options stonith-enabled="true" symmetric-cluster="true" no-quorum-policy="stop" maintenance-mode="false"/>
+  </summary>
+  <nodes>
+    <node name="cluster02" id="2" online="true" standby="false" standby_onfail="false" maintenance="false" pending="false" unclean="false" shutdown="false" expected_up="true" is_dc="true" resources_running="2" type="member"/>
+  </nodes>
+  <resources>
+    <clone id="ping-clone" multi_state="false" unique="false" managed="true" failed="false" failure_ignored="false">
+      <resource id="ping" resource_agent="ocf::pacemaker:ping" role="Started" active="true" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="1">
+        <node name="cluster02" id="2" cached="true"/>
+      </resource>
+    </clone>
+    <resource id="dummy" resource_agent="ocf::pacemaker:Dummy" role="Started" active="true" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="1">
+      <node name="cluster02" id="2" cached="true"/>
+    </resource>
+    <resource id="inactive-dummy" resource_agent="ocf::pacemaker:Dummy" role="Stopped" target_role="stopped" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
+    <clone id="inactive-clone-master" multi_state="true" unique="false" managed="true" failed="false" failure_ignored="false">
+      <resource id="inactive-clone" resource_agent="ocf::pacemaker:Stateful" role="Stopped" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
+      <resource id="inactive-clone" resource_agent="ocf::pacemaker:Stateful" role="Stopped" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
+    </clone>
+    <bundle id="httpd-bundle" type="docker" image="pcmk:http" unique="false" managed="true" failed="false">
+      <replica id="0">
+        <resource id="httpd-bundle-ip-192.168.122.131" resource_agent="ocf::heartbeat:IPaddr2" role="Stopped" target_role="Started" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
+        <resource id="httpd" resource_agent="ocf::heartbeat:apache" role="Stopped" target_role="Started" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
+        <resource id="httpd-bundle-docker-0" resource_agent="ocf::heartbeat:docker" role="Stopped" target_role="Started" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
+        <resource id="httpd-bundle-0" resource_agent="ocf::pacemaker:remote" role="Stopped" target_role="Started" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
+      </replica>
+      <replica id="1">
+        <resource id="httpd-bundle-ip-192.168.122.132" resource_agent="ocf::heartbeat:IPaddr2" role="Stopped" target_role="Started" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
+        <resource id="httpd" resource_agent="ocf::heartbeat:apache" role="Stopped" target_role="Started" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
+        <resource id="httpd-bundle-docker-1" resource_agent="ocf::heartbeat:docker" role="Stopped" target_role="Started" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
+        <resource id="httpd-bundle-1" resource_agent="ocf::pacemaker:remote" role="Stopped" target_role="Started" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
+      </replica>
+      <replica id="2">
+        <resource id="httpd-bundle-ip-192.168.122.133" resource_agent="ocf::heartbeat:IPaddr2" role="Stopped" target_role="Started" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
+        <resource id="httpd" resource_agent="ocf::heartbeat:apache" role="Stopped" target_role="Started" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
+        <resource id="httpd-bundle-docker-2" resource_agent="ocf::heartbeat:docker" role="Stopped" target_role="Started" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
+        <resource id="httpd-bundle-2" resource_agent="ocf::pacemaker:remote" role="Stopped" target_role="Started" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
+      </replica>
+    </bundle>
+  </resources>
+  <node_attributes>
+    <node name="cluster02">
+      <attribute name="pingd" value="1000" expected="1000"/>
+    </node>
+  </node_attributes>
+  <node_history>
+    <node name="cluster02">
+      <resource_history id="ping" orphan="false" migration-threshold="1000000">
+        <operation_history call="11" task="start" exec-time="2044ms" queue-time="0ms" rc="0" rc_text="ok"/>
+        <operation_history call="12" task="monitor" interval="10000ms" exec-time="2031ms" queue-time="0ms" rc="0" rc_text="ok"/>
+      </resource_history>
+      <resource_history id="dummy" orphan="false" migration-threshold="1000000">
+        <operation_history call="18" task="start" exec-time="6020ms" queue-time="0ms" rc="0" rc_text="ok"/>
+        <operation_history call="19" task="monitor" interval="60000ms" exec-time="6015ms" queue-time="0ms" rc="0" rc_text="ok"/>
+      </resource_history>
+    </node>
+  </node_history>
+  <status code="0" message="OK"/>
+</pacemaker-result>
+=#=#=#= End test: XML output filtered by tag - OK (0) =#=#=#=
+* Passed: crm_mon        - XML output filtered by tag
+=#=#=#= Begin test: Basic text output filtered by node that doesn't exist =#=#=#=
+Cluster Summary:
+  * Stack: corosync
+  * Current DC: cluster02 (version) - partition with quorum
+  * Last updated:
+  * Last change:
+  * 5 nodes configured
+  * 19 resource instances configured (1 DISABLED)
+
+Active Resources:
+  * No active resources
+=#=#=#= End test: Basic text output filtered by node that doesn't exist - OK (0) =#=#=#=
+* Passed: crm_mon        - Basic text output filtered by node that doesn't exist
+=#=#=#= Begin test: XML output filtered by node that doesn't exist =#=#=#=
+<pacemaker-result api-version="2.2" request="crm_mon --output-as=xml --node=blah">
+  <summary>
+    <stack type="corosync"/>
+    <current_dc present="true" version="" with_quorum="true"/>
+    <last_update time=""/>
+    <last_change time=""/>
+    <nodes_configured number="5"/>
+    <resources_configured number="19" disabled="1" blocked="0"/>
+    <cluster_options stonith-enabled="true" symmetric-cluster="true" no-quorum-policy="stop" maintenance-mode="false"/>
+  </summary>
+  <nodes/>
+  <resources>
+    <resource id="inactive-dummy" resource_agent="ocf::pacemaker:Dummy" role="Stopped" target_role="stopped" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
+    <clone id="inactive-clone-master" multi_state="true" unique="false" managed="true" failed="false" failure_ignored="false">
+      <resource id="inactive-clone" resource_agent="ocf::pacemaker:Stateful" role="Stopped" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
+      <resource id="inactive-clone" resource_agent="ocf::pacemaker:Stateful" role="Stopped" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
+    </clone>
+    <bundle id="httpd-bundle" type="docker" image="pcmk:http" unique="false" managed="true" failed="false">
+      <replica id="0">
+        <resource id="httpd-bundle-ip-192.168.122.131" resource_agent="ocf::heartbeat:IPaddr2" role="Stopped" target_role="Started" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
+        <resource id="httpd" resource_agent="ocf::heartbeat:apache" role="Stopped" target_role="Started" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
+        <resource id="httpd-bundle-docker-0" resource_agent="ocf::heartbeat:docker" role="Stopped" target_role="Started" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
+        <resource id="httpd-bundle-0" resource_agent="ocf::pacemaker:remote" role="Stopped" target_role="Started" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
+      </replica>
+      <replica id="1">
+        <resource id="httpd-bundle-ip-192.168.122.132" resource_agent="ocf::heartbeat:IPaddr2" role="Stopped" target_role="Started" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
+        <resource id="httpd" resource_agent="ocf::heartbeat:apache" role="Stopped" target_role="Started" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
+        <resource id="httpd-bundle-docker-1" resource_agent="ocf::heartbeat:docker" role="Stopped" target_role="Started" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
+        <resource id="httpd-bundle-1" resource_agent="ocf::pacemaker:remote" role="Stopped" target_role="Started" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
+      </replica>
+      <replica id="2">
+        <resource id="httpd-bundle-ip-192.168.122.133" resource_agent="ocf::heartbeat:IPaddr2" role="Stopped" target_role="Started" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
+        <resource id="httpd" resource_agent="ocf::heartbeat:apache" role="Stopped" target_role="Started" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
+        <resource id="httpd-bundle-docker-2" resource_agent="ocf::heartbeat:docker" role="Stopped" target_role="Started" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
+        <resource id="httpd-bundle-2" resource_agent="ocf::pacemaker:remote" role="Stopped" target_role="Started" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
+      </replica>
+    </bundle>
+  </resources>
+  <status code="0" message="OK"/>
+</pacemaker-result>
+=#=#=#= End test: XML output filtered by node that doesn't exist - OK (0) =#=#=#=
+* Passed: crm_mon        - XML output filtered by node that doesn't exist
+=#=#=#= Begin test: Detailed text output =#=#=#=
+Cluster Summary:
+  * Stack: corosync
+  * Current DC: cluster02 (2) (version) - partition with quorum
+  * Last updated:
+  * Last change:
+  * 5 nodes configured
+  * 19 resource instances configured (1 DISABLED)
+
+Node List:
+  * Online: [ cluster01 (1) cluster02 (2) ]
+
+Active Resources:
+  * Clone Set: ping-clone [ping]:
+    * ping	(ocf::pacemaker:ping):	 Started cluster02
+    * ping	(ocf::pacemaker:ping):	 Started cluster01
+  * Fencing	(stonith:fence_xvm):	 Started cluster01
+  * dummy	(ocf::pacemaker:Dummy):	 Started cluster02
+=#=#=#= End test: Detailed text output - OK (0) =#=#=#=
+* Passed: crm_mon        - Detailed text output
+=#=#=#= Begin test: Basic text output with inactive resources =#=#=#=
+Cluster Summary:
+  * Stack: corosync
+  * Current DC: cluster02 (version) - partition with quorum
+  * Last updated:
+  * Last change:
+  * 5 nodes configured
+  * 19 resource instances configured (1 DISABLED)
+
+Node List:
+  * Online: [ cluster01 cluster02 ]
+
+Full List of Resources:
+  * Clone Set: ping-clone [ping]:
+    * Started: [ cluster01 cluster02 ]
+  * Fencing	(stonith:fence_xvm):	 Started cluster01
+  * dummy	(ocf::pacemaker:Dummy):	 Started cluster02
+  * inactive-dummy	(ocf::pacemaker:Dummy):	 Stopped (disabled)
+  * Clone Set: inactive-clone-master [inactive-clone] (promotable):
+    * Stopped: [ cluster01 cluster02 ]
+  * Container bundle set: httpd-bundle [pcmk:http]:
+    * httpd-bundle-0 (192.168.122.131)	(ocf::heartbeat:apache):	 Stopped
+    * httpd-bundle-1 (192.168.122.132)	(ocf::heartbeat:apache):	 Stopped
+    * httpd-bundle-2 (192.168.122.133)	(ocf::heartbeat:apache):	 Stopped
+=#=#=#= End test: Basic text output with inactive resources - OK (0) =#=#=#=
+* Passed: crm_mon        - Basic text output with inactive resources
+=#=#=#= Begin test: Basic text output with inactive resources, filtered by node =#=#=#=
+Cluster Summary:
+  * Stack: corosync
+  * Current DC: cluster02 (version) - partition with quorum
+  * Last updated:
+  * Last change:
+  * 5 nodes configured
+  * 19 resource instances configured (1 DISABLED)
+
+Node List:
+  * Online: [ cluster02 ]
+
+Full List of Resources:
+  * Clone Set: ping-clone [ping]:
+    * Started: [ cluster02 ]
+  * dummy	(ocf::pacemaker:Dummy):	 Started cluster02
+  * inactive-dummy	(ocf::pacemaker:Dummy):	 Stopped (disabled)
+  * Clone Set: inactive-clone-master [inactive-clone] (promotable):
+    * Stopped: [ cluster02 ]
+  * Container bundle set: httpd-bundle [pcmk:http]:
+    * httpd-bundle-0 (192.168.122.131)	(ocf::heartbeat:apache):	 Stopped
+    * httpd-bundle-1 (192.168.122.132)	(ocf::heartbeat:apache):	 Stopped
+    * httpd-bundle-2 (192.168.122.133)	(ocf::heartbeat:apache):	 Stopped
+=#=#=#= End test: Basic text output with inactive resources, filtered by node - OK (0) =#=#=#=
+* Passed: crm_mon        - Basic text output with inactive resources, filtered by node
+=#=#=#= Begin test: Text output of partially active resources =#=#=#=
+Cluster Summary:
+  * Stack: corosync
+  * Current DC: cluster02 (version) - partition with quorum
+  * Last updated:
+  * Last change:
+  * 4 nodes configured
+  * 11 resource instances configured
+
+Node List:
+  * Online: [ cluster01 cluster02 ]
+  * GuestOnline: [ httpd-bundle-0@cluster02 httpd-bundle-1@cluster01 ]
+
+Active Resources:
+  * Clone Set: ping-clone [ping]:
+    * Started: [ cluster01 ]
+  * Fencing	(stonith:fence_xvm):	 Started cluster01
+  * Container bundle set: httpd-bundle [pcmk:http]:
+    * httpd-bundle-0 (192.168.122.131)	(ocf::heartbeat:apache):	 Started
+    * httpd-bundle-1 (192.168.122.132)	(ocf::heartbeat:apache):	 Stopped
+=#=#=#= End test: Text output of partially active resources - OK (0) =#=#=#=
+* Passed: crm_mon        - Text output of partially active resources
+=#=#=#= Begin test: XML output of partially active resources =#=#=#=
+<pacemaker-result api-version="2.2" request="crm_mon -1 --output-as=xml">
+  <summary>
+    <stack type="corosync"/>
+    <current_dc present="true" version="" with_quorum="true"/>
+    <last_update time=""/>
+    <last_change time=""/>
+    <nodes_configured number="4"/>
+    <resources_configured number="11" disabled="0" blocked="0"/>
+    <cluster_options stonith-enabled="true" symmetric-cluster="true" no-quorum-policy="stop" maintenance-mode="false"/>
+  </summary>
+  <nodes>
+    <node name="cluster01" id="1" online="true" standby="false" standby_onfail="false" maintenance="false" pending="false" unclean="false" shutdown="false" expected_up="true" is_dc="false" resources_running="5" type="member"/>
+    <node name="cluster02" id="2" online="true" standby="false" standby_onfail="false" maintenance="false" pending="false" unclean="false" shutdown="false" expected_up="true" is_dc="true" resources_running="3" type="member"/>
+    <node name="httpd-bundle-0" id="httpd-bundle-0" online="true" standby="false" standby_onfail="false" maintenance="false" pending="false" unclean="false" shutdown="false" expected_up="false" is_dc="false" resources_running="1" type="remote" id_as_resource="httpd-bundle-docker-0"/>
+    <node name="httpd-bundle-1" id="httpd-bundle-1" online="true" standby="false" standby_onfail="false" maintenance="false" pending="false" unclean="false" shutdown="false" expected_up="false" is_dc="false" resources_running="0" type="remote" id_as_resource="httpd-bundle-docker-1"/>
+  </nodes>
+  <resources>
+    <clone id="ping-clone" multi_state="false" unique="false" managed="true" failed="false" failure_ignored="false">
+      <resource id="ping" resource_agent="ocf::pacemaker:ping" role="Started" active="true" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="1">
+        <node name="cluster01" id="1" cached="true"/>
+      </resource>
+      <resource id="ping" resource_agent="ocf::pacemaker:ping" role="Stopped" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
+    </clone>
+    <resource id="Fencing" resource_agent="stonith:fence_xvm" role="Started" active="true" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="1">
+      <node name="cluster01" id="1" cached="true"/>
+    </resource>
+    <bundle id="httpd-bundle" type="docker" image="pcmk:http" unique="false" managed="true" failed="false">
+      <replica id="0">
+        <resource id="httpd-bundle-ip-192.168.122.131" resource_agent="ocf::heartbeat:IPaddr2" role="Started" target_role="Started" active="true" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="1">
+          <node name="cluster02" id="2" cached="true"/>
+        </resource>
+        <resource id="httpd" resource_agent="ocf::heartbeat:apache" role="Started" target_role="Started" active="true" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="1">
+          <node name="httpd-bundle-0" id="httpd-bundle-0" cached="true"/>
+        </resource>
+        <resource id="httpd-bundle-docker-0" resource_agent="ocf::heartbeat:docker" role="Started" target_role="Started" active="true" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="1">
+          <node name="cluster02" id="2" cached="true"/>
+        </resource>
+        <resource id="httpd-bundle-0" resource_agent="ocf::pacemaker:remote" role="Started" target_role="Started" active="true" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="1">
+          <node name="cluster02" id="2" cached="true"/>
+        </resource>
+      </replica>
+      <replica id="1">
+        <resource id="httpd-bundle-ip-192.168.122.132" resource_agent="ocf::heartbeat:IPaddr2" role="Started" target_role="Started" active="true" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="1">
+          <node name="cluster01" id="1" cached="true"/>
+        </resource>
+        <resource id="httpd" resource_agent="ocf::heartbeat:apache" role="Stopped" target_role="Started" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
+        <resource id="httpd-bundle-docker-1" resource_agent="ocf::heartbeat:docker" role="Started" target_role="Started" active="true" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="1">
+          <node name="cluster01" id="1" cached="true"/>
+        </resource>
+        <resource id="httpd-bundle-1" resource_agent="ocf::pacemaker:remote" role="Started" target_role="Started" active="true" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="1">
+          <node name="cluster01" id="1" cached="true"/>
+        </resource>
+      </replica>
+    </bundle>
+  </resources>
+  <node_attributes>
+    <node name="cluster01">
+      <attribute name="pingd" value="1000"/>
+    </node>
+    <node name="cluster02">
+      <attribute name="pingd" value="1000"/>
+    </node>
+  </node_attributes>
+  <node_history>
+    <node name="cluster02">
+      <resource_history id="httpd-bundle-ip-192.168.122.131" orphan="false" migration-threshold="1000000">
+        <operation_history call="2" task="start" exec-time="0ms" queue-time="0ms" rc="0" rc_text="ok"/>
+        <operation_history call="3" task="monitor" interval="60000ms" exec-time="0ms" queue-time="0ms" rc="0" rc_text="ok"/>
+      </resource_history>
+      <resource_history id="httpd-bundle-docker-0" orphan="false" migration-threshold="1000000">
+        <operation_history call="2" task="start" exec-time="0ms" queue-time="0ms" rc="0" rc_text="ok"/>
+        <operation_history call="3" task="monitor" interval="60000ms" exec-time="0ms" queue-time="0ms" rc="0" rc_text="ok"/>
+      </resource_history>
+      <resource_history id="httpd-bundle-0" orphan="false" migration-threshold="1000000">
+        <operation_history call="2" task="start" exec-time="0ms" queue-time="0ms" rc="0" rc_text="ok"/>
+        <operation_history call="3" task="monitor" interval="30000ms" exec-time="0ms" queue-time="0ms" rc="0" rc_text="ok"/>
+      </resource_history>
+    </node>
+    <node name="cluster01">
+      <resource_history id="Fencing" orphan="false" migration-threshold="1000000">
+        <operation_history call="15" task="start" exec-time="36ms" queue-time="0ms" rc="0" rc_text="ok"/>
+        <operation_history call="20" task="monitor" interval="60000ms" exec-time="0ms" queue-time="0ms" rc="0" rc_text="ok"/>
+      </resource_history>
+      <resource_history id="ping" orphan="false" migration-threshold="1000000">
+        <operation_history call="2" task="start" exec-time="0ms" queue-time="0ms" rc="0" rc_text="ok"/>
+        <operation_history call="3" task="monitor" interval="10000ms" exec-time="0ms" queue-time="0ms" rc="0" rc_text="ok"/>
+      </resource_history>
+      <resource_history id="httpd-bundle-ip-192.168.122.132" orphan="false" migration-threshold="1000000">
+        <operation_history call="2" task="start" exec-time="0ms" queue-time="0ms" rc="0" rc_text="ok"/>
+        <operation_history call="3" task="monitor" interval="60000ms" exec-time="0ms" queue-time="0ms" rc="0" rc_text="ok"/>
+      </resource_history>
+      <resource_history id="httpd-bundle-docker-1" orphan="false" migration-threshold="1000000">
+        <operation_history call="2" task="start" exec-time="0ms" queue-time="0ms" rc="0" rc_text="ok"/>
+        <operation_history call="3" task="monitor" interval="60000ms" exec-time="0ms" queue-time="0ms" rc="0" rc_text="ok"/>
+      </resource_history>
+      <resource_history id="httpd-bundle-1" orphan="false" migration-threshold="1000000">
+        <operation_history call="2" task="start" exec-time="0ms" queue-time="0ms" rc="0" rc_text="ok"/>
+        <operation_history call="3" task="monitor" interval="30000ms" exec-time="0ms" queue-time="0ms" rc="0" rc_text="ok"/>
+      </resource_history>
+    </node>
+    <node name="httpd-bundle-0">
+      <resource_history id="httpd" orphan="false" migration-threshold="1000000">
+        <operation_history call="1" task="start" exec-time="0ms" queue-time="0ms" rc="0" rc_text="ok"/>
+      </resource_history>
+    </node>
+  </node_history>
+  <status code="0" message="OK"/>
+</pacemaker-result>
+=#=#=#= End test: XML output of partially active resources - OK (0) =#=#=#=
+* Passed: crm_mon        - XML output of partially active resources
+=#=#=#= Begin test: Text output of partially active resources, with inactive resources =#=#=#=
+Cluster Summary:
+  * Stack: corosync
+  * Current DC: cluster02 (version) - partition with quorum
+  * Last updated:
+  * Last change:
+  * 4 nodes configured
+  * 11 resource instances configured
+
+Node List:
+  * Online: [ cluster01 cluster02 ]
+  * GuestOnline: [ httpd-bundle-0@cluster02 httpd-bundle-1@cluster01 ]
+
+Full List of Resources:
+  * Clone Set: ping-clone [ping]:
+    * Started: [ cluster01 ]
+    * Stopped: [ cluster02 ]
+  * Fencing	(stonith:fence_xvm):	 Started cluster01
+  * Container bundle set: httpd-bundle [pcmk:http]:
+    * httpd-bundle-0 (192.168.122.131)	(ocf::heartbeat:apache):	 Started
+    * httpd-bundle-1 (192.168.122.132)	(ocf::heartbeat:apache):	 Stopped
+=#=#=#= End test: Text output of partially active resources, with inactive resources - OK (0) =#=#=#=
+* Passed: crm_mon        - Text output of partially active resources, with inactive resources
+=#=#=#= Begin test: Text output of partially active resources, with inactive resources, filtered by node =#=#=#=
+Cluster Summary:
+  * Stack: corosync
+  * Current DC: cluster02 (version) - partition with quorum
+  * Last updated:
+  * Last change:
+  * 4 nodes configured
+  * 11 resource instances configured
+
+Node List:
+  * Online: [ cluster01 ]
+
+Full List of Resources:
+  * Clone Set: ping-clone [ping]:
+    * Started: [ cluster01 ]
+  * Fencing	(stonith:fence_xvm):	 Started cluster01
+  * Container bundle set: httpd-bundle [pcmk:http]:
+    * httpd-bundle-1 (192.168.122.132)	(ocf::heartbeat:apache):	 Stopped
+=#=#=#= End test: Text output of partially active resources, with inactive resources, filtered by node - OK (0) =#=#=#=
+* Passed: crm_mon        - Text output of partially active resources, with inactive resources, filtered by node
+=#=#=#= Begin test: Text output of partially active resources, filtered by node =#=#=#=
+<pacemaker-result api-version="2.2" request="crm_mon -1 --output-as=xml --node=cluster01">
+  <summary>
+    <stack type="corosync"/>
+    <current_dc present="true" version="" with_quorum="true"/>
+    <last_update time=""/>
+    <last_change time=""/>
+    <nodes_configured number="4"/>
+    <resources_configured number="11" disabled="0" blocked="0"/>
+    <cluster_options stonith-enabled="true" symmetric-cluster="true" no-quorum-policy="stop" maintenance-mode="false"/>
+  </summary>
+  <nodes>
+    <node name="cluster01" id="1" online="true" standby="false" standby_onfail="false" maintenance="false" pending="false" unclean="false" shutdown="false" expected_up="true" is_dc="false" resources_running="5" type="member"/>
+  </nodes>
+  <resources>
+    <clone id="ping-clone" multi_state="false" unique="false" managed="true" failed="false" failure_ignored="false">
+      <resource id="ping" resource_agent="ocf::pacemaker:ping" role="Started" active="true" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="1">
+        <node name="cluster01" id="1" cached="true"/>
+      </resource>
+      <resource id="ping" resource_agent="ocf::pacemaker:ping" role="Stopped" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
+    </clone>
+    <resource id="Fencing" resource_agent="stonith:fence_xvm" role="Started" active="true" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="1">
+      <node name="cluster01" id="1" cached="true"/>
+    </resource>
+    <bundle id="httpd-bundle" type="docker" image="pcmk:http" unique="false" managed="true" failed="false">
+      <replica id="1">
+        <resource id="httpd-bundle-ip-192.168.122.132" resource_agent="ocf::heartbeat:IPaddr2" role="Started" target_role="Started" active="true" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="1">
+          <node name="cluster01" id="1" cached="true"/>
+        </resource>
+        <resource id="httpd" resource_agent="ocf::heartbeat:apache" role="Stopped" target_role="Started" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0"/>
+        <resource id="httpd-bundle-docker-1" resource_agent="ocf::heartbeat:docker" role="Started" target_role="Started" active="true" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="1">
+          <node name="cluster01" id="1" cached="true"/>
+        </resource>
+        <resource id="httpd-bundle-1" resource_agent="ocf::pacemaker:remote" role="Started" target_role="Started" active="true" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="1">
+          <node name="cluster01" id="1" cached="true"/>
+        </resource>
+      </replica>
+    </bundle>
+  </resources>
+  <node_attributes>
+    <node name="cluster01">
+      <attribute name="pingd" value="1000"/>
+    </node>
+  </node_attributes>
+  <node_history>
+    <node name="cluster01">
+      <resource_history id="Fencing" orphan="false" migration-threshold="1000000">
+        <operation_history call="15" task="start" exec-time="36ms" queue-time="0ms" rc="0" rc_text="ok"/>
+        <operation_history call="20" task="monitor" interval="60000ms" exec-time="0ms" queue-time="0ms" rc="0" rc_text="ok"/>
+      </resource_history>
+      <resource_history id="ping" orphan="false" migration-threshold="1000000">
+        <operation_history call="2" task="start" exec-time="0ms" queue-time="0ms" rc="0" rc_text="ok"/>
+        <operation_history call="3" task="monitor" interval="10000ms" exec-time="0ms" queue-time="0ms" rc="0" rc_text="ok"/>
+      </resource_history>
+      <resource_history id="httpd-bundle-ip-192.168.122.132" orphan="false" migration-threshold="1000000">
+        <operation_history call="2" task="start" exec-time="0ms" queue-time="0ms" rc="0" rc_text="ok"/>
+        <operation_history call="3" task="monitor" interval="60000ms" exec-time="0ms" queue-time="0ms" rc="0" rc_text="ok"/>
+      </resource_history>
+      <resource_history id="httpd-bundle-docker-1" orphan="false" migration-threshold="1000000">
+        <operation_history call="2" task="start" exec-time="0ms" queue-time="0ms" rc="0" rc_text="ok"/>
+        <operation_history call="3" task="monitor" interval="60000ms" exec-time="0ms" queue-time="0ms" rc="0" rc_text="ok"/>
+      </resource_history>
+      <resource_history id="httpd-bundle-1" orphan="false" migration-threshold="1000000">
+        <operation_history call="2" task="start" exec-time="0ms" queue-time="0ms" rc="0" rc_text="ok"/>
+        <operation_history call="3" task="monitor" interval="30000ms" exec-time="0ms" queue-time="0ms" rc="0" rc_text="ok"/>
+      </resource_history>
+    </node>
+  </node_history>
+  <status code="0" message="OK"/>
+</pacemaker-result>
+=#=#=#= End test: Text output of partially active resources, filtered by node - OK (0) =#=#=#=
+* Passed: crm_mon        - Text output of partially active resources, filtered by node
diff --git a/cts/crm_mon-partial.xml b/cts/crm_mon-partial.xml
new file mode 100644
index 0000000000..bc60e3a22a
--- /dev/null
+++ b/cts/crm_mon-partial.xml
@@ -0,0 +1,144 @@
+<cib crm_feature_set="3.3.0" validate-with="pacemaker-3.3" epoch="1" num_updates="37" admin_epoch="1" cib-last-written="Tue May  5 12:04:36 2020" update-origin="cluster01" update-client="crmd" update-user="hacluster" have-quorum="1" dc-uuid="2">
+  <configuration>
+    <crm_config>
+      <cluster_property_set id="cib-bootstrap-options">
+        <nvpair id="cib-bootstrap-options-have-watchdog" name="have-watchdog" value="false"/>
+        <nvpair id="cib-bootstrap-options-dc-version" name="dc-version" value="2.0.4-1.e97f9675f.git.el7-e97f9675f"/>
+        <nvpair id="cib-bootstrap-options-cluster-infrastructure" name="cluster-infrastructure" value="corosync"/>
+        <nvpair id="cib-bootstrap-options-cluster-name" name="cluster-name" value="test-cluster"/>
+        <nvpair id="cib-bootstrap-options-stonith-enabled" name="stonith-enabled" value="true"/>
+        <nvpair id="cib-bootstrap-options-maintenance-mode" name="maintenance-mode" value="false"/>
+      </cluster_property_set>
+    </crm_config>
+    <nodes>
+      <node id="1" uname="cluster01"/>
+      <node id="2" uname="cluster02"/>
+    </nodes>
+    <resources>
+      <clone id="ping-clone">
+        <primitive class="ocf" id="ping" provider="pacemaker" type="ping">
+          <instance_attributes id="ping-instance_attributes">
+            <nvpair id="ping-instance_attributes-dampen" name="dampen" value="5s"/>
+            <nvpair id="ping-instance_attributes-host_list" name="host_list" value="192.168.122.1"/>
+            <nvpair id="ping-instance_attributes-multiplier" name="multiplier" value="1000"/>
+          </instance_attributes>
+          <operations>
+            <op id="ping-monitor-interval-10s" interval="10s" name="monitor" timeout="60s"/>
+            <op id="ping-start-interval-0s" interval="0s" name="start" timeout="60s"/>
+            <op id="ping-stop-interval-0s" interval="0s" name="stop" timeout="20s"/>
+          </operations>
+        </primitive>
+      </clone>
+      <primitive class="stonith" id="Fencing" type="fence_xvm">
+        <instance_attributes id="Fencing-instance_attributes">
+          <nvpair id="Fencing-instance_attributes-ip_family" name="ip_family" value="ipv4"/>
+        </instance_attributes>
+        <operations>
+          <op id="Fencing-monitor-interval-60s" interval="60s" name="monitor"/>
+        </operations>
+      </primitive>
+      <bundle id="httpd-bundle">
+        <docker image="pcmk:http" replicas="2"/>
+        <network ip-range-start="192.168.122.131" host-netmask="24" host-interface="eth0">
+          <port-mapping id="httpd-port" port="80"/>
+        </network>
+        <storage>
+          <storage-mapping id="httpd-syslog" source-dir="/dev/log" target-dir="/dev/log" options="rw"/>
+          <storage-mapping id="httpd-root" source-dir="/srv/html" target-dir="/var/www/html" options="rw"/>
+          <storage-mapping id="httpd-logs" source-dir-root="/var/log/pacemaker/bundles" target-dir="/etc/httpd/logs" options="rw"/>
+        </storage>
+        <primitive class="ocf" id="httpd" provider="heartbeat" type="apache"/>
+        <meta_attributes id="bundle-meta_attributes">
+          <nvpair id="bundle-meta_attributes-target-role" name="target-role" value="Started"/>
+        </meta_attributes>
+      </bundle>
+    </resources>
+    <constraints/>
+  </configuration>
+  <status>
+    <node_state id="2" uname="cluster02" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
+      <lrm id="2">
+        <lrm_resources>
+          <lrm_resource id="Fencing" type="fence_xvm" class="stonith">
+            <lrm_rsc_op id="Fencing_last_0" operation_key="Fencing_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.3.0" transition-key="5:0:7:4a9e64d6-e1dd-4395-917c-1596312eafe4" transition-magic="0:7;5:0:7:4a9e64d6-e1dd-4395-917c-1596312eafe4" exit-reason="" on_node="cluster02" call-id="10" rc-code="7" op-status="0" interval="0" last-rc-change="1588951263" last-run="1588951263" exec-time="3" queue-time="0" op-digest="7da16842ab2328e41f737cab5e5fc89c"/>
+          </lrm_resource>
+          <lrm_resource id="httpd-bundle-ip-192.168.122.131" class="ocf" provider="heartbeat" type="IPaddr2">
+            <lrm_rsc_op id="httpd-bundle-ip-192.168.122.131_last_0" operation_key="httpd-bundle-ip-192.168.122.131_start_0" operation="start" crm-debug-origin="crm_simulate" crm_feature_set="3.3.0" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="2" rc-code="0" op-status="0" interval="0" last-rc-change="1590608589" last-run="1590608589" exec-time="0" queue-time="0" op-digest="8656419d4ed26465c724189832393477"/>
+            <lrm_rsc_op id="httpd-bundle-ip-192.168.122.131_monitor_60000" operation_key="httpd-bundle-ip-192.168.122.131_monitor_60000" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.3.0" transition-key="3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="3" rc-code="0" op-status="0" interval="60000" last-rc-change="1590608589" exec-time="0" queue-time="0" op-digest="617f27ac5fff521f401e6707063e2b5e"/>
+          </lrm_resource>
+          <lrm_resource id="httpd-bundle-docker-0" class="ocf" provider="heartbeat" type="docker">
+            <lrm_rsc_op id="httpd-bundle-docker-0_last_0" operation_key="httpd-bundle-docker-0_start_0" operation="start" crm-debug-origin="crm_simulate" crm_feature_set="3.3.0" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="2" rc-code="0" op-status="0" interval="0" last-rc-change="1590608589" last-run="1590608589" exec-time="0" queue-time="0" op-digest="02a1a0b2dfa1cade1893713b56939c55"/>
+            <lrm_rsc_op id="httpd-bundle-docker-0_monitor_60000" operation_key="httpd-bundle-docker-0_monitor_60000" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.3.0" transition-key="3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="3" rc-code="0" op-status="0" interval="60000" last-rc-change="1590608589" exec-time="0" queue-time="0" op-digest="775c93499f09f739ccbabe79d043f5ef"/>
+          </lrm_resource>
+          <lrm_resource id="httpd-bundle-ip-192.168.122.132" class="ocf" provider="heartbeat" type="IPaddr2">
+            <lrm_rsc_op id="httpd-bundle-ip-192.168.122.132_last_0" operation_key="httpd-bundle-ip-192.168.122.132_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.3.0" transition-key="1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:7;1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="1" rc-code="7" op-status="0" interval="0" last-rc-change="1590608589" last-run="1590608589" exec-time="0" queue-time="0" op-digest="c3d96a2922c2946905f760df9a177cd1"/>
+          </lrm_resource>
+          <lrm_resource id="httpd-bundle-docker-1" class="ocf" provider="heartbeat" type="docker">
+            <lrm_rsc_op id="httpd-bundle-docker-1_last_0" operation_key="httpd-bundle-docker-1_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.3.0" transition-key="1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:7;1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="1" rc-code="7" op-status="0" interval="0" last-rc-change="1590608589" last-run="1590608589" exec-time="0" queue-time="0" op-digest="2edb33b196e2261c6b3e30ce579e0590"/>
+          </lrm_resource>
+          <lrm_resource id="httpd-bundle-0" class="ocf" provider="pacemaker" type="remote">
+            <lrm_rsc_op id="httpd-bundle-0_last_0" operation_key="httpd-bundle-0_start_0" operation="start" crm-debug-origin="crm_simulate" crm_feature_set="3.3.0" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="2" rc-code="0" op-status="0" interval="0" last-rc-change="1590608589" last-run="1590608589" exec-time="0" queue-time="0" op-digest="c535429017a9ee0785106fbef2858a41"/>
+            <lrm_rsc_op id="httpd-bundle-0_monitor_30000" operation_key="httpd-bundle-0_monitor_30000" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.3.0" transition-key="3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="3" rc-code="0" op-status="0" interval="30000" last-rc-change="1590608589" exec-time="0" queue-time="0" op-digest="6d63e20548871f169e287d33f3711637"/>
+          </lrm_resource>
+          <lrm_resource id="httpd-bundle-1" class="ocf" provider="pacemaker" type="remote">
+            <lrm_rsc_op id="httpd-bundle-1_last_0" operation_key="httpd-bundle-1_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.3.0" transition-key="1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:7;1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="1" rc-code="7" op-status="0" interval="0" last-rc-change="1590608589" last-run="1590608589" exec-time="0" queue-time="0" op-digest="791bcda8f6693465cc318cba5302a8df"/>
+          </lrm_resource>
+        </lrm_resources>
+      </lrm>
+      <transient_attributes id="2">
+        <instance_attributes id="status-2">
+          <nvpair id="status-2-pingd" name="pingd" value="1000"/>
+        </instance_attributes>
+      </transient_attributes>
+    </node_state>
+    <node_state id="1" uname="cluster01" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
+      <lrm id="1">
+        <lrm_resources>
+          <lrm_resource id="Fencing" type="fence_xvm" class="stonith">
+            <lrm_rsc_op id="Fencing_last_0" operation_key="Fencing_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.3.0" transition-key="12:1:0:4a9e64d6-e1dd-4395-917c-1596312eafe4" transition-magic="0:0;12:1:0:4a9e64d6-e1dd-4395-917c-1596312eafe4" exit-reason="" on_node="cluster01" call-id="15" rc-code="0" op-status="0" interval="0" last-rc-change="1588951272" last-run="1588951272" exec-time="36" queue-time="0" op-digest="7da16842ab2328e41f737cab5e5fc89c"/>
+            <lrm_rsc_op id="Fencing_monitor_60000" operation_key="Fencing_monitor_60000" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.3.0" transition-key="20:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;20:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" on_node="cluster01" call-id="20" rc-code="0" op-status="0" interval="60000" last-rc-change="1590608589" exec-time="0" queue-time="0" op-digest="a88218bb6c7dc47e6586fc75fc2a8d69"/>
+          </lrm_resource>
+          <lrm_resource id="ping" class="ocf" provider="pacemaker" type="ping">
+            <lrm_rsc_op id="ping_last_0" operation_key="ping_start_0" operation="start" crm-debug-origin="crm_simulate" crm_feature_set="3.3.0" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="2" rc-code="0" op-status="0" interval="0" last-rc-change="1590608589" last-run="1590608589" exec-time="0" queue-time="0" op-digest="769dd6f95f1494d416ae9dc690960e17"/>
+            <lrm_rsc_op id="ping_monitor_10000" operation_key="ping_monitor_10000" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.3.0" transition-key="3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="3" rc-code="0" op-status="0" interval="10000" last-rc-change="1590608589" exec-time="0" queue-time="0" op-digest="7beffd8be749b787fabea4aef5df21c9"/>
+          </lrm_resource>
+          <lrm_resource id="httpd-bundle-ip-192.168.122.131" class="ocf" provider="heartbeat" type="IPaddr2">
+            <lrm_rsc_op id="httpd-bundle-ip-192.168.122.131_last_0" operation_key="httpd-bundle-ip-192.168.122.131_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.3.0" transition-key="1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:7;1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="1" rc-code="7" op-status="0" interval="0" last-rc-change="1590608589" last-run="1590608589" exec-time="0" queue-time="0" op-digest="8656419d4ed26465c724189832393477"/>
+          </lrm_resource>
+          <lrm_resource id="httpd-bundle-docker-0" class="ocf" provider="heartbeat" type="docker">
+            <lrm_rsc_op id="httpd-bundle-docker-0_last_0" operation_key="httpd-bundle-docker-0_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.3.0" transition-key="1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:7;1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="1" rc-code="7" op-status="0" interval="0" last-rc-change="1590608589" last-run="1590608589" exec-time="0" queue-time="0" op-digest="02a1a0b2dfa1cade1893713b56939c55"/>
+          </lrm_resource>
+          <lrm_resource id="httpd-bundle-ip-192.168.122.132" class="ocf" provider="heartbeat" type="IPaddr2">
+            <lrm_rsc_op id="httpd-bundle-ip-192.168.122.132_last_0" operation_key="httpd-bundle-ip-192.168.122.132_start_0" operation="start" crm-debug-origin="crm_simulate" crm_feature_set="3.3.0" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="2" rc-code="0" op-status="0" interval="0" last-rc-change="1590608589" last-run="1590608589" exec-time="0" queue-time="0" op-digest="c3d96a2922c2946905f760df9a177cd1"/>
+            <lrm_rsc_op id="httpd-bundle-ip-192.168.122.132_monitor_60000" operation_key="httpd-bundle-ip-192.168.122.132_monitor_60000" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.3.0" transition-key="3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="3" rc-code="0" op-status="0" interval="60000" last-rc-change="1590608589" exec-time="0" queue-time="0" op-digest="33ef2404fd1954b12433f676cffd08ec"/>
+          </lrm_resource>
+          <lrm_resource id="httpd-bundle-docker-1" class="ocf" provider="heartbeat" type="docker">
+            <lrm_rsc_op id="httpd-bundle-docker-1_last_0" operation_key="httpd-bundle-docker-1_start_0" operation="start" crm-debug-origin="crm_simulate" crm_feature_set="3.3.0" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="2" rc-code="0" op-status="0" interval="0" last-rc-change="1590608589" last-run="1590608589" exec-time="0" queue-time="0" op-digest="2edb33b196e2261c6b3e30ce579e0590"/>
+            <lrm_rsc_op id="httpd-bundle-docker-1_monitor_60000" operation_key="httpd-bundle-docker-1_monitor_60000" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.3.0" transition-key="3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="3" rc-code="0" op-status="0" interval="60000" last-rc-change="1590608589" exec-time="0" queue-time="0" op-digest="a2605826ef42e23316e4d27d9cb28f8e"/>
+          </lrm_resource>
+          <lrm_resource id="httpd-bundle-0" class="ocf" provider="pacemaker" type="remote">
+            <lrm_rsc_op id="httpd-bundle-0_last_0" operation_key="httpd-bundle-0_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.3.0" transition-key="1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:7;1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="1" rc-code="7" op-status="0" interval="0" last-rc-change="1590608589" last-run="1590608589" exec-time="0" queue-time="0" op-digest="c535429017a9ee0785106fbef2858a41"/>
+          </lrm_resource>
+          <lrm_resource id="httpd-bundle-1" class="ocf" provider="pacemaker" type="remote">
+            <lrm_rsc_op id="httpd-bundle-1_last_0" operation_key="httpd-bundle-1_start_0" operation="start" crm-debug-origin="crm_simulate" crm_feature_set="3.3.0" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="2" rc-code="0" op-status="0" interval="0" last-rc-change="1590608589" last-run="1590608589" exec-time="0" queue-time="0" op-digest="791bcda8f6693465cc318cba5302a8df"/>
+            <lrm_rsc_op id="httpd-bundle-1_monitor_30000" operation_key="httpd-bundle-1_monitor_30000" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.3.0" transition-key="3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="3" rc-code="0" op-status="0" interval="30000" last-rc-change="1590608589" exec-time="0" queue-time="0" op-digest="7592cb10fa1499772a031adfd385f558"/>
+          </lrm_resource>
+        </lrm_resources>
+      </lrm>
+      <transient_attributes id="1">
+        <instance_attributes id="status-1">
+          <nvpair id="status-1-pingd" name="pingd" value="1000"/>
+        </instance_attributes>
+      </transient_attributes>
+    </node_state>
+    <node_state id="httpd-bundle-0" uname="httpd-bundle-0">
+      <lrm id="httpd-bundle-0">
+        <lrm_resources>
+          <lrm_resource id="httpd" class="ocf" provider="heartbeat" type="apache">
+            <lrm_rsc_op id="httpd_last_0" operation_key="httpd_start_0" operation="start" crm-debug-origin="crm_simulate" crm_feature_set="3.3.0" transition-key="1:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;1:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" exit-reason="" call-id="1" rc-code="0" op-status="0" interval="0" last-rc-change="1590608589" last-run="1590608589" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+          </lrm_resource>
+        </lrm_resources>
+      </lrm>
+    </node_state>
+  </status>
+</cib>
diff --git a/cts/crm_mon.xml b/cts/crm_mon.xml
new file mode 100644
index 0000000000..007477c33a
--- /dev/null
+++ b/cts/crm_mon.xml
@@ -0,0 +1,151 @@
+<cib crm_feature_set="3.3.0" validate-with="pacemaker-3.3" epoch="1" num_updates="1" admin_epoch="1" cib-last-written="Tue May  5 12:04:36 2020" update-origin="cluster01" update-client="crmd" update-user="hacluster" have-quorum="1" dc-uuid="2">
+  <configuration>
+    <crm_config>
+      <cluster_property_set id="cib-bootstrap-options">
+        <nvpair id="cib-bootstrap-options-have-watchdog" name="have-watchdog" value="false"/>
+        <nvpair id="cib-bootstrap-options-dc-version" name="dc-version" value="2.0.4-1.e97f9675f.git.el7-e97f9675f"/>
+        <nvpair id="cib-bootstrap-options-cluster-infrastructure" name="cluster-infrastructure" value="corosync"/>
+        <nvpair id="cib-bootstrap-options-cluster-name" name="cluster-name" value="test-cluster"/>
+        <nvpair id="cib-bootstrap-options-stonith-enabled" name="stonith-enabled" value="true"/>
+        <nvpair id="cib-bootstrap-options-maintenance-mode" name="maintenance-mode" value="false"/>
+      </cluster_property_set>
+    </crm_config>
+    <nodes>
+      <node id="1" uname="cluster01">
+        <instance_attributes id="nodes-1">
+          <nvpair id="nodes-1-location" name="location" value="office"/>
+        </instance_attributes>
+      </node>
+      <node id="2" uname="cluster02"/>
+    </nodes>
+    <resources>
+      <clone id="ping-clone">
+        <primitive class="ocf" id="ping" provider="pacemaker" type="ping">
+          <instance_attributes id="ping-instance_attributes">
+            <nvpair id="ping-instance_attributes-dampen" name="dampen" value="5s"/>
+            <nvpair id="ping-instance_attributes-host_list" name="host_list" value="192.168.122.1"/>
+            <nvpair id="ping-instance_attributes-multiplier" name="multiplier" value="1000"/>
+          </instance_attributes>
+          <operations>
+            <op id="ping-monitor-interval-10s" interval="10s" name="monitor" timeout="60s"/>
+            <op id="ping-start-interval-0s" interval="0s" name="start" timeout="60s"/>
+            <op id="ping-stop-interval-0s" interval="0s" name="stop" timeout="20s"/>
+          </operations>
+        </primitive>
+      </clone>
+      <primitive class="stonith" id="Fencing" type="fence_xvm">
+        <instance_attributes id="Fencing-instance_attributes">
+          <nvpair id="Fencing-instance_attributes-ip_family" name="ip_family" value="ipv4"/>
+        </instance_attributes>
+        <operations>
+          <op id="Fencing-monitor-interval-60s" interval="60s" name="monitor"/>
+        </operations>
+      </primitive>
+      <primitive class="ocf" id="dummy" provider="pacemaker" type="Dummy">
+        <instance_attributes id="dummy-instance_attributes">
+          <nvpair id="dummy-instance_attributes-op_sleep" name="op_sleep" value="6"/>
+        </instance_attributes>
+        <operations>
+          <op id="dummy-migrate_from-interval-0s" interval="0s" name="migrate_from" timeout="20s"/>
+          <op id="dummy-migrate_to-interval-0s" interval="0s" name="migrate_to" timeout="20s"/>
+          <op id="dummy-monitor-interval-60s" interval="60s" name="monitor" on-fail="stop"/>
+          <op id="dummy-reload-interval-0s" interval="0s" name="reload" timeout="20s"/>
+          <op id="dummy-start-interval-0s" interval="0s" name="start" timeout="20s"/>
+          <op id="dummy-stop-interval-0s" interval="0s" name="stop" timeout="20s"/>
+        </operations>
+      </primitive>
+      <primitive class="ocf" id="inactive-dummy" provider="pacemaker" type="Dummy">
+        <meta_attributes id="inactive-dummy.meta">
+          <nvpair id="inactive-dummy.meta.target" name="target-role" value="stopped"/>
+        </meta_attributes>
+      </primitive>
+      <master id="inactive-clone-master">
+        <primitive class="ocf" id="inactive-clone" provider="pacemaker" type="Stateful">
+          <operations>
+            <op id="inactive-clone-monitor-interval-10" interval="10" name="monitor" role="Master" timeout="20"/>
+            <op id="inactive-clone-monitor-interval-11" interval="11" name="monitor" role="Slave" timeout="20"/>
+          </operations>
+        </primitive>
+      </master>
+      <bundle id="httpd-bundle">
+        <docker image="pcmk:http" replicas="3"/>
+        <network ip-range-start="192.168.122.131" host-netmask="24" host-interface="eth0">
+          <port-mapping id="httpd-port" port="80"/>
+        </network>
+        <storage>
+          <storage-mapping id="httpd-syslog" source-dir="/dev/log" target-dir="/dev/log" options="rw"/>
+          <storage-mapping id="httpd-root" source-dir="/srv/html" target-dir="/var/www/html" options="rw"/>
+          <storage-mapping id="httpd-logs" source-dir-root="/var/log/pacemaker/bundles" target-dir="/etc/httpd/logs" options="rw"/>
+        </storage>
+        <primitive class="ocf" id="httpd" provider="heartbeat" type="apache"/>
+        <meta_attributes id="bundle-meta_attributes">
+          <nvpair id="bundle-meta_attributes-target-role" name="target-role" value="Started"/>
+        </meta_attributes>
+      </bundle>
+    </resources>
+    <constraints/>
+    <tags>
+      <tag id="all-nodes">
+        <obj_ref id="1"/>
+        <obj_ref id="2"/>
+      </tag>
+      <tag id="even-nodes">
+        <obj_ref id="2"/>
+      </tag>
+      <tag id="odd-nodes">
+        <obj_ref id="1"/>
+      </tag>
+    </tags>
+    <op_defaults>
+      <meta_attributes id="op_defaults-options">
+        <nvpair id="op_defaults-options-timeout" name="timeout" value="5s"/>
+      </meta_attributes>
+    </op_defaults>
+  </configuration>
+  <status>
+    <node_state id="2" uname="cluster02" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
+      <lrm id="2">
+        <lrm_resources>
+          <lrm_resource id="ping" type="ping" class="ocf" provider="pacemaker">
+            <lrm_rsc_op id="ping_last_0" operation_key="ping_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.3.0" transition-key="9:0:0:4a9e64d6-e1dd-4395-917c-1596312eafe4" transition-magic="0:0;9:0:0:4a9e64d6-e1dd-4395-917c-1596312eafe4" exit-reason="" on_node="cluster02" call-id="11" rc-code="0" op-status="0" interval="0" last-rc-change="1588951263" last-run="1588951263" exec-time="2044" queue-time="0" op-digest="769dd6f95f1494d416ae9dc690960e17"/>
+            <lrm_rsc_op id="ping_monitor_10000" operation_key="ping_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.3.0" transition-key="10:0:0:4a9e64d6-e1dd-4395-917c-1596312eafe4" transition-magic="0:0;10:0:0:4a9e64d6-e1dd-4395-917c-1596312eafe4" exit-reason="" on_node="cluster02" call-id="12" rc-code="0" op-status="0" interval="10000" last-rc-change="1588951265" exec-time="2031" queue-time="0" op-digest="7beffd8be749b787fabea4aef5df21c9"/>
+          </lrm_resource>
+          <lrm_resource id="Fencing" type="fence_xvm" class="stonith">
+            <lrm_rsc_op id="Fencing_last_0" operation_key="Fencing_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.3.0" transition-key="5:0:7:4a9e64d6-e1dd-4395-917c-1596312eafe4" transition-magic="0:7;5:0:7:4a9e64d6-e1dd-4395-917c-1596312eafe4" exit-reason="" on_node="cluster02" call-id="10" rc-code="7" op-status="0" interval="0" last-rc-change="1588951263" last-run="1588951263" exec-time="3" queue-time="0" op-digest="7da16842ab2328e41f737cab5e5fc89c"/>
+          </lrm_resource>
+          <lrm_resource id="dummy" type="Dummy" class="ocf" provider="pacemaker">
+            <lrm_rsc_op id="dummy_last_0" operation_key="dummy_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.3.0" transition-key="14:1:0:4a9e64d6-e1dd-4395-917c-1596312eafe4" transition-magic="0:0;14:1:0:4a9e64d6-e1dd-4395-917c-1596312eafe4" exit-reason="" on_node="cluster02" call-id="18" rc-code="0" op-status="0" interval="0" last-rc-change="1588951278" last-run="1588951278" exec-time="6020" queue-time="0" op-digest="aa0f9b7caf28600646551adb55bd9b95" op-force-restart=" envfile  op_sleep  passwd  state " op-restart-digest="aa0f9b7caf28600646551adb55bd9b95" op-secure-params=" passwd " op-secure-digest="aa0f9b7caf28600646551adb55bd9b95"/>
+            <lrm_rsc_op id="dummy_monitor_60000" operation_key="dummy_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.3.0" transition-key="16:2:0:4a9e64d6-e1dd-4395-917c-1596312eafe4" transition-magic="0:0;16:2:0:4a9e64d6-e1dd-4395-917c-1596312eafe4" exit-reason="" on_node="cluster02" call-id="19" rc-code="0" op-status="0" interval="60000" last-rc-change="1588951284" exec-time="6015" queue-time="0" op-digest="ccfee4afbb0618907016c9bef210b8b6" op-secure-params=" passwd " op-secure-digest="aa0f9b7caf28600646551adb55bd9b95"/>
+          </lrm_resource>
+        </lrm_resources>
+      </lrm>
+      <transient_attributes id="2">
+        <instance_attributes id="status-2">
+          <nvpair id="status-2-pingd" name="pingd" value="1000"/>
+        </instance_attributes>
+      </transient_attributes>
+    </node_state>
+    <node_state id="1" uname="cluster01" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
+      <lrm id="1">
+        <lrm_resources>
+          <lrm_resource id="ping" type="ping" class="ocf" provider="pacemaker">
+            <lrm_rsc_op id="ping_last_0" operation_key="ping_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.3.0" transition-key="6:1:0:4a9e64d6-e1dd-4395-917c-1596312eafe4" transition-magic="0:0;6:1:0:4a9e64d6-e1dd-4395-917c-1596312eafe4" exit-reason="" on_node="cluster01" call-id="17" rc-code="0" op-status="0" interval="0" last-rc-change="1588951272" last-run="1588951272" exec-time="2038" queue-time="0" op-digest="769dd6f95f1494d416ae9dc690960e17"/>
+            <lrm_rsc_op id="ping_monitor_10000" operation_key="ping_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.3.0" transition-key="7:1:0:4a9e64d6-e1dd-4395-917c-1596312eafe4" transition-magic="0:0;7:1:0:4a9e64d6-e1dd-4395-917c-1596312eafe4" exit-reason="" on_node="cluster01" call-id="18" rc-code="0" op-status="0" interval="10000" last-rc-change="1588951274" exec-time="2034" queue-time="0" op-digest="7beffd8be749b787fabea4aef5df21c9"/>
+          </lrm_resource>
+          <lrm_resource id="Fencing" type="fence_xvm" class="stonith">
+            <lrm_rsc_op id="Fencing_last_0" operation_key="Fencing_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.3.0" transition-key="12:1:0:4a9e64d6-e1dd-4395-917c-1596312eafe4" transition-magic="0:0;12:1:0:4a9e64d6-e1dd-4395-917c-1596312eafe4" exit-reason="" on_node="cluster01" call-id="15" rc-code="0" op-status="0" interval="0" last-rc-change="1588951272" last-run="1588951272" exec-time="36" queue-time="0" op-digest="7da16842ab2328e41f737cab5e5fc89c"/>
+            <lrm_rsc_op id="Fencing_monitor_60000" operation_key="Fencing_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.3.0" transition-key="13:1:0:4a9e64d6-e1dd-4395-917c-1596312eafe4" transition-magic="0:0;13:1:0:4a9e64d6-e1dd-4395-917c-1596312eafe4" exit-reason="" on_node="cluster01" call-id="19" rc-code="0" op-status="0" interval="60000" last-rc-change="1588951276" exec-time="24" queue-time="0" op-digest="f85d77708ad4ea02a9099e1e548aff0d"/>
+          </lrm_resource>
+          <lrm_resource id="dummy" type="Dummy" class="ocf" provider="pacemaker">
+            <lrm_rsc_op id="dummy_last_0" operation_key="dummy_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.3.0" transition-key="3:1:0:4a9e64d6-e1dd-4395-917c-1596312eafe4" transition-magic="0:0;3:1:0:4a9e64d6-e1dd-4395-917c-1596312eafe4" exit-reason="" on_node="cluster01" call-id="16" rc-code="0" op-status="0" interval="0" last-rc-change="1588951272" last-run="1588951272" exec-time="6048" queue-time="0" op-digest="aa0f9b7caf28600646551adb55bd9b95" op-force-restart=" envfile  op_sleep  passwd  state " op-restart-digest="aa0f9b7caf28600646551adb55bd9b95" op-secure-params=" passwd " op-secure-digest="aa0f9b7caf28600646551adb55bd9b95"/>
+          </lrm_resource>
+        </lrm_resources>
+      </lrm>
+      <transient_attributes id="1">
+        <instance_attributes id="status-1">
+          <nvpair id="status-1-pingd" name="pingd" value="1000"/>
+        </instance_attributes>
+      </transient_attributes>
+    </node_state>
+  </status>
+</cib>
diff --git a/cts/cts-cli.in b/cts/cts-cli.in
index b6d873b42d..12f3311dd9 100755
--- a/cts/cts-cli.in
+++ b/cts/cts-cli.in
@@ -1,1223 +1,1339 @@
 #!@BASH_PATH@
 #
 # Copyright 2008-2020 the Pacemaker project contributors
 #
 # The version control history for this file may have further details.
 #
 # This source code is licensed under the GNU General Public License version 2
 # or later (GPLv2+) WITHOUT ANY WARRANTY.
 #
 
 #
 # Note on portable usage of sed: GNU/POSIX/*BSD sed have a limited subset of
 # compatible functionality. Do not use the -i option, alternation (\|),
 # \0, or character sequences such as \n or \s.
 #
 
 USAGE_TEXT="Usage: cts-cli [<options>]
 Options:
  --help          Display this text, then exit
  -V, --verbose   Display any differences from expected output
- -t 'TEST [...]' Run only specified tests (default: 'dates tools acls validity upgrade rules')
+ -t 'TEST [...]' Run only specified tests (default: 'dates tools crm_mon acls validity upgrade rules')
  -p DIR          Look for executables in DIR (may be specified multiple times)
  -v, --valgrind  Run all commands under valgrind
  -s              Save actual output as expected output"
 
 # If readlink supports -e (i.e. GNU), use it
 readlink -e / >/dev/null 2>/dev/null
 if [ $? -eq 0 ]; then
     test_home="$(dirname "$(readlink -e "$0")")"
 else
     test_home="$(dirname "$0")"
 fi
 
 : ${shadow=cts-cli}
 shadow_dir=$(mktemp -d ${TMPDIR:-/tmp}/cts-cli.shadow.XXXXXXXXXX)
 num_errors=0
 num_passed=0
 verbose=0
-tests="dates tools acls validity upgrade rules"
+tests="dates tools crm_mon acls validity upgrade rules"
 do_save=0
 VALGRIND_CMD=
 VALGRIND_OPTS="
     -q
     --gen-suppressions=all
     --show-reachable=no
     --leak-check=full
     --trace-children=no
     --time-stamp=yes
     --num-callers=20
     --suppressions=$test_home/valgrind-pcmk.suppressions
 "
 
 # These constants must track crm_exit_t values
 CRM_EX_OK=0
 CRM_EX_ERROR=1
 CRM_EX_INVALID_PARAM=2
 CRM_EX_UNIMPLEMENT_FEATURE=3
 CRM_EX_INSUFFICIENT_PRIV=4
 CRM_EX_USAGE=64
 CRM_EX_CONFIG=78
 CRM_EX_OLD=103
 CRM_EX_DIGEST=104
 CRM_EX_NOSUCH=105
 CRM_EX_UNSAFE=107
 CRM_EX_EXISTS=108
 CRM_EX_MULTIPLE=109
 CRM_EX_EXPIRED=110
 CRM_EX_NOT_YET_IN_EFFECT=111
 CRM_EX_INDETERMINATE=112
 CRM_EX_UNSATISFIED=113
 
 function test_assert() {
     target=$1; shift
     cib=$1; shift
     app=`echo "$cmd" | sed 's/\ .*//'`
     printf "* Running: $app - $desc\n" 1>&2
 
     printf "=#=#=#= Begin test: $desc =#=#=#=\n"
     eval $VALGRIND_CMD $cmd 2>&1
     rc=$?
 
     if [ x$cib != x0 ]; then
         printf "=#=#=#= Current cib after: $desc =#=#=#=\n"
         CIB_user=root cibadmin -Q
     fi
 
     printf "=#=#=#= End test: $desc - $(crm_error --exit $rc) ($rc) =#=#=#=\n"
 
     if [ $rc -ne $target ]; then
         num_errors=$(( $num_errors + 1 ))
         printf "* Failed (rc=%.3d): %-14s - %s\n" $rc $app "$desc"
         printf "* Failed (rc=%.3d): %-14s - %s\n" $rc $app "$desc (`which $app`)" 1>&2
         return
         exit $CRM_EX_ERROR
     else
         printf "* Passed: %-14s - %s\n" $app "$desc"
         num_passed=$(( $num_passed + 1 ))
     fi
 }
 
+function test_crm_mon() {
+    export CIB_file="$SRCDIR/cts/crm_mon.xml"
+
+    desc="Basic text output"
+    cmd="crm_mon -1"
+    test_assert $CRM_EX_OK 0
+
+    desc="XML output"
+    cmd="crm_mon --output-as=xml"
+    test_assert $CRM_EX_OK 0
+
+    desc="Basic text output without node section"
+    cmd="crm_mon -1 --exclude=nodes"
+    test_assert $CRM_EX_OK 0
+
+    desc="XML output without the node section"
+    cmd="crm_mon --output-as=xml --exclude=nodes"
+    test_assert $CRM_EX_OK 0
+
+    desc="Text output with only the node section"
+    cmd="crm_mon -1 --exclude=all --include=nodes"
+    test_assert $CRM_EX_OK 0
+
+    # The above test doesn't need to be performed for other output formats.  It's
+    # really just a test to make sure that blank lines are correct.
+
+    desc="Complete text output"
+    cmd="crm_mon -1 --include=all"
+    test_assert $CRM_EX_OK 0
+
+    # XML includes everything already so there's no need for a complete test
+
+    desc="Complete text output filtered by node"
+    cmd="crm_mon -1 --include=all --node=cluster01"
+    test_assert $CRM_EX_OK 0
+
+    desc="XML output filtered by node"
+    cmd="crm_mon --output-as xml --include=all --node=cluster01"
+    test_assert $CRM_EX_OK 0
+
+    desc="Complete text output filtered by tag"
+    cmd="crm_mon -1 --include=all --node=even-nodes"
+    test_assert $CRM_EX_OK 0
+
+    desc="XML output filtered by tag"
+    cmd="crm_mon --output-as=xml --include=all --node=even-nodes"
+    test_assert $CRM_EX_OK 0
+
+    desc="Basic text output filtered by node that doesn't exist"
+    cmd="crm_mon -1 --node=blah"
+    test_assert $CRM_EX_OK 0
+
+    desc="XML output filtered by node that doesn't exist"
+    cmd="crm_mon --output-as=xml --node=blah"
+    test_assert $CRM_EX_OK 0
+
+    desc="Detailed text output"
+    cmd="crm_mon -1 --show-detail"
+    test_assert $CRM_EX_OK 0
+
+    # XML already includes extra detail
+
+    desc="Basic text output with inactive resources"
+    cmd="crm_mon -1 -r"
+    test_assert $CRM_EX_OK 0
+
+    # XML already includes inactive resources
+
+    desc="Basic text output with inactive resources, filtered by node"
+    cmd="crm_mon -1 -r --node=cluster02"
+    test_assert $CRM_EX_OK 0
+
+    # XML already includes inactive resources
+
+    unset CIB_file
+
+    export CIB_file="$SRCDIR/cts/crm_mon-partial.xml"
+
+    desc="Text output of partially active resources"
+    cmd="crm_mon -1"
+    test_assert $CRM_EX_OK 0
+
+    desc="XML output of partially active resources"
+    cmd="crm_mon -1 --output-as=xml"
+    test_assert $CRM_EX_OK 0
+
+    desc="Text output of partially active resources, with inactive resources"
+    cmd="crm_mon -1 -r"
+    test_assert $CRM_EX_OK 0
+
+    # XML already includes inactive resources
+
+    desc="Text output of partially active resources, with inactive resources, filtered by node"
+    cmd="crm_mon -1 -r --node=cluster01"
+    test_assert $CRM_EX_OK 0
+
+    desc="Text output of partially active resources, filtered by node"
+    cmd="crm_mon -1 --output-as=xml --node=cluster01"
+    test_assert $CRM_EX_OK 0
+
+    unset CIB_file
+}
+
 function test_tools() {
     local TMPXML
     local TMPORIG
 
     TMPXML=$(mktemp ${TMPDIR:-/tmp}/cts-cli.tools.xml.XXXXXXXXXX)
     TMPORIG=$(mktemp ${TMPDIR:-/tmp}/cts-cli.tools.existing.xml.XXXXXXXXXX)
     export CIB_shadow_dir="${shadow_dir}"
 
     $VALGRIND_CMD crm_shadow --batch --force --create-empty $shadow  2>&1
     export CIB_shadow=$shadow
 
     desc="Validate CIB"
     cmd="cibadmin -Q"
     test_assert $CRM_EX_OK
 
     desc="Configure something before erasing"
     cmd="crm_attribute -n cluster-delay -v 60s"
     test_assert $CRM_EX_OK
 
     desc="Require --force for CIB erasure"
     cmd="cibadmin -E"
     test_assert $CRM_EX_UNSAFE
 
     desc="Allow CIB erasure with --force"
     cmd="cibadmin -E --force"
     test_assert $CRM_EX_OK
 
     desc="Query CIB"
     cmd="cibadmin -Q > $TMPORIG"
     test_assert $CRM_EX_OK
 
     desc="Set cluster option"
     cmd="crm_attribute -n cluster-delay -v 60s"
     test_assert $CRM_EX_OK
 
     desc="Query new cluster option"
     cmd="cibadmin -Q -o crm_config | grep cib-bootstrap-options-cluster-delay"
     test_assert $CRM_EX_OK
 
     desc="Query cluster options"
     cmd="cibadmin -Q -o crm_config > $TMPXML"
     test_assert $CRM_EX_OK
 
     desc="Set no-quorum policy"
     cmd="crm_attribute -n no-quorum-policy -v ignore"
     test_assert $CRM_EX_OK
 
     desc="Delete nvpair"
     cmd="cibadmin -D -o crm_config --xml-text '<nvpair id=\"cib-bootstrap-options-cluster-delay\"/>'"
     test_assert $CRM_EX_OK
 
     desc="Create operation should fail"
     cmd="cibadmin -C -o crm_config --xml-file $TMPXML"
     test_assert $CRM_EX_EXISTS
 
     desc="Modify cluster options section"
     cmd="cibadmin -M -o crm_config --xml-file $TMPXML"
     test_assert $CRM_EX_OK
 
     desc="Query updated cluster option"
     cmd="cibadmin -Q -o crm_config | grep cib-bootstrap-options-cluster-delay"
     test_assert $CRM_EX_OK
 
     desc="Set duplicate cluster option"
     cmd="crm_attribute -n cluster-delay -v 40s -s duplicate"
     test_assert $CRM_EX_OK
 
     desc="Setting multiply defined cluster option should fail"
     cmd="crm_attribute -n cluster-delay -v 30s"
     test_assert $CRM_EX_MULTIPLE
 
     desc="Set cluster option with -s"
     cmd="crm_attribute -n cluster-delay -v 30s -s duplicate"
     test_assert $CRM_EX_OK
 
     desc="Delete cluster option with -i"
     cmd="crm_attribute -n cluster-delay -D -i cib-bootstrap-options-cluster-delay"
     test_assert $CRM_EX_OK
 
     desc="Create node1 and bring it online"
     cmd="crm_simulate --live-check --in-place --node-up=node1"
     test_assert $CRM_EX_OK
 
     desc="Create node attribute"
     cmd="crm_attribute -n ram -v 1024M -N node1 -t nodes"
     test_assert $CRM_EX_OK
 
     desc="Query new node attribute"
     cmd="cibadmin -Q -o nodes | grep node1-ram"
     test_assert $CRM_EX_OK
 
     desc="Set a transient (fail-count) node attribute"
     cmd="crm_attribute -n fail-count-foo -v 3 -N node1 -t status"
     test_assert $CRM_EX_OK
 
     desc="Query a fail count"
     cmd="crm_failcount --query -r foo -N node1"
     test_assert $CRM_EX_OK
 
     desc="Delete a transient (fail-count) node attribute"
     cmd="crm_attribute -n fail-count-foo -D -N node1 -t status"
     test_assert $CRM_EX_OK
 
     desc="Digest calculation"
     cmd="cibadmin -Q | cibadmin -5 -p 2>&1 > /dev/null"
     test_assert $CRM_EX_OK
 
     # This update will fail because it has version numbers
     desc="Replace operation should fail"
     cmd="cibadmin -R --xml-file $TMPORIG"
     test_assert $CRM_EX_OLD
 
     desc="Default standby value"
     cmd="crm_standby -N node1 -G"
     test_assert $CRM_EX_OK
  
     desc="Set standby status"
     cmd="crm_standby -N node1 -v true"
     test_assert $CRM_EX_OK
  
     desc="Query standby value"
     cmd="crm_standby -N node1 -G"
     test_assert $CRM_EX_OK
  
     desc="Delete standby value"
     cmd="crm_standby -N node1 -D"
     test_assert $CRM_EX_OK
 
     desc="Create a resource"
     cmd="cibadmin -C -o resources --xml-text '<primitive id=\"dummy\" class=\"ocf\" provider=\"pacemaker\" type=\"Dummy\"/>'"
     test_assert $CRM_EX_OK
 
     desc="Create a resource meta attribute"
     cmd="crm_resource -r dummy --meta -p is-managed -v false"
     test_assert $CRM_EX_OK
 
     desc="Query a resource meta attribute"
     cmd="crm_resource -r dummy --meta -g is-managed"
     test_assert $CRM_EX_OK
 
     desc="Remove a resource meta attribute"
     cmd="crm_resource -r dummy --meta -d is-managed"
     test_assert $CRM_EX_OK
 
     desc="Create a resource attribute"
     cmd="crm_resource -r dummy -p delay -v 10s"
     test_assert $CRM_EX_OK
 
     desc="List the configured resources"
     cmd="crm_resource -L"
     test_assert $CRM_EX_OK
 
     desc="Require a destination when migrating a resource that is stopped"
     cmd="crm_resource -r dummy -M"
     test_assert $CRM_EX_USAGE
 
     desc="Don't support migration to non-existent locations"
     cmd="crm_resource -r dummy -M -N i.do.not.exist"
     test_assert $CRM_EX_NOSUCH
 
     desc="Create a fencing resource"
     cmd="cibadmin -C -o resources --xml-text '<primitive id=\"Fence\" class=\"stonith\" type=\"fence_true\"/>'"
     test_assert $CRM_EX_OK
 
     desc="Bring resources online"
     cmd="crm_simulate --live-check --in-place -S"
     test_assert $CRM_EX_OK
 
     desc="Try to move a resource to its existing location"
     cmd="crm_resource -r dummy --move --host node1"
     test_assert $CRM_EX_EXISTS
 
     desc="Move a resource from its existing location"
     cmd="crm_resource -r dummy --move"
     test_assert $CRM_EX_OK
 
     desc="Clear out constraints generated by --move"
     cmd="crm_resource -r dummy --clear"
     test_assert $CRM_EX_OK
 
     desc="Default ticket granted state"
     cmd="crm_ticket -t ticketA -G granted -d false"
     test_assert $CRM_EX_OK
 
     desc="Set ticket granted state"
     cmd="crm_ticket -t ticketA -r --force"
     test_assert $CRM_EX_OK
 
     desc="Query ticket granted state"
     cmd="crm_ticket -t ticketA -G granted"
     test_assert $CRM_EX_OK
 
     desc="Delete ticket granted state"
     cmd="crm_ticket -t ticketA -D granted --force"
     test_assert $CRM_EX_OK
 
     desc="Make a ticket standby"
     cmd="crm_ticket -t ticketA -s"
     test_assert $CRM_EX_OK
 
     desc="Query ticket standby state"
     cmd="crm_ticket -t ticketA -G standby"
     test_assert $CRM_EX_OK
 
     desc="Activate a ticket"
     cmd="crm_ticket -t ticketA -a"
     test_assert $CRM_EX_OK
 
     desc="Delete ticket standby state"
     cmd="crm_ticket -t ticketA -D standby"
     test_assert $CRM_EX_OK
 
     desc="Ban a resource on unknown node"
     cmd="crm_resource -r dummy -B -N host1"
     test_assert $CRM_EX_NOSUCH
 
     desc="Create two more nodes and bring them online"
     cmd="crm_simulate --live-check --in-place --node-up=node2 --node-up=node3"
     test_assert $CRM_EX_OK
 
     desc="Ban dummy from node1"
     cmd="crm_resource -r dummy -B -N node1"
     test_assert $CRM_EX_OK
 
     desc="Ban dummy from node2"
     cmd="crm_resource -r dummy -B -N node2"
     test_assert $CRM_EX_OK
 
     desc="Relocate resources due to ban"
     cmd="crm_simulate --live-check --in-place -S"
     test_assert $CRM_EX_OK
 
     desc="Move dummy to node1"
     cmd="crm_resource -r dummy -M -N node1"
     test_assert $CRM_EX_OK
 
     desc="Clear implicit constraints for dummy on node2"
     cmd="crm_resource -r dummy -U -N node2"
     test_assert $CRM_EX_OK
 
     desc="Drop the status section"
     cmd="cibadmin -R -o status --xml-text '<status/>'"
     test_assert $CRM_EX_OK 0
     
     desc="Create a clone"
     cmd="cibadmin -C -o resources --xml-text '<clone id=\"test-clone\"><primitive id=\"test-primitive\" class=\"ocf\" provider=\"pacemaker\" type=\"Dummy\"/></clone>'"
     test_assert $CRM_EX_OK 0
 
     desc="Create a resource meta attribute"
     cmd="crm_resource -r test-primitive --meta -p is-managed -v false"
     test_assert $CRM_EX_OK
 
     desc="Create a resource meta attribute in the primitive"
     cmd="crm_resource -r test-primitive --meta -p is-managed -v false --force"
     test_assert $CRM_EX_OK
 
     desc="Update resource meta attribute with duplicates"
     cmd="crm_resource -r test-clone --meta -p is-managed -v true"
     test_assert $CRM_EX_OK
 
     desc="Update resource meta attribute with duplicates (force clone)"
     cmd="crm_resource -r test-clone --meta -p is-managed -v true --force"
     test_assert $CRM_EX_OK
 
     desc="Update child resource meta attribute with duplicates"
     cmd="crm_resource -r test-primitive --meta -p is-managed -v false"
     test_assert $CRM_EX_OK
 
     desc="Delete resource meta attribute with duplicates"
     cmd="crm_resource -r test-clone --meta -d is-managed"
     test_assert $CRM_EX_OK
 
     desc="Delete resource meta attribute in parent"
     cmd="crm_resource -r test-primitive --meta -d is-managed"
     test_assert $CRM_EX_OK
 
     desc="Create a resource meta attribute in the primitive"
     cmd="crm_resource -r test-primitive --meta -p is-managed -v false --force"
     test_assert $CRM_EX_OK
 
     desc="Update existing resource meta attribute"
     cmd="crm_resource -r test-clone --meta -p is-managed -v true"
     test_assert $CRM_EX_OK
     
     desc="Create a resource meta attribute in the parent"
     cmd="crm_resource -r test-clone --meta -p is-managed -v true --force"
     test_assert $CRM_EX_OK
 
     desc="Copy resources"
     cmd="cibadmin -Q -o resources > $TMPXML"
     test_assert $CRM_EX_OK 0
 
     desc="Delete resource paremt meta attribute (force)"
     cmd="crm_resource -r test-clone --meta -d is-managed --force"
     test_assert $CRM_EX_OK
 
     desc="Restore duplicates"
     cmd="cibadmin -R -o resources --xml-file $TMPXML"
     test_assert $CRM_EX_OK
 
     desc="Delete resource child meta attribute"
     cmd="crm_resource -r test-primitive --meta -d is-managed"
     test_assert $CRM_EX_OK
 
     cibadmin -C -o resources --xml-text '<group id="dummy-group"> \
         <primitive id="dummy1" class="ocf" provider="pacemaker" type="Dummy"\/> \
         <primitive id="dummy2" class="ocf" provider="pacemaker" type="Dummy"\/> \
       </group>'
 
     desc="Create a resource meta attribute in dummy1"
     cmd="crm_resource -r dummy1 --meta -p is-managed -v true"
     test_assert $CRM_EX_OK
 
     desc="Create a resource meta attribute in dummy-group"
     cmd="crm_resource -r dummy-group --meta -p is-managed -v false"
     test_assert $CRM_EX_OK
 
     cibadmin -D -o resource --xml-text '<group id="dummy-group">'
 
     desc="Specify a lifetime when moving a resource"
     cmd="crm_resource -r dummy --move --node node2 --lifetime=PT1H"
     test_assert $CRM_EX_OK
 
     desc="Try to move a resource previously moved with a lifetime"
     cmd="crm_resource -r dummy --move --node node1"
     test_assert $CRM_EX_OK
 
     desc="Ban dummy from node1 for a short time"
     cmd="crm_resource -r dummy -B -N node1 --lifetime=PT1S"
     test_assert $CRM_EX_OK
 
     desc="Remove expired constraints"
     sleep 2
     cmd="crm_resource --clear --expired"
     test_assert $CRM_EX_OK
 
+    unset CIB_shadow
     unset CIB_shadow_dir
     rm -f "$TMPXML" "$TMPORIG"
 
     desc="Create an XML patchset"
     cmd="crm_diff -o $test_home/cli/crm_diff_old.xml -n $test_home/cli/crm_diff_new.xml"
     test_assert $CRM_EX_ERROR 0
 }
 
 INVALID_PERIODS=(
     "2019-01-01 00:00:00Z"              # Start with no end
     "2019-01-01 00:00:00Z/"             # Start with only a trailing slash
     "PT2S/P1M"                          # Two durations
     "2019-13-01 00:00:00Z/P1M"          # Out-of-range month
     "20191077T15/P1M"                   # Out-of-range day
     "2019-10-01T25:00:00Z/P1M"          # Out-of-range hour
     "2019-10-01T24:00:01Z/P1M"          # Hour 24 with anything but :00:00
     "PT5H/20191001T007000Z"             # Out-of-range minute
     "2019-10-01 00:00:80Z/P1M"          # Out-of-range second
     "2019-10-01 00:00:10 +25:00/P1M"    # Out-of-range offset hour
     "20191001T000010 -00:61/P1M"        # Out-of-range offset minute
     "P1Y/2019-02-29 00:00:00Z"          # Feb. 29 in non-leap-year
     "2019-01-01 00:00:00Z/P"            # Duration with no values
     "P1Z/2019-02-20 00:00:00Z"          # Invalid duration unit
     "P1YM/2019-02-20 00:00:00Z"         # No number for duration unit
 )
         
 function test_dates() {
     # Ensure invalid period specifications are rejected
     for spec in '' "${INVALID_PERIODS[@]}"; do
         desc="Invalid period - [$spec]"
         cmd="iso8601 -p \"$spec\""
         test_assert $CRM_EX_INVALID_PARAM 0
     done
 
     desc="2014-01-01 00:30:00 - 1 Hour"
     cmd="iso8601 -d '2014-01-01 00:30:00Z' -D P-1H -E '2013-12-31 23:30:00Z'"
     test_assert $CRM_EX_OK 0
 
     desc="Valid date - Feb 29 in leap year"
     cmd="iso8601 -d '2020-02-29 00:00:00Z' -E '2020-02-29 00:00:00Z'"
     test_assert $CRM_EX_OK 0
 
     desc="Valid date - using 'T' and offset"
     cmd="iso8601 -d '20191201T131211 -05:00' -E '2019-12-01 18:12:11Z'"
     test_assert $CRM_EX_OK 0
 
     desc="24:00:00 equivalent to 00:00:00 of next day"
     cmd="iso8601 -d '2019-12-31 24:00:00Z' -E '2020-01-01 00:00:00Z'"
     test_assert $CRM_EX_OK 0
 
     for y in 06 07 08 09 10 11 12 13 14 15 16 17 18 40; do
         desc="20$y-W01-7"
         cmd="iso8601 -d '20$y-W01-7 00Z'"
         test_assert $CRM_EX_OK 0
 
         desc="20$y-W01-7 - round-trip"
         cmd="iso8601 -d '20$y-W01-7 00Z' -W -E '20$y-W01-7 00:00:00Z'"
         test_assert $CRM_EX_OK 0
 
         desc="20$y-W01-1"
         cmd="iso8601 -d '20$y-W01-1 00Z'"
         test_assert $CRM_EX_OK 0
 
         desc="20$y-W01-1 - round-trip"
         cmd="iso8601 -d '20$y-W01-1 00Z' -W -E '20$y-W01-1 00:00:00Z'"
         test_assert $CRM_EX_OK 0
     done
 
     desc="2009-W53-07"
     cmd="iso8601 -d '2009-W53-7 00:00:00Z' -W -E '2009-W53-7 00:00:00Z'"
     test_assert $CRM_EX_OK 0
 
     desc="epoch + 2 Years 5 Months 6 Minutes"
     cmd="iso8601 -d 'epoch' -D P2Y5MT6M -E '1972-06-01 00:06:00Z'"
     test_assert $CRM_EX_OK 0
 
     desc="2009-01-31 + 1 Month"
     cmd="iso8601 -d '20090131T000000Z' -D P1M -E '2009-02-28 00:00:00Z'"
     test_assert $CRM_EX_OK 0
 
     desc="2009-01-31 + 2 Months"
     cmd="iso8601 -d '2009-01-31 00:00:00Z' -D P2M -E '2009-03-31 00:00:00Z'"
     test_assert $CRM_EX_OK 0
 
     desc="2009-01-31 + 3 Months"
     cmd="iso8601 -d '2009-01-31 00:00:00Z' -D P3M -E '2009-04-30 00:00:00Z'"
     test_assert $CRM_EX_OK 0
 
     desc="2009-03-31 - 1 Month"
     cmd="iso8601 -d '2009-03-31 01:00:00 +01:00' -D P-1M -E '2009-02-28 00:00:00Z'"
     test_assert $CRM_EX_OK 0
 
     desc="2038-01-01 + 3 Months"
     cmd="iso8601 -d '2038-01-01 00:00:00Z' -D P3M -E '2038-04-01 00:00:00Z'"
     test_assert $CRM_EX_OK 0
 }
 
 function test_acl_loop() {
     local TMPXML
 
     TMPXML="$1"
 
     # Make sure we're rejecting things for the right reasons
     export PCMK_trace_functions=pcmk__check_acl,pcmk__apply_creation_acl
     export PCMK_stderr=1
 
     CIB_user=root cibadmin --replace --xml-text '<resources/>'
 
     export CIB_user=unknownguy
     desc="$CIB_user: Query configuration"
     cmd="cibadmin -Q"
     test_assert $CRM_EX_INSUFFICIENT_PRIV 0
 
     desc="$CIB_user: Set enable-acl"
     cmd="crm_attribute -n enable-acl -v false"
     test_assert $CRM_EX_INSUFFICIENT_PRIV 0
 
     desc="$CIB_user: Set stonith-enabled"
     cmd="crm_attribute -n stonith-enabled -v false"
     test_assert $CRM_EX_INSUFFICIENT_PRIV 0
 
     desc="$CIB_user: Create a resource"
     cmd="cibadmin -C -o resources --xml-text '<primitive id=\"dummy\" class=\"ocf\" provider=\"pacemaker\" type=\"Dummy\"/>'"
     test_assert $CRM_EX_INSUFFICIENT_PRIV 0
 
     export CIB_user=l33t-haxor
     desc="$CIB_user: Query configuration"
     cmd="cibadmin -Q"
     test_assert $CRM_EX_INSUFFICIENT_PRIV 0
 
     desc="$CIB_user: Set enable-acl"
     cmd="crm_attribute -n enable-acl -v false"
     test_assert $CRM_EX_INSUFFICIENT_PRIV 0
 
     desc="$CIB_user: Set stonith-enabled"
     cmd="crm_attribute -n stonith-enabled -v false"
     test_assert $CRM_EX_INSUFFICIENT_PRIV 0
 
     desc="$CIB_user: Create a resource"
     cmd="cibadmin -C -o resources --xml-text '<primitive id=\"dummy\" class=\"ocf\" provider=\"pacemaker\" type=\"Dummy\"/>'"
     test_assert $CRM_EX_INSUFFICIENT_PRIV 0
 
     export CIB_user=niceguy
     desc="$CIB_user: Query configuration"
     cmd="cibadmin -Q"
     test_assert $CRM_EX_OK 0
 
     desc="$CIB_user: Set enable-acl"
     cmd="crm_attribute -n enable-acl -v false"
     test_assert $CRM_EX_INSUFFICIENT_PRIV 0
 
     desc="$CIB_user: Set stonith-enabled"
     cmd="crm_attribute -n stonith-enabled -v false"
     test_assert $CRM_EX_OK
 
     desc="$CIB_user: Create a resource"
     cmd="cibadmin -C -o resources --xml-text '<primitive id=\"dummy\" class=\"ocf\" provider=\"pacemaker\" type=\"Dummy\"/>'"
     test_assert $CRM_EX_INSUFFICIENT_PRIV 0
 
     export CIB_user=root
     desc="$CIB_user: Query configuration"
     cmd="cibadmin -Q"
     test_assert $CRM_EX_OK 0
 
     desc="$CIB_user: Set stonith-enabled"
     cmd="crm_attribute -n stonith-enabled -v true"
     test_assert $CRM_EX_OK
 
     desc="$CIB_user: Create a resource"
     cmd="cibadmin -C -o resources --xml-text '<primitive id=\"dummy\" class=\"ocf\" provider=\"pacemaker\" type=\"Dummy\"/>'"
     test_assert $CRM_EX_OK
 
     export CIB_user=l33t-haxor
 
     desc="$CIB_user: Create a resource meta attribute"
     cmd="crm_resource -r dummy --meta -p target-role -v Stopped"
     test_assert $CRM_EX_INSUFFICIENT_PRIV 0
 
     desc="$CIB_user: Query a resource meta attribute"
     cmd="crm_resource -r dummy --meta -g target-role"
     test_assert $CRM_EX_INSUFFICIENT_PRIV 0
 
     desc="$CIB_user: Remove a resource meta attribute"
     cmd="crm_resource -r dummy --meta -d target-role"
     test_assert $CRM_EX_INSUFFICIENT_PRIV 0
 
     export CIB_user=niceguy
 
     desc="$CIB_user: Create a resource meta attribute"
     cmd="crm_resource -r dummy --meta -p target-role -v Stopped"
     test_assert $CRM_EX_OK
 
     desc="$CIB_user: Query a resource meta attribute"
     cmd="crm_resource -r dummy --meta -g target-role"
     test_assert $CRM_EX_OK
 
     desc="$CIB_user: Remove a resource meta attribute"
     cmd="crm_resource -r dummy --meta -d target-role"
     test_assert $CRM_EX_OK
 
     desc="$CIB_user: Create a resource meta attribute"
     cmd="crm_resource -r dummy --meta -p target-role -v Started"
     test_assert $CRM_EX_OK
 
     export CIB_user=badidea
     desc="$CIB_user: Query configuration - implied deny"
     cmd="cibadmin -Q"
     test_assert $CRM_EX_OK 0
 
     export CIB_user=betteridea
     desc="$CIB_user: Query configuration - explicit deny"
     cmd="cibadmin -Q"
     test_assert $CRM_EX_OK 0
 
     CIB_user=root cibadmin -Q > "$TMPXML"
     CIB_user=root CIB_file="$TMPXML" CIB_shadow="" cibadmin --delete --xml-text '<acls/>'
     CIB_user=root CIB_file="$TMPXML" CIB_shadow="" cibadmin -Ql
 
     export CIB_user=niceguy
     desc="$CIB_user: Replace - remove acls"
     cmd="cibadmin --replace --xml-file $TMPXML"
     test_assert $CRM_EX_INSUFFICIENT_PRIV 0
 
     CIB_user=root cibadmin -Q > "$TMPXML"
     CIB_user=root CIB_file="$TMPXML" CIB_shadow="" cibadmin -C -o resources --xml-text '<primitive id="dummy2" class="ocf" provider="pacemaker" type="Dummy"/>'
     CIB_user=root CIB_file="$TMPXML" CIB_shadow="" cibadmin -Ql
 
     desc="$CIB_user: Replace - create resource"
     cmd="cibadmin --replace --xml-file $TMPXML"
     test_assert $CRM_EX_INSUFFICIENT_PRIV 0
 
     CIB_user=root cibadmin -Q > "$TMPXML"
     CIB_user=root CIB_file="$TMPXML" CIB_shadow="" crm_attribute -n enable-acl -v false
     CIB_user=root CIB_file="$TMPXML" CIB_shadow="" cibadmin -Ql
 
     desc="$CIB_user: Replace - modify attribute (deny)"
     cmd="cibadmin --replace --xml-file $TMPXML"
     test_assert $CRM_EX_INSUFFICIENT_PRIV 0
 
     CIB_user=root cibadmin -Q > "$TMPXML"
     CIB_user=root CIB_file="$TMPXML" CIB_shadow="" cibadmin --replace --xml-text '<nvpair id="cib-bootstrap-options-enable-acl" name="enable-acl"/>'
     CIB_user=root CIB_file="$TMPXML" CIB_shadow="" cibadmin -Ql
 
     desc="$CIB_user: Replace - delete attribute (deny)"
     cmd="cibadmin --replace --xml-file $TMPXML"
     test_assert $CRM_EX_INSUFFICIENT_PRIV 0
 
     CIB_user=root cibadmin -Q > "$TMPXML"
     CIB_user=root CIB_file="$TMPXML" CIB_shadow="" cibadmin --modify --xml-text '<primitive id="dummy" description="nothing interesting"/>'
     CIB_user=root CIB_file="$TMPXML" CIB_shadow="" cibadmin -Ql
 
     desc="$CIB_user: Replace - create attribute (deny)"
     cmd="cibadmin --replace --xml-file $TMPXML"
     test_assert $CRM_EX_INSUFFICIENT_PRIV 0
 
     CIB_user=bob
     CIB_user=root cibadmin -Q > "$TMPXML"
     CIB_user=root CIB_file="$TMPXML" CIB_shadow="" cibadmin --modify --xml-text '<primitive id="dummy" description="nothing interesting"/>'
     CIB_user=root CIB_file="$TMPXML" CIB_shadow="" cibadmin -Ql
 
     desc="$CIB_user: Replace - create attribute (allow)"
     cmd="cibadmin --replace -o resources --xml-file $TMPXML"
     test_assert $CRM_EX_OK 0
 
     CIB_user=root cibadmin -Q > "$TMPXML"
     CIB_user=root CIB_file="$TMPXML" CIB_shadow="" cibadmin --modify --xml-text '<primitive id="dummy" description="something interesting"/>'
     CIB_user=root CIB_file="$TMPXML" CIB_shadow="" cibadmin -Ql
 
     desc="$CIB_user: Replace - modify attribute (allow)"
     cmd="cibadmin --replace -o resources --xml-file $TMPXML"
     test_assert $CRM_EX_OK 0
 
     CIB_user=root cibadmin -Q > "$TMPXML"
     CIB_user=root CIB_file="$TMPXML" CIB_shadow="" cibadmin --replace -o resources --xml-text '<primitive id="dummy" class="ocf" provider="pacemaker" type="Dummy"/>'
     CIB_user=root CIB_file="$TMPXML" CIB_shadow="" cibadmin -Ql
 
     desc="$CIB_user: Replace - delete attribute (allow)"
     cmd="cibadmin --replace -o resources --xml-file $TMPXML"
     test_assert $CRM_EX_OK 0
 }
 
 function test_acls() {
     local SHADOWPATH
     local TMPXML
 
     TMPXML=$(mktemp ${TMPDIR:-/tmp}/cts-cli.acls.xml.XXXXXXXXXX)
     export CIB_shadow_dir="${shadow_dir}"
 
     $VALGRIND_CMD crm_shadow --batch --force --create-empty $shadow --validate-with pacemaker-1.3 2>&1
     export CIB_shadow=$shadow
 
     cat <<EOF > "$TMPXML"
     <acls>
       <acl_user id="l33t-haxor">
         <deny id="crook-nothing" xpath="/cib"/>
       </acl_user>
       <acl_user id="niceguy">
         <role_ref id="observer"/>
       </acl_user>
       <acl_user id="bob">
         <role_ref id="admin"/>
       </acl_user>
       <acl_role id="observer">
         <read id="observer-read-1" xpath="/cib"/>
         <write id="observer-write-1" xpath="//nvpair[@name=&apos;stonith-enabled&apos;]"/>
         <write id="observer-write-2" xpath="//nvpair[@name=&apos;target-role&apos;]"/>
       </acl_role>
       <acl_role id="admin">
         <read id="admin-read-1" xpath="/cib"/>
         <write id="admin-write-1" xpath="//resources"/>
       </acl_role>
     </acls>
 EOF
 
     desc="Configure some ACLs"
     cmd="cibadmin -M -o acls --xml-file $TMPXML"
     test_assert $CRM_EX_OK
 
     desc="Enable ACLs"
     cmd="crm_attribute -n enable-acl -v true"
     test_assert $CRM_EX_OK
 
     desc="Set cluster option"
     cmd="crm_attribute -n no-quorum-policy -v ignore"
     test_assert $CRM_EX_OK
 
     desc="New ACL"
     cmd="cibadmin --create -o acls --xml-text '<acl_user id=\"badidea\"><read id=\"badidea-resources\" xpath=\"//meta_attributes\"/></acl_user>'"
     test_assert $CRM_EX_OK
 
     desc="Another ACL"
     cmd="cibadmin --create -o acls --xml-text '<acl_user id=\"betteridea\"><read id=\"betteridea-resources\" xpath=\"//meta_attributes\"/></acl_user>'"
     test_assert $CRM_EX_OK
 
     desc="Updated ACL"
     cmd="cibadmin --replace -o acls --xml-text '<acl_user id=\"betteridea\"><deny id=\"betteridea-nothing\" xpath=\"/cib\"/><read id=\"betteridea-resources\" xpath=\"//meta_attributes\"/></acl_user>'"
     test_assert $CRM_EX_OK
 
     test_acl_loop "$TMPXML"
 
     printf "\n\n    !#!#!#!#! Upgrading to latest CIB schema and re-testing !#!#!#!#!\n"
     printf "\nUpgrading to latest CIB schema and re-testing\n" 1>&2
 
     export CIB_user=root
     desc="$CIB_user: Upgrade to latest CIB schema"
     cmd="cibadmin --upgrade --force -V"
     test_assert $CRM_EX_OK
 
     SHADOWPATH="$(crm_shadow --file)"
     # sed -i isn't portable :-(
     cp -p "$SHADOWPATH" "${SHADOWPATH}.$$" # to keep permissions
     sed -e 's/epoch=.2/epoch=\"6/g' -e 's/admin_epoch=.1/admin_epoch=\"0/g' \
         "$SHADOWPATH" > "${SHADOWPATH}.$$"
     mv -- "${SHADOWPATH}.$$" "$SHADOWPATH"
 
     test_acl_loop "$TMPXML"
 
     unset CIB_shadow_dir
     rm -f "$TMPXML"
 }
 
 function test_validity() {
     local TMPGOOD
     local TMPBAD
 
     TMPGOOD=$(mktemp ${TMPDIR:-/tmp}/cts-cli.validity.good.xml.XXXXXXXXXX)
     TMPBAD=$(mktemp ${TMPDIR:-/tmp}/cts-cli.validity.bad.xml.XXXXXXXXXX)
     export CIB_shadow_dir="${shadow_dir}"
 
     $VALGRIND_CMD crm_shadow --batch --force --create-empty $shadow --validate-with pacemaker-1.2 2>&1
     export CIB_shadow=$shadow
     export PCMK_trace_functions=apply_upgrade,update_validation,cli_config_update
     export PCMK_stderr=1
 
     cibadmin -C -o resources --xml-text '<primitive id="dummy1" class="ocf" provider="pacemaker" type="Dummy"/>'
     cibadmin -C -o resources --xml-text '<primitive id="dummy2" class="ocf" provider="pacemaker" type="Dummy"/>'
     cibadmin -C -o constraints --xml-text '<rsc_order id="ord_1-2" first="dummy1" first-action="start" then="dummy2"/>'
     cibadmin -Q > "$TMPGOOD"
 
 
     desc="Try to make resulting CIB invalid (enum violation)"
     cmd="cibadmin -M -o constraints --xml-text '<rsc_order id=\"ord_1-2\" first=\"dummy1\" first-action=\"break\" then=\"dummy2\"/>'"
     test_assert $CRM_EX_CONFIG
 
     sed 's|"start"|"break"|' "$TMPGOOD" > "$TMPBAD"
     desc="Run crm_simulate with invalid CIB (enum violation)"
     cmd="crm_simulate -x $TMPBAD -S"
     test_assert $CRM_EX_CONFIG 0
 
 
     desc="Try to make resulting CIB invalid (unrecognized validate-with)"
     cmd="cibadmin -M --xml-text '<cib validate-with=\"pacemaker-9999.0\"/>'"
     test_assert $CRM_EX_CONFIG
 
     sed 's|"pacemaker-1.2"|"pacemaker-9999.0"|' "$TMPGOOD" > "$TMPBAD"
     desc="Run crm_simulate with invalid CIB (unrecognized validate-with)"
     cmd="crm_simulate -x $TMPBAD -S"
     test_assert $CRM_EX_CONFIG 0
 
 
     desc="Try to make resulting CIB invalid, but possibly recoverable (valid with X.Y+1)"
     cmd="cibadmin -C -o configuration --xml-text '<tags/>'"
     test_assert $CRM_EX_CONFIG
 
     sed 's|</configuration>|<tags/></configuration>|' "$TMPGOOD" > "$TMPBAD"
     desc="Run crm_simulate with invalid, but possibly recoverable CIB (valid with X.Y+1)"
     cmd="crm_simulate -x $TMPBAD -S"
     test_assert $CRM_EX_OK 0
 
 
     sed 's|[ 	][ 	]*validate-with="[^"]*"||' "$TMPGOOD" > "$TMPBAD"
     desc="Make resulting CIB valid, although without validate-with attribute"
     cmd="cibadmin -R --xml-file $TMPBAD"
     test_assert $CRM_EX_OK
 
     desc="Run crm_simulate with valid CIB, but without validate-with attribute"
     cmd="crm_simulate -x $TMPBAD -S"
     test_assert $CRM_EX_OK 0
 
 
     # this will just disable validation and accept the config, outputting
     # validation errors
     sed -e 's|[ 	][ 	]*validate-with="[^"]*"||' \
         -e 's|\([ 	][ 	]*epoch="[^"]*\)"|\10"|' -e 's|"start"|"break"|' \
         "$TMPGOOD" > "$TMPBAD"
     desc="Make resulting CIB invalid, and without validate-with attribute"
     cmd="cibadmin -R --xml-file $TMPBAD"
     test_assert $CRM_EX_OK
 
     desc="Run crm_simulate with invalid CIB, also without validate-with attribute"
     cmd="crm_simulate -x $TMPBAD -S"
     test_assert $CRM_EX_OK 0
 
     unset CIB_shadow_dir
     rm -f "$TMPGOOD" "$TMPBAD"
 }
 
 test_upgrade() {
     local TMPXML
 
     TMPXML=$(mktemp ${TMPDIR:-/tmp}/cts-cli.tools.xml.XXXXXXXXXX)
     export CIB_shadow_dir="${shadow_dir}"
 
     $VALGRIND_CMD crm_shadow --batch --force --create-empty $shadow --validate-with pacemaker-2.10 2>&1
     export CIB_shadow=$shadow
 
     desc="Set stonith-enabled=false"
     cmd="crm_attribute -n stonith-enabled -v false"
     test_assert $CRM_EX_OK
 
     cat <<EOF > "$TMPXML"
     <resources>
       <primitive id="mySmartFuse" class="ocf" provider="experiment" type="SmartFuse">
         <operations>
           <op id="mySmartFuse-start" name="start" interval="0" timeout="40s"/>
           <op id="mySmartFuse-monitor-inputpower" name="monitor" interval="30s">
             <instance_attributes id="mySmartFuse-inputpower-instanceparams">
               <nvpair id="mySmartFuse-inputpower-requires" name="requires" value="inputpower"/>
             </instance_attributes>
           </op>
           <op id="mySmartFuse-monitor-outputpower" name="monitor" interval="2s">
             <instance_attributes id="mySmartFuse-outputpower-instanceparams">
               <nvpair id="mySmartFuse-outputpower-requires" name="requires" value="outputpower"/>
             </instance_attributes>
           </op>
         </operations>
         <instance_attributes id="mySmartFuse-params">
           <nvpair id="mySmartFuse-params-ip" name="ip" value="192.0.2.10"/>
         </instance_attributes>
 	<!-- a bit hairy but valid -->
         <instance_attributes id-ref="mySmartFuse-outputpower-instanceparams"/>
       </primitive>
     </resources>
 EOF
 
     desc="Configure the initial resource"
     cmd="cibadmin -M -o resources --xml-file $TMPXML"
     test_assert $CRM_EX_OK
 
     desc="Upgrade to latest CIB schema (trigger 2.10.xsl + the wrapping)"
     cmd="cibadmin --upgrade --force -V -V"
     test_assert $CRM_EX_OK
 
     desc="Query a resource instance attribute (shall survive)"
     cmd="crm_resource -r mySmartFuse -g requires"
     test_assert $CRM_EX_OK
 
     unset CIB_shadow_dir
     rm -f "$TMPXML"
 }
 
 test_rules() {
     local TMPXML
 
     export CIB_shadow_dir="${shadow_dir}"
     $VALGRIND_CMD crm_shadow --batch --force --create-empty $shadow 2>&1
     export CIB_shadow=$shadow
 
     cibadmin -C -o resources   --xml-text '<primitive class="ocf" id="dummy" provider="heartbeat" type="Dummy" />'
 
     TMPXML=$(mktemp ${TMPDIR:-/tmp}/cts-cli.tools.xml.XXXXXXXXXX)
     cat <<EOF > "$TMPXML"
 <rsc_location id="cli-too-many-date-expressions" rsc="dummy">
   <rule id="cli-rule-too-many-date-expressions" score="INFINITY" boolean-op="or">
     <date_expression id="cli-date-expression-1" operation="gt" start="2020-01-01 01:00:00 -0500"/>
     <date_expression id="cli-date-expression-2" operation="lt" end="2019-01-01 01:00:00 -0500"/>
   </rule>
 </rsc_location>
 EOF
 
     cibadmin -C -o constraints -x "$TMPXML"
     rm -f "$TMPXML"
 
     TMPXML=$(mktemp ${TMPDIR:-/tmp}/cts-cli.tools.xml.XXXXXXXXXX)
     cat <<EOF > "$TMPXML"
 <rsc_location id="cli-prefer-dummy-expired" rsc="dummy">
   <rule id="cli-prefer-rule-dummy-expired" score="INFINITY">
     <date_expression id="cli-prefer-lifetime-end-dummy-expired" operation="lt" end="2019-01-01 12:00:00 -05:00"/>
   </rule>
 </rsc_location>
 EOF
 
     cibadmin -C -o constraints -x "$TMPXML"
     rm -f "$TMPXML"
 
     if [ "$(uname)" == "FreeBSD" ]; then
         tomorrow=$(date -v+1d +"%F %T %z")
     else
         tomorrow=$(date --date=tomorrow +"%F %T %z")
     fi
 
     TMPXML=$(mktemp ${TMPDIR:-/tmp}/cts-cli.tools.xml.XXXXXXXXXX)
     cat <<EOF > "$TMPXML"
 <rsc_location id="cli-prefer-dummy-not-yet" rsc="dummy">
   <rule id="cli-prefer-rule-dummy-not-yet" score="INFINITY">
     <date_expression id="cli-prefer-lifetime-end-dummy-not-yet" operation="gt" start="${tomorrow}"/>
   </rule>
 </rsc_location>
 EOF
 
     cibadmin -C -o constraints -x "$TMPXML"
     rm -f "$TMPXML"
 
     TMPXML=$(mktemp ${TMPDIR:-/tmp}/cts-cli.tools.xml.XXXXXXXXXX)
     cat <<EOF > "$TMPXML"
 <rsc_location id="cli-prefer-dummy-date_spec-only-years" rsc="dummy">
   <rule id="cli-prefer-rule-dummy-date_spec-only-years" score="INFINITY">
     <date_expression id="cli-prefer-dummy-date_spec-only-years-expr" operation="date_spec">
       <date_spec id="cli-prefer-dummy-date_spec-only-years-spec" years="2019"/>
     </date_expression>
   </rule>
 </rsc_location>
 EOF
 
     cibadmin -C -o constraints -x "$TMPXML"
     rm -f "$TMPXML"
 
     TMPXML=$(mktemp ${TMPDIR:-/tmp}/cts-cli.tools.xml.XXXXXXXXXX)
     cat <<EOF > "$TMPXML"
 <rsc_location id="cli-prefer-dummy-date_spec-without-years" rsc="dummy">
   <rule id="cli-prefer-rule-dummy-date_spec-without-years" score="INFINITY">
     <date_expression id="cli-prefer-dummy-date_spec-without-years-expr" operation="date_spec">
       <date_spec id="cli-prefer-dummy-date_spec-without-years-spec" hours="20" months="1,3,5,7"/>
     </date_expression>
   </rule>
 </rsc_location>
 EOF
 
     cibadmin -C -o constraints -x "$TMPXML"
     rm -f "$TMPXML"
 
     TMPXML=$(mktemp ${TMPDIR:-/tmp}/cts-cli.tools.xml.XXXXXXXXXX)
     cat <<EOF > "$TMPXML"
 <rsc_location id="cli-prefer-dummy-date_spec-years-moon" rsc="dummy">
   <rule id="cli-prefer-rule-dummy-date_spec-years-moon" score="INFINITY">
     <date_expression id="cli-prefer-dummy-date_spec-years-moon-expr" operation="date_spec">
       <date_spec id="cli-prefer-dummy-date_spec-years-moon-spec" years="2019" moon="1"/>
     </date_expression>
   </rule>
 </rsc_location>
 EOF
 
     cibadmin -C -o constraints -x "$TMPXML"
     rm -f "$TMPXML"
 
     TMPXML=$(mktemp ${TMPDIR:-/tmp}/cts-cli.tools.xml.XXXXXXXXXX)
     cat <<EOF > "$TMPXML"
 <rsc_location id="cli-no-date_expression" rsc="dummy">
   <rule id="cli-no-date_expression-rule" score="INFINITY">
     <expression id="ban-apache-expr" attribute="#uname" operation="eq" value="node3"/>
   </rule>
 </rsc_location>
 EOF
 
     cibadmin -C -o constraints -x "$TMPXML"
     rm -f "$TMPXML"
 
     desc="Try to check a rule that doesn't exist"
     cmd="crm_rule -c -r blahblah"
     test_assert $CRM_EX_NOSUCH
 
     desc="Try to check a rule that has too many date_expressions"
     cmd="crm_rule -c -r cli-rule-too-many-date-expressions"
     test_assert $CRM_EX_UNIMPLEMENT_FEATURE
 
     desc="Verify basic rule is expired"
     cmd="crm_rule -c -r cli-prefer-rule-dummy-expired"
     test_assert $CRM_EX_EXPIRED
 
     desc="Verify basic rule worked in the past"
     cmd="crm_rule -c -r cli-prefer-rule-dummy-expired -d 20180101"
     test_assert $CRM_EX_OK
 
     desc="Verify basic rule is not yet in effect"
     cmd="crm_rule -c -r cli-prefer-rule-dummy-not-yet"
     test_assert $CRM_EX_NOT_YET_IN_EFFECT
 
     desc="Verify date_spec rule with years has expired"
     cmd="crm_rule -c -r cli-prefer-rule-dummy-date_spec-only-years"
     test_assert $CRM_EX_EXPIRED
 
     desc="Verify date_spec rule with years is in effect"
     cmd="crm_rule -c -r cli-prefer-rule-dummy-date_spec-only-years -d 20190201"
     test_assert $CRM_EX_OK
 
     desc="Try to check a rule whose date_spec does not contain years="
     cmd="crm_rule -c -r cli-prefer-rule-dummy-date_spec-without-years"
     test_assert $CRM_EX_NOSUCH
 
     desc="Try to check a rule whose date_spec contains years= and moon="
     cmd="crm_rule -c -r cli-prefer-rule-dummy-date_spec-years-moon"
     test_assert $CRM_EX_NOSUCH
 
     desc="Try to check a rule with no date_expression"
     cmd="crm_rule -c -r cli-no-date_expression-rule"
     test_assert $CRM_EX_UNIMPLEMENT_FEATURE
 
     unset CIB_shadow_dir
 }
 
 # Process command-line arguments
 while [ $# -gt 0 ]; do
     case "$1" in
         -t)
             tests="$2"
             shift 2
             ;;
         -V|--verbose)
             verbose=1
             shift
             ;;
         -v|--valgrind)
             export G_SLICE=always-malloc
             VALGRIND_CMD="valgrind $VALGRIND_OPTS"
             shift
             ;;
         -s)
             do_save=1
             shift
             ;;
         -p)
             export PATH="$2:$PATH"
             shift
             ;;
         --help)
             echo "$USAGE_TEXT"
             exit $CRM_EX_OK
             ;;
         *)
             echo "error: unknown option $1"
             echo
             echo "$USAGE_TEXT"
             exit $CRM_EX_USAGE
             ;;
     esac
 done
 
 for t in $tests; do
     case "$t" in
         dates) ;;
         tools) ;;
         acls) ;;
         validity) ;;
         upgrade) ;;
         rules) ;;
+        crm_mon) ;;
         *)
             echo "error: unknown test $t"
             echo
             echo "$USAGE_TEXT"
             exit $CRM_EX_USAGE
             ;;
     esac
 done
 
 # Check whether we're running from source directory
 SRCDIR=$(dirname $test_home)
 if [ -x "$SRCDIR/tools/crm_simulate" ]; then
     export PATH="$SRCDIR/tools:$PATH"
     echo "Using local binaries from: $SRCDIR/tools"
 
     if [ -x "$SRCDIR/xml" ]; then
         export PCMK_schema_directory="$SRCDIR/xml"
         echo "Using local schemas from: $PCMK_schema_directory"
     fi
 fi
 
 for t in $tests; do
     echo "Testing $t"
     TMPFILE=$(mktemp ${TMPDIR:-/tmp}/cts-cli.$t.XXXXXXXXXX)
     eval TMPFILE_$t="$TMPFILE"
     test_$t > "$TMPFILE"
 
+    # last-run= and last-rc-change= are always numeric in the CIB.  However,
+    # for the crm_mon test we also need to compare against the XML output of
+    # the crm_mon program.  There, these are shown as human readable strings
+    # (like the output of the `date` command).
     sed -e 's/cib-last-written.*>/>/'\
-        -e 's/ last-run=\"[0-9]*\"//'\
+        -e 's/ last-run=\"[A-Za-z0-9: ]*\"//'\
+        -e 's/Last updated: .*/Last updated:/' \
+        -e 's/Last change: .*/Last change:/' \
+        -e 's/(version .*)/(version)/' \
+        -e 's/last_update time=\".*\"/last_update time=\"\"/' \
+        -e 's/last_change time=\".*\"/last_change time=\"\"/' \
+        -e 's/ version=\".*\" / version=\"\" /' \
+        -e 's/request=\".*crm_mon/request=\"crm_mon/' \
         -e 's/crm_feature_set="[^"]*" //'\
         -e 's/validate-with="[^"]*" //'\
         -e 's/Created new pacemaker-.* configuration/Created new pacemaker configuration/'\
         -e 's/.*\(pcmk__.*\)@.*\.c:[0-9][0-9]*)/\1/g' \
         -e 's/.*\(unpack_.*\)@.*\.c:[0-9][0-9]*)/\1/g' \
         -e 's/.*\(update_validation\)@.*\.c:[0-9][0-9]*)/\1/g' \
         -e 's/.*\(apply_upgrade\)@.*\.c:[0-9][0-9]*)/\1/g' \
-        -e 's/ last-rc-change=\"[0-9]*\"//'\
+        -e 's/ last-rc-change=\"[A-Za-z0-9: ]*\"//'\
         -e 's|^/tmp/cts-cli\.validity\.bad.xml\.[^:]*:|validity.bad.xml:|'\
         -e 's/^Entity: line [0-9][0-9]*: //'\
         -e 's/\(validation ([0-9][0-9]* of \)[0-9][0-9]*\().*\)/\1X\2/' \
         -e 's/^Migration will take effect until: .*/Migration will take effect until:/' \
         -e 's/ end=\"[0-9][-+: 0-9]*Z*\"/ end=\"\"/' \
         -e 's/ start=\"[0-9][-+: 0-9]*Z*\"/ start=\"\"/' \
         -e 's/^Error checking rule: Device not configured/Error checking rule: No such device or address/' \
         "$TMPFILE" > "${TMPFILE}.$$"
     mv -- "${TMPFILE}.$$" "$TMPFILE"
 
     if [ $do_save -eq 1 ]; then
         cp "$TMPFILE" $test_home/cli/regression.$t.exp
     fi
 done
 
 rm -rf "${shadow_dir}"
     
 failed=0
 
 if [ $verbose -eq 1 ]; then
     echo -e "\n\nResults"
 fi
 for t in $tests; do
     eval TMPFILE="\$TMPFILE_$t"
     if [ $verbose -eq 1 ]; then
         diff -wu $test_home/cli/regression.$t.exp "$TMPFILE"
     else
         diff -w $test_home/cli/regression.$t.exp "$TMPFILE" >/dev/null 2>&1
     fi
     if [ $? -ne 0 ]; then
         failed=1
     fi
 done
 
 echo -e "\n\nSummary"
 for t in $tests; do
     eval TMPFILE="\$TMPFILE_$t"
     grep -e '^\*' "$TMPFILE"
 done
 
 if [ $num_errors -ne 0 ]; then
     echo "$num_errors tests failed; see output in:"
     for t in $tests; do
         eval TMPFILE="\$TMPFILE_$t"
         echo "    $TMPFILE"
     done
     exit $CRM_EX_ERROR
 
 elif [ $failed -eq 1 ]; then
     echo "$num_passed tests passed but output was unexpected; see output in:"
     for t in $tests; do
         eval TMPFILE="\$TMPFILE_$t"
         echo "    $TMPFILE"
     done
     exit $CRM_EX_DIGEST
 
 else
     echo $num_passed tests passed
     for t in $tests; do
         eval TMPFILE="\$TMPFILE_$t"
         rm -f "$TMPFILE"
     done
     crm_shadow --force --delete $shadow >/dev/null 2>&1
     exit $CRM_EX_OK
 fi
diff --git a/daemons/based/pacemaker-based.c b/daemons/based/pacemaker-based.c
index 91124db22d..2609742ca2 100644
--- a/daemons/based/pacemaker-based.c
+++ b/daemons/based/pacemaker-based.c
@@ -1,375 +1,375 @@
 /*
  * Copyright 2004-2020 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU General Public License version 2
  * or later (GPLv2+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #include <stdio.h>
 #include <stdlib.h>
 #include <pwd.h>
 #include <grp.h>
 #include <bzlib.h>
 #include <sys/types.h>
 
 #include <libxml/parser.h>
 
 #include <crm/crm.h>
 #include <crm/cib/internal.h>
 #include <crm/msg_xml.h>
 #include <crm/cluster/internal.h>
 #include <crm/common/xml.h>
 #include <crm/common/mainloop.h>
 
 #include <pacemaker-based.h>
 
 extern int init_remote_listener(int port, gboolean encrypted);
 gboolean cib_shutdown_flag = FALSE;
 int cib_status = pcmk_ok;
 
 crm_cluster_t crm_cluster;
 
 GMainLoop *mainloop = NULL;
 const char *cib_root = NULL;
 char *cib_our_uname = NULL;
 static gboolean preserve_status = FALSE;
 
 gboolean cib_writes_enabled = TRUE;
 
 int remote_fd = 0;
 int remote_tls_fd = 0;
 
 GHashTable *config_hash = NULL;
 GHashTable *local_notify_queue = NULL;
 
 static void cib_init(void);
 void cib_shutdown(int nsig);
 static bool startCib(const char *filename);
 extern int write_cib_contents(gpointer p);
 void cib_cleanup(void);
 
 static void
 cib_enable_writes(int nsig)
 {
     crm_info("(Re)enabling disk writes");
     cib_writes_enabled = TRUE;
 }
 
 static pcmk__cli_option_t long_options[] = {
     // long option, argument type, storage, short option, description, flags
     {
         "help", no_argument, 0, '?',
         "\tThis text", pcmk__option_default
     },
     {
         "verbose", no_argument, NULL, 'V',
         "\tIncrease debug output", pcmk__option_default
     },
     {
         "stand-alone", no_argument, NULL, 's',
         "\tAdvanced use only", pcmk__option_default
     },
     {
         "disk-writes", no_argument, NULL, 'w',
         "\tAdvanced use only", pcmk__option_default
     },
     {
         "cib-root", required_argument, NULL, 'r',
         "\tAdvanced use only", pcmk__option_default
     },
     { 0, 0, 0, 0 }
 };
 
 int
 main(int argc, char **argv)
 {
     int flag;
     int rc = 0;
     int index = 0;
     int argerr = 0;
     struct passwd *pwentry = NULL;
     crm_ipc_t *old_instance = NULL;
 
     crm_log_preinit(NULL, argc, argv);
     pcmk__set_cli_options(NULL, "[options]", long_options,
                           "daemon for managing the configuration "
                           "of a Pacemaker cluster");
 
     mainloop_add_signal(SIGTERM, cib_shutdown);
     mainloop_add_signal(SIGPIPE, cib_enable_writes);
 
     cib_writer = mainloop_add_trigger(G_PRIORITY_LOW, write_cib_contents, NULL);
 
     while (1) {
         flag = pcmk__next_cli_option(argc, argv, &index, NULL);
         if (flag == -1)
             break;
 
         switch (flag) {
             case 'V':
                 crm_bump_log_level(argc, argv);
                 break;
             case 's':
                 stand_alone = TRUE;
                 preserve_status = TRUE;
                 cib_writes_enabled = FALSE;
 
                 pwentry = getpwnam(CRM_DAEMON_USER);
                 CRM_CHECK(pwentry != NULL,
                           crm_perror(LOG_ERR, "Invalid uid (%s) specified", CRM_DAEMON_USER);
                           return CRM_EX_FATAL);
 
                 rc = setgid(pwentry->pw_gid);
                 if (rc < 0) {
                     crm_perror(LOG_ERR, "Could not set group to %d", pwentry->pw_gid);
                     return CRM_EX_FATAL;
                 }
 
                 rc = initgroups(CRM_DAEMON_USER, pwentry->pw_gid);
                 if (rc < 0) {
                     crm_perror(LOG_ERR, "Could not setup groups for user %d", pwentry->pw_uid);
                     return CRM_EX_FATAL;
                 }
 
                 rc = setuid(pwentry->pw_uid);
                 if (rc < 0) {
                     crm_perror(LOG_ERR, "Could not set user to %d", pwentry->pw_uid);
                     return CRM_EX_FATAL;
                 }
                 break;
             case '?':          /* Help message */
                 pcmk__cli_help(flag, CRM_EX_OK);
                 break;
             case 'w':
                 cib_writes_enabled = TRUE;
                 break;
             case 'r':
                 cib_root = optarg;
                 break;
             case 'm':
                 cib_metadata();
                 return CRM_EX_OK;
             default:
                 ++argerr;
                 break;
         }
     }
     if (argc - optind == 1 && safe_str_eq("metadata", argv[optind])) {
         cib_metadata();
         return CRM_EX_OK;
     }
 
     if (optind > argc) {
         ++argerr;
     }
 
     if (argerr) {
         pcmk__cli_help('?', CRM_EX_USAGE);
     }
 
     crm_log_init(NULL, LOG_INFO, TRUE, FALSE, argc, argv, FALSE);
 
     crm_notice("Starting Pacemaker CIB manager");
 
     old_instance = crm_ipc_new(PCMK__SERVER_BASED_RO, 0);
     if (crm_ipc_connect(old_instance)) {
         /* IPC end-point already up */
         crm_ipc_close(old_instance);
         crm_ipc_destroy(old_instance);
         crm_err("pacemaker-based is already active, aborting startup");
         crm_exit(CRM_EX_OK);
     } else {
         /* not up or not authentic, we'll proceed either way */
         crm_ipc_destroy(old_instance);
         old_instance = NULL;
     }
 
     if (cib_root == NULL) {
         cib_root = CRM_CONFIG_DIR;
     } else {
         crm_notice("Using custom config location: %s", cib_root);
     }
 
     if (pcmk__daemon_can_write(cib_root, NULL) == FALSE) {
         crm_err("Terminating due to bad permissions on %s", cib_root);
         fprintf(stderr, "ERROR: Bad permissions on %s (see logs for details)\n",
                 cib_root);
         fflush(stderr);
         return CRM_EX_FATAL;
     }
 
     crm_peer_init();
 
     // Read initial CIB, connect to cluster, and start IPC servers
     cib_init();
 
     // Run the main loop
     mainloop = g_main_loop_new(NULL, FALSE);
     crm_notice("Pacemaker CIB manager successfully started and accepting connections");
     g_main_loop_run(mainloop);
 
     /* If main loop returned, clean up and exit. We disconnect in case
      * terminate_cib() was called with fast=-1.
      */
     crm_cluster_disconnect(&crm_cluster);
     pcmk__stop_based_ipc(ipcs_ro, ipcs_rw, ipcs_shm);
-    return CRM_EX_OK;
+    crm_exit(CRM_EX_OK);
 }
 
 void
 cib_cleanup(void)
 {
     crm_peer_destroy();
     if (local_notify_queue) {
         g_hash_table_destroy(local_notify_queue);
     }
     pcmk__client_cleanup();
     g_hash_table_destroy(config_hash);
     free(cib_our_uname);
 }
 
 #if SUPPORT_COROSYNC
 static void
 cib_cs_dispatch(cpg_handle_t handle,
                  const struct cpg_name *groupName,
                  uint32_t nodeid, uint32_t pid, void *msg, size_t msg_len)
 {
     uint32_t kind = 0;
     xmlNode *xml = NULL;
     const char *from = NULL;
     char *data = pcmk_message_common_cs(handle, nodeid, pid, msg, &kind, &from);
 
     if(data == NULL) {
         return;
     }
     if (kind == crm_class_cluster) {
         xml = string2xml(data);
         if (xml == NULL) {
             crm_err("Invalid XML: '%.120s'", data);
             free(data);
             return;
         }
         crm_xml_add(xml, F_ORIG, from);
         /* crm_xml_add_int(xml, F_SEQ, wrapper->id); */
         cib_peer_callback(xml, NULL);
     }
 
     free_xml(xml);
     free(data);
 }
 
 static void
 cib_cs_destroy(gpointer user_data)
 {
     if (cib_shutdown_flag) {
         crm_info("Corosync disconnection complete");
     } else {
         crm_crit("Lost connection to cluster layer, shutting down");
         terminate_cib(__FUNCTION__, CRM_EX_DISCONNECT);
     }
 }
 #endif
 
 static void
 cib_peer_update_callback(enum crm_status_type type, crm_node_t * node, const void *data)
 {
     switch (type) {
         case crm_status_processes:
             if (cib_legacy_mode()
                 && is_not_set(node->processes, crm_get_cluster_proc())) {
 
                 uint32_t old = data? *(const uint32_t *)data : 0;
 
                 if ((node->processes ^ old) & crm_proc_cpg) {
                     crm_info("Attempting to disable legacy mode after %s left the cluster",
                              node->uname);
                     legacy_mode = FALSE;
                 }
             }
             break;
 
         case crm_status_uname:
         case crm_status_nstate:
             if (cib_shutdown_flag && (crm_active_peers() < 2)
                 && (pcmk__ipc_client_count() == 0)) {
 
                 crm_info("No more peers");
                 terminate_cib(__FUNCTION__, -1);
             }
             break;
     }
 }
 
 static void
 cib_init(void)
 {
     if (is_corosync_cluster()) {
 #if SUPPORT_COROSYNC
         crm_cluster.destroy = cib_cs_destroy;
         crm_cluster.cpg.cpg_deliver_fn = cib_cs_dispatch;
         crm_cluster.cpg.cpg_confchg_fn = pcmk_cpg_membership;
 #endif
     }
 
     config_hash = crm_str_table_new();
 
     if (startCib("cib.xml") == FALSE) {
         crm_crit("Cannot start CIB... terminating");
         crm_exit(CRM_EX_NOINPUT);
     }
 
     if (stand_alone == FALSE) {
         if (is_corosync_cluster()) {
             crm_set_status_callback(&cib_peer_update_callback);
         }
 
         if (crm_cluster_connect(&crm_cluster) == FALSE) {
             crm_crit("Cannot sign in to the cluster... terminating");
             crm_exit(CRM_EX_FATAL);
         }
         cib_our_uname = crm_cluster.uname;
 
     } else {
         cib_our_uname = strdup("localhost");
     }
 
     pcmk__serve_based_ipc(&ipcs_ro, &ipcs_rw, &ipcs_shm, &ipc_ro_callbacks,
                           &ipc_rw_callbacks);
 
     if (stand_alone) {
         cib_is_master = TRUE;
     }
 }
 
 static bool
 startCib(const char *filename)
 {
     gboolean active = FALSE;
     xmlNode *cib = readCibXmlFile(cib_root, filename, !preserve_status);
 
     if (activateCibXml(cib, TRUE, "start") == 0) {
         int port = 0;
         const char *port_s = NULL;
 
         active = TRUE;
 
         cib_read_config(config_hash, cib);
 
         port_s = crm_element_value(cib, "remote-tls-port");
         if (port_s) {
             port = crm_parse_int(port_s, "0");
             remote_tls_fd = init_remote_listener(port, TRUE);
         }
 
         port_s = crm_element_value(cib, "remote-clear-port");
         if (port_s) {
             port = crm_parse_int(port_s, "0");
             remote_fd = init_remote_listener(port, FALSE);
         }
     }
     return active;
 }
diff --git a/extra/resources/attribute b/extra/resources/attribute
index 8b3ed80acd..893b1b04d4 100755
--- a/extra/resources/attribute
+++ b/extra/resources/attribute
@@ -1,236 +1,236 @@
 #!/bin/sh
 #
 # ocf:pacemaker:attribute resource agent
 #
 # Copyright 2016-2019 the Pacemaker project contributors
 #
 # The version control history for this file may have further details.
 #
 # This source code is licensed under the GNU General Public License version 2
 # or later (GPLv2+) WITHOUT ANY WARRANTY.
 #
 
 USAGE="Usage: $0 {start|stop|monitor|migrate_to|migrate_from|validate-all|meta-data}
 
 Expects to have a fully populated OCF RA-compliant environment set."
 
 # Load OCF helper functions
 : ${OCF_FUNCTIONS:="${OCF_ROOT}/resource.d/heartbeat/.ocf-shellfuncs"}
 . "${OCF_FUNCTIONS}"
 : ${__OCF_ACTION:="$1"}
 
 # Ensure certain variables are set and not empty
 : ${HA_VARRUN:="/var/run"}
 : ${OCF_RESKEY_CRM_meta_globally_unique:="false"}
 : ${OCF_RESOURCE_INSTANCE:="undef"}
 
 DEFAULT_STATE_FILE="${HA_VARRUN%%/}/opa-${OCF_RESOURCE_INSTANCE}.state"
 if [ "${OCF_RESKEY_CRM_meta_globally_unique}" = "false" ]; then
     # Strip off any trailing clone marker (note + is not portable in sed)
     DEFAULT_STATE_FILE=$(echo "$DEFAULT_STATE_FILE" | sed s/:[0-9][0-9]*\.state/.state/)
 fi
 
 DEFAULT_ATTR_NAME="opa-${OCF_RESOURCE_INSTANCE}"
 DEFAULT_ACTIVE_VALUE="1"
 DEFAULT_INACTIVE_VALUE="0"
 
 : ${OCF_RESKEY_state:="$DEFAULT_STATE_FILE"}
 : ${OCF_RESKEY_name:="$DEFAULT_ATTR_NAME"}
 
 # If the user did not set a value, use the default. If the user explicitly set
 # a value to the empty string, use that (-z "${V+x}" tests whether $V was set).
 if [ -z "${OCF_RESKEY_active_value+x}" ]; then
     OCF_RESKEY_active_value="$DEFAULT_ACTIVE_VALUE"
 fi
 if [ -z "${OCF_RESKEY_inactive_value+x}" ]; then
     OCF_RESKEY_inactive_value="$DEFAULT_INACTIVE_VALUE"
 fi
 
 usage() {
     USAGE_RC=$1
     cat <<END
 $USAGE
 END
     return $USAGE_RC
 }
 
 meta_data() {
     cat <<END
 <?xml version="1.0"?>
 <!DOCTYPE resource-agent SYSTEM "ra-api-1.dtd">
 <resource-agent name="attribute" version="1.0">
   <version>1.0</version>
   <shortdesc lang="en">Manages a node attribute</shortdesc>
   <longdesc lang="en">
 This resource agent controls a node attribute for the node it's running on.
 It sets the attribute one way when started, and another way when stopped,
 according to the configuration parameters.
   </longdesc>
   <parameters>
 
     <parameter name="state" unique="1">
       <longdesc lang="en">
 Full path of a temporary file to store the resource state in
       </longdesc>
       <shortdesc lang="en">State file</shortdesc>
       <content type="string" default="${DEFAULT_STATE_FILE}" />
     </parameter>
 
-    <parameter name="name" unique="1">
+    <parameter name="name" unique="0">
       <longdesc lang="en">
 Name of node attribute to manage
       </longdesc>
       <shortdesc lang="en">Attribute name</shortdesc>
       <content type="string" default="${DEFAULT_ATTR_NAME}" />
     </parameter>
 
     <parameter name="active_value" unique="0">
       <longdesc lang="en">
 Value to use for node attribute when resource becomes active (empty string is
 discouraged, because monitor cannot distinguish it from a query error)
       </longdesc>
       <shortdesc lang="en">Attribute value when active</shortdesc>
       <content type="string" default="$DEFAULT_ACTIVE_VALUE" />
     </parameter>
 
     <parameter name="inactive_value" unique="0">
       <longdesc lang="en">
 Value to use for node attribute when resource becomes inactive
       </longdesc>
       <shortdesc lang="en">Attribute value when inactive</shortdesc>
       <content type="string" default="$DEFAULT_INACTIVE_VALUE" />
     </parameter>
 
   </parameters>
   <actions>
     <action name="start"        timeout="20s" />
     <action name="stop"         timeout="20s" />
     <action name="monitor"      timeout="20s" interval="10s" depth="0"/>
     <action name="reload"       timeout="20s" />
     <action name="migrate_to"   timeout="20s" />
     <action name="migrate_from" timeout="20s" />
     <action name="validate-all" timeout="20s" />
     <action name="meta-data"    timeout="5s" />
   </actions>
 </resource-agent>
 END
     return $OCF_SUCCESS
 }
 
 validate() {
     if [ "$OCF_RESKEY_active_value" = "$OCF_RESKEY_inactive_value" ]; then
         ocf_exit_reason "active value '%s' must be different from inactive value '%s'" \
             "$OCF_RESKEY_active_value" "$OCF_RESKEY_inactive_value"
         return $OCF_ERR_CONFIGURED
     fi
 
     VALIDATE_DIR=$(dirname "${OCF_RESKEY_state}")
     if [ ! -d "$VALIDATE_DIR" ]; then
         ocf_exit_reason "state file '$OCF_RESKEY_state' does not have a valid directory"
         return $OCF_ERR_PERM
     fi
     if [ ! -w "$VALIDATE_DIR" ] || [ ! -x "$VALIDATE_DIR" ]; then
         ocf_exit_reason "insufficient privileges on directory of state file '$OCF_RESKEY_state'"
         return $OCF_ERR_PERM
     fi
 
     return $OCF_SUCCESS
 }
 
 get_attribute() {
     GET_LINE=$(attrd_updater -n "$OCF_RESKEY_name" -Q 2>/dev/null)
     if [ $? -ne 0 ]; then
         echo ""
     else
         echo "$GET_LINE" | sed -e "s/.* value=\"\(.*\)\"$/\1/"
     fi
 }
 
 set_attribute() {
     attrd_updater -n "$OCF_RESKEY_name" -U "$1" 2>/dev/null
     # TODO if above call is async, loop until get_attribute returns expected value
 }
 
 check_attribute() {
     CHECK_VALUE=$(get_attribute)
     CHECK_REASON=""
     if [ ! -f "$OCF_RESKEY_state" ]; then
         if [ "$CHECK_VALUE" != "" ] && [ "$CHECK_VALUE" != "$OCF_RESKEY_inactive_value" ]; then
             CHECK_REASON="Node attribute $OCF_RESKEY_name='$CHECK_VALUE' differs from expected value '$OCF_RESKEY_inactive_value'"
             return $OCF_ERR_GENERIC
         fi
         return $OCF_NOT_RUNNING
     fi
     if [ "$CHECK_VALUE" != "$OCF_RESKEY_active_value" ]; then
         CHECK_REASON="Node attribute $OCF_RESKEY_name='$CHECK_VALUE' differs from expected value '$OCF_RESKEY_active_value'"
         return $OCF_ERR_GENERIC
     fi
     return $OCF_SUCCESS
 }
 
 monitor() {
     check_attribute
     MONITOR_RC=$?
     if [ $MONITOR_RC -eq $OCF_ERR_GENERIC ]; then
         ocf_exit_reason "$CHECK_REASON"
     fi
     return $MONITOR_RC
 }
 
 start() {
     check_attribute
     if [ $? -eq $OCF_SUCCESS ]; then
         return $OCF_SUCCESS
     fi
 
     touch "${OCF_RESKEY_state}" 2>/dev/null
     if [ $? -ne 0 ]; then
         ocf_exit_reason "Unable to manage state file $OCF_RESKEY_state"
         return $OCF_ERR_GENERIC
     fi
 
     set_attribute "${OCF_RESKEY_active_value}"
     if [ $? -ne 0 ]; then
         rm -f "${OCF_RESKEY_state}"
         ocf_exit_reason "Unable to set node attribute $OCF_RESKEY_name='$OCF_RESKEY_active_value'"
         return $OCF_ERR_GENERIC
     fi
 
     return $OCF_SUCCESS
 }
 
 stop() {
     check_attribute
     if [ $? -eq $OCF_NOT_RUNNING ]; then
         return $OCF_SUCCESS
     fi
 
     rm -f ${OCF_RESKEY_state}
 
     set_attribute "${OCF_RESKEY_inactive_value}"
     if [ $? -ne 0 ]; then
         ocf_exit_reason "Unable to set node attribute $OCF_RESKEY_name='$OCF_RESKEY_inactive_value'"
         return $OCF_ERR_GENERIC
     fi
 
     return $OCF_SUCCESS
 }
 
 case $__OCF_ACTION in
 meta-data)      meta_data ;;
 start)          start ;;
 stop)           stop ;;
 monitor)        monitor ;;
 # We don't do anything special for live migration, but we support it so that
 # other resources that live migrate can depend on this one.
 migrate_to)     stop ;;
 migrate_from)   start ;;
 reload)         start ;;
 validate-all)   validate ;;
 usage|help)     usage $OCF_SUCCESS ;;
 *)              usage $OCF_ERR_UNIMPLEMENTED ;;
 esac
 
 exit $?
 
 # vim: set filetype=sh expandtab tabstop=4 softtabstop=4 shiftwidth=4 textwidth=80:
diff --git a/include/crm/common/logging.h b/include/crm/common/logging.h
index 9b0bc06380..8f69e8b9aa 100644
--- a/include/crm/common/logging.h
+++ b/include/crm/common/logging.h
@@ -1,422 +1,424 @@
 /*
  * Copyright 2004-2020 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU General Public License version 2
  * or later (GPLv2+) WITHOUT ANY WARRANTY.
  */
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
 /**
  * \file
  * \brief Wrappers for and extensions to libqb logging
  * \ingroup core
  */
 
 #ifndef CRM_LOGGING__H
 #  define CRM_LOGGING__H
 
 #  include <stdio.h>
+#  include <glib.h>
 #  include <qb/qblog.h>
+#  include <libxml/tree.h>
 
 /* Define custom log priorities.
  *
  * syslog(3) uses int for priorities, but libqb's struct qb_log_callsite uses
  * uint8_t, so make sure they fit in the latter.
  */
 
 // Define something even less desired than debug
 #  ifndef LOG_TRACE
 #    define LOG_TRACE   (LOG_DEBUG+1)
 #  endif
 
 // Print message to stdout instead of logging it
 #  ifndef LOG_STDOUT
 #    define LOG_STDOUT  254
 #  endif
 
 // Don't send message anywhere
 #  ifndef LOG_NEVER
 #    define LOG_NEVER   255
 #  endif
 
 /* "Extended information" logging support */
 #ifdef QB_XS
 #  define CRM_XS QB_XS
 #  define crm_extended_logging(t, e) qb_log_ctl((t), QB_LOG_CONF_EXTENDED, (e))
 #else
 #  define CRM_XS "|"
 
 /* A caller might want to check the return value, so we can't define this as a
  * no-op, and we can't simply define it to be 0 because gcc will then complain
  * when the value isn't checked.
  */
 static inline int
 crm_extended_logging(int t, int e)
 {
     return 0;
 }
 #endif
 
 extern unsigned int crm_log_level;
 extern gboolean crm_config_error;
 extern gboolean crm_config_warning;
 extern unsigned int crm_trace_nonlog;
 
 enum xml_log_options
 {
     xml_log_option_filtered   = 0x0001,
     xml_log_option_formatted  = 0x0002,
     xml_log_option_text       = 0x0004, /* add this option to dump text into xml */
     xml_log_option_full_fledged = 0x0008, /* crm_xml_dump: serialize using libxml */
     xml_log_option_diff_plus  = 0x0010,
     xml_log_option_diff_minus = 0x0020,
     xml_log_option_diff_short = 0x0040,
     xml_log_option_diff_all   = 0x0100,
     xml_log_option_dirty_add  = 0x1000,
     xml_log_option_open       = 0x2000,
     xml_log_option_children   = 0x4000,
     xml_log_option_close      = 0x8000,
 };
 
 void crm_enable_blackbox(int nsig);
 void crm_disable_blackbox(int nsig);
 void crm_write_blackbox(int nsig, struct qb_log_callsite *callsite);
 
 void crm_update_callsites(void);
 
 void crm_log_deinit(void);
 
 gboolean crm_log_cli_init(const char *entity);
 
 void crm_log_preinit(const char *entity, int argc, char **argv);
 gboolean crm_log_init(const char *entity, uint8_t level, gboolean daemon,
                       gboolean to_stderr, int argc, char **argv, gboolean quiet);
 
 void crm_log_args(int argc, char **argv);
 void crm_log_output_fn(const char *file, const char *function, int line, int level,
                        const char *prefix, const char *output);
 
 // Log a block of text line by line
 #  define crm_log_output(level, prefix, output) crm_log_output_fn(__FILE__, __FUNCTION__, __LINE__, level, prefix, output)
 
 gboolean crm_add_logfile(const char *filename);
 
 void crm_bump_log_level(int argc, char **argv);
 
 void crm_enable_stderr(int enable);
 
 gboolean crm_is_callsite_active(struct qb_log_callsite *cs, uint8_t level, uint32_t tags);
 
 void log_data_element(int log_level, const char *file, const char *function, int line,
                       const char *prefix, xmlNode * data, int depth, gboolean formatted);
 
 /* returns the old value */
 unsigned int set_crm_log_level(unsigned int level);
 
 unsigned int get_crm_log_level(void);
 
 /*
  * Throughout the macros below, note the leading, pre-comma, space in the
  * various ' , ##args' occurrences to aid portability across versions of 'gcc'.
  * https://gcc.gnu.org/onlinedocs/cpp/Variadic-Macros.html#Variadic-Macros
  */
 #if defined(__clang__)
 #    define CRM_TRACE_INIT_DATA(name)
 #  else
 #    include <assert.h> // required by QB_LOG_INIT_DATA() macro
 #    define CRM_TRACE_INIT_DATA(name) QB_LOG_INIT_DATA(name)
 #endif
 
 /* Using "switch" instead of "if" in these macro definitions keeps
  * static analysis from complaining about constant evaluations
  */
 
 /*!
  * \brief Log a message
  *
  * \param[in] level  Priority at which to log the message
  * \param[in] fmt    printf-style format string literal for message
  * \param[in] args   Any arguments needed by format string
  *
  * \note This is a macro, and \p level may be evaluated more than once.
  */
 #  define do_crm_log(level, fmt, args...) do {                              \
         switch (level) {                                                    \
             case LOG_STDOUT:                                                \
                 printf(fmt "\n" , ##args);                                  \
                 break;                                                      \
             case LOG_NEVER:                                                 \
                 break;                                                      \
             default:                                                        \
                 qb_log_from_external_source(__func__, __FILE__, fmt,        \
                     (level),   __LINE__, 0 , ##args);                       \
                 break;                                                      \
         }                                                                   \
     } while (0)
 
 /*!
  * \brief Log a message that is likely to be filtered out
  *
  * \param[in] level  Priority at which to log the message
  * \param[in] fmt    printf-style format string for message
  * \param[in] args   Any arguments needed by format string
  *
  * \note This is a macro, and \p level may be evaluated more than once.
  *       This does nothing when level is LOG_STDOUT.
  */
 #  define do_crm_log_unlikely(level, fmt, args...) do {                     \
         switch (level) {                                                    \
             case LOG_STDOUT: case LOG_NEVER:                                \
                 break;                                                      \
             default: {                                                      \
                 static struct qb_log_callsite *trace_cs = NULL;             \
                 if (trace_cs == NULL) {                                     \
                     trace_cs = qb_log_callsite_get(__func__, __FILE__, fmt, \
                                                    (level), __LINE__, 0);   \
                 }                                                           \
                 if (crm_is_callsite_active(trace_cs, (level), 0)) {         \
                     qb_log_from_external_source(__func__, __FILE__, fmt,    \
                         (level), __LINE__, 0 , ##args);                     \
                 }                                                           \
             }                                                               \
             break;                                                          \
         }                                                                   \
     } while (0)
 
 /*!
  * \internal
  * \brief Execute code depending on whether message would be logged
  *
  * This is similar to do_crm_log_unlikely() except instead of logging, it either
  * continues past this statement or executes else_action depending on whether a
  * message of the given severity would be logged or not. This allows whole
  * blocks of code to be skipped if tracing or debugging is turned off.
  *
  * \param[in] level        Severity at which to continue past this statement
  * \param[in] else_action  Code block to execute if severity would not be logged
  *
  * \note else_action must not contain a break or continue statement
  */
 #  define pcmk__log_else(level, else_action) do {                           \
         static struct qb_log_callsite *trace_cs = NULL;                     \
                                                                             \
         if (trace_cs == NULL) {                                             \
             trace_cs = qb_log_callsite_get(__func__, __FILE__, "log_else",  \
                                            level, __LINE__, 0);             \
         }                                                                   \
         if (!crm_is_callsite_active(trace_cs, level, 0)) {                  \
             else_action;                                                    \
         }                                                                   \
     } while(0)
 
 #  define CRM_LOG_ASSERT(expr) do {					\
         if(__unlikely((expr) == FALSE)) {				\
             static struct qb_log_callsite *core_cs = NULL;              \
             if(core_cs == NULL) {                                       \
                 core_cs = qb_log_callsite_get(__func__, __FILE__, "log-assert", LOG_TRACE, __LINE__, 0); \
             }                                                           \
             crm_abort(__FILE__, __FUNCTION__, __LINE__, #expr,   \
                       core_cs?core_cs->targets:FALSE, TRUE);            \
         }                                                               \
     } while(0)
 
 /* 'failure_action' MUST NOT be 'continue' as it will apply to the
  * macro's do-while loop
  */
 #  define CRM_CHECK(expr, failure_action) do {				\
 	if(__unlikely((expr) == FALSE)) {				\
             static struct qb_log_callsite *core_cs = NULL;              \
             if(core_cs == NULL) {                                       \
                 core_cs = qb_log_callsite_get(__func__, __FILE__, "check-assert", LOG_TRACE, __LINE__, 0); \
             }                                                           \
 	    crm_abort(__FILE__, __FUNCTION__, __LINE__, #expr,	\
 		      core_cs?core_cs->targets:FALSE, TRUE);            \
 	    failure_action;						\
 	}								\
     } while(0)
 
 /*!
  * \brief Log XML line-by-line in a formatted fashion
  *
  * \param[in] level  Priority at which to log the messages
  * \param[in] text   Prefix for each line
  * \param[in] xml    XML to log
  *
  * \note This is a macro, and \p level may be evaluated more than once.
  *       This does nothing when level is LOG_STDOUT.
  */
 #  define do_crm_log_xml(level, text, xml) do {                             \
         switch (level) {                                                    \
             case LOG_STDOUT: case LOG_NEVER:                                \
                 break;                                                      \
             default: {                                                      \
                 static struct qb_log_callsite *xml_cs = NULL;               \
                 if (xml_cs == NULL) {                                       \
                     xml_cs = qb_log_callsite_get(__func__, __FILE__,        \
                                         "xml-blob", (level), __LINE__, 0);  \
                 }                                                           \
                 if (crm_is_callsite_active(xml_cs, (level), 0)) {           \
                     log_data_element((level), __FILE__, __FUNCTION__,       \
                          __LINE__, text, xml, 1, xml_log_option_formatted); \
                 }                                                           \
             }                                                               \
             break;                                                          \
         }                                                                   \
     } while(0)
 
 /*!
  * \brief Log a message as if it came from a different code location
  *
  * \param[in] level     Priority at which to log the message
  * \param[in] file      Source file name to use instead of __FILE__
  * \param[in] function  Source function name to use instead of __func__
  * \param[in] line      Source line number to use instead of __line__
  * \param[in] fmt       printf-style format string literal for message
  * \param[in] args      Any arguments needed by format string
  *
  * \note This is a macro, and \p level may be evaluated more than once.
  */
 #  define do_crm_log_alias(level, file, function, line, fmt, args...) do {  \
         switch (level) {                                                    \
             case LOG_STDOUT:                                                \
                 printf(fmt "\n" , ##args);                                  \
                 break;                                                      \
             case LOG_NEVER:                                                 \
                 break;                                                      \
             default:                                                        \
                 qb_log_from_external_source(function, file, fmt, (level),   \
                                             line, 0 , ##args);              \
                 break;                                                      \
         }                                                                   \
     } while (0)
 
 /*!
  * \brief Send a system error message to both the log and stderr
  *
  * \param[in] level  Priority at which to log the message
  * \param[in] fmt    printf-style format string for message
  * \param[in] args   Any arguments needed by format string
  *
  * \deprecated One of the other logging functions should be used with
  *             pcmk_strerror() instead.
  * \note This is a macro, and \p level may be evaluated more than once.
  * \note Because crm_perror() adds the system error message and error number
  *       onto the end of fmt, that information will become extended information
  *       if CRM_XS is used inside fmt and will not show up in syslog.
  */
 #  define crm_perror(level, fmt, args...) do {                              \
         switch (level) {                                                    \
             case LOG_NEVER:                                                 \
                 break;                                                      \
             default: {                                                      \
                 const char *err = strerror(errno);                          \
                 /* cast to int makes coverity happy when level == 0 */      \
                 if ((level) <= (int) crm_log_level) {                       \
                     fprintf(stderr, fmt ": %s (%d)\n" , ##args, err, errno);\
                 }                                                           \
                 do_crm_log((level), fmt ": %s (%d)" , ##args, err, errno);  \
             }                                                               \
             break;                                                          \
         }                                                                   \
     } while (0)
 
 /*!
  * \brief Log a message with a tag (for use with PCMK_trace_tags)
  *
  * \param[in] level  Priority at which to log the message
  * \param[in] tag    String to tag message with
  * \param[in] fmt    printf-style format string for message
  * \param[in] args   Any arguments needed by format string
  *
  * \note This is a macro, and \p level may be evaluated more than once.
  *       This does nothing when level is LOG_STDOUT.
  */
 #  define crm_log_tag(level, tag, fmt, args...)    do {                     \
         switch (level) {                                                    \
             case LOG_STDOUT: case LOG_NEVER:                                \
                 break;                                                      \
             default: {                                                      \
                 static struct qb_log_callsite *trace_tag_cs = NULL;         \
                 int converted_tag = g_quark_try_string(tag);                \
                 if (trace_tag_cs == NULL) {                                 \
                     trace_tag_cs = qb_log_callsite_get(__func__, __FILE__,  \
                                     fmt, (level), __LINE__, converted_tag); \
                 }                                                           \
                 if (crm_is_callsite_active(trace_tag_cs, (level),           \
                                            converted_tag)) {                \
                     qb_log_from_external_source(__func__, __FILE__, fmt,    \
                                 (level), __LINE__, converted_tag , ##args); \
                 }                                                           \
             }                                                               \
         }                                                                   \
     } while (0)
 
 #  define crm_emerg(fmt, args...)   qb_log(LOG_EMERG,       fmt , ##args)
 #  define crm_crit(fmt, args...)    qb_logt(LOG_CRIT,    0, fmt , ##args)
 #  define crm_err(fmt, args...)     qb_logt(LOG_ERR,     0, fmt , ##args)
 #  define crm_warn(fmt, args...)    qb_logt(LOG_WARNING, 0, fmt , ##args)
 #  define crm_notice(fmt, args...)  qb_logt(LOG_NOTICE,  0, fmt , ##args)
 #  define crm_info(fmt, args...)    qb_logt(LOG_INFO,    0, fmt , ##args)
 
 #  define crm_debug(fmt, args...)   do_crm_log_unlikely(LOG_DEBUG, fmt , ##args)
 #  define crm_trace(fmt, args...)   do_crm_log_unlikely(LOG_TRACE, fmt , ##args)
 
 #  define crm_log_xml_crit(xml, text)    do_crm_log_xml(LOG_CRIT,    text, xml)
 #  define crm_log_xml_err(xml, text)     do_crm_log_xml(LOG_ERR,     text, xml)
 #  define crm_log_xml_warn(xml, text)    do_crm_log_xml(LOG_WARNING, text, xml)
 #  define crm_log_xml_notice(xml, text)  do_crm_log_xml(LOG_NOTICE,  text, xml)
 #  define crm_log_xml_info(xml, text)    do_crm_log_xml(LOG_INFO,    text, xml)
 #  define crm_log_xml_debug(xml, text)   do_crm_log_xml(LOG_DEBUG,   text, xml)
 #  define crm_log_xml_trace(xml, text)   do_crm_log_xml(LOG_TRACE,   text, xml)
 
 #  define crm_log_xml_explicit(xml, text)  do {                 \
         static struct qb_log_callsite *digest_cs = NULL;        \
         digest_cs = qb_log_callsite_get(                        \
             __func__, __FILE__, text, LOG_TRACE, __LINE__,      \
             crm_trace_nonlog);                                  \
         if (digest_cs && digest_cs->targets) {                  \
             do_crm_log_xml(LOG_TRACE,   text, xml);             \
         }                                                       \
     } while(0)
 
 #  define crm_str(x)    (const char*)(x?x:"<null>")
 
 #ifndef PCMK__NO_COMPAT
 
 /* Everything here is deprecated and kept only for public API backward
  * compatibility. It will be moved to compatibility.h when 2.1.0 is released.
  */
 
 /*!
  * \brief Log a message using constant priority
  *
  * \param[in] level     Priority at which to log the message
  * \param[in] fmt       printf-style format string literal for message
  * \param[in] args      Any arguments needed by format string
  *
  * \deprecated Use one of the other logging functions instead
  * \note This is a macro, and \p level may be evaluated more than once.
  *       This does nothing when level is LOG_STDOUT.
  */
 #  define do_crm_log_always(level, fmt, args...) do {                       \
         switch (level) {                                                    \
             case LOG_STDOUT: case LOG_NEVER:                                \
                 break;                                                      \
             default:                                                        \
                 qb_log((level), fmt , ##args);                              \
                 break;                                                      \
         }                                                                   \
     } while (0)
 
 #endif // PCMK__NO_COMPAT
 
 #ifdef __cplusplus
 }
 #endif
 
 #endif
diff --git a/include/crm/pengine/internal.h b/include/crm/pengine/internal.h
index 1ebd1139cc..25554cf266 100644
--- a/include/crm/pengine/internal.h
+++ b/include/crm/pengine/internal.h
@@ -1,476 +1,478 @@
 /*
  * Copyright 2004-2020 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU Lesser General Public License
  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
  */
 
 #ifndef PE_INTERNAL__H
 #  define PE_INTERNAL__H
 #  include <string.h>
 #  include <crm/pengine/status.h>
 #  include <crm/pengine/remote_internal.h>
 #  include <crm/common/output.h>
 
 #  define pe_rsc_info(rsc, fmt, args...)  crm_log_tag(LOG_INFO,  rsc ? rsc->id : "<NULL>", fmt, ##args)
 #  define pe_rsc_debug(rsc, fmt, args...) crm_log_tag(LOG_DEBUG, rsc ? rsc->id : "<NULL>", fmt, ##args)
 #  define pe_rsc_trace(rsc, fmt, args...) crm_log_tag(LOG_TRACE, rsc ? rsc->id : "<NULL>", fmt, ##args)
 
 #  define pe_err(fmt...) { was_processing_error = TRUE; crm_config_error = TRUE; crm_err(fmt); }
 #  define pe_warn(fmt...) { was_processing_warning = TRUE; crm_config_warning = TRUE; crm_warn(fmt); }
 #  define pe_proc_err(fmt...) { was_processing_error = TRUE; crm_err(fmt); }
 #  define pe_proc_warn(fmt...) { was_processing_warning = TRUE; crm_warn(fmt); }
 #  define pe_set_action_bit(action, bit) action->flags = crm_set_bit(__FUNCTION__, __LINE__, action->uuid, action->flags, bit)
 #  define pe_clear_action_bit(action, bit) action->flags = crm_clear_bit(__FUNCTION__, __LINE__, action->uuid, action->flags, bit)
 
 // Some warnings we don't want to print every transition
 
 enum pe_warn_once_e {
     pe_wo_blind         = 0x0001,
     pe_wo_restart_type  = 0x0002,
     pe_wo_role_after    = 0x0004,
     pe_wo_poweroff      = 0x0008,
     pe_wo_require_all   = 0x0010,
     pe_wo_order_score   = 0x0020,
     pe_wo_neg_threshold = 0x0040,
 };
 
 extern uint32_t pe_wo;
 
 #define pe_warn_once(pe_wo_bit, fmt...) do {    \
         if (is_not_set(pe_wo, pe_wo_bit)) {     \
             if (pe_wo_bit == pe_wo_blind) {     \
                 crm_warn(fmt);                  \
             } else {                            \
                 pe_warn(fmt);                   \
             }                                   \
             set_bit(pe_wo, pe_wo_bit);          \
         }                                       \
     } while (0);
 
 
 typedef struct pe__location_constraint_s {
     char *id;                           // Constraint XML ID
     pe_resource_t *rsc_lh;              // Resource being located
     enum rsc_role_e role_filter;        // Role to locate
     enum pe_discover_e discover_mode;   // Resource discovery
     GListPtr node_list_rh;              // List of pe_node_t*
 } pe__location_t;
 
 typedef struct pe__order_constraint_s {
     int id;
     enum pe_ordering type;
 
     void *lh_opaque;
     pe_resource_t *lh_rsc;
     pe_action_t *lh_action;
     char *lh_action_task;
 
     void *rh_opaque;
     pe_resource_t *rh_rsc;
     pe_action_t *rh_action;
     char *rh_action_task;
 } pe__ordering_t;
 
 typedef struct notify_data_s {
     GSList *keys;               // Environment variable name/value pairs
 
     const char *action;
 
     pe_action_t *pre;
     pe_action_t *post;
     pe_action_t *pre_done;
     pe_action_t *post_done;
 
     GListPtr active;            /* notify_entry_t*  */
     GListPtr inactive;          /* notify_entry_t*  */
     GListPtr start;             /* notify_entry_t*  */
     GListPtr stop;              /* notify_entry_t*  */
     GListPtr demote;            /* notify_entry_t*  */
     GListPtr promote;           /* notify_entry_t*  */
     GListPtr master;            /* notify_entry_t*  */
     GListPtr slave;             /* notify_entry_t*  */
     GHashTable *allowed_nodes;
 
 } notify_data_t;
 
 bool pe_can_fence(pe_working_set_t *data_set, pe_node_t *node);
 
 int pe__add_scores(int score1, int score2);
 void add_hash_param(GHashTable * hash, const char *name, const char *value);
 
 char *native_parameter(pe_resource_t * rsc, pe_node_t * node, gboolean create, const char *name,
                        pe_working_set_t * data_set);
 pe_node_t *native_location(const pe_resource_t *rsc, GList **list, int current);
 
 void pe_metadata(void);
 void verify_pe_options(GHashTable * options);
 
 void common_update_score(pe_resource_t * rsc, const char *id, int score);
 void native_add_running(pe_resource_t * rsc, pe_node_t * node, pe_working_set_t * data_set);
 
 gboolean native_unpack(pe_resource_t * rsc, pe_working_set_t * data_set);
 gboolean group_unpack(pe_resource_t * rsc, pe_working_set_t * data_set);
 gboolean clone_unpack(pe_resource_t * rsc, pe_working_set_t * data_set);
 gboolean pe__unpack_bundle(pe_resource_t *rsc, pe_working_set_t *data_set);
 
 pe_resource_t *native_find_rsc(pe_resource_t *rsc, const char *id, const pe_node_t *node,
                                int flags);
 
 gboolean native_active(pe_resource_t * rsc, gboolean all);
 gboolean group_active(pe_resource_t * rsc, gboolean all);
 gboolean clone_active(pe_resource_t * rsc, gboolean all);
 gboolean pe__bundle_active(pe_resource_t *rsc, gboolean all);
 
 void native_print(pe_resource_t * rsc, const char *pre_text, long options, void *print_data);
 void group_print(pe_resource_t * rsc, const char *pre_text, long options, void *print_data);
 void clone_print(pe_resource_t * rsc, const char *pre_text, long options, void *print_data);
 void pe__print_bundle(pe_resource_t *rsc, const char *pre_text, long options,
                       void *print_data);
 
 int pe__name_and_nvpairs_xml(pcmk__output_t *out, bool is_list, const char *tag_name
                          , size_t pairs_count, ...);
 char *pe__node_display_name(pe_node_t *node, bool print_detail);
 
 int pe__ban_html(pcmk__output_t *out, va_list args);
 int pe__ban_text(pcmk__output_t *out, va_list args);
 int pe__ban_xml(pcmk__output_t *out, va_list args);
 int pe__clone_xml(pcmk__output_t *out, va_list args);
 int pe__clone_html(pcmk__output_t *out, va_list args);
 int pe__clone_text(pcmk__output_t *out, va_list args);
 int pe__cluster_counts_html(pcmk__output_t *out, va_list args);
 int pe__cluster_counts_text(pcmk__output_t *out, va_list args);
 int pe__cluster_counts_xml(pcmk__output_t *out, va_list args);
 int pe__cluster_dc_html(pcmk__output_t *out, va_list args);
 int pe__cluster_dc_text(pcmk__output_t *out, va_list args);
 int pe__cluster_dc_xml(pcmk__output_t *out, va_list args);
 int pe__cluster_maint_mode_html(pcmk__output_t *out, va_list args);
 int pe__cluster_maint_mode_text(pcmk__output_t *out, va_list args);
 int pe__cluster_options_html(pcmk__output_t *out, va_list args);
 int pe__cluster_options_log(pcmk__output_t *out, va_list args);
 int pe__cluster_options_text(pcmk__output_t *out, va_list args);
 int pe__cluster_options_xml(pcmk__output_t *out, va_list args);
 int pe__cluster_stack_html(pcmk__output_t *out, va_list args);
 int pe__cluster_stack_text(pcmk__output_t *out, va_list args);
 int pe__cluster_stack_xml(pcmk__output_t *out, va_list args);
 int pe__cluster_summary(pcmk__output_t *out, va_list args);
 int pe__cluster_summary_html(pcmk__output_t *out, va_list args);
 int pe__cluster_times_html(pcmk__output_t *out, va_list args);
 int pe__cluster_times_xml(pcmk__output_t *out, va_list args);
 int pe__cluster_times_text(pcmk__output_t *out, va_list args);
 int pe__failed_action_text(pcmk__output_t *out, va_list args);
 int pe__failed_action_xml(pcmk__output_t *out, va_list args);
 int pe__group_xml(pcmk__output_t *out, va_list args);
 int pe__group_html(pcmk__output_t *out, va_list args);
 int pe__group_text(pcmk__output_t *out, va_list args);
 int pe__bundle_xml(pcmk__output_t *out, va_list args);
 int pe__bundle_html(pcmk__output_t *out, va_list args);
 int pe__bundle_text(pcmk__output_t *out, va_list args);
 int pe__node_html(pcmk__output_t *out, va_list args);
 int pe__node_text(pcmk__output_t *out, va_list args);
 int pe__node_xml(pcmk__output_t *out, va_list args);
 int pe__node_attribute_html(pcmk__output_t *out, va_list args);
 int pe__node_attribute_text(pcmk__output_t *out, va_list args);
 int pe__node_attribute_xml(pcmk__output_t *out, va_list args);
 int pe__node_list_html(pcmk__output_t *out, va_list args);
 int pe__node_list_text(pcmk__output_t *out, va_list args);
 int pe__node_list_xml(pcmk__output_t *out, va_list args);
 int pe__op_history_text(pcmk__output_t *out, va_list args);
 int pe__op_history_xml(pcmk__output_t *out, va_list args);
 int pe__resource_history_text(pcmk__output_t *out, va_list args);
 int pe__resource_history_xml(pcmk__output_t *out, va_list args);
 int pe__resource_xml(pcmk__output_t *out, va_list args);
 int pe__resource_html(pcmk__output_t *out, va_list args);
 int pe__resource_text(pcmk__output_t *out, va_list args);
 int pe__ticket_html(pcmk__output_t *out, va_list args);
 int pe__ticket_text(pcmk__output_t *out, va_list args);
 int pe__ticket_xml(pcmk__output_t *out, va_list args);
 
 void native_free(pe_resource_t * rsc);
 void group_free(pe_resource_t * rsc);
 void clone_free(pe_resource_t * rsc);
 void pe__free_bundle(pe_resource_t *rsc);
 
 enum rsc_role_e native_resource_state(const pe_resource_t * rsc, gboolean current);
 enum rsc_role_e group_resource_state(const pe_resource_t * rsc, gboolean current);
 enum rsc_role_e clone_resource_state(const pe_resource_t * rsc, gboolean current);
 enum rsc_role_e pe__bundle_resource_state(const pe_resource_t *rsc,
                                           gboolean current);
 
 void pe__count_common(pe_resource_t *rsc);
 void pe__count_bundle(pe_resource_t *rsc);
 
 gboolean common_unpack(xmlNode * xml_obj, pe_resource_t ** rsc, pe_resource_t * parent,
                        pe_working_set_t * data_set);
 void common_free(pe_resource_t * rsc);
 
 pe_node_t *pe__copy_node(const pe_node_t *this_node);
 extern time_t get_effective_time(pe_working_set_t * data_set);
 
 /* Failure handling utilities (from failcounts.c) */
 
 // bit flags for fail count handling options
 enum pe_fc_flags_e {
     pe_fc_default   = 0x00,
     pe_fc_effective = 0x01, // don't count expired failures
     pe_fc_fillers   = 0x02, // if container, include filler failures in count
 };
 
 int pe_get_failcount(pe_node_t *node, pe_resource_t *rsc, time_t *last_failure,
                      uint32_t flags, xmlNode *xml_op,
                      pe_working_set_t *data_set);
 
 pe_action_t *pe__clear_failcount(pe_resource_t *rsc, pe_node_t *node,
                                  const char *reason,
                                  pe_working_set_t *data_set);
 
 /* Functions for finding/counting a resource's active nodes */
 
 pe_node_t *pe__find_active_on(const pe_resource_t *rsc,
                               unsigned int *count_all,
                               unsigned int *count_clean);
 pe_node_t *pe__find_active_requires(const pe_resource_t *rsc,
                                     unsigned int *count);
 
 static inline pe_node_t *
 pe__current_node(const pe_resource_t *rsc)
 {
     return pe__find_active_on(rsc, NULL, NULL);
 }
 
 
 /* Binary like operators for lists of nodes */
 extern void node_list_exclude(GHashTable * list, GListPtr list2, gboolean merge_scores);
 
 GHashTable *pe__node_list2table(GList *list);
 
 static inline gpointer
 pe_hash_table_lookup(GHashTable * hash, gconstpointer key)
 {
     if (hash) {
         return g_hash_table_lookup(hash, key);
     }
     return NULL;
 }
 
 extern pe_action_t *get_pseudo_op(const char *name, pe_working_set_t * data_set);
 extern gboolean order_actions(pe_action_t * lh_action, pe_action_t * rh_action, enum pe_ordering order);
 
 /* Printing functions for debug */
 extern void print_node(const char *pre_text, pe_node_t * node, gboolean details);
 extern void print_str_str(gpointer key, gpointer value, gpointer user_data);
 extern void pe__output_node(pe_node_t * node, gboolean details, pcmk__output_t *out);
 
 extern void dump_node_capacity(int level, const char *comment, pe_node_t * node);
 extern void dump_rsc_utilization(int level, const char *comment, pe_resource_t * rsc, pe_node_t * node);
 
 void pe__show_node_weights_as(const char *file, const char *function,
                               int line, bool to_log, pe_resource_t *rsc,
                               const char *comment, GHashTable *nodes);
 
 #define pe__show_node_weights(level, rsc, text, nodes)              \
         pe__show_node_weights_as(__FILE__, __FUNCTION__, __LINE__,  \
                                  (level), (rsc), (text), (nodes))
 
 /* Sorting functions */
 extern gint sort_rsc_priority(gconstpointer a, gconstpointer b);
 extern gint sort_rsc_index(gconstpointer a, gconstpointer b);
 
 extern xmlNode *find_rsc_op_entry(pe_resource_t * rsc, const char *key);
 
 extern pe_action_t *custom_action(pe_resource_t * rsc, char *key, const char *task, pe_node_t * on_node,
                                   gboolean optional, gboolean foo, pe_working_set_t * data_set);
 
 #  define delete_key(rsc) pcmk__op_key(rsc->id, CRMD_ACTION_DELETE, 0)
 #  define delete_action(rsc, node, optional) custom_action(		\
 		rsc, delete_key(rsc), CRMD_ACTION_DELETE, node,		\
 		optional, TRUE, data_set);
 
 #  define stopped_key(rsc) pcmk__op_key(rsc->id, CRMD_ACTION_STOPPED, 0)
 #  define stopped_action(rsc, node, optional) custom_action(		\
 		rsc, stopped_key(rsc), CRMD_ACTION_STOPPED, node,	\
 		optional, TRUE, data_set);
 
 #  define stop_key(rsc) pcmk__op_key(rsc->id, CRMD_ACTION_STOP, 0)
 #  define stop_action(rsc, node, optional) custom_action(			\
 		rsc, stop_key(rsc), CRMD_ACTION_STOP, node,		\
 		optional, TRUE, data_set);
 
 #  define reload_key(rsc) pcmk__op_key(rsc->id, CRMD_ACTION_RELOAD, 0)
 #  define start_key(rsc) pcmk__op_key(rsc->id, CRMD_ACTION_START, 0)
 #  define start_action(rsc, node, optional) custom_action(		\
 		rsc, start_key(rsc), CRMD_ACTION_START, node,		\
 		optional, TRUE, data_set)
 
 #  define started_key(rsc) pcmk__op_key(rsc->id, CRMD_ACTION_STARTED, 0)
 #  define started_action(rsc, node, optional) custom_action(		\
 		rsc, started_key(rsc), CRMD_ACTION_STARTED, node,	\
 		optional, TRUE, data_set)
 
 #  define promote_key(rsc) pcmk__op_key(rsc->id, CRMD_ACTION_PROMOTE, 0)
 #  define promote_action(rsc, node, optional) custom_action(		\
 		rsc, promote_key(rsc), CRMD_ACTION_PROMOTE, node,	\
 		optional, TRUE, data_set)
 
 #  define promoted_key(rsc) pcmk__op_key(rsc->id, CRMD_ACTION_PROMOTED, 0)
 #  define promoted_action(rsc, node, optional) custom_action(		\
 		rsc, promoted_key(rsc), CRMD_ACTION_PROMOTED, node,	\
 		optional, TRUE, data_set)
 
 #  define demote_key(rsc) pcmk__op_key(rsc->id, CRMD_ACTION_DEMOTE, 0)
 #  define demote_action(rsc, node, optional) custom_action(		\
 		rsc, demote_key(rsc), CRMD_ACTION_DEMOTE, node,		\
 		optional, TRUE, data_set)
 
 #  define demoted_key(rsc) pcmk__op_key(rsc->id, CRMD_ACTION_DEMOTED, 0)
 #  define demoted_action(rsc, node, optional) custom_action(		\
 		rsc, demoted_key(rsc), CRMD_ACTION_DEMOTED, node,	\
 		optional, TRUE, data_set)
 
 extern int pe_get_configured_timeout(pe_resource_t *rsc, const char *action,
                                      pe_working_set_t *data_set);
 
 extern pe_action_t *find_first_action(GListPtr input, const char *uuid, const char *task,
                                       pe_node_t * on_node);
 extern enum action_tasks get_complex_task(pe_resource_t * rsc, const char *name,
                                           gboolean allow_non_atomic);
 
 extern GListPtr find_actions(GListPtr input, const char *key, const pe_node_t *on_node);
 GList *find_actions_exact(GList *input, const char *key,
                           const pe_node_t *on_node);
 extern GListPtr find_recurring_actions(GListPtr input, pe_node_t * not_on_node);
 GList *pe__resource_actions(const pe_resource_t *rsc, const pe_node_t *node,
                             const char *task, bool require_node);
 
 extern void pe_free_action(pe_action_t * action);
 
 extern void resource_location(pe_resource_t * rsc, pe_node_t * node, int score, const char *tag,
                               pe_working_set_t * data_set);
 
 extern gint sort_op_by_callid(gconstpointer a, gconstpointer b);
 extern gboolean get_target_role(pe_resource_t * rsc, enum rsc_role_e *role);
 
 extern pe_resource_t *find_clone_instance(pe_resource_t * rsc, const char *sub_id,
                                           pe_working_set_t * data_set);
 
 extern void destroy_ticket(gpointer data);
 extern pe_ticket_t *ticket_new(const char *ticket_id, pe_working_set_t * data_set);
 
 // Resources for manipulating resource names
 const char *pe_base_name_end(const char *id);
 char *clone_strip(const char *last_rsc_id);
 char *clone_zero(const char *last_rsc_id);
 
 static inline bool
 pe_base_name_eq(pe_resource_t *rsc, const char *id)
 {
     if (id && rsc && rsc->id) {
         // Number of characters in rsc->id before any clone suffix
         size_t base_len = pe_base_name_end(rsc->id) - rsc->id + 1;
 
         return (strlen(id) == base_len) && !strncmp(id, rsc->id, base_len);
     }
     return FALSE;
 }
 
 int pe__target_rc_from_xml(xmlNode *xml_op);
 
 gint sort_node_uname(gconstpointer a, gconstpointer b);
 bool is_set_recursive(pe_resource_t * rsc, long long flag, bool any);
 
 enum rsc_digest_cmp_val {
     /*! Digests are the same */
     RSC_DIGEST_MATCH = 0,
     /*! Params that require a restart changed */
     RSC_DIGEST_RESTART,
     /*! Some parameter changed.  */
     RSC_DIGEST_ALL,
     /*! rsc op didn't have a digest associated with it, so
      *  it is unknown if parameters changed or not. */
     RSC_DIGEST_UNKNOWN,
 };
 
 typedef struct op_digest_cache_s {
     enum rsc_digest_cmp_val rc;
     xmlNode *params_all;
     xmlNode *params_secure;
     xmlNode *params_restart;
     char *digest_all_calc;
     char *digest_secure_calc;
     char *digest_restart_calc;
 } op_digest_cache_t;
 
 op_digest_cache_t *rsc_action_digest_cmp(pe_resource_t * rsc, xmlNode * xml_op, pe_node_t * node,
                                          pe_working_set_t * data_set);
 
 pe_action_t *pe_fence_op(pe_node_t * node, const char *op, bool optional, const char *reason, bool priority_delay, pe_working_set_t * data_set);
 void trigger_unfencing(
     pe_resource_t * rsc, pe_node_t *node, const char *reason, pe_action_t *dependency, pe_working_set_t * data_set);
 
 void pe_action_set_reason(pe_action_t *action, const char *reason, bool overwrite);
 void pe_action_set_flag_reason(const char *function, long line, pe_action_t *action, pe_action_t *reason, const char *text, enum pe_action_flags flags, bool overwrite);
 
 #define pe_action_required(action, reason, text) pe_action_set_flag_reason(__FUNCTION__, __LINE__, action, reason, text, pe_action_optional, FALSE)
 #define pe_action_implies(action, reason, flag) pe_action_set_flag_reason(__FUNCTION__, __LINE__, action, reason, NULL, flag, FALSE)
 
 void set_bit_recursive(pe_resource_t * rsc, unsigned long long flag);
 void clear_bit_recursive(pe_resource_t * rsc, unsigned long long flag);
 
 gboolean add_tag_ref(GHashTable * tags, const char * tag_name,  const char * obj_ref);
 
 void print_rscs_brief(GListPtr rsc_list, const char * pre_text, long options,
                       void * print_data, gboolean print_all);
 int pe__rscs_brief_output(pcmk__output_t *out, GListPtr rsc_list, long options, gboolean print_all);
 void pe_fence_node(pe_working_set_t * data_set, pe_node_t * node, const char *reason, bool priority_delay);
 
 pe_node_t *pe_create_node(const char *id, const char *uname, const char *type,
                           const char *score, pe_working_set_t * data_set);
 void common_print(pe_resource_t * rsc, const char *pre_text, const char *name, pe_node_t *node, long options, void *print_data);
 int pe__common_output_text(pcmk__output_t *out, pe_resource_t * rsc, const char *name, pe_node_t *node, long options);
 int pe__common_output_html(pcmk__output_t *out, pe_resource_t * rsc, const char *name, pe_node_t *node, long options);
 pe_resource_t *pe__find_bundle_replica(const pe_resource_t *bundle,
                                        const pe_node_t *node);
 bool pe__bundle_needs_remote_name(pe_resource_t *rsc);
 const char *pe__add_bundle_remote_name(pe_resource_t *rsc, xmlNode *xml,
                                        const char *field);
 const char *pe_node_attribute_calculated(const pe_node_t *node,
                                          const char *name,
                                          const pe_resource_t *rsc);
 const char *pe_node_attribute_raw(pe_node_t *node, const char *name);
 bool pe__is_universal_clone(pe_resource_t *rsc,
                             pe_working_set_t *data_set);
 void pe__add_param_check(xmlNode *rsc_op, pe_resource_t *rsc, pe_node_t *node,
                          enum pe_check_parameters, pe_working_set_t *data_set);
 void pe__foreach_param_check(pe_working_set_t *data_set,
                              void (*cb)(pe_resource_t*, pe_node_t*, xmlNode*,
                                         enum pe_check_parameters,
                                         pe_working_set_t*));
 void pe__free_param_checks(pe_working_set_t *data_set);
 
 bool pe__shutdown_requested(pe_node_t *node);
 void pe__update_recheck_time(time_t recheck, pe_working_set_t *data_set);
 
 #define BOOL2STR(x) ((x) ? "true" : "false")
 /*!
  * \internal
  * \brief Register xml formatting message functions.
  */
 void pe__register_messages(pcmk__output_t *out);
 
 void pe__unpack_dataset_nvpairs(xmlNode *xml_obj, const char *set_name,
                                 pe_rule_eval_data_t *rule_data, GHashTable *hash,
                                 const char *always_first, gboolean overwrite,
                                 pe_working_set_t *data_set);
 
 bool pe__resource_is_disabled(pe_resource_t *rsc);
 pe_action_t *pe__clear_resource_history(pe_resource_t *rsc, pe_node_t *node,
                                         pe_working_set_t *data_set);
 
 GListPtr pe__unames_with_tag(pe_working_set_t *data_set, const char *tag_name);
 bool pe__uname_has_tag(pe_working_set_t *data_set, const char *node, const char *tag);
 
 bool pe__rsc_running_on_any_node_in_list(pe_resource_t *rsc, GListPtr node_list);
 
+bool pcmk__rsc_is_filtered(pe_resource_t *rsc, GListPtr only_show);
+
 #endif
diff --git a/lib/common/ipc_server.c b/lib/common/ipc_server.c
index b747be39b0..0697f3c412 100644
--- a/lib/common/ipc_server.c
+++ b/lib/common/ipc_server.c
@@ -1,903 +1,903 @@
 /*
  * Copyright 2004-2020 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU Lesser General Public License
  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #include <stdio.h>
 #include <errno.h>
 #include <bzlib.h>
 #include <sys/stat.h>
 #include <sys/types.h>
 
 #include <crm/crm.h>
 #include <crm/msg_xml.h>
 #include <crm/common/ipc.h>
 #include <crm/common/ipc_internal.h>
 #include "crmcommon_private.h"
 
 /* Evict clients whose event queue grows this large (by default) */
 #define PCMK_IPC_DEFAULT_QUEUE_MAX 500
 
 static GHashTable *client_connections = NULL;
 
 /*!
  * \internal
  * \brief Count IPC clients
  *
  * \return Number of active IPC client connections
  */
 guint
 pcmk__ipc_client_count()
 {
     return client_connections? g_hash_table_size(client_connections) : 0;
 }
 
 /*!
  * \internal
  * \brief Execute a function for each active IPC client connection
  *
  * \param[in] func       Function to call
  * \param[in] user_data  Pointer to pass to function
  *
  * \note The parameters are the same as for g_hash_table_foreach().
  */
 void
 pcmk__foreach_ipc_client(GHFunc func, gpointer user_data)
 {
     if ((func != NULL) && (client_connections != NULL)) {
         g_hash_table_foreach(client_connections, func, user_data);
     }
 }
 
 /*!
  * \internal
  * \brief Remote IPC clients based on iterative function result
  *
  * \param[in] func       Function to call for each active IPC client
  * \param[in] user_data  Pointer to pass to function
  *
  * \note The parameters are the same as for g_hash_table_foreach_remove().
  */
 void
 pcmk__foreach_ipc_client_remove(GHRFunc func, gpointer user_data)
 {
     if ((func != NULL) && (client_connections != NULL)) {
         g_hash_table_foreach_remove(client_connections, func, user_data);
     }
 }
 
 pcmk__client_t *
 pcmk__find_client(qb_ipcs_connection_t *c)
 {
     if (client_connections) {
         return g_hash_table_lookup(client_connections, c);
     }
 
     crm_trace("No client found for %p", c);
     return NULL;
 }
 
 pcmk__client_t *
 pcmk__find_client_by_id(const char *id)
 {
     gpointer key;
     pcmk__client_t *client;
     GHashTableIter iter;
 
     if (client_connections && id) {
         g_hash_table_iter_init(&iter, client_connections);
         while (g_hash_table_iter_next(&iter, &key, (gpointer *) & client)) {
             if (strcmp(client->id, id) == 0) {
                 return client;
             }
         }
     }
 
     crm_trace("No client found with id=%s", id);
     return NULL;
 }
 
 const char *
 pcmk__client_name(pcmk__client_t *c)
 {
     if (c == NULL) {
         return "null";
     } else if (c->name == NULL && c->id == NULL) {
         return "unknown";
     } else if (c->name == NULL) {
         return c->id;
     } else {
         return c->name;
     }
 }
 
 void
 pcmk__client_cleanup(void)
 {
     if (client_connections != NULL) {
         int active = g_hash_table_size(client_connections);
 
         if (active) {
             crm_err("Exiting with %d active IPC client%s",
                     active, pcmk__plural_s(active));
         }
         g_hash_table_destroy(client_connections); client_connections = NULL;
     }
 }
 
 void
 pcmk__drop_all_clients(qb_ipcs_service_t *service)
 {
     qb_ipcs_connection_t *c = NULL;
 
     if (service == NULL) {
         return;
     }
 
     c = qb_ipcs_connection_first_get(service);
 
     while (c != NULL) {
         qb_ipcs_connection_t *last = c;
 
         c = qb_ipcs_connection_next_get(service, last);
 
         /* There really shouldn't be anyone connected at this point */
         crm_notice("Disconnecting client %p, pid=%d...",
                    last, pcmk__client_pid(last));
         qb_ipcs_disconnect(last);
         qb_ipcs_connection_unref(last);
     }
 }
 
 /*!
  * \internal
  * \brief Allocate a new pcmk__client_t object based on an IPC connection
  *
  * \param[in] c           IPC connection (or NULL to allocate generic client)
  * \param[in] key         Connection table key (or NULL to use sane default)
  * \param[in] uid_client  UID corresponding to c (ignored if c is NULL)
  *
  * \return Pointer to new pcmk__client_t (or NULL on error)
  */
 static pcmk__client_t *
 client_from_connection(qb_ipcs_connection_t *c, void *key, uid_t uid_client)
 {
     pcmk__client_t *client = calloc(1, sizeof(pcmk__client_t));
 
     if (client == NULL) {
         crm_perror(LOG_ERR, "Allocating client");
         return NULL;
     }
 
     if (c) {
 #if ENABLE_ACL
         client->user = pcmk__uid2username(uid_client);
         if (client->user == NULL) {
             client->user = strdup("#unprivileged");
             CRM_CHECK(client->user != NULL, free(client); return NULL);
             crm_err("Unable to enforce ACLs for user ID %d, assuming unprivileged",
                     uid_client);
         }
 #endif
         client->ipcs = c;
         client->kind = PCMK__CLIENT_IPC;
         client->pid = pcmk__client_pid(c);
         if (key == NULL) {
             key = c;
         }
     }
 
     client->id = crm_generate_uuid();
     if (client->id == NULL) {
         crm_err("Could not generate UUID for client");
         free(client->user);
         free(client);
         return NULL;
     }
     if (key == NULL) {
         key = client->id;
     }
     if (client_connections == NULL) {
         crm_trace("Creating IPC client table");
         client_connections = g_hash_table_new(g_direct_hash, g_direct_equal);
     }
     g_hash_table_insert(client_connections, key, client);
     return client;
 }
 
 /*!
  * \brief Allocate a new pcmk__client_t object and generate its ID
  *
  * \param[in] key  What to use as connections hash table key (NULL to use ID)
  *
  * \return Pointer to new pcmk__client_t (asserts on failure)
  */
 pcmk__client_t *
 pcmk__new_unauth_client(void *key)
 {
     pcmk__client_t *client = client_from_connection(NULL, key, 0);
 
     CRM_ASSERT(client != NULL);
     return client;
 }
 
 pcmk__client_t *
 pcmk__new_client(qb_ipcs_connection_t *c, uid_t uid_client, gid_t gid_client)
 {
     gid_t uid_cluster = 0;
     gid_t gid_cluster = 0;
 
     pcmk__client_t *client = NULL;
 
     CRM_CHECK(c != NULL, return NULL);
 
     if (pcmk_daemon_user(&uid_cluster, &gid_cluster) < 0) {
         static bool need_log = TRUE;
 
         if (need_log) {
             crm_warn("Could not find user and group IDs for user %s",
                      CRM_DAEMON_USER);
             need_log = FALSE;
         }
     }
 
     if (uid_client != 0) {
         crm_trace("Giving group %u access to new IPC connection", gid_cluster);
         /* Passing -1 to chown(2) means don't change */
         qb_ipcs_connection_auth_set(c, -1, gid_cluster, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP);
     }
 
     /* TODO: Do our own auth checking, return NULL if unauthorized */
     client = client_from_connection(c, NULL, uid_client);
     if (client == NULL) {
         return NULL;
     }
 
     if ((uid_client == 0) || (uid_client == uid_cluster)) {
         /* Remember when a connection came from root or hacluster */
         set_bit(client->flags, pcmk__client_privileged);
     }
 
     crm_debug("New IPC client %s for PID %u with uid %d and gid %d",
               client->id, client->pid, uid_client, gid_client);
     return client;
 }
 
 static struct iovec *
 pcmk__new_ipc_event(void)
 {
     struct iovec *iov = calloc(2, sizeof(struct iovec));
 
     CRM_ASSERT(iov != NULL);
     return iov;
 }
 
 /*!
  * \brief Free an I/O vector created by pcmk__ipc_prepare_iov()
  *
  * \param[in] event  I/O vector to free
  */
 void
 pcmk_free_ipc_event(struct iovec *event)
 {
     if (event != NULL) {
         free(event[0].iov_base);
         free(event[1].iov_base);
         free(event);
     }
 }
 
 static void
 free_event(gpointer data)
 {
     pcmk_free_ipc_event((struct iovec *) data);
 }
 
 static void
 add_event(pcmk__client_t *c, struct iovec *iov)
 {
     if (c->event_queue == NULL) {
         c->event_queue = g_queue_new();
     }
     g_queue_push_tail(c->event_queue, iov);
 }
 
 void
 pcmk__free_client(pcmk__client_t *c)
 {
     if (c == NULL) {
         return;
     }
 
     if (client_connections) {
         if (c->ipcs) {
             crm_trace("Destroying %p/%p (%d remaining)",
                       c, c->ipcs, g_hash_table_size(client_connections) - 1);
             g_hash_table_remove(client_connections, c->ipcs);
 
         } else {
             crm_trace("Destroying remote connection %p (%d remaining)",
                       c, g_hash_table_size(client_connections) - 1);
             g_hash_table_remove(client_connections, c->id);
         }
     }
 
     if (c->event_timer) {
         g_source_remove(c->event_timer);
     }
 
     if (c->event_queue) {
         crm_debug("Destroying %d events", g_queue_get_length(c->event_queue));
         g_queue_free_full(c->event_queue, free_event);
     }
 
     free(c->id);
     free(c->name);
     free(c->user);
     if (c->remote) {
         if (c->remote->auth_timeout) {
             g_source_remove(c->remote->auth_timeout);
         }
         free(c->remote->buffer);
         free(c->remote);
     }
     free(c);
 }
 
 /*!
  * \internal
  * \brief Raise IPC eviction threshold for a client, if allowed
  *
  * \param[in,out] client     Client to modify
  * \param[in]     qmax       New threshold (as non-NULL string)
  *
  * \return TRUE if change was allowed, FALSE otherwise
  */
 bool
 pcmk__set_client_queue_max(pcmk__client_t *client, const char *qmax)
 {
     if (is_set(client->flags, pcmk__client_privileged)) {
         long long qmax_int;
 
         errno = 0;
         qmax_int = crm_parse_ll(qmax, NULL);
         if ((errno == 0) && (qmax_int > 0)) {
             client->queue_max = (unsigned int) qmax_int;
             return TRUE;
         }
     }
     return FALSE;
 }
 
 int
 pcmk__client_pid(qb_ipcs_connection_t *c)
 {
     struct qb_ipcs_connection_stats stats;
 
     stats.client_pid = 0;
     qb_ipcs_connection_stats_get(c, &stats, 0);
     return stats.client_pid;
 }
 
 /*!
  * \internal
  * \brief Retrieve message XML from data read from client IPC
  *
  * \param[in]  c       IPC client connection
  * \param[in]  data    Data read from client connection
  * \param[out] id      Where to store message ID from libqb header
  * \param[out] flags   Where to store flags from libqb header
  *
  * \return Message XML on success, NULL otherwise
  */
 xmlNode *
 pcmk__client_data2xml(pcmk__client_t *c, void *data, uint32_t *id,
                       uint32_t *flags)
 {
     xmlNode *xml = NULL;
     char *uncompressed = NULL;
     char *text = ((char *)data) + sizeof(pcmk__ipc_header_t);
     pcmk__ipc_header_t *header = data;
 
     if (!pcmk__valid_ipc_header(header)) {
         return NULL;
     }
 
     if (id) {
         *id = ((struct qb_ipc_response_header *)data)->id;
     }
     if (flags) {
         *flags = header->flags;
     }
 
     if (is_set(header->flags, crm_ipc_proxied)) {
         /* Mark this client as being the endpoint of a proxy connection.
          * Proxy connections responses are sent on the event channel, to avoid
          * blocking the controller serving as proxy.
          */
         c->flags |= pcmk__client_proxied;
     }
 
     if (header->size_compressed) {
         int rc = 0;
         unsigned int size_u = 1 + header->size_uncompressed;
         uncompressed = calloc(1, size_u);
 
         crm_trace("Decompressing message data %u bytes into %u bytes",
                   header->size_compressed, size_u);
 
         rc = BZ2_bzBuffToBuffDecompress(uncompressed, &size_u, text, header->size_compressed, 1, 0);
         text = uncompressed;
 
         if (rc != BZ_OK) {
             crm_err("Decompression failed: %s " CRM_XS " bzerror=%d",
                     bz2_strerror(rc), rc);
             free(uncompressed);
             return NULL;
         }
     }
 
     CRM_ASSERT(text[header->size_uncompressed - 1] == 0);
 
     xml = string2xml(text);
     crm_log_xml_trace(xml, "[IPC received]");
 
     free(uncompressed);
     return xml;
 }
 
 static int crm_ipcs_flush_events(pcmk__client_t *c);
 
 static gboolean
 crm_ipcs_flush_events_cb(gpointer data)
 {
     pcmk__client_t *c = data;
 
     c->event_timer = 0;
     crm_ipcs_flush_events(c);
     return FALSE;
 }
 
 /*!
  * \internal
  * \brief Add progressive delay before next event queue flush
  *
  * \param[in,out] c          Client connection to add delay to
  * \param[in]     queue_len  Current event queue length
  */
 static inline void
 delay_next_flush(pcmk__client_t *c, unsigned int queue_len)
 {
     /* Delay a maximum of 1.5 seconds */
     guint delay = (queue_len < 5)? (1000 + 100 * queue_len) : 1500;
 
     c->event_timer = g_timeout_add(delay, crm_ipcs_flush_events_cb, c);
 }
 
 /*!
  * \internal
  * \brief Send client any messages in its queue
  *
  * \param[in]  c  Client to flush
  *
  * \return Standard Pacemaker return value
  */
 static int
 crm_ipcs_flush_events(pcmk__client_t *c)
 {
     int rc = pcmk_rc_ok;
     ssize_t qb_rc = 0;
     unsigned int sent = 0;
     unsigned int queue_len = 0;
 
     if (c == NULL) {
         return rc;
 
     } else if (c->event_timer) {
         /* There is already a timer, wait until it goes off */
         crm_trace("Timer active for %p - %d", c->ipcs, c->event_timer);
         return rc;
     }
 
     if (c->event_queue) {
         queue_len = g_queue_get_length(c->event_queue);
     }
     while (sent < 100) {
         pcmk__ipc_header_t *header = NULL;
         struct iovec *event = NULL;
 
         if (c->event_queue) {
             // We don't pop unless send is successful
             event = g_queue_peek_head(c->event_queue);
         }
         if (event == NULL) { // Queue is empty
             break;
         }
 
         qb_rc = qb_ipcs_event_sendv(c->ipcs, event, 2);
         if (qb_rc < 0) {
             rc = (int) -qb_rc;
             break;
         }
         event = g_queue_pop_head(c->event_queue);
 
         sent++;
         header = event[0].iov_base;
         if (header->size_compressed) {
             crm_trace("Event %d to %p[%d] (%lld compressed bytes) sent",
                       header->qb.id, c->ipcs, c->pid, (long long) qb_rc);
         } else {
             crm_trace("Event %d to %p[%d] (%lld bytes) sent: %.120s",
                       header->qb.id, c->ipcs, c->pid, (long long) qb_rc,
                       (char *) (event[1].iov_base));
         }
         pcmk_free_ipc_event(event);
     }
 
     queue_len -= sent;
     if (sent > 0 || queue_len) {
         crm_trace("Sent %d events (%d remaining) for %p[%d]: %s (%lld)",
                   sent, queue_len, c->ipcs, c->pid,
                   pcmk_rc_str(rc), (long long) qb_rc);
     }
 
     if (queue_len) {
 
         /* Allow clients to briefly fall behind on processing incoming messages,
          * but drop completely unresponsive clients so the connection doesn't
          * consume resources indefinitely.
          */
         if (queue_len > QB_MAX(c->queue_max, PCMK_IPC_DEFAULT_QUEUE_MAX)) {
             if ((c->queue_backlog <= 1) || (queue_len < c->queue_backlog)) {
                 /* Don't evict for a new or shrinking backlog */
                 crm_warn("Client with process ID %u has a backlog of %u messages "
                          CRM_XS " %p", c->pid, queue_len, c->ipcs);
             } else {
                 crm_err("Evicting client with process ID %u due to backlog of %u messages "
                          CRM_XS " %p", c->pid, queue_len, c->ipcs);
                 c->queue_backlog = 0;
                 qb_ipcs_disconnect(c->ipcs);
                 return rc;
             }
         }
 
         c->queue_backlog = queue_len;
         delay_next_flush(c, queue_len);
 
     } else {
         /* Event queue is empty, there is no backlog */
         c->queue_backlog = 0;
     }
 
     return rc;
 }
 
 /*!
  * \internal
  * \brief Create an I/O vector for sending an IPC XML message
  *
  * \param[in]  request        Identifier for libqb response header
  * \param[in]  message        XML message to send
  * \param[in]  max_send_size  If 0, default IPC buffer size is used
  * \param[out] result         Where to store prepared I/O vector
  * \param[out] bytes          Size of prepared data in bytes
  *
  * \return Standard Pacemaker return code
  */
 int
 pcmk__ipc_prepare_iov(uint32_t request, xmlNode *message,
                       uint32_t max_send_size, struct iovec **result,
                       ssize_t *bytes)
 {
     static unsigned int biggest = 0;
     struct iovec *iov;
     unsigned int total = 0;
     char *compressed = NULL;
     char *buffer = NULL;
     pcmk__ipc_header_t *header = NULL;
 
     if ((message == NULL) || (result == NULL)) {
         return EINVAL;
     }
 
     header = calloc(1, sizeof(pcmk__ipc_header_t));
     if (header == NULL) {
-        return errno;
+       return ENOMEM; /* errno mightn't be set by allocator */
     }
 
     buffer = dump_xml_unformatted(message);
 
     if (max_send_size == 0) {
         max_send_size = crm_ipc_default_buffer_size();
     }
     CRM_LOG_ASSERT(max_send_size != 0);
 
     *result = NULL;
     iov = pcmk__new_ipc_event();
     iov[0].iov_len = sizeof(pcmk__ipc_header_t);
     iov[0].iov_base = header;
 
     header->version = PCMK__IPC_VERSION;
     header->size_uncompressed = 1 + strlen(buffer);
     total = iov[0].iov_len + header->size_uncompressed;
 
     if (total < max_send_size) {
         iov[1].iov_base = buffer;
         iov[1].iov_len = header->size_uncompressed;
 
     } else {
         unsigned int new_size = 0;
 
         if (pcmk__compress(buffer, (unsigned int) header->size_uncompressed,
                            (unsigned int) max_send_size, &compressed,
                            &new_size) == pcmk_rc_ok) {
 
             header->flags |= crm_ipc_compressed;
             header->size_compressed = new_size;
 
             iov[1].iov_len = header->size_compressed;
             iov[1].iov_base = compressed;
 
             free(buffer);
 
             biggest = QB_MAX(header->size_compressed, biggest);
 
         } else {
             crm_log_xml_trace(message, "EMSGSIZE");
             biggest = QB_MAX(header->size_uncompressed, biggest);
 
             crm_err("Could not compress %u-byte message into less than IPC "
                     "limit of %u bytes; set PCMK_ipc_buffer to higher value "
                     "(%u bytes suggested)",
                     header->size_uncompressed, max_send_size, 4 * biggest);
 
             free(compressed);
             free(buffer);
             pcmk_free_ipc_event(iov);
             return EMSGSIZE;
         }
     }
 
     header->qb.size = iov[0].iov_len + iov[1].iov_len;
     header->qb.id = (int32_t)request;    /* Replying to a specific request */
 
     *result = iov;
     CRM_ASSERT(header->qb.size > 0);
     if (bytes != NULL) {
         *bytes = header->qb.size;
     }
     return pcmk_rc_ok;
 }
 
 int
 pcmk__ipc_send_iov(pcmk__client_t *c, struct iovec *iov, uint32_t flags)
 {
     int rc = pcmk_rc_ok;
     static uint32_t id = 1;
     pcmk__ipc_header_t *header = iov[0].iov_base;
 
     if (c->flags & pcmk__client_proxied) {
         /* _ALL_ replies to proxied connections need to be sent as events */
         if (is_not_set(flags, crm_ipc_server_event)) {
             flags |= crm_ipc_server_event;
             /* this flag lets us know this was originally meant to be a response.
              * even though we're sending it over the event channel. */
             flags |= crm_ipc_proxied_relay_response;
         }
     }
 
     header->flags |= flags;
     if (flags & crm_ipc_server_event) {
         header->qb.id = id++;   /* We don't really use it, but doesn't hurt to set one */
 
         if (flags & crm_ipc_server_free) {
             crm_trace("Sending the original to %p[%d]", c->ipcs, c->pid);
             add_event(c, iov);
 
         } else {
             struct iovec *iov_copy = pcmk__new_ipc_event();
 
             crm_trace("Sending a copy to %p[%d]", c->ipcs, c->pid);
             iov_copy[0].iov_len = iov[0].iov_len;
             iov_copy[0].iov_base = malloc(iov[0].iov_len);
             memcpy(iov_copy[0].iov_base, iov[0].iov_base, iov[0].iov_len);
 
             iov_copy[1].iov_len = iov[1].iov_len;
             iov_copy[1].iov_base = malloc(iov[1].iov_len);
             memcpy(iov_copy[1].iov_base, iov[1].iov_base, iov[1].iov_len);
 
             add_event(c, iov_copy);
         }
 
     } else {
         ssize_t qb_rc;
 
         CRM_LOG_ASSERT(header->qb.id != 0);     /* Replying to a specific request */
 
         qb_rc = qb_ipcs_response_sendv(c->ipcs, iov, 2);
         if (qb_rc < header->qb.size) {
             if (qb_rc < 0) {
                 rc = (int) -qb_rc;
             }
             crm_notice("Response %d to pid %d failed: %s "
                        CRM_XS " bytes=%u rc=%lld ipcs=%p",
                        header->qb.id, c->pid, pcmk_rc_str(rc),
                        header->qb.size, (long long) qb_rc, c->ipcs);
 
         } else {
             crm_trace("Response %d sent, %lld bytes to %p[%d]",
                       header->qb.id, (long long) qb_rc, c->ipcs, c->pid);
         }
 
         if (flags & crm_ipc_server_free) {
             pcmk_free_ipc_event(iov);
         }
     }
 
     if (flags & crm_ipc_server_event) {
         rc = crm_ipcs_flush_events(c);
     } else {
         crm_ipcs_flush_events(c);
     }
 
     if ((rc == EPIPE) || (rc == ENOTCONN)) {
         crm_trace("Client %p disconnected", c->ipcs);
     }
     return rc;
 }
 
 int
 pcmk__ipc_send_xml(pcmk__client_t *c, uint32_t request, xmlNode *message,
                    uint32_t flags)
 {
     struct iovec *iov = NULL;
     int rc = pcmk_rc_ok;
 
     if (c == NULL) {
         return EINVAL;
     }
     rc = pcmk__ipc_prepare_iov(request, message, crm_ipc_default_buffer_size(),
                                &iov, NULL);
     if (rc == pcmk_rc_ok) {
         rc = pcmk__ipc_send_iov(c, iov, flags | crm_ipc_server_free);
     } else {
         pcmk_free_ipc_event(iov);
         crm_notice("IPC message to pid %d failed: %s " CRM_XS " rc=%d",
                    c->pid, pcmk_rc_str(rc), rc);
     }
     return rc;
 }
 
 void
 pcmk__ipc_send_ack_as(const char *function, int line, pcmk__client_t *c,
                       uint32_t request, uint32_t flags, const char *tag)
 {
     if (flags & crm_ipc_client_response) {
         xmlNode *ack = create_xml_node(NULL, tag);
 
         crm_trace("Ack'ing IPC message from %s", pcmk__client_name(c));
         c->request_id = 0;
         crm_xml_add(ack, "function", function);
         crm_xml_add_int(ack, "line", line);
         pcmk__ipc_send_xml(c, request, ack, flags);
         free_xml(ack);
     }
 }
 
 /*!
  * \internal
  * \brief Add an IPC server to the main loop for the pacemaker-based API
  *
  * \param[out] ipcs_ro   New IPC server for read-only pacemaker-based API
  * \param[out] ipcs_rw   New IPC server for read/write pacemaker-based API
  * \param[out] ipcs_shm  New IPC server for shared-memory pacemaker-based API
  * \param[in]  ro_cb     IPC callbacks for read-only API
  * \param[in]  rw_cb     IPC callbacks for read/write and shared-memory APIs
  *
  * \note This function exits fatally if unable to create the servers.
  */
 void pcmk__serve_based_ipc(qb_ipcs_service_t **ipcs_ro,
                            qb_ipcs_service_t **ipcs_rw,
                            qb_ipcs_service_t **ipcs_shm,
                            struct qb_ipcs_service_handlers *ro_cb,
                            struct qb_ipcs_service_handlers *rw_cb)
 {
     *ipcs_ro = mainloop_add_ipc_server(PCMK__SERVER_BASED_RO,
                                        QB_IPC_NATIVE, ro_cb);
 
     *ipcs_rw = mainloop_add_ipc_server(PCMK__SERVER_BASED_RW,
                                        QB_IPC_NATIVE, rw_cb);
 
     *ipcs_shm = mainloop_add_ipc_server(PCMK__SERVER_BASED_SHM,
                                         QB_IPC_SHM, rw_cb);
 
     if (*ipcs_ro == NULL || *ipcs_rw == NULL || *ipcs_shm == NULL) {
         crm_err("Failed to create the CIB manager: exiting and inhibiting respawn");
         crm_warn("Verify pacemaker and pacemaker_remote are not both enabled");
         crm_exit(CRM_EX_FATAL);
     }
 }
 
 /*!
  * \internal
  * \brief Destroy IPC servers for pacemaker-based API
  *
  * \param[out] ipcs_ro   IPC server for read-only pacemaker-based API
  * \param[out] ipcs_rw   IPC server for read/write pacemaker-based API
  * \param[out] ipcs_shm  IPC server for shared-memory pacemaker-based API
  *
  * \note This is a convenience function for calling qb_ipcs_destroy() for each
  *       argument.
  */
 void
 pcmk__stop_based_ipc(qb_ipcs_service_t *ipcs_ro,
                      qb_ipcs_service_t *ipcs_rw,
                      qb_ipcs_service_t *ipcs_shm)
 {
     qb_ipcs_destroy(ipcs_ro);
     qb_ipcs_destroy(ipcs_rw);
     qb_ipcs_destroy(ipcs_shm);
 }
 
 /*!
  * \internal
  * \brief Add an IPC server to the main loop for the pacemaker-controld API
  *
  * \param[in] cb  IPC callbacks
  *
  * \return Newly created IPC server
  */
 qb_ipcs_service_t *
 pcmk__serve_controld_ipc(struct qb_ipcs_service_handlers *cb)
 {
     return mainloop_add_ipc_server(CRM_SYSTEM_CRMD, QB_IPC_NATIVE, cb);
 }
 
 /*!
  * \internal
  * \brief Add an IPC server to the main loop for the pacemaker-attrd API
  *
  * \param[in] cb  IPC callbacks
  *
  * \note This function exits fatally if unable to create the servers.
  */
 void
 pcmk__serve_attrd_ipc(qb_ipcs_service_t **ipcs,
                       struct qb_ipcs_service_handlers *cb)
 {
     *ipcs = mainloop_add_ipc_server(T_ATTRD, QB_IPC_NATIVE, cb);
 
     if (*ipcs == NULL) {
         crm_err("Failed to create pacemaker-attrd server: exiting and inhibiting respawn");
         crm_warn("Verify pacemaker and pacemaker_remote are not both enabled.");
         crm_exit(CRM_EX_FATAL);
     }
 }
 
 /*!
  * \internal
  * \brief Add an IPC server to the main loop for the pacemaker-fenced API
  *
  * \param[in] cb  IPC callbacks
  *
  * \note This function exits fatally if unable to create the servers.
  */
 void
 pcmk__serve_fenced_ipc(qb_ipcs_service_t **ipcs,
                        struct qb_ipcs_service_handlers *cb)
 {
     *ipcs = mainloop_add_ipc_server_with_prio("stonith-ng", QB_IPC_NATIVE, cb,
                                               QB_LOOP_HIGH);
 
     if (*ipcs == NULL) {
         crm_err("Failed to create fencer: exiting and inhibiting respawn.");
         crm_warn("Verify pacemaker and pacemaker_remote are not both enabled.");
         crm_exit(CRM_EX_FATAL);
     }
 }
diff --git a/lib/pengine/bundle.c b/lib/pengine/bundle.c
index 776f85aabe..6a72cc5835 100644
--- a/lib/pengine/bundle.c
+++ b/lib/pengine/bundle.c
@@ -1,1965 +1,1965 @@
 /*
  * Copyright 2004-2020 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU Lesser General Public License
  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #include <ctype.h>
 
 #include <crm/pengine/rules.h>
 #include <crm/pengine/status.h>
 #include <crm/pengine/internal.h>
 #include <crm/msg_xml.h>
 #include <pe_status_private.h>
 
 #define PE__VARIANT_BUNDLE 1
 #include "./variant.h"
 
 static char *
 next_ip(const char *last_ip)
 {
     unsigned int oct1 = 0;
     unsigned int oct2 = 0;
     unsigned int oct3 = 0;
     unsigned int oct4 = 0;
     int rc = sscanf(last_ip, "%u.%u.%u.%u", &oct1, &oct2, &oct3, &oct4);
 
     if (rc != 4) {
         /*@ TODO check for IPv6 */
         return NULL;
 
     } else if (oct3 > 253) {
         return NULL;
 
     } else if (oct4 > 253) {
         ++oct3;
         oct4 = 1;
 
     } else {
         ++oct4;
     }
 
     return crm_strdup_printf("%u.%u.%u.%u", oct1, oct2, oct3, oct4);
 }
 
 static int
 allocate_ip(pe__bundle_variant_data_t *data, pe__bundle_replica_t *replica,
             char *buffer, int max)
 {
     if(data->ip_range_start == NULL) {
         return 0;
 
     } else if(data->ip_last) {
         replica->ipaddr = next_ip(data->ip_last);
 
     } else {
         replica->ipaddr = strdup(data->ip_range_start);
     }
 
     data->ip_last = replica->ipaddr;
     switch (data->agent_type) {
         case PE__CONTAINER_AGENT_DOCKER:
         case PE__CONTAINER_AGENT_PODMAN:
             if (data->add_host) {
                 return snprintf(buffer, max, " --add-host=%s-%d:%s",
                                 data->prefix, replica->offset,
                                 replica->ipaddr);
             }
         case PE__CONTAINER_AGENT_RKT:
             return snprintf(buffer, max, " --hosts-entry=%s=%s-%d",
                             replica->ipaddr, data->prefix, replica->offset);
         default: // PE__CONTAINER_AGENT_UNKNOWN
             break;
     }
     return 0;
 }
 
 static xmlNode *
 create_resource(const char *name, const char *provider, const char *kind)
 {
     xmlNode *rsc = create_xml_node(NULL, XML_CIB_TAG_RESOURCE);
 
     crm_xml_add(rsc, XML_ATTR_ID, name);
     crm_xml_add(rsc, XML_AGENT_ATTR_CLASS, PCMK_RESOURCE_CLASS_OCF);
     crm_xml_add(rsc, XML_AGENT_ATTR_PROVIDER, provider);
     crm_xml_add(rsc, XML_ATTR_TYPE, kind);
 
     return rsc;
 }
 
 /*!
  * \internal
  * \brief Check whether cluster can manage resource inside container
  *
  * \param[in] data  Container variant data
  *
  * \return TRUE if networking configuration is acceptable, FALSE otherwise
  *
  * \note The resource is manageable if an IP range or control port has been
  *       specified. If a control port is used without an IP range, replicas per
  *       host must be 1.
  */
 static bool
 valid_network(pe__bundle_variant_data_t *data)
 {
     if(data->ip_range_start) {
         return TRUE;
     }
     if(data->control_port) {
         if(data->nreplicas_per_host > 1) {
             pe_err("Specifying the 'control-port' for %s requires 'replicas-per-host=1'", data->prefix);
             data->nreplicas_per_host = 1;
             /* @TODO to be sure: clear_bit(rsc->flags, pe_rsc_unique); */
         }
         return TRUE;
     }
     return FALSE;
 }
 
 static bool
 create_ip_resource(pe_resource_t *parent, pe__bundle_variant_data_t *data,
                    pe__bundle_replica_t *replica, pe_working_set_t *data_set)
 {
     if(data->ip_range_start) {
         char *id = NULL;
         xmlNode *xml_ip = NULL;
         xmlNode *xml_obj = NULL;
 
         id = crm_strdup_printf("%s-ip-%s", data->prefix, replica->ipaddr);
         crm_xml_sanitize_id(id);
         xml_ip = create_resource(id, "heartbeat", "IPaddr2");
         free(id);
 
         xml_obj = create_xml_node(xml_ip, XML_TAG_ATTR_SETS);
         crm_xml_set_id(xml_obj, "%s-attributes-%d",
                        data->prefix, replica->offset);
 
         crm_create_nvpair_xml(xml_obj, NULL, "ip", replica->ipaddr);
         if(data->host_network) {
             crm_create_nvpair_xml(xml_obj, NULL, "nic", data->host_network);
         }
 
         if(data->host_netmask) {
             crm_create_nvpair_xml(xml_obj, NULL,
                                   "cidr_netmask", data->host_netmask);
 
         } else {
             crm_create_nvpair_xml(xml_obj, NULL, "cidr_netmask", "32");
         }
 
         xml_obj = create_xml_node(xml_ip, "operations");
         crm_create_op_xml(xml_obj, ID(xml_ip), "monitor", "60s", NULL);
 
         // TODO: Other ops? Timeouts and intervals from underlying resource?
 
         if (!common_unpack(xml_ip, &replica->ip, parent, data_set)) {
             return FALSE;
         }
 
         parent->children = g_list_append(parent->children, replica->ip);
     }
     return TRUE;
 }
 
 static bool
 create_docker_resource(pe_resource_t *parent, pe__bundle_variant_data_t *data,
                        pe__bundle_replica_t *replica,
                        pe_working_set_t *data_set)
 {
         int offset = 0, max = 4096;
         char *buffer = calloc(1, max+1);
 
         int doffset = 0, dmax = 1024;
         char *dbuffer = calloc(1, dmax+1);
 
         char *id = NULL;
         xmlNode *xml_container = NULL;
         xmlNode *xml_obj = NULL;
 
         id = crm_strdup_printf("%s-docker-%d", data->prefix, replica->offset);
         crm_xml_sanitize_id(id);
         xml_container = create_resource(id, "heartbeat",
                                         PE__CONTAINER_AGENT_DOCKER_S);
         free(id);
 
         xml_obj = create_xml_node(xml_container, XML_TAG_ATTR_SETS);
         crm_xml_set_id(xml_obj, "%s-attributes-%d",
                        data->prefix, replica->offset);
 
         crm_create_nvpair_xml(xml_obj, NULL, "image", data->image);
         crm_create_nvpair_xml(xml_obj, NULL, "allow_pull", XML_BOOLEAN_TRUE);
         crm_create_nvpair_xml(xml_obj, NULL, "force_kill", XML_BOOLEAN_FALSE);
         crm_create_nvpair_xml(xml_obj, NULL, "reuse", XML_BOOLEAN_FALSE);
 
         offset += snprintf(buffer+offset, max-offset, " --restart=no");
 
         /* Set a container hostname only if we have an IP to map it to.
          * The user can set -h or --uts=host themselves if they want a nicer
          * name for logs, but this makes applications happy who need their
          * hostname to match the IP they bind to.
          */
         if (data->ip_range_start != NULL) {
             offset += snprintf(buffer+offset, max-offset, " -h %s-%d",
                                data->prefix, replica->offset);
         }
 
         offset += snprintf(buffer+offset, max-offset, " -e PCMK_stderr=1");
 
         if (data->container_network) {
 #if 0
             offset += snprintf(buffer+offset, max-offset, " --link-local-ip=%s",
                                replica->ipaddr);
 #endif
             offset += snprintf(buffer+offset, max-offset, " --net=%s",
                                data->container_network);
         }
 
         if(data->control_port) {
             offset += snprintf(buffer+offset, max-offset, " -e PCMK_remote_port=%s", data->control_port);
         } else {
             offset += snprintf(buffer+offset, max-offset, " -e PCMK_remote_port=%d", DEFAULT_REMOTE_PORT);
         }
 
         for(GListPtr pIter = data->mounts; pIter != NULL; pIter = pIter->next) {
             pe__bundle_mount_t *mount = pIter->data;
 
             if (is_set(mount->flags, pe__bundle_mount_subdir)) {
                 char *source = crm_strdup_printf(
                     "%s/%s-%d", mount->source, data->prefix, replica->offset);
 
                 if(doffset > 0) {
                     doffset += snprintf(dbuffer+doffset, dmax-doffset, ",");
                 }
                 doffset += snprintf(dbuffer+doffset, dmax-doffset, "%s", source);
                 offset += snprintf(buffer+offset, max-offset, " -v %s:%s", source, mount->target);
                 free(source);
 
             } else {
                 offset += snprintf(buffer+offset, max-offset, " -v %s:%s", mount->source, mount->target);
             }
             if(mount->options) {
                 offset += snprintf(buffer+offset, max-offset, ":%s", mount->options);
             }
         }
 
         for(GListPtr pIter = data->ports; pIter != NULL; pIter = pIter->next) {
             pe__bundle_port_t *port = pIter->data;
 
             if (replica->ipaddr) {
                 offset += snprintf(buffer+offset, max-offset, " -p %s:%s:%s",
                                    replica->ipaddr, port->source,
                                    port->target);
             } else if(safe_str_neq(data->container_network, "host")) {
                 // No need to do port mapping if net=host
                 offset += snprintf(buffer+offset, max-offset, " -p %s:%s", port->source, port->target);
             }
         }
 
         if (data->launcher_options) {
             offset += snprintf(buffer+offset, max-offset, " %s",
                                data->launcher_options);
         }
 
         if (data->container_host_options) {
             offset += snprintf(buffer + offset, max - offset, " %s",
                                data->container_host_options);
         }
 
         crm_create_nvpair_xml(xml_obj, NULL, "run_opts", buffer);
         free(buffer);
 
         crm_create_nvpair_xml(xml_obj, NULL, "mount_points", dbuffer);
         free(dbuffer);
 
         if (replica->child) {
             if (data->container_command) {
                 crm_create_nvpair_xml(xml_obj, NULL,
                                       "run_cmd", data->container_command);
             } else {
                 crm_create_nvpair_xml(xml_obj, NULL,
                                       "run_cmd", SBIN_DIR "/pacemaker-remoted");
             }
 
             /* TODO: Allow users to specify their own?
              *
              * We just want to know if the container is alive, we'll
              * monitor the child independently
              */
             crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd", "/bin/true");
         /* } else if(child && data->untrusted) {
          * Support this use-case?
          *
          * The ability to have resources started/stopped by us, but
          * unable to set attributes, etc.
          *
          * Arguably better to control API access this with ACLs like
          * "normal" remote nodes
          *
          *     crm_create_nvpair_xml(xml_obj, NULL,
          *                           "run_cmd",
          *                           "/usr/libexec/pacemaker/pacemaker-execd");
          *     crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd",
          *         "/usr/libexec/pacemaker/lrmd_internal_ctl -c poke");
          */
         } else {
             if (data->container_command) {
                 crm_create_nvpair_xml(xml_obj, NULL,
                                       "run_cmd", data->container_command);
             }
 
             /* TODO: Allow users to specify their own?
              *
              * We don't know what's in the container, so we just want
              * to know if it is alive
              */
             crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd", "/bin/true");
         }
 
 
         xml_obj = create_xml_node(xml_container, "operations");
         crm_create_op_xml(xml_obj, ID(xml_container), "monitor", "60s", NULL);
 
         // TODO: Other ops? Timeouts and intervals from underlying resource?
         if (!common_unpack(xml_container, &replica->container, parent, data_set)) {
             return FALSE;
         }
         parent->children = g_list_append(parent->children, replica->container);
         return TRUE;
 }
 
 static bool
 create_podman_resource(pe_resource_t *parent, pe__bundle_variant_data_t *data,
                        pe__bundle_replica_t *replica,
                        pe_working_set_t *data_set)
 {
         int offset = 0, max = 4096;
         char *buffer = calloc(1, max+1);
 
         int doffset = 0, dmax = 1024;
         char *dbuffer = calloc(1, dmax+1);
 
         char *id = NULL;
         xmlNode *xml_container = NULL;
         xmlNode *xml_obj = NULL;
 
         id = crm_strdup_printf("%s-podman-%d", data->prefix, replica->offset);
         crm_xml_sanitize_id(id);
         xml_container = create_resource(id, "heartbeat",
                                         PE__CONTAINER_AGENT_PODMAN_S);
         free(id);
 
         xml_obj = create_xml_node(xml_container, XML_TAG_ATTR_SETS);
         crm_xml_set_id(xml_obj, "%s-attributes-%d",
                        data->prefix, replica->offset);
 
         crm_create_nvpair_xml(xml_obj, NULL, "image", data->image);
         crm_create_nvpair_xml(xml_obj, NULL, "allow_pull", XML_BOOLEAN_TRUE);
         crm_create_nvpair_xml(xml_obj, NULL, "force_kill", XML_BOOLEAN_FALSE);
         crm_create_nvpair_xml(xml_obj, NULL, "reuse", XML_BOOLEAN_FALSE);
 
         // FIXME: (bandini 2018-08) podman has no restart policies
         //offset += snprintf(buffer+offset, max-offset, " --restart=no");
 
         /* Set a container hostname only if we have an IP to map it to.
          * The user can set -h or --uts=host themselves if they want a nicer
          * name for logs, but this makes applications happy who need their
          * hostname to match the IP they bind to.
          */
         if (data->ip_range_start != NULL) {
             offset += snprintf(buffer+offset, max-offset, " -h %s-%d",
                                data->prefix, replica->offset);
         }
 
         offset += snprintf(buffer+offset, max-offset, " -e PCMK_stderr=1");
 
         if (data->container_network) {
 #if 0
             // podman has no support for --link-local-ip
             offset += snprintf(buffer+offset, max-offset, " --link-local-ip=%s",
                                replica->ipaddr);
 #endif
             offset += snprintf(buffer+offset, max-offset, " --net=%s",
                                data->container_network);
         }
 
         if(data->control_port) {
             offset += snprintf(buffer+offset, max-offset, " -e PCMK_remote_port=%s", data->control_port);
         } else {
             offset += snprintf(buffer+offset, max-offset, " -e PCMK_remote_port=%d", DEFAULT_REMOTE_PORT);
         }
 
         for(GListPtr pIter = data->mounts; pIter != NULL; pIter = pIter->next) {
             pe__bundle_mount_t *mount = pIter->data;
 
             if (is_set(mount->flags, pe__bundle_mount_subdir)) {
                 char *source = crm_strdup_printf(
                     "%s/%s-%d", mount->source, data->prefix, replica->offset);
 
                 if(doffset > 0) {
                     doffset += snprintf(dbuffer+doffset, dmax-doffset, ",");
                 }
                 doffset += snprintf(dbuffer+doffset, dmax-doffset, "%s", source);
                 offset += snprintf(buffer+offset, max-offset, " -v %s:%s", source, mount->target);
                 free(source);
 
             } else {
                 offset += snprintf(buffer+offset, max-offset, " -v %s:%s", mount->source, mount->target);
             }
             if(mount->options) {
                 offset += snprintf(buffer+offset, max-offset, ":%s", mount->options);
             }
         }
 
         for(GListPtr pIter = data->ports; pIter != NULL; pIter = pIter->next) {
             pe__bundle_port_t *port = pIter->data;
 
             if (replica->ipaddr) {
                 offset += snprintf(buffer+offset, max-offset, " -p %s:%s:%s",
                                    replica->ipaddr, port->source,
                                    port->target);
             } else if(safe_str_neq(data->container_network, "host")) {
                 // No need to do port mapping if net=host
                 offset += snprintf(buffer+offset, max-offset, " -p %s:%s", port->source, port->target);
             }
         }
 
         if (data->launcher_options) {
             offset += snprintf(buffer+offset, max-offset, " %s",
                                data->launcher_options);
         }
 
         if (data->container_host_options) {
             offset += snprintf(buffer + offset, max - offset, " %s",
                                data->container_host_options);
         }
 
         crm_create_nvpair_xml(xml_obj, NULL, "run_opts", buffer);
         free(buffer);
 
         crm_create_nvpair_xml(xml_obj, NULL, "mount_points", dbuffer);
         free(dbuffer);
 
         if (replica->child) {
             if (data->container_command) {
                 crm_create_nvpair_xml(xml_obj, NULL,
                                       "run_cmd", data->container_command);
             } else {
                 crm_create_nvpair_xml(xml_obj, NULL,
                                       "run_cmd", SBIN_DIR "/pacemaker-remoted");
             }
 
             /* TODO: Allow users to specify their own?
              *
              * We just want to know if the container is alive, we'll
              * monitor the child independently
              */
             crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd", "/bin/true");
         /* } else if(child && data->untrusted) {
          * Support this use-case?
          *
          * The ability to have resources started/stopped by us, but
          * unable to set attributes, etc.
          *
          * Arguably better to control API access this with ACLs like
          * "normal" remote nodes
          *
          *     crm_create_nvpair_xml(xml_obj, NULL,
          *                           "run_cmd",
          *                           "/usr/libexec/pacemaker/pacemaker-execd");
          *     crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd",
          *         "/usr/libexec/pacemaker/lrmd_internal_ctl -c poke");
          */
         } else {
             if (data->container_command) {
                 crm_create_nvpair_xml(xml_obj, NULL,
                                       "run_cmd", data->container_command);
             }
 
             /* TODO: Allow users to specify their own?
              *
              * We don't know what's in the container, so we just want
              * to know if it is alive
              */
             crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd", "/bin/true");
         }
 
 
         xml_obj = create_xml_node(xml_container, "operations");
         crm_create_op_xml(xml_obj, ID(xml_container), "monitor", "60s", NULL);
 
         // TODO: Other ops? Timeouts and intervals from underlying resource?
         if (!common_unpack(xml_container, &replica->container, parent,
                            data_set)) {
             return FALSE;
         }
         parent->children = g_list_append(parent->children, replica->container);
         return TRUE;
 }
 
 static bool
 create_rkt_resource(pe_resource_t *parent, pe__bundle_variant_data_t *data,
                     pe__bundle_replica_t *replica, pe_working_set_t *data_set)
 {
         int offset = 0, max = 4096;
         char *buffer = calloc(1, max+1);
 
         int doffset = 0, dmax = 1024;
         char *dbuffer = calloc(1, dmax+1);
 
         char *id = NULL;
         xmlNode *xml_container = NULL;
         xmlNode *xml_obj = NULL;
 
         int volid = 0;
 
         id = crm_strdup_printf("%s-rkt-%d", data->prefix, replica->offset);
         crm_xml_sanitize_id(id);
         xml_container = create_resource(id, "heartbeat",
                                         PE__CONTAINER_AGENT_RKT_S);
         free(id);
 
         xml_obj = create_xml_node(xml_container, XML_TAG_ATTR_SETS);
         crm_xml_set_id(xml_obj, "%s-attributes-%d",
                        data->prefix, replica->offset);
 
         crm_create_nvpair_xml(xml_obj, NULL, "image", data->image);
         crm_create_nvpair_xml(xml_obj, NULL, "allow_pull", "true");
         crm_create_nvpair_xml(xml_obj, NULL, "force_kill", "false");
         crm_create_nvpair_xml(xml_obj, NULL, "reuse", "false");
 
         /* Set a container hostname only if we have an IP to map it to.
          * The user can set -h or --uts=host themselves if they want a nicer
          * name for logs, but this makes applications happy who need their
          * hostname to match the IP they bind to.
          */
         if (data->ip_range_start != NULL) {
             offset += snprintf(buffer+offset, max-offset, " --hostname=%s-%d",
                                data->prefix, replica->offset);
         }
 
         offset += snprintf(buffer+offset, max-offset, " --environment=PCMK_stderr=1");
 
         if (data->container_network) {
 #if 0
             offset += snprintf(buffer+offset, max-offset, " --link-local-ip=%s",
                                replica->ipaddr);
 #endif
             offset += snprintf(buffer+offset, max-offset, " --net=%s",
                                data->container_network);
         }
 
         if(data->control_port) {
             offset += snprintf(buffer+offset, max-offset, " --environment=PCMK_remote_port=%s", data->control_port);
         } else {
             offset += snprintf(buffer+offset, max-offset, " --environment=PCMK_remote_port=%d", DEFAULT_REMOTE_PORT);
         }
 
         for(GListPtr pIter = data->mounts; pIter != NULL; pIter = pIter->next) {
             pe__bundle_mount_t *mount = pIter->data;
 
             if (is_set(mount->flags, pe__bundle_mount_subdir)) {
                 char *source = crm_strdup_printf(
                     "%s/%s-%d", mount->source, data->prefix, replica->offset);
 
                 if(doffset > 0) {
                     doffset += snprintf(dbuffer+doffset, dmax-doffset, ",");
                 }
                 doffset += snprintf(dbuffer+doffset, dmax-doffset, "%s", source);
                 offset += snprintf(buffer+offset, max-offset, " --volume vol%d,kind=host,source=%s", volid, source);
                 if(mount->options) {
                     offset += snprintf(buffer+offset, max-offset, ",%s", mount->options);
                 }
                 offset += snprintf(buffer+offset, max-offset, " --mount volume=vol%d,target=%s", volid, mount->target);
                 free(source);
 
             } else {
                 offset += snprintf(buffer+offset, max-offset, " --volume vol%d,kind=host,source=%s", volid, mount->source);
                 if(mount->options) {
                     offset += snprintf(buffer+offset, max-offset, ",%s", mount->options);
                 }
                 offset += snprintf(buffer+offset, max-offset, " --mount volume=vol%d,target=%s", volid, mount->target);
             }
             volid++;
         }
 
         for(GListPtr pIter = data->ports; pIter != NULL; pIter = pIter->next) {
             pe__bundle_port_t *port = pIter->data;
 
             if (replica->ipaddr) {
                 offset += snprintf(buffer+offset, max-offset,
                                    " --port=%s:%s:%s", port->target,
                                    replica->ipaddr, port->source);
             } else {
                 offset += snprintf(buffer+offset, max-offset, " --port=%s:%s", port->target, port->source);
             }
         }
 
         if (data->launcher_options) {
             offset += snprintf(buffer+offset, max-offset, " %s",
                                data->launcher_options);
         }
 
         if (data->container_host_options) {
             offset += snprintf(buffer + offset, max - offset, " %s",
                                data->container_host_options);
         }
 
         crm_create_nvpair_xml(xml_obj, NULL, "run_opts", buffer);
         free(buffer);
 
         crm_create_nvpair_xml(xml_obj, NULL, "mount_points", dbuffer);
         free(dbuffer);
 
         if (replica->child) {
             if (data->container_command) {
                 crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
                                       data->container_command);
             } else {
                 crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
                                       SBIN_DIR "/pacemaker-remoted");
             }
 
             /* TODO: Allow users to specify their own?
              *
              * We just want to know if the container is alive, we'll
              * monitor the child independently
              */
             crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd", "/bin/true");
         /* } else if(child && data->untrusted) {
          * Support this use-case?
          *
          * The ability to have resources started/stopped by us, but
          * unable to set attributes, etc.
          *
          * Arguably better to control API access this with ACLs like
          * "normal" remote nodes
          *
          *     crm_create_nvpair_xml(xml_obj, NULL,
          *                           "run_cmd",
          *                           "/usr/libexec/pacemaker/pacemaker-execd");
          *     crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd",
          *         "/usr/libexec/pacemaker/lrmd_internal_ctl -c poke");
          */
         } else {
             if (data->container_command) {
                 crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
                                       data->container_command);
             }
 
             /* TODO: Allow users to specify their own?
              *
              * We don't know what's in the container, so we just want
              * to know if it is alive
              */
             crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd", "/bin/true");
         }
 
 
         xml_obj = create_xml_node(xml_container, "operations");
         crm_create_op_xml(xml_obj, ID(xml_container), "monitor", "60s", NULL);
 
         // TODO: Other ops? Timeouts and intervals from underlying resource?
 
         if (!common_unpack(xml_container, &replica->container, parent, data_set)) {
             return FALSE;
         }
         parent->children = g_list_append(parent->children, replica->container);
         return TRUE;
 }
 
 /*!
  * \brief Ban a node from a resource's (and its children's) allowed nodes list
  *
  * \param[in,out] rsc    Resource to modify
  * \param[in]     uname  Name of node to ban
  */
 static void
 disallow_node(pe_resource_t *rsc, const char *uname)
 {
     gpointer match = g_hash_table_lookup(rsc->allowed_nodes, uname);
 
     if (match) {
         ((pe_node_t *) match)->weight = -INFINITY;
         ((pe_node_t *) match)->rsc_discover_mode = pe_discover_never;
     }
     if (rsc->children) {
         GListPtr child;
 
         for (child = rsc->children; child != NULL; child = child->next) {
             disallow_node((pe_resource_t *) (child->data), uname);
         }
     }
 }
 
 static bool
 create_remote_resource(pe_resource_t *parent, pe__bundle_variant_data_t *data,
                        pe__bundle_replica_t *replica,
                        pe_working_set_t *data_set)
 {
     if (replica->child && valid_network(data)) {
         GHashTableIter gIter;
         GListPtr rsc_iter = NULL;
         pe_node_t *node = NULL;
         xmlNode *xml_remote = NULL;
         char *id = crm_strdup_printf("%s-%d", data->prefix, replica->offset);
         char *port_s = NULL;
         const char *uname = NULL;
         const char *connect_name = NULL;
 
         if (pe_find_resource(data_set->resources, id) != NULL) {
             free(id);
             // The biggest hammer we have
             id = crm_strdup_printf("pcmk-internal-%s-remote-%d",
                                    replica->child->id, replica->offset);
             //@TODO return false instead of asserting?
             CRM_ASSERT(pe_find_resource(data_set->resources, id) == NULL);
         }
 
         /* REMOTE_CONTAINER_HACK: Using "#uname" as the server name when the
          * connection does not have its own IP is a magic string that we use to
          * support nested remotes (i.e. a bundle running on a remote node).
          */
         connect_name = (replica->ipaddr? replica->ipaddr : "#uname");
 
         if (data->control_port == NULL) {
             port_s = crm_itoa(DEFAULT_REMOTE_PORT);
         }
 
         /* This sets replica->container as replica->remote's container, which is
          * similar to what happens with guest nodes. This is how the scheduler
          * knows that the bundle node is fenced by recovering the container, and
          * that remote should be ordered relative to the container.
          */
         xml_remote = pe_create_remote_xml(NULL, id, replica->container->id,
                                           NULL, NULL, NULL,
                                           connect_name, (data->control_port?
                                           data->control_port : port_s));
         free(port_s);
 
         /* Abandon our created ID, and pull the copy from the XML, because we
          * need something that will get freed during data set cleanup to use as
          * the node ID and uname.
          */
         free(id);
         id = NULL;
         uname = ID(xml_remote);
 
         /* Ensure a node has been created for the guest (it may have already
          * been, if it has a permanent node attribute), and ensure its weight is
          * -INFINITY so no other resources can run on it.
          */
         node = pe_find_node(data_set->nodes, uname);
         if (node == NULL) {
             node = pe_create_node(uname, uname, "remote", "-INFINITY",
                                   data_set);
         } else {
             node->weight = -INFINITY;
         }
         node->rsc_discover_mode = pe_discover_never;
 
         /* unpack_remote_nodes() ensures that each remote node and guest node
          * has a pe_node_t entry. Ideally, it would do the same for bundle nodes.
          * Unfortunately, a bundle has to be mostly unpacked before it's obvious
          * what nodes will be needed, so we do it just above.
          *
          * Worse, that means that the node may have been utilized while
          * unpacking other resources, without our weight correction. The most
          * likely place for this to happen is when common_unpack() calls
          * resource_location() to set a default score in symmetric clusters.
          * This adds a node *copy* to each resource's allowed nodes, and these
          * copies will have the wrong weight.
          *
          * As a hacky workaround, fix those copies here.
          *
          * @TODO Possible alternative: ensure bundles are unpacked before other
          * resources, so the weight is correct before any copies are made.
          */
         for (rsc_iter = data_set->resources; rsc_iter; rsc_iter = rsc_iter->next) {
             disallow_node((pe_resource_t *) (rsc_iter->data), uname);
         }
 
         replica->node = pe__copy_node(node);
         replica->node->weight = 500;
         replica->node->rsc_discover_mode = pe_discover_exclusive;
 
         /* Ensure the node shows up as allowed and with the correct discovery set */
         if (replica->child->allowed_nodes != NULL) {
             g_hash_table_destroy(replica->child->allowed_nodes);
         }
         replica->child->allowed_nodes = g_hash_table_new_full(crm_str_hash,
                                                               g_str_equal,
                                                               NULL, free);
         g_hash_table_insert(replica->child->allowed_nodes,
                             (gpointer) replica->node->details->id,
                             pe__copy_node(replica->node));
 
         {
             pe_node_t *copy = pe__copy_node(replica->node);
             copy->weight = -INFINITY;
             g_hash_table_insert(replica->child->parent->allowed_nodes,
                                 (gpointer) replica->node->details->id, copy);
         }
         if (!common_unpack(xml_remote, &replica->remote, parent, data_set)) {
             return FALSE;
         }
 
         g_hash_table_iter_init(&gIter, replica->remote->allowed_nodes);
         while (g_hash_table_iter_next(&gIter, NULL, (void **)&node)) {
             if (pe__is_guest_or_remote_node(node)) {
                 /* Remote resources can only run on 'normal' cluster node */
                 node->weight = -INFINITY;
             }
         }
 
         replica->node->details->remote_rsc = replica->remote;
 
         // Ensure pe__is_guest_node() functions correctly immediately
         replica->remote->container = replica->container;
 
         /* A bundle's #kind is closer to "container" (guest node) than the
          * "remote" set by pe_create_node().
          */
         g_hash_table_insert(replica->node->details->attrs,
                             strdup(CRM_ATTR_KIND), strdup("container"));
 
         /* One effect of this is that setup_container() will add
          * replica->remote to replica->container's fillers, which will make
          * pe__resource_contains_guest_node() true for replica->container.
          *
          * replica->child does NOT get added to replica->container's fillers.
          * The only noticeable effect if it did would be for its fail count to
          * be taken into account when checking replica->container's migration
          * threshold.
          */
         parent->children = g_list_append(parent->children, replica->remote);
     }
     return TRUE;
 }
 
 static bool
 create_container(pe_resource_t *parent, pe__bundle_variant_data_t *data,
                  pe__bundle_replica_t *replica, pe_working_set_t *data_set)
 {
 
     switch (data->agent_type) {
         case PE__CONTAINER_AGENT_DOCKER:
             if (!create_docker_resource(parent, data, replica, data_set)) {
                 return FALSE;
             }
             break;
 
         case PE__CONTAINER_AGENT_PODMAN:
             if (!create_podman_resource(parent, data, replica, data_set)) {
                 return FALSE;
             }
             break;
 
         case PE__CONTAINER_AGENT_RKT:
             if (!create_rkt_resource(parent, data, replica, data_set)) {
                 return FALSE;
             }
             break;
         default: // PE__CONTAINER_AGENT_UNKNOWN
             return FALSE;
     }
 
     if (create_ip_resource(parent, data, replica, data_set) == FALSE) {
         return FALSE;
     }
     if(create_remote_resource(parent, data, replica, data_set) == FALSE) {
         return FALSE;
     }
     if (replica->child && replica->ipaddr) {
         add_hash_param(replica->child->meta, "external-ip", replica->ipaddr);
     }
 
     if (replica->remote) {
         /*
          * Allow the remote connection resource to be allocated to a
          * different node than the one on which the container is active.
          *
          * This makes it possible to have Pacemaker Remote nodes running
          * containers with pacemaker-remoted inside in order to start
          * services inside those containers.
          */
         set_bit(replica->remote->flags, pe_rsc_allow_remote_remotes);
     }
 
     return TRUE;
 }
 
 static void
 mount_add(pe__bundle_variant_data_t *bundle_data, const char *source,
           const char *target, const char *options, uint32_t flags)
 {
     pe__bundle_mount_t *mount = calloc(1, sizeof(pe__bundle_mount_t));
 
     mount->source = strdup(source);
     mount->target = strdup(target);
     if (options) {
         mount->options = strdup(options);
     }
     mount->flags = flags;
     bundle_data->mounts = g_list_append(bundle_data->mounts, mount);
 }
 
 static void
 mount_free(pe__bundle_mount_t *mount)
 {
     free(mount->source);
     free(mount->target);
     free(mount->options);
     free(mount);
 }
 
 static void
 port_free(pe__bundle_port_t *port)
 {
     free(port->source);
     free(port->target);
     free(port);
 }
 
 static pe__bundle_replica_t *
 replica_for_remote(pe_resource_t *remote)
 {
     pe_resource_t *top = remote;
     pe__bundle_variant_data_t *bundle_data = NULL;
 
     if (top == NULL) {
         return NULL;
     }
 
     while (top->parent != NULL) {
         top = top->parent;
     }
 
     get_bundle_variant_data(bundle_data, top);
     for (GList *gIter = bundle_data->replicas; gIter != NULL;
          gIter = gIter->next) {
         pe__bundle_replica_t *replica = gIter->data;
 
         if (replica->remote == remote) {
             return replica;
         }
     }
     CRM_LOG_ASSERT(FALSE);
     return NULL;
 }
 
 bool
 pe__bundle_needs_remote_name(pe_resource_t *rsc)
 {
     const char *value;
 
     if (rsc == NULL) {
         return FALSE;
     }
 
     value = g_hash_table_lookup(rsc->parameters, XML_RSC_ATTR_REMOTE_RA_ADDR);
     if (safe_str_eq(value, "#uname") == FALSE) {
         return FALSE;
 
     } else {
         const char *match[3][2] = {
             { XML_ATTR_TYPE,           "remote"                },
             { XML_AGENT_ATTR_CLASS,    PCMK_RESOURCE_CLASS_OCF },
             { XML_AGENT_ATTR_PROVIDER, "pacemaker"             },
         };
 
         for (int m = 0; m < 3; m++) {
             value = crm_element_value(rsc->xml, match[m][0]);
             if (safe_str_neq(value, match[m][1])) {
                 return FALSE;
             }
         }
     }
     return TRUE;
 }
 
 const char *
 pe__add_bundle_remote_name(pe_resource_t *rsc, xmlNode *xml, const char *field)
 {
     // REMOTE_CONTAINER_HACK: Allow remote nodes that start containers with pacemaker remote inside
 
     pe_node_t *node = NULL;
     pe__bundle_replica_t *replica = NULL;
 
     if (!pe__bundle_needs_remote_name(rsc)) {
         return NULL;
     }
 
     replica = replica_for_remote(rsc);
     if (replica == NULL) {
         return NULL;
     }
 
     node = replica->container->allocated_to;
     if (node == NULL) {
         /* If it won't be running anywhere after the
          * transition, go with where it's running now.
          */
         node = pe__current_node(replica->container);
     }
 
     if(node == NULL) {
         crm_trace("Cannot determine address for bundle connection %s", rsc->id);
         return NULL;
     }
 
     crm_trace("Setting address for bundle connection %s to bundle host %s",
               rsc->id, node->details->uname);
     if(xml != NULL && field != NULL) {
         crm_xml_add(xml, field, node->details->uname);
     }
 
     return node->details->uname;
 }
 
 gboolean
 pe__unpack_bundle(pe_resource_t *rsc, pe_working_set_t *data_set)
 {
     const char *value = NULL;
     xmlNode *xml_obj = NULL;
     xmlNode *xml_resource = NULL;
     pe__bundle_variant_data_t *bundle_data = NULL;
     bool need_log_mount = TRUE;
 
     CRM_ASSERT(rsc != NULL);
     pe_rsc_trace(rsc, "Processing resource %s...", rsc->id);
 
     bundle_data = calloc(1, sizeof(pe__bundle_variant_data_t));
     rsc->variant_opaque = bundle_data;
     bundle_data->prefix = strdup(rsc->id);
 
     xml_obj = first_named_child(rsc->xml, PE__CONTAINER_AGENT_DOCKER_S);
     if (xml_obj != NULL) {
         bundle_data->agent_type = PE__CONTAINER_AGENT_DOCKER;
     } else {
         xml_obj = first_named_child(rsc->xml, PE__CONTAINER_AGENT_RKT_S);
         if (xml_obj != NULL) {
             bundle_data->agent_type = PE__CONTAINER_AGENT_RKT;
         } else {
             xml_obj = first_named_child(rsc->xml, PE__CONTAINER_AGENT_PODMAN_S);
             if (xml_obj != NULL) {
                 bundle_data->agent_type = PE__CONTAINER_AGENT_PODMAN;
             } else {
                 return FALSE;
             }
         }
     }
 
     value = crm_element_value(xml_obj, XML_RSC_ATTR_PROMOTED_MAX);
     if (value == NULL) {
         // @COMPAT deprecated since 2.0.0
         value = crm_element_value(xml_obj, "masters");
     }
     bundle_data->promoted_max = crm_parse_int(value, "0");
     if (bundle_data->promoted_max < 0) {
         pe_err("%s for %s must be nonnegative integer, using 0",
                XML_RSC_ATTR_PROMOTED_MAX, rsc->id);
         bundle_data->promoted_max = 0;
     }
 
     value = crm_element_value(xml_obj, "replicas");
     if ((value == NULL) && bundle_data->promoted_max) {
         bundle_data->nreplicas = bundle_data->promoted_max;
     } else {
         bundle_data->nreplicas = crm_parse_int(value, "1");
     }
     if (bundle_data->nreplicas < 1) {
         pe_err("'replicas' for %s must be positive integer, using 1", rsc->id);
         bundle_data->nreplicas = 1;
     }
 
     /*
      * Communication between containers on the same host via the
      * floating IPs only works if the container is started with:
      *   --userland-proxy=false --ip-masq=false
      */
     value = crm_element_value(xml_obj, "replicas-per-host");
     bundle_data->nreplicas_per_host = crm_parse_int(value, "1");
     if (bundle_data->nreplicas_per_host < 1) {
         pe_err("'replicas-per-host' for %s must be positive integer, using 1",
                rsc->id);
         bundle_data->nreplicas_per_host = 1;
     }
     if (bundle_data->nreplicas_per_host == 1) {
         clear_bit(rsc->flags, pe_rsc_unique);
     }
 
     bundle_data->container_command = crm_element_value_copy(xml_obj, "run-command");
     bundle_data->launcher_options = crm_element_value_copy(xml_obj, "options");
     bundle_data->image = crm_element_value_copy(xml_obj, "image");
     bundle_data->container_network = crm_element_value_copy(xml_obj, "network");
 
     xml_obj = first_named_child(rsc->xml, "network");
     if(xml_obj) {
 
         bundle_data->ip_range_start = crm_element_value_copy(xml_obj, "ip-range-start");
         bundle_data->host_netmask = crm_element_value_copy(xml_obj, "host-netmask");
         bundle_data->host_network = crm_element_value_copy(xml_obj, "host-interface");
         bundle_data->control_port = crm_element_value_copy(xml_obj, "control-port");
         value = crm_element_value(xml_obj, "add-host");
         if (crm_str_to_boolean(value, &bundle_data->add_host) != 1) {
             bundle_data->add_host = TRUE;
         }
 
         for (xmlNode *xml_child = __xml_first_child_element(xml_obj); xml_child != NULL;
              xml_child = __xml_next_element(xml_child)) {
 
             pe__bundle_port_t *port = calloc(1, sizeof(pe__bundle_port_t));
             port->source = crm_element_value_copy(xml_child, "port");
 
             if(port->source == NULL) {
                 port->source = crm_element_value_copy(xml_child, "range");
             } else {
                 port->target = crm_element_value_copy(xml_child, "internal-port");
             }
 
             if(port->source != NULL && strlen(port->source) > 0) {
                 if(port->target == NULL) {
                     port->target = strdup(port->source);
                 }
                 bundle_data->ports = g_list_append(bundle_data->ports, port);
 
             } else {
                 pe_err("Invalid port directive %s", ID(xml_child));
                 port_free(port);
             }
         }
     }
 
     xml_obj = first_named_child(rsc->xml, "storage");
     for (xmlNode *xml_child = __xml_first_child_element(xml_obj); xml_child != NULL;
          xml_child = __xml_next_element(xml_child)) {
 
         const char *source = crm_element_value(xml_child, "source-dir");
         const char *target = crm_element_value(xml_child, "target-dir");
         const char *options = crm_element_value(xml_child, "options");
         int flags = pe__bundle_mount_none;
 
         if (source == NULL) {
             source = crm_element_value(xml_child, "source-dir-root");
             set_bit(flags, pe__bundle_mount_subdir);
         }
 
         if (source && target) {
             mount_add(bundle_data, source, target, options, flags);
             if (strcmp(target, "/var/log") == 0) {
                 need_log_mount = FALSE;
             }
         } else {
             pe_err("Invalid mount directive %s", ID(xml_child));
         }
     }
 
     xml_obj = first_named_child(rsc->xml, "primitive");
     if (xml_obj && valid_network(bundle_data)) {
         char *value = NULL;
         xmlNode *xml_set = NULL;
 
         xml_resource = create_xml_node(NULL, XML_CIB_TAG_INCARNATION);
 
         /* @COMPAT We no longer use the <master> tag, but we need to keep it as
          * part of the resource name, so that bundles don't restart in a rolling
          * upgrade. (It also avoids needing to change regression tests.)
          */
         crm_xml_set_id(xml_resource, "%s-%s", bundle_data->prefix,
                       (bundle_data->promoted_max? "master"
                       : (const char *)xml_resource->name));
 
         xml_set = create_xml_node(xml_resource, XML_TAG_META_SETS);
         crm_xml_set_id(xml_set, "%s-%s-meta", bundle_data->prefix, xml_resource->name);
 
         crm_create_nvpair_xml(xml_set, NULL,
                               XML_RSC_ATTR_ORDERED, XML_BOOLEAN_TRUE);
 
         value = crm_itoa(bundle_data->nreplicas);
         crm_create_nvpair_xml(xml_set, NULL,
                               XML_RSC_ATTR_INCARNATION_MAX, value);
         free(value);
 
         value = crm_itoa(bundle_data->nreplicas_per_host);
         crm_create_nvpair_xml(xml_set, NULL,
                               XML_RSC_ATTR_INCARNATION_NODEMAX, value);
         free(value);
 
         crm_create_nvpair_xml(xml_set, NULL, XML_RSC_ATTR_UNIQUE,
                 (bundle_data->nreplicas_per_host > 1)?
                 XML_BOOLEAN_TRUE : XML_BOOLEAN_FALSE);
 
         if (bundle_data->promoted_max) {
             crm_create_nvpair_xml(xml_set, NULL,
                                   XML_RSC_ATTR_PROMOTABLE, XML_BOOLEAN_TRUE);
 
             value = crm_itoa(bundle_data->promoted_max);
             crm_create_nvpair_xml(xml_set, NULL,
                                   XML_RSC_ATTR_PROMOTED_MAX, value);
             free(value);
         }
 
         //crm_xml_add(xml_obj, XML_ATTR_ID, bundle_data->prefix);
         add_node_copy(xml_resource, xml_obj);
 
     } else if(xml_obj) {
         pe_err("Cannot control %s inside %s without either ip-range-start or control-port",
                rsc->id, ID(xml_obj));
         return FALSE;
     }
 
     if(xml_resource) {
         int lpc = 0;
         GListPtr childIter = NULL;
         pe_resource_t *new_rsc = NULL;
         pe__bundle_port_t *port = NULL;
 
         int offset = 0, max = 1024;
         char *buffer = NULL;
 
         if (common_unpack(xml_resource, &new_rsc, rsc, data_set) == FALSE) {
             pe_err("Failed unpacking resource %s", ID(rsc->xml));
             if (new_rsc != NULL && new_rsc->fns != NULL) {
                 new_rsc->fns->free(new_rsc);
             }
             return FALSE;
         }
 
         bundle_data->child = new_rsc;
 
         /* Currently, we always map the default authentication key location
          * into the same location inside the container.
          *
          * Ideally, we would respect the host's PCMK_authkey_location, but:
          * - it may be different on different nodes;
          * - the actual connection will do extra checking to make sure the key
          *   file exists and is readable, that we can't do here on the DC
          * - tools such as crm_resource and crm_simulate may not have the same
          *   environment variables as the cluster, causing operation digests to
          *   differ
          *
          * Always using the default location inside the container is fine,
          * because we control the pacemaker_remote environment, and it avoids
          * having to pass another environment variable to the container.
          *
          * @TODO A better solution may be to have only pacemaker_remote use the
          * environment variable, and have the cluster nodes use a new
          * cluster option for key location. This would introduce the limitation
          * of the location being the same on all cluster nodes, but that's
          * reasonable.
          */
         mount_add(bundle_data, DEFAULT_REMOTE_KEY_LOCATION,
                   DEFAULT_REMOTE_KEY_LOCATION, NULL, pe__bundle_mount_none);
 
         if (need_log_mount) {
             mount_add(bundle_data, CRM_BUNDLE_DIR, "/var/log", NULL,
                       pe__bundle_mount_subdir);
         }
 
         port = calloc(1, sizeof(pe__bundle_port_t));
         if(bundle_data->control_port) {
             port->source = strdup(bundle_data->control_port);
         } else {
             /* If we wanted to respect PCMK_remote_port, we could use
              * crm_default_remote_port() here and elsewhere in this file instead
              * of DEFAULT_REMOTE_PORT.
              *
              * However, it gains nothing, since we control both the container
              * environment and the connection resource parameters, and the user
              * can use a different port if desired by setting control-port.
              */
             port->source = crm_itoa(DEFAULT_REMOTE_PORT);
         }
         port->target = strdup(port->source);
         bundle_data->ports = g_list_append(bundle_data->ports, port);
 
         buffer = calloc(1, max+1);
         for (childIter = bundle_data->child->children; childIter != NULL;
              childIter = childIter->next) {
 
             pe__bundle_replica_t *replica = calloc(1, sizeof(pe__bundle_replica_t));
 
             replica->child = childIter->data;
             replica->child->exclusive_discover = TRUE;
             replica->offset = lpc++;
 
             // Ensure the child's notify gets set based on the underlying primitive's value
             if (is_set(replica->child->flags, pe_rsc_notify)) {
                 set_bit(bundle_data->child->flags, pe_rsc_notify);
             }
 
             offset += allocate_ip(bundle_data, replica, buffer+offset,
                                   max-offset);
             bundle_data->replicas = g_list_append(bundle_data->replicas,
                                                   replica);
             bundle_data->attribute_target = g_hash_table_lookup(replica->child->meta,
                                                                 XML_RSC_ATTR_TARGET);
         }
         bundle_data->container_host_options = buffer;
         if (bundle_data->attribute_target) {
             g_hash_table_replace(rsc->meta, strdup(XML_RSC_ATTR_TARGET),
                                  strdup(bundle_data->attribute_target));
             g_hash_table_replace(bundle_data->child->meta,
                                  strdup(XML_RSC_ATTR_TARGET),
                                  strdup(bundle_data->attribute_target));
         }
 
     } else {
         // Just a naked container, no pacemaker-remote
         int offset = 0, max = 1024;
         char *buffer = calloc(1, max+1);
 
         for (int lpc = 0; lpc < bundle_data->nreplicas; lpc++) {
             pe__bundle_replica_t *replica = calloc(1, sizeof(pe__bundle_replica_t));
 
             replica->offset = lpc;
             offset += allocate_ip(bundle_data, replica, buffer+offset,
                                   max-offset);
             bundle_data->replicas = g_list_append(bundle_data->replicas,
                                                   replica);
         }
         bundle_data->container_host_options = buffer;
     }
 
     for (GList *gIter = bundle_data->replicas; gIter != NULL;
          gIter = gIter->next) {
         pe__bundle_replica_t *replica = gIter->data;
 
         if (!create_container(rsc, bundle_data, replica, data_set)) {
             pe_err("Failed unpacking resource %s", rsc->id);
             rsc->fns->free(rsc);
             return FALSE;
         }
     }
 
     if (bundle_data->child) {
         rsc->children = g_list_append(rsc->children, bundle_data->child);
     }
     return TRUE;
 }
 
 static int
 replica_resource_active(pe_resource_t *rsc, gboolean all)
 {
     if (rsc) {
         gboolean child_active = rsc->fns->active(rsc, all);
 
         if (child_active && !all) {
             return TRUE;
         } else if (!child_active && all) {
             return FALSE;
         }
     }
     return -1;
 }
 
 gboolean
 pe__bundle_active(pe_resource_t *rsc, gboolean all)
 {
     pe__bundle_variant_data_t *bundle_data = NULL;
     GListPtr iter = NULL;
 
     get_bundle_variant_data(bundle_data, rsc);
     for (iter = bundle_data->replicas; iter != NULL; iter = iter->next) {
         pe__bundle_replica_t *replica = iter->data;
         int rsc_active;
 
         rsc_active = replica_resource_active(replica->ip, all);
         if (rsc_active >= 0) {
             return (gboolean) rsc_active;
         }
 
         rsc_active = replica_resource_active(replica->child, all);
         if (rsc_active >= 0) {
             return (gboolean) rsc_active;
         }
 
         rsc_active = replica_resource_active(replica->container, all);
         if (rsc_active >= 0) {
             return (gboolean) rsc_active;
         }
 
         rsc_active = replica_resource_active(replica->remote, all);
         if (rsc_active >= 0) {
             return (gboolean) rsc_active;
         }
     }
 
     /* If "all" is TRUE, we've already checked that no resources were inactive,
      * so return TRUE; if "all" is FALSE, we didn't find any active resources,
      * so return FALSE.
      */
     return all;
 }
 
 /*!
  * \internal
  * \brief Find the bundle replica corresponding to a given node
  *
  * \param[in] bundle  Top-level bundle resource
  * \param[in] node    Node to search for
  *
  * \return Bundle replica if found, NULL otherwise
  */
 pe_resource_t *
 pe__find_bundle_replica(const pe_resource_t *bundle, const pe_node_t *node)
 {
     pe__bundle_variant_data_t *bundle_data = NULL;
     CRM_ASSERT(bundle && node);
 
     get_bundle_variant_data(bundle_data, bundle);
     for (GList *gIter = bundle_data->replicas; gIter != NULL;
          gIter = gIter->next) {
         pe__bundle_replica_t *replica = gIter->data;
 
         CRM_ASSERT(replica && replica->node);
         if (replica->node->details == node->details) {
             return replica->child;
         }
     }
     return NULL;
 }
 
 static void
 print_rsc_in_list(pe_resource_t *rsc, const char *pre_text, long options,
                   void *print_data)
 {
     if (rsc != NULL) {
         if (options & pe_print_html) {
             status_print("<li>");
         }
         rsc->fns->print(rsc, pre_text, options, print_data);
         if (options & pe_print_html) {
             status_print("</li>\n");
         }
     }
 }
 
 static const char*
 container_agent_str(enum pe__container_agent t)
 {
     switch (t) {
         case PE__CONTAINER_AGENT_DOCKER: return PE__CONTAINER_AGENT_DOCKER_S;
         case PE__CONTAINER_AGENT_RKT:    return PE__CONTAINER_AGENT_RKT_S;
         case PE__CONTAINER_AGENT_PODMAN: return PE__CONTAINER_AGENT_PODMAN_S;
         default: // PE__CONTAINER_AGENT_UNKNOWN
             break;
     }
     return PE__CONTAINER_AGENT_UNKNOWN_S;
 }
 
 static void
 bundle_print_xml(pe_resource_t *rsc, const char *pre_text, long options,
                  void *print_data)
 {
     pe__bundle_variant_data_t *bundle_data = NULL;
     char *child_text = NULL;
     CRM_CHECK(rsc != NULL, return);
 
     if (pre_text == NULL) {
         pre_text = "";
     }
     child_text = crm_strdup_printf("%s        ", pre_text);
 
     get_bundle_variant_data(bundle_data, rsc);
 
     status_print("%s<bundle ", pre_text);
     status_print("id=\"%s\" ", rsc->id);
     status_print("type=\"%s\" ", container_agent_str(bundle_data->agent_type));
     status_print("image=\"%s\" ", bundle_data->image);
     status_print("unique=\"%s\" ", is_set(rsc->flags, pe_rsc_unique)? "true" : "false");
     status_print("managed=\"%s\" ", is_set(rsc->flags, pe_rsc_managed) ? "true" : "false");
     status_print("failed=\"%s\" ", is_set(rsc->flags, pe_rsc_failed) ? "true" : "false");
     status_print(">\n");
 
     for (GList *gIter = bundle_data->replicas; gIter != NULL;
          gIter = gIter->next) {
         pe__bundle_replica_t *replica = gIter->data;
 
         CRM_ASSERT(replica);
         status_print("%s    <replica id=\"%d\">\n", pre_text, replica->offset);
         print_rsc_in_list(replica->ip, child_text, options, print_data);
         print_rsc_in_list(replica->child, child_text, options, print_data);
         print_rsc_in_list(replica->container, child_text, options, print_data);
         print_rsc_in_list(replica->remote, child_text, options, print_data);
         status_print("%s    </replica>\n", pre_text);
     }
     status_print("%s</bundle>\n", pre_text);
     free(child_text);
 }
 
 PCMK__OUTPUT_ARGS("bundle", "unsigned int", "struct pe_resource_t *", "GListPtr")
 int
 pe__bundle_xml(pcmk__output_t *out, va_list args)
 {
     unsigned int options = va_arg(args, unsigned int);
     pe_resource_t *rsc = va_arg(args, pe_resource_t *);
     GListPtr only_show = va_arg(args, GListPtr);
 
     pe__bundle_variant_data_t *bundle_data = NULL;
     int rc = pcmk_rc_no_output;
     gboolean printed_header = FALSE;
 
     CRM_ASSERT(rsc != NULL);
 
     get_bundle_variant_data(bundle_data, rsc);
 
     for (GList *gIter = bundle_data->replicas; gIter != NULL;
          gIter = gIter->next) {
         pe__bundle_replica_t *replica = gIter->data;
         char *id = crm_itoa(replica->offset);
 
         CRM_ASSERT(replica);
 
-        if (!pe__rsc_running_on_any_node_in_list(replica->container, only_show)) {
+        if (pcmk__rsc_is_filtered(replica->container, only_show)) {
             continue;
         }
 
         if (!printed_header) {
             printed_header = TRUE;
 
             rc = pe__name_and_nvpairs_xml(out, true, "bundle", 6
                          , "id", rsc->id
                          , "type", container_agent_str(bundle_data->agent_type)
                          , "image", bundle_data->image
                          , "unique", BOOL2STR(is_set(rsc->flags, pe_rsc_unique))
                          , "managed", BOOL2STR(is_set(rsc->flags, pe_rsc_managed))
                          , "failed", BOOL2STR(is_set(rsc->flags, pe_rsc_failed)));
             CRM_ASSERT(rc == pcmk_rc_ok);
         }
 
         rc = pe__name_and_nvpairs_xml(out, true, "replica", 1, "id", id);
         free(id);
         CRM_ASSERT(rc == pcmk_rc_ok);
 
         if (replica->ip != NULL) {
             out->message(out, crm_map_element_name(replica->ip->xml), options, replica->ip, only_show);
         }
 
         if (replica->child != NULL) {
             out->message(out, crm_map_element_name(replica->child->xml), options, replica->child, only_show);
         }
 
         out->message(out, crm_map_element_name(replica->container->xml), options, replica->container, only_show);
 
         if (replica->remote != NULL) {
             out->message(out, crm_map_element_name(replica->remote->xml), options, replica->remote, only_show);
         }
 
         pcmk__output_xml_pop_parent(out); // replica
     }
 
     if (printed_header) {
         pcmk__output_xml_pop_parent(out); // bundle
     }
 
     return rc;
 }
 
 static void
 pe__bundle_replica_output_html(pcmk__output_t *out, pe__bundle_replica_t *replica,
                                long options)
 {
     pe_node_t *node = NULL;
     pe_resource_t *rsc = replica->child;
 
     int offset = 0;
     char buffer[LINE_MAX];
 
     if(rsc == NULL) {
         rsc = replica->container;
     }
 
     if (replica->remote) {
         offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
                            rsc_printable_id(replica->remote));
     } else {
         offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
                            rsc_printable_id(replica->container));
     }
     if (replica->ipaddr) {
         offset += snprintf(buffer + offset, LINE_MAX - offset, " (%s)",
                            replica->ipaddr);
     }
 
     pe__common_output_html(out, rsc, buffer, node, options);
 }
 
 PCMK__OUTPUT_ARGS("bundle", "unsigned int", "struct pe_resource_t *", "GListPtr")
 int
 pe__bundle_html(pcmk__output_t *out, va_list args)
 {
     unsigned int options = va_arg(args, unsigned int);
     pe_resource_t *rsc = va_arg(args, pe_resource_t *);
     GListPtr only_show = va_arg(args, GListPtr);
 
     pe__bundle_variant_data_t *bundle_data = NULL;
     char buffer[LINE_MAX];
 
     CRM_ASSERT(rsc != NULL);
 
     get_bundle_variant_data(bundle_data, rsc);
 
     pcmk__output_create_xml_node(out, "br");
     out->begin_list(out, NULL, NULL, "Container bundle%s: %s [%s]%s%s",
                     (bundle_data->nreplicas > 1)? " set" : "",
                     rsc->id, bundle_data->image,
                     is_set(rsc->flags, pe_rsc_unique) ? " (unique)" : "",
                     is_set(rsc->flags, pe_rsc_managed) ? "" : " (unmanaged)");
 
     for (GList *gIter = bundle_data->replicas; gIter != NULL;
          gIter = gIter->next) {
         pe__bundle_replica_t *replica = gIter->data;
 
         CRM_ASSERT(replica);
 
-        if (!pe__rsc_running_on_any_node_in_list(replica->container, only_show)) {
+        if (pcmk__rsc_is_filtered(replica->container, only_show)) {
             continue;
         }
 
         pcmk__output_xml_create_parent(out, "li");
         if (is_set(options, pe_print_implicit)) {
             if (pcmk__list_of_multiple(bundle_data->replicas)) {
                 snprintf(buffer, LINE_MAX, " Replica[%d]", replica->offset);
                 xmlNodeSetContent(pcmk__output_xml_peek_parent(out), (pcmkXmlStr) buffer);
             }
             pcmk__output_create_xml_node(out, "br");
             out->begin_list(out, NULL, NULL, NULL);
 
             if (replica->ip != NULL) {
                 out->message(out, crm_map_element_name(replica->ip->xml), options, replica->ip, only_show);
             }
 
             if (replica->child != NULL) {
                 out->message(out, crm_map_element_name(replica->child->xml), options, replica->child, only_show);
             }
 
             out->message(out, crm_map_element_name(replica->container->xml), options, replica->container, only_show);
 
             if (replica->remote != NULL) {
                 out->message(out, crm_map_element_name(replica->remote->xml), options, replica->remote, only_show);
             }
 
             out->end_list(out);
         } else {
             pe__bundle_replica_output_html(out, replica, options);
         }
 
         pcmk__output_xml_pop_parent(out);
     }
 
     out->end_list(out);
     return pcmk_rc_ok;
 }
 
 static void
 pe__bundle_replica_output_text(pcmk__output_t *out, pe__bundle_replica_t *replica,
                                long options)
 {
     pe_node_t *node = NULL;
     pe_resource_t *rsc = replica->child;
 
     int offset = 0;
     char buffer[LINE_MAX];
 
     if(rsc == NULL) {
         rsc = replica->container;
     }
 
     if (replica->remote) {
         offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
                            rsc_printable_id(replica->remote));
     } else {
         offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
                            rsc_printable_id(replica->container));
     }
     if (replica->ipaddr) {
         offset += snprintf(buffer + offset, LINE_MAX - offset, " (%s)",
                            replica->ipaddr);
     }
 
     pe__common_output_text(out, rsc, buffer, node, options);
 }
 
 PCMK__OUTPUT_ARGS("bundle", "unsigned int", "struct pe_resource_t *", "GListPtr")
 int
 pe__bundle_text(pcmk__output_t *out, va_list args)
 {
     unsigned int options = va_arg(args, unsigned int);
     pe_resource_t *rsc = va_arg(args, pe_resource_t *);
     GListPtr only_show = va_arg(args, GListPtr);
 
     pe__bundle_variant_data_t *bundle_data = NULL;
 
     CRM_ASSERT(rsc != NULL);
 
     get_bundle_variant_data(bundle_data, rsc);
 
     out->begin_list(out, NULL, NULL, "Container bundle%s: %s [%s]%s%s",
                     (bundle_data->nreplicas > 1)? " set" : "",
                     rsc->id, bundle_data->image,
                     is_set(rsc->flags, pe_rsc_unique) ? " (unique)" : "",
                     is_set(rsc->flags, pe_rsc_managed) ? "" : " (unmanaged)");
 
     for (GList *gIter = bundle_data->replicas; gIter != NULL;
          gIter = gIter->next) {
         pe__bundle_replica_t *replica = gIter->data;
 
         CRM_ASSERT(replica);
 
-        if (!pe__rsc_running_on_any_node_in_list(replica->container, only_show)) {
+        if (pcmk__rsc_is_filtered(replica->container, only_show)) {
             continue;
         }
 
         if (is_set(options, pe_print_implicit)) {
             if (pcmk__list_of_multiple(bundle_data->replicas)) {
                 out->list_item(out, NULL, "Replica[%d]", replica->offset);
             }
 
             out->begin_list(out, NULL, NULL, NULL);
 
             if (replica->ip != NULL) {
                 out->message(out, crm_map_element_name(replica->ip->xml), options, replica->ip, only_show);
             }
 
             if (replica->child != NULL) {
                 out->message(out, crm_map_element_name(replica->child->xml), options, replica->child, only_show);
             }
 
             out->message(out, crm_map_element_name(replica->container->xml), options, replica->container, only_show);
 
             if (replica->remote != NULL) {
                 out->message(out, crm_map_element_name(replica->remote->xml), options, replica->remote, only_show);
             }
 
             out->end_list(out);
         } else {
             pe__bundle_replica_output_text(out, replica, options);
         }
     }
 
     out->end_list(out);
     return pcmk_rc_ok;
 }
 
 static void
 print_bundle_replica(pe__bundle_replica_t *replica, const char *pre_text,
                      long options, void *print_data)
 {
     pe_node_t *node = NULL;
     pe_resource_t *rsc = replica->child;
 
     int offset = 0;
     char buffer[LINE_MAX];
 
     if(rsc == NULL) {
         rsc = replica->container;
     }
 
     if (replica->remote) {
         offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
                            rsc_printable_id(replica->remote));
     } else {
         offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
                            rsc_printable_id(replica->container));
     }
     if (replica->ipaddr) {
         offset += snprintf(buffer + offset, LINE_MAX - offset, " (%s)",
                            replica->ipaddr);
     }
 
     node = pe__current_node(replica->container);
     common_print(rsc, pre_text, buffer, node, options, print_data);
 }
 
 void
 pe__print_bundle(pe_resource_t *rsc, const char *pre_text, long options,
                  void *print_data)
 {
     pe__bundle_variant_data_t *bundle_data = NULL;
     char *child_text = NULL;
     CRM_CHECK(rsc != NULL, return);
 
     if (options & pe_print_xml) {
         bundle_print_xml(rsc, pre_text, options, print_data);
         return;
     }
 
     get_bundle_variant_data(bundle_data, rsc);
 
     if (pre_text == NULL) {
         pre_text = " ";
     }
 
     status_print("%sContainer bundle%s: %s [%s]%s%s\n",
                  pre_text, ((bundle_data->nreplicas > 1)? " set" : ""),
                  rsc->id, bundle_data->image,
                  is_set(rsc->flags, pe_rsc_unique) ? " (unique)" : "",
                  is_set(rsc->flags, pe_rsc_managed) ? "" : " (unmanaged)");
     if (options & pe_print_html) {
         status_print("<br />\n<ul>\n");
     }
 
 
     for (GList *gIter = bundle_data->replicas; gIter != NULL;
          gIter = gIter->next) {
         pe__bundle_replica_t *replica = gIter->data;
 
         CRM_ASSERT(replica);
         if (options & pe_print_html) {
             status_print("<li>");
         }
 
         if (is_set(options, pe_print_implicit)) {
             child_text = crm_strdup_printf("     %s", pre_text);
             if (pcmk__list_of_multiple(bundle_data->replicas)) {
                 status_print("  %sReplica[%d]\n", pre_text, replica->offset);
             }
             if (options & pe_print_html) {
                 status_print("<br />\n<ul>\n");
             }
             print_rsc_in_list(replica->ip, child_text, options, print_data);
             print_rsc_in_list(replica->container, child_text, options, print_data);
             print_rsc_in_list(replica->remote, child_text, options, print_data);
             print_rsc_in_list(replica->child, child_text, options, print_data);
             if (options & pe_print_html) {
                 status_print("</ul>\n");
             }
         } else {
             child_text = crm_strdup_printf("%s  ", pre_text);
             print_bundle_replica(replica, child_text, options, print_data);
         }
         free(child_text);
 
         if (options & pe_print_html) {
             status_print("</li>\n");
         }
     }
     if (options & pe_print_html) {
         status_print("</ul>\n");
     }
 }
 
 static void
 free_bundle_replica(pe__bundle_replica_t *replica)
 {
     if (replica == NULL) {
         return;
     }
 
     if (replica->node) {
         free(replica->node);
         replica->node = NULL;
     }
 
     if (replica->ip) {
         free_xml(replica->ip->xml);
         replica->ip->xml = NULL;
         replica->ip->fns->free(replica->ip);
         replica->ip = NULL;
     }
     if (replica->container) {
         free_xml(replica->container->xml);
         replica->container->xml = NULL;
         replica->container->fns->free(replica->container);
         replica->container = NULL;
     }
     if (replica->remote) {
         free_xml(replica->remote->xml);
         replica->remote->xml = NULL;
         replica->remote->fns->free(replica->remote);
         replica->remote = NULL;
     }
     free(replica->ipaddr);
     free(replica);
 }
 
 void
 pe__free_bundle(pe_resource_t *rsc)
 {
     pe__bundle_variant_data_t *bundle_data = NULL;
     CRM_CHECK(rsc != NULL, return);
 
     get_bundle_variant_data(bundle_data, rsc);
     pe_rsc_trace(rsc, "Freeing %s", rsc->id);
 
     free(bundle_data->prefix);
     free(bundle_data->image);
     free(bundle_data->control_port);
     free(bundle_data->host_network);
     free(bundle_data->host_netmask);
     free(bundle_data->ip_range_start);
     free(bundle_data->container_network);
     free(bundle_data->launcher_options);
     free(bundle_data->container_command);
     free(bundle_data->container_host_options);
 
     g_list_free_full(bundle_data->replicas,
                      (GDestroyNotify) free_bundle_replica);
     g_list_free_full(bundle_data->mounts, (GDestroyNotify)mount_free);
     g_list_free_full(bundle_data->ports, (GDestroyNotify)port_free);
     g_list_free(rsc->children);
 
     if(bundle_data->child) {
         free_xml(bundle_data->child->xml);
         bundle_data->child->xml = NULL;
         bundle_data->child->fns->free(bundle_data->child);
     }
     common_free(rsc);
 }
 
 enum rsc_role_e
 pe__bundle_resource_state(const pe_resource_t *rsc, gboolean current)
 {
     enum rsc_role_e container_role = RSC_ROLE_UNKNOWN;
     return container_role;
 }
 
 /*!
  * \brief Get the number of configured replicas in a bundle
  *
  * \param[in] rsc  Bundle resource
  *
  * \return Number of configured replicas, or 0 on error
  */
 int
 pe_bundle_replicas(const pe_resource_t *rsc)
 {
     if ((rsc == NULL) || (rsc->variant != pe_container)) {
         return 0;
     } else {
         pe__bundle_variant_data_t *bundle_data = NULL;
 
         get_bundle_variant_data(bundle_data, rsc);
         return bundle_data->nreplicas;
     }
 }
 
 void
 pe__count_bundle(pe_resource_t *rsc)
 {
     pe__bundle_variant_data_t *bundle_data = NULL;
 
     get_bundle_variant_data(bundle_data, rsc);
     for (GList *item = bundle_data->replicas; item != NULL; item = item->next) {
         pe__bundle_replica_t *replica = item->data;
 
         if (replica->ip) {
             replica->ip->fns->count(replica->ip);
         }
         if (replica->child) {
             replica->child->fns->count(replica->child);
         }
         if (replica->container) {
             replica->container->fns->count(replica->container);
         }
         if (replica->remote) {
             replica->remote->fns->count(replica->remote);
         }
     }
 }
diff --git a/lib/pengine/clone.c b/lib/pengine/clone.c
index 8292d9087a..83c58288fa 100644
--- a/lib/pengine/clone.c
+++ b/lib/pengine/clone.c
@@ -1,1107 +1,1107 @@
 /*
  * Copyright 2004-2020 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU Lesser General Public License
  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #include <crm/pengine/rules.h>
 #include <crm/pengine/status.h>
 #include <crm/pengine/internal.h>
 #include <pe_status_private.h>
 #include <crm/msg_xml.h>
 
 #define VARIANT_CLONE 1
 #include "./variant.h"
 
 void
 pe__force_anon(const char *standard, pe_resource_t *rsc, const char *rid,
                pe_working_set_t *data_set)
 {
     if (pe_rsc_is_clone(rsc)) {
         clone_variant_data_t *clone_data = NULL;
 
         get_clone_variant_data(clone_data, rsc);
 
         pe_warn("Ignoring " XML_RSC_ATTR_UNIQUE " for %s because %s resources "
                 "such as %s can be used only as anonymous clones",
                 rsc->id, standard, rid);
 
         clone_data->clone_node_max = 1;
         clone_data->clone_max = QB_MIN(clone_data->clone_max,
                                        g_list_length(data_set->nodes));
     }
 }
 
 pe_resource_t *
 find_clone_instance(pe_resource_t * rsc, const char *sub_id, pe_working_set_t * data_set)
 {
     char *child_id = NULL;
     pe_resource_t *child = NULL;
     const char *child_base = NULL;
     clone_variant_data_t *clone_data = NULL;
 
     get_clone_variant_data(clone_data, rsc);
 
     child_base = ID(clone_data->xml_obj_child);
     child_id = crm_strdup_printf("%s:%s", child_base, sub_id);
     child = pe_find_resource(rsc->children, child_id);
 
     free(child_id);
     return child;
 }
 
 pe_resource_t *
 pe__create_clone_child(pe_resource_t *rsc, pe_working_set_t *data_set)
 {
     gboolean as_orphan = FALSE;
     char *inc_num = NULL;
     char *inc_max = NULL;
     pe_resource_t *child_rsc = NULL;
     xmlNode *child_copy = NULL;
     clone_variant_data_t *clone_data = NULL;
 
     get_clone_variant_data(clone_data, rsc);
 
     CRM_CHECK(clone_data->xml_obj_child != NULL, return FALSE);
 
     if (clone_data->total_clones >= clone_data->clone_max) {
         // If we've already used all available instances, this is an orphan
         as_orphan = TRUE;
     }
 
     // Allocate instance numbers in numerical order (starting at 0)
     inc_num = crm_itoa(clone_data->total_clones);
     inc_max = crm_itoa(clone_data->clone_max);
 
     child_copy = copy_xml(clone_data->xml_obj_child);
 
     crm_xml_add(child_copy, XML_RSC_ATTR_INCARNATION, inc_num);
 
     if (common_unpack(child_copy, &child_rsc, rsc, data_set) == FALSE) {
         pe_err("Failed unpacking resource %s", crm_element_value(child_copy, XML_ATTR_ID));
         child_rsc = NULL;
         goto bail;
     }
 /*  child_rsc->globally_unique = rsc->globally_unique; */
 
     CRM_ASSERT(child_rsc);
     clone_data->total_clones += 1;
     pe_rsc_trace(child_rsc, "Setting clone attributes for: %s", child_rsc->id);
     rsc->children = g_list_append(rsc->children, child_rsc);
     if (as_orphan) {
         set_bit_recursive(child_rsc, pe_rsc_orphan);
     }
 
     add_hash_param(child_rsc->meta, XML_RSC_ATTR_INCARNATION_MAX, inc_max);
     pe_rsc_trace(rsc, "Added %s instance %s", rsc->id, child_rsc->id);
 
   bail:
     free(inc_num);
     free(inc_max);
 
     return child_rsc;
 }
 
 gboolean
 clone_unpack(pe_resource_t * rsc, pe_working_set_t * data_set)
 {
     int lpc = 0;
     xmlNode *a_child = NULL;
     xmlNode *xml_obj = rsc->xml;
     clone_variant_data_t *clone_data = NULL;
 
     const char *ordered = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_ORDERED);
     const char *max_clones = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_INCARNATION_MAX);
     const char *max_clones_node = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_INCARNATION_NODEMAX);
 
     pe_rsc_trace(rsc, "Processing resource %s...", rsc->id);
 
     clone_data = calloc(1, sizeof(clone_variant_data_t));
     rsc->variant_opaque = clone_data;
 
     if (is_set(rsc->flags, pe_rsc_promotable)) {
         const char *promoted_max = NULL;
         const char *promoted_node_max = NULL;
 
         promoted_max = g_hash_table_lookup(rsc->meta,
                                            XML_RSC_ATTR_PROMOTED_MAX);
         if (promoted_max == NULL) {
             // @COMPAT deprecated since 2.0.0
             promoted_max = g_hash_table_lookup(rsc->meta,
                                                XML_RSC_ATTR_MASTER_MAX);
         }
 
         promoted_node_max = g_hash_table_lookup(rsc->meta,
                                                 XML_RSC_ATTR_PROMOTED_NODEMAX);
         if (promoted_node_max == NULL) {
             // @COMPAT deprecated since 2.0.0
             promoted_node_max = g_hash_table_lookup(rsc->meta,
                                                     XML_RSC_ATTR_MASTER_NODEMAX);
         }
 
         clone_data->promoted_max = crm_parse_int(promoted_max, "1");
         clone_data->promoted_node_max = crm_parse_int(promoted_node_max, "1");
     }
 
     // Implied by calloc()
     /* clone_data->xml_obj_child = NULL; */
 
     clone_data->clone_node_max = crm_parse_int(max_clones_node, "1");
 
     if (max_clones) {
         clone_data->clone_max = crm_parse_int(max_clones, "1");
 
     } else if (pcmk__list_of_multiple(data_set->nodes)) {
         clone_data->clone_max = g_list_length(data_set->nodes);
 
     } else {
         clone_data->clone_max = 1;      /* Handy during crm_verify */
     }
 
     clone_data->ordered = crm_is_true(ordered);
 
     if ((rsc->flags & pe_rsc_unique) == 0 && clone_data->clone_node_max > 1) {
         pcmk__config_err("Ignoring " XML_RSC_ATTR_PROMOTED_MAX " for %s "
                          "because anonymous clones support only one instance "
                          "per node", rsc->id);
         clone_data->clone_node_max = 1;
     }
 
     pe_rsc_trace(rsc, "Options for %s", rsc->id);
     pe_rsc_trace(rsc, "\tClone max: %d", clone_data->clone_max);
     pe_rsc_trace(rsc, "\tClone node max: %d", clone_data->clone_node_max);
     pe_rsc_trace(rsc, "\tClone is unique: %s",
                  is_set(rsc->flags, pe_rsc_unique) ? "true" : "false");
     pe_rsc_trace(rsc, "\tClone is promotable: %s",
                  is_set(rsc->flags, pe_rsc_promotable) ? "true" : "false");
 
     // Clones may contain a single group or primitive
     for (a_child = __xml_first_child_element(xml_obj); a_child != NULL;
          a_child = __xml_next_element(a_child)) {
 
         if (crm_str_eq((const char *)a_child->name, XML_CIB_TAG_RESOURCE, TRUE)
         || crm_str_eq((const char *)a_child->name, XML_CIB_TAG_GROUP, TRUE)) {
             clone_data->xml_obj_child = a_child;
             break;
         }
     }
 
     if (clone_data->xml_obj_child == NULL) {
         pcmk__config_err("%s has nothing to clone", rsc->id);
         return FALSE;
     }
 
     /*
      * Make clones ever so slightly sticky by default
      *
      * This helps ensure clone instances are not shuffled around the cluster
      * for no benefit in situations when pre-allocation is not appropriate
      */
     if (g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_STICKINESS) == NULL) {
         add_hash_param(rsc->meta, XML_RSC_ATTR_STICKINESS, "1");
     }
 
     /* This ensures that the globally-unique value always exists for children to
      * inherit when being unpacked, as well as in resource agents' environment.
      */
     add_hash_param(rsc->meta, XML_RSC_ATTR_UNIQUE,
                    is_set(rsc->flags, pe_rsc_unique) ? XML_BOOLEAN_TRUE : XML_BOOLEAN_FALSE);
 
     if (clone_data->clone_max <= 0) {
         /* Create one child instance so that unpack_find_resource() will hook up
          * any orphans up to the parent correctly.
          */
         if (pe__create_clone_child(rsc, data_set) == NULL) {
             return FALSE;
         }
 
     } else {
         // Create a child instance for each available instance number
         for (lpc = 0; lpc < clone_data->clone_max; lpc++) {
             if (pe__create_clone_child(rsc, data_set) == NULL) {
                 return FALSE;
             }
         }
     }
 
     pe_rsc_trace(rsc, "Added %d children to resource %s...", clone_data->clone_max, rsc->id);
     return TRUE;
 }
 
 gboolean
 clone_active(pe_resource_t * rsc, gboolean all)
 {
     GListPtr gIter = rsc->children;
 
     for (; gIter != NULL; gIter = gIter->next) {
         pe_resource_t *child_rsc = (pe_resource_t *) gIter->data;
         gboolean child_active = child_rsc->fns->active(child_rsc, all);
 
         if (all == FALSE && child_active) {
             return TRUE;
         } else if (all && child_active == FALSE) {
             return FALSE;
         }
     }
 
     if (all) {
         return TRUE;
     } else {
         return FALSE;
     }
 }
 
 static void
 short_print(char *list, const char *prefix, const char *type, const char *suffix, long options, void *print_data)
 {
     if(suffix == NULL) {
         suffix = "";
     }
 
     if (list) {
         if (options & pe_print_html) {
             status_print("<li>");
         }
         status_print("%s%s: [%s ]%s", prefix, type, list, suffix);
 
         if (options & pe_print_html) {
             status_print("</li>\n");
 
         } else if (options & pe_print_suppres_nl) {
             /* nothing */
         } else if ((options & pe_print_printf) || (options & pe_print_ncurses)) {
             status_print("\n");
         }
 
     }
 }
 
 static const char *
 configured_role_str(pe_resource_t * rsc)
 {
     const char *target_role = g_hash_table_lookup(rsc->meta,
                                                   XML_RSC_ATTR_TARGET_ROLE);
 
     if ((target_role == NULL) && rsc->children && rsc->children->data) {
         target_role = g_hash_table_lookup(((pe_resource_t*)rsc->children->data)->meta,
                                           XML_RSC_ATTR_TARGET_ROLE);
     }
     return target_role;
 }
 
 static enum rsc_role_e
 configured_role(pe_resource_t * rsc)
 {
     const char *target_role = configured_role_str(rsc);
 
     if (target_role) {
         return text2role(target_role);
     }
     return RSC_ROLE_UNKNOWN;
 }
 
 static void
 clone_print_xml(pe_resource_t * rsc, const char *pre_text, long options, void *print_data)
 {
     char *child_text = crm_strdup_printf("%s    ", pre_text);
     const char *target_role = configured_role_str(rsc);
     GListPtr gIter = rsc->children;
 
     status_print("%s<clone ", pre_text);
     status_print("id=\"%s\" ", rsc->id);
     status_print("multi_state=\"%s\" ", is_set(rsc->flags, pe_rsc_promotable)? "true" : "false");
     status_print("unique=\"%s\" ", is_set(rsc->flags, pe_rsc_unique) ? "true" : "false");
     status_print("managed=\"%s\" ", is_set(rsc->flags, pe_rsc_managed) ? "true" : "false");
     status_print("failed=\"%s\" ", is_set(rsc->flags, pe_rsc_failed) ? "true" : "false");
     status_print("failure_ignored=\"%s\" ",
                  is_set(rsc->flags, pe_rsc_failure_ignored) ? "true" : "false");
     if (target_role) {
         status_print("target_role=\"%s\" ", target_role);
     }
     status_print(">\n");
 
     for (; gIter != NULL; gIter = gIter->next) {
         pe_resource_t *child_rsc = (pe_resource_t *) gIter->data;
 
         child_rsc->fns->print(child_rsc, child_text, options, print_data);
     }
 
     status_print("%s</clone>\n", pre_text);
     free(child_text);
 }
 
 bool is_set_recursive(pe_resource_t * rsc, long long flag, bool any)
 {
     GListPtr gIter;
     bool all = !any;
 
     if(is_set(rsc->flags, flag)) {
         if(any) {
             return TRUE;
         }
     } else if(all) {
         return FALSE;
     }
 
     for (gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
         if(is_set_recursive(gIter->data, flag, any)) {
             if(any) {
                 return TRUE;
             }
 
         } else if(all) {
             return FALSE;
         }
     }
 
     if(all) {
         return TRUE;
     }
     return FALSE;
 }
 
 void
 clone_print(pe_resource_t * rsc, const char *pre_text, long options, void *print_data)
 {
     char *list_text = NULL;
     char *child_text = NULL;
     char *stopped_list = NULL;
 
     GListPtr master_list = NULL;
     GListPtr started_list = NULL;
     GListPtr gIter = rsc->children;
 
     clone_variant_data_t *clone_data = NULL;
     int active_instances = 0;
 
     if (pre_text == NULL) {
         pre_text = " ";
     }
 
     if (options & pe_print_xml) {
         clone_print_xml(rsc, pre_text, options, print_data);
         return;
     }
 
     get_clone_variant_data(clone_data, rsc);
 
     child_text = crm_strdup_printf("%s    ", pre_text);
 
     status_print("%sClone Set: %s [%s]%s%s%s",
                  pre_text ? pre_text : "", rsc->id, ID(clone_data->xml_obj_child),
                  is_set(rsc->flags, pe_rsc_promotable) ? " (promotable)" : "",
                  is_set(rsc->flags, pe_rsc_unique) ? " (unique)" : "",
                  is_set(rsc->flags, pe_rsc_managed) ? "" : " (unmanaged)");
 
     if (options & pe_print_html) {
         status_print("\n<ul>\n");
 
     } else if ((options & pe_print_log) == 0) {
         status_print("\n");
     }
 
     for (; gIter != NULL; gIter = gIter->next) {
         gboolean print_full = FALSE;
         pe_resource_t *child_rsc = (pe_resource_t *) gIter->data;
         gboolean partially_active = child_rsc->fns->active(child_rsc, FALSE);
 
         if (options & pe_print_clone_details) {
             print_full = TRUE;
         }
 
         if (is_set(rsc->flags, pe_rsc_unique)) {
             // Print individual instance when unique (except stopped orphans)
             if (partially_active || is_not_set(rsc->flags, pe_rsc_orphan)) {
                 print_full = TRUE;
             }
 
         // Everything else in this block is for anonymous clones
 
         } else if (is_set(options, pe_print_pending)
                    && (child_rsc->pending_task != NULL)
                    && strcmp(child_rsc->pending_task, "probe")) {
             // Print individual instance when non-probe action is pending
             print_full = TRUE;
 
         } else if (partially_active == FALSE) {
             // List stopped instances when requested (except orphans)
             if (is_not_set(child_rsc->flags, pe_rsc_orphan)
                 && is_not_set(options, pe_print_clone_active)) {
                 stopped_list = pcmk__add_word(stopped_list, child_rsc->id);
             }
 
         } else if (is_set_recursive(child_rsc, pe_rsc_orphan, TRUE)
                    || is_set_recursive(child_rsc, pe_rsc_managed, FALSE) == FALSE
                    || is_set_recursive(child_rsc, pe_rsc_failed, TRUE)) {
 
             // Print individual instance when active orphaned/unmanaged/failed
             print_full = TRUE;
 
         } else if (child_rsc->fns->active(child_rsc, TRUE)) {
             // Instance of fully active anonymous clone
 
             pe_node_t *location = child_rsc->fns->location(child_rsc, NULL, TRUE);
 
             if (location) {
                 // Instance is active on a single node
 
                 enum rsc_role_e a_role = child_rsc->fns->state(child_rsc, TRUE);
 
                 if (location->details->online == FALSE && location->details->unclean) {
                     print_full = TRUE;
 
                 } else if (a_role > RSC_ROLE_SLAVE) {
                     master_list = g_list_append(master_list, location);
 
                 } else {
                     started_list = g_list_append(started_list, location);
                 }
 
             } else {
                 /* uncolocated group - bleh */
                 print_full = TRUE;
             }
 
         } else {
             // Instance of partially active anonymous clone
             print_full = TRUE;
         }
 
         if (print_full) {
             if (options & pe_print_html) {
                 status_print("<li>\n");
             }
             child_rsc->fns->print(child_rsc, child_text, options, print_data);
             if (options & pe_print_html) {
                 status_print("</li>\n");
             }
         }
     }
 
     /* Masters */
     master_list = g_list_sort(master_list, sort_node_uname);
     for (gIter = master_list; gIter; gIter = gIter->next) {
         pe_node_t *host = gIter->data;
 
         list_text = pcmk__add_word(list_text, host->details->uname);
 	active_instances++;
     }
 
     short_print(list_text, child_text, "Masters", NULL, options, print_data);
     g_list_free(master_list);
     free(list_text);
     list_text = NULL;
 
     /* Started/Slaves */
     started_list = g_list_sort(started_list, sort_node_uname);
     for (gIter = started_list; gIter; gIter = gIter->next) {
         pe_node_t *host = gIter->data;
 
         list_text = pcmk__add_word(list_text, host->details->uname);
 	active_instances++;
     }
 
     if (is_set(rsc->flags, pe_rsc_promotable)) {
         enum rsc_role_e role = configured_role(rsc);
 
         if(role == RSC_ROLE_SLAVE) {
             short_print(list_text, child_text, "Slaves (target-role)", NULL, options, print_data);
         } else {
             short_print(list_text, child_text, "Slaves", NULL, options, print_data);
         }
 
     } else {
         short_print(list_text, child_text, "Started", NULL, options, print_data);
     }
 
     g_list_free(started_list);
     free(list_text);
     list_text = NULL;
 
     if (is_not_set(options, pe_print_clone_active)) {
         const char *state = "Stopped";
         enum rsc_role_e role = configured_role(rsc);
 
         if (role == RSC_ROLE_STOPPED) {
             state = "Stopped (disabled)";
         }
 
         if (is_not_set(rsc->flags, pe_rsc_unique)
             && (clone_data->clone_max > active_instances)) {
 
             GListPtr nIter;
             GListPtr list = g_hash_table_get_values(rsc->allowed_nodes);
 
             /* Custom stopped list for non-unique clones */
             free(stopped_list); stopped_list = NULL;
 
             if (list == NULL) {
                 /* Clusters with symmetrical=false haven't calculated allowed_nodes yet
                  * If we've not probed for them yet, the Stopped list will be empty
                  */
                 list = g_hash_table_get_values(rsc->known_on);
             }
 
             list = g_list_sort(list, sort_node_uname);
             for (nIter = list; nIter != NULL; nIter = nIter->next) {
                 pe_node_t *node = (pe_node_t *)nIter->data;
 
                 if (pe_find_node(rsc->running_on, node->details->uname) == NULL) {
                     stopped_list = pcmk__add_word(stopped_list,
                                                   node->details->uname);
                 }
             }
             g_list_free(list);
         }
 
         short_print(stopped_list, child_text, state, NULL, options, print_data);
         free(stopped_list);
     }
 
     if (options & pe_print_html) {
         status_print("</ul>\n");
     }
 
     free(child_text);
 }
 
 PCMK__OUTPUT_ARGS("clone", "unsigned int", "struct pe_resource_t *", "GListPtr")
 int
 pe__clone_xml(pcmk__output_t *out, va_list args)
 {
     unsigned int options = va_arg(args, unsigned int);
     pe_resource_t *rsc = va_arg(args, pe_resource_t *);
     GListPtr only_show = va_arg(args, GListPtr);
 
     GListPtr gIter = rsc->children;
 
     int rc = pcmk_rc_no_output;
     gboolean printed_header = FALSE;
 
     for (; gIter != NULL; gIter = gIter->next) {
         pe_resource_t *child_rsc = (pe_resource_t *) gIter->data;
 
-        if (!pe__rsc_running_on_any_node_in_list(child_rsc, only_show)) {
+        if (pcmk__rsc_is_filtered(child_rsc, only_show)) {
             continue;
         }
 
         if (!printed_header) {
             printed_header = TRUE;
 
             rc = pe__name_and_nvpairs_xml(out, true, "clone", 7
                      , "id", rsc->id
                      , "multi_state", BOOL2STR(is_set(rsc->flags, pe_rsc_promotable))
                      , "unique", BOOL2STR(is_set(rsc->flags, pe_rsc_unique))
                      , "managed", BOOL2STR(is_set(rsc->flags, pe_rsc_managed))
                      , "failed", BOOL2STR(is_set(rsc->flags, pe_rsc_failed))
                      , "failure_ignored", BOOL2STR(is_set(rsc->flags, pe_rsc_failure_ignored))
                      , "target_role", configured_role_str(rsc));
             CRM_ASSERT(rc == pcmk_rc_ok);
         }
 
         out->message(out, crm_map_element_name(child_rsc->xml), options, child_rsc, only_show);
     }
 
     if (printed_header) {
         pcmk__output_xml_pop_parent(out);
     }
 
     return rc;
 }
 
 PCMK__OUTPUT_ARGS("clone", "unsigned int", "struct pe_resource_t *", "GListPtr")
 int
 pe__clone_html(pcmk__output_t *out, va_list args)
 {
     unsigned int options = va_arg(args, unsigned int);
     pe_resource_t *rsc = va_arg(args, pe_resource_t *);
     GListPtr only_show = va_arg(args, GListPtr);
 
     char *list_text = NULL;
     char *stopped_list = NULL;
 
     GListPtr master_list = NULL;
     GListPtr started_list = NULL;
     GListPtr gIter = rsc->children;
 
     clone_variant_data_t *clone_data = NULL;
     int active_instances = 0;
 
     get_clone_variant_data(clone_data, rsc);
 
     out->begin_list(out, NULL, NULL, "Clone Set: %s [%s]%s%s%s",
                     rsc->id, ID(clone_data->xml_obj_child),
                     is_set(rsc->flags, pe_rsc_promotable) ? " (promotable)" : "",
                     is_set(rsc->flags, pe_rsc_unique) ? " (unique)" : "",
                     is_set(rsc->flags, pe_rsc_managed) ? "" : " (unmanaged)");
 
     for (; gIter != NULL; gIter = gIter->next) {
         gboolean print_full = FALSE;
         pe_resource_t *child_rsc = (pe_resource_t *) gIter->data;
         gboolean partially_active = child_rsc->fns->active(child_rsc, FALSE);
 
-        if (!pe__rsc_running_on_any_node_in_list(child_rsc, only_show)) {
+        if (pcmk__rsc_is_filtered(child_rsc, only_show)) {
             continue;
         }
 
         if (options & pe_print_clone_details) {
             print_full = TRUE;
         }
 
         if (is_set(rsc->flags, pe_rsc_unique)) {
             // Print individual instance when unique (except stopped orphans)
             if (partially_active || is_not_set(rsc->flags, pe_rsc_orphan)) {
                 print_full = TRUE;
             }
 
         // Everything else in this block is for anonymous clones
 
         } else if (is_set(options, pe_print_pending)
                    && (child_rsc->pending_task != NULL)
                    && strcmp(child_rsc->pending_task, "probe")) {
             // Print individual instance when non-probe action is pending
             print_full = TRUE;
 
         } else if (partially_active == FALSE) {
             // List stopped instances when requested (except orphans)
             if (is_not_set(child_rsc->flags, pe_rsc_orphan)
                 && is_not_set(options, pe_print_clone_active)) {
                 stopped_list = pcmk__add_word(stopped_list, child_rsc->id);
             }
 
         } else if (is_set_recursive(child_rsc, pe_rsc_orphan, TRUE)
                    || is_set_recursive(child_rsc, pe_rsc_managed, FALSE) == FALSE
                    || is_set_recursive(child_rsc, pe_rsc_failed, TRUE)) {
 
             // Print individual instance when active orphaned/unmanaged/failed
             print_full = TRUE;
 
         } else if (child_rsc->fns->active(child_rsc, TRUE)) {
             // Instance of fully active anonymous clone
 
             pe_node_t *location = child_rsc->fns->location(child_rsc, NULL, TRUE);
 
             if (location) {
                 // Instance is active on a single node
 
                 enum rsc_role_e a_role = child_rsc->fns->state(child_rsc, TRUE);
 
                 if (location->details->online == FALSE && location->details->unclean) {
                     print_full = TRUE;
 
                 } else if (a_role > RSC_ROLE_SLAVE) {
                     master_list = g_list_append(master_list, location);
 
                 } else {
                     started_list = g_list_append(started_list, location);
                 }
 
             } else {
                 /* uncolocated group - bleh */
                 print_full = TRUE;
             }
 
         } else {
             // Instance of partially active anonymous clone
             print_full = TRUE;
         }
 
         if (print_full) {
             out->message(out, crm_map_element_name(child_rsc->xml), options, child_rsc, only_show);
         }
     }
 
     if (is_set(options, pe_print_clone_details)) {
         free(stopped_list);
         out->end_list(out);
         return pcmk_rc_ok;
     }
 
     /* Masters */
     master_list = g_list_sort(master_list, sort_node_uname);
     for (gIter = master_list; gIter; gIter = gIter->next) {
         pe_node_t *host = gIter->data;
 
         if (!pcmk__str_in_list(only_show, host->details->uname)) {
             continue;
         }
 
         list_text = pcmk__add_word(list_text, host->details->uname);
         active_instances++;
     }
 
     if (list_text != NULL) {
         out->list_item(out, NULL, " Masters: [%s ]", list_text);
         g_list_free(master_list);
         free(list_text);
         list_text = NULL;
     }
 
     /* Started/Slaves */
     started_list = g_list_sort(started_list, sort_node_uname);
     for (gIter = started_list; gIter; gIter = gIter->next) {
         pe_node_t *host = gIter->data;
 
         if (!pcmk__str_in_list(only_show, host->details->uname)) {
             continue;
         }
 
         list_text = pcmk__add_word(list_text, host->details->uname);
         active_instances++;
     }
 
     if (list_text != NULL) {
         if (is_set(rsc->flags, pe_rsc_promotable)) {
             enum rsc_role_e role = configured_role(rsc);
 
             if(role == RSC_ROLE_SLAVE) {
                 out->list_item(out, NULL, " Slaves (target-role): [%s ]", list_text);
             } else {
                 out->list_item(out, NULL, " Slaves: [%s ]", list_text);
             }
 
         } else {
             out->list_item(out, NULL, " Started: [%s ]", list_text);
         }
 
         g_list_free(started_list);
         free(list_text);
         list_text = NULL;
     }
 
     if (is_not_set(options, pe_print_clone_active)) {
         const char *state = "Stopped";
         enum rsc_role_e role = configured_role(rsc);
 
         if (role == RSC_ROLE_STOPPED) {
             state = "Stopped (disabled)";
         }
 
         if (is_not_set(rsc->flags, pe_rsc_unique)
             && (clone_data->clone_max > active_instances)) {
 
             GListPtr nIter;
             GListPtr list = g_hash_table_get_values(rsc->allowed_nodes);
 
             /* Custom stopped list for non-unique clones */
             free(stopped_list);
             stopped_list = NULL;
 
             if (list == NULL) {
                 /* Clusters with symmetrical=false haven't calculated allowed_nodes yet
                  * If we've not probed for them yet, the Stopped list will be empty
                  */
                 list = g_hash_table_get_values(rsc->known_on);
             }
 
             list = g_list_sort(list, sort_node_uname);
             for (nIter = list; nIter != NULL; nIter = nIter->next) {
                 pe_node_t *node = (pe_node_t *)nIter->data;
 
                 if (pe_find_node(rsc->running_on, node->details->uname) == NULL &&
                     pcmk__str_in_list(only_show, node->details->uname)) {
                     stopped_list = pcmk__add_word(stopped_list,
                                                   node->details->uname);
                 }
             }
             g_list_free(list);
         }
 
         if (stopped_list != NULL) {
             out->list_item(out, NULL, " %s: [%s ]", state, stopped_list);
             free(stopped_list);
         }
     }
 
     out->end_list(out);
 
     return pcmk_rc_ok;
 }
 
 PCMK__OUTPUT_ARGS("clone", "unsigned int", "struct pe_resource_t *", "GListPtr")
 int
 pe__clone_text(pcmk__output_t *out, va_list args)
 {
     unsigned int options = va_arg(args, unsigned int);
     pe_resource_t *rsc = va_arg(args, pe_resource_t *);
     GListPtr only_show = va_arg(args, GListPtr);
 
     char *list_text = NULL;
     char *stopped_list = NULL;
 
     GListPtr master_list = NULL;
     GListPtr started_list = NULL;
     GListPtr gIter = rsc->children;
 
     clone_variant_data_t *clone_data = NULL;
     int active_instances = 0;
 
     get_clone_variant_data(clone_data, rsc);
 
     out->begin_list(out, NULL, NULL, "Clone Set: %s [%s]%s%s%s",
                     rsc->id, ID(clone_data->xml_obj_child),
                     is_set(rsc->flags, pe_rsc_promotable) ? " (promotable)" : "",
                     is_set(rsc->flags, pe_rsc_unique) ? " (unique)" : "",
                     is_set(rsc->flags, pe_rsc_managed) ? "" : " (unmanaged)");
 
     for (; gIter != NULL; gIter = gIter->next) {
         gboolean print_full = FALSE;
         pe_resource_t *child_rsc = (pe_resource_t *) gIter->data;
         gboolean partially_active = child_rsc->fns->active(child_rsc, FALSE);
 
-        if (!pe__rsc_running_on_any_node_in_list(child_rsc, only_show)) {
+        if (pcmk__rsc_is_filtered(child_rsc, only_show)) {
             continue;
         }
 
         if (options & pe_print_clone_details) {
             print_full = TRUE;
         }
 
         if (is_set(rsc->flags, pe_rsc_unique)) {
             // Print individual instance when unique (except stopped orphans)
             if (partially_active || is_not_set(rsc->flags, pe_rsc_orphan)) {
                 print_full = TRUE;
             }
 
         // Everything else in this block is for anonymous clones
 
         } else if (is_set(options, pe_print_pending)
                    && (child_rsc->pending_task != NULL)
                    && strcmp(child_rsc->pending_task, "probe")) {
             // Print individual instance when non-probe action is pending
             print_full = TRUE;
 
         } else if (partially_active == FALSE) {
             // List stopped instances when requested (except orphans)
             if (is_not_set(child_rsc->flags, pe_rsc_orphan)
                 && is_not_set(options, pe_print_clone_active)) {
                 stopped_list = pcmk__add_word(stopped_list, child_rsc->id);
             }
 
         } else if (is_set_recursive(child_rsc, pe_rsc_orphan, TRUE)
                    || is_set_recursive(child_rsc, pe_rsc_managed, FALSE) == FALSE
                    || is_set_recursive(child_rsc, pe_rsc_failed, TRUE)) {
 
             // Print individual instance when active orphaned/unmanaged/failed
             print_full = TRUE;
 
         } else if (child_rsc->fns->active(child_rsc, TRUE)) {
             // Instance of fully active anonymous clone
 
             pe_node_t *location = child_rsc->fns->location(child_rsc, NULL, TRUE);
 
             if (location) {
                 // Instance is active on a single node
 
                 enum rsc_role_e a_role = child_rsc->fns->state(child_rsc, TRUE);
 
                 if (location->details->online == FALSE && location->details->unclean) {
                     print_full = TRUE;
 
                 } else if (a_role > RSC_ROLE_SLAVE) {
                     master_list = g_list_append(master_list, location);
 
                 } else {
                     started_list = g_list_append(started_list, location);
                 }
 
             } else {
                 /* uncolocated group - bleh */
                 print_full = TRUE;
             }
 
         } else {
             // Instance of partially active anonymous clone
             print_full = TRUE;
         }
 
         if (print_full) {
             out->message(out, crm_map_element_name(child_rsc->xml), options, child_rsc, only_show);
         }
     }
 
     if (is_set(options, pe_print_clone_details)) {
         free(stopped_list);
         out->end_list(out);
         return pcmk_rc_ok;
     }
 
     /* Masters */
     master_list = g_list_sort(master_list, sort_node_uname);
     for (gIter = master_list; gIter; gIter = gIter->next) {
         pe_node_t *host = gIter->data;
 
         if (!pcmk__str_in_list(only_show, host->details->uname)) {
             continue;
         }
 
         list_text = pcmk__add_word(list_text, host->details->uname);
         active_instances++;
     }
 
     if (list_text != NULL) {
         out->list_item(out, "Masters", "[%s ]", list_text);
         g_list_free(master_list);
         free(list_text);
         list_text = NULL;
     }
 
     /* Started/Slaves */
     started_list = g_list_sort(started_list, sort_node_uname);
     for (gIter = started_list; gIter; gIter = gIter->next) {
         pe_node_t *host = gIter->data;
 
         if (!pcmk__str_in_list(only_show, host->details->uname)) {
             continue;
         }
 
         list_text = pcmk__add_word(list_text, host->details->uname);
         active_instances++;
     }
 
     if (list_text != NULL) {
         if (is_set(rsc->flags, pe_rsc_promotable)) {
             enum rsc_role_e role = configured_role(rsc);
 
             if(role == RSC_ROLE_SLAVE) {
                 out->list_item(out, "Slaves (target-role)", "[%s ]", list_text);
             } else {
                 out->list_item(out, "Slaves", "[%s ]", list_text);
             }
         } else {
             out->list_item(out, "Started", "[%s ]", list_text);
         }
 
         g_list_free(started_list);
         free(list_text);
         list_text = NULL;
     }
 
     if (is_not_set(options, pe_print_clone_active)) {
         const char *state = "Stopped";
         enum rsc_role_e role = configured_role(rsc);
 
         if (role == RSC_ROLE_STOPPED) {
             state = "Stopped (disabled)";
         }
 
         if (is_not_set(rsc->flags, pe_rsc_unique)
             && (clone_data->clone_max > active_instances)) {
 
             GListPtr nIter;
             GListPtr list = g_hash_table_get_values(rsc->allowed_nodes);
 
             /* Custom stopped list for non-unique clones */
             free(stopped_list);
             stopped_list = NULL;
 
             if (list == NULL) {
                 /* Clusters with symmetrical=false haven't calculated allowed_nodes yet
                  * If we've not probed for them yet, the Stopped list will be empty
                  */
                 list = g_hash_table_get_values(rsc->known_on);
             }
 
             list = g_list_sort(list, sort_node_uname);
             for (nIter = list; nIter != NULL; nIter = nIter->next) {
                 pe_node_t *node = (pe_node_t *)nIter->data;
 
                 if (pe_find_node(rsc->running_on, node->details->uname) == NULL &&
                     pcmk__str_in_list(only_show, node->details->uname)) {
                     stopped_list = pcmk__add_word(stopped_list,
                                                   node->details->uname);
                 }
             }
             g_list_free(list);
         }
 
         if (stopped_list != NULL) {
             out->list_item(out, state, "[%s ]", stopped_list);
             free(stopped_list);
         }
     }
 
     out->end_list(out);
 
     return pcmk_rc_ok;
 }
 
 void
 clone_free(pe_resource_t * rsc)
 {
     clone_variant_data_t *clone_data = NULL;
 
     get_clone_variant_data(clone_data, rsc);
 
     pe_rsc_trace(rsc, "Freeing %s", rsc->id);
 
     for (GListPtr gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
         pe_resource_t *child_rsc = (pe_resource_t *) gIter->data;
 
         CRM_ASSERT(child_rsc);
         pe_rsc_trace(child_rsc, "Freeing child %s", child_rsc->id);
         free_xml(child_rsc->xml);
         child_rsc->xml = NULL;
         /* There could be a saved unexpanded xml */
         free_xml(child_rsc->orig_xml);
         child_rsc->orig_xml = NULL;
         child_rsc->fns->free(child_rsc);
     }
 
     g_list_free(rsc->children);
 
     if (clone_data) {
         CRM_ASSERT(clone_data->demote_notify == NULL);
         CRM_ASSERT(clone_data->stop_notify == NULL);
         CRM_ASSERT(clone_data->start_notify == NULL);
         CRM_ASSERT(clone_data->promote_notify == NULL);
     }
 
     common_free(rsc);
 }
 
 enum rsc_role_e
 clone_resource_state(const pe_resource_t * rsc, gboolean current)
 {
     enum rsc_role_e clone_role = RSC_ROLE_UNKNOWN;
     GListPtr gIter = rsc->children;
 
     for (; gIter != NULL; gIter = gIter->next) {
         pe_resource_t *child_rsc = (pe_resource_t *) gIter->data;
         enum rsc_role_e a_role = child_rsc->fns->state(child_rsc, current);
 
         if (a_role > clone_role) {
             clone_role = a_role;
         }
     }
 
     pe_rsc_trace(rsc, "%s role: %s", rsc->id, role2text(clone_role));
     return clone_role;
 }
 
 /*!
  * \internal
  * \brief Check whether a clone has an instance for every node
  *
  * \param[in] rsc       Clone to check
  * \param[in] data_set  Cluster state
  */
 bool
 pe__is_universal_clone(pe_resource_t *rsc,
                        pe_working_set_t *data_set)
 {
     if (pe_rsc_is_clone(rsc)) {
         clone_variant_data_t *clone_data = NULL;
 
         get_clone_variant_data(clone_data, rsc);
         if (clone_data->clone_max == g_list_length(data_set->nodes)) {
             return TRUE;
         }
     }
     return FALSE;
 }
diff --git a/lib/pengine/unpack.c b/lib/pengine/unpack.c
index 8784857173..3c6606b92d 100644
--- a/lib/pengine/unpack.c
+++ b/lib/pengine/unpack.c
@@ -1,3896 +1,3896 @@
 /*
  * Copyright 2004-2020 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU Lesser General Public License
  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #include <stdio.h>
 #include <string.h>
 #include <glib.h>
 #include <time.h>
 
 #include <crm/crm.h>
 #include <crm/services.h>
 #include <crm/msg_xml.h>
 #include <crm/common/xml.h>
 
 #include <crm/common/util.h>
 #include <crm/pengine/rules.h>
 #include <crm/pengine/internal.h>
 #include <crm/common/iso8601_internal.h>
 #include <pe_status_private.h>
 
 CRM_TRACE_INIT_DATA(pe_status);
 
 #define set_config_flag(data_set, option, flag) do {			\
 	const char *tmp = pe_pref(data_set->config_hash, option);	\
 	if(tmp) {							\
 	    if(crm_is_true(tmp)) {					\
 		set_bit(data_set->flags, flag);			\
 	    } else {							\
 		clear_bit(data_set->flags, flag);		\
 	    }								\
 	}								\
     } while(0)
 
 static void unpack_rsc_op(pe_resource_t *rsc, pe_node_t *node, xmlNode *xml_op,
                           xmlNode **last_failure,
                           enum action_fail_response *failed,
                           pe_working_set_t *data_set);
 static void determine_remote_online_status(pe_working_set_t *data_set,
                                            pe_node_t *this_node);
 static void add_node_attrs(xmlNode *attrs, pe_node_t *node, bool overwrite,
                            pe_working_set_t *data_set);
 static void determine_online_status(xmlNode *node_state, pe_node_t *this_node,
                                     pe_working_set_t *data_set);
 
 static void unpack_lrm_resources(pe_node_t *node, xmlNode *lrm_state,
                                  pe_working_set_t *data_set);
 
 
 // Bitmask for warnings we only want to print once
 uint32_t pe_wo = 0;
 
 static gboolean
 is_dangling_guest_node(pe_node_t *node)
 {
     /* we are looking for a remote-node that was supposed to be mapped to a
      * container resource, but all traces of that container have disappeared 
      * from both the config and the status section. */
     if (pe__is_guest_or_remote_node(node) &&
         node->details->remote_rsc &&
         node->details->remote_rsc->container == NULL &&
         is_set(node->details->remote_rsc->flags, pe_rsc_orphan_container_filler)) {
         return TRUE;
     }
 
     return FALSE;
 }
 
 
 /*!
  * \brief Schedule a fence action for a node
  *
  * \param[in,out] data_set  Current working set of cluster
  * \param[in,out] node      Node to fence
  * \param[in]     reason    Text description of why fencing is needed
  * \param[in]     priority_delay  Whether to consider `priority-fencing-delay`
  */
 void
 pe_fence_node(pe_working_set_t * data_set, pe_node_t * node,
               const char *reason, bool priority_delay)
 {
     CRM_CHECK(node, return);
 
     /* A guest node is fenced by marking its container as failed */
     if (pe__is_guest_node(node)) {
         pe_resource_t *rsc = node->details->remote_rsc->container;
 
         if (is_set(rsc->flags, pe_rsc_failed) == FALSE) {
             if (!is_set(rsc->flags, pe_rsc_managed)) {
                 crm_notice("Not fencing guest node %s "
                            "(otherwise would because %s): "
                            "its guest resource %s is unmanaged",
                            node->details->uname, reason, rsc->id);
             } else {
                 crm_warn("Guest node %s will be fenced "
                          "(by recovering its guest resource %s): %s",
                          node->details->uname, rsc->id, reason);
 
                 /* We don't mark the node as unclean because that would prevent the
                  * node from running resources. We want to allow it to run resources
                  * in this transition if the recovery succeeds.
                  */
                 node->details->remote_requires_reset = TRUE;
                 set_bit(rsc->flags, pe_rsc_failed);
             }
         }
 
     } else if (is_dangling_guest_node(node)) {
         crm_info("Cleaning up dangling connection for guest node %s: "
                  "fencing was already done because %s, "
                  "and guest resource no longer exists",
                  node->details->uname, reason);
         set_bit(node->details->remote_rsc->flags, pe_rsc_failed);
 
     } else if (pe__is_remote_node(node)) {
         pe_resource_t *rsc = node->details->remote_rsc;
 
         if (rsc && (!is_set(rsc->flags, pe_rsc_managed))) {
             crm_notice("Not fencing remote node %s "
                        "(otherwise would because %s): connection is unmanaged",
                        node->details->uname, reason);
         } else if(node->details->remote_requires_reset == FALSE) {
             node->details->remote_requires_reset = TRUE;
             crm_warn("Remote node %s %s: %s",
                      node->details->uname,
                      pe_can_fence(data_set, node)? "will be fenced" : "is unclean",
                      reason);
         }
         node->details->unclean = TRUE;
         // No need to apply `priority-fencing-delay` for remote nodes
         pe_fence_op(node, NULL, TRUE, reason, FALSE, data_set);
 
     } else if (node->details->unclean) {
         crm_trace("Cluster node %s %s because %s",
                   node->details->uname,
                   pe_can_fence(data_set, node)? "would also be fenced" : "also is unclean",
                   reason);
 
     } else {
         crm_warn("Cluster node %s %s: %s",
                  node->details->uname,
                  pe_can_fence(data_set, node)? "will be fenced" : "is unclean",
                  reason);
         node->details->unclean = TRUE;
         pe_fence_op(node, NULL, TRUE, reason, priority_delay, data_set);
     }
 }
 
 // @TODO xpaths can't handle templates, rules, or id-refs
 
 // nvpair with provides or requires set to unfencing
 #define XPATH_UNFENCING_NVPAIR XML_CIB_TAG_NVPAIR                \
     "[(@" XML_NVPAIR_ATTR_NAME "='" XML_RSC_ATTR_PROVIDES "'"    \
     "or @" XML_NVPAIR_ATTR_NAME "='" XML_RSC_ATTR_REQUIRES "') " \
     "and @" XML_NVPAIR_ATTR_VALUE "='unfencing']"
 
 // unfencing in rsc_defaults or any resource
 #define XPATH_ENABLE_UNFENCING \
     "/" XML_TAG_CIB "/" XML_CIB_TAG_CONFIGURATION "/" XML_CIB_TAG_RESOURCES   \
     "//" XML_TAG_META_SETS "/" XPATH_UNFENCING_NVPAIR                                               \
     "|/" XML_TAG_CIB "/" XML_CIB_TAG_CONFIGURATION "/" XML_CIB_TAG_RSCCONFIG  \
     "/" XML_TAG_META_SETS "/" XPATH_UNFENCING_NVPAIR
 
 static
 void set_if_xpath(unsigned long long flag, const char *xpath,
                   pe_working_set_t *data_set)
 {
     xmlXPathObjectPtr result = NULL;
 
     if (is_not_set(data_set->flags, flag)) {
         result = xpath_search(data_set->input, xpath);
         if (result && (numXpathResults(result) > 0)) {
             set_bit(data_set->flags, flag);
         }
         freeXpathObject(result);
     }
 }
 
 gboolean
 unpack_config(xmlNode * config, pe_working_set_t * data_set)
 {
     const char *value = NULL;
     GHashTable *config_hash = crm_str_table_new();
 
     pe_rule_eval_data_t rule_data = {
         .node_hash = NULL,
         .role = RSC_ROLE_UNKNOWN,
         .now = data_set->now,
         .match_data = NULL,
         .rsc_data = NULL,
         .op_data = NULL
     };
 
     data_set->config_hash = config_hash;
 
     pe__unpack_dataset_nvpairs(config, XML_CIB_TAG_PROPSET, &rule_data, config_hash,
                                CIB_OPTIONS_FIRST, FALSE, data_set);
 
     verify_pe_options(data_set->config_hash);
 
     set_config_flag(data_set, "enable-startup-probes", pe_flag_startup_probes);
     if(is_not_set(data_set->flags, pe_flag_startup_probes)) {
         crm_info("Startup probes: disabled (dangerous)");
     }
 
     value = pe_pref(data_set->config_hash, XML_ATTR_HAVE_WATCHDOG);
     if (value && crm_is_true(value)) {
         crm_notice("Watchdog will be used via SBD if fencing is required "
                    "and stonith-watchdog-timeout is nonzero");
         set_bit(data_set->flags, pe_flag_have_stonith_resource);
     }
 
     /* Set certain flags via xpath here, so they can be used before the relevant
      * configuration sections are unpacked.
      */
     set_if_xpath(pe_flag_enable_unfencing, XPATH_ENABLE_UNFENCING, data_set);
 
     value = pe_pref(data_set->config_hash, "stonith-timeout");
     data_set->stonith_timeout = (int) crm_parse_interval_spec(value);
     crm_debug("STONITH timeout: %d", data_set->stonith_timeout);
 
     set_config_flag(data_set, "stonith-enabled", pe_flag_stonith_enabled);
     crm_debug("STONITH of failed nodes is %s",
               is_set(data_set->flags, pe_flag_stonith_enabled) ? "enabled" : "disabled");
 
     data_set->stonith_action = pe_pref(data_set->config_hash, "stonith-action");
     if (!strcmp(data_set->stonith_action, "poweroff")) {
         pe_warn_once(pe_wo_poweroff,
                      "Support for stonith-action of 'poweroff' is deprecated "
                      "and will be removed in a future release (use 'off' instead)");
         data_set->stonith_action = "off";
     }
     crm_trace("STONITH will %s nodes", data_set->stonith_action);
 
     set_config_flag(data_set, "concurrent-fencing", pe_flag_concurrent_fencing);
     crm_debug("Concurrent fencing is %s",
               is_set(data_set->flags, pe_flag_concurrent_fencing) ? "enabled" : "disabled");
 
     value = pe_pref(data_set->config_hash,
                     XML_CONFIG_ATTR_PRIORITY_FENCING_DELAY);
     if (value) {
         data_set->priority_fencing_delay = crm_parse_interval_spec(value) / 1000;
         crm_trace("Priority fencing delay is %ds", data_set->priority_fencing_delay);
     }
 
     set_config_flag(data_set, "stop-all-resources", pe_flag_stop_everything);
     crm_debug("Stop all active resources: %s",
               is_set(data_set->flags, pe_flag_stop_everything) ? "true" : "false");
 
     set_config_flag(data_set, "symmetric-cluster", pe_flag_symmetric_cluster);
     if (is_set(data_set->flags, pe_flag_symmetric_cluster)) {
         crm_debug("Cluster is symmetric" " - resources can run anywhere by default");
     }
 
     value = pe_pref(data_set->config_hash, "no-quorum-policy");
 
     if (safe_str_eq(value, "ignore")) {
         data_set->no_quorum_policy = no_quorum_ignore;
 
     } else if (safe_str_eq(value, "freeze")) {
         data_set->no_quorum_policy = no_quorum_freeze;
 
     } else if (safe_str_eq(value, "suicide")) {
         if (is_set(data_set->flags, pe_flag_stonith_enabled)) {
             int do_panic = 0;
 
             crm_element_value_int(data_set->input, XML_ATTR_QUORUM_PANIC,
                                   &do_panic);
             if (do_panic || is_set(data_set->flags, pe_flag_have_quorum)) {
                 data_set->no_quorum_policy = no_quorum_suicide;
             } else {
                 crm_notice("Resetting no-quorum-policy to 'stop': cluster has never had quorum");
                 data_set->no_quorum_policy = no_quorum_stop;
             }
         } else {
             pcmk__config_err("Resetting no-quorum-policy to 'stop' because "
                              "fencing is disabled");
             data_set->no_quorum_policy = no_quorum_stop;
         }
 
     } else {
         data_set->no_quorum_policy = no_quorum_stop;
     }
 
     switch (data_set->no_quorum_policy) {
         case no_quorum_freeze:
             crm_debug("On loss of quorum: Freeze resources");
             break;
         case no_quorum_stop:
             crm_debug("On loss of quorum: Stop ALL resources");
             break;
         case no_quorum_suicide:
             crm_notice("On loss of quorum: Fence all remaining nodes");
             break;
         case no_quorum_ignore:
             crm_notice("On loss of quorum: Ignore");
             break;
     }
 
     set_config_flag(data_set, "stop-orphan-resources", pe_flag_stop_rsc_orphans);
     crm_trace("Orphan resources are %s",
               is_set(data_set->flags, pe_flag_stop_rsc_orphans) ? "stopped" : "ignored");
 
     set_config_flag(data_set, "stop-orphan-actions", pe_flag_stop_action_orphans);
     crm_trace("Orphan resource actions are %s",
               is_set(data_set->flags, pe_flag_stop_action_orphans) ? "stopped" : "ignored");
 
     set_config_flag(data_set, "remove-after-stop", pe_flag_remove_after_stop);
     crm_trace("Stopped resources are removed from the status section: %s",
               is_set(data_set->flags, pe_flag_remove_after_stop) ? "true" : "false");
 
     set_config_flag(data_set, "maintenance-mode", pe_flag_maintenance_mode);
     crm_trace("Maintenance mode: %s",
               is_set(data_set->flags, pe_flag_maintenance_mode) ? "true" : "false");
 
     set_config_flag(data_set, "start-failure-is-fatal", pe_flag_start_failure_fatal);
     crm_trace("Start failures are %s",
               is_set(data_set->flags,
                      pe_flag_start_failure_fatal) ? "always fatal" : "handled by failcount");
 
     if (is_set(data_set->flags, pe_flag_stonith_enabled)) {
         set_config_flag(data_set, "startup-fencing", pe_flag_startup_fencing);
     }
     if (is_set(data_set->flags, pe_flag_startup_fencing)) {
         crm_trace("Unseen nodes will be fenced");
     } else {
         pe_warn_once(pe_wo_blind, "Blind faith: not fencing unseen nodes");
     }
 
     pcmk__score_red = char2score(pe_pref(data_set->config_hash, "node-health-red"));
     pcmk__score_green = char2score(pe_pref(data_set->config_hash, "node-health-green"));
     pcmk__score_yellow = char2score(pe_pref(data_set->config_hash, "node-health-yellow"));
 
     crm_debug("Node scores: 'red' = %s, 'yellow' = %s, 'green' = %s",
              pe_pref(data_set->config_hash, "node-health-red"),
              pe_pref(data_set->config_hash, "node-health-yellow"),
              pe_pref(data_set->config_hash, "node-health-green"));
 
     data_set->placement_strategy = pe_pref(data_set->config_hash, "placement-strategy");
     crm_trace("Placement strategy: %s", data_set->placement_strategy);
 
     set_config_flag(data_set, "shutdown-lock", pe_flag_shutdown_lock);
     crm_trace("Resources will%s be locked to cleanly shut down nodes",
               (is_set(data_set->flags, pe_flag_shutdown_lock)? "" : " not"));
     if (is_set(data_set->flags, pe_flag_shutdown_lock)) {
         value = pe_pref(data_set->config_hash,
                         XML_CONFIG_ATTR_SHUTDOWN_LOCK_LIMIT);
         data_set->shutdown_lock = crm_parse_interval_spec(value) / 1000;
         crm_trace("Shutdown locks expire after %us", data_set->shutdown_lock);
     }
 
     return TRUE;
 }
 
 static void
 destroy_digest_cache(gpointer ptr)
 {
     op_digest_cache_t *data = ptr;
 
     free_xml(data->params_all);
     free_xml(data->params_secure);
     free_xml(data->params_restart);
 
     free(data->digest_all_calc);
     free(data->digest_restart_calc);
     free(data->digest_secure_calc);
 
     free(data);
 }
 
 pe_node_t *
 pe_create_node(const char *id, const char *uname, const char *type,
                const char *score, pe_working_set_t * data_set)
 {
     pe_node_t *new_node = NULL;
 
     if (pe_find_node(data_set->nodes, uname) != NULL) {
         pcmk__config_warn("More than one node entry has name '%s'", uname);
     }
 
     new_node = calloc(1, sizeof(pe_node_t));
     if (new_node == NULL) {
         return NULL;
     }
 
     new_node->weight = char2score(score);
     new_node->fixed = FALSE;
     new_node->details = calloc(1, sizeof(struct pe_node_shared_s));
 
     if (new_node->details == NULL) {
         free(new_node);
         return NULL;
     }
 
     crm_trace("Creating node for entry %s/%s", uname, id);
     new_node->details->id = id;
     new_node->details->uname = uname;
     new_node->details->online = FALSE;
     new_node->details->shutdown = FALSE;
     new_node->details->rsc_discovery_enabled = TRUE;
     new_node->details->running_rsc = NULL;
     new_node->details->type = node_ping;
 
     if (safe_str_eq(type, "remote")) {
         new_node->details->type = node_remote;
         set_bit(data_set->flags, pe_flag_have_remote_nodes);
     } else if ((type == NULL) || safe_str_eq(type, "member")) {
         new_node->details->type = node_member;
     }
 
     new_node->details->attrs = crm_str_table_new();
 
     if (pe__is_guest_or_remote_node(new_node)) {
         g_hash_table_insert(new_node->details->attrs, strdup(CRM_ATTR_KIND),
                             strdup("remote"));
     } else {
         g_hash_table_insert(new_node->details->attrs, strdup(CRM_ATTR_KIND),
                             strdup("cluster"));
     }
 
     new_node->details->utilization = crm_str_table_new();
 
     new_node->details->digest_cache = g_hash_table_new_full(crm_str_hash,
                                                             g_str_equal, free,
                                                             destroy_digest_cache);
 
     data_set->nodes = g_list_insert_sorted(data_set->nodes, new_node, sort_node_uname);
     return new_node;
 }
 
 static const char *
 expand_remote_rsc_meta(xmlNode *xml_obj, xmlNode *parent, pe_working_set_t *data)
 {
     xmlNode *attr_set = NULL;
     xmlNode *attr = NULL;
 
     const char *container_id = ID(xml_obj);
     const char *remote_name = NULL;
     const char *remote_server = NULL;
     const char *remote_port = NULL;
     const char *connect_timeout = "60s";
     const char *remote_allow_migrate=NULL;
     const char *is_managed = NULL;
 
     for (attr_set = __xml_first_child_element(xml_obj); attr_set != NULL;
          attr_set = __xml_next_element(attr_set)) {
         if (safe_str_neq((const char *)attr_set->name, XML_TAG_META_SETS)) {
             continue;
         }
 
         for (attr = __xml_first_child_element(attr_set); attr != NULL;
              attr = __xml_next_element(attr)) {
             const char *value = crm_element_value(attr, XML_NVPAIR_ATTR_VALUE);
             const char *name = crm_element_value(attr, XML_NVPAIR_ATTR_NAME);
 
             if (safe_str_eq(name, XML_RSC_ATTR_REMOTE_NODE)) {
                 remote_name = value;
             } else if (safe_str_eq(name, "remote-addr")) {
                 remote_server = value;
             } else if (safe_str_eq(name, "remote-port")) {
                 remote_port = value;
             } else if (safe_str_eq(name, "remote-connect-timeout")) {
                 connect_timeout = value;
             } else if (safe_str_eq(name, "remote-allow-migrate")) {
                 remote_allow_migrate=value;
             } else if (safe_str_eq(name, XML_RSC_ATTR_MANAGED)) {
                 is_managed = value;
             }
         }
     }
 
     if (remote_name == NULL) {
         return NULL;
     }
 
     if (pe_find_resource(data->resources, remote_name) != NULL) {
         return NULL;
     }
 
     pe_create_remote_xml(parent, remote_name, container_id,
                          remote_allow_migrate, is_managed,
                          connect_timeout, remote_server, remote_port);
     return remote_name;
 }
 
 static void
 handle_startup_fencing(pe_working_set_t *data_set, pe_node_t *new_node)
 {
     if ((new_node->details->type == node_remote) && (new_node->details->remote_rsc == NULL)) {
         /* Ignore fencing for remote nodes that don't have a connection resource
          * associated with them. This happens when remote node entries get left
          * in the nodes section after the connection resource is removed.
          */
         return;
     }
 
     if (is_set(data_set->flags, pe_flag_startup_fencing)) {
         // All nodes are unclean until we've seen their status entry
         new_node->details->unclean = TRUE;
 
     } else {
         // Blind faith ...
         new_node->details->unclean = FALSE;
     }
 
     /* We need to be able to determine if a node's status section
      * exists or not separate from whether the node is unclean. */
     new_node->details->unseen = TRUE;
 }
 
 gboolean
 unpack_nodes(xmlNode * xml_nodes, pe_working_set_t * data_set)
 {
     xmlNode *xml_obj = NULL;
     pe_node_t *new_node = NULL;
     const char *id = NULL;
     const char *uname = NULL;
     const char *type = NULL;
     const char *score = NULL;
 
     pe_rule_eval_data_t rule_data = {
         .node_hash = NULL,
         .role = RSC_ROLE_UNKNOWN,
         .now = data_set->now,
         .match_data = NULL,
         .rsc_data = NULL,
         .op_data = NULL
     };
 
     for (xml_obj = __xml_first_child_element(xml_nodes); xml_obj != NULL;
          xml_obj = __xml_next_element(xml_obj)) {
 
         if (crm_str_eq((const char *)xml_obj->name, XML_CIB_TAG_NODE, TRUE)) {
             new_node = NULL;
 
             id = crm_element_value(xml_obj, XML_ATTR_ID);
             uname = crm_element_value(xml_obj, XML_ATTR_UNAME);
             type = crm_element_value(xml_obj, XML_ATTR_TYPE);
             score = crm_element_value(xml_obj, XML_RULE_ATTR_SCORE);
             crm_trace("Processing node %s/%s", uname, id);
 
             if (id == NULL) {
                 pcmk__config_err("Ignoring <" XML_CIB_TAG_NODE
                                  "> entry in configuration without id");
                 continue;
             }
             new_node = pe_create_node(id, uname, type, score, data_set);
 
             if (new_node == NULL) {
                 return FALSE;
             }
 
 /* 		if(data_set->have_quorum == FALSE */
 /* 		   && data_set->no_quorum_policy == no_quorum_stop) { */
 /* 			/\* start shutting resources down *\/ */
 /* 			new_node->weight = -INFINITY; */
 /* 		} */
 
             handle_startup_fencing(data_set, new_node);
 
             add_node_attrs(xml_obj, new_node, FALSE, data_set);
             pe__unpack_dataset_nvpairs(xml_obj, XML_TAG_UTILIZATION, &rule_data,
                                        new_node->details->utilization, NULL,
                                        FALSE, data_set);
 
             crm_trace("Done with node %s", crm_element_value(xml_obj, XML_ATTR_UNAME));
         }
     }
 
     if (data_set->localhost && pe_find_node(data_set->nodes, data_set->localhost) == NULL) {
         crm_info("Creating a fake local node");
         pe_create_node(data_set->localhost, data_set->localhost, NULL, 0,
                        data_set);
     }
 
     return TRUE;
 }
 
 static void
 setup_container(pe_resource_t * rsc, pe_working_set_t * data_set)
 {
     const char *container_id = NULL;
 
     if (rsc->children) {
         GListPtr gIter = rsc->children;
 
         for (; gIter != NULL; gIter = gIter->next) {
             pe_resource_t *child_rsc = (pe_resource_t *) gIter->data;
 
             setup_container(child_rsc, data_set);
         }
         return;
     }
 
     container_id = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_CONTAINER);
     if (container_id && safe_str_neq(container_id, rsc->id)) {
         pe_resource_t *container = pe_find_resource(data_set->resources, container_id);
 
         if (container) {
             rsc->container = container;
             set_bit(container->flags, pe_rsc_is_container);
             container->fillers = g_list_append(container->fillers, rsc);
             pe_rsc_trace(rsc, "Resource %s's container is %s", rsc->id, container_id);
         } else {
             pe_err("Resource %s: Unknown resource container (%s)", rsc->id, container_id);
         }
     }
 }
 
 gboolean
 unpack_remote_nodes(xmlNode * xml_resources, pe_working_set_t * data_set)
 {
     xmlNode *xml_obj = NULL;
 
     /* Create remote nodes and guest nodes from the resource configuration
      * before unpacking resources.
      */
     for (xml_obj = __xml_first_child_element(xml_resources); xml_obj != NULL;
          xml_obj = __xml_next_element(xml_obj)) {
 
         const char *new_node_id = NULL;
 
         /* Check for remote nodes, which are defined by ocf:pacemaker:remote
          * primitives.
          */
         if (xml_contains_remote_node(xml_obj)) {
             new_node_id = ID(xml_obj);
             /* The "pe_find_node" check is here to make sure we don't iterate over
              * an expanded node that has already been added to the node list. */
             if (new_node_id && pe_find_node(data_set->nodes, new_node_id) == NULL) {
                 crm_trace("Found remote node %s defined by resource %s",
                           new_node_id, ID(xml_obj));
                 pe_create_node(new_node_id, new_node_id, "remote", NULL,
                                data_set);
             }
             continue;
         }
 
         /* Check for guest nodes, which are defined by special meta-attributes
          * of a primitive of any type (for example, VirtualDomain or Xen).
          */
         if (crm_str_eq((const char *)xml_obj->name, XML_CIB_TAG_RESOURCE, TRUE)) {
             /* This will add an ocf:pacemaker:remote primitive to the
              * configuration for the guest node's connection, to be unpacked
              * later.
              */
             new_node_id = expand_remote_rsc_meta(xml_obj, xml_resources, data_set);
             if (new_node_id && pe_find_node(data_set->nodes, new_node_id) == NULL) {
                 crm_trace("Found guest node %s in resource %s",
                           new_node_id, ID(xml_obj));
                 pe_create_node(new_node_id, new_node_id, "remote", NULL,
                                data_set);
             }
             continue;
         }
 
         /* Check for guest nodes inside a group. Clones are currently not
          * supported as guest nodes.
          */
         if (crm_str_eq((const char *)xml_obj->name, XML_CIB_TAG_GROUP, TRUE)) {
             xmlNode *xml_obj2 = NULL;
             for (xml_obj2 = __xml_first_child_element(xml_obj); xml_obj2 != NULL;
                  xml_obj2 = __xml_next_element(xml_obj2)) {
 
                 new_node_id = expand_remote_rsc_meta(xml_obj2, xml_resources, data_set);
 
                 if (new_node_id && pe_find_node(data_set->nodes, new_node_id) == NULL) {
                     crm_trace("Found guest node %s in resource %s inside group %s",
                               new_node_id, ID(xml_obj2), ID(xml_obj));
                     pe_create_node(new_node_id, new_node_id, "remote", NULL,
                                    data_set);
                 }
             }
         }
     }
     return TRUE;
 }
 
 /* Call this after all the nodes and resources have been
  * unpacked, but before the status section is read.
  *
  * A remote node's online status is reflected by the state
  * of the remote node's connection resource. We need to link
  * the remote node to this connection resource so we can have
  * easy access to the connection resource during the scheduler calculations.
  */
 static void
 link_rsc2remotenode(pe_working_set_t *data_set, pe_resource_t *new_rsc)
 {
     pe_node_t *remote_node = NULL;
 
     if (new_rsc->is_remote_node == FALSE) {
         return;
     }
 
     if (is_set(data_set->flags, pe_flag_quick_location)) {
         /* remote_nodes and remote_resources are not linked in quick location calculations */
         return;
     }
 
     remote_node = pe_find_node(data_set->nodes, new_rsc->id);
     CRM_CHECK(remote_node != NULL, return;);
 
     pe_rsc_trace(new_rsc, "Linking remote connection resource %s to node %s",
                  new_rsc->id, remote_node->details->uname);
     remote_node->details->remote_rsc = new_rsc;
 
     if (new_rsc->container == NULL) {
         /* Handle start-up fencing for remote nodes (as opposed to guest nodes)
          * the same as is done for cluster nodes.
          */
         handle_startup_fencing(data_set, remote_node);
 
     } else {
         /* pe_create_node() marks the new node as "remote" or "cluster"; now
          * that we know the node is a guest node, update it correctly.
          */
         g_hash_table_replace(remote_node->details->attrs, strdup(CRM_ATTR_KIND),
                              strdup("container"));
     }
 }
 
 static void
 destroy_tag(gpointer data)
 {
     pe_tag_t *tag = data;
 
     if (tag) {
         free(tag->id);
         g_list_free_full(tag->refs, free);
         free(tag);
     }
 }
 
 /*!
  * \internal
  * \brief Parse configuration XML for resource information
  *
  * \param[in]     xml_resources  Top of resource configuration XML
  * \param[in,out] data_set       Where to put resource information
  *
  * \return TRUE
  *
  * \note unpack_remote_nodes() MUST be called before this, so that the nodes can
  *       be used when common_unpack() calls resource_location()
  */
 gboolean
 unpack_resources(xmlNode * xml_resources, pe_working_set_t * data_set)
 {
     xmlNode *xml_obj = NULL;
     GListPtr gIter = NULL;
 
     data_set->template_rsc_sets = g_hash_table_new_full(crm_str_hash,
                                                         g_str_equal, free,
                                                         destroy_tag);
 
     for (xml_obj = __xml_first_child_element(xml_resources); xml_obj != NULL;
          xml_obj = __xml_next_element(xml_obj)) {
 
         pe_resource_t *new_rsc = NULL;
 
         if (crm_str_eq((const char *)xml_obj->name, XML_CIB_TAG_RSC_TEMPLATE, TRUE)) {
             const char *template_id = ID(xml_obj);
 
             if (template_id && g_hash_table_lookup_extended(data_set->template_rsc_sets,
                                                             template_id, NULL, NULL) == FALSE) {
                 /* Record the template's ID for the knowledge of its existence anyway. */
                 g_hash_table_insert(data_set->template_rsc_sets, strdup(template_id), NULL);
             }
             continue;
         }
 
         crm_trace("Beginning unpack... <%s id=%s... >", crm_element_name(xml_obj), ID(xml_obj));
-        if (common_unpack(xml_obj, &new_rsc, NULL, data_set)) {
+        if (common_unpack(xml_obj, &new_rsc, NULL, data_set) && (new_rsc != NULL)) {
             data_set->resources = g_list_append(data_set->resources, new_rsc);
             pe_rsc_trace(new_rsc, "Added resource %s", new_rsc->id);
 
         } else {
             pcmk__config_err("Ignoring <%s> resource '%s' "
                              "because configuration is invalid",
                              crm_element_name(xml_obj), crm_str(ID(xml_obj)));
             if (new_rsc != NULL && new_rsc->fns != NULL) {
                 new_rsc->fns->free(new_rsc);
             }
         }
     }
 
     for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
         pe_resource_t *rsc = (pe_resource_t *) gIter->data;
 
         setup_container(rsc, data_set);
         link_rsc2remotenode(data_set, rsc);
     }
 
     data_set->resources = g_list_sort(data_set->resources, sort_rsc_priority);
     if (is_set(data_set->flags, pe_flag_quick_location)) {
         /* Ignore */
 
     } else if (is_set(data_set->flags, pe_flag_stonith_enabled)
                && is_set(data_set->flags, pe_flag_have_stonith_resource) == FALSE) {
 
         pcmk__config_err("Resource start-up disabled since no STONITH resources have been defined");
         pcmk__config_err("Either configure some or disable STONITH with the stonith-enabled option");
         pcmk__config_err("NOTE: Clusters with shared data need STONITH to ensure data integrity");
     }
 
     return TRUE;
 }
 
 gboolean
 unpack_tags(xmlNode * xml_tags, pe_working_set_t * data_set)
 {
     xmlNode *xml_tag = NULL;
 
     data_set->tags = g_hash_table_new_full(crm_str_hash, g_str_equal, free,
                                            destroy_tag);
 
     for (xml_tag = __xml_first_child_element(xml_tags); xml_tag != NULL;
          xml_tag = __xml_next_element(xml_tag)) {
 
         xmlNode *xml_obj_ref = NULL;
         const char *tag_id = ID(xml_tag);
 
         if (crm_str_eq((const char *)xml_tag->name, XML_CIB_TAG_TAG, TRUE) == FALSE) {
             continue;
         }
 
         if (tag_id == NULL) {
             pcmk__config_err("Ignoring <%s> without " XML_ATTR_ID,
                              crm_element_name(xml_tag));
             continue;
         }
 
         for (xml_obj_ref = __xml_first_child_element(xml_tag); xml_obj_ref != NULL;
              xml_obj_ref = __xml_next_element(xml_obj_ref)) {
 
             const char *obj_ref = ID(xml_obj_ref);
 
             if (crm_str_eq((const char *)xml_obj_ref->name, XML_CIB_TAG_OBJ_REF, TRUE) == FALSE) {
                 continue;
             }
 
             if (obj_ref == NULL) {
                 pcmk__config_err("Ignoring <%s> for tag '%s' without " XML_ATTR_ID,
                                  crm_element_name(xml_obj_ref), tag_id);
                 continue;
             }
 
             if (add_tag_ref(data_set->tags, tag_id, obj_ref) == FALSE) {
                 return FALSE;
             }
         }
     }
 
     return TRUE;
 }
 
 /* The ticket state section:
  * "/cib/status/tickets/ticket_state" */
 static gboolean
 unpack_ticket_state(xmlNode * xml_ticket, pe_working_set_t * data_set)
 {
     const char *ticket_id = NULL;
     const char *granted = NULL;
     const char *last_granted = NULL;
     const char *standby = NULL;
     xmlAttrPtr xIter = NULL;
 
     pe_ticket_t *ticket = NULL;
 
     ticket_id = ID(xml_ticket);
     if (ticket_id == NULL || strlen(ticket_id) == 0) {
         return FALSE;
     }
 
     crm_trace("Processing ticket state for %s", ticket_id);
 
     ticket = g_hash_table_lookup(data_set->tickets, ticket_id);
     if (ticket == NULL) {
         ticket = ticket_new(ticket_id, data_set);
         if (ticket == NULL) {
             return FALSE;
         }
     }
 
     for (xIter = xml_ticket->properties; xIter; xIter = xIter->next) {
         const char *prop_name = (const char *)xIter->name;
         const char *prop_value = crm_element_value(xml_ticket, prop_name);
 
         if (crm_str_eq(prop_name, XML_ATTR_ID, TRUE)) {
             continue;
         }
         g_hash_table_replace(ticket->state, strdup(prop_name), strdup(prop_value));
     }
 
     granted = g_hash_table_lookup(ticket->state, "granted");
     if (granted && crm_is_true(granted)) {
         ticket->granted = TRUE;
         crm_info("We have ticket '%s'", ticket->id);
     } else {
         ticket->granted = FALSE;
         crm_info("We do not have ticket '%s'", ticket->id);
     }
 
     last_granted = g_hash_table_lookup(ticket->state, "last-granted");
     if (last_granted) {
         ticket->last_granted = crm_parse_int(last_granted, 0);
     }
 
     standby = g_hash_table_lookup(ticket->state, "standby");
     if (standby && crm_is_true(standby)) {
         ticket->standby = TRUE;
         if (ticket->granted) {
             crm_info("Granted ticket '%s' is in standby-mode", ticket->id);
         }
     } else {
         ticket->standby = FALSE;
     }
 
     crm_trace("Done with ticket state for %s", ticket_id);
 
     return TRUE;
 }
 
 static gboolean
 unpack_tickets_state(xmlNode * xml_tickets, pe_working_set_t * data_set)
 {
     xmlNode *xml_obj = NULL;
 
     for (xml_obj = __xml_first_child_element(xml_tickets); xml_obj != NULL;
          xml_obj = __xml_next_element(xml_obj)) {
 
         if (crm_str_eq((const char *)xml_obj->name, XML_CIB_TAG_TICKET_STATE, TRUE) == FALSE) {
             continue;
         }
         unpack_ticket_state(xml_obj, data_set);
     }
 
     return TRUE;
 }
 
 static void
 unpack_handle_remote_attrs(pe_node_t *this_node, xmlNode *state, pe_working_set_t * data_set) 
 {
     const char *resource_discovery_enabled = NULL;
     xmlNode *attrs = NULL;
     pe_resource_t *rsc = NULL;
 
     if (crm_str_eq((const char *)state->name, XML_CIB_TAG_STATE, TRUE) == FALSE) {
         return;
     }
 
     if ((this_node == NULL) || !pe__is_guest_or_remote_node(this_node)) {
         return;
     }
     crm_trace("Processing remote node id=%s, uname=%s", this_node->details->id, this_node->details->uname);
 
     this_node->details->remote_maintenance =
         crm_atoi(crm_element_value(state, XML_NODE_IS_MAINTENANCE), "0");
 
     rsc = this_node->details->remote_rsc;
     if (this_node->details->remote_requires_reset == FALSE) {
         this_node->details->unclean = FALSE;
         this_node->details->unseen = FALSE;
     }
     attrs = find_xml_node(state, XML_TAG_TRANSIENT_NODEATTRS, FALSE);
     add_node_attrs(attrs, this_node, TRUE, data_set);
 
     if (pe__shutdown_requested(this_node)) {
         crm_info("Node %s is shutting down", this_node->details->uname);
         this_node->details->shutdown = TRUE;
         if (rsc) {
             rsc->next_role = RSC_ROLE_STOPPED;
         }
     }
  
     if (crm_is_true(pe_node_attribute_raw(this_node, "standby"))) {
         crm_info("Node %s is in standby-mode", this_node->details->uname);
         this_node->details->standby = TRUE;
     }
 
     if (crm_is_true(pe_node_attribute_raw(this_node, "maintenance")) ||
         (rsc && !is_set(rsc->flags, pe_rsc_managed))) {
         crm_info("Node %s is in maintenance-mode", this_node->details->uname);
         this_node->details->maintenance = TRUE;
     }
 
     resource_discovery_enabled = pe_node_attribute_raw(this_node, XML_NODE_ATTR_RSC_DISCOVERY);
     if (resource_discovery_enabled && !crm_is_true(resource_discovery_enabled)) {
         if (pe__is_remote_node(this_node)
             && is_not_set(data_set->flags, pe_flag_stonith_enabled)) {
             crm_warn("Ignoring %s attribute on remote node %s because stonith is disabled",
                      XML_NODE_ATTR_RSC_DISCOVERY, this_node->details->uname);
         } else {
             /* This is either a remote node with fencing enabled, or a guest
              * node. We don't care whether fencing is enabled when fencing guest
              * nodes, because they are "fenced" by recovering their containing
              * resource.
              */
             crm_info("Node %s has resource discovery disabled", this_node->details->uname);
             this_node->details->rsc_discovery_enabled = FALSE;
         }
     }
 }
 
 static bool
 unpack_node_loop(xmlNode * status, bool fence, pe_working_set_t * data_set) 
 {
     bool changed = false;
     xmlNode *lrm_rsc = NULL;
 
     for (xmlNode *state = __xml_first_child_element(status); state != NULL;
          state = __xml_next_element(state)) {
 
         const char *id = NULL;
         const char *uname = NULL;
         pe_node_t *this_node = NULL;
         bool process = FALSE;
 
         if (crm_str_eq((const char *)state->name, XML_CIB_TAG_STATE, TRUE) == FALSE) {
             continue;
         }
 
         id = crm_element_value(state, XML_ATTR_ID);
         uname = crm_element_value(state, XML_ATTR_UNAME);
         this_node = pe_find_node_any(data_set->nodes, id, uname);
 
         if (this_node == NULL) {
             crm_info("Node %s is unknown", id);
             continue;
 
         } else if (this_node->details->unpacked) {
             crm_trace("Node %s was already processed", id);
             continue;
 
         } else if (!pe__is_guest_or_remote_node(this_node)
                    && is_set(data_set->flags, pe_flag_stonith_enabled)) {
             // A redundant test, but preserves the order for regression tests
             process = TRUE;
 
         } else if (pe__is_guest_or_remote_node(this_node)) {
             bool check = FALSE;
             pe_resource_t *rsc = this_node->details->remote_rsc;
 
             if(fence) {
                 check = TRUE;
 
             } else if(rsc == NULL) {
                 /* Not ready yet */
 
             } else if (pe__is_guest_node(this_node)
                        && rsc->role == RSC_ROLE_STARTED
                        && rsc->container->role == RSC_ROLE_STARTED) {
                 /* Both the connection and its containing resource need to be
                  * known to be up before we process resources running in it.
                  */
                 check = TRUE;
                 crm_trace("Checking node %s/%s/%s status %d/%d/%d", id, rsc->id, rsc->container->id, fence, rsc->role, RSC_ROLE_STARTED);
 
             } else if (!pe__is_guest_node(this_node)
                        && ((rsc->role == RSC_ROLE_STARTED)
                            || is_set(data_set->flags, pe_flag_shutdown_lock))) {
                 check = TRUE;
                 crm_trace("Checking node %s/%s status %d/%d/%d", id, rsc->id, fence, rsc->role, RSC_ROLE_STARTED);
             }
 
             if (check) {
                 determine_remote_online_status(data_set, this_node);
                 unpack_handle_remote_attrs(this_node, state, data_set);
                 process = TRUE;
             }
 
         } else if (this_node->details->online) {
             process = TRUE;
 
         } else if (fence) {
             process = TRUE;
 
         } else if (is_set(data_set->flags, pe_flag_shutdown_lock)) {
             process = TRUE;
         }
 
         if(process) {
             crm_trace("Processing lrm resource entries on %shealthy%s node: %s",
                       fence?"un":"",
                       (pe__is_guest_or_remote_node(this_node)? " remote" : ""),
                       this_node->details->uname);
             changed = TRUE;
             this_node->details->unpacked = TRUE;
 
             lrm_rsc = find_xml_node(state, XML_CIB_TAG_LRM, FALSE);
             lrm_rsc = find_xml_node(lrm_rsc, XML_LRM_TAG_RESOURCES, FALSE);
             unpack_lrm_resources(this_node, lrm_rsc, data_set);
         }
     }
     return changed;
 }
 
 /* remove nodes that are down, stopping */
 /* create positive rsc_to_node constraints between resources and the nodes they are running on */
 /* anything else? */
 gboolean
 unpack_status(xmlNode * status, pe_working_set_t * data_set)
 {
     const char *id = NULL;
     const char *uname = NULL;
 
     xmlNode *state = NULL;
     pe_node_t *this_node = NULL;
 
     crm_trace("Beginning unpack");
 
     if (data_set->tickets == NULL) {
         data_set->tickets = g_hash_table_new_full(crm_str_hash, g_str_equal,
                                                   free, destroy_ticket);
     }
 
     for (state = __xml_first_child_element(status); state != NULL;
          state = __xml_next_element(state)) {
 
         if (crm_str_eq((const char *)state->name, XML_CIB_TAG_TICKETS, TRUE)) {
             unpack_tickets_state((xmlNode *) state, data_set);
 
         } else if (crm_str_eq((const char *)state->name, XML_CIB_TAG_STATE, TRUE)) {
             xmlNode *attrs = NULL;
             const char *resource_discovery_enabled = NULL;
 
             id = crm_element_value(state, XML_ATTR_ID);
             uname = crm_element_value(state, XML_ATTR_UNAME);
             this_node = pe_find_node_any(data_set->nodes, id, uname);
 
             if (uname == NULL) {
                 /* error */
                 continue;
 
             } else if (this_node == NULL) {
                 pcmk__config_warn("Ignoring recorded node status for '%s' "
                                   "because no longer in configuration", uname);
                 continue;
 
             } else if (pe__is_guest_or_remote_node(this_node)) {
                 /* online state for remote nodes is determined by the
                  * rsc state after all the unpacking is done. we do however
                  * need to mark whether or not the node has been fenced as this plays
                  * a role during unpacking cluster node resource state */
                 this_node->details->remote_was_fenced = 
                     crm_atoi(crm_element_value(state, XML_NODE_IS_FENCED), "0");
                 continue;
             }
 
             crm_trace("Processing node id=%s, uname=%s", id, uname);
 
             /* Mark the node as provisionally clean
              * - at least we have seen it in the current cluster's lifetime
              */
             this_node->details->unclean = FALSE;
             this_node->details->unseen = FALSE;
             attrs = find_xml_node(state, XML_TAG_TRANSIENT_NODEATTRS, FALSE);
             add_node_attrs(attrs, this_node, TRUE, data_set);
 
             if (crm_is_true(pe_node_attribute_raw(this_node, "standby"))) {
                 crm_info("Node %s is in standby-mode", this_node->details->uname);
                 this_node->details->standby = TRUE;
             }
 
             if (crm_is_true(pe_node_attribute_raw(this_node, "maintenance"))) {
                 crm_info("Node %s is in maintenance-mode", this_node->details->uname);
                 this_node->details->maintenance = TRUE;
             }
 
             resource_discovery_enabled = pe_node_attribute_raw(this_node, XML_NODE_ATTR_RSC_DISCOVERY);
             if (resource_discovery_enabled && !crm_is_true(resource_discovery_enabled)) {
                 crm_warn("ignoring %s attribute on node %s, disabling resource discovery is not allowed on cluster nodes",
                     XML_NODE_ATTR_RSC_DISCOVERY, this_node->details->uname);
             }
 
             crm_trace("determining node state");
             determine_online_status(state, this_node, data_set);
 
             if (is_not_set(data_set->flags, pe_flag_have_quorum)
                 && this_node->details->online
                 && (data_set->no_quorum_policy == no_quorum_suicide)) {
                 /* Everything else should flow from this automatically
                  * (at least until the scheduler becomes able to migrate off
                  * healthy resources)
                  */
                 pe_fence_node(data_set, this_node, "cluster does not have quorum", FALSE);
             }
         }
     }
 
 
     while(unpack_node_loop(status, FALSE, data_set)) {
         crm_trace("Start another loop");
     }
 
     // Now catch any nodes we didn't see
     unpack_node_loop(status, is_set(data_set->flags, pe_flag_stonith_enabled), data_set);
 
     /* Now that we know where resources are, we can schedule stops of containers
      * with failed bundle connections
      */
     if (data_set->stop_needed != NULL) {
         for (GList *item = data_set->stop_needed; item; item = item->next) {
             pe_resource_t *container = item->data;
             pe_node_t *node = pe__current_node(container);
 
             if (node) {
                 stop_action(container, node, FALSE);
             }
         }
         g_list_free(data_set->stop_needed);
         data_set->stop_needed = NULL;
     }
 
     for (GListPtr gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
         pe_node_t *this_node = gIter->data;
 
         if (this_node == NULL) {
             continue;
         } else if (!pe__is_guest_or_remote_node(this_node)) {
             continue;
         } else if(this_node->details->unpacked) {
             continue;
         }
         determine_remote_online_status(data_set, this_node);
     }
 
     return TRUE;
 }
 
 static gboolean
 determine_online_status_no_fencing(pe_working_set_t * data_set, xmlNode * node_state,
                                    pe_node_t * this_node)
 {
     gboolean online = FALSE;
     const char *join = crm_element_value(node_state, XML_NODE_JOIN_STATE);
     const char *is_peer = crm_element_value(node_state, XML_NODE_IS_PEER);
     const char *in_cluster = crm_element_value(node_state, XML_NODE_IN_CLUSTER);
     const char *exp_state = crm_element_value(node_state, XML_NODE_EXPECTED);
 
     if (!crm_is_true(in_cluster)) {
         crm_trace("Node is down: in_cluster=%s", crm_str(in_cluster));
 
     } else if (safe_str_eq(is_peer, ONLINESTATUS)) {
         if (safe_str_eq(join, CRMD_JOINSTATE_MEMBER)) {
             online = TRUE;
         } else {
             crm_debug("Node is not ready to run resources: %s", join);
         }
 
     } else if (this_node->details->expected_up == FALSE) {
         crm_trace("Controller is down: in_cluster=%s", crm_str(in_cluster));
         crm_trace("\tis_peer=%s, join=%s, expected=%s",
                   crm_str(is_peer), crm_str(join), crm_str(exp_state));
 
     } else {
         /* mark it unclean */
         pe_fence_node(data_set, this_node, "peer is unexpectedly down", FALSE);
         crm_info("\tin_cluster=%s, is_peer=%s, join=%s, expected=%s",
                  crm_str(in_cluster), crm_str(is_peer), crm_str(join), crm_str(exp_state));
     }
     return online;
 }
 
 static gboolean
 determine_online_status_fencing(pe_working_set_t * data_set, xmlNode * node_state,
                                 pe_node_t * this_node)
 {
     gboolean online = FALSE;
     gboolean do_terminate = FALSE;
     bool crmd_online = FALSE;
     const char *join = crm_element_value(node_state, XML_NODE_JOIN_STATE);
     const char *is_peer = crm_element_value(node_state, XML_NODE_IS_PEER);
     const char *in_cluster = crm_element_value(node_state, XML_NODE_IN_CLUSTER);
     const char *exp_state = crm_element_value(node_state, XML_NODE_EXPECTED);
     const char *terminate = pe_node_attribute_raw(this_node, "terminate");
 
 /*
   - XML_NODE_IN_CLUSTER    ::= true|false
   - XML_NODE_IS_PEER       ::= online|offline
   - XML_NODE_JOIN_STATE    ::= member|down|pending|banned
   - XML_NODE_EXPECTED      ::= member|down
 */
 
     if (crm_is_true(terminate)) {
         do_terminate = TRUE;
 
     } else if (terminate != NULL && strlen(terminate) > 0) {
         /* could be a time() value */
         char t = terminate[0];
 
         if (t != '0' && isdigit(t)) {
             do_terminate = TRUE;
         }
     }
 
     crm_trace("%s: in_cluster=%s, is_peer=%s, join=%s, expected=%s, term=%d",
               this_node->details->uname, crm_str(in_cluster), crm_str(is_peer),
               crm_str(join), crm_str(exp_state), do_terminate);
 
     online = crm_is_true(in_cluster);
     crmd_online = safe_str_eq(is_peer, ONLINESTATUS);
     if (exp_state == NULL) {
         exp_state = CRMD_JOINSTATE_DOWN;
     }
 
     if (this_node->details->shutdown) {
         crm_debug("%s is shutting down", this_node->details->uname);
 
         /* Slightly different criteria since we can't shut down a dead peer */
         online = crmd_online;
 
     } else if (in_cluster == NULL) {
         pe_fence_node(data_set, this_node, "peer has not been seen by the cluster", FALSE);
 
     } else if (safe_str_eq(join, CRMD_JOINSTATE_NACK)) {
         pe_fence_node(data_set, this_node, "peer failed the pacemaker membership criteria", FALSE);
 
     } else if (do_terminate == FALSE && safe_str_eq(exp_state, CRMD_JOINSTATE_DOWN)) {
 
         if (crm_is_true(in_cluster) || crmd_online) {
             crm_info("- Node %s is not ready to run resources", this_node->details->uname);
             this_node->details->standby = TRUE;
             this_node->details->pending = TRUE;
 
         } else {
             crm_trace("%s is down or still coming up", this_node->details->uname);
         }
 
     } else if (do_terminate && safe_str_eq(join, CRMD_JOINSTATE_DOWN)
                && crm_is_true(in_cluster) == FALSE && !crmd_online) {
         crm_info("Node %s was just shot", this_node->details->uname);
         online = FALSE;
 
     } else if (crm_is_true(in_cluster) == FALSE) {
         // Consider `priority-fencing-delay` for lost nodes
         pe_fence_node(data_set, this_node, "peer is no longer part of the cluster", TRUE);
 
     } else if (!crmd_online) {
         pe_fence_node(data_set, this_node, "peer process is no longer available", FALSE);
 
         /* Everything is running at this point, now check join state */
     } else if (do_terminate) {
         pe_fence_node(data_set, this_node, "termination was requested", FALSE);
 
     } else if (safe_str_eq(join, CRMD_JOINSTATE_MEMBER)) {
         crm_info("Node %s is active", this_node->details->uname);
 
     } else if (safe_str_eq(join, CRMD_JOINSTATE_PENDING)
                || safe_str_eq(join, CRMD_JOINSTATE_DOWN)) {
         crm_info("Node %s is not ready to run resources", this_node->details->uname);
         this_node->details->standby = TRUE;
         this_node->details->pending = TRUE;
 
     } else {
         pe_fence_node(data_set, this_node, "peer was in an unknown state", FALSE);
         crm_warn("%s: in-cluster=%s, is-peer=%s, join=%s, expected=%s, term=%d, shutdown=%d",
                  this_node->details->uname, crm_str(in_cluster), crm_str(is_peer),
                  crm_str(join), crm_str(exp_state), do_terminate, this_node->details->shutdown);
     }
 
     return online;
 }
 
 static void
 determine_remote_online_status(pe_working_set_t * data_set, pe_node_t * this_node)
 {
     pe_resource_t *rsc = this_node->details->remote_rsc;
     pe_resource_t *container = NULL;
     pe_node_t *host = NULL;
 
     /* If there is a node state entry for a (former) Pacemaker Remote node
      * but no resource creating that node, the node's connection resource will
      * be NULL. Consider it an offline remote node in that case.
      */
     if (rsc == NULL) {
         this_node->details->online = FALSE;
         goto remote_online_done;
     }
 
     container = rsc->container;
 
     if (container && pcmk__list_of_1(rsc->running_on)) {
         host = rsc->running_on->data;
     }
 
     /* If the resource is currently started, mark it online. */
     if (rsc->role == RSC_ROLE_STARTED) {
         crm_trace("%s node %s presumed ONLINE because connection resource is started",
                   (container? "Guest" : "Remote"), this_node->details->id);
         this_node->details->online = TRUE;
     }
 
     /* consider this node shutting down if transitioning start->stop */
     if (rsc->role == RSC_ROLE_STARTED && rsc->next_role == RSC_ROLE_STOPPED) {
         crm_trace("%s node %s shutting down because connection resource is stopping",
                   (container? "Guest" : "Remote"), this_node->details->id);
         this_node->details->shutdown = TRUE;
     }
 
     /* Now check all the failure conditions. */
     if(container && is_set(container->flags, pe_rsc_failed)) {
         crm_trace("Guest node %s UNCLEAN because guest resource failed",
                   this_node->details->id);
         this_node->details->online = FALSE;
         this_node->details->remote_requires_reset = TRUE;
 
     } else if(is_set(rsc->flags, pe_rsc_failed)) {
         crm_trace("%s node %s OFFLINE because connection resource failed",
                   (container? "Guest" : "Remote"), this_node->details->id);
         this_node->details->online = FALSE;
 
     } else if (rsc->role == RSC_ROLE_STOPPED
         || (container && container->role == RSC_ROLE_STOPPED)) {
 
         crm_trace("%s node %s OFFLINE because its resource is stopped",
                   (container? "Guest" : "Remote"), this_node->details->id);
         this_node->details->online = FALSE;
         this_node->details->remote_requires_reset = FALSE;
 
     } else if (host && (host->details->online == FALSE)
                && host->details->unclean) {
         crm_trace("Guest node %s UNCLEAN because host is unclean",
                   this_node->details->id);
         this_node->details->online = FALSE;
         this_node->details->remote_requires_reset = TRUE;
     }
 
 remote_online_done:
     crm_trace("Remote node %s online=%s",
         this_node->details->id, this_node->details->online ? "TRUE" : "FALSE");
 }
 
 static void
 determine_online_status(xmlNode * node_state, pe_node_t * this_node, pe_working_set_t * data_set)
 {
     gboolean online = FALSE;
     const char *exp_state = crm_element_value(node_state, XML_NODE_EXPECTED);
 
     CRM_CHECK(this_node != NULL, return);
 
     this_node->details->shutdown = FALSE;
     this_node->details->expected_up = FALSE;
 
     if (pe__shutdown_requested(this_node)) {
         this_node->details->shutdown = TRUE;
 
     } else if (safe_str_eq(exp_state, CRMD_JOINSTATE_MEMBER)) {
         this_node->details->expected_up = TRUE;
     }
 
     if (this_node->details->type == node_ping) {
         this_node->details->unclean = FALSE;
         online = FALSE;         /* As far as resource management is concerned,
                                  * the node is safely offline.
                                  * Anyone caught abusing this logic will be shot
                                  */
 
     } else if (is_set(data_set->flags, pe_flag_stonith_enabled) == FALSE) {
         online = determine_online_status_no_fencing(data_set, node_state, this_node);
 
     } else {
         online = determine_online_status_fencing(data_set, node_state, this_node);
     }
 
     if (online) {
         this_node->details->online = TRUE;
 
     } else {
         /* remove node from contention */
         this_node->fixed = TRUE;
         this_node->weight = -INFINITY;
     }
 
     if (online && this_node->details->shutdown) {
         /* don't run resources here */
         this_node->fixed = TRUE;
         this_node->weight = -INFINITY;
     }
 
     if (this_node->details->type == node_ping) {
         crm_info("Node %s is not a pacemaker node", this_node->details->uname);
 
     } else if (this_node->details->unclean) {
         pe_proc_warn("Node %s is unclean", this_node->details->uname);
 
     } else if (this_node->details->online) {
         crm_info("Node %s is %s", this_node->details->uname,
                  this_node->details->shutdown ? "shutting down" :
                  this_node->details->pending ? "pending" :
                  this_node->details->standby ? "standby" :
                  this_node->details->maintenance ? "maintenance" : "online");
 
     } else {
         crm_trace("Node %s is offline", this_node->details->uname);
     }
 }
 
 /*!
  * \internal
  * \brief Find the end of a resource's name, excluding any clone suffix
  *
  * \param[in] id  Resource ID to check
  *
  * \return Pointer to last character of resource's base name
  */
 const char *
 pe_base_name_end(const char *id)
 {
     if (!pcmk__str_empty(id)) {
         const char *end = id + strlen(id) - 1;
 
         for (const char *s = end; s > id; --s) {
             switch (*s) {
                 case '0':
                 case '1':
                 case '2':
                 case '3':
                 case '4':
                 case '5':
                 case '6':
                 case '7':
                 case '8':
                 case '9':
                     break;
                 case ':':
                     return (s == end)? s : (s - 1);
                 default:
                     return end;
             }
         }
         return end;
     }
     return NULL;
 }
 
 /*!
  * \internal
  * \brief Get a resource name excluding any clone suffix
  *
  * \param[in] last_rsc_id  Resource ID to check
  *
  * \return Pointer to newly allocated string with resource's base name
  * \note It is the caller's responsibility to free() the result.
  *       This asserts on error, so callers can assume result is not NULL.
  */
 char *
 clone_strip(const char *last_rsc_id)
 {
     const char *end = pe_base_name_end(last_rsc_id);
     char *basename = NULL;
 
     CRM_ASSERT(end);
     basename = strndup(last_rsc_id, end - last_rsc_id + 1);
     CRM_ASSERT(basename);
     return basename;
 }
 
 /*!
  * \internal
  * \brief Get the name of the first instance of a cloned resource
  *
  * \param[in] last_rsc_id  Resource ID to check
  *
  * \return Pointer to newly allocated string with resource's base name plus :0
  * \note It is the caller's responsibility to free() the result.
  *       This asserts on error, so callers can assume result is not NULL.
  */
 char *
 clone_zero(const char *last_rsc_id)
 {
     const char *end = pe_base_name_end(last_rsc_id);
     size_t base_name_len = end - last_rsc_id + 1;
     char *zero = NULL;
 
     CRM_ASSERT(end);
     zero = calloc(base_name_len + 3, sizeof(char));
     CRM_ASSERT(zero);
     memcpy(zero, last_rsc_id, base_name_len);
     zero[base_name_len] = ':';
     zero[base_name_len + 1] = '0';
     return zero;
 }
 
 static pe_resource_t *
 create_fake_resource(const char *rsc_id, xmlNode * rsc_entry, pe_working_set_t * data_set)
 {
     pe_resource_t *rsc = NULL;
     xmlNode *xml_rsc = create_xml_node(NULL, XML_CIB_TAG_RESOURCE);
 
     copy_in_properties(xml_rsc, rsc_entry);
     crm_xml_add(xml_rsc, XML_ATTR_ID, rsc_id);
     crm_log_xml_debug(xml_rsc, "Orphan resource");
 
     if (!common_unpack(xml_rsc, &rsc, NULL, data_set)) {
         return NULL;
     }
 
     if (xml_contains_remote_node(xml_rsc)) {
         pe_node_t *node;
 
         crm_debug("Detected orphaned remote node %s", rsc_id);
         node = pe_find_node(data_set->nodes, rsc_id);
         if (node == NULL) {
 	        node = pe_create_node(rsc_id, rsc_id, "remote", NULL, data_set);
         }
         link_rsc2remotenode(data_set, rsc);
 
         if (node) {
             crm_trace("Setting node %s as shutting down due to orphaned connection resource", rsc_id);
             node->details->shutdown = TRUE;
         }
     }
 
     if (crm_element_value(rsc_entry, XML_RSC_ATTR_CONTAINER)) {
         /* This orphaned rsc needs to be mapped to a container. */
         crm_trace("Detected orphaned container filler %s", rsc_id);
         set_bit(rsc->flags, pe_rsc_orphan_container_filler);
     }
     set_bit(rsc->flags, pe_rsc_orphan);
     data_set->resources = g_list_append(data_set->resources, rsc);
     return rsc;
 }
 
 /*!
  * \internal
  * \brief Create orphan instance for anonymous clone resource history
  */
 static pe_resource_t *
 create_anonymous_orphan(pe_resource_t *parent, const char *rsc_id,
                         pe_node_t *node, pe_working_set_t *data_set)
 {
     pe_resource_t *top = pe__create_clone_child(parent, data_set);
 
     // find_rsc() because we might be a cloned group
     pe_resource_t *orphan = top->fns->find_rsc(top, rsc_id, NULL, pe_find_clone);
 
     pe_rsc_debug(parent, "Created orphan %s for %s: %s on %s",
                  top->id, parent->id, rsc_id, node->details->uname);
     return orphan;
 }
 
 /*!
  * \internal
  * \brief Check a node for an instance of an anonymous clone
  *
  * Return a child instance of the specified anonymous clone, in order of
  * preference: (1) the instance running on the specified node, if any;
  * (2) an inactive instance (i.e. within the total of clone-max instances);
  * (3) a newly created orphan (i.e. clone-max instances are already active).
  *
  * \param[in] data_set  Cluster information
  * \param[in] node      Node on which to check for instance
  * \param[in] parent    Clone to check
  * \param[in] rsc_id    Name of cloned resource in history (without instance)
  */
 static pe_resource_t *
 find_anonymous_clone(pe_working_set_t * data_set, pe_node_t * node, pe_resource_t * parent,
                      const char *rsc_id)
 {
     GListPtr rIter = NULL;
     pe_resource_t *rsc = NULL;
     pe_resource_t *inactive_instance = NULL;
     gboolean skip_inactive = FALSE;
 
     CRM_ASSERT(parent != NULL);
     CRM_ASSERT(pe_rsc_is_clone(parent));
     CRM_ASSERT(is_not_set(parent->flags, pe_rsc_unique));
 
     // Check for active (or partially active, for cloned groups) instance
     pe_rsc_trace(parent, "Looking for %s on %s in %s", rsc_id, node->details->uname, parent->id);
     for (rIter = parent->children; rsc == NULL && rIter; rIter = rIter->next) {
         GListPtr locations = NULL;
         pe_resource_t *child = rIter->data;
 
         /* Check whether this instance is already known to be active or pending
          * anywhere, at this stage of unpacking. Because this function is called
          * for a resource before the resource's individual operation history
          * entries are unpacked, locations will generally not contain the
          * desired node.
          *
          * However, there are three exceptions:
          * (1) when child is a cloned group and we have already unpacked the
          *     history of another member of the group on the same node;
          * (2) when we've already unpacked the history of another numbered
          *     instance on the same node (which can happen if globally-unique
          *     was flipped from true to false); and
          * (3) when we re-run calculations on the same data set as part of a
          *     simulation.
          */
         child->fns->location(child, &locations, 2);
         if (locations) {
             /* We should never associate the same numbered anonymous clone
              * instance with multiple nodes, and clone instances can't migrate,
              * so there must be only one location, regardless of history.
              */
             CRM_LOG_ASSERT(locations->next == NULL);
 
             if (((pe_node_t *)locations->data)->details == node->details) {
                 /* This child instance is active on the requested node, so check
                  * for a corresponding configured resource. We use find_rsc()
                  * instead of child because child may be a cloned group, and we
                  * need the particular member corresponding to rsc_id.
                  *
                  * If the history entry is orphaned, rsc will be NULL.
                  */
                 rsc = parent->fns->find_rsc(child, rsc_id, NULL, pe_find_clone);
                 if (rsc) {
                     /* If there are multiple instance history entries for an
                      * anonymous clone in a single node's history (which can
                      * happen if globally-unique is switched from true to
                      * false), we want to consider the instances beyond the
                      * first as orphans, even if there are inactive instance
                      * numbers available.
                      */
                     if (rsc->running_on) {
                         crm_notice("Active (now-)anonymous clone %s has "
                                    "multiple (orphan) instance histories on %s",
                                    parent->id, node->details->uname);
                         skip_inactive = TRUE;
                         rsc = NULL;
                     } else {
                         pe_rsc_trace(parent, "Resource %s, active", rsc->id);
                     }
                 }
             }
             g_list_free(locations);
 
         } else {
             pe_rsc_trace(parent, "Resource %s, skip inactive", child->id);
             if (!skip_inactive && !inactive_instance
                 && is_not_set(child->flags, pe_rsc_block)) {
                 // Remember one inactive instance in case we don't find active
                 inactive_instance = parent->fns->find_rsc(child, rsc_id, NULL,
                                                           pe_find_clone);
 
                 /* ... but don't use it if it was already associated with a
                  * pending action on another node
                  */
                 if (inactive_instance && inactive_instance->pending_node
                     && (inactive_instance->pending_node->details != node->details)) {
                     inactive_instance = NULL;
                 }
             }
         }
     }
 
     if ((rsc == NULL) && !skip_inactive && (inactive_instance != NULL)) {
         pe_rsc_trace(parent, "Resource %s, empty slot", inactive_instance->id);
         rsc = inactive_instance;
     }
 
     /* If the resource has "requires" set to "quorum" or "nothing", and we don't
      * have a clone instance for every node, we don't want to consume a valid
      * instance number for unclean nodes. Such instances may appear to be active
      * according to the history, but should be considered inactive, so we can
      * start an instance elsewhere. Treat such instances as orphans.
      *
      * An exception is instances running on guest nodes -- since guest node
      * "fencing" is actually just a resource stop, requires shouldn't apply.
      *
      * @TODO Ideally, we'd use an inactive instance number if it is not needed
      * for any clean instances. However, we don't know that at this point.
      */
     if ((rsc != NULL) && is_not_set(rsc->flags, pe_rsc_needs_fencing)
         && (!node->details->online || node->details->unclean)
         && !pe__is_guest_node(node)
         && !pe__is_universal_clone(parent, data_set)) {
 
         rsc = NULL;
     }
 
     if (rsc == NULL) {
         rsc = create_anonymous_orphan(parent, rsc_id, node, data_set);
         pe_rsc_trace(parent, "Resource %s, orphan", rsc->id);
     }
     return rsc;
 }
 
 static pe_resource_t *
 unpack_find_resource(pe_working_set_t * data_set, pe_node_t * node, const char *rsc_id,
                      xmlNode * rsc_entry)
 {
     pe_resource_t *rsc = NULL;
     pe_resource_t *parent = NULL;
 
     crm_trace("looking for %s", rsc_id);
     rsc = pe_find_resource(data_set->resources, rsc_id);
 
     if (rsc == NULL) {
         /* If we didn't find the resource by its name in the operation history,
          * check it again as a clone instance. Even when clone-max=0, we create
          * a single :0 orphan to match against here.
          */
         char *clone0_id = clone_zero(rsc_id);
         pe_resource_t *clone0 = pe_find_resource(data_set->resources, clone0_id);
 
         if (clone0 && is_not_set(clone0->flags, pe_rsc_unique)) {
             rsc = clone0;
             parent = uber_parent(clone0);
             crm_trace("%s found as %s (%s)", rsc_id, clone0_id, parent->id);
         } else {
             crm_trace("%s is not known as %s either (orphan)",
                       rsc_id, clone0_id);
         }
         free(clone0_id);
 
     } else if (rsc->variant > pe_native) {
         crm_trace("Resource history for %s is orphaned because it is no longer primitive",
                   rsc_id);
         return NULL;
 
     } else {
         parent = uber_parent(rsc);
     }
 
     if (pe_rsc_is_anon_clone(parent)) {
 
         if (pe_rsc_is_bundled(parent)) {
             rsc = pe__find_bundle_replica(parent->parent, node);
         } else {
             char *base = clone_strip(rsc_id);
 
             rsc = find_anonymous_clone(data_set, node, parent, base);
             free(base);
             CRM_ASSERT(rsc != NULL);
         }
     }
 
     if (rsc && safe_str_neq(rsc_id, rsc->id)
         && safe_str_neq(rsc_id, rsc->clone_name)) {
 
         free(rsc->clone_name);
         rsc->clone_name = strdup(rsc_id);
         pe_rsc_debug(rsc, "Internally renamed %s on %s to %s%s",
                      rsc_id, node->details->uname, rsc->id,
                      (is_set(rsc->flags, pe_rsc_orphan)? " (ORPHAN)" : ""));
     }
     return rsc;
 }
 
 static pe_resource_t *
 process_orphan_resource(xmlNode * rsc_entry, pe_node_t * node, pe_working_set_t * data_set)
 {
     pe_resource_t *rsc = NULL;
     const char *rsc_id = crm_element_value(rsc_entry, XML_ATTR_ID);
 
     crm_debug("Detected orphan resource %s on %s", rsc_id, node->details->uname);
     rsc = create_fake_resource(rsc_id, rsc_entry, data_set);
 
     if (is_set(data_set->flags, pe_flag_stop_rsc_orphans) == FALSE) {
         clear_bit(rsc->flags, pe_rsc_managed);
 
     } else {
         CRM_CHECK(rsc != NULL, return NULL);
         pe_rsc_trace(rsc, "Added orphan %s", rsc->id);
         resource_location(rsc, NULL, -INFINITY, "__orphan_do_not_run__", data_set);
     }
     return rsc;
 }
 
 static void
 process_rsc_state(pe_resource_t * rsc, pe_node_t * node,
                   enum action_fail_response on_fail,
                   xmlNode * migrate_op, pe_working_set_t * data_set)
 {
     pe_node_t *tmpnode = NULL;
     char *reason = NULL;
 
     CRM_ASSERT(rsc);
     pe_rsc_trace(rsc, "Resource %s is %s on %s: on_fail=%s",
                  rsc->id, role2text(rsc->role), node->details->uname, fail2text(on_fail));
 
     /* process current state */
     if (rsc->role != RSC_ROLE_UNKNOWN) {
         pe_resource_t *iter = rsc;
 
         while (iter) {
             if (g_hash_table_lookup(iter->known_on, node->details->id) == NULL) {
                 pe_node_t *n = pe__copy_node(node);
 
                 pe_rsc_trace(rsc, "%s (aka. %s) known on %s", rsc->id, rsc->clone_name,
                              n->details->uname);
                 g_hash_table_insert(iter->known_on, (gpointer) n->details->id, n);
             }
             if (is_set(iter->flags, pe_rsc_unique)) {
                 break;
             }
             iter = iter->parent;
         }
     }
 
     /* If a managed resource is believed to be running, but node is down ... */
     if (rsc->role > RSC_ROLE_STOPPED
         && node->details->online == FALSE
         && node->details->maintenance == FALSE
         && is_set(rsc->flags, pe_rsc_managed)) {
 
         gboolean should_fence = FALSE;
 
         /* If this is a guest node, fence it (regardless of whether fencing is
          * enabled, because guest node fencing is done by recovery of the
          * container resource rather than by the fencer). Mark the resource
          * we're processing as failed. When the guest comes back up, its
          * operation history in the CIB will be cleared, freeing the affected
          * resource to run again once we are sure we know its state.
          */
         if (pe__is_guest_node(node)) {
             set_bit(rsc->flags, pe_rsc_failed);
             should_fence = TRUE;
 
         } else if (is_set(data_set->flags, pe_flag_stonith_enabled)) {
             if (pe__is_remote_node(node) && node->details->remote_rsc
                 && is_not_set(node->details->remote_rsc->flags, pe_rsc_failed)) {
 
                 /* Setting unseen means that fencing of the remote node will
                  * occur only if the connection resource is not going to start
                  * somewhere. This allows connection resources on a failed
                  * cluster node to move to another node without requiring the
                  * remote nodes to be fenced as well.
                  */
                 node->details->unseen = TRUE;
                 reason = crm_strdup_printf("%s is active there (fencing will be"
                                            " revoked if remote connection can "
                                            "be re-established elsewhere)",
                                            rsc->id);
             }
             should_fence = TRUE;
         }
 
         if (should_fence) {
             if (reason == NULL) {
                reason = crm_strdup_printf("%s is thought to be active there", rsc->id);
             }
             pe_fence_node(data_set, node, reason, FALSE);
         }
         free(reason);
     }
 
     if (node->details->unclean) {
         /* No extra processing needed
          * Also allows resources to be started again after a node is shot
          */
         on_fail = action_fail_ignore;
     }
 
     switch (on_fail) {
         case action_fail_ignore:
             /* nothing to do */
             break;
 
         case action_fail_fence:
             /* treat it as if it is still running
              * but also mark the node as unclean
              */
             reason = crm_strdup_printf("%s failed there", rsc->id);
             pe_fence_node(data_set, node, reason, FALSE);
             free(reason);
             break;
 
         case action_fail_standby:
             node->details->standby = TRUE;
             node->details->standby_onfail = TRUE;
             break;
 
         case action_fail_block:
             /* is_managed == FALSE will prevent any
              * actions being sent for the resource
              */
             clear_bit(rsc->flags, pe_rsc_managed);
             set_bit(rsc->flags, pe_rsc_block);
             break;
 
         case action_fail_migrate:
             /* make sure it comes up somewhere else
              * or not at all
              */
             resource_location(rsc, node, -INFINITY, "__action_migration_auto__", data_set);
             break;
 
         case action_fail_stop:
             rsc->next_role = RSC_ROLE_STOPPED;
             break;
 
         case action_fail_recover:
             if (rsc->role != RSC_ROLE_STOPPED && rsc->role != RSC_ROLE_UNKNOWN) {
                 set_bit(rsc->flags, pe_rsc_failed);
                 stop_action(rsc, node, FALSE);
             }
             break;
 
         case action_fail_restart_container:
             set_bit(rsc->flags, pe_rsc_failed);
 
             if (rsc->container && pe_rsc_is_bundled(rsc)) {
                 /* A bundle's remote connection can run on a different node than
                  * the bundle's container. We don't necessarily know where the
                  * container is running yet, so remember it and add a stop
                  * action for it later.
                  */
                 data_set->stop_needed = g_list_prepend(data_set->stop_needed,
                                                        rsc->container);
             } else if (rsc->container) {
                 stop_action(rsc->container, node, FALSE);
             } else if (rsc->role != RSC_ROLE_STOPPED && rsc->role != RSC_ROLE_UNKNOWN) {
                 stop_action(rsc, node, FALSE);
             }
             break;
 
         case action_fail_reset_remote:
             set_bit(rsc->flags, pe_rsc_failed);
             if (is_set(data_set->flags, pe_flag_stonith_enabled)) {
                 tmpnode = NULL;
                 if (rsc->is_remote_node) {
                     tmpnode = pe_find_node(data_set->nodes, rsc->id);
                 }
                 if (tmpnode &&
                     pe__is_remote_node(tmpnode) &&
                     tmpnode->details->remote_was_fenced == 0) {
 
                     /* The remote connection resource failed in a way that
                      * should result in fencing the remote node.
                      */
                     pe_fence_node(data_set, tmpnode,
                                   "remote connection is unrecoverable", FALSE);
                 }
             }
 
             /* require the stop action regardless if fencing is occurring or not. */
             if (rsc->role > RSC_ROLE_STOPPED) {
                 stop_action(rsc, node, FALSE);
             }
 
             /* if reconnect delay is in use, prevent the connection from exiting the
              * "STOPPED" role until the failure is cleared by the delay timeout. */
             if (rsc->remote_reconnect_ms) {
                 rsc->next_role = RSC_ROLE_STOPPED;
             }
             break;
     }
 
     /* ensure a remote-node connection failure forces an unclean remote-node
      * to be fenced. By setting unseen = FALSE, the remote-node failure will
      * result in a fencing operation regardless if we're going to attempt to 
      * reconnect to the remote-node in this transition or not. */
     if (is_set(rsc->flags, pe_rsc_failed) && rsc->is_remote_node) {
         tmpnode = pe_find_node(data_set->nodes, rsc->id);
         if (tmpnode && tmpnode->details->unclean) {
             tmpnode->details->unseen = FALSE;
         }
     }
 
     if (rsc->role != RSC_ROLE_STOPPED && rsc->role != RSC_ROLE_UNKNOWN) {
         if (is_set(rsc->flags, pe_rsc_orphan)) {
             if (is_set(rsc->flags, pe_rsc_managed)) {
                 pcmk__config_warn("Detected active orphan %s running on %s",
                                   rsc->id, node->details->uname);
             } else {
                 pcmk__config_warn("Resource '%s' must be stopped manually on "
                                   "%s because cluster is configured not to "
                                   "stop active orphans",
                                   rsc->id, node->details->uname);
             }
         }
 
         native_add_running(rsc, node, data_set);
         if (on_fail != action_fail_ignore) {
             set_bit(rsc->flags, pe_rsc_failed);
         }
 
     } else if (rsc->clone_name && strchr(rsc->clone_name, ':') != NULL) {
         /* Only do this for older status sections that included instance numbers
          * Otherwise stopped instances will appear as orphans
          */
         pe_rsc_trace(rsc, "Resetting clone_name %s for %s (stopped)", rsc->clone_name, rsc->id);
         free(rsc->clone_name);
         rsc->clone_name = NULL;
 
     } else {
         GList *possible_matches = pe__resource_actions(rsc, node, RSC_STOP,
                                                        FALSE);
         GListPtr gIter = possible_matches;
 
         for (; gIter != NULL; gIter = gIter->next) {
             pe_action_t *stop = (pe_action_t *) gIter->data;
 
             stop->flags |= pe_action_optional;
         }
 
         g_list_free(possible_matches);
     }
 }
 
 /* create active recurring operations as optional */
 static void
 process_recurring(pe_node_t * node, pe_resource_t * rsc,
                   int start_index, int stop_index,
                   GListPtr sorted_op_list, pe_working_set_t * data_set)
 {
     int counter = -1;
     const char *task = NULL;
     const char *status = NULL;
     GListPtr gIter = sorted_op_list;
 
     CRM_ASSERT(rsc);
     pe_rsc_trace(rsc, "%s: Start index %d, stop index = %d", rsc->id, start_index, stop_index);
 
     for (; gIter != NULL; gIter = gIter->next) {
         xmlNode *rsc_op = (xmlNode *) gIter->data;
 
         guint interval_ms = 0;
         char *key = NULL;
         const char *id = ID(rsc_op);
 
         counter++;
 
         if (node->details->online == FALSE) {
             pe_rsc_trace(rsc, "Skipping %s/%s: node is offline", rsc->id, node->details->uname);
             break;
 
             /* Need to check if there's a monitor for role="Stopped" */
         } else if (start_index < stop_index && counter <= stop_index) {
             pe_rsc_trace(rsc, "Skipping %s/%s: resource is not active", id, node->details->uname);
             continue;
 
         } else if (counter < start_index) {
             pe_rsc_trace(rsc, "Skipping %s/%s: old %d", id, node->details->uname, counter);
             continue;
         }
 
         crm_element_value_ms(rsc_op, XML_LRM_ATTR_INTERVAL_MS, &interval_ms);
         if (interval_ms == 0) {
             pe_rsc_trace(rsc, "Skipping %s/%s: non-recurring", id, node->details->uname);
             continue;
         }
 
         status = crm_element_value(rsc_op, XML_LRM_ATTR_OPSTATUS);
         if (safe_str_eq(status, "-1")) {
             pe_rsc_trace(rsc, "Skipping %s/%s: status", id, node->details->uname);
             continue;
         }
         task = crm_element_value(rsc_op, XML_LRM_ATTR_TASK);
         /* create the action */
         key = pcmk__op_key(rsc->id, task, interval_ms);
         pe_rsc_trace(rsc, "Creating %s/%s", key, node->details->uname);
         custom_action(rsc, key, task, node, TRUE, TRUE, data_set);
     }
 }
 
 void
 calculate_active_ops(GListPtr sorted_op_list, int *start_index, int *stop_index)
 {
     int counter = -1;
     int implied_monitor_start = -1;
     int implied_clone_start = -1;
     const char *task = NULL;
     const char *status = NULL;
     GListPtr gIter = sorted_op_list;
 
     *stop_index = -1;
     *start_index = -1;
 
     for (; gIter != NULL; gIter = gIter->next) {
         xmlNode *rsc_op = (xmlNode *) gIter->data;
 
         counter++;
 
         task = crm_element_value(rsc_op, XML_LRM_ATTR_TASK);
         status = crm_element_value(rsc_op, XML_LRM_ATTR_OPSTATUS);
 
         if (safe_str_eq(task, CRMD_ACTION_STOP)
             && safe_str_eq(status, "0")) {
             *stop_index = counter;
 
         } else if (safe_str_eq(task, CRMD_ACTION_START) || safe_str_eq(task, CRMD_ACTION_MIGRATED)) {
             *start_index = counter;
 
         } else if ((implied_monitor_start <= *stop_index) && safe_str_eq(task, CRMD_ACTION_STATUS)) {
             const char *rc = crm_element_value(rsc_op, XML_LRM_ATTR_RC);
 
             if (safe_str_eq(rc, "0") || safe_str_eq(rc, "8")) {
                 implied_monitor_start = counter;
             }
         } else if (safe_str_eq(task, CRMD_ACTION_PROMOTE) || safe_str_eq(task, CRMD_ACTION_DEMOTE)) {
             implied_clone_start = counter;
         }
     }
 
     if (*start_index == -1) {
         if (implied_clone_start != -1) {
             *start_index = implied_clone_start;
         } else if (implied_monitor_start != -1) {
             *start_index = implied_monitor_start;
         }
     }
 }
 
 // If resource history entry has shutdown lock, remember lock node and time
 static void
 unpack_shutdown_lock(xmlNode *rsc_entry, pe_resource_t *rsc, pe_node_t *node,
                      pe_working_set_t *data_set)
 {
     time_t lock_time = 0;   // When lock started (i.e. node shutdown time)
 
     if ((crm_element_value_epoch(rsc_entry, XML_CONFIG_ATTR_SHUTDOWN_LOCK,
                                  &lock_time) == pcmk_ok) && (lock_time != 0)) {
 
         if ((data_set->shutdown_lock > 0)
             && (get_effective_time(data_set)
                 > (lock_time + data_set->shutdown_lock))) {
             pe_rsc_info(rsc, "Shutdown lock for %s on %s expired",
                         rsc->id, node->details->uname);
             pe__clear_resource_history(rsc, node, data_set);
         } else {
             rsc->lock_node = node;
             rsc->lock_time = lock_time;
         }
     }
 }
 
 static pe_resource_t *
 unpack_lrm_rsc_state(pe_node_t * node, xmlNode * rsc_entry, pe_working_set_t * data_set)
 {
     GListPtr gIter = NULL;
     int stop_index = -1;
     int start_index = -1;
     enum rsc_role_e req_role = RSC_ROLE_UNKNOWN;
 
     const char *task = NULL;
     const char *rsc_id = crm_element_value(rsc_entry, XML_ATTR_ID);
 
     pe_resource_t *rsc = NULL;
     GListPtr op_list = NULL;
     GListPtr sorted_op_list = NULL;
 
     xmlNode *migrate_op = NULL;
     xmlNode *rsc_op = NULL;
     xmlNode *last_failure = NULL;
 
     enum action_fail_response on_fail = FALSE;
     enum rsc_role_e saved_role = RSC_ROLE_UNKNOWN;
 
     crm_trace("[%s] Processing %s on %s",
               crm_element_name(rsc_entry), rsc_id, node->details->uname);
 
     /* extract operations */
     op_list = NULL;
     sorted_op_list = NULL;
 
     for (rsc_op = __xml_first_child_element(rsc_entry); rsc_op != NULL;
          rsc_op = __xml_next_element(rsc_op)) {
         if (crm_str_eq((const char *)rsc_op->name, XML_LRM_TAG_RSC_OP, TRUE)) {
             op_list = g_list_prepend(op_list, rsc_op);
         }
     }
 
     if (is_not_set(data_set->flags, pe_flag_shutdown_lock)) {
         if (op_list == NULL) {
             // If there are no operations, there is nothing to do
             return NULL;
         }
     }
 
     /* find the resource */
     rsc = unpack_find_resource(data_set, node, rsc_id, rsc_entry);
     if (rsc == NULL) {
         if (op_list == NULL) {
             // If there are no operations, there is nothing to do
             return NULL;
         } else {
             rsc = process_orphan_resource(rsc_entry, node, data_set);
         }
     }
     CRM_ASSERT(rsc != NULL);
 
     // Check whether the resource is "shutdown-locked" to this node
     if (is_set(data_set->flags, pe_flag_shutdown_lock)) {
         unpack_shutdown_lock(rsc_entry, rsc, node, data_set);
     }
 
     /* process operations */
     saved_role = rsc->role;
     on_fail = action_fail_ignore;
     rsc->role = RSC_ROLE_UNKNOWN;
     sorted_op_list = g_list_sort(op_list, sort_op_by_callid);
 
     for (gIter = sorted_op_list; gIter != NULL; gIter = gIter->next) {
         xmlNode *rsc_op = (xmlNode *) gIter->data;
 
         task = crm_element_value(rsc_op, XML_LRM_ATTR_TASK);
         if (safe_str_eq(task, CRMD_ACTION_MIGRATED)) {
             migrate_op = rsc_op;
         }
 
         unpack_rsc_op(rsc, node, rsc_op, &last_failure, &on_fail, data_set);
     }
 
     /* create active recurring operations as optional */
     calculate_active_ops(sorted_op_list, &start_index, &stop_index);
     process_recurring(node, rsc, start_index, stop_index, sorted_op_list, data_set);
 
     /* no need to free the contents */
     g_list_free(sorted_op_list);
 
     process_rsc_state(rsc, node, on_fail, migrate_op, data_set);
 
     if (get_target_role(rsc, &req_role)) {
         if (rsc->next_role == RSC_ROLE_UNKNOWN || req_role < rsc->next_role) {
             pe_rsc_debug(rsc, "%s: Overwriting calculated next role %s"
                          " with requested next role %s",
                          rsc->id, role2text(rsc->next_role), role2text(req_role));
             rsc->next_role = req_role;
 
         } else if (req_role > rsc->next_role) {
             pe_rsc_info(rsc, "%s: Not overwriting calculated next role %s"
                         " with requested next role %s",
                         rsc->id, role2text(rsc->next_role), role2text(req_role));
         }
     }
 
     if (saved_role > rsc->role) {
         rsc->role = saved_role;
     }
 
     return rsc;
 }
 
 static void
 handle_orphaned_container_fillers(xmlNode * lrm_rsc_list, pe_working_set_t * data_set)
 {
     xmlNode *rsc_entry = NULL;
     for (rsc_entry = __xml_first_child_element(lrm_rsc_list); rsc_entry != NULL;
          rsc_entry = __xml_next_element(rsc_entry)) {
 
         pe_resource_t *rsc;
         pe_resource_t *container;
         const char *rsc_id;
         const char *container_id;
 
         if (safe_str_neq((const char *)rsc_entry->name, XML_LRM_TAG_RESOURCE)) {
             continue;
         }
 
         container_id = crm_element_value(rsc_entry, XML_RSC_ATTR_CONTAINER);
         rsc_id = crm_element_value(rsc_entry, XML_ATTR_ID);
         if (container_id == NULL || rsc_id == NULL) {
             continue;
         }
 
         container = pe_find_resource(data_set->resources, container_id);
         if (container == NULL) {
             continue;
         }
 
         rsc = pe_find_resource(data_set->resources, rsc_id);
         if (rsc == NULL ||
             is_set(rsc->flags, pe_rsc_orphan_container_filler) == FALSE ||
             rsc->container != NULL) {
             continue;
         }
 
         pe_rsc_trace(rsc, "Mapped container of orphaned resource %s to %s",
                      rsc->id, container_id);
         rsc->container = container;
         container->fillers = g_list_append(container->fillers, rsc);
     }
 }
 
 static void
 unpack_lrm_resources(pe_node_t *node, xmlNode *lrm_rsc_list,
                      pe_working_set_t *data_set)
 {
     xmlNode *rsc_entry = NULL;
     gboolean found_orphaned_container_filler = FALSE;
 
     for (rsc_entry = __xml_first_child_element(lrm_rsc_list); rsc_entry != NULL;
          rsc_entry = __xml_next_element(rsc_entry)) {
 
         if (crm_str_eq((const char *)rsc_entry->name, XML_LRM_TAG_RESOURCE, TRUE)) {
             pe_resource_t *rsc = unpack_lrm_rsc_state(node, rsc_entry, data_set);
             if (!rsc) {
                 continue;
             }
             if (is_set(rsc->flags, pe_rsc_orphan_container_filler)) {
                 found_orphaned_container_filler = TRUE;
             }
         }
     }
 
     /* now that all the resource state has been unpacked for this node
      * we have to go back and map any orphaned container fillers to their
      * container resource */
     if (found_orphaned_container_filler) {
         handle_orphaned_container_fillers(lrm_rsc_list, data_set);
     }
 }
 
 static void
 set_active(pe_resource_t * rsc)
 {
     pe_resource_t *top = uber_parent(rsc);
 
     if (top && is_set(top->flags, pe_rsc_promotable)) {
         rsc->role = RSC_ROLE_SLAVE;
     } else {
         rsc->role = RSC_ROLE_STARTED;
     }
 }
 
 static void
 set_node_score(gpointer key, gpointer value, gpointer user_data)
 {
     pe_node_t *node = value;
     int *score = user_data;
 
     node->weight = *score;
 }
 
 #define STATUS_PATH_MAX 1024
 static xmlNode *
 find_lrm_op(const char *resource, const char *op, const char *node, const char *source,
             bool success_only, pe_working_set_t *data_set)
 {
     int offset = 0;
     char xpath[STATUS_PATH_MAX];
     xmlNode *xml = NULL;
 
     offset += snprintf(xpath + offset, STATUS_PATH_MAX - offset, "//node_state[@uname='%s']", node);
     offset +=
         snprintf(xpath + offset, STATUS_PATH_MAX - offset, "//" XML_LRM_TAG_RESOURCE "[@id='%s']",
                  resource);
 
     /* Need to check against transition_magic too? */
     if (source && safe_str_eq(op, CRMD_ACTION_MIGRATE)) {
         offset +=
             snprintf(xpath + offset, STATUS_PATH_MAX - offset,
                      "/" XML_LRM_TAG_RSC_OP "[@operation='%s' and @migrate_target='%s']", op,
                      source);
     } else if (source && safe_str_eq(op, CRMD_ACTION_MIGRATED)) {
         offset +=
             snprintf(xpath + offset, STATUS_PATH_MAX - offset,
                      "/" XML_LRM_TAG_RSC_OP "[@operation='%s' and @migrate_source='%s']", op,
                      source);
     } else {
         offset +=
             snprintf(xpath + offset, STATUS_PATH_MAX - offset,
                      "/" XML_LRM_TAG_RSC_OP "[@operation='%s']", op);
     }
 
     CRM_LOG_ASSERT(offset > 0);
     xml = get_xpath_object(xpath, data_set->input, LOG_DEBUG);
 
     if (xml && success_only) {
         int rc = PCMK_OCF_UNKNOWN_ERROR;
         int status = PCMK_LRM_OP_ERROR;
 
         crm_element_value_int(xml, XML_LRM_ATTR_RC, &rc);
         crm_element_value_int(xml, XML_LRM_ATTR_OPSTATUS, &status);
         if ((rc != PCMK_OCF_OK) || (status != PCMK_LRM_OP_DONE)) {
             return NULL;
         }
     }
     return xml;
 }
 
 static int
 pe__call_id(xmlNode *op_xml)
 {
     int id = 0;
 
     if (op_xml) {
         crm_element_value_int(op_xml, XML_LRM_ATTR_CALLID, &id);
     }
     return id;
 }
 
 /*!
  * \brief Check whether a stop happened on the same node after some event
  *
  * \param[in] rsc       Resource being checked
  * \param[in] node      Node being checked
  * \param[in] xml_op    Event that stop is being compared to
  * \param[in] data_set  Cluster working set
  *
  * \return TRUE if stop happened after event, FALSE otherwise
  *
  * \note This is really unnecessary, but kept as a safety mechanism. We
  *       currently don't save more than one successful event in history, so this
  *       only matters when processing really old CIB files that we don't
  *       technically support anymore, or as preparation for logging an extended
  *       history in the future.
  */
 static bool
 stop_happened_after(pe_resource_t *rsc, pe_node_t *node, xmlNode *xml_op,
                     pe_working_set_t *data_set)
 {
     xmlNode *stop_op = find_lrm_op(rsc->id, CRMD_ACTION_STOP,
                                    node->details->uname, NULL, TRUE, data_set);
 
     return (stop_op && (pe__call_id(stop_op) > pe__call_id(xml_op)));
 }
 
 static void
 unpack_migrate_to_success(pe_resource_t *rsc, pe_node_t *node, xmlNode *xml_op,
                           pe_working_set_t *data_set)
 {
     /* A successful migration sequence is:
      *    migrate_to on source node
      *    migrate_from on target node
      *    stop on source node
      *
      * If a migrate_to is followed by a stop, the entire migration (successful
      * or failed) is complete, and we don't care what happened on the target.
      *
      * If no migrate_from has happened, the migration is considered to be
      * "partial". If the migrate_from failed, make sure the resource gets
      * stopped on both source and target (if up).
      *
      * If the migrate_to and migrate_from both succeeded (which also implies the
      * resource is no longer running on the source), but there is no stop, the
      * migration is considered to be "dangling". Schedule a stop on the source
      * in this case.
      */
     int from_rc = 0;
     int from_status = 0;
     pe_node_t *target_node = NULL;
     pe_node_t *source_node = NULL;
     xmlNode *migrate_from = NULL;
     const char *source = crm_element_value(xml_op, XML_LRM_ATTR_MIGRATE_SOURCE);
     const char *target = crm_element_value(xml_op, XML_LRM_ATTR_MIGRATE_TARGET);
 
     // Sanity check
     CRM_CHECK(source && target && !strcmp(source, node->details->uname), return);
 
     if (stop_happened_after(rsc, node, xml_op, data_set)) {
         return;
     }
 
     // Clones are not allowed to migrate, so role can't be master
     rsc->role = RSC_ROLE_STARTED;
 
     target_node = pe_find_node(data_set->nodes, target);
     source_node = pe_find_node(data_set->nodes, source);
 
     // Check whether there was a migrate_from action on the target
     migrate_from = find_lrm_op(rsc->id, CRMD_ACTION_MIGRATED, target,
                                source, FALSE, data_set);
     if (migrate_from) {
         crm_element_value_int(migrate_from, XML_LRM_ATTR_RC, &from_rc);
         crm_element_value_int(migrate_from, XML_LRM_ATTR_OPSTATUS, &from_status);
         pe_rsc_trace(rsc, "%s op on %s exited with status=%d, rc=%d",
                      ID(migrate_from), target, from_status, from_rc);
     }
 
     if (migrate_from && from_rc == PCMK_OCF_OK
         && from_status == PCMK_LRM_OP_DONE) {
         /* The migrate_to and migrate_from both succeeded, so mark the migration
          * as "dangling". This will be used to schedule a stop action on the
          * source without affecting the target.
          */
         pe_rsc_trace(rsc, "Detected dangling migration op: %s on %s", ID(xml_op),
                      source);
         rsc->role = RSC_ROLE_STOPPED;
         rsc->dangling_migrations = g_list_prepend(rsc->dangling_migrations, node);
 
     } else if (migrate_from && (from_status != PCMK_LRM_OP_PENDING)) { // Failed
         if (target_node && target_node->details->online) {
             pe_rsc_trace(rsc, "Marking active on %s %p %d", target, target_node,
                          target_node->details->online);
             native_add_running(rsc, target_node, data_set);
         }
 
     } else { // Pending, or complete but erased
         if (target_node && target_node->details->online) {
             pe_rsc_trace(rsc, "Marking active on %s %p %d", target, target_node,
                          target_node->details->online);
 
             native_add_running(rsc, target_node, data_set);
             if (source_node && source_node->details->online) {
                 /* This is a partial migration: the migrate_to completed
                  * successfully on the source, but the migrate_from has not
                  * completed. Remember the source and target; if the newly
                  * chosen target remains the same when we schedule actions
                  * later, we may continue with the migration.
                  */
                 rsc->partial_migration_target = target_node;
                 rsc->partial_migration_source = source_node;
             }
         } else {
             /* Consider it failed here - forces a restart, prevents migration */
             set_bit(rsc->flags, pe_rsc_failed);
             clear_bit(rsc->flags, pe_rsc_allow_migrate);
         }
     }
 }
 
 static void
 unpack_migrate_to_failure(pe_resource_t *rsc, pe_node_t *node, xmlNode *xml_op,
                           pe_working_set_t *data_set)
 {
     int target_stop_id = 0;
     int target_migrate_from_id = 0;
     xmlNode *target_stop = NULL;
     xmlNode *target_migrate_from = NULL;
     const char *source = crm_element_value(xml_op, XML_LRM_ATTR_MIGRATE_SOURCE);
     const char *target = crm_element_value(xml_op, XML_LRM_ATTR_MIGRATE_TARGET);
 
     // Sanity check
     CRM_CHECK(source && target && !strcmp(source, node->details->uname), return);
 
     /* If a migration failed, we have to assume the resource is active. Clones
      * are not allowed to migrate, so role can't be master.
      */
     rsc->role = RSC_ROLE_STARTED;
 
     // Check for stop on the target
     target_stop = find_lrm_op(rsc->id, CRMD_ACTION_STOP, target, NULL,
                               TRUE, data_set);
     target_stop_id = pe__call_id(target_stop);
 
     // Check for migrate_from on the target
     target_migrate_from = find_lrm_op(rsc->id, CRMD_ACTION_MIGRATED, target,
                                       source, TRUE, data_set);
     target_migrate_from_id = pe__call_id(target_migrate_from);
 
     if ((target_stop == NULL) || (target_stop_id < target_migrate_from_id)) {
         /* There was no stop on the source, or a stop that happened before a
          * migrate_from, so assume the resource is still active on the target
          * (if it is up).
          */
         pe_node_t *target_node = pe_find_node(data_set->nodes, target);
 
         pe_rsc_trace(rsc, "stop (%d) + migrate_from (%d)",
                      target_stop_id, target_migrate_from_id);
         if (target_node && target_node->details->online) {
             native_add_running(rsc, target_node, data_set);
         }
 
     } else if (target_migrate_from == NULL) {
         /* We know there was a stop on the target, but there may not have been a
          * migrate_from (the stop could have happened before migrate_from was
          * scheduled or attempted).
          *
          * That means this could be a "dangling" migration. But first, check
          * whether there is a newer migrate_from or start on the source node --
          * it's possible the failed migration was followed by a successful
          * full restart or migration in the reverse direction, in which case we
          * don't want to force it to stop.
          */
         xmlNode *source_migrate_from = NULL;
         xmlNode *source_start = NULL;
         int source_migrate_to_id = pe__call_id(xml_op);
 
         source_migrate_from = find_lrm_op(rsc->id, CRMD_ACTION_MIGRATED, source,
                                           NULL, TRUE, data_set);
         if (pe__call_id(source_migrate_from) > source_migrate_to_id) {
             return;
         }
 
         source_start = find_lrm_op(rsc->id, CRMD_ACTION_START, source, NULL,
                                    TRUE, data_set);
         if (pe__call_id(source_start) > source_migrate_to_id) {
             return;
         }
 
         // Mark node as having dangling migration so we can force a stop later
         rsc->dangling_migrations = g_list_prepend(rsc->dangling_migrations, node);
     }
 }
 
 static void
 unpack_migrate_from_failure(pe_resource_t *rsc, pe_node_t *node,
                             xmlNode *xml_op, pe_working_set_t *data_set)
 {
     xmlNode *source_stop = NULL;
     xmlNode *source_migrate_to = NULL;
     const char *source = crm_element_value(xml_op, XML_LRM_ATTR_MIGRATE_SOURCE);
     const char *target = crm_element_value(xml_op, XML_LRM_ATTR_MIGRATE_TARGET);
 
     // Sanity check
     CRM_CHECK(source && target && !strcmp(target, node->details->uname), return);
 
     /* If a migration failed, we have to assume the resource is active. Clones
      * are not allowed to migrate, so role can't be master.
      */
     rsc->role = RSC_ROLE_STARTED;
 
     // Check for a stop on the source
     source_stop = find_lrm_op(rsc->id, CRMD_ACTION_STOP, source, NULL,
                               TRUE, data_set);
 
     // Check for a migrate_to on the source
     source_migrate_to = find_lrm_op(rsc->id, CRMD_ACTION_MIGRATE,
                                     source, target, TRUE, data_set);
 
     if ((source_stop == NULL)
         || (pe__call_id(source_stop) < pe__call_id(source_migrate_to))) {
         /* There was no stop on the source, or a stop that happened before
          * migrate_to, so assume the resource is still active on the source (if
          * it is up).
          */
         pe_node_t *source_node = pe_find_node(data_set->nodes, source);
 
         if (source_node && source_node->details->online) {
             native_add_running(rsc, source_node, data_set);
         }
     }
 }
 
 static void
 record_failed_op(xmlNode *op, const pe_node_t *node,
                  const pe_resource_t *rsc, pe_working_set_t *data_set)
 {
     xmlNode *xIter = NULL;
     const char *op_key = crm_element_value(op, XML_LRM_ATTR_TASK_KEY);
 
     if (node->details->online == FALSE) {
         return;
     }
 
     for (xIter = data_set->failed->children; xIter; xIter = xIter->next) {
         const char *key = crm_element_value(xIter, XML_LRM_ATTR_TASK_KEY);
         const char *uname = crm_element_value(xIter, XML_ATTR_UNAME);
 
         if(safe_str_eq(op_key, key) && safe_str_eq(uname, node->details->uname)) {
             crm_trace("Skipping duplicate entry %s on %s", op_key, node->details->uname);
             return;
         }
     }
 
     crm_trace("Adding entry %s on %s", op_key, node->details->uname);
     crm_xml_add(op, XML_ATTR_UNAME, node->details->uname);
     crm_xml_add(op, XML_LRM_ATTR_RSCID, rsc->id);
     add_node_copy(data_set->failed, op);
 }
 
 static const char *get_op_key(xmlNode *xml_op)
 {
     const char *key = crm_element_value(xml_op, XML_LRM_ATTR_TASK_KEY);
     if(key == NULL) {
         key = ID(xml_op);
     }
     return key;
 }
 
 static const char *
 last_change_str(xmlNode *xml_op)
 {
     time_t when;
     const char *when_s = NULL;
 
     if (crm_element_value_epoch(xml_op, XML_RSC_OP_LAST_CHANGE,
                                 &when) == pcmk_ok) {
         when_s = pcmk__epoch2str(&when);
         if (when_s) {
             // Skip day of week to make message shorter
             when_s = strchr(when_s, ' ');
             if (when_s) {
                 ++when_s;
             }
         }
     }
     return ((when_s && *when_s)? when_s : "unknown time");
 }
 
 static void
 unpack_rsc_op_failure(pe_resource_t * rsc, pe_node_t * node, int rc, xmlNode * xml_op, xmlNode ** last_failure,
                       enum action_fail_response * on_fail, pe_working_set_t * data_set)
 {
     guint interval_ms = 0;
     bool is_probe = false;
     pe_action_t *action = NULL;
 
     const char *key = get_op_key(xml_op);
     const char *task = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
     const char *exit_reason = crm_element_value(xml_op,
                                                 XML_LRM_ATTR_EXIT_REASON);
 
     CRM_ASSERT(rsc);
     CRM_CHECK(task != NULL, return);
 
     *last_failure = xml_op;
 
     crm_element_value_ms(xml_op, XML_LRM_ATTR_INTERVAL_MS, &interval_ms);
     if ((interval_ms == 0) && !strcmp(task, CRMD_ACTION_STATUS)) {
         is_probe = true;
     }
 
     if (exit_reason == NULL) {
         exit_reason = "";
     }
 
     if (is_not_set(data_set->flags, pe_flag_symmetric_cluster)
         && (rc == PCMK_OCF_NOT_INSTALLED)) {
         crm_trace("Unexpected result (%s%s%s) was recorded for "
                   "%s of %s on %s at %s " CRM_XS " rc=%d id=%s",
                   services_ocf_exitcode_str(rc),
                   (*exit_reason? ": " : ""), exit_reason,
                   (is_probe? "probe" : task), rsc->id, node->details->uname,
                   last_change_str(xml_op), rc, ID(xml_op));
     } else {
         crm_warn("Unexpected result (%s%s%s) was recorded for "
                   "%s of %s on %s at %s " CRM_XS " rc=%d id=%s",
                  services_ocf_exitcode_str(rc),
                  (*exit_reason? ": " : ""), exit_reason,
                  (is_probe? "probe" : task), rsc->id, node->details->uname,
                  last_change_str(xml_op), rc, ID(xml_op));
 
         if (is_probe && (rc != PCMK_OCF_OK)
             && (rc != PCMK_OCF_NOT_RUNNING)
             && (rc != PCMK_OCF_RUNNING_MASTER)) {
 
             /* A failed (not just unexpected) probe result could mean the user
              * didn't know resources will be probed even where they can't run.
              */
             crm_notice("If it is not possible for %s to run on %s, see "
                        "the resource-discovery option for location constraints",
                        rsc->id, node->details->uname);
         }
 
         record_failed_op(xml_op, node, rsc, data_set);
     }
 
     action = custom_action(rsc, strdup(key), task, NULL, TRUE, FALSE, data_set);
     if ((action->on_fail <= action_fail_fence && *on_fail < action->on_fail) ||
         (action->on_fail == action_fail_reset_remote && *on_fail <= action_fail_recover) ||
         (action->on_fail == action_fail_restart_container && *on_fail <= action_fail_recover) ||
         (*on_fail == action_fail_restart_container && action->on_fail >= action_fail_migrate)) {
         pe_rsc_trace(rsc, "on-fail %s -> %s for %s (%s)", fail2text(*on_fail),
                      fail2text(action->on_fail), action->uuid, key);
         *on_fail = action->on_fail;
     }
 
     if (!strcmp(task, CRMD_ACTION_STOP)) {
         resource_location(rsc, node, -INFINITY, "__stop_fail__", data_set);
 
     } else if (!strcmp(task, CRMD_ACTION_MIGRATE)) {
         unpack_migrate_to_failure(rsc, node, xml_op, data_set);
 
     } else if (!strcmp(task, CRMD_ACTION_MIGRATED)) {
         unpack_migrate_from_failure(rsc, node, xml_op, data_set);
 
     } else if (!strcmp(task, CRMD_ACTION_PROMOTE)) {
         rsc->role = RSC_ROLE_MASTER;
 
     } else if (!strcmp(task, CRMD_ACTION_DEMOTE)) {
         if (action->on_fail == action_fail_block) {
             rsc->role = RSC_ROLE_MASTER;
             rsc->next_role = RSC_ROLE_STOPPED;
 
         } else if(rc == PCMK_OCF_NOT_RUNNING) {
             rsc->role = RSC_ROLE_STOPPED;
 
         } else {
             /* Staying in master role would put the scheduler and controller
              * into a loop. Setting slave role is not dangerous because the
              * resource will be stopped as part of recovery, and any master
              * promotion will be ordered after that stop.
              */
             rsc->role = RSC_ROLE_SLAVE;
         }
     }
 
     if(is_probe && rc == PCMK_OCF_NOT_INSTALLED) {
         /* leave stopped */
         pe_rsc_trace(rsc, "Leaving %s stopped", rsc->id);
         rsc->role = RSC_ROLE_STOPPED;
 
     } else if (rsc->role < RSC_ROLE_STARTED) {
         pe_rsc_trace(rsc, "Setting %s active", rsc->id);
         set_active(rsc);
     }
 
     pe_rsc_trace(rsc, "Resource %s: role=%s, unclean=%s, on_fail=%s, fail_role=%s",
                  rsc->id, role2text(rsc->role),
                  node->details->unclean ? "true" : "false",
                  fail2text(action->on_fail), role2text(action->fail_role));
 
     if (action->fail_role != RSC_ROLE_STARTED && rsc->next_role < action->fail_role) {
         rsc->next_role = action->fail_role;
     }
 
     if (action->fail_role == RSC_ROLE_STOPPED) {
         int score = -INFINITY;
 
         pe_resource_t *fail_rsc = rsc;
 
         if (fail_rsc->parent) {
             pe_resource_t *parent = uber_parent(fail_rsc);
 
             if (pe_rsc_is_clone(parent)
                 && is_not_set(parent->flags, pe_rsc_unique)) {
                 /* For clone resources, if a child fails on an operation
                  * with on-fail = stop, all the resources fail.  Do this by preventing
                  * the parent from coming up again. */
                 fail_rsc = parent;
             }
         }
         crm_notice("%s will not be started under current conditions",
                    fail_rsc->id);
         /* make sure it doesn't come up again */
         if (fail_rsc->allowed_nodes != NULL) {
             g_hash_table_destroy(fail_rsc->allowed_nodes);
         }
         fail_rsc->allowed_nodes = pe__node_list2table(data_set->nodes);
         g_hash_table_foreach(fail_rsc->allowed_nodes, set_node_score, &score);
     }
 
     pe_free_action(action);
 }
 
 /*!
  * \internal
  * \brief Remap operation status based on action result
  *
  * Given an action result, determine an appropriate operation status for the
  * purposes of responding to the action (the status provided by the executor is
  * not directly usable since the executor does not know what was expected).
  *
  * \param[in,out] rsc        Resource that operation history entry is for
  * \param[in]     rc         Actual return code of operation
  * \param[in]     target_rc  Expected return code of operation
  * \param[in]     node       Node where operation was executed
  * \param[in]     xml_op     Operation history entry XML from CIB status
  * \param[in,out] on_fail    What should be done about the result
  * \param[in]     data_set   Current cluster working set
  *
  * \return Operation status based on return code and action info
  * \note This may update the resource's current and next role.
  */
 static int
 determine_op_status(
     pe_resource_t *rsc, int rc, int target_rc, pe_node_t * node, xmlNode * xml_op, enum action_fail_response * on_fail, pe_working_set_t * data_set) 
 {
     guint interval_ms = 0;
     bool is_probe = false;
     int result = PCMK_LRM_OP_DONE;
     const char *key = get_op_key(xml_op);
     const char *task = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
     const char *exit_reason = crm_element_value(xml_op,
                                                 XML_LRM_ATTR_EXIT_REASON);
 
     CRM_ASSERT(rsc);
     CRM_CHECK(task != NULL, return PCMK_LRM_OP_ERROR);
 
     if (exit_reason == NULL) {
         exit_reason = "";
     }
 
     crm_element_value_ms(xml_op, XML_LRM_ATTR_INTERVAL_MS, &interval_ms);
     if ((interval_ms == 0) && !strcmp(task, CRMD_ACTION_STATUS)) {
         is_probe = true;
         task = "probe";
     }
 
     if (target_rc < 0) {
         /* Pre-1.0 Pacemaker versions, and Pacemaker 1.1.6 or earlier with
          * Heartbeat 2.0.7 or earlier as the cluster layer, did not include the
          * target_rc in the transition key, which (along with the similar case
          * of a corrupted transition key in the CIB) will be reported to this
          * function as -1. Pacemaker 2.0+ does not support rolling upgrades from
          * those versions or processing of saved CIB files from those versions,
          * so we do not need to care much about this case.
          */
         result = PCMK_LRM_OP_ERROR;
         crm_warn("Expected result not found for %s on %s (corrupt or obsolete CIB?)",
                  key, node->details->uname);
 
     } else if (target_rc != rc) {
         result = PCMK_LRM_OP_ERROR;
         pe_rsc_debug(rsc, "%s on %s: expected %d (%s), got %d (%s%s%s)",
                      key, node->details->uname,
                      target_rc, services_ocf_exitcode_str(target_rc),
                      rc, services_ocf_exitcode_str(rc),
                      (*exit_reason? ": " : ""), exit_reason);
     }
 
     switch (rc) {
         case PCMK_OCF_OK:
             if (is_probe && (target_rc == PCMK_OCF_NOT_RUNNING)) {
                 result = PCMK_LRM_OP_DONE;
                 pe_rsc_info(rsc, "Probe found %s active on %s at %s",
                             rsc->id, node->details->uname,
                             last_change_str(xml_op));
             }
             break;
 
         case PCMK_OCF_NOT_RUNNING:
             if (is_probe || target_rc == rc || is_not_set(rsc->flags, pe_rsc_managed)) {
                 result = PCMK_LRM_OP_DONE;
                 rsc->role = RSC_ROLE_STOPPED;
 
                 /* clear any previous failure actions */
                 *on_fail = action_fail_ignore;
                 rsc->next_role = RSC_ROLE_UNKNOWN;
             }
             break;
 
         case PCMK_OCF_RUNNING_MASTER:
             if (is_probe && (rc != target_rc)) {
                 result = PCMK_LRM_OP_DONE;
                 pe_rsc_info(rsc,
                             "Probe found %s active and promoted on %s at %s",
                             rsc->id, node->details->uname,
                             last_change_str(xml_op));
             }
             rsc->role = RSC_ROLE_MASTER;
             break;
 
         case PCMK_OCF_DEGRADED_MASTER:
         case PCMK_OCF_FAILED_MASTER:
             rsc->role = RSC_ROLE_MASTER;
             result = PCMK_LRM_OP_ERROR;
             break;
 
         case PCMK_OCF_NOT_CONFIGURED:
             result = PCMK_LRM_OP_ERROR_FATAL;
             break;
 
         case PCMK_OCF_UNIMPLEMENT_FEATURE:
             if (interval_ms > 0) {
                 result = PCMK_LRM_OP_NOTSUPPORTED;
                 break;
             }
             // fall through
         case PCMK_OCF_NOT_INSTALLED:
         case PCMK_OCF_INVALID_PARAM:
         case PCMK_OCF_INSUFFICIENT_PRIV:
             if (!pe_can_fence(data_set, node)
                 && !strcmp(task, CRMD_ACTION_STOP)) {
                 /* If a stop fails and we can't fence, there's nothing else we can do */
                 pe_proc_err("No further recovery can be attempted for %s "
                             "because %s on %s failed (%s%s%s) at %s "
                             CRM_XS " rc=%d id=%s", rsc->id, task,
                             node->details->uname, services_ocf_exitcode_str(rc),
                             (*exit_reason? ": " : ""), exit_reason,
                             last_change_str(xml_op), rc, ID(xml_op));
                 clear_bit(rsc->flags, pe_rsc_managed);
                 set_bit(rsc->flags, pe_rsc_block);
             }
             result = PCMK_LRM_OP_ERROR_HARD;
             break;
 
         default:
             if (result == PCMK_LRM_OP_DONE) {
                 crm_info("Treating unknown exit status %d from %s of %s "
                          "on %s at %s as failure",
                          rc, task, rsc->id, node->details->uname,
                          last_change_str(xml_op));
                 result = PCMK_LRM_OP_ERROR;
             }
             break;
     }
     return result;
 }
 
 // return TRUE if start or monitor last failure but parameters changed
 static bool
 should_clear_for_param_change(xmlNode *xml_op, const char *task,
                               pe_resource_t *rsc, pe_node_t *node,
                               pe_working_set_t *data_set)
 {
     if (!strcmp(task, "start") || !strcmp(task, "monitor")) {
 
         if (pe__bundle_needs_remote_name(rsc)) {
             /* We haven't allocated resources yet, so we can't reliably
              * substitute addr parameters for the REMOTE_CONTAINER_HACK.
              * When that's needed, defer the check until later.
              */
             pe__add_param_check(xml_op, rsc, node, pe_check_last_failure,
                                 data_set);
 
         } else {
             op_digest_cache_t *digest_data = NULL;
 
             digest_data = rsc_action_digest_cmp(rsc, xml_op, node, data_set);
             switch (digest_data->rc) {
                 case RSC_DIGEST_UNKNOWN:
                     crm_trace("Resource %s history entry %s on %s"
                               " has no digest to compare",
                               rsc->id, get_op_key(xml_op), node->details->id);
                     break;
                 case RSC_DIGEST_MATCH:
                     break;
                 default:
                     return TRUE;
             }
         }
     }
     return FALSE;
 }
 
 // Order action after fencing of remote node, given connection rsc
 static void
 order_after_remote_fencing(pe_action_t *action, pe_resource_t *remote_conn,
                            pe_working_set_t *data_set)
 {
     pe_node_t *remote_node = pe_find_node(data_set->nodes, remote_conn->id);
 
     if (remote_node) {
         pe_action_t *fence = pe_fence_op(remote_node, NULL, TRUE, NULL,
                                          FALSE, data_set);
 
         order_actions(fence, action, pe_order_implies_then);
     }
 }
 
 static bool
 should_ignore_failure_timeout(pe_resource_t *rsc, xmlNode *xml_op,
                               const char *task, guint interval_ms,
                               bool is_last_failure, pe_working_set_t *data_set)
 {
     /* Clearing failures of recurring monitors has special concerns. The
      * executor reports only changes in the monitor result, so if the
      * monitor is still active and still getting the same failure result,
      * that will go undetected after the failure is cleared.
      *
      * Also, the operation history will have the time when the recurring
      * monitor result changed to the given code, not the time when the
      * result last happened.
      *
      * @TODO We probably should clear such failures only when the failure
      * timeout has passed since the last occurrence of the failed result.
      * However we don't record that information. We could maybe approximate
      * that by clearing only if there is a more recent successful monitor or
      * stop result, but we don't even have that information at this point
      * since we are still unpacking the resource's operation history.
      *
      * This is especially important for remote connection resources with a
      * reconnect interval, so in that case, we skip clearing failures
      * if the remote node hasn't been fenced.
      */
     if (rsc->remote_reconnect_ms
         && is_set(data_set->flags, pe_flag_stonith_enabled)
         && (interval_ms != 0) && safe_str_eq(task, CRMD_ACTION_STATUS)) {
 
         pe_node_t *remote_node = pe_find_node(data_set->nodes, rsc->id);
 
         if (remote_node && !remote_node->details->remote_was_fenced) {
             if (is_last_failure) {
                 crm_info("Waiting to clear monitor failure for remote node %s"
                          " until fencing has occurred", rsc->id);
             }
             return TRUE;
         }
     }
     return FALSE;
 }
 
 /*!
  * \internal
  * \brief Check operation age and schedule failure clearing when appropriate
  *
  * This function has two distinct purposes. The first is to check whether an
  * operation history entry is expired (i.e. the resource has a failure timeout,
  * the entry is older than the timeout, and the resource either has no fail
  * count or its fail count is entirely older than the timeout). The second is to
  * schedule fail count clearing when appropriate (i.e. the operation is expired
  * and either the resource has an expired fail count or the operation is a
  * last_failure for a remote connection resource with a reconnect interval,
  * or the operation is a last_failure for a start or monitor operation and the
  * resource's parameters have changed since the operation).
  *
  * \param[in] rsc       Resource that operation happened to
  * \param[in] node      Node that operation happened on
  * \param[in] rc        Actual result of operation
  * \param[in] xml_op    Operation history entry XML
  * \param[in] data_set  Current working set
  *
  * \return TRUE if operation history entry is expired, FALSE otherwise
  */
 static bool
 check_operation_expiry(pe_resource_t *rsc, pe_node_t *node, int rc,
                        xmlNode *xml_op, pe_working_set_t *data_set)
 {
     bool expired = FALSE;
     bool is_last_failure = pcmk__ends_with(ID(xml_op), "_last_failure_0");
     time_t last_run = 0;
     guint interval_ms = 0;
     int unexpired_fail_count = 0;
     const char *task = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
     const char *clear_reason = NULL;
 
     crm_element_value_ms(xml_op, XML_LRM_ATTR_INTERVAL_MS, &interval_ms);
 
     if ((rsc->failure_timeout > 0)
         && (crm_element_value_epoch(xml_op, XML_RSC_OP_LAST_CHANGE,
                                     &last_run) == 0)) {
 
         // Resource has a failure-timeout, and history entry has a timestamp
 
         time_t now = get_effective_time(data_set);
         time_t last_failure = 0;
 
         // Is this particular operation history older than the failure timeout?
         if ((now >= (last_run + rsc->failure_timeout))
             && !should_ignore_failure_timeout(rsc, xml_op, task, interval_ms,
                                               is_last_failure, data_set)) {
             expired = TRUE;
         }
 
         // Does the resource as a whole have an unexpired fail count?
         unexpired_fail_count = pe_get_failcount(node, rsc, &last_failure,
                                                 pe_fc_effective, xml_op,
                                                 data_set);
 
         // Update scheduler recheck time according to *last* failure
         crm_trace("%s@%lld is %sexpired @%lld with unexpired_failures=%d timeout=%ds"
                   " last-failure@%lld",
                   ID(xml_op), (long long) last_run, (expired? "" : "not "),
                   (long long) now, unexpired_fail_count, rsc->failure_timeout,
                   (long long) last_failure);
         last_failure += rsc->failure_timeout + 1;
         if (unexpired_fail_count && (now < last_failure)) {
             pe__update_recheck_time(last_failure, data_set);
         }
     }
 
     if (expired) {
         if (pe_get_failcount(node, rsc, NULL, pe_fc_default, xml_op, data_set)) {
 
             // There is a fail count ignoring timeout
 
             if (unexpired_fail_count == 0) {
                 // There is no fail count considering timeout
                 clear_reason = "it expired";
 
             } else {
                 /* This operation is old, but there is an unexpired fail count.
                  * In a properly functioning cluster, this should only be
                  * possible if this operation is not a failure (otherwise the
                  * fail count should be expired too), so this is really just a
                  * failsafe.
                  */
                 expired = FALSE;
             }
 
         } else if (is_last_failure && rsc->remote_reconnect_ms) {
             /* Clear any expired last failure when reconnect interval is set,
              * even if there is no fail count.
              */
             clear_reason = "reconnect interval is set";
         }
     }
 
     if (!expired && is_last_failure
         && should_clear_for_param_change(xml_op, task, rsc, node, data_set)) {
         clear_reason = "resource parameters have changed";
     }
 
     if (clear_reason != NULL) {
         // Schedule clearing of the fail count
         pe_action_t *clear_op = pe__clear_failcount(rsc, node, clear_reason,
                                                     data_set);
 
         if (is_set(data_set->flags, pe_flag_stonith_enabled)
             && rsc->remote_reconnect_ms) {
             /* If we're clearing a remote connection due to a reconnect
              * interval, we want to wait until any scheduled fencing
              * completes.
              *
              * We could limit this to remote_node->details->unclean, but at
              * this point, that's always true (it won't be reliable until
              * after unpack_node_loop() is done).
              */
             crm_info("Clearing %s failure will wait until any scheduled "
                      "fencing of %s completes", task, rsc->id);
             order_after_remote_fencing(clear_op, rsc, data_set);
         }
     }
 
     if (expired && (interval_ms == 0) && safe_str_eq(task, CRMD_ACTION_STATUS)) {
         switch(rc) {
             case PCMK_OCF_OK:
             case PCMK_OCF_NOT_RUNNING:
             case PCMK_OCF_RUNNING_MASTER:
             case PCMK_OCF_DEGRADED:
             case PCMK_OCF_DEGRADED_MASTER:
                 // Don't expire probes that return these values
                 expired = FALSE;
                 break;
         }
     }
 
     return expired;
 }
 
 int pe__target_rc_from_xml(xmlNode *xml_op)
 {
     int target_rc = 0;
     const char *key = crm_element_value(xml_op, XML_ATTR_TRANSITION_KEY);
 
     if (key == NULL) {
         return -1;
     }
     decode_transition_key(key, NULL, NULL, NULL, &target_rc);
     return target_rc;
 }
 
 static enum action_fail_response
 get_action_on_fail(pe_resource_t *rsc, const char *key, const char *task, pe_working_set_t * data_set) 
 {
     int result = action_fail_recover;
     pe_action_t *action = custom_action(rsc, strdup(key), task, NULL, TRUE, FALSE, data_set);
 
     result = action->on_fail;
     pe_free_action(action);
 
     return result;
 }
 
 static void
 update_resource_state(pe_resource_t * rsc, pe_node_t * node, xmlNode * xml_op, const char * task, int rc,
                       xmlNode * last_failure, enum action_fail_response * on_fail, pe_working_set_t * data_set)
 {
     gboolean clear_past_failure = FALSE;
 
     CRM_ASSERT(rsc);
     CRM_ASSERT(xml_op);
 
     if (rc == PCMK_OCF_NOT_RUNNING) {
         clear_past_failure = TRUE;
 
     } else if (rc == PCMK_OCF_NOT_INSTALLED) {
         rsc->role = RSC_ROLE_STOPPED;
 
     } else if (safe_str_eq(task, CRMD_ACTION_STATUS)) {
         if (last_failure) {
             const char *op_key = get_op_key(xml_op);
             const char *last_failure_key = get_op_key(last_failure);
 
             if (safe_str_eq(op_key, last_failure_key)) {
                 clear_past_failure = TRUE;
             }
         }
 
         if (rsc->role < RSC_ROLE_STARTED) {
             set_active(rsc);
         }
 
     } else if (safe_str_eq(task, CRMD_ACTION_START)) {
         rsc->role = RSC_ROLE_STARTED;
         clear_past_failure = TRUE;
 
     } else if (safe_str_eq(task, CRMD_ACTION_STOP)) {
         rsc->role = RSC_ROLE_STOPPED;
         clear_past_failure = TRUE;
 
     } else if (safe_str_eq(task, CRMD_ACTION_PROMOTE)) {
         rsc->role = RSC_ROLE_MASTER;
         clear_past_failure = TRUE;
 
     } else if (safe_str_eq(task, CRMD_ACTION_DEMOTE)) {
         /* Demote from Master does not clear an error */
         rsc->role = RSC_ROLE_SLAVE;
 
     } else if (safe_str_eq(task, CRMD_ACTION_MIGRATED)) {
         rsc->role = RSC_ROLE_STARTED;
         clear_past_failure = TRUE;
 
     } else if (safe_str_eq(task, CRMD_ACTION_MIGRATE)) {
         unpack_migrate_to_success(rsc, node, xml_op, data_set);
 
     } else if (rsc->role < RSC_ROLE_STARTED) {
         pe_rsc_trace(rsc, "%s active on %s", rsc->id, node->details->uname);
         set_active(rsc);
     }
 
     /* clear any previous failure actions */
     if (clear_past_failure) {
         switch (*on_fail) {
             case action_fail_stop:
             case action_fail_fence:
             case action_fail_migrate:
             case action_fail_standby:
                 pe_rsc_trace(rsc, "%s.%s is not cleared by a completed stop",
                              rsc->id, fail2text(*on_fail));
                 break;
 
             case action_fail_block:
             case action_fail_ignore:
             case action_fail_recover:
             case action_fail_restart_container:
                 *on_fail = action_fail_ignore;
                 rsc->next_role = RSC_ROLE_UNKNOWN;
                 break;
             case action_fail_reset_remote:
                 if (rsc->remote_reconnect_ms == 0) {
                     /* With no reconnect interval, the connection is allowed to
                      * start again after the remote node is fenced and
                      * completely stopped. (With a reconnect interval, we wait
                      * for the failure to be cleared entirely before attempting
                      * to reconnect.)
                      */
                     *on_fail = action_fail_ignore;
                     rsc->next_role = RSC_ROLE_UNKNOWN;
                 }
                 break;
         }
     }
 }
 
 /*!
  * \internal
  * \brief Remap informational monitor results to usual values
  *
  * Certain OCF result codes are for providing extended information to the
  * user about services that aren't yet failed but not entirely healthy either.
  * These must be treated as the "normal" result by pacemaker.
  *
  * \param[in] rc        Actual result of a monitor action
  * \param[in] xml_op    Operation history XML
  * \param[in] node      Node that operation happened on
  * \param[in] rsc       Resource that operation happened to
  * \param[in] data_set  Cluster working set
  *
  * \return Result code that pacemaker should use
  *
  * \note If the result is remapped, and the node is not shutting down or failed,
  *       the operation will be recorded in the data set's list of failed
  *       operations, to highlight it for the user.
  */
 static int
 remap_monitor_rc(int rc, xmlNode *xml_op, const pe_node_t *node,
                  const pe_resource_t *rsc, pe_working_set_t *data_set)
 {
     int remapped_rc = rc;
 
     switch (rc) {
         case PCMK_OCF_DEGRADED:
             remapped_rc = PCMK_OCF_OK;
             break;
 
         case PCMK_OCF_DEGRADED_MASTER:
             remapped_rc = PCMK_OCF_RUNNING_MASTER;
             break;
 
         default:
             break;
     }
 
     if (rc != remapped_rc) {
         crm_trace("Remapping monitor result %d to %d", rc, remapped_rc);
         if (!node->details->shutdown || node->details->online) {
             record_failed_op(xml_op, node, rsc, data_set);
         }
     }
     return remapped_rc;
 }
 
 static void
 unpack_rsc_op(pe_resource_t *rsc, pe_node_t *node, xmlNode *xml_op,
               xmlNode **last_failure, enum action_fail_response *on_fail,
               pe_working_set_t *data_set)
 {
     int rc = 0;
     int task_id = 0;
     int target_rc = 0;
     int status = PCMK_LRM_OP_UNKNOWN;
     guint interval_ms = 0;
     const char *task = NULL;
     const char *task_key = NULL;
     const char *exit_reason = NULL;
     bool expired = FALSE;
     pe_resource_t *parent = rsc;
     enum action_fail_response failure_strategy = action_fail_recover;
 
     CRM_CHECK(rsc && node && xml_op, return);
 
     target_rc = pe__target_rc_from_xml(xml_op);
     task_key = get_op_key(xml_op);
     task = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
     exit_reason = crm_element_value(xml_op, XML_LRM_ATTR_EXIT_REASON);
     if (exit_reason == NULL) {
         exit_reason = "";
     }
 
     crm_element_value_int(xml_op, XML_LRM_ATTR_RC, &rc);
     crm_element_value_int(xml_op, XML_LRM_ATTR_CALLID, &task_id);
     crm_element_value_int(xml_op, XML_LRM_ATTR_OPSTATUS, &status);
     crm_element_value_ms(xml_op, XML_LRM_ATTR_INTERVAL_MS, &interval_ms);
 
     CRM_CHECK(task != NULL, return);
     CRM_CHECK(status <= PCMK_LRM_OP_INVALID, return);
     CRM_CHECK(status >= PCMK_LRM_OP_PENDING, return);
 
     if (!strcmp(task, CRMD_ACTION_NOTIFY) ||
         !strcmp(task, CRMD_ACTION_METADATA)) {
         /* safe to ignore these */
         return;
     }
 
     if (is_not_set(rsc->flags, pe_rsc_unique)) {
         parent = uber_parent(rsc);
     }
 
     pe_rsc_trace(rsc, "Unpacking task %s/%s (call_id=%d, status=%d, rc=%d) on %s (role=%s)",
                  task_key, task, task_id, status, rc, node->details->uname, role2text(rsc->role));
 
     if (node->details->unclean) {
         pe_rsc_trace(rsc, "Node %s (where %s is running) is unclean."
                      " Further action depends on the value of the stop's on-fail attribute",
                      node->details->uname, rsc->id);
     }
 
     /* It should be possible to call remap_monitor_rc() first then call
      * check_operation_expiry() only if rc != target_rc, because there should
      * never be a fail count without at least one unexpected result in the
      * resource history. That would be more efficient by avoiding having to call
      * check_operation_expiry() for expected results.
      *
      * However, we do have such configurations in the scheduler regression
      * tests, even if it shouldn't be possible with the current code. It's
      * probably a good idea anyway, but that would require updating the test
      * inputs to something currently possible.
      */
 
     if ((status != PCMK_LRM_OP_NOT_INSTALLED)
         && check_operation_expiry(rsc, node, rc, xml_op, data_set)) {
         expired = TRUE;
     }
 
     if (!strcmp(task, CRMD_ACTION_STATUS)) {
         rc = remap_monitor_rc(rc, xml_op, node, rsc, data_set);
     }
 
     if (expired && (rc != target_rc)) {
         const char *magic = crm_element_value(xml_op, XML_ATTR_TRANSITION_MAGIC);
 
         if (interval_ms == 0) {
             crm_notice("Ignoring expired %s failure on %s "
                        CRM_XS " actual=%d expected=%d magic=%s",
                        task_key, node->details->uname, rc, target_rc, magic);
             goto done;
 
         } else if(node->details->online && node->details->unclean == FALSE) {
             /* Reschedule the recurring monitor. CancelXmlOp() won't work at
              * this stage, so as a hacky workaround, forcibly change the restart
              * digest so check_action_definition() does what we want later.
              *
              * @TODO We should skip this if there is a newer successful monitor.
              *       Also, this causes rescheduling only if the history entry
              *       has an op-digest (which the expire-non-blocked-failure
              *       scheduler regression test doesn't, but that may not be a
              *       realistic scenario in production).
              */
             crm_notice("Rescheduling %s after failure expired on %s "
                        CRM_XS " actual=%d expected=%d magic=%s",
                        task_key, node->details->uname, rc, target_rc, magic);
             crm_xml_add(xml_op, XML_LRM_ATTR_RESTART_DIGEST, "calculated-failure-timeout");
             goto done;
         }
     }
 
     /* If the executor reported an operation status of anything but done or
      * error, consider that final. But for done or error, we know better whether
      * it should be treated as a failure or not, because we know the expected
      * result.
      */
     if(status == PCMK_LRM_OP_DONE || status == PCMK_LRM_OP_ERROR) {
         status = determine_op_status(rsc, rc, target_rc, node, xml_op, on_fail, data_set);
         pe_rsc_trace(rsc, "Remapped %s status to %d", task_key, status);
     }
 
     switch (status) {
         case PCMK_LRM_OP_CANCELLED:
             // Should never happen
             pe_err("Resource history contains cancellation '%s' "
                    "(%s of %s on %s at %s)",
                    ID(xml_op), task, rsc->id, node->details->uname,
                    last_change_str(xml_op));
             break;
 
         case PCMK_LRM_OP_PENDING:
             if (!strcmp(task, CRMD_ACTION_START)) {
                 set_bit(rsc->flags, pe_rsc_start_pending);
                 set_active(rsc);
 
             } else if (!strcmp(task, CRMD_ACTION_PROMOTE)) {
                 rsc->role = RSC_ROLE_MASTER;
 
             } else if (!strcmp(task, CRMD_ACTION_MIGRATE) && node->details->unclean) {
                 /* If a pending migrate_to action is out on a unclean node,
                  * we have to force the stop action on the target. */
                 const char *migrate_target = crm_element_value(xml_op, XML_LRM_ATTR_MIGRATE_TARGET);
                 pe_node_t *target = pe_find_node(data_set->nodes, migrate_target);
                 if (target) {
                     stop_action(rsc, target, FALSE);
                 }
             }
 
             if (rsc->pending_task == NULL) {
                 if ((interval_ms != 0) || strcmp(task, CRMD_ACTION_STATUS)) {
                     rsc->pending_task = strdup(task);
                     rsc->pending_node = node;
                 } else {
                     /* Pending probes are not printed, even if pending
                      * operations are requested. If someone ever requests that
                      * behavior, enable the below and the corresponding part of
                      * native.c:native_pending_task().
                      */
 #if 0
                     rsc->pending_task = strdup("probe");
                     rsc->pending_node = node;
 #endif
                 }
             }
             break;
 
         case PCMK_LRM_OP_DONE:
             pe_rsc_trace(rsc, "%s of %s on %s completed at %s " CRM_XS " id=%s",
                          task, rsc->id, node->details->uname,
                          last_change_str(xml_op), ID(xml_op));
             update_resource_state(rsc, node, xml_op, task, rc, *last_failure, on_fail, data_set);
             break;
 
         case PCMK_LRM_OP_NOT_INSTALLED:
             failure_strategy = get_action_on_fail(rsc, task_key, task, data_set);
             if (failure_strategy == action_fail_ignore) {
                 crm_warn("Cannot ignore failed %s of %s on %s: "
                          "Resource agent doesn't exist "
                          CRM_XS " status=%d rc=%d id=%s",
                          task, rsc->id, node->details->uname, status, rc,
                          ID(xml_op));
                 /* Also for printing it as "FAILED" by marking it as pe_rsc_failed later */
                 *on_fail = action_fail_migrate;
             }
             resource_location(parent, node, -INFINITY, "hard-error", data_set);
             unpack_rsc_op_failure(rsc, node, rc, xml_op, last_failure, on_fail, data_set);
             break;
 
         case PCMK_LRM_OP_NOT_CONNECTED:
             if (pe__is_guest_or_remote_node(node)
                 && is_set(node->details->remote_rsc->flags, pe_rsc_managed)) {
                 /* We should never get into a situation where a managed remote
                  * connection resource is considered OK but a resource action
                  * behind the connection gets a "not connected" status. But as a
                  * fail-safe in case a bug or unusual circumstances do lead to
                  * that, ensure the remote connection is considered failed.
                  */
                 set_bit(node->details->remote_rsc->flags, pe_rsc_failed);
             }
 
             // fall through
 
         case PCMK_LRM_OP_ERROR:
         case PCMK_LRM_OP_ERROR_HARD:
         case PCMK_LRM_OP_ERROR_FATAL:
         case PCMK_LRM_OP_TIMEOUT:
         case PCMK_LRM_OP_NOTSUPPORTED:
         case PCMK_LRM_OP_INVALID:
 
             failure_strategy = get_action_on_fail(rsc, task_key, task, data_set);
             if ((failure_strategy == action_fail_ignore)
                 || (failure_strategy == action_fail_restart_container
                     && !strcmp(task, CRMD_ACTION_STOP))) {
 
                 crm_warn("Pretending failed %s (%s%s%s) of %s on %s at %s "
                          "succeeded " CRM_XS " rc=%d id=%s",
                          task, services_ocf_exitcode_str(rc),
                          (*exit_reason? ": " : ""), exit_reason, rsc->id,
                          node->details->uname, last_change_str(xml_op), rc,
                          ID(xml_op));
 
                 update_resource_state(rsc, node, xml_op, task, target_rc, *last_failure, on_fail, data_set);
                 crm_xml_add(xml_op, XML_ATTR_UNAME, node->details->uname);
                 set_bit(rsc->flags, pe_rsc_failure_ignored);
 
                 record_failed_op(xml_op, node, rsc, data_set);
 
                 if (failure_strategy == action_fail_restart_container && *on_fail <= action_fail_recover) {
                     *on_fail = failure_strategy;
                 }
 
             } else {
                 unpack_rsc_op_failure(rsc, node, rc, xml_op, last_failure, on_fail, data_set);
 
                 if(status == PCMK_LRM_OP_ERROR_HARD) {
                     do_crm_log(rc != PCMK_OCF_NOT_INSTALLED?LOG_ERR:LOG_NOTICE,
                                "Preventing %s from restarting on %s because "
                                "of hard failure (%s%s%s)" CRM_XS " rc=%d id=%s",
                                parent->id, node->details->uname,
                                services_ocf_exitcode_str(rc),
                                (*exit_reason? ": " : ""), exit_reason,
                                rc, ID(xml_op));
                     resource_location(parent, node, -INFINITY, "hard-error", data_set);
 
                 } else if(status == PCMK_LRM_OP_ERROR_FATAL) {
                     crm_err("Preventing %s from restarting anywhere because "
                             "of fatal failure (%s%s%s) " CRM_XS " rc=%d id=%s",
                             parent->id, services_ocf_exitcode_str(rc),
                             (*exit_reason? ": " : ""), exit_reason,
                             rc, ID(xml_op));
                     resource_location(parent, NULL, -INFINITY, "fatal-error", data_set);
                 }
             }
             break;
     }
 
   done:
     pe_rsc_trace(rsc, "Resource %s after %s: role=%s, next=%s",
                  rsc->id, task, role2text(rsc->role),
                  role2text(rsc->next_role));
 }
 
 static void
 add_node_attrs(xmlNode *xml_obj, pe_node_t *node, bool overwrite,
                pe_working_set_t *data_set)
 {
     const char *cluster_name = NULL;
 
     pe_rule_eval_data_t rule_data = {
         .node_hash = NULL,
         .role = RSC_ROLE_UNKNOWN,
         .now = data_set->now,
         .match_data = NULL,
         .rsc_data = NULL,
         .op_data = NULL
     };
 
     g_hash_table_insert(node->details->attrs,
                         strdup(CRM_ATTR_UNAME), strdup(node->details->uname));
 
     g_hash_table_insert(node->details->attrs, strdup(CRM_ATTR_ID),
                         strdup(node->details->id));
     if (safe_str_eq(node->details->id, data_set->dc_uuid)) {
         data_set->dc_node = node;
         node->details->is_dc = TRUE;
         g_hash_table_insert(node->details->attrs,
                             strdup(CRM_ATTR_IS_DC), strdup(XML_BOOLEAN_TRUE));
     } else {
         g_hash_table_insert(node->details->attrs,
                             strdup(CRM_ATTR_IS_DC), strdup(XML_BOOLEAN_FALSE));
     }
 
     cluster_name = g_hash_table_lookup(data_set->config_hash, "cluster-name");
     if (cluster_name) {
         g_hash_table_insert(node->details->attrs, strdup(CRM_ATTR_CLUSTER_NAME),
                             strdup(cluster_name));
     }
 
     pe__unpack_dataset_nvpairs(xml_obj, XML_TAG_ATTR_SETS, &rule_data,
                                node->details->attrs, NULL, overwrite, data_set);
 
     if (pe_node_attribute_raw(node, CRM_ATTR_SITE_NAME) == NULL) {
         const char *site_name = pe_node_attribute_raw(node, "site-name");
 
         if (site_name) {
             g_hash_table_insert(node->details->attrs,
                                 strdup(CRM_ATTR_SITE_NAME),
                                 strdup(site_name));
 
         } else if (cluster_name) {
             /* Default to cluster-name if unset */
             g_hash_table_insert(node->details->attrs,
                                 strdup(CRM_ATTR_SITE_NAME),
                                 strdup(cluster_name));
         }
     }
 }
 
 static GListPtr
 extract_operations(const char *node, const char *rsc, xmlNode * rsc_entry, gboolean active_filter)
 {
     int counter = -1;
     int stop_index = -1;
     int start_index = -1;
 
     xmlNode *rsc_op = NULL;
 
     GListPtr gIter = NULL;
     GListPtr op_list = NULL;
     GListPtr sorted_op_list = NULL;
 
     /* extract operations */
     op_list = NULL;
     sorted_op_list = NULL;
 
     for (rsc_op = __xml_first_child_element(rsc_entry);
          rsc_op != NULL; rsc_op = __xml_next_element(rsc_op)) {
         if (crm_str_eq((const char *)rsc_op->name, XML_LRM_TAG_RSC_OP, TRUE)) {
             crm_xml_add(rsc_op, "resource", rsc);
             crm_xml_add(rsc_op, XML_ATTR_UNAME, node);
             op_list = g_list_prepend(op_list, rsc_op);
         }
     }
 
     if (op_list == NULL) {
         /* if there are no operations, there is nothing to do */
         return NULL;
     }
 
     sorted_op_list = g_list_sort(op_list, sort_op_by_callid);
 
     /* create active recurring operations as optional */
     if (active_filter == FALSE) {
         return sorted_op_list;
     }
 
     op_list = NULL;
 
     calculate_active_ops(sorted_op_list, &start_index, &stop_index);
 
     for (gIter = sorted_op_list; gIter != NULL; gIter = gIter->next) {
         xmlNode *rsc_op = (xmlNode *) gIter->data;
 
         counter++;
 
         if (start_index < stop_index) {
             crm_trace("Skipping %s: not active", ID(rsc_entry));
             break;
 
         } else if (counter < start_index) {
             crm_trace("Skipping %s: old", ID(rsc_op));
             continue;
         }
         op_list = g_list_append(op_list, rsc_op);
     }
 
     g_list_free(sorted_op_list);
     return op_list;
 }
 
 GListPtr
 find_operations(const char *rsc, const char *node, gboolean active_filter,
                 pe_working_set_t * data_set)
 {
     GListPtr output = NULL;
     GListPtr intermediate = NULL;
 
     xmlNode *tmp = NULL;
     xmlNode *status = find_xml_node(data_set->input, XML_CIB_TAG_STATUS, TRUE);
 
     pe_node_t *this_node = NULL;
 
     xmlNode *node_state = NULL;
 
     for (node_state = __xml_first_child_element(status); node_state != NULL;
          node_state = __xml_next_element(node_state)) {
 
         if (crm_str_eq((const char *)node_state->name, XML_CIB_TAG_STATE, TRUE)) {
             const char *uname = crm_element_value(node_state, XML_ATTR_UNAME);
 
             if (node != NULL && safe_str_neq(uname, node)) {
                 continue;
             }
 
             this_node = pe_find_node(data_set->nodes, uname);
             if(this_node == NULL) {
                 CRM_LOG_ASSERT(this_node != NULL);
                 continue;
 
             } else if (pe__is_guest_or_remote_node(this_node)) {
                 determine_remote_online_status(data_set, this_node);
 
             } else {
                 determine_online_status(node_state, this_node, data_set);
             }
 
             if (this_node->details->online || is_set(data_set->flags, pe_flag_stonith_enabled)) {
                 /* offline nodes run no resources...
                  * unless stonith is enabled in which case we need to
                  *   make sure rsc start events happen after the stonith
                  */
                 xmlNode *lrm_rsc = NULL;
 
                 tmp = find_xml_node(node_state, XML_CIB_TAG_LRM, FALSE);
                 tmp = find_xml_node(tmp, XML_LRM_TAG_RESOURCES, FALSE);
 
                 for (lrm_rsc = __xml_first_child_element(tmp); lrm_rsc != NULL;
                      lrm_rsc = __xml_next_element(lrm_rsc)) {
                     if (crm_str_eq((const char *)lrm_rsc->name, XML_LRM_TAG_RESOURCE, TRUE)) {
 
                         const char *rsc_id = crm_element_value(lrm_rsc, XML_ATTR_ID);
 
                         if (rsc != NULL && safe_str_neq(rsc_id, rsc)) {
                             continue;
                         }
 
                         intermediate = extract_operations(uname, rsc_id, lrm_rsc, active_filter);
                         output = g_list_concat(output, intermediate);
                     }
                 }
             }
         }
     }
 
     return output;
 }
diff --git a/lib/pengine/utils.c b/lib/pengine/utils.c
index a59d3a7a8a..20a8db5cbd 100644
--- a/lib/pengine/utils.c
+++ b/lib/pengine/utils.c
@@ -1,2809 +1,2800 @@
 /*
  * Copyright 2004-2020 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU Lesser General Public License
  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 #include <crm/crm.h>
 #include <crm/msg_xml.h>
 #include <crm/common/xml.h>
 #include <crm/common/util.h>
 
 #include <ctype.h>
 #include <glib.h>
 #include <stdbool.h>
 
 #include <crm/pengine/rules.h>
 #include <crm/pengine/internal.h>
 
 extern xmlNode *get_object_root(const char *object_type, xmlNode * the_root);
 void print_str_str(gpointer key, gpointer value, gpointer user_data);
 gboolean ghash_free_str_str(gpointer key, gpointer value, gpointer user_data);
 static void unpack_operation(pe_action_t * action, xmlNode * xml_obj, pe_resource_t * container,
                              pe_working_set_t * data_set, guint interval_ms);
 static xmlNode *find_rsc_op_entry_helper(pe_resource_t * rsc, const char *key,
                                          gboolean include_disabled);
 
 #if ENABLE_VERSIONED_ATTRS
 pe_rsc_action_details_t *
 pe_rsc_action_details(pe_action_t *action)
 {
     pe_rsc_action_details_t *details;
 
     CRM_CHECK(action != NULL, return NULL);
 
     if (action->action_details == NULL) {
         action->action_details = calloc(1, sizeof(pe_rsc_action_details_t));
         CRM_CHECK(action->action_details != NULL, return NULL);
     }
 
     details = (pe_rsc_action_details_t *) action->action_details;
     if (details->versioned_parameters == NULL) {
         details->versioned_parameters = create_xml_node(NULL,
                                                         XML_TAG_OP_VER_ATTRS);
     }
     if (details->versioned_meta == NULL) {
         details->versioned_meta = create_xml_node(NULL, XML_TAG_OP_VER_META);
     }
     return details;
 }
 
 static void
 pe_free_rsc_action_details(pe_action_t *action)
 {
     pe_rsc_action_details_t *details;
 
     if ((action == NULL) || (action->action_details == NULL)) {
         return;
     }
 
     details = (pe_rsc_action_details_t *) action->action_details;
 
     if (details->versioned_parameters) {
         free_xml(details->versioned_parameters);
     }
     if (details->versioned_meta) {
         free_xml(details->versioned_meta);
     }
 
     action->action_details = NULL;
 }
 #endif
 
 /*!
  * \internal
  * \brief Check whether we can fence a particular node
  *
  * \param[in] data_set  Working set for cluster
  * \param[in] node      Name of node to check
  *
  * \return true if node can be fenced, false otherwise
  */
 bool
 pe_can_fence(pe_working_set_t *data_set, pe_node_t *node)
 {
     if (pe__is_guest_node(node)) {
         /* Guest nodes are fenced by stopping their container resource. We can
          * do that if the container's host is either online or fenceable.
          */
         pe_resource_t *rsc = node->details->remote_rsc->container;
 
         for (GList *n = rsc->running_on; n != NULL; n = n->next) {
             pe_node_t *container_node = n->data;
 
             if (!container_node->details->online
                 && !pe_can_fence(data_set, container_node)) {
                 return false;
             }
         }
         return true;
 
     } else if(is_not_set(data_set->flags, pe_flag_stonith_enabled)) {
         return false; /* Turned off */
 
     } else if (is_not_set(data_set->flags, pe_flag_have_stonith_resource)) {
         return false; /* No devices */
 
     } else if (is_set(data_set->flags, pe_flag_have_quorum)) {
         return true;
 
     } else if (data_set->no_quorum_policy == no_quorum_ignore) {
         return true;
 
     } else if(node == NULL) {
         return false;
 
     } else if(node->details->online) {
         crm_notice("We can fence %s without quorum because they're in our membership", node->details->uname);
         return true;
     }
 
     crm_trace("Cannot fence %s", node->details->uname);
     return false;
 }
 
 /*!
  * \internal
  * \brief Copy a node object
  *
  * \param[in] this_node  Node object to copy
  *
  * \return Newly allocated shallow copy of this_node
  * \note This function asserts on errors and is guaranteed to return non-NULL.
  */
 pe_node_t *
 pe__copy_node(const pe_node_t *this_node)
 {
     pe_node_t *new_node = NULL;
 
     CRM_ASSERT(this_node != NULL);
 
     new_node = calloc(1, sizeof(pe_node_t));
     CRM_ASSERT(new_node != NULL);
 
     new_node->rsc_discover_mode = this_node->rsc_discover_mode;
     new_node->weight = this_node->weight;
     new_node->fixed = this_node->fixed;
     new_node->details = this_node->details;
 
     return new_node;
 }
 
 /* any node in list1 or list2 and not in the other gets a score of -INFINITY */
 void
 node_list_exclude(GHashTable * hash, GListPtr list, gboolean merge_scores)
 {
     GHashTable *result = hash;
     pe_node_t *other_node = NULL;
     GListPtr gIter = list;
 
     GHashTableIter iter;
     pe_node_t *node = NULL;
 
     g_hash_table_iter_init(&iter, hash);
     while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
 
         other_node = pe_find_node_id(list, node->details->id);
         if (other_node == NULL) {
             node->weight = -INFINITY;
         } else if (merge_scores) {
             node->weight = pe__add_scores(node->weight, other_node->weight);
         }
     }
 
     for (; gIter != NULL; gIter = gIter->next) {
         pe_node_t *node = (pe_node_t *) gIter->data;
 
         other_node = pe_hash_table_lookup(result, node->details->id);
 
         if (other_node == NULL) {
             pe_node_t *new_node = pe__copy_node(node);
 
             new_node->weight = -INFINITY;
             g_hash_table_insert(result, (gpointer) new_node->details->id, new_node);
         }
     }
 }
 
 /*!
  * \internal
  * \brief Create a node hash table from a node list
  *
  * \param[in] list  Node list
  *
  * \return Hash table equivalent of node list
  */
 GHashTable *
 pe__node_list2table(GList *list)
 {
     GHashTable *result = NULL;
 
     result = g_hash_table_new_full(crm_str_hash, g_str_equal, NULL, free);
     for (GList *gIter = list; gIter != NULL; gIter = gIter->next) {
         pe_node_t *new_node = pe__copy_node((pe_node_t *) gIter->data);
 
         g_hash_table_insert(result, (gpointer) new_node->details->id, new_node);
     }
     return result;
 }
 
 gint
 sort_node_uname(gconstpointer a, gconstpointer b)
 {
     const char *name_a = ((const pe_node_t *) a)->details->uname;
     const char *name_b = ((const pe_node_t *) b)->details->uname;
 
     while (*name_a && *name_b) {
         if (isdigit(*name_a) && isdigit(*name_b)) {
             // If node names contain a number, sort numerically
 
             char *end_a = NULL;
             char *end_b = NULL;
             long num_a = strtol(name_a, &end_a, 10);
             long num_b = strtol(name_b, &end_b, 10);
 
             // allow ordering e.g. 007 > 7
             size_t len_a = end_a - name_a;
             size_t len_b = end_b - name_b;
 
             if (num_a < num_b) {
                 return -1;
             } else if (num_a > num_b) {
                 return 1;
             } else if (len_a < len_b) {
                 return -1;
             } else if (len_a > len_b) {
                 return 1;
             }
             name_a = end_a;
             name_b = end_b;
         } else {
             // Compare non-digits case-insensitively
             int lower_a = tolower(*name_a);
             int lower_b = tolower(*name_b);
 
             if (lower_a < lower_b) {
                 return -1;
             } else if (lower_a > lower_b) {
                 return 1;
             }
             ++name_a;
             ++name_b;
         }
     }
     if (!*name_a && *name_b) {
         return -1;
     } else if (*name_a && !*name_b) {
         return 1;
     }
     return 0;
 }
 
 /*!
  * \internal
  * \brief Output node weights to stdout
  *
  * \param[in] rsc       Use allowed nodes for this resource
  * \param[in] comment   Text description to prefix lines with
  * \param[in] nodes     If rsc is not specified, use these nodes
  */
 static void
 pe__output_node_weights(pe_resource_t *rsc, const char *comment,
                         GHashTable *nodes)
 {
     char score[128]; // Stack-allocated since this is called frequently
 
     // Sort the nodes so the output is consistent for regression tests
     GList *list = g_list_sort(g_hash_table_get_values(nodes), sort_node_uname);
 
     for (GList *gIter = list; gIter != NULL; gIter = gIter->next) {
         pe_node_t *node = (pe_node_t *) gIter->data;
 
         score2char_stack(node->weight, score, sizeof(score));
         if (rsc) {
             printf("%s: %s allocation score on %s: %s\n",
                    comment, rsc->id, node->details->uname, score);
         } else {
             printf("%s: %s = %s\n", comment, node->details->uname, score);
         }
     }
     g_list_free(list);
 }
 
 /*!
  * \internal
  * \brief Log node weights at trace level
  *
  * \param[in] file      Caller's filename
  * \param[in] function  Caller's function name
  * \param[in] line      Caller's line number
  * \param[in] rsc       Use allowed nodes for this resource
  * \param[in] comment   Text description to prefix lines with
  * \param[in] nodes     If rsc is not specified, use these nodes
  */
 static void
 pe__log_node_weights(const char *file, const char *function, int line,
                      pe_resource_t *rsc, const char *comment, GHashTable *nodes)
 {
     GHashTableIter iter;
     pe_node_t *node = NULL;
     char score[128]; // Stack-allocated since this is called frequently
 
     // Don't waste time if we're not tracing at this point
     pcmk__log_else(LOG_TRACE, return);
 
     g_hash_table_iter_init(&iter, nodes);
     while (g_hash_table_iter_next(&iter, NULL, (void **) &node)) {
         score2char_stack(node->weight, score, sizeof(score));
         if (rsc) {
             qb_log_from_external_source(function, file,
                                         "%s: %s allocation score on %s: %s",
                                         LOG_TRACE, line, 0,
                                         comment, rsc->id,
                                         node->details->uname, score);
         } else {
             qb_log_from_external_source(function, file, "%s: %s = %s",
                                         LOG_TRACE, line, 0,
                                         comment, node->details->uname,
                                         score);
         }
     }
 }
 
 /*!
  * \internal
  * \brief Log or output node weights
  *
  * \param[in] file      Caller's filename
  * \param[in] function  Caller's function name
  * \param[in] line      Caller's line number
  * \param[in] to_log    Log if true, otherwise output
  * \param[in] rsc       Use allowed nodes for this resource
  * \param[in] comment   Text description to prefix lines with
  * \param[in] nodes     If rsc is not specified, use these nodes
  */
 void
 pe__show_node_weights_as(const char *file, const char *function, int line,
                          bool to_log, pe_resource_t *rsc, const char *comment,
                          GHashTable *nodes)
 {
     if (rsc != NULL) {
         if (is_set(rsc->flags, pe_rsc_orphan)) {
             // Don't show allocation scores for orphans
             return;
         }
         nodes = rsc->allowed_nodes;
     }
     if (nodes == NULL) {
         // Nothing to show
         return;
     }
 
     if (to_log) {
         pe__log_node_weights(file, function, line, rsc, comment, nodes);
     } else {
         pe__output_node_weights(rsc, comment, nodes);
     }
 
     // If this resource has children, repeat recursively for each
     if (rsc && rsc->children) {
         for (GList *gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
             pe_resource_t *child = (pe_resource_t *) gIter->data;
 
             pe__show_node_weights_as(file, function, line, to_log, child,
                                      comment, nodes);
         }
     }
 }
 
 static void
 append_dump_text(gpointer key, gpointer value, gpointer user_data)
 {
     char **dump_text = user_data;
     char *new_text = crm_strdup_printf("%s %s=%s",
                                        *dump_text, (char *)key, (char *)value);
 
     free(*dump_text);
     *dump_text = new_text;
 }
 
 void
 dump_node_capacity(int level, const char *comment, pe_node_t * node)
 {
     char *dump_text = crm_strdup_printf("%s: %s capacity:",
                                         comment, node->details->uname);
 
     g_hash_table_foreach(node->details->utilization, append_dump_text, &dump_text);
 
     if (level == LOG_STDOUT) {
         fprintf(stdout, "%s\n", dump_text);
     } else {
         crm_trace("%s", dump_text);
     }
 
     free(dump_text);
 }
 
 void
 dump_rsc_utilization(int level, const char *comment, pe_resource_t * rsc, pe_node_t * node)
 {
     char *dump_text = crm_strdup_printf("%s: %s utilization on %s:",
                                         comment, rsc->id, node->details->uname);
 
     g_hash_table_foreach(rsc->utilization, append_dump_text, &dump_text);
     switch (level) {
         case LOG_STDOUT:
             fprintf(stdout, "%s\n", dump_text);
             break;
         case LOG_NEVER:
             break;
         default:
             crm_trace("%s", dump_text);
     }
     free(dump_text);
 }
 
 gint
 sort_rsc_index(gconstpointer a, gconstpointer b)
 {
     const pe_resource_t *resource1 = (const pe_resource_t *)a;
     const pe_resource_t *resource2 = (const pe_resource_t *)b;
 
     if (a == NULL && b == NULL) {
         return 0;
     }
     if (a == NULL) {
         return 1;
     }
     if (b == NULL) {
         return -1;
     }
 
     if (resource1->sort_index > resource2->sort_index) {
         return -1;
     }
 
     if (resource1->sort_index < resource2->sort_index) {
         return 1;
     }
 
     return 0;
 }
 
 gint
 sort_rsc_priority(gconstpointer a, gconstpointer b)
 {
     const pe_resource_t *resource1 = (const pe_resource_t *)a;
     const pe_resource_t *resource2 = (const pe_resource_t *)b;
 
     if (a == NULL && b == NULL) {
         return 0;
     }
     if (a == NULL) {
         return 1;
     }
     if (b == NULL) {
         return -1;
     }
 
     if (resource1->priority > resource2->priority) {
         return -1;
     }
 
     if (resource1->priority < resource2->priority) {
         return 1;
     }
 
     return 0;
 }
 
 pe_action_t *
 custom_action(pe_resource_t * rsc, char *key, const char *task,
               pe_node_t * on_node, gboolean optional, gboolean save_action,
               pe_working_set_t * data_set)
 {
     pe_action_t *action = NULL;
     GListPtr possible_matches = NULL;
 
     CRM_CHECK(key != NULL, return NULL);
     CRM_CHECK(task != NULL, free(key); return NULL);
 
     if (save_action && rsc != NULL) {
         possible_matches = find_actions(rsc->actions, key, on_node);
     } else if(save_action) {
 #if 0
         action = g_hash_table_lookup(data_set->singletons, key);
 #else
         /* More expensive but takes 'node' into account */
         possible_matches = find_actions(data_set->actions, key, on_node);
 #endif
     }
 
     if(data_set->singletons == NULL) {
         data_set->singletons = g_hash_table_new_full(crm_str_hash, g_str_equal, NULL, NULL);
     }
 
     if (possible_matches != NULL) {
         if (pcmk__list_of_multiple(possible_matches)) {
             pe_warn("Action %s for %s on %s exists %d times",
                     task, rsc ? rsc->id : "<NULL>",
                     on_node ? on_node->details->uname : "<NULL>", g_list_length(possible_matches));
         }
 
         action = g_list_nth_data(possible_matches, 0);
         pe_rsc_trace(rsc, "Found existing action %d (%s) for %s (%s) on %s",
                      action->id, action->uuid,
                      (rsc? rsc->id : "no resource"), task,
                      (on_node? on_node->details->uname : "no node"));
         g_list_free(possible_matches);
     }
 
     if (action == NULL) {
         if (save_action) {
             pe_rsc_trace(rsc, "Creating %s action %d: %s for %s (%s) on %s",
                          (optional? "optional" : "mandatory"),
                          data_set->action_id, key,
                          (rsc? rsc->id : "no resource"), task,
                          (on_node? on_node->details->uname : "no node"));
         }
 
         action = calloc(1, sizeof(pe_action_t));
         if (save_action) {
             action->id = data_set->action_id++;
         } else {
             action->id = 0;
         }
         action->rsc = rsc;
         CRM_ASSERT(task != NULL);
         action->task = strdup(task);
         if (on_node) {
             action->node = pe__copy_node(on_node);
         }
         action->uuid = strdup(key);
 
         if (safe_str_eq(task, CRM_OP_LRM_DELETE)) {
             // Resource history deletion for a node can be done on the DC
             pe_set_action_bit(action, pe_action_dc);
         }
 
         pe_set_action_bit(action, pe_action_runnable);
         if (optional) {
             pe_set_action_bit(action, pe_action_optional);
         } else {
             pe_clear_action_bit(action, pe_action_optional);
         }
 
         action->extra = crm_str_table_new();
         action->meta = crm_str_table_new();
 
         if (save_action) {
             data_set->actions = g_list_prepend(data_set->actions, action);
             if(rsc == NULL) {
                 g_hash_table_insert(data_set->singletons, action->uuid, action);
             }
         }
 
         if (rsc != NULL) {
             guint interval_ms = 0;
 
             action->op_entry = find_rsc_op_entry_helper(rsc, key, TRUE);
             parse_op_key(key, NULL, NULL, &interval_ms);
 
             unpack_operation(action, action->op_entry, rsc->container, data_set,
                              interval_ms);
 
             if (save_action) {
                 rsc->actions = g_list_prepend(rsc->actions, action);
             }
         }
 
         if (save_action) {
             pe_rsc_trace(rsc, "Action %d created", action->id);
         }
     }
 
     if (!optional && is_set(action->flags, pe_action_optional)) {
         pe_rsc_trace(rsc, "Unset optional on action %d", action->id);
         pe_clear_action_bit(action, pe_action_optional);
     }
 
     if (rsc != NULL) {
         enum action_tasks a_task = text2task(action->task);
         int warn_level = LOG_TRACE;
 
         if (save_action) {
             warn_level = LOG_WARNING;
         }
 
         if (is_set(action->flags, pe_action_have_node_attrs) == FALSE
             && action->node != NULL && action->op_entry != NULL) {
             pe_rule_eval_data_t rule_data = {
                 .node_hash = action->node->details->attrs,
                 .role = RSC_ROLE_UNKNOWN,
                 .now = data_set->now,
                 .match_data = NULL,
                 .rsc_data = NULL,
                 .op_data = NULL
             };
 
             pe_set_action_bit(action, pe_action_have_node_attrs);
             pe__unpack_dataset_nvpairs(action->op_entry, XML_TAG_ATTR_SETS,
                                        &rule_data, action->extra, NULL,
                                        FALSE, data_set);
         }
 
         if (is_set(action->flags, pe_action_pseudo)) {
             /* leave untouched */
 
         } else if (action->node == NULL) {
             pe_rsc_trace(rsc, "Unset runnable on %s", action->uuid);
             pe_clear_action_bit(action, pe_action_runnable);
 
         } else if (is_not_set(rsc->flags, pe_rsc_managed)
                    && g_hash_table_lookup(action->meta,
                                           XML_LRM_ATTR_INTERVAL_MS) == NULL) {
             crm_debug("Action %s (unmanaged)", action->uuid);
             pe_rsc_trace(rsc, "Set optional on %s", action->uuid);
             pe_set_action_bit(action, pe_action_optional);
 /*   			action->runnable = FALSE; */
 
         } else if (is_not_set(action->flags, pe_action_dc)
                    && !(action->node->details->online)
                    && (!pe__is_guest_node(action->node)
                        || action->node->details->remote_requires_reset)) {
             pe_clear_action_bit(action, pe_action_runnable);
             do_crm_log(warn_level, "Action %s on %s is unrunnable (offline)",
                        action->uuid, action->node->details->uname);
             if (is_set(action->rsc->flags, pe_rsc_managed)
                 && save_action && a_task == stop_rsc
                 && action->node->details->unclean == FALSE) {
                 pe_fence_node(data_set, action->node, "resource actions are unrunnable", FALSE);
             }
 
         } else if (is_not_set(action->flags, pe_action_dc)
                    && action->node->details->pending) {
             pe_clear_action_bit(action, pe_action_runnable);
             do_crm_log(warn_level, "Action %s on %s is unrunnable (pending)",
                        action->uuid, action->node->details->uname);
 
         } else if (action->needs == rsc_req_nothing) {
             pe_rsc_trace(rsc, "Action %s does not require anything", action->uuid);
             pe_action_set_reason(action, NULL, TRUE);
             if (pe__is_guest_node(action->node)
                 && !pe_can_fence(data_set, action->node)) {
                 /* An action that requires nothing usually does not require any
                  * fencing in order to be runnable. However, there is an
                  * exception: an action cannot be completed if it is on a guest
                  * node whose host is unclean and cannot be fenced.
                  */
                 pe_clear_action_bit(action, pe_action_runnable);
                 crm_debug("%s\t%s (cancelled : host cannot be fenced)",
                           action->node->details->uname, action->uuid);
             } else {
                 pe_set_action_bit(action, pe_action_runnable);
             }
 #if 0
             /*
              * No point checking this
              * - if we don't have quorum we can't stonith anyway
              */
         } else if (action->needs == rsc_req_stonith) {
             crm_trace("Action %s requires only stonith", action->uuid);
             action->runnable = TRUE;
 #endif
         } else if (is_set(data_set->flags, pe_flag_have_quorum) == FALSE
                    && data_set->no_quorum_policy == no_quorum_stop) {
             pe_action_set_flag_reason(__FUNCTION__, __LINE__, action, NULL, "no quorum", pe_action_runnable, TRUE);
             crm_debug("%s\t%s (cancelled : quorum)", action->node->details->uname, action->uuid);
 
         } else if (is_set(data_set->flags, pe_flag_have_quorum) == FALSE
                    && data_set->no_quorum_policy == no_quorum_freeze) {
             pe_rsc_trace(rsc, "Check resource is already active: %s %s %s %s", rsc->id, action->uuid, role2text(rsc->next_role), role2text(rsc->role));
             if (rsc->fns->active(rsc, TRUE) == FALSE || rsc->next_role > rsc->role) {
                 pe_action_set_flag_reason(__FUNCTION__, __LINE__, action, NULL, "quorum freeze", pe_action_runnable, TRUE);
                 pe_rsc_debug(rsc, "%s\t%s (cancelled : quorum freeze)",
                              action->node->details->uname, action->uuid);
             }
 
         } else if(is_not_set(action->flags, pe_action_runnable)) {
             pe_rsc_trace(rsc, "Action %s is runnable", action->uuid);
             //pe_action_set_reason(action, NULL, TRUE);
             pe_set_action_bit(action, pe_action_runnable);
         }
 
         if (save_action) {
             switch (a_task) {
                 case stop_rsc:
                     set_bit(rsc->flags, pe_rsc_stopping);
                     break;
                 case start_rsc:
                     clear_bit(rsc->flags, pe_rsc_starting);
                     if (is_set(action->flags, pe_action_runnable)) {
                         set_bit(rsc->flags, pe_rsc_starting);
                     }
                     break;
                 default:
                     break;
             }
         }
     }
 
     free(key);
     return action;
 }
 
 static const char *
 unpack_operation_on_fail(pe_action_t * action)
 {
 
     const char *value = g_hash_table_lookup(action->meta, XML_OP_ATTR_ON_FAIL);
 
     if (safe_str_eq(action->task, CRMD_ACTION_STOP) && safe_str_eq(value, "standby")) {
         pcmk__config_err("Resetting '" XML_OP_ATTR_ON_FAIL "' for %s stop "
                          "action to default value because 'standby' is not "
                          "allowed for stop", action->rsc->id);
         return NULL;
     } else if (safe_str_eq(action->task, CRMD_ACTION_DEMOTE) && !value) {
         /* demote on_fail defaults to master monitor value if present */
         xmlNode *operation = NULL;
         const char *name = NULL;
         const char *role = NULL;
         const char *on_fail = NULL;
         const char *interval_spec = NULL;
         const char *enabled = NULL;
 
         CRM_CHECK(action->rsc != NULL, return NULL);
 
         for (operation = __xml_first_child_element(action->rsc->ops_xml);
              operation && !value; operation = __xml_next_element(operation)) {
 
             if (!crm_str_eq((const char *)operation->name, "op", TRUE)) {
                 continue;
             }
             name = crm_element_value(operation, "name");
             role = crm_element_value(operation, "role");
             on_fail = crm_element_value(operation, XML_OP_ATTR_ON_FAIL);
             enabled = crm_element_value(operation, "enabled");
             interval_spec = crm_element_value(operation, XML_LRM_ATTR_INTERVAL);
             if (!on_fail) {
                 continue;
             } else if (enabled && !crm_is_true(enabled)) {
                 continue;
             } else if (safe_str_neq(name, "monitor") || safe_str_neq(role, "Master")) {
                 continue;
             } else if (crm_parse_interval_spec(interval_spec) == 0) {
                 continue;
             }
 
             value = on_fail;
         }
     } else if (safe_str_eq(action->task, CRM_OP_LRM_DELETE)) {
         value = "ignore";
     }
 
     return value;
 }
 
 static xmlNode *
 find_min_interval_mon(pe_resource_t * rsc, gboolean include_disabled)
 {
     guint interval_ms = 0;
     guint min_interval_ms = G_MAXUINT;
     const char *name = NULL;
     const char *value = NULL;
     const char *interval_spec = NULL;
     xmlNode *op = NULL;
     xmlNode *operation = NULL;
 
     for (operation = __xml_first_child_element(rsc->ops_xml); operation != NULL;
          operation = __xml_next_element(operation)) {
 
         if (crm_str_eq((const char *)operation->name, "op", TRUE)) {
             name = crm_element_value(operation, "name");
             interval_spec = crm_element_value(operation, XML_LRM_ATTR_INTERVAL);
             value = crm_element_value(operation, "enabled");
             if (!include_disabled && value && crm_is_true(value) == FALSE) {
                 continue;
             }
 
             if (safe_str_neq(name, RSC_STATUS)) {
                 continue;
             }
 
             interval_ms = crm_parse_interval_spec(interval_spec);
 
             if (interval_ms && (interval_ms < min_interval_ms)) {
                 min_interval_ms = interval_ms;
                 op = operation;
             }
         }
     }
 
     return op;
 }
 
 static int
 unpack_start_delay(const char *value, GHashTable *meta)
 {
     int start_delay = 0;
 
     if (value != NULL) {
         start_delay = crm_get_msec(value);
 
         if (start_delay < 0) {
             start_delay = 0;
         }
 
         if (meta) {
             g_hash_table_replace(meta, strdup(XML_OP_ATTR_START_DELAY), crm_itoa(start_delay));
         }
     }
 
     return start_delay;
 }
 
 // true if value contains valid, non-NULL interval origin for recurring op
 static bool
 unpack_interval_origin(const char *value, xmlNode *xml_obj, guint interval_ms,
                        crm_time_t *now, long long *start_delay)
 {
     long long result = 0;
     guint interval_sec = interval_ms / 1000;
     crm_time_t *origin = NULL;
 
     // Ignore unspecified values and non-recurring operations
     if ((value == NULL) || (interval_ms == 0) || (now == NULL)) {
         return false;
     }
 
     // Parse interval origin from text
     origin = crm_time_new(value);
     if (origin == NULL) {
         pcmk__config_err("Ignoring '" XML_OP_ATTR_ORIGIN "' for operation "
                          "'%s' because '%s' is not valid",
                          (ID(xml_obj)? ID(xml_obj) : "(missing ID)"), value);
         return false;
     }
 
     // Get seconds since origin (negative if origin is in the future)
     result = crm_time_get_seconds(now) - crm_time_get_seconds(origin);
     crm_time_free(origin);
 
     // Calculate seconds from closest interval to now
     result = result % interval_sec;
 
     // Calculate seconds remaining until next interval
     result = ((result <= 0)? 0 : interval_sec) - result;
     crm_info("Calculated a start delay of %llds for operation '%s'",
              result,
              (ID(xml_obj)? ID(xml_obj) : "(unspecified)"));
 
     if (start_delay != NULL) {
         *start_delay = result * 1000; // milliseconds
     }
     return true;
 }
 
 static int
 unpack_timeout(const char *value)
 {
     int timeout = crm_get_msec(value);
 
     if (timeout < 0) {
         timeout = crm_get_msec(CRM_DEFAULT_OP_TIMEOUT_S);
     }
     return timeout;
 }
 
 int
 pe_get_configured_timeout(pe_resource_t *rsc, const char *action, pe_working_set_t *data_set)
 {
     xmlNode *child = NULL;
     const char *timeout = NULL;
     int timeout_ms = 0;
 
     pe_rule_eval_data_t rule_data = {
         .node_hash = NULL,
         .role = RSC_ROLE_UNKNOWN,
         .now = data_set->now,
         .match_data = NULL,
         .rsc_data = NULL,
         .op_data = NULL
     };
 
     for (child = first_named_child(rsc->ops_xml, XML_ATTR_OP);
          child != NULL; child = crm_next_same_xml(child)) {
         if (safe_str_eq(action, crm_element_value(child, XML_NVPAIR_ATTR_NAME))) {
             timeout = crm_element_value(child, XML_ATTR_TIMEOUT);
             break;
         }
     }
 
     if (timeout == NULL && data_set->op_defaults) {
         GHashTable *action_meta = crm_str_table_new();
         pe__unpack_dataset_nvpairs(data_set->op_defaults, XML_TAG_META_SETS,
                                    &rule_data, action_meta, NULL, FALSE, data_set);
         timeout = g_hash_table_lookup(action_meta, XML_ATTR_TIMEOUT);
     }
 
     // @TODO check meta-attributes (including versioned meta-attributes)
     // @TODO maybe use min-interval monitor timeout as default for monitors
 
     timeout_ms = crm_get_msec(timeout);
     if (timeout_ms < 0) {
         timeout_ms = crm_get_msec(CRM_DEFAULT_OP_TIMEOUT_S);
     }
     return timeout_ms;
 }
 
 #if ENABLE_VERSIONED_ATTRS
 static void
 unpack_versioned_meta(xmlNode *versioned_meta, xmlNode *xml_obj,
                       guint interval_ms, crm_time_t *now)
 {
     xmlNode *attrs = NULL;
     xmlNode *attr = NULL;
 
     for (attrs = __xml_first_child_element(versioned_meta); attrs != NULL;
          attrs = __xml_next_element(attrs)) {
 
         for (attr = __xml_first_child_element(attrs); attr != NULL;
              attr = __xml_next_element(attr)) {
 
             const char *name = crm_element_value(attr, XML_NVPAIR_ATTR_NAME);
             const char *value = crm_element_value(attr, XML_NVPAIR_ATTR_VALUE);
 
             if (safe_str_eq(name, XML_OP_ATTR_START_DELAY)) {
                 int start_delay = unpack_start_delay(value, NULL);
 
                 crm_xml_add_int(attr, XML_NVPAIR_ATTR_VALUE, start_delay);
             } else if (safe_str_eq(name, XML_OP_ATTR_ORIGIN)) {
                 long long start_delay = 0;
 
                 if (unpack_interval_origin(value, xml_obj, interval_ms, now,
                                            &start_delay)) {
                     crm_xml_add(attr, XML_NVPAIR_ATTR_NAME,
                                 XML_OP_ATTR_START_DELAY);
                     crm_xml_add_ll(attr, XML_NVPAIR_ATTR_VALUE, start_delay);
                 }
             } else if (safe_str_eq(name, XML_ATTR_TIMEOUT)) {
                 int timeout = unpack_timeout(value);
 
                 crm_xml_add_int(attr, XML_NVPAIR_ATTR_VALUE, timeout);
             }
         }
     }
 }
 #endif
 
 /*!
  * \brief Unpack operation XML into an action structure
  *
  * Unpack an operation's meta-attributes (normalizing the interval, timeout,
  * and start delay values as integer milliseconds), requirements, and
  * failure policy.
  *
  * \param[in,out] action      Action to unpack into
  * \param[in]     xml_obj     Operation XML (or NULL if all defaults)
  * \param[in]     container   Resource that contains affected resource, if any
  * \param[in]     data_set    Cluster state
  * \param[in]     interval_ms How frequently to perform the operation
  */
 static void
 unpack_operation(pe_action_t * action, xmlNode * xml_obj, pe_resource_t * container,
                  pe_working_set_t * data_set, guint interval_ms)
 {
     int timeout = 0;
     char *value_ms = NULL;
     const char *value = NULL;
     const char *field = XML_LRM_ATTR_INTERVAL;
     char *default_timeout = NULL;
 #if ENABLE_VERSIONED_ATTRS
     pe_rsc_action_details_t *rsc_details = NULL;
 #endif
 
     pe_rsc_eval_data_t rsc_rule_data = {
         .standard = crm_element_value(action->rsc->xml, XML_AGENT_ATTR_CLASS),
         .provider = crm_element_value(action->rsc->xml, XML_AGENT_ATTR_PROVIDER),
         .agent = crm_element_value(action->rsc->xml, XML_EXPR_ATTR_TYPE)
     };
 
     pe_op_eval_data_t op_rule_data = {
         .op_name = action->task,
         .interval = interval_ms
     };
 
     pe_rule_eval_data_t rule_data = {
         .node_hash = NULL,
         .role = RSC_ROLE_UNKNOWN,
         .now = data_set->now,
         .match_data = NULL,
         .rsc_data = &rsc_rule_data,
         .op_data = &op_rule_data
     };
 
     CRM_CHECK(action && action->rsc, return);
 
     // Cluster-wide <op_defaults> <meta_attributes>
     pe__unpack_dataset_nvpairs(data_set->op_defaults, XML_TAG_META_SETS, &rule_data,
                                action->meta, NULL, FALSE, data_set);
 
     // Probe timeouts default differently, so handle timeout default later
     default_timeout = g_hash_table_lookup(action->meta, XML_ATTR_TIMEOUT);
     if (default_timeout) {
         default_timeout = strdup(default_timeout);
         g_hash_table_remove(action->meta, XML_ATTR_TIMEOUT);
     }
 
     if (xml_obj) {
         xmlAttrPtr xIter = NULL;
 
         // <op> <meta_attributes> take precedence over defaults
         pe__unpack_dataset_nvpairs(xml_obj, XML_TAG_META_SETS, &rule_data,
                                    action->meta, NULL, TRUE, data_set);
 
 #if ENABLE_VERSIONED_ATTRS
         rsc_details = pe_rsc_action_details(action);
 
         pe_eval_versioned_attributes(data_set->input, xml_obj,
                                      XML_TAG_ATTR_SETS, &rule_data,
                                      rsc_details->versioned_parameters,
                                      NULL);
         pe_eval_versioned_attributes(data_set->input, xml_obj,
                                      XML_TAG_META_SETS, &rule_data,
                                      rsc_details->versioned_meta,
                                      NULL);
 #endif
 
         /* Anything set as an <op> XML property has highest precedence.
          * This ensures we use the name and interval from the <op> tag.
          */
         for (xIter = xml_obj->properties; xIter; xIter = xIter->next) {
             const char *prop_name = (const char *)xIter->name;
             const char *prop_value = crm_element_value(xml_obj, prop_name);
 
             g_hash_table_replace(action->meta, strdup(prop_name), strdup(prop_value));
         }
     }
 
     g_hash_table_remove(action->meta, "id");
 
     // Normalize interval to milliseconds
     if (interval_ms > 0) {
         value_ms = crm_strdup_printf("%u", interval_ms);
         g_hash_table_replace(action->meta, strdup(field), value_ms);
 
     } else if (g_hash_table_lookup(action->meta, field) != NULL) {
         g_hash_table_remove(action->meta, field);
     }
 
     // Handle timeout default, now that we know the interval
     if (g_hash_table_lookup(action->meta, XML_ATTR_TIMEOUT)) {
         free(default_timeout);
 
     } else {
         // Probe timeouts default to minimum-interval monitor's
         if (safe_str_eq(action->task, RSC_STATUS) && (interval_ms == 0)) {
 
             xmlNode *min_interval_mon = find_min_interval_mon(action->rsc, FALSE);
 
             if (min_interval_mon) {
                 value = crm_element_value(min_interval_mon, XML_ATTR_TIMEOUT);
                 if (value) {
                     crm_trace("\t%s defaults to minimum-interval monitor's timeout '%s'",
                               action->uuid, value);
                     free(default_timeout);
                     default_timeout = strdup(value);
                 }
             }
         }
 
         if (default_timeout) {
             g_hash_table_insert(action->meta, strdup(XML_ATTR_TIMEOUT),
                                 default_timeout);
         }
     }
 
     if (safe_str_neq(action->task, RSC_START)
         && safe_str_neq(action->task, RSC_PROMOTE)) {
         action->needs = rsc_req_nothing;
         value = "nothing (not start/promote)";
 
     } else if (is_set(action->rsc->flags, pe_rsc_needs_fencing)) {
         action->needs = rsc_req_stonith;
         value = "fencing (resource)";
 
     } else if (is_set(action->rsc->flags, pe_rsc_needs_quorum)) {
         action->needs = rsc_req_quorum;
         value = "quorum (resource)";
 
     } else {
         action->needs = rsc_req_nothing;
         value = "nothing (resource)";
     }
 
     pe_rsc_trace(action->rsc, "\tAction %s requires: %s", action->uuid, value);
 
     value = unpack_operation_on_fail(action);
 
     if (value == NULL) {
 
     } else if (safe_str_eq(value, "block")) {
         action->on_fail = action_fail_block;
         g_hash_table_insert(action->meta, strdup(XML_OP_ATTR_ON_FAIL), strdup("block"));
         value = "block"; // The above could destroy the original string
 
     } else if (safe_str_eq(value, "fence")) {
         action->on_fail = action_fail_fence;
         value = "node fencing";
 
         if (is_set(data_set->flags, pe_flag_stonith_enabled) == FALSE) {
             pcmk__config_err("Resetting '" XML_OP_ATTR_ON_FAIL "' for "
                              "operation '%s' to 'stop' because 'fence' is not "
                              "valid when fencing is disabled", action->uuid);
             action->on_fail = action_fail_stop;
             action->fail_role = RSC_ROLE_STOPPED;
             value = "stop resource";
         }
 
     } else if (safe_str_eq(value, "standby")) {
         action->on_fail = action_fail_standby;
         value = "node standby";
 
     } else if (safe_str_eq(value, "ignore")
                || safe_str_eq(value, "nothing")) {
         action->on_fail = action_fail_ignore;
         value = "ignore";
 
     } else if (safe_str_eq(value, "migrate")) {
         action->on_fail = action_fail_migrate;
         value = "force migration";
 
     } else if (safe_str_eq(value, "stop")) {
         action->on_fail = action_fail_stop;
         action->fail_role = RSC_ROLE_STOPPED;
         value = "stop resource";
 
     } else if (safe_str_eq(value, "restart")) {
         action->on_fail = action_fail_recover;
         value = "restart (and possibly migrate)";
 
     } else if (safe_str_eq(value, "restart-container")) {
         if (container) {
             action->on_fail = action_fail_restart_container;
             value = "restart container (and possibly migrate)";
 
         } else {
             value = NULL;
         }
 
     } else {
         pe_err("Resource %s: Unknown failure type (%s)", action->rsc->id, value);
         value = NULL;
     }
 
     /* defaults */
     if (value == NULL && container) {
         action->on_fail = action_fail_restart_container;
         value = "restart container (and possibly migrate) (default)";
 
     /* For remote nodes, ensure that any failure that results in dropping an
      * active connection to the node results in fencing of the node.
      *
      * There are only two action failures that don't result in fencing.
      * 1. probes - probe failures are expected.
      * 2. start - a start failure indicates that an active connection does not already
      * exist. The user can set op on-fail=fence if they really want to fence start
      * failures. */
     } else if (((value == NULL) || !is_set(action->rsc->flags, pe_rsc_managed)) &&
                 (pe__resource_is_remote_conn(action->rsc, data_set) &&
                !(safe_str_eq(action->task, CRMD_ACTION_STATUS) && (interval_ms == 0)) &&
                 (safe_str_neq(action->task, CRMD_ACTION_START)))) {
 
         if (!is_set(action->rsc->flags, pe_rsc_managed)) {
             action->on_fail = action_fail_stop;
             action->fail_role = RSC_ROLE_STOPPED;
             value = "stop unmanaged remote node (enforcing default)";
 
         } else {
             if (is_set(data_set->flags, pe_flag_stonith_enabled)) {
                 value = "fence remote node (default)";
             } else {
                 value = "recover remote node connection (default)";
             }
 
             if (action->rsc->remote_reconnect_ms) {
                 action->fail_role = RSC_ROLE_STOPPED;
             }
             action->on_fail = action_fail_reset_remote;
         }
 
     } else if (value == NULL && safe_str_eq(action->task, CRMD_ACTION_STOP)) {
         if (is_set(data_set->flags, pe_flag_stonith_enabled)) {
             action->on_fail = action_fail_fence;
             value = "resource fence (default)";
 
         } else {
             action->on_fail = action_fail_block;
             value = "resource block (default)";
         }
 
     } else if (value == NULL) {
         action->on_fail = action_fail_recover;
         value = "restart (and possibly migrate) (default)";
     }
 
     pe_rsc_trace(action->rsc, "\t%s failure handling: %s", action->task, value);
 
     value = NULL;
     if (xml_obj != NULL) {
         value = g_hash_table_lookup(action->meta, "role_after_failure");
         if (value) {
             pe_warn_once(pe_wo_role_after,
                         "Support for role_after_failure is deprecated and will be removed in a future release");
         }
     }
     if (value != NULL && action->fail_role == RSC_ROLE_UNKNOWN) {
         action->fail_role = text2role(value);
     }
     /* defaults */
     if (action->fail_role == RSC_ROLE_UNKNOWN) {
         if (safe_str_eq(action->task, CRMD_ACTION_PROMOTE)) {
             action->fail_role = RSC_ROLE_SLAVE;
         } else {
             action->fail_role = RSC_ROLE_STARTED;
         }
     }
     pe_rsc_trace(action->rsc, "\t%s failure results in: %s", action->task,
                  role2text(action->fail_role));
 
     value = g_hash_table_lookup(action->meta, XML_OP_ATTR_START_DELAY);
     if (value) {
         unpack_start_delay(value, action->meta);
     } else {
         long long start_delay = 0;
 
         value = g_hash_table_lookup(action->meta, XML_OP_ATTR_ORIGIN);
         if (unpack_interval_origin(value, xml_obj, interval_ms, data_set->now,
                                    &start_delay)) {
             g_hash_table_replace(action->meta, strdup(XML_OP_ATTR_START_DELAY),
                                  crm_strdup_printf("%lld", start_delay));
         }
     }
 
     value = g_hash_table_lookup(action->meta, XML_ATTR_TIMEOUT);
     timeout = unpack_timeout(value);
     g_hash_table_replace(action->meta, strdup(XML_ATTR_TIMEOUT), crm_itoa(timeout));
 
 #if ENABLE_VERSIONED_ATTRS
     unpack_versioned_meta(rsc_details->versioned_meta, xml_obj, interval_ms,
                           data_set->now);
 #endif
 }
 
 static xmlNode *
 find_rsc_op_entry_helper(pe_resource_t * rsc, const char *key, gboolean include_disabled)
 {
     guint interval_ms = 0;
     gboolean do_retry = TRUE;
     char *local_key = NULL;
     const char *name = NULL;
     const char *value = NULL;
     const char *interval_spec = NULL;
     char *match_key = NULL;
     xmlNode *op = NULL;
     xmlNode *operation = NULL;
 
   retry:
     for (operation = __xml_first_child_element(rsc->ops_xml); operation != NULL;
          operation = __xml_next_element(operation)) {
         if (crm_str_eq((const char *)operation->name, "op", TRUE)) {
             name = crm_element_value(operation, "name");
             interval_spec = crm_element_value(operation, XML_LRM_ATTR_INTERVAL);
             value = crm_element_value(operation, "enabled");
             if (!include_disabled && value && crm_is_true(value) == FALSE) {
                 continue;
             }
 
             interval_ms = crm_parse_interval_spec(interval_spec);
             match_key = pcmk__op_key(rsc->id, name, interval_ms);
             if (safe_str_eq(key, match_key)) {
                 op = operation;
             }
             free(match_key);
 
             if (rsc->clone_name) {
                 match_key = pcmk__op_key(rsc->clone_name, name, interval_ms);
                 if (safe_str_eq(key, match_key)) {
                     op = operation;
                 }
                 free(match_key);
             }
 
             if (op != NULL) {
                 free(local_key);
                 return op;
             }
         }
     }
 
     free(local_key);
     if (do_retry == FALSE) {
         return NULL;
     }
 
     do_retry = FALSE;
     if (strstr(key, CRMD_ACTION_MIGRATE) || strstr(key, CRMD_ACTION_MIGRATED)) {
         local_key = pcmk__op_key(rsc->id, "migrate", 0);
         key = local_key;
         goto retry;
 
     } else if (strstr(key, "_notify_")) {
         local_key = pcmk__op_key(rsc->id, "notify", 0);
         key = local_key;
         goto retry;
     }
 
     return NULL;
 }
 
 xmlNode *
 find_rsc_op_entry(pe_resource_t * rsc, const char *key)
 {
     return find_rsc_op_entry_helper(rsc, key, FALSE);
 }
 
 void
 print_node(const char *pre_text, pe_node_t * node, gboolean details)
 {
     if (node == NULL) {
         crm_trace("%s%s: <NULL>", pre_text == NULL ? "" : pre_text, pre_text == NULL ? "" : ": ");
         return;
     }
 
     CRM_ASSERT(node->details);
     crm_trace("%s%s%sNode %s: (weight=%d, fixed=%s)",
               pre_text == NULL ? "" : pre_text,
               pre_text == NULL ? "" : ": ",
               node->details->online ? "" : "Unavailable/Unclean ",
               node->details->uname, node->weight, node->fixed ? "True" : "False");
 
     if (details) {
         int log_level = LOG_TRACE;
 
         char *pe_mutable = strdup("\t\t");
         GListPtr gIter = node->details->running_rsc;
 
         crm_trace("\t\t===Node Attributes");
         g_hash_table_foreach(node->details->attrs, print_str_str, pe_mutable);
         free(pe_mutable);
 
         crm_trace("\t\t=== Resources");
 
         for (; gIter != NULL; gIter = gIter->next) {
             pe_resource_t *rsc = (pe_resource_t *) gIter->data;
 
             rsc->fns->print(rsc, "\t\t", pe_print_log|pe_print_pending,
                             &log_level);
         }
     }
 }
 
 /*
  * Used by the HashTable for-loop
  */
 void
 print_str_str(gpointer key, gpointer value, gpointer user_data)
 {
     crm_trace("%s%s %s ==> %s",
               user_data == NULL ? "" : (char *)user_data,
               user_data == NULL ? "" : ": ", (char *)key, (char *)value);
 }
 
 void
 pe_free_action(pe_action_t * action)
 {
     if (action == NULL) {
         return;
     }
     g_list_free_full(action->actions_before, free);     /* pe_action_wrapper_t* */
     g_list_free_full(action->actions_after, free);      /* pe_action_wrapper_t* */
     if (action->extra) {
         g_hash_table_destroy(action->extra);
     }
     if (action->meta) {
         g_hash_table_destroy(action->meta);
     }
 #if ENABLE_VERSIONED_ATTRS
     if (action->rsc) {
         pe_free_rsc_action_details(action);
     }
 #endif
     free(action->cancel_task);
     free(action->reason);
     free(action->task);
     free(action->uuid);
     free(action->node);
     free(action);
 }
 
 GListPtr
 find_recurring_actions(GListPtr input, pe_node_t * not_on_node)
 {
     const char *value = NULL;
     GListPtr result = NULL;
     GListPtr gIter = input;
 
     CRM_CHECK(input != NULL, return NULL);
 
     for (; gIter != NULL; gIter = gIter->next) {
         pe_action_t *action = (pe_action_t *) gIter->data;
 
         value = g_hash_table_lookup(action->meta, XML_LRM_ATTR_INTERVAL_MS);
         if (value == NULL) {
             /* skip */
         } else if (safe_str_eq(value, "0")) {
             /* skip */
         } else if (safe_str_eq(CRMD_ACTION_CANCEL, action->task)) {
             /* skip */
         } else if (not_on_node == NULL) {
             crm_trace("(null) Found: %s", action->uuid);
             result = g_list_prepend(result, action);
 
         } else if (action->node == NULL) {
             /* skip */
         } else if (action->node->details != not_on_node->details) {
             crm_trace("Found: %s", action->uuid);
             result = g_list_prepend(result, action);
         }
     }
 
     return result;
 }
 
 enum action_tasks
 get_complex_task(pe_resource_t * rsc, const char *name, gboolean allow_non_atomic)
 {
     enum action_tasks task = text2task(name);
 
     if (rsc == NULL) {
         return task;
 
     } else if (allow_non_atomic == FALSE || rsc->variant == pe_native) {
         switch (task) {
             case stopped_rsc:
             case started_rsc:
             case action_demoted:
             case action_promoted:
                 crm_trace("Folding %s back into its atomic counterpart for %s", name, rsc->id);
                 return task - 1;
                 break;
             default:
                 break;
         }
     }
     return task;
 }
 
 pe_action_t *
 find_first_action(GListPtr input, const char *uuid, const char *task, pe_node_t * on_node)
 {
     GListPtr gIter = NULL;
 
     CRM_CHECK(uuid || task, return NULL);
 
     for (gIter = input; gIter != NULL; gIter = gIter->next) {
         pe_action_t *action = (pe_action_t *) gIter->data;
 
         if (uuid != NULL && safe_str_neq(uuid, action->uuid)) {
             continue;
 
         } else if (task != NULL && safe_str_neq(task, action->task)) {
             continue;
 
         } else if (on_node == NULL) {
             return action;
 
         } else if (action->node == NULL) {
             continue;
 
         } else if (on_node->details == action->node->details) {
             return action;
         }
     }
 
     return NULL;
 }
 
 GListPtr
 find_actions(GListPtr input, const char *key, const pe_node_t *on_node)
 {
     GListPtr gIter = input;
     GListPtr result = NULL;
 
     CRM_CHECK(key != NULL, return NULL);
 
     for (; gIter != NULL; gIter = gIter->next) {
         pe_action_t *action = (pe_action_t *) gIter->data;
 
         if (safe_str_neq(key, action->uuid)) {
             crm_trace("%s does not match action %s", key, action->uuid);
             continue;
 
         } else if (on_node == NULL) {
             crm_trace("Action %s matches (ignoring node)", key);
             result = g_list_prepend(result, action);
 
         } else if (action->node == NULL) {
             crm_trace("Action %s matches (unallocated, assigning to %s)",
                       key, on_node->details->uname);
 
             action->node = pe__copy_node(on_node);
             result = g_list_prepend(result, action);
 
         } else if (on_node->details == action->node->details) {
             crm_trace("Action %s on %s matches", key, on_node->details->uname);
             result = g_list_prepend(result, action);
 
         } else {
             crm_trace("Action %s on node %s does not match requested node %s",
                       key, action->node->details->uname,
                       on_node->details->uname);
         }
     }
 
     return result;
 }
 
 GList *
 find_actions_exact(GList *input, const char *key, const pe_node_t *on_node)
 {
     GList *result = NULL;
 
     CRM_CHECK(key != NULL, return NULL);
 
     if (on_node == NULL) {
         crm_trace("Not searching for action %s because node not specified",
                   key);
         return NULL;
     }
 
     for (GList *gIter = input; gIter != NULL; gIter = gIter->next) {
         pe_action_t *action = (pe_action_t *) gIter->data;
 
         if (action->node == NULL) {
             crm_trace("Skipping comparison of %s vs action %s without node",
                       key, action->uuid);
 
         } else if (safe_str_neq(key, action->uuid)) {
             crm_trace("Desired action %s doesn't match %s", key, action->uuid);
 
         } else if (safe_str_neq(on_node->details->id,
                                 action->node->details->id)) {
             crm_trace("Action %s desired node ID %s doesn't match %s",
                       key, on_node->details->id, action->node->details->id);
 
         } else {
             crm_trace("Action %s matches", key);
             result = g_list_prepend(result, action);
         }
     }
 
     return result;
 }
 
 /*!
  * \brief Find all actions of given type for a resource
  *
  * \param[in] rsc           Resource to search
  * \param[in] node          Find only actions scheduled on this node
  * \param[in] task          Action name to search for
  * \param[in] require_node  If TRUE, NULL node or action node will not match
  *
  * \return List of actions found (or NULL if none)
  * \note If node is not NULL and require_node is FALSE, matching actions
  *       without a node will be assigned to node.
  */
 GList *
 pe__resource_actions(const pe_resource_t *rsc, const pe_node_t *node,
                      const char *task, bool require_node)
 {
     GList *result = NULL;
     char *key = pcmk__op_key(rsc->id, task, 0);
 
     if (require_node) {
         result = find_actions_exact(rsc->actions, key, node);
     } else {
         result = find_actions(rsc->actions, key, node);
     }
     free(key);
     return result;
 }
 
 static void
 resource_node_score(pe_resource_t * rsc, pe_node_t * node, int score, const char *tag)
 {
     pe_node_t *match = NULL;
 
     if ((rsc->exclusive_discover || (node->rsc_discover_mode == pe_discover_never))
         && safe_str_eq(tag, "symmetric_default")) {
         /* This string comparision may be fragile, but exclusive resources and
          * exclusive nodes should not have the symmetric_default constraint
          * applied to them.
          */
         return;
 
     } else if (rsc->children) {
         GListPtr gIter = rsc->children;
 
         for (; gIter != NULL; gIter = gIter->next) {
             pe_resource_t *child_rsc = (pe_resource_t *) gIter->data;
 
             resource_node_score(child_rsc, node, score, tag);
         }
     }
 
     pe_rsc_trace(rsc, "Setting %s for %s on %s: %d", tag, rsc->id, node->details->uname, score);
     match = pe_hash_table_lookup(rsc->allowed_nodes, node->details->id);
     if (match == NULL) {
         match = pe__copy_node(node);
         g_hash_table_insert(rsc->allowed_nodes, (gpointer) match->details->id, match);
     }
     match->weight = pe__add_scores(match->weight, score);
 }
 
 void
 resource_location(pe_resource_t * rsc, pe_node_t * node, int score, const char *tag,
                   pe_working_set_t * data_set)
 {
     if (node != NULL) {
         resource_node_score(rsc, node, score, tag);
 
     } else if (data_set != NULL) {
         GListPtr gIter = data_set->nodes;
 
         for (; gIter != NULL; gIter = gIter->next) {
             pe_node_t *node_iter = (pe_node_t *) gIter->data;
 
             resource_node_score(rsc, node_iter, score, tag);
         }
 
     } else {
         GHashTableIter iter;
         pe_node_t *node_iter = NULL;
 
         g_hash_table_iter_init(&iter, rsc->allowed_nodes);
         while (g_hash_table_iter_next(&iter, NULL, (void **)&node_iter)) {
             resource_node_score(rsc, node_iter, score, tag);
         }
     }
 
     if (node == NULL && score == -INFINITY) {
         if (rsc->allocated_to) {
             crm_info("Deallocating %s from %s", rsc->id, rsc->allocated_to->details->uname);
             free(rsc->allocated_to);
             rsc->allocated_to = NULL;
         }
     }
 }
 
 #define sort_return(an_int, why) do {					\
 	free(a_uuid);						\
 	free(b_uuid);						\
 	crm_trace("%s (%d) %c %s (%d) : %s",				\
 		  a_xml_id, a_call_id, an_int>0?'>':an_int<0?'<':'=',	\
 		  b_xml_id, b_call_id, why);				\
 	return an_int;							\
     } while(0)
 
 gint
 sort_op_by_callid(gconstpointer a, gconstpointer b)
 {
     int a_call_id = -1;
     int b_call_id = -1;
 
     char *a_uuid = NULL;
     char *b_uuid = NULL;
 
     const xmlNode *xml_a = a;
     const xmlNode *xml_b = b;
 
     const char *a_xml_id = crm_element_value(xml_a, XML_ATTR_ID);
     const char *b_xml_id = crm_element_value(xml_b, XML_ATTR_ID);
 
     if (safe_str_eq(a_xml_id, b_xml_id)) {
         /* We have duplicate lrm_rsc_op entries in the status
          * section which is unlikely to be a good thing
          *    - we can handle it easily enough, but we need to get
          *    to the bottom of why it's happening.
          */
         pe_err("Duplicate lrm_rsc_op entries named %s", a_xml_id);
         sort_return(0, "duplicate");
     }
 
     crm_element_value_int(xml_a, XML_LRM_ATTR_CALLID, &a_call_id);
     crm_element_value_int(xml_b, XML_LRM_ATTR_CALLID, &b_call_id);
 
     if (a_call_id == -1 && b_call_id == -1) {
         /* both are pending ops so it doesn't matter since
          *   stops are never pending
          */
         sort_return(0, "pending");
 
     } else if (a_call_id >= 0 && a_call_id < b_call_id) {
         sort_return(-1, "call id");
 
     } else if (b_call_id >= 0 && a_call_id > b_call_id) {
         sort_return(1, "call id");
 
     } else if (b_call_id >= 0 && a_call_id == b_call_id) {
         /*
          * The op and last_failed_op are the same
          * Order on last-rc-change
          */
         time_t last_a = -1;
         time_t last_b = -1;
 
         crm_element_value_epoch(xml_a, XML_RSC_OP_LAST_CHANGE, &last_a);
         crm_element_value_epoch(xml_b, XML_RSC_OP_LAST_CHANGE, &last_b);
 
         crm_trace("rc-change: %lld vs %lld",
                   (long long) last_a, (long long) last_b);
         if (last_a >= 0 && last_a < last_b) {
             sort_return(-1, "rc-change");
 
         } else if (last_b >= 0 && last_a > last_b) {
             sort_return(1, "rc-change");
         }
         sort_return(0, "rc-change");
 
     } else {
         /* One of the inputs is a pending operation
          * Attempt to use XML_ATTR_TRANSITION_MAGIC to determine its age relative to the other
          */
 
         int a_id = -1;
         int b_id = -1;
 
         const char *a_magic = crm_element_value(xml_a, XML_ATTR_TRANSITION_MAGIC);
         const char *b_magic = crm_element_value(xml_b, XML_ATTR_TRANSITION_MAGIC);
 
         CRM_CHECK(a_magic != NULL && b_magic != NULL, sort_return(0, "No magic"));
         if (!decode_transition_magic(a_magic, &a_uuid, &a_id, NULL, NULL, NULL,
                                      NULL)) {
             sort_return(0, "bad magic a");
         }
         if (!decode_transition_magic(b_magic, &b_uuid, &b_id, NULL, NULL, NULL,
                                      NULL)) {
             sort_return(0, "bad magic b");
         }
         /* try to determine the relative age of the operation...
          * some pending operations (e.g. a start) may have been superseded
          *   by a subsequent stop
          *
          * [a|b]_id == -1 means it's a shutdown operation and _always_ comes last
          */
         if (safe_str_neq(a_uuid, b_uuid) || a_id == b_id) {
             /*
              * some of the logic in here may be redundant...
              *
              * if the UUID from the TE doesn't match then one better
              *   be a pending operation.
              * pending operations don't survive between elections and joins
              *   because we query the LRM directly
              */
 
             if (b_call_id == -1) {
                 sort_return(-1, "transition + call");
 
             } else if (a_call_id == -1) {
                 sort_return(1, "transition + call");
             }
 
         } else if ((a_id >= 0 && a_id < b_id) || b_id == -1) {
             sort_return(-1, "transition");
 
         } else if ((b_id >= 0 && a_id > b_id) || a_id == -1) {
             sort_return(1, "transition");
         }
     }
 
     /* we should never end up here */
     CRM_CHECK(FALSE, sort_return(0, "default"));
 
 }
 
 time_t
 get_effective_time(pe_working_set_t * data_set)
 {
     if(data_set) {
         if (data_set->now == NULL) {
             crm_trace("Recording a new 'now'");
             data_set->now = crm_time_new(NULL);
         }
         return crm_time_get_seconds_since_epoch(data_set->now);
     }
 
     crm_trace("Defaulting to 'now'");
     return time(NULL);
 }
 
 gboolean
 get_target_role(pe_resource_t * rsc, enum rsc_role_e * role)
 {
     enum rsc_role_e local_role = RSC_ROLE_UNKNOWN;
     const char *value = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_TARGET_ROLE);
 
     CRM_CHECK(role != NULL, return FALSE);
 
     if (value == NULL || safe_str_eq("started", value)
         || safe_str_eq("default", value)) {
         return FALSE;
     }
 
     local_role = text2role(value);
     if (local_role == RSC_ROLE_UNKNOWN) {
         pcmk__config_err("Ignoring '" XML_RSC_ATTR_TARGET_ROLE "' for %s "
                          "because '%s' is not valid", rsc->id, value);
         return FALSE;
 
     } else if (local_role > RSC_ROLE_STARTED) {
         if (is_set(uber_parent(rsc)->flags, pe_rsc_promotable)) {
             if (local_role > RSC_ROLE_SLAVE) {
                 /* This is what we'd do anyway, just leave the default to avoid messing up the placement algorithm */
                 return FALSE;
             }
 
         } else {
             pcmk__config_err("Ignoring '" XML_RSC_ATTR_TARGET_ROLE "' for %s "
                              "because '%s' only makes sense for promotable "
                              "clones", rsc->id, value);
             return FALSE;
         }
     }
 
     *role = local_role;
     return TRUE;
 }
 
 gboolean
 order_actions(pe_action_t * lh_action, pe_action_t * rh_action, enum pe_ordering order)
 {
     GListPtr gIter = NULL;
     pe_action_wrapper_t *wrapper = NULL;
     GListPtr list = NULL;
 
     if (order == pe_order_none) {
         return FALSE;
     }
 
     if (lh_action == NULL || rh_action == NULL) {
         return FALSE;
     }
 
     crm_trace("Ordering Action %s before %s", lh_action->uuid, rh_action->uuid);
 
     /* Ensure we never create a dependency on ourselves... it's happened */
     CRM_ASSERT(lh_action != rh_action);
 
     /* Filter dups, otherwise update_action_states() has too much work to do */
     gIter = lh_action->actions_after;
     for (; gIter != NULL; gIter = gIter->next) {
         pe_action_wrapper_t *after = (pe_action_wrapper_t *) gIter->data;
 
         if (after->action == rh_action && (after->type & order)) {
             return FALSE;
         }
     }
 
     wrapper = calloc(1, sizeof(pe_action_wrapper_t));
     wrapper->action = rh_action;
     wrapper->type = order;
 
     list = lh_action->actions_after;
     list = g_list_prepend(list, wrapper);
     lh_action->actions_after = list;
 
     wrapper = NULL;
 
 /* 	order |= pe_order_implies_then; */
 /* 	order ^= pe_order_implies_then; */
 
     wrapper = calloc(1, sizeof(pe_action_wrapper_t));
     wrapper->action = lh_action;
     wrapper->type = order;
     list = rh_action->actions_before;
     list = g_list_prepend(list, wrapper);
     rh_action->actions_before = list;
     return TRUE;
 }
 
 pe_action_t *
 get_pseudo_op(const char *name, pe_working_set_t * data_set)
 {
     pe_action_t *op = NULL;
 
     if(data_set->singletons) {
         op = g_hash_table_lookup(data_set->singletons, name);
     }
     if (op == NULL) {
         op = custom_action(NULL, strdup(name), name, NULL, TRUE, TRUE, data_set);
         set_bit(op->flags, pe_action_pseudo);
         set_bit(op->flags, pe_action_runnable);
     }
 
     return op;
 }
 
 void
 destroy_ticket(gpointer data)
 {
     pe_ticket_t *ticket = data;
 
     if (ticket->state) {
         g_hash_table_destroy(ticket->state);
     }
     free(ticket->id);
     free(ticket);
 }
 
 pe_ticket_t *
 ticket_new(const char *ticket_id, pe_working_set_t * data_set)
 {
     pe_ticket_t *ticket = NULL;
 
     if (ticket_id == NULL || strlen(ticket_id) == 0) {
         return NULL;
     }
 
     if (data_set->tickets == NULL) {
         data_set->tickets =
             g_hash_table_new_full(crm_str_hash, g_str_equal, free,
                                   destroy_ticket);
     }
 
     ticket = g_hash_table_lookup(data_set->tickets, ticket_id);
     if (ticket == NULL) {
 
         ticket = calloc(1, sizeof(pe_ticket_t));
         if (ticket == NULL) {
             crm_err("Cannot allocate ticket '%s'", ticket_id);
             return NULL;
         }
 
         crm_trace("Creaing ticket entry for %s", ticket_id);
 
         ticket->id = strdup(ticket_id);
         ticket->granted = FALSE;
         ticket->last_granted = -1;
         ticket->standby = FALSE;
         ticket->state = crm_str_table_new();
 
         g_hash_table_insert(data_set->tickets, strdup(ticket->id), ticket);
     }
 
     return ticket;
 }
 
 static void
 filter_parameters(xmlNode * param_set, const char *param_string, bool need_present)
 {
     if (param_set && param_string) {
         xmlAttrPtr xIter = param_set->properties;
 
         while (xIter) {
             const char *prop_name = (const char *)xIter->name;
             char *name = crm_strdup_printf(" %s ", prop_name);
             char *match = strstr(param_string, name);
 
             free(name);
 
             //  Do now, because current entry might get removed below
             xIter = xIter->next;
 
             if (need_present && match == NULL) {
                 crm_trace("%s not found in %s", prop_name, param_string);
                 xml_remove_prop(param_set, prop_name);
 
             } else if (need_present == FALSE && match) {
                 crm_trace("%s found in %s", prop_name, param_string);
                 xml_remove_prop(param_set, prop_name);
             }
         }
     }
 }
 
 #if ENABLE_VERSIONED_ATTRS
 static void
 append_versioned_params(xmlNode *versioned_params, const char *ra_version, xmlNode *params)
 {
     GHashTable *hash = pe_unpack_versioned_parameters(versioned_params, ra_version);
     char *key = NULL;
     char *value = NULL;
     GHashTableIter iter;
 
     g_hash_table_iter_init(&iter, hash);
     while (g_hash_table_iter_next(&iter, (gpointer *) &key, (gpointer *) &value)) {
         crm_xml_add(params, key, value);
     }
     g_hash_table_destroy(hash);
 }
 #endif
 
 /*!
  * \internal
  * \brief Calculate action digests and store in node's digest cache
  *
  * \param[in] rsc          Resource that action was for
  * \param[in] task         Name of action performed
  * \param[in] key          Action's task key
  * \param[in] node         Node action was performed on
  * \param[in] xml_op       XML of operation in CIB status (if available)
  * \param[in] calc_secure  Whether to calculate secure digest
  * \param[in] data_set     Cluster working set
  *
  * \return Pointer to node's digest cache entry
  */
 static op_digest_cache_t *
 rsc_action_digest(pe_resource_t *rsc, const char *task, const char *key,
                   pe_node_t *node, xmlNode *xml_op, bool calc_secure,
                   pe_working_set_t *data_set)
 {
     op_digest_cache_t *data = NULL;
 
     data = g_hash_table_lookup(node->details->digest_cache, key);
     if (data == NULL) {
         GHashTable *local_rsc_params = crm_str_table_new();
         pe_action_t *action = custom_action(rsc, strdup(key), task, node, TRUE, FALSE, data_set);
 #if ENABLE_VERSIONED_ATTRS
         xmlNode *local_versioned_params = create_xml_node(NULL, XML_TAG_RSC_VER_ATTRS);
         const char *ra_version = NULL;
 #endif
 
         const char *op_version;
         const char *restart_list = NULL;
         const char *secure_list = " passwd password ";
 
         data = calloc(1, sizeof(op_digest_cache_t));
         CRM_ASSERT(data != NULL);
 
         get_rsc_attributes(local_rsc_params, rsc, node, data_set);
 #if ENABLE_VERSIONED_ATTRS
         pe_get_versioned_attributes(local_versioned_params, rsc, node, data_set);
 #endif
 
         data->params_all = create_xml_node(NULL, XML_TAG_PARAMS);
 
         // REMOTE_CONTAINER_HACK: Allow remote nodes that start containers with pacemaker remote inside
         if (pe__add_bundle_remote_name(rsc, data->params_all,
                                        XML_RSC_ATTR_REMOTE_RA_ADDR)) {
             crm_trace("Set address for bundle connection %s (on %s)",
                       rsc->id, node->details->uname);
         }
 
         g_hash_table_foreach(local_rsc_params, hash2field, data->params_all);
         g_hash_table_foreach(action->extra, hash2field, data->params_all);
         g_hash_table_foreach(rsc->parameters, hash2field, data->params_all);
         g_hash_table_foreach(action->meta, hash2metafield, data->params_all);
 
         if(xml_op) {
             secure_list = crm_element_value(xml_op, XML_LRM_ATTR_OP_SECURE);
             restart_list = crm_element_value(xml_op, XML_LRM_ATTR_OP_RESTART);
 
             op_version = crm_element_value(xml_op, XML_ATTR_CRM_VERSION);
 #if ENABLE_VERSIONED_ATTRS
             ra_version = crm_element_value(xml_op, XML_ATTR_RA_VERSION);
 #endif
 
         } else {
             op_version = CRM_FEATURE_SET;
         }
 
 #if ENABLE_VERSIONED_ATTRS
         append_versioned_params(local_versioned_params, ra_version, data->params_all);
         append_versioned_params(rsc->versioned_parameters, ra_version, data->params_all);
 
         {
             pe_rsc_action_details_t *details = pe_rsc_action_details(action);
             append_versioned_params(details->versioned_parameters, ra_version, data->params_all);
         }
 #endif
 
         pcmk__filter_op_for_digest(data->params_all);
 
         g_hash_table_destroy(local_rsc_params);
         pe_free_action(action);
 
         data->digest_all_calc = calculate_operation_digest(data->params_all, op_version);
 
         if (calc_secure) {
             data->params_secure = copy_xml(data->params_all);
             if(secure_list) {
                 filter_parameters(data->params_secure, secure_list, FALSE);
             }
             data->digest_secure_calc = calculate_operation_digest(data->params_secure, op_version);
         }
 
         if(xml_op && crm_element_value(xml_op, XML_LRM_ATTR_RESTART_DIGEST) != NULL) {
             data->params_restart = copy_xml(data->params_all);
             if (restart_list) {
                 filter_parameters(data->params_restart, restart_list, TRUE);
             }
             data->digest_restart_calc = calculate_operation_digest(data->params_restart, op_version);
         }
 
         g_hash_table_insert(node->details->digest_cache, strdup(key), data);
     }
 
     return data;
 }
 
 op_digest_cache_t *
 rsc_action_digest_cmp(pe_resource_t * rsc, xmlNode * xml_op, pe_node_t * node,
                       pe_working_set_t * data_set)
 {
     op_digest_cache_t *data = NULL;
 
     char *key = NULL;
     guint interval_ms = 0;
 
     const char *op_version;
     const char *task = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
     const char *digest_all;
     const char *digest_restart;
 
     CRM_ASSERT(node != NULL);
 
     op_version = crm_element_value(xml_op, XML_ATTR_CRM_VERSION);
     digest_all = crm_element_value(xml_op, XML_LRM_ATTR_OP_DIGEST);
     digest_restart = crm_element_value(xml_op, XML_LRM_ATTR_RESTART_DIGEST);
 
     crm_element_value_ms(xml_op, XML_LRM_ATTR_INTERVAL_MS, &interval_ms);
     key = pcmk__op_key(rsc->id, task, interval_ms);
     data = rsc_action_digest(rsc, task, key, node, xml_op,
                              is_set(data_set->flags, pe_flag_sanitized),
                              data_set);
 
     data->rc = RSC_DIGEST_MATCH;
     if (digest_restart && data->digest_restart_calc && strcmp(data->digest_restart_calc, digest_restart) != 0) {
         pe_rsc_info(rsc, "Parameters to %s on %s changed: was %s vs. now %s (restart:%s) %s",
                  key, node->details->uname,
                  crm_str(digest_restart), data->digest_restart_calc,
                  op_version, crm_element_value(xml_op, XML_ATTR_TRANSITION_MAGIC));
         data->rc = RSC_DIGEST_RESTART;
 
     } else if (digest_all == NULL) {
         /* it is unknown what the previous op digest was */
         data->rc = RSC_DIGEST_UNKNOWN;
 
     } else if (strcmp(digest_all, data->digest_all_calc) != 0) {
         pe_rsc_info(rsc, "Parameters to %s on %s changed: was %s vs. now %s (%s:%s) %s",
                  key, node->details->uname,
                  crm_str(digest_all), data->digest_all_calc,
                  (interval_ms > 0)? "reschedule" : "reload",
                  op_version, crm_element_value(xml_op, XML_ATTR_TRANSITION_MAGIC));
         data->rc = RSC_DIGEST_ALL;
     }
 
     free(key);
     return data;
 }
 
 /*!
  * \internal
  * \brief Create an unfencing summary for use in special node attribute
  *
  * Create a string combining a fence device's resource ID, agent type, and
  * parameter digest (whether for all parameters or just non-private parameters).
  * This can be stored in a special node attribute, allowing us to detect changes
  * in either the agent type or parameters, to know whether unfencing must be
  * redone or can be safely skipped when the device's history is cleaned.
  *
  * \param[in] rsc_id        Fence device resource ID
  * \param[in] agent_type    Fence device agent
  * \param[in] param_digest  Fence device parameter digest
  *
  * \return Newly allocated string with unfencing digest
  * \note The caller is responsible for freeing the result.
  */
 static inline char *
 create_unfencing_summary(const char *rsc_id, const char *agent_type,
                          const char *param_digest)
 {
     return crm_strdup_printf("%s:%s:%s", rsc_id, agent_type, param_digest);
 }
 
 /*!
  * \internal
  * \brief Check whether a node can skip unfencing
  *
  * Check whether a fence device's current definition matches a node's
  * stored summary of when it was last unfenced by the device.
  *
  * \param[in] rsc_id        Fence device's resource ID
  * \param[in] agent         Fence device's agent type
  * \param[in] digest_calc   Fence device's current parameter digest
  * \param[in] node_summary  Value of node's special unfencing node attribute
  *                          (a comma-separated list of unfencing summaries for
  *                          all devices that have unfenced this node)
  *
  * \return TRUE if digest matches, FALSE otherwise
  */
 static bool
 unfencing_digest_matches(const char *rsc_id, const char *agent,
                          const char *digest_calc, const char *node_summary)
 {
     bool matches = FALSE;
 
     if (rsc_id && agent && digest_calc && node_summary) {
         char *search_secure = create_unfencing_summary(rsc_id, agent,
                                                        digest_calc);
 
         /* The digest was calculated including the device ID and agent,
          * so there is no risk of collision using strstr().
          */
         matches = (strstr(node_summary, search_secure) != NULL);
         crm_trace("Calculated unfencing digest '%s' %sfound in '%s'",
                   search_secure, matches? "" : "not ", node_summary);
         free(search_secure);
     }
     return matches;
 }
 
 /* Magic string to use as action name for digest cache entries used for
  * unfencing checks. This is not a real action name (i.e. "on"), so
  * check_action_definition() won't confuse these entries with real actions.
  */
 #define STONITH_DIGEST_TASK "stonith-on"
 
 /*!
  * \internal
  * \brief Calculate fence device digests and digest comparison result
  *
  * \param[in] rsc       Fence device resource
  * \param[in] agent     Fence device's agent type
  * \param[in] node      Node with digest cache to use
  * \param[in] data_set  Cluster working set
  *
  * \return Node's digest cache entry
  */
 static op_digest_cache_t *
 fencing_action_digest_cmp(pe_resource_t *rsc, const char *agent,
                           pe_node_t *node, pe_working_set_t *data_set)
 {
     const char *node_summary = NULL;
 
     // Calculate device's current parameter digests
     char *key = pcmk__op_key(rsc->id, STONITH_DIGEST_TASK, 0);
     op_digest_cache_t *data = rsc_action_digest(rsc, STONITH_DIGEST_TASK, key,
                                                 node, NULL, TRUE, data_set);
 
     free(key);
 
     // Check whether node has special unfencing summary node attribute
     node_summary = pe_node_attribute_raw(node, CRM_ATTR_DIGESTS_ALL);
     if (node_summary == NULL) {
         data->rc = RSC_DIGEST_UNKNOWN;
         return data;
     }
 
     // Check whether full parameter digest matches
     if (unfencing_digest_matches(rsc->id, agent, data->digest_all_calc,
                                  node_summary)) {
         data->rc = RSC_DIGEST_MATCH;
         return data;
     }
 
     // Check whether secure parameter digest matches
     node_summary = pe_node_attribute_raw(node, CRM_ATTR_DIGESTS_SECURE);
     if (unfencing_digest_matches(rsc->id, agent, data->digest_secure_calc,
                                  node_summary)) {
         data->rc = RSC_DIGEST_MATCH;
         if (is_set(data_set->flags, pe_flag_stdout)) {
             printf("Only 'private' parameters to %s for unfencing %s changed\n",
                    rsc->id, node->details->uname);
         }
         return data;
     }
 
     // Parameters don't match
     data->rc = RSC_DIGEST_ALL;
     if (is_set(data_set->flags, (pe_flag_sanitized|pe_flag_stdout))
         && data->digest_secure_calc) {
         char *digest = create_unfencing_summary(rsc->id, agent,
                                                 data->digest_secure_calc);
 
         printf("Parameters to %s for unfencing %s changed, try '%s'\n",
                rsc->id, node->details->uname, digest);
         free(digest);
     }
     return data;
 }
 
 const char *rsc_printable_id(pe_resource_t *rsc)
 {
     if (is_not_set(rsc->flags, pe_rsc_unique)) {
         return ID(rsc->xml);
     }
     return rsc->id;
 }
 
 void
 clear_bit_recursive(pe_resource_t * rsc, unsigned long long flag)
 {
     GListPtr gIter = rsc->children;
 
     clear_bit(rsc->flags, flag);
     for (; gIter != NULL; gIter = gIter->next) {
         pe_resource_t *child_rsc = (pe_resource_t *) gIter->data;
 
         clear_bit_recursive(child_rsc, flag);
     }
 }
 
 void
 set_bit_recursive(pe_resource_t * rsc, unsigned long long flag)
 {
     GListPtr gIter = rsc->children;
 
     set_bit(rsc->flags, flag);
     for (; gIter != NULL; gIter = gIter->next) {
         pe_resource_t *child_rsc = (pe_resource_t *) gIter->data;
 
         set_bit_recursive(child_rsc, flag);
     }
 }
 
 static GListPtr
 find_unfencing_devices(GListPtr candidates, GListPtr matches) 
 {
     for (GListPtr gIter = candidates; gIter != NULL; gIter = gIter->next) {
         pe_resource_t *candidate = gIter->data;
         const char *provides = g_hash_table_lookup(candidate->meta, XML_RSC_ATTR_PROVIDES);
         const char *requires = g_hash_table_lookup(candidate->meta, XML_RSC_ATTR_REQUIRES);
 
         if(candidate->children) {
             matches = find_unfencing_devices(candidate->children, matches);
         } else if (is_not_set(candidate->flags, pe_rsc_fence_device)) {
             continue;
 
         } else if (crm_str_eq(provides, "unfencing", FALSE) || crm_str_eq(requires, "unfencing", FALSE)) {
             matches = g_list_prepend(matches, candidate);
         }
     }
     return matches;
 }
 
 static int
 node_priority_fencing_delay(pe_node_t * node, pe_working_set_t * data_set)
 {
     int member_count = 0;
     int online_count = 0;
     int top_priority = 0;
     int lowest_priority = 0;
     GListPtr gIter = NULL;
 
     // `priority-fencing-delay` is disabled
     if (data_set->priority_fencing_delay <= 0) {
         return 0;
     }
 
     /* No need to request a delay if the fencing target is not a normal cluster
      * member, for example if it's a remote node or a guest node. */
     if (node->details->type != node_member) {
         return 0;
     }
 
     // No need to request a delay if the fencing target is in our partition
     if (node->details->online) {
         return 0;
     }
 
     for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
         pe_node_t *n =  gIter->data;
 
         if (n->details->type != node_member) {
             continue;
         }
 
         member_count ++;
 
         if (n->details->online) {
             online_count++;
         }
 
         if (member_count == 1
             || n->details->priority > top_priority) {
             top_priority = n->details->priority;
         }
 
         if (member_count == 1
             || n->details->priority < lowest_priority) {
             lowest_priority = n->details->priority;
         }
     }
 
     // No need to delay if we have more than half of the cluster members
     if (online_count > member_count / 2) {
         return 0;
     }
 
     /* All the nodes have equal priority.
      * Any configured corresponding `pcmk_delay_base/max` will be applied. */
     if (lowest_priority == top_priority) {
         return 0;
     }
 
     if (node->details->priority < top_priority) {
         return 0;
     }
 
     return data_set->priority_fencing_delay;
 }
 
 pe_action_t *
 pe_fence_op(pe_node_t * node, const char *op, bool optional, const char *reason,
             bool priority_delay, pe_working_set_t * data_set)
 {
     char *op_key = NULL;
     pe_action_t *stonith_op = NULL;
 
     if(op == NULL) {
         op = data_set->stonith_action;
     }
 
     op_key = crm_strdup_printf("%s-%s-%s", CRM_OP_FENCE, node->details->uname, op);
 
     if(data_set->singletons) {
         stonith_op = g_hash_table_lookup(data_set->singletons, op_key);
     }
 
     if(stonith_op == NULL) {
         stonith_op = custom_action(NULL, op_key, CRM_OP_FENCE, node, TRUE, TRUE, data_set);
 
         add_hash_param(stonith_op->meta, XML_LRM_ATTR_TARGET, node->details->uname);
         add_hash_param(stonith_op->meta, XML_LRM_ATTR_TARGET_UUID, node->details->id);
         add_hash_param(stonith_op->meta, "stonith_action", op);
 
         if (pe__is_guest_or_remote_node(node)
             && is_set(data_set->flags, pe_flag_enable_unfencing)) {
             /* Extra work to detect device changes on remotes
              *
              * We may do this for all nodes in the future, but for now
              * the check_action_definition() based stuff works fine.
              */
             long max = 1024;
             long digests_all_offset = 0;
             long digests_secure_offset = 0;
 
             char *digests_all = calloc(max, sizeof(char));
             char *digests_secure = calloc(max, sizeof(char));
             GListPtr matches = find_unfencing_devices(data_set->resources, NULL);
 
             for (GListPtr gIter = matches; gIter != NULL; gIter = gIter->next) {
                 pe_resource_t *match = gIter->data;
                 const char *agent = g_hash_table_lookup(match->meta,
                                                         XML_ATTR_TYPE);
                 op_digest_cache_t *data = NULL;
 
                 data = fencing_action_digest_cmp(match, agent, node, data_set);
                 if(data->rc == RSC_DIGEST_ALL) {
                     optional = FALSE;
                     crm_notice("Unfencing %s (remote): because the definition of %s changed", node->details->uname, match->id);
                     if (is_set(data_set->flags, pe_flag_stdout)) {
                         fprintf(stdout, "  notice: Unfencing %s (remote): because the definition of %s changed\n", node->details->uname, match->id);
                     }
                 }
 
                 digests_all_offset += snprintf(
                     digests_all+digests_all_offset, max-digests_all_offset,
                     "%s:%s:%s,", match->id, agent, data->digest_all_calc);
 
                 digests_secure_offset += snprintf(
                     digests_secure+digests_secure_offset, max-digests_secure_offset,
                     "%s:%s:%s,", match->id, agent, data->digest_secure_calc);
             }
             g_hash_table_insert(stonith_op->meta,
                                 strdup(XML_OP_ATTR_DIGESTS_ALL),
                                 digests_all);
             g_hash_table_insert(stonith_op->meta,
                                 strdup(XML_OP_ATTR_DIGESTS_SECURE),
                                 digests_secure);
         }
 
     } else {
         free(op_key);
     }
 
     if (data_set->priority_fencing_delay > 0
 
             /* It's a suitable case where `priority-fencing-delay` applies.
              * At least add `priority-fencing-delay` field as an indicator. */
         && (priority_delay
 
             /* Re-calculate priority delay for the suitable case when
              * pe_fence_op() is called again by stage6() after node priority has
              * been actually calculated with native_add_running() */
             || g_hash_table_lookup(stonith_op->meta,
                                    XML_CONFIG_ATTR_PRIORITY_FENCING_DELAY) != NULL)) {
 
             /* Add `priority-fencing-delay` to the fencing op even if it's 0 for
              * the targeting node. So that it takes precedence over any possible
              * `pcmk_delay_base/max`.
              */
             char *delay_s = crm_itoa(node_priority_fencing_delay(node, data_set));
 
             g_hash_table_insert(stonith_op->meta,
                                 strdup(XML_CONFIG_ATTR_PRIORITY_FENCING_DELAY),
                                 delay_s);
     }
 
     if(optional == FALSE && pe_can_fence(data_set, node)) {
         pe_action_required(stonith_op, NULL, reason);
     } else if(reason && stonith_op->reason == NULL) {
         stonith_op->reason = strdup(reason);
     }
 
     return stonith_op;
 }
 
 void
 trigger_unfencing(
     pe_resource_t * rsc, pe_node_t *node, const char *reason, pe_action_t *dependency, pe_working_set_t * data_set) 
 {
     if(is_not_set(data_set->flags, pe_flag_enable_unfencing)) {
         /* No resources require it */
         return;
 
     } else if (rsc != NULL && is_not_set(rsc->flags, pe_rsc_fence_device)) {
         /* Wasn't a stonith device */
         return;
 
     } else if(node
               && node->details->online
               && node->details->unclean == FALSE
               && node->details->shutdown == FALSE) {
         pe_action_t *unfence = pe_fence_op(node, "on", FALSE, reason, FALSE, data_set);
 
         if(dependency) {
             order_actions(unfence, dependency, pe_order_optional);
         }
 
     } else if(rsc) {
         GHashTableIter iter;
 
         g_hash_table_iter_init(&iter, rsc->allowed_nodes);
         while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
             if(node->details->online && node->details->unclean == FALSE && node->details->shutdown == FALSE) {
                 trigger_unfencing(rsc, node, reason, dependency, data_set);
             }
         }
     }
 }
 
 gboolean
 add_tag_ref(GHashTable * tags, const char * tag_name,  const char * obj_ref)
 {
     pe_tag_t *tag = NULL;
     GListPtr gIter = NULL;
     gboolean is_existing = FALSE;
 
     CRM_CHECK(tags && tag_name && obj_ref, return FALSE);
 
     tag = g_hash_table_lookup(tags, tag_name);
     if (tag == NULL) {
         tag = calloc(1, sizeof(pe_tag_t));
         if (tag == NULL) {
             return FALSE;
         }
         tag->id = strdup(tag_name);
         tag->refs = NULL;
         g_hash_table_insert(tags, strdup(tag_name), tag);
     }
 
     for (gIter = tag->refs; gIter != NULL; gIter = gIter->next) {
         const char *existing_ref = (const char *) gIter->data;
 
         if (crm_str_eq(existing_ref, obj_ref, TRUE)){
             is_existing = TRUE;
             break;
         }
     }
 
     if (is_existing == FALSE) {
         tag->refs = g_list_append(tag->refs, strdup(obj_ref));
         crm_trace("Added: tag=%s ref=%s", tag->id, obj_ref);
     }
 
     return TRUE;
 }
 
 void pe_action_set_flag_reason(const char *function, long line,
                                pe_action_t *action, pe_action_t *reason, const char *text,
                                enum pe_action_flags flags, bool overwrite)
 {
     bool unset = FALSE;
     bool update = FALSE;
     const char *change = NULL;
 
     if(is_set(flags, pe_action_runnable)) {
         unset = TRUE;
         change = "unrunnable";
     } else if(is_set(flags, pe_action_optional)) {
         unset = TRUE;
         change = "required";
     } else if(is_set(flags, pe_action_migrate_runnable)) {
         unset = TRUE;
         overwrite = TRUE;
         change = "unrunnable";
     } else if(is_set(flags, pe_action_dangle)) {
         change = "dangling";
     } else if(is_set(flags, pe_action_requires_any)) {
         change = "required";
     } else {
         crm_err("Unknown flag change to %x by %s: 0x%s",
                 flags, action->uuid, (reason? reason->uuid : "0"));
     }
 
     if(unset) {
         if(is_set(action->flags, flags)) {
             action->flags = crm_clear_bit(function, line, action->uuid, action->flags, flags);
             update = TRUE;
         }
 
     } else {
         if(is_not_set(action->flags, flags)) {
             action->flags = crm_set_bit(function, line, action->uuid, action->flags, flags);
             update = TRUE;
         }
     }
 
     if((change && update) || text) {
         char *reason_text = NULL;
         if(reason == NULL) {
             pe_action_set_reason(action, text, overwrite);
 
         } else if(reason->rsc == NULL) {
             reason_text = crm_strdup_printf("%s %s%c %s", change, reason->task, text?':':0, text?text:"");
         } else {
             reason_text = crm_strdup_printf("%s %s %s%c %s", change, reason->rsc->id, reason->task, text?':':0, text?text:"NA");
         }
 
         if(reason_text && action->rsc != reason->rsc) {
             pe_action_set_reason(action, reason_text, overwrite);
         }
         free(reason_text);
     }
  }
 
 void pe_action_set_reason(pe_action_t *action, const char *reason, bool overwrite) 
 {
     if (action->reason != NULL && overwrite) {
         pe_rsc_trace(action->rsc, "Changing %s reason from '%s' to '%s'",
                      action->uuid, action->reason, crm_str(reason));
         free(action->reason);
     } else if (action->reason == NULL) {
         pe_rsc_trace(action->rsc, "Set %s reason to '%s'",
                      action->uuid, crm_str(reason));
     } else {
         // crm_assert(action->reason != NULL && !overwrite);
         return;
     }
 
     if (reason != NULL) {
         action->reason = strdup(reason);
     } else {
         action->reason = NULL;
     }
 }
 
 /*!
  * \internal
  * \brief Check whether shutdown has been requested for a node
  *
  * \param[in] node  Node to check
  *
  * \return TRUE if node has shutdown attribute set and nonzero, FALSE otherwise
  * \note This differs from simply using node->details->shutdown in that it can
  *       be used before that has been determined (and in fact to determine it),
  *       and it can also be used to distinguish requested shutdown from implicit
  *       shutdown of remote nodes by virtue of their connection stopping.
  */
 bool
 pe__shutdown_requested(pe_node_t *node)
 {
     const char *shutdown = pe_node_attribute_raw(node, XML_CIB_ATTR_SHUTDOWN);
 
     return shutdown && strcmp(shutdown, "0");
 }
 
 /*!
  * \internal
  * \brief Update a data set's "recheck by" time
  *
  * \param[in]     recheck   Epoch time when recheck should happen
  * \param[in,out] data_set  Current working set
  */
 void
 pe__update_recheck_time(time_t recheck, pe_working_set_t *data_set)
 {
     if ((recheck > get_effective_time(data_set))
         && ((data_set->recheck_by == 0)
             || (data_set->recheck_by > recheck))) {
         data_set->recheck_by = recheck;
     }
 }
 
 /*!
  * \internal
  * \brief Wrapper for pe_unpack_nvpairs() using a cluster working set
  */
 void
 pe__unpack_dataset_nvpairs(xmlNode *xml_obj, const char *set_name,
                            pe_rule_eval_data_t *rule_data, GHashTable *hash,
                            const char *always_first, gboolean overwrite,
                            pe_working_set_t *data_set)
 {
     crm_time_t *next_change = crm_time_new_undefined();
 
     pe_eval_nvpairs(data_set->input, xml_obj, set_name, rule_data, hash,
                     always_first, overwrite, next_change);
     if (crm_time_is_defined(next_change)) {
         time_t recheck = (time_t) crm_time_get_seconds_since_epoch(next_change);
 
         pe__update_recheck_time(recheck, data_set);
     }
     crm_time_free(next_change);
 }
 
 bool
 pe__resource_is_disabled(pe_resource_t *rsc)
 {
     const char *target_role = NULL;
 
     CRM_CHECK(rsc != NULL, return false);
     target_role = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_TARGET_ROLE);
     if (target_role) {
         enum rsc_role_e target_role_e = text2role(target_role);
 
         if ((target_role_e == RSC_ROLE_STOPPED)
             || ((target_role_e == RSC_ROLE_SLAVE)
                 && is_set(uber_parent(rsc)->flags, pe_rsc_promotable))) {
             return true;
         }
     }
     return false;
 }
 
 /*!
  * \internal
  * \brief Create an action to clear a resource's history from CIB
  *
  * \param[in] rsc   Resource to clear
  * \param[in] node  Node to clear history on
  *
  * \return New action to clear resource history
  */
 pe_action_t *
 pe__clear_resource_history(pe_resource_t *rsc, pe_node_t *node,
                            pe_working_set_t *data_set)
 {
     char *key = NULL;
 
     CRM_ASSERT(rsc && node);
     key = pcmk__op_key(rsc->id, CRM_OP_LRM_DELETE, 0);
     return custom_action(rsc, key, CRM_OP_LRM_DELETE, node, FALSE, TRUE,
                          data_set);
 }
 
 bool
 pe__rsc_running_on_any_node_in_list(pe_resource_t *rsc, GListPtr node_list)
 {
-    /* If this resource is inactive, we will always return false unless
-     * node_list contains just '*'.  Inactive resources aren't running on
-     * any node.
-     */
-    gboolean is_active = rsc->fns->active(rsc, TRUE);
-    gboolean partially_active = rsc->fns->active(rsc, FALSE);
-
-    if (!is_active && !partially_active &&
-        node_list != NULL && strcmp(node_list->data, "*") == 0 && node_list->next == NULL) {
-        return true;
-    }
-
-    /* Otherwise, this resource must be running on one of the nodes in the
-     * given list.
-     */
     for (GListPtr ele = rsc->running_on; ele; ele = ele->next) {
         pe_node_t *node = (pe_node_t *) ele->data;
         if (pcmk__str_in_list(node_list, node->details->uname)) {
             return true;
         }
     }
 
     return false;
 }
+
+bool
+pcmk__rsc_is_filtered(pe_resource_t *rsc, GListPtr only_show)
+{
+    return (rsc->fns->active(rsc, FALSE) && !pe__rsc_running_on_any_node_in_list(rsc, only_show));
+}
diff --git a/maint/Makefile.am b/maint/Makefile.am
index ef8e9f25f7..11e4dd1f8f 100644
--- a/maint/Makefile.am
+++ b/maint/Makefile.am
@@ -1,115 +1,116 @@
 #
 # Copyright 2019 the Pacemaker project contributors
 #
 # The version control history for this file may have further details.
 #
 # This source code is licensed under the GNU General Public License version 2
 # or later (GPLv2+) WITHOUT ANY WARRANTY.
 #
 
 TESTCC_TARGETS          = testcc_common_acl_h testcc_common_ipc_h \
     testcc_common_iso8601_h testcc_common_mainloop_h testcc_common_nvpair_h \
     testcc_common_results_h testcc_common_util_h testcc_common_xml_h \
     testcc_compatibility_h testcc_pengine_common_h testcc_pengine_complex_h \
     testcc_pe_types_h testcc_cib_types_h testcc_cib_util_h testcc_crm_h \
     testcc_cib_h testcc_cluster_h testcc_lrmd_h testcc_msg_xml_h \
     testcc_services_h testcc_stonith_ng_h testcc_pengine_status_h \
-    testcc_pengine_rules_h
-CLEANFILES		= $(TESTCC_TARGETS)
+    testcc_pengine_rules_h testcc_common_logging_h
+CLEANFILES		= $(TESTCC_TARGETS) testcc_helper.cc
 
 EXTRA_SCRIPTS		= bumplibs.sh				\
 			  travisci_build_coverity_scan.sh
 EXTRA_PROGRAMS		= $(TESTCC_TARGETS)
 EXTRA_DIST		= README
 
+testcc_helper.cc:
+	echo "int main() { return 0; }" > $@
+
 nodist_testcc_common_acl_h_SOURCES = testcc_helper.cc
 testcc_common_acl_h_CXXFLAGS = -include "crm/common/acl.h"
 
 nodist_testcc_common_ipc_h_SOURCES = testcc_helper.cc
 testcc_common_ipc_h_CXXFLAGS = -include "crm/common/ipc.h"
 
 nodist_testcc_common_iso8601_h_SOURCES = testcc_helper.cc
 testcc_common_iso8601_h_CXXFLAGS = -include "crm/common/iso8601.h"
 
 nodist_testcc_common_mainloop_h_SOURCES = testcc_helper.cc
 testcc_common_mainloop_h_CXXFLAGS = -include "crm/common/mainloop.h"
 
 nodist_testcc_common_nvpair_h_SOURCES = testcc_helper.cc
 testcc_common_nvpair_h_CXXFLAGS = -include "crm/common/nvpair.h"
 
 nodist_testcc_common_results_h_SOURCES = testcc_helper.cc
 testcc_common_results_h_CXXFLAGS = -include "crm/common/results.h"
 
 nodist_testcc_common_util_h_SOURCES = testcc_helper.cc
 testcc_common_util_h_CXXFLAGS = -include "crm/common/util.h"
 
 nodist_testcc_common_xml_h_SOURCES = testcc_helper.cc
 testcc_common_xml_h_CXXFLAGS = -include "crm/common/xml.h"
 
 nodist_testcc_compatibility_h_SOURCES = testcc_helper.cc
 testcc_compatibility_h_CXXFLAGS = -include "crm/compatibility.h"
 
 nodist_testcc_pengine_common_h_SOURCES = testcc_helper.cc
 testcc_pengine_common_h_CXXFLAGS = -include "crm/pengine/common.h"
 
 nodist_testcc_pengine_complex_h_SOURCES = testcc_helper.cc
 testcc_pengine_complex_h_CXXFLAGS = -include "crm/pengine/complex.h"
 
 nodist_testcc_pe_types_h_SOURCES = testcc_helper.cc
 testcc_pe_types_h_CXXFLAGS = -include "crm/pengine/pe_types.h"
 
 nodist_testcc_cib_types_h_SOURCES = testcc_helper.cc
 testcc_cib_types_h_CXXFLAGS = -include "crm/cib/cib_types.h"
 
 nodist_testcc_cib_util_h_SOURCES = testcc_helper.cc
 testcc_cib_util_h_CXXFLAGS = -include "crm/cib/util.h"
 
-# the initial ones except election.h not packaged
+nodist_testcc_common_logging_h_SOURCES = testcc_helper.cc
+testcc_common_logging_h_CXXFLAGS = -include "crm/common/logging.h"
+
+# ones previously in testcc.cc except election.h not packaged
 
 nodist_testcc_crm_h_SOURCES = testcc_helper.cc
 testcc_crm_h_CXXFLAGS = -include "crm/crm.h"
 
 nodist_testcc_cib_h_SOURCES = testcc_helper.cc
 testcc_cib_h_CXXFLAGS = -include "crm/cib.h"
 
 nodist_testcc_cluster_h_SOURCES = testcc_helper.cc
 testcc_cluster_h_CXXFLAGS = -include "crm/cluster.h"
 
 nodist_testcc_lrmd_h_SOURCES = testcc_helper.cc
 testcc_lrmd_h_CXXFLAGS = -include "crm/lrmd.h"
 
 nodist_testcc_msg_xml_h_SOURCES = testcc_helper.cc
 testcc_msg_xml_h_CXXFLAGS = -include "crm/msg_xml.h"
 
 nodist_testcc_services_h_SOURCES = testcc_helper.cc
 testcc_services_h_CXXFLAGS = -include "crm/services.h"
 
 nodist_testcc_stonith_ng_h_SOURCES = testcc_helper.cc
 testcc_stonith_ng_h_CXXFLAGS = -include "crm/stonith-ng.h"
 
 nodist_testcc_pengine_status_h_SOURCES = testcc_helper.cc
 testcc_pengine_status_h_CXXFLAGS = -include "crm/pengine/status.h"
 
 nodist_testcc_pengine_rules_h_SOURCES = testcc_helper.cc
 testcc_pengine_rules_h_CXXFLAGS = -include "crm/pengine/rules.h"
 
-# headers not pulling in everything they need
-
-nodist_testcc_common_logging_h_SOURCES = testcc_helper.cc
-testcc_common_logging_h_CXXFLAGS = -include "crm/common/logging.h"
-
 # c++-headers but not packaged
 # daemons/controld/controld_membership.h
 # include/crm/pengine/remote_internal.h
 # include/crm/common/cmdline_internal.h
 # include/crm/common/output.h
 # include/crm/common/ipcs_internal.h
 # include/crm/common/attrd_internal.h
 # include/pacemaker.h
 # include/pcmki/pcmki_output.h
 # include/pcmki/pcmki_transition.h
 # include/crm/cluster/election.h
 # lib/gnu/md5.h
 # tools/crm_resource_controller.h
 
 testcc: $(TESTCC_TARGETS)
diff --git a/maint/testcc_helper.cc b/maint/testcc_helper.cc
deleted file mode 100644
index 031a74e1c1..0000000000
--- a/maint/testcc_helper.cc
+++ /dev/null
@@ -1,13 +0,0 @@
-/*
- * Copyright 2019 the Pacemaker project contributors
- *
- * The version control history for this file may have further details.
- *
- * This source code is licensed under the GNU Lesser General Public License
- * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
- */
-
-int main()
-{
-    return 0;
-}
diff --git a/rpm/pacemaker.spec.in b/rpm/pacemaker.spec.in
index 270fb92445..ba5f808b3a 100644
--- a/rpm/pacemaker.spec.in
+++ b/rpm/pacemaker.spec.in
@@ -1,885 +1,888 @@
-# Define conditionals so that "rpmbuild --with <feature>" and
-# "rpmbuild --without <feature>" can enable and disable specific features
-
-## Add option to enable support for stonith/external fencing agents
-%bcond_with stonithd
-
-## Add option to create binaries suitable for use with profiling tools
-%bcond_with profiling
-
-## Add option to create binaries with coverage analysis
-%bcond_with coverage
-
-## Add option to skip generating documentation
-## (the build tools aren't available everywhere)
-%bcond_without doc
-
-## Add option to prefix package version with "0."
-## (so later "official" packages will be considered updates)
-%bcond_with pre_release
-
-## Add option to ship Upstart job files
-%bcond_with upstart_job
-
-## Add option to turn off hardening of libraries and daemon executables
-%bcond_without hardening
-
-## Add option to disable links for legacy daemon names
-%bcond_without legacy_links
-
-# Globals and defines to control package behavior (configure these as desired)
+# User-configurable globals and defines to control package behavior
+# (these should not test {with X} values, which are declared later)
 
 ## User and group to use for nonprivileged services
 %global uname hacluster
 %global gname haclient
 
 ## Where to install Pacemaker documentation
 %if 0%{?suse_version} > 0
 %global pcmk_docdir %{_docdir}/%{name}-%{version}
 %else
 %if 0%{?rhel} > 7
 %global pcmk_docdir %{_docdir}/%{name}-doc
 %else
 %global pcmk_docdir %{_docdir}/%{name}
 %endif
 %endif
 
 ## GitHub entity that distributes source (for ease of using a fork)
 %global github_owner ClusterLabs
 
 ## Upstream pacemaker version, and its package version (specversion
 ## can be incremented to build packages reliably considered "newer"
 ## than previously built packages with the same pcmkversion)
 %global pcmkversion X.Y.Z
 %global specversion 1
 
 ## Upstream commit (full commit ID, abbreviated commit ID, or tag) to build
 %global commit HEAD
 ## Since git v2.11, the extent of abbreviation is autoscaled by default
 ## (used to be constant of 7), so we need to convey it for non-tags, too.
 %global commit_abbrev 7
 
 ## Python major version to use (2, 3, or 0 for auto-detect)
 %global python_major 0
 
 
+# Define conditionals so that "rpmbuild --with <feature>" and
+# "rpmbuild --without <feature>" can enable and disable specific features
+
+## Add option to enable support for stonith/external fencing agents
+%bcond_with stonithd
+
+## Add option to create binaries suitable for use with profiling tools
+%bcond_with profiling
+
+## Add option to create binaries with coverage analysis
+%bcond_with coverage
+
+## Add option to skip generating documentation
+## (the build tools aren't available everywhere)
+%bcond_without doc
+
+## Add option to prefix package version with "0."
+## (so later "official" packages will be considered updates)
+%bcond_with pre_release
+
+## Add option to ship Upstart job files
+%bcond_with upstart_job
+
+## Add option to turn off hardening of libraries and daemon executables
+%bcond_without hardening
+
+## Add option to disable links for legacy daemon names
+%bcond_without legacy_links
+
 # Define globals for convenient use later
 
 ## Workaround to use parentheses in other globals
 %global lparen (
 %global rparen )
 
 ## Whether this is a tagged release (final or release candidate)
 %define tag_release %(c=%{commit}; case ${c} in Pacemaker-*%{rparen} echo 1 ;;
                       *%{rparen} echo 0 ;; esac)
 
 ## Portion of export/dist tarball name after "pacemaker-", and release version
 %if 0%{tag_release}
 %define archive_version %(c=%{commit}; echo ${c:10})
+%define archive_github_url %{commit}#/%{name}-%{archive_version}.tar.gz
 %define pcmk_release %(c=%{commit}; case $c in *-rc[[:digit:]]*%{rparen}
                        echo 0.%{specversion}.${c: -3} ;;
                        *%{rparen} echo %{specversion} ;; esac)
 %else
 %define archive_version %(c=%{commit}; echo ${c:0:%{commit_abbrev}})
+%define archive_github_url %{archive_version}#/%{name}-%{archive_version}.tar.gz
 %if %{with pre_release}
 %define pcmk_release 0.%{specversion}.%{archive_version}.git
 %else
 %define pcmk_release %{specversion}.%{archive_version}.git
 %endif
 %endif
 
 ## Heuristic used to infer bleeding-edge deployments that are
 ## less likely to have working versions of the documentation tools
 %define bleeding %(test ! -e /etc/yum.repos.d/fedora-rawhide.repo; echo $?)
 
 ## Whether this platform defaults to using systemd as an init system
 ## (needs to be evaluated prior to BuildRequires being enumerated and
 ## installed as it's intended to conditionally select some of these, and
 ## for that there are only few indicators with varying reliability:
 ## - presence of systemd-defined macros (when building in a full-fledged
 ##   environment, which is not the case with ordinary mock-based builds)
 ## - systemd-aware rpm as manifested with the presence of particular
 ##   macro (rpm itself will trivially always be present when building)
 ## - existence of /usr/lib/os-release file, which is something heavily
 ##   propagated by systemd project
 ## - when not good enough, there's always a possibility to check
 ##   particular distro-specific macros (incl. version comparison)
 %define systemd_native (%{?_unitdir:1}%{!?_unitdir:0}%{nil \
   } || %{?__transaction_systemd_inhibit:1}%{!?__transaction_systemd_inhibit:0}%{nil \
   } || %(test -f /usr/lib/os-release; test $? -ne 0; echo $?))
 
 %if 0%{?fedora} > 20 || 0%{?rhel} > 7
 ## Base GnuTLS cipher priorities (presumably only the initial, required keyword)
 ## overridable with "rpmbuild --define 'pcmk_gnutls_priorities PRIORITY-SPEC'"
 %define gnutls_priorities %{?pcmk_gnutls_priorities}%{!?pcmk_gnutls_priorities:@SYSTEM}
 %endif
 
 %if !%{defined _rundir}
 %if 0%{?fedora} >= 15 || 0%{?rhel} >= 7 || 0%{?suse_version} >= 1200
 %define _rundir /run
 %else
 %define _rundir /var/run
 %endif
 %endif
 
 %if 0%{?fedora} > 22 || 0%{?rhel} > 7
 %global supports_recommends 1
 %endif
 
 ## Different distros name certain packages differently
 ## (note: corosync libraries also differ, but all provide corosync-devel)
 %if 0%{?suse_version} > 0
 %global pkgname_bzip2_devel libbz2-devel
 %global pkgname_docbook_xsl docbook-xsl-stylesheets
 %global pkgname_gnutls_devel libgnutls-devel
 %global pkgname_shadow_utils shadow
 %global pkgname_procps procps
 %global pkgname_glue_libs libglue
 %global pkgname_pcmk_libs lib%{name}3
 %global hacluster_id 90
 %else
 %global pkgname_libtool_devel libtool-ltdl-devel
 %global pkgname_libtool_devel_arch libtool-ltdl-devel%{?_isa}
 %global pkgname_bzip2_devel bzip2-devel
 %global pkgname_docbook_xsl docbook-style-xsl
 %global pkgname_gnutls_devel gnutls-devel
 %global pkgname_shadow_utils shadow-utils
 %global pkgname_procps procps-ng
 %global pkgname_publican publican
 %global pkgname_glue_libs cluster-glue-libs
 %global pkgname_pcmk_libs %{name}-libs
 %global hacluster_id 189
 %endif
 
 # Python-related definitions
 
 ## Use Python 3 on certain platforms if major version not specified
 %if %{?python_major} == 0
 %if 0%{?fedora} > 26 || 0%{?rhel} > 7
 %global python_major 3
 %endif
 %endif
 
 ## Turn off auto-compilation of Python files outside Python specific paths,
 ## so there's no risk that unexpected "__python" macro gets picked to do the
 ## RPM-native byte-compiling there (only "{_datadir}/pacemaker/tests" affected)
 ## -- distro-dependent tricks or automake's fallback to be applied there
 %if %{defined _python_bytecompile_extra}
 %global _python_bytecompile_extra 0
 %else
 ### the statement effectively means no RPM-native byte-compiling will occur at
 ### all, so distro-dependent tricks for Python-specific packages to be applied
 %global __os_install_post %(echo '%{__os_install_post}' | {
                             sed -e 's!/usr/lib[^[:space:]]*/brp-python-bytecompile[[:space:]].*$!!g'; })
 %endif
 
 ## Values that differ by Python major version
 %if 0%{?python_major} > 2
 %global python_name python3
 %global python_path %{?__python3}%{!?__python3:/usr/bin/python%{?python3_pkgversion}%{!?python3_pkgversion:3}}
 %define python_site %{?python3_sitelib}%{!?python3_sitelib:%(
   %{python_path} -c 'from distutils.sysconfig import get_python_lib as gpl; print(gpl(1))' 2>/dev/null)}
 %else
 %if 0%{?python_major} > 1
 %global python_name python2
 %global python_path %{?__python2}%{!?__python2:/usr/bin/python%{?python2_pkgversion}%{!?python2_pkgversion:2}}
 %define python_site %{?python2_sitelib}%{!?python2_sitelib:%(
   %{python_path} -c 'from distutils.sysconfig import get_python_lib as gpl; print(gpl(1))' 2>/dev/null)}
 %else
 %global python_name python
 %global python_path %{?__python}%{!?__python:/usr/bin/python%{?python_pkgversion}}
 %define python_site %{?python_sitelib}%{!?python_sitelib:%(
   python -c 'from distutils.sysconfig import get_python_lib as gpl; print(gpl(1))' 2>/dev/null)}
 %endif
 %endif
 
 
 # Definitions for backward compatibility with older RPM versions
 
 ## Ensure the license macro behaves consistently (older RPM will otherwise
 ## overwrite it once it encounters "License:"). Courtesy Jason Tibbitts:
 ## https://pkgs.fedoraproject.org/cgit/rpms/epel-rpm-macros.git/tree/macros.zzz-epel?h=el6&id=e1adcb77
 %if !%{defined _licensedir}
 %define description %{lua:
     rpm.define("license %doc")
     print("%description")
 }
 %endif
 
 # Keep sane profiling data if requested
 %if %{with profiling}
 
 ## Disable -debuginfo package and stripping binaries/libraries
 %define debug_package %{nil}
 
 %endif
 
 
 Name:          pacemaker
 Summary:       Scalable High-Availability cluster resource manager
 Version:       %{pcmkversion}
 Release:       %{pcmk_release}%{?dist}
 %if %{defined _unitdir}
 License:       GPLv2+ and LGPLv2+
 %else
 # initscript is Revised BSD
 License:       GPLv2+ and LGPLv2+ and BSD
 %endif
 Url:           http://www.clusterlabs.org
 Group:         System Environment/Daemons
 
 # Example: https://codeload.github.com/ClusterLabs/pacemaker/tar.gz/e91769e
 # will download pacemaker-e91769e.tar.gz
 #
 # The ending part starting with '#' is ignored by github but necessary for
 # rpmbuild to know what the tar archive name is. (The downloaded file will be
 # named correctly only for commit IDs, not tagged releases.)
 #
 # You can use "spectool -s 0 pacemaker.spec" (rpmdevtools) to show final URL.
-Source0:       https://codeload.github.com/%{github_owner}/%{name}/tar.gz/%{commit}#/%{name}-%{archive_version}.tar.gz
+Source0:       https://codeload.github.com/%{github_owner}/%{name}/tar.gz/%{archive_github_url}
 Requires:      resource-agents
 Requires:      %{pkgname_pcmk_libs}%{?_isa} = %{version}-%{release}
 Requires:      %{name}-cluster-libs%{?_isa} = %{version}-%{release}
 Requires:      %{name}-cli = %{version}-%{release}
 %if !%{defined _unitdir}
 Requires:      %{pkgname_procps}
 Requires:      psmisc
 %endif
 %{?systemd_requires}
 
 Requires:      %{python_path}
 BuildRequires: %{python_name}-devel
 
 # Pacemaker requires a minimum libqb functionality
 Requires:      libqb >= 0.13.0
 BuildRequires: libqb-devel >= 0.13.0
 
 # Basics required for the build (even if usually satisfied through other BRs)
 BuildRequires: coreutils findutils grep sed
 
 # Required for core functionality
 BuildRequires: automake autoconf gcc libtool pkgconfig %{?pkgname_libtool_devel}
 BuildRequires: pkgconfig(glib-2.0) >= 2.16
 BuildRequires: libxml2-devel libxslt-devel libuuid-devel
 BuildRequires: %{pkgname_bzip2_devel}
 
 # Enables optional functionality
 BuildRequires: ncurses-devel %{pkgname_docbook_xsl}
 BuildRequires: help2man %{pkgname_gnutls_devel} pam-devel pkgconfig(dbus-1)
 
 %if %{systemd_native}
 BuildRequires: pkgconfig(systemd)
 %endif
 
 Requires:      corosync >= 2.0.0
 BuildRequires: corosync-devel >= 2.0.0
 
 %if %{with stonithd}
 BuildRequires: %{pkgname_glue_libs}-devel
 %endif
 
 ## (note no avoiding effect when building through non-customized mock)
 %if !%{bleeding}
 %if %{with doc}
 BuildRequires: inkscape asciidoc %{?pkgname_publican}
 %endif
 %endif
 
 Provides:      pcmk-cluster-manager = %{version}-%{release}
 Provides:      pcmk-cluster-manager%{?_isa} = %{version}-%{release}
 
 # Bundled bits
 ## Pacemaker uses the crypto/md5-buffer module from gnulib
 %if 0%{?fedora} || 0%{?rhel}
 Provides:      bundled(gnulib)
 %endif
 
 %description
 Pacemaker is an advanced, scalable High-Availability cluster resource
 manager.
 
 It supports more than 16 node clusters with significant capabilities
 for managing resources and dependencies.
 
 It will run scripts at initialization, when machines go up or down,
 when related resources fail and can be configured to periodically check
 resource health.
 
 Available rpmbuild rebuild options:
   --with(out) : coverage doc stonithd hardening pre_release profiling
                 upstart_job
 
 %package cli
 License:       GPLv2+ and LGPLv2+
 Summary:       Command line tools for controlling Pacemaker clusters
 Group:         System Environment/Daemons
 Requires:      %{pkgname_pcmk_libs}%{?_isa} = %{version}-%{release}
 %if 0%{?supports_recommends}
 Recommends:    pcmk-cluster-manager = %{version}-%{release}
 # For crm_report
 Recommends:    tar
 Recommends:    bzip2
 %endif
 Requires:      perl-TimeDate
 Requires:      %{pkgname_procps}
 Requires:      psmisc
 Requires(post):coreutils
 
 %description cli
 Pacemaker is an advanced, scalable High-Availability cluster resource
 manager.
 
 The %{name}-cli package contains command line tools that can be used
 to query and control the cluster from machines that may, or may not,
 be part of the cluster.
 
 %package -n %{pkgname_pcmk_libs}
 License:       GPLv2+ and LGPLv2+
 Summary:       Core Pacemaker libraries
 Group:         System Environment/Daemons
 Requires(pre): %{pkgname_shadow_utils}
 Requires:      %{name}-schemas = %{version}-%{release}
 # sbd 1.4.0+ supports the libpe_status API for pe_working_set_t
 Conflicts:     sbd < 1.4.0
 
 %description -n %{pkgname_pcmk_libs}
 Pacemaker is an advanced, scalable High-Availability cluster resource
 manager.
 
 The %{pkgname_pcmk_libs} package contains shared libraries needed for cluster
 nodes and those just running the CLI tools.
 
 %package cluster-libs
 License:       GPLv2+ and LGPLv2+
 Summary:       Cluster Libraries used by Pacemaker
 Group:         System Environment/Daemons
 Requires:      %{pkgname_pcmk_libs}%{?_isa} = %{version}-%{release}
 
 %description cluster-libs
 Pacemaker is an advanced, scalable High-Availability cluster resource
 manager.
 
 The %{name}-cluster-libs package contains cluster-aware shared
 libraries needed for nodes that will form part of the cluster nodes.
 
 %package remote
 %if %{defined _unitdir}
 License:       GPLv2+ and LGPLv2+
 %else
 # initscript is Revised BSD
 License:       GPLv2+ and LGPLv2+ and BSD
 %endif
 Summary:       Pacemaker remote daemon for non-cluster nodes
 Group:         System Environment/Daemons
 Requires:      %{pkgname_pcmk_libs}%{?_isa} = %{version}-%{release}
 Requires:      %{name}-cli = %{version}-%{release}
 Requires:      resource-agents
 %if !%{defined _unitdir}
 Requires:      %{pkgname_procps}
 %endif
 # -remote can be fully independent of systemd
 %{?systemd_ordering}%{!?systemd_ordering:%{?systemd_requires}}
 Provides:      pcmk-cluster-manager = %{version}-%{release}
 Provides:      pcmk-cluster-manager%{?_isa} = %{version}-%{release}
 
 %description remote
 Pacemaker is an advanced, scalable High-Availability cluster resource
 manager.
 
 The %{name}-remote package contains the Pacemaker Remote daemon
 which is capable of extending pacemaker functionality to remote
 nodes not running the full corosync/cluster stack.
 
 %package -n %{pkgname_pcmk_libs}-devel
 License:       GPLv2+ and LGPLv2+
 Summary:       Pacemaker development package
 Group:         Development/Libraries
 Requires:      %{pkgname_pcmk_libs}%{?_isa} = %{version}-%{release}
 Requires:      %{name}-cluster-libs%{?_isa} = %{version}-%{release}
 Requires:      libuuid-devel%{?_isa} %{?pkgname_libtool_devel_arch}
 Requires:      libxml2-devel%{?_isa} libxslt-devel%{?_isa}
 Requires:      %{pkgname_bzip2_devel}%{?_isa} glib2-devel%{?_isa}
 Requires:      libqb-devel%{?_isa}
 Requires:      corosync-devel >= 2.0.0
 
 %description -n %{pkgname_pcmk_libs}-devel
 Pacemaker is an advanced, scalable High-Availability cluster resource
 manager.
 
 The %{pkgname_pcmk_libs}-devel package contains headers and shared libraries
 for developing tools for Pacemaker.
 
 %package       cts
 License:       GPLv2+ and LGPLv2+
 Summary:       Test framework for cluster-related technologies like Pacemaker
 Group:         System Environment/Daemons
 Requires:      %{python_path}
 Requires:      %{pkgname_pcmk_libs} = %{version}-%{release}
 Requires:      %{name}-cli = %{version}-%{release}
 Requires:      %{pkgname_procps}
 Requires:      psmisc
 BuildArch:     noarch
 
 # systemd python bindings are separate package in some distros
 %if %{defined systemd_requires}
 
 %if 0%{?fedora} > 22 || 0%{?rhel} > 7
 Requires:      %{python_name}-systemd
 %else
 %if 0%{?fedora} > 20 || 0%{?rhel} > 6
 Requires:      systemd-python
 %endif
 %endif
 
 %endif
 
 %description   cts
 Test framework for cluster-related technologies like Pacemaker
 
 %package       doc
 License:       CC-BY-SA-4.0
 Summary:       Documentation for Pacemaker
 Group:         Documentation
 BuildArch:     noarch
 
 %description   doc
 Documentation for Pacemaker.
 
 Pacemaker is an advanced, scalable High-Availability cluster resource
 manager.
 
 %package       schemas
 License:       GPLv2+
 Summary:       Schemas and upgrade stylesheets for Pacemaker
 BuildArch:     noarch
 
 %description   schemas
 Schemas and upgrade stylesheets for Pacemaker
 
 Pacemaker is an advanced, scalable High-Availability cluster resource
 manager.
 
 %prep
 %setup -q -n %{name}-%{archive_version}
 
 %build
 
 # Early versions of autotools (e.g. RHEL <= 5) do not support --docdir
 export docdir=%{pcmk_docdir}
 
 export systemdsystemunitdir=%{?_unitdir}%{!?_unitdir:no}
 
 %if %{with hardening}
 # prefer distro-provided hardening flags in case they are defined
 # through _hardening_{c,ld}flags macros, configure script will
 # use its own defaults otherwise; if such hardenings are completely
 # undesired, rpmbuild using "--without hardening"
 # (or "--define '_without_hardening 1'")
 export CFLAGS_HARDENED_EXE="%{?_hardening_cflags}"
 export CFLAGS_HARDENED_LIB="%{?_hardening_cflags}"
 export LDFLAGS_HARDENED_EXE="%{?_hardening_ldflags}"
 export LDFLAGS_HARDENED_LIB="%{?_hardening_ldflags}"
 %endif
 
 ./autogen.sh
 
 %{configure}                                                                    \
         PYTHON=%{python_path}                                                   \
         %{!?with_hardening:    --disable-hardening}                             \
         %{!?with_legacy_links: --disable-legacy-links}                          \
         %{?with_profiling:     --with-profiling}                                \
         %{?with_coverage:      --with-coverage}                                 \
         %{!?with_doc:          --with-brand=}                                   \
         %{?gnutls_priorities:  --with-gnutls-priorities="%{gnutls_priorities}"} \
         --with-initdir=%{_initrddir}                                            \
         --with-runstatedir=%{_rundir}                                           \
         --localstatedir=%{_var}                                                 \
         --with-version=%{version}-%{release}
 
 %if 0%{?suse_version} >= 1200
 # Fedora handles rpath removal automagically
 sed -i 's|^hardcode_libdir_flag_spec=.*|hardcode_libdir_flag_spec=""|g' libtool
 sed -i 's|^runpath_var=LD_RUN_PATH|runpath_var=DIE_RPATH_DIE|g' libtool
 %endif
 
 make %{_smp_mflags} V=1
 
 %check
 make %{_smp_mflags} check
 { cts/cts-scheduler --run load-stopped-loop \
   && cts/cts-cli \
   && touch .CHECKED
 } 2>&1 | sed 's/[fF]ail/faiil/g'  # prevent false positives in rpmlint
 [ -f .CHECKED ] && rm -f -- .CHECKED
 exit $?  # TODO remove when rpm<4.14 compatibility irrelevant
 
 %install
 # skip automake-native Python byte-compilation, since RPM-native one (possibly
 # distro-confined to Python-specific directories, which is currently the only
 # relevant place, anyway) assures proper intrinsic alignment with wider system
 # (such as with py_byte_compile macro, which is concurrent Fedora/EL specific)
 make install \
   DESTDIR=%{buildroot} V=1 docdir=%{pcmk_docdir} \
   %{?_python_bytecompile_extra:%{?py_byte_compile:am__py_compile=true}}
 
 mkdir -p ${RPM_BUILD_ROOT}%{_sysconfdir}/sysconfig
 install -m 644 daemons/pacemakerd/pacemaker.sysconfig ${RPM_BUILD_ROOT}%{_sysconfdir}/sysconfig/pacemaker
 install -m 644 tools/crm_mon.sysconfig ${RPM_BUILD_ROOT}%{_sysconfdir}/sysconfig/crm_mon
 
 %if %{with upstart_job}
 mkdir -p ${RPM_BUILD_ROOT}%{_sysconfdir}/init
 install -m 644 pacemakerd/pacemaker.upstart ${RPM_BUILD_ROOT}%{_sysconfdir}/init/pacemaker.conf
 install -m 644 pacemakerd/pacemaker.combined.upstart ${RPM_BUILD_ROOT}%{_sysconfdir}/init/pacemaker.combined.conf
 install -m 644 tools/crm_mon.upstart ${RPM_BUILD_ROOT}%{_sysconfdir}/init/crm_mon.conf
 %endif
 
 %if %{defined _unitdir}
 mkdir -p ${RPM_BUILD_ROOT}%{_localstatedir}/lib/rpm-state/%{name}
 %endif
 
 # Don't package static libs
 find %{buildroot} -name '*.a' -type f -print0 | xargs -0 rm -f
 find %{buildroot} -name '*.la' -type f -print0 | xargs -0 rm -f
 
 # For now, don't package the servicelog-related binaries built only for
 # ppc64le when certain dependencies are installed. If they get more exercise by
 # advanced users, we can reconsider.
 rm -f %{buildroot}/%{_sbindir}/notifyServicelogEvent
 rm -f %{buildroot}/%{_sbindir}/ipmiservicelogd
 
 # Don't ship init scripts for systemd based platforms
 %if %{defined _unitdir}
 rm -f %{buildroot}/%{_initrddir}/pacemaker
 rm -f %{buildroot}/%{_initrddir}/pacemaker_remote
 %endif
 
 # Byte-compile Python sources where suitable and the distro procedures known
 %if %{defined py_byte_compile}
 %{py_byte_compile %{python_path} %{buildroot}%{_datadir}/pacemaker/tests}
 %if !%{defined _python_bytecompile_extra}
 %{py_byte_compile %{python_path} %{buildroot}%{python_site}/cts}
 %endif
 %endif
 
 %if %{with coverage}
 GCOV_BASE=%{buildroot}/%{_var}/lib/pacemaker/gcov
 mkdir -p $GCOV_BASE
 find . -name '*.gcno' -type f | while read F ; do
         D=`dirname $F`
         mkdir -p ${GCOV_BASE}/$D
         cp $F ${GCOV_BASE}/$D
 done
 %endif
 
 %post
 %if %{defined _unitdir}
 %systemd_post pacemaker.service
 %else
 /sbin/chkconfig --add pacemaker || :
 %endif
 
 %preun
 %if %{defined _unitdir}
 %systemd_preun pacemaker.service
 %else
 /sbin/service pacemaker stop >/dev/null 2>&1 || :
 if [ "$1" -eq 0 ]; then
     # Package removal, not upgrade
     /sbin/chkconfig --del pacemaker || :
 fi
 %endif
 
 %postun
 %if %{defined _unitdir}
 %systemd_postun_with_restart pacemaker.service
 %endif
 
 %pre remote
 %if %{defined _unitdir}
 # Stop the service before anything is touched, and remember to restart
 # it as one of the last actions (compared to using systemd_postun_with_restart,
 # this avoids suicide when sbd is in use)
 systemctl --quiet is-active pacemaker_remote
 if [ $? -eq 0 ] ; then
     mkdir -p %{_localstatedir}/lib/rpm-state/%{name}
     touch %{_localstatedir}/lib/rpm-state/%{name}/restart_pacemaker_remote
     systemctl stop pacemaker_remote >/dev/null 2>&1
 else
     rm -f %{_localstatedir}/lib/rpm-state/%{name}/restart_pacemaker_remote
 fi
 %endif
 
 %post remote
 %if %{defined _unitdir}
 %systemd_post pacemaker_remote.service
 %else
 /sbin/chkconfig --add pacemaker_remote || :
 %endif
 
 %preun remote
 %if %{defined _unitdir}
 %systemd_preun pacemaker_remote.service
 %else
 /sbin/service pacemaker_remote stop >/dev/null 2>&1 || :
 if [ "$1" -eq 0 ]; then
     # Package removal, not upgrade
     /sbin/chkconfig --del pacemaker_remote || :
 fi
 %endif
 
 %postun remote
 %if %{defined _unitdir}
 # This next line is a no-op, because we stopped the service earlier, but
 # we leave it here because it allows us to revert to the standard behavior
 # in the future if desired
 %systemd_postun_with_restart pacemaker_remote.service
 # Explicitly take care of removing the flag-file(s) upon final removal
 if [ "$1" -eq 0 ] ; then
     rm -f %{_localstatedir}/lib/rpm-state/%{name}/restart_pacemaker_remote
 fi
 %endif
 
 %posttrans remote
 %if %{defined _unitdir}
 if [ -e %{_localstatedir}/lib/rpm-state/%{name}/restart_pacemaker_remote ] ; then
     systemctl start pacemaker_remote >/dev/null 2>&1
     rm -f %{_localstatedir}/lib/rpm-state/%{name}/restart_pacemaker_remote
 fi
 %endif
 
 %post cli
 %if %{defined _unitdir}
 %systemd_post crm_mon.service
 %endif
 if [ "$1" -eq 2 ]; then
     # Package upgrade, not initial install:
     # Move any pre-2.0 logs to new location to ensure they get rotated
     { mv -fbS.rpmsave %{_var}/log/pacemaker.log* %{_var}/log/pacemaker \
       || mv -f %{_var}/log/pacemaker.log* %{_var}/log/pacemaker
     } >/dev/null 2>/dev/null || :
 fi
 
 %preun cli
 %if %{defined _unitdir}
 %systemd_preun crm_mon.service
 %endif
 
 %postun cli
 %if %{defined _unitdir}
 %systemd_postun_with_restart crm_mon.service
 %endif
 
 %pre -n %{pkgname_pcmk_libs}
 getent group %{gname} >/dev/null || groupadd -r %{gname} -g %{hacluster_id}
 getent passwd %{uname} >/dev/null || useradd -r -g %{gname} -u %{hacluster_id} -s /sbin/nologin -c "cluster user" %{uname}
 exit 0
 
 %if %{defined ldconfig_scriptlets}
 %ldconfig_scriptlets -n %{pkgname_pcmk_libs}
 %ldconfig_scriptlets cluster-libs
 %else
 %post -n %{pkgname_pcmk_libs} -p /sbin/ldconfig
 %postun -n %{pkgname_pcmk_libs} -p /sbin/ldconfig
 
 %post cluster-libs -p /sbin/ldconfig
 %postun cluster-libs -p /sbin/ldconfig
 %endif
 
 %files
 ###########################################################
 %config(noreplace) %{_sysconfdir}/sysconfig/pacemaker
 %{_sbindir}/pacemakerd
 
 %if %{defined _unitdir}
 %{_unitdir}/pacemaker.service
 %else
 %{_initrddir}/pacemaker
 %endif
 
 %exclude %{_libexecdir}/pacemaker/cts-log-watcher
 %exclude %{_libexecdir}/pacemaker/cts-support
 %exclude %{_sbindir}/pacemaker-remoted
 %if %{with legacy_links}
 %exclude %{_sbindir}/pacemaker_remoted
 %endif
 %{_libexecdir}/pacemaker/*
 
 %{_sbindir}/crm_attribute
 %{_sbindir}/crm_master
 %{_sbindir}/fence_legacy
 
 %doc %{_mandir}/man7/pacemaker-controld.*
 %doc %{_mandir}/man7/pacemaker-schedulerd.*
 %doc %{_mandir}/man7/pacemaker-fenced.*
 %doc %{_mandir}/man7/ocf_pacemaker_controld.*
 %doc %{_mandir}/man7/ocf_pacemaker_o2cb.*
 %doc %{_mandir}/man7/ocf_pacemaker_remote.*
 %doc %{_mandir}/man8/crm_attribute.*
 %doc %{_mandir}/man8/crm_master.*
 %doc %{_mandir}/man8/fence_legacy.*
 %doc %{_mandir}/man8/pacemakerd.*
 
 %doc %{_datadir}/pacemaker/alerts
 
 %license licenses/GPLv2
 %doc COPYING
 %doc ChangeLog
 
 %dir %attr (750, %{uname}, %{gname}) %{_var}/lib/pacemaker/cib
 %dir %attr (750, %{uname}, %{gname}) %{_var}/lib/pacemaker/pengine
 /usr/lib/ocf/resource.d/pacemaker/controld
 /usr/lib/ocf/resource.d/pacemaker/o2cb
 /usr/lib/ocf/resource.d/pacemaker/remote
 
 %if %{with upstart_job}
 %config(noreplace) %{_sysconfdir}/init/pacemaker.conf
 %config(noreplace) %{_sysconfdir}/init/pacemaker.combined.conf
 %endif
 
 %files cli
 %dir %attr (750, root, %{gname}) %{_sysconfdir}/pacemaker
 %config(noreplace) %{_sysconfdir}/logrotate.d/pacemaker
 %config(noreplace) %{_sysconfdir}/sysconfig/crm_mon
 
 %if %{defined _unitdir}
 %{_unitdir}/crm_mon.service
 %endif
 
 %if %{with upstart_job}
 %config(noreplace) %{_sysconfdir}/init/crm_mon.conf
 %endif
 
 %{_sbindir}/attrd_updater
 %{_sbindir}/cibadmin
 %{_sbindir}/crm_diff
 %{_sbindir}/crm_error
 %{_sbindir}/crm_failcount
 %{_sbindir}/crm_mon
 %{_sbindir}/crm_node
 %{_sbindir}/crm_resource
 %{_sbindir}/crm_rule
 %{_sbindir}/crm_standby
 %{_sbindir}/crm_verify
 %{_sbindir}/crmadmin
 %{_sbindir}/iso8601
 %{_sbindir}/crm_shadow
 %{_sbindir}/crm_simulate
 %{_sbindir}/crm_report
 %{_sbindir}/crm_ticket
 %{_sbindir}/stonith_admin
 # "dirname" is owned by -schemas, which is a prerequisite
 %{_datadir}/pacemaker/report.collector
 %{_datadir}/pacemaker/report.common
 # XXX "dirname" is not owned by any prerequisite
 %{_datadir}/snmp/mibs/PCMK-MIB.txt
 
 %exclude /usr/lib/ocf/resource.d/pacemaker/controld
 %exclude /usr/lib/ocf/resource.d/pacemaker/o2cb
 %exclude /usr/lib/ocf/resource.d/pacemaker/remote
 
 %dir /usr/lib/ocf
 %dir /usr/lib/ocf/resource.d
 /usr/lib/ocf/resource.d/pacemaker
 
 %doc %{_mandir}/man7/*
 %exclude %{_mandir}/man7/pacemaker-controld.*
 %exclude %{_mandir}/man7/pacemaker-schedulerd.*
 %exclude %{_mandir}/man7/pacemaker-fenced.*
 %exclude %{_mandir}/man7/ocf_pacemaker_controld.*
 %exclude %{_mandir}/man7/ocf_pacemaker_o2cb.*
 %exclude %{_mandir}/man7/ocf_pacemaker_remote.*
 %doc %{_mandir}/man8/*
 %exclude %{_mandir}/man8/crm_attribute.*
 %exclude %{_mandir}/man8/crm_master.*
 %exclude %{_mandir}/man8/fence_legacy.*
 %exclude %{_mandir}/man8/pacemakerd.*
 %exclude %{_mandir}/man8/pacemaker-remoted.*
 
 %license licenses/GPLv2
 %doc COPYING
 %doc ChangeLog
 
 %dir %attr (750, %{uname}, %{gname}) %{_var}/lib/pacemaker
 %dir %attr (750, %{uname}, %{gname}) %{_var}/lib/pacemaker/blackbox
 %dir %attr (750, %{uname}, %{gname}) %{_var}/lib/pacemaker/cores
 %dir %attr (770, %{uname}, %{gname}) %{_var}/log/pacemaker
 %dir %attr (770, %{uname}, %{gname}) %{_var}/log/pacemaker/bundles
 
 %files -n %{pkgname_pcmk_libs}
 %{_libdir}/libcib.so.*
 %{_libdir}/liblrmd.so.*
 %{_libdir}/libcrmservice.so.*
 %{_libdir}/libcrmcommon.so.*
 %{_libdir}/libpe_status.so.*
 %{_libdir}/libpe_rules.so.*
 %{_libdir}/libpacemaker.so.*
 %{_libdir}/libstonithd.so.*
 %license licenses/LGPLv2.1
 %doc COPYING
 %doc ChangeLog
 
 %files cluster-libs
 %{_libdir}/libcrmcluster.so.*
 %license licenses/LGPLv2.1
 %doc COPYING
 %doc ChangeLog
 
 %files remote
 %config(noreplace) %{_sysconfdir}/sysconfig/pacemaker
 %if %{defined _unitdir}
 # state directory is shared between the subpackets
 # let rpm take care of removing it once it isn't
 # referenced anymore and empty
 %ghost %dir %{_localstatedir}/lib/rpm-state/%{name}
 %{_unitdir}/pacemaker_remote.service
 %else
 %{_initrddir}/pacemaker_remote
 %endif
 
 %{_sbindir}/pacemaker-remoted
 %if %{with legacy_links}
 %{_sbindir}/pacemaker_remoted
 %endif
 %{_mandir}/man8/pacemaker-remoted.*
 %license licenses/GPLv2
 %doc COPYING
 %doc ChangeLog
 
 %files doc
 %doc %{pcmk_docdir}
 %license licenses/CC-BY-SA-4.0
 
 %files cts
 %{python_site}/cts
 %{_datadir}/pacemaker/tests
 
 %{_libexecdir}/pacemaker/cts-log-watcher
 %{_libexecdir}/pacemaker/cts-support
 
 %license licenses/GPLv2
 %doc COPYING
 %doc ChangeLog
 
 %files -n %{pkgname_pcmk_libs}-devel
 %{_includedir}/pacemaker
 %{_libdir}/*.so
 %if %{with coverage}
 %{_var}/lib/pacemaker/gcov
 %endif
 %{_libdir}/pkgconfig/*.pc
 %license licenses/LGPLv2.1
 %doc COPYING
 %doc ChangeLog
 
 %files schemas
 %license licenses/GPLv2
 %dir %{_datadir}/pacemaker
 %{_datadir}/pacemaker/*.rng
 %{_datadir}/pacemaker/*.xsl
 %{_datadir}/pacemaker/api
 %{_datadir}/pkgconfig/pacemaker-schemas.pc
 
 %changelog
 
 * PACKAGE_DATE ClusterLabs <admin@clusterlabs.org> PACKAGE_VERSION-1
  - See included ChangeLog file for details
diff --git a/tools/attrd_updater.c b/tools/attrd_updater.c
index d186886f07..3fa6ef2eaf 100644
--- a/tools/attrd_updater.c
+++ b/tools/attrd_updater.c
@@ -1,466 +1,467 @@
 /*
  * Copyright 2004-2020 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU General Public License version 2
  * or later (GPLv2+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #include <stdio.h>
 #include <unistd.h>
 #include <stdlib.h>
 #include <libgen.h>
 
 #include <sys/param.h>
 #include <sys/types.h>
 
 #include <crm/crm.h>
 #include <crm/msg_xml.h>
 #include <crm/common/ipc.h>
 
 #include <crm/common/attrd_internal.h>
 
 static pcmk__cli_option_t long_options[] = {
     // long option, argument type, storage, short option, description, flags
     {
         "help", no_argument, NULL, '?',
         "\tThis text", pcmk__option_default
     },
     {
         "version", no_argument, NULL, '$',
         "\tVersion information", pcmk__option_default
     },
     {
         "verbose", no_argument, NULL, 'V',
         "\tIncrease debug output\n", pcmk__option_default
     },
     {
         "name", required_argument, NULL, 'n',
         "The attribute's name", pcmk__option_default
     },
     {
         "-spacer-", no_argument, NULL, '-',
         "\nCommands:", pcmk__option_default
     },
     {
         "update", required_argument, NULL, 'U',
         "Update attribute's value in pacemaker-attrd. If this causes the value "
             "to change, it will also be updated in the cluster configuration.",
         pcmk__option_default
     },
     {
         "update-both", required_argument, NULL, 'B',
         "Update attribute's value and time to wait (dampening) in "
             "pacemaker-attrd. If this causes the value or dampening to change, "
             "the attribute will also be written to the cluster configuration, "
             "so be aware that repeatedly changing the dampening reduces its "
             "effectiveness.",
         pcmk__option_default
     },
     {
         "update-delay", no_argument, NULL, 'Y',
         "Update attribute's dampening in pacemaker-attrd (requires "
             "-d/--delay). If this causes the dampening to change, the "
             "attribute will also be written to the cluster configuration, so "
             "be aware that repeatedly changing the dampening reduces its "
             "effectiveness.",
         pcmk__option_default
     },
     {
         "query", no_argument, NULL, 'Q',
         "\tQuery the attribute's value from pacemaker-attrd",
         pcmk__option_default
     },
     {
         "delete", no_argument, NULL, 'D',
         "\tDelete attribute from pacemaker-attrd. If a value was previously "
             "set, it will also be removed from the cluster configuration",
         pcmk__option_default
     },
     {
         "refresh", no_argument, NULL, 'R',
         "\t(Advanced) Force the pacemaker-attrd daemon to resend all current "
             "values to the CIB",
         pcmk__option_default
     },
 
     {
         "-spacer-", no_argument, NULL, '-',
         "\nAdditional options:", pcmk__option_default
     },
     {
         "delay", required_argument, NULL, 'd',
         "The time to wait (dampening) in seconds for further changes "
             "before writing",
         pcmk__option_default
     },
     {
         "set", required_argument, NULL, 's',
         "(Advanced) The attribute set in which to place the value",
         pcmk__option_default
     },
     {
         "node", required_argument, NULL, 'N',
         "Set the attribute for the named node (instead of the local one)",
         pcmk__option_default
     },
     {
         "all", no_argument, NULL, 'A',
         "Show values of the attribute for all nodes (query only)",
         pcmk__option_default
     },
 
     // @TODO Implement --lifetime
     {
         "lifetime", required_argument, NULL, 'l',
         "(Not yet implemented) Lifetime of the node attribute (silently "
             "ignored by cluster)",
         pcmk__option_default
     },
     {
         "private", no_argument, NULL, 'p',
         "\tIf this creates a new attribute, never write the attribute to CIB",
         pcmk__option_default
     },
 
     /* Legacy options */
     {
         "quiet", no_argument, NULL, 'q',
         NULL, pcmk__option_hidden
     },
     {
         "update", required_argument, NULL, 'v',
         NULL, pcmk__option_hidden
     },
     {
         "section", required_argument, NULL, 'S',
         NULL, pcmk__option_hidden
     },
     { 0, 0, 0, 0 }
 };
 
 static int do_query(const char *attr_name, const char *attr_node, gboolean query_all);
 static int do_update(char command, const char *attr_node, const char *attr_name,
                      const char *attr_value, const char *attr_section,
                      const char *attr_set, const char *attr_dampen, int attr_options);
 
 // Free memory at exit to make analyzers happy
 #define cleanup_memory() \
     free(attr_dampen); \
     free(attr_name); \
     free(attr_node); \
     free(attr_section); \
     free(attr_set);
 
 #define set_option(option_var) \
     if (option_var) { \
         free(option_var); \
     } \
     option_var = strdup(optarg);
 
 int
 main(int argc, char **argv)
 {
     int index = 0;
     int argerr = 0;
     int attr_options = pcmk__node_attr_none;
     int flag;
     crm_exit_t exit_code = CRM_EX_OK;
     char *attr_node = NULL;
     char *attr_name = NULL;
     char *attr_set = NULL;
     char *attr_section = NULL;
     char *attr_dampen = NULL;
     const char *attr_value = NULL;
     char command = 'Q';
 
     gboolean query_all = FALSE;
 
     crm_log_cli_init("attrd_updater");
     pcmk__set_cli_options(NULL, "-n <attribute> <command> [options]",
                           long_options,
                           "query and update Pacemaker node attributes");
 
     if (argc < 2) {
         pcmk__cli_help('?', CRM_EX_USAGE);
     }
 
     while (1) {
         flag = pcmk__next_cli_option(argc, argv, &index, NULL);
         if (flag == -1)
             break;
 
         switch (flag) {
             case 'V':
                 crm_bump_log_level(argc, argv);
                 break;
             case '?':
             case '$':
                 cleanup_memory();
                 pcmk__cli_help(flag, CRM_EX_OK);
                 break;
             case 'n':
                 set_option(attr_name);
                 break;
             case 's':
                 set_option(attr_set);
                 break;
             case 'd':
                 set_option(attr_dampen);
                 break;
             case 'l':
             case 'S':
                 set_option(attr_section);
                 break;
             case 'N':
                 set_option(attr_node);
                 break;
             case 'A':
                 query_all = TRUE;
                 break;
             case 'p':
                 set_bit(attr_options, pcmk__node_attr_private);
                 break;
             case 'q':
                 break;
             case 'Y':
                 command = flag;
                 crm_log_args(argc, argv); /* Too much? */
                 break;
             case 'Q':
             case 'B':
             case 'R':
             case 'D':
             case 'U':
             case 'v':
                 command = flag;
                 attr_value = optarg;
                 crm_log_args(argc, argv); /* Too much? */
                 break;
             default:
                 ++argerr;
                 break;
         }
     }
 
     if (optind > argc) {
         ++argerr;
     }
 
     if (command != 'R' && attr_name == NULL) {
         ++argerr;
     }
 
     if (argerr) {
         cleanup_memory();
         pcmk__cli_help('?', CRM_EX_USAGE);
     }
 
     if (command == 'Q') {
         exit_code = crm_errno2exit(do_query(attr_name, attr_node, query_all));
     } else {
         /* @TODO We don't know whether the specified node is a Pacemaker Remote
          * node or not, so we can't set pcmk__node_attr_remote when appropriate.
          * However, it's not a big problem, because pacemaker-attrd will learn
          * and remember a node's "remoteness".
          */
         exit_code = pcmk_rc2exitc(do_update(command,
                                             pcmk__node_attr_target(attr_node),
                                             attr_name, attr_value,
                                             attr_section, attr_set,
                                             attr_dampen, attr_options));
     }
 
     cleanup_memory();
     crm_exit(exit_code);
 }
 
 /*!
  * \internal
  * \brief Submit a query request to pacemaker-attrd and wait for reply
  *
  * \param[in] name    Name of attribute to query
  * \param[in] host    Query applies to this host only (or all hosts if NULL)
  * \param[out] reply  On success, will be set to new XML tree with reply
  *
  * \return pcmk_ok on success, -errno on error
  * \note On success, caller is responsible for freeing result via free_xml(*reply)
  */
 static int
 send_attrd_query(const char *name, const char *host, xmlNode **reply)
 {
     int rc;
     crm_ipc_t *ipc;
     xmlNode *query;
 
     /* Build the query XML */
     query = create_xml_node(NULL, __FUNCTION__);
     if (query == NULL) {
         return -ENOMEM;
     }
     crm_xml_add(query, F_TYPE, T_ATTRD);
     crm_xml_add(query, F_ORIG, crm_system_name);
     crm_xml_add(query, PCMK__XA_ATTR_NODE_NAME, host);
     crm_xml_add(query, PCMK__XA_TASK, PCMK__ATTRD_CMD_QUERY);
     crm_xml_add(query, PCMK__XA_ATTR_NAME, name);
 
     /* Connect to pacemaker-attrd, send query XML and get reply */
     crm_debug("Sending query for value of %s on %s", name, (host? host : "all nodes"));
     ipc = crm_ipc_new(T_ATTRD, 0);
     if (crm_ipc_connect(ipc) == FALSE) {
         crm_perror(LOG_ERR, "Connection to cluster attribute manager failed");
         rc = -ENOTCONN;
     } else {
         rc = crm_ipc_send(ipc, query, crm_ipc_flags_none|crm_ipc_client_response, 0, reply);
         if (rc > 0) {
             rc = pcmk_ok;
         }
         crm_ipc_close(ipc);
     }
+    crm_ipc_destroy(ipc);
 
     free_xml(query);
     return(rc);
 }
 
 /*!
  * \brief Validate pacemaker-attrd's XML reply to an query
  *
  * param[in] reply      Root of reply XML tree to validate
  * param[in] attr_name  Name of attribute that was queried
  *
  * \return pcmk_ok on success,
  *         -errno on error (-ENXIO = requested attribute does not exist)
  */
 static int
 validate_attrd_reply(xmlNode *reply, const char *attr_name)
 {
     const char *reply_attr;
 
     if (reply == NULL) {
         fprintf(stderr, "Could not query value of %s: reply did not contain valid XML\n",
                 attr_name);
         return -pcmk_err_schema_validation;
     }
     crm_log_xml_trace(reply, "Reply");
 
     reply_attr = crm_element_value(reply, PCMK__XA_ATTR_NAME);
     if (reply_attr == NULL) {
         fprintf(stderr, "Could not query value of %s: attribute does not exist\n",
                 attr_name);
         return -ENXIO;
     }
 
     if (safe_str_neq(crm_element_value(reply, F_TYPE), T_ATTRD)
         || (crm_element_value(reply, PCMK__XA_ATTR_VERSION) == NULL)
         || strcmp(reply_attr, attr_name)) {
             fprintf(stderr,
                     "Could not query value of %s: reply did not contain expected identification\n",
                     attr_name);
             return -pcmk_err_schema_validation;
     }
     return pcmk_ok;
 }
 
 /*!
  * \brief Print the attribute values in a pacemaker-attrd XML query reply
  *
  * \param[in] reply     Root of XML tree with query reply
  * \param[in] attr_name Name of attribute that was queried
  *
  * \return TRUE if any values were printed
  */
 static gboolean
 print_attrd_values(xmlNode *reply, const char *attr_name)
 {
     xmlNode *child;
     const char *reply_host, *reply_value;
     gboolean have_values = FALSE;
 
     /* Iterate through reply's XML tags (a node tag for each host-value pair) */
     for (child = __xml_first_child(reply); child != NULL; child = __xml_next(child)) {
         if (safe_str_neq((const char*)child->name, XML_CIB_TAG_NODE)) {
             crm_warn("Ignoring unexpected %s tag in query reply", child->name);
         } else {
             reply_host = crm_element_value(child, PCMK__XA_ATTR_NODE_NAME);
             reply_value = crm_element_value(child, PCMK__XA_ATTR_VALUE);
 
             if (reply_host == NULL) {
                 crm_warn("Ignoring %s tag without %s attribute in query reply",
                          XML_CIB_TAG_NODE, PCMK__XA_ATTR_NODE_NAME);
             } else {
                 printf("name=\"%s\" host=\"%s\" value=\"%s\"\n",
                        attr_name, reply_host, (reply_value? reply_value : ""));
                 have_values = TRUE;
             }
         }
     }
     return have_values;
 }
 
 /*!
  * \brief Submit a query to pacemaker-attrd and print reply
  *
  * \param[in] attr_name  Name of attribute to be affected by request
  * \param[in] attr_node  Name of host to query for (or NULL for localhost)
  * \param[in] query_all  If TRUE, ignore attr_node and query all nodes instead
  *
  * \return pcmk_ok on success, -errno on error
  */
 static int
 do_query(const char *attr_name, const char *attr_node, gboolean query_all)
 {
     xmlNode *reply = NULL;
     int rc;
 
     /* Decide which node(s) to query */
     if (query_all == TRUE) {
         attr_node = NULL;
     } else {
         attr_node = pcmk__node_attr_target(attr_node);
     }
 
     /* Build and send pacemaker-attrd request, and get XML reply */
     rc = send_attrd_query(attr_name, attr_node, &reply);
     if (rc != pcmk_ok) {
         fprintf(stderr, "Could not query value of %s: %s (%d)\n", attr_name, pcmk_strerror(rc), rc);
         return rc;
     }
 
     /* Validate the XML reply */
     rc = validate_attrd_reply(reply, attr_name);
     if (rc != pcmk_ok) {
         if (reply != NULL) {
             free_xml(reply);
         }
         return rc;
     }
 
     /* Print the values from the reply */
     if (print_attrd_values(reply, attr_name) == FALSE) {
         fprintf(stderr,
                 "Could not query value of %s: reply had attribute name but no host values\n",
                 attr_name);
         free_xml(reply);
         return -pcmk_err_schema_validation;
     }
 
     return pcmk_ok;
 }
 
 static int
 do_update(char command, const char *attr_node, const char *attr_name,
           const char *attr_value, const char *attr_section,
           const char *attr_set, const char *attr_dampen, int attr_options)
 {
     int rc = pcmk__node_attr_request(NULL, command, attr_node, attr_name,
                                      attr_value, attr_section, attr_set,
                                      attr_dampen, NULL, attr_options);
     if (rc != pcmk_rc_ok) {
         fprintf(stderr, "Could not update %s=%s: %s (%d)\n",
                 attr_name, attr_value, pcmk_rc_str(rc), rc);
     }
     return rc;
 }
diff --git a/tools/crm_mon_print.c b/tools/crm_mon_print.c
index e2fd631f3f..92ddbd959e 100644
--- a/tools/crm_mon_print.c
+++ b/tools/crm_mon_print.c
@@ -1,1093 +1,1092 @@
 /*
  * Copyright 2019-2020 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU General Public License version 2
  * or later (GPLv2+) WITHOUT ANY WARRANTY.
  */
 
 #include <glib.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <time.h>
 
 #include <config.h>
 #include <crm/cib/util.h>
 #include <crm/common/curses_internal.h>
 #include <crm/common/iso8601_internal.h>
 #include <crm/common/xml.h>
 #include <crm/msg_xml.h>
 #include <crm/pengine/internal.h>
 #include <crm/pengine/pe_types.h>
 #include <crm/stonith-ng.h>
 #include <crm/common/internal.h>
 #include <crm/common/util.h>
 #include <crm/fencing/internal.h>
 
 #include "crm_mon.h"
 
 static void print_resources_heading(pcmk__output_t *out, unsigned int mon_ops);
 static void print_resources_closing(pcmk__output_t *out, unsigned int mon_ops);
 static int print_resources(pcmk__output_t *out, pe_working_set_t *data_set,
                            unsigned int print_opts, unsigned int mon_ops, gboolean brief_output,
                            gboolean print_summary, GListPtr only_show, gboolean print_spacer);
 static int print_rsc_history(pcmk__output_t *out, pe_working_set_t *data_set,
                              pe_node_t *node, xmlNode *rsc_entry, unsigned int mon_ops,
                              GListPtr op_list);
 static int print_node_history(pcmk__output_t *out, pe_working_set_t *data_set,
                               pe_node_t *node, xmlNode *node_state, gboolean operations,
                               unsigned int mon_ops, GListPtr only_show);
 static gboolean add_extra_info(pcmk__output_t *out, pe_node_t * node, GListPtr rsc_list,
                                const char *attrname, int *expected_score);
 static void print_node_attribute(gpointer name, gpointer user_data);
 static int print_node_summary(pcmk__output_t *out, pe_working_set_t * data_set,
                               gboolean operations, unsigned int mon_ops,
                               GListPtr only_snow, gboolean print_spacer);
 static int print_cluster_tickets(pcmk__output_t *out, pe_working_set_t * data_set,
                                  gboolean print_spacer);
 static int print_neg_locations(pcmk__output_t *out, pe_working_set_t *data_set,
                                unsigned int mon_ops, const char *prefix,
                                gboolean print_spacer);
 static int print_node_attributes(pcmk__output_t *out, pe_working_set_t *data_set,
                                  unsigned int mon_ops, GListPtr only_show,
                                  gboolean print_spacer);
 static int print_failed_actions(pcmk__output_t *out, pe_working_set_t *data_set,
                                 GListPtr only_show, gboolean print_spacer);
 
 static GListPtr
 build_uname_list(pe_working_set_t *data_set, const char *s) {
     GListPtr unames = NULL;
 
     if (s == NULL || strcmp(s, "*") == 0) {
         /* Nothing was given so return a list of all node names.  Or, '*' was
          * given.  This would normally fall into the pe__unames_with_tag branch
          * where it will return an empty list.  Catch it here instead.
          */
         unames = g_list_prepend(unames, strdup("*"));
     } else {
         pe_node_t *node = pe_find_node(data_set->nodes, s);
 
         if (node) {
             /* The given string was a valid uname for a node.  Return a
              * singleton list containing just that uname.
              */
             unames = g_list_prepend(unames, strdup(s));
         } else {
             /* The given string was not a valid uname.  It's either a tag or
              * it's a typo or something.  In the first case, we'll return a
              * list of all the unames of the nodes with the given tag.  In the
              * second case, we'll return a NULL pointer and nothing will
              * get displayed.
              */
             unames = pe__unames_with_tag(data_set, s);
         }
     }
 
     return unames;
 }
 
 /*!
  * \internal
  * \brief Print resources section heading appropriate to options
  *
  * \param[in] out     The output functions structure.
  * \param[in] mon_ops Bitmask of mon_op_*.
  */
 static void
 print_resources_heading(pcmk__output_t *out, unsigned int mon_ops)
 {
     const char *heading;
 
     if (is_set(mon_ops, mon_op_group_by_node)) {
 
         /* Active resources have already been printed by node */
         heading = is_set(mon_ops, mon_op_inactive_resources) ? "Inactive Resources" : NULL;
 
     } else if (is_set(mon_ops, mon_op_inactive_resources)) {
         heading = "Full List of Resources";
 
     } else {
         heading = "Active Resources";
     }
 
     /* Print section heading */
     out->begin_list(out, NULL, NULL, "%s", heading);
 }
 
 /*!
  * \internal
  * \brief Print whatever resource section closing is appropriate
  *
  * \param[in] out     The output functions structure.
  * \param[in] mon_ops Bitmask of mon_op_*.
  */
 static void
 print_resources_closing(pcmk__output_t *out, unsigned int mon_ops)
 {
     const char *heading;
 
     /* What type of resources we did or did not display */
     if (is_set(mon_ops, mon_op_group_by_node)) {
         heading = "inactive ";
     } else if (is_set(mon_ops, mon_op_inactive_resources)) {
         heading = "";
     } else {
         heading = "active ";
     }
 
     out->list_item(out, NULL, "No %sresources", heading);
 }
 
 /*!
  * \internal
  * \brief Print whatever resource section(s) are appropriate
  *
  * \param[in] out           The output functions structure.
  * \param[in] data_set      Cluster state to display.
  * \param[in] print_opts    Bitmask of pe_print_*.
  * \param[in] mon_ops       Bitmask of mon_op_*.
  * \param[in] brief_output  Whether to display full or brief output.
  * \param[in] print_summary Whether to display a failure summary.
  */
 static int
 print_resources(pcmk__output_t *out, pe_working_set_t *data_set,
                 unsigned int print_opts, unsigned int mon_ops, gboolean brief_output,
                 gboolean print_summary, GListPtr only_show, gboolean print_spacer)
 {
     GListPtr rsc_iter;
     int rc = pcmk_rc_no_output;
 
     /* If we already showed active resources by node, and
      * we're not showing inactive resources, we have nothing to do
      */
     if (is_set(mon_ops, mon_op_group_by_node) && is_not_set(mon_ops, mon_op_inactive_resources)) {
         return rc;
     }
 
     /* Add a blank line between this section and the one before it. */
     if (print_spacer) {
         out->info(out, "%s", "");
     }
 
     print_resources_heading(out, mon_ops);
 
     /* If we haven't already printed resources grouped by node,
      * and brief output was requested, print resource summary */
     if (brief_output && is_not_set(mon_ops, mon_op_group_by_node)) {
         pe__rscs_brief_output(out, data_set->resources, print_opts,
                               is_set(mon_ops, mon_op_inactive_resources));
     }
 
     /* For each resource, display it if appropriate */
     for (rsc_iter = data_set->resources; rsc_iter != NULL; rsc_iter = rsc_iter->next) {
         pe_resource_t *rsc = (pe_resource_t *) rsc_iter->data;
 
         /* Complex resources may have some sub-resources active and some inactive */
         gboolean is_active = rsc->fns->active(rsc, TRUE);
         gboolean partially_active = rsc->fns->active(rsc, FALSE);
 
         /* Skip inactive orphans (deleted but still in CIB) */
         if (is_set(rsc->flags, pe_rsc_orphan) && !is_active) {
             continue;
 
         /* Skip active resources if we already displayed them by node */
         } else if (is_set(mon_ops, mon_op_group_by_node)) {
             if (is_active) {
                 continue;
             }
 
         /* Skip primitives already counted in a brief summary */
         } else if (brief_output && (rsc->variant == pe_native)) {
             continue;
 
         /* Skip resources that aren't at least partially active,
          * unless we're displaying inactive resources
          */
         } else if (!partially_active && is_not_set(mon_ops, mon_op_inactive_resources)) {
             continue;
-        }
 
-        if (!pe__rsc_running_on_any_node_in_list(rsc, only_show)) {
+        } else if (partially_active && !pe__rsc_running_on_any_node_in_list(rsc, only_show)) {
             continue;
         }
 
         /* Print this resource */
         rc = pcmk_rc_ok;
         out->message(out, crm_map_element_name(rsc->xml), print_opts, rsc, only_show);
     }
 
     if (print_summary && rc != pcmk_rc_ok) {
         print_resources_closing(out, mon_ops);
     }
 
     out->end_list(out);
     return pcmk_rc_ok;
 }
 
 static int
 failure_count(pe_working_set_t *data_set, pe_node_t *node, pe_resource_t *rsc, time_t *last_failure) {
     return rsc ? pe_get_failcount(node, rsc, last_failure, pe_fc_default,
                                   NULL, data_set)
                : 0;
 }
 
 static GListPtr
 get_operation_list(xmlNode *rsc_entry) {
     GListPtr op_list = NULL;
     xmlNode *rsc_op = NULL;
 
     for (rsc_op = __xml_first_child_element(rsc_entry); rsc_op != NULL;
          rsc_op = __xml_next_element(rsc_op)) {
         if (crm_str_eq((const char *) rsc_op->name, XML_LRM_TAG_RSC_OP, TRUE)) {
             op_list = g_list_append(op_list, rsc_op);
         }
     }
 
     op_list = g_list_sort(op_list, sort_op_by_callid);
     return op_list;
 }
 
 /*!
  * \internal
  * \brief Print resource operation/failure history
  *
  * \param[in] out       The output functions structure.
  * \param[in] data_set  Cluster state to display.
  * \param[in] node      Node that ran this resource.
  * \param[in] rsc_entry Root of XML tree describing resource status.
  * \param[in] mon_ops   Bitmask of mon_op_*.
  * \param[in] op_list   A list of operations to print.
  */
 static int
 print_rsc_history(pcmk__output_t *out, pe_working_set_t *data_set, pe_node_t *node,
                   xmlNode *rsc_entry, unsigned int mon_ops, GListPtr op_list)
 {
     GListPtr gIter = NULL;
     int rc = pcmk_rc_no_output;
     const char *rsc_id = crm_element_value(rsc_entry, XML_ATTR_ID);
     pe_resource_t *rsc = pe_find_resource(data_set->resources, rsc_id);
 
     /* Print each operation */
     for (gIter = op_list; gIter != NULL; gIter = gIter->next) {
         xmlNode *xml_op = (xmlNode *) gIter->data;
         const char *task = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
         const char *interval_ms_s = crm_element_value(xml_op,
                                                       XML_LRM_ATTR_INTERVAL_MS);
         const char *op_rc = crm_element_value(xml_op, XML_LRM_ATTR_RC);
         int op_rc_i = crm_parse_int(op_rc, "0");
 
         /* Display 0-interval monitors as "probe" */
         if (safe_str_eq(task, CRMD_ACTION_STATUS)
             && ((interval_ms_s == NULL) || safe_str_eq(interval_ms_s, "0"))) {
             task = "probe";
         }
 
         /* Ignore notifies and some probes */
         if (safe_str_eq(task, CRMD_ACTION_NOTIFY) || (safe_str_eq(task, "probe") && (op_rc_i == 7))) {
             continue;
         }
 
         /* If this is the first printed operation, print heading for resource */
         if (rc == pcmk_rc_no_output) {
             time_t last_failure = 0;
             int failcount = failure_count(data_set, node, rsc, &last_failure);
 
             out->message(out, "resource-history", rsc, rsc_id, TRUE, failcount, last_failure, TRUE);
             rc = pcmk_rc_ok;
         }
 
         /* Print the operation */
         out->message(out, "op-history", xml_op, task, interval_ms_s,
                      op_rc_i, is_set(mon_ops, mon_op_print_timing));
     }
 
     /* Free the list we created (no need to free the individual items) */
     g_list_free(op_list);
 
     /* If we printed anything, close the resource */
     if (rc == pcmk_rc_ok) {
         out->end_list(out);
     }
 
     return rc;
 }
 
 /*!
  * \internal
  * \brief Print node operation/failure history
  *
  * \param[in] out        The output functions structure.
  * \param[in] data_set   Cluster state to display.
  * \param[in] node_state Root of XML tree describing node status.
  * \param[in] operations Whether to print operations or just failcounts.
  * \param[in] mon_ops    Bitmask of mon_op_*.
  */
 static int
 print_node_history(pcmk__output_t *out, pe_working_set_t *data_set,
                    pe_node_t *node, xmlNode *node_state, gboolean operations,
                    unsigned int mon_ops, GListPtr only_show)
 {
     xmlNode *lrm_rsc = NULL;
     xmlNode *rsc_entry = NULL;
     int rc = pcmk_rc_no_output;
 
     lrm_rsc = find_xml_node(node_state, XML_CIB_TAG_LRM, FALSE);
     lrm_rsc = find_xml_node(lrm_rsc, XML_LRM_TAG_RESOURCES, FALSE);
 
     /* Print history of each of the node's resources */
     for (rsc_entry = __xml_first_child_element(lrm_rsc); rsc_entry != NULL;
          rsc_entry = __xml_next_element(rsc_entry)) {
 
         if (!crm_str_eq((const char *)rsc_entry->name, XML_LRM_TAG_RESOURCE, TRUE)) {
             continue;
         }
 
         if (operations == FALSE) {
             const char *rsc_id = crm_element_value(rsc_entry, XML_ATTR_ID);
             pe_resource_t *rsc = pe_find_resource(data_set->resources, rsc_id);
             time_t last_failure = 0;
             int failcount = failure_count(data_set, node, rsc, &last_failure);
 
             if (failcount > 0) {
                 if (rc == pcmk_rc_no_output) {
                     rc = pcmk_rc_ok;
                     out->message(out, "node", node, get_resource_display_options(mon_ops),
                                  FALSE, NULL, is_set(mon_ops, mon_op_print_clone_detail),
                                  is_set(mon_ops, mon_op_print_brief), is_set(mon_ops, mon_op_group_by_node),
                                  only_show);
                 }
 
                 out->message(out, "resource-history", rsc, rsc_id, FALSE,
                              failcount, last_failure, FALSE);
             }
         } else {
             GListPtr op_list = get_operation_list(rsc_entry);
 
             if (rc == pcmk_rc_no_output) {
                 rc = pcmk_rc_ok;
                 out->message(out, "node", node, get_resource_display_options(mon_ops),
                              FALSE, NULL, is_set(mon_ops, mon_op_print_clone_detail),
                              is_set(mon_ops, mon_op_print_brief), is_set(mon_ops, mon_op_group_by_node),
                              only_show);
             }
 
             if (op_list != NULL) {
                 print_rsc_history(out, data_set, node, rsc_entry, mon_ops, op_list);
             }
         }
     }
 
     if (rc == pcmk_rc_ok) {
         out->end_list(out);
     }
 
     return rc;
 }
 
 /*!
  * \internal
  * \brief Determine whether extended information about an attribute should be added.
  *
  * \param[in]  out            The output functions structure.
  * \param[in]  node           Node that ran this resource.
  * \param[in]  rsc_list       The list of resources for this node.
  * \param[in]  attrname       The attribute to find.
  * \param[out] expected_score The expected value for this attribute.
  *
  * \return TRUE if extended information should be printed, FALSE otherwise
  * \note Currently, extended information is only supported for ping/pingd
  *       resources, for which a message will be printed if connectivity is lost
  *       or degraded.
  */
 static gboolean
 add_extra_info(pcmk__output_t *out, pe_node_t *node, GListPtr rsc_list,
                const char *attrname, int *expected_score)
 {
     GListPtr gIter = NULL;
 
     for (gIter = rsc_list; gIter != NULL; gIter = gIter->next) {
         pe_resource_t *rsc = (pe_resource_t *) gIter->data;
         const char *type = g_hash_table_lookup(rsc->meta, "type");
         const char *name = NULL;
 
         if (rsc->children != NULL) {
             if (add_extra_info(out, node, rsc->children, attrname, expected_score)) {
                 return TRUE;
             }
         }
 
         if (safe_str_neq(type, "ping") && safe_str_neq(type, "pingd")) {
             return FALSE;
         }
 
         name = g_hash_table_lookup(rsc->parameters, "name");
 
         if (name == NULL) {
             name = "pingd";
         }
 
         /* To identify the resource with the attribute name. */
         if (safe_str_eq(name, attrname)) {
             int host_list_num = 0;
             /* int value = crm_parse_int(attrvalue, "0"); */
             const char *hosts = g_hash_table_lookup(rsc->parameters, "host_list");
             const char *multiplier = g_hash_table_lookup(rsc->parameters, "multiplier");
 
             if (hosts) {
                 char **host_list = g_strsplit(hosts, " ", 0);
                 host_list_num = g_strv_length(host_list);
                 g_strfreev(host_list);
             }
 
             /* pingd multiplier is the same as the default value. */
             *expected_score = host_list_num * crm_parse_int(multiplier, "1");
 
             return TRUE;
         }
     }
     return FALSE;
 }
 
 /* structure for passing multiple user data to g_list_foreach() */
 struct mon_attr_data {
     pcmk__output_t *out;
     pe_node_t *node;
 };
 
 static void
 print_node_attribute(gpointer name, gpointer user_data)
 {
     const char *value = NULL;
     int expected_score = 0;
     gboolean add_extra = FALSE;
     struct mon_attr_data *data = (struct mon_attr_data *) user_data;
 
     value = pe_node_attribute_raw(data->node, name);
 
     add_extra = add_extra_info(data->out, data->node, data->node->details->running_rsc,
                                name, &expected_score);
 
     /* Print attribute name and value */
     data->out->message(data->out, "node-attribute", name, value, add_extra,
                        expected_score);
 }
 
 /*!
  * \internal
  * \brief Print history for all nodes.
  *
  * \param[in] out        The output functions structure.
  * \param[in] data_set   Cluster state to display.
  * \param[in] operations Whether to print operations or just failcounts.
  * \param[in] mon_ops    Bitmask of mon_op_*.
  */
 static int
 print_node_summary(pcmk__output_t *out, pe_working_set_t * data_set,
                    gboolean operations, unsigned int mon_ops, GListPtr only_show,
                    gboolean print_spacer)
 {
     xmlNode *node_state = NULL;
     xmlNode *cib_status = get_object_root(XML_CIB_TAG_STATUS, data_set->input);
     int rc = pcmk_rc_no_output;
 
     if (xmlChildElementCount(cib_status) == 0) {
         return rc;
     }
 
     /* Print each node in the CIB status */
     for (node_state = __xml_first_child_element(cib_status); node_state != NULL;
          node_state = __xml_next_element(node_state)) {
         pe_node_t *node;
 
         if (!crm_str_eq((const char *)node_state->name, XML_CIB_TAG_STATE, TRUE)) {
             continue;
         }
 
         node = pe_find_node_id(data_set->nodes, ID(node_state));
 
         if (!node || !node->details || !node->details->online) {
             continue;
         }
 
         if (!pcmk__str_in_list(only_show, node->details->uname)) {
             continue;
         }
 
         if (rc == pcmk_rc_no_output) {
             /* Add a blank line between this section and the one before it. */
             if (print_spacer) {
                 out->info(out, "%s", "");
             }
 
             if (operations) {
                 out->begin_list(out, NULL, NULL, "Operations");
             } else {
                 out->begin_list(out, NULL, NULL, "Migration Summary");
             }
 
             rc = pcmk_rc_ok;
         }
 
         print_node_history(out, data_set, node, node_state, operations, mon_ops,
                            only_show);
     }
 
     if (rc == pcmk_rc_ok) {
         out->end_list(out);
     }
 
     return rc;
 }
 
 /*!
  * \internal
  * \brief Print all tickets.
  *
  * \param[in] out      The output functions structure.
  * \param[in] data_set Cluster state to display.
  */
 static int
 print_cluster_tickets(pcmk__output_t *out, pe_working_set_t * data_set,
                       gboolean print_spacer)
 {
     GHashTableIter iter;
     gpointer key, value;
 
     if (g_hash_table_size(data_set->tickets) == 0) {
         return pcmk_rc_no_output;
     }
 
     /* Add a blank line between this section and the one before it. */
     if (print_spacer) {
         out->info(out, "%s", "");
     }
 
     /* Print section heading */
     out->begin_list(out, NULL, NULL, "Tickets");
 
     /* Print each ticket */
     g_hash_table_iter_init(&iter, data_set->tickets);
     while (g_hash_table_iter_next(&iter, &key, &value)) {
         pe_ticket_t *ticket = (pe_ticket_t *) value;
         out->message(out, "ticket", ticket);
     }
 
     /* Close section */
     out->end_list(out);
     return pcmk_rc_ok;
 }
 
 /*!
  * \internal
  * \brief Print section for negative location constraints
  *
  * \param[in] out      The output functions structure.
  * \param[in] data_set Cluster state to display.
  * \param[in] mon_ops  Bitmask of mon_op_*.
  * \param[in] prefix   ID prefix to filter results by.
  */
 static int
 print_neg_locations(pcmk__output_t *out, pe_working_set_t *data_set,
                     unsigned int mon_ops, const char *prefix, gboolean print_spacer)
 {
     GListPtr gIter, gIter2;
     int rc = pcmk_rc_no_output;
 
     /* Print each ban */
     for (gIter = data_set->placement_constraints; gIter != NULL; gIter = gIter->next) {
         pe__location_t *location = gIter->data;
         if (prefix != NULL && !g_str_has_prefix(location->id, prefix))
             continue;
         for (gIter2 = location->node_list_rh; gIter2 != NULL; gIter2 = gIter2->next) {
             pe_node_t *node = (pe_node_t *) gIter2->data;
 
             if (node->weight < 0) {
                 if (rc == pcmk_rc_no_output) {
                     /* Add a blank line between this section and the one before it. */
                     if (print_spacer) {
                         out->info(out, "%s", "");
                     }
 
                     rc = pcmk_rc_ok;
                     out->begin_list(out, NULL, NULL, "Negative Location Constraints");
                 }
 
                 out->message(out, "ban", node, location, is_set(mon_ops, mon_op_print_clone_detail));
             }
         }
     }
 
     if (rc == pcmk_rc_ok) {
         out->end_list(out);
     }
 
     return rc;
 }
 
 /*!
  * \internal
  * \brief Print node attributes section
  *
  * \param[in] out      The output functions structure.
  * \param[in] data_set Cluster state to display.
  * \param[in] mon_ops  Bitmask of mon_op_*.
  */
 static int
 print_node_attributes(pcmk__output_t *out, pe_working_set_t *data_set,
                       unsigned int mon_ops, GListPtr only_show,
                       gboolean print_spacer)
 {
     GListPtr gIter = NULL;
     int rc = pcmk_rc_no_output;
 
     /* Unpack all resource parameters (it would be more efficient to do this
      * only when needed for the first time in add_extra_info())
      */
     for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
         crm_mon_get_parameters(gIter->data, data_set);
     }
 
     /* Display each node's attributes */
     for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
         struct mon_attr_data data;
 
         data.out = out;
         data.node = (pe_node_t *) gIter->data;
 
         if (data.node && data.node->details && data.node->details->online) {
             GList *attr_list = NULL;
             GHashTableIter iter;
             gpointer key, value;
 
             g_hash_table_iter_init(&iter, data.node->details->attrs);
             while (g_hash_table_iter_next (&iter, &key, &value)) {
                 attr_list = append_attr_list(attr_list, key);
             }
 
             if (attr_list == NULL) {
                 continue;
             }
 
             if (!pcmk__str_in_list(only_show, data.node->details->uname)) {
                 continue;
             }
 
             if (rc == pcmk_rc_no_output) {
                 /* Add a blank line between this section and the one before it. */
                 if (print_spacer) {
                     out->info(out, "%s", "");
                 }
 
                 rc = pcmk_rc_ok;
                 out->begin_list(out, NULL, NULL, "Node Attributes");
             }
 
             out->message(out, "node", data.node, get_resource_display_options(mon_ops),
                          FALSE, NULL, is_set(mon_ops, mon_op_print_clone_detail),
                          is_set(mon_ops, mon_op_print_brief), is_set(mon_ops, mon_op_group_by_node),
                          only_show);
             g_list_foreach(attr_list, print_node_attribute, &data);
             g_list_free(attr_list);
             out->end_list(out);
         }
     }
 
     /* Print section footer */
     if (rc == pcmk_rc_ok) {
         out->end_list(out);
     }
 
     return rc;
 }
 
 /*!
  * \internal
  * \brief Print a section for failed actions
  *
  * \param[in] out      The output functions structure.
  * \param[in] data_set Cluster state to display.
  */
 static int
 print_failed_actions(pcmk__output_t *out, pe_working_set_t *data_set,
                      GListPtr only_show, gboolean print_spacer)
 {
     xmlNode *xml_op = NULL;
     int rc = pcmk_rc_no_output;
 
     if (xmlChildElementCount(data_set->failed) == 0) {
         return rc;
     }
 
     for (xml_op = __xml_first_child(data_set->failed); xml_op != NULL;
          xml_op = __xml_next(xml_op)) {
         if (!pcmk__str_in_list(only_show, crm_element_value(xml_op, XML_ATTR_UNAME))) {
             continue;
         }
 
         if (rc == pcmk_rc_no_output) {
             /* Add a blank line between this section and the one before it. */
             if (print_spacer) {
                 out->info(out, "%s", "");
             }
 
             rc = pcmk_rc_ok;
             out->begin_list(out, NULL, NULL, "Failed Resource Actions");
         }
 
         out->message(out, "failed-action", xml_op);
     }
 
     if (rc == pcmk_rc_ok) {
         out->end_list(out);
     }
 
     return rc;
 }
 
 /*!
  * \internal
  * \brief Top-level printing function for text/curses output.
  *
  * \param[in] out             The output functions structure.
  * \param[in] data_set        Cluster state to display.
  * \param[in] stonith_history List of stonith actions.
  * \param[in] mon_ops         Bitmask of mon_op_*.
  * \param[in] show            Bitmask of mon_show_*.
  * \param[in] prefix          ID prefix to filter results by.
  */
 void
 print_status(pcmk__output_t *out, pe_working_set_t *data_set,
              stonith_history_t *stonith_history, unsigned int mon_ops,
              unsigned int show, char *prefix, char *only_show)
 {
     GListPtr unames = NULL;
 
     unsigned int print_opts = get_resource_display_options(mon_ops);
     int rc = pcmk_rc_no_output;
     int x;
 
     x = out->message(out, "cluster-summary", data_set,
                      is_set(mon_ops, mon_op_print_clone_detail),
                      is_set(show, mon_show_stack),
                      is_set(show, mon_show_dc),
                      is_set(show, mon_show_times),
                      is_set(show, mon_show_counts),
                      is_set(show, mon_show_options));
 
     if (x == pcmk_rc_ok) {
         rc = pcmk_rc_ok;
     }
 
     unames = build_uname_list(data_set, only_show);
 
     if (is_set(show, mon_show_nodes) && unames) {
         if (rc == pcmk_rc_ok) {
             out->info(out, "%s", "");
         }
 
         x = out->message(out, "node-list", data_set->nodes, unames, print_opts,
                          is_set(mon_ops, mon_op_print_clone_detail),
                          is_set(mon_ops, mon_op_print_brief),
                          is_set(mon_ops, mon_op_group_by_node));
 
         if (x == pcmk_rc_ok) {
             rc = pcmk_rc_ok;
         }
     }
 
     /* Print resources section, if needed */
     if (is_set(show, mon_show_resources)) {
         x = print_resources(out, data_set, print_opts, mon_ops,
                             is_set(mon_ops, mon_op_print_brief), TRUE,
                             unames, rc == pcmk_rc_ok);
         if (x == pcmk_rc_ok) {
             rc = pcmk_rc_ok;
         }
     }
 
     /* print Node Attributes section if requested */
     if (is_set(show, mon_show_attributes)) {
         x =  print_node_attributes(out, data_set, mon_ops, unames,
                                    rc == pcmk_rc_ok);
         if (x == pcmk_rc_ok) {
             rc = pcmk_rc_ok;
         }
     }
 
     /* If requested, print resource operations (which includes failcounts)
      * or just failcounts
      */
     if (is_set(show, mon_show_operations) || is_set(show, mon_show_failcounts)) {
         x = print_node_summary(out, data_set, is_set(show, mon_show_operations),
                                mon_ops, unames, rc == pcmk_rc_ok);
         if (x == pcmk_rc_ok) {
             rc = pcmk_rc_ok;
         }
     }
 
     /* If there were any failed actions, print them */
     if (is_set(show, mon_show_failures) && xml_has_children(data_set->failed)) {
         x = print_failed_actions(out, data_set, unames, rc == pcmk_rc_ok);
         if (x == pcmk_rc_ok) {
             rc = pcmk_rc_ok;
         }
     }
 
     /* Print failed stonith actions */
     if (is_set(show, mon_show_fence_failed) && is_set(mon_ops, mon_op_fence_history)) {
         for (stonith_history_t *hp = stonith_history; hp; hp = hp->next) {
             if (hp->state == st_failed) {
                 x = out->message(out, "failed-fencing-history", hp, unames,
                                  is_set(mon_ops, mon_op_fence_full_history),
                                  rc == pcmk_rc_ok);
 
                 if (x == pcmk_rc_ok) {
                     rc = pcmk_rc_ok;
                 }
 
                 break;
             }
         }
     }
 
     /* Print tickets if requested */
     if (is_set(show, mon_show_tickets)) {
         x = print_cluster_tickets(out, data_set, rc == pcmk_rc_ok);
         if (x == pcmk_rc_ok) {
             rc = pcmk_rc_ok;
         }
     }
 
     /* Print negative location constraints if requested */
     if (is_set(show, mon_show_bans)) {
         x = print_neg_locations(out, data_set, mon_ops, prefix, rc == pcmk_rc_ok);
         if (x == pcmk_rc_ok) {
             rc = pcmk_rc_ok;
         }
     }
 
     /* Print stonith history */
     if (is_set(mon_ops, mon_op_fence_history)) {
         if (is_set(show, mon_show_fence_worked)) {
             for (stonith_history_t *hp = stonith_history; hp; hp = hp->next) {
                 if (hp->state != st_failed) {
                     x = out->message(out, "fencing-history", hp, unames,
                                      is_set(mon_ops, mon_op_fence_full_history),
                                      rc == pcmk_rc_ok);
 
                     if (x == pcmk_rc_ok) {
                         rc = pcmk_rc_ok;
                     }
 
                     break;
                 }
             }
         } else if (is_set(show, mon_show_fence_pending)) {
             for (stonith_history_t *hp = stonith_history; hp; hp = hp->next) {
                 if (hp->state != st_failed && hp->state != st_done) {
                     x = out->message(out, "pending-fencing-actions", hp, unames,
                                      is_set(mon_ops, mon_op_fence_full_history),
                                      rc == pcmk_rc_ok);
 
                     if (x == pcmk_rc_ok) {
                         rc = pcmk_rc_ok;
                     }
 
                     break;
                 }
             }
         }
     }
 
     g_list_free_full(unames, free);
 }
 
 /*!
  * \internal
  * \brief Top-level printing function for XML output.
  *
  * \param[in] out             The output functions structure.
  * \param[in] data_set        Cluster state to display.
  * \param[in] stonith_history List of stonith actions.
  * \param[in] mon_ops         Bitmask of mon_op_*.
  * \param[in] show            Bitmask of mon_show_*.
  * \param[in] prefix          ID prefix to filter results by.
  */
 void
 print_xml_status(pcmk__output_t *out, pe_working_set_t *data_set,
                  crm_exit_t history_rc, stonith_history_t *stonith_history,
                  unsigned int mon_ops, unsigned int show, char *prefix,
                  char *only_show)
 {
     GListPtr unames = NULL;
     unsigned int print_opts = get_resource_display_options(mon_ops);
 
     out->message(out, "cluster-summary", data_set,
                  is_set(mon_ops, mon_op_print_clone_detail),
                  is_set(show, mon_show_stack),
                  is_set(show, mon_show_dc),
                  is_set(show, mon_show_times),
                  is_set(show, mon_show_counts),
                  is_set(show, mon_show_options));
 
     unames = build_uname_list(data_set, only_show);
 
     /*** NODES ***/
     if (is_set(show, mon_show_nodes)) {
         out->message(out, "node-list", data_set->nodes, unames, print_opts,
                      is_set(mon_ops, mon_op_print_clone_detail),
                      is_set(mon_ops, mon_op_print_brief),
                      is_set(mon_ops, mon_op_group_by_node));
     }
 
     /* Print resources section, if needed */
     if (is_set(show, mon_show_resources)) {
         print_resources(out, data_set, print_opts, mon_ops, FALSE, FALSE, unames, FALSE);
     }
 
     /* print Node Attributes section if requested */
     if (is_set(show, mon_show_attributes)) {
         print_node_attributes(out, data_set, mon_ops, unames, FALSE);
     }
 
     /* If requested, print resource operations (which includes failcounts)
      * or just failcounts
      */
     if (is_set(show, mon_show_operations) || is_set(show, mon_show_failcounts)) {
         print_node_summary(out, data_set, is_set(show, mon_show_operations),
                            mon_ops, unames, FALSE);
     }
 
     /* If there were any failed actions, print them */
     if (is_set(show, mon_show_failures) && xml_has_children(data_set->failed)) {
         print_failed_actions(out, data_set, unames, FALSE);
     }
 
     /* Print stonith history */
     if (is_set(show, mon_show_fencing_all) && is_set(mon_ops, mon_op_fence_history)) {
         out->message(out, "full-fencing-history", history_rc, stonith_history,
                      unames, is_set(mon_ops, mon_op_fence_full_history), FALSE);
     }
 
     /* Print tickets if requested */
     if (is_set(show, mon_show_tickets)) {
         print_cluster_tickets(out, data_set, FALSE);
     }
 
     /* Print negative location constraints if requested */
     if (is_set(show, mon_show_bans)) {
         print_neg_locations(out, data_set, mon_ops, prefix, FALSE);
     }
 
     g_list_free_full(unames, free);
 }
 
 /*!
  * \internal
  * \brief Top-level printing function for HTML output.
  *
  * \param[in] out             The output functions structure.
  * \param[in] data_set        Cluster state to display.
  * \param[in] stonith_history List of stonith actions.
  * \param[in] mon_ops         Bitmask of mon_op_*.
  * \param[in] show            Bitmask of mon_show_*.
  * \param[in] prefix          ID prefix to filter results by.
  */
 int
 print_html_status(pcmk__output_t *out, pe_working_set_t *data_set,
                   stonith_history_t *stonith_history, unsigned int mon_ops,
                   unsigned int show, char *prefix, char *only_show)
 {
     GListPtr unames = NULL;
     unsigned int print_opts = get_resource_display_options(mon_ops);
 
     out->message(out, "cluster-summary", data_set,
                  is_set(mon_ops, mon_op_print_clone_detail),
                  is_set(show, mon_show_stack),
                  is_set(show, mon_show_dc),
                  is_set(show, mon_show_times),
                  is_set(show, mon_show_counts),
                  is_set(show, mon_show_options));
 
     unames = build_uname_list(data_set, only_show);
 
     /*** NODE LIST ***/
     if (is_set(show, mon_show_nodes) && unames) {
         out->message(out, "node-list", data_set->nodes, unames, print_opts,
                      is_set(mon_ops, mon_op_print_clone_detail),
                      is_set(mon_ops, mon_op_print_brief),
                      is_set(mon_ops, mon_op_group_by_node));
     }
 
     /* Print resources section, if needed */
     if (is_set(show, mon_show_resources)) {
         print_resources(out, data_set, print_opts, mon_ops,
                         is_set(mon_ops, mon_op_print_brief), TRUE, unames, FALSE);
     }
 
     /* print Node Attributes section if requested */
     if (is_set(show, mon_show_attributes)) {
         print_node_attributes(out, data_set, mon_ops, unames, FALSE);
     }
 
     /* If requested, print resource operations (which includes failcounts)
      * or just failcounts
      */
     if (is_set(show, mon_show_operations) || is_set(show, mon_show_failcounts)) {
         print_node_summary(out, data_set, is_set(show, mon_show_operations),
                            mon_ops, unames, FALSE);
     }
 
     /* If there were any failed actions, print them */
     if (is_set(show, mon_show_failures) && xml_has_children(data_set->failed)) {
         print_failed_actions(out, data_set, unames, FALSE);
     }
 
     /* Print failed stonith actions */
     if (is_set(show, mon_show_fence_failed) && is_set(mon_ops, mon_op_fence_history)) {
         for (stonith_history_t *hp = stonith_history; hp; hp = hp->next) {
             if (hp->state == st_failed) {
                 out->message(out, "failed-fencing-history", hp, unames,
                              is_set(mon_ops, mon_op_fence_full_history), FALSE);
                 break;
             }
         }
     }
 
     /* Print stonith history */
     if (is_set(mon_ops, mon_op_fence_history)) {
         if (is_set(show, mon_show_fence_worked)) {
             for (stonith_history_t *hp = stonith_history; hp; hp = hp->next) {
                 if (hp->state != st_failed) {
                     out->message(out, "fencing-history", hp, unames,
                                  is_set(mon_ops, mon_op_fence_full_history),
                                  FALSE);
                     break;
                 }
             }
         } else if (is_set(show, mon_show_fence_pending)) {
             for (stonith_history_t *hp = stonith_history; hp; hp = hp->next) {
                 if (hp->state != st_failed && hp->state != st_done) {
                     out->message(out, "pending-fencing-actions", hp, unames,
                                  is_set(mon_ops, mon_op_fence_full_history),
                                  FALSE);
                     break;
                 }
             }
         }
     }
 
     /* Print tickets if requested */
     if (is_set(show, mon_show_tickets)) {
         print_cluster_tickets(out, data_set, FALSE);
     }
 
     /* Print negative location constraints if requested */
     if (is_set(show, mon_show_bans)) {
         print_neg_locations(out, data_set, mon_ops, prefix, FALSE);
     }
 
     g_list_free_full(unames, free);
     return 0;
 }
diff --git a/tools/crm_resource.c b/tools/crm_resource.c
index ab5c45ca2a..52ed9cb232 100644
--- a/tools/crm_resource.c
+++ b/tools/crm_resource.c
@@ -1,1564 +1,1564 @@
 /*
  * Copyright 2004-2020 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU General Public License version 2
  * or later (GPLv2+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_resource.h>
 #include <pacemaker-internal.h>
 
 #include <sys/param.h>
 
 #include <crm/crm.h>
 #include <crm/stonith-ng.h>
 
 #include <stdio.h>
 #include <sys/types.h>
 #include <unistd.h>
 
 #include <stdlib.h>
 #include <errno.h>
 #include <fcntl.h>
 #include <libgen.h>
 #include <time.h>
 
 bool BE_QUIET = FALSE;
 bool scope_master = FALSE;
 int cib_options = cib_sync_call;
 
 static GMainLoop *mainloop = NULL;
 
 // Things that should be cleaned up on exit
 static cib_t *cib_conn = NULL;
 static pcmk_controld_api_t *controld_api = NULL;
 static pe_working_set_t *data_set = NULL;
 
 #define MESSAGE_TIMEOUT_S 60
 
 // Clean up and exit
 static crm_exit_t
 bye(crm_exit_t exit_code)
 {
     static bool crm_resource_shutdown_flag = false;
 
     if (crm_resource_shutdown_flag) {
         // Allow usage like "return bye(...);"
         return exit_code;
     }
     crm_resource_shutdown_flag = true;
 
     if (cib_conn != NULL) {
         cib_t *save_cib_conn = cib_conn;
 
         cib_conn = NULL;
         save_cib_conn->cmds->signoff(save_cib_conn);
         cib_delete(save_cib_conn);
     }
     if (controld_api != NULL) {
         pcmk_controld_api_t *save_controld_api = controld_api;
 
         controld_api = NULL;
         pcmk_free_controld_api(save_controld_api);
     }
     pe_free_working_set(data_set);
     data_set = NULL;
     crm_exit(exit_code);
     return exit_code;
 }
 
 static gboolean
 resource_ipc_timeout(gpointer data)
 {
     // Start with newline because "Waiting for ..." message doesn't have one
     fprintf(stderr, "\nAborting because no messages received in %d seconds\n",
             MESSAGE_TIMEOUT_S);
     crm_err("No messages received in %d seconds", MESSAGE_TIMEOUT_S);
     bye(CRM_EX_TIMEOUT);
     return FALSE;
 }
 
 static void
 handle_controller_reply(pcmk_controld_api_t *capi, void *api_data,
                         void *user_data)
 {
     fprintf(stderr, ".");
     if ((capi->replies_expected(capi) == 0)
         && mainloop && g_main_loop_is_running(mainloop)) {
         fprintf(stderr, " OK\n");
         crm_debug("Got all the replies we expected");
         bye(CRM_EX_OK);
     }
 }
 
 static void
 handle_controller_drop(pcmk_controld_api_t *capi, void *api_data,
                        void *user_data)
 {
     crm_info("Connection to controller was terminated");
     bye(CRM_EX_DISCONNECT);
 }
 
 static void
 start_mainloop(pcmk_controld_api_t *capi)
 {
     if (capi->replies_expected(capi) > 0) {
         unsigned int count = capi->replies_expected(capi);
 
         fprintf(stderr, "Waiting for %d %s from the controller",
                 count, pcmk__plural_alt(count, "reply", "replies"));
         mainloop = g_main_loop_new(NULL, FALSE);
         g_timeout_add(MESSAGE_TIMEOUT_S * 1000, resource_ipc_timeout, NULL);
         g_main_loop_run(mainloop);
     }
 }
 
 static int
 compare_id(gconstpointer a, gconstpointer b)
 {
     return strcmp((const char *)a, (const char *)b);
 }
 
 static GListPtr
 build_constraint_list(xmlNode *root)
 {
     GListPtr retval = NULL;
     xmlNode *cib_constraints = NULL;
     xmlXPathObjectPtr xpathObj = NULL;
     int ndx = 0;
 
     cib_constraints = get_object_root(XML_CIB_TAG_CONSTRAINTS, root);
     xpathObj = xpath_search(cib_constraints, "//" XML_CONS_TAG_RSC_LOCATION);
 
     for (ndx = 0; ndx < numXpathResults(xpathObj); ndx++) {
         xmlNode *match = getXpathResult(xpathObj, ndx);
         retval = g_list_insert_sorted(retval, (gpointer) ID(match), compare_id);
     }
 
     freeXpathObject(xpathObj);
     return retval;
 }
 
 /* short option letters still available: eEJkKXyYZ */
 
 static pcmk__cli_option_t long_options[] = {
     // long option, argument type, storage, short option, description, flags
     {
         "help", no_argument, NULL, '?',
         "\t\tDisplay this text and exit", pcmk__option_default
     },
     {
         "version", no_argument, NULL, '$',
         "\t\tDisplay version information and exit", pcmk__option_default
     },
     {
         "verbose", no_argument, NULL, 'V',
         "\t\tIncrease debug output (may be specified multiple times)",
         pcmk__option_default
     },
     {
         "quiet", no_argument, NULL, 'Q',
         "\t\tBe less descriptive in results", pcmk__option_default
     },
     {
         "resource", required_argument, NULL, 'r',
         "\tResource ID", pcmk__option_default
     },
     {
         "-spacer-", no_argument, NULL, '-',
         "\nQueries:", pcmk__option_default
     },
     {
         "list", no_argument, NULL, 'L',
         "\t\tList all cluster resources with status", pcmk__option_default
     },
     {
         "list-raw", no_argument, NULL, 'l',
         "\t\tList IDs of all instantiated resources (individual members rather "
             "than groups etc.)",
         pcmk__option_default
     },
     {
         "list-cts", no_argument, NULL, 'c',
         NULL, pcmk__option_hidden
     },
     {
         "list-operations", no_argument, NULL, 'O',
         "\tList active resource operations, optionally filtered by --resource "
             "and/or --node",
         pcmk__option_default
     },
     {
         "list-all-operations", no_argument, NULL, 'o',
         "List all resource operations, optionally filtered by --resource "
             "and/or --node",
         pcmk__option_default
     },
     {
         "list-standards", no_argument, NULL, 0,
         "\tList supported standards", pcmk__option_default
     },
     {
         "list-ocf-providers", no_argument, NULL, 0,
         "List all available OCF providers", pcmk__option_default
     },
     {
         "list-agents", required_argument, NULL, 0,
         "List all agents available for the named standard and/or provider",
         pcmk__option_default
     },
     {
         "list-ocf-alternatives", required_argument, NULL, 0,
         "List all available providers for the named OCF agent",
         pcmk__option_default
     },
     {
         "show-metadata", required_argument, NULL, 0,
         "Show the metadata for the named class:provider:agent",
         pcmk__option_default
     },
     {
         "query-xml", no_argument, NULL, 'q',
         "\tShow XML configuration of resource (after any template expansion)",
         pcmk__option_default
     },
     {
         "query-xml-raw", no_argument, NULL, 'w',
         "\tShow XML configuration of resource (before any template expansion)",
         pcmk__option_default
     },
     {
         "get-parameter", required_argument, NULL, 'g',
         "Display named parameter for resource (use instance attribute unless "
             "--meta or --utilization is specified)",
         pcmk__option_default
     },
     {
         "get-property", required_argument, NULL, 'G',
         "Display named property of resource ('class', 'type', or 'provider') "
             "(requires --resource)",
         pcmk__option_hidden
     },
     {
         "locate", no_argument, NULL, 'W',
         "\t\tShow node(s) currently running resource",
         pcmk__option_default
     },
     {
         "stack", no_argument, NULL, 'A',
         "\t\tDisplay the prerequisites and dependents of a resource",
         pcmk__option_default
     },
     {
         "constraints", no_argument, NULL, 'a',
         "\tDisplay the (co)location constraints that apply to a resource",
         pcmk__option_default
     },
     {
         "why", no_argument, NULL, 'Y',
         "\t\tShow why resources are not running, optionally filtered by "
             "--resource and/or --node",
         pcmk__option_default
     },
     {
         "-spacer-", no_argument, NULL, '-',
         "\nCommands:", pcmk__option_default
     },
     {
         "validate", no_argument, NULL, 0,
         "\t\tValidate resource configuration by calling agent's validate-all "
             "action. The configuration may be specified either by giving an "
             "existing resource name with -r, or by specifying --class, "
             "--agent, and --provider arguments, along with any number of "
             "--option arguments.",
         pcmk__option_default
     },
     {
         "cleanup", no_argument, NULL, 'C',
         "\t\tIf resource has any past failures, clear its history and "
             "fail count. Optionally filtered by --resource, --node, "
             "--operation, and --interval (otherwise all). --operation and "
             "--interval apply to fail counts, but entire history is always "
             "cleared, to allow current state to be rechecked. If the named "
             "resource is part of a group, or one numbered instance of a clone "
             "or bundled resource, the clean-up applies to the whole collective "
             "resource unless --force is given.",
         pcmk__option_default
     },
     {
         "refresh", no_argument, NULL, 'R',
         "\t\tDelete resource's history (including failures) so its current "
             "state is rechecked. Optionally filtered by --resource and --node "
             "(otherwise all). If the named resource is part of a group, or one "
-            "numbered instance of a clone or bundled resource, the clean-up "
+            "numbered instance of a clone or bundled resource, the refresh "
             "applies to the whole collective resource unless --force is given.",
         pcmk__option_default
     },
     {
         "set-parameter", required_argument, NULL, 'p',
         "Set named parameter for resource (requires -v). Use instance "
             "attribute unless --meta or --utilization is specified.",
         pcmk__option_default
     },
     {
         "delete-parameter", required_argument, NULL, 'd',
         "Delete named parameter for resource. Use instance attribute unless "
             "--meta or --utilization is specified.",
         pcmk__option_default
     },
     {
         "set-property", required_argument, NULL, 'S',
         "Set named property of resource ('class', 'type', or 'provider') "
             "(requires -r, -t, -v)",
         pcmk__option_hidden
     },
     {
         "-spacer-", no_argument, NULL, '-',
         "\nResource location:", pcmk__option_default
     },
     {
         "move", no_argument, NULL, 'M',
         "\t\tCreate a constraint to move resource. If --node is specified, the "
             "constraint will be to move to that node, otherwise it will be to "
             "ban the current node. Unless --force is specified, this will "
             "return an error if the resource is already running on the "
             "specified node. If --force is specified, this will always ban the "
             "current node. Optional: --lifetime, --master. NOTE: This may "
             "prevent the resource from running on its previous location until "
             "the implicit constraint expires or is removed with --clear.",
         pcmk__option_default
     },
     {
         "ban", no_argument, NULL, 'B',
         "\t\tCreate a constraint to keep resource off a node. Optional: "
             "--node, --lifetime, --master. NOTE: This will prevent the "
             "resource from running on the affected node until the implicit "
             "constraint expires or is removed with --clear. If --node is not "
             "specified, it defaults to the node currently running the resource "
             "for primitives and groups, or the master for promotable clones "
             "with promoted-max=1 (all other situations result in an error as "
             "there is no sane default).",
         pcmk__option_default
     },
     {
         "clear", no_argument, NULL, 'U',
         "\t\tRemove all constraints created by the --ban and/or --move "
             "commands. Requires: --resource. Optional: --node, --master, "
             "--expired. If --node is not specified, all constraints created "
             "by --ban and --move will be removed for the named resource. If "
             "--node and --force are specified, any constraint created by "
             "--move will be cleared, even if it is not for the specified node. "
             "If --expired is specified, only those constraints whose lifetimes "
             "have expired will be removed.",
         pcmk__option_default
     },
     {
         "expired", no_argument, NULL, 'e',
         "\t\tModifies the --clear argument to remove constraints with "
             "expired lifetimes.",
         pcmk__option_default
     },
     {
         "lifetime", required_argument, NULL, 'u',
         "\tLifespan (as ISO 8601 duration) of created constraints (with -B, "
             "-M) (see https://en.wikipedia.org/wiki/ISO_8601#Durations)",
         pcmk__option_default
     },
     {
         "master", no_argument, NULL, 0,
         "\t\tLimit scope of command to Master role (with -B, -M, -U). For -B "
             "and -M, the previous master may remain active in the Slave role.",
         pcmk__option_default
     },
     {
         "-spacer-", no_argument, NULL, '-',
         "\nAdvanced Commands:", pcmk__option_default
     },
     {
         "delete", no_argument, NULL, 'D',
         "\t\t(Advanced) Delete a resource from the CIB. Required: -t",
         pcmk__option_default
     },
     {
         "fail", no_argument, NULL, 'F',
         "\t\t(Advanced) Tell the cluster this resource has failed",
         pcmk__option_default
     },
     {
         "restart", no_argument, NULL, 0,
         "\t\t(Advanced) Tell the cluster to restart this resource and "
             "anything that depends on it",
         pcmk__option_default
     },
     {
         "wait", no_argument, NULL, 0,
         "\t\t(Advanced) Wait until the cluster settles into a stable state",
         pcmk__option_default
     },
     {
         "force-demote", no_argument, NULL, 0,
         "\t(Advanced) Bypass the cluster and demote a resource on the local "
             "node. Unless --force is specified, this will refuse to do so if "
             "the cluster believes the resource is a clone instance already "
             "running on the local node.",
         pcmk__option_default
     },
     {
         "force-stop", no_argument, NULL, 0,
         "\t(Advanced) Bypass the cluster and stop a resource on the local node",
         pcmk__option_default
     },
     {
         "force-start", no_argument, NULL, 0,
         "\t(Advanced) Bypass the cluster and start a resource on the local "
             "node. Unless --force is specified, this will refuse to do so if "
             "the cluster believes the resource is a clone instance already "
             "running on the local node.",
         pcmk__option_default
     },
     {
         "force-promote", no_argument, NULL, 0,
         "\t(Advanced) Bypass the cluster and promote a resource on the local "
             "node. Unless --force is specified, this will refuse to do so if "
             "the cluster believes the resource is a clone instance already "
             "running on the local node.",
         pcmk__option_default
     },
     {
         "force-check", no_argument, NULL, 0,
         "\t(Advanced) Bypass the cluster and check the state of a resource on "
             "the local node",
         pcmk__option_default
     },
     {
         "-spacer-", no_argument, NULL, '-',
         "\nValidate Options:", pcmk__option_default
     },
     {
         "class", required_argument, NULL, 0,
         "\tThe standard the resource agent confirms to (for example, ocf). "
             "Use with --agent, --provider, --option, and --validate.",
         pcmk__option_default
     },
     {
         "agent", required_argument, NULL, 0,
         "\tThe agent to use (for example, IPaddr). Use with --class, "
             "--provider, --option, and --validate.",
         pcmk__option_default
     },
     {
         "provider", required_argument, NULL, 0,
         "\tThe vendor that supplies the resource agent (for example, "
             "heartbeat). Use with --class, --agent, --option, and --validate.",
         pcmk__option_default
     },
     {
         "option", required_argument, NULL, 0,
         "\tSpecify a device configuration parameter as NAME=VALUE (may be "
             "specified multiple times). Use with --validate and without the "
             "-r option.",
         pcmk__option_default
     },
     {
         "-spacer-", no_argument, NULL, '-',
         "\nAdditional Options:", pcmk__option_default
     },
     {
         "node", required_argument, NULL, 'N',
         "\tNode name", pcmk__option_default
     },
     {
         "recursive", no_argument, NULL, 0,
         "\tFollow colocation chains when using --set-parameter",
         pcmk__option_default
     },
     {
         "resource-type", required_argument, NULL, 't',
         "Resource XML element (primitive, group, etc.) (with -D)",
         pcmk__option_default
     },
     {
         "parameter-value", required_argument, NULL, 'v',
         "Value to use with -p", pcmk__option_default
     },
     {
         "meta", no_argument, NULL, 'm',
         "\t\tUse resource meta-attribute instead of instance attribute "
             "(with -p, -g, -d)",
         pcmk__option_default
     },
     {
         "utilization", no_argument, NULL, 'z',
         "\tUse resource utilization attribute instead of instance attribute "
             "(with -p, -g, -d)",
         pcmk__option_default
     },
     {
         "operation", required_argument, NULL, 'n',
         "\tOperation to clear instead of all (with -C -r)",
         pcmk__option_default
     },
     {
         "interval", required_argument, NULL, 'I',
         "\tInterval of operation to clear (default 0) (with -C -r -n)",
         pcmk__option_default
     },
     {
         "set-name", required_argument, NULL, 's',
         "\t(Advanced) XML ID of attributes element to use (with -p, -d)",
         pcmk__option_default
     },
     {
         "nvpair", required_argument, NULL, 'i',
         "\t(Advanced) XML ID of nvpair element to use (with -p, -d)",
         pcmk__option_default
     },
     {
         "timeout", required_argument, NULL, 'T',
         "\t(Advanced) Abort if command does not finish in this time (with "
             "--restart, --wait, --force-*)",
         pcmk__option_default
     },
     {
         "force", no_argument, NULL, 'f',
         "\t\tIf making CIB changes, do so regardless of quorum. See help for "
             "individual commands for additional behavior.",
         pcmk__option_default
     },
     {
         "xml-file", required_argument, NULL, 'x',
         NULL, pcmk__option_hidden
     },
 
     /* legacy options */
     {
         "host-uname", required_argument, NULL, 'H',
         NULL, pcmk__option_hidden
     },
 
     {
         "-spacer-", 1, NULL, '-',
         "\nExamples:", pcmk__option_paragraph
     },
     {
         "-spacer-", 1, NULL, '-',
         "List the available OCF agents:", pcmk__option_paragraph
     },
     {
         "-spacer-", 1, NULL, '-',
         " crm_resource --list-agents ocf", pcmk__option_example
     },
     {
         "-spacer-", 1, NULL, '-',
         "List the available OCF agents from the linux-ha project:",
         pcmk__option_paragraph
     },
     {
         "-spacer-", 1, NULL, '-',
         " crm_resource --list-agents ocf:heartbeat", pcmk__option_example
     },
     {
         "-spacer-", 1, NULL, '-',
         "Move 'myResource' to a specific node:", pcmk__option_paragraph
     },
     {
         "-spacer-", 1, NULL, '-',
         " crm_resource --resource myResource --move --node altNode",
         pcmk__option_example
     },
     {
         "-spacer-", 1, NULL, '-',
         "Allow (but not force) 'myResource' to move back to its original "
             "location:",
         pcmk__option_paragraph
     },
     {
         "-spacer-", 1, NULL, '-',
         " crm_resource --resource myResource --clear", pcmk__option_example
     },
     {
         "-spacer-", 1, NULL, '-',
         "Stop 'myResource' (and anything that depends on it):",
         pcmk__option_paragraph
     },
     {
         "-spacer-", 1, NULL, '-',
         " crm_resource --resource myResource --set-parameter target-role "
             "--meta --parameter-value Stopped",
         pcmk__option_example
     },
     {
         "-spacer-", 1, NULL, '-',
         "Tell the cluster not to manage 'myResource' (the cluster will not "
             "attempt to start or stop the resource under any circumstances; "
             "useful when performing maintenance tasks on a resource):",
         pcmk__option_paragraph
     },
     {
         "-spacer-", 1, NULL, '-',
         " crm_resource --resource myResource --set-parameter is-managed "
             "--meta --parameter-value false",
         pcmk__option_example
     },
     {
         "-spacer-", 1, NULL, '-',
         "Erase the operation history of 'myResource' on 'aNode' (the cluster "
             "will 'forget' the existing resource state, including any "
             "errors, and attempt to recover the resource; useful when a "
             "resource had failed permanently and has been repaired "
             "by an administrator):",
         pcmk__option_paragraph
     },
     {
         "-spacer-", 1, NULL, '-',
         " crm_resource --resource myResource --cleanup --node aNode",
         pcmk__option_example
     },
     { 0, 0, 0, 0 }
 };
 
 
 int
 main(int argc, char **argv)
 {
     char rsc_cmd = 'L';
 
     const char *v_class = NULL;
     const char *v_agent = NULL;
     const char *v_provider = NULL;
     char *name = NULL;
     char *value = NULL;
     GHashTable *validate_options = NULL;
 
     const char *rsc_id = NULL;
     const char *host_uname = NULL;
     const char *prop_name = NULL;
     const char *prop_value = NULL;
     const char *rsc_type = NULL;
     const char *prop_id = NULL;
     const char *prop_set = NULL;
     const char *rsc_long_cmd = NULL;
     const char *longname = NULL;
     const char *operation = NULL;
     const char *interval_spec = NULL;
     const char *cib_file = getenv("CIB_file");
     const char *xml_file = NULL;
     GHashTable *override_params = NULL;
 
     xmlNode *cib_xml_copy = NULL;
     pe_resource_t *rsc = NULL;
     bool recursive = FALSE;
 
     bool validate_cmdline = FALSE; /* whether we are just validating based on command line options */
     bool require_resource = TRUE; /* whether command requires that resource be specified */
     bool require_dataset = TRUE;  /* whether command requires populated dataset instance */
     bool require_crmd = FALSE;    // whether command requires controller connection
     bool clear_expired = FALSE;
 
     int rc = pcmk_ok;
     int is_ocf_rc = 0;
     int option_index = 0;
     int timeout_ms = 0;
     int argerr = 0;
     int flag;
     int find_flags = 0;           // Flags to use when searching for resource
     crm_exit_t exit_code = CRM_EX_OK;
 
     crm_log_cli_init("crm_resource");
     pcmk__set_cli_options(NULL, "<query>|<command> [options]", long_options,
                           "perform tasks related to Pacemaker "
                           "cluster resources");
 
     validate_options = crm_str_table_new();
 
     while (1) {
         flag = pcmk__next_cli_option(argc, argv, &option_index, &longname);
         if (flag == -1)
             break;
 
         switch (flag) {
             case 0: /* long options with no short equivalent */
                 if (safe_str_eq("master", longname)) {
                     scope_master = TRUE;
 
                 } else if(safe_str_eq(longname, "recursive")) {
                     recursive = TRUE;
 
                 } else if (safe_str_eq("wait", longname)) {
                     rsc_cmd = flag;
                     rsc_long_cmd = longname;
                     require_resource = FALSE;
                     require_dataset = FALSE;
 
                 } else if (
                     safe_str_eq("validate", longname)
                     || safe_str_eq("restart", longname)
                     || safe_str_eq("force-demote",  longname)
                     || safe_str_eq("force-stop",    longname)
                     || safe_str_eq("force-start",   longname)
                     || safe_str_eq("force-promote", longname)
                     || safe_str_eq("force-check",   longname)) {
                     rsc_cmd = flag;
                     rsc_long_cmd = longname;
                     find_flags = pe_find_renamed|pe_find_anon;
                     crm_log_args(argc, argv);
 
                 } else if (safe_str_eq("list-ocf-providers", longname)
                            || safe_str_eq("list-ocf-alternatives", longname)
                            || safe_str_eq("list-standards", longname)) {
                     const char *text = NULL;
                     lrmd_list_t *list = NULL;
                     lrmd_list_t *iter = NULL;
                     lrmd_t *lrmd_conn = lrmd_api_new();
 
                     if (safe_str_eq("list-ocf-providers", longname)
                         || safe_str_eq("list-ocf-alternatives", longname)) {
                         rc = lrmd_conn->cmds->list_ocf_providers(lrmd_conn, optarg, &list);
                         text = "OCF providers";
 
                     } else if (safe_str_eq("list-standards", longname)) {
                         rc = lrmd_conn->cmds->list_standards(lrmd_conn, &list);
                         text = "standards";
                     }
 
                     if (rc > 0) {
                         for (iter = list; iter != NULL; iter = iter->next) {
                             printf("%s\n", iter->val);
                         }
                         lrmd_list_freeall(list);
 
                     } else if (optarg) {
                         fprintf(stderr, "No %s found for %s\n", text, optarg);
                         exit_code = CRM_EX_NOSUCH;
 
                     } else {
                         fprintf(stderr, "No %s found\n", text);
                         exit_code = CRM_EX_NOSUCH;
                     }
 
                     lrmd_api_delete(lrmd_conn);
                     return bye(exit_code);
 
                 } else if (safe_str_eq("show-metadata", longname)) {
                     char *standard = NULL;
                     char *provider = NULL;
                     char *type = NULL;
                     char *metadata = NULL;
                     lrmd_t *lrmd_conn = lrmd_api_new();
 
                     rc = crm_parse_agent_spec(optarg, &standard, &provider, &type);
                     if (rc == pcmk_ok) {
                         rc = lrmd_conn->cmds->get_metadata(lrmd_conn, standard,
                                                            provider, type,
                                                            &metadata, 0);
                     } else {
                         fprintf(stderr,
                                 "'%s' is not a valid agent specification\n",
                                 optarg);
                         rc = -ENXIO;
                     }
 
                     if (metadata) {
                         printf("%s\n", metadata);
                     } else {
                         fprintf(stderr, "Metadata query for %s failed: %s\n",
                                 optarg, pcmk_strerror(rc));
                         exit_code = crm_errno2exit(rc);
                     }
                     lrmd_api_delete(lrmd_conn);
                     return bye(exit_code);
 
                 } else if (safe_str_eq("list-agents", longname)) {
                     lrmd_list_t *list = NULL;
                     lrmd_list_t *iter = NULL;
                     char *provider = strchr (optarg, ':');
                     lrmd_t *lrmd_conn = lrmd_api_new();
 
                     if (provider) {
                         *provider++ = 0;
                     }
                     rc = lrmd_conn->cmds->list_agents(lrmd_conn, &list, optarg, provider);
 
                     if (rc > 0) {
                         for (iter = list; iter != NULL; iter = iter->next) {
                             printf("%s\n", iter->val);
                         }
                         lrmd_list_freeall(list);
                     } else {
                         fprintf(stderr, "No agents found for standard=%s, provider=%s\n",
                                 optarg, (provider? provider : "*"));
                         exit_code = CRM_EX_NOSUCH;
                     }
                     lrmd_api_delete(lrmd_conn);
                     return bye(exit_code);
 
                 } else if (safe_str_eq("class", longname)) {
                     if (!(pcmk_get_ra_caps(optarg) & pcmk_ra_cap_params)) {
                         if (BE_QUIET == FALSE) {
                             fprintf(stdout, "Standard %s does not support parameters\n",
                                     optarg);
                         }
                         return bye(exit_code);
 
                     } else {
                         v_class = optarg;
                     }
 
                     validate_cmdline = TRUE;
                     require_resource = FALSE;
 
                 } else if (safe_str_eq("agent", longname)) {
                     validate_cmdline = TRUE;
                     require_resource = FALSE;
                     v_agent = optarg;
 
                 } else if (safe_str_eq("provider", longname)) {
                     validate_cmdline = TRUE;
                     require_resource = FALSE;
                     v_provider = optarg;
 
                 } else if (safe_str_eq("option", longname)) {
                     crm_info("Scanning: --option %s", optarg);
                     rc = pcmk_scan_nvpair(optarg, &name, &value);
                     if (rc != 2) {
                         fprintf(stderr, "Invalid option: --option %s: %s", optarg, pcmk_strerror(rc));
                         argerr++;
                     } else {
                         crm_info("Got: '%s'='%s'", name, value);
                     }
 
                     g_hash_table_replace(validate_options, name, value);
 
                 } else {
                     crm_err("Unhandled long option: %s", longname);
                 }
                 break;
             case 'V':
                 resource_verbose++;
                 crm_bump_log_level(argc, argv);
                 break;
             case '$':
             case '?':
                 pcmk__cli_help(flag, CRM_EX_OK);
                 break;
             case 'x':
                 xml_file = optarg;
                 break;
             case 'Q':
                 BE_QUIET = TRUE;
                 break;
             case 'm':
                 attr_set_type = XML_TAG_META_SETS;
                 break;
             case 'z':
                 attr_set_type = XML_TAG_UTILIZATION;
                 break;
             case 'u':
                 move_lifetime = strdup(optarg);
                 break;
             case 'f':
                 do_force = TRUE;
                 crm_log_args(argc, argv);
                 break;
             case 'i':
                 prop_id = optarg;
                 break;
             case 's':
                 prop_set = optarg;
                 break;
             case 'r':
                 rsc_id = optarg;
                 break;
             case 'v':
                 prop_value = optarg;
                 break;
             case 't':
                 rsc_type = optarg;
                 break;
             case 'T':
                 timeout_ms = crm_get_msec(optarg);
                 break;
             case 'e':
                 clear_expired = TRUE;
                 require_resource = FALSE;
                 break;
 
             case 'C':
             case 'R':
                 crm_log_args(argc, argv);
                 require_resource = FALSE;
                 if (cib_file == NULL) {
                     require_crmd = TRUE;
                 }
                 rsc_cmd = flag;
                 find_flags = pe_find_renamed|pe_find_anon;
                 break;
 
             case 'n':
                 operation = optarg;
                 break;
 
             case 'I':
                 interval_spec = optarg;
                 break;
 
             case 'D':
                 require_dataset = FALSE;
                 crm_log_args(argc, argv);
                 rsc_cmd = flag;
                 find_flags = pe_find_renamed|pe_find_any;
                 break;
 
             case 'F':
                 require_crmd = TRUE;
                 crm_log_args(argc, argv);
                 rsc_cmd = flag;
                 break;
 
             case 'U':
             case 'B':
             case 'M':
                 crm_log_args(argc, argv);
                 rsc_cmd = flag;
                 find_flags = pe_find_renamed|pe_find_anon;
                 break;
 
             case 'c':
             case 'L':
             case 'l':
             case 'O':
             case 'o':
                 require_resource = FALSE;
                 rsc_cmd = flag;
                 break;
 
             case 'Y':
                 require_resource = FALSE;
                 rsc_cmd = flag;
                 find_flags = pe_find_renamed|pe_find_anon;
                 break;
 
             case 'q':
             case 'w':
                 rsc_cmd = flag;
                 find_flags = pe_find_renamed|pe_find_any;
                 break;
 
             case 'W':
             case 'A':
             case 'a':
                 rsc_cmd = flag;
                 find_flags = pe_find_renamed|pe_find_anon;
                 break;
 
             case 'S':
                 require_dataset = FALSE;
                 crm_log_args(argc, argv);
                 prop_name = optarg;
                 rsc_cmd = flag;
                 find_flags = pe_find_renamed|pe_find_any;
                 break;
 
             case 'p':
             case 'd':
                 crm_log_args(argc, argv);
                 prop_name = optarg;
                 rsc_cmd = flag;
                 find_flags = pe_find_renamed|pe_find_any;
                 break;
 
             case 'G':
             case 'g':
                 prop_name = optarg;
                 rsc_cmd = flag;
                 find_flags = pe_find_renamed|pe_find_any;
                 break;
 
             case 'H':
             case 'N':
                 crm_trace("Option %c => %s", flag, optarg);
                 host_uname = optarg;
                 break;
 
             default:
                 CMD_ERR("Argument code 0%o (%c) is not (?yet?) supported", flag, flag);
                 ++argerr;
                 break;
         }
     }
 
     // Catch the case where the user didn't specify a command
     if (rsc_cmd == 'L') {
         require_resource = FALSE;
     }
 
     // --expired without --clear/-U doesn't make sense
     if (clear_expired == TRUE && rsc_cmd != 'U') {
         CMD_ERR("--expired requires --clear or -U");
         argerr++;
     }
 
     if (optind < argc
         && argv[optind] != NULL
         && rsc_cmd == 0
         && rsc_long_cmd) {
 
         override_params = crm_str_table_new();
         while (optind < argc && argv[optind] != NULL) {
             char *name = calloc(1, strlen(argv[optind]));
             char *value = calloc(1, strlen(argv[optind]));
             int rc = sscanf(argv[optind], "%[^=]=%s", name, value);
 
             if(rc == 2) {
                 g_hash_table_replace(override_params, name, value);
 
             } else {
                 CMD_ERR("Error parsing '%s' as a name=value pair for --%s", argv[optind], rsc_long_cmd);
                 free(value);
                 free(name);
                 argerr++;
             }
             optind++;
         }
 
     } else if (optind < argc && argv[optind] != NULL && rsc_cmd == 0) {
         CMD_ERR("non-option ARGV-elements: ");
         while (optind < argc && argv[optind] != NULL) {
             CMD_ERR("[%d of %d] %s ", optind, argc, argv[optind]);
             optind++;
             argerr++;
         }
     }
 
     if (optind > argc) {
         ++argerr;
     }
 
     // Sanity check validating from command line parameters.  If everything checks out,
     // go ahead and run the validation.  This way we don't need a CIB connection.
     if (validate_cmdline == TRUE) {
         // -r cannot be used with any of --class, --agent, or --provider
         if (rsc_id != NULL) {
             CMD_ERR("--resource cannot be used with --class, --agent, and --provider");
             argerr++;
 
         // If --class, --agent, or --provider are given, --validate must also be given.
         } else if (!safe_str_eq(rsc_long_cmd, "validate")) {
             CMD_ERR("--class, --agent, and --provider require --validate");
             argerr++;
 
         // Not all of --class, --agent, and --provider need to be given.  Not all
         // classes support the concept of a provider.  Check that what we were given
         // is valid.
         } else if (crm_str_eq(v_class, "stonith", TRUE)) {
             if (v_provider != NULL) {
                 CMD_ERR("stonith does not support providers");
                 argerr++;
 
             } else if (stonith_agent_exists(v_agent, 0) == FALSE) {
                 CMD_ERR("%s is not a known stonith agent", v_agent ? v_agent : "");
                 argerr++;
             }
 
         } else if (resources_agent_exists(v_class, v_provider, v_agent) == FALSE) {
             CMD_ERR("%s:%s:%s is not a known resource",
                     v_class ? v_class : "",
                     v_provider ? v_provider : "",
                     v_agent ? v_agent : "");
             argerr++;
         }
 
         if (argerr == 0) {
             rc = cli_resource_execute_from_params("test", v_class, v_provider, v_agent,
                                                   "validate-all", validate_options,
                                                   override_params, timeout_ms);
             exit_code = crm_errno2exit(rc);
             return bye(exit_code);
         }
     }
 
     if (argerr) {
         CMD_ERR("Invalid option(s) supplied, use --help for valid usage");
         return bye(CRM_EX_USAGE);
     }
 
     if (do_force) {
         crm_debug("Forcing...");
         cib_options |= cib_quorum_override;
     }
 
     if (require_resource && !rsc_id) {
         CMD_ERR("Must supply a resource id with -r");
         rc = -ENXIO;
         goto bail;
     }
 
     if (find_flags && rsc_id) {
         require_dataset = TRUE;
     }
 
     // Establish a connection to the CIB
     cib_conn = cib_new();
     rc = cib_conn->cmds->signon(cib_conn, crm_system_name, cib_command);
     if (rc != pcmk_ok) {
         CMD_ERR("Could not connect to the CIB: %s", pcmk_strerror(rc));
         goto bail;
     }
 
     /* Populate working set from XML file if specified or CIB query otherwise */
     if (require_dataset) {
         if (xml_file != NULL) {
             cib_xml_copy = filename2xml(xml_file);
         } else {
             rc = cib_conn->cmds->query(cib_conn, NULL, &cib_xml_copy, cib_scope_local | cib_sync_call);
         }
 
         if(rc != pcmk_ok) {
             goto bail;
         }
 
         /* Populate the working set instance */
         data_set = pe_new_working_set();
         if (data_set == NULL) {
             rc = -ENOMEM;
             goto bail;
         }
         set_bit(data_set->flags, pe_flag_no_counts);
         set_bit(data_set->flags, pe_flag_no_compat);
         rc = update_working_set_xml(data_set, &cib_xml_copy);
         if (rc != pcmk_ok) {
             goto bail;
         }
         cluster_status(data_set);
     }
 
     // If command requires that resource exist if specified, find it
     if (find_flags && rsc_id) {
         rsc = pe_find_resource_with_flags(data_set->resources, rsc_id,
                                           find_flags);
         if (rsc == NULL) {
             CMD_ERR("Resource '%s' not found", rsc_id);
             rc = -ENXIO;
             goto bail;
         }
     }
 
     // Establish a connection to the controller if needed
     if (require_crmd) {
         char *client_uuid;
         pcmk_controld_api_cb_t dispatch_cb = {
             handle_controller_reply, NULL
         };
         pcmk_controld_api_cb_t destroy_cb = {
             handle_controller_drop, NULL
         };
 
 
         client_uuid = pcmk__getpid_s();
         controld_api = pcmk_new_controld_api(crm_system_name, client_uuid);
         free(client_uuid);
 
         rc = controld_api->connect(controld_api, true, &dispatch_cb,
                                    &destroy_cb);
         if (rc != pcmk_rc_ok) {
             CMD_ERR("Error connecting to the controller: %s", pcmk_rc_str(rc));
             rc = pcmk_rc2legacy(rc);
             goto bail;
         }
     }
 
     /* Handle rsc_cmd appropriately */
     if (rsc_cmd == 'L') {
         rc = pcmk_ok;
         cli_resource_print_list(data_set, FALSE);
 
     } else if (rsc_cmd == 'l') {
         int found = 0;
         GListPtr lpc = NULL;
 
         rc = pcmk_ok;
         for (lpc = data_set->resources; lpc != NULL; lpc = lpc->next) {
             rsc = (pe_resource_t *) lpc->data;
 
             found++;
             cli_resource_print_raw(rsc);
         }
 
         if (found == 0) {
             printf("NO resources configured\n");
             rc = -ENXIO;
         }
 
     } else if (rsc_cmd == 0 && rsc_long_cmd && safe_str_eq(rsc_long_cmd, "restart")) {
         /* We don't pass data_set because rsc needs to stay valid for the entire
          * lifetime of cli_resource_restart(), but it will reset and update the
          * working set multiple times, so it needs to use its own copy.
          */
         rc = cli_resource_restart(rsc, host_uname, timeout_ms, cib_conn);
 
     } else if (rsc_cmd == 0 && rsc_long_cmd && safe_str_eq(rsc_long_cmd, "wait")) {
         rc = wait_till_stable(timeout_ms, cib_conn);
 
     } else if (rsc_cmd == 0 && rsc_long_cmd) {
         // validate, force-(stop|start|demote|promote|check)
         rc = cli_resource_execute(rsc, rsc_id, rsc_long_cmd, override_params,
                                   timeout_ms, cib_conn, data_set);
         if (rc >= 0) {
             is_ocf_rc = 1;
         }
 
     } else if (rsc_cmd == 'A' || rsc_cmd == 'a') {
         GListPtr lpc = NULL;
         xmlNode *cib_constraints = get_object_root(XML_CIB_TAG_CONSTRAINTS,
                                                    data_set->input);
 
         unpack_constraints(cib_constraints, data_set);
 
         // Constraints apply to group/clone, not member/instance
         rsc = uber_parent(rsc);
 
         for (lpc = data_set->resources; lpc != NULL; lpc = lpc->next) {
             pe_resource_t *r = (pe_resource_t *) lpc->data;
 
             clear_bit(r->flags, pe_rsc_allocating);
         }
 
         cli_resource_print_colocation(rsc, TRUE, rsc_cmd == 'A', 1);
 
         fprintf(stdout, "* %s\n", rsc->id);
         cli_resource_print_location(rsc, NULL);
 
         for (lpc = data_set->resources; lpc != NULL; lpc = lpc->next) {
             pe_resource_t *r = (pe_resource_t *) lpc->data;
 
             clear_bit(r->flags, pe_rsc_allocating);
         }
 
         cli_resource_print_colocation(rsc, FALSE, rsc_cmd == 'A', 1);
 
     } else if (rsc_cmd == 'c') {
         GListPtr lpc = NULL;
 
         rc = pcmk_ok;
         for (lpc = data_set->resources; lpc != NULL; lpc = lpc->next) {
             rsc = (pe_resource_t *) lpc->data;
             cli_resource_print_cts(rsc);
         }
         cli_resource_print_cts_constraints(data_set);
 
     } else if (rsc_cmd == 'F') {
         rc = cli_resource_fail(controld_api, host_uname, rsc_id, data_set);
         if (rc == pcmk_rc_ok) {
             start_mainloop(controld_api);
         }
         rc = pcmk_rc2legacy(rc);
 
     } else if (rsc_cmd == 'O') {
         rc = cli_resource_print_operations(rsc_id, host_uname, TRUE, data_set);
 
     } else if (rsc_cmd == 'o') {
         rc = cli_resource_print_operations(rsc_id, host_uname, FALSE, data_set);
 
     } else if (rsc_cmd == 'W') {
         rc = cli_resource_search(rsc, rsc_id, data_set);
         if (rc >= 0) {
             rc = pcmk_ok;
         }
 
     } else if (rsc_cmd == 'q') {
         rc = cli_resource_print(rsc, data_set, TRUE);
 
     } else if (rsc_cmd == 'w') {
         rc = cli_resource_print(rsc, data_set, FALSE);
 
     } else if (rsc_cmd == 'Y') {
         pe_node_t *dest = NULL;
 
         if (host_uname) {
             dest = pe_find_node(data_set->nodes, host_uname);
             if (dest == NULL) {
                 rc = -pcmk_err_node_unknown;
                 goto bail;
             }
         }
         cli_resource_why(cib_conn, data_set->resources, rsc, dest);
         rc = pcmk_ok;
 
     } else if (rsc_cmd == 'U') {
         GListPtr before = NULL;
         GListPtr after = NULL;
         GListPtr remaining = NULL;
         GListPtr ele = NULL;
         pe_node_t *dest = NULL;
 
         if (BE_QUIET == FALSE) {
             before = build_constraint_list(data_set->input);
         }
 
         if (clear_expired == TRUE) {
             rc = cli_resource_clear_all_expired(data_set->input, cib_conn, rsc_id, host_uname, scope_master);
 
         } else if (host_uname) {
             dest = pe_find_node(data_set->nodes, host_uname);
             if (dest == NULL) {
                 rc = -pcmk_err_node_unknown;
                 if (BE_QUIET == FALSE) {
                     g_list_free(before);
                 }
                 goto bail;
             }
             rc = cli_resource_clear(rsc_id, dest->details->uname, NULL, cib_conn, TRUE);
 
         } else {
             rc = cli_resource_clear(rsc_id, NULL, data_set->nodes, cib_conn, TRUE);
         }
 
         if (BE_QUIET == FALSE) {
             rc = cib_conn->cmds->query(cib_conn, NULL, &cib_xml_copy, cib_scope_local | cib_sync_call);
             if (rc != pcmk_ok) {
                 CMD_ERR("Could not get modified CIB: %s\n", pcmk_strerror(rc));
                 g_list_free(before);
                 goto bail;
             }
 
             data_set->input = cib_xml_copy;
             cluster_status(data_set);
 
             after = build_constraint_list(data_set->input);
             remaining = subtract_lists(before, after, (GCompareFunc) strcmp);
 
             for (ele = remaining; ele != NULL; ele = ele->next) {
                 printf("Removing constraint: %s\n", (char *) ele->data);
             }
 
             g_list_free(before);
             g_list_free(after);
             g_list_free(remaining);
         }
 
     } else if (rsc_cmd == 'M' && host_uname) {
         rc = cli_resource_move(rsc, rsc_id, host_uname, cib_conn, data_set);
 
     } else if (rsc_cmd == 'B' && host_uname) {
         pe_node_t *dest = pe_find_node(data_set->nodes, host_uname);
 
         if (dest == NULL) {
             rc = -pcmk_err_node_unknown;
             goto bail;
         }
         rc = cli_resource_ban(rsc_id, dest->details->uname, NULL, cib_conn);
 
     } else if (rsc_cmd == 'B' || rsc_cmd == 'M') {
         pe_node_t *current = NULL;
         unsigned int nactive = 0;
 
         current = pe__find_active_requires(rsc, &nactive);
 
         if (nactive == 1) {
             rc = cli_resource_ban(rsc_id, current->details->uname, NULL, cib_conn);
 
         } else if (is_set(rsc->flags, pe_rsc_promotable)) {
             int count = 0;
             GListPtr iter = NULL;
 
             current = NULL;
             for(iter = rsc->children; iter; iter = iter->next) {
                 pe_resource_t *child = (pe_resource_t *)iter->data;
                 enum rsc_role_e child_role = child->fns->state(child, TRUE);
 
                 if(child_role == RSC_ROLE_MASTER) {
                     count++;
                     current = pe__current_node(child);
                 }
             }
 
             if(count == 1 && current) {
                 rc = cli_resource_ban(rsc_id, current->details->uname, NULL, cib_conn);
 
             } else {
                 rc = -EINVAL;
                 exit_code = CRM_EX_USAGE;
                 CMD_ERR("Resource '%s' not moved: active in %d locations (promoted in %d).",
                         rsc_id, nactive, count);
                 CMD_ERR("To prevent '%s' from running on a specific location, "
                         "specify a node.", rsc_id);
                 CMD_ERR("To prevent '%s' from being promoted at a specific "
                         "location, specify a node and the master option.",
                         rsc_id);
             }
 
         } else {
             rc = -EINVAL;
             exit_code = CRM_EX_USAGE;
             CMD_ERR("Resource '%s' not moved: active in %d locations.", rsc_id, nactive);
             CMD_ERR("To prevent '%s' from running on a specific location, "
                     "specify a node.", rsc_id);
         }
 
     } else if (rsc_cmd == 'G') {
         rc = cli_resource_print_property(rsc, prop_name, data_set);
 
     } else if (rsc_cmd == 'S') {
         xmlNode *msg_data = NULL;
 
         if ((rsc_type == NULL) || !strlen(rsc_type)) {
             CMD_ERR("Must specify -t with resource type");
             rc = -ENXIO;
             goto bail;
 
         } else if ((prop_value == NULL) || !strlen(prop_value)) {
             CMD_ERR("Must supply -v with new value");
             rc = -EINVAL;
             goto bail;
         }
 
         CRM_LOG_ASSERT(prop_name != NULL);
 
         msg_data = create_xml_node(NULL, rsc_type);
         crm_xml_add(msg_data, XML_ATTR_ID, rsc_id);
         crm_xml_add(msg_data, prop_name, prop_value);
 
         rc = cib_conn->cmds->modify(cib_conn, XML_CIB_TAG_RESOURCES, msg_data, cib_options);
         free_xml(msg_data);
 
     } else if (rsc_cmd == 'g') {
         rc = cli_resource_print_attribute(rsc, prop_name, data_set);
 
     } else if (rsc_cmd == 'p') {
         if (prop_value == NULL || strlen(prop_value) == 0) {
             CMD_ERR("You need to supply a value with the -v option");
             rc = -EINVAL;
             goto bail;
         }
 
         /* coverity[var_deref_model] False positive */
         rc = cli_resource_update_attribute(rsc, rsc_id, prop_set, prop_id,
                                            prop_name, prop_value, recursive,
                                            cib_conn, data_set);
 
     } else if (rsc_cmd == 'd') {
         /* coverity[var_deref_model] False positive */
         rc = cli_resource_delete_attribute(rsc, rsc_id, prop_set, prop_id,
                                            prop_name, cib_conn, data_set);
 
     } else if ((rsc_cmd == 'C') && rsc) {
         if (do_force == FALSE) {
             rsc = uber_parent(rsc);
         }
 
         crm_debug("Erasing failures of %s (%s requested) on %s",
                   rsc->id, rsc_id, (host_uname? host_uname: "all nodes"));
         rc = cli_resource_delete(controld_api, host_uname, rsc, operation,
                                  interval_spec, TRUE, data_set);
 
         if ((rc == pcmk_ok) && !BE_QUIET) {
             // Show any reasons why resource might stay stopped
             cli_resource_check(cib_conn, rsc);
         }
 
         if (rc == pcmk_ok) {
             start_mainloop(controld_api);
         }
 
     } else if (rsc_cmd == 'C') {
         rc = cli_cleanup_all(controld_api, host_uname, operation, interval_spec,
                              data_set);
         if (rc == pcmk_ok) {
             start_mainloop(controld_api);
         }
 
     } else if ((rsc_cmd == 'R') && rsc) {
         if (do_force == FALSE) {
             rsc = uber_parent(rsc);
         }
 
         crm_debug("Re-checking the state of %s (%s requested) on %s",
                   rsc->id, rsc_id, (host_uname? host_uname: "all nodes"));
         rc = cli_resource_delete(controld_api, host_uname, rsc, NULL, 0, FALSE,
                                  data_set);
 
         if ((rc == pcmk_ok) && !BE_QUIET) {
             // Show any reasons why resource might stay stopped
             cli_resource_check(cib_conn, rsc);
         }
 
         if (rc == pcmk_ok) {
             start_mainloop(controld_api);
         }
 
     } else if (rsc_cmd == 'R') {
         const char *router_node = host_uname;
         int attr_options = pcmk__node_attr_none;
 
         if (host_uname) {
             pe_node_t *node = pe_find_node(data_set->nodes, host_uname);
 
             if (pe__is_guest_or_remote_node(node)) {
                 node = pe__current_node(node->details->remote_rsc);
                 if (node == NULL) {
                     CMD_ERR("No cluster connection to Pacemaker Remote node %s detected",
                             host_uname);
                     rc = -ENXIO;
                     goto bail;
                 }
                 router_node = node->details->uname;
                 attr_options |= pcmk__node_attr_remote;
             }
         }
 
         if (controld_api == NULL) {
             printf("Dry run: skipping clean-up of %s due to CIB_file\n",
                    host_uname? host_uname : "all nodes");
             rc = pcmk_ok;
             goto bail;
         }
 
         crm_debug("Re-checking the state of all resources on %s", host_uname?host_uname:"all nodes");
 
         rc = pcmk_rc2legacy(pcmk__node_attr_request_clear(NULL, host_uname,
                                                           NULL, NULL, NULL,
                                                           NULL, attr_options));
 
         if (controld_api->reprobe(controld_api, host_uname,
                                   router_node) == pcmk_rc_ok) {
             start_mainloop(controld_api);
         }
 
     } else if (rsc_cmd == 'D') {
         xmlNode *msg_data = NULL;
 
         if (rsc_type == NULL) {
             CMD_ERR("You need to specify a resource type with -t");
             rc = -ENXIO;
             goto bail;
         }
 
         msg_data = create_xml_node(NULL, rsc_type);
         crm_xml_add(msg_data, XML_ATTR_ID, rsc_id);
 
         rc = cib_conn->cmds->remove(cib_conn, XML_CIB_TAG_RESOURCES, msg_data, cib_options);
         free_xml(msg_data);
 
     } else {
         CMD_ERR("Unknown command: %c", rsc_cmd);
     }
 
   bail:
 
     if (is_ocf_rc) {
         exit_code = rc;
 
     } else if (rc != pcmk_ok) {
         CMD_ERR("Error performing operation: %s", pcmk_strerror(rc));
         if (rc == -pcmk_err_no_quorum) {
             CMD_ERR("To ignore quorum, use the force option");
         }
         if (exit_code == CRM_EX_OK) {
             exit_code = crm_errno2exit(rc);
         }
     }
 
     return bye(exit_code);
 }