diff --git a/cts/CTS.py.in b/cts/CTS.py.in
index c4183183e6..091bb1f8c1 100644
--- a/cts/CTS.py.in
+++ b/cts/CTS.py.in
@@ -1,939 +1,939 @@
 """ Main classes for Pacemaker's Cluster Test Suite (CTS)
 """
 
 # Pacemaker targets compatibility with Python 2.7 and 3.2+
 from __future__ import print_function, unicode_literals, absolute_import, division
 
 __copyright__ = "Copyright 2000-2018 the Pacemaker project contributors"
 __license__ = "GNU General Public License version 2 or later (GPLv2+) WITHOUT ANY WARRANTY"
 
 import os
 import re
 import sys
 import time
 import traceback
 
 if sys.version_info > (3,):
     from collections import UserDict
 else:
     from UserDict import UserDict
 
 from cts.CTSvars     import *
 from cts.logging     import LogFactory
 from cts.watcher     import LogWatcher
 from cts.remote      import RemoteFactory, input_wrapper
 from cts.environment import EnvFactory
 from cts.patterns    import PatternSelector
 
 has_log_stats = {}
 log_stats_bin = CTSvars.CRM_DAEMON_DIR + "/cts_log_stats.sh"
 log_stats = """
 #!@BASH_PATH@
 # Tool for generating system load reports while CTS runs
 
 trap "" 1
 
 f=$1; shift
 action=$1; shift
 base=`basename $0`
 
 if [ ! -e $f ]; then
     echo "Time, Load 1, Load 5, Load 15, Test Marker" > $f
 fi
 
 function killpid() {
     if [ -e $f.pid ]; then
        kill -9 `cat $f.pid`
        rm -f $f.pid
     fi
 }
 
 function status() {
     if [ -e $f.pid ]; then
        kill -0 `cat $f.pid`
        return $?
     else
        return 1
     fi
 }
 
 function start() {
     # Is it already running?
     if
         status
     then
         return
     fi
 
     echo Active as $$
     echo $$ > $f.pid
 
     while [ 1 = 1 ]; do
         uptime | sed s/up.*:/,/ | tr '\\n' ',' >> $f
         #top -b -c -n1 | grep -e usr/libexec/pacemaker | grep -v -e grep -e python | head -n 1 | sed s@/usr/libexec/pacemaker/@@ | awk '{print " 0, "$9", "$10", "$12}' | tr '\\n' ',' >> $f
         echo 0 >> $f
         sleep 5
     done
 }
 
 case $action in
     start)
         start
         ;;
     start-bg|bg)
         # Use c --ssh -- ./stats.sh file start-bg
         nohup $0 $f start >/dev/null 2>&1 </dev/null &
         ;;
     stop)
         killpid
         ;;
     delete)
         killpid
         rm -f $f
         ;;
     mark)
         uptime | sed s/up.*:/,/ | tr '\\n' ',' >> $f
         echo " $*" >> $f
         start
         ;;
     *)
         echo "Unknown action: $action."
         ;;
 esac
 """
 
 class CtsLab(object):
     '''This class defines the Lab Environment for the Cluster Test System.
     It defines those things which are expected to change from test
     environment to test environment for the same cluster manager.
 
     It is where you define the set of nodes that are in your test lab
     what kind of reset mechanism you use, etc.
 
     This class is derived from a UserDict because we hold many
     different parameters of different kinds, and this provides
     a uniform and extensible interface useful for any kind of
     communication between the user/administrator/tester and CTS.
 
     At this point in time, it is the intent of this class to model static
     configuration and/or environmental data about the environment which
     doesn't change as the tests proceed.
 
     Well-known names (keys) are an important concept in this class.
     The HasMinimalKeys member function knows the minimal set of
     well-known names for the class.
 
     The following names are standard (well-known) at this time:
 
         nodes           An array of the nodes in the cluster
         reset           A ResetMechanism object
         logger          An array of objects that log strings...
         CMclass         The type of ClusterManager we are running
                         (This is a class object, not a class instance)
         RandSeed        Random seed.  It is a triple of bytes. (optional)
 
     The CTS code ignores names it doesn't know about/need.
     The individual tests have access to this information, and it is
     perfectly acceptable to provide hints, tweaks, fine-tuning
     directions or other information to the tests through this mechanism.
     '''
 
     def __init__(self, args=None):
         self.Env = EnvFactory().getInstance(args)
         self.Scenario = None
         self.logger = LogFactory()
         self.rsh = RemoteFactory().getInstance()
 
     def dump(self):
         self.Env.dump()
 
     def has_key(self, key):
         return key in list(self.Env.keys())
 
     def __getitem__(self, key):
         return self.Env[key]
 
     def __setitem__(self, key, value):
         self.Env[key] = value
 
     def run(self, Scenario, Iterations):
         if not Scenario:
             self.logger.log("No scenario was defined")
             return 1
 
         self.logger.log("Cluster nodes: ")
         for node in self.Env["nodes"]:
             self.logger.log("    * %s" % (node))
 
         if not Scenario.SetUp():
             return 1
 
         try :
             Scenario.run(Iterations)
         except :
             self.logger.log("Exception by %s" % sys.exc_info()[0])
             self.logger.traceback(traceback)
 
             Scenario.summarize()
             Scenario.TearDown()
             return 1
 
         #ClusterManager.oprofileSave(Iterations)
         Scenario.TearDown()
 
         Scenario.summarize()
         if Scenario.Stats["failure"] > 0:
             return Scenario.Stats["failure"]
 
         elif Scenario.Stats["success"] != Iterations:
             self.logger.log("No failure count but success != requested iterations")
             return 1
 
         return 0
 
     def __CheckNode(self, node):
         "Raise a ValueError if the given node isn't valid"
 
         if not self.IsValidNode(node):
             raise ValueError("Invalid node [%s] in CheckNode" % node)
 
 class NodeStatus(object):
     def __init__(self, env):
         self.Env = env
 
     def IsNodeBooted(self, node):
         '''Return TRUE if the given node is booted (responds to pings)'''
         if self.Env["docker"]:
             return RemoteFactory().getInstance()("localhost", "docker inspect --format {{.State.Running}} %s | grep -q true" % node, silent=True) == 0
 
         return RemoteFactory().getInstance()("localhost", "ping -nq -c1 -w1 %s" % node, silent=True) == 0
 
     def IsSshdUp(self, node):
         rc = RemoteFactory().getInstance()(node, "true", silent=True)
         return rc == 0
 
     def WaitForNodeToComeUp(self, node, Timeout=300):
         '''Return TRUE when given node comes up, or None/FALSE if timeout'''
         timeout = Timeout
         anytimeouts = 0
         while timeout > 0:
             if self.IsNodeBooted(node) and self.IsSshdUp(node):
                 if anytimeouts:
                      # Fudge to wait for the system to finish coming up
                      time.sleep(30)
                      LogFactory().debug("Node %s now up" % node)
                 return 1
 
             time.sleep(30)
             if (not anytimeouts):
                 LogFactory().debug("Waiting for node %s to come up" % node)
 
             anytimeouts = 1
             timeout = timeout - 1
 
         LogFactory().log("%s did not come up within %d tries" % (node, Timeout))
         if self.Env["continue"] == 1:
             answer = "Y"
         else:
             try:
                 answer = input_wrapper('Continue? [nY]')
             except EOFError as e:
                 answer = "n"
         if answer and answer == "n":
             raise ValueError("%s did not come up within %d tries" % (node, Timeout))
 
     def WaitForAllNodesToComeUp(self, nodes, timeout=300):
         '''Return TRUE when all nodes come up, or FALSE if timeout'''
 
         for node in nodes:
             if not self.WaitForNodeToComeUp(node, timeout):
                 return None
         return 1
 
 
 class ClusterManager(UserDict):
     '''The Cluster Manager class.
     This is an subclass of the Python dictionary class.
     (this is because it contains lots of {name,value} pairs,
     not because it's behavior is that terribly similar to a
     dictionary in other ways.)
 
     This is an abstract class which class implements high-level
     operations on the cluster and/or its cluster managers.
     Actual cluster managers classes are subclassed from this type.
 
     One of the things we do is track the state we think every node should
     be in.
     '''
 
     def __InitialConditions(self):
         #if os.geteuid() != 0:
         #  raise ValueError("Must Be Root!")
         None
 
     def _finalConditions(self):
         for key in list(self.keys()):
             if self[key] == None:
                 raise ValueError("Improper derivation: self[" + key +   "] must be overridden by subclass.")
 
     def __init__(self, Environment, randseed=None):
         self.Env = EnvFactory().getInstance()
         self.templates = PatternSelector(self.Env["Name"])
         self.__InitialConditions()
         self.logger = LogFactory()
         self.TestLoggingLevel=0
         self.data = {}
         self.name = self.Env["Name"]
 
         self.rsh = RemoteFactory().getInstance()
         self.ShouldBeStatus={}
         self.ns = NodeStatus(self.Env)
         self.OurNode = os.uname()[1].lower()
         self.__instance_errorstoignore = []
 
     def __getitem__(self, key):
         if key == "Name":
             return self.name
 
         print("FIXME: Getting %s from %s" % (key, repr(self)))
         if key in self.data:
             return self.data[key]
 
         return self.templates.get_patterns(self.Env["Name"], key)
 
     def __setitem__(self, key, value):
         print("FIXME: Setting %s=%s on %s" % (key, value, repr(self)))
         self.data[key] = value
 
     def key_for_node(self, node):
         return node
 
     def instance_errorstoignore_clear(self):
         '''Allows the test scenario to reset instance errors to ignore on each iteration.'''
         self.__instance_errorstoignore = []
 
     def instance_errorstoignore(self):
         '''Return list of errors which are 'normal' for a specific test instance'''
         return self.__instance_errorstoignore
 
     def errorstoignore(self):
         '''Return list of errors which are 'normal' and should be ignored'''
         return []
 
     def log(self, args):
         self.logger.log(args)
 
     def debug(self, args):
         self.logger.debug(args)
 
     def prepare(self):
         '''Finish the Initialization process. Prepare to test...'''
 
         print(repr(self)+"prepare")
         for node in self.Env["nodes"]:
             if self.StataCM(node):
                 self.ShouldBeStatus[node] = "up"
             else:
                 self.ShouldBeStatus[node] = "down"
 
             if self.Env["experimental-tests"]:
                 self.unisolate_node(node)
 
     def upcount(self):
         '''How many nodes are up?'''
         count = 0
         for node in self.Env["nodes"]:
           if self.ShouldBeStatus[node] == "up":
             count = count + 1
         return count
 
     def install_support(self, command="install"):
         for node in self.Env["nodes"]:
             self.rsh(node, CTSvars.CRM_DAEMON_DIR + "/cts-support " + command)
 
     def install_config(self, node):
         return None
 
     def prepare_fencing_watcher(self, name):
         # If we don't have quorum now but get it as a result of starting this node,
         # then a bunch of nodes might get fenced
         upnode = None
         if self.HasQuorum(None):
             self.debug("Have quorum")
             return None
 
         if not self.templates["Pat:Fencing_start"]:
             print("No start pattern")
             return None
 
         if not self.templates["Pat:Fencing_ok"]:
             print("No ok pattern")
             return None
 
         stonith = None
         stonithPats = []
         for peer in self.Env["nodes"]:
             if self.ShouldBeStatus[peer] != "up":
                 stonithPats.append(self.templates["Pat:Fencing_ok"] % peer)
                 stonithPats.append(self.templates["Pat:Fencing_start"] % peer)
 
         stonith = LogWatcher(self.Env["LogFileName"], stonithPats, "StartupFencing", 0, hosts=self.Env["nodes"], kind=self.Env["LogWatcher"])
         stonith.setwatch()
         return stonith
 
     def fencing_cleanup(self, node, stonith):
         peer_list = []
         peer_state = {}
 
         self.debug("Looking for nodes that were fenced as a result of %s starting" % node)
 
         # If we just started a node, we may now have quorum (and permission to fence)
         if not stonith:
             self.debug("Nothing to do")
             return peer_list
 
         q = self.HasQuorum(None)
         if not q and len(self.Env["nodes"]) > 2:
             # We didn't gain quorum - we shouldn't have shot anyone
             self.debug("Quorum: %d Len: %d" % (q, len(self.Env["nodes"])))
             return peer_list
 
         for n in self.Env["nodes"]:
             peer_state[n] = "unknown"
 
         # Now see if any states need to be updated
         self.debug("looking for: " + repr(stonith.regexes))
         shot = stonith.look(0)
         while shot:
             line = repr(shot)
             self.debug("Found: " + line)
             del stonith.regexes[stonith.whichmatch]
 
             # Extract node name
             for n in self.Env["nodes"]:
                 if re.search(self.templates["Pat:Fencing_ok"] % n, shot):
                     peer = n
                     peer_state[peer] = "complete"
                     self.__instance_errorstoignore.append(self.templates["Pat:Fencing_ok"] % peer)
 
                 elif peer_state[n] != "complete" and re.search(self.templates["Pat:Fencing_start"] % n, shot):
                     # TODO: Correctly detect multiple fencing operations for the same host
                     peer = n
                     peer_state[peer] = "in-progress"
                     self.__instance_errorstoignore.append(self.templates["Pat:Fencing_start"] % peer)
 
             if not peer:
                 self.logger.log("ERROR: Unknown stonith match: %s" % line)
 
             elif not peer in peer_list:
                 self.debug("Found peer: " + peer)
                 peer_list.append(peer)
 
             # Get the next one
             shot = stonith.look(60)
 
         for peer in peer_list:
 
             self.debug("   Peer %s was fenced as a result of %s starting: %s" % (peer, node, peer_state[peer]))
             if self.Env["at-boot"]:
                 self.ShouldBeStatus[peer] = "up"
             else:
                 self.ShouldBeStatus[peer] = "down"
 
             if peer_state[peer] == "in-progress":
                 # Wait for any in-progress operations to complete
                 shot = stonith.look(60)
                 while len(stonith.regexes) and shot:
                     line = repr(shot)
                     self.debug("Found: " + line)
                     del stonith.regexes[stonith.whichmatch]
                     shot = stonith.look(60)
 
             # Now make sure the node is alive too
             self.ns.WaitForNodeToComeUp(peer, self.Env["DeadTime"])
 
             # Poll until it comes up
             if self.Env["at-boot"]:
                 if not self.StataCM(peer):
                     time.sleep(self.Env["StartTime"])
 
                 if not self.StataCM(peer):
                     self.logger.log("ERROR: Peer %s failed to restart after being fenced" % peer)
                     return None
 
         return peer_list
 
     def StartaCM(self, node, verbose=False):
 
         '''Start up the cluster manager on a given node'''
         if verbose: self.logger.log("Starting %s on node %s" % (self.templates["Name"], node))
         else: self.debug("Starting %s on node %s" % (self.templates["Name"], node))
         ret = 1
 
         if not node in self.ShouldBeStatus:
             self.ShouldBeStatus[node] = "down"
 
         if self.ShouldBeStatus[node] != "down":
             return 1
 
         patterns = []
         # Technically we should always be able to notice ourselves starting
         patterns.append(self.templates["Pat:Local_started"] % node)
         if self.upcount() == 0:
             patterns.append(self.templates["Pat:DC_started"] % node)
         else:
             patterns.append(self.templates["Pat:NonDC_started"] % node)
 
         watch = LogWatcher(
             self.Env["LogFileName"], patterns, "StartaCM", self.Env["StartTime"]+10, hosts=self.Env["nodes"], kind=self.Env["LogWatcher"])
 
         self.install_config(node)
 
         self.ShouldBeStatus[node] = "any"
         if self.StataCM(node) and self.cluster_stable(self.Env["DeadTime"]):
             self.logger.log ("%s was already started" % (node))
             return 1
 
         stonith = self.prepare_fencing_watcher(node)
         watch.setwatch()
 
         if self.rsh(node, self.templates["StartCmd"]) != 0:
             self.logger.log ("Warn: Start command failed on node %s" % (node))
             self.fencing_cleanup(node, stonith)
             return None
 
         self.ShouldBeStatus[node] = "up"
         watch_result = watch.lookforall()
 
         if watch.unmatched:
             for regex in watch.unmatched:
                 self.logger.log ("Warn: Startup pattern not found: %s" % (regex))
 
         if watch_result and self.cluster_stable(self.Env["DeadTime"]):
             #self.debug("Found match: "+ repr(watch_result))
             self.fencing_cleanup(node, stonith)
             return 1
 
         elif self.StataCM(node) and self.cluster_stable(self.Env["DeadTime"]):
             self.fencing_cleanup(node, stonith)
             return 1
 
         self.logger.log ("Warn: Start failed for node %s" % (node))
         return None
 
     def StartaCMnoBlock(self, node, verbose=False):
 
         '''Start up the cluster manager on a given node with none-block mode'''
 
         if verbose: self.logger.log("Starting %s on node %s" % (self["Name"], node))
         else: self.debug("Starting %s on node %s" % (self["Name"], node))
 
         self.install_config(node)
         self.rsh(node, self.templates["StartCmd"], synchronous=0)
         self.ShouldBeStatus[node] = "up"
         return 1
 
     def StopaCM(self, node, verbose=False, force=False):
 
         '''Stop the cluster manager on a given node'''
 
         if verbose: self.logger.log("Stopping %s on node %s" % (self["Name"], node))
         else: self.debug("Stopping %s on node %s" % (self["Name"], node))
 
         if self.ShouldBeStatus[node] != "up" and force == False:
             return 1
 
         if self.rsh(node, self.templates["StopCmd"]) == 0:
             # Make sure we can continue even if corosync leaks
             # fdata-* is the old name
-            #self.rsh(node, "rm -f /dev/shm/qb-* /dev/shm/fdata-*")
+            #self.rsh(node, "rm -rf /dev/shm/qb-* /dev/shm/fdata-*")
             self.ShouldBeStatus[node] = "down"
             self.cluster_stable(self.Env["DeadTime"])
             return 1
         else:
             self.logger.log ("ERROR: Could not stop %s on node %s" % (self["Name"], node))
 
         return None
 
     def StopaCMnoBlock(self, node):
 
         '''Stop the cluster manager on a given node with none-block mode'''
 
         self.debug("Stopping %s on node %s" % (self["Name"], node))
 
         self.rsh(node, self.templates["StopCmd"], synchronous=0)
         self.ShouldBeStatus[node] = "down"
         return 1
 
     def cluster_stable(self, timeout = None):
         time.sleep(self.Env["StableTime"])
         return 1
 
     def node_stable(self, node):
         return 1
 
     def RereadCM(self, node):
 
         '''Force the cluster manager on a given node to reread its config
            This may be a no-op on certain cluster managers.
         '''
         rc=self.rsh(node, self.templates["RereadCmd"])
         if rc == 0:
             return 1
         else:
             self.logger.log ("Could not force %s on node %s to reread its config"
             %        (self["Name"], node))
         return None
 
     def StataCM(self, node):
 
         '''Report the status of the cluster manager on a given node'''
 
         out=self.rsh(node, self.templates["StatusCmd"] % node, 1)
         ret= (str.find(out, 'stopped') == -1)
 
         try:
             if ret:
                 if self.ShouldBeStatus[node] == "down":
                     self.logger.log(
                     "Node status for %s is %s but we think it should be %s"
                     %        (node, "up", self.ShouldBeStatus[node]))
             else:
                 if self.ShouldBeStatus[node] == "up":
                     self.logger.log(
                     "Node status for %s is %s but we think it should be %s"
                     %        (node, "down", self.ShouldBeStatus[node]))
         except KeyError:        pass
 
         if ret:
             self.ShouldBeStatus[node] = "up"
         else:
             self.ShouldBeStatus[node] = "down"
         return ret
 
     def startall(self, nodelist=None, verbose=False, quick=False):
 
         '''Start the cluster manager on every node in the cluster.
         We can do it on a subset of the cluster if nodelist is not None.
         '''
         map = {}
         if not nodelist:
             nodelist = self.Env["nodes"]
 
         for node in nodelist:
             if self.ShouldBeStatus[node] == "down":
                 self.ns.WaitForAllNodesToComeUp(nodelist, 300)
 
         if not quick:
             # This is used for "basic sanity checks", so only start one node ...
             if not self.StartaCM(node, verbose=verbose):
                 return 0
             return 1
 
         # Approximation of SimulStartList for --boot 
         watchpats = [ ]
         watchpats.append(self.templates["Pat:DC_IDLE"])
         for node in nodelist:
             watchpats.append(self.templates["Pat:Local_started"] % node)
             watchpats.append(self.templates["Pat:InfraUp"] % node)
             watchpats.append(self.templates["Pat:PacemakerUp"] % node)
 
         #   Start all the nodes - at about the same time...
         watch = LogWatcher(self.Env["LogFileName"], watchpats, "fast-start", self.Env["DeadTime"]+10, hosts=self.Env["nodes"], kind=self.Env["LogWatcher"])
         watch.setwatch()
 
         if not self.StartaCM(nodelist[0], verbose=verbose):
             return 0
         for node in nodelist:
             self.StartaCMnoBlock(node, verbose=verbose)
 
         watch.lookforall()
         if watch.unmatched:
             for regex in watch.unmatched:
                 self.logger.log ("Warn: Startup pattern not found: %s" % (regex))
 
         if not self.cluster_stable():
             self.logger.log("Cluster did not stabilize")
             return 0
 
         return 1
 
     def stopall(self, nodelist=None, verbose=False, force=False):
 
         '''Stop the cluster managers on every node in the cluster.
         We can do it on a subset of the cluster if nodelist is not None.
         '''
 
         ret = 1
         map = {}
         if not nodelist:
             nodelist = self.Env["nodes"]
         for node in self.Env["nodes"]:
             if self.ShouldBeStatus[node] == "up" or force == True:
                 if not self.StopaCM(node, verbose=verbose, force=force):
                     ret = 0
         return ret
 
     def rereadall(self, nodelist=None):
 
         '''Force the cluster managers on every node in the cluster
         to reread their config files.  We can do it on a subset of the
         cluster if nodelist is not None.
         '''
 
         map = {}
         if not nodelist:
             nodelist = self.Env["nodes"]
         for node in self.Env["nodes"]:
             if self.ShouldBeStatus[node] == "up":
                 self.RereadCM(node)
 
     def statall(self, nodelist=None):
 
         '''Return the status of the cluster managers in the cluster.
         We can do it on a subset of the cluster if nodelist is not None.
         '''
 
         result = {}
         if not nodelist:
             nodelist = self.Env["nodes"]
         for node in nodelist:
             if self.StataCM(node):
                 result[node] = "up"
             else:
                 result[node] = "down"
         return result
 
     def isolate_node(self, target, nodes=None):
         '''isolate the communication between the nodes'''
         if not nodes:
             nodes = self.Env["nodes"]
 
         for node in nodes:
             if node != target:
                 rc = self.rsh(target, self.templates["BreakCommCmd"] % self.key_for_node(node))
                 if rc != 0:
                     self.logger.log("Could not break the communication between %s and %s: %d" % (target, node, rc))
                     return None
                 else:
                     self.debug("Communication cut between %s and %s" % (target, node))
         return 1
 
     def unisolate_node(self, target, nodes=None):
         '''fix the communication between the nodes'''
         if not nodes:
             nodes = self.Env["nodes"]
 
         for node in nodes:
             if node != target:
                 restored = 0
 
                 # Limit the amount of time we have asynchronous connectivity for
                 # Restore both sides as simultaneously as possible
                 self.rsh(target, self.templates["FixCommCmd"] % self.key_for_node(node), synchronous=0)
                 self.rsh(node, self.templates["FixCommCmd"] % self.key_for_node(target), synchronous=0)
                 self.debug("Communication restored between %s and %s" % (target, node))
 
     def reducecomm_node(self,node):
         '''reduce the communication between the nodes'''
         rc = self.rsh(node, self.templates["ReduceCommCmd"]%(self.Env["XmitLoss"],self.Env["RecvLoss"]))
         if rc == 0:
             return 1
         else:
             self.logger.log("Could not reduce the communication between the nodes from node: %s" % node)
         return None
 
     def restorecomm_node(self,node):
         '''restore the saved communication between the nodes'''
         rc = 0
         if float(self.Env["XmitLoss"]) != 0 or float(self.Env["RecvLoss"]) != 0 :
             rc = self.rsh(node, self.templates["RestoreCommCmd"]);
         if rc == 0:
             return 1
         else:
             self.logger.log("Could not restore the communication between the nodes from node: %s" % node)
         return None
 
     def HasQuorum(self, node_list):
         "Return TRUE if the cluster currently has quorum"
         # If we are auditing a partition, then one side will
         #   have quorum and the other not.
         # So the caller needs to tell us which we are checking
         # If no value for node_list is specified... assume all nodes
         raise ValueError("Abstract Class member (HasQuorum)")
 
     def Components(self):
         raise ValueError("Abstract Class member (Components)")
 
     def oprofileStart(self, node=None):
         if not node:
             for n in self.Env["oprofile"]:
                 self.oprofileStart(n)
 
         elif node in self.Env["oprofile"]:
             self.debug("Enabling oprofile on %s" % node)
             self.rsh(node, "opcontrol --init")
             self.rsh(node, "opcontrol --setup --no-vmlinux --separate=lib --callgraph=20 --image=all")
             self.rsh(node, "opcontrol --start")
             self.rsh(node, "opcontrol --reset")
 
     def oprofileSave(self, test, node=None):
         if not node:
             for n in self.Env["oprofile"]:
                 self.oprofileSave(test, n)
 
         elif node in self.Env["oprofile"]:
             self.rsh(node, "opcontrol --dump")
             self.rsh(node, "opcontrol --save=cts.%d" % test)
             # Read back with: opreport -l session:cts.0 image:<directory>/c*
             if None:
                 self.rsh(node, "opcontrol --reset")
             else:
                 self.oprofileStop(node)
                 self.oprofileStart(node)
 
     def oprofileStop(self, node=None):
         if not node:
             for n in self.Env["oprofile"]:
                 self.oprofileStop(n)
 
         elif node in self.Env["oprofile"]:
             self.debug("Stopping oprofile on %s" % node)
             self.rsh(node, "opcontrol --reset")
             self.rsh(node, "opcontrol --shutdown 2>&1 > /dev/null")
 
 
     def StatsExtract(self):
         if not self.Env["stats"]:
             return
 
         for host in self.Env["nodes"]:
             log_stats_file = "%s/cts-stats.csv" % CTSvars.CRM_DAEMON_DIR
             if host in has_log_stats:
                 self.rsh(host, '''bash %s %s stop''' % (log_stats_bin, log_stats_file))
                 (rc, lines) = self.rsh(host, '''cat %s''' % log_stats_file, stdout=2)
                 self.rsh(host, '''bash %s %s delete''' % (log_stats_bin, log_stats_file))
 
                 fname = "cts-stats-%d-nodes-%s.csv" % (len(self.Env["nodes"]), host)
                 print("Extracted stats: %s" % fname)
                 fd = open(fname, "a")
                 fd.writelines(lines)
                 fd.close()
 
     def StatsMark(self, testnum):
         '''Mark the test number in the stats log'''
 
         global has_log_stats
         if not self.Env["stats"]:
             return
 
         for host in self.Env["nodes"]:
             log_stats_file = "%s/cts-stats.csv" % CTSvars.CRM_DAEMON_DIR
             if not host in has_log_stats:
 
                 global log_stats
                 global log_stats_bin
                 script=log_stats
                 #script = re.sub("\\\\", "\\\\", script)
                 script = re.sub('\"', '\\\"', script)
                 script = re.sub("'", "\'", script)
                 script = re.sub("`", "\`", script)
                 script = re.sub("\$", "\\\$", script)
 
                 self.debug("Installing %s on %s" % (log_stats_bin, host))
                 self.rsh(host, '''echo "%s" > %s''' % (script, log_stats_bin), silent=True)
                 self.rsh(host, '''bash %s %s delete''' % (log_stats_bin, log_stats_file))
                 has_log_stats[host] = 1
 
             # Now mark it
             self.rsh(host, '''bash %s %s mark %s''' % (log_stats_bin, log_stats_file, testnum), synchronous=0)
 
 
 class Resource(object):
     '''
     This is an HA resource (not a resource group).
     A resource group is just an ordered list of Resource objects.
     '''
 
     def __init__(self, cm, rsctype=None, instance=None):
         self.CM = cm
         self.ResourceType = rsctype
         self.Instance = instance
         self.needs_quorum = 1
 
     def Type(self):
         return self.ResourceType
 
     def Instance(self, nodename):
         return self.Instance
 
     def IsRunningOn(self, nodename):
         '''
         This member function returns true if our resource is running
         on the given node in the cluster.
         It is analagous to the "status" operation on SystemV init scripts and
         heartbeat scripts.  FailSafe calls it the "exclusive" operation.
         '''
         raise ValueError("Abstract Class member (IsRunningOn)")
         return None
 
     def IsWorkingCorrectly(self, nodename):
         '''
         This member function returns true if our resource is operating
         correctly on the given node in the cluster.
         OCF does not require this operation, but it might be called
         the Monitor operation, which is what FailSafe calls it.
         For remotely monitorable resources (like IP addresses), they *should*
         be monitored remotely for testing.
         '''
         raise ValueError("Abstract Class member (IsWorkingCorrectly)")
         return None
 
     def Start(self, nodename):
         '''
         This member function starts or activates the resource.
         '''
         raise ValueError("Abstract Class member (Start)")
         return None
 
     def Stop(self, nodename):
         '''
         This member function stops or deactivates the resource.
         '''
         raise ValueError("Abstract Class member (Stop)")
         return None
 
     def __repr__(self):
         if (self.Instance and len(self.Instance) > 1):
                 return "{" + self.ResourceType + "::" + self.Instance + "}"
         else:
                 return "{" + self.ResourceType + "}"
 
 
 class Component(object):
     def kill(self, node):
         None
 
 
 class Process(Component):
     def __init__(self, cm, name, process=None, dc_only=0, pats=[], dc_pats=[], badnews_ignore=[], common_ignore=[], triggersreboot=0):
         self.name = str(name)
         self.dc_only = dc_only
         self.pats = pats
         self.dc_pats = dc_pats
         self.CM = cm
         self.badnews_ignore = badnews_ignore
         self.badnews_ignore.extend(common_ignore)
         self.triggersreboot = triggersreboot
 
         if process:
             self.proc = str(process)
         else:
             self.proc = str(name)
         self.KillCmd = "killall -9 " + self.proc
 
     def kill(self, node):
         if self.CM.rsh(node, self.KillCmd) != 0:
             self.CM.log ("ERROR: Kill %s failed on node %s" % (self.name,node))
             return None
         return 1
diff --git a/cts/CTSaudits.py b/cts/CTSaudits.py
index b7e082729b..cc82171081 100755
--- a/cts/CTSaudits.py
+++ b/cts/CTSaudits.py
@@ -1,865 +1,865 @@
 """ Auditing classes for Pacemaker's Cluster Test Suite (CTS)
 """
 
 # Pacemaker targets compatibility with Python 2.7 and 3.2+
 from __future__ import print_function, unicode_literals, absolute_import, division
 
 __copyright__ = "Copyright 2000-2018 the Pacemaker project contributors"
 __license__ = "GNU General Public License version 2 or later (GPLv2+) WITHOUT ANY WARRANTY"
 
 import time, re, uuid
 from cts.watcher import LogWatcher
 from cts.remote import input_wrapper
 from cts.CTSvars import *
 
 class ClusterAudit(object):
 
     def __init__(self, cm):
         self.CM = cm
 
     def __call__(self):
          raise ValueError("Abstract Class member (__call__)")
     
     def is_applicable(self):
         '''Return TRUE if we are applicable in the current test configuration'''
         raise ValueError("Abstract Class member (is_applicable)")
         return 1
 
     def log(self, args):
         self.CM.log("audit: %s" % args)
 
     def debug(self, args):
         self.CM.debug("audit: %s" % args)
 
     def name(self):
          raise ValueError("Abstract Class member (name)")
 
 AllAuditClasses = [ ]
 
 
 class LogAudit(ClusterAudit):
 
     def name(self):
         return "LogAudit"
 
     def __init__(self, cm):
         self.CM = cm
         self.kinds = [ "combined syslog", "journal", "remote" ]
 
     def RestartClusterLogging(self, nodes=None):
         if not nodes:
             nodes = self.CM.Env["nodes"]
 
         self.CM.debug("Restarting logging on: %s" % repr(nodes))
 
         for node in nodes:
             if self.CM.Env["have_systemd"]:
                 if self.CM.rsh(node, "systemctl stop systemd-journald.socket") != 0:
                     self.CM.log ("ERROR: Cannot stop 'systemd-journald' on %s" % node)
                 if self.CM.rsh(node, "systemctl start systemd-journald.service") != 0:
                     self.CM.log ("ERROR: Cannot start 'systemd-journald' on %s" % node)
 
             if self.CM.rsh(node, "service %s restart" % self.CM.Env["syslogd"]) != 0:
                 self.CM.log ("ERROR: Cannot restart '%s' on %s" % (self.CM.Env["syslogd"], node))
 
     def TestLogging(self):
         patterns = []
         prefix   = "Test message from"
         suffix   = str(uuid.uuid4())
         watch    = {}
 
         for node in self.CM.Env["nodes"]:
             # Look for the node name in two places to make sure 
             # that syslog is logging with the correct hostname
             m = re.search("^([^.]+).*", node)
             if m:
                 simple = m.group(1)
             else:
                 simple = node
             patterns.append("%s.*%s %s %s" % (simple, prefix, node, suffix))
 
         watch_pref = self.CM.Env["LogWatcher"]
         if watch_pref == "any": 
             for k in self.kinds:
                 watch[k] = LogWatcher(self.CM.Env["LogFileName"], patterns, "LogAudit", 5, silent=True, hosts=self.CM.Env["nodes"], kind=k)
                 watch[k].setwatch()
         else:
             k = watch_pref
             watch[k] = LogWatcher(self.CM.Env["LogFileName"], patterns, "LogAudit", 5, silent=True, hosts=self.CM.Env["nodes"], kind=k)
             watch[k].setwatch()
 
         if watch_pref == "any": self.CM.log("Writing log with key: %s" % (suffix))
         for node in self.CM.Env["nodes"]:
             cmd = "logger -p %s.info %s %s %s" % (self.CM.Env["SyslogFacility"], prefix, node, suffix)
             if self.CM.rsh(node, cmd, synchronous=0, silent=True) != 0:
                 self.CM.log ("ERROR: Cannot execute remote command [%s] on %s" % (cmd, node))
 
         for k in self.kinds:
             if k in watch:
                 w = watch[k]
                 if watch_pref == "any": self.CM.log("Testing for %s logs" % (k))
                 w.lookforall(silent=True)
                 if not w.unmatched:
                     if watch_pref == "any": 
                         self.CM.log ("Continuing with %s-based log reader" % (w.kind))
                         self.CM.Env["LogWatcher"] = w.kind
                     return 1
 
         for k in list(watch.keys()):
             w = watch[k]
             if w.unmatched:
                 for regex in w.unmatched:
                     self.CM.log ("Test message [%s] not found in %s logs." % (regex, w.kind))
 
         return 0
 
     def __call__(self):
         max = 3
         attempt = 0
 
         self.CM.ns.WaitForAllNodesToComeUp(self.CM.Env["nodes"])
         while attempt <= max and self.TestLogging() == 0:
             attempt = attempt + 1
             self.RestartClusterLogging()
             time.sleep(60*attempt)
 
         if attempt > max:
             self.CM.log("ERROR: Cluster logging unrecoverable.")
             return 0
 
         return 1
     
     def is_applicable(self):
         if self.CM.Env["DoBSC"]:
             return 0
         if self.CM.Env["LogAuditDisabled"]:
             return 0
         return 1
 
 
 class DiskAudit(ClusterAudit):
 
     def name(self):
         return "DiskspaceAudit"
 
     def __init__(self, cm):
         self.CM = cm
 
     def __call__(self):
         result = 1
         # @TODO Use directory of PCMK_logfile if set on host
         dfcmd = "df -BM " + CTSvars.CRM_LOG_DIR + " | tail -1 | awk '{print $(NF-1)\" \"$(NF-2)}' | tr -d 'M%'"
 
         self.CM.ns.WaitForAllNodesToComeUp(self.CM.Env["nodes"])
         for node in self.CM.Env["nodes"]:
             dfout = self.CM.rsh(node, dfcmd, 1)
             if not dfout:
                 self.CM.log ("ERROR: Cannot execute remote df command [%s] on %s" % (dfcmd, node))
             else:
                 try:
                     (used, remain) = dfout.split()
                     used_percent = int(used)
                     remaining_mb = int(remain)
                 except (ValueError, TypeError):
                     self.CM.log("Warning: df output '%s' from %s was invalid [%s, %s]"
                                 % (dfout, node, used, remain))
                 else:
                     if remaining_mb < 10 or used_percent > 95:
                         self.CM.log("CRIT: Out of log disk space on %s (%d%% / %dMB)"
                                     % (node, used_percent, remaining_mb))
                         result = None
                         if self.CM.Env["continue"] == 1:
                             answer = "Y"
                         else:
                             try:
                                 answer = input_wrapper('Continue? [nY]')
                             except EOFError as e:
                                 answer = "n"
 
                         if answer and answer == "n":
                             raise ValueError("Disk full on %s" % (node))
 
                     elif remaining_mb < 100 or used_percent > 90:
                         self.CM.log("WARN: Low on log disk space (%dMB) on %s" % (remaining_mb, node))
         return result
 
     def is_applicable(self):
         if self.CM.Env["DoBSC"]:
             return 0
         return 1
 
 
 class FileAudit(ClusterAudit):
 
     def name(self):
         return "FileAudit"
 
     def __init__(self, cm):
         self.CM = cm
         self.known = []
 
     def __call__(self):
         result = 1
 
         self.CM.ns.WaitForAllNodesToComeUp(self.CM.Env["nodes"])
         for node in self.CM.Env["nodes"]:
 
             (rc, lsout) = self.CM.rsh(node, "ls -al /var/lib/pacemaker/cores/* | grep core.[0-9]", None)
             for line in lsout:
                 line = line.strip()
                 if line not in self.known:
                     result = 0
                     self.known.append(line)
                     self.CM.log("Warning: Pacemaker core file on %s: %s" % (node, line))
 
             (rc, lsout) = self.CM.rsh(node, "ls -al /var/lib/corosync | grep core.[0-9]", None)
             for line in lsout:
                 line = line.strip()
                 if line not in self.known:
                     result = 0
                     self.known.append(line)
                     self.CM.log("Warning: Corosync core file on %s: %s" % (node, line))
 
             if node in self.CM.ShouldBeStatus and self.CM.ShouldBeStatus[node] == "down":
                 clean = 0
                 (rc, lsout) = self.CM.rsh(node, "ls -al /dev/shm | grep qb-", None)
                 for line in lsout:
                     result = 0
                     clean = 1
                     self.CM.log("Warning: Stale IPC file on %s: %s" % (node, line))
 
                 if clean:
                     (rc, lsout) = self.CM.rsh(node, "ps axf | grep -e pacemaker -e corosync", None)
                     for line in lsout:
                         self.CM.debug("ps[%s]: %s" % (node, line))
 
-                    self.CM.rsh(node, "rm -f /dev/shm/qb-*")
+                    self.CM.rsh(node, "rm -rf /dev/shm/qb-*")
 
             else:
                 self.CM.debug("Skipping %s" % node)
 
         return result
     
     def is_applicable(self):
         return 1
 
 
 class AuditResource(object):
     def __init__(self, cm, line):
         fields = line.split()
         self.CM = cm
         self.line = line
         self.type = fields[1]
         self.id = fields[2]
         self.clone_id = fields[3]
         self.parent = fields[4]
         self.rprovider = fields[5]
         self.rclass = fields[6]
         self.rtype = fields[7]
         self.host = fields[8]
         self.needs_quorum = fields[9]
         self.flags = int(fields[10])
         self.flags_s = fields[11]
 
         if self.parent == "NA":
             self.parent = None
 
     def unique(self):
         if self.flags & int("0x00000020", 16):
             return 1
         return 0
 
     def orphan(self):
         if self.flags & int("0x00000001", 16):
             return 1
         return 0
 
     def managed(self):
         if self.flags & int("0x00000002", 16):
             return 1
         return 0
             
 
 class AuditConstraint(object):
     def __init__(self, cm, line):
         fields = line.split()
         self.CM = cm
         self.line = line
         self.type = fields[1]
         self.id = fields[2]
         self.rsc = fields[3]
         self.target = fields[4]
         self.score = fields[5]
         self.rsc_role = fields[6]
         self.target_role = fields[7]
 
         if self.rsc_role == "NA":
             self.rsc_role = None
         if self.target_role == "NA":
             self.target_role = None
 
 
 class PrimitiveAudit(ClusterAudit):
     def name(self):
         return "PrimitiveAudit"
 
     def __init__(self, cm):
         self.CM = cm
 
     def doResourceAudit(self, resource, quorum):
         rc = 1
         active = self.CM.ResourceLocation(resource.id)
 
         if len(active) == 1:
             if quorum:
                 self.debug("Resource %s active on %s" % (resource.id, repr(active)))
                 
             elif resource.needs_quorum == 1:
                 self.CM.log("Resource %s active without quorum: %s" 
                             % (resource.id, repr(active)))
                 rc = 0
 
         elif not resource.managed():
             self.CM.log("Resource %s not managed. Active on %s"
                         % (resource.id, repr(active)))
 
         elif not resource.unique():
             # TODO: Figure out a clever way to actually audit these resource types
             if len(active) > 1:
                 self.debug("Non-unique resource %s is active on: %s" 
                               % (resource.id, repr(active)))
             else:
                 self.debug("Non-unique resource %s is not active" % resource.id)
 
         elif len(active) > 1:
             self.CM.log("Resource %s is active multiple times: %s" 
                         % (resource.id, repr(active)))
             rc = 0
             
         elif resource.orphan():
             self.debug("Resource %s is an inactive orphan" % resource.id)
 
         elif len(self.inactive_nodes) == 0:
             self.CM.log("WARN: Resource %s not served anywhere" % resource.id)
             rc = 0
 
         elif self.CM.Env["warn-inactive"] == 1:
             if quorum or not resource.needs_quorum:
                 self.CM.log("WARN: Resource %s not served anywhere (Inactive nodes: %s)" 
                             % (resource.id, repr(self.inactive_nodes)))
             else:
                 self.debug("Resource %s not served anywhere (Inactive nodes: %s)" 
                               % (resource.id, repr(self.inactive_nodes)))
 
         elif quorum or not resource.needs_quorum:
             self.debug("Resource %s not served anywhere (Inactive nodes: %s)" 
                           % (resource.id, repr(self.inactive_nodes)))
 
         return rc
 
     def setup(self):
         self.target = None
         self.resources = []
         self.constraints = []
         self.active_nodes = []
         self.inactive_nodes = []
 
         for node in self.CM.Env["nodes"]:
             if self.CM.ShouldBeStatus[node] == "up":
                 self.active_nodes.append(node)
             else:
                 self.inactive_nodes.append(node)
 
         for node in self.CM.Env["nodes"]:
             if self.target == None and self.CM.ShouldBeStatus[node] == "up":
                 self.target = node
 
         if not self.target:
             # TODO: In Pacemaker 1.0 clusters we'll be able to run crm_resource 
             # with CIB_file=/path/to/cib.xml even when the cluster isn't running
             self.debug("No nodes active - skipping %s" % self.name())
             return 0
 
         (rc, lines) = self.CM.rsh(self.target, "crm_resource -c", None)
 
         for line in lines:
             if re.search("^Resource", line):
                 self.resources.append(AuditResource(self.CM, line))
             elif re.search("^Constraint", line):
                 self.constraints.append(AuditConstraint(self.CM, line))
             else:
                 self.CM.log("Unknown entry: %s" % line);
 
         return 1
 
     def __call__(self):
         rc = 1
                 
         if not self.setup():
             return 1
 
         quorum = self.CM.HasQuorum(None)
         for resource in self.resources:
             if resource.type == "primitive":
                 if self.doResourceAudit(resource, quorum) == 0:
                     rc = 0
         return rc
 
     def is_applicable(self):
         # @TODO Due to long-ago refactoring, this name test would never match,
         # so this audit (and those derived from it) would never run.
         # Uncommenting the next lines fixes the name test, but that then
         # exposes pre-existing bugs that need to be fixed.
         #if self.CM["Name"] == "crm-corosync":
         #    return 1
         return 0
 
 
 class GroupAudit(PrimitiveAudit):
     def name(self):
         return "GroupAudit"
 
     def __call__(self):
         rc = 1
         if not self.setup():
             return 1
 
         for group in self.resources:
             if group.type == "group":
                 first_match = 1
                 group_location = None
                 for child in self.resources:
                     if child.parent == group.id:
                         nodes = self.CM.ResourceLocation(child.id)
 
                         if first_match and len(nodes) > 0:
                             group_location = nodes[0]
 
                         first_match = 0
 
                         if len(nodes) > 1:
                             rc = 0
                             self.CM.log("Child %s of %s is active more than once: %s" 
                                         % (child.id, group.id, repr(nodes)))
 
                         elif len(nodes) == 0:
                             # Groups are allowed to be partially active
                             # However we do need to make sure later children aren't running
                             group_location = None
                             self.debug("Child %s of %s is stopped" % (child.id, group.id))
 
                         elif nodes[0] != group_location:  
                             rc = 0
                             self.CM.log("Child %s of %s is active on the wrong node (%s) expected %s" 
                                         % (child.id, group.id, nodes[0], group_location))
                         else:
                             self.debug("Child %s of %s is active on %s" % (child.id, group.id, nodes[0]))
 
         return rc
     
 
 class CloneAudit(PrimitiveAudit):
     def name(self):
         return "CloneAudit"
 
     def __call__(self):
         rc = 1
         if not self.setup():
             return 1
 
         for clone in self.resources:
             if clone.type == "clone":
                 for child in self.resources:
                     if child.parent == clone.id and child.type == "primitive":
                         self.debug("Checking child %s of %s..." % (child.id, clone.id))
                         # Check max and node_max
                         # Obtain with:
                         #    crm_resource -g clone_max --meta -r child.id
                         #    crm_resource -g clone_node_max --meta -r child.id
 
         return rc
     
 
 class ColocationAudit(PrimitiveAudit):
     def name(self):
         return "ColocationAudit"
 
     def crm_location(self, resource):
         (rc, lines) = self.CM.rsh(self.target, "crm_resource -W -r %s -Q"%resource, None)
         hosts = []
         if rc == 0:
             for line in lines:
                 fields = line.split()
                 hosts.append(fields[0])
 
         return hosts
 
     def __call__(self):
         rc = 1
         if not self.setup():
             return 1
 
         for coloc in self.constraints:
             if coloc.type == "rsc_colocation":
                 source = self.crm_location(coloc.rsc)
                 target = self.crm_location(coloc.target)
                 if len(source) == 0:
                     self.debug("Colocation audit (%s): %s not running" % (coloc.id, coloc.rsc))
                 else:
                     for node in source:
                         if not node in target:
                             rc = 0
                             self.CM.log("Colocation audit (%s): %s running on %s (not in %s)" 
                                         % (coloc.id, coloc.rsc, node, repr(target)))
                         else:
                             self.debug("Colocation audit (%s): %s running on %s (in %s)" 
                                           % (coloc.id, coloc.rsc, node, repr(target)))
 
         return rc
 
 
 class ControllerStateAudit(ClusterAudit):
     def __init__(self, cm):
         self.CM = cm
         self.Stats = {"calls":0
         ,        "success":0
         ,        "failure":0
         ,        "skipped":0
         ,        "auditfail":0}
 
     def has_key(self, key):
         return key in self.Stats
 
     def __setitem__(self, key, value):
         self.Stats[key] = value
         
     def __getitem__(self, key):
         return self.Stats[key]
 
     def incr(self, name):
         '''Increment (or initialize) the value associated with the given name'''
         if not name in self.Stats:
             self.Stats[name] = 0
         self.Stats[name] = self.Stats[name]+1
 
     def __call__(self):
         passed = 1
         up_are_down = 0
         down_are_up = 0
         unstable_list = []
 
         for node in self.CM.Env["nodes"]:
             should_be = self.CM.ShouldBeStatus[node]
             rc = self.CM.test_node_CM(node)
             if rc > 0:
                 if should_be == "down":
                     down_are_up = down_are_up + 1
                 if rc == 1:
                     unstable_list.append(node)
             elif should_be == "up":
                 up_are_down = up_are_down + 1
 
         if len(unstable_list) > 0:
             passed = 0
             self.CM.log("Cluster is not stable: %d (of %d): %s" 
                      % (len(unstable_list), self.CM.upcount(), repr(unstable_list)))
 
         if up_are_down > 0:
             passed = 0
             self.CM.log("%d (of %d) nodes expected to be up were down."
                      % (up_are_down, len(self.CM.Env["nodes"])))
 
         if down_are_up > 0:
             passed = 0
             self.CM.log("%d (of %d) nodes expected to be down were up." 
                      % (down_are_up, len(self.CM.Env["nodes"])))
             
         return passed
 
     def name(self):
         return "ControllerStateAudit"
     
     def is_applicable(self):
         # @TODO Due to long-ago refactoring, this name test would never match,
         # so this audit (and those derived from it) would never run.
         # Uncommenting the next lines fixes the name test, but that then
         # exposes pre-existing bugs that need to be fixed.
         #if self.CM["Name"] == "crm-corosync":
         #    return 1
         return 0
 
 
 class CIBAudit(ClusterAudit):
     def __init__(self, cm):
         self.CM = cm
         self.Stats = {"calls":0
         ,        "success":0
         ,        "failure":0
         ,        "skipped":0
         ,        "auditfail":0}
 
     def has_key(self, key):
         return key in self.Stats
 
     def __setitem__(self, key, value):
         self.Stats[key] = value
         
     def __getitem__(self, key):
         return self.Stats[key]
     
     def incr(self, name):
         '''Increment (or initialize) the value associated with the given name'''
         if not name in self.Stats:
             self.Stats[name] = 0
         self.Stats[name] = self.Stats[name]+1
 
     def __call__(self):
         passed = 1
         ccm_partitions = self.CM.find_partitions()
 
         if len(ccm_partitions) == 0:
             self.debug("\tNo partitions to audit")
             return 1
         
         for partition in ccm_partitions:
             self.debug("\tAuditing CIB consistency for: %s" % partition)
             partition_passed = 0
             if self.audit_cib_contents(partition) == 0:
                 passed = 0
         
         return passed
 
     def audit_cib_contents(self, hostlist):
         passed = 1
         node0 = None
         node0_xml = None
 
         partition_hosts = hostlist.split()
         for node in partition_hosts:
             node_xml = self.store_remote_cib(node, node0)
 
             if node_xml == None:
                 self.CM.log("Could not perform audit: No configuration from %s" % node)
                 passed = 0
                 
             elif node0 == None:
                 node0 = node
                 node0_xml = node_xml
                 
             elif node0_xml == None: 
                 self.CM.log("Could not perform audit: No configuration from %s" % node0)
                 passed = 0
                 
             else:
                 (rc, result) = self.CM.rsh(
                     node0, "crm_diff -VV -cf --new %s --original %s" % (node_xml, node0_xml), None)
                 
                 if rc != 0:
                     self.CM.log("Diff between %s and %s failed: %d" % (node0_xml, node_xml, rc))
                     passed = 0
                     
                 for line in result:
                     if not re.search("<diff/>", line):
                         passed = 0
                         self.debug("CibDiff[%s-%s]: %s" % (node0, node, line)) 
                     else:
                         self.debug("CibDiff[%s-%s] Ignoring: %s" % (node0, node, line)) 
                         
 #            self.CM.rsh(node0, "rm -f %s" % node_xml)                        
 #        self.CM.rsh(node0, "rm -f %s" % node0_xml) 
         return passed
                 
     def store_remote_cib(self, node, target):
         combined = ""
         filename = "/tmp/ctsaudit.%s.xml" % node
 
         if not target:
             target = node
 
         (rc, lines) = self.CM.rsh(node, self.CM["CibQuery"], None)
         if rc != 0:
             self.CM.log("Could not retrieve configuration")
             return None
 
         self.CM.rsh("localhost", "rm -f %s" % filename)
         for line in lines:
             self.CM.rsh("localhost", "echo \'%s\' >> %s" % (line[:-1], filename), silent=True)
 
         if self.CM.rsh.cp(filename, "root@%s:%s" % (target, filename), silent=True) != 0:
             self.CM.log("Could not store configuration")
             return None
         return filename
 
     def name(self):
         return "CibAudit"
     
     def is_applicable(self):
         # @TODO Due to long-ago refactoring, this name test would never match,
         # so this audit (and those derived from it) would never run.
         # Uncommenting the next lines fixes the name test, but that then
         # exposes pre-existing bugs that need to be fixed.
         #if self.CM["Name"] == "crm-corosync":
         #    return 1
         return 0
 
 
 class PartitionAudit(ClusterAudit):
     def __init__(self, cm):
         self.CM = cm
         self.Stats = {"calls":0
         ,        "success":0
         ,        "failure":0
         ,        "skipped":0
         ,        "auditfail":0}
         self.NodeEpoch = {}
         self.NodeState = {}
         self.NodeQuorum = {}
 
     def has_key(self, key):
         return key in self.Stats
 
     def __setitem__(self, key, value):
         self.Stats[key] = value
         
     def __getitem__(self, key):
         return self.Stats[key]
     
     def incr(self, name):
         '''Increment (or initialize) the value associated with the given name'''
         if not name in self.Stats:
             self.Stats[name] = 0
         self.Stats[name] = self.Stats[name]+1
 
     def __call__(self):
         passed = 1
         ccm_partitions = self.CM.find_partitions()
 
         if ccm_partitions == None or len(ccm_partitions) == 0:
             return 1
 
         self.CM.cluster_stable(double_check=True)
 
         if len(ccm_partitions) != self.CM.partitions_expected:
             self.CM.log("ERROR: %d cluster partitions detected:" % len(ccm_partitions))
             passed = 0
             for partition in ccm_partitions:
                 self.CM.log("\t %s" % partition)
 
         for partition in ccm_partitions:
             partition_passed = 0
             if self.audit_partition(partition) == 0:
                 passed = 0
 
         return passed
 
     def trim_string(self, avalue):
         if not avalue:
             return None
         if len(avalue) > 1:
             return avalue[:-1]
 
     def trim2int(self, avalue):
         if not avalue:
             return None
         if len(avalue) > 1:
             return int(avalue[:-1])
 
     def audit_partition(self, partition):
         passed = 1
         dc_found = []
         dc_allowed_list = []
         lowest_epoch = None
         node_list = partition.split()
 
         self.debug("Auditing partition: %s" % (partition))
         for node in node_list:
             if self.CM.ShouldBeStatus[node] != "up":
                 self.CM.log("Warn: Node %s appeared out of nowhere" % (node))
                 self.CM.ShouldBeStatus[node] = "up"
                 # not in itself a reason to fail the audit (not what we're
                 #  checking for in this audit)
 
             self.NodeState[node]  = self.CM.rsh(node, self.CM["StatusCmd"] % node, 1)
             self.NodeEpoch[node] = self.CM.rsh(node, self.CM["EpochCmd"], 1)
             self.NodeQuorum[node] = self.CM.rsh(node, self.CM["QuorumCmd"], 1)
             
             self.debug("Node %s: %s - %s - %s." % (node, self.NodeState[node], self.NodeEpoch[node], self.NodeQuorum[node]))
             self.NodeState[node]  = self.trim_string(self.NodeState[node])
             self.NodeEpoch[node] = self.trim2int(self.NodeEpoch[node])
             self.NodeQuorum[node] = self.trim_string(self.NodeQuorum[node])
 
             if not self.NodeEpoch[node]:
                 self.CM.log("Warn: Node %s dissappeared: cant determin epoch" % (node))
                 self.CM.ShouldBeStatus[node] = "down"
                 # not in itself a reason to fail the audit (not what we're
                 #  checking for in this audit)
             elif lowest_epoch == None or self.NodeEpoch[node] < lowest_epoch:
                 lowest_epoch = self.NodeEpoch[node]
                 
         if not lowest_epoch:
             self.CM.log("Lowest epoch not determined in %s" % (partition))
             passed = 0
 
         for node in node_list:
             if self.CM.ShouldBeStatus[node] == "up":
                 if self.CM.is_node_dc(node, self.NodeState[node]):
                     dc_found.append(node)
                     if self.NodeEpoch[node] == lowest_epoch:
                         self.debug("%s: OK" % node)
                     elif not self.NodeEpoch[node]:
                         self.debug("Check on %s ignored: no node epoch" % node)
                     elif not lowest_epoch:
                         self.debug("Check on %s ignored: no lowest epoch" % node)
                     else:
                         self.CM.log("DC %s is not the oldest node (%d vs. %d)"
                             % (node, self.NodeEpoch[node], lowest_epoch))
                         passed = 0
 
         if len(dc_found) == 0:
             self.CM.log("DC not found on any of the %d allowed nodes: %s (of %s)"
                         % (len(dc_allowed_list), str(dc_allowed_list), str(node_list)))
 
         elif len(dc_found) > 1:
             self.CM.log("%d DCs (%s) found in cluster partition: %s"
                         % (len(dc_found), str(dc_found), str(node_list)))
             passed = 0
 
         if passed == 0:
             for node in node_list:
                 if self.CM.ShouldBeStatus[node] == "up":
                     self.CM.log("epoch %s : %s"  
                                 % (self.NodeEpoch[node], self.NodeState[node]))
 
         return passed
 
     def name(self):
         return "PartitionAudit"
     
     def is_applicable(self):
         # @TODO Due to long-ago refactoring, this name test would never match,
         # so this audit (and those derived from it) would never run.
         # Uncommenting the next lines fixes the name test, but that then
         # exposes pre-existing bugs that need to be fixed.
         #if self.CM["Name"] == "crm-corosync":
         #    return 1
         return 0
 
 AllAuditClasses.append(DiskAudit)
 AllAuditClasses.append(FileAudit)
 AllAuditClasses.append(LogAudit)
 AllAuditClasses.append(ControllerStateAudit)
 AllAuditClasses.append(PartitionAudit)
 AllAuditClasses.append(PrimitiveAudit)
 AllAuditClasses.append(GroupAudit)
 AllAuditClasses.append(CloneAudit)
 AllAuditClasses.append(ColocationAudit)
 AllAuditClasses.append(CIBAudit)
 
 
 def AuditList(cm):
     result = []
     for auditclass in AllAuditClasses:
         a = auditclass(cm)
         if a.is_applicable():
             result.append(a)
     return result
diff --git a/cts/patterns.py b/cts/patterns.py
index 96d6471c38..7eed90cd7d 100644
--- a/cts/patterns.py
+++ b/cts/patterns.py
@@ -1,413 +1,417 @@
 """ Pattern-holding classes for Pacemaker's Cluster Test Suite (CTS)
 """
 
 # Pacemaker targets compatibility with Python 2.7 and 3.2+
 from __future__ import print_function, unicode_literals, absolute_import, division
 
 __copyright__ = "Copyright 2008-2019 the Pacemaker project contributors"
 __license__ = "GNU General Public License version 2 or later (GPLv2+) WITHOUT ANY WARRANTY"
 
 import sys, os
 
 from cts.CTSvars import *
 
 patternvariants = {}
 class BasePatterns(object):
     def __init__(self, name):
         self.name = name
         patternvariants[name] = self
         self.ignore = [
             "avoid confusing Valgrind",
 
             # Logging bug in some versions of libvirtd
             r"libvirtd.*: internal error: Failed to parse PCI config address",
+
+            # pcs can log this when node is fenced, but fencing is OK in some
+            # tests (and we will catch it in pacemaker logs when not OK)
+            r"pcs.daemon:No response from: .* request: get_configs, error:",
         ]
         self.BadNews = []
         self.components = {}
         self.commands = {
             "StatusCmd"      : "crmadmin -t 60000 -S %s 2>/dev/null",
             "CibQuery"       : "cibadmin -Ql",
             "CibAddXml"      : "cibadmin --modify -c --xml-text %s",
             "CibDelXpath"    : "cibadmin --delete --xpath %s",
             # 300,000 == 5 minutes
             "RscRunning"     : CTSvars.CRM_DAEMON_DIR + "/cts-exec-helper -R -r %s",
             "CIBfile"        : "%s:"+CTSvars.CRM_CONFIG_DIR+"/cib.xml",
             "TmpDir"         : "/tmp",
 
             "BreakCommCmd"   : "iptables -A INPUT -s %s -j DROP >/dev/null 2>&1",
             "FixCommCmd"     : "iptables -D INPUT -s %s -j DROP >/dev/null 2>&1",
 
 # tc qdisc add dev lo root handle 1: cbq avpkt 1000 bandwidth 1000mbit
 # tc class add dev lo parent 1: classid 1:1 cbq rate "$RATE"kbps allot 17000 prio 5 bounded isolated
 # tc filter add dev lo parent 1: protocol ip prio 16 u32 match ip dst 127.0.0.1 match ip sport $PORT 0xFFFF flowid 1:1
 # tc qdisc add dev lo parent 1: netem delay "$LATENCY"msec "$(($LATENCY/4))"msec 10% 2> /dev/null > /dev/null
             "ReduceCommCmd"  : "",
             "RestoreCommCmd" : "tc qdisc del dev lo root",
 
             "MaintenanceModeOn"    : "cibadmin --modify -c --xml-text '<cluster_property_set id=\"cib-bootstrap-options\"><nvpair id=\"cts-maintenance-mode-setting\" name=\"maintenance-mode\" value=\"true\"/></cluster_property_set>'",
             "MaintenanceModeOff"    : "cibadmin --delete --xpath \"//nvpair[@name='maintenance-mode']\"",
 
             "StandbyCmd"      : "crm_attribute -Vq  -U %s -n standby -l forever -v %s 2>/dev/null",
             "StandbyQueryCmd" : "crm_attribute -qG -U %s -n standby -l forever -d off 2>/dev/null",
         }
         self.search = {
             "Pat:DC_IDLE"      : "pacemaker-controld.*State transition.*-> S_IDLE",
             
             # This won't work if we have multiple partitions
             "Pat:Local_started" : "%s\W.*controller successfully started",
             "Pat:NonDC_started" : r"%s\W.*State transition.*-> S_NOT_DC",
             "Pat:DC_started"    : r"%s\W.*State transition.*-> S_IDLE",
             "Pat:We_stopped"    : "%s\W.*OVERRIDE THIS PATTERN",
             "Pat:They_stopped"  : "%s\W.*LOST:.* %s ",
             "Pat:They_dead"     : "node %s.*: is dead",
             "Pat:TransitionComplete" : "Transition status: Complete: complete",
 
             "Pat:Fencing_start"   : r"Requesting peer fencing .* targeting %s",
             "Pat:Fencing_ok"      : r"pacemaker-fenced.*:\s*Operation .* targeting %s on .* for .*@.*: OK",
             "Pat:Fencing_recover" : r"pacemaker-schedulerd.*: Recover %s",
             "Pat:Fencing_active"  : r"pacemaker-schedulerd.*: Resource %s is active on .* nodes",
             "Pat:Fencing_probe"   : r"pacemaker-controld.* Result of probe operation for %s on .*: Error",
 
             "Pat:RscOpOK"       : r"pacemaker-controld.*:\s+Result of %s operation for %s.*: (0 \()?ok",
             "Pat:RscOpFail"     : r"pacemaker-schedulerd.*:.*Unexpected result .* recorded for %s of %s ",
             "Pat:CloneOpFail"   : r"pacemaker-schedulerd.*:.*Unexpected result .* recorded for %s of (%s|%s) ",
             "Pat:RscRemoteOpOK" : r"pacemaker-controld.*:\s+Result of %s operation for %s on %s: (0 \()?ok",
             "Pat:NodeFenced"    : r"pacemaker-controld.*:\s* Peer %s was terminated \(.*\) by .* on behalf of .*: OK",
             "Pat:FenceOpOK"     : "Operation .* for host '%s' with device .* returned: 0",
         }
 
     def get_component(self, key):
         if key in self.components:
             return self.components[key]
         print("Unknown component '%s' for %s" % (key, self.name))
         return []
 
     def get_patterns(self, key):
         if key == "BadNews":
             return self.BadNews
         elif key == "BadNewsIgnore":
             return self.ignore
         elif key == "Commands":
             return self.commands
         elif key == "Search":
             return self.search
         elif key == "Components":
             return self.components
 
     def __getitem__(self, key):
         if key == "Name":
             return self.name
         elif key in self.commands:
             return self.commands[key]
         elif key in self.search:
             return self.search[key]
         else:
             print("Unknown template '%s' for %s" % (key, self.name))
             return None
 
 
 class crm_corosync(BasePatterns):
     '''
     Patterns for Corosync version 2 cluster manager class
     '''
 
     def __init__(self, name):
         BasePatterns.__init__(self, name)
 
         self.commands.update({
             "StartCmd"       : "service corosync start && service pacemaker start",
             "StopCmd"        : "service pacemaker stop; [ ! -e /usr/sbin/pacemaker-remoted ] || service pacemaker_remote stop; service corosync stop",
 
             "EpochCmd"      : "crm_node -e",
             "QuorumCmd"      : "crm_node -q",
             "PartitionCmd"    : "crm_node -p",
         })
 
         self.search.update({
             # Close enough ... "Corosync Cluster Engine exiting normally" isn't
             # printed reliably.
             "Pat:We_stopped"   : "%s\W.*Unloading all Corosync service engines",
             "Pat:They_stopped" : "%s\W.*pacemaker-controld.*Node %s(\[|\s).*state is now lost",
             "Pat:They_dead"    : "pacemaker-controld.*Node %s(\[|\s).*state is now lost",
 
             "Pat:ChildExit"    : r"\[[0-9]+\] exited with status [0-9]+ \(",
             # "with signal 9" == pcmk_child_exit(), "$" == check_active_before_startup_processes()
             "Pat:ChildKilled"  : r"%s\W.*pacemakerd.*%s\[[0-9]+\] terminated( with signal 9|$)",
             "Pat:ChildRespawn" : "%s\W.*pacemakerd.*Respawning failed child process: %s",
 
             "Pat:InfraUp"      : "%s\W.*corosync.*Initializing transport",
             "Pat:PacemakerUp"  : "%s\W.*pacemakerd.*Starting Pacemaker",
         })
 
         self.ignore = self.ignore + [
             r"crm_mon:",
             r"crmadmin:",
             r"update_trace_data",
             r"async_notify:.*strange, client not found",
             r"Parse error: Ignoring unknown option .*nodename",
             r"error.*: Operation 'reboot' .* with device 'FencingFail' returned:",
             r"getinfo response error: 1$",
             r"sbd.* error: inquisitor_child: DEBUG MODE IS ACTIVE",
             r"sbd.* pcmk:\s*error:.*Connection to cib_ro.* (failed|closed)",
         ]
 
         self.BadNews = [
             r"[^(]error:",
             r"crit:",
             r"ERROR:",
             r"CRIT:",
             r"Shutting down...NOW",
             r"Timer I_TERMINATE just popped",
             r"input=I_ERROR",
             r"input=I_FAIL",
             r"input=I_INTEGRATED cause=C_TIMER_POPPED",
             r"input=I_FINALIZED cause=C_TIMER_POPPED",
             r"input=I_ERROR",
             r"(pacemakerd|pacemaker-execd|pacemaker-controld):.*, exiting",
             r"schedulerd.*Attempting recovery of resource",
             r"is taking more than 2x its timeout",
             r"Confirm not received from",
             r"Welcome reply not received from",
             r"Attempting to schedule .* after a stop",
             r"Resource .* was active at shutdown",
             r"duplicate entries for call_id",
             r"Search terminated:",
             r":global_timer_callback",
             r"Faking parameter digest creation",
             r"Parameters to .* action changed:",
             r"Parameters to .* changed",
             r"pacemakerd.*\[[0-9]+\] terminated( with signal| as IPC server|$)",
             r"pacemaker-schedulerd.*Recover .*\(.* -\> .*\)",
             r"rsyslogd.* imuxsock lost .* messages from pid .* due to rate-limiting",
             r"Peer is not part of our cluster",
             r"We appear to be in an election loop",
             r"Unknown node -> we will not deliver message",
             r"(Blackbox dump requested|Problem detected)",
             r"pacemakerd.*Could not connect to Cluster Configuration Database API",
             r"Receiving messages from a node we think is dead",
             r"share the same cluster nodeid",
             r"share the same name",
 
             #r"crm_ipc_send:.*Request .* failed",
             #r"crm_ipc_send:.*Sending to .* is disabled until pending reply is received",
 
                 # Not inherently bad, but worth tracking
             #r"No need to invoke the TE",
             #r"ping.*: DEBUG: Updated connected = 0",
             #r"Digest mis-match:",
             r"pacemaker-controld:.*Transition failed: terminated",
             r"Local CIB .* differs from .*:",
             r"warn.*:\s*Continuing but .* will NOT be used",
             r"warn.*:\s*Cluster configuration file .* is corrupt",
             #r"Executing .* fencing operation",
             r"Election storm",
             r"stalled the FSA with pending inputs",
         ]
 
         self.components["common-ignore"] = [
             r"Pending action:",
             r"resource( was|s were) active at shutdown",
             r"pending LRM operations at shutdown",
             r"Lost connection to the CIB manager",
             r"pacemaker-controld.*:\s*Action A_RECOVER .* not supported",
             r"pacemaker-controld.*:\s*Performing A_EXIT_1 - forcefully exiting ",
             r".*:\s*Executing .* fencing operation \(.*\) on ",
             r".*:\s*Requesting fencing \([^)]+\) of node ",
             r"(Blackbox dump requested|Problem detected)",
 #           "Resource .*stonith::.* is active on 2 nodes attempting recovery",
 #           "Transition .* ERRORs found during PE processing",
         ]
         
         self.components["corosync-ignore"] = [
             r"error:.*Connection to the CPG API failed: Library error",
             r"\[[0-9]+\] exited with status [0-9]+ \(",
             r"pacemaker-based.*error:.*Corosync connection lost",
             r"pacemaker-fenced.*error:.*Corosync connection terminated",
             r"pacemaker-controld.*State transition .* S_RECOVERY",
             r"pacemaker-controld.*error:.*Input (I_ERROR|I_TERMINATE ) .*received in state",
             r"pacemaker-controld.*error:.*Could not recover from internal error",
             r"error:.*Connection to cib_(shm|rw).* (failed|closed)",
             r"error:.*Connection to (fencer|stonith-ng).* (closed|failed|lost)",
             r"crit: Fencing daemon connection failed",
             # This is overbroad, but we don't have a way to say that only
             # certain transition errors are acceptable (if the fencer respawns,
             # fence devices may appear multiply active). We have to rely on
             # other causes of a transition error logging their own error
             # message, which is the usual practice.
             r"pacemaker-schedulerd.* Calculated transition .*/pe-error",
             ]
 
         self.components["corosync"] = [
             # We expect each daemon to lose its cluster connection.
             # However, if the CIB manager loses its connection first,
             # it's possible for another daemon to lose that connection and
             # exit before losing the cluster connection.
             r"pacemakerd.*:\s*(crit|error):.*Lost connection to cluster layer",
             r"pacemaker-attrd.*:\s*(crit|error):.*Lost connection to (cluster layer|the CIB manager)",
             r"pacemaker-based.*:\s*(crit|error):.*Lost connection to cluster layer",
             r"pacemaker-controld.*:\s*(crit|error):.*Lost connection to (cluster layer|the CIB manager)",
             r"pacemaker-fenced.*:\s*(crit|error):.*Lost connection to (cluster layer|the CIB manager)",
             r"schedulerd.*Scheduling Node .* for STONITH",
             r"pacemaker-controld.*:\s*Peer .* was terminated \(.*\) by .* on behalf of .*:\s*OK",
         ]
 
         self.components["pacemaker-based"] = [
             r"pacemakerd.* pacemaker-attrd\[[0-9]+\] exited with status 102",
             r"pacemakerd.* pacemaker-controld\[[0-9]+\] exited with status 1",
             r"pacemakerd.* Respawning failed child process: pacemaker-attrd",
             r"pacemakerd.* Respawning failed child process: pacemaker-based",
             r"pacemakerd.* Respawning failed child process: pacemaker-controld",
             r"pacemakerd.* Respawning failed child process: pacemaker-fenced",
             r"pacemaker-.* Connection to cib_.* (failed|closed)",
             r"pacemaker-attrd.*:.*Lost connection to the CIB manager",
             r"pacemaker-controld.*:.*Lost connection to the CIB manager",
             r"pacemaker-controld.*I_ERROR.*crmd_cib_connection_destroy",
             r"pacemaker-controld.* State transition .* S_RECOVERY",
             r"pacemaker-controld.*: Input I_TERMINATE .*from do_recover",
             r"pacemaker-controld.*Could not recover from internal error",
         ]
         self.components["pacemaker-based-ignore"] = [
             r"pacemaker-execd.*Connection to (fencer|stonith-ng).* (closed|failed|lost)",
             # This is overbroad, but we don't have a way to say that only
             # certain transition errors are acceptable (if the fencer respawns,
             # fence devices may appear multiply active). We have to rely on
             # other causes of a transition error logging their own error
             # message, which is the usual practice.
             r"pacemaker-schedulerd.* Calculated transition .*/pe-error",
         ]
 
         self.components["pacemaker-execd"] = [
             r"pacemaker-controld.*Connection to (pacemaker-execd|lrmd|executor) (failed|closed)",
             r"pacemaker-controld.*I_ERROR.*lrm_connection_destroy",
             r"pacemaker-controld.*State transition .* S_RECOVERY",
             r"pacemaker-controld.*: Input I_TERMINATE .*from do_recover",
             r"pacemaker-controld.*Could not recover from internal error",
             r"pacemakerd.*pacemaker-controld\[[0-9]+\] exited with status 1",
             r"pacemakerd.*Respawning failed child process: pacemaker-execd",
             r"pacemakerd.*Respawning failed child process: pacemaker-controld",
         ]
         self.components["pacemaker-execd-ignore"] = [
             r"pacemaker-attrd.*Connection to lrmd (failed|closed)",
             r"pacemaker-(attrd|controld).*Could not execute alert",
         ]
 
         self.components["pacemaker-controld"] = [
 #                    "WARN: determine_online_status: Node .* is unclean",
 #                    "Scheduling Node .* for STONITH",
 #                    "Executing .* fencing operation",
 # Only if the node wasn't the DC:  "State transition S_IDLE",
                     "State transition .* -> S_IDLE",
                     ]
         self.components["pacemaker-controld-ignore"] = []
 
         self.components["pacemaker-attrd"] = []
         self.components["pacemaker-attrd-ignore"] = []
 
         self.components["pacemaker-schedulerd"] = [
                     "State transition .* S_RECOVERY",
                     r"Respawning failed child process: pacemaker-controld",
                     r"pacemaker-controld\[[0-9]+\] exited with status 1 \(",
                     "Connection to pengine failed",
                     "Connection to pengine.* closed",
                     r"Connection to the scheduler failed",
                     "pacemaker-controld.*I_ERROR.*save_cib_contents",
                     r"pacemaker-controld.*: Input I_TERMINATE .*from do_recover",
                     "pacemaker-controld.*Could not recover from internal error",
                     ]
         self.components["pacemaker-schedulerd-ignore"] = []
 
         self.components["pacemaker-fenced"] = [
             r"error:.*Connection to (fencer|stonith-ng).* (closed|failed|lost)",
             r"Fencing daemon connection failed",
             r"pacemaker-controld.*Fencer successfully connected",
         ]
         self.components["pacemaker-fenced-ignore"] = [
             r"error:.*Connection to (fencer|stonith-ng).* (closed|failed|lost)",
             r"crit:.*Fencing daemon connection failed",
             r"error:.*Fencer connection failed \(will retry\)",
             r"Connection to (fencer|stonith-ng) failed, finalizing .* pending operations",
             r"pacemaker-controld.*:\s+Result of .* operation for Fencing.*Error",
             # This is overbroad, but we don't have a way to say that only
             # certain transition errors are acceptable (if the fencer respawns,
             # fence devices may appear multiply active). We have to rely on
             # other causes of a transition error logging their own error
             # message, which is the usual practice.
             r"pacemaker-schedulerd.* Calculated transition .*/pe-error",
         ]
         self.components["pacemaker-fenced-ignore"].extend(self.components["common-ignore"])
 
 
 class crm_corosync_docker(crm_corosync):
     '''
     Patterns for Corosync version 2 cluster manager class
     '''
     def __init__(self, name):
         crm_corosync.__init__(self, name)
 
         self.commands.update({
             "StartCmd"       : "pcmk_start",
             "StopCmd"        : "pcmk_stop",
         })
 
 
 class PatternSelector(object):
 
     def __init__(self, name=None):
         self.name = name
         self.base = BasePatterns("crm-base")
 
         if not name:
             crm_corosync("crm-corosync")
         elif name == "crm-corosync":
             crm_corosync(name)
         elif name == "crm-corosync-docker":
             crm_corosync_docker(name)
 
     def get_variant(self, variant):
         if variant in patternvariants:
             return patternvariants[variant]
         print("defaulting to crm-base for %s" % variant)
         return self.base
 
     def get_patterns(self, variant, kind):
         return self.get_variant(variant).get_patterns(kind)
 
     def get_template(self, variant, key):
         v = self.get_variant(variant)
         return v[key]
 
     def get_component(self, variant, kind):
         return self.get_variant(variant).get_component(kind)
 
     def __getitem__(self, key):
         return self.get_template(self.name, key)
 
 # python cts/CTSpatt.py -k crm-corosync -t StartCmd
 if __name__ == '__main__':
 
     pdir=os.path.dirname(sys.path[0])
     sys.path.insert(0, pdir) # So that things work from the source directory
 
     kind=None
     template=None
 
     skipthis=None
     args=sys.argv[1:]
     for i in range(0, len(args)):
        if skipthis:
            skipthis=None
            continue
 
        elif args[i] == "-k" or args[i] == "--kind":
            skipthis=1
            kind = args[i+1]
 
        elif args[i] == "-t" or args[i] == "--template":
            skipthis=1
            template = args[i+1]
 
        else:
            print("Illegal argument " + args[i])
 
 
     print(PatternSelector(kind)[template])
diff --git a/daemons/fenced/fenced_commands.c b/daemons/fenced/fenced_commands.c
index 05c5437117..9c27d6176f 100644
--- a/daemons/fenced/fenced_commands.c
+++ b/daemons/fenced/fenced_commands.c
@@ -1,2812 +1,2821 @@
 /*
  * Copyright 2009-2020 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU General Public License version 2
  * or later (GPLv2+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #include <sys/param.h>
 #include <stdio.h>
 #include <sys/types.h>
 #include <sys/wait.h>
 #include <sys/stat.h>
 #include <unistd.h>
 #include <sys/utsname.h>
 
 #include <stdlib.h>
 #include <errno.h>
 #include <fcntl.h>
 #include <ctype.h>
 
 #include <crm/crm.h>
 #include <crm/msg_xml.h>
 #include <crm/common/ipc.h>
 #include <crm/common/ipc_internal.h>
 #include <crm/cluster/internal.h>
 #include <crm/common/mainloop.h>
 
 #include <crm/stonith-ng.h>
 #include <crm/fencing/internal.h>
 #include <crm/common/xml.h>
 
 #include <pacemaker-fenced.h>
 
 GHashTable *device_list = NULL;
 GHashTable *topology = NULL;
 GList *cmd_list = NULL;
 
 struct device_search_s {
     /* target of fence action */
     char *host;
     /* requested fence action */
     char *action;
     /* timeout to use if a device is queried dynamically for possible targets */
     int per_device_timeout;
     /* number of registered fencing devices at time of request */
     int replies_needed;
     /* number of device replies received so far */
     int replies_received;
     /* whether the target is eligible to perform requested action (or off) */
     bool allow_suicide;
 
     /* private data to pass to search callback function */
     void *user_data;
     /* function to call when all replies have been received */
     void (*callback) (GList * devices, void *user_data);
     /* devices capable of performing requested action (or off if remapping) */
     GListPtr capable;
 };
 
 static gboolean stonith_device_dispatch(gpointer user_data);
 static void st_child_done(GPid pid, int rc, const char *output, gpointer user_data);
 static void stonith_send_reply(xmlNode * reply, int call_options, const char *remote_peer,
                                const char *client_id);
 
 static void search_devices_record_result(struct device_search_s *search, const char *device,
                                          gboolean can_fence);
 
 typedef struct async_command_s {
 
     int id;
     int pid;
     int fd_stdout;
     int options;
     int default_timeout; /* seconds */
     int timeout; /* seconds */
 
     int start_delay; /* seconds */
     int delay_id;
 
     char *op;
     char *origin;
     char *client;
     char *client_name;
     char *remote_op_id;
 
     char *victim;
     uint32_t victim_nodeid;
     char *action;
     char *device;
     char *mode;
 
     GListPtr device_list;
     GListPtr device_next;
 
     void *internal_user_data;
     void (*done_cb) (GPid pid, int rc, const char *output, gpointer user_data);
     guint timer_sigterm;
     guint timer_sigkill;
     /*! If the operation timed out, this is the last signal
      *  we sent to the process to get it to terminate */
     int last_timeout_signo;
 
     stonith_device_t *active_on;
     stonith_device_t *activating_on;
 } async_command_t;
 
 static xmlNode *stonith_construct_async_reply(async_command_t * cmd, const char *output,
                                               xmlNode * data, int rc);
 
 static gboolean
 is_action_required(const char *action, stonith_device_t *device)
 {
     return device && device->automatic_unfencing && safe_str_eq(action, "on");
 }
 
 static int
 get_action_delay_max(stonith_device_t * device, const char * action)
 {
     const char *value = NULL;
     int delay_max = 0;
 
     if (pcmk__str_none_of(action, "off", "reboot", NULL)) {
         return 0;
     }
 
     value = g_hash_table_lookup(device->params, STONITH_ATTR_DELAY_MAX);
     if (value) {
        delay_max = crm_parse_interval_spec(value) / 1000;
     }
 
     return delay_max;
 }
 
 static int
 get_action_delay_base(stonith_device_t * device, const char * action)
 {
     const char *value = NULL;
     int delay_base = 0;
 
     if (pcmk__str_none_of(action, "off", "reboot", NULL)) {
         return 0;
     }
 
     value = g_hash_table_lookup(device->params, STONITH_ATTR_DELAY_BASE);
     if (value) {
        delay_base = crm_parse_interval_spec(value) / 1000;
     }
 
     return delay_base;
 }
 
 /*!
  * \internal
  * \brief Override STONITH timeout with pcmk_*_timeout if available
  *
  * \param[in] device           STONITH device to use
  * \param[in] action           STONITH action name
  * \param[in] default_timeout  Timeout to use if device does not have
  *                             a pcmk_*_timeout parameter for action
  *
  * \return Value of pcmk_(action)_timeout if available, otherwise default_timeout
  * \note For consistency, it would be nice if reboot/off/on timeouts could be
  *       set the same way as start/stop/monitor timeouts, i.e. with an
  *       <operation> entry in the fencing resource configuration. However that
  *       is insufficient because fencing devices may be registered directly via
  *       the fencer's register_device() API instead of going through the CIB
  *       (e.g. stonith_admin uses it for its -R option, and the executor uses it
  *       to ensure a device is registered when a command is issued). As device
  *       properties, pcmk_*_timeout parameters can be grabbed by the fencer when
  *       the device is registered, whether by CIB change or API call.
  */
 static int
 get_action_timeout(stonith_device_t * device, const char *action, int default_timeout)
 {
     if (action && device && device->params) {
         char buffer[64] = { 0, };
         const char *value = NULL;
 
         /* If "reboot" was requested but the device does not support it,
          * we will remap to "off", so check timeout for "off" instead
          */
         if (safe_str_eq(action, "reboot")
             && is_not_set(device->flags, st_device_supports_reboot)) {
             crm_trace("%s doesn't support reboot, using timeout for off instead",
                       device->id);
             action = "off";
         }
 
         /* If the device config specified an action-specific timeout, use it */
         snprintf(buffer, sizeof(buffer), "pcmk_%s_timeout", action);
         value = g_hash_table_lookup(device->params, buffer);
         if (value) {
             return atoi(value);
         }
     }
     return default_timeout;
 }
 
 static void
 free_async_command(async_command_t * cmd)
 {
     if (!cmd) {
         return;
     }
 
     if (cmd->delay_id) {
         g_source_remove(cmd->delay_id);
     }
 
     cmd_list = g_list_remove(cmd_list, cmd);
 
     g_list_free_full(cmd->device_list, free);
     free(cmd->device);
     free(cmd->action);
     free(cmd->victim);
     free(cmd->remote_op_id);
     free(cmd->client);
     free(cmd->client_name);
     free(cmd->origin);
     free(cmd->mode);
     free(cmd->op);
     free(cmd);
 }
 
 static async_command_t *
 create_async_command(xmlNode * msg)
 {
     async_command_t *cmd = NULL;
     xmlNode *op = get_xpath_object("//@" F_STONITH_ACTION, msg, LOG_ERR);
     const char *action = crm_element_value(op, F_STONITH_ACTION);
 
     CRM_CHECK(action != NULL, crm_log_xml_warn(msg, "NoAction"); return NULL);
 
     crm_log_xml_trace(msg, "Command");
     cmd = calloc(1, sizeof(async_command_t));
     crm_element_value_int(msg, F_STONITH_CALLID, &(cmd->id));
     crm_element_value_int(msg, F_STONITH_CALLOPTS, &(cmd->options));
     crm_element_value_int(msg, F_STONITH_TIMEOUT, &(cmd->default_timeout));
     cmd->timeout = cmd->default_timeout;
     // Value -1 means disable any static/random fencing delays
     crm_element_value_int(msg, F_STONITH_DELAY, &(cmd->start_delay));
 
     cmd->origin = crm_element_value_copy(msg, F_ORIG);
     cmd->remote_op_id = crm_element_value_copy(msg, F_STONITH_REMOTE_OP_ID);
     cmd->client = crm_element_value_copy(msg, F_STONITH_CLIENTID);
     cmd->client_name = crm_element_value_copy(msg, F_STONITH_CLIENTNAME);
     cmd->op = crm_element_value_copy(msg, F_STONITH_OPERATION);
     cmd->action = strdup(action);
     cmd->victim = crm_element_value_copy(op, F_STONITH_TARGET);
     cmd->mode = crm_element_value_copy(op, F_STONITH_MODE);
     cmd->device = crm_element_value_copy(op, F_STONITH_DEVICE);
 
     CRM_CHECK(cmd->op != NULL, crm_log_xml_warn(msg, "NoOp"); free_async_command(cmd); return NULL);
     CRM_CHECK(cmd->client != NULL, crm_log_xml_warn(msg, "NoClient"));
 
     cmd->done_cb = st_child_done;
     cmd_list = g_list_append(cmd_list, cmd);
     return cmd;
 }
 
 static int
 get_action_limit(stonith_device_t * device)
 {
     const char *value = NULL;
     int action_limit = 1;
 
     value = g_hash_table_lookup(device->params, STONITH_ATTR_ACTION_LIMIT);
     if (value) {
        action_limit = crm_parse_int(value, "1");
        if (action_limit == 0) {
            /* pcmk_action_limit should not be 0. Enforce it to be 1. */
            action_limit = 1;
        }
     }
 
     return action_limit;
 }
 
 static int
 get_active_cmds(stonith_device_t * device)
 {
     int counter = 0;
     GListPtr gIter = NULL;
     GListPtr gIterNext = NULL;
 
     CRM_CHECK(device != NULL, return 0);
 
     for (gIter = cmd_list; gIter != NULL; gIter = gIterNext) {
         async_command_t *cmd = gIter->data;
 
         gIterNext = gIter->next;
 
         if (cmd->active_on == device) {
             counter++;
         }
     }
 
     return counter;
 }
 
 static void
 fork_cb(GPid pid, gpointer user_data)
 {
     async_command_t *cmd = (async_command_t *) user_data;
     stonith_device_t * device =
         /* in case of a retry we've done the move from
            activating_on to active_on already
          */
         cmd->activating_on?cmd->activating_on:cmd->active_on;
 
     CRM_ASSERT(device);
     crm_debug("Operation '%s'%s%s on %s now running with pid=%d, timeout=%ds",
                   cmd->action, cmd->victim ? " targeting " : "", cmd->victim ? cmd->victim : "",
                   device->id, pid, cmd->timeout);
     cmd->active_on = device;
     cmd->activating_on = NULL;
 }
 
 static gboolean
 stonith_device_execute(stonith_device_t * device)
 {
     int exec_rc = 0;
     const char *action_str = NULL;
     const char *host_arg = NULL;
     async_command_t *cmd = NULL;
     stonith_action_t *action = NULL;
     int active_cmds = 0;
     int action_limit = 0;
     GListPtr gIter = NULL;
     GListPtr gIterNext = NULL;
 
     CRM_CHECK(device != NULL, return FALSE);
 
     active_cmds = get_active_cmds(device);
     action_limit = get_action_limit(device);
     if (action_limit > -1 && active_cmds >= action_limit) {
         crm_trace("%s is over its action limit of %d (%u active action%s)",
                   device->id, action_limit, active_cmds, active_cmds > 1 ? "s" : "");
         return TRUE;
     }
 
     for (gIter = device->pending_ops; gIter != NULL; gIter = gIterNext) {
         async_command_t *pending_op = gIter->data;
 
         gIterNext = gIter->next;
 
         if (pending_op && pending_op->delay_id) {
             crm_trace
                 ("Operation '%s'%s%s on %s was asked to run too early, waiting for start_delay timeout of %ds",
                  pending_op->action, pending_op->victim ? " targeting " : "",
                  pending_op->victim ? pending_op->victim : "",
                  device->id, pending_op->start_delay);
             continue;
         }
 
         device->pending_ops = g_list_remove_link(device->pending_ops, gIter);
         g_list_free_1(gIter);
 
         cmd = pending_op;
         break;
     }
 
     if (cmd == NULL) {
         crm_trace("Nothing further to do for %s for now", device->id);
         return TRUE;
     }
 
     if(safe_str_eq(device->agent, STONITH_WATCHDOG_AGENT)) {
         if(safe_str_eq(cmd->action, "reboot")) {
             pcmk_panic(__FUNCTION__);
             goto done;
 
         } else if(safe_str_eq(cmd->action, "off")) {
             pcmk_panic(__FUNCTION__);
             goto done;
 
         } else {
             crm_info("Faking success for %s watchdog operation", cmd->action);
             cmd->done_cb(0, 0, NULL, cmd);
             goto done;
         }
     }
 
 #if SUPPORT_CIBSECRETS
     if (pcmk__substitute_secrets(device->id, device->params) != pcmk_rc_ok) {
         /* replacing secrets failed! */
         if (safe_str_eq(cmd->action,"stop")) {
             /* don't fail on stop! */
             crm_info("proceeding with the stop operation for %s", device->id);
 
         } else {
             crm_err("failed to get secrets for %s, "
                     "considering resource not configured", device->id);
             exec_rc = PCMK_OCF_NOT_CONFIGURED;
             cmd->done_cb(0, exec_rc, NULL, cmd);
             goto done;
         }
     }
 #endif
 
     action_str = cmd->action;
     if (safe_str_eq(cmd->action, "reboot") && is_not_set(device->flags, st_device_supports_reboot)) {
         crm_warn("Agent '%s' does not advertise support for 'reboot', performing 'off' action instead", device->agent);
         action_str = "off";
     }
 
     if (is_set(device->flags, st_device_supports_parameter_port)) {
         host_arg = "port";
 
     } else if (is_set(device->flags, st_device_supports_parameter_plug)) {
         host_arg = "plug";
     }
 
     action = stonith_action_create(device->agent,
                                    action_str,
                                    cmd->victim,
                                    cmd->victim_nodeid,
                                    cmd->timeout, device->params,
                                    device->aliases, host_arg);
 
     /* for async exec, exec_rc is negative for early error exit
        otherwise handling of success/errors is done via callbacks */
     cmd->activating_on = device;
     exec_rc = stonith_action_execute_async(action, (void *)cmd,
                                            cmd->done_cb, fork_cb);
 
     if (exec_rc < 0) {
         crm_warn("Operation '%s'%s%s on %s failed: %s (%d)",
                  cmd->action, cmd->victim ? " targeting " : "", cmd->victim ? cmd->victim : "",
                  device->id, pcmk_strerror(exec_rc), exec_rc);
         cmd->activating_on = NULL;
         cmd->done_cb(0, exec_rc, NULL, cmd);
     }
 
 done:
     /* Device might get triggered to work by multiple fencing commands
      * simultaneously. Trigger the device again to make sure any
      * remaining concurrent commands get executed. */
     if (device->pending_ops) {
         mainloop_set_trigger(device->work);
     }
     return TRUE;
 }
 
 static gboolean
 stonith_device_dispatch(gpointer user_data)
 {
     return stonith_device_execute(user_data);
 }
 
 static gboolean
 start_delay_helper(gpointer data)
 {
     async_command_t *cmd = data;
     stonith_device_t *device = NULL;
 
     cmd->delay_id = 0;
     device = cmd->device ? g_hash_table_lookup(device_list, cmd->device) : NULL;
 
     if (device) {
         mainloop_set_trigger(device->work);
     }
 
     return FALSE;
 }
 
 static void
 schedule_stonith_command(async_command_t * cmd, stonith_device_t * device)
 {
     int delay_max = 0;
     int delay_base = 0;
     int requested_delay = cmd->start_delay;
 
     CRM_CHECK(cmd != NULL, return);
     CRM_CHECK(device != NULL, return);
 
     if (cmd->device) {
         free(cmd->device);
     }
 
     if (device->include_nodeid && cmd->victim) {
         crm_node_t *node = crm_get_peer(0, cmd->victim);
 
         cmd->victim_nodeid = node->id;
     }
 
     cmd->device = strdup(device->id);
     cmd->timeout = get_action_timeout(device, cmd->action, cmd->default_timeout);
 
     if (cmd->remote_op_id) {
         crm_debug("Scheduling '%s' action%s%s on %s for remote peer %s with op id (%s) (timeout=%ds)",
                   cmd->action,
                   cmd->victim ? " targeting " : "", cmd->victim ? cmd->victim : "",
                   device->id, cmd->origin, cmd->remote_op_id, cmd->timeout);
     } else {
         crm_debug("Scheduling '%s' action%s%s on %s for %s (timeout=%ds)",
                   cmd->action,
                   cmd->victim ? " targeting " : "", cmd->victim ? cmd->victim : "",
                   device->id, cmd->client, cmd->timeout);
     }
 
     device->pending_ops = g_list_append(device->pending_ops, cmd);
     mainloop_set_trigger(device->work);
 
     // Value -1 means disable any static/random fencing delays
     if (requested_delay < 0) {
         return;
     }
 
     delay_max = get_action_delay_max(device, cmd->action);
     delay_base = get_action_delay_base(device, cmd->action);
     if (delay_max == 0) {
         delay_max = delay_base;
     }
     if (delay_max < delay_base) {
         crm_warn("Base-delay (%ds) is larger than max-delay (%ds) "
                  "for %s on %s - limiting to max-delay",
                  delay_base, delay_max, cmd->action, device->id);
         delay_base = delay_max;
     }
     if (delay_max > 0) {
         // coverity[dont_call] We're not using rand() for security
         cmd->start_delay +=
             ((delay_max != delay_base)?(rand() % (delay_max - delay_base)):0)
             + delay_base;
     }
 
     if (cmd->start_delay > 0) {
         crm_notice("Delaying '%s' action%s%s on %s for %ds (timeout=%ds, "
                    "requested_delay=%ds, base=%ds, max=%ds)",
                    cmd->action,
                    cmd->victim ? " targeting " : "", cmd->victim ? cmd->victim : "",
                    device->id, cmd->start_delay, cmd->timeout,
                    requested_delay, delay_base, delay_max);
         cmd->delay_id =
             g_timeout_add_seconds(cmd->start_delay, start_delay_helper, cmd);
     }
 }
 
 static void
 free_device(gpointer data)
 {
     GListPtr gIter = NULL;
     stonith_device_t *device = data;
 
     g_hash_table_destroy(device->params);
     g_hash_table_destroy(device->aliases);
 
     for (gIter = device->pending_ops; gIter != NULL; gIter = gIter->next) {
         async_command_t *cmd = gIter->data;
 
         crm_warn("Removal of device '%s' purged operation '%s'", device->id, cmd->action);
         cmd->done_cb(0, -ENODEV, NULL, cmd);
     }
     g_list_free(device->pending_ops);
 
     g_list_free_full(device->targets, free);
 
     mainloop_destroy_trigger(device->work);
 
     free_xml(device->agent_metadata);
     free(device->namespace);
     free(device->on_target_actions);
     free(device->agent);
     free(device->id);
     free(device);
 }
 
 void free_device_list(void)
 {
     if (device_list != NULL) {
         g_hash_table_destroy(device_list);
         device_list = NULL;
     }
 }
 
 void
 init_device_list(void)
 {
     if (device_list == NULL) {
         device_list = g_hash_table_new_full(crm_str_hash, g_str_equal, NULL,
                                             free_device);
     }
 }
 
 static GHashTable *
 build_port_aliases(const char *hostmap, GListPtr * targets)
 {
     char *name = NULL;
     int last = 0, lpc = 0, max = 0, added = 0;
     GHashTable *aliases = crm_strcase_table_new();
 
     if (hostmap == NULL) {
         return aliases;
     }
 
     max = strlen(hostmap);
     for (; lpc <= max; lpc++) {
         switch (hostmap[lpc]) {
                 /* Assignment chars */
             case '=':
             case ':':
                 if (lpc > last) {
                     free(name);
                     name = calloc(1, 1 + lpc - last);
                     memcpy(name, hostmap + last, lpc - last);
                 }
                 last = lpc + 1;
                 break;
 
                 /* Delimeter chars */
                 /* case ',': Potentially used to specify multiple ports */
             case 0:
             case ';':
             case ' ':
             case '\t':
                 if (name) {
                     char *value = NULL;
 
                     value = calloc(1, 1 + lpc - last);
                     memcpy(value, hostmap + last, lpc - last);
 
                     crm_debug("Adding alias '%s'='%s'", name, value);
                     g_hash_table_replace(aliases, name, value);
                     if (targets) {
                         *targets = g_list_append(*targets, strdup(value));
                     }
                     value = NULL;
                     name = NULL;
                     added++;
 
                 } else if (lpc > last) {
                     crm_debug("Parse error at offset %d near '%s'", lpc - last, hostmap + last);
                 }
 
                 last = lpc + 1;
                 break;
         }
 
         if (hostmap[lpc] == 0) {
             break;
         }
     }
 
     if (added == 0) {
         crm_info("No host mappings detected in '%s'", hostmap);
     }
 
     free(name);
     return aliases;
 }
 
 GHashTable *metadata_cache = NULL;
 
 void
 free_metadata_cache(void) {
     if (metadata_cache != NULL) {
         g_hash_table_destroy(metadata_cache);
         metadata_cache = NULL;
     }
 }
 
 static void
 init_metadata_cache(void) {
     if (metadata_cache == NULL) {
         metadata_cache = crm_str_table_new();
     }
 }
 
 static xmlNode *
 get_agent_metadata(const char *agent)
 {
     xmlNode *xml = NULL;
     char *buffer = NULL;
 
     init_metadata_cache();
     buffer = g_hash_table_lookup(metadata_cache, agent);
     if(safe_str_eq(agent, STONITH_WATCHDOG_AGENT)) {
         return NULL;
 
     } else if(buffer == NULL) {
         stonith_t *st = stonith_api_new();
         int rc;
 
         if (st == NULL) {
             crm_warn("Could not get agent meta-data: "
                      "API memory allocation failed");
             return NULL;
         }
         rc = st->cmds->metadata(st, st_opt_sync_call, agent, NULL, &buffer, 10);
         stonith_api_delete(st);
         if (rc || !buffer) {
             crm_err("Could not retrieve metadata for fencing agent %s", agent);
             return NULL;
         }
         g_hash_table_replace(metadata_cache, strdup(agent), buffer);
     }
 
     xml = string2xml(buffer);
 
     return xml;
 }
 
 static gboolean
 is_nodeid_required(xmlNode * xml)
 {
     xmlXPathObjectPtr xpath = NULL;
 
     if (stand_alone) {
         return FALSE;
     }
 
     if (!xml) {
         return FALSE;
     }
 
     xpath = xpath_search(xml, "//parameter[@name='nodeid']");
     if (numXpathResults(xpath)  <= 0) {
         freeXpathObject(xpath);
         return FALSE;
     }
 
     freeXpathObject(xpath);
     return TRUE;
 }
 
 #define MAX_ACTION_LEN 256
 
 static char *
 add_action(char *actions, const char *action)
 {
     int offset = 0;
 
     if (actions == NULL) {
         actions = calloc(1, MAX_ACTION_LEN);
     } else {
         offset = strlen(actions);
     }
 
     if (offset > 0) {
         offset += snprintf(actions+offset, MAX_ACTION_LEN - offset, " ");
     }
     offset += snprintf(actions+offset, MAX_ACTION_LEN - offset, "%s", action);
 
     return actions;
 }
 
 static void
 read_action_metadata(stonith_device_t *device)
 {
     xmlXPathObjectPtr xpath = NULL;
     int max = 0;
     int lpc = 0;
 
     if (device->agent_metadata == NULL) {
         return;
     }
 
     xpath = xpath_search(device->agent_metadata, "//action");
     max = numXpathResults(xpath);
 
     if (max <= 0) {
         freeXpathObject(xpath);
         return;
     }
 
     for (lpc = 0; lpc < max; lpc++) {
         const char *on_target = NULL;
         const char *action = NULL;
         xmlNode *match = getXpathResult(xpath, lpc);
 
         CRM_LOG_ASSERT(match != NULL);
         if(match == NULL) { continue; };
 
         on_target = crm_element_value(match, "on_target");
         action = crm_element_value(match, "name");
 
         if(safe_str_eq(action, "list")) {
             set_bit(device->flags, st_device_supports_list);
         } else if(safe_str_eq(action, "status")) {
             set_bit(device->flags, st_device_supports_status);
         } else if(safe_str_eq(action, "reboot")) {
             set_bit(device->flags, st_device_supports_reboot);
         } else if (safe_str_eq(action, "on")) {
             /* "automatic" means the cluster will unfence node when it joins */
             const char *automatic = crm_element_value(match, "automatic");
 
             /* "required" is a deprecated synonym for "automatic" */
             const char *required = crm_element_value(match, "required");
 
             if (crm_is_true(automatic) || crm_is_true(required)) {
                 device->automatic_unfencing = TRUE;
             }
         }
 
         if (action && crm_is_true(on_target)) {
             device->on_target_actions = add_action(device->on_target_actions, action);
         }
     }
 
     freeXpathObject(xpath);
 }
 
 /*!
  * \internal
  * \brief Set a pcmk_*_action parameter if not already set
  *
  * \param[in,out] params  Device parameters
  * \param[in]     action  Name of action
  * \param[in]     value   Value to use if action is not already set
  */
 static void
 map_action(GHashTable *params, const char *action, const char *value)
 {
     char *key = crm_strdup_printf("pcmk_%s_action", action);
 
     if (g_hash_table_lookup(params, key)) {
         crm_warn("Ignoring %s='%s', see %s instead",
                  STONITH_ATTR_ACTION_OP, value, key);
         free(key);
     } else {
         crm_warn("Mapping %s='%s' to %s='%s'",
                  STONITH_ATTR_ACTION_OP, value, key, value);
         g_hash_table_insert(params, key, strdup(value));
     }
 }
 
 /*!
  * \internal
  * \brief Create device parameter table from XML
  *
  * \param[in]     name    Device name (used for logging only)
  * \param[in,out] params  Device parameters
  */
 static GHashTable *
 xml2device_params(const char *name, xmlNode *dev)
 {
     GHashTable *params = xml2list(dev);
     const char *value;
 
     /* Action should never be specified in the device configuration,
      * but we support it for users who are familiar with other software
      * that worked that way.
      */
     value = g_hash_table_lookup(params, STONITH_ATTR_ACTION_OP);
     if (value != NULL) {
         crm_warn("%s has '%s' parameter, which should never be specified in configuration",
                  name, STONITH_ATTR_ACTION_OP);
 
         if (*value == '\0') {
             crm_warn("Ignoring empty '%s' parameter", STONITH_ATTR_ACTION_OP);
 
         } else if (strcmp(value, "reboot") == 0) {
             crm_warn("Ignoring %s='reboot' (see stonith-action cluster property instead)",
                      STONITH_ATTR_ACTION_OP);
 
         } else if (strcmp(value, "off") == 0) {
             map_action(params, "reboot", value);
 
         } else {
             map_action(params, "off", value);
             map_action(params, "reboot", value);
         }
 
         g_hash_table_remove(params, STONITH_ATTR_ACTION_OP);
     }
 
     return params;
 }
 
 static stonith_device_t *
 build_device_from_xml(xmlNode * msg)
 {
     const char *value;
     xmlNode *dev = get_xpath_object("//" F_STONITH_DEVICE, msg, LOG_ERR);
     stonith_device_t *device = NULL;
     char *agent = crm_element_value_copy(dev, "agent");
 
     CRM_CHECK(agent != NULL, return device);
 
     device = calloc(1, sizeof(stonith_device_t));
 
     CRM_CHECK(device != NULL, {free(agent); return device;});
 
     device->id = crm_element_value_copy(dev, XML_ATTR_ID);
     device->agent = agent;
     device->namespace = crm_element_value_copy(dev, "namespace");
     device->params = xml2device_params(device->id, dev);
 
     value = g_hash_table_lookup(device->params, STONITH_ATTR_HOSTLIST);
     if (value) {
         device->targets = stonith__parse_targets(value);
     }
 
     value = g_hash_table_lookup(device->params, STONITH_ATTR_HOSTMAP);
     device->aliases = build_port_aliases(value, &(device->targets));
 
     device->agent_metadata = get_agent_metadata(device->agent);
     if (device->agent_metadata) {
         read_action_metadata(device);
         set_bit(device->flags,
                 stonith__device_parameter_flags(device->agent_metadata));
     }
 
     value = g_hash_table_lookup(device->params, "nodeid");
     if (!value) {
         device->include_nodeid = is_nodeid_required(device->agent_metadata);
     }
 
     value = crm_element_value(dev, "rsc_provides");
     if (safe_str_eq(value, "unfencing")) {
         device->automatic_unfencing = TRUE;
     }
 
     if (is_action_required("on", device)) {
         crm_info("The fencing device '%s' requires unfencing", device->id);
     }
 
     if (device->on_target_actions) {
         crm_info("The fencing device '%s' requires actions (%s) to be executed on the target node",
                  device->id, device->on_target_actions);
     }
 
     device->work = mainloop_add_trigger(G_PRIORITY_HIGH, stonith_device_dispatch, device);
     /* TODO: Hook up priority */
 
     return device;
 }
 
 static const char *
 target_list_type(stonith_device_t * dev)
 {
     const char *check_type = NULL;
 
     check_type = g_hash_table_lookup(dev->params, STONITH_ATTR_HOSTCHECK);
 
     if (check_type == NULL) {
 
         if (g_hash_table_lookup(dev->params, STONITH_ATTR_HOSTLIST)) {
             check_type = "static-list";
         } else if (g_hash_table_lookup(dev->params, STONITH_ATTR_HOSTMAP)) {
             check_type = "static-list";
         } else if(is_set(dev->flags, st_device_supports_list)){
             check_type = "dynamic-list";
         } else if(is_set(dev->flags, st_device_supports_status)){
             check_type = "status";
         } else {
             check_type = "none";
         }
     }
 
     return check_type;
 }
 
 static void
 schedule_internal_command(const char *origin,
                           stonith_device_t * device,
                           const char *action,
                           const char *victim,
                           int timeout,
                           void *internal_user_data,
                           void (*done_cb) (GPid pid, int rc, const char *output,
                                            gpointer user_data))
 {
     async_command_t *cmd = NULL;
 
     cmd = calloc(1, sizeof(async_command_t));
 
     cmd->id = -1;
     cmd->default_timeout = timeout ? timeout : 60;
     cmd->timeout = cmd->default_timeout;
     cmd->action = strdup(action);
     cmd->victim = victim ? strdup(victim) : NULL;
     cmd->device = strdup(device->id);
     cmd->origin = strdup(origin);
     cmd->client = strdup(crm_system_name);
     cmd->client_name = strdup(crm_system_name);
 
     cmd->internal_user_data = internal_user_data;
     cmd->done_cb = done_cb; /* cmd, not internal_user_data, is passed to 'done_cb' as the userdata */
 
     schedule_stonith_command(cmd, device);
 }
 
 gboolean
 string_in_list(GListPtr list, const char *item)
 {
     int lpc = 0;
     int max = g_list_length(list);
 
     for (lpc = 0; lpc < max; lpc++) {
         const char *value = g_list_nth_data(list, lpc);
 
         if (safe_str_eq(item, value)) {
             return TRUE;
         } else {
             crm_trace("%d: '%s' != '%s'", lpc, item, value);
         }
     }
     return FALSE;
 }
 
 static void
 status_search_cb(GPid pid, int rc, const char *output, gpointer user_data)
 {
     async_command_t *cmd = user_data;
     struct device_search_s *search = cmd->internal_user_data;
     stonith_device_t *dev = cmd->device ? g_hash_table_lookup(device_list, cmd->device) : NULL;
     gboolean can = FALSE;
 
     free_async_command(cmd);
 
     if (!dev) {
         search_devices_record_result(search, NULL, FALSE);
         return;
     }
 
     mainloop_set_trigger(dev->work);
 
     if (rc == 1 /* unknown */ ) {
         crm_trace("Host %s is not known by %s", search->host, dev->id);
 
     } else if (rc == 0 /* active */  || rc == 2 /* inactive */ ) {
         crm_trace("Host %s is known by %s", search->host, dev->id);
         can = TRUE;
 
     } else {
         crm_notice("Unknown result when testing if %s can fence %s: rc=%d", dev->id, search->host,
                    rc);
     }
     search_devices_record_result(search, dev->id, can);
 }
 
 static void
 dynamic_list_search_cb(GPid pid, int rc, const char *output, gpointer user_data)
 {
     async_command_t *cmd = user_data;
     struct device_search_s *search = cmd->internal_user_data;
     stonith_device_t *dev = cmd->device ? g_hash_table_lookup(device_list, cmd->device) : NULL;
     gboolean can_fence = FALSE;
 
     free_async_command(cmd);
 
     /* Host/alias must be in the list output to be eligible to be fenced
      *
      * Will cause problems if down'd nodes aren't listed or (for virtual nodes)
      *  if the guest is still listed despite being moved to another machine
      */
     if (!dev) {
         search_devices_record_result(search, NULL, FALSE);
         return;
     }
 
     mainloop_set_trigger(dev->work);
 
     /* If we successfully got the targets earlier, don't disable. */
     if (rc != 0 && !dev->targets) {
         crm_notice("Disabling port list queries for %s (%d): %s", dev->id, rc, output);
         /* Fall back to status */
         g_hash_table_replace(dev->params, strdup(STONITH_ATTR_HOSTCHECK), strdup("status"));
 
         g_list_free_full(dev->targets, free);
         dev->targets = NULL;
     } else if (!rc) {
         crm_info("Refreshing port list for %s", dev->id);
         g_list_free_full(dev->targets, free);
         dev->targets = stonith__parse_targets(output);
         dev->targets_age = time(NULL);
     }
 
     if (dev->targets) {
         const char *alias = g_hash_table_lookup(dev->aliases, search->host);
 
         if (!alias) {
             alias = search->host;
         }
         if (string_in_list(dev->targets, alias)) {
             can_fence = TRUE;
         }
     }
     search_devices_record_result(search, dev->id, can_fence);
 }
 
 /*!
  * \internal
  * \brief Returns true if any key in first is not in second or second has a different value for key
  */
 static int
 device_params_diff(GHashTable *first, GHashTable *second) {
     char *key = NULL;
     char *value = NULL;
     GHashTableIter gIter;
 
     g_hash_table_iter_init(&gIter, first);
     while (g_hash_table_iter_next(&gIter, (void **)&key, (void **)&value)) {
 
         if(strstr(key, "CRM_meta") == key) {
             continue;
         } else if(strcmp(key, "crm_feature_set") == 0) {
             continue;
         } else {
             char *other_value = g_hash_table_lookup(second, key);
 
             if (!other_value || safe_str_neq(other_value, value)) {
                 crm_trace("Different value for %s: %s != %s", key, other_value, value);
                 return 1;
             }
         }
     }
 
     return 0;
 }
 
 /*!
  * \internal
  * \brief Checks to see if an identical device already exists in the device_list
  */
 static stonith_device_t *
 device_has_duplicate(stonith_device_t * device)
 {
     stonith_device_t *dup = g_hash_table_lookup(device_list, device->id);
 
     if (!dup) {
         crm_trace("No match for %s", device->id);
         return NULL;
 
     } else if (safe_str_neq(dup->agent, device->agent)) {
         crm_trace("Different agent: %s != %s", dup->agent, device->agent);
         return NULL;
     }
 
     /* Use calculate_operation_digest() here? */
     if (device_params_diff(device->params, dup->params) ||
         device_params_diff(dup->params, device->params)) {
         return NULL;
     }
 
     crm_trace("Match");
     return dup;
 }
 
 int
 stonith_device_register(xmlNode * msg, const char **desc, gboolean from_cib)
 {
     stonith_device_t *dup = NULL;
     stonith_device_t *device = build_device_from_xml(msg);
 
     CRM_CHECK(device != NULL, return -ENOMEM);
 
     dup = device_has_duplicate(device);
     if (dup) {
         crm_debug("Device '%s' already existed in device list (%d active devices)", device->id,
                    g_hash_table_size(device_list));
         free_device(device);
         device = dup;
 
     } else {
         stonith_device_t *old = g_hash_table_lookup(device_list, device->id);
 
         if (from_cib && old && old->api_registered) {
             /* If the cib is writing over an entry that is shared with a stonith client,
              * copy any pending ops that currently exist on the old entry to the new one.
              * Otherwise the pending ops will be reported as failures
              */
             crm_info("Overwriting an existing entry for %s from the cib", device->id);
             device->pending_ops = old->pending_ops;
             device->api_registered = TRUE;
             old->pending_ops = NULL;
             if (device->pending_ops) {
                 mainloop_set_trigger(device->work);
             }
         }
         g_hash_table_replace(device_list, device->id, device);
 
         crm_notice("Added '%s' to the device list (%d active devices)", device->id,
                    g_hash_table_size(device_list));
     }
     if (desc) {
         *desc = device->id;
     }
 
     if (from_cib) {
         device->cib_registered = TRUE;
     } else {
         device->api_registered = TRUE;
     }
 
     return pcmk_ok;
 }
 
 int
 stonith_device_remove(const char *id, gboolean from_cib)
 {
     stonith_device_t *device = g_hash_table_lookup(device_list, id);
 
     if (!device) {
         crm_info("Device '%s' not found (%d active devices)", id, g_hash_table_size(device_list));
         return pcmk_ok;
     }
 
     if (from_cib) {
         device->cib_registered = FALSE;
     } else {
         device->verified = FALSE;
         device->api_registered = FALSE;
     }
 
     if (!device->cib_registered && !device->api_registered) {
         g_hash_table_remove(device_list, id);
         crm_info("Removed '%s' from the device list (%d active devices)",
                  id, g_hash_table_size(device_list));
     } else {
         crm_trace("Not removing '%s' from the device list (%d active devices) "
                   "- still %s%s_registered", id, g_hash_table_size(device_list),
                   device->cib_registered?"cib":"", device->api_registered?"api":"");
     }
     return pcmk_ok;
 }
 
 /*!
  * \internal
  * \brief Return the number of stonith levels registered for a node
  *
  * \param[in] tp  Node's topology table entry
  *
  * \return Number of non-NULL levels in topology entry
  * \note This function is used only for log messages.
  */
 static int
 count_active_levels(stonith_topology_t * tp)
 {
     int lpc = 0;
     int count = 0;
 
     for (lpc = 0; lpc < ST_LEVEL_MAX; lpc++) {
         if (tp->levels[lpc] != NULL) {
             count++;
         }
     }
     return count;
 }
 
 static void
 free_topology_entry(gpointer data)
 {
     stonith_topology_t *tp = data;
 
     int lpc = 0;
 
     for (lpc = 0; lpc < ST_LEVEL_MAX; lpc++) {
         if (tp->levels[lpc] != NULL) {
             g_list_free_full(tp->levels[lpc], free);
         }
     }
     free(tp->target);
     free(tp->target_value);
     free(tp->target_pattern);
     free(tp->target_attribute);
     free(tp);
 }
 
 void
 free_topology_list(void)
 {
     if (topology != NULL) {
         g_hash_table_destroy(topology);
         topology = NULL;
     }
 }
 
 void
 init_topology_list(void)
 {
     if (topology == NULL) {
         topology = g_hash_table_new_full(crm_str_hash, g_str_equal, NULL,
                                          free_topology_entry);
     }
 }
 
 char *stonith_level_key(xmlNode *level, int mode)
 {
     if(mode == -1) {
         mode = stonith_level_kind(level);
     }
 
     switch(mode) {
         case 0:
             return crm_element_value_copy(level, XML_ATTR_STONITH_TARGET);
         case 1:
             return crm_element_value_copy(level, XML_ATTR_STONITH_TARGET_PATTERN);
         case 2:
             {
                 const char *name = crm_element_value(level, XML_ATTR_STONITH_TARGET_ATTRIBUTE);
                 const char *value = crm_element_value(level, XML_ATTR_STONITH_TARGET_VALUE);
 
                 if(name && value) {
                     return crm_strdup_printf("%s=%s", name, value);
                 }
             }
         default:
             return crm_strdup_printf("Unknown-%d-%s", mode, ID(level));
     }
 }
 
 int stonith_level_kind(xmlNode * level)
 {
     int mode = 0;
     const char *target = crm_element_value(level, XML_ATTR_STONITH_TARGET);
 
     if(target == NULL) {
         mode++;
         target = crm_element_value(level, XML_ATTR_STONITH_TARGET_PATTERN);
     }
 
     if(stand_alone == FALSE && target == NULL) {
 
         mode++;
 
         if(crm_element_value(level, XML_ATTR_STONITH_TARGET_ATTRIBUTE) == NULL) {
             mode++;
 
         } else if(crm_element_value(level, XML_ATTR_STONITH_TARGET_VALUE) == NULL) {
             mode++;
         }
     }
 
     return mode;
 }
 
 static stonith_key_value_t *
 parse_device_list(const char *devices)
 {
     int lpc = 0;
     int max = 0;
     int last = 0;
     stonith_key_value_t *output = NULL;
 
     if (devices == NULL) {
         return output;
     }
 
     max = strlen(devices);
     for (lpc = 0; lpc <= max; lpc++) {
         if (devices[lpc] == ',' || devices[lpc] == 0) {
             char *line = strndup(devices + last, lpc - last);
 
             output = stonith_key_value_add(output, NULL, line);
             free(line);
 
             last = lpc + 1;
         }
     }
 
     return output;
 }
 
 /*!
  * \internal
  * \brief Register a STONITH level for a target
  *
  * Given an XML request specifying the target name, level index, and device IDs
  * for the level, this will create an entry for the target in the global topology
  * table if one does not already exist, then append the specified device IDs to
  * the entry's device list for the specified level.
  *
  * \param[in]  msg   XML request for STONITH level registration
  * \param[out] desc  If not NULL, will be set to string representation ("TARGET[LEVEL]")
  *
  * \return pcmk_ok on success, -EINVAL if XML does not specify valid level index
  */
 int
 stonith_level_register(xmlNode *msg, char **desc)
 {
     int id = 0;
     xmlNode *level;
     int mode;
     char *target;
 
     stonith_topology_t *tp;
     stonith_key_value_t *dIter = NULL;
     stonith_key_value_t *devices = NULL;
 
     /* Allow the XML here to point to the level tag directly, or wrapped in
      * another tag. If directly, don't search by xpath, because it might give
      * multiple hits (e.g. if the XML is the CIB).
      */
     if (safe_str_eq(TYPE(msg), XML_TAG_FENCING_LEVEL)) {
         level = msg;
     } else {
         level = get_xpath_object("//" XML_TAG_FENCING_LEVEL, msg, LOG_ERR);
     }
     CRM_CHECK(level != NULL, return -EINVAL);
 
     mode = stonith_level_kind(level);
     target = stonith_level_key(level, mode);
     crm_element_value_int(level, XML_ATTR_STONITH_INDEX, &id);
 
     if (desc) {
         *desc = crm_strdup_printf("%s[%d]", target, id);
     }
 
     /* Sanity-check arguments */
     if (mode >= 3 || (id <= 0) || (id >= ST_LEVEL_MAX)) {
         crm_trace("Could not add %s[%d] (%d) to the topology (%d active entries)", target, id, mode, g_hash_table_size(topology));
         free(target);
         crm_log_xml_err(level, "Bad topology");
         return -EINVAL;
     }
 
     /* Find or create topology table entry */
     tp = g_hash_table_lookup(topology, target);
     if (tp == NULL) {
         tp = calloc(1, sizeof(stonith_topology_t));
         tp->kind = mode;
         tp->target = target;
         tp->target_value = crm_element_value_copy(level, XML_ATTR_STONITH_TARGET_VALUE);
         tp->target_pattern = crm_element_value_copy(level, XML_ATTR_STONITH_TARGET_PATTERN);
         tp->target_attribute = crm_element_value_copy(level, XML_ATTR_STONITH_TARGET_ATTRIBUTE);
 
         g_hash_table_replace(topology, tp->target, tp);
         crm_trace("Added %s (%d) to the topology (%d active entries)",
                   target, mode, g_hash_table_size(topology));
     } else {
         free(target);
     }
 
     if (tp->levels[id] != NULL) {
         crm_info("Adding to the existing %s[%d] topology entry",
                  tp->target, id);
     }
 
     devices = parse_device_list(crm_element_value(level, XML_ATTR_STONITH_DEVICES));
     for (dIter = devices; dIter; dIter = dIter->next) {
         const char *device = dIter->value;
 
         crm_trace("Adding device '%s' for %s[%d]", device, tp->target, id);
         tp->levels[id] = g_list_append(tp->levels[id], strdup(device));
     }
     stonith_key_value_freeall(devices, 1, 1);
 
     crm_info("Target %s has %d active fencing levels",
              tp->target, count_active_levels(tp));
     return pcmk_ok;
 }
 
 int
 stonith_level_remove(xmlNode *msg, char **desc)
 {
     int id = 0;
     stonith_topology_t *tp;
     char *target;
 
     /* Unlike additions, removal requests should always have one level tag */
     xmlNode *level = get_xpath_object("//" XML_TAG_FENCING_LEVEL, msg, LOG_ERR);
 
     CRM_CHECK(level != NULL, return -EINVAL);
 
     target = stonith_level_key(level, -1);
     crm_element_value_int(level, XML_ATTR_STONITH_INDEX, &id);
     if (desc) {
         *desc = crm_strdup_printf("%s[%d]", target, id);
     }
 
     /* Sanity-check arguments */
     if (id >= ST_LEVEL_MAX) {
         free(target);
         return -EINVAL;
     }
 
     tp = g_hash_table_lookup(topology, target);
     if (tp == NULL) {
         crm_info("Topology for %s not found (%d active entries)",
                  target, g_hash_table_size(topology));
 
     } else if (id == 0 && g_hash_table_remove(topology, target)) {
         crm_info("Removed all %s related entries from the topology (%d active entries)",
                  target, g_hash_table_size(topology));
 
     } else if (id > 0 && tp->levels[id] != NULL) {
         g_list_free_full(tp->levels[id], free);
         tp->levels[id] = NULL;
 
         crm_info("Removed level '%d' from topology for %s (%d active levels remaining)",
                  id, target, count_active_levels(tp));
     }
 
     free(target);
     return pcmk_ok;
 }
 
 /*!
  * \internal
  * \brief Schedule an (asynchronous) action directly on a stonith device
  *
  * Handle a STONITH_OP_EXEC API message by scheduling a requested agent action
  * directly on a specified device. Only list, monitor, and status actions are
  * expected to use this call, though it should work with any agent command.
  *
  * \param[in]  msg     API message XML with desired action
  * \param[out] output  Unused
  *
  * \return -EINPROGRESS on success, -errno otherwise
  * \note If the action is monitor, the device must be registered via the API
  *       (CIB registration is not sufficient), because monitor should not be
  *       possible unless the device is "started" (API registered).
  */
 static int
 stonith_device_action(xmlNode * msg, char **output)
 {
     xmlNode *dev = get_xpath_object("//" F_STONITH_DEVICE, msg, LOG_ERR);
     xmlNode *op = get_xpath_object("//@" F_STONITH_ACTION, msg, LOG_ERR);
     const char *id = crm_element_value(dev, F_STONITH_DEVICE);
     const char *action = crm_element_value(op, F_STONITH_ACTION);
     async_command_t *cmd = NULL;
     stonith_device_t *device = NULL;
 
     if ((id == NULL) || (action == NULL)) {
         crm_info("Malformed API action request: device %s, action %s",
                  (id? id : "not specified"),
                  (action? action : "not specified"));
         return -EPROTO;
     }
 
     device = g_hash_table_lookup(device_list, id);
     if ((device == NULL)
         || (!device->api_registered && !strcmp(action, "monitor"))) {
 
         // Monitors may run only on "started" (API-registered) devices
         crm_info("Ignoring API '%s' action request because device %s not found",
                  action, id);
         return -ENODEV;
     }
 
     cmd = create_async_command(msg);
     if (cmd == NULL) {
         return -EPROTO;
     }
 
     schedule_stonith_command(cmd, device);
     return -EINPROGRESS;
 }
 
 static void
 search_devices_record_result(struct device_search_s *search, const char *device, gboolean can_fence)
 {
     search->replies_received++;
 
     if (can_fence && device) {
         search->capable = g_list_append(search->capable, strdup(device));
     }
 
     if (search->replies_needed == search->replies_received) {
 
         crm_debug("Finished Search. %d devices can perform action (%s) on node %s",
                   g_list_length(search->capable),
                   search->action ? search->action : "<unknown>",
                   search->host ? search->host : "<anyone>");
 
         search->callback(search->capable, search->user_data);
         free(search->host);
         free(search->action);
         free(search);
     }
 }
 
 /*!
  * \internal
  * \brief Check whether the local host is allowed to execute a fencing action
  *
  * \param[in] device         Fence device to check
  * \param[in] action         Fence action to check
  * \param[in] target         Hostname of fence target
  * \param[in] allow_suicide  Whether self-fencing is allowed for this operation
  *
  * \return TRUE if local host is allowed to execute action, FALSE otherwise
  */
 static gboolean
 localhost_is_eligible(const stonith_device_t *device, const char *action,
                       const char *target, gboolean allow_suicide)
 {
     gboolean localhost_is_target = safe_str_eq(target, stonith_our_uname);
 
     if (device && action && device->on_target_actions
         && strstr(device->on_target_actions, action)) {
         if (!localhost_is_target) {
             crm_trace("'%s' operation with %s can only be executed for localhost not %s",
                       action, device->id, target);
             return FALSE;
         }
 
     } else if (localhost_is_target && !allow_suicide) {
         crm_trace("'%s' operation does not support self-fencing", action);
         return FALSE;
     }
     return TRUE;
 }
 
 static void
 can_fence_host_with_device(stonith_device_t * dev, struct device_search_s *search)
 {
     gboolean can = FALSE;
     const char *check_type = NULL;
     const char *host = search->host;
     const char *alias = NULL;
 
     CRM_LOG_ASSERT(dev != NULL);
 
     if (dev == NULL) {
         goto search_report_results;
     } else if (host == NULL) {
         can = TRUE;
         goto search_report_results;
     }
 
     /* Short-circuit query if this host is not allowed to perform the action */
     if (safe_str_eq(search->action, "reboot")) {
         /* A "reboot" *might* get remapped to "off" then "on", so short-circuit
          * only if all three are disallowed. If only one or two are disallowed,
          * we'll report that with the results. We never allow suicide for
          * remapped "on" operations because the host is off at that point.
          */
         if (!localhost_is_eligible(dev, "reboot", host, search->allow_suicide)
             && !localhost_is_eligible(dev, "off", host, search->allow_suicide)
             && !localhost_is_eligible(dev, "on", host, FALSE)) {
             goto search_report_results;
         }
     } else if (!localhost_is_eligible(dev, search->action, host,
                                       search->allow_suicide)) {
         goto search_report_results;
     }
 
     alias = g_hash_table_lookup(dev->aliases, host);
     if (alias == NULL) {
         alias = host;
     }
 
     check_type = target_list_type(dev);
 
     if (safe_str_eq(check_type, "none")) {
         can = TRUE;
 
     } else if (safe_str_eq(check_type, "static-list")) {
 
         /* Presence in the hostmap is sufficient
          * Only use if all hosts on which the device can be active can always fence all listed hosts
          */
 
         if (string_in_list(dev->targets, host)) {
             can = TRUE;
         } else if (g_hash_table_lookup(dev->params, STONITH_ATTR_HOSTMAP)
                    && g_hash_table_lookup(dev->aliases, host)) {
             can = TRUE;
         }
 
     } else if (safe_str_eq(check_type, "dynamic-list")) {
         time_t now = time(NULL);
 
         if (dev->targets == NULL || dev->targets_age + 60 < now) {
             crm_trace("Running '%s' to check whether %s is eligible to fence %s (%s)",
                       check_type, dev->id, search->host, search->action);
 
             schedule_internal_command(__FUNCTION__, dev, "list", NULL,
                                       search->per_device_timeout, search, dynamic_list_search_cb);
 
             /* we'll respond to this search request async in the cb */
             return;
         }
 
         if (string_in_list(dev->targets, alias)) {
             can = TRUE;
         }
 
     } else if (safe_str_eq(check_type, "status")) {
         crm_trace("Running '%s' to check whether %s is eligible to fence %s (%s)",
                   check_type, dev->id, search->host, search->action);
         schedule_internal_command(__FUNCTION__, dev, "status", search->host,
                                   search->per_device_timeout, search, status_search_cb);
         /* we'll respond to this search request async in the cb */
         return;
     } else {
         crm_err("Invalid value for " STONITH_ATTR_HOSTCHECK ": %s", check_type);
         check_type = "Invalid " STONITH_ATTR_HOSTCHECK;
     }
 
     if (safe_str_eq(host, alias)) {
         crm_notice("%s is%s eligible to fence (%s) %s: %s",
                    dev->id, (can? "" : " not"), search->action, host,
                    check_type);
     } else {
         crm_notice("%s is%s eligible to fence (%s) %s (aka. '%s'): %s",
                    dev->id, (can? "" : " not"), search->action, host, alias,
                    check_type);
     }
 
   search_report_results:
     search_devices_record_result(search, dev ? dev->id : NULL, can);
 }
 
 static void
 search_devices(gpointer key, gpointer value, gpointer user_data)
 {
     stonith_device_t *dev = value;
     struct device_search_s *search = user_data;
 
     can_fence_host_with_device(dev, search);
 }
 
 #define DEFAULT_QUERY_TIMEOUT 20
 static void
 get_capable_devices(const char *host, const char *action, int timeout, bool suicide, void *user_data,
                     void (*callback) (GList * devices, void *user_data))
 {
     struct device_search_s *search;
     int per_device_timeout = DEFAULT_QUERY_TIMEOUT;
     int devices_needing_async_query = 0;
     char *key = NULL;
     const char *check_type = NULL;
     GHashTableIter gIter;
     stonith_device_t *device = NULL;
 
     if (!g_hash_table_size(device_list)) {
         callback(NULL, user_data);
         return;
     }
 
     search = calloc(1, sizeof(struct device_search_s));
     if (!search) {
         callback(NULL, user_data);
         return;
     }
 
     g_hash_table_iter_init(&gIter, device_list);
     while (g_hash_table_iter_next(&gIter, (void **)&key, (void **)&device)) {
         check_type = target_list_type(device);
         if (pcmk__str_any_of(check_type, "status", "dynamic-list", NULL)) {
             devices_needing_async_query++;
         }
     }
 
     /* If we have devices that require an async event in order to know what
      * nodes they can fence, we have to give the events a timeout. The total
      * query timeout is divided among those events. */
     if (devices_needing_async_query) {
         per_device_timeout = timeout / devices_needing_async_query;
         if (!per_device_timeout) {
             crm_err("STONITH timeout %ds is too low; using %ds, but consider raising to at least %ds",
                     timeout, DEFAULT_QUERY_TIMEOUT,
                     DEFAULT_QUERY_TIMEOUT * devices_needing_async_query);
             per_device_timeout = DEFAULT_QUERY_TIMEOUT;
         } else if (per_device_timeout < DEFAULT_QUERY_TIMEOUT) {
             crm_notice("STONITH timeout %ds is low for the current configuration;"
                        " consider raising to at least %ds",
                        timeout, DEFAULT_QUERY_TIMEOUT * devices_needing_async_query);
         }
     }
 
     search->host = host ? strdup(host) : NULL;
     search->action = action ? strdup(action) : NULL;
     search->per_device_timeout = per_device_timeout;
     /* We are guaranteed this many replies. Even if a device gets
      * unregistered some how during the async search, we will get
      * the correct number of replies. */
     search->replies_needed = g_hash_table_size(device_list);
     search->allow_suicide = suicide;
     search->callback = callback;
     search->user_data = user_data;
     /* kick off the search */
 
     crm_debug("Searching through %d devices to see what is capable of action (%s) for target %s",
               search->replies_needed,
               search->action ? search->action : "<unknown>",
               search->host ? search->host : "<anyone>");
     g_hash_table_foreach(device_list, search_devices, search);
 }
 
 struct st_query_data {
     xmlNode *reply;
     char *remote_peer;
     char *client_id;
     char *target;
     char *action;
     int call_options;
 };
 
 /*!
  * \internal
  * \brief Add action-specific attributes to query reply XML
  *
  * \param[in,out] xml     XML to add attributes to
  * \param[in]     action  Fence action
  * \param[in]     device  Fence device
  */
 static void
 add_action_specific_attributes(xmlNode *xml, const char *action,
                                stonith_device_t *device)
 {
     int action_specific_timeout;
     int delay_max;
     int delay_base;
 
     CRM_CHECK(xml && action && device, return);
 
     if (is_action_required(action, device)) {
         crm_trace("Action '%s' is required on %s", action, device->id);
         crm_xml_add_int(xml, F_STONITH_DEVICE_REQUIRED, 1);
     }
 
     action_specific_timeout = get_action_timeout(device, action, 0);
     if (action_specific_timeout) {
         crm_trace("Action '%s' has timeout %dms on %s",
                   action, action_specific_timeout, device->id);
         crm_xml_add_int(xml, F_STONITH_ACTION_TIMEOUT, action_specific_timeout);
     }
 
     delay_max = get_action_delay_max(device, action);
     if (delay_max > 0) {
         crm_trace("Action '%s' has maximum random delay %dms on %s",
                   action, delay_max, device->id);
         crm_xml_add_int(xml, F_STONITH_DELAY_MAX, delay_max / 1000);
     }
 
     delay_base = get_action_delay_base(device, action);
     if (delay_base > 0) {
         crm_xml_add_int(xml, F_STONITH_DELAY_BASE, delay_base / 1000);
     }
 
     if ((delay_max > 0) && (delay_base == 0)) {
         crm_trace("Action '%s' has maximum random delay %dms on %s",
                   action, delay_max, device->id);
     } else if ((delay_max == 0) && (delay_base > 0)) {
         crm_trace("Action '%s' has a static delay of %dms on %s",
                   action, delay_base, device->id);
     } else if ((delay_max > 0) && (delay_base > 0)) {
         crm_trace("Action '%s' has a minimum delay of %dms and a randomly chosen "
                   "maximum delay of %dms on %s",
                   action, delay_base, delay_max, device->id);
     }
 }
 
 /*!
  * \internal
  * \brief Add "disallowed" attribute to query reply XML if appropriate
  *
  * \param[in,out] xml            XML to add attribute to
  * \param[in]     action         Fence action
  * \param[in]     device         Fence device
  * \param[in]     target         Fence target
  * \param[in]     allow_suicide  Whether self-fencing is allowed
  */
 static void
 add_disallowed(xmlNode *xml, const char *action, stonith_device_t *device,
                const char *target, gboolean allow_suicide)
 {
     if (!localhost_is_eligible(device, action, target, allow_suicide)) {
         crm_trace("Action '%s' on %s is disallowed for local host",
                   action, device->id);
         crm_xml_add(xml, F_STONITH_ACTION_DISALLOWED, XML_BOOLEAN_TRUE);
     }
 }
 
 /*!
  * \internal
  * \brief Add child element with action-specific values to query reply XML
  *
  * \param[in,out] xml            XML to add attribute to
  * \param[in]     action         Fence action
  * \param[in]     device         Fence device
  * \param[in]     target         Fence target
  * \param[in]     allow_suicide  Whether self-fencing is allowed
  */
 static void
 add_action_reply(xmlNode *xml, const char *action, stonith_device_t *device,
                const char *target, gboolean allow_suicide)
 {
     xmlNode *child = create_xml_node(xml, F_STONITH_ACTION);
 
     crm_xml_add(child, XML_ATTR_ID, action);
     add_action_specific_attributes(child, action, device);
     add_disallowed(child, action, device, target, allow_suicide);
 }
 
 static void
 stonith_query_capable_device_cb(GList * devices, void *user_data)
 {
     struct st_query_data *query = user_data;
     int available_devices = 0;
     xmlNode *dev = NULL;
     xmlNode *list = NULL;
     GListPtr lpc = NULL;
 
     /* Pack the results into XML */
     list = create_xml_node(NULL, __FUNCTION__);
     crm_xml_add(list, F_STONITH_TARGET, query->target);
     for (lpc = devices; lpc != NULL; lpc = lpc->next) {
         stonith_device_t *device = g_hash_table_lookup(device_list, lpc->data);
         const char *action = query->action;
 
         if (!device) {
             /* It is possible the device got unregistered while
              * determining who can fence the target */
             continue;
         }
 
         available_devices++;
 
         dev = create_xml_node(list, F_STONITH_DEVICE);
         crm_xml_add(dev, XML_ATTR_ID, device->id);
         crm_xml_add(dev, "namespace", device->namespace);
         crm_xml_add(dev, "agent", device->agent);
         crm_xml_add_int(dev, F_STONITH_DEVICE_VERIFIED, device->verified);
 
         /* If the originating fencer wants to reboot the node, and we have a
          * capable device that doesn't support "reboot", remap to "off" instead.
          */
         if (is_not_set(device->flags, st_device_supports_reboot)
             && safe_str_eq(query->action, "reboot")) {
             crm_trace("%s doesn't support reboot, using values for off instead",
                       device->id);
             action = "off";
         }
 
         /* Add action-specific values if available */
         add_action_specific_attributes(dev, action, device);
         if (safe_str_eq(query->action, "reboot")) {
             /* A "reboot" *might* get remapped to "off" then "on", so after
              * sending the "reboot"-specific values in the main element, we add
              * sub-elements for "off" and "on" values.
              *
              * We short-circuited earlier if "reboot", "off" and "on" are all
              * disallowed for the local host. However if only one or two are
              * disallowed, we send back the results and mark which ones are
              * disallowed. If "reboot" is disallowed, this might cause problems
              * with older fencer versions, which won't check for it. Older
              * versions will ignore "off" and "on", so they are not a problem.
              */
             add_disallowed(dev, action, device, query->target,
                            is_set(query->call_options, st_opt_allow_suicide));
             add_action_reply(dev, "off", device, query->target,
                              is_set(query->call_options, st_opt_allow_suicide));
             add_action_reply(dev, "on", device, query->target, FALSE);
         }
 
         /* A query without a target wants device parameters */
         if (query->target == NULL) {
             xmlNode *attrs = create_xml_node(dev, XML_TAG_ATTRS);
 
             g_hash_table_foreach(device->params, hash2field, attrs);
         }
     }
 
     crm_xml_add_int(list, F_STONITH_AVAILABLE_DEVICES, available_devices);
     if (query->target) {
         crm_debug("Found %d matching devices for '%s'", available_devices, query->target);
     } else {
         crm_debug("%d devices installed", available_devices);
     }
 
     if (list != NULL) {
         crm_log_xml_trace(list, "Add query results");
         add_message_xml(query->reply, F_STONITH_CALLDATA, list);
     }
     stonith_send_reply(query->reply, query->call_options, query->remote_peer, query->client_id);
 
     free_xml(query->reply);
     free(query->remote_peer);
     free(query->client_id);
     free(query->target);
     free(query->action);
     free(query);
     free_xml(list);
     g_list_free_full(devices, free);
 }
 
 static void
 stonith_query(xmlNode * msg, const char *remote_peer, const char *client_id, int call_options)
 {
     struct st_query_data *query = NULL;
     const char *action = NULL;
     const char *target = NULL;
     int timeout = 0;
     xmlNode *dev = get_xpath_object("//@" F_STONITH_ACTION, msg, LOG_NEVER);
 
     crm_element_value_int(msg, F_STONITH_TIMEOUT, &timeout);
     if (dev) {
         const char *device = crm_element_value(dev, F_STONITH_DEVICE);
 
         target = crm_element_value(dev, F_STONITH_TARGET);
         action = crm_element_value(dev, F_STONITH_ACTION);
         if (device && safe_str_eq(device, "manual_ack")) {
             /* No query or reply necessary */
             return;
         }
     }
 
     crm_log_xml_debug(msg, "Query");
     query = calloc(1, sizeof(struct st_query_data));
 
     query->reply = stonith_construct_reply(msg, NULL, NULL, pcmk_ok);
     query->remote_peer = remote_peer ? strdup(remote_peer) : NULL;
     query->client_id = client_id ? strdup(client_id) : NULL;
     query->target = target ? strdup(target) : NULL;
     query->action = action ? strdup(action) : NULL;
     query->call_options = call_options;
 
     get_capable_devices(target, action, timeout,
                         is_set(call_options, st_opt_allow_suicide),
                         query, stonith_query_capable_device_cb);
 }
 
 #define ST_LOG_OUTPUT_MAX 512
 static void
 log_operation(async_command_t * cmd, int rc, int pid, const char *next, const char *output, gboolean op_merged)
 {
     if (rc == 0) {
         next = NULL;
     }
 
     if (cmd->victim != NULL) {
         do_crm_log(rc == 0 ? LOG_NOTICE : LOG_ERR,
                    "Operation '%s' [%d] (call %d from %s) for host '%s' with device '%s' returned%s: %d (%s)%s%s",
                    cmd->action, pid, cmd->id, cmd->client_name, cmd->victim,
                    cmd->device, (op_merged? " (merged)" : ""),
                    rc, pcmk_strerror(rc),
                    (next? ", retrying with " : ""), (next ? next : ""));
     } else {
         do_crm_log_unlikely(rc == 0 ? LOG_DEBUG : LOG_NOTICE,
                             "Operation '%s' [%d] for device '%s' returned%s: %d (%s)%s%s",
                             cmd->action, pid, cmd->device,
                             (op_merged? " (merged)" : ""),
                             rc, pcmk_strerror(rc),
                             (next? ", retrying with " : ""), (next ? next : ""));
     }
 
     if (output) {
         /* Logging the whole string confuses syslog when the string is xml */
         char *prefix = crm_strdup_printf("%s:%d", cmd->device, pid);
 
         crm_log_output(rc == 0 ? LOG_DEBUG : LOG_WARNING, prefix, output);
         free(prefix);
     }
 }
 
 static void
 stonith_send_async_reply(async_command_t * cmd, const char *output, int rc, GPid pid, int options)
 {
     xmlNode *reply = NULL;
     gboolean bcast = FALSE;
 
     reply = stonith_construct_async_reply(cmd, output, NULL, rc);
 
     if (safe_str_eq(cmd->action, "metadata")) {
         /* Too verbose to log */
         crm_trace("Metadata query for %s", cmd->device);
         output = NULL;
 
     } else if (crm_str_eq(cmd->action, "monitor", TRUE) ||
                crm_str_eq(cmd->action, "list", TRUE) || crm_str_eq(cmd->action, "status", TRUE)) {
         crm_trace("Never broadcast '%s' replies", cmd->action);
 
     } else if (!stand_alone && safe_str_eq(cmd->origin, cmd->victim) && safe_str_neq(cmd->action, "on")) {
         crm_trace("Broadcast '%s' reply for %s", cmd->action, cmd->victim);
         crm_xml_add(reply, F_SUBTYPE, "broadcast");
         bcast = TRUE;
     }
 
     log_operation(cmd, rc, pid, NULL, output, (options & st_reply_opt_merged ? TRUE : FALSE));
     crm_log_xml_trace(reply, "Reply");
 
     if (options & st_reply_opt_merged) {
         crm_xml_add(reply, F_STONITH_MERGED, "true");
     }
 
     if (bcast) {
         crm_xml_add(reply, F_STONITH_OPERATION, T_STONITH_NOTIFY);
         send_cluster_message(NULL, crm_msg_stonith_ng, reply, FALSE);
 
     } else if (cmd->origin) {
         crm_trace("Directed reply to %s", cmd->origin);
         send_cluster_message(crm_get_peer(0, cmd->origin), crm_msg_stonith_ng, reply, FALSE);
 
     } else {
         crm_trace("Directed local %ssync reply to %s",
                   (cmd->options & st_opt_sync_call) ? "" : "a-", cmd->client_name);
         do_local_reply(reply, cmd->client, cmd->options & st_opt_sync_call, FALSE);
     }
 
     if (stand_alone) {
         /* Do notification with a clean data object */
         xmlNode *notify_data = create_xml_node(NULL, T_STONITH_NOTIFY_FENCE);
 
         crm_xml_add_int(notify_data, F_STONITH_RC, rc);
         crm_xml_add(notify_data, F_STONITH_TARGET, cmd->victim);
         crm_xml_add(notify_data, F_STONITH_OPERATION, cmd->op);
         crm_xml_add(notify_data, F_STONITH_DELEGATE, "localhost");
         crm_xml_add(notify_data, F_STONITH_DEVICE, cmd->device);
         crm_xml_add(notify_data, F_STONITH_REMOTE_OP_ID, cmd->remote_op_id);
         crm_xml_add(notify_data, F_STONITH_ORIGIN, cmd->client);
 
         do_stonith_notify(0, T_STONITH_NOTIFY_FENCE, rc, notify_data);
         do_stonith_notify(0, T_STONITH_NOTIFY_HISTORY, 0, NULL);
     }
 
     free_xml(reply);
 }
 
 static void
 cancel_stonith_command(async_command_t * cmd)
 {
     stonith_device_t *device;
 
     CRM_CHECK(cmd != NULL, return);
 
     if (!cmd->device) {
         return;
     }
 
     device = g_hash_table_lookup(device_list, cmd->device);
 
     if (device) {
         crm_trace("Cancel scheduled '%s' action on %s", cmd->action, device->id);
         device->pending_ops = g_list_remove(device->pending_ops, cmd);
     }
 }
 
 static void
 st_child_done(GPid pid, int rc, const char *output, gpointer user_data)
 {
     stonith_device_t *device = NULL;
     stonith_device_t *next_device = NULL;
     async_command_t *cmd = user_data;
 
     GListPtr gIter = NULL;
     GListPtr gIterNext = NULL;
 
     CRM_CHECK(cmd != NULL, return);
 
     cmd->active_on = NULL;
 
     /* The device is ready to do something else now */
     device = g_hash_table_lookup(device_list, cmd->device);
     if (device) {
         if (!device->verified && (rc == pcmk_ok) &&
             (safe_str_eq(cmd->action, "list") ||
              safe_str_eq(cmd->action, "monitor") || safe_str_eq(cmd->action, "status"))) {
 
             device->verified = TRUE;
         }
 
         mainloop_set_trigger(device->work);
     }
 
     crm_debug("Operation '%s' on '%s' completed with rc=%d (%d remaining)",
               cmd->action, cmd->device, rc, g_list_length(cmd->device_next));
 
     if (rc == 0) {
         GListPtr iter;
         /* see if there are any required devices left to execute for this op */
         for (iter = cmd->device_next; iter != NULL; iter = iter->next) {
             next_device = g_hash_table_lookup(device_list, iter->data);
 
             if (next_device != NULL && is_action_required(cmd->action, next_device)) {
                 cmd->device_next = iter->next;
                 break;
             }
             next_device = NULL;
         }
 
     } else if (rc != 0 && cmd->device_next && (is_action_required(cmd->action, device) == FALSE)) {
         /* if this device didn't work out, see if there are any others we can try.
          * if the failed device was 'required', we can't pick another device. */
         next_device = g_hash_table_lookup(device_list, cmd->device_next->data);
         cmd->device_next = cmd->device_next->next;
     }
 
     /* this operation requires more fencing, hooray! */
     if (next_device) {
         log_operation(cmd, rc, pid, next_device->id, output, FALSE);
 
         schedule_stonith_command(cmd, next_device);
         /* Prevent cmd from being freed */
         cmd = NULL;
         goto done;
     }
 
     stonith_send_async_reply(cmd, output, rc, pid, st_reply_opt_none);
 
     if (rc != 0) {
         goto done;
     }
 
     /* Check to see if any operations are scheduled to do the exact
      * same thing that just completed.  If so, rather than
      * performing the same fencing operation twice, return the result
      * of this operation for all pending commands it matches. */
     for (gIter = cmd_list; gIter != NULL; gIter = gIterNext) {
         async_command_t *cmd_other = gIter->data;
 
         gIterNext = gIter->next;
 
         if (cmd == cmd_other) {
             continue;
         }
 
         /* A pending scheduled command matches the command that just finished if.
          * 1. The client connections are different.
          * 2. The node victim is the same.
          * 3. The fencing action is the same.
          * 4. The device scheduled to execute the action is the same.
          */
         if (safe_str_eq(cmd->client, cmd_other->client) ||
             safe_str_neq(cmd->victim, cmd_other->victim) ||
             safe_str_neq(cmd->action, cmd_other->action) ||
             safe_str_neq(cmd->device, cmd_other->device)) {
 
             continue;
         }
 
         /* Duplicate merging will do the right thing for either type of remapped
          * reboot. If the executing fencer remapped an unsupported reboot to
          * off, then cmd->action will be reboot and will be merged with any
          * other reboot requests. If the originating fencer remapped a
          * topology reboot to off then on, we will get here once with
          * cmd->action "off" and once with "on", and they will be merged
          * separately with similar requests.
          */
         crm_notice
             ("Merging stonith action '%s' targeting %s originating from client %s with identical stonith request from client %s",
              cmd_other->action, cmd_other->victim, cmd_other->client_name, cmd->client_name);
 
         cmd_list = g_list_remove_link(cmd_list, gIter);
 
         stonith_send_async_reply(cmd_other, output, rc, pid, st_reply_opt_merged);
         cancel_stonith_command(cmd_other);
 
         free_async_command(cmd_other);
         g_list_free_1(gIter);
     }
 
   done:
     free_async_command(cmd);
 }
 
 static gint
 sort_device_priority(gconstpointer a, gconstpointer b)
 {
     const stonith_device_t *dev_a = a;
     const stonith_device_t *dev_b = b;
 
     if (dev_a->priority > dev_b->priority) {
         return -1;
     } else if (dev_a->priority < dev_b->priority) {
         return 1;
     }
     return 0;
 }
 
 static void
 stonith_fence_get_devices_cb(GList * devices, void *user_data)
 {
     async_command_t *cmd = user_data;
     stonith_device_t *device = NULL;
 
     crm_info("Found %d matching devices for '%s'", g_list_length(devices), cmd->victim);
 
     if (devices != NULL) {
         /* Order based on priority */
         devices = g_list_sort(devices, sort_device_priority);
         device = g_hash_table_lookup(device_list, devices->data);
 
         if (device) {
             cmd->device_list = devices;
             cmd->device_next = devices->next;
             devices = NULL;     /* list owned by cmd now */
         }
     }
 
     /* we have a device, schedule it for fencing. */
     if (device) {
         schedule_stonith_command(cmd, device);
         /* in progress */
         return;
     }
 
     /* no device found! */
     stonith_send_async_reply(cmd, NULL, -ENODEV, 0, st_reply_opt_none);
 
     free_async_command(cmd);
     g_list_free_full(devices, free);
 }
 
 static int
 stonith_fence(xmlNode * msg)
 {
     const char *device_id = NULL;
     stonith_device_t *device = NULL;
     async_command_t *cmd = create_async_command(msg);
     xmlNode *dev = get_xpath_object("//@" F_STONITH_TARGET, msg, LOG_ERR);
 
     if (cmd == NULL) {
         return -EPROTO;
     }
 
     device_id = crm_element_value(dev, F_STONITH_DEVICE);
     if (device_id) {
         device = g_hash_table_lookup(device_list, device_id);
         if (device == NULL) {
             crm_err("Requested device '%s' is not available", device_id);
             return -ENODEV;
         }
         schedule_stonith_command(cmd, device);
 
     } else {
         const char *host = crm_element_value(dev, F_STONITH_TARGET);
 
         if (cmd->options & st_opt_cs_nodeid) {
             int nodeid = crm_atoi(host, NULL);
             crm_node_t *node = crm_find_known_peer_full(nodeid, NULL, CRM_GET_PEER_ANY);
 
             if (node) {
                 host = node->uname;
             }
         }
 
         /* If we get to here, then self-fencing is implicitly allowed */
         get_capable_devices(host, cmd->action, cmd->default_timeout,
                             TRUE, cmd, stonith_fence_get_devices_cb);
     }
 
     return -EINPROGRESS;
 }
 
 xmlNode *
 stonith_construct_reply(xmlNode * request, const char *output, xmlNode * data, int rc)
 {
-    int lpc = 0;
     xmlNode *reply = NULL;
 
-    const char *name = NULL;
-    const char *value = NULL;
-
-    const char *names[] = {
-        F_STONITH_OPERATION,
-        F_STONITH_CALLID,
-        F_STONITH_CLIENTID,
-        F_STONITH_CLIENTNAME,
-        F_STONITH_REMOTE_OP_ID,
-        F_STONITH_CALLOPTS
-    };
-
-    crm_trace("Creating a basic reply");
     reply = create_xml_node(NULL, T_STONITH_REPLY);
 
     crm_xml_add(reply, "st_origin", __FUNCTION__);
     crm_xml_add(reply, F_TYPE, T_STONITH_NG);
     crm_xml_add(reply, "st_output", output);
     crm_xml_add_int(reply, F_STONITH_RC, rc);
 
-    CRM_CHECK(request != NULL, crm_warn("Can't create a sane reply"); return reply);
-    for (lpc = 0; lpc < DIMOF(names); lpc++) {
-        name = names[lpc];
-        value = crm_element_value(request, name);
-        crm_xml_add(reply, name, value);
-    }
+    if (request == NULL) {
+        /* Most likely, this is the result of a stonith operation that was
+         * initiated before we came up. Unfortunately that means we lack enough
+         * information to provide clients with a full result.
+         *
+         * @TODO Maybe synchronize this information at start-up?
+         */
+        crm_warn("Missing request information for client notifications for "
+                 "operation with result %d (initiated before we came up?)", rc);
 
-    if (data != NULL) {
-        crm_trace("Attaching reply output");
-        add_message_xml(reply, F_STONITH_CALLDATA, data);
+    } else {
+        const char *name = NULL;
+        const char *value = NULL;
+
+        const char *names[] = {
+            F_STONITH_OPERATION,
+            F_STONITH_CALLID,
+            F_STONITH_CLIENTID,
+            F_STONITH_CLIENTNAME,
+            F_STONITH_REMOTE_OP_ID,
+            F_STONITH_CALLOPTS
+        };
+
+        crm_trace("Creating a result reply with%s reply output (rc=%d)",
+                  (data? "" : "out"), rc);
+        for (int lpc = 0; lpc < DIMOF(names); lpc++) {
+            name = names[lpc];
+            value = crm_element_value(request, name);
+            crm_xml_add(reply, name, value);
+        }
+        if (data != NULL) {
+            add_message_xml(reply, F_STONITH_CALLDATA, data);
+        }
     }
     return reply;
 }
 
 static xmlNode *
 stonith_construct_async_reply(async_command_t * cmd, const char *output, xmlNode * data, int rc)
 {
     xmlNode *reply = NULL;
 
     crm_trace("Creating a basic reply");
     reply = create_xml_node(NULL, T_STONITH_REPLY);
 
     crm_xml_add(reply, "st_origin", __FUNCTION__);
     crm_xml_add(reply, F_TYPE, T_STONITH_NG);
 
     crm_xml_add(reply, F_STONITH_OPERATION, cmd->op);
     crm_xml_add(reply, F_STONITH_DEVICE, cmd->device);
     crm_xml_add(reply, F_STONITH_REMOTE_OP_ID, cmd->remote_op_id);
     crm_xml_add(reply, F_STONITH_CLIENTID, cmd->client);
     crm_xml_add(reply, F_STONITH_CLIENTNAME, cmd->client_name);
     crm_xml_add(reply, F_STONITH_TARGET, cmd->victim);
     crm_xml_add(reply, F_STONITH_ACTION, cmd->op);
     crm_xml_add(reply, F_STONITH_ORIGIN, cmd->origin);
     crm_xml_add_int(reply, F_STONITH_CALLID, cmd->id);
     crm_xml_add_int(reply, F_STONITH_CALLOPTS, cmd->options);
 
     crm_xml_add_int(reply, F_STONITH_RC, rc);
 
     crm_xml_add(reply, "st_output", output);
 
     if (data != NULL) {
         crm_info("Attaching reply output");
         add_message_xml(reply, F_STONITH_CALLDATA, data);
     }
     return reply;
 }
 
 bool fencing_peer_active(crm_node_t *peer)
 {
     if (peer == NULL) {
         return FALSE;
     } else if (peer->uname == NULL) {
         return FALSE;
     } else if (is_set(peer->processes, crm_get_cluster_proc())) {
         return TRUE;
     }
     return FALSE;
 }
 
 /*!
  * \internal
  * \brief Determine if we need to use an alternate node to
  * fence the target. If so return that node's uname
  *
  * \retval NULL, no alternate host
  * \retval uname, uname of alternate host to use
  */
 static const char *
 check_alternate_host(const char *target)
 {
     const char *alternate_host = NULL;
 
     crm_trace("Checking if we (%s) can fence %s", stonith_our_uname, target);
     if (find_topology_for_host(target) && safe_str_eq(target, stonith_our_uname)) {
         GHashTableIter gIter;
         crm_node_t *entry = NULL;
 
         g_hash_table_iter_init(&gIter, crm_peer_cache);
         while (g_hash_table_iter_next(&gIter, NULL, (void **)&entry)) {
             crm_trace("Checking for %s.%d != %s", entry->uname, entry->id, target);
             if (fencing_peer_active(entry)
                 && safe_str_neq(entry->uname, target)) {
                 alternate_host = entry->uname;
                 break;
             }
         }
         if (alternate_host == NULL) {
             crm_err("No alternate host available to handle complex self fencing request");
             g_hash_table_iter_init(&gIter, crm_peer_cache);
             while (g_hash_table_iter_next(&gIter, NULL, (void **)&entry)) {
                 crm_notice("Peer[%d] %s", entry->id, entry->uname);
             }
         }
     }
 
     return alternate_host;
 }
 
 static void
 stonith_send_reply(xmlNode * reply, int call_options, const char *remote_peer,
                    const char *client_id)
 {
     if (remote_peer) {
         send_cluster_message(crm_get_peer(0, remote_peer), crm_msg_stonith_ng, reply, FALSE);
     } else {
         do_local_reply(reply, client_id, is_set(call_options, st_opt_sync_call), remote_peer != NULL);
     }
 }
 
 static void 
 remove_relay_op(xmlNode * request)
 {
     xmlNode *dev = get_xpath_object("//@" F_STONITH_ACTION, request, LOG_TRACE);
     const char *relay_op_id = NULL; 
     const char *op_id = NULL;
     const char *client_name = NULL;
     const char *target = NULL; 
     remote_fencing_op_t *relay_op = NULL; 
 
     if (dev) { 
         target = crm_element_value(dev, F_STONITH_TARGET); 
     }
 
     relay_op_id = crm_element_value(request, F_STONITH_REMOTE_OP_ID_RELAY);
     op_id = crm_element_value(request, F_STONITH_REMOTE_OP_ID);
     client_name = crm_element_value(request, F_STONITH_CLIENTNAME);
 
     /* Delete RELAY operation. */
     if (relay_op_id && target && safe_str_eq(target, stonith_our_uname)) {
         relay_op = g_hash_table_lookup(stonith_remote_op_list, relay_op_id);
 
         if (relay_op) {
             GHashTableIter iter;
             remote_fencing_op_t *list_op = NULL; 
             g_hash_table_iter_init(&iter, stonith_remote_op_list);
 
             /* If the operation to be deleted is registered as a duplicate, delete the registration. */
             while (g_hash_table_iter_next(&iter, NULL, (void **)&list_op)) {
                 GListPtr dup_iter = NULL;
                 if (list_op != relay_op) {
                     for (dup_iter = list_op->duplicates; dup_iter != NULL; dup_iter = dup_iter->next) {
                         remote_fencing_op_t *other = dup_iter->data;
                         if (other == relay_op) {
                             other->duplicates = g_list_remove(other->duplicates, relay_op);
                             break;
                         }
                     }
                 }
             }
             crm_info("Delete the relay op : %s - %s of %s for %s.(replaced by op : %s - %s of %s for %s)",
                   relay_op->id, relay_op->action, relay_op->target, relay_op->client_name,
                   op_id, relay_op->action, target, client_name);
 
             g_hash_table_remove(stonith_remote_op_list, relay_op_id);
         }
     }
 }
 
 static int
 handle_request(pcmk__client_t *client, uint32_t id, uint32_t flags,
                xmlNode *request, const char *remote_peer)
 {
     int call_options = 0;
     int rc = -EOPNOTSUPP;
 
     xmlNode *data = NULL;
     xmlNode *reply = NULL;
 
     char *output = NULL;
     const char *op = crm_element_value(request, F_STONITH_OPERATION);
     const char *client_id = crm_element_value(request, F_STONITH_CLIENTID);
 
     crm_element_value_int(request, F_STONITH_CALLOPTS, &call_options);
 
     if (is_set(call_options, st_opt_sync_call)) {
         CRM_ASSERT(client == NULL || client->request_id == id);
     }
 
     if (crm_str_eq(op, CRM_OP_REGISTER, TRUE)) {
         xmlNode *reply = create_xml_node(NULL, "reply");
 
         CRM_ASSERT(client);
         crm_xml_add(reply, F_STONITH_OPERATION, CRM_OP_REGISTER);
         crm_xml_add(reply, F_STONITH_CLIENTID, client->id);
         pcmk__ipc_send_xml(client, id, reply, flags);
         client->request_id = 0;
         free_xml(reply);
         return 0;
 
     } else if (crm_str_eq(op, STONITH_OP_EXEC, TRUE)) {
         rc = stonith_device_action(request, &output);
 
     } else if (crm_str_eq(op, STONITH_OP_TIMEOUT_UPDATE, TRUE)) {
         const char *call_id = crm_element_value(request, F_STONITH_CALLID);
         const char *client_id = crm_element_value(request, F_STONITH_CLIENTID);
         int op_timeout = 0;
 
         crm_element_value_int(request, F_STONITH_TIMEOUT, &op_timeout);
         do_stonith_async_timeout_update(client_id, call_id, op_timeout);
         return 0;
 
     } else if (crm_str_eq(op, STONITH_OP_QUERY, TRUE)) {
         if (remote_peer) {
             create_remote_stonith_op(client_id, request, TRUE); /* Record it for the future notification */
         }
 
         /* Delete the DC node RELAY operation. */
         remove_relay_op(request);
 
         stonith_query(request, remote_peer, client_id, call_options);
         return 0;
 
     } else if (crm_str_eq(op, T_STONITH_NOTIFY, TRUE)) {
         const char *flag_name = NULL;
 
         CRM_ASSERT(client);
         flag_name = crm_element_value(request, F_STONITH_NOTIFY_ACTIVATE);
         if (flag_name) {
             crm_debug("Setting %s callbacks for %s (%s): ON", flag_name, client->name, client->id);
             client->options |= get_stonith_flag(flag_name);
         }
 
         flag_name = crm_element_value(request, F_STONITH_NOTIFY_DEACTIVATE);
         if (flag_name) {
             crm_debug("Setting %s callbacks for %s (%s): off", flag_name, client->name, client->id);
             client->options |= get_stonith_flag(flag_name);
         }
 
         if (flags & crm_ipc_client_response) {
             pcmk__ipc_send_ack(client, id, flags, "ack");
         }
         return 0;
 
     } else if (crm_str_eq(op, STONITH_OP_RELAY, TRUE)) {
         xmlNode *dev = get_xpath_object("//@" F_STONITH_TARGET, request, LOG_TRACE);
 
         crm_notice("Peer %s has received a forwarded fencing request from %s to fence (%s) peer %s",
                    stonith_our_uname,
                    client ? client->name : remote_peer,
                    crm_element_value(dev, F_STONITH_ACTION),
                    crm_element_value(dev, F_STONITH_TARGET));
 
         if (initiate_remote_stonith_op(NULL, request, FALSE) != NULL) {
             rc = -EINPROGRESS;
         }
 
     } else if (crm_str_eq(op, STONITH_OP_FENCE, TRUE)) {
 
         if (remote_peer || stand_alone) {
             rc = stonith_fence(request);
 
         } else if (call_options & st_opt_manual_ack) {
             remote_fencing_op_t *rop = NULL;
             xmlNode *dev = get_xpath_object("//@" F_STONITH_TARGET, request, LOG_TRACE);
             const char *target = crm_element_value(dev, F_STONITH_TARGET);
 
             crm_notice("Received manual confirmation that %s is fenced", target);
             rop = initiate_remote_stonith_op(client, request, TRUE);
             rc = stonith_manual_ack(request, rop);
 
         } else {
             const char *alternate_host = NULL;
             xmlNode *dev = get_xpath_object("//@" F_STONITH_TARGET, request, LOG_TRACE);
             const char *target = crm_element_value(dev, F_STONITH_TARGET);
             const char *action = crm_element_value(dev, F_STONITH_ACTION);
             const char *device = crm_element_value(dev, F_STONITH_DEVICE);
 
             if (client) {
                 int tolerance = 0;
 
                 crm_notice("Client %s.%.8s wants to fence (%s) '%s' with device '%s'",
                            client->name, client->id, action, target, device ? device : "(any)");
 
                 crm_element_value_int(dev, F_STONITH_TOLERANCE, &tolerance);
 
                 if (stonith_check_fence_tolerance(tolerance, target, action)) {
                     rc = 0;
                     goto done;
                 }
 
             } else {
                 crm_notice("Peer %s wants to fence (%s) '%s' with device '%s'",
                            remote_peer, action, target, device ? device : "(any)");
             }
 
             alternate_host = check_alternate_host(target);
 
             if (alternate_host && client) {
                 const char *client_id = NULL;
                 remote_fencing_op_t *op = NULL;
 
                 crm_notice("Forwarding complex self fencing request to peer %s", alternate_host);
 
                 if (client->id) {
                     client_id = client->id;
                 } else {
                     client_id = crm_element_value(request, F_STONITH_CLIENTID);
                 }
 
                 /* Create an operation for RELAY and send the ID in the RELAY message. */
                 /* When a QUERY response is received, delete the RELAY operation to avoid the existence of duplicate operations. */
                 op = create_remote_stonith_op(client_id, request, FALSE);
 
                 crm_xml_add(request, F_STONITH_OPERATION, STONITH_OP_RELAY);
                 crm_xml_add(request, F_STONITH_CLIENTID, client->id);
                 crm_xml_add(request, F_STONITH_REMOTE_OP_ID, op->id);
                 send_cluster_message(crm_get_peer(0, alternate_host), crm_msg_stonith_ng, request,
                                      FALSE);
                 rc = -EINPROGRESS;
 
             } else if (initiate_remote_stonith_op(client, request, FALSE) != NULL) {
                 rc = -EINPROGRESS;
             }
         }
 
     } else if (crm_str_eq(op, STONITH_OP_FENCE_HISTORY, TRUE)) {
         rc = stonith_fence_history(request, &data, remote_peer, call_options);
         if (call_options & st_opt_discard_reply) {
             /* we don't expect answers to the broadcast
              * we might have sent out
              */
             free_xml(data);
             return pcmk_ok;
         }
 
     } else if (crm_str_eq(op, STONITH_OP_DEVICE_ADD, TRUE)) {
         const char *device_id = NULL;
 
         rc = stonith_device_register(request, &device_id, FALSE);
         do_stonith_notify_device(call_options, op, rc, device_id);
 
     } else if (crm_str_eq(op, STONITH_OP_DEVICE_DEL, TRUE)) {
         xmlNode *dev = get_xpath_object("//" F_STONITH_DEVICE, request, LOG_ERR);
         const char *device_id = crm_element_value(dev, XML_ATTR_ID);
 
         rc = stonith_device_remove(device_id, FALSE);
         do_stonith_notify_device(call_options, op, rc, device_id);
 
     } else if (crm_str_eq(op, STONITH_OP_LEVEL_ADD, TRUE)) {
         char *device_id = NULL;
 
         rc = stonith_level_register(request, &device_id);
         do_stonith_notify_level(call_options, op, rc, device_id);
         free(device_id);
 
     } else if (crm_str_eq(op, STONITH_OP_LEVEL_DEL, TRUE)) {
         char *device_id = NULL;
 
         rc = stonith_level_remove(request, &device_id);
         do_stonith_notify_level(call_options, op, rc, device_id);
 
     } else if(safe_str_eq(op, CRM_OP_RM_NODE_CACHE)) {
         int node_id = 0;
         const char *name = NULL;
 
         crm_element_value_int(request, XML_ATTR_ID, &node_id);
         name = crm_element_value(request, XML_ATTR_UNAME);
         reap_crm_member(node_id, name);
 
         return pcmk_ok;
 
     } else {
         crm_err("Unknown IPC request %s from %s",
                 op, (client? client->name : remote_peer));
     }
 
   done:
 
     /* Always reply unless the request is in process still.
      * If in progress, a reply will happen async after the request
      * processing is finished */
     if (rc != -EINPROGRESS) {
         crm_trace("Reply handling: %p %u %u %d %d %s", client, client?client->request_id:0,
                   id, is_set(call_options, st_opt_sync_call), call_options,
                   crm_element_value(request, F_STONITH_CALLOPTS));
 
         if (is_set(call_options, st_opt_sync_call)) {
             CRM_ASSERT(client == NULL || client->request_id == id);
         }
         reply = stonith_construct_reply(request, output, data, rc);
         stonith_send_reply(reply, call_options, remote_peer, client_id);
     }
 
     free(output);
     free_xml(data);
     free_xml(reply);
 
     return rc;
 }
 
 static void
 handle_reply(pcmk__client_t *client, xmlNode *request, const char *remote_peer)
 {
     const char *op = crm_element_value(request, F_STONITH_OPERATION);
 
     if (crm_str_eq(op, STONITH_OP_QUERY, TRUE)) {
         process_remote_stonith_query(request);
     } else if (crm_str_eq(op, T_STONITH_NOTIFY, TRUE)) {
         process_remote_stonith_exec(request);
     } else if (crm_str_eq(op, STONITH_OP_FENCE, TRUE)) {
         /* Reply to a complex fencing op */
         process_remote_stonith_exec(request);
     } else {
         crm_err("Unknown %s reply from %s", op, client ? client->name : remote_peer);
         crm_log_xml_warn(request, "UnknownOp");
     }
 }
 
 void
 stonith_command(pcmk__client_t *client, uint32_t id, uint32_t flags,
                 xmlNode *request, const char *remote_peer)
 {
     int call_options = 0;
     int rc = 0;
     gboolean is_reply = FALSE;
 
     /* Copy op for reporting. The original might get freed by handle_reply()
      * before we use it in crm_debug():
      *     handle_reply()
      *     |- process_remote_stonith_exec()
      *     |-- remote_op_done()
      *     |--- handle_local_reply_and_notify()
      *     |---- crm_xml_add(...F_STONITH_OPERATION...)
      *     |--- free_xml(op->request)
      */
     char *op = crm_element_value_copy(request, F_STONITH_OPERATION);
 
     if (get_xpath_object("//" T_STONITH_REPLY, request, LOG_NEVER)) {
         is_reply = TRUE;
     }
 
     crm_element_value_int(request, F_STONITH_CALLOPTS, &call_options);
     crm_debug("Processing %s%s %u from %s (%16x)", op, is_reply ? " reply" : "",
               id, client ? client->name : remote_peer, call_options);
 
     if (is_set(call_options, st_opt_sync_call)) {
         CRM_ASSERT(client == NULL || client->request_id == id);
     }
 
     if (is_reply) {
         handle_reply(client, request, remote_peer);
     } else {
         rc = handle_request(client, id, flags, request, remote_peer);
     }
 
     crm_debug("Processed %s%s from %s: %s (%d)", op,
               is_reply ? " reply" : "", client ? client->name : remote_peer,
               rc > 0 ? "" : pcmk_strerror(rc), rc);
 
     free(op);
 }
diff --git a/include/crm/cluster/internal.h b/include/crm/cluster/internal.h
index 612efcbda5..8f647e0270 100644
--- a/include/crm/cluster/internal.h
+++ b/include/crm/cluster/internal.h
@@ -1,337 +1,349 @@
 /*
  * Copyright 2004-2018 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU Lesser General Public License
  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
  */
 
 #ifndef CRM_CLUSTER_INTERNAL__H
 #  define CRM_CLUSTER_INTERNAL__H
 
 #  include <crm/cluster.h>
 
 typedef struct crm_ais_host_s AIS_Host;
 typedef struct crm_ais_msg_s AIS_Message;
 
 struct crm_ais_host_s {
     uint32_t id;
     uint32_t pid;
     gboolean local;
     enum crm_ais_msg_types type;
     uint32_t size;
     char uname[MAX_NAME];
 
 } __attribute__ ((packed));
 
+#if SUPPORT_COROSYNC
+#  include <qb/qbipc_common.h>
+#  include <corosync/corotypes.h>
+typedef struct qb_ipc_response_header cs_ipc_header_response_t;
+#else
+typedef struct {
+    int size __attribute__ ((aligned(8)));
+    int id __attribute__ ((aligned(8)));
+    int error __attribute__ ((aligned(8)));
+} __attribute__ ((aligned(8))) cs_ipc_header_response_t;
+#endif
+
 struct crm_ais_msg_s {
     cs_ipc_header_response_t header __attribute__ ((aligned(8)));
     uint32_t id;
     gboolean is_compressed;
 
     AIS_Host host;
     AIS_Host sender;
 
     uint32_t size;
     uint32_t compressed_size;
     /* 584 bytes */
     char data[0];
 
 } __attribute__ ((packed));
 
 /* *INDENT-OFF* */
 enum crm_proc_flag {
     crm_proc_none       = 0x00000001,
 
     // Cluster layers
     crm_proc_cpg        = 0x04000000,
 
     // Daemons
     crm_proc_execd      = 0x00000010,
     crm_proc_based      = 0x00000100,
     crm_proc_controld   = 0x00000200,
     crm_proc_attrd      = 0x00001000,
     crm_proc_schedulerd = 0x00010000,
     crm_proc_fenced     = 0x00100000,
 };
 /* *INDENT-ON* */
 
 /*!
  * \internal
  * \brief Return the process bit corresponding to the current cluster stack
  *
  * \return Process flag if detectable, otherwise 0
  */
 static inline uint32_t
 crm_get_cluster_proc(void)
 {
     switch (get_cluster_type()) {
         case pcmk_cluster_corosync:
             return crm_proc_cpg;
 
         default:
             break;
     }
     return crm_proc_none;
 }
 
 static inline const char *
 peer2text(enum crm_proc_flag proc)
 {
     const char *text = "unknown";
 
     switch (proc) {
         case crm_proc_none:
             text = "none";
             break;
         case crm_proc_based:
             text = "pacemaker-based";
             break;
         case crm_proc_controld:
             text = "pacemaker-controld";
             break;
         case crm_proc_schedulerd:
             text = "pacemaker-schedulerd";
             break;
         case crm_proc_execd:
             text = "pacemaker-execd";
             break;
         case crm_proc_attrd:
             text = "pacemaker-attrd";
             break;
         case crm_proc_fenced:
             text = "pacemaker-fenced";
             break;
         case crm_proc_cpg:
             text = "corosync-cpg";
             break;
     }
     return text;
 }
 
 static inline const char *
 ais_dest(const AIS_Host *host)
 {
     if (host->local) {
         return "local";
     } else if (host->size > 0) {
         return host->uname;
     } else {
         return "<all>";
     }
 }
 
 #  define ais_data_len(msg) (msg->is_compressed?msg->compressed_size:msg->size)
 
 /*
 typedef enum {
    CS_OK = 1,
    CS_ERR_LIBRARY = 2,
    CS_ERR_VERSION = 3,
    CS_ERR_INIT = 4,
    CS_ERR_TIMEOUT = 5,
    CS_ERR_TRY_AGAIN = 6,
    CS_ERR_INVALID_PARAM = 7,
    CS_ERR_NO_MEMORY = 8,
    CS_ERR_BAD_HANDLE = 9,
    CS_ERR_BUSY = 10,
    CS_ERR_ACCESS = 11,
    CS_ERR_NOT_EXIST = 12,
    CS_ERR_NAME_TOO_LONG = 13,
    CS_ERR_EXIST = 14,
    CS_ERR_NO_SPACE = 15,
    CS_ERR_INTERRUPT = 16,
    CS_ERR_NAME_NOT_FOUND = 17,
    CS_ERR_NO_RESOURCES = 18,
    CS_ERR_NOT_SUPPORTED = 19,
    CS_ERR_BAD_OPERATION = 20,
    CS_ERR_FAILED_OPERATION = 21,
    CS_ERR_MESSAGE_ERROR = 22,
    CS_ERR_QUEUE_FULL = 23,
    CS_ERR_QUEUE_NOT_AVAILABLE = 24,
    CS_ERR_BAD_FLAGS = 25,
    CS_ERR_TOO_BIG = 26,
    CS_ERR_NO_SECTIONS = 27,
    CS_ERR_CONTEXT_NOT_FOUND = 28,
    CS_ERR_TOO_MANY_GROUPS = 30,
    CS_ERR_SECURITY = 100
 } cs_error_t;
  */
 static inline const char *
 ais_error2text(int error)
 {
     const char *text = "unknown";
 
 #  if SUPPORT_COROSYNC
     switch (error) {
         case CS_OK:
             text = "OK";
             break;
         case CS_ERR_LIBRARY:
             text = "Library error";
             break;
         case CS_ERR_VERSION:
             text = "Version error";
             break;
         case CS_ERR_INIT:
             text = "Initialization error";
             break;
         case CS_ERR_TIMEOUT:
             text = "Timeout";
             break;
         case CS_ERR_TRY_AGAIN:
             text = "Try again";
             break;
         case CS_ERR_INVALID_PARAM:
             text = "Invalid parameter";
             break;
         case CS_ERR_NO_MEMORY:
             text = "No memory";
             break;
         case CS_ERR_BAD_HANDLE:
             text = "Bad handle";
             break;
         case CS_ERR_BUSY:
             text = "Busy";
             break;
         case CS_ERR_ACCESS:
             text = "Access error";
             break;
         case CS_ERR_NOT_EXIST:
             text = "Doesn't exist";
             break;
         case CS_ERR_NAME_TOO_LONG:
             text = "Name too long";
             break;
         case CS_ERR_EXIST:
             text = "Exists";
             break;
         case CS_ERR_NO_SPACE:
             text = "No space";
             break;
         case CS_ERR_INTERRUPT:
             text = "Interrupt";
             break;
         case CS_ERR_NAME_NOT_FOUND:
             text = "Name not found";
             break;
         case CS_ERR_NO_RESOURCES:
             text = "No resources";
             break;
         case CS_ERR_NOT_SUPPORTED:
             text = "Not supported";
             break;
         case CS_ERR_BAD_OPERATION:
             text = "Bad operation";
             break;
         case CS_ERR_FAILED_OPERATION:
             text = "Failed operation";
             break;
         case CS_ERR_MESSAGE_ERROR:
             text = "Message error";
             break;
         case CS_ERR_QUEUE_FULL:
             text = "Queue full";
             break;
         case CS_ERR_QUEUE_NOT_AVAILABLE:
             text = "Queue not available";
             break;
         case CS_ERR_BAD_FLAGS:
             text = "Bad flags";
             break;
         case CS_ERR_TOO_BIG:
             text = "Too big";
             break;
         case CS_ERR_NO_SECTIONS:
             text = "No sections";
             break;
     }
 #  endif
     return text;
 }
 
 static inline const char *
 msg_type2text(enum crm_ais_msg_types type)
 {
     const char *text = "unknown";
 
     switch (type) {
         case crm_msg_none:
             text = "unknown";
             break;
         case crm_msg_ais:
             text = "ais";
             break;
         case crm_msg_cib:
             text = "cib";
             break;
         case crm_msg_crmd:
             text = "crmd";
             break;
         case crm_msg_pe:
             text = "pengine";
             break;
         case crm_msg_te:
             text = "tengine";
             break;
         case crm_msg_lrmd:
             text = "lrmd";
             break;
         case crm_msg_attrd:
             text = "attrd";
             break;
         case crm_msg_stonithd:
             text = "stonithd";
             break;
         case crm_msg_stonith_ng:
             text = "stonith-ng";
             break;
     }
     return text;
 }
 
 gboolean check_message_sanity(const AIS_Message * msg, const char *data);
 
 #  if SUPPORT_COROSYNC
 
 gboolean send_cpg_iov(struct iovec * iov);
 
 char *get_corosync_uuid(crm_node_t *peer);
 char *corosync_node_name(uint64_t /*cmap_handle_t */ cmap_handle, uint32_t nodeid);
 char *corosync_cluster_name(void);
 int corosync_cmap_has_config(const char *prefix);
 
 gboolean corosync_initialize_nodelist(void *cluster, gboolean force_member, xmlNode * xml_parent);
 
 gboolean send_cluster_message_cs(xmlNode * msg, gboolean local,
                                  crm_node_t * node, enum crm_ais_msg_types dest);
 
 enum cluster_type_e find_corosync_variant(void);
 
 void terminate_cs_connection(crm_cluster_t * cluster);
 gboolean init_cs_connection(crm_cluster_t * cluster);
 gboolean init_cs_connection_once(crm_cluster_t * cluster);
 #  endif
 
 crm_node_t *crm_update_peer_proc(const char *source, crm_node_t * peer,
                                  uint32_t flag, const char *status);
 crm_node_t *crm_update_peer_state(const char *source, crm_node_t * node,
                                   const char *state, uint64_t membership);
 
 void crm_update_peer_uname(crm_node_t *node, const char *uname);
 void crm_update_peer_expected(const char *source, crm_node_t * node, const char *expected);
 void crm_reap_unseen_nodes(uint64_t ring_id);
 
 gboolean cluster_connect_quorum(gboolean(*dispatch) (unsigned long long, gboolean),
                                 void (*destroy) (gpointer));
 
 gboolean node_name_is_valid(const char *key, const char *name);
 
 crm_node_t * crm_find_peer_full(unsigned int id, const char *uname, int flags);
 crm_node_t * crm_find_peer(unsigned int id, const char *uname);
 
 void crm_peer_caches_refresh(xmlNode *cib);
 crm_node_t *crm_find_known_peer_full(unsigned int id, const char *uname, int flags);
 
 #endif
diff --git a/include/crm/common/internal.h b/include/crm/common/internal.h
index f5ec562a6e..41abdbbee8 100644
--- a/include/crm/common/internal.h
+++ b/include/crm/common/internal.h
@@ -1,310 +1,310 @@
 /*
  * Copyright 2015-2020 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU Lesser General Public License
  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
  */
 
 #ifndef CRM_COMMON_INTERNAL__H
 #define CRM_COMMON_INTERNAL__H
 
 #include <unistd.h>             // getpid()
 #include <stdbool.h>            // bool
 #include <string.h>             // strcmp()
 #include <sys/types.h>          // uid_t, gid_t, pid_t
 
 #include <glib.h>               // guint, GList, GHashTable
 #include <libxml/tree.h>        // xmlNode
 
 #include <crm/common/util.h>    // crm_strdup_printf()
 #include <crm/common/mainloop.h> // mainloop_io_t, struct ipc_client_callbacks
 
 // Internal ACL-related utilities (from acl.c)
 
 char *pcmk__uid2username(uid_t uid);
 const char *pcmk__update_acl_user(xmlNode *request, const char *field,
                                   const char *peer_user);
 
 #if ENABLE_ACL
 #  include <string.h>
 static inline bool
 pcmk__is_privileged(const char *user)
 {
     return user && (!strcmp(user, CRM_DAEMON_USER) || !strcmp(user, "root"));
 }
 #endif
 
 
 #if SUPPORT_CIBSECRETS
 // Internal CIB utilities (from cib_secrets.c) */
 
 int pcmk__substitute_secrets(const char *rsc_id, GHashTable *params);
 #endif
 
 
 /* internal digest-related utilities (from digest.c) */
 
 bool pcmk__verify_digest(xmlNode *input, const char *expected);
 
 
 /* internal I/O utilities (from io.c) */
 
 int pcmk__real_path(const char *path, char **resolved_path);
 
 char *pcmk__series_filename(const char *directory, const char *series,
                             int sequence, bool bzip);
 int pcmk__read_series_sequence(const char *directory, const char *series,
                                unsigned int *seq);
 void pcmk__write_series_sequence(const char *directory, const char *series,
                                  unsigned int sequence, int max);
 int pcmk__chown_series_sequence(const char *directory, const char *series,
                                 uid_t uid, gid_t gid);
 
 int pcmk__build_path(const char *path_c, mode_t mode);
 bool pcmk__daemon_can_write(const char *dir, const char *file);
 void pcmk__sync_directory(const char *name);
 
 int pcmk__file_contents(const char *filename, char **contents);
 int pcmk__write_sync(int fd, const char *contents);
 int pcmk__set_nonblocking(int fd);
 const char *pcmk__get_tmpdir(void);
 
 void pcmk__close_fds_in_child(bool);
 
 /*!
  * \internal
  * \brief Open /dev/null to consume next available file descriptor
  *
  * Open /dev/null, disregarding the result. This is intended when daemonizing to
  * be able to null stdin, stdout, and stderr.
  *
  * \param[in] flags  O_RDONLY (stdin) or O_WRONLY (stdout and stderr)
  */
 static inline void
 pcmk__open_devnull(int flags)
 {
     // Static analysis clutter
     // cppcheck-suppress leakReturnValNotUsed
     (void) open("/dev/null", flags);
 }
 
 
 /* internal logging utilities */
 
 #  define pcmk__config_err(fmt...) do {     \
         crm_config_error = TRUE;            \
         crm_err(fmt);                       \
     } while (0)
 
 #  define pcmk__config_warn(fmt...) do {    \
         crm_config_warning = TRUE;          \
         crm_warn(fmt);                      \
     } while (0)
 
 /*!
  * \internal
  * \brief Execute code depending on whether message would be logged
  *
  * This is similar to do_crm_log_unlikely() except instead of logging, it either
  * continues past this statement or executes else_action depending on whether a
  * message of the given severity would be logged or not. This allows whole
  * blocks of code to be skipped if tracing or debugging is turned off.
  *
  * \param[in] level        Severity at which to continue past this statement
  * \param[in] else_action  Code block to execute if severity would not be logged
  *
  * \note else_action must not contain a break or continue statement
  */
 #  define pcmk__log_else(level, else_action) do {                           \
         static struct qb_log_callsite *trace_cs = NULL;                     \
                                                                             \
         if (trace_cs == NULL) {                                             \
             trace_cs = qb_log_callsite_get(__func__, __FILE__, "log_else",  \
                                            level, __LINE__, 0);             \
         }                                                                   \
         if (!crm_is_callsite_active(trace_cs, level, 0)) {                  \
             else_action;                                                    \
         }                                                                   \
     } while(0)
 
 
 /* internal main loop utilities (from mainloop.c) */
 
 int pcmk__add_mainloop_ipc(crm_ipc_t *ipc, int priority, void *userdata,
                            struct ipc_client_callbacks *callbacks,
                            mainloop_io_t **source);
 
 
 /* internal procfs utilities (from procfs.c) */
 
 pid_t pcmk__procfs_pid_of(const char *name);
 unsigned int pcmk__procfs_num_cores(void);
 
 
 /* internal XML schema functions (from xml.c) */
 
 void crm_schema_init(void);
 void crm_schema_cleanup(void);
 
 
 /* internal functions related to process IDs (from pid.c) */
 
 /*!
  * \internal
  * \brief Check whether process exists (by PID and optionally executable path)
  *
  * \param[in] pid     PID of process to check
  * \param[in] daemon  If not NULL, path component to match with procfs entry
  *
  * \return Standard Pacemaker return code
  * \note Particular return codes of interest include pcmk_rc_ok for alive,
  *       ESRCH for process is not alive (verified by kill and/or executable path
  *       match), EACCES for caller unable or not allowed to check. A result of
  *       "alive" is less reliable when \p daemon is not provided or procfs is
  *       not available, since there is no guarantee that the PID has not been
  *       recycled for another process.
  * \note This function cannot be used to verify \e authenticity of the process.
  */
 int pcmk__pid_active(pid_t pid, const char *daemon);
 
 int pcmk__read_pidfile(const char *filename, pid_t *pid);
 int pcmk__pidfile_matches(const char *filename, pid_t expected_pid,
                           const char *expected_name, pid_t *pid);
 int pcmk__lock_pidfile(const char *filename, const char *name);
 
 
-/* interal functions related to resource operations (from operations.c) */
+/* internal functions related to resource operations (from operations.c) */
 
 // printf-style format to create operation ID from resource, action, interval
 #define PCMK__OP_FMT "%s_%s_%u"
 
 char *pcmk__op_key(const char *rsc_id, const char *op_type, guint interval_ms);
 char *pcmk__notify_key(const char *rsc_id, const char *notify_type,
                        const char *op_type);
 char *pcmk__transition_key(int transition_id, int action_id, int target_rc,
                            const char *node);
 void pcmk__filter_op_for_digest(xmlNode *param_set);
 
 
 // miscellaneous utilities (from utils.c)
 
 const char *pcmk_message_name(const char *name);
 
 extern int pcmk__score_red;
 extern int pcmk__score_green;
 extern int pcmk__score_yellow;
 
 
 /* internal generic string functions (from strings.c) */
 
 int pcmk__guint_from_hash(GHashTable *table, const char *key, guint default_val,
                           guint *result);
 bool pcmk__starts_with(const char *str, const char *prefix);
 bool pcmk__ends_with(const char *s, const char *match);
 bool pcmk__ends_with_ext(const char *s, const char *match);
 char *pcmk__add_word(char *list, const char *word);
 int pcmk__compress(const char *data, unsigned int length, unsigned int max,
                    char **result, unsigned int *result_len);
 
 int pcmk__parse_ll_range(const char *srcstring, long long *start, long long *end);
 gboolean pcmk__str_in_list(GList *lst, const gchar *s);
 
 bool pcmk__str_any_of(const char *s, ...) G_GNUC_NULL_TERMINATED;
 bool pcmk__str_none_of(const char *s, ...) G_GNUC_NULL_TERMINATED;
 
 /* Correctly displaying singular or plural is complicated; consider "1 node has"
  * vs. "2 nodes have". A flexible solution is to pluralize entire strings, e.g.
  *
  * if (a == 1) {
  *     crm_info("singular message"):
  * } else {
  *     crm_info("plural message");
  * }
  *
  * though even that's not sufficient for all languages besides English (if we
  * ever desire to do translations of output and log messages). But the following
  * convenience macros are "good enough" and more concise for many cases.
  */
 
 /* Example:
  * crm_info("Found %d %s", nentries,
  *          pcmk__plural_alt(nentries, "entry", "entries"));
  */
 #define pcmk__plural_alt(i, s1, s2) (((i) == 1)? (s1) : (s2))
 
 // Example: crm_info("Found %d node%s", nnodes, pcmk__plural_s(nnodes));
 #define pcmk__plural_s(i) pcmk__plural_alt(i, "", "s")
 
 static inline int
 pcmk__str_empty(const char *s)
 {
     return (s == NULL) || (s[0] == '\0');
 }
 
 static inline char *
 pcmk__getpid_s(void)
 {
     return crm_strdup_printf("%lu", (unsigned long) getpid());
 }
 
 // More efficient than g_list_length(list) == 1
 static inline bool
 pcmk__list_of_1(GList *list)
 {
     return list && (list->next == NULL);
 }
 
 // More efficient than g_list_length(list) > 1
 static inline bool
 pcmk__list_of_multiple(GList *list)
 {
     return list && (list->next != NULL);
 }
 
 /* convenience functions for failure-related node attributes */
 
 #define PCMK__FAIL_COUNT_PREFIX   "fail-count"
 #define PCMK__LAST_FAILURE_PREFIX "last-failure"
 
 /*!
  * \internal
  * \brief Generate a failure-related node attribute name for a resource
  *
  * \param[in] prefix       Start of attribute name
  * \param[in] rsc_id       Resource name
  * \param[in] op           Operation name
  * \param[in] interval_ms  Operation interval
  *
  * \return Newly allocated string with attribute name
  *
  * \note Failure attributes are named like PREFIX-RSC#OP_INTERVAL (for example,
  *       "fail-count-myrsc#monitor_30000"). The '#' is used because it is not
  *       a valid character in a resource ID, to reliably distinguish where the
  *       operation name begins. The '_' is used simply to be more comparable to
  *       action labels like "myrsc_monitor_30000".
  */
 static inline char *
 pcmk__fail_attr_name(const char *prefix, const char *rsc_id, const char *op,
                    guint interval_ms)
 {
     CRM_CHECK(prefix && rsc_id && op, return NULL);
     return crm_strdup_printf("%s-%s#%s_%u", prefix, rsc_id, op, interval_ms);
 }
 
 static inline char *
 pcmk__failcount_name(const char *rsc_id, const char *op, guint interval_ms)
 {
     return pcmk__fail_attr_name(PCMK__FAIL_COUNT_PREFIX, rsc_id, op,
                                 interval_ms);
 }
 
 static inline char *
 pcmk__lastfailure_name(const char *rsc_id, const char *op, guint interval_ms)
 {
     return pcmk__fail_attr_name(PCMK__LAST_FAILURE_PREFIX, rsc_id, op,
                                 interval_ms);
 }
 
 #endif /* CRM_COMMON_INTERNAL__H */
diff --git a/include/crm_internal.h b/include/crm_internal.h
index cf8999fff4..efc7d957d7 100644
--- a/include/crm_internal.h
+++ b/include/crm_internal.h
@@ -1,179 +1,160 @@
 /*
  * Copyright 2006-2020 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU Lesser General Public License
  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
  */
 
 #ifndef CRM_INTERNAL__H
 #  define CRM_INTERNAL__H
 
 #  include <config.h>
 #  include <portability.h>
 
 #  include <glib.h>
 #  include <stdbool.h>
 #  include <libxml/tree.h>
 
 /* Public API headers can guard deprecated code with this symbol, thus
  * preventing internal code (which includes this header) from using it, while
  * still allowing external code (which can't include this header) to use it,
  * for backward compatibility.
  */
 #define PCMK__NO_COMPAT
 
 #  include <crm/lrmd.h>
 #  include <crm/common/logging.h>
 #  include <crm/common/ipc_internal.h>
 #  include <crm/common/options_internal.h>
 #  include <crm/common/internal.h>
 
 /* Assorted convenience functions */
 void crm_make_daemon(const char *name, gboolean daemonize, const char *pidfile);
 
 static inline long long
 crm_clear_bit(const char *function, int line, const char *target, long long word, long long bit)
 {
     long long rc = (word & ~bit);
 
     if (rc == word) {
         /* Unchanged */
     } else if (target) {
         crm_trace("Bit 0x%.8llx for %s cleared by %s:%d", bit, target, function, line);
     } else {
         crm_trace("Bit 0x%.8llx cleared by %s:%d", bit, function, line);
     }
 
     return rc;
 }
 
 static inline long long
 crm_set_bit(const char *function, int line, const char *target, long long word, long long bit)
 {
     long long rc = (word | bit);
 
     if (rc == word) {
         /* Unchanged */
     } else if (target) {
         crm_trace("Bit 0x%.8llx for %s set by %s:%d", bit, target, function, line);
     } else {
         crm_trace("Bit 0x%.8llx set by %s:%d", bit, function, line);
     }
 
     return rc;
 }
 
 #  define set_bit(word, bit) word = crm_set_bit(__FUNCTION__, __LINE__, NULL, word, bit)
 #  define clear_bit(word, bit) word = crm_clear_bit(__FUNCTION__, __LINE__, NULL, word, bit)
 
 void strip_text_nodes(xmlNode * xml);
 void pcmk_panic(const char *origin);
 pid_t pcmk_locate_sbd(void);
 
 
 /*
  * XML attribute names used only by internal code
  */
 
 #define PCMK__XA_ATTR_DAMPENING         "attr_dampening"
 #define PCMK__XA_ATTR_FORCE             "attrd_is_force_write"
 #define PCMK__XA_ATTR_INTERVAL          "attr_clear_interval"
 #define PCMK__XA_ATTR_IS_PRIVATE        "attr_is_private"
 #define PCMK__XA_ATTR_IS_REMOTE         "attr_is_remote"
 #define PCMK__XA_ATTR_NAME              "attr_name"
 #define PCMK__XA_ATTR_NODE_ID           "attr_host_id"
 #define PCMK__XA_ATTR_NODE_NAME         "attr_host"
 #define PCMK__XA_ATTR_OPERATION         "attr_clear_operation"
 #define PCMK__XA_ATTR_PATTERN           "attr_regex"
 #define PCMK__XA_ATTR_RESOURCE          "attr_resource"
 #define PCMK__XA_ATTR_SECTION           "attr_section"
 #define PCMK__XA_ATTR_SET               "attr_set"
 #define PCMK__XA_ATTR_USER              "attr_user"
 #define PCMK__XA_ATTR_UUID              "attr_key"
 #define PCMK__XA_ATTR_VALUE             "attr_value"
 #define PCMK__XA_ATTR_VERSION           "attr_version"
 #define PCMK__XA_ATTR_WRITER            "attr_writer"
 #define PCMK__XA_MODE                   "mode"
 #define PCMK__XA_TASK                   "task"
 
 
 /*
  * IPC service names that are only used internally
  */
 
 #  define PCMK__SERVER_BASED_RO		"cib_ro"
 #  define PCMK__SERVER_BASED_RW		"cib_rw"
 #  define PCMK__SERVER_BASED_SHM		"cib_shm"
 
 /*
  * IPC commands that can be sent to Pacemaker daemons
  */
 
 #define PCMK__ATTRD_CMD_PEER_REMOVE     "peer-remove"
 #define PCMK__ATTRD_CMD_UPDATE          "update"
 #define PCMK__ATTRD_CMD_UPDATE_BOTH     "update-both"
 #define PCMK__ATTRD_CMD_UPDATE_DELAY    "update-delay"
 #define PCMK__ATTRD_CMD_QUERY           "query"
 #define PCMK__ATTRD_CMD_REFRESH         "refresh"
 #define PCMK__ATTRD_CMD_FLUSH           "flush"
 #define PCMK__ATTRD_CMD_SYNC            "sync"
 #define PCMK__ATTRD_CMD_SYNC_RESPONSE   "sync-response"
 #define PCMK__ATTRD_CMD_CLEAR_FAILURE   "clear-failure"
 
 #define PCMK__CONTROLD_CMD_NODES        "list-nodes"
 
 /*
  * Environment variables used by Pacemaker
  */
 
 #define PCMK__ENV_PHYSICAL_HOST         "physical_host"
 
 
-#  if SUPPORT_COROSYNC
-#    include <qb/qbipc_common.h>
-#    include <corosync/corotypes.h>
-typedef struct qb_ipc_request_header cs_ipc_header_request_t;
-typedef struct qb_ipc_response_header cs_ipc_header_response_t;
-#  else
-typedef struct {
-    int size __attribute__ ((aligned(8)));
-    int id __attribute__ ((aligned(8)));
-} __attribute__ ((aligned(8))) cs_ipc_header_request_t;
-
-typedef struct {
-    int size __attribute__ ((aligned(8)));
-    int id __attribute__ ((aligned(8)));
-    int error __attribute__ ((aligned(8)));
-} __attribute__ ((aligned(8))) cs_ipc_header_response_t;
-
-#  endif
-
 static inline void *
 realloc_safe(void *ptr, size_t size)
 {
     void *new_ptr;
 
     // realloc(p, 0) can replace free(p) but this wrapper can't
     CRM_ASSERT(size > 0);
 
     new_ptr = realloc(ptr, size);
     if (new_ptr == NULL) {
         free(ptr);
         abort();
     }
     return new_ptr;
 }
 
 const char *crm_xml_add_last_written(xmlNode *xml_node);
 void crm_xml_dump(xmlNode * data, int options, char **buffer, int *offset, int *max, int depth);
 void crm_buffer_add_char(char **buffer, int *offset, int *max, char c);
 
 #if defined(PCMK__WITH_ATTRIBUTE_OUTPUT_ARGS)
 #  define PCMK__OUTPUT_ARGS(ARGS...) __attribute__((output_args(ARGS)))
 #else
 #  define PCMK__OUTPUT_ARGS(ARGS...)
 #endif
 
 #endif                          /* CRM_INTERNAL__H */
diff --git a/lib/common/ipc_controld.c b/lib/common/ipc_controld.c
index 5917cc5dc0..22cb9e0385 100644
--- a/lib/common/ipc_controld.c
+++ b/lib/common/ipc_controld.c
@@ -1,668 +1,666 @@
 /*
  * Copyright 2020 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU Lesser General Public License
  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #include <stdio.h>
 #include <stdbool.h>
 #include <errno.h>
 #include <libxml/tree.h>
 
 #include <crm/crm.h>
 #include <crm/msg_xml.h>
 #include <crm/common/xml.h>
 #include <crm/common/ipc.h>
 #include <crm/common/ipc_internal.h>
 #include <crm/common/ipc_controld.h>
 #include "crmcommon_private.h"
 
 struct controld_api_private_s {
     char *client_uuid;
     unsigned int replies_expected;
 };
 
 // \return Standard Pacemaker return code
 static int
 new_data(pcmk_ipc_api_t *api)
 {
     struct controld_api_private_s *private = NULL;
 
     api->api_data = calloc(1, sizeof(struct controld_api_private_s));
 
     if (api->api_data == NULL) {
         return errno;
     }
 
     private = api->api_data;
 
     /* This is set to the PID because that's how it was always done, but PIDs
      * are not unique because clients can be remote. The value appears to be
      * unused other than as part of F_CRM_SYS_FROM in IPC requests, which is
      * only compared against the internal system names (CRM_SYSTEM_TENGINE,
      * etc.), so it shouldn't be a problem.
      */
     private->client_uuid = pcmk__getpid_s();
 
     /* @TODO Implement a call ID model similar to the CIB, executor, and fencer
      *       IPC APIs, so that requests and replies can be matched, and
      *       duplicate replies can be discarded.
      */
     return pcmk_rc_ok;
 }
 
 static void
 free_data(void *data)
 {
     free(((struct controld_api_private_s *) data)->client_uuid);
     free(data);
 }
 
 // \return Standard Pacemaker return code
 static int
 post_connect(pcmk_ipc_api_t *api)
 {
     /* The controller currently requires clients to register via a hello
      * request, but does not reply back.
      */
     struct controld_api_private_s *private = api->api_data;
     const char *client_name = crm_system_name? crm_system_name : "client";
     xmlNode *hello;
     int rc;
 
     hello = create_hello_message(private->client_uuid, client_name,
                                  PCMK__CONTROLD_API_MAJOR,
                                  PCMK__CONTROLD_API_MINOR);
     rc = pcmk__send_ipc_request(api, hello);
     free_xml(hello);
     if (rc != pcmk_rc_ok) {
         crm_info("Could not send IPC hello to %s: %s " CRM_XS " rc=%s",
                  pcmk_ipc_name(api, true), pcmk_rc_str(rc), rc);
     } else {
         crm_debug("Sent IPC hello to %s", pcmk_ipc_name(api, true));
     }
     return rc;
 }
 
 #define xml_true(xml, field) crm_is_true(crm_element_value(xml, field))
 
 static void
 set_node_info_data(pcmk_controld_api_reply_t *data, xmlNode *msg_data)
 {
     data->reply_type = pcmk_controld_reply_info;
     if (msg_data == NULL) {
         return;
     }
     data->data.node_info.have_quorum = xml_true(msg_data, XML_ATTR_HAVE_QUORUM);
     data->data.node_info.is_remote = xml_true(msg_data, XML_NODE_IS_REMOTE);
     crm_element_value_int(msg_data, XML_ATTR_ID, &(data->data.node_info.id));
     data->data.node_info.uuid = crm_element_value(msg_data, XML_ATTR_UUID);
     data->data.node_info.uname = crm_element_value(msg_data, XML_ATTR_UNAME);
     data->data.node_info.state = crm_element_value(msg_data, XML_NODE_IS_PEER);
 }
 
 static void
 set_ping_data(pcmk_controld_api_reply_t *data, xmlNode *msg_data)
 {
     data->reply_type = pcmk_controld_reply_ping;
     if (msg_data == NULL) {
         return;
     }
     data->data.ping.sys_from = crm_element_value(msg_data,
                                                  XML_PING_ATTR_SYSFROM);
     data->data.ping.fsa_state = crm_element_value(msg_data,
                                                   XML_PING_ATTR_CRMDSTATE);
     data->data.ping.result = crm_element_value(msg_data, XML_PING_ATTR_STATUS);
 }
 
 static void
 set_nodes_data(pcmk_controld_api_reply_t *data, xmlNode *msg_data)
 {
     pcmk_controld_api_node_t *node_info;
 
     data->reply_type = pcmk_controld_reply_nodes;
     for (xmlNode *node = first_named_child(msg_data, XML_CIB_TAG_NODE);
          node != NULL; node = crm_next_same_xml(node)) {
 
         long long id_ll = 0;
 
         node_info = calloc(1, sizeof(pcmk_controld_api_node_t));
         crm_element_value_ll(node, XML_ATTR_ID, &id_ll);
         if (id_ll > 0) {
             node_info->id = id_ll;
         }
         node_info->uname = crm_element_value(node, XML_ATTR_UNAME);
         node_info->state = crm_element_value(node, XML_NODE_IN_CLUSTER);
         data->data.nodes = g_list_prepend(data->data.nodes, node_info);
     }
 }
 
 static bool
 reply_expected(pcmk_ipc_api_t *api, xmlNode *request)
 {
     const char *command = crm_element_value(request, F_CRM_TASK);
 
     if (command == NULL) {
         return false;
     }
 
     // We only need to handle commands that functions in this file can send
     return !strcmp(command, CRM_OP_REPROBE)
            || !strcmp(command, CRM_OP_NODE_INFO)
            || !strcmp(command, CRM_OP_PING)
            || !strcmp(command, CRM_OP_LRM_FAIL)
            || !strcmp(command, CRM_OP_LRM_DELETE);
 }
 
 static void
 dispatch(pcmk_ipc_api_t *api, xmlNode *reply)
 {
     struct controld_api_private_s *private = api->api_data;
     crm_exit_t status = CRM_EX_OK;
     xmlNode *msg_data = NULL;
     const char *value = NULL;
     pcmk_controld_api_reply_t reply_data = {
         pcmk_controld_reply_unknown, NULL, NULL,
     };
 
     if (private->replies_expected > 0) {
         private->replies_expected--;
     }
 
     // Do some basic validation of the reply
 
     /* @TODO We should be able to verify that value is always a response, but
      *       currently the controller doesn't always properly set the type. Even
      *       if we fix the controller, we'll still need to handle replies from
      *       old versions (feature set could be used to differentiate).
      */
     value = crm_element_value(reply, F_CRM_MSG_TYPE);
     if ((value == NULL) || (strcmp(value, XML_ATTR_REQUEST)
                             && strcmp(value, XML_ATTR_RESPONSE))) {
         crm_debug("Unrecognizable controller message: invalid message type '%s'",
                   crm_str(value));
         status = CRM_EX_PROTOCOL;
-        reply = NULL;
+        goto done;
     }
 
     if (crm_element_value(reply, XML_ATTR_REFERENCE) == NULL) {
         crm_debug("Unrecognizable controller message: no reference");
         status = CRM_EX_PROTOCOL;
-        reply = NULL;
+        goto done;
     }
 
     value = crm_element_value(reply, F_CRM_TASK);
     if (value == NULL) {
         crm_debug("Unrecognizable controller message: no command name");
         status = CRM_EX_PROTOCOL;
-        reply = NULL;
+        goto done;
     }
 
     // Parse useful info from reply
 
-    if (reply != NULL) {
-        reply_data.feature_set = crm_element_value(reply, XML_ATTR_VERSION);
-        reply_data.host_from = crm_element_value(reply, F_CRM_HOST_FROM);
-        msg_data = get_message_xml(reply, F_CRM_DATA);
+    reply_data.feature_set = crm_element_value(reply, XML_ATTR_VERSION);
+    reply_data.host_from = crm_element_value(reply, F_CRM_HOST_FROM);
+    msg_data = get_message_xml(reply, F_CRM_DATA);
 
-        if (!strcmp(value, CRM_OP_REPROBE)) {
-            reply_data.reply_type = pcmk_controld_reply_reprobe;
+    if (!strcmp(value, CRM_OP_REPROBE)) {
+        reply_data.reply_type = pcmk_controld_reply_reprobe;
 
-        } else if (!strcmp(value, CRM_OP_NODE_INFO)) {
-            set_node_info_data(&reply_data, msg_data);
+    } else if (!strcmp(value, CRM_OP_NODE_INFO)) {
+        set_node_info_data(&reply_data, msg_data);
 
-        } else if (!strcmp(value, CRM_OP_INVOKE_LRM)) {
-            reply_data.reply_type = pcmk_controld_reply_resource;
-            reply_data.data.resource.node_state = msg_data;
+    } else if (!strcmp(value, CRM_OP_INVOKE_LRM)) {
+        reply_data.reply_type = pcmk_controld_reply_resource;
+        reply_data.data.resource.node_state = msg_data;
 
-        } else if (!strcmp(value, CRM_OP_PING)) {
-            set_ping_data(&reply_data, msg_data);
+    } else if (!strcmp(value, CRM_OP_PING)) {
+        set_ping_data(&reply_data, msg_data);
 
-        } else if (!strcmp(value, PCMK__CONTROLD_CMD_NODES)) {
-            set_nodes_data(&reply_data, msg_data);
+    } else if (!strcmp(value, PCMK__CONTROLD_CMD_NODES)) {
+        set_nodes_data(&reply_data, msg_data);
 
-        } else {
-            crm_debug("Unrecognizable controller message: unknown command '%s'",
-                      value);
-            status = CRM_EX_PROTOCOL;
-            reply = NULL;
-        }
+    } else {
+        crm_debug("Unrecognizable controller message: unknown command '%s'",
+                  value);
+        status = CRM_EX_PROTOCOL;
     }
 
+done:
     pcmk__call_ipc_callback(api, pcmk_ipc_event_reply, status, &reply_data);
 
     // Free any reply data that was allocated
     if (safe_str_eq(value, PCMK__CONTROLD_CMD_NODES)) {
         g_list_free_full(reply_data.data.nodes, free);
     }
 }
 
 pcmk__ipc_methods_t *
 pcmk__controld_api_methods()
 {
     pcmk__ipc_methods_t *cmds = calloc(1, sizeof(pcmk__ipc_methods_t));
 
     if (cmds != NULL) {
         cmds->new_data = new_data;
         cmds->free_data = free_data;
         cmds->post_connect = post_connect;
         cmds->reply_expected = reply_expected;
         cmds->dispatch = dispatch;
     }
     return cmds;
 }
 
 /*!
  * \internal
  * \brief Create XML for a controller IPC request
  *
  * \param[in] api       Controller connection
  * \param[in] op        Controller IPC command name
  * \param[in] node      Node name to set as destination host
  * \param[in] msg_data  XML to attach to request as message data
  *
  * \return Newly allocated XML for request
  */
 static xmlNode *
 create_controller_request(pcmk_ipc_api_t *api, const char *op,
                           const char *node, xmlNode *msg_data)
 {
     struct controld_api_private_s *private = api->api_data;
     const char *sys_to = NULL;
 
     if ((node == NULL) && !strcmp(op, CRM_OP_PING)) {
         sys_to = CRM_SYSTEM_DC;
     } else {
         sys_to = CRM_SYSTEM_CRMD;
     }
     return create_request(op, msg_data, node, sys_to,
                           (crm_system_name? crm_system_name : "client"),
                           private->client_uuid);
 }
 
 // \return Standard Pacemaker return code
 static int
 send_controller_request(pcmk_ipc_api_t *api, xmlNode *request,
                         bool reply_is_expected)
 {
     int rc;
 
     if (crm_element_value(request, XML_ATTR_REFERENCE) == NULL) {
         return EINVAL;
     }
     rc = pcmk__send_ipc_request(api, request);
     if ((rc == pcmk_rc_ok) && reply_is_expected) {
         struct controld_api_private_s *private = api->api_data;
 
         private->replies_expected++;
     }
     return rc;
 }
 
 static xmlNode *
 create_reprobe_message_data(const char *target_node, const char *router_node)
 {
     xmlNode *msg_data;
 
     msg_data = create_xml_node(NULL, "data_for_" CRM_OP_REPROBE);
     crm_xml_add(msg_data, XML_LRM_ATTR_TARGET, target_node);
     if ((router_node != NULL) && safe_str_neq(router_node, target_node)) {
         crm_xml_add(msg_data, XML_LRM_ATTR_ROUTER_NODE, router_node);
     }
     return msg_data;
 }
 
 /*!
  * \brief Send a reprobe controller operation
  *
  * \param[in] api          Controller connection
  * \param[in] target_node  Name of node to reprobe
  * \param[in] router_node  Router node for host
  *
  * \return Standard Pacemaker return code
  * \note Event callback will get a reply of type pcmk_controld_reply_reprobe.
  */
 int
 pcmk_controld_api_reprobe(pcmk_ipc_api_t *api, const char *target_node,
                           const char *router_node)
 {
     xmlNode *request;
     xmlNode *msg_data;
     int rc = pcmk_rc_ok;
 
     if (api == NULL) {
         return EINVAL;
     }
     if (router_node == NULL) {
         router_node = target_node;
     }
     crm_debug("Sending %s IPC request to reprobe %s via %s",
               pcmk_ipc_name(api, true), crm_str(target_node),
               crm_str(router_node));
     msg_data = create_reprobe_message_data(target_node, router_node);
     request = create_controller_request(api, CRM_OP_REPROBE, router_node,
                                         msg_data);
     rc = send_controller_request(api, request, true);
     free_xml(msg_data);
     free_xml(request);
     return rc;
 }
 
 /*!
  * \brief Send a "node info" controller operation
  *
  * \param[in] api          Controller connection
  * \param[in] nodeid       ID of node to get info for (or 0 for local node)
  *
  * \return Standard Pacemaker return code
  * \note Event callback will get a reply of type pcmk_controld_reply_info.
  */
 int
 pcmk_controld_api_node_info(pcmk_ipc_api_t *api, uint32_t nodeid)
 {
     xmlNode *request;
     int rc = pcmk_rc_ok;
 
     request = create_controller_request(api, CRM_OP_NODE_INFO, NULL, NULL);
     if (request == NULL) {
         return EINVAL;
     }
     if (nodeid > 0) {
         crm_xml_set_id(request, "%lu", (unsigned long) nodeid);
     }
 
     rc = send_controller_request(api, request, true);
     free_xml(request);
     return rc;
 }
 
 /*!
  * \brief Ask the controller for status
  *
  * \param[in] api          Controller connection
  * \param[in] node_name    Name of node whose status is desired (or NULL for DC)
  *
  * \return Standard Pacemaker return code
  * \note Event callback will get a reply of type pcmk_controld_reply_ping.
  */
 int
 pcmk_controld_api_ping(pcmk_ipc_api_t *api, const char *node_name)
 {
     xmlNode *request;
     int rc = pcmk_rc_ok;
 
     request = create_controller_request(api, CRM_OP_PING, node_name, NULL);
     if (request == NULL) {
         return EINVAL;
     }
     rc = send_controller_request(api, request, true);
     free_xml(request);
     return rc;
 }
 
 /*!
  * \brief Ask the controller for cluster information
  *
  * \param[in] api          Controller connection
  *
  * \return Standard Pacemaker return code
  * \note Event callback will get a reply of type pcmk_controld_reply_nodes.
  */
 int
 pcmk_controld_api_list_nodes(pcmk_ipc_api_t *api)
 {
     xmlNode *request;
     int rc = EINVAL;
 
     request = create_controller_request(api, PCMK__CONTROLD_CMD_NODES, NULL,
                                         NULL);
     if (request != NULL) {
         rc = send_controller_request(api, request, true);
         free_xml(request);
     }
     return rc;
 }
 
 /*!
  * \internal
  * \brief Ask the controller to shut down
  *
  * \param[in] api          Controller connection
  * \param[in] node_name    Name of node whose controller should shut down
  *
  * \return Standard Pacemaker return code
  *
  * \note This capability currently does not work, so the function is considered
  *       internal. It will likely be removed.
  * \note Event callback will not get a reply.
  */
 int
 pcmk_controld_api_shutdown(pcmk_ipc_api_t *api, const char *node_name)
 {
     xmlNode *request;
     int rc = pcmk_rc_ok;
 
     request = create_controller_request(api, CRM_OP_SHUTDOWN, NULL, NULL);
     if (request == NULL) {
         return EINVAL;
     }
     rc = send_controller_request(api, request, false);
     free_xml(request);
     return rc;
 }
 
 /*!
  * \internal
  * \brief Ask the controller to start a DC election
  *
  * \param[in] api          Controller connection
  *
  * \return Standard Pacemaker return code
  *
  * \note This capability currently does not work, so the function is considered
  *       internal. It will likely be removed.
  * \note Event callback will not get a reply.
  */
 int
 pcmk_controld_api_start_election(pcmk_ipc_api_t *api)
 {
     xmlNode *request;
     int rc = pcmk_rc_ok;
 
     request = create_controller_request(api, CRM_OP_VOTE, NULL, NULL);
     if (request == NULL) {
         return EINVAL;
     }
     rc = send_controller_request(api, request, false);
     free_xml(request);
     return rc;
 }
 
 // \return Standard Pacemaker return code
 static int
 controller_resource_op(pcmk_ipc_api_t *api, const char *op,
                        const char *target_node, const char *router_node,
                        bool cib_only, const char *rsc_id,
                        const char *rsc_long_id, const char *standard,
                        const char *provider, const char *type)
 {
     int rc = pcmk_rc_ok;
     char *key;
     xmlNode *request, *msg_data, *xml_rsc, *params;
 
     if (api == NULL) {
         return EINVAL;
     }
     if (router_node == NULL) {
         router_node = target_node;
     }
 
     msg_data = create_xml_node(NULL, XML_GRAPH_TAG_RSC_OP);
 
     /* The controller logs the transition key from resource op requests, so we
      * need to have *something* for it.
      * @TODO don't use "crm-resource"
      */
     key = pcmk__transition_key(0, getpid(), 0,
                                "xxxxxxxx-xrsc-opxx-xcrm-resourcexxxx");
     crm_xml_add(msg_data, XML_ATTR_TRANSITION_KEY, key);
     free(key);
 
     crm_xml_add(msg_data, XML_LRM_ATTR_TARGET, target_node);
     if (safe_str_neq(router_node, target_node)) {
         crm_xml_add(msg_data, XML_LRM_ATTR_ROUTER_NODE, router_node);
     }
 
     if (cib_only) {
         // Indicate that only the CIB needs to be cleaned
         crm_xml_add(msg_data, PCMK__XA_MODE, XML_TAG_CIB);
     }
 
     xml_rsc = create_xml_node(msg_data, XML_CIB_TAG_RESOURCE);
     crm_xml_add(xml_rsc, XML_ATTR_ID, rsc_id);
     crm_xml_add(xml_rsc, XML_ATTR_ID_LONG, rsc_long_id);
     crm_xml_add(xml_rsc, XML_AGENT_ATTR_CLASS, standard);
     crm_xml_add(xml_rsc, XML_AGENT_ATTR_PROVIDER, provider);
     crm_xml_add(xml_rsc, XML_ATTR_TYPE, type);
 
     params = create_xml_node(msg_data, XML_TAG_ATTRS);
     crm_xml_add(params, XML_ATTR_CRM_VERSION, CRM_FEATURE_SET);
 
     // The controller parses the timeout from the request
     key = crm_meta_name(XML_ATTR_TIMEOUT);
     crm_xml_add(params, key, "60000");  /* 1 minute */ //@TODO pass as arg
     free(key);
 
     request = create_controller_request(api, op, router_node, msg_data);
     rc = send_controller_request(api, request, true);
     free_xml(msg_data);
     free_xml(request);
     return rc;
 }
 
 /*!
  * \brief Ask the controller to fail a resource
  *
  * \param[in] api          Controller connection
  * \param[in] target_node  Name of node resource is on
  * \param[in] router_node  Router node for target
  * \param[in] rsc_id       ID of resource to fail
  * \param[in] rsc_long_id  Long ID of resource (if any)
  * \param[in] standard     Standard of resource
  * \param[in] provider     Provider of resource (if any)
  * \param[in] type         Type of resource to fail
  *
  * \return Standard Pacemaker return code
  * \note Event callback will get a reply of type pcmk_controld_reply_resource.
  */
 int
 pcmk_controld_api_fail(pcmk_ipc_api_t *api,
                        const char *target_node, const char *router_node,
                        const char *rsc_id, const char *rsc_long_id,
                        const char *standard, const char *provider,
                        const char *type)
 {
     crm_debug("Sending %s IPC request to fail %s (a.k.a. %s) on %s via %s",
               pcmk_ipc_name(api, true), crm_str(rsc_id), crm_str(rsc_long_id),
               crm_str(target_node), crm_str(router_node));
     return controller_resource_op(api, CRM_OP_LRM_FAIL, target_node,
                                   router_node, false, rsc_id, rsc_long_id,
                                   standard, provider, type);
 }
 
 /*!
  * \brief Ask the controller to refresh a resource
  *
  * \param[in] api          Controller connection
  * \param[in] target_node  Name of node resource is on
  * \param[in] router_node  Router node for target
  * \param[in] rsc_id       ID of resource to refresh
  * \param[in] rsc_long_id  Long ID of resource (if any)
  * \param[in] standard     Standard of resource
  * \param[in] provider     Provider of resource (if any)
  * \param[in] type         Type of resource
  * \param[in] cib_only     If true, clean resource from CIB only
  *
  * \return Standard Pacemaker return code
  * \note Event callback will get a reply of type pcmk_controld_reply_resource.
  */
 int
 pcmk_controld_api_refresh(pcmk_ipc_api_t *api, const char *target_node,
                           const char *router_node,
                           const char *rsc_id, const char *rsc_long_id,
                           const char *standard, const char *provider,
                           const char *type, bool cib_only)
 {
     crm_debug("Sending %s IPC request to refresh %s (a.k.a. %s) on %s via %s",
               pcmk_ipc_name(api, true), crm_str(rsc_id), crm_str(rsc_long_id),
               crm_str(target_node), crm_str(router_node));
     return controller_resource_op(api, CRM_OP_LRM_DELETE, target_node,
                                   router_node, cib_only, rsc_id, rsc_long_id,
                                   standard, provider, type);
 }
 
 /*!
  * \brief Get the number of IPC replies currently expected from the controller
  *
  * \param[in] api  Controller IPC API connection
  *
  * \return Number of replies expected
  */
 unsigned int
 pcmk_controld_api_replies_expected(pcmk_ipc_api_t *api)
 {
     struct controld_api_private_s *private = api->api_data;
 
     return private->replies_expected;
 }
 
 /*!
  * \brief Create XML for a controller IPC "hello" message
  *
  * \deprecated This function is deprecated as part of the public C API.
  */
 // \todo make this static to this file when breaking API backward compatibility
 xmlNode *
 create_hello_message(const char *uuid, const char *client_name,
                      const char *major_version, const char *minor_version)
 {
     xmlNode *hello_node = NULL;
     xmlNode *hello = NULL;
 
     if (pcmk__str_empty(uuid) || pcmk__str_empty(client_name)
         || pcmk__str_empty(major_version) || pcmk__str_empty(minor_version)) {
         crm_err("Could not create IPC hello message from %s (UUID %s): "
                 "missing information",
                 client_name? client_name : "unknown client",
                 uuid? uuid : "unknown");
         return NULL;
     }
 
     hello_node = create_xml_node(NULL, XML_TAG_OPTIONS);
     if (hello_node == NULL) {
         crm_err("Could not create IPC hello message from %s (UUID %s): "
                 "Message data creation failed", client_name, uuid);
         return NULL;
     }
 
     crm_xml_add(hello_node, "major_version", major_version);
     crm_xml_add(hello_node, "minor_version", minor_version);
     crm_xml_add(hello_node, "client_name", client_name);
     crm_xml_add(hello_node, "client_uuid", uuid);
 
     hello = create_request(CRM_OP_HELLO, hello_node, NULL, NULL, client_name, uuid);
     if (hello == NULL) {
         crm_err("Could not create IPC hello message from %s (UUID %s): "
                 "Request creation failed", client_name, uuid);
         return NULL;
     }
     free_xml(hello_node);
 
     crm_trace("Created hello message from %s (UUID %s)", client_name, uuid);
     return hello;
 }
diff --git a/m4/CHECK_ENUM_VALUE.m4 b/m4/CHECK_ENUM_VALUE.m4
index 8c4e3d22e2..50bbe23a74 100644
--- a/m4/CHECK_ENUM_VALUE.m4
+++ b/m4/CHECK_ENUM_VALUE.m4
@@ -1,19 +1,21 @@
 # CHECK_ENUM_VALUE([HEADER], [ENUM_NAME], [ENUM_VALUE])
 #
 # Define HAVE_[ENUM_NAME]_[ENUM_VALUE] if the specified enum value is
 # available.
 dnl
-dnl Copyright 2018 Andrew Beekhof <andrew@beekhof.net>
+dnl Copyright 2018-2020 the Pacemaker project contributors
+dnl
+dnl The version control history for this file may have further details.
 dnl
 dnl This source code is licensed under the GNU General Public License version 2
 dnl or later (GPLv2+) WITHOUT ANY WARRANTY.
 
 AC_DEFUN([CHECK_ENUM_VALUE], [
     AC_MSG_CHECKING(whether $1 defines enum $2 value $3)
     AC_COMPILE_IFELSE([AC_LANG_PROGRAM([#include <$1>],
                                        [enum $2 check_$2_$3 = $3])],
                       [AC_DEFINE_UNQUOTED(HAVE_$2_$3, 1,
                                           [Whether enum $2 supports $3])
                        AC_MSG_RESULT(yes)],
                       [AC_MSG_RESULT(no)])
 ])