No OneTemporary
Actions

Size

899 KB

Referenced Files

None

Subscribers

None

View Options

This file is larger than 256 KB, so syntax highlighting was skipped.

	diff --git a/cts/CTStests.py b/cts/CTStests.py
	index 3a91dc7082..8b92d0429a 100644
	--- a/cts/CTStests.py
	+++ b/cts/CTStests.py
	@@ -1,3129 +1,3127 @@
	""" Test-specific classes for Pacemaker's Cluster Test Suite (CTS)
	"""

	# Pacemaker targets compatibility with Python 2.7 and 3.2+
	from __future__ import print_function, unicode_literals, absolute_import, division

	__copyright__ = "Copyright 2000-2019 the Pacemaker project contributors"
	__license__ = "GNU General Public License version 2 or later (GPLv2+) WITHOUT ANY WARRANTY"

	#
	# SPECIAL NOTE:
	#
	# Tests may NOT implement any cluster-manager-specific code in them.
	# EXTEND the ClusterManager object to provide the base capabilities
	# the test needs if you need to do something that the current CM classes
	# do not. Otherwise you screw up the whole point of the object structure
	# in CTS.
	#
	# Thank you.
	#

	import os
	import re
	import time
	import subprocess
	import tempfile

	from stat import *
	from cts import CTS
	from cts.CTSaudits import *
	from cts.CTSvars import *
	from cts.patterns import PatternSelector
	from cts.logging import LogFactory
	from cts.remote import RemoteFactory, input_wrapper
	from cts.watcher import LogWatcher
	from cts.environment import EnvFactory

	AllTestClasses = [ ]


	class CTSTest(object):
	'''
	A Cluster test.
	We implement the basic set of properties and behaviors for a generic
	cluster test.

	Cluster tests track their own statistics.
	We keep each of the kinds of counts we track as separate {name,value}
	pairs.
	'''

	def __init__(self, cm):
	#self.name="the unnamed test"
	self.Stats = {"calls":0
	, "success":0
	, "failure":0
	, "skipped":0
	, "auditfail":0}

	# if not issubclass(cm.__class__, ClusterManager):
	# raise ValueError("Must be a ClusterManager object")
	self.CM = cm
	self.Env = EnvFactory().getInstance()
	self.rsh = RemoteFactory().getInstance()
	self.logger = LogFactory()
	self.templates = PatternSelector(cm["Name"])
	self.Audits = []
	self.timeout = 120
	self.passed = 1
	self.is_loop = 0
	self.is_unsafe = 0
	self.is_docker_unsafe = 0
	self.is_experimental = 0
	self.is_container = 0
	self.is_valgrind = 0
	self.benchmark = 0 # which tests to benchmark
	self.timer = {} # timers

	def log(self, args):
	self.logger.log(args)

	def debug(self, args):
	self.logger.debug(args)

	def has_key(self, key):
	return key in self.Stats

	def __setitem__(self, key, value):
	self.Stats[key] = value

	def __getitem__(self, key):
	if str(key) == "0":
	raise ValueError("Bad call to 'foo in X', should reference 'foo in X.Stats' instead")

	if key in self.Stats:
	return self.Stats[key]
	return None

	def log_mark(self, msg):
	self.debug("MARK: test %s %s %d" % (self.name,msg,time.time()))
	return

	def get_timer(self,key = "test"):
	try: return self.timer[key]
	except: return 0

	def set_timer(self,key = "test"):
	self.timer[key] = time.time()
	return self.timer[key]

	def log_timer(self,key = "test"):
	elapsed = 0
	if key in self.timer:
	elapsed = time.time() - self.timer[key]
	s = key == "test" and self.name or "%s:%s" % (self.name,key)
	self.debug("%s runtime: %.2f" % (s, elapsed))
	del self.timer[key]
	return elapsed

	def incr(self, name):
	'''Increment (or initialize) the value associated with the given name'''
	if not name in self.Stats:
	self.Stats[name] = 0
	self.Stats[name] = self.Stats[name]+1

	# Reset the test passed boolean
	if name == "calls":
	self.passed = 1

	def failure(self, reason="none"):
	'''Increment the failure count'''
	self.passed = 0
	self.incr("failure")
	self.logger.log(("Test %s" % self.name).ljust(35) + " FAILED: %s" % reason)
	return None

	def success(self):
	'''Increment the success count'''
	self.incr("success")
	return 1

	def skipped(self):
	'''Increment the skipped count'''
	self.incr("skipped")
	return 1

	def __call__(self, node):
	'''Perform the given test'''
	raise ValueError("Abstract Class member (__call__)")
	self.incr("calls")
	return self.failure()

	def audit(self):
	passed = 1
	if len(self.Audits) > 0:
	for audit in self.Audits:
	if not audit():
	self.logger.log("Internal %s Audit %s FAILED." % (self.name, audit.name()))
	self.incr("auditfail")
	passed = 0
	return passed

	def setup(self, node):
	'''Setup the given test'''
	return self.success()

	def teardown(self, node):
	'''Tear down the given test'''
	return self.success()

	def create_watch(self, patterns, timeout, name=None):
	if not name:
	name = self.name
	return LogWatcher(self.Env["LogFileName"], patterns, name, timeout, kind=self.Env["LogWatcher"], hosts=self.Env["nodes"])

	def local_badnews(self, prefix, watch, local_ignore=[]):
	errcount = 0
	if not prefix:
	prefix = "LocalBadNews:"

	ignorelist = []
	ignorelist.append(" CTS: ")
	ignorelist.append(prefix)
	ignorelist.extend(local_ignore)

	while errcount < 100:
	match = watch.look(0)
	if match:
	add_err = 1
	for ignore in ignorelist:
	if add_err == 1 and re.search(ignore, match):
	add_err = 0
	if add_err == 1:
	self.logger.log(prefix + " " + match)
	errcount = errcount + 1
	else:
	break
	else:
	self.logger.log("Too many errors!")

	watch.end()
	return errcount

	def is_applicable(self):
	return self.is_applicable_common()

	def is_applicable_common(self):
	'''Return TRUE if we are applicable in the current test configuration'''
	#raise ValueError("Abstract Class member (is_applicable)")

	if self.is_loop and not self.Env["loop-tests"]:
	return 0
	elif self.is_unsafe and not self.Env["unsafe-tests"]:
	return 0
	elif self.is_valgrind and not self.Env["valgrind-tests"]:
	return 0
	elif self.is_experimental and not self.Env["experimental-tests"]:
	return 0
	elif self.is_docker_unsafe and self.Env["docker"]:
	return 0
	elif self.is_container and not self.Env["container-tests"]:
	return 0
	elif self.Env["benchmark"] and self.benchmark == 0:
	return 0

	return 1

	def find_ocfs2_resources(self, node):
	self.r_o2cb = None
	self.r_ocfs2 = []

	(rc, lines) = self.rsh(node, "crm_resource -c", None)
	for line in lines:
	if re.search("^Resource", line):
	r = AuditResource(self.CM, line)
	if r.rtype == "o2cb" and r.parent != "NA":
	self.debug("Found o2cb: %s" % self.r_o2cb)
	self.r_o2cb = r.parent
	if re.search("^Constraint", line):
	c = AuditConstraint(self.CM, line)
	if c.type == "rsc_colocation" and c.target == self.r_o2cb:
	self.r_ocfs2.append(c.rsc)

	self.debug("Found ocfs2 filesystems: %s" % repr(self.r_ocfs2))
	return len(self.r_ocfs2)

	def canrunnow(self, node):
	'''Return TRUE if we can meaningfully run right now'''
	return 1

	def errorstoignore(self):
	'''Return list of errors which are 'normal' and should be ignored'''
	return []


	class StopTest(CTSTest):
	'''Stop (deactivate) the cluster manager on a node'''
	def __init__(self, cm):
	CTSTest.__init__(self, cm)
	self.name = "Stop"

	def __call__(self, node):
	'''Perform the 'stop' test. '''
	self.incr("calls")
	if self.CM.ShouldBeStatus[node] != "up":
	return self.skipped()

	patterns = []
	# Technically we should always be able to notice ourselves stopping
	patterns.append(self.templates["Pat:We_stopped"] % node)

	# Any active node needs to notice this one left
	# (note that this won't work if we have multiple partitions)
	for other in self.Env["nodes"]:
	if self.CM.ShouldBeStatus[other] == "up" and other != node:
	patterns.append(self.templates["Pat:They_stopped"] %(other, self.CM.key_for_node(node)))
	#self.debug("Checking %s will notice %s left"%(other, node))

	watch = self.create_watch(patterns, self.Env["DeadTime"])
	watch.setwatch()

	if node == self.CM.OurNode:
	self.incr("us")
	else:
	if self.CM.upcount() <= 1:
	self.incr("all")
	else:
	self.incr("them")

	self.CM.StopaCM(node)
	watch_result = watch.lookforall()

	failreason = None
	UnmatchedList = "\|\|"
	if watch.unmatched:
	(rc, output) = self.rsh(node, "/bin/ps axf", None)
	for line in output:
	self.debug(line)

	(rc, output) = self.rsh(node, "/usr/sbin/dlm_tool dump 2>/dev/null", None)
	for line in output:
	self.debug(line)

	for regex in watch.unmatched:
	self.logger.log ("ERROR: Shutdown pattern not found: %s" % (regex))
	UnmatchedList += regex + "\|\|";
	failreason = "Missing shutdown pattern"

	self.CM.cluster_stable(self.Env["DeadTime"])

	if not watch.unmatched or self.CM.upcount() == 0:
	return self.success()

	if len(watch.unmatched) >= self.CM.upcount():
	return self.failure("no match against (%s)" % UnmatchedList)

	if failreason == None:
	return self.success()
	else:
	return self.failure(failreason)
	#
	# We don't register StopTest because it's better when called by
	# another test...
	#


	class StartTest(CTSTest):
	'''Start (activate) the cluster manager on a node'''
	def __init__(self, cm, debug=None):
	CTSTest.__init__(self,cm)
	self.name = "start"
	self.debug = debug

	def __call__(self, node):
	'''Perform the 'start' test. '''
	self.incr("calls")

	if self.CM.upcount() == 0:
	self.incr("us")
	else:
	self.incr("them")

	if self.CM.ShouldBeStatus[node] != "down":
	return self.skipped()
	elif self.CM.StartaCM(node):
	return self.success()
	else:
	return self.failure("Startup %s on node %s failed"
	% (self.Env["Name"], node))

	#
	# We don't register StartTest because it's better when called by
	# another test...
	#


	class FlipTest(CTSTest):
	'''If it's running, stop it. If it's stopped start it.
	Overthrow the status quo...
	'''
	def __init__(self, cm):
	CTSTest.__init__(self,cm)
	self.name = "Flip"
	self.start = StartTest(cm)
	self.stop = StopTest(cm)

	def __call__(self, node):
	'''Perform the 'Flip' test. '''
	self.incr("calls")
	if self.CM.ShouldBeStatus[node] == "up":
	self.incr("stopped")
	ret = self.stop(node)
	type = "up->down"
	# Give the cluster time to recognize it's gone...
	time.sleep(self.Env["StableTime"])
	elif self.CM.ShouldBeStatus[node] == "down":
	self.incr("started")
	ret = self.start(node)
	type = "down->up"
	else:
	return self.skipped()

	self.incr(type)
	if ret:
	return self.success()
	else:
	return self.failure("%s failure" % type)

	# Register FlipTest as a good test to run
	AllTestClasses.append(FlipTest)


	class RestartTest(CTSTest):
	'''Stop and restart a node'''
	def __init__(self, cm):
	CTSTest.__init__(self,cm)
	self.name = "Restart"
	self.start = StartTest(cm)
	self.stop = StopTest(cm)
	self.benchmark = 1

	def __call__(self, node):
	'''Perform the 'restart' test. '''
	self.incr("calls")

	self.incr("node:" + node)

	ret1 = 1
	if self.CM.StataCM(node):
	self.incr("WasStopped")
	if not self.start(node):
	return self.failure("start (setup) failure: "+node)

	self.set_timer()
	if not self.stop(node):
	return self.failure("stop failure: "+node)
	if not self.start(node):
	return self.failure("start failure: "+node)
	return self.success()

	# Register RestartTest as a good test to run
	AllTestClasses.append(RestartTest)


	class StonithdTest(CTSTest):
	def __init__(self, cm):
	CTSTest.__init__(self, cm)
	self.name = "Stonithd"
	self.startall = SimulStartLite(cm)
	self.benchmark = 1

	def __call__(self, node):
	self.incr("calls")
	if len(self.Env["nodes"]) < 2:
	return self.skipped()

	ret = self.startall(None)
	if not ret:
	return self.failure("Setup failed")

	is_dc = self.CM.is_node_dc(node)

	watchpats = []
	watchpats.append(self.templates["Pat:FenceOpOK"] % node)
	watchpats.append(self.templates["Pat:NodeFenced"] % node)

	if self.Env["at-boot"] == 0:
	self.debug("Expecting %s to stay down" % node)
	self.CM.ShouldBeStatus[node] = "down"
	else:
	self.debug("Expecting %s to come up again %d" % (node, self.Env["at-boot"]))
	watchpats.append("%s.* S_STARTING -> S_PENDING" % node)
	watchpats.append("%s.* S_PENDING -> S_NOT_DC" % node)

	watch = self.create_watch(watchpats, 30 + self.Env["DeadTime"] + self.Env["StableTime"] + self.Env["StartTime"])
	watch.setwatch()

	origin = self.Env.RandomGen.choice(self.Env["nodes"])

	rc = self.rsh(origin, "stonith_admin --reboot %s -VVVVVV" % node)

	if rc == 194:
	# 194 - 256 = -62 = Timer expired
	#
	# Look for the patterns, usually this means the required
	# device was running on the node to be fenced - or that
	# the required devices were in the process of being loaded
	# and/or moved
	#
	# Effectively the node committed suicide so there will be
	# no confirmation, but pacemaker should be watching and
	# fence the node again

	self.logger.log("Fencing command on %s to fence %s timed out" % (origin, node))

	elif origin != node and rc != 0:
	self.debug("Waiting for the cluster to recover")
	self.CM.cluster_stable()

	self.debug("Waiting for fenced node to come back up")
	self.CM.ns.WaitForAllNodesToComeUp(self.Env["nodes"], 600)

	self.logger.log("Fencing command on %s failed to fence %s (rc=%d)" % (origin, node, rc))

	elif origin == node and rc != 255:
	# 255 == broken pipe, ie. the node was fenced as expected
	self.logger.log("Locally originated fencing returned %d" % rc)

	self.set_timer("fence")
	matched = watch.lookforall()
	self.log_timer("fence")
	self.set_timer("reform")
	if watch.unmatched:
	self.logger.log("Patterns not found: " + repr(watch.unmatched))

	self.debug("Waiting for the cluster to recover")
	self.CM.cluster_stable()

	self.debug("Waiting for fenced node to come back up")
	self.CM.ns.WaitForAllNodesToComeUp(self.Env["nodes"], 600)

	self.debug("Waiting for the cluster to re-stabilize with all nodes")
	is_stable = self.CM.cluster_stable(self.Env["StartTime"])

	if not matched:
	return self.failure("Didn't find all expected patterns")
	elif not is_stable:
	return self.failure("Cluster did not become stable")

	self.log_timer("reform")
	return self.success()

	def errorstoignore(self):
	return [
	self.templates["Pat:Fencing_start"] % ".*",
	self.templates["Pat:Fencing_ok"] % ".*",
	self.templates["Pat:Fencing_active"],
	r"error.: Operation 'reboot' targeting . by .* for stonith_admin.*: Timer expired",
	]

	def is_applicable(self):
	if not self.is_applicable_common():
	return 0

	if "DoFencing" in list(self.Env.keys()):
	return self.Env["DoFencing"]

	return 1

	AllTestClasses.append(StonithdTest)


	class StartOnebyOne(CTSTest):
	'''Start all the nodes ~ one by one'''
	def __init__(self, cm):
	CTSTest.__init__(self,cm)
	self.name = "StartOnebyOne"
	self.stopall = SimulStopLite(cm)
	self.start = StartTest(cm)
	self.ns = CTS.NodeStatus(cm.Env)

	def __call__(self, dummy):
	'''Perform the 'StartOnebyOne' test. '''
	self.incr("calls")

	# We ignore the "node" parameter...

	# Shut down all the nodes...
	ret = self.stopall(None)
	if not ret:
	return self.failure("Test setup failed")

	failed = []
	self.set_timer()
	for node in self.Env["nodes"]:
	if not self.start(node):
	failed.append(node)

	if len(failed) > 0:
	return self.failure("Some node failed to start: " + repr(failed))

	return self.success()

	# Register StartOnebyOne as a good test to run
	AllTestClasses.append(StartOnebyOne)


	class SimulStart(CTSTest):
	'''Start all the nodes ~ simultaneously'''
	def __init__(self, cm):
	CTSTest.__init__(self,cm)
	self.name = "SimulStart"
	self.stopall = SimulStopLite(cm)
	self.startall = SimulStartLite(cm)

	def __call__(self, dummy):
	'''Perform the 'SimulStart' test. '''
	self.incr("calls")

	# We ignore the "node" parameter...

	# Shut down all the nodes...
	ret = self.stopall(None)
	if not ret:
	return self.failure("Setup failed")

	if not self.startall(None):
	return self.failure("Startall failed")

	return self.success()

	# Register SimulStart as a good test to run
	AllTestClasses.append(SimulStart)


	class SimulStop(CTSTest):
	'''Stop all the nodes ~ simultaneously'''
	def __init__(self, cm):
	CTSTest.__init__(self,cm)
	self.name = "SimulStop"
	self.startall = SimulStartLite(cm)
	self.stopall = SimulStopLite(cm)

	def __call__(self, dummy):
	'''Perform the 'SimulStop' test. '''
	self.incr("calls")

	# We ignore the "node" parameter...

	# Start up all the nodes...
	ret = self.startall(None)
	if not ret:
	return self.failure("Setup failed")

	if not self.stopall(None):
	return self.failure("Stopall failed")

	return self.success()

	# Register SimulStop as a good test to run
	AllTestClasses.append(SimulStop)


	class StopOnebyOne(CTSTest):
	'''Stop all the nodes in order'''
	def __init__(self, cm):
	CTSTest.__init__(self,cm)
	self.name = "StopOnebyOne"
	self.startall = SimulStartLite(cm)
	self.stop = StopTest(cm)

	def __call__(self, dummy):
	'''Perform the 'StopOnebyOne' test. '''
	self.incr("calls")

	# We ignore the "node" parameter...

	# Start up all the nodes...
	ret = self.startall(None)
	if not ret:
	return self.failure("Setup failed")

	failed = []
	self.set_timer()
	for node in self.Env["nodes"]:
	if not self.stop(node):
	failed.append(node)

	if len(failed) > 0:
	return self.failure("Some node failed to stop: " + repr(failed))

	return self.success()

	# Register StopOnebyOne as a good test to run
	AllTestClasses.append(StopOnebyOne)


	class RestartOnebyOne(CTSTest):
	'''Restart all the nodes in order'''
	def __init__(self, cm):
	CTSTest.__init__(self,cm)
	self.name = "RestartOnebyOne"
	self.startall = SimulStartLite(cm)

	def __call__(self, dummy):
	'''Perform the 'RestartOnebyOne' test. '''
	self.incr("calls")

	# We ignore the "node" parameter...

	# Start up all the nodes...
	ret = self.startall(None)
	if not ret:
	return self.failure("Setup failed")

	did_fail = []
	self.set_timer()
	self.restart = RestartTest(self.CM)
	for node in self.Env["nodes"]:
	if not self.restart(node):
	did_fail.append(node)

	if did_fail:
	return self.failure("Could not restart %d nodes: %s"
	% (len(did_fail), repr(did_fail)))
	return self.success()

	# Register StopOnebyOne as a good test to run
	AllTestClasses.append(RestartOnebyOne)


	class PartialStart(CTSTest):
	'''Start a node - but tell it to stop before it finishes starting up'''
	def __init__(self, cm):
	CTSTest.__init__(self,cm)
	self.name = "PartialStart"
	self.startall = SimulStartLite(cm)
	self.stopall = SimulStopLite(cm)
	self.stop = StopTest(cm)
	#self.is_unsafe = 1

	def __call__(self, node):
	'''Perform the 'PartialStart' test. '''
	self.incr("calls")

	ret = self.stopall(None)
	if not ret:
	return self.failure("Setup failed")

	-# FIXME! This should use the CM class to get the pattern
	-# then it would be applicable in general
	watchpats = []
	- watchpats.append("pacemaker-controld.*Connecting to cluster infrastructure")
	+ watchpats.append("pacemaker-controld.Connecting to . cluster infrastructure")
	watch = self.create_watch(watchpats, self.Env["DeadTime"]+10)
	watch.setwatch()

	self.CM.StartaCMnoBlock(node)
	ret = watch.lookforall()
	if not ret:
	self.logger.log("Patterns not found: " + repr(watch.unmatched))
	return self.failure("Setup of %s failed" % node)

	ret = self.stop(node)
	if not ret:
	return self.failure("%s did not stop in time" % node)

	return self.success()

	def errorstoignore(self):
	'''Return list of errors which should be ignored'''

	# We might do some fencing in the 2-node case if we make it up far enough
	return [
	r"Executing reboot fencing operation",
	r"Requesting fencing $[^)]+$ of node ",
	]

	# Register StopOnebyOne as a good test to run
	AllTestClasses.append(PartialStart)


	class StandbyTest(CTSTest):
	def __init__(self, cm):
	CTSTest.__init__(self,cm)
	self.name = "Standby"
	self.benchmark = 1

	self.start = StartTest(cm)
	self.startall = SimulStartLite(cm)

	# make sure the node is active
	# set the node to standby mode
	# check resources, none resource should be running on the node
	# set the node to active mode
	# check resouces, resources should have been migrated back (SHOULD THEY?)

	def __call__(self, node):

	self.incr("calls")
	ret = self.startall(None)
	if not ret:
	return self.failure("Start all nodes failed")

	self.debug("Make sure node %s is active" % node)
	if self.CM.StandbyStatus(node) != "off":
	if not self.CM.SetStandbyMode(node, "off"):
	return self.failure("can't set node %s to active mode" % node)

	self.CM.cluster_stable()

	status = self.CM.StandbyStatus(node)
	if status != "off":
	return self.failure("standby status of %s is [%s] but we expect [off]" % (node, status))

	self.debug("Getting resources running on node %s" % node)
	rsc_on_node = self.CM.active_resources(node)

	watchpats = []
	watchpats.append(r"State transition .* -> S_POLICY_ENGINE")
	watch = self.create_watch(watchpats, self.Env["DeadTime"]+10)
	watch.setwatch()

	self.debug("Setting node %s to standby mode" % node)
	if not self.CM.SetStandbyMode(node, "on"):
	return self.failure("can't set node %s to standby mode" % node)

	self.set_timer("on")

	ret = watch.lookforall()
	if not ret:
	self.logger.log("Patterns not found: " + repr(watch.unmatched))
	self.CM.SetStandbyMode(node, "off")
	return self.failure("cluster didn't react to standby change on %s" % node)

	self.CM.cluster_stable()

	status = self.CM.StandbyStatus(node)
	if status != "on":
	return self.failure("standby status of %s is [%s] but we expect [on]" % (node, status))
	self.log_timer("on")

	self.debug("Checking resources")
	bad_run = self.CM.active_resources(node)
	if len(bad_run) > 0:
	rc = self.failure("%s set to standby, %s is still running on it" % (node, repr(bad_run)))
	self.debug("Setting node %s to active mode" % node)
	self.CM.SetStandbyMode(node, "off")
	return rc

	self.debug("Setting node %s to active mode" % node)
	if not self.CM.SetStandbyMode(node, "off"):
	return self.failure("can't set node %s to active mode" % node)

	self.set_timer("off")
	self.CM.cluster_stable()

	status = self.CM.StandbyStatus(node)
	if status != "off":
	return self.failure("standby status of %s is [%s] but we expect [off]" % (node, status))
	self.log_timer("off")

	return self.success()

	AllTestClasses.append(StandbyTest)


	class ValgrindTest(CTSTest):
	'''Check for memory leaks'''
	def __init__(self, cm):
	CTSTest.__init__(self,cm)
	self.name = "Valgrind"
	self.stopall = SimulStopLite(cm)
	self.startall = SimulStartLite(cm)
	self.is_valgrind = 1
	self.is_loop = 1

	def setup(self, node):
	self.incr("calls")

	ret = self.stopall(None)
	if not ret:
	return self.failure("Stop all nodes failed")

	# @TODO Edit /etc/sysconfig/pacemaker on all nodes to enable valgrind,
	# and clear any valgrind logs from previous runs. For now, we rely on
	# the user to do this manually.

	ret = self.startall(None)
	if not ret:
	return self.failure("Start all nodes failed")

	return self.success()

	def teardown(self, node):
	# Return all nodes to normal
	# @TODO Edit /etc/sysconfig/pacemaker on all nodes to disable valgrind
	ret = self.stopall(None)
	if not ret:
	return self.failure("Stop all nodes failed")

	return self.success()

	def find_leaks(self):
	# Check for leaks
	# (no longer used but kept in case feature is restored)
	leaked = []
	self.stop = StopTest(self.CM)

	for node in self.Env["nodes"]:
	rc = self.stop(node)
	if not rc:
	self.failure("Couldn't shut down %s" % node)

	rc = self.rsh(node, "grep -e indirectly.lost:.[1-9] -e definitely.lost:.[1-9] -e (ERROR\|error).SUMMARY:.[1-9].*errors %s" % self.logger.logPat, 0)
	if rc != 1:
	leaked.append(node)
	self.failure("Valgrind errors detected on %s" % node)
	(rc, output) = self.rsh(node, "grep -e lost: -e SUMMARY: %s" % self.logger.logPat, None)
	for line in output:
	self.logger.log(line)
	(rc, output) = self.rsh(node, "cat %s" % self.logger.logPat, None)
	for line in output:
	self.debug(line)

	self.rsh(node, "rm -f %s" % self.logger.logPat, None)
	return leaked

	def __call__(self, node):
	#leaked = self.find_leaks()
	#if len(leaked) > 0:
	# return self.failure("Nodes %s leaked" % repr(leaked))

	return self.success()

	def errorstoignore(self):
	'''Return list of errors which should be ignored'''
	return [
	r"pacemaker-based.: \\\\\\\\\\\\\",
	r"pacemaker-based.: . avoid confusing Valgrind",
	r"HA_VALGRIND_ENABLED",
	]


	class StandbyLoopTest(ValgrindTest):
	'''Check for memory leaks by putting a node in and out of standby for an hour'''
	# @TODO This is not a useful test for memory leaks
	def __init__(self, cm):
	ValgrindTest.__init__(self,cm)
	self.name = "StandbyLoop"

	def __call__(self, node):

	lpc = 0
	delay = 2
	failed = 0
	done = time.time() + self.Env["loop-minutes"] * 60
	while time.time() <= done and not failed:
	lpc = lpc + 1

	time.sleep(delay)
	if not self.CM.SetStandbyMode(node, "on"):
	self.failure("can't set node %s to standby mode" % node)
	failed = lpc

	time.sleep(delay)
	if not self.CM.SetStandbyMode(node, "off"):
	self.failure("can't set node %s to active mode" % node)
	failed = lpc

	leaked = self.find_leaks()
	if failed:
	return self.failure("Iteration %d failed" % failed)
	elif len(leaked) > 0:
	return self.failure("Nodes %s leaked" % repr(leaked))

	return self.success()

	#AllTestClasses.append(StandbyLoopTest)


	class BandwidthTest(CTSTest):
	# Tests should not be cluster-manager-specific
	# If you need to find out cluster manager configuration to do this, then
	# it should be added to the generic cluster manager API.
	'''Test the bandwidth which the cluster uses'''
	def __init__(self, cm):
	CTSTest.__init__(self, cm)
	self.name = "Bandwidth"
	self.start = StartTest(cm)
	self.__setitem__("min",0)
	self.__setitem__("max",0)
	self.__setitem__("totalbandwidth",0)
	(handle, self.tempfile) = tempfile.mkstemp(".cts")
	os.close(handle)
	self.startall = SimulStartLite(cm)

	def __call__(self, node):
	'''Perform the Bandwidth test'''
	self.incr("calls")

	if self.CM.upcount() < 1:
	return self.skipped()

	Path = self.CM.InternalCommConfig()
	if "ip" not in Path["mediatype"]:
	return self.skipped()

	port = Path["port"][0]
	port = int(port)

	ret = self.startall(None)
	if not ret:
	return self.failure("Test setup failed")
	time.sleep(5) # We get extra messages right after startup.

	fstmpfile = "/var/run/band_estimate"
	dumpcmd = "tcpdump -p -n -c 102 -i any udp port %d > %s 2>&1" \
	% (port, fstmpfile)

	rc = self.rsh(node, dumpcmd)
	if rc == 0:
	farfile = "root@%s:%s" % (node, fstmpfile)
	self.rsh.cp(farfile, self.tempfile)
	Bandwidth = self.countbandwidth(self.tempfile)
	if not Bandwidth:
	self.logger.log("Could not compute bandwidth.")
	return self.success()
	intband = int(Bandwidth + 0.5)
	self.logger.log("...bandwidth: %d bits/sec" % intband)
	self.Stats["totalbandwidth"] = self.Stats["totalbandwidth"] + Bandwidth
	if self.Stats["min"] == 0:
	self.Stats["min"] = Bandwidth
	if Bandwidth > self.Stats["max"]:
	self.Stats["max"] = Bandwidth
	if Bandwidth < self.Stats["min"]:
	self.Stats["min"] = Bandwidth
	self.rsh(node, "rm -f %s" % fstmpfile)
	os.unlink(self.tempfile)
	return self.success()
	else:
	return self.failure("no response from tcpdump command [%d]!" % rc)

	def countbandwidth(self, file):
	fp = open(file, "r")
	fp.seek(0)
	count = 0
	sum = 0
	while 1:
	line = fp.readline()
	if not line:
	return None
	if re.search("udp",line) or re.search("UDP,", line):
	count = count + 1
	linesplit = line.split(" ")
	for j in range(len(linesplit)-1):
	if linesplit[j] == "udp": break
	if linesplit[j] == "length:": break

	try:
	sum = sum + int(linesplit[j+1])
	except ValueError:
	self.logger.log("Invalid tcpdump line: %s" % line)
	return None
	T1 = linesplit[0]
	timesplit = T1.split(":")
	time2split = timesplit[2].split(".")
	time1 = (int(timesplit[0])60+int(timesplit[1]))60+int(time2split[0])+int(time2split[1])*0.000001
	break

	while count < 100:
	line = fp.readline()
	if not line:
	return None
	if re.search("udp",line) or re.search("UDP,", line):
	count = count+1
	linessplit = line.split(" ")
	for j in range(len(linessplit)-1):
	if linessplit[j] == "udp": break
	if linessplit[j] == "length:": break
	try:
	sum = int(linessplit[j+1]) + sum
	except ValueError:
	self.logger.log("Invalid tcpdump line: %s" % line)
	return None

	T2 = linessplit[0]
	timesplit = T2.split(":")
	time2split = timesplit[2].split(".")
	time2 = (int(timesplit[0])60+int(timesplit[1]))60+int(time2split[0])+int(time2split[1])*0.000001
	time = time2-time1
	if (time <= 0):
	return 0
	return int((sum*8)/time)

	def is_applicable(self):
	'''BandwidthTest never applicable'''
	return 0

	AllTestClasses.append(BandwidthTest)


	###################################################################
	class MaintenanceMode(CTSTest):
	###################################################################
	def __init__(self, cm):
	CTSTest.__init__(self,cm)
	self.name = "MaintenanceMode"
	self.start = StartTest(cm)
	self.startall = SimulStartLite(cm)
	self.max = 30
	#self.is_unsafe = 1
	self.benchmark = 1
	self.action = "asyncmon"
	self.interval = 0
	self.rid = "maintenanceDummy"

	def toggleMaintenanceMode(self, node, action):
	pats = []
	pats.append(self.templates["Pat:DC_IDLE"])

	# fail the resource right after turning Maintenance mode on
	# verify it is not recovered until maintenance mode is turned off
	if action == "On":
	pats.append(self.templates["Pat:RscOpFail"] % (self.action, self.rid))
	else:
	pats.append(self.templates["Pat:RscOpOK"] % ("stop", self.rid))
	pats.append(self.templates["Pat:RscOpOK"] % ("start", self.rid))

	watch = self.create_watch(pats, 60)
	watch.setwatch()

	self.debug("Turning maintenance mode %s" % action)
	self.rsh(node, self.templates["MaintenanceMode%s" % (action)])
	if (action == "On"):
	self.rsh(node, "crm_resource -V -F -r %s -H %s &>/dev/null" % (self.rid, node))

	self.set_timer("recover%s" % (action))
	watch.lookforall()
	self.log_timer("recover%s" % (action))
	if watch.unmatched:
	self.debug("Failed to find patterns when turning maintenance mode %s" % action)
	return repr(watch.unmatched)

	return ""

	def insertMaintenanceDummy(self, node):
	pats = []
	pats.append(("%s.*" % node) + (self.templates["Pat:RscOpOK"] % ("start", self.rid)))

	watch = self.create_watch(pats, 60)
	watch.setwatch()

	self.CM.AddDummyRsc(node, self.rid)

	self.set_timer("addDummy")
	watch.lookforall()
	self.log_timer("addDummy")

	if watch.unmatched:
	self.debug("Failed to find patterns when adding maintenance dummy resource")
	return repr(watch.unmatched)
	return ""

	def removeMaintenanceDummy(self, node):
	pats = []
	pats.append(self.templates["Pat:RscOpOK"] % ("stop", self.rid))

	watch = self.create_watch(pats, 60)
	watch.setwatch()
	self.CM.RemoveDummyRsc(node, self.rid)

	self.set_timer("removeDummy")
	watch.lookforall()
	self.log_timer("removeDummy")

	if watch.unmatched:
	self.debug("Failed to find patterns when removing maintenance dummy resource")
	return repr(watch.unmatched)
	return ""

	def managedRscList(self, node):
	rscList = []
	(rc, lines) = self.rsh(node, "crm_resource -c", None)
	for line in lines:
	if re.search("^Resource", line):
	tmp = AuditResource(self.CM, line)
	if tmp.managed():
	rscList.append(tmp.id)

	return rscList

	def verifyResources(self, node, rscList, managed):
	managedList = list(rscList)
	managed_str = "managed"
	if not managed:
	managed_str = "unmanaged"

	(rc, lines) = self.rsh(node, "crm_resource -c", None)
	for line in lines:
	if re.search("^Resource", line):
	tmp = AuditResource(self.CM, line)
	if managed and not tmp.managed():
	continue
	elif not managed and tmp.managed():
	continue
	elif managedList.count(tmp.id):
	managedList.remove(tmp.id)

	if len(managedList) == 0:
	self.debug("Found all %s resources on %s" % (managed_str, node))
	return True

	self.logger.log("Could not find all %s resources on %s. %s" % (managed_str, node, managedList))
	return False

	def __call__(self, node):
	'''Perform the 'MaintenanceMode' test. '''
	self.incr("calls")
	verify_managed = False
	verify_unmanaged = False
	failPat = ""

	ret = self.startall(None)
	if not ret:
	return self.failure("Setup failed")

	# get a list of all the managed resources. We use this list
	# after enabling maintenance mode to verify all managed resources
	# become un-managed. After maintenance mode is turned off, we use
	# this list to verify all the resources become managed again.
	managedResources = self.managedRscList(node)
	if len(managedResources) == 0:
	self.logger.log("No managed resources on %s" % node)
	return self.skipped()

	# insert a fake resource we can fail during maintenance mode
	# so we can verify recovery does not take place until after maintenance
	# mode is disabled.
	failPat = failPat + self.insertMaintenanceDummy(node)

	# toggle maintenance mode ON, then fail dummy resource.
	failPat = failPat + self.toggleMaintenanceMode(node, "On")

	# verify all the resources are now unmanaged
	if self.verifyResources(node, managedResources, False):
	verify_unmanaged = True

	# Toggle maintenance mode OFF, verify dummy is recovered.
	failPat = failPat + self.toggleMaintenanceMode(node, "Off")

	# verify all the resources are now managed again
	if self.verifyResources(node, managedResources, True):
	verify_managed = True

	# Remove our maintenance dummy resource.
	failPat = failPat + self.removeMaintenanceDummy(node)

	self.CM.cluster_stable()

	if failPat != "":
	return self.failure("Unmatched patterns: %s" % (failPat))
	elif verify_unmanaged is False:
	return self.failure("Failed to verify resources became unmanaged during maintenance mode")
	elif verify_managed is False:
	return self.failure("Failed to verify resources switched back to managed after disabling maintenance mode")

	return self.success()

	def errorstoignore(self):
	'''Return list of errors which should be ignored'''
	return [
	r"Updating failcount for %s" % self.rid,
	r"schedulerd.: Recover %s\s$.*$" % self.rid,
	r"Unknown operation: fail",
	self.templates["Pat:RscOpOK"] % (self.action, self.rid),
	r"(ERROR\|error).: Action %s_%s_%d . initiated outside of a transition" % (self.rid, self.action, self.interval),
	]

	AllTestClasses.append(MaintenanceMode)


	class ResourceRecover(CTSTest):
	def __init__(self, cm):
	CTSTest.__init__(self,cm)
	self.name = "ResourceRecover"
	self.start = StartTest(cm)
	self.startall = SimulStartLite(cm)
	self.max = 30
	self.rid = None
	self.rid_alt = None
	#self.is_unsafe = 1
	self.benchmark = 1

	# these are the values used for the new LRM API call
	self.action = "asyncmon"
	self.interval = 0

	def __call__(self, node):
	'''Perform the 'ResourceRecover' test. '''
	self.incr("calls")

	ret = self.startall(None)
	if not ret:
	return self.failure("Setup failed")

	resourcelist = self.CM.active_resources(node)
	# if there are no resourcelist, return directly
	if len(resourcelist) == 0:
	self.logger.log("No active resources on %s" % node)
	return self.skipped()

	self.rid = self.Env.RandomGen.choice(resourcelist)
	self.rid_alt = self.rid

	rsc = None
	(rc, lines) = self.rsh(node, "crm_resource -c", None)
	for line in lines:
	if re.search("^Resource", line):
	tmp = AuditResource(self.CM, line)
	if tmp.id == self.rid:
	rsc = tmp
	# Handle anonymous clones that get renamed
	self.rid = rsc.clone_id
	break

	if not rsc:
	return self.failure("Could not find %s in the resource list" % self.rid)

	self.debug("Shooting %s aka. %s" % (rsc.clone_id, rsc.id))

	pats = []
	pats.append(self.templates["Pat:CloneOpFail"] % (self.action, rsc.id, rsc.clone_id))

	if rsc.managed():
	pats.append(self.templates["Pat:RscOpOK"] % ("stop", self.rid))
	if rsc.unique():
	pats.append(self.templates["Pat:RscOpOK"] % ("start", self.rid))
	else:
	# Anonymous clones may get restarted with a different clone number
	pats.append(self.templates["Pat:RscOpOK"] % ("start", ".*"))

	watch = self.create_watch(pats, 60)
	watch.setwatch()

	self.rsh(node, "crm_resource -V -F -r %s -H %s &>/dev/null" % (self.rid, node))

	self.set_timer("recover")
	watch.lookforall()
	self.log_timer("recover")

	self.CM.cluster_stable()
	recovered = self.CM.ResourceLocation(self.rid)

	if watch.unmatched:
	return self.failure("Patterns not found: %s" % repr(watch.unmatched))

	elif rsc.unique() and len(recovered) > 1:
	return self.failure("%s is now active on more than one node: %s"%(self.rid, repr(recovered)))

	elif len(recovered) > 0:
	self.debug("%s is running on: %s" % (self.rid, repr(recovered)))

	elif rsc.managed():
	return self.failure("%s was not recovered and is inactive" % self.rid)

	return self.success()

	def errorstoignore(self):
	'''Return list of errors which should be ignored'''
	return [
	r"Updating failcount for %s" % self.rid,
	r"schedulerd.: Recover (%s\|%s)\s$.*$" % (self.rid, self.rid_alt),
	r"Unknown operation: fail",
	self.templates["Pat:RscOpOK"] % (self.action, self.rid),
	r"(ERROR\|error).: Action %s_%s_%d . initiated outside of a transition" % (self.rid, self.action, self.interval),
	]

	AllTestClasses.append(ResourceRecover)


	class ComponentFail(CTSTest):
	def __init__(self, cm):
	CTSTest.__init__(self,cm)
	self.name = "ComponentFail"
	# TODO make this work correctly in docker.
	self.is_docker_unsafe = 1
	self.startall = SimulStartLite(cm)
	self.complist = cm.Components()
	self.patterns = []
	self.okerrpatterns = []
	self.is_unsafe = 1

	def __call__(self, node):
	'''Perform the 'ComponentFail' test. '''
	self.incr("calls")
	self.patterns = []
	self.okerrpatterns = []

	# start all nodes
	ret = self.startall(None)
	if not ret:
	return self.failure("Setup failed")

	if not self.CM.cluster_stable(self.Env["StableTime"]):
	return self.failure("Setup failed - unstable")

	node_is_dc = self.CM.is_node_dc(node, None)

	# select a component to kill
	chosen = self.Env.RandomGen.choice(self.complist)
	while chosen.dc_only == 1 and node_is_dc == 0:
	chosen = self.Env.RandomGen.choice(self.complist)

	self.debug("...component %s (dc=%d,boot=%d)" % (chosen.name, node_is_dc,chosen.triggersreboot))
	self.incr(chosen.name)

	if chosen.name != "corosync":
	self.patterns.append(self.templates["Pat:ChildKilled"] %(node, chosen.name))
	self.patterns.append(self.templates["Pat:ChildRespawn"] %(node, chosen.name))

	self.patterns.extend(chosen.pats)
	if node_is_dc:
	self.patterns.extend(chosen.dc_pats)

	# @TODO this should be a flag in the Component
	if chosen.name in [ "corosync", "pacemaker-based", "pacemaker-fenced" ]:
	# Ignore actions for fence devices if fencer will respawn
	# (their registration will be lost, and probes will fail)
	self.okerrpatterns = [ self.templates["Pat:Fencing_active"] ]
	(rc, lines) = self.rsh(node, "crm_resource -c", None)
	for line in lines:
	if re.search("^Resource", line):
	r = AuditResource(self.CM, line)
	if r.rclass == "stonith":
	self.okerrpatterns.append(self.templates["Pat:Fencing_recover"] % r.id)
	self.okerrpatterns.append(self.templates["Pat:Fencing_probe"] % r.id)

	# supply a copy so self.patterns doesn't end up empty
	tmpPats = []
	tmpPats.extend(self.patterns)
	self.patterns.extend(chosen.badnews_ignore)

	# Look for STONITH ops, depending on Env["at-boot"] we might need to change the nodes status
	stonithPats = []
	stonithPats.append(self.templates["Pat:Fencing_ok"] % node)
	stonith = self.create_watch(stonithPats, 0)
	stonith.setwatch()

	# set the watch for stable
	watch = self.create_watch(
	tmpPats, self.Env["DeadTime"] + self.Env["StableTime"] + self.Env["StartTime"])
	watch.setwatch()

	# kill the component
	chosen.kill(node)

	self.debug("Waiting for the cluster to recover")
	self.CM.cluster_stable()

	self.debug("Waiting for any fenced node to come back up")
	self.CM.ns.WaitForAllNodesToComeUp(self.Env["nodes"], 600)

	self.debug("Waiting for the cluster to re-stabilize with all nodes")
	self.CM.cluster_stable(self.Env["StartTime"])

	self.debug("Checking if %s was shot" % node)
	shot = stonith.look(60)
	if shot:
	self.debug("Found: " + repr(shot))
	self.okerrpatterns.append(self.templates["Pat:Fencing_start"] % node)

	if self.Env["at-boot"] == 0:
	self.CM.ShouldBeStatus[node] = "down"

	# If fencing occurred, chances are many (if not all) the expected logs
	# will not be sent - or will be lost when the node reboots
	return self.success()

	# check for logs indicating a graceful recovery
	matched = watch.lookforall(allow_multiple_matches=1)
	if watch.unmatched:
	self.logger.log("Patterns not found: " + repr(watch.unmatched))

	self.debug("Waiting for the cluster to re-stabilize with all nodes")
	is_stable = self.CM.cluster_stable(self.Env["StartTime"])

	if not matched:
	return self.failure("Didn't find all expected %s patterns" % chosen.name)
	elif not is_stable:
	return self.failure("Cluster did not become stable after killing %s" % chosen.name)

	return self.success()

	def errorstoignore(self):
	'''Return list of errors which should be ignored'''
	# Note that okerrpatterns refers to the last time we ran this test
	# The good news is that this works fine for us...
	self.okerrpatterns.extend(self.patterns)
	return self.okerrpatterns

	AllTestClasses.append(ComponentFail)


	class SplitBrainTest(CTSTest):
	'''It is used to test split-brain. when the path between the two nodes break
	check the two nodes both take over the resource'''
	def __init__(self,cm):
	CTSTest.__init__(self,cm)
	self.name = "SplitBrain"
	self.start = StartTest(cm)
	self.startall = SimulStartLite(cm)
	self.is_experimental = 1

	def isolate_partition(self, partition):
	other_nodes = []
	other_nodes.extend(self.Env["nodes"])

	for node in partition:
	try:
	other_nodes.remove(node)
	except ValueError:
	self.logger.log("Node "+node+" not in " + repr(self.Env["nodes"]) + " from " +repr(partition))

	if len(other_nodes) == 0:
	return 1

	self.debug("Creating partition: " + repr(partition))
	self.debug("Everyone else: " + repr(other_nodes))

	for node in partition:
	if not self.CM.isolate_node(node, other_nodes):
	self.logger.log("Could not isolate %s" % node)
	return 0

	return 1

	def heal_partition(self, partition):
	other_nodes = []
	other_nodes.extend(self.Env["nodes"])

	for node in partition:
	try:
	other_nodes.remove(node)
	except ValueError:
	self.logger.log("Node "+node+" not in " + repr(self.Env["nodes"]))

	if len(other_nodes) == 0:
	return 1

	self.debug("Healing partition: " + repr(partition))
	self.debug("Everyone else: " + repr(other_nodes))

	for node in partition:
	self.CM.unisolate_node(node, other_nodes)

	def __call__(self, node):
	'''Perform split-brain test'''
	self.incr("calls")
	self.passed = 1
	partitions = {}

	ret = self.startall(None)
	if not ret:
	return self.failure("Setup failed")

	while 1:
	# Retry until we get multiple partitions
	partitions = {}
	p_max = len(self.Env["nodes"])
	for node in self.Env["nodes"]:
	p = self.Env.RandomGen.randint(1, p_max)
	if not p in partitions:
	partitions[p] = []
	partitions[p].append(node)
	p_max = len(list(partitions.keys()))
	if p_max > 1:
	break
	# else, try again

	self.debug("Created %d partitions" % p_max)
	for key in list(partitions.keys()):
	self.debug("Partition["+str(key)+"]:\t"+repr(partitions[key]))

	# Disabling STONITH to reduce test complexity for now
	self.rsh(node, "crm_attribute -V -n stonith-enabled -v false")

	for key in list(partitions.keys()):
	self.isolate_partition(partitions[key])

	count = 30
	while count > 0:
	if len(self.CM.find_partitions()) != p_max:
	time.sleep(10)
	else:
	break
	else:
	self.failure("Expected partitions were not created")

	# Target number of partitions formed - wait for stability
	if not self.CM.cluster_stable():
	self.failure("Partitioned cluster not stable")

	# Now audit the cluster state
	self.CM.partitions_expected = p_max
	if not self.audit():
	self.failure("Audits failed")
	self.CM.partitions_expected = 1

	# And heal them again
	for key in list(partitions.keys()):
	self.heal_partition(partitions[key])

	# Wait for a single partition to form
	count = 30
	while count > 0:
	if len(self.CM.find_partitions()) != 1:
	time.sleep(10)
	count -= 1
	else:
	break
	else:
	self.failure("Cluster did not reform")

	# Wait for it to have the right number of members
	count = 30
	while count > 0:
	members = []

	partitions = self.CM.find_partitions()
	if len(partitions) > 0:
	members = partitions[0].split()

	if len(members) != len(self.Env["nodes"]):
	time.sleep(10)
	count -= 1
	else:
	break
	else:
	self.failure("Cluster did not completely reform")

	# Wait up to 20 minutes - the delay is more preferable than
	# trying to continue with in a messed up state
	if not self.CM.cluster_stable(1200):
	self.failure("Reformed cluster not stable")
	if self.Env["continue"] == 1:
	answer = "Y"
	else:
	try:
	answer = input_wrapper('Continue? [nY]')
	except EOFError as e:
	answer = "n"
	if answer and answer == "n":
	raise ValueError("Reformed cluster not stable")

	# Turn fencing back on
	if self.Env["DoFencing"]:
	self.rsh(node, "crm_attribute -V -D -n stonith-enabled")

	self.CM.cluster_stable()

	if self.passed:
	return self.success()
	return self.failure("See previous errors")

	def errorstoignore(self):
	'''Return list of errors which are 'normal' and should be ignored'''
	return [
	r"Another DC detected:",
	r"(ERROR\|error).: .Application of an update diff failed",
	r"pacemaker-controld.:.not in our membership list",
	r"CRIT:.node.returning after partition",
	]

	def is_applicable(self):
	if not self.is_applicable_common():
	return 0
	return len(self.Env["nodes"]) > 2

	AllTestClasses.append(SplitBrainTest)


	class Reattach(CTSTest):
	def __init__(self, cm):
	CTSTest.__init__(self,cm)
	self.name = "Reattach"
	self.startall = SimulStartLite(cm)
	self.restart1 = RestartTest(cm)
	self.stopall = SimulStopLite(cm)
	self.is_unsafe = 0 # Handled by canrunnow()

	def _is_managed(self, node):
	is_managed = self.rsh(node, "crm_attribute -t rsc_defaults -n is-managed -q -G -d true", 1)
	is_managed = is_managed[:-1] # Strip off the newline
	return is_managed == "true"

	def _set_unmanaged(self, node):
	self.debug("Disable resource management")
	self.rsh(node, "crm_attribute -t rsc_defaults -n is-managed -v false")

	def _set_managed(self, node):
	self.debug("Re-enable resource management")
	self.rsh(node, "crm_attribute -t rsc_defaults -n is-managed -D")

	def setup(self, node):
	attempt = 0
	if not self.startall(None):
	return None

	# Make sure we are really _really_ stable and that all
	# resources, including those that depend on transient node
	# attributes, are started
	while not self.CM.cluster_stable(double_check=True):
	if attempt < 5:
	attempt += 1
	self.debug("Not stable yet, re-testing")
	else:
	self.logger.log("Cluster is not stable")
	return None

	return 1

	def teardown(self, node):

	# Make sure 'node' is up
	start = StartTest(self.CM)
	start(node)

	if not self._is_managed(node):
	self.logger.log("Attempting to re-enable resource management on %s" % node)
	self._set_managed(node)
	self.CM.cluster_stable()
	if not self._is_managed(node):
	self.logger.log("Could not re-enable resource management")
	return 0

	return 1

	def canrunnow(self, node):
	'''Return TRUE if we can meaningfully run right now'''
	if self.find_ocfs2_resources(node):
	self.logger.log("Detach/Reattach scenarios are not possible with OCFS2 services present")
	return 0
	return 1

	def __call__(self, node):
	self.incr("calls")

	pats = []
	# Conveniently, the scheduler will display this message when disabling
	# management, even if fencing is not enabled, so we can rely on it.
	managed = self.create_watch(["Delaying fencing operations"], 60)
	managed.setwatch()

	self._set_unmanaged(node)

	if not managed.lookforall():
	self.logger.log("Patterns not found: " + repr(managed.unmatched))
	return self.failure("Resource management not disabled")

	pats = []
	pats.append(self.templates["Pat:RscOpOK"] % ("start", ".*"))
	pats.append(self.templates["Pat:RscOpOK"] % ("stop", ".*"))
	pats.append(self.templates["Pat:RscOpOK"] % ("promote", ".*"))
	pats.append(self.templates["Pat:RscOpOK"] % ("demote", ".*"))
	pats.append(self.templates["Pat:RscOpOK"] % ("migrate", ".*"))

	watch = self.create_watch(pats, 60, "ShutdownActivity")
	watch.setwatch()

	self.debug("Shutting down the cluster")
	ret = self.stopall(None)
	if not ret:
	self._set_managed(node)
	return self.failure("Couldn't shut down the cluster")

	self.debug("Bringing the cluster back up")
	ret = self.startall(None)
	time.sleep(5) # allow ping to update the CIB
	if not ret:
	self._set_managed(node)
	return self.failure("Couldn't restart the cluster")

	if self.local_badnews("ResourceActivity:", watch):
	self._set_managed(node)
	return self.failure("Resources stopped or started during cluster restart")

	watch = self.create_watch(pats, 60, "StartupActivity")
	watch.setwatch()

	# Re-enable resource management (and verify it happened).
	self._set_managed(node)
	self.CM.cluster_stable()
	if not self._is_managed(node):
	return self.failure("Could not re-enable resource management")

	# Ignore actions for STONITH resources
	ignore = []
	(rc, lines) = self.rsh(node, "crm_resource -c", None)
	for line in lines:
	if re.search("^Resource", line):
	r = AuditResource(self.CM, line)
	if r.rclass == "stonith":

	self.debug("Ignoring start actions for %s" % r.id)
	ignore.append(self.templates["Pat:RscOpOK"] % ("start", r.id))

	if self.local_badnews("ResourceActivity:", watch, ignore):
	return self.failure("Resources stopped or started after resource management was re-enabled")

	return ret

	def errorstoignore(self):
	'''Return list of errors which should be ignored'''
	return [
	r"resource( was\|s were) active at shutdown",
	]

	def is_applicable(self):
	return 1

	AllTestClasses.append(Reattach)


	class SpecialTest1(CTSTest):
	'''Set up a custom test to cause quorum failure issues for Andrew'''
	def __init__(self, cm):
	CTSTest.__init__(self,cm)
	self.name = "SpecialTest1"
	self.startall = SimulStartLite(cm)
	self.restart1 = RestartTest(cm)
	self.stopall = SimulStopLite(cm)

	def __call__(self, node):
	'''Perform the 'SpecialTest1' test for Andrew. '''
	self.incr("calls")

	# Shut down all the nodes...
	ret = self.stopall(None)
	if not ret:
	return self.failure("Could not stop all nodes")

	# Test config recovery when the other nodes come up
	self.rsh(node, "rm -f "+CTSvars.CRM_CONFIG_DIR+"/cib*")

	# Start the selected node
	ret = self.restart1(node)
	if not ret:
	return self.failure("Could not start "+node)

	# Start all remaining nodes
	ret = self.startall(None)
	if not ret:
	return self.failure("Could not start the remaining nodes")

	return self.success()

	def errorstoignore(self):
	'''Return list of errors which should be ignored'''
	# Errors that occur as a result of the CIB being wiped
	return [
	r"error.*: v1 patchset error, patch failed to apply: Application of an update diff failed",
	r"error.*: Resource start-up disabled since no STONITH resources have been defined",
	r"error.*: Either configure some or disable STONITH with the stonith-enabled option",
	r"error.*: NOTE: Clusters with shared data need STONITH to ensure data integrity",
	]

	AllTestClasses.append(SpecialTest1)


	class HAETest(CTSTest):
	'''Set up a custom test to cause quorum failure issues for Andrew'''
	def __init__(self, cm):
	CTSTest.__init__(self,cm)
	self.name = "HAETest"
	self.stopall = SimulStopLite(cm)
	self.startall = SimulStartLite(cm)
	self.is_loop = 1

	def setup(self, node):
	# Start all remaining nodes
	ret = self.startall(None)
	if not ret:
	return self.failure("Couldn't start all nodes")
	return self.success()

	def teardown(self, node):
	# Stop everything
	ret = self.stopall(None)
	if not ret:
	return self.failure("Couldn't stop all nodes")
	return self.success()

	def wait_on_state(self, node, resource, expected_clones, attempts=240):
	while attempts > 0:
	active = 0
	(rc, lines) = self.rsh(node, "crm_resource -r %s -W -Q" % resource, stdout=None)

	# Hack until crm_resource does the right thing
	if rc == 0 and lines:
	active = len(lines)

	if len(lines) == expected_clones:
	return 1

	elif rc == 1:
	self.debug("Resource %s is still inactive" % resource)

	elif rc == 234:
	self.logger.log("Unknown resource %s" % resource)
	return 0

	elif rc == 246:
	self.logger.log("Cluster is inactive")
	return 0

	elif rc != 0:
	self.logger.log("Call to crm_resource failed, rc=%d" % rc)
	return 0

	else:
	self.debug("Resource %s is active on %d times instead of %d" % (resource, active, expected_clones))

	attempts -= 1
	time.sleep(1)

	return 0

	def find_dlm(self, node):
	self.r_dlm = None

	(rc, lines) = self.rsh(node, "crm_resource -c", None)
	for line in lines:
	if re.search("^Resource", line):
	r = AuditResource(self.CM, line)
	if r.rtype == "controld" and r.parent != "NA":
	self.debug("Found dlm: %s" % self.r_dlm)
	self.r_dlm = r.parent
	return 1
	return 0

	def find_hae_resources(self, node):
	self.r_dlm = None
	self.r_o2cb = None
	self.r_ocfs2 = []

	if self.find_dlm(node):
	self.find_ocfs2_resources(node)

	def is_applicable(self):
	if not self.is_applicable_common():
	return 0
	if self.Env["Schema"] == "hae":
	return 1
	return None


	class HAERoleTest(HAETest):
	def __init__(self, cm):
	'''Lars' mount/unmount test for the HA extension. '''
	HAETest.__init__(self,cm)
	self.name = "HAERoleTest"

	def change_state(self, node, resource, target):
	rc = self.rsh(node, "crm_resource -V -r %s -p target-role -v %s --meta" % (resource, target))
	return rc

	def __call__(self, node):
	self.incr("calls")
	lpc = 0
	failed = 0
	delay = 2
	done = time.time() + self.Env["loop-minutes"]*60
	self.find_hae_resources(node)

	clone_max = len(self.Env["nodes"])
	while time.time() <= done and not failed:
	lpc = lpc + 1

	self.change_state(node, self.r_dlm, "Stopped")
	if not self.wait_on_state(node, self.r_dlm, 0):
	self.failure("%s did not go down correctly" % self.r_dlm)
	failed = lpc

	self.change_state(node, self.r_dlm, "Started")
	if not self.wait_on_state(node, self.r_dlm, clone_max):
	self.failure("%s did not come up correctly" % self.r_dlm)
	failed = lpc

	if not self.wait_on_state(node, self.r_o2cb, clone_max):
	self.failure("%s did not come up correctly" % self.r_o2cb)
	failed = lpc

	for fs in self.r_ocfs2:
	if not self.wait_on_state(node, fs, clone_max):
	self.failure("%s did not come up correctly" % fs)
	failed = lpc

	if failed:
	return self.failure("iteration %d failed" % failed)
	return self.success()

	AllTestClasses.append(HAERoleTest)


	class HAEStandbyTest(HAETest):
	'''Set up a custom test to cause quorum failure issues for Andrew'''
	def __init__(self, cm):
	HAETest.__init__(self,cm)
	self.name = "HAEStandbyTest"

	def change_state(self, node, resource, target):
	rc = self.rsh(node, "crm_standby -V -l reboot -v %s" % (target))
	return rc

	def __call__(self, node):
	self.incr("calls")

	lpc = 0
	failed = 0
	done = time.time() + self.Env["loop-minutes"]*60
	self.find_hae_resources(node)

	clone_max = len(self.Env["nodes"])
	while time.time() <= done and not failed:
	lpc = lpc + 1

	self.change_state(node, self.r_dlm, "true")
	if not self.wait_on_state(node, self.r_dlm, clone_max-1):
	self.failure("%s did not go down correctly" % self.r_dlm)
	failed = lpc

	self.change_state(node, self.r_dlm, "false")
	if not self.wait_on_state(node, self.r_dlm, clone_max):
	self.failure("%s did not come up correctly" % self.r_dlm)
	failed = lpc

	if not self.wait_on_state(node, self.r_o2cb, clone_max):
	self.failure("%s did not come up correctly" % self.r_o2cb)
	failed = lpc

	for fs in self.r_ocfs2:
	if not self.wait_on_state(node, fs, clone_max):
	self.failure("%s did not come up correctly" % fs)
	failed = lpc

	if failed:
	return self.failure("iteration %d failed" % failed)
	return self.success()

	AllTestClasses.append(HAEStandbyTest)


	class NearQuorumPointTest(CTSTest):
	'''
	This test brings larger clusters near the quorum point (50%).
	In addition, it will test doing starts and stops at the same time.

	Here is how I think it should work:
	- loop over the nodes and decide randomly which will be up and which
	will be down Use a 50% probability for each of up/down.
	- figure out what to do to get into that state from the current state
	- in parallel, bring up those going up and bring those going down.
	'''

	def __init__(self, cm):
	CTSTest.__init__(self,cm)
	self.name = "NearQuorumPoint"

	def __call__(self, dummy):
	'''Perform the 'NearQuorumPoint' test. '''
	self.incr("calls")
	startset = []
	stopset = []

	stonith = self.CM.prepare_fencing_watcher("NearQuorumPoint")
	#decide what to do with each node
	for node in self.Env["nodes"]:
	action = self.Env.RandomGen.choice(["start","stop"])
	#action = self.Env.RandomGen.choice(["start","stop","no change"])
	if action == "start" :
	startset.append(node)
	elif action == "stop" :
	stopset.append(node)

	self.debug("start nodes:" + repr(startset))
	self.debug("stop nodes:" + repr(stopset))

	#add search patterns
	watchpats = [ ]
	for node in stopset:
	if self.CM.ShouldBeStatus[node] == "up":
	watchpats.append(self.templates["Pat:We_stopped"] % node)

	for node in startset:
	if self.CM.ShouldBeStatus[node] == "down":
	#watchpats.append(self.templates["Pat:NonDC_started"] % node)
	watchpats.append(self.templates["Pat:Local_started"] % node)
	else:
	for stopping in stopset:
	if self.CM.ShouldBeStatus[stopping] == "up":
	watchpats.append(self.templates["Pat:They_stopped"] % (node, self.CM.key_for_node(stopping)))

	if len(watchpats) == 0:
	return self.skipped()

	if len(startset) != 0:
	watchpats.append(self.templates["Pat:DC_IDLE"])

	watch = self.create_watch(watchpats, self.Env["DeadTime"]+10)

	watch.setwatch()

	#begin actions
	for node in stopset:
	if self.CM.ShouldBeStatus[node] == "up":
	self.CM.StopaCMnoBlock(node)

	for node in startset:
	if self.CM.ShouldBeStatus[node] == "down":
	self.CM.StartaCMnoBlock(node)

	#get the result
	if watch.lookforall():
	self.CM.cluster_stable()
	self.CM.fencing_cleanup("NearQuorumPoint", stonith)
	return self.success()

	self.logger.log("Warn: Patterns not found: " + repr(watch.unmatched))

	#get the "bad" nodes
	upnodes = []
	for node in stopset:
	if self.CM.StataCM(node) == 1:
	upnodes.append(node)

	downnodes = []
	for node in startset:
	if self.CM.StataCM(node) == 0:
	downnodes.append(node)

	self.CM.fencing_cleanup("NearQuorumPoint", stonith)
	if upnodes == [] and downnodes == []:
	self.CM.cluster_stable()

	# Make sure they're completely down with no residule
	for node in stopset:
	self.rsh(node, self.templates["StopCmd"])

	return self.success()

	if len(upnodes) > 0:
	self.logger.log("Warn: Unstoppable nodes: " + repr(upnodes))

	if len(downnodes) > 0:
	self.logger.log("Warn: Unstartable nodes: " + repr(downnodes))

	return self.failure()

	def is_applicable(self):
	return 1

	AllTestClasses.append(NearQuorumPointTest)


	class RollingUpgradeTest(CTSTest):
	'''Perform a rolling upgrade of the cluster'''
	def __init__(self, cm):
	CTSTest.__init__(self,cm)
	self.name = "RollingUpgrade"
	self.start = StartTest(cm)
	self.stop = StopTest(cm)
	self.stopall = SimulStopLite(cm)
	self.startall = SimulStartLite(cm)

	def setup(self, node):
	# Start all remaining nodes
	ret = self.stopall(None)
	if not ret:
	return self.failure("Couldn't stop all nodes")

	for node in self.Env["nodes"]:
	if not self.downgrade(node, None):
	return self.failure("Couldn't downgrade %s" % node)

	ret = self.startall(None)
	if not ret:
	return self.failure("Couldn't start all nodes")
	return self.success()

	def teardown(self, node):
	# Stop everything
	ret = self.stopall(None)
	if not ret:
	return self.failure("Couldn't stop all nodes")

	for node in self.Env["nodes"]:
	if not self.upgrade(node, None):
	return self.failure("Couldn't upgrade %s" % node)

	return self.success()

	def install(self, node, version, start=1, flags="--force"):

	target_dir = "/tmp/rpm-%s" % version
	src_dir = "%s/%s" % (self.Env["rpm-dir"], version)

	self.logger.log("Installing %s on %s with %s" % (version, node, flags))
	if not self.stop(node):
	return self.failure("stop failure: "+node)

	rc = self.rsh(node, "mkdir -p %s" % target_dir)
	rc = self.rsh(node, "rm -f %s/*.rpm" % target_dir)
	(rc, lines) = self.rsh(node, "ls -1 %s/*.rpm" % src_dir, None)
	for line in lines:
	line = line[:-1]
	rc = self.rsh.cp("%s" % (line), "%s:%s/" % (node, target_dir))
	rc = self.rsh(node, "rpm -Uvh %s %s/*.rpm" % (flags, target_dir))

	if start and not self.start(node):
	return self.failure("start failure: "+node)

	return self.success()

	def upgrade(self, node, start=1):
	return self.install(node, self.Env["current-version"], start)

	def downgrade(self, node, start=1):
	return self.install(node, self.Env["previous-version"], start, "--force --nodeps")

	def __call__(self, node):
	'''Perform the 'Rolling Upgrade' test. '''
	self.incr("calls")

	for node in self.Env["nodes"]:
	if self.upgrade(node):
	return self.failure("Couldn't upgrade %s" % node)

	self.CM.cluster_stable()

	return self.success()

	def is_applicable(self):
	if not self.is_applicable_common():
	return None

	if not "rpm-dir" in list(self.Env.keys()):
	return None
	if not "current-version" in list(self.Env.keys()):
	return None
	if not "previous-version" in list(self.Env.keys()):
	return None

	return 1

	# Register RestartTest as a good test to run
	AllTestClasses.append(RollingUpgradeTest)


	class BSC_AddResource(CTSTest):
	'''Add a resource to the cluster'''
	def __init__(self, cm):
	CTSTest.__init__(self, cm)
	self.name = "AddResource"
	self.resource_offset = 0
	self.cib_cmd = """cibadmin -C -o %s -X '%s' """

	def __call__(self, node):
	self.incr("calls")
	self.resource_offset = self.resource_offset + 1

	r_id = "bsc-rsc-%s-%d" % (node, self.resource_offset)
	start_pat = "pacemaker-controld.%s_start_0.confirmed.*ok"

	patterns = []
	patterns.append(start_pat % r_id)

	watch = self.create_watch(patterns, self.Env["DeadTime"])
	watch.setwatch()

	ip = self.NextIP()
	if not self.make_ip_resource(node, r_id, "ocf", "IPaddr", ip):
	return self.failure("Make resource %s failed" % r_id)

	failed = 0
	watch_result = watch.lookforall()
	if watch.unmatched:
	for regex in watch.unmatched:
	self.logger.log ("Warn: Pattern not found: %s" % (regex))
	failed = 1

	if failed:
	return self.failure("Resource pattern(s) not found")

	if not self.CM.cluster_stable(self.Env["DeadTime"]):
	return self.failure("Unstable cluster")

	return self.success()

	def NextIP(self):
	ip = self.Env["IPBase"]
	if ":" in ip:
	fields = ip.rpartition(":")
	fields[2] = str(hex(int(fields[2], 16)+1))
	print(str(hex(int(f[2], 16)+1)))
	else:
	fields = ip.rpartition('.')
	fields[2] = str(int(fields[2])+1)

	ip = fields[0] + fields[1] + fields[3];
	self.Env["IPBase"] = ip
	return ip.strip()

	def make_ip_resource(self, node, id, rclass, type, ip):
	self.logger.log("Creating %s::%s:%s (%s) on %s" % (rclass,type,id,ip,node))
	rsc_xml="""
	<primitive id="%s" class="%s" type="%s" provider="heartbeat">
	<instance_attributes id="%s"><attributes>
	<nvpair id="%s" name="ip" value="%s"/>
	</attributes></instance_attributes>
	</primitive>""" % (id, rclass, type, id, id, ip)

	node_constraint = """
	<rsc_location id="run_%s" rsc="%s">
	<rule id="pref_run_%s" score="100">
	<expression id="%s_loc_expr" attribute="#uname" operation="eq" value="%s"/>
	</rule>
	</rsc_location>""" % (id, id, id, id, node)

	rc = 0
	(rc, lines) = self.rsh(node, self.cib_cmd % ("constraints", node_constraint), None)
	if rc != 0:
	self.logger.log("Constraint creation failed: %d" % rc)
	return None

	(rc, lines) = self.rsh(node, self.cib_cmd % ("resources", rsc_xml), None)
	if rc != 0:
	self.logger.log("Resource creation failed: %d" % rc)
	return None

	return 1

	def is_applicable(self):
	if self.Env["DoBSC"]:
	return 1
	return None

	AllTestClasses.append(BSC_AddResource)


	class SimulStopLite(CTSTest):
	'''Stop any active nodes ~ simultaneously'''
	def __init__(self, cm):
	CTSTest.__init__(self,cm)
	self.name = "SimulStopLite"

	def __call__(self, dummy):
	'''Perform the 'SimulStopLite' setup work. '''
	self.incr("calls")

	self.debug("Setup: " + self.name)

	# We ignore the "node" parameter...
	watchpats = [ ]

	for node in self.Env["nodes"]:
	if self.CM.ShouldBeStatus[node] == "up":
	self.incr("WasStarted")
	watchpats.append(self.templates["Pat:We_stopped"] % node)

	if len(watchpats) == 0:
	return self.success()

	# Stop all the nodes - at about the same time...
	watch = self.create_watch(watchpats, self.Env["DeadTime"]+10)

	watch.setwatch()
	self.set_timer()
	for node in self.Env["nodes"]:
	if self.CM.ShouldBeStatus[node] == "up":
	self.CM.StopaCMnoBlock(node)
	if watch.lookforall():
	# Make sure they're completely down with no residule
	for node in self.Env["nodes"]:
	self.rsh(node, self.templates["StopCmd"])

	return self.success()

	did_fail = 0
	up_nodes = []
	for node in self.Env["nodes"]:
	if self.CM.StataCM(node) == 1:
	did_fail = 1
	up_nodes.append(node)

	if did_fail:
	return self.failure("Active nodes exist: " + repr(up_nodes))

	self.logger.log("Warn: All nodes stopped but CTS didnt detect: "
	+ repr(watch.unmatched))

	return self.failure("Missing log message: "+repr(watch.unmatched))

	def is_applicable(self):
	'''SimulStopLite is a setup test and never applicable'''
	return 0


	class SimulStartLite(CTSTest):
	'''Start any stopped nodes ~ simultaneously'''
	def __init__(self, cm):
	CTSTest.__init__(self,cm)
	self.name = "SimulStartLite"

	def __call__(self, dummy):
	'''Perform the 'SimulStartList' setup work. '''
	self.incr("calls")
	self.debug("Setup: " + self.name)

	# We ignore the "node" parameter...
	node_list = []
	for node in self.Env["nodes"]:
	if self.CM.ShouldBeStatus[node] == "down":
	self.incr("WasStopped")
	node_list.append(node)

	self.set_timer()
	while len(node_list) > 0:
	# Repeat until all nodes come up
	watchpats = [ ]

	uppat = self.templates["Pat:NonDC_started"]
	if self.CM.upcount() == 0:
	uppat = self.templates["Pat:Local_started"]

	watchpats.append(self.templates["Pat:DC_IDLE"])
	for node in node_list:
	watchpats.append(uppat % node)
	watchpats.append(self.templates["Pat:InfraUp"] % node)
	watchpats.append(self.templates["Pat:PacemakerUp"] % node)

	# Start all the nodes - at about the same time...
	watch = self.create_watch(watchpats, self.Env["DeadTime"]+10)
	watch.setwatch()

	stonith = self.CM.prepare_fencing_watcher(self.name)

	for node in node_list:
	self.CM.StartaCMnoBlock(node)

	watch.lookforall()

	node_list = self.CM.fencing_cleanup(self.name, stonith)

	if node_list == None:
	return self.failure("Cluster did not stabilize")

	# Remove node_list messages from watch.unmatched
	for node in node_list:
	self.logger.debug("Dealing with stonith operations for %s" % repr(node_list))
	if watch.unmatched:
	try:
	watch.unmatched.remove(uppat % node)
	except:
	self.debug("Already matched: %s" % (uppat % node))
	try:
	watch.unmatched.remove(self.templates["Pat:InfraUp"] % node)
	except:
	self.debug("Already matched: %s" % (self.templates["Pat:InfraUp"] % node))
	try:
	watch.unmatched.remove(self.templates["Pat:PacemakerUp"] % node)
	except:
	self.debug("Already matched: %s" % (self.templates["Pat:PacemakerUp"] % node))

	if watch.unmatched:
	for regex in watch.unmatched:
	self.logger.log ("Warn: Startup pattern not found: %s" %(regex))

	if not self.CM.cluster_stable():
	return self.failure("Cluster did not stabilize")

	did_fail = 0
	unstable = []
	for node in self.Env["nodes"]:
	if self.CM.StataCM(node) == 0:
	did_fail = 1
	unstable.append(node)

	if did_fail:
	return self.failure("Unstarted nodes exist: " + repr(unstable))

	unstable = []
	for node in self.Env["nodes"]:
	if not self.CM.node_stable(node):
	did_fail = 1
	unstable.append(node)

	if did_fail:
	return self.failure("Unstable cluster nodes exist: " + repr(unstable))

	return self.success()

	def is_applicable(self):
	'''SimulStartLite is a setup test and never applicable'''
	return 0


	def TestList(cm, audits):
	result = []
	for testclass in AllTestClasses:
	bound_test = testclass(cm)
	if bound_test.is_applicable():
	bound_test.Audits = audits
	result.append(bound_test)
	return result


	class RemoteLXC(CTSTest):
	def __init__(self, cm):
	CTSTest.__init__(self,cm)
	self.name = "RemoteLXC"
	self.start = StartTest(cm)
	self.startall = SimulStartLite(cm)
	self.num_containers = 2
	self.is_container = 1
	self.is_docker_unsafe = 1
	self.failed = 0
	self.fail_string = ""

	def start_lxc_simple(self, node):

	# restore any artifacts laying around from a previous test.
	self.rsh(node, "/usr/share/pacemaker/tests/cts/lxc_autogen.sh -s -R &>/dev/null")

	# generate the containers, put them in the config, add some resources to them
	pats = [ ]
	watch = self.create_watch(pats, 120)
	watch.setwatch()
	pats.append(self.templates["Pat:RscOpOK"] % ("start", "lxc1"))
	pats.append(self.templates["Pat:RscOpOK"] % ("start", "lxc2"))
	pats.append(self.templates["Pat:RscOpOK"] % ("start", "lxc-ms"))
	pats.append(self.templates["Pat:RscOpOK"] % ("promote", "lxc-ms"))

	self.rsh(node, "/usr/share/pacemaker/tests/cts/lxc_autogen.sh -g -a -m -s -c %d &>/dev/null" % self.num_containers)
	self.set_timer("remoteSimpleInit")
	watch.lookforall()
	self.log_timer("remoteSimpleInit")
	if watch.unmatched:
	self.fail_string = "Unmatched patterns: %s" % (repr(watch.unmatched))
	self.failed = 1

	def cleanup_lxc_simple(self, node):

	pats = [ ]
	# if the test failed, attempt to clean up the cib and libvirt environment
	# as best as possible
	if self.failed == 1:
	# restore libvirt and cib
	self.rsh(node, "/usr/share/pacemaker/tests/cts/lxc_autogen.sh -s -R &>/dev/null")
	return

	watch = self.create_watch(pats, 120)
	watch.setwatch()

	pats.append(self.templates["Pat:RscOpOK"] % ("stop", "container1"))
	pats.append(self.templates["Pat:RscOpOK"] % ("stop", "container2"))

	self.rsh(node, "/usr/share/pacemaker/tests/cts/lxc_autogen.sh -p &>/dev/null")
	self.set_timer("remoteSimpleCleanup")
	watch.lookforall()
	self.log_timer("remoteSimpleCleanup")

	if watch.unmatched:
	self.fail_string = "Unmatched patterns: %s" % (repr(watch.unmatched))
	self.failed = 1

	# cleanup libvirt
	self.rsh(node, "/usr/share/pacemaker/tests/cts/lxc_autogen.sh -s -R &>/dev/null")

	def __call__(self, node):
	'''Perform the 'RemoteLXC' test. '''
	self.incr("calls")

	ret = self.startall(None)
	if not ret:
	return self.failure("Setup failed, start all nodes failed.")

	rc = self.rsh(node, "/usr/share/pacemaker/tests/cts/lxc_autogen.sh -v &>/dev/null")
	if rc == 1:
	self.log("Environment test for lxc support failed.")
	return self.skipped()

	self.start_lxc_simple(node)
	self.cleanup_lxc_simple(node)

	self.debug("Waiting for the cluster to recover")
	self.CM.cluster_stable()

	if self.failed == 1:
	return self.failure(self.fail_string)

	return self.success()

	def errorstoignore(self):
	'''Return list of errors which should be ignored'''
	return [
	r"Updating failcount for ping",
	r"schedulerd.: Recover (ping\|lxc-ms\|container)\s$.*$",
	# The orphaned lxc-ms resource causes an expected transition error
	# that is a result of the scheduler not having knowledge that the
	# promotable resource used to be a clone. As a result, it looks like that
	# resource is running in multiple locations when it shouldn't... But in
	# this instance we know why this error is occurring and that it is expected.
	r"Calculated [Tt]ransition .*pe-error",
	r"Resource lxc-ms .* is active on 2 nodes attempting recovery",
	r"Unknown operation: fail",
	r"VirtualDomain.*ERROR: Unable to determine emulator",
	]

	AllTestClasses.append(RemoteLXC)


	class RemoteDriver(CTSTest):

	def __init__(self, cm):
	CTSTest.__init__(self,cm)
	self.name = self.__class__.__name__
	self.is_docker_unsafe = 1
	self.start = StartTest(cm)
	self.startall = SimulStartLite(cm)
	self.stop = StopTest(cm)
	self.remote_rsc = "remote-rsc"
	self.cib_cmd = """cibadmin -C -o %s -X '%s' """
	self.reset()

	def reset(self):
	self.pcmk_started = 0
	self.failed = False
	self.fail_string = ""
	self.remote_node_added = 0
	self.remote_rsc_added = 0
	self.remote_use_reconnect_interval = self.Env.RandomGen.choice([True,False])

	def fail(self, msg):
	""" Mark test as failed. """

	self.failed = True

	# Always log the failure.
	self.logger.log(msg)

	# Use first failure as test status, as it's likely to be most useful.
	if not self.fail_string:
	self.fail_string = msg

	def get_othernode(self, node):
	for othernode in self.Env["nodes"]:
	if othernode == node:
	# we don't want to try and use the cib that we just shutdown.
	# find a cluster node that is not our soon to be remote-node.
	continue
	else:
	return othernode

	def del_rsc(self, node, rsc):
	othernode = self.get_othernode(node)
	rc = self.rsh(othernode, "crm_resource -D -r %s -t primitive" % (rsc))
	if rc != 0:
	self.fail("Removal of resource '%s' failed" % rsc)

	def add_rsc(self, node, rsc_xml):
	othernode = self.get_othernode(node)
	rc = self.rsh(othernode, self.cib_cmd % ("resources", rsc_xml))
	if rc != 0:
	self.fail("resource creation failed")

	def add_primitive_rsc(self, node):
	rsc_xml = """
	<primitive class="ocf" id="%(node)s" provider="heartbeat" type="Dummy">
	<meta_attributes id="%(node)s-meta_attributes"/>
	<operations>
	<op id="%(node)s-monitor-interval-20s" interval="20s" name="monitor"/>
	</operations>
	</primitive>""" % { "node": self.remote_rsc }
	self.add_rsc(node, rsc_xml)
	if not self.failed:
	self.remote_rsc_added = 1

	def add_connection_rsc(self, node):
	rsc_xml = """
	<primitive class="ocf" id="%(node)s" provider="pacemaker" type="remote">
	<instance_attributes id="%(node)s-instance_attributes">
	<nvpair id="%(node)s-instance_attributes-server" name="server" value="%(server)s"/>
	""" % { "node": self.remote_node, "server": node }

	if self.remote_use_reconnect_interval:
	# Set reconnect interval on resource
	rsc_xml = rsc_xml + """
	<nvpair id="%s-instance_attributes-reconnect_interval" name="reconnect_interval" value="60s"/>
	""" % (self.remote_node)

	rsc_xml = rsc_xml + """
	</instance_attributes>
	<operations>
	<op id="%(node)s-start" name="start" interval="0" timeout="120s"/>
	<op id="%(node)s-monitor-20s" name="monitor" interval="20s" timeout="45s"/>
	</operations>
	</primitive>
	""" % { "node": self.remote_node }

	self.add_rsc(node, rsc_xml)
	if not self.failed:
	self.remote_node_added = 1

	def disable_services(self, node):
	self.corosync_enabled = self.Env.service_is_enabled(node, "corosync")
	if self.corosync_enabled:
	self.Env.disable_service(node, "corosync")

	self.pacemaker_enabled = self.Env.service_is_enabled(node, "pacemaker")
	if self.pacemaker_enabled:
	self.Env.disable_service(node, "pacemaker")

	def restore_services(self, node):
	if self.corosync_enabled:
	self.Env.enable_service(node, "corosync")

	if self.pacemaker_enabled:
	self.Env.enable_service(node, "pacemaker")

	def stop_pcmk_remote(self, node):
	# disable pcmk remote
	for i in range(10):
	rc = self.rsh(node, "service pacemaker_remote stop")
	if rc != 0:
	time.sleep(6)
	else:
	break

	def start_pcmk_remote(self, node):
	for i in range(10):
	rc = self.rsh(node, "service pacemaker_remote start")
	if rc != 0:
	time.sleep(6)
	else:
	self.pcmk_started = 1
	break

	def freeze_pcmk_remote(self, node):
	""" Simulate a Pacemaker Remote daemon failure. """

	# We freeze the process.
	self.rsh(node, "killall -STOP pacemaker-remoted")

	def resume_pcmk_remote(self, node):
	# We resume the process.
	self.rsh(node, "killall -CONT pacemaker-remoted")

	def start_metal(self, node):
	# Cluster nodes are reused as remote nodes in remote tests. If cluster
	# services were enabled at boot, in case the remote node got fenced, the
	# cluster node would join instead of the expected remote one. Meanwhile
	# pacemaker_remote would not be able to start. Depending on the chances,
	# the situations might not be able to be orchestrated gracefully any more.
	#
	# Temporarily disable any enabled cluster serivces.
	self.disable_services(node)

	pcmk_started = 0

	# make sure the resource doesn't already exist for some reason
	self.rsh(node, "crm_resource -D -r %s -t primitive" % (self.remote_rsc))
	self.rsh(node, "crm_resource -D -r %s -t primitive" % (self.remote_node))

	if not self.stop(node):
	self.fail("Failed to shutdown cluster node %s" % node)
	return

	self.start_pcmk_remote(node)

	if self.pcmk_started == 0:
	self.fail("Failed to start pacemaker_remote on node %s" % node)
	return

	# Convert node to baremetal now that it has shutdown the cluster stack
	pats = [ ]
	watch = self.create_watch(pats, 120)
	watch.setwatch()
	pats.append(self.templates["Pat:RscOpOK"] % ("start", self.remote_node))
	pats.append(self.templates["Pat:DC_IDLE"])

	self.add_connection_rsc(node)

	self.set_timer("remoteMetalInit")
	watch.lookforall()
	self.log_timer("remoteMetalInit")
	if watch.unmatched:
	self.fail("Unmatched patterns: %s" % watch.unmatched)

	def migrate_connection(self, node):
	if self.failed:
	return

	pats = [ ]
	pats.append(self.templates["Pat:RscOpOK"] % ("migrate_to", self.remote_node))
	pats.append(self.templates["Pat:RscOpOK"] % ("migrate_from", self.remote_node))
	pats.append(self.templates["Pat:DC_IDLE"])
	watch = self.create_watch(pats, 120)
	watch.setwatch()

	(rc, lines) = self.rsh(node, "crm_resource -M -r %s" % (self.remote_node), None)
	if rc != 0:
	self.fail("failed to move remote node connection resource")
	return

	self.set_timer("remoteMetalMigrate")
	watch.lookforall()
	self.log_timer("remoteMetalMigrate")

	if watch.unmatched:
	self.fail("Unmatched patterns: %s" % watch.unmatched)
	return

	def fail_rsc(self, node):
	if self.failed:
	return

	watchpats = [ ]
	watchpats.append(self.templates["Pat:RscRemoteOpOK"] % ("stop", self.remote_rsc, self.remote_node))
	watchpats.append(self.templates["Pat:RscRemoteOpOK"] % ("start", self.remote_rsc, self.remote_node))
	watchpats.append(self.templates["Pat:DC_IDLE"])

	watch = self.create_watch(watchpats, 120)
	watch.setwatch()

	self.debug("causing dummy rsc to fail.")

	rc = self.rsh(node, "rm -f /var/run/resource-agents/Dummy*")

	self.set_timer("remoteRscFail")
	watch.lookforall()
	self.log_timer("remoteRscFail")
	if watch.unmatched:
	self.fail("Unmatched patterns during rsc fail: %s" % watch.unmatched)

	def fail_connection(self, node):
	if self.failed:
	return

	watchpats = [ ]
	watchpats.append(self.templates["Pat:FenceOpOK"] % self.remote_node)
	watchpats.append(self.templates["Pat:NodeFenced"] % self.remote_node)

	watch = self.create_watch(watchpats, 120)
	watch.setwatch()

	# freeze the pcmk remote daemon. this will result in fencing
	self.debug("Force stopped active remote node")
	self.freeze_pcmk_remote(node)

	self.debug("Waiting for remote node to be fenced.")
	self.set_timer("remoteMetalFence")
	watch.lookforall()
	self.log_timer("remoteMetalFence")
	if watch.unmatched:
	self.fail("Unmatched patterns: %s" % watch.unmatched)
	return

	self.debug("Waiting for the remote node to come back up")
	self.CM.ns.WaitForNodeToComeUp(node, 120);

	pats = [ ]
	watch = self.create_watch(pats, 240)
	watch.setwatch()
	pats.append(self.templates["Pat:RscOpOK"] % ("start", self.remote_node))
	if self.remote_rsc_added == 1:
	pats.append(self.templates["Pat:RscRemoteOpOK"] % ("start", self.remote_rsc, self.remote_node))

	# start the remote node again watch it integrate back into cluster.
	self.start_pcmk_remote(node)
	if self.pcmk_started == 0:
	self.fail("Failed to start pacemaker_remote on node %s" % node)
	return

	self.debug("Waiting for remote node to rejoin cluster after being fenced.")
	self.set_timer("remoteMetalRestart")
	watch.lookforall()
	self.log_timer("remoteMetalRestart")
	if watch.unmatched:
	self.fail("Unmatched patterns: %s" % watch.unmatched)
	return

	def add_dummy_rsc(self, node):
	if self.failed:
	return

	# verify we can put a resource on the remote node
	pats = [ ]
	watch = self.create_watch(pats, 120)
	watch.setwatch()
	pats.append(self.templates["Pat:RscRemoteOpOK"] % ("start", self.remote_rsc, self.remote_node))
	pats.append(self.templates["Pat:DC_IDLE"])

	# Add a resource that must live on remote-node
	self.add_primitive_rsc(node)

	# force that rsc to prefer the remote node.
	(rc, line) = self.CM.rsh(node, "crm_resource -M -r %s -N %s -f" % (self.remote_rsc, self.remote_node), None)
	if rc != 0:
	self.fail("Failed to place remote resource on remote node.")
	return

	self.set_timer("remoteMetalRsc")
	watch.lookforall()
	self.log_timer("remoteMetalRsc")
	if watch.unmatched:
	self.fail("Unmatched patterns: %s" % watch.unmatched)

	def test_attributes(self, node):
	if self.failed:
	return

	# This verifies permanent attributes can be set on a remote-node. It also
	# verifies the remote-node can edit its own cib node section remotely.
	(rc, line) = self.CM.rsh(node, "crm_attribute -l forever -n testattr -v testval -N %s" % (self.remote_node), None)
	if rc != 0:
	self.fail("Failed to set remote-node attribute. rc:%s output:%s" % (rc, line))
	return

	(rc, line) = self.CM.rsh(node, "crm_attribute -l forever -n testattr -q -N %s" % (self.remote_node), None)
	if rc != 0:
	self.fail("Failed to get remote-node attribute")
	return

	(rc, line) = self.CM.rsh(node, "crm_attribute -l forever -n testattr -D -N %s" % (self.remote_node), None)
	if rc != 0:
	self.fail("Failed to delete remote-node attribute")
	return

	def cleanup_metal(self, node):
	self.restore_services(node)

	if self.pcmk_started == 0:
	return

	pats = [ ]

	watch = self.create_watch(pats, 120)
	watch.setwatch()

	if self.remote_rsc_added == 1:
	pats.append(self.templates["Pat:RscOpOK"] % ("stop", self.remote_rsc))
	if self.remote_node_added == 1:
	pats.append(self.templates["Pat:RscOpOK"] % ("stop", self.remote_node))

	self.set_timer("remoteMetalCleanup")

	self.resume_pcmk_remote(node)

	if self.remote_rsc_added == 1:

	# Remove dummy resource added for remote node tests
	self.debug("Cleaning up dummy rsc put on remote node")
	self.rsh(self.get_othernode(node), "crm_resource -U -r %s" % self.remote_rsc)
	self.del_rsc(node, self.remote_rsc)

	if self.remote_node_added == 1:

	# Remove remote node's connection resource
	self.debug("Cleaning up remote node connection resource")
	self.rsh(self.get_othernode(node), "crm_resource -U -r %s" % (self.remote_node))
	self.del_rsc(node, self.remote_node)

	watch.lookforall()
	self.log_timer("remoteMetalCleanup")

	if watch.unmatched:
	self.fail("Unmatched patterns: %s" % watch.unmatched)

	self.stop_pcmk_remote(node)

	self.debug("Waiting for the cluster to recover")
	self.CM.cluster_stable()

	if self.remote_node_added == 1:
	# Remove remote node itself
	self.debug("Cleaning up node entry for remote node")
	self.rsh(self.get_othernode(node), "crm_node --force --remove %s" % self.remote_node)

	def setup_env(self, node):

	self.remote_node = "remote-%s" % (node)

	# we are assuming if all nodes have a key, that it is
	# the right key... If any node doesn't have a remote
	# key, we regenerate it everywhere.
	if self.rsh.exists_on_all("/etc/pacemaker/authkey", self.Env["nodes"]):
	return

	# create key locally
	(handle, keyfile) = tempfile.mkstemp(".cts")
	os.close(handle)
	devnull = open(os.devnull, 'wb')
	subprocess.check_call(["dd", "if=/dev/urandom", "of=%s" % keyfile, "bs=4096", "count=1"],
	stdout=devnull, stderr=devnull)
	devnull.close()

	# sync key throughout the cluster
	for node in self.Env["nodes"]:
	self.rsh(node, "mkdir -p --mode=0750 /etc/pacemaker")
	self.rsh.cp(keyfile, "root@%s:/etc/pacemaker/authkey" % node)
	self.rsh(node, "chgrp haclient /etc/pacemaker /etc/pacemaker/authkey")
	self.rsh(node, "chmod 0640 /etc/pacemaker/authkey")
	os.unlink(keyfile)

	def is_applicable(self):
	if not self.is_applicable_common():
	return False

	for node in self.Env["nodes"]:
	rc = self.rsh(node, "which pacemaker-remoted >/dev/null 2>&1")
	if rc != 0:
	return False
	return True

	def start_new_test(self, node):
	self.incr("calls")
	self.reset()

	ret = self.startall(None)
	if not ret:
	return self.failure("setup failed: could not start all nodes")

	self.setup_env(node)
	self.start_metal(node)
	self.add_dummy_rsc(node)
	return True

	def __call__(self, node):
	return self.failure("This base class is not meant to be called directly.")

	def errorstoignore(self):
	'''Return list of errors which should be ignored'''
	return [ r"""is running on remote.*which isn't allowed""",
	r"""Connection terminated""",
	r"""Could not send remote""",
	]

	# RemoteDriver is just a base class for other tests, so it is not added to AllTestClasses


	class RemoteBasic(RemoteDriver):

	def __call__(self, node):
	'''Perform the 'RemoteBaremetal' test. '''

	if not self.start_new_test(node):
	return self.failure(self.fail_string)

	self.test_attributes(node)
	self.cleanup_metal(node)

	self.debug("Waiting for the cluster to recover")
	self.CM.cluster_stable()
	if self.failed:
	return self.failure(self.fail_string)

	return self.success()

	AllTestClasses.append(RemoteBasic)

	class RemoteStonithd(RemoteDriver):

	def __call__(self, node):
	'''Perform the 'RemoteStonithd' test. '''

	if not self.start_new_test(node):
	return self.failure(self.fail_string)

	self.fail_connection(node)
	self.cleanup_metal(node)

	self.debug("Waiting for the cluster to recover")
	self.CM.cluster_stable()
	if self.failed:
	return self.failure(self.fail_string)

	return self.success()

	def is_applicable(self):
	if not RemoteDriver.is_applicable(self):
	return False

	if "DoFencing" in list(self.Env.keys()):
	return self.Env["DoFencing"]

	return True

	def errorstoignore(self):
	ignore_pats = [
	r"Lost connection to Pacemaker Remote node",
	r"Software caused connection abort",
	r"pacemaker-controld.:\s+error.: Operation remote-.*_monitor",
	r"pacemaker-controld.:\s+error.: Result of monitor operation for remote-.*",
	r"schedulerd.:\s+Recover remote-.\s$.$",
	r"error: Result of monitor operation for .* on remote-.*: No executor connection",
	]

	ignore_pats.extend(RemoteDriver.errorstoignore(self))
	return ignore_pats

	AllTestClasses.append(RemoteStonithd)


	class RemoteMigrate(RemoteDriver):

	def __call__(self, node):
	'''Perform the 'RemoteMigrate' test. '''

	if not self.start_new_test(node):
	return self.failure(self.fail_string)

	self.migrate_connection(node)
	self.cleanup_metal(node)

	self.debug("Waiting for the cluster to recover")
	self.CM.cluster_stable()
	if self.failed:
	return self.failure(self.fail_string)

	return self.success()

	AllTestClasses.append(RemoteMigrate)


	class RemoteRscFailure(RemoteDriver):

	def __call__(self, node):
	'''Perform the 'RemoteRscFailure' test. '''

	if not self.start_new_test(node):
	return self.failure(self.fail_string)

	# This is an important step. We are migrating the connection
	# before failing the resource. This verifies that the migration
	# has properly maintained control over the remote-node.
	self.migrate_connection(node)

	self.fail_rsc(node)
	self.cleanup_metal(node)

	self.debug("Waiting for the cluster to recover")
	self.CM.cluster_stable()
	if self.failed:
	return self.failure(self.fail_string)

	return self.success()

	def errorstoignore(self):
	ignore_pats = [
	r"schedulerd.: Recover remote-rsc\s$.*$",
	r"Dummy.*: No process state file found",
	]

	ignore_pats.extend(RemoteDriver.errorstoignore(self))
	return ignore_pats

	AllTestClasses.append(RemoteRscFailure)

	# vim:ts=4:sw=4:et:
	diff --git a/daemons/attrd/attrd_commands.c b/daemons/attrd/attrd_commands.c
	index f8a01e286b..3ba006c88d 100644
	--- a/daemons/attrd/attrd_commands.c
	+++ b/daemons/attrd/attrd_commands.c
	@@ -1,1314 +1,1314 @@
	/*
	* Copyright 2013-2020 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <sys/types.h>
	#include <regex.h>
	#include <glib.h>

	#include <crm/msg_xml.h>
	#include <crm/cluster.h>
	#include <crm/cib.h>
	#include <crm/common/xml_internal.h>
	#include <crm/cluster/internal.h>
	#include <crm/cluster/election.h>
	#include <crm/cib/internal.h>

	#include "pacemaker-attrd.h"

	/*
	* Legacy attrd (all pre-1.1.11 Pacemaker versions, plus all versions when used
	* with the no-longer-supported CMAN or corosync-plugin stacks) is unversioned.
	*
	* With atomic attrd, each attrd will send ATTRD_PROTOCOL_VERSION with every
	* peer request and reply. As of Pacemaker 2.0.0, at start-up each attrd will
	* also set a private attribute for itself with its version, so any attrd can
	* determine the minimum version supported by all peers.
	*
	* Protocol Pacemaker Significant changes
	* -------- --------- -------------------
	* 1 1.1.11 PCMK__ATTRD_CMD_UPDATE (PCMK__XA_ATTR_NAME only),
	* PCMK__ATTRD_CMD_PEER_REMOVE, PCMK__ATTRD_CMD_REFRESH,
	* PCMK__ATTRD_CMD_FLUSH, PCMK__ATTRD_CMD_SYNC,
	* PCMK__ATTRD_CMD_SYNC_RESPONSE
	* 1 1.1.13 PCMK__ATTRD_CMD_UPDATE (with PCMK__XA_ATTR_PATTERN),
	* PCMK__ATTRD_CMD_QUERY
	* 1 1.1.15 PCMK__ATTRD_CMD_UPDATE_BOTH,
	* PCMK__ATTRD_CMD_UPDATE_DELAY
	* 2 1.1.17 PCMK__ATTRD_CMD_CLEAR_FAILURE
	*/
	#define ATTRD_PROTOCOL_VERSION "2"

	int last_cib_op_done = 0;
	GHashTable *attributes = NULL;

	void write_attribute(attribute_t *a, bool ignore_delay);
	void write_or_elect_attribute(attribute_t *a);
	void attrd_current_only_attribute_update(crm_node_t peer, xmlNode xml);
	void attrd_peer_update(crm_node_t peer, xmlNode xml, const char *host, bool filter);
	void attrd_peer_sync(crm_node_t peer, xmlNode xml);
	void attrd_peer_remove(const char host, gboolean uncache, const char source);

	static gboolean
	send_attrd_message(crm_node_t * node, xmlNode * data)
	{
	crm_xml_add(data, F_TYPE, T_ATTRD);
	crm_xml_add(data, PCMK__XA_ATTR_VERSION, ATTRD_PROTOCOL_VERSION);
	attrd_xml_add_writer(data);
	return send_cluster_message(node, crm_msg_attrd, data, TRUE);
	}

	static gboolean
	attribute_timer_cb(gpointer data)
	{
	attribute_t *a = data;
	crm_trace("Dampen interval expired for %s", a->id);
	write_or_elect_attribute(a);
	return FALSE;
	}

	static void
	free_attribute_value(gpointer data)
	{
	attribute_value_t *v = data;

	free(v->nodename);
	free(v->current);
	free(v->requested);
	free(v);
	}

	void
	free_attribute(gpointer data)
	{
	attribute_t *a = data;
	if(a) {
	free(a->id);
	free(a->set);
	free(a->uuid);
	free(a->user);

	mainloop_timer_del(a->timer);
	g_hash_table_destroy(a->values);

	free(a);
	}
	}

	static xmlNode *
	build_attribute_xml(
	xmlNode parent, const char name, const char set, const char uuid, unsigned int timeout_ms, const char *user,
	gboolean is_private, const char peer, uint32_t peerid, const char value, gboolean is_force_write)
	{
	xmlNode *xml = create_xml_node(parent, __func__);

	crm_xml_add(xml, PCMK__XA_ATTR_NAME, name);
	crm_xml_add(xml, PCMK__XA_ATTR_SET, set);
	crm_xml_add(xml, PCMK__XA_ATTR_UUID, uuid);
	crm_xml_add(xml, PCMK__XA_ATTR_USER, user);
	crm_xml_add(xml, PCMK__XA_ATTR_NODE_NAME, peer);
	crm_xml_add_int(xml, PCMK__XA_ATTR_NODE_ID, peerid);
	crm_xml_add(xml, PCMK__XA_ATTR_VALUE, value);
	crm_xml_add_int(xml, PCMK__XA_ATTR_DAMPENING, timeout_ms/1000);
	crm_xml_add_int(xml, PCMK__XA_ATTR_IS_PRIVATE, is_private);
	crm_xml_add_int(xml, PCMK__XA_ATTR_FORCE, is_force_write);

	return xml;
	}

	static void
	clear_attribute_value_seen(void)
	{
	GHashTableIter aIter;
	GHashTableIter vIter;
	attribute_t *a;
	attribute_value_t *v = NULL;

	g_hash_table_iter_init(&aIter, attributes);
	while (g_hash_table_iter_next(&aIter, NULL, (gpointer *) & a)) {
	g_hash_table_iter_init(&vIter, a->values);
	while (g_hash_table_iter_next(&vIter, NULL, (gpointer *) & v)) {
	v->seen = FALSE;
	crm_trace("Clear seen flag %s[%s] = %s.", a->id, v->nodename, v->current);
	}
	}
	}

	static attribute_t *
	create_attribute(xmlNode *xml)
	{
	int dampen = 0;
	const char *value = crm_element_value(xml, PCMK__XA_ATTR_DAMPENING);
	attribute_t *a = calloc(1, sizeof(attribute_t));

	a->id = crm_element_value_copy(xml, PCMK__XA_ATTR_NAME);
	a->set = crm_element_value_copy(xml, PCMK__XA_ATTR_SET);
	a->uuid = crm_element_value_copy(xml, PCMK__XA_ATTR_UUID);
	a->values = g_hash_table_new_full(crm_strcase_hash, crm_strcase_equal, NULL, free_attribute_value);

	crm_element_value_int(xml, PCMK__XA_ATTR_IS_PRIVATE, &a->is_private);

	#if ENABLE_ACL
	a->user = crm_element_value_copy(xml, PCMK__XA_ATTR_USER);
	crm_trace("Performing all %s operations as user '%s'", a->id, a->user);
	#endif

	if(value) {
	dampen = crm_get_msec(value);
	crm_trace("Created attribute %s with delay %dms (%s)", a->id, dampen, value);
	} else {
	crm_trace("Created attribute %s with no delay", a->id);
	}

	if(dampen > 0) {
	a->timeout_ms = dampen;
	a->timer = mainloop_timer_add(a->id, a->timeout_ms, FALSE, attribute_timer_cb, a);
	} else if (dampen < 0) {
	crm_warn("Ignoring invalid delay %s for attribute %s", value, a->id);
	}

	g_hash_table_replace(attributes, a->id, a);
	return a;
	}

	/*!
	* \internal
	* \brief Respond to a client peer-remove request (i.e. propagate to all peers)
	*
	* \param[in] client_name Name of client that made request (for log messages)
	* \param[in] xml Root of request XML
	*
	* \return void
	*/
	void
	attrd_client_peer_remove(pcmk__client_t client, xmlNode xml)
	{
	// Host and ID are not used in combination, rather host has precedence
	const char *host = crm_element_value(xml, PCMK__XA_ATTR_NODE_NAME);
	char *host_alloc = NULL;

	if (host == NULL) {
	int nodeid = 0;

	crm_element_value_int(xml, PCMK__XA_ATTR_NODE_ID, &nodeid);
	if (nodeid > 0) {
	- crm_node_t *node = crm_find_peer(nodeid, NULL);
	+ crm_node_t *node = pcmk__search_cluster_node_cache(nodeid, NULL);
	char *host_alloc = NULL;

	if (node && node->uname) {
	// Use cached name if available
	host = node->uname;
	} else {
	// Otherwise ask cluster layer
	host_alloc = get_node_name(nodeid);
	host = host_alloc;
	}
	crm_xml_add(xml, PCMK__XA_ATTR_NODE_NAME, host);
	}
	}

	if (host) {
	crm_info("Client %s is requesting all values for %s be removed",
	pcmk__client_name(client), host);
	send_attrd_message(NULL, xml); /* ends up at attrd_peer_message() */
	free(host_alloc);
	} else {
	crm_info("Ignoring request by client %s to remove all peer values without specifying peer",
	pcmk__client_name(client));
	}
	}

	/*!
	* \internal
	* \brief Respond to a client update request
	*
	* \param[in] xml Root of request XML
	*
	* \return void
	*/
	void
	attrd_client_update(xmlNode *xml)
	{
	attribute_t *a = NULL;
	char *host = crm_element_value_copy(xml, PCMK__XA_ATTR_NODE_NAME);
	const char *attr = crm_element_value(xml, PCMK__XA_ATTR_NAME);
	const char *value = crm_element_value(xml, PCMK__XA_ATTR_VALUE);
	const char *regex = crm_element_value(xml, PCMK__XA_ATTR_PATTERN);

	/* If a regex was specified, broadcast a message for each match */
	if ((attr == NULL) && regex) {
	GHashTableIter aIter;
	regex_t *r_patt = calloc(1, sizeof(regex_t));

	crm_debug("Setting %s to %s", regex, value);
	if (regcomp(r_patt, regex, REG_EXTENDED\|REG_NOSUB)) {
	crm_err("Bad regex '%s' for update", regex);

	} else {
	g_hash_table_iter_init(&aIter, attributes);
	while (g_hash_table_iter_next(&aIter, (gpointer *) & attr, NULL)) {
	int status = regexec(r_patt, attr, 0, NULL, 0);

	if (status == 0) {
	crm_trace("Matched %s with %s", attr, regex);
	crm_xml_add(xml, PCMK__XA_ATTR_NAME, attr);
	send_attrd_message(NULL, xml);
	}
	}
	}

	free(host);
	regfree(r_patt);
	free(r_patt);
	return;

	} else if (attr == NULL) {
	crm_err("Update request did not specify attribute or regular expression");
	free(host);
	return;
	}

	if (host == NULL) {
	crm_trace("Inferring host");
	host = strdup(attrd_cluster->uname);
	crm_xml_add(xml, PCMK__XA_ATTR_NODE_NAME, host);
	crm_xml_add_int(xml, PCMK__XA_ATTR_NODE_ID, attrd_cluster->nodeid);
	}

	a = g_hash_table_lookup(attributes, attr);

	/* If value was specified using ++ or += notation, expand to real value */
	if (value) {
	if (attrd_value_needs_expansion(value)) {
	int int_value;
	attribute_value_t *v = NULL;

	if (a) {
	v = g_hash_table_lookup(a->values, host);
	}
	int_value = attrd_expand_value(value, (v? v->current : NULL));

	crm_info("Expanded %s=%s to %d", attr, value, int_value);
	crm_xml_add_int(xml, PCMK__XA_ATTR_VALUE, int_value);

	/* Replacing the value frees the previous memory, so re-query it */
	value = crm_element_value(xml, PCMK__XA_ATTR_VALUE);
	}
	}

	crm_debug("Broadcasting %s[%s]=%s%s", attr, host, value,
	(attrd_election_won()? " (writer)" : ""));

	free(host);

	send_attrd_message(NULL, xml); /* ends up at attrd_peer_message() */
	}

	/*!
	* \internal
	* \brief Respond to client clear-failure request
	*
	* \param[in] xml Request XML
	*/
	void
	attrd_client_clear_failure(xmlNode *xml)
	{
	#if 0
	/* @TODO Track the minimum supported protocol version across all nodes,
	* then enable this more-efficient code.
	*/
	if (compare_version("2", minimum_protocol_version) <= 0) {
	/* Propagate to all peers (including ourselves).
	* This ends up at attrd_peer_message().
	*/
	send_attrd_message(NULL, xml);
	return;
	}
	#endif

	const char *rsc = crm_element_value(xml, PCMK__XA_ATTR_RESOURCE);
	const char *op = crm_element_value(xml, PCMK__XA_ATTR_OPERATION);
	const char *interval_spec = crm_element_value(xml, PCMK__XA_ATTR_INTERVAL);

	/* Map this to an update */
	crm_xml_add(xml, PCMK__XA_TASK, PCMK__ATTRD_CMD_UPDATE);

	/* Add regular expression matching desired attributes */

	if (rsc) {
	char *pattern;

	if (op == NULL) {
	pattern = crm_strdup_printf(ATTRD_RE_CLEAR_ONE, rsc);

	} else {
	guint interval_ms = crm_parse_interval_spec(interval_spec);

	pattern = crm_strdup_printf(ATTRD_RE_CLEAR_OP,
	rsc, op, interval_ms);
	}

	crm_xml_add(xml, PCMK__XA_ATTR_PATTERN, pattern);
	free(pattern);

	} else {
	crm_xml_add(xml, PCMK__XA_ATTR_PATTERN, ATTRD_RE_CLEAR_ALL);
	}

	/* Make sure attribute and value are not set, so we delete via regex */
	if (crm_element_value(xml, PCMK__XA_ATTR_NAME)) {
	crm_xml_replace(xml, PCMK__XA_ATTR_NAME, NULL);
	}
	if (crm_element_value(xml, PCMK__XA_ATTR_VALUE)) {
	crm_xml_replace(xml, PCMK__XA_ATTR_VALUE, NULL);
	}

	attrd_client_update(xml);
	}

	/*!
	* \internal
	* \brief Respond to a client refresh request (i.e. write out all attributes)
	*
	* \return void
	*/
	void
	attrd_client_refresh(void)
	{
	crm_info("Updating all attributes");
	write_attributes(TRUE, TRUE);
	}

	/*!
	* \internal
	* \brief Build the XML reply to a client query
	*
	* param[in] attr Name of requested attribute
	* param[in] host Name of requested host (or NULL for all hosts)
	*
	* \return New XML reply
	* \note Caller is responsible for freeing the resulting XML
	*/
	static xmlNode build_query_reply(const char attr, const char *host)
	{
	xmlNode *reply = create_xml_node(NULL, __func__);
	attribute_t *a;

	if (reply == NULL) {
	return NULL;
	}
	crm_xml_add(reply, F_TYPE, T_ATTRD);
	crm_xml_add(reply, PCMK__XA_ATTR_VERSION, ATTRD_PROTOCOL_VERSION);

	/* If desired attribute exists, add its value(s) to the reply */
	a = g_hash_table_lookup(attributes, attr);
	if (a) {
	attribute_value_t *v;
	xmlNode *host_value;

	crm_xml_add(reply, PCMK__XA_ATTR_NAME, attr);

	/* Allow caller to use "localhost" to refer to local node */
	if (pcmk__str_eq(host, "localhost", pcmk__str_casei)) {
	host = attrd_cluster->uname;
	crm_trace("Mapped localhost to %s", host);
	}

	/* If a specific node was requested, add its value */
	if (host) {
	v = g_hash_table_lookup(a->values, host);
	host_value = create_xml_node(reply, XML_CIB_TAG_NODE);
	if (host_value == NULL) {
	free_xml(reply);
	return NULL;
	}
	crm_xml_add(host_value, PCMK__XA_ATTR_NODE_NAME, host);
	crm_xml_add(host_value, PCMK__XA_ATTR_VALUE,
	(v? v->current : NULL));

	/* Otherwise, add all nodes' values */
	} else {
	GHashTableIter iter;

	g_hash_table_iter_init(&iter, a->values);
	while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &v)) {
	host_value = create_xml_node(reply, XML_CIB_TAG_NODE);
	if (host_value == NULL) {
	free_xml(reply);
	return NULL;
	}
	crm_xml_add(host_value, PCMK__XA_ATTR_NODE_NAME, v->nodename);
	crm_xml_add(host_value, PCMK__XA_ATTR_VALUE, v->current);
	}
	}
	}
	return reply;
	}

	/*!
	* \internal
	* \brief Respond to a client query
	*
	* \param[in] client Who queried us
	* \param[in] query Root of query XML
	*
	* \return void
	*/
	void
	attrd_client_query(pcmk__client_t *client, uint32_t id, uint32_t flags,
	xmlNode *query)
	{
	const char *attr;
	const char *origin = crm_element_value(query, F_ORIG);
	xmlNode *reply;

	if (origin == NULL) {
	origin = "unknown client";
	}
	crm_debug("Query arrived from %s", origin);

	/* Request must specify attribute name to query */
	attr = crm_element_value(query, PCMK__XA_ATTR_NAME);
	if (attr == NULL) {
	crm_warn("Ignoring malformed query from %s (no attribute name given)",
	origin);
	return;
	}

	/* Build the XML reply */
	reply = build_query_reply(attr, crm_element_value(query,
	PCMK__XA_ATTR_NODE_NAME));
	if (reply == NULL) {
	crm_err("Could not respond to query from %s: could not create XML reply",
	origin);
	return;
	}
	crm_log_xml_trace(reply, "Reply");

	/* Send the reply to the client */
	client->request_id = 0;
	{
	int rc = pcmk__ipc_send_xml(client, id, reply, flags);

	if (rc != pcmk_rc_ok) {
	crm_err("Could not respond to query from %s: %s " CRM_XS " rc=%d",
	origin, pcmk_rc_str(rc), rc);
	}
	}
	free_xml(reply);
	}

	/*!
	* \internal
	* \brief Clear failure-related attributes
	*
	* \param[in] peer Peer that sent clear request
	* \param[in] xml Request XML
	*/
	static void
	attrd_peer_clear_failure(crm_node_t peer, xmlNode xml)
	{
	const char *rsc = crm_element_value(xml, PCMK__XA_ATTR_RESOURCE);
	const char *host = crm_element_value(xml, PCMK__XA_ATTR_NODE_NAME);
	const char *op = crm_element_value(xml, PCMK__XA_ATTR_OPERATION);
	const char *interval_spec = crm_element_value(xml, PCMK__XA_ATTR_INTERVAL);
	guint interval_ms = crm_parse_interval_spec(interval_spec);
	char *attr = NULL;
	GHashTableIter iter;
	regex_t regex;

	if (attrd_failure_regex(&regex, rsc, op, interval_ms) != pcmk_ok) {
	crm_info("Ignoring invalid request to clear failures for %s",
	(rsc? rsc : "all resources"));
	return;
	}

	crm_xml_add(xml, PCMK__XA_TASK, PCMK__ATTRD_CMD_UPDATE);

	/* Make sure value is not set, so we delete */
	if (crm_element_value(xml, PCMK__XA_ATTR_VALUE)) {
	crm_xml_replace(xml, PCMK__XA_ATTR_VALUE, NULL);
	}

	g_hash_table_iter_init(&iter, attributes);
	while (g_hash_table_iter_next(&iter, (gpointer *) &attr, NULL)) {
	if (regexec(&regex, attr, 0, NULL, 0) == 0) {
	crm_trace("Matched %s when clearing %s",
	attr, (rsc? rsc : "all resources"));
	crm_xml_add(xml, PCMK__XA_ATTR_NAME, attr);
	attrd_peer_update(peer, xml, host, FALSE);
	}
	}
	regfree(&regex);
	}

	/*!
	\internal
	\brief Broadcast private attribute for local node with protocol version
	*/
	void
	attrd_broadcast_protocol()
	{
	xmlNode *attrd_op = create_xml_node(NULL, __func__);

	crm_xml_add(attrd_op, F_TYPE, T_ATTRD);
	crm_xml_add(attrd_op, F_ORIG, crm_system_name);
	crm_xml_add(attrd_op, PCMK__XA_TASK, PCMK__ATTRD_CMD_UPDATE);
	crm_xml_add(attrd_op, PCMK__XA_ATTR_NAME, CRM_ATTR_PROTOCOL);
	crm_xml_add(attrd_op, PCMK__XA_ATTR_VALUE, ATTRD_PROTOCOL_VERSION);
	crm_xml_add_int(attrd_op, PCMK__XA_ATTR_IS_PRIVATE, 1);
	attrd_client_update(attrd_op);
	free_xml(attrd_op);
	}

	void
	attrd_peer_message(crm_node_t peer, xmlNode xml)
	{
	const char *op = crm_element_value(xml, PCMK__XA_TASK);
	const char *election_op = crm_element_value(xml, F_CRM_TASK);
	const char *host = crm_element_value(xml, PCMK__XA_ATTR_NODE_NAME);
	bool peer_won = FALSE;

	if (election_op) {
	attrd_handle_election_op(peer, xml);
	return;
	}

	if (attrd_shutting_down()) {
	/* If we're shutting down, we want to continue responding to election
	* ops as long as we're a cluster member (because our vote may be
	* needed). Ignore all other messages.
	*/
	return;
	}

	peer_won = attrd_check_for_new_writer(peer, xml);

	if (pcmk__strcase_any_of(op, PCMK__ATTRD_CMD_UPDATE, PCMK__ATTRD_CMD_UPDATE_BOTH,
	PCMK__ATTRD_CMD_UPDATE_DELAY, NULL)) {
	attrd_peer_update(peer, xml, host, FALSE);

	} else if (pcmk__str_eq(op, PCMK__ATTRD_CMD_SYNC, pcmk__str_casei)) {
	attrd_peer_sync(peer, xml);

	} else if (pcmk__str_eq(op, PCMK__ATTRD_CMD_PEER_REMOVE, pcmk__str_casei)) {
	attrd_peer_remove(host, TRUE, peer->uname);

	} else if (pcmk__str_eq(op, PCMK__ATTRD_CMD_CLEAR_FAILURE, pcmk__str_casei)) {
	/* It is not currently possible to receive this as a peer command,
	* but will be, if we one day enable propagating this operation.
	*/
	attrd_peer_clear_failure(peer, xml);

	} else if (pcmk__str_eq(op, PCMK__ATTRD_CMD_SYNC_RESPONSE, pcmk__str_casei)
	&& !pcmk__str_eq(peer->uname, attrd_cluster->uname, pcmk__str_casei)) {
	xmlNode *child = NULL;

	crm_info("Processing %s from %s", op, peer->uname);

	/* Clear the seen flag for attribute processing held only in the own node. */
	if (peer_won) {
	clear_attribute_value_seen();
	}

	for (child = pcmk__xml_first_child(xml); child != NULL;
	child = pcmk__xml_next(child)) {
	host = crm_element_value(child, PCMK__XA_ATTR_NODE_NAME);
	attrd_peer_update(peer, child, host, TRUE);
	}

	if (peer_won) {
	/* Synchronize if there is an attribute held only by own node that Writer does not have. */
	attrd_current_only_attribute_update(peer, xml);
	}
	}
	}

	void
	attrd_peer_sync(crm_node_t peer, xmlNode xml)
	{
	GHashTableIter aIter;
	GHashTableIter vIter;

	attribute_t *a = NULL;
	attribute_value_t *v = NULL;
	xmlNode *sync = create_xml_node(NULL, __func__);

	crm_xml_add(sync, PCMK__XA_TASK, PCMK__ATTRD_CMD_SYNC_RESPONSE);

	g_hash_table_iter_init(&aIter, attributes);
	while (g_hash_table_iter_next(&aIter, NULL, (gpointer *) & a)) {
	g_hash_table_iter_init(&vIter, a->values);
	while (g_hash_table_iter_next(&vIter, NULL, (gpointer *) & v)) {
	crm_debug("Syncing %s[%s] = %s to %s", a->id, v->nodename, v->current, peer?peer->uname:"everyone");
	build_attribute_xml(sync, a->id, a->set, a->uuid, a->timeout_ms, a->user, a->is_private,
	v->nodename, v->nodeid, v->current, FALSE);
	}
	}

	crm_debug("Syncing values to %s", peer?peer->uname:"everyone");
	send_attrd_message(peer, sync);
	free_xml(sync);
	}

	/*!
	* \internal
	* \brief Remove all attributes and optionally peer cache entries for a node
	*
	* \param[in] host Name of node to purge
	* \param[in] uncache If TRUE, remove node from peer caches
	* \param[in] source Who requested removal (only used for logging)
	*/
	void
	attrd_peer_remove(const char host, gboolean uncache, const char source)
	{
	attribute_t *a = NULL;
	GHashTableIter aIter;

	CRM_CHECK(host != NULL, return);
	crm_notice("Removing all %s attributes for peer %s", host, source);

	g_hash_table_iter_init(&aIter, attributes);
	while (g_hash_table_iter_next(&aIter, NULL, (gpointer *) & a)) {
	if(g_hash_table_remove(a->values, host)) {
	crm_debug("Removed %s[%s] for peer %s", a->id, host, source);
	}
	}

	if (uncache) {
	crm_remote_peer_cache_remove(host);
	reap_crm_member(0, host);
	}
	}

	/*!
	* \internal
	* \brief Return host's hash table entry (creating one if needed)
	*
	* \param[in] values Hash table of values
	* \param[in] host Name of peer to look up
	* \param[in] xml XML describing the attribute
	*
	* \return Pointer to new or existing hash table entry
	*/
	static attribute_value_t *
	attrd_lookup_or_create_value(GHashTable values, const char host, xmlNode *xml)
	{
	attribute_value_t *v = g_hash_table_lookup(values, host);
	int is_remote = 0;

	crm_element_value_int(xml, PCMK__XA_ATTR_IS_REMOTE, &is_remote);
	if (is_remote) {
	/* If we previously assumed this node was an unseen cluster node,
	* remove its entry from the cluster peer cache.
	*/
	- crm_node_t *dup = crm_find_peer(0, host);
	+ crm_node_t *dup = pcmk__search_cluster_node_cache(0, host);

	if (dup && (dup->uuid == NULL)) {
	reap_crm_member(0, host);
	}

	/* Ensure this host is in the remote peer cache */
	CRM_ASSERT(crm_remote_peer_get(host) != NULL);
	}

	if (v == NULL) {
	v = calloc(1, sizeof(attribute_value_t));
	CRM_ASSERT(v != NULL);

	v->nodename = strdup(host);
	CRM_ASSERT(v->nodename != NULL);

	v->is_remote = is_remote;
	g_hash_table_replace(values, v->nodename, v);
	}
	return(v);
	}

	void
	attrd_current_only_attribute_update(crm_node_t peer, xmlNode xml)
	{
	GHashTableIter aIter;
	GHashTableIter vIter;
	attribute_t *a;
	attribute_value_t *v = NULL;
	xmlNode *sync = create_xml_node(NULL, __func__);
	gboolean build = FALSE;

	crm_xml_add(sync, PCMK__XA_TASK, PCMK__ATTRD_CMD_SYNC_RESPONSE);

	g_hash_table_iter_init(&aIter, attributes);
	while (g_hash_table_iter_next(&aIter, NULL, (gpointer *) & a)) {
	g_hash_table_iter_init(&vIter, a->values);
	while (g_hash_table_iter_next(&vIter, NULL, (gpointer *) & v)) {
	if (pcmk__str_eq(v->nodename, attrd_cluster->uname, pcmk__str_casei) && v->seen == FALSE) {
	crm_trace("Syncing %s[%s] = %s to everyone.(from local only attributes)", a->id, v->nodename, v->current);

	build = TRUE;
	build_attribute_xml(sync, a->id, a->set, a->uuid, a->timeout_ms, a->user, a->is_private,
	v->nodename, v->nodeid, v->current, (a->timeout_ms && a->timer ? TRUE : FALSE));
	} else {
	crm_trace("Local attribute(%s[%s] = %s) was ignore.(another host) : [%s]", a->id, v->nodename, v->current, attrd_cluster->uname);
	continue;
	}
	}
	}

	if (build) {
	crm_debug("Syncing values to everyone.(from local only attributes)");
	send_attrd_message(NULL, sync);
	}
	free_xml(sync);
	}

	void
	attrd_peer_update(crm_node_t peer, xmlNode xml, const char *host, bool filter)
	{
	bool update_both = FALSE;
	attribute_t *a;
	attribute_value_t *v = NULL;
	gboolean is_force_write = FALSE;

	const char *op = crm_element_value(xml, PCMK__XA_TASK);
	const char *attr = crm_element_value(xml, PCMK__XA_ATTR_NAME);
	const char *value = crm_element_value(xml, PCMK__XA_ATTR_VALUE);
	crm_element_value_int(xml, PCMK__XA_ATTR_FORCE, &is_force_write);

	if (attr == NULL) {
	crm_warn("Could not update attribute: peer did not specify name");
	return;
	}

	// NULL because PCMK__ATTRD_CMD_SYNC_RESPONSE has no PCMK__XA_TASK
	update_both = pcmk__str_eq(op, PCMK__ATTRD_CMD_UPDATE_BOTH,
	pcmk__str_null_matches \| pcmk__str_casei);

	// Look up or create attribute entry
	a = g_hash_table_lookup(attributes, attr);
	if (a == NULL) {
	if (update_both \|\| pcmk__str_eq(op, PCMK__ATTRD_CMD_UPDATE, pcmk__str_casei)) {
	a = create_attribute(xml);
	} else {
	crm_warn("Could not update %s: attribute not found", attr);
	return;
	}
	}

	// Update attribute dampening
	if (update_both \|\| pcmk__str_eq(op, PCMK__ATTRD_CMD_UPDATE_DELAY, pcmk__str_casei)) {
	const char *dvalue = crm_element_value(xml, PCMK__XA_ATTR_DAMPENING);
	int dampen = 0;

	if (dvalue == NULL) {
	crm_warn("Could not update %s: peer did not specify value for delay",
	attr);
	return;
	}

	dampen = crm_get_msec(dvalue);
	if (dampen < 0) {
	crm_warn("Could not update %s: invalid delay value %dms (%s)",
	attr, dampen, dvalue);
	return;
	}

	if (a->timeout_ms != dampen) {
	mainloop_timer_del(a->timer);
	a->timeout_ms = dampen;
	if (dampen > 0) {
	a->timer = mainloop_timer_add(attr, a->timeout_ms, FALSE,
	attribute_timer_cb, a);
	crm_info("Update attribute %s delay to %dms (%s)",
	attr, dampen, dvalue);
	} else {
	a->timer = NULL;
	crm_info("Update attribute %s to remove delay", attr);
	}

	/* If dampening changed, do an immediate write-out,
	* otherwise repeated dampening changes would prevent write-outs
	*/
	write_or_elect_attribute(a);
	}

	if (!update_both) {
	return;
	}
	}

	// If no host was specified, update all hosts recursively
	if (host == NULL) {
	GHashTableIter vIter;

	crm_debug("Setting %s for all hosts to %s", attr, value);
	xml_remove_prop(xml, PCMK__XA_ATTR_NODE_ID);
	g_hash_table_iter_init(&vIter, a->values);
	while (g_hash_table_iter_next(&vIter, (gpointer *) & host, NULL)) {
	attrd_peer_update(peer, xml, host, filter);
	}
	return;
	}

	// Update attribute value for one host

	v = attrd_lookup_or_create_value(a->values, host, xml);

	if (filter && !pcmk__str_eq(v->current, value, pcmk__str_casei)
	&& pcmk__str_eq(host, attrd_cluster->uname, pcmk__str_casei)) {

	xmlNode *sync = create_xml_node(NULL, __func__);

	crm_notice("%s[%s]: local value '%s' takes priority over '%s' from %s",
	attr, host, v->current, value, peer->uname);

	crm_xml_add(sync, PCMK__XA_TASK, PCMK__ATTRD_CMD_SYNC_RESPONSE);
	v = g_hash_table_lookup(a->values, host);
	build_attribute_xml(sync, attr, a->set, a->uuid, a->timeout_ms, a->user,
	a->is_private, v->nodename, v->nodeid, v->current, FALSE);

	attrd_xml_add_writer(sync);

	/* Broadcast in case any other nodes had the inconsistent value */
	send_attrd_message(NULL, sync);
	free_xml(sync);

	} else if (!pcmk__str_eq(v->current, value, pcmk__str_casei)) {
	crm_notice("Setting %s[%s]: %s -> %s " CRM_XS " from %s",
	attr, host, v->current? v->current : "(unset)", value? value : "(unset)", peer->uname);
	free(v->current);
	v->current = (value? strdup(value) : NULL);
	a->changed = TRUE;

	// Write out new value or start dampening timer
	if (a->timeout_ms && a->timer) {
	crm_trace("Delayed write out (%dms) for %s", a->timeout_ms, attr);
	mainloop_timer_start(a->timer);
	} else {
	write_or_elect_attribute(a);
	}

	} else {
	if (is_force_write && a->timeout_ms && a->timer) {
	/* Save forced writing and set change flag. */
	/* The actual attribute is written by Writer after election. */
	crm_trace("Unchanged %s[%s] from %s is %s(Set the forced write flag)", attr, host, peer->uname, value);
	a->force_write = TRUE;
	} else {
	crm_trace("Unchanged %s[%s] from %s is %s", attr, host, peer->uname, value);
	}
	}

	/* Set the seen flag for attribute processing held only in the own node. */
	v->seen = TRUE;

	/* If this is a cluster node whose node ID we are learning, remember it */
	if ((v->nodeid == 0) && (v->is_remote == FALSE)
	&& (crm_element_value_int(xml, PCMK__XA_ATTR_NODE_ID,
	(int*)&v->nodeid) == 0)) {

	crm_node_t *known_peer = crm_get_peer(v->nodeid, host);

	crm_trace("Learned %s has node id %s",
	known_peer->uname, known_peer->uuid);
	if (attrd_election_won()) {
	write_attributes(FALSE, FALSE);
	}
	}
	}

	void
	write_or_elect_attribute(attribute_t *a)
	{
	if (attrd_election_won()) {
	write_attribute(a, FALSE);
	} else {
	attrd_start_election_if_needed();
	}
	}

	gboolean
	attrd_election_cb(gpointer user_data)
	{
	attrd_declare_winner();

	/* Update the peers after an election */
	attrd_peer_sync(NULL, NULL);

	/* Update the CIB after an election */
	write_attributes(TRUE, FALSE);
	return FALSE;
	}


	void
	attrd_peer_change_cb(enum crm_status_type kind, crm_node_t peer, const void data)
	{
	bool remove_voter = FALSE;

	switch (kind) {
	case crm_status_uname:
	break;

	case crm_status_processes:
	if (!pcmk_is_set(peer->processes, crm_get_cluster_proc())) {
	remove_voter = TRUE;
	}
	break;

	case crm_status_nstate:
	if (pcmk__str_eq(peer->state, CRM_NODE_MEMBER, pcmk__str_casei)) {
	/* If we're the writer, send new peers a list of all attributes
	* (unless it's a remote node, which doesn't run its own attrd)
	*/
	if (attrd_election_won()
	&& !pcmk_is_set(peer->flags, crm_remote_node)) {
	attrd_peer_sync(peer, NULL);
	}
	} else {
	// Remove all attribute values associated with lost nodes
	attrd_peer_remove(peer->uname, FALSE, "loss");
	remove_voter = TRUE;
	}
	break;
	}

	// In case an election is in progress, remove any vote by the node
	if (remove_voter) {
	attrd_remove_voter(peer);
	}
	}

	static void
	attrd_cib_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
	{
	int level = LOG_ERR;
	GHashTableIter iter;
	const char *peer = NULL;
	attribute_value_t *v = NULL;

	char *name = user_data;
	attribute_t *a = g_hash_table_lookup(attributes, name);

	if(a == NULL) {
	crm_info("Attribute %s no longer exists", name);
	return;
	}

	a->update = 0;
	if (rc == pcmk_ok && call_id < 0) {
	rc = call_id;
	}

	switch (rc) {
	case pcmk_ok:
	level = LOG_INFO;
	last_cib_op_done = call_id;
	if (a->timer && !a->timeout_ms) {
	// Remove temporary dampening for failed writes
	mainloop_timer_del(a->timer);
	a->timer = NULL;
	}
	break;

	case -pcmk_err_diff_failed: /* When an attr changes while the CIB is syncing */
	case -ETIME: /* When an attr changes while there is a DC election */
	case -ENXIO: /* When an attr changes while the CIB is syncing a
	* newer config from a node that just came up
	*/
	level = LOG_WARNING;
	break;
	}

	do_crm_log(level, "CIB update %d result for %s: %s " CRM_XS " rc=%d",
	call_id, a->id, pcmk_strerror(rc), rc);

	g_hash_table_iter_init(&iter, a->values);
	while (g_hash_table_iter_next(&iter, (gpointer ) & peer, (gpointer ) & v)) {
	do_crm_log(level, "* %s[%s]=%s", a->id, peer, v->requested);
	free(v->requested);
	v->requested = NULL;
	if (rc != pcmk_ok) {
	a->changed = TRUE; /* Attempt write out again */
	}
	}

	if (a->changed && attrd_election_won()) {
	if (rc == pcmk_ok) {
	/* We deferred a write of a new update because this update was in
	* progress. Write out the new value without additional delay.
	*/
	write_attribute(a, FALSE);

	/* We're re-attempting a write because the original failed; delay
	* the next attempt so we don't potentially flood the CIB manager
	* and logs with a zillion attempts per second.
	*
	* @TODO We could elect a new writer instead. However, we'd have to
	* somehow downgrade our vote, and we'd still need something like this
	* if all peers similarly fail to write this attribute (which may
	* indicate a corrupted attribute entry rather than a CIB issue).
	*/
	} else if (a->timer) {
	// Attribute has a dampening value, so use that as delay
	if (!mainloop_timer_running(a->timer)) {
	crm_trace("Delayed re-attempted write (%dms) for %s",
	a->timeout_ms, name);
	mainloop_timer_start(a->timer);
	}
	} else {
	/* Set a temporary dampening of 2 seconds (timer will continue
	* to exist until the attribute's dampening gets set or the
	* write succeeds).
	*/
	a->timer = mainloop_timer_add(a->id, 2000, FALSE,
	attribute_timer_cb, a);
	mainloop_timer_start(a->timer);
	}
	}
	}

	void
	write_attributes(bool all, bool ignore_delay)
	{
	GHashTableIter iter;
	attribute_t *a = NULL;

	crm_debug("Writing out %s attributes", all? "all" : "changed");
	g_hash_table_iter_init(&iter, attributes);
	while (g_hash_table_iter_next(&iter, NULL, (gpointer *) & a)) {
	if (!all && a->unknown_peer_uuids) {
	// Try writing this attribute again, in case peer ID was learned
	a->changed = TRUE;
	} else if (a->force_write) {
	/* If the force_write flag is set, write the attribute. */
	a->changed = TRUE;
	}

	if(all \|\| a->changed) {
	/* When forced write flag is set, ignore delay. */
	write_attribute(a, (a->force_write ? TRUE : ignore_delay));
	} else {
	crm_trace("Skipping unchanged attribute %s", a->id);
	}
	}
	}

	static void
	build_update_element(xmlNode parent, attribute_t a, const char nodeid, const char value)
	{
	const char *set = NULL;
	xmlNode *xml_obj = NULL;

	xml_obj = create_xml_node(parent, XML_CIB_TAG_STATE);
	crm_xml_add(xml_obj, XML_ATTR_ID, nodeid);

	xml_obj = create_xml_node(xml_obj, XML_TAG_TRANSIENT_NODEATTRS);
	crm_xml_add(xml_obj, XML_ATTR_ID, nodeid);

	xml_obj = create_xml_node(xml_obj, XML_TAG_ATTR_SETS);
	if (a->set) {
	crm_xml_set_id(xml_obj, "%s", a->set);
	} else {
	crm_xml_set_id(xml_obj, "%s-%s", XML_CIB_TAG_STATUS, nodeid);
	}
	set = ID(xml_obj);

	xml_obj = create_xml_node(xml_obj, XML_CIB_TAG_NVPAIR);
	if (a->uuid) {
	crm_xml_set_id(xml_obj, "%s", a->uuid);
	} else {
	crm_xml_set_id(xml_obj, "%s-%s", set, a->id);
	}
	crm_xml_add(xml_obj, XML_NVPAIR_ATTR_NAME, a->id);

	if(value) {
	crm_xml_add(xml_obj, XML_NVPAIR_ATTR_VALUE, value);

	} else {
	crm_xml_add(xml_obj, XML_NVPAIR_ATTR_VALUE, "");
	crm_xml_add(xml_obj, "__delete__", XML_NVPAIR_ATTR_VALUE);
	}
	}

	static void
	set_alert_attribute_value(GHashTable t, attribute_value_t v)
	{
	attribute_value_t *a_v = NULL;
	a_v = calloc(1, sizeof(attribute_value_t));
	CRM_ASSERT(a_v != NULL);

	a_v->nodeid = v->nodeid;
	a_v->nodename = strdup(v->nodename);

	if (v->current != NULL) {
	a_v->current = strdup(v->current);
	}

	g_hash_table_replace(t, a_v->nodename, a_v);
	}

	static void
	send_alert_attributes_value(attribute_t a, GHashTable t)
	{
	int rc = 0;
	attribute_value_t *at = NULL;
	GHashTableIter vIter;

	g_hash_table_iter_init(&vIter, t);

	while (g_hash_table_iter_next(&vIter, NULL, (gpointer *) & at)) {
	rc = attrd_send_attribute_alert(at->nodename, at->nodeid,
	a->id, at->current);
	crm_trace("Sent alerts for %s[%s]=%s: nodeid=%d rc=%d",
	a->id, at->nodename, at->current, at->nodeid, rc);
	}
	}

	void
	write_attribute(attribute_t *a, bool ignore_delay)
	{
	int private_updates = 0, cib_updates = 0;
	xmlNode *xml_top = NULL;
	attribute_value_t *v = NULL;
	GHashTableIter iter;
	enum cib_call_options flags = cib_quorum_override;
	GHashTable *alert_attribute_value = NULL;

	if (a == NULL) {
	return;
	}

	/* If this attribute will be written to the CIB ... */
	if (!a->is_private) {

	/* Defer the write if now's not a good time */
	CRM_CHECK(the_cib != NULL, return);
	if (a->update && (a->update < last_cib_op_done)) {
	crm_info("Write out of '%s' continuing: update %d considered lost", a->id, a->update);
	a->update = 0; // Don't log this message again

	} else if (a->update) {
	crm_info("Write out of '%s' delayed: update %d in progress", a->id, a->update);
	return;

	} else if (mainloop_timer_running(a->timer)) {
	if (ignore_delay) {
	/* 'refresh' forces a write of the current value of all attributes
	* Cancel any existing timers, we're writing it NOW
	*/
	mainloop_timer_stop(a->timer);
	crm_debug("Write out of '%s': timer is running but ignore delay", a->id);
	} else {
	crm_info("Write out of '%s' delayed: timer is running", a->id);
	return;
	}
	}

	/* Initialize the status update XML */
	xml_top = create_xml_node(NULL, XML_CIB_TAG_STATUS);
	}

	/* Attribute will be written shortly, so clear changed flag */
	a->changed = FALSE;

	/* We will check all peers' uuids shortly, so initialize this to false */
	a->unknown_peer_uuids = FALSE;

	/* Attribute will be written shortly, so clear forced write flag */
	a->force_write = FALSE;

	/* Make the table for the attribute trap */
	alert_attribute_value = g_hash_table_new_full(crm_strcase_hash,
	crm_strcase_equal, NULL,
	free_attribute_value);

	/* Iterate over each peer value of this attribute */
	g_hash_table_iter_init(&iter, a->values);
	while (g_hash_table_iter_next(&iter, NULL, (gpointer *) & v)) {
	crm_node_t *peer = crm_get_peer_full(v->nodeid, v->nodename, CRM_GET_PEER_ANY);

	/* If the value's peer info does not correspond to a peer, ignore it */
	if (peer == NULL) {
	crm_notice("Cannot update %s[%s]=%s because peer not known",
	a->id, v->nodename, v->current);
	continue;
	}

	/* If we're just learning the peer's node id, remember it */
	if (peer->id && (v->nodeid == 0)) {
	crm_trace("Learned ID %u for node %s", peer->id, v->nodename);
	v->nodeid = peer->id;
	}

	/* If this is a private attribute, no update needs to be sent */
	if (a->is_private) {
	private_updates++;
	continue;
	}

	/* If the peer is found, but its uuid is unknown, defer write */
	if (peer->uuid == NULL) {
	a->unknown_peer_uuids = TRUE;
	crm_notice("Cannot update %s[%s]=%s because peer UUID not known "
	"(will retry if learned)",
	a->id, v->nodename, v->current);
	continue;
	}

	/* Add this value to status update XML */
	crm_debug("Updating %s[%s]=%s (peer known as %s, UUID %s, ID %u/%u)",
	a->id, v->nodename, v->current,
	peer->uname, peer->uuid, peer->id, v->nodeid);
	build_update_element(xml_top, a, peer->uuid, v->current);
	cib_updates++;

	/* Preservation of the attribute to transmit alert */
	set_alert_attribute_value(alert_attribute_value, v);

	free(v->requested);
	v->requested = NULL;
	if (v->current) {
	v->requested = strdup(v->current);
	} else {
	/* Older attrd versions don't know about the cib_mixed_update
	* flag so make sure it goes to the local cib which does
	*/
	cib__set_call_options(flags, crm_system_name,
	cib_mixed_update\|cib_scope_local);
	}
	}

	if (private_updates) {
	crm_info("Processed %d private change%s for %s, id=%s, set=%s",
	private_updates, pcmk__plural_s(private_updates),
	a->id, (a->uuid? a->uuid : "n/a"), (a->set? a->set : "n/a"));
	}
	if (cib_updates) {
	crm_log_xml_trace(xml_top, __func__);

	a->update = cib_internal_op(the_cib, CIB_OP_MODIFY, NULL, XML_CIB_TAG_STATUS, xml_top, NULL,
	flags, a->user);

	crm_info("Sent CIB request %d with %d change%s for %s (id %s, set %s)",
	a->update, cib_updates, pcmk__plural_s(cib_updates),
	a->id, (a->uuid? a->uuid : "n/a"), (a->set? a->set : "n/a"));

	the_cib->cmds->register_callback_full(the_cib, a->update,
	CIB_OP_TIMEOUT_S, FALSE,
	strdup(a->id),
	"attrd_cib_callback",
	attrd_cib_callback, free);
	/* Transmit alert of the attribute */
	send_alert_attributes_value(a, alert_attribute_value);
	}

	g_hash_table_destroy(alert_attribute_value);
	free_xml(xml_top);
	}
	diff --git a/daemons/based/based_messages.c b/daemons/based/based_messages.c
	index ffa4529131..c0bd578367 100644
	--- a/daemons/based/based_messages.c
	+++ b/daemons/based/based_messages.c
	@@ -1,540 +1,540 @@
	/*
	* Copyright 2004-2020 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdio.h>
	#include <unistd.h>
	#include <stdlib.h>
	#include <errno.h>
	#include <fcntl.h>
	#include <time.h>

	#include <sys/param.h>
	#include <sys/types.h>

	#include <crm/crm.h>
	#include <crm/cib/internal.h>
	#include <crm/msg_xml.h>

	#include <crm/common/xml.h>
	#include <crm/common/ipc_internal.h>
	#include <crm/common/xml_internal.h>
	#include <crm/cluster/internal.h>

	#include <pacemaker-based.h>

	/* Maximum number of diffs to ignore while waiting for a resync */
	#define MAX_DIFF_RETRY 5

	gboolean cib_is_master = FALSE;

	xmlNode *the_cib = NULL;
	int revision_check(xmlNode * cib_update, xmlNode * cib_copy, int flags);
	int get_revision(xmlNode * xml_obj, int cur_revision);

	int updateList(xmlNode * local_cib, xmlNode * update_command, xmlNode * failed,
	int operation, const char *section);

	gboolean update_results(xmlNode * failed, xmlNode * target, const char *operation, int return_code);

	int cib_update_counter(xmlNode * xml_obj, const char *field, gboolean reset);

	int sync_our_cib(xmlNode * request, gboolean all);

	int
	cib_process_shutdown_req(const char op, int options, const char section, xmlNode * req,
	xmlNode * input, xmlNode * existing_cib, xmlNode ** result_cib,
	xmlNode ** answer)
	{
	const char *host = crm_element_value(req, F_ORIG);

	*answer = NULL;

	if (crm_element_value(req, F_CIB_ISREPLY) == NULL) {
	crm_info("Peer %s is requesting to shut down", host);
	return pcmk_ok;
	}

	if (cib_shutdown_flag == FALSE) {
	crm_err("Peer %s mistakenly thinks we wanted to shut down", host);
	return -EINVAL;
	}

	crm_info("Peer %s has acknowledged our shutdown request", host);
	terminate_cib(__func__, 0);
	return pcmk_ok;
	}

	int
	cib_process_default(const char op, int options, const char section, xmlNode * req,
	xmlNode * input, xmlNode * existing_cib, xmlNode ** result_cib,
	xmlNode ** answer)
	{
	int result = pcmk_ok;

	crm_trace("Processing \"%s\" event", op);
	*answer = NULL;

	if (op == NULL) {
	result = -EINVAL;
	crm_err("No operation specified");

	} else if (strcasecmp(CRM_OP_NOOP, op) == 0) {
	;

	} else {
	result = -EPROTONOSUPPORT;
	crm_err("Action [%s] is not supported by the CIB manager", op);
	}
	return result;
	}

	int
	cib_process_readwrite(const char op, int options, const char section, xmlNode * req,
	xmlNode * input, xmlNode * existing_cib, xmlNode ** result_cib,
	xmlNode ** answer)
	{
	int result = pcmk_ok;

	crm_trace("Processing \"%s\" event", op);

	if (pcmk__str_eq(op, CIB_OP_ISMASTER, pcmk__str_casei)) {
	if (cib_is_master == TRUE) {
	result = pcmk_ok;
	} else {
	result = -EPERM;
	}
	return result;
	}

	if (pcmk__str_eq(op, CIB_OP_MASTER, pcmk__str_casei)) {
	if (cib_is_master == FALSE) {
	crm_info("We are now in R/W mode");
	cib_is_master = TRUE;
	} else {
	crm_debug("We are still in R/W mode");
	}

	} else if (cib_is_master) {
	crm_info("We are now in R/O mode");
	cib_is_master = FALSE;
	}

	return result;
	}

	/* Set to 1 when a sync is requested, incremented when a diff is ignored,
	* reset to 0 when a sync is received
	*/
	static int sync_in_progress = 0;

	void
	send_sync_request(const char *host)
	{
	xmlNode *sync_me = create_xml_node(NULL, "sync-me");

	crm_info("Requesting re-sync from %s", (host? host : "all peers"));
	sync_in_progress = 1;

	crm_xml_add(sync_me, F_TYPE, "cib");
	crm_xml_add(sync_me, F_CIB_OPERATION, CIB_OP_SYNC_ONE);
	crm_xml_add(sync_me, F_CIB_DELEGATED, cib_our_uname);

	send_cluster_message(host ? crm_get_peer(0, host) : NULL, crm_msg_cib, sync_me, FALSE);
	free_xml(sync_me);
	}

	int
	cib_process_ping(const char op, int options, const char section, xmlNode * req, xmlNode * input,
	xmlNode * existing_cib, xmlNode result_cib, xmlNode answer)
	{
	const char *host = crm_element_value(req, F_ORIG);
	const char *seq = crm_element_value(req, F_CIB_PING_ID);
	char *digest = calculate_xml_versioned_digest(the_cib, FALSE, TRUE, CRM_FEATURE_SET);

	static struct qb_log_callsite *cs = NULL;

	crm_trace("Processing \"%s\" event %s from %s", op, seq, host);
	*answer = create_xml_node(NULL, XML_CRM_TAG_PING);

	crm_xml_add(*answer, XML_ATTR_CRM_VERSION, CRM_FEATURE_SET);
	crm_xml_add(*answer, XML_ATTR_DIGEST, digest);
	crm_xml_add(*answer, F_CIB_PING_ID, seq);

	if (cs == NULL) {
	cs = qb_log_callsite_get(__func__, __FILE__, __func__, LOG_TRACE,
	__LINE__, crm_trace_nonlog);
	}
	if (cs && cs->targets) {
	/* Append additional detail so the reciever can log the differences */
	add_message_xml(*answer, F_CIB_CALLDATA, the_cib);

	} else {
	/* Always include at least the version details */
	const char *tag = TYPE(the_cib);
	xmlNode *shallow = create_xml_node(NULL, tag);

	copy_in_properties(shallow, the_cib);
	add_message_xml(*answer, F_CIB_CALLDATA, shallow);
	free_xml(shallow);
	}

	crm_info("Reporting our current digest to %s: %s for %s.%s.%s (%p %d)",
	host, digest,
	crm_element_value(existing_cib, XML_ATTR_GENERATION_ADMIN),
	crm_element_value(existing_cib, XML_ATTR_GENERATION),
	crm_element_value(existing_cib, XML_ATTR_NUMUPDATES),
	existing_cib,
	cs && cs->targets);

	free(digest);

	return pcmk_ok;
	}

	int
	cib_process_sync(const char op, int options, const char section, xmlNode * req, xmlNode * input,
	xmlNode * existing_cib, xmlNode result_cib, xmlNode answer)
	{
	return sync_our_cib(req, TRUE);
	}

	int
	cib_process_upgrade_server(const char op, int options, const char section, xmlNode * req, xmlNode * input,
	xmlNode * existing_cib, xmlNode result_cib, xmlNode answer)
	{
	int rc = pcmk_ok;

	*answer = NULL;

	if(crm_element_value(req, F_CIB_SCHEMA_MAX)) {
	/* The originator of an upgrade request sends it to the DC, without
	* F_CIB_SCHEMA_MAX. If an upgrade is needed, the DC re-broadcasts the
	* request with F_CIB_SCHEMA_MAX, and each node performs the upgrade
	* (and notifies its local clients) here.
	*/
	return cib_process_upgrade(
	op, options, section, req, input, existing_cib, result_cib, answer);

	} else {
	int new_version = 0;
	int current_version = 0;
	xmlNode *scratch = copy_xml(existing_cib);
	const char *host = crm_element_value(req, F_ORIG);
	const char *value = crm_element_value(existing_cib, XML_ATTR_VALIDATION);
	const char *client_id = crm_element_value(req, F_CIB_CLIENTID);
	const char *call_opts = crm_element_value(req, F_CIB_CALLOPTS);
	const char *call_id = crm_element_value(req, F_CIB_CALLID);

	crm_trace("Processing \"%s\" event", op);
	if (value != NULL) {
	current_version = get_schema_version(value);
	}

	rc = update_validation(&scratch, &new_version, 0, TRUE, TRUE);
	if (new_version > current_version) {
	xmlNode *up = create_xml_node(NULL, __func__);

	rc = pcmk_ok;
	crm_notice("Upgrade request from %s verified", host);

	crm_xml_add(up, F_TYPE, "cib");
	crm_xml_add(up, F_CIB_OPERATION, CIB_OP_UPGRADE);
	crm_xml_add(up, F_CIB_SCHEMA_MAX, get_schema_name(new_version));
	crm_xml_add(up, F_CIB_DELEGATED, host);
	crm_xml_add(up, F_CIB_CLIENTID, client_id);
	crm_xml_add(up, F_CIB_CALLOPTS, call_opts);
	crm_xml_add(up, F_CIB_CALLID, call_id);

	if (cib_legacy_mode() && cib_is_master) {
	rc = cib_process_upgrade(
	op, options, section, up, input, existing_cib, result_cib, answer);

	} else {
	send_cluster_message(NULL, crm_msg_cib, up, FALSE);
	}

	free_xml(up);

	} else if(rc == pcmk_ok) {
	rc = -pcmk_err_schema_unchanged;
	}

	if (rc != pcmk_ok) {
	// Notify originating peer so it can notify its local clients
	- crm_node_t *origin = crm_find_peer(0, host);
	+ crm_node_t *origin = pcmk__search_cluster_node_cache(0, host);

	crm_info("Rejecting upgrade request from %s: %s "
	CRM_XS " rc=%d peer=%s", host, pcmk_strerror(rc), rc,
	(origin? origin->uname : "lost"));

	if (origin) {
	xmlNode *up = create_xml_node(NULL, __func__);

	crm_xml_add(up, F_TYPE, "cib");
	crm_xml_add(up, F_CIB_OPERATION, CIB_OP_UPGRADE);
	crm_xml_add(up, F_CIB_DELEGATED, host);
	crm_xml_add(up, F_CIB_ISREPLY, host);
	crm_xml_add(up, F_CIB_CLIENTID, client_id);
	crm_xml_add(up, F_CIB_CALLOPTS, call_opts);
	crm_xml_add(up, F_CIB_CALLID, call_id);
	crm_xml_add_int(up, F_CIB_UPGRADE_RC, rc);
	if (send_cluster_message(origin, crm_msg_cib, up, TRUE)
	== FALSE) {
	crm_warn("Could not send CIB upgrade result to %s", host);
	}
	free_xml(up);
	}
	}
	free_xml(scratch);
	}
	return rc;
	}

	int
	cib_process_sync_one(const char op, int options, const char section, xmlNode * req,
	xmlNode * input, xmlNode * existing_cib, xmlNode ** result_cib,
	xmlNode ** answer)
	{
	return sync_our_cib(req, FALSE);
	}

	int
	cib_server_process_diff(const char op, int options, const char section, xmlNode * req,
	xmlNode * input, xmlNode * existing_cib, xmlNode ** result_cib,
	xmlNode ** answer)
	{
	int rc = pcmk_ok;

	if (sync_in_progress > MAX_DIFF_RETRY) {
	/* Don't ignore diffs forever; the last request may have been lost.
	* If the diff fails, we'll ask for another full resync.
	*/
	sync_in_progress = 0;
	}

	/* The master should never ignore a diff */
	if (sync_in_progress && !cib_is_master) {
	int diff_add_updates = 0;
	int diff_add_epoch = 0;
	int diff_add_admin_epoch = 0;

	int diff_del_updates = 0;
	int diff_del_epoch = 0;
	int diff_del_admin_epoch = 0;

	cib_diff_version_details(input,
	&diff_add_admin_epoch, &diff_add_epoch, &diff_add_updates,
	&diff_del_admin_epoch, &diff_del_epoch, &diff_del_updates);

	sync_in_progress++;
	crm_notice("Not applying diff %d.%d.%d -> %d.%d.%d (sync in progress)",
	diff_del_admin_epoch, diff_del_epoch, diff_del_updates,
	diff_add_admin_epoch, diff_add_epoch, diff_add_updates);
	return -pcmk_err_diff_resync;
	}

	rc = cib_process_diff(op, options, section, req, input, existing_cib, result_cib, answer);
	crm_trace("result: %s (%d), %s", pcmk_strerror(rc), rc, cib_is_master?"master":"slave");

	if (rc == -pcmk_err_diff_resync && cib_is_master == FALSE) {
	free_xml(*result_cib);
	*result_cib = NULL;
	send_sync_request(NULL);

	} else if (rc == -pcmk_err_diff_resync) {
	rc = -pcmk_err_diff_failed;
	if (options & cib_force_diff) {
	crm_warn("Not requesting full refresh in R/W mode");
	}

	} else if ((rc != pcmk_ok) && !cib_is_master && cib_legacy_mode()) {
	crm_warn("Requesting full CIB refresh because update failed: %s"
	CRM_XS " rc=%d", pcmk_strerror(rc), rc);
	xml_log_patchset(LOG_INFO, __func__, input);
	free_xml(*result_cib);
	*result_cib = NULL;
	send_sync_request(NULL);
	}

	return rc;
	}

	int
	cib_process_replace_svr(const char op, int options, const char section, xmlNode * req,
	xmlNode * input, xmlNode * existing_cib, xmlNode ** result_cib,
	xmlNode ** answer)
	{
	const char *tag = crm_element_name(input);
	int rc =
	cib_process_replace(op, options, section, req, input, existing_cib, result_cib, answer);
	if (rc == pcmk_ok && pcmk__str_eq(tag, XML_TAG_CIB, pcmk__str_casei)) {
	sync_in_progress = 0;
	}
	return rc;
	}

	static int
	delete_cib_object(xmlNode * parent, xmlNode * delete_spec)
	{
	const char *object_name = NULL;
	const char *object_id = NULL;
	xmlNode *equiv_node = NULL;
	int result = pcmk_ok;

	if (delete_spec != NULL) {
	object_name = crm_element_name(delete_spec);
	}
	object_id = crm_element_value(delete_spec, XML_ATTR_ID);

	crm_trace("Processing: <%s id=%s>", crm_str(object_name), crm_str(object_id));

	if (delete_spec == NULL) {
	result = -EINVAL;

	} else if (parent == NULL) {
	result = -EINVAL;

	} else if (object_id == NULL) {
	/* placeholder object */
	equiv_node = find_xml_node(parent, object_name, FALSE);

	} else {
	equiv_node = pcmk__xe_match(parent, object_name, XML_ATTR_ID,
	object_id);
	}

	if (result != pcmk_ok) {
	; /* nothing */

	} else if (equiv_node == NULL) {
	result = pcmk_ok;

	} else if (xml_has_children(delete_spec) == FALSE) {
	/* only leaves are deleted */
	crm_debug("Removing leaf: <%s id=%s>", crm_str(object_name), crm_str(object_id));
	free_xml(equiv_node);
	equiv_node = NULL;

	} else {
	xmlNode *child = NULL;

	for (child = pcmk__xml_first_child(delete_spec); child != NULL;
	child = pcmk__xml_next(child)) {
	int tmp_result = delete_cib_object(equiv_node, child);

	/* only the first error is likely to be interesting */
	if (tmp_result != pcmk_ok && result == pcmk_ok) {
	result = tmp_result;
	}
	}
	}

	return result;
	}

	int
	cib_process_delete_absolute(const char op, int options, const char section, xmlNode * req,
	xmlNode * input, xmlNode * existing_cib, xmlNode ** result_cib,
	xmlNode ** answer)
	{
	xmlNode *failed = NULL;
	int result = pcmk_ok;
	xmlNode *update_section = NULL;

	crm_trace("Processing \"%s\" event for section=%s", op, crm_str(section));
	if (pcmk__str_eq(XML_CIB_TAG_SECTION_ALL, section, pcmk__str_casei)) {
	section = NULL;

	} else if (pcmk__str_eq(XML_TAG_CIB, section, pcmk__str_casei)) {
	section = NULL;

	} else if (pcmk__str_eq(crm_element_name(input), XML_TAG_CIB, pcmk__str_casei)) {
	section = NULL;
	}

	CRM_CHECK(strcasecmp(CIB_OP_DELETE, op) == 0, return -EINVAL);

	if (input == NULL) {
	crm_err("Cannot perform modification with no data");
	return -EINVAL;
	}

	failed = create_xml_node(NULL, XML_TAG_FAILED);

	update_section = get_object_root(section, *result_cib);
	result = delete_cib_object(update_section, input);
	update_results(failed, input, op, result);

	if ((result == pcmk_ok) && xml_has_children(failed)) {
	result = -EINVAL;
	}

	if (result != pcmk_ok) {
	crm_log_xml_err(failed, "CIB Update failures");
	*answer = failed;

	} else {
	free_xml(failed);
	}

	return result;
	}

	int
	sync_our_cib(xmlNode * request, gboolean all)
	{
	int result = pcmk_ok;
	char *digest = NULL;
	const char *host = crm_element_value(request, F_ORIG);
	const char *op = crm_element_value(request, F_CIB_OPERATION);

	xmlNode *replace_request = cib_msg_copy(request, FALSE);

	CRM_CHECK(the_cib != NULL,;);
	CRM_CHECK(replace_request != NULL,;);

	crm_debug("Syncing CIB to %s", all ? "all peers" : host);
	if (all == FALSE && host == NULL) {
	crm_log_xml_err(request, "bad sync");
	}

	/* remove the "all == FALSE" condition
	*
	* sync_from was failing, the local client wasn't being notified
	* because it didn't know it was a reply
	* setting this does not prevent the other nodes from applying it
	* if all == TRUE
	*/
	if (host != NULL) {
	crm_xml_add(replace_request, F_CIB_ISREPLY, host);
	}
	if (all) {
	xml_remove_prop(replace_request, F_CIB_HOST);
	}

	crm_xml_add(replace_request, F_CIB_OPERATION, CIB_OP_REPLACE);
	crm_xml_add(replace_request, "original_" F_CIB_OPERATION, op);
	crm_xml_add(replace_request, F_CIB_GLOBAL_UPDATE, XML_BOOLEAN_TRUE);

	crm_xml_add(replace_request, XML_ATTR_CRM_VERSION, CRM_FEATURE_SET);
	digest = calculate_xml_versioned_digest(the_cib, FALSE, TRUE, CRM_FEATURE_SET);
	crm_xml_add(replace_request, XML_ATTR_DIGEST, digest);

	add_message_xml(replace_request, F_CIB_CALLDATA, the_cib);

	if (send_cluster_message
	(all ? NULL : crm_get_peer(0, host), crm_msg_cib, replace_request, FALSE) == FALSE) {
	result = -ENOTCONN;
	}
	free_xml(replace_request);
	free(digest);
	return result;
	}
	diff --git a/daemons/controld/controld_corosync.c b/daemons/controld/controld_corosync.c
	index d104127370..db99630fbc 100644
	--- a/daemons/controld/controld_corosync.c
	+++ b/daemons/controld/controld_corosync.c
	@@ -1,109 +1,110 @@
	/*
	* Copyright 2004-2019 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <sys/param.h>
	#include <sys/types.h>
	#include <sys/stat.h>

	#include <crm/crm.h>
	#include <crm/cluster/internal.h>
	#include <crm/common/xml.h>

	#include <pacemaker-controld.h>

	#if SUPPORT_COROSYNC

	extern void post_cache_update(int seq);

	/* A_HA_CONNECT */

	static void
	crmd_cs_dispatch(cpg_handle_t handle, const struct cpg_name *groupName,
	uint32_t nodeid, uint32_t pid, void *msg, size_t msg_len)
	{
	uint32_t kind = 0;
	const char *from = NULL;
	char *data = pcmk_message_common_cs(handle, nodeid, pid, msg, &kind, &from);

	if(data == NULL) {
	return;
	}
	if (kind == crm_class_cluster) {
	crm_node_t *peer = NULL;
	xmlNode *xml = string2xml(data);

	if (xml == NULL) {
	crm_err("Could not parse message content (%d): %.100s", kind, data);
	free(data);
	return;
	}

	crm_xml_add(xml, F_ORIG, from);
	/* crm_xml_add_int(xml, F_SEQ, wrapper->id); Fake? */

	peer = crm_get_peer(0, from);
	if (!pcmk_is_set(peer->processes, crm_proc_cpg)) {
	/* If we can still talk to our peer process on that node,
	* then it must be part of the corosync membership
	*/
	crm_warn("Receiving messages from a node we think is dead: %s[%d]",
	peer->uname, peer->id);
	crm_update_peer_proc(__func__, peer, crm_proc_cpg,
	ONLINESTATUS);
	}
	crmd_ha_msg_filter(xml);
	free_xml(xml);
	} else {
	crm_err("Invalid message class (%d): %.100s", kind, data);
	}
	free(data);
	}

	static gboolean
	crmd_quorum_callback(unsigned long long seq, gboolean quorate)
	{
	crm_update_quorum(quorate, FALSE);
	post_cache_update(seq);
	return TRUE;
	}

	static void
	crmd_cs_destroy(gpointer user_data)
	{
	if (!pcmk_is_set(fsa_input_register, R_HA_DISCONNECTED)) {
	crm_crit("Lost connection to cluster layer, shutting down");
	crmd_exit(CRM_EX_DISCONNECT);

	} else {
	crm_info("Corosync connection closed");
	}
	}

	extern gboolean crm_connect_corosync(crm_cluster_t * cluster);

	gboolean
	crm_connect_corosync(crm_cluster_t * cluster)
	{
	if (is_corosync_cluster()) {
	crm_set_status_callback(&peer_update_callback);
	cluster->cpg.cpg_deliver_fn = crmd_cs_dispatch;
	cluster->cpg.cpg_confchg_fn = pcmk_cpg_membership;
	cluster->destroy = crmd_cs_destroy;

	if (crm_cluster_connect(cluster)) {
	- cluster_connect_quorum(crmd_quorum_callback, crmd_cs_destroy);
	+ pcmk__corosync_quorum_connect(crmd_quorum_callback,
	+ crmd_cs_destroy);
	return TRUE;
	}
	}
	return FALSE;
	}

	#endif
	diff --git a/daemons/controld/controld_election.c b/daemons/controld/controld_election.c
	index 7e241bdc96..afcdbf27b5 100644
	--- a/daemons/controld/controld_election.c
	+++ b/daemons/controld/controld_election.c
	@@ -1,275 +1,275 @@
	/*
	* Copyright 2004-2020 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <sys/time.h>
	#include <sys/resource.h>

	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>
	#include <crm/cluster/internal.h>
	#include <crm/cluster/election.h>
	#include <crm/crm.h>

	#include <pacemaker-controld.h>

	static election_t *fsa_election = NULL;

	static gboolean
	election_win_cb(gpointer data)
	{
	register_fsa_input(C_FSA_INTERNAL, I_ELECTION_DC, NULL);
	return FALSE;
	}

	void
	controld_election_init(const char *uname)
	{
	fsa_election = election_init("DC", uname, 60000 /60s/, election_win_cb);
	}

	void
	controld_remove_voter(const char *uname)
	{
	election_remove(fsa_election, uname);

	if (pcmk__str_eq(uname, fsa_our_dc, pcmk__str_casei)) {
	/* Clear any election dampening in effect. Otherwise, if the lost DC had
	* just won, an immediate new election could fizzle out with no new DC.
	*/
	election_clear_dampening(fsa_election);
	}
	}

	void
	controld_election_fini()
	{
	election_fini(fsa_election);
	fsa_election = NULL;
	}

	void
	controld_set_election_period(const char *value)
	{
	election_timeout_set_period(fsa_election, crm_parse_interval_spec(value));
	}

	void
	controld_stop_election_timer()
	{
	election_timeout_stop(fsa_election);
	}

	/* A_ELECTION_VOTE */
	void
	do_election_vote(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	gboolean not_voting = FALSE;

	/* don't vote if we're in one of these states or wanting to shut down */
	switch (cur_state) {
	case S_STARTING:
	case S_RECOVERY:
	case S_STOPPING:
	case S_TERMINATE:
	crm_warn("Not voting in election, we're in state %s", fsa_state2string(cur_state));
	not_voting = TRUE;
	break;
	case S_ELECTION:
	case S_INTEGRATION:
	case S_RELEASE_DC:
	break;
	default:
	crm_err("Broken? Voting in state %s", fsa_state2string(cur_state));
	break;
	}

	if (not_voting == FALSE) {
	if (pcmk_is_set(fsa_input_register, R_STARTING)) {
	not_voting = TRUE;
	}
	}

	if (not_voting) {
	if (AM_I_DC) {
	register_fsa_input(C_FSA_INTERNAL, I_RELEASE_DC, NULL);

	} else {
	register_fsa_input(C_FSA_INTERNAL, I_PENDING, NULL);
	}
	return;
	}

	election_vote(fsa_election);
	return;
	}

	void
	do_election_check(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	if (fsa_state == S_ELECTION) {
	election_check(fsa_election);
	} else {
	crm_debug("Ignoring election check because we are not in an election");
	}
	}

	/* A_ELECTION_COUNT */
	void
	do_election_count_vote(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	enum election_result rc = 0;
	ha_msg_input_t *vote = fsa_typed_data(fsa_dt_ha_msg);

	if(crm_peer_cache == NULL) {
	if (!pcmk_is_set(fsa_input_register, R_SHUTDOWN)) {
	crm_err("Internal error, no peer cache");
	}
	return;
	}

	rc = election_count_vote(fsa_election, vote->msg, cur_state != S_STARTING);
	switch(rc) {
	case election_start:
	election_reset(fsa_election);
	register_fsa_input(C_FSA_INTERNAL, I_ELECTION, NULL);
	break;

	case election_lost:
	update_dc(NULL);

	if (fsa_input_register & R_THE_DC) {
	register_fsa_input(C_FSA_INTERNAL, I_RELEASE_DC, NULL);
	fsa_cib_conn->cmds->set_slave(fsa_cib_conn, cib_scope_local);

	} else if (cur_state != S_STARTING) {
	register_fsa_input(C_FSA_INTERNAL, I_PENDING, NULL);
	}
	break;

	default:
	crm_trace("Election message resulted in state %d", rc);
	}
	}

	static void
	feature_update_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
	{
	if (rc != pcmk_ok) {
	fsa_data_t *msg_data = NULL;

	crm_notice("Feature update failed: %s "CRM_XS" rc=%d",
	pcmk_strerror(rc), rc);
	register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
	}
	}

	/* A_DC_TAKEOVER */
	void
	do_dc_takeover(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	int rc = pcmk_ok;
	xmlNode *cib = NULL;
	const char *cluster_type = name_for_cluster_type(get_cluster_type());
	pid_t watchdog = pcmk__locate_sbd();

	crm_info("Taking over DC status for this partition");
	controld_set_fsa_input_flags(R_THE_DC);
	execute_stonith_cleanup();

	election_reset(fsa_election);
	controld_set_fsa_input_flags(R_JOIN_OK\|R_INVOKE_PE);

	fsa_cib_conn->cmds->set_master(fsa_cib_conn, cib_scope_local);

	cib = create_xml_node(NULL, XML_TAG_CIB);
	crm_xml_add(cib, XML_ATTR_CRM_VERSION, CRM_FEATURE_SET);
	fsa_cib_update(XML_TAG_CIB, cib, cib_quorum_override, rc, NULL);
	fsa_register_cib_callback(rc, FALSE, NULL, feature_update_callback);

	update_attr_delegate(fsa_cib_conn, cib_none, XML_CIB_TAG_CRMCONFIG, NULL,
	NULL, NULL, NULL, XML_ATTR_HAVE_WATCHDOG,
	pcmk__btoa(watchdog), FALSE, NULL, NULL);

	update_attr_delegate(fsa_cib_conn, cib_none, XML_CIB_TAG_CRMCONFIG, NULL, NULL, NULL, NULL,
	"dc-version", PACEMAKER_VERSION "-" BUILD_VERSION, FALSE, NULL, NULL);

	update_attr_delegate(fsa_cib_conn, cib_none, XML_CIB_TAG_CRMCONFIG, NULL, NULL, NULL, NULL,
	"cluster-infrastructure", cluster_type, FALSE, NULL, NULL);

	#if SUPPORT_COROSYNC
	if (fsa_cluster_name == NULL && is_corosync_cluster()) {
	- char *cluster_name = corosync_cluster_name();
	+ char *cluster_name = pcmk__corosync_cluster_name();

	if (cluster_name) {
	update_attr_delegate(fsa_cib_conn, cib_none, XML_CIB_TAG_CRMCONFIG, NULL, NULL, NULL, NULL,
	"cluster-name", cluster_name, FALSE, NULL, NULL);
	}
	free(cluster_name);
	}
	#endif

	mainloop_set_trigger(config_read);
	free_xml(cib);
	}

	/* A_DC_RELEASE */
	void
	do_dc_release(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	if (action & A_DC_RELEASE) {
	crm_debug("Releasing the role of DC");
	controld_clear_fsa_input_flags(R_THE_DC);
	controld_expect_sched_reply(NULL);

	} else if (action & A_DC_RELEASED) {
	crm_info("DC role released");
	#if 0
	if (are there errors) {
	/* we can't stay up if not healthy */
	/* or perhaps I_ERROR and go to S_RECOVER? */
	result = I_SHUTDOWN;
	}
	#endif
	if (pcmk_is_set(fsa_input_register, R_SHUTDOWN)) {
	xmlNode *update = NULL;
	crm_node_t *node = crm_get_peer(0, fsa_our_uname);

	- crm_update_peer_expected(__func__, node, CRMD_JOINSTATE_DOWN);
	+ pcmk__update_peer_expected(__func__, node, CRMD_JOINSTATE_DOWN);
	update = create_node_state_update(node, node_update_expected, NULL,
	__func__);
	/* Don't need a based response because controld will stop. */
	fsa_cib_anon_update_discard_reply(XML_CIB_TAG_STATUS, update);
	free_xml(update);
	}
	register_fsa_input(C_FSA_INTERNAL, I_RELEASE_SUCCESS, NULL);

	} else {
	crm_err("Unknown DC action %s", fsa_action2string(action));
	}

	crm_trace("Am I still the DC? %s", AM_I_DC ? XML_BOOLEAN_YES : XML_BOOLEAN_NO);

	}
	diff --git a/daemons/controld/controld_fencing.c b/daemons/controld/controld_fencing.c
	index e2c318243b..dcc23b4c91 100644
	--- a/daemons/controld/controld_fencing.c
	+++ b/daemons/controld/controld_fencing.c
	@@ -1,968 +1,969 @@
	/*
	* Copyright 2004-2020 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>
	#include <crm/crm.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>
	#include <crm/fencing/internal.h>

	#include <pacemaker-controld.h>

	static void
	tengine_stonith_history_synced(stonith_t st, stonith_event_t st_event);

	/*
	* stonith failure counting
	*
	* We don't want to get stuck in a permanent fencing loop. Keep track of the
	* number of fencing failures for each target node, and the most we'll restart a
	* transition for.
	*/

	struct st_fail_rec {
	int count;
	};

	static bool fence_reaction_panic = FALSE;
	static unsigned long int stonith_max_attempts = 10;
	static GHashTable *stonith_failures = NULL;

	// crmd_opts defines default for stonith-max-attempts, so value is never NULL
	void
	update_stonith_max_attempts(const char *value)
	{
	if (pcmk__str_eq(value, CRM_INFINITY_S, pcmk__str_casei)) {
	stonith_max_attempts = CRM_SCORE_INFINITY;
	} else {
	stonith_max_attempts = (unsigned long int) crm_parse_ll(value, NULL);
	}
	}

	void
	set_fence_reaction(const char *reaction_s)
	{
	if (pcmk__str_eq(reaction_s, "panic", pcmk__str_casei)) {
	fence_reaction_panic = TRUE;

	} else {
	if (!pcmk__str_eq(reaction_s, "stop", pcmk__str_casei)) {
	crm_warn("Invalid value '%s' for %s, using 'stop'",
	reaction_s, XML_CONFIG_ATTR_FENCE_REACTION);
	}
	fence_reaction_panic = FALSE;
	}
	}

	static gboolean
	too_many_st_failures(const char *target)
	{
	GHashTableIter iter;
	const char *key = NULL;
	struct st_fail_rec *value = NULL;

	if (stonith_failures == NULL) {
	return FALSE;
	}

	if (target == NULL) {
	g_hash_table_iter_init(&iter, stonith_failures);
	while (g_hash_table_iter_next(&iter, (gpointer *) &key,
	(gpointer *) &value)) {

	if (value->count >= stonith_max_attempts) {
	target = (const char*)key;
	goto too_many;
	}
	}
	} else {
	value = g_hash_table_lookup(stonith_failures, target);
	if ((value != NULL) && (value->count >= stonith_max_attempts)) {
	goto too_many;
	}
	}
	return FALSE;

	too_many:
	crm_warn("Too many failures (%d) to fence %s, giving up",
	value->count, target);
	return TRUE;
	}

	/*!
	* \internal
	* \brief Reset a stonith fail count
	*
	* \param[in] target Name of node to reset, or NULL for all
	*/
	void
	st_fail_count_reset(const char *target)
	{
	if (stonith_failures == NULL) {
	return;
	}

	if (target) {
	struct st_fail_rec *rec = NULL;

	rec = g_hash_table_lookup(stonith_failures, target);
	if (rec) {
	rec->count = 0;
	}
	} else {
	GHashTableIter iter;
	const char *key = NULL;
	struct st_fail_rec *rec = NULL;

	g_hash_table_iter_init(&iter, stonith_failures);
	while (g_hash_table_iter_next(&iter, (gpointer *) &key,
	(gpointer *) &rec)) {
	rec->count = 0;
	}
	}
	}

	static void
	st_fail_count_increment(const char *target)
	{
	struct st_fail_rec *rec = NULL;

	if (stonith_failures == NULL) {
	stonith_failures = crm_str_table_new();
	}

	rec = g_hash_table_lookup(stonith_failures, target);
	if (rec) {
	rec->count++;
	} else {
	rec = malloc(sizeof(struct st_fail_rec));
	if(rec == NULL) {
	return;
	}

	rec->count = 1;
	g_hash_table_insert(stonith_failures, strdup(target), rec);
	}
	}

	/* end stonith fail count functions */


	static void
	cib_fencing_updated(xmlNode msg, int call_id, int rc, xmlNode output,
	void *user_data)
	{
	if (rc < pcmk_ok) {
	crm_err("Fencing update %d for %s: failed - %s (%d)",
	call_id, (char *)user_data, pcmk_strerror(rc), rc);
	crm_log_xml_warn(msg, "Failed update");
	abort_transition(INFINITY, tg_shutdown, "CIB update failed", NULL);

	} else {
	crm_info("Fencing update %d for %s: complete", call_id, (char *)user_data);
	}
	}

	static void
	send_stonith_update(crm_action_t action, const char target, const char *uuid)
	{
	int rc = pcmk_ok;
	crm_node_t *peer = NULL;

	/* We (usually) rely on the membership layer to do node_update_cluster,
	* and the peer status callback to do node_update_peer, because the node
	* might have already rejoined before we get the stonith result here.
	*/
	int flags = node_update_join \| node_update_expected;

	/* zero out the node-status & remove all LRM status info */
	xmlNode *node_state = NULL;

	CRM_CHECK(target != NULL, return);
	CRM_CHECK(uuid != NULL, return);

	/* Make sure the membership and join caches are accurate */
	peer = crm_get_peer_full(0, target, CRM_GET_PEER_ANY);

	CRM_CHECK(peer != NULL, return);

	if (peer->state == NULL) {
	/* Usually, we rely on the membership layer to update the cluster state
	* in the CIB. However, if the node has never been seen, do it here, so
	* the node is not considered unclean.
	*/
	flags \|= node_update_cluster;
	}

	if (peer->uuid == NULL) {
	crm_info("Recording uuid '%s' for node '%s'", uuid, target);
	peer->uuid = strdup(uuid);
	}

	crmd_peer_down(peer, TRUE);

	/* Generate a node state update for the CIB */
	node_state = create_node_state_update(peer, flags, NULL, __func__);

	/* we have to mark whether or not remote nodes have already been fenced */
	if (peer->flags & crm_remote_node) {
	time_t now = time(NULL);
	char *now_s = crm_itoa(now);
	crm_xml_add(node_state, XML_NODE_IS_FENCED, now_s);
	free(now_s);
	}

	/* Force our known ID */
	crm_xml_add(node_state, XML_ATTR_UUID, uuid);

	rc = fsa_cib_conn->cmds->update(fsa_cib_conn, XML_CIB_TAG_STATUS, node_state,
	cib_quorum_override \| cib_scope_local \| cib_can_create);

	/* Delay processing the trigger until the update completes */
	crm_debug("Sending fencing update %d for %s", rc, target);
	fsa_register_cib_callback(rc, FALSE, strdup(target), cib_fencing_updated);

	/* Make sure it sticks */
	/* fsa_cib_conn->cmds->bump_epoch(fsa_cib_conn, cib_quorum_override\|cib_scope_local); */

	controld_delete_node_state(peer->uname, controld_section_all,
	cib_scope_local);
	free_xml(node_state);
	return;
	}

	/*!
	* \internal
	* \brief Abort transition due to stonith failure
	*
	* \param[in] abort_action Whether to restart or stop transition
	* \param[in] target Don't restart if this (NULL for any) has too many failures
	* \param[in] reason Log this stonith action XML as abort reason (or NULL)
	*/
	static void
	abort_for_stonith_failure(enum transition_action abort_action,
	const char target, xmlNode reason)
	{
	/* If stonith repeatedly fails, we eventually give up on starting a new
	* transition for that reason.
	*/
	if ((abort_action != tg_stop) && too_many_st_failures(target)) {
	abort_action = tg_stop;
	}
	abort_transition(INFINITY, abort_action, "Stonith failed", reason);
	}


	/*
	* stonith cleanup list
	*
	* If the DC is shot, proper notifications might not go out.
	* The stonith cleanup list allows the cluster to (re-)send
	* notifications once a new DC is elected.
	*/

	static GListPtr stonith_cleanup_list = NULL;

	/*!
	* \internal
	* \brief Add a node to the stonith cleanup list
	*
	* \param[in] target Name of node to add
	*/
	void
	add_stonith_cleanup(const char *target) {
	stonith_cleanup_list = g_list_append(stonith_cleanup_list, strdup(target));
	}

	/*!
	* \internal
	* \brief Remove a node from the stonith cleanup list
	*
	* \param[in] Name of node to remove
	*/
	void
	remove_stonith_cleanup(const char *target)
	{
	GListPtr iter = stonith_cleanup_list;

	while (iter != NULL) {
	GListPtr tmp = iter;
	char *iter_name = tmp->data;

	iter = iter->next;
	if (pcmk__str_eq(target, iter_name, pcmk__str_casei)) {
	crm_trace("Removing %s from the cleanup list", iter_name);
	stonith_cleanup_list = g_list_delete_link(stonith_cleanup_list, tmp);
	free(iter_name);
	}
	}
	}

	/*!
	* \internal
	* \brief Purge all entries from the stonith cleanup list
	*/
	void
	purge_stonith_cleanup()
	{
	if (stonith_cleanup_list) {
	GListPtr iter = NULL;

	for (iter = stonith_cleanup_list; iter != NULL; iter = iter->next) {
	char *target = iter->data;

	crm_info("Purging %s from stonith cleanup list", target);
	free(target);
	}
	g_list_free(stonith_cleanup_list);
	stonith_cleanup_list = NULL;
	}
	}

	/*!
	* \internal
	* \brief Send stonith updates for all entries in cleanup list, then purge it
	*/
	void
	execute_stonith_cleanup()
	{
	GListPtr iter;

	for (iter = stonith_cleanup_list; iter != NULL; iter = iter->next) {
	char *target = iter->data;
	crm_node_t *target_node = crm_get_peer(0, target);
	const char *uuid = crm_peer_uuid(target_node);

	crm_notice("Marking %s, target of a previous stonith action, as clean", target);
	send_stonith_update(NULL, target, uuid);
	free(target);
	}
	g_list_free(stonith_cleanup_list);
	stonith_cleanup_list = NULL;
	}

	/* end stonith cleanup list functions */


	/* stonith API client
	*
	* Functions that need to interact directly with the fencer via its API
	*/

	static stonith_t *stonith_api = NULL;
	static crm_trigger_t *stonith_reconnect = NULL;
	static char *te_client_id = NULL;

	static gboolean
	fail_incompletable_stonith(crm_graph_t *graph)
	{
	GListPtr lpc = NULL;
	const char *task = NULL;
	xmlNode *last_action = NULL;

	if (graph == NULL) {
	return FALSE;
	}

	for (lpc = graph->synapses; lpc != NULL; lpc = lpc->next) {
	GListPtr lpc2 = NULL;
	synapse_t synapse = (synapse_t ) lpc->data;

	if (synapse->confirmed) {
	continue;
	}

	for (lpc2 = synapse->actions; lpc2 != NULL; lpc2 = lpc2->next) {
	crm_action_t action = (crm_action_t ) lpc2->data;

	if (action->type != action_type_crm \|\| action->confirmed) {
	continue;
	}

	task = crm_element_value(action->xml, XML_LRM_ATTR_TASK);
	if (task && pcmk__str_eq(task, CRM_OP_FENCE, pcmk__str_casei)) {
	action->failed = TRUE;
	last_action = action->xml;
	update_graph(graph, action);
	crm_notice("Failing action %d (%s): fencer terminated",
	action->id, ID(action->xml));
	}
	}
	}

	if (last_action != NULL) {
	crm_warn("Fencer failure resulted in unrunnable actions");
	abort_for_stonith_failure(tg_restart, NULL, last_action);
	return TRUE;
	}

	return FALSE;
	}

	static void
	tengine_stonith_connection_destroy(stonith_t st, stonith_event_t e)
	{
	te_cleanup_stonith_history_sync(st, FALSE);

	if (pcmk_is_set(fsa_input_register, R_ST_REQUIRED)) {
	crm_crit("Fencing daemon connection failed");
	mainloop_set_trigger(stonith_reconnect);

	} else {
	crm_info("Fencing daemon disconnected");
	}

	if (stonith_api) {
	/* the client API won't properly reconnect notifications
	* if they are still in the table - so remove them
	*/
	if (stonith_api->state != stonith_disconnected) {
	stonith_api->cmds->disconnect(st);
	}
	stonith_api->cmds->remove_notification(stonith_api, T_STONITH_NOTIFY_DISCONNECT);
	stonith_api->cmds->remove_notification(stonith_api, T_STONITH_NOTIFY_FENCE);
	stonith_api->cmds->remove_notification(stonith_api, T_STONITH_NOTIFY_HISTORY_SYNCED);
	}

	if (AM_I_DC) {
	fail_incompletable_stonith(transition_graph);
	trigger_graph();
	}
	}

	static void
	tengine_stonith_notify(stonith_t st, stonith_event_t st_event)
	{
	if (te_client_id == NULL) {
	te_client_id = crm_strdup_printf("%s.%lu", crm_system_name,
	(unsigned long) getpid());
	}

	if (st_event == NULL) {
	crm_err("Notify data not found");
	return;
	}

	crmd_alert_fencing_op(st_event);

	if ((st_event->result == pcmk_ok) && pcmk__str_eq("on", st_event->action, pcmk__str_casei)) {
	crm_notice("%s was successfully unfenced by %s (at the request of %s)",
	st_event->target,
	st_event->executioner? st_event->executioner : "<anyone>",
	st_event->origin);
	/* TODO: Hook up st_event->device */
	return;

	} else if (pcmk__str_eq("on", st_event->action, pcmk__str_casei)) {
	crm_err("Unfencing of %s by %s failed: %s (%d)",
	st_event->target,
	st_event->executioner? st_event->executioner : "<anyone>",
	pcmk_strerror(st_event->result), st_event->result);
	return;

	} else if ((st_event->result == pcmk_ok)
	&& pcmk__str_eq(st_event->target, fsa_our_uname, pcmk__str_none)) {

	/* We were notified of our own fencing. Most likely, either fencing was
	* misconfigured, or fabric fencing that doesn't cut cluster
	* communication is in use.
	*
	* Either way, shutting down the local host is a good idea, to require
	* administrator intervention. Also, other nodes would otherwise likely
	* set our status to lost because of the fencing callback and discard
	* our subsequent election votes as "not part of our cluster".
	*/
	crm_crit("We were allegedly just fenced by %s for %s!",
	st_event->executioner? st_event->executioner : "the cluster",
	st_event->origin); /* Dumps blackbox if enabled */
	if (fence_reaction_panic) {
	pcmk__panic(__func__);
	} else {
	crm_exit(CRM_EX_FATAL);
	}
	return;
	}

	/* Update the count of stonith failures for this target, in case we become
	* DC later. The current DC has already updated its fail count in
	* tengine_stonith_callback().
	*/
	if (!AM_I_DC && pcmk__str_eq(st_event->operation, T_STONITH_NOTIFY_FENCE, pcmk__str_casei)) {
	if (st_event->result == pcmk_ok) {
	st_fail_count_reset(st_event->target);
	} else {
	st_fail_count_increment(st_event->target);
	}
	}

	crm_notice("Peer %s was%s terminated (%s) by %s on behalf of %s: %s "
	CRM_XS " initiator=%s ref=%s",
	st_event->target, st_event->result == pcmk_ok ? "" : " not",
	st_event->action,
	st_event->executioner ? st_event->executioner : "<anyone>",
	(st_event->client_origin? st_event->client_origin : "<unknown>"),
	pcmk_strerror(st_event->result),
	st_event->origin, st_event->id);

	if (st_event->result == pcmk_ok) {
	- crm_node_t *peer = crm_find_known_peer_full(0, st_event->target, CRM_GET_PEER_ANY);
	+ crm_node_t *peer = pcmk__search_known_node_cache(0, st_event->target,
	+ CRM_GET_PEER_ANY);
	const char *uuid = NULL;
	gboolean we_are_executioner = pcmk__str_eq(st_event->executioner,
	fsa_our_uname,
	pcmk__str_casei);

	if (peer == NULL) {
	return;
	}

	uuid = crm_peer_uuid(peer);

	crm_trace("target=%s dc=%s", st_event->target, fsa_our_dc);
	if(AM_I_DC) {
	/* The DC always sends updates */
	send_stonith_update(NULL, st_event->target, uuid);

	/* @TODO Ideally, at this point, we'd check whether the fenced node
	* hosted any guest nodes, and call remote_node_down() for them.
	* Unfortunately, the controller doesn't have a simple, reliable way
	* to map hosts to guests. It might be possible to track this in the
	* peer cache via crm_remote_peer_cache_refresh(). For now, we rely
	* on the scheduler creating fence pseudo-events for the guests.
	*/

	if (st_event->client_origin
	&& !pcmk__str_eq(st_event->client_origin, te_client_id, pcmk__str_casei)) {

	/* Abort the current transition graph if it wasn't us
	* that invoked stonith to fence someone
	*/
	crm_info("External fencing operation from %s fenced %s", st_event->client_origin, st_event->target);
	abort_transition(INFINITY, tg_restart, "External Fencing Operation", NULL);
	}

	/* Assume it was our leader if we don't currently have one */
	} else if (pcmk__str_eq(fsa_our_dc, st_event->target, pcmk__str_null_matches \| pcmk__str_casei)
	&& !pcmk_is_set(peer->flags, crm_remote_node)) {

	crm_notice("Fencing target %s %s our leader",
	st_event->target, (fsa_our_dc? "was" : "may have been"));

	/* Given the CIB resyncing that occurs around elections,
	* have one node update the CIB now and, if the new DC is different,
	* have them do so too after the election
	*/
	if (we_are_executioner) {
	send_stonith_update(NULL, st_event->target, uuid);
	}
	add_stonith_cleanup(st_event->target);
	}

	/* If the target is a remote node, and we host its connection,
	* immediately fail all monitors so it can be recovered quickly.
	* The connection won't necessarily drop when a remote node is fenced,
	* so the failure might not otherwise be detected until the next poke.
	*/
	if (pcmk_is_set(peer->flags, crm_remote_node)) {
	remote_ra_fail(st_event->target);
	}

	crmd_peer_down(peer, TRUE);
	}
	}

	/*!
	* \brief Connect to fencer
	*
	* \param[in] user_data If NULL, retry failures now, otherwise retry in main loop
	*
	* \return TRUE
	* \note If user_data is NULL, this will wait 2s between attempts, for up to
	* 30 attempts, meaning the controller could be blocked as long as 58s.
	*/
	static gboolean
	te_connect_stonith(gpointer user_data)
	{
	int rc = pcmk_ok;

	if (stonith_api == NULL) {
	stonith_api = stonith_api_new();
	if (stonith_api == NULL) {
	crm_err("Could not connect to fencer: API memory allocation failed");
	return TRUE;
	}
	}

	if (stonith_api->state != stonith_disconnected) {
	crm_trace("Already connected to fencer, no need to retry");
	return TRUE;
	}

	if (user_data == NULL) {
	// Blocking (retry failures now until successful)
	rc = stonith_api_connect_retry(stonith_api, crm_system_name, 30);
	if (rc != pcmk_ok) {
	crm_err("Could not connect to fencer in 30 attempts: %s "
	CRM_XS " rc=%d", pcmk_strerror(rc), rc);
	}
	} else {
	// Non-blocking (retry failures later in main loop)
	rc = stonith_api->cmds->connect(stonith_api, crm_system_name, NULL);
	if (rc != pcmk_ok) {
	if (pcmk_is_set(fsa_input_register, R_ST_REQUIRED)) {
	crm_notice("Fencer connection failed (will retry): %s "
	CRM_XS " rc=%d", pcmk_strerror(rc), rc);
	mainloop_set_trigger(stonith_reconnect);
	} else {
	crm_info("Fencer connection failed (ignoring because no longer required): %s "
	CRM_XS " rc=%d", pcmk_strerror(rc), rc);
	}
	return TRUE;
	}
	}

	if (rc == pcmk_ok) {
	stonith_api->cmds->register_notification(stonith_api,
	T_STONITH_NOTIFY_DISCONNECT,
	tengine_stonith_connection_destroy);
	stonith_api->cmds->register_notification(stonith_api,
	T_STONITH_NOTIFY_FENCE,
	tengine_stonith_notify);
	stonith_api->cmds->register_notification(stonith_api,
	T_STONITH_NOTIFY_HISTORY_SYNCED,
	tengine_stonith_history_synced);
	te_trigger_stonith_history_sync(TRUE);
	crm_notice("Fencer successfully connected");
	}

	return TRUE;
	}

	/*!
	\internal
	\brief Schedule fencer connection attempt in main loop
	*/
	void
	controld_trigger_fencer_connect()
	{
	if (stonith_reconnect == NULL) {
	stonith_reconnect = mainloop_add_trigger(G_PRIORITY_LOW,
	te_connect_stonith,
	GINT_TO_POINTER(TRUE));
	}
	controld_set_fsa_input_flags(R_ST_REQUIRED);
	mainloop_set_trigger(stonith_reconnect);
	}

	void
	controld_disconnect_fencer(bool destroy)
	{
	if (stonith_api) {
	// Prevent fencer connection from coming up again
	controld_clear_fsa_input_flags(R_ST_REQUIRED);

	if (stonith_api->state != stonith_disconnected) {
	stonith_api->cmds->disconnect(stonith_api);
	}
	stonith_api->cmds->remove_notification(stonith_api, T_STONITH_NOTIFY_DISCONNECT);
	stonith_api->cmds->remove_notification(stonith_api, T_STONITH_NOTIFY_FENCE);
	stonith_api->cmds->remove_notification(stonith_api, T_STONITH_NOTIFY_HISTORY_SYNCED);
	}
	if (destroy) {
	if (stonith_api) {
	stonith_api->cmds->free(stonith_api);
	stonith_api = NULL;
	}
	if (stonith_reconnect) {
	mainloop_destroy_trigger(stonith_reconnect);
	stonith_reconnect = NULL;
	}
	if (te_client_id) {
	free(te_client_id);
	te_client_id = NULL;
	}
	}
	}

	static gboolean
	do_stonith_history_sync(gpointer user_data)
	{
	if (stonith_api && (stonith_api->state != stonith_disconnected)) {
	stonith_history_t *history = NULL;

	te_cleanup_stonith_history_sync(stonith_api, FALSE);
	stonith_api->cmds->history(stonith_api,
	st_opt_sync_call \| st_opt_broadcast,
	NULL, &history, 5);
	stonith_history_free(history);
	return TRUE;
	} else {
	crm_info("Skip triggering stonith history-sync as stonith is disconnected");
	return FALSE;
	}
	}

	static void
	tengine_stonith_callback(stonith_t stonith, stonith_callback_data_t data)
	{
	char *uuid = NULL;
	int stonith_id = -1;
	int transition_id = -1;
	crm_action_t *action = NULL;
	int call_id = data->call_id;
	int rc = data->rc;
	char *userdata = data->userdata;

	CRM_CHECK(userdata != NULL, return);
	crm_notice("Stonith operation %d/%s: %s (%d)", call_id, (char *)userdata,
	pcmk_strerror(rc), rc);

	if (AM_I_DC == FALSE) {
	return;
	}

	/* crm_info("call=%d, optype=%d, node_name=%s, result=%d, node_list=%s, action=%s", */
	/* op->call_id, op->optype, op->node_name, op->op_result, */
	/* (char )op->node_list, op->private_data); /

	/* filter out old STONITH actions */
	CRM_CHECK(decode_transition_key(userdata, &uuid, &transition_id, &stonith_id, NULL),
	goto bail);

	if (transition_graph->complete \|\| stonith_id < 0 \|\| !pcmk__str_eq(uuid, te_uuid, pcmk__str_casei)
	\|\| transition_graph->id != transition_id) {
	crm_info("Ignoring STONITH action initiated outside of the current transition");
	goto bail;
	}

	action = controld_get_action(stonith_id);
	if (action == NULL) {
	crm_err("Stonith action not matched");
	goto bail;
	}

	stop_te_timer(action->timer);
	if (rc == pcmk_ok) {
	const char *target = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
	const char *uuid = crm_element_value(action->xml, XML_LRM_ATTR_TARGET_UUID);
	const char *op = crm_meta_value(action->params, "stonith_action");

	crm_info("Stonith operation %d for %s passed", call_id, target);
	if (action->confirmed == FALSE) {
	te_action_confirmed(action, NULL);
	if (pcmk__str_eq("on", op, pcmk__str_casei)) {
	const char *value = NULL;
	char *now = crm_ttoa(time(NULL));

	update_attrd(target, CRM_ATTR_UNFENCED, now, NULL, FALSE);
	free(now);

	value = crm_meta_value(action->params, XML_OP_ATTR_DIGESTS_ALL);
	update_attrd(target, CRM_ATTR_DIGESTS_ALL, value, NULL, FALSE);

	value = crm_meta_value(action->params, XML_OP_ATTR_DIGESTS_SECURE);
	update_attrd(target, CRM_ATTR_DIGESTS_SECURE, value, NULL, FALSE);

	} else if (action->sent_update == FALSE) {
	send_stonith_update(action, target, uuid);
	action->sent_update = TRUE;
	}
	}
	st_fail_count_reset(target);

	} else {
	const char *target = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
	enum transition_action abort_action = tg_restart;

	action->failed = TRUE;
	crm_notice("Stonith operation %d for %s failed (%s): aborting transition.",
	call_id, target, pcmk_strerror(rc));

	/* If no fence devices were available, there's no use in immediately
	* checking again, so don't start a new transition in that case.
	*/
	if (rc == -ENODEV) {
	crm_warn("No devices found in cluster to fence %s, giving up",
	target);
	abort_action = tg_stop;
	}

	/* Increment the fail count now, so abort_for_stonith_failure() can
	* check it. Non-DC nodes will increment it in tengine_stonith_notify().
	*/
	st_fail_count_increment(target);
	abort_for_stonith_failure(abort_action, target, NULL);
	}

	update_graph(transition_graph, action);
	trigger_graph();

	bail:
	free(userdata);
	free(uuid);
	return;
	}

	static int
	fence_with_delay(const char target, const char type, const char *delay)
	{
	uint32_t options = st_opt_none; // Group of enum stonith_call_options
	int timeout_sec = (int) (transition_graph->stonith_timeout / 1000);

	if (crmd_join_phase_count(crm_join_confirmed) == 1) {
	stonith__set_call_options(options, target, st_opt_allow_suicide);
	}
	return stonith_api->cmds->fence_with_delay(stonith_api, options, target,
	type, timeout_sec, 0,
	crm_atoi(delay, "0"));
	}

	gboolean
	te_fence_node(crm_graph_t graph, crm_action_t action)
	{
	int rc = 0;
	const char *id = NULL;
	const char *uuid = NULL;
	const char *target = NULL;
	const char *type = NULL;
	char *transition_key = NULL;
	const char *priority_delay = NULL;
	gboolean invalid_action = FALSE;

	id = ID(action->xml);
	target = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
	uuid = crm_element_value(action->xml, XML_LRM_ATTR_TARGET_UUID);
	type = crm_meta_value(action->params, "stonith_action");

	CRM_CHECK(id != NULL, invalid_action = TRUE);
	CRM_CHECK(uuid != NULL, invalid_action = TRUE);
	CRM_CHECK(type != NULL, invalid_action = TRUE);
	CRM_CHECK(target != NULL, invalid_action = TRUE);

	if (invalid_action) {
	crm_log_xml_warn(action->xml, "BadAction");
	return FALSE;
	}

	priority_delay = crm_meta_value(action->params, XML_CONFIG_ATTR_PRIORITY_FENCING_DELAY);

	crm_notice("Requesting fencing (%s) of node %s "
	CRM_XS " action=%s timeout=%u%s%s",
	type, target, id, transition_graph->stonith_timeout,
	priority_delay ? " priority_delay=" : "",
	priority_delay ? priority_delay : "");

	/* Passing NULL means block until we can connect... */
	te_connect_stonith(NULL);

	rc = fence_with_delay(target, type, priority_delay);
	transition_key = pcmk__transition_key(transition_graph->id, action->id, 0,
	te_uuid),
	stonith_api->cmds->register_callback(stonith_api, rc,
	(int) (transition_graph->stonith_timeout / 1000),
	st_opt_timeout_updates, transition_key,
	"tengine_stonith_callback", tengine_stonith_callback);

	return TRUE;
	}

	/* end stonith API client functions */


	/*
	* stonith history synchronization
	*
	* Each node's fencer keeps track of a cluster-wide fencing history. When a node
	* joins or leaves, we need to synchronize the history across all nodes.
	*/

	static crm_trigger_t *stonith_history_sync_trigger = NULL;
	static mainloop_timer_t *stonith_history_sync_timer_short = NULL;
	static mainloop_timer_t *stonith_history_sync_timer_long = NULL;

	void
	te_cleanup_stonith_history_sync(stonith_t *st, bool free_timers)
	{
	if (free_timers) {
	mainloop_timer_del(stonith_history_sync_timer_short);
	stonith_history_sync_timer_short = NULL;
	mainloop_timer_del(stonith_history_sync_timer_long);
	stonith_history_sync_timer_long = NULL;
	} else {
	mainloop_timer_stop(stonith_history_sync_timer_short);
	mainloop_timer_stop(stonith_history_sync_timer_long);
	}

	if (st) {
	st->cmds->remove_notification(st, T_STONITH_NOTIFY_HISTORY_SYNCED);
	}
	}

	static void
	tengine_stonith_history_synced(stonith_t st, stonith_event_t st_event)
	{
	te_cleanup_stonith_history_sync(st, FALSE);
	crm_debug("Fence-history synced - cancel all timers");
	}

	static gboolean
	stonith_history_sync_set_trigger(gpointer user_data)
	{
	mainloop_set_trigger(stonith_history_sync_trigger);
	return FALSE;
	}

	void
	te_trigger_stonith_history_sync(bool long_timeout)
	{
	/* trigger a sync in 5s to give more nodes the
	* chance to show up so that we don't create
	* unnecessary stonith-history-sync traffic
	*
	* the long timeout of 30s is there as a fallback
	* so that after a successful connection to fenced
	* we will wait for 30s for the DC to trigger a
	* history-sync
	* if this doesn't happen we trigger a sync locally
	* (e.g. fenced segfaults and is restarted by pacemakerd)
	*/

	/* as we are finally checking the stonith-connection
	* in do_stonith_history_sync we should be fine
	* leaving stonith_history_sync_time & stonith_history_sync_trigger
	* around
	*/
	if (stonith_history_sync_trigger == NULL) {
	stonith_history_sync_trigger =
	mainloop_add_trigger(G_PRIORITY_LOW,
	do_stonith_history_sync, NULL);
	}

	if (long_timeout) {
	if(stonith_history_sync_timer_long == NULL) {
	stonith_history_sync_timer_long =
	mainloop_timer_add("history_sync_long", 30000,
	FALSE, stonith_history_sync_set_trigger,
	NULL);
	}
	crm_info("Fence history will be synchronized cluster-wide within 30 seconds");
	mainloop_timer_start(stonith_history_sync_timer_long);
	} else {
	if(stonith_history_sync_timer_short == NULL) {
	stonith_history_sync_timer_short =
	mainloop_timer_add("history_sync_short", 5000,
	FALSE, stonith_history_sync_set_trigger,
	NULL);
	}
	crm_info("Fence history will be synchronized cluster-wide within 5 seconds");
	mainloop_timer_start(stonith_history_sync_timer_short);
	}

	}

	/* end stonith history synchronization functions */
	diff --git a/daemons/controld/controld_join_client.c b/daemons/controld/controld_join_client.c
	index 0499a25b64..c62170d94e 100644
	--- a/daemons/controld/controld_join_client.c
	+++ b/daemons/controld/controld_join_client.c
	@@ -1,312 +1,312 @@
	/*
	* Copyright 2004-2020 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <crm/crm.h>
	#include <crm/cib.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>

	#include <pacemaker-controld.h>

	int reannounce_count = 0;
	void join_query_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data);

	extern ha_msg_input_t copy_ha_msg_input(ha_msg_input_t orig);

	/*!
	* \internal
	* \brief Remember if DC is shutting down as we join
	*
	* If we're joining while the current DC is shutting down, update its expected
	* state, so we don't fence it if we become the new DC. (We weren't a peer
	* when it broadcast its shutdown request.)
	*
	* \param[in] msg A join message from the DC
	*/
	static void
	update_dc_expected(xmlNode *msg)
	{
	if (fsa_our_dc && crm_is_true(crm_element_value(msg, F_CRM_DC_LEAVING))) {
	crm_node_t *dc_node = crm_get_peer(0, fsa_our_dc);

	- crm_update_peer_expected(__func__, dc_node, CRMD_JOINSTATE_DOWN);
	+ pcmk__update_peer_expected(__func__, dc_node, CRMD_JOINSTATE_DOWN);
	}
	}

	/* A_CL_JOIN_QUERY */
	/* is there a DC out there? */
	void
	do_cl_join_query(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	xmlNode *req = create_request(CRM_OP_JOIN_ANNOUNCE, NULL, NULL,
	CRM_SYSTEM_DC, CRM_SYSTEM_CRMD, NULL);

	sleep(1); // Give the cluster layer time to propagate to the DC
	update_dc(NULL); /* Unset any existing value so that the result is not discarded */
	crm_debug("Querying for a DC");
	send_cluster_message(NULL, crm_msg_crmd, req, FALSE);
	free_xml(req);
	}

	/* A_CL_JOIN_ANNOUNCE */

	/* this is kind of a workaround for the fact that we may not be around or
	* are otherwise unable to reply when the DC sends out A_DC_JOIN_OFFER_ALL
	*/
	void
	do_cl_join_announce(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	/* don't announce if we're in one of these states */
	if (cur_state != S_PENDING) {
	crm_warn("Not announcing cluster join because in state %s",
	fsa_state2string(cur_state));
	return;
	}

	if (AM_I_OPERATIONAL) {
	/* send as a broadcast */
	xmlNode *req = create_request(CRM_OP_JOIN_ANNOUNCE, NULL, NULL,
	CRM_SYSTEM_DC, CRM_SYSTEM_CRMD, NULL);

	crm_debug("Announcing availability");
	update_dc(NULL);
	send_cluster_message(NULL, crm_msg_crmd, req, FALSE);
	free_xml(req);

	} else {
	/* Delay announce until we have finished local startup */
	crm_warn("Delaying announce of cluster join until local startup is complete");
	return;
	}
	}

	static int query_call_id = 0;

	/* A_CL_JOIN_REQUEST */
	/* aka. accept the welcome offer */
	void
	do_cl_join_offer_respond(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	ha_msg_input_t *input = fsa_typed_data(fsa_dt_ha_msg);
	const char *welcome_from;
	const char *join_id;

	CRM_CHECK(input != NULL, return);

	#if 0
	if (we are sick) {
	log error;

	/* save the request for later? */
	return;
	}
	#endif

	welcome_from = crm_element_value(input->msg, F_CRM_HOST_FROM);
	join_id = crm_element_value(input->msg, F_CRM_JOIN_ID);
	crm_trace("Accepting cluster join offer from node %s "CRM_XS" join-%s",
	welcome_from, crm_element_value(input->msg, F_CRM_JOIN_ID));

	/* we only ever want the last one */
	if (query_call_id > 0) {
	crm_trace("Cancelling previous join query: %d", query_call_id);
	remove_cib_op_callback(query_call_id, FALSE);
	query_call_id = 0;
	}

	if (update_dc(input->msg) == FALSE) {
	crm_warn("Discarding cluster join offer from node %s (expected %s)",
	welcome_from, fsa_our_dc);
	return;
	}

	update_dc_expected(input->msg);

	query_call_id =
	fsa_cib_conn->cmds->query(fsa_cib_conn, NULL, NULL, cib_scope_local \| cib_no_children);
	fsa_register_cib_callback(query_call_id, FALSE, strdup(join_id), join_query_callback);
	crm_trace("Registered join query callback: %d", query_call_id);

	controld_set_fsa_action_flags(A_DC_TIMER_STOP);
	trigger_fsa();
	}

	void
	join_query_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
	{
	char *join_id = user_data;
	xmlNode *generation = create_xml_node(NULL, XML_CIB_TAG_GENERATION_TUPPLE);

	CRM_LOG_ASSERT(join_id != NULL);

	if (query_call_id != call_id) {
	crm_trace("Query %d superseded", call_id);
	goto done;
	}

	query_call_id = 0;
	if(rc != pcmk_ok \|\| output == NULL) {
	crm_err("Could not retrieve version details for join-%s: %s (%d)",
	join_id, pcmk_strerror(rc), rc);
	register_fsa_error_adv(C_FSA_INTERNAL, I_ERROR, NULL, NULL, __func__);

	} else if (fsa_our_dc == NULL) {
	crm_debug("Membership is in flux, not continuing join-%s", join_id);

	} else {
	xmlNode *reply = NULL;

	crm_debug("Respond to join offer join-%s from %s", join_id, fsa_our_dc);
	copy_in_properties(generation, output);

	reply = create_request(CRM_OP_JOIN_REQUEST, generation, fsa_our_dc,
	CRM_SYSTEM_DC, CRM_SYSTEM_CRMD, NULL);

	crm_xml_add(reply, F_CRM_JOIN_ID, join_id);
	crm_xml_add(reply, XML_ATTR_CRM_VERSION, CRM_FEATURE_SET);
	send_cluster_message(crm_get_peer(0, fsa_our_dc), crm_msg_crmd, reply, TRUE);
	free_xml(reply);
	}

	done:
	free_xml(generation);
	}

	static void
	set_join_state(const char * start_state)
	{
	if (pcmk__str_eq(start_state, "standby", pcmk__str_casei)) {
	crm_notice("Forcing node %s to join in %s state per configured environment",
	fsa_our_uname, start_state);
	update_attr_delegate(fsa_cib_conn, cib_sync_call, XML_CIB_TAG_NODES, fsa_our_uuid,
	NULL, NULL, NULL, "standby", "on", TRUE, NULL, NULL);

	} else if (pcmk__str_eq(start_state, "online", pcmk__str_casei)) {
	crm_notice("Forcing node %s to join in %s state per configured environment",
	fsa_our_uname, start_state);
	update_attr_delegate(fsa_cib_conn, cib_sync_call, XML_CIB_TAG_NODES, fsa_our_uuid,
	NULL, NULL, NULL, "standby", "off", TRUE, NULL, NULL);

	} else if (pcmk__str_eq(start_state, "default", pcmk__str_casei)) {
	crm_debug("Not forcing a starting state on node %s", fsa_our_uname);

	} else {
	crm_warn("Unrecognized start state '%s', using 'default' (%s)",
	start_state, fsa_our_uname);
	}
	}

	/* A_CL_JOIN_RESULT */
	/* aka. this is notification that we have (or have not) been accepted */
	void
	do_cl_join_finalize_respond(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	xmlNode *tmp1 = NULL;
	gboolean was_nack = TRUE;
	static gboolean first_join = TRUE;
	ha_msg_input_t *input = fsa_typed_data(fsa_dt_ha_msg);
	const char *start_state = pcmk__env_option("node_start_state");

	int join_id = -1;
	const char *op = crm_element_value(input->msg, F_CRM_TASK);
	const char *ack_nack = crm_element_value(input->msg, CRM_OP_JOIN_ACKNAK);
	const char *welcome_from = crm_element_value(input->msg, F_CRM_HOST_FROM);

	if (!pcmk__str_eq(op, CRM_OP_JOIN_ACKNAK, pcmk__str_casei)) {
	crm_trace("Ignoring op=%s message", op);
	return;
	}

	/* calculate if it was an ack or a nack */
	if (crm_is_true(ack_nack)) {
	was_nack = FALSE;
	}

	crm_element_value_int(input->msg, F_CRM_JOIN_ID, &join_id);

	if (was_nack) {
	crm_err("Shutting down because cluster join with leader %s failed "
	CRM_XS" join-%d NACK'd", welcome_from, join_id);
	register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
	return;
	}

	if (AM_I_DC == FALSE && pcmk__str_eq(welcome_from, fsa_our_uname, pcmk__str_casei)) {
	crm_warn("Discarding our own welcome - we're no longer the DC");
	return;
	}

	if (update_dc(input->msg) == FALSE) {
	crm_warn("Discarding %s from node %s (expected from %s)",
	op, welcome_from, fsa_our_dc);
	return;
	}

	update_dc_expected(input->msg);

	/* send our status section to the DC */
	tmp1 = controld_query_executor_state(fsa_our_uname);
	if (tmp1 != NULL) {
	xmlNode *reply = create_request(CRM_OP_JOIN_CONFIRM, tmp1, fsa_our_dc,
	CRM_SYSTEM_DC, CRM_SYSTEM_CRMD, NULL);

	crm_xml_add_int(reply, F_CRM_JOIN_ID, join_id);

	crm_debug("Confirming join-%d: sending local operation history to %s",
	join_id, fsa_our_dc);

	/*
	* If this is the node's first join since the controller started on it,
	* set its initial state (standby or member) according to the user's
	* preference.
	*
	* We do not clear the LRM history here. Even if the DC failed to do it
	* when we last left, removing them here creates a race condition if the
	* controller is being recovered. Instead of a list of active resources
	* from the executor, we may end up with a blank status section. If we
	* are _NOT_ lucky, we will probe for the "wrong" instance of anonymous
	* clones and end up with multiple active instances on the machine.
	*/
	if (first_join && !pcmk_is_set(fsa_input_register, R_SHUTDOWN)) {
	first_join = FALSE;
	if (start_state) {
	set_join_state(start_state);
	}
	}

	send_cluster_message(crm_get_peer(0, fsa_our_dc), crm_msg_crmd, reply, TRUE);
	free_xml(reply);

	if (AM_I_DC == FALSE) {
	register_fsa_input_adv(cause, I_NOT_DC, NULL, A_NOTHING, TRUE,
	__func__);
	}

	free_xml(tmp1);

	} else {
	crm_err("Could not confirm join-%d with %s: Local operation history failed",
	join_id, fsa_our_dc);
	register_fsa_error(C_FSA_INTERNAL, I_FAIL, NULL);
	}
	}
	diff --git a/daemons/controld/controld_join_dc.c b/daemons/controld/controld_join_dc.c
	index 5a606dd16b..b17d9d5aa3 100644
	--- a/daemons/controld/controld_join_dc.c
	+++ b/daemons/controld/controld_join_dc.c
	@@ -1,773 +1,773 @@
	/*
	* Copyright 2004-2020 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <crm/crm.h>

	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>
	#include <crm/cluster.h>

	#include <pacemaker-controld.h>

	char *max_epoch = NULL;
	char *max_generation_from = NULL;
	xmlNode *max_generation_xml = NULL;

	void finalize_join_for(gpointer key, gpointer value, gpointer user_data);
	void finalize_sync_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data);
	gboolean check_join_state(enum crmd_fsa_state cur_state, const char *source);

	/* Numeric counter used to identify join rounds (an unsigned int would be
	* appropriate, except we get and set it in XML as int)
	*/
	static int current_join_id = 0;

	unsigned long long saved_ccm_membership_id = 0;

	void
	crm_update_peer_join(const char source, crm_node_t node, enum crm_join_phase phase)
	{
	enum crm_join_phase last = 0;

	CRM_CHECK(node != NULL, return);

	/* Remote nodes do not participate in joins */
	if (pcmk_is_set(node->flags, crm_remote_node)) {
	return;
	}

	last = node->join;

	if(phase == last) {
	crm_trace("Node %s join-%d phase is still %s "
	CRM_XS " nodeid=%u source=%s",
	node->uname, current_join_id, crm_join_phase_str(last),
	node->id, source);

	} else if ((phase <= crm_join_none) \|\| (phase == (last + 1))) {
	node->join = phase;
	crm_trace("Node %s join-%d phase is now %s (was %s) "
	CRM_XS " nodeid=%u source=%s",
	node->uname, current_join_id, crm_join_phase_str(phase),
	crm_join_phase_str(last), node->id, source);

	} else {
	crm_warn("Rejecting join-%d phase update for node %s because "
	"can't go from %s to %s " CRM_XS " nodeid=%u source=%s",
	current_join_id, node->uname, crm_join_phase_str(last),
	crm_join_phase_str(phase), node->id, source);
	}
	}

	static void
	start_join_round(void)
	{
	GHashTableIter iter;
	crm_node_t *peer = NULL;

	crm_debug("Starting new join round join-%d", current_join_id);

	g_hash_table_iter_init(&iter, crm_peer_cache);
	while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &peer)) {
	crm_update_peer_join(__func__, peer, crm_join_none);
	}
	if (max_generation_from != NULL) {
	free(max_generation_from);
	max_generation_from = NULL;
	}
	if (max_generation_xml != NULL) {
	free_xml(max_generation_xml);
	max_generation_xml = NULL;
	}
	controld_clear_fsa_input_flags(R_HAVE_CIB\|R_CIB_ASKED);
	}

	/*!
	* \internal
	* \brief Create a join message from the DC
	*
	* \param[in] join_op Join operation name
	* \param[in] host_to Recipient of message
	*/
	static xmlNode *
	create_dc_message(const char join_op, const char host_to)
	{
	xmlNode *msg = create_request(join_op, NULL, host_to, CRM_SYSTEM_CRMD,
	CRM_SYSTEM_DC, NULL);

	/* Identify which election this is a part of */
	crm_xml_add_int(msg, F_CRM_JOIN_ID, current_join_id);

	/* Add a field specifying whether the DC is shutting down. This keeps the
	* joining node from fencing the old DC if it becomes the new DC.
	*/
	crm_xml_add_boolean(msg, F_CRM_DC_LEAVING,
	pcmk_is_set(fsa_input_register, R_SHUTDOWN));
	return msg;
	}

	static void
	join_make_offer(gpointer key, gpointer value, gpointer user_data)
	{
	xmlNode *offer = NULL;
	crm_node_t member = (crm_node_t )value;

	CRM_ASSERT(member != NULL);
	if (crm_is_peer_active(member) == FALSE) {
	crm_info("Not making join-%d offer to inactive node %s",
	current_join_id,
	(member->uname? member->uname : "with unknown name"));
	if(member->expected == NULL && pcmk__str_eq(member->state, CRM_NODE_LOST, pcmk__str_casei)) {
	/* You would think this unsafe, but in fact this plus an
	* active resource is what causes it to be fenced.
	*
	* Yes, this does mean that any node that dies at the same
	* time as the old DC and is not running resource (still)
	* won't be fenced.
	*
	* I'm not happy about this either.
	*/
	- crm_update_peer_expected(__func__, member, CRMD_JOINSTATE_DOWN);
	+ pcmk__update_peer_expected(__func__, member, CRMD_JOINSTATE_DOWN);
	}
	return;
	}

	if (member->uname == NULL) {
	crm_info("Not making join-%d offer to node uuid %s with unknown name",
	current_join_id, member->uuid);
	return;
	}

	if (saved_ccm_membership_id != crm_peer_seq) {
	saved_ccm_membership_id = crm_peer_seq;
	crm_info("Making join-%d offers based on membership event %llu",
	current_join_id, crm_peer_seq);
	}

	if(user_data && member->join > crm_join_none) {
	crm_info("Not making join-%d offer to already known node %s (%s)",
	current_join_id, member->uname,
	crm_join_phase_str(member->join));
	return;
	}

	crm_update_peer_join(__func__, (crm_node_t*)member, crm_join_none);

	offer = create_dc_message(CRM_OP_JOIN_OFFER, member->uname);

	// Advertise our feature set so the joining node can bail if not compatible
	crm_xml_add(offer, XML_ATTR_CRM_VERSION, CRM_FEATURE_SET);

	crm_info("Sending join-%d offer to %s", current_join_id, member->uname);
	send_cluster_message(member, crm_msg_crmd, offer, TRUE);
	free_xml(offer);

	crm_update_peer_join(__func__, member, crm_join_welcomed);
	}

	/* A_DC_JOIN_OFFER_ALL */
	void
	do_dc_join_offer_all(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	int count;

	/* Reset everyone's status back to down or in_ccm in the CIB.
	* Any nodes that are active in the CIB but not in the cluster membership
	* will be seen as offline by the scheduler anyway.
	*/
	current_join_id++;
	start_join_round();
	// do_update_cib_nodes(TRUE, __func__);

	update_dc(NULL);
	if (cause == C_HA_MESSAGE && current_input == I_NODE_JOIN) {
	crm_info("A new node joined the cluster");
	}
	g_hash_table_foreach(crm_peer_cache, join_make_offer, NULL);

	count = crmd_join_phase_count(crm_join_welcomed);
	crm_info("Waiting on join-%d requests from %d outstanding node%s",
	current_join_id, count, pcmk__plural_s(count));

	// Don't waste time by invoking the scheduler yet
	}

	/* A_DC_JOIN_OFFER_ONE */
	void
	do_dc_join_offer_one(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	crm_node_t *member;
	ha_msg_input_t *welcome = NULL;
	int count;
	const char *join_to = NULL;

	if (msg_data->data == NULL) {
	crm_info("Making join-%d offers to any unconfirmed nodes "
	"because an unknown node joined", current_join_id);
	g_hash_table_foreach(crm_peer_cache, join_make_offer, &member);
	check_join_state(cur_state, __func__);
	return;
	}

	welcome = fsa_typed_data(fsa_dt_ha_msg);
	if (welcome == NULL) {
	// fsa_typed_data() already logged an error
	return;
	}

	join_to = crm_element_value(welcome->msg, F_CRM_HOST_FROM);
	if (join_to == NULL) {
	crm_err("Can't make join-%d offer to unknown node", current_join_id);
	return;
	}
	member = crm_get_peer(0, join_to);

	/* It is possible that a node will have been sick or starting up when the
	* original offer was made. However, it will either re-announce itself in
	* due course, or we can re-store the original offer on the client.
	*/

	crm_update_peer_join(__func__, member, crm_join_none);
	join_make_offer(NULL, member, NULL);

	/* If the offer isn't to the local node, make an offer to the local node as
	* well, to ensure the correct value for max_generation_from.
	*/
	if (strcmp(join_to, fsa_our_uname) != 0) {
	member = crm_get_peer(0, fsa_our_uname);
	join_make_offer(NULL, member, NULL);
	}

	/* This was a genuine join request; cancel any existing transition and
	* invoke the scheduler.
	*/
	abort_transition(INFINITY, tg_restart, "Node join", NULL);

	count = crmd_join_phase_count(crm_join_welcomed);
	crm_info("Waiting on join-%d requests from %d outstanding node%s",
	current_join_id, count, pcmk__plural_s(count));

	// Don't waste time by invoking the scheduler yet
	}

	static int
	compare_int_fields(xmlNode * left, xmlNode * right, const char *field)
	{
	const char *elem_l = crm_element_value(left, field);
	const char *elem_r = crm_element_value(right, field);

	long long int_elem_l = elem_l? crm_parse_ll(elem_l, NULL) : -1;
	long long int_elem_r = elem_r? crm_parse_ll(elem_r, NULL) : -1;

	if (int_elem_l < int_elem_r) {
	return -1;

	} else if (int_elem_l > int_elem_r) {
	return 1;
	}

	return 0;
	}

	/* A_DC_JOIN_PROCESS_REQ */
	void
	do_dc_join_filter_offer(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	xmlNode *generation = NULL;

	int cmp = 0;
	int join_id = -1;
	int count = 0;
	gboolean ack_nack_bool = TRUE;
	ha_msg_input_t *join_ack = fsa_typed_data(fsa_dt_ha_msg);

	const char *join_from = crm_element_value(join_ack->msg, F_CRM_HOST_FROM);
	const char *ref = crm_element_value(join_ack->msg, F_CRM_REFERENCE);
	const char *join_version = crm_element_value(join_ack->msg,
	XML_ATTR_CRM_VERSION);
	crm_node_t *join_node = NULL;

	if (join_from == NULL) {
	crm_err("Ignoring invalid join request without node name");
	return;
	}
	join_node = crm_get_peer(0, join_from);

	crm_element_value_int(join_ack->msg, F_CRM_JOIN_ID, &join_id);
	if (join_id != current_join_id) {
	crm_debug("Ignoring join-%d request from %s because we are on join-%d",
	join_id, join_from, current_join_id);
	check_join_state(cur_state, __func__);
	return;
	}

	generation = join_ack->xml;
	if (max_generation_xml != NULL && generation != NULL) {
	int lpc = 0;

	const char *attributes[] = {
	XML_ATTR_GENERATION_ADMIN,
	XML_ATTR_GENERATION,
	XML_ATTR_NUMUPDATES,
	};

	for (lpc = 0; cmp == 0 && lpc < DIMOF(attributes); lpc++) {
	cmp = compare_int_fields(max_generation_xml, generation, attributes[lpc]);
	}
	}

	if (ref == NULL) {
	ref = "none"; // for logging only
	}

	if (crm_is_peer_active(join_node) == FALSE) {
	crm_err("Rejecting join-%d request from inactive node %s "
	CRM_XS " ref=%s", join_id, join_from, ref);
	ack_nack_bool = FALSE;

	} else if (generation == NULL) {
	crm_err("Rejecting invalid join-%d request from node %s "
	"missing CIB generation " CRM_XS " ref=%s",
	join_id, join_from, ref);
	ack_nack_bool = FALSE;

	} else if ((join_version == NULL)
	\|\| !feature_set_compatible(CRM_FEATURE_SET, join_version)) {
	crm_err("Rejecting join-%d request from node %s because feature set %s"
	" is incompatible with ours (%s) " CRM_XS " ref=%s",
	join_id, join_from, (join_version? join_version : "pre-3.1.0"),
	CRM_FEATURE_SET, ref);
	ack_nack_bool = FALSE;

	} else if (max_generation_xml == NULL) {
	crm_debug("Accepting join-%d request from %s "
	"(with first CIB generation) " CRM_XS " ref=%s",
	join_id, join_from, ref);
	max_generation_xml = copy_xml(generation);
	max_generation_from = strdup(join_from);

	} else if (cmp < 0 \|\| (cmp == 0 && pcmk__str_eq(join_from, fsa_our_uname, pcmk__str_casei))) {
	crm_debug("Accepting join-%d request from %s (with better "
	"CIB generation than current best from %s) " CRM_XS " ref=%s",
	join_id, join_from, max_generation_from, ref);
	crm_log_xml_debug(max_generation_xml, "Old max generation");
	crm_log_xml_debug(generation, "New max generation");

	free(max_generation_from);
	free_xml(max_generation_xml);

	max_generation_from = strdup(join_from);
	max_generation_xml = copy_xml(join_ack->xml);

	} else {
	crm_debug("Accepting join-%d request from %s " CRM_XS " ref=%s",
	join_id, join_from, ref);
	}

	if (ack_nack_bool == FALSE) {
	crm_update_peer_join(__func__, join_node, crm_join_nack);
	- crm_update_peer_expected(__func__, join_node, CRMD_JOINSTATE_NACK);
	+ pcmk__update_peer_expected(__func__, join_node, CRMD_JOINSTATE_NACK);
	} else {
	crm_update_peer_join(__func__, join_node, crm_join_integrated);
	- crm_update_peer_expected(__func__, join_node, CRMD_JOINSTATE_MEMBER);
	+ pcmk__update_peer_expected(__func__, join_node, CRMD_JOINSTATE_MEMBER);
	}

	count = crmd_join_phase_count(crm_join_integrated);
	crm_debug("%d node%s currently integrated in join-%d",
	count, pcmk__plural_s(count), join_id);

	if (check_join_state(cur_state, __func__) == FALSE) {
	// Don't waste time by invoking the scheduler yet
	count = crmd_join_phase_count(crm_join_welcomed);
	crm_debug("Waiting on join-%d requests from %d outstanding node%s",
	join_id, count, pcmk__plural_s(count));
	}
	}

	/* A_DC_JOIN_FINALIZE */
	void
	do_dc_join_finalize(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	char *sync_from = NULL;
	int rc = pcmk_ok;
	int count_welcomed = crmd_join_phase_count(crm_join_welcomed);
	int count_integrated = crmd_join_phase_count(crm_join_integrated);

	/* This we can do straight away and avoid clients timing us out
	* while we compute the latest CIB
	*/
	if (count_welcomed != 0) {
	crm_debug("Waiting on join-%d requests from %d outstanding node%s "
	"before finalizing join", current_join_id, count_welcomed,
	pcmk__plural_s(count_welcomed));
	crmd_join_phase_log(LOG_DEBUG);
	/* crmd_fsa_stall(FALSE); Needed? */
	return;

	} else if (count_integrated == 0) {
	crm_debug("Finalization not needed for join-%d at the current time",
	current_join_id);
	crmd_join_phase_log(LOG_DEBUG);
	check_join_state(fsa_state, __func__);
	return;
	}

	controld_clear_fsa_input_flags(R_HAVE_CIB);
	if (pcmk__str_eq(max_generation_from, fsa_our_uname, pcmk__str_null_matches \| pcmk__str_casei)) {
	controld_set_fsa_input_flags(R_HAVE_CIB);
	}

	if (pcmk_is_set(fsa_input_register, R_IN_TRANSITION)) {
	crm_warn("Delaying join-%d finalization while transition in progress",
	current_join_id);
	crmd_join_phase_log(LOG_DEBUG);
	crmd_fsa_stall(FALSE);
	return;
	}

	if (max_generation_from && !pcmk_is_set(fsa_input_register, R_HAVE_CIB)) {
	/* ask for the agreed best CIB */
	sync_from = strdup(max_generation_from);
	controld_set_fsa_input_flags(R_CIB_ASKED);
	crm_notice("Finalizing join-%d for %d node%s (sync'ing CIB from %s)",
	current_join_id, count_integrated,
	pcmk__plural_s(count_integrated), sync_from);
	crm_log_xml_notice(max_generation_xml, "Requested CIB version");

	} else {
	/* Send _our_ CIB out to everyone */
	sync_from = strdup(fsa_our_uname);
	crm_debug("Finalizing join-%d for %d node%s (sync'ing from local CIB)",
	current_join_id, count_integrated,
	pcmk__plural_s(count_integrated));
	crm_log_xml_debug(max_generation_xml, "Requested CIB version");
	}
	crmd_join_phase_log(LOG_DEBUG);

	rc = fsa_cib_conn->cmds->sync_from(fsa_cib_conn, sync_from, NULL, cib_quorum_override);
	fsa_register_cib_callback(rc, FALSE, sync_from, finalize_sync_callback);
	}

	void
	finalize_sync_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
	{
	CRM_LOG_ASSERT(-EPERM != rc);
	controld_clear_fsa_input_flags(R_CIB_ASKED);
	if (rc != pcmk_ok) {
	do_crm_log(((rc == -pcmk_err_old_data)? LOG_WARNING : LOG_ERR),
	"Could not sync CIB from %s in join-%d: %s",
	(char *) user_data, current_join_id, pcmk_strerror(rc));

	/* restart the whole join process */
	register_fsa_error_adv(C_FSA_INTERNAL, I_ELECTION_DC, NULL, NULL,
	__func__);

	} else if (!AM_I_DC) {
	crm_debug("Sync'ed CIB for join-%d but no longer DC", current_join_id);

	} else if (fsa_state != S_FINALIZE_JOIN) {
	crm_debug("Sync'ed CIB for join-%d but no longer in S_FINALIZE_JOIN (%s)",
	current_join_id, fsa_state2string(fsa_state));

	} else {
	controld_set_fsa_input_flags(R_HAVE_CIB);
	controld_clear_fsa_input_flags(R_CIB_ASKED);

	/* make sure dc_uuid is re-set to us */
	if (check_join_state(fsa_state, __func__) == FALSE) {
	int count_integrated = crmd_join_phase_count(crm_join_integrated);

	crm_debug("Notifying %d node%s of join-%d results",
	count_integrated, pcmk__plural_s(count_integrated),
	current_join_id);
	g_hash_table_foreach(crm_peer_cache, finalize_join_for, NULL);
	}
	}
	}

	static void
	join_update_complete_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
	{
	fsa_data_t *msg_data = NULL;

	if (rc == pcmk_ok) {
	crm_debug("join-%d node history update (via CIB call %d) complete",
	current_join_id, call_id);
	check_join_state(fsa_state, __func__);

	} else {
	crm_err("join-%d node history update (via CIB call %d) failed: %s "
	"(next transition may determine resource status incorrectly)",
	current_join_id, call_id, pcmk_strerror(rc));
	crm_log_xml_debug(msg, "failed");
	register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
	}
	}

	/* A_DC_JOIN_PROCESS_ACK */
	void
	do_dc_join_ack(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	int join_id = -1;
	int call_id = 0;
	ha_msg_input_t *join_ack = fsa_typed_data(fsa_dt_ha_msg);
	enum controld_section_e section = controld_section_lrm;

	const char *op = crm_element_value(join_ack->msg, F_CRM_TASK);
	const char *join_from = crm_element_value(join_ack->msg, F_CRM_HOST_FROM);
	crm_node_t *peer = NULL;

	// Sanity checks
	if (join_from == NULL) {
	crm_warn("Ignoring message received without node identification");
	return;
	}
	if (op == NULL) {
	crm_warn("Ignoring message received from %s without task", join_from);
	return;
	}

	if (strcmp(op, CRM_OP_JOIN_CONFIRM)) {
	crm_debug("Ignoring '%s' message from %s while waiting for '%s'",
	op, join_from, CRM_OP_JOIN_CONFIRM);
	return;
	}

	if (crm_element_value_int(join_ack->msg, F_CRM_JOIN_ID, &join_id) != 0) {
	crm_warn("Ignoring join confirmation from %s without valid join ID",
	join_from);
	return;
	}

	peer = crm_get_peer(0, join_from);
	if (peer->join != crm_join_finalized) {
	crm_info("Ignoring out-of-sequence join-%d confirmation from %s "
	"(currently %s not %s)",
	join_id, join_from, crm_join_phase_str(peer->join),
	crm_join_phase_str(crm_join_finalized));
	return;
	}

	if (join_id != current_join_id) {
	crm_err("Rejecting join-%d confirmation from %s "
	"because currently on join-%d",
	join_id, join_from, current_join_id);
	crm_update_peer_join(__func__, peer, crm_join_nack);
	return;
	}

	crm_update_peer_join(__func__, peer, crm_join_confirmed);

	/* Update CIB with node's current executor state. A new transition will be
	* triggered later, when the CIB notifies us of the change.
	*/
	if (controld_shutdown_lock_enabled) {
	section = controld_section_lrm_unlocked;
	}
	controld_delete_node_state(join_from, section, cib_scope_local);
	if (pcmk__str_eq(join_from, fsa_our_uname, pcmk__str_casei)) {
	xmlNode *now_dc_lrmd_state = controld_query_executor_state(fsa_our_uname);

	if (now_dc_lrmd_state != NULL) {
	fsa_cib_update(XML_CIB_TAG_STATUS, now_dc_lrmd_state,
	cib_scope_local \| cib_quorum_override \| cib_can_create, call_id, NULL);
	free_xml(now_dc_lrmd_state);
	crm_debug("Updating local node history for join-%d "
	"from query result (via CIB call %d)", join_id, call_id);
	} else {
	fsa_cib_update(XML_CIB_TAG_STATUS, join_ack->xml,
	cib_scope_local \| cib_quorum_override \| cib_can_create, call_id, NULL);
	crm_warn("Updating local node history from join-%d confirmation "
	"because query failed (via CIB call %d)", join_id, call_id);
	}
	} else {
	fsa_cib_update(XML_CIB_TAG_STATUS, join_ack->xml,
	cib_scope_local \| cib_quorum_override \| cib_can_create, call_id, NULL);
	crm_debug("Updating node history for %s from join-%d confirmation "
	"(via CIB call %d)", join_from, join_id, call_id);
	}
	fsa_register_cib_callback(call_id, FALSE, NULL, join_update_complete_callback);
	}

	void
	finalize_join_for(gpointer key, gpointer value, gpointer user_data)
	{
	xmlNode *acknak = NULL;
	xmlNode *tmp1 = NULL;
	crm_node_t *join_node = value;
	const char *join_to = join_node->uname;

	if(join_node->join != crm_join_integrated) {
	crm_trace("Not updating non-integrated node %s (%s) for join-%d",
	join_to, crm_join_phase_str(join_node->join),
	current_join_id);
	return;
	}

	crm_trace("Updating node state for %s", join_to);
	tmp1 = create_xml_node(NULL, XML_CIB_TAG_NODE);
	set_uuid(tmp1, XML_ATTR_UUID, join_node);
	crm_xml_add(tmp1, XML_ATTR_UNAME, join_to);
	fsa_cib_anon_update(XML_CIB_TAG_NODES, tmp1);
	free_xml(tmp1);

	join_node = crm_get_peer(0, join_to);
	if (crm_is_peer_active(join_node) == FALSE) {
	/*
	* NACK'ing nodes that the membership layer doesn't know about yet
	* simply creates more churn
	*
	* Better to leave them waiting and let the join restart when
	* the new membership event comes in
	*
	* All other NACKs (due to versions etc) should still be processed
	*/
	- crm_update_peer_expected(__func__, join_node, CRMD_JOINSTATE_PENDING);
	+ pcmk__update_peer_expected(__func__, join_node, CRMD_JOINSTATE_PENDING);
	return;
	}

	// Acknowledge node's join request
	crm_debug("Acknowledging join-%d request from %s",
	current_join_id, join_to);
	acknak = create_dc_message(CRM_OP_JOIN_ACKNAK, join_to);
	crm_xml_add(acknak, CRM_OP_JOIN_ACKNAK, XML_BOOLEAN_TRUE);
	crm_update_peer_join(__func__, join_node, crm_join_finalized);
	- crm_update_peer_expected(__func__, join_node, CRMD_JOINSTATE_MEMBER);
	+ pcmk__update_peer_expected(__func__, join_node, CRMD_JOINSTATE_MEMBER);

	send_cluster_message(crm_get_peer(0, join_to), crm_msg_crmd, acknak, TRUE);
	free_xml(acknak);
	return;
	}

	gboolean
	check_join_state(enum crmd_fsa_state cur_state, const char *source)
	{
	static unsigned long long highest_seq = 0;

	if (saved_ccm_membership_id != crm_peer_seq) {
	crm_debug("join-%d: Membership changed from %llu to %llu "
	CRM_XS " highest=%llu state=%s for=%s",
	current_join_id, saved_ccm_membership_id, crm_peer_seq, highest_seq,
	fsa_state2string(cur_state), source);
	if(highest_seq < crm_peer_seq) {
	/* Don't spam the FSA with duplicates */
	highest_seq = crm_peer_seq;
	register_fsa_input_before(C_FSA_INTERNAL, I_NODE_JOIN, NULL);
	}

	} else if (cur_state == S_INTEGRATION) {
	if (crmd_join_phase_count(crm_join_welcomed) == 0) {
	int count = crmd_join_phase_count(crm_join_integrated);

	crm_debug("join-%d: Integration of %d peer%s complete "
	CRM_XS " state=%s for=%s",
	current_join_id, count, pcmk__plural_s(count),
	fsa_state2string(cur_state), source);
	register_fsa_input_before(C_FSA_INTERNAL, I_INTEGRATED, NULL);
	return TRUE;
	}

	} else if (cur_state == S_FINALIZE_JOIN) {
	if (!pcmk_is_set(fsa_input_register, R_HAVE_CIB)) {
	crm_debug("join-%d: Delaying finalization until we have CIB "
	CRM_XS " state=%s for=%s",
	current_join_id, fsa_state2string(cur_state), source);
	return TRUE;

	} else if (crmd_join_phase_count(crm_join_welcomed) != 0) {
	int count = crmd_join_phase_count(crm_join_welcomed);

	crm_debug("join-%d: Still waiting on %d welcomed node%s "
	CRM_XS " state=%s for=%s",
	current_join_id, count, pcmk__plural_s(count),
	fsa_state2string(cur_state), source);
	crmd_join_phase_log(LOG_DEBUG);

	} else if (crmd_join_phase_count(crm_join_integrated) != 0) {
	int count = crmd_join_phase_count(crm_join_integrated);

	crm_debug("join-%d: Still waiting on %d integrated node%s "
	CRM_XS " state=%s for=%s",
	current_join_id, count, pcmk__plural_s(count),
	fsa_state2string(cur_state), source);
	crmd_join_phase_log(LOG_DEBUG);

	} else if (crmd_join_phase_count(crm_join_finalized) != 0) {
	int count = crmd_join_phase_count(crm_join_finalized);

	crm_debug("join-%d: Still waiting on %d finalized node%s "
	CRM_XS " state=%s for=%s",
	current_join_id, count, pcmk__plural_s(count),
	fsa_state2string(cur_state), source);
	crmd_join_phase_log(LOG_DEBUG);

	} else {
	crm_debug("join-%d: Complete " CRM_XS " state=%s for=%s",
	current_join_id, fsa_state2string(cur_state), source);
	register_fsa_input_later(C_FSA_INTERNAL, I_FINALIZED, NULL);
	return TRUE;
	}
	}

	return FALSE;
	}

	void
	do_dc_join_final(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	crm_debug("Ensuring DC, quorum and node attributes are up-to-date");
	crm_update_quorum(crm_have_quorum, TRUE);
	}

	int crmd_join_phase_count(enum crm_join_phase phase)
	{
	int count = 0;
	crm_node_t *peer;
	GHashTableIter iter;

	g_hash_table_iter_init(&iter, crm_peer_cache);
	while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &peer)) {
	if(peer->join == phase) {
	count++;
	}
	}
	return count;
	}

	void crmd_join_phase_log(int level)
	{
	crm_node_t *peer;
	GHashTableIter iter;

	g_hash_table_iter_init(&iter, crm_peer_cache);
	while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &peer)) {
	do_crm_log(level, "join-%d: %s=%s", current_join_id, peer->uname,
	crm_join_phase_str(peer->join));
	}
	}
	diff --git a/daemons/controld/controld_membership.c b/daemons/controld/controld_membership.c
	index f5e20721e1..550c19c5cb 100644
	--- a/daemons/controld/controld_membership.c
	+++ b/daemons/controld/controld_membership.c
	@@ -1,439 +1,439 @@
	/*
	* Copyright 2004-2020 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	/* put these first so that uuid_t is defined without conflicts */
	#include <crm_internal.h>

	#include <string.h>

	#include <crm/crm.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>
	#include <crm/common/xml_internal.h>
	#include <crm/cluster/internal.h>

	#include <pacemaker-controld.h>

	gboolean membership_flux_hack = FALSE;
	void post_cache_update(int instance);

	int last_peer_update = 0;
	guint highest_born_on = -1;

	extern gboolean check_join_state(enum crmd_fsa_state cur_state, const char *source);

	static void
	reap_dead_nodes(gpointer key, gpointer value, gpointer user_data)
	{
	crm_node_t *node = value;

	if (crm_is_peer_active(node) == FALSE) {
	crm_update_peer_join(__func__, node, crm_join_none);

	if(node && node->uname) {
	if (pcmk__str_eq(fsa_our_uname, node->uname, pcmk__str_casei)) {
	crm_err("We're not part of the cluster anymore");
	register_fsa_input(C_FSA_INTERNAL, I_ERROR, NULL);

	} else if (AM_I_DC == FALSE && pcmk__str_eq(node->uname, fsa_our_dc, pcmk__str_casei)) {
	crm_warn("Our DC node (%s) left the cluster", node->uname);
	register_fsa_input(C_FSA_INTERNAL, I_ELECTION, NULL);
	}
	}

	if (fsa_state == S_INTEGRATION \|\| fsa_state == S_FINALIZE_JOIN) {
	check_join_state(fsa_state, __func__);
	}
	if(node && node->uuid) {
	fail_incompletable_actions(transition_graph, node->uuid);
	}
	}
	}

	gboolean ever_had_quorum = FALSE;

	void
	post_cache_update(int instance)
	{
	xmlNode *no_op = NULL;

	crm_peer_seq = instance;
	crm_debug("Updated cache after membership event %d.", instance);

	g_hash_table_foreach(crm_peer_cache, reap_dead_nodes, NULL);
	controld_set_fsa_input_flags(R_MEMBERSHIP);

	if (AM_I_DC) {
	populate_cib_nodes(node_update_quick \| node_update_cluster \| node_update_peer \|
	node_update_expected, __func__);
	}

	/*
	* If we lost nodes, we should re-check the election status
	* Safe to call outside of an election
	*/
	controld_set_fsa_action_flags(A_ELECTION_CHECK);
	trigger_fsa();

	/* Membership changed, remind everyone we're here.
	* This will aid detection of duplicate DCs
	*/
	no_op = create_request(CRM_OP_NOOP, NULL, NULL, CRM_SYSTEM_CRMD,
	AM_I_DC ? CRM_SYSTEM_DC : CRM_SYSTEM_CRMD, NULL);
	send_cluster_message(NULL, crm_msg_crmd, no_op, FALSE);
	free_xml(no_op);
	}

	static void
	crmd_node_update_complete(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
	{
	fsa_data_t *msg_data = NULL;

	last_peer_update = 0;

	if (rc == pcmk_ok) {
	crm_trace("Node update %d complete", call_id);

	} else if(call_id < pcmk_ok) {
	crm_err("Node update failed: %s (%d)", pcmk_strerror(call_id), call_id);
	crm_log_xml_debug(msg, "failed");
	register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);

	} else {
	crm_err("Node update %d failed: %s (%d)", call_id, pcmk_strerror(rc), rc);
	crm_log_xml_debug(msg, "failed");
	register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
	}
	}

	/*!
	* \internal
	* \brief Create an XML node state tag with updates
	*
	* \param[in,out] node Node whose state will be used for update
	* \param[in] flags Bitmask of node_update_flags indicating what to update
	* \param[in,out] parent XML node to contain update (or NULL)
	* \param[in] source Who requested the update (only used for logging)
	*
	* \return Pointer to created node state tag
	*/
	xmlNode *
	create_node_state_update(crm_node_t node, int flags, xmlNode parent,
	const char *source)
	{
	const char *value = NULL;
	xmlNode *node_state;

	if (!node->state) {
	crm_info("Node update for %s cancelled: no state, not seen yet", node->uname);
	return NULL;
	}

	node_state = create_xml_node(parent, XML_CIB_TAG_STATE);

	if (pcmk_is_set(node->flags, crm_remote_node)) {
	crm_xml_add(node_state, XML_NODE_IS_REMOTE, XML_BOOLEAN_TRUE);
	}

	set_uuid(node_state, XML_ATTR_UUID, node);

	if (crm_element_value(node_state, XML_ATTR_UUID) == NULL) {
	crm_info("Node update for %s cancelled: no id", node->uname);
	free_xml(node_state);
	return NULL;
	}

	crm_xml_add(node_state, XML_ATTR_UNAME, node->uname);

	if ((flags & node_update_cluster) && node->state) {
	crm_xml_add_boolean(node_state, XML_NODE_IN_CLUSTER,
	pcmk__str_eq(node->state, CRM_NODE_MEMBER, pcmk__str_casei));
	}

	if (!pcmk_is_set(node->flags, crm_remote_node)) {
	if (flags & node_update_peer) {
	value = OFFLINESTATUS;
	if (pcmk_is_set(node->processes, crm_get_cluster_proc())) {
	value = ONLINESTATUS;
	}
	crm_xml_add(node_state, XML_NODE_IS_PEER, value);
	}

	if (flags & node_update_join) {
	if (node->join <= crm_join_none) {
	value = CRMD_JOINSTATE_DOWN;
	} else {
	value = CRMD_JOINSTATE_MEMBER;
	}
	crm_xml_add(node_state, XML_NODE_JOIN_STATE, value);
	}

	if (flags & node_update_expected) {
	crm_xml_add(node_state, XML_NODE_EXPECTED, node->expected);
	}
	}

	crm_xml_add(node_state, XML_ATTR_ORIGIN, source);

	return node_state;
	}

	static void
	remove_conflicting_node_callback(xmlNode * msg, int call_id, int rc,
	xmlNode * output, void *user_data)
	{
	char *node_uuid = user_data;

	do_crm_log_unlikely(rc == 0 ? LOG_DEBUG : LOG_NOTICE,
	"Deletion of the unknown conflicting node \"%s\": %s (rc=%d)",
	node_uuid, pcmk_strerror(rc), rc);
	}

	static void
	search_conflicting_node_callback(xmlNode * msg, int call_id, int rc,
	xmlNode * output, void *user_data)
	{
	char *new_node_uuid = user_data;
	xmlNode *node_xml = NULL;

	if (rc != pcmk_ok) {
	if (rc != -ENXIO) {
	crm_notice("Searching conflicting nodes for %s failed: %s (%d)",
	new_node_uuid, pcmk_strerror(rc), rc);
	}
	return;

	} else if (output == NULL) {
	return;
	}

	if (pcmk__str_eq(crm_element_name(output), XML_CIB_TAG_NODE, pcmk__str_casei)) {
	node_xml = output;

	} else {
	node_xml = pcmk__xml_first_child(output);
	}

	for (; node_xml != NULL; node_xml = pcmk__xml_next(node_xml)) {
	const char *node_uuid = NULL;
	const char *node_uname = NULL;
	GHashTableIter iter;
	crm_node_t *node = NULL;
	gboolean known = FALSE;

	if (!pcmk__str_eq(crm_element_name(node_xml), XML_CIB_TAG_NODE, pcmk__str_casei)) {
	continue;
	}

	node_uuid = crm_element_value(node_xml, XML_ATTR_ID);
	node_uname = crm_element_value(node_xml, XML_ATTR_UNAME);

	if (node_uuid == NULL \|\| node_uname == NULL) {
	continue;
	}

	g_hash_table_iter_init(&iter, crm_peer_cache);
	while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &node)) {
	if (node->uuid
	&& pcmk__str_eq(node->uuid, node_uuid, pcmk__str_casei)
	&& node->uname
	&& pcmk__str_eq(node->uname, node_uname, pcmk__str_casei)) {

	known = TRUE;
	break;
	}
	}

	if (known == FALSE) {
	int delete_call_id = 0;
	xmlNode *node_state_xml = NULL;

	crm_notice("Deleting unknown node %s/%s which has conflicting uname with %s",
	node_uuid, node_uname, new_node_uuid);

	delete_call_id = fsa_cib_conn->cmds->remove(fsa_cib_conn, XML_CIB_TAG_NODES, node_xml,
	cib_scope_local \| cib_quorum_override);
	fsa_register_cib_callback(delete_call_id, FALSE, strdup(node_uuid),
	remove_conflicting_node_callback);

	node_state_xml = create_xml_node(NULL, XML_CIB_TAG_STATE);
	crm_xml_add(node_state_xml, XML_ATTR_ID, node_uuid);
	crm_xml_add(node_state_xml, XML_ATTR_UNAME, node_uname);

	delete_call_id = fsa_cib_conn->cmds->remove(fsa_cib_conn, XML_CIB_TAG_STATUS, node_state_xml,
	cib_scope_local \| cib_quorum_override);
	fsa_register_cib_callback(delete_call_id, FALSE, strdup(node_uuid),
	remove_conflicting_node_callback);
	free_xml(node_state_xml);
	}
	}
	}

	static void
	node_list_update_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
	{
	fsa_data_t *msg_data = NULL;

	if(call_id < pcmk_ok) {
	crm_err("Node list update failed: %s (%d)", pcmk_strerror(call_id), call_id);
	crm_log_xml_debug(msg, "update:failed");
	register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);

	} else if(rc < pcmk_ok) {
	crm_err("Node update %d failed: %s (%d)", call_id, pcmk_strerror(rc), rc);
	crm_log_xml_debug(msg, "update:failed");
	register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
	}
	}

	#define NODE_PATH_MAX 512

	void
	populate_cib_nodes(enum node_update_flags flags, const char *source)
	{
	int call_id = 0;
	gboolean from_hashtable = TRUE;
	int call_options = cib_scope_local \| cib_quorum_override;
	xmlNode *node_list = create_xml_node(NULL, XML_CIB_TAG_NODES);

	#if SUPPORT_COROSYNC
	if (!pcmk_is_set(flags, node_update_quick) && is_corosync_cluster()) {
	- from_hashtable = corosync_initialize_nodelist(NULL, FALSE, node_list);
	+ from_hashtable = pcmk__corosync_add_nodes(node_list);
	}
	#endif

	if (from_hashtable) {
	GHashTableIter iter;
	crm_node_t *node = NULL;

	g_hash_table_iter_init(&iter, crm_peer_cache);
	while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &node)) {
	xmlNode *new_node = NULL;

	crm_trace("Creating node entry for %s/%s", node->uname, node->uuid);
	if(node->uuid && node->uname) {
	char xpath[NODE_PATH_MAX];

	/* We need both to be valid */
	new_node = create_xml_node(node_list, XML_CIB_TAG_NODE);
	crm_xml_add(new_node, XML_ATTR_ID, node->uuid);
	crm_xml_add(new_node, XML_ATTR_UNAME, node->uname);

	/* Search and remove unknown nodes with the conflicting uname from CIB */
	snprintf(xpath, NODE_PATH_MAX,
	"/" XML_TAG_CIB "/" XML_CIB_TAG_CONFIGURATION "/" XML_CIB_TAG_NODES
	"/" XML_CIB_TAG_NODE "[@uname='%s'][@id!='%s']",
	node->uname, node->uuid);

	call_id = fsa_cib_conn->cmds->query(fsa_cib_conn, xpath, NULL,
	cib_scope_local \| cib_xpath);
	fsa_register_cib_callback(call_id, FALSE, strdup(node->uuid),
	search_conflicting_node_callback);
	}
	}
	}

	crm_trace("Populating <nodes> section from %s", from_hashtable ? "hashtable" : "cluster");

	fsa_cib_update(XML_CIB_TAG_NODES, node_list, call_options, call_id, NULL);
	fsa_register_cib_callback(call_id, FALSE, NULL, node_list_update_callback);

	free_xml(node_list);

	if (call_id >= pcmk_ok && crm_peer_cache != NULL && AM_I_DC) {
	/*
	* There is no need to update the local CIB with our values if
	* we've not seen valid membership data
	*/
	GHashTableIter iter;
	crm_node_t *node = NULL;

	node_list = create_xml_node(NULL, XML_CIB_TAG_STATUS);

	g_hash_table_iter_init(&iter, crm_peer_cache);
	while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &node)) {
	create_node_state_update(node, flags, node_list, source);
	}

	if (crm_remote_peer_cache) {
	g_hash_table_iter_init(&iter, crm_remote_peer_cache);
	while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &node)) {
	create_node_state_update(node, flags, node_list, source);
	}
	}

	fsa_cib_update(XML_CIB_TAG_STATUS, node_list, call_options, call_id, NULL);
	fsa_register_cib_callback(call_id, FALSE, NULL, crmd_node_update_complete);
	last_peer_update = call_id;

	free_xml(node_list);
	}
	}

	static void
	cib_quorum_update_complete(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
	{
	fsa_data_t *msg_data = NULL;

	if (rc == pcmk_ok) {
	crm_trace("Quorum update %d complete", call_id);

	} else {
	crm_err("Quorum update %d failed: %s (%d)", call_id, pcmk_strerror(rc), rc);
	crm_log_xml_debug(msg, "failed");
	register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
	}
	}

	void
	crm_update_quorum(gboolean quorum, gboolean force_update)
	{
	ever_had_quorum \|= quorum;

	if(ever_had_quorum && quorum == FALSE && no_quorum_suicide_escalation) {
	pcmk__panic(__func__);
	}

	if (AM_I_DC && (force_update \|\| fsa_has_quorum != quorum)) {
	int call_id = 0;
	xmlNode *update = NULL;
	int call_options = cib_scope_local \| cib_quorum_override;

	update = create_xml_node(NULL, XML_TAG_CIB);
	crm_xml_add_int(update, XML_ATTR_HAVE_QUORUM, quorum);
	crm_xml_add(update, XML_ATTR_DC_UUID, fsa_our_uuid);

	fsa_cib_update(XML_TAG_CIB, update, call_options, call_id, NULL);
	crm_debug("Updating quorum status to %s (call=%d)",
	pcmk__btoa(quorum), call_id);
	fsa_register_cib_callback(call_id, FALSE, NULL, cib_quorum_update_complete);
	free_xml(update);

	/* Quorum changes usually cause a new transition via other activity:
	* quorum gained via a node joining will abort via the node join,
	* and quorum lost via a node leaving will usually abort via resource
	* activity and/or fencing.
	*
	* However, it is possible that nothing else causes a transition (e.g.
	* someone forces quorum via corosync-cmaptcl, or quorum is lost due to
	* a node in standby shutting down cleanly), so here ensure a new
	* transition is triggered.
	*/
	if (quorum) {
	/* If quorum was gained, abort after a short delay, in case multiple
	* nodes are joining around the same time, so the one that brings us
	* to quorum doesn't cause all the remaining ones to be fenced.
	*/
	abort_after_delay(INFINITY, tg_restart, "Quorum gained", 5000);
	} else {
	abort_transition(INFINITY, tg_restart, "Quorum lost", NULL);
	}
	}
	fsa_has_quorum = quorum;
	}
	diff --git a/daemons/controld/controld_messages.c b/daemons/controld/controld_messages.c
	index 6913255d66..6cac5afc61 100644
	--- a/daemons/controld/controld_messages.c
	+++ b/daemons/controld/controld_messages.c
	@@ -1,1294 +1,1294 @@
	/*
	* Copyright 2004-2020 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <sys/param.h>
	#include <string.h>
	#include <time.h>

	#include <crm/crm.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>
	#include <crm/cluster/internal.h>
	#include <crm/cib.h>
	#include <crm/common/ipc_internal.h>

	#include <pacemaker-controld.h>

	GListPtr fsa_message_queue = NULL;
	extern void crm_shutdown(int nsig);

	static enum crmd_fsa_input handle_message(xmlNode *msg,
	enum crmd_fsa_cause cause);
	static void handle_response(xmlNode *stored_msg);
	static enum crmd_fsa_input handle_request(xmlNode *stored_msg,
	enum crmd_fsa_cause cause);
	static enum crmd_fsa_input handle_shutdown_request(xmlNode *stored_msg);
	static void send_msg_via_ipc(xmlNode * msg, const char *sys);

	/* debug only, can wrap all it likes */
	int last_data_id = 0;

	void
	register_fsa_error_adv(enum crmd_fsa_cause cause, enum crmd_fsa_input input,
	fsa_data_t * cur_data, void new_data, const char raised_from)
	{
	/* save the current actions if any */
	if (fsa_actions != A_NOTHING) {
	register_fsa_input_adv(cur_data ? cur_data->fsa_cause : C_FSA_INTERNAL,
	I_NULL, cur_data ? cur_data->data : NULL,
	fsa_actions, TRUE, __func__);
	}

	/* reset the action list */
	crm_info("Resetting the current action list");
	fsa_dump_actions(fsa_actions, "Drop");
	fsa_actions = A_NOTHING;

	/* register the error */
	register_fsa_input_adv(cause, input, new_data, A_NOTHING, TRUE, raised_from);
	}

	int
	register_fsa_input_adv(enum crmd_fsa_cause cause, enum crmd_fsa_input input,
	void *data, uint64_t with_actions,
	gboolean prepend, const char *raised_from)
	{
	unsigned old_len = g_list_length(fsa_message_queue);
	fsa_data_t *fsa_data = NULL;

	if (raised_from == NULL) {
	raised_from = "<unknown>";
	}

	if (input == I_NULL && with_actions == A_NOTHING /* && data == NULL */ ) {
	/* no point doing anything */
	crm_err("Cannot add entry to queue: no input and no action");
	return 0;
	}

	if (input == I_WAIT_FOR_EVENT) {
	do_fsa_stall = TRUE;
	crm_debug("Stalling the FSA pending further input: source=%s cause=%s data=%p queue=%d",
	raised_from, fsa_cause2string(cause), data, old_len);

	if (old_len > 0) {
	fsa_dump_queue(LOG_TRACE);
	prepend = FALSE;
	}

	if (data == NULL) {
	controld_set_fsa_action_flags(with_actions);
	fsa_dump_actions(with_actions, "Restored");
	return 0;
	}

	/* Store everything in the new event and reset fsa_actions */
	with_actions \|= fsa_actions;
	fsa_actions = A_NOTHING;
	}

	last_data_id++;
	crm_trace("%s %s FSA input %d (%s) due to %s, %s data",
	raised_from, (prepend? "prepended" : "appended"), last_data_id,
	fsa_input2string(input), fsa_cause2string(cause),
	(data? "with" : "without"));

	fsa_data = calloc(1, sizeof(fsa_data_t));
	fsa_data->id = last_data_id;
	fsa_data->fsa_input = input;
	fsa_data->fsa_cause = cause;
	fsa_data->origin = raised_from;
	fsa_data->data = NULL;
	fsa_data->data_type = fsa_dt_none;
	fsa_data->actions = with_actions;

	if (with_actions != A_NOTHING) {
	crm_trace("Adding actions %.16llx to input",
	(unsigned long long) with_actions);
	}

	if (data != NULL) {
	switch (cause) {
	case C_FSA_INTERNAL:
	case C_CRMD_STATUS_CALLBACK:
	case C_IPC_MESSAGE:
	case C_HA_MESSAGE:
	CRM_CHECK(((ha_msg_input_t *) data)->msg != NULL,
	crm_err("Bogus data from %s", raised_from));
	crm_trace("Copying %s data from %s as cluster message data",
	fsa_cause2string(cause), raised_from);
	fsa_data->data = copy_ha_msg_input(data);
	fsa_data->data_type = fsa_dt_ha_msg;
	break;

	case C_LRM_OP_CALLBACK:
	crm_trace("Copying %s data from %s as lrmd_event_data_t",
	fsa_cause2string(cause), raised_from);
	fsa_data->data = lrmd_copy_event((lrmd_event_data_t *) data);
	fsa_data->data_type = fsa_dt_lrm;
	break;

	case C_TIMER_POPPED:
	case C_SHUTDOWN:
	case C_UNKNOWN:
	case C_STARTUP:
	crm_crit("Copying %s data (from %s) is not yet implemented",
	fsa_cause2string(cause), raised_from);
	crmd_exit(CRM_EX_SOFTWARE);
	break;
	}
	}

	/* make sure to free it properly later */
	if (prepend) {
	fsa_message_queue = g_list_prepend(fsa_message_queue, fsa_data);
	} else {
	fsa_message_queue = g_list_append(fsa_message_queue, fsa_data);
	}

	crm_trace("FSA message queue length is %d",
	g_list_length(fsa_message_queue));

	/* fsa_dump_queue(LOG_TRACE); */

	if (old_len == g_list_length(fsa_message_queue)) {
	crm_err("Couldn't add message to the queue");
	}

	if (fsa_source && input != I_WAIT_FOR_EVENT) {
	crm_trace("Triggering FSA");
	mainloop_set_trigger(fsa_source);
	}
	return last_data_id;
	}

	void
	fsa_dump_queue(int log_level)
	{
	int offset = 0;
	GListPtr lpc = NULL;

	for (lpc = fsa_message_queue; lpc != NULL; lpc = lpc->next) {
	fsa_data_t data = (fsa_data_t ) lpc->data;

	do_crm_log_unlikely(log_level,
	"queue[%d.%d]: input %s raised by %s(%p.%d)\t(cause=%s)",
	offset++, data->id, fsa_input2string(data->fsa_input),
	data->origin, data->data, data->data_type,
	fsa_cause2string(data->fsa_cause));
	}
	}

	ha_msg_input_t *
	copy_ha_msg_input(ha_msg_input_t * orig)
	{
	ha_msg_input_t *copy = calloc(1, sizeof(ha_msg_input_t));

	CRM_ASSERT(copy != NULL);
	copy->msg = (orig && orig->msg)? copy_xml(orig->msg) : NULL;
	copy->xml = get_message_xml(copy->msg, F_CRM_DATA);
	return copy;
	}

	void
	delete_fsa_input(fsa_data_t * fsa_data)
	{
	lrmd_event_data_t *op = NULL;
	xmlNode *foo = NULL;

	if (fsa_data == NULL) {
	return;
	}
	crm_trace("About to free %s data", fsa_cause2string(fsa_data->fsa_cause));

	if (fsa_data->data != NULL) {
	switch (fsa_data->data_type) {
	case fsa_dt_ha_msg:
	delete_ha_msg_input(fsa_data->data);
	break;

	case fsa_dt_xml:
	foo = fsa_data->data;
	free_xml(foo);
	break;

	case fsa_dt_lrm:
	op = (lrmd_event_data_t *) fsa_data->data;
	lrmd_free_event(op);
	break;

	case fsa_dt_none:
	if (fsa_data->data != NULL) {
	crm_err("Don't know how to free %s data from %s",
	fsa_cause2string(fsa_data->fsa_cause), fsa_data->origin);
	crmd_exit(CRM_EX_SOFTWARE);
	}
	break;
	}
	crm_trace("%s data freed", fsa_cause2string(fsa_data->fsa_cause));
	}

	free(fsa_data);
	}

	/* returns the next message */
	fsa_data_t *
	get_message(void)
	{
	fsa_data_t *message = g_list_nth_data(fsa_message_queue, 0);

	fsa_message_queue = g_list_remove(fsa_message_queue, message);
	crm_trace("Processing input %d", message->id);
	return message;
	}

	void *
	fsa_typed_data_adv(fsa_data_t * fsa_data, enum fsa_data_type a_type, const char *caller)
	{
	void *ret_val = NULL;

	if (fsa_data == NULL) {
	crm_err("%s: No FSA data available", caller);

	} else if (fsa_data->data == NULL) {
	crm_err("%s: No message data available. Origin: %s", caller, fsa_data->origin);

	} else if (fsa_data->data_type != a_type) {
	crm_crit("%s: Message data was the wrong type! %d vs. requested=%d. Origin: %s",
	caller, fsa_data->data_type, a_type, fsa_data->origin);
	CRM_ASSERT(fsa_data->data_type == a_type);
	} else {
	ret_val = fsa_data->data;
	}

	return ret_val;
	}

	/* A_MSG_ROUTE */
	void
	do_msg_route(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	ha_msg_input_t *input = fsa_typed_data(fsa_dt_ha_msg);

	route_message(msg_data->fsa_cause, input->msg);
	}

	void
	route_message(enum crmd_fsa_cause cause, xmlNode * input)
	{
	ha_msg_input_t fsa_input;
	enum crmd_fsa_input result = I_NULL;

	fsa_input.msg = input;
	CRM_CHECK(cause == C_IPC_MESSAGE \|\| cause == C_HA_MESSAGE, return);

	/* try passing the buck first */
	if (relay_message(input, cause == C_IPC_MESSAGE)) {
	return;
	}

	/* handle locally */
	result = handle_message(input, cause);

	/* done or process later? */
	switch (result) {
	case I_NULL:
	case I_CIB_OP:
	case I_ROUTER:
	case I_NODE_JOIN:
	case I_JOIN_REQUEST:
	case I_JOIN_RESULT:
	break;
	default:
	/* Defering local processing of message */
	register_fsa_input_later(cause, result, &fsa_input);
	return;
	}

	if (result != I_NULL) {
	/* add to the front of the queue */
	register_fsa_input(cause, result, &fsa_input);
	}
	}

	gboolean
	relay_message(xmlNode * msg, gboolean originated_locally)
	{
	int dest = 1;
	int is_for_dc = 0;
	int is_for_dcib = 0;
	int is_for_te = 0;
	int is_for_crm = 0;
	int is_for_cib = 0;
	int is_local = 0;
	const char *host_to = crm_element_value(msg, F_CRM_HOST_TO);
	const char *sys_to = crm_element_value(msg, F_CRM_SYS_TO);
	const char *sys_from = crm_element_value(msg, F_CRM_SYS_FROM);
	const char *type = crm_element_value(msg, F_TYPE);
	const char *task = crm_element_value(msg, F_CRM_TASK);
	const char *ref = crm_element_value(msg, XML_ATTR_REFERENCE);

	if (ref == NULL) {
	ref = "without reference ID";
	}

	if (msg == NULL) {
	crm_warn("Cannot route empty message");
	return TRUE;

	} else if (pcmk__str_eq(task, CRM_OP_HELLO, pcmk__str_casei)) {
	/* quietly ignore */
	crm_trace("No routing needed for hello message %s", ref);
	return TRUE;

	} else if (!pcmk__str_eq(type, T_CRM, pcmk__str_casei)) {
	crm_warn("Cannot route message %s: Type is '%s' not '" T_CRM "'",
	ref, (type? type : "missing"));
	crm_log_xml_warn(msg, "[bad message type]");
	return TRUE;

	} else if (sys_to == NULL) {
	crm_warn("Cannot route message %s: No subsystem specified", ref);
	crm_log_xml_warn(msg, "[no subsystem]");
	return TRUE;
	}

	is_for_dc = (strcasecmp(CRM_SYSTEM_DC, sys_to) == 0);
	is_for_dcib = (strcasecmp(CRM_SYSTEM_DCIB, sys_to) == 0);
	is_for_te = (strcasecmp(CRM_SYSTEM_TENGINE, sys_to) == 0);
	is_for_cib = (strcasecmp(CRM_SYSTEM_CIB, sys_to) == 0);
	is_for_crm = (strcasecmp(CRM_SYSTEM_CRMD, sys_to) == 0);

	is_local = 0;
	if (pcmk__str_empty(host_to)) {
	if (is_for_dc \|\| is_for_te) {
	is_local = 0;

	} else if (is_for_crm) {
	if (pcmk__strcase_any_of(task, CRM_OP_NODE_INFO,
	PCMK__CONTROLD_CMD_NODES, NULL)) {
	/* Node info requests do not specify a host, which is normally
	* treated as "all hosts", because the whole point is that the
	* client may not know the local node name. Always handle these
	* requests locally.
	*/
	is_local = 1;
	} else {
	is_local = !originated_locally;
	}

	} else {
	is_local = 1;
	}

	} else if (pcmk__str_eq(fsa_our_uname, host_to, pcmk__str_casei)) {
	is_local = 1;
	} else if (is_for_crm && pcmk__str_eq(task, CRM_OP_LRM_DELETE, pcmk__str_casei)) {
	xmlNode *msg_data = get_message_xml(msg, F_CRM_DATA);
	const char *mode = crm_element_value(msg_data, PCMK__XA_MODE);

	if (pcmk__str_eq(mode, XML_TAG_CIB, pcmk__str_casei)) {
	// Local delete of an offline node's resource history
	is_local = 1;
	}
	}

	if (is_for_dc \|\| is_for_dcib \|\| is_for_te) {
	if (AM_I_DC && is_for_te) {
	crm_trace("Route message %s locally as transition request", ref);
	send_msg_via_ipc(msg, sys_to);

	} else if (AM_I_DC) {
	crm_trace("Route message %s locally as DC request", ref);
	return FALSE; // More to be done by caller

	} else if (originated_locally && !pcmk__strcase_any_of(sys_from, CRM_SYSTEM_PENGINE,
	CRM_SYSTEM_TENGINE, NULL)) {

	#if SUPPORT_COROSYNC
	if (is_corosync_cluster()) {
	dest = text2msg_type(sys_to);
	}
	#endif
	crm_trace("Relay message %s to DC", ref);
	send_cluster_message(host_to ? crm_get_peer(0, host_to) : NULL, dest, msg, TRUE);

	} else {
	/* Neither the TE nor the scheduler should be sending messages
	* to DCs on other nodes. By definition, if we are no longer the DC,
	* then the scheduler's or TE's data should be discarded.
	*/
	crm_trace("Discard message %s because we are not DC", ref);
	}

	} else if (is_local && (is_for_crm \|\| is_for_cib)) {
	crm_trace("Route message %s locally as controller request", ref);
	return FALSE; // More to be done by caller

	} else if (is_local) {
	crm_trace("Relay message %s locally to %s",
	ref, (sys_to? sys_to : "unknown client"));
	crm_log_xml_trace(msg, "[IPC relay]");
	send_msg_via_ipc(msg, sys_to);

	} else {
	crm_node_t *node_to = NULL;

	#if SUPPORT_COROSYNC
	if (is_corosync_cluster()) {
	dest = text2msg_type(sys_to);

	if (dest == crm_msg_none \|\| dest > crm_msg_stonith_ng) {
	dest = crm_msg_crmd;
	}
	}
	#endif

	if (host_to) {
	- node_to = crm_find_peer(0, host_to);
	+ node_to = pcmk__search_cluster_node_cache(0, host_to);
	if (node_to == NULL) {
	crm_warn("Cannot route message %s: Unknown node %s",
	ref, host_to);
	return TRUE;
	}
	crm_trace("Relay message %s to %s",
	ref, (node_to->uname? node_to->uname : "peer"));
	} else {
	crm_trace("Broadcast message %s to all peers", ref);
	}
	send_cluster_message(host_to ? node_to : NULL, dest, msg, TRUE);
	}

	return TRUE; // No further processing of message is needed
	}

	// Return true if field contains a positive integer
	static bool
	authorize_version(xmlNode message_data, const char field,
	const char client_name, const char ref, const char *uuid)
	{
	const char *version = crm_element_value(message_data, field);

	if (pcmk__str_empty(version)) {
	crm_warn("IPC hello from %s rejected: No protocol %s",
	CRM_XS " ref=%s uuid=%s",
	client_name, field, (ref? ref : "none"), uuid);
	return false;
	} else {
	int version_num = crm_parse_int(version, NULL);

	if (version_num < 0) {
	crm_warn("IPC hello from %s rejected: Protocol %s '%s' "
	"not recognized", CRM_XS " ref=%s uuid=%s",
	client_name, field, version, (ref? ref : "none"), uuid);
	return false;
	}
	}
	return true;
	}

	/*!
	* \internal
	* \brief Check whether a client IPC message is acceptable
	*
	* If a given client IPC message is a hello, "authorize" it by ensuring it has
	* valid information such as a protocol version, and return false indicating
	* that nothing further needs to be done with the message. If the message is not
	* a hello, just return true to indicate it needs further processing.
	*
	* \param[in] client_msg XML of IPC message
	* \param[in] curr_client If IPC is not proxied, client that sent message
	* \param[in] proxy_session If IPC is proxied, the session ID
	*
	* \return true if message needs further processing, false if it doesn't
	*/
	bool
	controld_authorize_ipc_message(xmlNode client_msg, pcmk__client_t curr_client,
	const char *proxy_session)
	{
	xmlNode *message_data = NULL;
	const char *client_name = NULL;
	const char *op = crm_element_value(client_msg, F_CRM_TASK);
	const char *ref = crm_element_value(client_msg, XML_ATTR_REFERENCE);
	const char *uuid = (curr_client? curr_client->id : proxy_session);

	if (uuid == NULL) {
	crm_warn("IPC message from client rejected: No client identifier "
	CRM_XS " ref=%s", (ref? ref : "none"));
	goto rejected;
	}

	if (!pcmk__str_eq(CRM_OP_HELLO, op, pcmk__str_casei)) {
	// Only hello messages need to be authorized
	return true;
	}

	message_data = get_message_xml(client_msg, F_CRM_DATA);

	client_name = crm_element_value(message_data, "client_name");
	if (pcmk__str_empty(client_name)) {
	crm_warn("IPC hello from client rejected: No client name",
	CRM_XS " ref=%s uuid=%s", (ref? ref : "none"), uuid);
	goto rejected;
	}
	if (!authorize_version(message_data, "major_version", client_name, ref,
	uuid)) {
	goto rejected;
	}
	if (!authorize_version(message_data, "minor_version", client_name, ref,
	uuid)) {
	goto rejected;
	}

	crm_trace("Validated IPC hello from client %s", client_name);
	if (curr_client) {
	curr_client->userdata = strdup(client_name);
	}
	mainloop_set_trigger(fsa_source);
	return false;

	rejected:
	if (curr_client) {
	qb_ipcs_disconnect(curr_client->ipcs);
	}
	return false;
	}

	static enum crmd_fsa_input
	handle_message(xmlNode *msg, enum crmd_fsa_cause cause)
	{
	const char *type = NULL;

	CRM_CHECK(msg != NULL, return I_NULL);

	type = crm_element_value(msg, F_CRM_MSG_TYPE);
	if (pcmk__str_eq(type, XML_ATTR_REQUEST, pcmk__str_none)) {
	return handle_request(msg, cause);

	} else if (pcmk__str_eq(type, XML_ATTR_RESPONSE, pcmk__str_none)) {
	handle_response(msg);
	return I_NULL;
	}

	crm_err("Unknown message type: %s", type);
	return I_NULL;
	}

	static enum crmd_fsa_input
	handle_failcount_op(xmlNode * stored_msg)
	{
	const char *rsc = NULL;
	const char *uname = NULL;
	const char *op = NULL;
	char *interval_spec = NULL;
	guint interval_ms = 0;
	gboolean is_remote_node = FALSE;
	xmlNode *xml_op = get_message_xml(stored_msg, F_CRM_DATA);

	if (xml_op) {
	xmlNode *xml_rsc = first_named_child(xml_op, XML_CIB_TAG_RESOURCE);
	xmlNode *xml_attrs = first_named_child(xml_op, XML_TAG_ATTRS);

	if (xml_rsc) {
	rsc = ID(xml_rsc);
	}
	if (xml_attrs) {
	op = crm_element_value(xml_attrs,
	CRM_META "_" XML_RSC_ATTR_CLEAR_OP);
	crm_element_value_ms(xml_attrs,
	CRM_META "_" XML_RSC_ATTR_CLEAR_INTERVAL,
	&interval_ms);
	}
	}
	uname = crm_element_value(xml_op, XML_LRM_ATTR_TARGET);

	if ((rsc == NULL) \|\| (uname == NULL)) {
	crm_log_xml_warn(stored_msg, "invalid failcount op");
	return I_NULL;
	}

	if (crm_element_value(xml_op, XML_LRM_ATTR_ROUTER_NODE)) {
	is_remote_node = TRUE;
	}

	if (interval_ms) {
	interval_spec = crm_strdup_printf("%ums", interval_ms);
	}
	update_attrd_clear_failures(uname, rsc, op, interval_spec, is_remote_node);
	free(interval_spec);

	lrm_clear_last_failure(rsc, uname, op, interval_ms);

	return I_NULL;
	}

	static enum crmd_fsa_input
	handle_lrm_delete(xmlNode *stored_msg)
	{
	const char *mode = NULL;
	xmlNode *msg_data = get_message_xml(stored_msg, F_CRM_DATA);

	CRM_CHECK(msg_data != NULL, return I_NULL);

	/* CRM_OP_LRM_DELETE has two distinct modes. The default behavior is to
	* relay the operation to the affected node, which will unregister the
	* resource from the local executor, clear the resource's history from the
	* CIB, and do some bookkeeping in the controller.
	*
	* However, if the affected node is offline, the client will specify
	* mode="cib" which means the controller receiving the operation should
	* clear the resource's history from the CIB and nothing else. This is used
	* to clear shutdown locks.
	*/
	mode = crm_element_value(msg_data, PCMK__XA_MODE);
	if ((mode == NULL) \|\| strcmp(mode, XML_TAG_CIB)) {
	// Relay to affected node
	crm_xml_add(stored_msg, F_CRM_SYS_TO, CRM_SYSTEM_LRMD);
	return I_ROUTER;

	} else {
	// Delete CIB history locally (compare with do_lrm_delete())
	const char *from_sys = NULL;
	const char *user_name = NULL;
	const char *rsc_id = NULL;
	const char *node = NULL;
	xmlNode *rsc_xml = NULL;
	int rc = pcmk_rc_ok;

	rsc_xml = first_named_child(msg_data, XML_CIB_TAG_RESOURCE);
	CRM_CHECK(rsc_xml != NULL, return I_NULL);

	rsc_id = ID(rsc_xml);
	from_sys = crm_element_value(stored_msg, F_CRM_SYS_FROM);
	node = crm_element_value(msg_data, XML_LRM_ATTR_TARGET);
	#if ENABLE_ACL
	user_name = pcmk__update_acl_user(stored_msg, F_CRM_USER, NULL);
	#endif
	crm_debug("Handling " CRM_OP_LRM_DELETE " for %s on %s locally%s%s "
	"(clearing CIB resource history only)", rsc_id, node,
	(user_name? " for user " : ""), (user_name? user_name : ""));
	#if ENABLE_ACL
	rc = controld_delete_resource_history(rsc_id, node, user_name,
	cib_dryrun\|cib_sync_call);
	#endif
	if (rc == pcmk_rc_ok) {
	rc = controld_delete_resource_history(rsc_id, node, user_name,
	crmd_cib_smart_opt());
	}

	//Notify client and tengine.(Only notify tengine if mode = "cib" and CRM_OP_LRM_DELETE.)
	if (from_sys) {
	lrmd_event_data_t *op = NULL;
	const char *from_host = crm_element_value(stored_msg,
	F_CRM_HOST_FROM);
	const char *transition;

	if (strcmp(from_sys, CRM_SYSTEM_TENGINE)) {
	transition = crm_element_value(msg_data,
	XML_ATTR_TRANSITION_KEY);
	} else {
	transition = crm_element_value(stored_msg,
	XML_ATTR_TRANSITION_KEY);
	}

	crm_info("Notifying %s on %s that %s was%s deleted",
	from_sys, (from_host? from_host : "local node"), rsc_id,
	((rc == pcmk_rc_ok)? "" : " not"));
	op = lrmd_new_event(rsc_id, CRMD_ACTION_DELETE, 0);
	op->type = lrmd_event_exec_complete;
	op->user_data = strdup(transition? transition : FAKE_TE_ID);
	op->params = crm_str_table_new();
	g_hash_table_insert(op->params, strdup(XML_ATTR_CRM_VERSION),
	strdup(CRM_FEATURE_SET));
	controld_rc2event(op, rc);
	controld_ack_event_directly(from_host, from_sys, NULL, op, rsc_id);
	lrmd_free_event(op);
	controld_trigger_delete_refresh(from_sys, rsc_id);
	}
	return I_NULL;
	}
	}

	/*!
	* \brief Handle a CRM_OP_REMOTE_STATE message by updating remote peer cache
	*
	* \param[in] msg Message XML
	*
	* \return Next FSA input
	*/
	static enum crmd_fsa_input
	handle_remote_state(xmlNode *msg)
	{
	const char *remote_uname = ID(msg);
	const char *remote_is_up = crm_element_value(msg, XML_NODE_IN_CLUSTER);
	crm_node_t *remote_peer;

	CRM_CHECK(remote_uname && remote_is_up, return I_NULL);

	remote_peer = crm_remote_peer_get(remote_uname);
	CRM_CHECK(remote_peer, return I_NULL);

	- crm_update_peer_state(__func__, remote_peer,
	- crm_is_true(remote_is_up)?
	- CRM_NODE_MEMBER : CRM_NODE_LOST, 0);
	+ pcmk__update_peer_state(__func__, remote_peer,
	+ crm_is_true(remote_is_up)? CRM_NODE_MEMBER : CRM_NODE_LOST,
	+ 0);
	return I_NULL;
	}

	/*!
	* \brief Handle a CRM_OP_PING message
	*
	* \param[in] msg Message XML
	*
	* \return Next FSA input
	*/
	static enum crmd_fsa_input
	handle_ping(xmlNode *msg)
	{
	const char *value = NULL;
	xmlNode *ping = NULL;

	// Build reply

	ping = create_xml_node(NULL, XML_CRM_TAG_PING);
	value = crm_element_value(msg, F_CRM_SYS_TO);
	crm_xml_add(ping, XML_PING_ATTR_SYSFROM, value);

	// Add controller state
	value = fsa_state2string(fsa_state);
	crm_xml_add(ping, XML_PING_ATTR_CRMDSTATE, value);
	crm_notice("Current ping state: %s", value); // CTS needs this

	// Add controller health
	// @TODO maybe do some checks to determine meaningful status
	crm_xml_add(ping, XML_PING_ATTR_STATUS, "ok");

	// Send reply
	msg = create_reply(msg, ping);
	free_xml(ping);
	if (msg) {
	(void) relay_message(msg, TRUE);
	free_xml(msg);
	}

	// Nothing further to do
	return I_NULL;
	}

	/*!
	* \brief Handle a PCMK__CONTROLD_CMD_NODES message
	*
	* \return Next FSA input
	*/
	static enum crmd_fsa_input
	handle_node_list(xmlNode *request)
	{
	GHashTableIter iter;
	crm_node_t *node = NULL;
	xmlNode *reply = NULL;
	xmlNode *reply_data = NULL;

	// Create message data for reply
	reply_data = create_xml_node(NULL, XML_CIB_TAG_NODES);
	g_hash_table_iter_init(&iter, crm_peer_cache);
	while (g_hash_table_iter_next(&iter, NULL, (gpointer *) & node)) {
	xmlNode *xml = create_xml_node(reply_data, XML_CIB_TAG_NODE);

	crm_xml_add_ll(xml, XML_ATTR_ID, (long long) node->id); // uint32_t
	crm_xml_add(xml, XML_ATTR_UNAME, node->uname);
	crm_xml_add(xml, XML_NODE_IN_CLUSTER, node->state);
	}

	// Create and send reply
	reply = create_reply(request, reply_data);
	free_xml(reply_data);
	if (reply) {
	(void) relay_message(reply, TRUE);
	free_xml(reply);
	}

	// Nothing further to do
	return I_NULL;
	}

	/*!
	* \brief Handle a CRM_OP_NODE_INFO request
	*
	* \param[in] msg Message XML
	*
	* \return Next FSA input
	*/
	static enum crmd_fsa_input
	handle_node_info_request(xmlNode *msg)
	{
	const char *value = NULL;
	crm_node_t *node = NULL;
	int node_id = 0;
	xmlNode *reply = NULL;

	// Build reply

	reply = create_xml_node(NULL, XML_CIB_TAG_NODE);
	crm_xml_add(reply, XML_PING_ATTR_SYSFROM, CRM_SYSTEM_CRMD);

	// Add whether current partition has quorum
	crm_xml_add_boolean(reply, XML_ATTR_HAVE_QUORUM, fsa_has_quorum);

	// Check whether client requested node info by ID and/or name
	crm_element_value_int(msg, XML_ATTR_ID, &node_id);
	if (node_id < 0) {
	node_id = 0;
	}
	value = crm_element_value(msg, XML_ATTR_UNAME);

	// Default to local node if none given
	if ((node_id == 0) && (value == NULL)) {
	value = fsa_our_uname;
	}

	- node = crm_find_peer_full(node_id, value, CRM_GET_PEER_ANY);
	+ node = pcmk__search_node_caches(node_id, value, CRM_GET_PEER_ANY);
	if (node) {
	crm_xml_add_int(reply, XML_ATTR_ID, node->id);
	crm_xml_add(reply, XML_ATTR_UUID, node->uuid);
	crm_xml_add(reply, XML_ATTR_UNAME, node->uname);
	crm_xml_add(reply, XML_NODE_IS_PEER, node->state);
	crm_xml_add_boolean(reply, XML_NODE_IS_REMOTE,
	node->flags & crm_remote_node);
	}

	// Send reply
	msg = create_reply(msg, reply);
	free_xml(reply);
	if (msg) {
	(void) relay_message(msg, TRUE);
	free_xml(msg);
	}

	// Nothing further to do
	return I_NULL;
	}

	static void
	verify_feature_set(xmlNode *msg)
	{
	const char *dc_version = crm_element_value(msg, XML_ATTR_CRM_VERSION);

	if (dc_version == NULL) {
	/* All we really know is that the DC feature set is older than 3.1.0,
	* but that's also all that really matters.
	*/
	dc_version = "3.0.14";
	}

	if (feature_set_compatible(dc_version, CRM_FEATURE_SET)) {
	crm_trace("Local feature set (%s) is compatible with DC's (%s)",
	CRM_FEATURE_SET, dc_version);
	} else {
	crm_err("Local feature set (%s) is incompatible with DC's (%s)",
	CRM_FEATURE_SET, dc_version);

	// Nothing is likely to improve without administrator involvement
	controld_set_fsa_input_flags(R_STAYDOWN);
	crmd_exit(CRM_EX_FATAL);
	}
	}

	// DC gets own shutdown all-clear
	static enum crmd_fsa_input
	handle_shutdown_self_ack(xmlNode *stored_msg)
	{
	const char *host_from = crm_element_value(stored_msg, F_CRM_HOST_FROM);

	if (pcmk_is_set(fsa_input_register, R_SHUTDOWN)) {
	// The expected case -- we initiated own shutdown sequence
	crm_info("Shutting down controller");
	return I_STOP;
	}

	if (pcmk__str_eq(host_from, fsa_our_dc, pcmk__str_casei)) {
	// Must be logic error -- DC confirming its own unrequested shutdown
	crm_err("Shutting down controller immediately due to "
	"unexpected shutdown confirmation");
	return I_TERMINATE;
	}

	if (fsa_state != S_STOPPING) {
	// Shouldn't happen -- non-DC confirming unrequested shutdown
	crm_err("Starting new DC election because %s is "
	"confirming shutdown we did not request",
	(host_from? host_from : "another node"));
	return I_ELECTION;
	}

	// Shouldn't happen, but we are already stopping anyway
	crm_debug("Ignoring unexpected shutdown confirmation from %s",
	(host_from? host_from : "another node"));
	return I_NULL;
	}

	// Non-DC gets shutdown all-clear from DC
	static enum crmd_fsa_input
	handle_shutdown_ack(xmlNode *stored_msg)
	{
	const char *host_from = crm_element_value(stored_msg, F_CRM_HOST_FROM);

	if (host_from == NULL) {
	crm_warn("Ignoring shutdown request without origin specified");
	return I_NULL;
	}

	if ((fsa_our_dc == NULL) \|\| (strcmp(host_from, fsa_our_dc) == 0)) {

	if (pcmk_is_set(fsa_input_register, R_SHUTDOWN)) {
	crm_info("Shutting down controller after confirmation from %s",
	host_from);
	} else {
	crm_err("Shutting down controller after unexpected "
	"shutdown request from %s", host_from);
	controld_set_fsa_input_flags(R_STAYDOWN);
	}
	return I_STOP;
	}

	crm_warn("Ignoring shutdown request from %s because DC is %s",
	host_from, fsa_our_dc);
	return I_NULL;
	}

	static enum crmd_fsa_input
	handle_request(xmlNode *stored_msg, enum crmd_fsa_cause cause)
	{
	xmlNode *msg = NULL;
	const char *op = crm_element_value(stored_msg, F_CRM_TASK);

	/* Optimize this for the DC - it has the most to do */

	if (op == NULL) {
	crm_log_xml_warn(stored_msg, "[request without " F_CRM_TASK "]");
	return I_NULL;
	}

	if (strcmp(op, CRM_OP_SHUTDOWN_REQ) == 0) {
	const char *from = crm_element_value(stored_msg, F_CRM_HOST_FROM);
	- crm_node_t *node = crm_find_peer(0, from);
	+ crm_node_t *node = pcmk__search_cluster_node_cache(0, from);

	- crm_update_peer_expected(__func__, node, CRMD_JOINSTATE_DOWN);
	+ pcmk__update_peer_expected(__func__, node, CRMD_JOINSTATE_DOWN);
	if(AM_I_DC == FALSE) {
	return I_NULL; /* Done */
	}
	}

	/========== DC-Only Actions ==========/
	if (AM_I_DC) {
	if (strcmp(op, CRM_OP_JOIN_ANNOUNCE) == 0) {
	return I_NODE_JOIN;

	} else if (strcmp(op, CRM_OP_JOIN_REQUEST) == 0) {
	return I_JOIN_REQUEST;

	} else if (strcmp(op, CRM_OP_JOIN_CONFIRM) == 0) {
	return I_JOIN_RESULT;

	} else if (strcmp(op, CRM_OP_SHUTDOWN) == 0) {
	return handle_shutdown_self_ack(stored_msg);

	} else if (strcmp(op, CRM_OP_SHUTDOWN_REQ) == 0) {
	/* a slave wants to shut down */
	/* create cib fragment and add to message */
	return handle_shutdown_request(stored_msg);

	} else if (strcmp(op, CRM_OP_REMOTE_STATE) == 0) {
	/* a remote connection host is letting us know the node state */
	return handle_remote_state(stored_msg);
	}
	}

	/========== common actions ==========/
	if (strcmp(op, CRM_OP_NOVOTE) == 0) {
	ha_msg_input_t fsa_input;

	fsa_input.msg = stored_msg;
	register_fsa_input_adv(C_HA_MESSAGE, I_NULL, &fsa_input,
	A_ELECTION_COUNT \| A_ELECTION_CHECK, FALSE,
	__func__);

	} else if (strcmp(op, CRM_OP_THROTTLE) == 0) {
	throttle_update(stored_msg);
	if (AM_I_DC && transition_graph != NULL) {
	if (transition_graph->complete == FALSE) {
	crm_debug("The throttle changed. Trigger a graph.");
	trigger_graph();
	}
	}
	return I_NULL;

	} else if (strcmp(op, CRM_OP_CLEAR_FAILCOUNT) == 0) {
	return handle_failcount_op(stored_msg);

	} else if (strcmp(op, CRM_OP_VOTE) == 0) {
	/* count the vote and decide what to do after that */
	ha_msg_input_t fsa_input;

	fsa_input.msg = stored_msg;
	register_fsa_input_adv(C_HA_MESSAGE, I_NULL, &fsa_input,
	A_ELECTION_COUNT \| A_ELECTION_CHECK, FALSE,
	__func__);

	/* Sometimes we _must_ go into S_ELECTION */
	if (fsa_state == S_HALT) {
	crm_debug("Forcing an election from S_HALT");
	return I_ELECTION;
	#if 0
	} else if (AM_I_DC) {
	/* This is the old way of doing things but what is gained? */
	return I_ELECTION;
	#endif
	}

	} else if (strcmp(op, CRM_OP_JOIN_OFFER) == 0) {
	verify_feature_set(stored_msg);
	crm_debug("Raising I_JOIN_OFFER: join-%s", crm_element_value(stored_msg, F_CRM_JOIN_ID));
	return I_JOIN_OFFER;

	} else if (strcmp(op, CRM_OP_JOIN_ACKNAK) == 0) {
	crm_debug("Raising I_JOIN_RESULT: join-%s", crm_element_value(stored_msg, F_CRM_JOIN_ID));
	return I_JOIN_RESULT;

	} else if (strcmp(op, CRM_OP_LRM_DELETE) == 0) {
	return handle_lrm_delete(stored_msg);

	} else if ((strcmp(op, CRM_OP_LRM_FAIL) == 0)
	\|\| (strcmp(op, CRM_OP_LRM_REFRESH) == 0)
	\|\| (strcmp(op, CRM_OP_REPROBE) == 0)) {

	crm_xml_add(stored_msg, F_CRM_SYS_TO, CRM_SYSTEM_LRMD);
	return I_ROUTER;

	} else if (strcmp(op, CRM_OP_NOOP) == 0) {
	return I_NULL;

	} else if (strcmp(op, CRM_OP_LOCAL_SHUTDOWN) == 0) {

	crm_shutdown(SIGTERM);
	/return I_SHUTDOWN; /
	return I_NULL;

	} else if (strcmp(op, CRM_OP_PING) == 0) {
	return handle_ping(stored_msg);

	} else if (strcmp(op, CRM_OP_NODE_INFO) == 0) {
	return handle_node_info_request(stored_msg);

	} else if (strcmp(op, CRM_OP_RM_NODE_CACHE) == 0) {
	int id = 0;
	const char *name = NULL;

	crm_element_value_int(stored_msg, XML_ATTR_ID, &id);
	name = crm_element_value(stored_msg, XML_ATTR_UNAME);

	if(cause == C_IPC_MESSAGE) {
	msg = create_request(CRM_OP_RM_NODE_CACHE, NULL, NULL, CRM_SYSTEM_CRMD, CRM_SYSTEM_CRMD, NULL);
	if (send_cluster_message(NULL, crm_msg_crmd, msg, TRUE) == FALSE) {
	crm_err("Could not instruct peers to remove references to node %s/%u", name, id);
	} else {
	crm_notice("Instructing peers to remove references to node %s/%u", name, id);
	}
	free_xml(msg);

	} else {
	reap_crm_member(id, name);

	/* If we're forgetting this node, also forget any failures to fence
	* it, so we don't carry that over to any node added later with the
	* same name.
	*/
	st_fail_count_reset(name);
	}

	} else if (strcmp(op, CRM_OP_MAINTENANCE_NODES) == 0) {
	xmlNode *xml = get_message_xml(stored_msg, F_CRM_DATA);

	remote_ra_process_maintenance_nodes(xml);

	} else if (strcmp(op, PCMK__CONTROLD_CMD_NODES) == 0) {
	return handle_node_list(stored_msg);

	/========== (NOT_DC)-Only Actions ==========/
	} else if (!AM_I_DC) {

	if (strcmp(op, CRM_OP_SHUTDOWN) == 0) {
	return handle_shutdown_ack(stored_msg);
	}

	} else {
	crm_err("Unexpected request (%s) sent to %s", op, AM_I_DC ? "the DC" : "non-DC node");
	crm_log_xml_err(stored_msg, "Unexpected");
	}

	return I_NULL;
	}

	static void
	handle_response(xmlNode *stored_msg)
	{
	const char *op = crm_element_value(stored_msg, F_CRM_TASK);

	if (op == NULL) {
	crm_log_xml_err(stored_msg, "Bad message");

	} else if (AM_I_DC && strcmp(op, CRM_OP_PECALC) == 0) {
	// Check whether scheduler answer been superseded by subsequent request
	const char *msg_ref = crm_element_value(stored_msg, XML_ATTR_REFERENCE);

	if (msg_ref == NULL) {
	crm_err("%s - Ignoring calculation with no reference", op);

	} else if (pcmk__str_eq(msg_ref, fsa_pe_ref, pcmk__str_casei)) {
	ha_msg_input_t fsa_input;

	controld_stop_sched_timer();
	fsa_input.msg = stored_msg;
	register_fsa_input_later(C_IPC_MESSAGE, I_PE_SUCCESS, &fsa_input);

	} else {
	crm_info("%s calculation %s is obsolete", op, msg_ref);
	}

	} else if (strcmp(op, CRM_OP_VOTE) == 0
	\|\| strcmp(op, CRM_OP_SHUTDOWN_REQ) == 0 \|\| strcmp(op, CRM_OP_SHUTDOWN) == 0) {

	} else {
	const char *host_from = crm_element_value(stored_msg, F_CRM_HOST_FROM);

	crm_err("Unexpected response (op=%s, src=%s) sent to the %s",
	op, host_from, AM_I_DC ? "DC" : "controller");
	}
	}

	static enum crmd_fsa_input
	handle_shutdown_request(xmlNode * stored_msg)
	{
	/* handle here to avoid potential version issues
	* where the shutdown message/procedure may have
	* been changed in later versions.
	*
	* This way the DC is always in control of the shutdown
	*/

	char *now_s = NULL;
	time_t now = time(NULL);
	const char *host_from = crm_element_value(stored_msg, F_CRM_HOST_FROM);

	if (host_from == NULL) {
	/* we're shutting down and the DC */
	host_from = fsa_our_uname;
	}

	crm_info("Creating shutdown request for %s (state=%s)", host_from, fsa_state2string(fsa_state));
	crm_log_xml_trace(stored_msg, "message");

	now_s = crm_itoa(now);
	update_attrd(host_from, XML_CIB_ATTR_SHUTDOWN, now_s, NULL, FALSE);
	free(now_s);

	/* will be picked up by the TE as long as its running */
	return I_NULL;
	}

	/* msg is deleted by the time this returns */
	extern gboolean process_te_message(xmlNode * msg, xmlNode * xml_data);

	static void
	send_msg_via_ipc(xmlNode * msg, const char *sys)
	{
	pcmk__client_t *client_channel = pcmk__find_client_by_id(sys);

	if (crm_element_value(msg, F_CRM_HOST_FROM) == NULL) {
	crm_xml_add(msg, F_CRM_HOST_FROM, fsa_our_uname);
	}

	if (client_channel != NULL) {
	/* Transient clients such as crmadmin */
	pcmk__ipc_send_xml(client_channel, 0, msg, crm_ipc_server_event);

	} else if (sys != NULL && strcmp(sys, CRM_SYSTEM_TENGINE) == 0) {
	xmlNode *data = get_message_xml(msg, F_CRM_DATA);

	process_te_message(msg, data);

	} else if (sys != NULL && strcmp(sys, CRM_SYSTEM_LRMD) == 0) {
	fsa_data_t fsa_data;
	ha_msg_input_t fsa_input;

	fsa_input.msg = msg;
	fsa_input.xml = get_message_xml(msg, F_CRM_DATA);

	fsa_data.id = 0;
	fsa_data.actions = 0;
	fsa_data.data = &fsa_input;
	fsa_data.fsa_input = I_MESSAGE;
	fsa_data.fsa_cause = C_IPC_MESSAGE;
	fsa_data.origin = __func__;
	fsa_data.data_type = fsa_dt_ha_msg;

	do_lrm_invoke(A_LRM_INVOKE, C_IPC_MESSAGE, fsa_state, I_MESSAGE, &fsa_data);

	} else if (sys != NULL && crmd_is_proxy_session(sys)) {
	crmd_proxy_send(sys, msg);

	} else {
	crm_debug("Unknown Sub-system (%s)... discarding message.", crm_str(sys));
	}
	}

	void
	delete_ha_msg_input(ha_msg_input_t * orig)
	{
	if (orig == NULL) {
	return;
	}
	free_xml(orig->msg);
	free(orig);
	}

	/*!
	* \internal
	* \brief Notify the DC of a remote node state change
	*
	* \param[in] node_name Node's name
	* \param[in] node_up TRUE if node is up, FALSE if down
	*/
	void
	send_remote_state_message(const char *node_name, gboolean node_up)
	{
	/* If we don't have a DC, or the message fails, we have a failsafe:
	* the DC will eventually pick up the change via the CIB node state.
	* The message allows it to happen sooner if possible.
	*/
	if (fsa_our_dc) {
	xmlNode *msg = create_request(CRM_OP_REMOTE_STATE, NULL, fsa_our_dc,
	CRM_SYSTEM_DC, CRM_SYSTEM_CRMD, NULL);

	crm_info("Notifying DC %s of pacemaker_remote node %s %s",
	fsa_our_dc, node_name, (node_up? "coming up" : "going down"));
	crm_xml_add(msg, XML_ATTR_ID, node_name);
	crm_xml_add_boolean(msg, XML_NODE_IN_CLUSTER, node_up);
	send_cluster_message(crm_get_peer(0, fsa_our_dc), crm_msg_crmd, msg,
	TRUE);
	free_xml(msg);
	} else {
	crm_debug("No DC to notify of pacemaker_remote node %s %s",
	node_name, (node_up? "coming up" : "going down"));
	}
	}

	diff --git a/daemons/controld/controld_remote_ra.c b/daemons/controld/controld_remote_ra.c
	index ad1808798b..9b111fe78a 100644
	--- a/daemons/controld/controld_remote_ra.c
	+++ b/daemons/controld/controld_remote_ra.c
	@@ -1,1299 +1,1299 @@
	/*
	* Copyright 2013-2020 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <crm/crm.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml_internal.h>
	#include <crm/lrmd.h>
	#include <crm/services.h>

	#include <pacemaker-controld.h>

	#define REMOTE_LRMD_RA "remote"

	/* The max start timeout before cmd retry */
	#define MAX_START_TIMEOUT_MS 10000

	typedef struct remote_ra_cmd_s {
	/! the local node the cmd is issued from /
	char *owner;
	/! the remote node the cmd is executed on /
	char *rsc_id;
	/! the action to execute /
	char *action;
	/! some string the client wants us to give it back /
	char *userdata;
	char *exit_reason; // descriptive text on error
	/! start delay in ms /
	int start_delay;
	/! timer id used for start delay. /
	int delay_id;
	/! timeout in ms for cmd /
	int timeout;
	int remaining_timeout;
	/! recurring interval in ms /
	guint interval_ms;
	/! interval timer id /
	int interval_id;
	int reported_success;
	int monitor_timeout_id;
	int takeover_timeout_id;
	/! action parameters /
	lrmd_key_value_t *params;
	/! executed rc /
	int rc;
	int op_status;
	int call_id;
	time_t start_time;
	gboolean cancel;
	} remote_ra_cmd_t;

	enum remote_migration_status {
	expect_takeover = 1,
	takeover_complete,
	};

	typedef struct remote_ra_data_s {
	crm_trigger_t *work;
	remote_ra_cmd_t *cur_cmd;
	GList *cmds;
	GList *recurring_cmds;

	enum remote_migration_status migrate_status;

	gboolean active;

	/* Maintenance mode is difficult to determine from the controller's context,
	* so we have it signalled back with the transition from the scheduler.
	*/
	gboolean is_maintenance;

	/* Similar for whether we are controlling a guest node or remote node.
	* Fortunately there is a meta-attribute in the transition already and
	* as the situation doesn't change over time we can use the
	* resource start for noting down the information for later use when
	* the attributes aren't at hand.
	*/
	gboolean controlling_guest;
	} remote_ra_data_t;

	static int handle_remote_ra_start(lrm_state_t * lrm_state, remote_ra_cmd_t * cmd, int timeout_ms);
	static void handle_remote_ra_stop(lrm_state_t * lrm_state, remote_ra_cmd_t * cmd);
	static GList fail_all_monitor_cmds(GList list);

	static void
	free_cmd(gpointer user_data)
	{
	remote_ra_cmd_t *cmd = user_data;

	if (!cmd) {
	return;
	}
	if (cmd->delay_id) {
	g_source_remove(cmd->delay_id);
	}
	if (cmd->interval_id) {
	g_source_remove(cmd->interval_id);
	}
	if (cmd->monitor_timeout_id) {
	g_source_remove(cmd->monitor_timeout_id);
	}
	if (cmd->takeover_timeout_id) {
	g_source_remove(cmd->takeover_timeout_id);
	}
	free(cmd->owner);
	free(cmd->rsc_id);
	free(cmd->action);
	free(cmd->userdata);
	free(cmd->exit_reason);
	lrmd_key_value_freeall(cmd->params);
	free(cmd);
	}

	static int
	generate_callid(void)
	{
	static int remote_ra_callid = 0;

	remote_ra_callid++;
	if (remote_ra_callid <= 0) {
	remote_ra_callid = 1;
	}

	return remote_ra_callid;
	}

	static gboolean
	recurring_helper(gpointer data)
	{
	remote_ra_cmd_t *cmd = data;
	lrm_state_t *connection_rsc = NULL;

	cmd->interval_id = 0;
	connection_rsc = lrm_state_find(cmd->rsc_id);
	if (connection_rsc && connection_rsc->remote_ra_data) {
	remote_ra_data_t *ra_data = connection_rsc->remote_ra_data;

	ra_data->recurring_cmds = g_list_remove(ra_data->recurring_cmds, cmd);

	ra_data->cmds = g_list_append(ra_data->cmds, cmd);
	mainloop_set_trigger(ra_data->work);
	}
	return FALSE;
	}

	static gboolean
	start_delay_helper(gpointer data)
	{
	remote_ra_cmd_t *cmd = data;
	lrm_state_t *connection_rsc = NULL;

	cmd->delay_id = 0;
	connection_rsc = lrm_state_find(cmd->rsc_id);
	if (connection_rsc && connection_rsc->remote_ra_data) {
	remote_ra_data_t *ra_data = connection_rsc->remote_ra_data;

	mainloop_set_trigger(ra_data->work);
	}
	return FALSE;
	}

	/*!
	* \internal
	* \brief Handle cluster communication related to pacemaker_remote node joining
	*
	* \param[in] node_name Name of newly integrated pacemaker_remote node
	*/
	static void
	remote_node_up(const char *node_name)
	{
	int call_opt, call_id = 0;
	xmlNode update, state;
	crm_node_t *node;
	enum controld_section_e section = controld_section_all;

	CRM_CHECK(node_name != NULL, return);
	crm_info("Announcing pacemaker_remote node %s", node_name);

	/* Clear node's entire state (resource history and transient attributes)
	* other than shutdown locks. The transient attributes should and normally
	* will be cleared when the node leaves, but since remote node state has a
	* number of corner cases, clear them here as well, to be sure.
	*/
	call_opt = crmd_cib_smart_opt();
	if (controld_shutdown_lock_enabled) {
	section = controld_section_all_unlocked;
	}
	controld_delete_node_state(node_name, section, call_opt);

	/* Clear node's probed attribute */
	update_attrd(node_name, CRM_OP_PROBED, NULL, NULL, TRUE);

	/* Ensure node is in the remote peer cache with member status */
	node = crm_remote_peer_get(node_name);
	CRM_CHECK(node != NULL, return);
	- crm_update_peer_state(__func__, node, CRM_NODE_MEMBER, 0);
	+ pcmk__update_peer_state(__func__, node, CRM_NODE_MEMBER, 0);

	/* pacemaker_remote nodes don't participate in the membership layer,
	* so cluster nodes don't automatically get notified when they come and go.
	* We send a cluster message to the DC, and update the CIB node state entry,
	* so the DC will get it sooner (via message) or later (via CIB refresh),
	* and any other interested parties can query the CIB.
	*/
	send_remote_state_message(node_name, TRUE);

	update = create_xml_node(NULL, XML_CIB_TAG_STATUS);
	state = create_node_state_update(node, node_update_cluster, update,
	__func__);

	/* Clear the XML_NODE_IS_FENCED flag in the node state. If the node ever
	* needs to be fenced, this flag will allow various actions to determine
	* whether the fencing has happened yet.
	*/
	crm_xml_add(state, XML_NODE_IS_FENCED, "0");

	/* TODO: If the remote connection drops, and this (async) CIB update either
	* failed or has not yet completed, later actions could mistakenly think the
	* node has already been fenced (if the XML_NODE_IS_FENCED attribute was
	* previously set, because it won't have been cleared). This could prevent
	* actual fencing or allow recurring monitor failures to be cleared too
	* soon. Ideally, we wouldn't rely on the CIB for the fenced status.
	*/
	fsa_cib_update(XML_CIB_TAG_STATUS, update, call_opt, call_id, NULL);
	if (call_id < 0) {
	crm_perror(LOG_WARNING, "%s CIB node state setup", node_name);
	}
	free_xml(update);
	}

	enum down_opts {
	DOWN_KEEP_LRM,
	DOWN_ERASE_LRM
	};

	/*!
	* \internal
	* \brief Handle cluster communication related to pacemaker_remote node leaving
	*
	* \param[in] node_name Name of lost node
	* \param[in] opts Whether to keep or erase LRM history
	*/
	static void
	remote_node_down(const char *node_name, const enum down_opts opts)
	{
	xmlNode *update;
	int call_id = 0;
	int call_opt = crmd_cib_smart_opt();
	crm_node_t *node;

	/* Purge node from attrd's memory */
	update_attrd_remote_node_removed(node_name, NULL);

	/* Normally, only node attributes should be erased, and the resource history
	* should be kept until the node comes back up. However, after a successful
	* fence, we want to clear the history as well, so we don't think resources
	* are still running on the node.
	*/
	if (opts == DOWN_ERASE_LRM) {
	controld_delete_node_state(node_name, controld_section_all, call_opt);
	} else {
	controld_delete_node_state(node_name, controld_section_attrs, call_opt);
	}

	/* Ensure node is in the remote peer cache with lost state */
	node = crm_remote_peer_get(node_name);
	CRM_CHECK(node != NULL, return);
	- crm_update_peer_state(__func__, node, CRM_NODE_LOST, 0);
	+ pcmk__update_peer_state(__func__, node, CRM_NODE_LOST, 0);

	/* Notify DC */
	send_remote_state_message(node_name, FALSE);

	/* Update CIB node state */
	update = create_xml_node(NULL, XML_CIB_TAG_STATUS);
	create_node_state_update(node, node_update_cluster, update, __func__);
	fsa_cib_update(XML_CIB_TAG_STATUS, update, call_opt, call_id, NULL);
	if (call_id < 0) {
	crm_perror(LOG_ERR, "%s CIB node state update", node_name);
	}
	free_xml(update);
	}

	/*!
	* \internal
	* \brief Handle effects of a remote RA command on node state
	*
	* \param[in] cmd Completed remote RA command
	*/
	static void
	check_remote_node_state(remote_ra_cmd_t *cmd)
	{
	/* Only successful actions can change node state */
	if (cmd->rc != PCMK_OCF_OK) {
	return;
	}

	if (pcmk__str_eq(cmd->action, "start", pcmk__str_casei)) {
	remote_node_up(cmd->rsc_id);

	} else if (pcmk__str_eq(cmd->action, "migrate_from", pcmk__str_casei)) {
	/* After a successful migration, we don't need to do remote_node_up()
	* because the DC already knows the node is up, and we don't want to
	* clear LRM history etc. We do need to add the remote node to this
	* host's remote peer cache, because (unless it happens to be DC)
	* it hasn't been tracking the remote node, and other code relies on
	* the cache to distinguish remote nodes from unseen cluster nodes.
	*/
	crm_node_t *node = crm_remote_peer_get(cmd->rsc_id);

	CRM_CHECK(node != NULL, return);
	- crm_update_peer_state(__func__, node, CRM_NODE_MEMBER, 0);
	+ pcmk__update_peer_state(__func__, node, CRM_NODE_MEMBER, 0);

	} else if (pcmk__str_eq(cmd->action, "stop", pcmk__str_casei)) {
	lrm_state_t *lrm_state = lrm_state_find(cmd->rsc_id);
	remote_ra_data_t *ra_data = lrm_state? lrm_state->remote_ra_data : NULL;

	if (ra_data) {
	if (ra_data->migrate_status != takeover_complete) {
	/* Stop means down if we didn't successfully migrate elsewhere */
	remote_node_down(cmd->rsc_id, DOWN_KEEP_LRM);
	} else if (AM_I_DC == FALSE) {
	/* Only the connection host and DC track node state,
	* so if the connection migrated elsewhere and we aren't DC,
	* un-cache the node, so we don't have stale info
	*/
	crm_remote_peer_cache_remove(cmd->rsc_id);
	}
	}
	}

	/* We don't do anything for successful monitors, which is correct for
	* routine recurring monitors, and for monitors on nodes where the
	* connection isn't supposed to be (the cluster will stop the connection in
	* that case). However, if the initial probe finds the connection already
	* active on the node where we want it, we probably should do
	* remote_node_up(). Unfortunately, we can't distinguish that case here.
	* Given that connections have to be initiated by the cluster, the chance of
	* that should be close to zero.
	*/
	}

	static void
	report_remote_ra_result(remote_ra_cmd_t * cmd)
	{
	lrmd_event_data_t op = { 0, };

	check_remote_node_state(cmd);

	op.type = lrmd_event_exec_complete;
	op.rsc_id = cmd->rsc_id;
	op.op_type = cmd->action;
	op.user_data = cmd->userdata;
	op.exit_reason = cmd->exit_reason;
	op.timeout = cmd->timeout;
	op.interval_ms = cmd->interval_ms;
	op.rc = cmd->rc;
	op.op_status = cmd->op_status;
	op.t_run = (unsigned int) cmd->start_time;
	op.t_rcchange = (unsigned int) cmd->start_time;
	if (cmd->reported_success && cmd->rc != PCMK_OCF_OK) {
	op.t_rcchange = (unsigned int) time(NULL);
	/* This edge case will likely never ever occur, but if it does the
	* result is that a failure will not be processed correctly. This is only
	* remotely possible because we are able to detect a connection resource's tcp
	* connection has failed at any moment after start has completed. The actual
	* recurring operation is just a connectivity ping.
	*
	* basically, we are not guaranteed that the first successful monitor op and
	* a subsequent failed monitor op will not occur in the same timestamp. We have to
	* make it look like the operations occurred at separate times though. */
	if (op.t_rcchange == op.t_run) {
	op.t_rcchange++;
	}
	}

	if (cmd->params) {
	lrmd_key_value_t *tmp;

	op.params = crm_str_table_new();
	for (tmp = cmd->params; tmp; tmp = tmp->next) {
	g_hash_table_insert(op.params, strdup(tmp->key), strdup(tmp->value));
	}

	}
	op.call_id = cmd->call_id;
	op.remote_nodename = cmd->owner;

	lrm_op_callback(&op);

	if (op.params) {
	g_hash_table_destroy(op.params);
	}
	}

	static void
	update_remaining_timeout(remote_ra_cmd_t * cmd)
	{
	cmd->remaining_timeout = ((cmd->timeout / 1000) - (time(NULL) - cmd->start_time)) * 1000;
	}

	static gboolean
	retry_start_cmd_cb(gpointer data)
	{
	lrm_state_t *lrm_state = data;
	remote_ra_data_t *ra_data = lrm_state->remote_ra_data;
	remote_ra_cmd_t *cmd = NULL;
	int rc = -1;

	if (!ra_data \|\| !ra_data->cur_cmd) {
	return FALSE;
	}
	cmd = ra_data->cur_cmd;
	if (!pcmk__strcase_any_of(cmd->action, "start", "migrate_from", NULL)) {
	return FALSE;
	}
	update_remaining_timeout(cmd);

	if (cmd->remaining_timeout > 0) {
	rc = handle_remote_ra_start(lrm_state, cmd, cmd->remaining_timeout);
	}

	if (rc != 0) {
	cmd->rc = PCMK_OCF_UNKNOWN_ERROR;
	cmd->op_status = PCMK_LRM_OP_ERROR;
	report_remote_ra_result(cmd);

	if (ra_data->cmds) {
	mainloop_set_trigger(ra_data->work);
	}
	ra_data->cur_cmd = NULL;
	free_cmd(cmd);
	} else {
	/* wait for connection event */
	}

	return FALSE;
	}


	static gboolean
	connection_takeover_timeout_cb(gpointer data)
	{
	lrm_state_t *lrm_state = NULL;
	remote_ra_cmd_t *cmd = data;

	crm_info("takeover event timed out for node %s", cmd->rsc_id);
	cmd->takeover_timeout_id = 0;

	lrm_state = lrm_state_find(cmd->rsc_id);

	handle_remote_ra_stop(lrm_state, cmd);
	free_cmd(cmd);

	return FALSE;
	}

	static gboolean
	monitor_timeout_cb(gpointer data)
	{
	lrm_state_t *lrm_state = NULL;
	remote_ra_cmd_t *cmd = data;

	lrm_state = lrm_state_find(cmd->rsc_id);

	crm_info("Timed out waiting for remote poke response from %s%s",
	cmd->rsc_id, (lrm_state? "" : " (no LRM state)"));
	cmd->monitor_timeout_id = 0;
	cmd->op_status = PCMK_LRM_OP_TIMEOUT;
	cmd->rc = PCMK_OCF_UNKNOWN_ERROR;

	if (lrm_state && lrm_state->remote_ra_data) {
	remote_ra_data_t *ra_data = lrm_state->remote_ra_data;

	if (ra_data->cur_cmd == cmd) {
	ra_data->cur_cmd = NULL;
	}
	if (ra_data->cmds) {
	mainloop_set_trigger(ra_data->work);
	}
	}

	report_remote_ra_result(cmd);
	free_cmd(cmd);

	if(lrm_state) {
	lrm_state_disconnect(lrm_state);
	}
	return FALSE;
	}

	static void
	synthesize_lrmd_success(lrm_state_t lrm_state, const char rsc_id, const char *op_type)
	{
	lrmd_event_data_t op = { 0, };

	if (lrm_state == NULL) {
	/* if lrm_state not given assume local */
	lrm_state = lrm_state_find(fsa_our_uname);
	}
	CRM_ASSERT(lrm_state != NULL);

	op.type = lrmd_event_exec_complete;
	op.rsc_id = rsc_id;
	op.op_type = op_type;
	op.rc = PCMK_OCF_OK;
	op.op_status = PCMK_LRM_OP_DONE;
	op.t_run = (unsigned int) time(NULL);
	op.t_rcchange = op.t_run;
	op.call_id = generate_callid();
	process_lrm_event(lrm_state, &op, NULL, NULL);
	}

	void
	remote_lrm_op_callback(lrmd_event_data_t * op)
	{
	gboolean cmd_handled = FALSE;
	lrm_state_t *lrm_state = NULL;
	remote_ra_data_t *ra_data = NULL;
	remote_ra_cmd_t *cmd = NULL;

	crm_debug("Processing '%s%s%s' event on remote connection to %s: %s "
	"(%d) status=%s (%d)",
	(op->op_type? op->op_type : ""), (op->op_type? " " : ""),
	lrmd_event_type2str(op->type), op->remote_nodename,
	services_ocf_exitcode_str(op->rc), op->rc,
	services_lrm_status_str(op->op_status), op->op_status);

	lrm_state = lrm_state_find(op->remote_nodename);
	if (!lrm_state \|\| !lrm_state->remote_ra_data) {
	crm_debug("No state information found for remote connection event");
	return;
	}
	ra_data = lrm_state->remote_ra_data;

	if (op->type == lrmd_event_new_client) {
	// Another client has connected to the remote daemon

	if (ra_data->migrate_status == expect_takeover) {
	// Great, we knew this was coming
	ra_data->migrate_status = takeover_complete;

	} else {
	crm_err("Unexpected pacemaker_remote client takeover for %s. Disconnecting", op->remote_nodename);
	/* In this case, lrmd_tls_connection_destroy() will be called under the control of mainloop. */
	/* Do not free lrm_state->conn yet. */
	/* It'll be freed in the following stop action. */
	lrm_state_disconnect_only(lrm_state);
	}
	return;
	}

	/* filter all EXEC events up */
	if (op->type == lrmd_event_exec_complete) {
	if (ra_data->migrate_status == takeover_complete) {
	crm_debug("ignoring event, this connection is taken over by another node");
	} else {
	lrm_op_callback(op);
	}
	return;
	}

	if ((op->type == lrmd_event_disconnect) && (ra_data->cur_cmd == NULL)) {

	if (ra_data->active == FALSE) {
	crm_debug("Disconnection from Pacemaker Remote node %s complete",
	lrm_state->node_name);

	} else if (!remote_ra_is_in_maintenance(lrm_state)) {
	crm_err("Lost connection to Pacemaker Remote node %s",
	lrm_state->node_name);
	ra_data->recurring_cmds = fail_all_monitor_cmds(ra_data->recurring_cmds);
	ra_data->cmds = fail_all_monitor_cmds(ra_data->cmds);

	} else {
	crm_notice("Unmanaged Pacemaker Remote node %s disconnected",
	lrm_state->node_name);
	/* Do roughly what a 'stop' on the remote-resource would do */
	handle_remote_ra_stop(lrm_state, NULL);
	remote_node_down(lrm_state->node_name, DOWN_KEEP_LRM);
	/* now fake the reply of a successful 'stop' */
	synthesize_lrmd_success(NULL, lrm_state->node_name, "stop");
	}
	return;
	}

	if (!ra_data->cur_cmd) {
	crm_debug("no event to match");
	return;
	}

	cmd = ra_data->cur_cmd;

	/* Start actions and migrate from actions complete after connection
	* comes back to us. */
	if (op->type == lrmd_event_connect && pcmk__strcase_any_of(cmd->action, "start",
	"migrate_from", NULL)) {
	if (op->connection_rc < 0) {
	update_remaining_timeout(cmd);

	if (op->connection_rc == -ENOKEY) {
	// Hard error, don't retry
	cmd->op_status = PCMK_LRM_OP_ERROR;
	cmd->rc = PCMK_OCF_INVALID_PARAM;
	cmd->exit_reason = strdup("Authentication key not readable");

	} else if (cmd->remaining_timeout > 3000) {
	crm_trace("rescheduling start, remaining timeout %d", cmd->remaining_timeout);
	g_timeout_add(1000, retry_start_cmd_cb, lrm_state);
	return;

	} else {
	crm_trace("can't reschedule start, remaining timeout too small %d",
	cmd->remaining_timeout);
	cmd->op_status = PCMK_LRM_OP_TIMEOUT;
	cmd->rc = PCMK_OCF_UNKNOWN_ERROR;
	}

	} else {
	lrm_state_reset_tables(lrm_state, TRUE);
	cmd->rc = PCMK_OCF_OK;
	cmd->op_status = PCMK_LRM_OP_DONE;
	ra_data->active = TRUE;
	}

	crm_debug("Remote connection event matched %s action", cmd->action);
	report_remote_ra_result(cmd);
	cmd_handled = TRUE;

	} else if (op->type == lrmd_event_poke && pcmk__str_eq(cmd->action, "monitor", pcmk__str_casei)) {

	if (cmd->monitor_timeout_id) {
	g_source_remove(cmd->monitor_timeout_id);
	cmd->monitor_timeout_id = 0;
	}

	/* Only report success the first time, after that only worry about failures.
	* For this function, if we get the poke pack, it is always a success. Pokes
	* only fail if the send fails, or the response times out. */
	if (!cmd->reported_success) {
	cmd->rc = PCMK_OCF_OK;
	cmd->op_status = PCMK_LRM_OP_DONE;
	report_remote_ra_result(cmd);
	cmd->reported_success = 1;
	}

	crm_debug("Remote poke event matched %s action", cmd->action);

	/* success, keep rescheduling if interval is present. */
	if (cmd->interval_ms && (cmd->cancel == FALSE)) {
	ra_data->recurring_cmds = g_list_append(ra_data->recurring_cmds, cmd);
	cmd->interval_id = g_timeout_add(cmd->interval_ms,
	recurring_helper, cmd);
	cmd = NULL; /* prevent free */
	}
	cmd_handled = TRUE;

	} else if (op->type == lrmd_event_disconnect && pcmk__str_eq(cmd->action, "monitor", pcmk__str_casei)) {
	if (ra_data->active == TRUE && (cmd->cancel == FALSE)) {
	cmd->rc = PCMK_OCF_UNKNOWN_ERROR;
	cmd->op_status = PCMK_LRM_OP_ERROR;
	report_remote_ra_result(cmd);
	crm_err("Remote connection to %s unexpectedly dropped during monitor",
	lrm_state->node_name);
	}
	cmd_handled = TRUE;

	} else if (op->type == lrmd_event_new_client && pcmk__str_eq(cmd->action, "stop", pcmk__str_casei)) {

	handle_remote_ra_stop(lrm_state, cmd);
	cmd_handled = TRUE;

	} else {
	crm_debug("Event did not match %s action", ra_data->cur_cmd->action);
	}

	if (cmd_handled) {
	ra_data->cur_cmd = NULL;
	if (ra_data->cmds) {
	mainloop_set_trigger(ra_data->work);
	}
	free_cmd(cmd);
	}
	}

	static void
	handle_remote_ra_stop(lrm_state_t * lrm_state, remote_ra_cmd_t * cmd)
	{
	remote_ra_data_t *ra_data = NULL;

	CRM_ASSERT(lrm_state);
	ra_data = lrm_state->remote_ra_data;

	if (ra_data->migrate_status != takeover_complete) {
	/* delete pending ops when ever the remote connection is intentionally stopped */
	g_hash_table_remove_all(lrm_state->pending_ops);
	} else {
	/* we no longer hold the history if this connection has been migrated,
	* however, we keep metadata cache for future use */
	lrm_state_reset_tables(lrm_state, FALSE);
	}

	ra_data->active = FALSE;
	lrm_state_disconnect(lrm_state);

	if (ra_data->cmds) {
	g_list_free_full(ra_data->cmds, free_cmd);
	}
	if (ra_data->recurring_cmds) {
	g_list_free_full(ra_data->recurring_cmds, free_cmd);
	}
	ra_data->cmds = NULL;
	ra_data->recurring_cmds = NULL;
	ra_data->cur_cmd = NULL;

	if (cmd) {
	cmd->rc = PCMK_OCF_OK;
	cmd->op_status = PCMK_LRM_OP_DONE;

	report_remote_ra_result(cmd);
	}
	}

	static int
	handle_remote_ra_start(lrm_state_t * lrm_state, remote_ra_cmd_t * cmd, int timeout_ms)
	{
	const char *server = NULL;
	lrmd_key_value_t *tmp = NULL;
	int port = 0;
	remote_ra_data_t *ra_data = lrm_state->remote_ra_data;
	int timeout_used = timeout_ms > MAX_START_TIMEOUT_MS ? MAX_START_TIMEOUT_MS : timeout_ms;

	for (tmp = cmd->params; tmp; tmp = tmp->next) {
	if (pcmk__strcase_any_of(tmp->key, XML_RSC_ATTR_REMOTE_RA_ADDR,
	XML_RSC_ATTR_REMOTE_RA_SERVER, NULL)) {
	server = tmp->value;
	} else if (pcmk__str_eq(tmp->key, XML_RSC_ATTR_REMOTE_RA_PORT, pcmk__str_casei)) {
	port = atoi(tmp->value);
	} else if (pcmk__str_eq(tmp->key, CRM_META "_" XML_RSC_ATTR_CONTAINER, pcmk__str_casei)) {
	ra_data->controlling_guest = TRUE;
	}
	}

	return lrm_state_remote_connect_async(lrm_state, server, port, timeout_used);
	}

	static gboolean
	handle_remote_ra_exec(gpointer user_data)
	{
	int rc = 0;
	lrm_state_t *lrm_state = user_data;
	remote_ra_data_t *ra_data = lrm_state->remote_ra_data;
	remote_ra_cmd_t *cmd;
	GList *first = NULL;

	if (ra_data->cur_cmd) {
	/* still waiting on previous cmd */
	return TRUE;
	}

	while (ra_data->cmds) {
	first = ra_data->cmds;
	cmd = first->data;
	if (cmd->delay_id) {
	/* still waiting for start delay timer to trip */
	return TRUE;
	}

	ra_data->cmds = g_list_remove_link(ra_data->cmds, first);
	g_list_free_1(first);

	if (!strcmp(cmd->action, "start") \|\| !strcmp(cmd->action, "migrate_from")) {
	ra_data->migrate_status = 0;
	rc = handle_remote_ra_start(lrm_state, cmd, cmd->timeout);
	if (rc == 0) {
	/* take care of this later when we get async connection result */
	crm_debug("Initiated async remote connection, %s action will complete after connect event",
	cmd->action);
	ra_data->cur_cmd = cmd;
	return TRUE;
	} else {
	crm_debug("Could not initiate remote connection for %s action",
	cmd->action);
	cmd->rc = PCMK_OCF_UNKNOWN_ERROR;
	cmd->op_status = PCMK_LRM_OP_ERROR;
	}
	report_remote_ra_result(cmd);

	} else if (!strcmp(cmd->action, "monitor")) {

	if (lrm_state_is_connected(lrm_state) == TRUE) {
	rc = lrm_state_poke_connection(lrm_state);
	if (rc < 0) {
	cmd->rc = PCMK_OCF_UNKNOWN_ERROR;
	cmd->op_status = PCMK_LRM_OP_ERROR;
	}
	} else {
	rc = -1;
	cmd->op_status = PCMK_LRM_OP_DONE;
	cmd->rc = PCMK_OCF_NOT_RUNNING;
	}

	if (rc == 0) {
	crm_debug("Poked Pacemaker Remote at node %s, waiting for async response",
	cmd->rsc_id);
	ra_data->cur_cmd = cmd;
	cmd->monitor_timeout_id = g_timeout_add(cmd->timeout, monitor_timeout_cb, cmd);
	return TRUE;
	}
	report_remote_ra_result(cmd);

	} else if (!strcmp(cmd->action, "stop")) {

	if (ra_data->migrate_status == expect_takeover) {
	/* briefly wait on stop for the takeover event to occur. If the
	* takeover event does not occur during the wait period, that's fine.
	* It just means that the remote-node's lrm_status section is going to get
	* cleared which will require all the resources running in the remote-node
	* to be explicitly re-detected via probe actions. If the takeover does occur
	* successfully, then we can leave the status section intact. */
	cmd->takeover_timeout_id = g_timeout_add((cmd->timeout/2), connection_takeover_timeout_cb, cmd);
	ra_data->cur_cmd = cmd;
	return TRUE;
	}

	handle_remote_ra_stop(lrm_state, cmd);

	} else if (!strcmp(cmd->action, "migrate_to")) {
	ra_data->migrate_status = expect_takeover;
	cmd->rc = PCMK_OCF_OK;
	cmd->op_status = PCMK_LRM_OP_DONE;
	report_remote_ra_result(cmd);
	} else if (!strcmp(cmd->action, "reload")) {
	/* reloads are a no-op right now, add logic here when they become important */
	cmd->rc = PCMK_OCF_OK;
	cmd->op_status = PCMK_LRM_OP_DONE;
	report_remote_ra_result(cmd);
	}

	free_cmd(cmd);
	}

	return TRUE;
	}

	static void
	remote_ra_data_init(lrm_state_t * lrm_state)
	{
	remote_ra_data_t *ra_data = NULL;

	if (lrm_state->remote_ra_data) {
	return;
	}

	ra_data = calloc(1, sizeof(remote_ra_data_t));
	ra_data->work = mainloop_add_trigger(G_PRIORITY_HIGH, handle_remote_ra_exec, lrm_state);
	lrm_state->remote_ra_data = ra_data;
	}

	void
	remote_ra_cleanup(lrm_state_t * lrm_state)
	{
	remote_ra_data_t *ra_data = lrm_state->remote_ra_data;

	if (!ra_data) {
	return;
	}

	if (ra_data->cmds) {
	g_list_free_full(ra_data->cmds, free_cmd);
	}

	if (ra_data->recurring_cmds) {
	g_list_free_full(ra_data->recurring_cmds, free_cmd);
	}
	mainloop_destroy_trigger(ra_data->work);
	free(ra_data);
	lrm_state->remote_ra_data = NULL;
	}

	gboolean
	is_remote_lrmd_ra(const char agent, const char provider, const char *id)
	{
	if (agent && provider && !strcmp(agent, REMOTE_LRMD_RA) && !strcmp(provider, "pacemaker")) {
	return TRUE;
	}
	if (id && lrm_state_find(id) && !pcmk__str_eq(id, fsa_our_uname, pcmk__str_casei)) {
	return TRUE;
	}

	return FALSE;
	}

	lrmd_rsc_info_t *
	remote_ra_get_rsc_info(lrm_state_t * lrm_state, const char *rsc_id)
	{
	lrmd_rsc_info_t *info = NULL;

	if ((lrm_state_find(rsc_id))) {
	info = calloc(1, sizeof(lrmd_rsc_info_t));

	info->id = strdup(rsc_id);
	info->type = strdup(REMOTE_LRMD_RA);
	info->standard = strdup(PCMK_RESOURCE_CLASS_OCF);
	info->provider = strdup("pacemaker");
	}

	return info;
	}

	static gboolean
	is_remote_ra_supported_action(const char *action)
	{
	if (!action) {
	return FALSE;
	} else if (strcmp(action, "start") &&
	strcmp(action, "stop") &&
	strcmp(action, "reload") &&
	strcmp(action, "migrate_to") &&
	strcmp(action, "migrate_from") && strcmp(action, "monitor")) {
	return FALSE;
	}

	return TRUE;
	}

	static GList *
	fail_all_monitor_cmds(GList * list)
	{
	GList *rm_list = NULL;
	remote_ra_cmd_t *cmd = NULL;
	GListPtr gIter = NULL;

	for (gIter = list; gIter != NULL; gIter = gIter->next) {
	cmd = gIter->data;
	if ((cmd->interval_ms > 0) && pcmk__str_eq(cmd->action, "monitor", pcmk__str_casei)) {
	rm_list = g_list_append(rm_list, cmd);
	}
	}

	for (gIter = rm_list; gIter != NULL; gIter = gIter->next) {
	cmd = gIter->data;

	cmd->rc = PCMK_OCF_UNKNOWN_ERROR;
	cmd->op_status = PCMK_LRM_OP_ERROR;
	crm_trace("Pre-emptively failing %s %s (interval=%u, %s)",
	cmd->action, cmd->rsc_id, cmd->interval_ms, cmd->userdata);
	report_remote_ra_result(cmd);

	list = g_list_remove(list, cmd);
	free_cmd(cmd);
	}

	/* frees only the list data, not the cmds */
	g_list_free(rm_list);
	return list;
	}

	static GList *
	remove_cmd(GList * list, const char *action, guint interval_ms)
	{
	remote_ra_cmd_t *cmd = NULL;
	GListPtr gIter = NULL;

	for (gIter = list; gIter != NULL; gIter = gIter->next) {
	cmd = gIter->data;
	if ((cmd->interval_ms == interval_ms)
	&& pcmk__str_eq(cmd->action, action, pcmk__str_casei)) {
	break;
	}
	cmd = NULL;
	}
	if (cmd) {
	list = g_list_remove(list, cmd);
	free_cmd(cmd);
	}
	return list;
	}

	int
	remote_ra_cancel(lrm_state_t lrm_state, const char rsc_id,
	const char *action, guint interval_ms)
	{
	lrm_state_t *connection_rsc = NULL;
	remote_ra_data_t *ra_data = NULL;

	connection_rsc = lrm_state_find(rsc_id);
	if (!connection_rsc \|\| !connection_rsc->remote_ra_data) {
	return -EINVAL;
	}

	ra_data = connection_rsc->remote_ra_data;
	ra_data->cmds = remove_cmd(ra_data->cmds, action, interval_ms);
	ra_data->recurring_cmds = remove_cmd(ra_data->recurring_cmds, action,
	interval_ms);
	if (ra_data->cur_cmd &&
	(ra_data->cur_cmd->interval_ms == interval_ms) &&
	(pcmk__str_eq(ra_data->cur_cmd->action, action, pcmk__str_casei))) {

	ra_data->cur_cmd->cancel = TRUE;
	}

	return 0;
	}

	static remote_ra_cmd_t *
	handle_dup_monitor(remote_ra_data_t *ra_data, guint interval_ms,
	const char *userdata)
	{
	GList *gIter = NULL;
	remote_ra_cmd_t *cmd = NULL;

	/* there are 3 places a potential duplicate monitor operation
	* could exist.
	* 1. recurring_cmds list. where the op is waiting for its next interval
	* 2. cmds list, where the op is queued to get executed immediately
	* 3. cur_cmd, which means the monitor op is in flight right now.
	*/
	if (interval_ms == 0) {
	return NULL;
	}

	if (ra_data->cur_cmd &&
	ra_data->cur_cmd->cancel == FALSE &&
	(ra_data->cur_cmd->interval_ms == interval_ms) &&
	pcmk__str_eq(ra_data->cur_cmd->action, "monitor", pcmk__str_casei)) {

	cmd = ra_data->cur_cmd;
	goto handle_dup;
	}

	for (gIter = ra_data->recurring_cmds; gIter != NULL; gIter = gIter->next) {
	cmd = gIter->data;
	if ((cmd->interval_ms == interval_ms)
	&& pcmk__str_eq(cmd->action, "monitor", pcmk__str_casei)) {
	goto handle_dup;
	}
	}

	for (gIter = ra_data->cmds; gIter != NULL; gIter = gIter->next) {
	cmd = gIter->data;
	if ((cmd->interval_ms == interval_ms)
	&& pcmk__str_eq(cmd->action, "monitor", pcmk__str_casei)) {
	goto handle_dup;
	}
	}

	return NULL;

	handle_dup:

	crm_trace("merging duplicate monitor cmd " PCMK__OP_FMT,
	cmd->rsc_id, "monitor", interval_ms);

	/* update the userdata */
	if (userdata) {
	free(cmd->userdata);
	cmd->userdata = strdup(userdata);
	}

	/* if we've already reported success, generate a new call id */
	if (cmd->reported_success) {
	cmd->start_time = time(NULL);
	cmd->call_id = generate_callid();
	cmd->reported_success = 0;
	}

	/* if we have an interval_id set, that means we are in the process of
	* waiting for this cmd's next interval. instead of waiting, cancel
	* the timer and execute the action immediately */
	if (cmd->interval_id) {
	g_source_remove(cmd->interval_id);
	cmd->interval_id = 0;
	recurring_helper(cmd);
	}

	return cmd;
	}

	int
	remote_ra_exec(lrm_state_t lrm_state, const char rsc_id, const char *action,
	const char *userdata, guint interval_ms,
	int timeout, /* ms */
	int start_delay, /* ms */
	lrmd_key_value_t * params)
	{
	int rc = 0;
	lrm_state_t *connection_rsc = NULL;
	remote_ra_cmd_t *cmd = NULL;
	remote_ra_data_t *ra_data = NULL;

	if (is_remote_ra_supported_action(action) == FALSE) {
	rc = -EINVAL;
	goto exec_done;
	}

	connection_rsc = lrm_state_find(rsc_id);
	if (!connection_rsc) {
	rc = -EINVAL;
	goto exec_done;
	}

	remote_ra_data_init(connection_rsc);
	ra_data = connection_rsc->remote_ra_data;

	cmd = handle_dup_monitor(ra_data, interval_ms, userdata);
	if (cmd) {
	rc = cmd->call_id;
	goto exec_done;
	}

	cmd = calloc(1, sizeof(remote_ra_cmd_t));
	cmd->owner = strdup(lrm_state->node_name);
	cmd->rsc_id = strdup(rsc_id);
	cmd->action = strdup(action);
	cmd->userdata = strdup(userdata);
	cmd->interval_ms = interval_ms;
	cmd->timeout = timeout;
	cmd->start_delay = start_delay;
	cmd->params = params;
	cmd->start_time = time(NULL);

	cmd->call_id = generate_callid();

	if (cmd->start_delay) {
	cmd->delay_id = g_timeout_add(cmd->start_delay, start_delay_helper, cmd);
	}

	ra_data->cmds = g_list_append(ra_data->cmds, cmd);
	mainloop_set_trigger(ra_data->work);

	return cmd->call_id;
	exec_done:

	lrmd_key_value_freeall(params);
	return rc;
	}

	/*!
	* \internal
	* \brief Immediately fail all monitors of a remote node, if proxied here
	*
	* \param[in] node_name Name of pacemaker_remote node
	*/
	void
	remote_ra_fail(const char *node_name)
	{
	lrm_state_t *lrm_state = lrm_state_find(node_name);

	if (lrm_state && lrm_state_is_connected(lrm_state)) {
	remote_ra_data_t *ra_data = lrm_state->remote_ra_data;

	crm_info("Failing monitors on pacemaker_remote node %s", node_name);
	ra_data->recurring_cmds = fail_all_monitor_cmds(ra_data->recurring_cmds);
	ra_data->cmds = fail_all_monitor_cmds(ra_data->cmds);
	}
	}

	/* A guest node fencing implied by host fencing looks like:
	*
	* <pseudo_event id="103" operation="stonith" operation_key="stonith-lxc1-off"
	* on_node="lxc1" on_node_uuid="lxc1">
	* <attributes CRM_meta_master_lxc_ms="10" CRM_meta_on_node="lxc1"
	* CRM_meta_on_node_uuid="lxc1" CRM_meta_stonith_action="off"
	* crm_feature_set="3.0.12"/>
	* <downed>
	* <node id="lxc1"/>
	* </downed>
	* </pseudo_event>
	*/
	#define XPATH_PSEUDO_FENCE "//" XML_GRAPH_TAG_PSEUDO_EVENT \
	"[@" XML_LRM_ATTR_TASK "='stonith']/" XML_GRAPH_TAG_DOWNED \
	"/" XML_CIB_TAG_NODE

	/*!
	* \internal
	* \brief Check a pseudo-action for Pacemaker Remote node side effects
	*
	* \param[in] xml XML of pseudo-action to check
	*/
	void
	remote_ra_process_pseudo(xmlNode *xml)
	{
	xmlXPathObjectPtr search = xpath_search(xml, XPATH_PSEUDO_FENCE);

	if (numXpathResults(search) == 1) {
	xmlNode *result = getXpathResult(search, 0);

	/* Normally, we handle the necessary side effects of a guest node stop
	* action when reporting the remote agent's result. However, if the stop
	* is implied due to fencing, it will be a fencing pseudo-event, and
	* there won't be a result to report. Handle that case here.
	*
	* This will result in a duplicate call to remote_node_down() if the
	* guest stop was real instead of implied, but that shouldn't hurt.
	*
	* There is still one corner case that isn't handled: if a guest node
	* isn't running any resources when its host is fenced, it will appear
	* to be cleanly stopped, so there will be no pseudo-fence, and our
	* peer cache state will be incorrect unless and until the guest is
	* recovered.
	*/
	if (result) {
	const char *remote = ID(result);

	if (remote) {
	remote_node_down(remote, DOWN_ERASE_LRM);
	}
	}
	}
	freeXpathObject(search);
	}

	static void
	remote_ra_maintenance(lrm_state_t * lrm_state, gboolean maintenance)
	{
	remote_ra_data_t *ra_data = lrm_state->remote_ra_data;
	xmlNode update, state;
	int call_opt, call_id = 0;
	crm_node_t *node;

	call_opt = crmd_cib_smart_opt();
	node = crm_remote_peer_get(lrm_state->node_name);
	CRM_CHECK(node != NULL, return);
	update = create_xml_node(NULL, XML_CIB_TAG_STATUS);
	state = create_node_state_update(node, node_update_none, update,
	__func__);
	crm_xml_add(state, XML_NODE_IS_MAINTENANCE, maintenance?"1":"0");
	fsa_cib_update(XML_CIB_TAG_STATUS, update, call_opt, call_id, NULL);
	if (call_id < 0) {
	crm_perror(LOG_WARNING, "%s CIB node state update failed", lrm_state->node_name);
	} else {
	/* TODO: still not 100% sure that async update will succeed ... */
	ra_data->is_maintenance = maintenance;
	}
	free_xml(update);
	}

	#define XPATH_PSEUDO_MAINTENANCE "//" XML_GRAPH_TAG_PSEUDO_EVENT \
	"[@" XML_LRM_ATTR_TASK "='" CRM_OP_MAINTENANCE_NODES "']/" \
	XML_GRAPH_TAG_MAINTENANCE

	/*!
	* \internal
	* \brief Check a pseudo-action holding updates for maintenance state
	*
	* \param[in] xml XML of pseudo-action to check
	*/

	void
	remote_ra_process_maintenance_nodes(xmlNode *xml)
	{
	xmlXPathObjectPtr search = xpath_search(xml, XPATH_PSEUDO_MAINTENANCE);

	if (numXpathResults(search) == 1) {
	xmlNode *node;
	int cnt = 0, cnt_remote = 0;

	for (node =
	first_named_child(getXpathResult(search, 0), XML_CIB_TAG_NODE);
	node != NULL; node = pcmk__xml_next(node)) {
	lrm_state_t *lrm_state = lrm_state_find(ID(node));

	cnt++;
	if (lrm_state && lrm_state->remote_ra_data &&
	((remote_ra_data_t *) lrm_state->remote_ra_data)->active) {
	cnt_remote++;
	remote_ra_maintenance(lrm_state,
	crm_atoi(crm_element_value(node,
	XML_NODE_IS_MAINTENANCE), "0"));

	}
	}
	crm_trace("Action holds %d nodes (%d remotes found) "
	"adjusting maintenance-mode", cnt, cnt_remote);
	}
	freeXpathObject(search);
	}

	gboolean
	remote_ra_is_in_maintenance(lrm_state_t * lrm_state)
	{
	remote_ra_data_t *ra_data = lrm_state->remote_ra_data;

	return ra_data->is_maintenance;
	}

	gboolean
	remote_ra_controlling_guest(lrm_state_t * lrm_state)
	{
	remote_ra_data_t *ra_data = lrm_state->remote_ra_data;

	return ra_data->controlling_guest;
	}
	diff --git a/daemons/controld/controld_schedulerd.c b/daemons/controld/controld_schedulerd.c
	index 73a6ae2207..2444da7930 100644
	--- a/daemons/controld/controld_schedulerd.c
	+++ b/daemons/controld/controld_schedulerd.c
	@@ -1,480 +1,480 @@
	/*
	* Copyright 2004-2020 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <unistd.h> /* pid_t, sleep, ssize_t */

	#include <crm/cib.h>
	#include <crm/cluster.h>
	#include <crm/common/xml.h>
	#include <crm/crm.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml_internal.h>

	#include <pacemaker-controld.h>

	static mainloop_io_t *pe_subsystem = NULL;

	/*!
	* \internal
	* \brief Close any scheduler connection and free associated memory
	*/
	void
	pe_subsystem_free(void)
	{
	controld_clear_fsa_input_flags(R_PE_REQUIRED);
	if (pe_subsystem) {
	controld_expect_sched_reply(NULL);
	mainloop_del_ipc_client(pe_subsystem);
	pe_subsystem = NULL;
	controld_clear_fsa_input_flags(R_PE_CONNECTED);
	}
	}

	/*!
	* \internal
	* \brief Save CIB query result to file, raising FSA error
	*
	* \param[in] msg Ignored
	* \param[in] call_id Call ID of CIB query
	* \param[in] rc Return code of CIB query
	* \param[in] output Result of CIB query
	* \param[in] user_data Unique identifier for filename (will be freed)
	*
	* \note This is intended to be called after a scheduler connection fails.
	*/
	static void
	save_cib_contents(xmlNode msg, int call_id, int rc, xmlNode output,
	void *user_data)
	{
	char *id = user_data;

	register_fsa_error_adv(C_FSA_INTERNAL, I_ERROR, NULL, NULL, __func__);
	CRM_CHECK(id != NULL, return);

	if (rc == pcmk_ok) {
	char *filename = crm_strdup_printf(PE_STATE_DIR "/pe-core-%s.bz2", id);

	if (write_xml_file(output, filename, TRUE) < 0) {
	crm_err("Could not save Cluster Information Base to %s after scheduler crash",
	filename);
	} else {
	crm_notice("Saved Cluster Information Base to %s after scheduler crash",
	filename);
	}
	free(filename);
	}
	}

	/*!
	* \internal
	* \brief Respond to scheduler connection failure
	*
	* \param[in] user_data Ignored
	*/
	static void
	pe_ipc_destroy(gpointer user_data)
	{
	// If we aren't connected to the scheduler, we can't expect a reply
	controld_expect_sched_reply(NULL);

	if (pcmk_is_set(fsa_input_register, R_PE_REQUIRED)) {
	int rc = pcmk_ok;
	char *uuid_str = crm_generate_uuid();

	crm_crit("Connection to the scheduler failed "
	CRM_XS " uuid=%s", uuid_str);

	/*
	* The scheduler died...
	*
	* Save the current CIB so that we have a chance of
	* figuring out what killed it.
	*
	* Delay raising the I_ERROR until the query below completes or
	* 5s is up, whichever comes first.
	*
	*/
	rc = fsa_cib_conn->cmds->query(fsa_cib_conn, NULL, NULL, cib_scope_local);
	fsa_register_cib_callback(rc, FALSE, uuid_str, save_cib_contents);

	} else {
	crm_info("Connection to the scheduler released");
	}

	controld_clear_fsa_input_flags(R_PE_CONNECTED);
	pe_subsystem = NULL;
	mainloop_set_trigger(fsa_source);
	return;
	}

	/*!
	* \internal
	* \brief Handle message from scheduler connection
	*
	* \param[in] buffer XML message (will be freed)
	* \param[in] length Ignored
	* \param[in] userdata Ignored
	*
	* \return 0
	*/
	static int
	pe_ipc_dispatch(const char *buffer, ssize_t length, gpointer userdata)
	{
	xmlNode *msg = string2xml(buffer);

	if (msg) {
	route_message(C_IPC_MESSAGE, msg);
	}
	free_xml(msg);
	return 0;
	}

	/*!
	* \internal
	* \brief Make new connection to scheduler
	*
	* \return TRUE on success, FALSE otherwise
	*/
	static bool
	pe_subsystem_new(void)
	{
	struct ipc_client_callbacks pe_callbacks = {
	.dispatch = pe_ipc_dispatch,
	.destroy = pe_ipc_destroy
	};
	static bool retry_one = TRUE;

	controld_set_fsa_input_flags(R_PE_REQUIRED);
	retry:
	pe_subsystem = mainloop_add_ipc_client(CRM_SYSTEM_PENGINE,
	G_PRIORITY_DEFAULT,
	5 * 1024 * 1024 /* 5MB */,
	NULL, &pe_callbacks);
	if (pe_subsystem == NULL) {
	crm_debug("Could not connect to scheduler : %s(%d)", pcmk_rc_str(errno), errno);
	if (errno == EAGAIN && retry_one) {
	/* In rare cases, a SIGTERM may be received and the connection may fail when the cluster shuts down. */
	/* At this time, the connection will be retried only once. */
	crm_debug("Scheduler connection attempt.");
	retry_one = FALSE;
	goto retry;
	}
	return FALSE;
	}
	controld_set_fsa_input_flags(R_PE_CONNECTED);
	return TRUE;
	}

	/*!
	* \internal
	* \brief Send an XML message to the scheduler
	*
	* \param[in] cmd XML message to send
	*
	* \return pcmk_ok on success, -errno otherwise
	*/
	static int
	pe_subsystem_send(xmlNode *cmd)
	{
	if (pe_subsystem) {
	int sent = crm_ipc_send(mainloop_get_ipc_client(pe_subsystem), cmd,
	0, 0, NULL);

	if (sent == 0) {
	sent = -ENODATA;
	} else if (sent > 0) {
	sent = pcmk_ok;
	}
	return sent;
	}
	return -ENOTCONN;
	}

	static void do_pe_invoke_callback(xmlNode *msg, int call_id, int rc,
	xmlNode output, void user_data);

	/* A_PE_START, A_PE_STOP, O_PE_RESTART */
	void
	do_pe_control(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	if (action & A_PE_STOP) {
	pe_subsystem_free();
	}
	if ((action & A_PE_START)
	&& !pcmk_is_set(fsa_input_register, R_PE_CONNECTED)) {

	if (cur_state == S_STOPPING) {
	crm_info("Ignoring request to connect to scheduler while shutting down");

	} else if (!pe_subsystem_new()) {
	crm_warn("Could not connect to scheduler");
	register_fsa_error(C_FSA_INTERNAL, I_FAIL, NULL);
	}
	}
	}

	int fsa_pe_query = 0;
	char *fsa_pe_ref = NULL;
	static mainloop_timer_t *controld_sched_timer = NULL;

	// @TODO Make this a configurable cluster option if there's demand for it
	#define SCHED_TIMEOUT_MS (120000)

	/*!
	* \internal
	* \brief Handle a timeout waiting for scheduler reply
	*
	* \param[in] user_data Ignored
	*
	* \return FALSE (indicating that timer should not be restarted)
	*/
	static gboolean
	controld_sched_timeout(gpointer user_data)
	{
	if (AM_I_DC) {
	/* If this node is the DC but can't communicate with the scheduler, just
	* exit (and likely get fenced) so this node doesn't interfere with any
	* further DC elections.
	*
	* @TODO We could try something less drastic first, like disconnecting
	* and reconnecting to the scheduler, but something is likely going
	* seriously wrong, so perhaps it's better to just fail as quickly as
	* possible.
	*/
	crmd_exit(CRM_EX_FATAL);
	}
	return FALSE;
	}

	void
	controld_stop_sched_timer(void)
	{
	if (controld_sched_timer && fsa_pe_ref) {
	crm_trace("Stopping timer for scheduler reply %s", fsa_pe_ref);
	}
	mainloop_timer_stop(controld_sched_timer);
	}

	/*!
	* \internal
	* \brief Set the scheduler request currently being waited on
	*
	* \param[in] msg Request to expect reply to (or NULL for none)
	*/
	void
	controld_expect_sched_reply(xmlNode *msg)
	{
	char *ref = NULL;

	if (msg) {
	ref = crm_element_value_copy(msg, XML_ATTR_REFERENCE);
	CRM_ASSERT(ref != NULL);

	if (controld_sched_timer == NULL) {
	controld_sched_timer = mainloop_timer_add("scheduler_reply_timer",
	SCHED_TIMEOUT_MS, FALSE,
	controld_sched_timeout,
	NULL);
	}
	mainloop_timer_start(controld_sched_timer);
	} else {
	controld_stop_sched_timer();
	}
	free(fsa_pe_ref);
	fsa_pe_ref = ref;
	}

	/*!
	* \internal
	* \brief Free the scheduler reply timer
	*/
	void
	controld_free_sched_timer(void)
	{
	if (controld_sched_timer != NULL) {
	mainloop_timer_del(controld_sched_timer);
	controld_sched_timer = NULL;
	}
	}

	/* A_PE_INVOKE */
	void
	do_pe_invoke(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	if (AM_I_DC == FALSE) {
	crm_err("Not invoking scheduler because not DC: %s",
	fsa_action2string(action));
	return;
	}

	if (!pcmk_is_set(fsa_input_register, R_PE_CONNECTED)) {
	if (pcmk_is_set(fsa_input_register, R_SHUTDOWN)) {
	crm_err("Cannot shut down gracefully without the scheduler");
	register_fsa_input_before(C_FSA_INTERNAL, I_TERMINATE, NULL);

	} else {
	crm_info("Waiting for the scheduler to connect");
	crmd_fsa_stall(FALSE);
	controld_set_fsa_action_flags(A_PE_START);
	trigger_fsa();
	}
	return;
	}

	if (cur_state != S_POLICY_ENGINE) {
	crm_notice("Not invoking scheduler because in state %s",
	fsa_state2string(cur_state));
	return;
	}
	if (!pcmk_is_set(fsa_input_register, R_HAVE_CIB)) {
	crm_err("Attempted to invoke scheduler without consistent Cluster Information Base!");

	/* start the join from scratch */
	register_fsa_input_before(C_FSA_INTERNAL, I_ELECTION, NULL);
	return;
	}

	fsa_pe_query = fsa_cib_conn->cmds->query(fsa_cib_conn, NULL, NULL, cib_scope_local);

	crm_debug("Query %d: Requesting the current CIB: %s", fsa_pe_query,
	fsa_state2string(fsa_state));

	controld_expect_sched_reply(NULL);
	fsa_register_cib_callback(fsa_pe_query, FALSE, NULL, do_pe_invoke_callback);
	}

	static void
	force_local_option(xmlNode xml, const char attr_name, const char *attr_value)
	{
	int max = 0;
	int lpc = 0;
	char *xpath_string = NULL;
	xmlXPathObjectPtr xpathObj = NULL;

	xpath_string = crm_strdup_printf("%.128s//%s//nvpair[@name='%.128s']",
	get_object_path(XML_CIB_TAG_CRMCONFIG),
	XML_CIB_TAG_PROPSET, attr_name);
	xpathObj = xpath_search(xml, xpath_string);
	max = numXpathResults(xpathObj);
	free(xpath_string);

	for (lpc = 0; lpc < max; lpc++) {
	xmlNode *match = getXpathResult(xpathObj, lpc);
	crm_trace("Forcing %s/%s = %s", ID(match), attr_name, attr_value);
	crm_xml_add(match, XML_NVPAIR_ATTR_VALUE, attr_value);
	}

	if(max == 0) {
	xmlNode *configuration = NULL;
	xmlNode *crm_config = NULL;
	xmlNode *cluster_property_set = NULL;

	crm_trace("Creating %s-%s for %s=%s",
	CIB_OPTIONS_FIRST, attr_name, attr_name, attr_value);

	configuration = pcmk__xe_match(xml, XML_CIB_TAG_CONFIGURATION, NULL,
	NULL);
	if (configuration == NULL) {
	configuration = create_xml_node(xml, XML_CIB_TAG_CONFIGURATION);
	}

	crm_config = pcmk__xe_match(configuration, XML_CIB_TAG_CRMCONFIG, NULL,
	NULL);
	if (crm_config == NULL) {
	crm_config = create_xml_node(configuration, XML_CIB_TAG_CRMCONFIG);
	}

	cluster_property_set = pcmk__xe_match(crm_config, XML_CIB_TAG_PROPSET,
	NULL, NULL);
	if (cluster_property_set == NULL) {
	cluster_property_set = create_xml_node(crm_config, XML_CIB_TAG_PROPSET);
	crm_xml_add(cluster_property_set, XML_ATTR_ID, CIB_OPTIONS_FIRST);
	}

	xml = create_xml_node(cluster_property_set, XML_CIB_TAG_NVPAIR);

	crm_xml_set_id(xml, "%s-%s", CIB_OPTIONS_FIRST, attr_name);
	crm_xml_add(xml, XML_NVPAIR_ATTR_NAME, attr_name);
	crm_xml_add(xml, XML_NVPAIR_ATTR_VALUE, attr_value);
	}
	freeXpathObject(xpathObj);
	}

	static void
	do_pe_invoke_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
	{
	xmlNode *cmd = NULL;
	pid_t watchdog = pcmk__locate_sbd();

	if (rc != pcmk_ok) {
	crm_err("Could not retrieve the Cluster Information Base: %s "
	CRM_XS " rc=%d call=%d", pcmk_strerror(rc), rc, call_id);
	register_fsa_error_adv(C_FSA_INTERNAL, I_ERROR, NULL, NULL, __func__);
	return;

	} else if (call_id != fsa_pe_query) {
	crm_trace("Skipping superseded CIB query: %d (current=%d)", call_id, fsa_pe_query);
	return;

	} else if (!AM_I_DC \|\| !pcmk_is_set(fsa_input_register, R_PE_CONNECTED)) {
	crm_debug("No need to invoke the scheduler anymore");
	return;

	} else if (fsa_state != S_POLICY_ENGINE) {
	crm_debug("Discarding scheduler request in state: %s",
	fsa_state2string(fsa_state));
	return;

	/* this callback counts as 1 */
	} else if (num_cib_op_callbacks() > 1) {
	crm_debug("Re-asking for the CIB: %d other peer updates still pending",
	(num_cib_op_callbacks() - 1));
	sleep(1);
	controld_set_fsa_action_flags(A_PE_INVOKE);
	trigger_fsa();
	return;
	}

	CRM_LOG_ASSERT(output != NULL);

	/* Refresh the remote node cache and the known node cache when the
	* scheduler is invoked */
	- crm_peer_caches_refresh(output);
	+ pcmk__refresh_node_caches_from_cib(output);

	crm_xml_add(output, XML_ATTR_DC_UUID, fsa_our_uuid);
	crm_xml_add_int(output, XML_ATTR_HAVE_QUORUM, fsa_has_quorum);

	force_local_option(output, XML_ATTR_HAVE_WATCHDOG, pcmk__btoa(watchdog));

	if (ever_had_quorum && crm_have_quorum == FALSE) {
	crm_xml_add_int(output, XML_ATTR_QUORUM_PANIC, 1);
	}

	cmd = create_request(CRM_OP_PECALC, output, NULL, CRM_SYSTEM_PENGINE, CRM_SYSTEM_DC, NULL);

	rc = pe_subsystem_send(cmd);
	if (rc < 0) {
	crm_err("Could not contact the scheduler: %s " CRM_XS " rc=%d",
	pcmk_strerror(rc), rc);
	register_fsa_error_adv(C_FSA_INTERNAL, I_ERROR, NULL, NULL, __func__);
	} else {
	controld_expect_sched_reply(cmd);
	crm_debug("Invoking the scheduler: query=%d, ref=%s, seq=%llu, quorate=%d",
	fsa_pe_query, fsa_pe_ref, crm_peer_seq, fsa_has_quorum);
	}
	free_xml(cmd);
	}
	diff --git a/daemons/controld/controld_te_actions.c b/daemons/controld/controld_te_actions.c
	index ff3cbd0d7b..c4cf76ec59 100644
	--- a/daemons/controld/controld_te_actions.c
	+++ b/daemons/controld/controld_te_actions.c
	@@ -1,643 +1,644 @@
	/*
	* Copyright 2004-2020 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <sys/param.h>
	#include <crm/crm.h>
	#include <crm/cib.h>
	#include <crm/lrmd.h> // lrmd_event_data_t, lrmd_free_event()
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>
	#include <crm/cluster.h>

	#include <pacemaker-internal.h>
	#include <pacemaker-controld.h>

	char *te_uuid = NULL;
	GHashTable *te_targets = NULL;
	void send_rsc_command(crm_action_t * action);
	static void te_update_job_count(crm_action_t * action, int offset);

	static void
	te_start_action_timer(crm_graph_t * graph, crm_action_t * action)
	{
	action->timer = calloc(1, sizeof(crm_action_timer_t));
	action->timer->timeout = action->timeout;
	action->timer->action = action;
	action->timer->source_id = g_timeout_add(action->timer->timeout + graph->network_delay,
	action_timer_callback, (void *)action->timer);

	CRM_ASSERT(action->timer->source_id != 0);
	}

	static gboolean
	te_pseudo_action(crm_graph_t * graph, crm_action_t * pseudo)
	{
	const char *task = crm_element_value(pseudo->xml, XML_LRM_ATTR_TASK);

	/* send to peers as well? */
	if (pcmk__str_eq(task, CRM_OP_MAINTENANCE_NODES, pcmk__str_casei)) {
	GHashTableIter iter;
	crm_node_t *node = NULL;

	g_hash_table_iter_init(&iter, crm_peer_cache);
	while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &node)) {
	xmlNode *cmd = NULL;

	if (pcmk__str_eq(fsa_our_uname, node->uname, pcmk__str_casei)) {
	continue;
	}

	cmd = create_request(task, pseudo->xml, node->uname,
	CRM_SYSTEM_CRMD, CRM_SYSTEM_TENGINE, NULL);
	send_cluster_message(node, crm_msg_crmd, cmd, FALSE);
	free_xml(cmd);
	}

	remote_ra_process_maintenance_nodes(pseudo->xml);
	} else {
	/* Check action for Pacemaker Remote node side effects */
	remote_ra_process_pseudo(pseudo->xml);
	}

	crm_debug("Pseudo-action %d (%s) fired and confirmed", pseudo->id,
	crm_element_value(pseudo->xml, XML_LRM_ATTR_TASK_KEY));
	te_action_confirmed(pseudo, graph);
	return TRUE;
	}

	static int
	get_target_rc(crm_action_t * action)
	{
	const char *target_rc_s = crm_meta_value(action->params, XML_ATTR_TE_TARGET_RC);

	if (target_rc_s != NULL) {
	return crm_parse_int(target_rc_s, "0");
	}
	return 0;
	}

	static gboolean
	te_crm_command(crm_graph_t * graph, crm_action_t * action)
	{
	char *counter = NULL;
	xmlNode *cmd = NULL;
	gboolean is_local = FALSE;

	const char *id = NULL;
	const char *task = NULL;
	const char *value = NULL;
	const char *on_node = NULL;
	const char *router_node = NULL;

	gboolean rc = TRUE;
	gboolean no_wait = FALSE;

	id = ID(action->xml);
	task = crm_element_value(action->xml, XML_LRM_ATTR_TASK);
	on_node = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
	router_node = crm_element_value(action->xml, XML_LRM_ATTR_ROUTER_NODE);

	if (!router_node) {
	router_node = on_node;
	if (pcmk__str_eq(task, CRM_OP_LRM_DELETE, pcmk__str_casei)) {
	const char *mode = crm_element_value(action->xml, PCMK__XA_MODE);

	if (pcmk__str_eq(mode, XML_TAG_CIB, pcmk__str_casei)) {
	router_node = fsa_our_uname;
	}
	}
	}

	CRM_CHECK(on_node != NULL && strlen(on_node) != 0,
	crm_err("Corrupted command (id=%s) %s: no node", crm_str(id), crm_str(task));
	return FALSE);

	if (pcmk__str_eq(router_node, fsa_our_uname, pcmk__str_casei)) {
	is_local = TRUE;
	}

	value = crm_meta_value(action->params, XML_ATTR_TE_NOWAIT);
	if (crm_is_true(value)) {
	no_wait = TRUE;
	}

	crm_info("Executing crm-event (%s)%s%s: %s on %s",
	crm_str(id), (is_local? " locally" : ""),
	(no_wait? " without waiting" : ""), crm_str(task), on_node);

	if (is_local && pcmk__str_eq(task, CRM_OP_SHUTDOWN, pcmk__str_casei)) {
	/* defer until everything else completes */
	crm_info("crm-event (%s) is a local shutdown", crm_str(id));
	graph->completion_action = tg_shutdown;
	graph->abort_reason = "local shutdown";
	te_action_confirmed(action, graph);
	return TRUE;

	} else if (pcmk__str_eq(task, CRM_OP_SHUTDOWN, pcmk__str_casei)) {
	crm_node_t *peer = crm_get_peer(0, router_node);
	- crm_update_peer_expected(__func__, peer, CRMD_JOINSTATE_DOWN);
	+
	+ pcmk__update_peer_expected(__func__, peer, CRMD_JOINSTATE_DOWN);
	}

	cmd = create_request(task, action->xml, router_node, CRM_SYSTEM_CRMD, CRM_SYSTEM_TENGINE, NULL);

	counter = pcmk__transition_key(transition_graph->id, action->id,
	get_target_rc(action), te_uuid);
	crm_xml_add(cmd, XML_ATTR_TRANSITION_KEY, counter);

	rc = send_cluster_message(crm_get_peer(0, router_node), crm_msg_crmd, cmd, TRUE);
	free(counter);
	free_xml(cmd);

	if (rc == FALSE) {
	crm_err("Action %d failed: send", action->id);
	return FALSE;

	} else if (no_wait) {
	te_action_confirmed(action, graph);

	} else {
	if (action->timeout <= 0) {
	crm_err("Action %d: %s on %s had an invalid timeout (%dms). Using %ums instead",
	action->id, task, on_node, action->timeout, graph->network_delay);
	action->timeout = (int) graph->network_delay;
	}
	te_start_action_timer(graph, action);
	}

	return TRUE;
	}

	void
	controld_record_action_timeout(crm_action_t *action)
	{
	lrmd_event_data_t *op = NULL;
	xmlNode *state = NULL;
	xmlNode *rsc = NULL;
	xmlNode *xml_op = NULL;
	xmlNode *action_rsc = NULL;

	int rc = pcmk_ok;

	const char *rsc_id = NULL;
	const char *target = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
	const char *task_uuid = crm_element_value(action->xml, XML_LRM_ATTR_TASK_KEY);
	const char *target_uuid = crm_element_value(action->xml, XML_LRM_ATTR_TARGET_UUID);

	int call_options = cib_quorum_override \| cib_scope_local;
	int target_rc = get_target_rc(action);

	crm_warn("%s %d: %s on %s timed out",
	crm_element_name(action->xml), action->id, task_uuid, target);

	action_rsc = find_xml_node(action->xml, XML_CIB_TAG_RESOURCE, TRUE);
	if (action_rsc == NULL) {
	return;
	}

	rsc_id = ID(action_rsc);
	CRM_CHECK(rsc_id != NULL,
	crm_log_xml_err(action->xml, "Bad:action"); return);

	/*
	update the CIB

	<node_state id="hadev">
	<lrm>
	<lrm_resources>
	<lrm_resource id="rsc2" last_op="start" op_code="0" target="hadev"/>
	*/

	state = create_xml_node(NULL, XML_CIB_TAG_STATE);

	crm_xml_add(state, XML_ATTR_UUID, target_uuid);
	crm_xml_add(state, XML_ATTR_UNAME, target);

	rsc = create_xml_node(state, XML_CIB_TAG_LRM);
	crm_xml_add(rsc, XML_ATTR_ID, target_uuid);

	rsc = create_xml_node(rsc, XML_LRM_TAG_RESOURCES);
	rsc = create_xml_node(rsc, XML_LRM_TAG_RESOURCE);
	crm_xml_add(rsc, XML_ATTR_ID, rsc_id);


	crm_copy_xml_element(action_rsc, rsc, XML_ATTR_TYPE);
	crm_copy_xml_element(action_rsc, rsc, XML_AGENT_ATTR_CLASS);
	crm_copy_xml_element(action_rsc, rsc, XML_AGENT_ATTR_PROVIDER);

	/* If the executor gets a timeout while waiting for the action to complete,
	* that will be reported via the usual callback. This timeout means that we
	* didn't hear from the executor or the controller that relayed the action
	* to the executor.
	*
	* @TODO Using PCMK_OCF_UNKNOWN_ERROR instead of PCMK_OCF_TIMEOUT is one way
	* to distinguish those situations, but perhaps PCMK_OCF_TIMEOUT would be
	* preferable anyway.
	*/
	op = convert_graph_action(NULL, action, PCMK_LRM_OP_TIMEOUT,
	PCMK_OCF_UNKNOWN_ERROR);
	op->call_id = -1;
	op->user_data = pcmk__transition_key(transition_graph->id, action->id,
	target_rc, te_uuid);

	xml_op = pcmk__create_history_xml(rsc, op, CRM_FEATURE_SET, target_rc,
	target, __func__, LOG_INFO);
	lrmd_free_event(op);

	crm_log_xml_trace(xml_op, "Action timeout");

	rc = fsa_cib_conn->cmds->update(fsa_cib_conn, XML_CIB_TAG_STATUS, state, call_options);
	fsa_register_cib_callback(rc, FALSE, NULL, cib_action_updated);
	free_xml(state);

	crm_trace("Sent CIB update (call ID %d) for timeout of action %d (%s on %s)",
	rc, action->id, task_uuid, target);
	action->sent_update = TRUE;
	}

	static gboolean
	te_rsc_command(crm_graph_t * graph, crm_action_t * action)
	{
	/* never overwrite stop actions in the CIB with
	* anything other than completed results
	*
	* Writing pending stops makes it look like the
	* resource is running again
	*/
	xmlNode *cmd = NULL;
	xmlNode *rsc_op = NULL;

	gboolean rc = TRUE;
	gboolean no_wait = FALSE;
	gboolean is_local = FALSE;

	char *counter = NULL;
	const char *task = NULL;
	const char *value = NULL;
	const char *on_node = NULL;
	const char *router_node = NULL;
	const char *task_uuid = NULL;

	CRM_ASSERT(action != NULL);
	CRM_ASSERT(action->xml != NULL);

	action->executed = FALSE;
	on_node = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);

	CRM_CHECK(on_node != NULL && strlen(on_node) != 0,
	crm_err("Corrupted command(id=%s) %s: no node", ID(action->xml), crm_str(task));
	return FALSE);

	rsc_op = action->xml;
	task = crm_element_value(rsc_op, XML_LRM_ATTR_TASK);
	task_uuid = crm_element_value(action->xml, XML_LRM_ATTR_TASK_KEY);
	router_node = crm_element_value(rsc_op, XML_LRM_ATTR_ROUTER_NODE);

	if (!router_node) {
	router_node = on_node;
	}

	counter = pcmk__transition_key(transition_graph->id, action->id,
	get_target_rc(action), te_uuid);
	crm_xml_add(rsc_op, XML_ATTR_TRANSITION_KEY, counter);

	if (pcmk__str_eq(router_node, fsa_our_uname, pcmk__str_casei)) {
	is_local = TRUE;
	}

	value = crm_meta_value(action->params, XML_ATTR_TE_NOWAIT);
	if (crm_is_true(value)) {
	no_wait = TRUE;
	}

	crm_notice("Initiating %s operation %s%s on %s%s "CRM_XS" action %d",
	task, task_uuid, (is_local? " locally" : ""), on_node,
	(no_wait? " without waiting" : ""), action->id);

	cmd = create_request(CRM_OP_INVOKE_LRM, rsc_op, router_node,
	CRM_SYSTEM_LRMD, CRM_SYSTEM_TENGINE, NULL);

	if (is_local) {
	/* shortcut local resource commands */
	ha_msg_input_t data = {
	.msg = cmd,
	.xml = rsc_op,
	};

	fsa_data_t msg = {
	.id = 0,
	.data = &data,
	.data_type = fsa_dt_ha_msg,
	.fsa_input = I_NULL,
	.fsa_cause = C_FSA_INTERNAL,
	.actions = A_LRM_INVOKE,
	.origin = __func__,
	};

	do_lrm_invoke(A_LRM_INVOKE, C_FSA_INTERNAL, fsa_state, I_NULL, &msg);

	} else {
	rc = send_cluster_message(crm_get_peer(0, router_node), crm_msg_lrmd, cmd, TRUE);
	}

	free(counter);
	free_xml(cmd);

	action->executed = TRUE;

	if (rc == FALSE) {
	crm_err("Action %d failed: send", action->id);
	return FALSE;

	} else if (no_wait) {
	crm_info("Action %d confirmed - no wait", action->id);
	action->confirmed = TRUE; /* Just mark confirmed.
	* Don't bump the job count only to immediately decrement it
	*/
	update_graph(transition_graph, action);
	trigger_graph();

	} else if (action->confirmed == TRUE) {
	crm_debug("Action %d: %s %s on %s(timeout %dms) was already confirmed.",
	action->id, task, task_uuid, on_node, action->timeout);
	} else {
	if (action->timeout <= 0) {
	crm_err("Action %d: %s %s on %s had an invalid timeout (%dms). Using %ums instead",
	action->id, task, task_uuid, on_node, action->timeout, graph->network_delay);
	action->timeout = (int) graph->network_delay;
	}
	te_update_job_count(action, 1);
	te_start_action_timer(graph, action);
	}

	return TRUE;
	}

	struct te_peer_s
	{
	char *name;
	int jobs;
	int migrate_jobs;
	};

	static void te_peer_free(gpointer p)
	{
	struct te_peer_s *peer = p;

	free(peer->name);
	free(peer);
	}

	void te_reset_job_counts(void)
	{
	GHashTableIter iter;
	struct te_peer_s *peer = NULL;

	if(te_targets == NULL) {
	te_targets = g_hash_table_new_full(crm_str_hash, g_str_equal, NULL, te_peer_free);
	}

	g_hash_table_iter_init(&iter, te_targets);
	while (g_hash_table_iter_next(&iter, NULL, (gpointer *) & peer)) {
	peer->jobs = 0;
	peer->migrate_jobs = 0;
	}
	}

	static void
	te_update_job_count_on(const char *target, int offset, bool migrate)
	{
	struct te_peer_s *r = NULL;

	if(target == NULL \|\| te_targets == NULL) {
	return;
	}

	r = g_hash_table_lookup(te_targets, target);
	if(r == NULL) {
	r = calloc(1, sizeof(struct te_peer_s));
	r->name = strdup(target);
	g_hash_table_insert(te_targets, r->name, r);
	}

	r->jobs += offset;
	if(migrate) {
	r->migrate_jobs += offset;
	}
	crm_trace("jobs[%s] = %d", target, r->jobs);
	}

	static void
	te_update_job_count(crm_action_t * action, int offset)
	{
	const char *task = crm_element_value(action->xml, XML_LRM_ATTR_TASK);
	const char *target = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);

	if (action->type != action_type_rsc \|\| target == NULL) {
	/* No limit on these */
	return;
	}

	/* if we have a router node, this means the action is performing
	* on a remote node. For now, we count all actions occurring on a
	* remote node against the job list on the cluster node hosting
	* the connection resources */
	target = crm_element_value(action->xml, XML_LRM_ATTR_ROUTER_NODE);

	if ((target == NULL) && pcmk__strcase_any_of(task, CRMD_ACTION_MIGRATE,
	CRMD_ACTION_MIGRATED, NULL)) {

	const char *t1 = crm_meta_value(action->params, XML_LRM_ATTR_MIGRATE_SOURCE);
	const char *t2 = crm_meta_value(action->params, XML_LRM_ATTR_MIGRATE_TARGET);

	te_update_job_count_on(t1, offset, TRUE);
	te_update_job_count_on(t2, offset, TRUE);
	return;
	} else if (target == NULL) {
	target = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
	}

	te_update_job_count_on(target, offset, FALSE);
	}

	static gboolean
	te_should_perform_action_on(crm_graph_t * graph, crm_action_t * action, const char *target)
	{
	int limit = 0;
	struct te_peer_s *r = NULL;
	const char *task = crm_element_value(action->xml, XML_LRM_ATTR_TASK);
	const char *id = crm_element_value(action->xml, XML_LRM_ATTR_TASK_KEY);

	if(target == NULL) {
	/* No limit on these */
	return TRUE;

	} else if(te_targets == NULL) {
	return FALSE;
	}

	r = g_hash_table_lookup(te_targets, target);
	limit = throttle_get_job_limit(target);

	if(r == NULL) {
	r = calloc(1, sizeof(struct te_peer_s));
	r->name = strdup(target);
	g_hash_table_insert(te_targets, r->name, r);
	}

	if(limit <= r->jobs) {
	crm_trace("Peer %s is over their job limit of %d (%d): deferring %s",
	target, limit, r->jobs, id);
	return FALSE;

	} else if(graph->migration_limit > 0 && r->migrate_jobs >= graph->migration_limit) {
	if (pcmk__strcase_any_of(task, CRMD_ACTION_MIGRATE, CRMD_ACTION_MIGRATED, NULL)) {
	crm_trace("Peer %s is over their migration job limit of %d (%d): deferring %s",
	target, graph->migration_limit, r->migrate_jobs, id);
	return FALSE;
	}
	}

	crm_trace("Peer %s has not hit their limit yet. current jobs = %d limit= %d limit", target, r->jobs, limit);

	return TRUE;
	}

	static gboolean
	te_should_perform_action(crm_graph_t * graph, crm_action_t * action)
	{
	const char *target = NULL;
	const char *task = crm_element_value(action->xml, XML_LRM_ATTR_TASK);

	if (action->type != action_type_rsc) {
	/* No limit on these */
	return TRUE;
	}

	/* if we have a router node, this means the action is performing
	* on a remote node. For now, we count all actions occurring on a
	* remote node against the job list on the cluster node hosting
	* the connection resources */
	target = crm_element_value(action->xml, XML_LRM_ATTR_ROUTER_NODE);

	if ((target == NULL) && pcmk__strcase_any_of(task, CRMD_ACTION_MIGRATE,
	CRMD_ACTION_MIGRATED, NULL)) {
	target = crm_meta_value(action->params, XML_LRM_ATTR_MIGRATE_SOURCE);
	if(te_should_perform_action_on(graph, action, target) == FALSE) {
	return FALSE;
	}

	target = crm_meta_value(action->params, XML_LRM_ATTR_MIGRATE_TARGET);

	} else if (target == NULL) {
	target = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
	}

	return te_should_perform_action_on(graph, action, target);
	}

	/*!
	* \brief Confirm a graph action (and optionally update graph)
	*
	* \param[in] action Action to confirm
	* \param[in] graph Update and trigger this graph (if non-NULL)
	*/
	void
	te_action_confirmed(crm_action_t action, crm_graph_t graph)
	{
	if (action->confirmed == FALSE) {
	if ((action->type == action_type_rsc)
	&& (crm_element_value(action->xml, XML_LRM_ATTR_TARGET) != NULL)) {
	te_update_job_count(action, -1);
	}
	action->confirmed = TRUE;
	}
	if (graph) {
	update_graph(graph, action);
	trigger_graph();
	}
	}


	crm_graph_functions_t te_graph_fns = {
	te_pseudo_action,
	te_rsc_command,
	te_crm_command,
	te_fence_node,
	te_should_perform_action,
	};

	void
	notify_crmd(crm_graph_t * graph)
	{
	const char *type = "unknown";
	enum crmd_fsa_input event = I_NULL;

	crm_debug("Processing transition completion in state %s", fsa_state2string(fsa_state));

	if (graph->complete == FALSE) {
	CRM_CHECK(graph->complete,);
	graph->complete = TRUE;
	}

	switch (graph->completion_action) {
	case tg_stop:
	type = "stop";
	if (fsa_state == S_TRANSITION_ENGINE) {
	event = I_TE_SUCCESS;
	}
	break;
	case tg_done:
	type = "done";
	if (fsa_state == S_TRANSITION_ENGINE) {
	event = I_TE_SUCCESS;
	}
	break;

	case tg_restart:
	type = "restart";
	if (fsa_state == S_TRANSITION_ENGINE) {
	if (transition_timer->period_ms > 0) {
	controld_stop_timer(transition_timer);
	controld_start_timer(transition_timer);
	} else {
	event = I_PE_CALC;
	}

	} else if (fsa_state == S_POLICY_ENGINE) {
	controld_set_fsa_action_flags(A_PE_INVOKE);
	trigger_fsa();
	}
	break;

	case tg_shutdown:
	type = "shutdown";
	if (pcmk_is_set(fsa_input_register, R_SHUTDOWN)) {
	event = I_STOP;

	} else {
	crm_err("We didn't ask to be shut down, yet the scheduler is telling us to");
	event = I_TERMINATE;
	}
	}

	crm_debug("Transition %d status: %s - %s", graph->id, type, crm_str(graph->abort_reason));

	graph->abort_reason = NULL;
	graph->completion_action = tg_done;
	controld_clear_fsa_input_flags(R_IN_TRANSITION);

	if (event != I_NULL) {
	register_fsa_input(C_FSA_INTERNAL, event, NULL);

	} else if (fsa_source) {
	mainloop_set_trigger(fsa_source);
	}
	}
	diff --git a/daemons/controld/controld_utils.c b/daemons/controld/controld_utils.c
	index 9c8a634773..ffd16bb319 100644
	--- a/daemons/controld/controld_utils.c
	+++ b/daemons/controld/controld_utils.c
	@@ -1,875 +1,875 @@
	/*
	* Copyright 2004-2020 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdlib.h>
	#include <stdint.h> // uint64_t

	#include <crm/crm.h>
	#include <crm/cib.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>

	#include <pacemaker-controld.h>

	const char *
	fsa_input2string(enum crmd_fsa_input input)
	{
	const char *inputAsText = NULL;

	switch (input) {
	case I_NULL:
	inputAsText = "I_NULL";
	break;
	case I_CIB_OP:
	inputAsText = "I_CIB_OP (unused)";
	break;
	case I_CIB_UPDATE:
	inputAsText = "I_CIB_UPDATE";
	break;
	case I_DC_TIMEOUT:
	inputAsText = "I_DC_TIMEOUT";
	break;
	case I_ELECTION:
	inputAsText = "I_ELECTION";
	break;
	case I_PE_CALC:
	inputAsText = "I_PE_CALC";
	break;
	case I_RELEASE_DC:
	inputAsText = "I_RELEASE_DC";
	break;
	case I_ELECTION_DC:
	inputAsText = "I_ELECTION_DC";
	break;
	case I_ERROR:
	inputAsText = "I_ERROR";
	break;
	case I_FAIL:
	inputAsText = "I_FAIL";
	break;
	case I_INTEGRATED:
	inputAsText = "I_INTEGRATED";
	break;
	case I_FINALIZED:
	inputAsText = "I_FINALIZED";
	break;
	case I_NODE_JOIN:
	inputAsText = "I_NODE_JOIN";
	break;
	case I_JOIN_OFFER:
	inputAsText = "I_JOIN_OFFER";
	break;
	case I_JOIN_REQUEST:
	inputAsText = "I_JOIN_REQUEST";
	break;
	case I_JOIN_RESULT:
	inputAsText = "I_JOIN_RESULT";
	break;
	case I_NOT_DC:
	inputAsText = "I_NOT_DC";
	break;
	case I_RECOVERED:
	inputAsText = "I_RECOVERED";
	break;
	case I_RELEASE_FAIL:
	inputAsText = "I_RELEASE_FAIL";
	break;
	case I_RELEASE_SUCCESS:
	inputAsText = "I_RELEASE_SUCCESS";
	break;
	case I_RESTART:
	inputAsText = "I_RESTART";
	break;
	case I_PE_SUCCESS:
	inputAsText = "I_PE_SUCCESS";
	break;
	case I_ROUTER:
	inputAsText = "I_ROUTER";
	break;
	case I_SHUTDOWN:
	inputAsText = "I_SHUTDOWN";
	break;
	case I_STARTUP:
	inputAsText = "I_STARTUP";
	break;
	case I_TE_SUCCESS:
	inputAsText = "I_TE_SUCCESS";
	break;
	case I_STOP:
	inputAsText = "I_STOP";
	break;
	case I_DC_HEARTBEAT:
	inputAsText = "I_DC_HEARTBEAT";
	break;
	case I_WAIT_FOR_EVENT:
	inputAsText = "I_WAIT_FOR_EVENT";
	break;
	case I_LRM_EVENT:
	inputAsText = "I_LRM_EVENT";
	break;
	case I_PENDING:
	inputAsText = "I_PENDING";
	break;
	case I_HALT:
	inputAsText = "I_HALT";
	break;
	case I_TERMINATE:
	inputAsText = "I_TERMINATE";
	break;
	case I_ILLEGAL:
	inputAsText = "I_ILLEGAL";
	break;
	}

	if (inputAsText == NULL) {
	crm_err("Input %d is unknown", input);
	inputAsText = "<UNKNOWN_INPUT>";
	}

	return inputAsText;
	}

	const char *
	fsa_state2string(enum crmd_fsa_state state)
	{
	const char *stateAsText = NULL;

	switch (state) {
	case S_IDLE:
	stateAsText = "S_IDLE";
	break;
	case S_ELECTION:
	stateAsText = "S_ELECTION";
	break;
	case S_INTEGRATION:
	stateAsText = "S_INTEGRATION";
	break;
	case S_FINALIZE_JOIN:
	stateAsText = "S_FINALIZE_JOIN";
	break;
	case S_NOT_DC:
	stateAsText = "S_NOT_DC";
	break;
	case S_POLICY_ENGINE:
	stateAsText = "S_POLICY_ENGINE";
	break;
	case S_RECOVERY:
	stateAsText = "S_RECOVERY";
	break;
	case S_RELEASE_DC:
	stateAsText = "S_RELEASE_DC";
	break;
	case S_PENDING:
	stateAsText = "S_PENDING";
	break;
	case S_STOPPING:
	stateAsText = "S_STOPPING";
	break;
	case S_TERMINATE:
	stateAsText = "S_TERMINATE";
	break;
	case S_TRANSITION_ENGINE:
	stateAsText = "S_TRANSITION_ENGINE";
	break;
	case S_STARTING:
	stateAsText = "S_STARTING";
	break;
	case S_HALT:
	stateAsText = "S_HALT";
	break;
	case S_ILLEGAL:
	stateAsText = "S_ILLEGAL";
	break;
	}

	if (stateAsText == NULL) {
	crm_err("State %d is unknown", state);
	stateAsText = "<UNKNOWN_STATE>";
	}

	return stateAsText;
	}

	const char *
	fsa_cause2string(enum crmd_fsa_cause cause)
	{
	const char *causeAsText = NULL;

	switch (cause) {
	case C_UNKNOWN:
	causeAsText = "C_UNKNOWN";
	break;
	case C_STARTUP:
	causeAsText = "C_STARTUP";
	break;
	case C_IPC_MESSAGE:
	causeAsText = "C_IPC_MESSAGE";
	break;
	case C_HA_MESSAGE:
	causeAsText = "C_HA_MESSAGE";
	break;
	case C_TIMER_POPPED:
	causeAsText = "C_TIMER_POPPED";
	break;
	case C_SHUTDOWN:
	causeAsText = "C_SHUTDOWN";
	break;
	case C_LRM_OP_CALLBACK:
	causeAsText = "C_LRM_OP_CALLBACK";
	break;
	case C_CRMD_STATUS_CALLBACK:
	causeAsText = "C_CRMD_STATUS_CALLBACK";
	break;
	case C_FSA_INTERNAL:
	causeAsText = "C_FSA_INTERNAL";
	break;
	}

	if (causeAsText == NULL) {
	crm_err("Cause %d is unknown", cause);
	causeAsText = "<UNKNOWN_CAUSE>";
	}

	return causeAsText;
	}

	const char *
	fsa_action2string(long long action)
	{
	const char *actionAsText = NULL;

	switch (action) {

	case A_NOTHING:
	actionAsText = "A_NOTHING";
	break;
	case A_ELECTION_START:
	actionAsText = "A_ELECTION_START";
	break;
	case A_DC_JOIN_FINAL:
	actionAsText = "A_DC_JOIN_FINAL";
	break;
	case A_READCONFIG:
	actionAsText = "A_READCONFIG";
	break;
	case O_RELEASE:
	actionAsText = "O_RELEASE";
	break;
	case A_STARTUP:
	actionAsText = "A_STARTUP";
	break;
	case A_STARTED:
	actionAsText = "A_STARTED";
	break;
	case A_HA_CONNECT:
	actionAsText = "A_HA_CONNECT";
	break;
	case A_HA_DISCONNECT:
	actionAsText = "A_HA_DISCONNECT";
	break;
	case A_LRM_CONNECT:
	actionAsText = "A_LRM_CONNECT";
	break;
	case A_LRM_EVENT:
	actionAsText = "A_LRM_EVENT";
	break;
	case A_LRM_INVOKE:
	actionAsText = "A_LRM_INVOKE";
	break;
	case A_LRM_DISCONNECT:
	actionAsText = "A_LRM_DISCONNECT";
	break;
	case O_LRM_RECONNECT:
	actionAsText = "O_LRM_RECONNECT";
	break;
	case A_CL_JOIN_QUERY:
	actionAsText = "A_CL_JOIN_QUERY";
	break;
	case A_DC_TIMER_STOP:
	actionAsText = "A_DC_TIMER_STOP";
	break;
	case A_DC_TIMER_START:
	actionAsText = "A_DC_TIMER_START";
	break;
	case A_INTEGRATE_TIMER_START:
	actionAsText = "A_INTEGRATE_TIMER_START";
	break;
	case A_INTEGRATE_TIMER_STOP:
	actionAsText = "A_INTEGRATE_TIMER_STOP";
	break;
	case A_FINALIZE_TIMER_START:
	actionAsText = "A_FINALIZE_TIMER_START";
	break;
	case A_FINALIZE_TIMER_STOP:
	actionAsText = "A_FINALIZE_TIMER_STOP";
	break;
	case A_ELECTION_COUNT:
	actionAsText = "A_ELECTION_COUNT";
	break;
	case A_ELECTION_VOTE:
	actionAsText = "A_ELECTION_VOTE";
	break;
	case A_ELECTION_CHECK:
	actionAsText = "A_ELECTION_CHECK";
	break;
	case A_CL_JOIN_ANNOUNCE:
	actionAsText = "A_CL_JOIN_ANNOUNCE";
	break;
	case A_CL_JOIN_REQUEST:
	actionAsText = "A_CL_JOIN_REQUEST";
	break;
	case A_CL_JOIN_RESULT:
	actionAsText = "A_CL_JOIN_RESULT";
	break;
	case A_DC_JOIN_OFFER_ALL:
	actionAsText = "A_DC_JOIN_OFFER_ALL";
	break;
	case A_DC_JOIN_OFFER_ONE:
	actionAsText = "A_DC_JOIN_OFFER_ONE";
	break;
	case A_DC_JOIN_PROCESS_REQ:
	actionAsText = "A_DC_JOIN_PROCESS_REQ";
	break;
	case A_DC_JOIN_PROCESS_ACK:
	actionAsText = "A_DC_JOIN_PROCESS_ACK";
	break;
	case A_DC_JOIN_FINALIZE:
	actionAsText = "A_DC_JOIN_FINALIZE";
	break;
	case A_MSG_PROCESS:
	actionAsText = "A_MSG_PROCESS";
	break;
	case A_MSG_ROUTE:
	actionAsText = "A_MSG_ROUTE";
	break;
	case A_RECOVER:
	actionAsText = "A_RECOVER";
	break;
	case A_DC_RELEASE:
	actionAsText = "A_DC_RELEASE";
	break;
	case A_DC_RELEASED:
	actionAsText = "A_DC_RELEASED";
	break;
	case A_DC_TAKEOVER:
	actionAsText = "A_DC_TAKEOVER";
	break;
	case A_SHUTDOWN:
	actionAsText = "A_SHUTDOWN";
	break;
	case A_SHUTDOWN_REQ:
	actionAsText = "A_SHUTDOWN_REQ";
	break;
	case A_STOP:
	actionAsText = "A_STOP ";
	break;
	case A_EXIT_0:
	actionAsText = "A_EXIT_0";
	break;
	case A_EXIT_1:
	actionAsText = "A_EXIT_1";
	break;
	case O_CIB_RESTART:
	actionAsText = "O_CIB_RESTART";
	break;
	case A_CIB_START:
	actionAsText = "A_CIB_START";
	break;
	case A_CIB_STOP:
	actionAsText = "A_CIB_STOP";
	break;
	case A_TE_INVOKE:
	actionAsText = "A_TE_INVOKE";
	break;
	case O_TE_RESTART:
	actionAsText = "O_TE_RESTART";
	break;
	case A_TE_START:
	actionAsText = "A_TE_START";
	break;
	case A_TE_STOP:
	actionAsText = "A_TE_STOP";
	break;
	case A_TE_HALT:
	actionAsText = "A_TE_HALT";
	break;
	case A_TE_CANCEL:
	actionAsText = "A_TE_CANCEL";
	break;
	case A_PE_INVOKE:
	actionAsText = "A_PE_INVOKE";
	break;
	case O_PE_RESTART:
	actionAsText = "O_PE_RESTART";
	break;
	case A_PE_START:
	actionAsText = "A_PE_START";
	break;
	case A_PE_STOP:
	actionAsText = "A_PE_STOP";
	break;
	case A_NODE_BLOCK:
	actionAsText = "A_NODE_BLOCK";
	break;
	case A_UPDATE_NODESTATUS:
	actionAsText = "A_UPDATE_NODESTATUS";
	break;
	case A_LOG:
	actionAsText = "A_LOG ";
	break;
	case A_ERROR:
	actionAsText = "A_ERROR ";
	break;
	case A_WARN:
	actionAsText = "A_WARN ";
	break;
	/* Composite actions */
	case A_DC_TIMER_START \| A_CL_JOIN_QUERY:
	actionAsText = "A_DC_TIMER_START\|A_CL_JOIN_QUERY";
	break;
	}

	if (actionAsText == NULL) {
	crm_err("Action %.16llx is unknown", action);
	actionAsText = "<UNKNOWN_ACTION>";
	}

	return actionAsText;
	}

	void
	fsa_dump_inputs(int log_level, const char *text, long long input_register)
	{
	if (input_register == A_NOTHING) {
	return;
	}
	if (text == NULL) {
	text = "Input register contents:";
	}

	if (pcmk_is_set(input_register, R_THE_DC)) {
	crm_trace("%s %.16llx (R_THE_DC)", text, R_THE_DC);
	}
	if (pcmk_is_set(input_register, R_STARTING)) {
	crm_trace("%s %.16llx (R_STARTING)", text, R_STARTING);
	}
	if (pcmk_is_set(input_register, R_SHUTDOWN)) {
	crm_trace("%s %.16llx (R_SHUTDOWN)", text, R_SHUTDOWN);
	}
	if (pcmk_is_set(input_register, R_STAYDOWN)) {
	crm_trace("%s %.16llx (R_STAYDOWN)", text, R_STAYDOWN);
	}
	if (pcmk_is_set(input_register, R_JOIN_OK)) {
	crm_trace("%s %.16llx (R_JOIN_OK)", text, R_JOIN_OK);
	}
	if (pcmk_is_set(input_register, R_READ_CONFIG)) {
	crm_trace("%s %.16llx (R_READ_CONFIG)", text, R_READ_CONFIG);
	}
	if (pcmk_is_set(input_register, R_INVOKE_PE)) {
	crm_trace("%s %.16llx (R_INVOKE_PE)", text, R_INVOKE_PE);
	}
	if (pcmk_is_set(input_register, R_CIB_CONNECTED)) {
	crm_trace("%s %.16llx (R_CIB_CONNECTED)", text, R_CIB_CONNECTED);
	}
	if (pcmk_is_set(input_register, R_PE_CONNECTED)) {
	crm_trace("%s %.16llx (R_PE_CONNECTED)", text, R_PE_CONNECTED);
	}
	if (pcmk_is_set(input_register, R_TE_CONNECTED)) {
	crm_trace("%s %.16llx (R_TE_CONNECTED)", text, R_TE_CONNECTED);
	}
	if (pcmk_is_set(input_register, R_LRM_CONNECTED)) {
	crm_trace("%s %.16llx (R_LRM_CONNECTED)", text, R_LRM_CONNECTED);
	}
	if (pcmk_is_set(input_register, R_CIB_REQUIRED)) {
	crm_trace("%s %.16llx (R_CIB_REQUIRED)", text, R_CIB_REQUIRED);
	}
	if (pcmk_is_set(input_register, R_PE_REQUIRED)) {
	crm_trace("%s %.16llx (R_PE_REQUIRED)", text, R_PE_REQUIRED);
	}
	if (pcmk_is_set(input_register, R_TE_REQUIRED)) {
	crm_trace("%s %.16llx (R_TE_REQUIRED)", text, R_TE_REQUIRED);
	}
	if (pcmk_is_set(input_register, R_REQ_PEND)) {
	crm_trace("%s %.16llx (R_REQ_PEND)", text, R_REQ_PEND);
	}
	if (pcmk_is_set(input_register, R_PE_PEND)) {
	crm_trace("%s %.16llx (R_PE_PEND)", text, R_PE_PEND);
	}
	if (pcmk_is_set(input_register, R_TE_PEND)) {
	crm_trace("%s %.16llx (R_TE_PEND)", text, R_TE_PEND);
	}
	if (pcmk_is_set(input_register, R_RESP_PEND)) {
	crm_trace("%s %.16llx (R_RESP_PEND)", text, R_RESP_PEND);
	}
	if (pcmk_is_set(input_register, R_CIB_DONE)) {
	crm_trace("%s %.16llx (R_CIB_DONE)", text, R_CIB_DONE);
	}
	if (pcmk_is_set(input_register, R_HAVE_CIB)) {
	crm_trace("%s %.16llx (R_HAVE_CIB)", text, R_HAVE_CIB);
	}
	if (pcmk_is_set(input_register, R_CIB_ASKED)) {
	crm_trace("%s %.16llx (R_CIB_ASKED)", text, R_CIB_ASKED);
	}
	if (pcmk_is_set(input_register, R_MEMBERSHIP)) {
	crm_trace("%s %.16llx (R_MEMBERSHIP)", text, R_MEMBERSHIP);
	}
	if (pcmk_is_set(input_register, R_PEER_DATA)) {
	crm_trace("%s %.16llx (R_PEER_DATA)", text, R_PEER_DATA);
	}
	if (pcmk_is_set(input_register, R_IN_RECOVERY)) {
	crm_trace("%s %.16llx (R_IN_RECOVERY)", text, R_IN_RECOVERY);
	}
	}

	void
	fsa_dump_actions(uint64_t action, const char *text)
	{
	if (pcmk_is_set(action, A_READCONFIG)) {
	crm_trace("Action %.16llx (A_READCONFIG) %s", A_READCONFIG, text);
	}
	if (pcmk_is_set(action, A_STARTUP)) {
	crm_trace("Action %.16llx (A_STARTUP) %s", A_STARTUP, text);
	}
	if (pcmk_is_set(action, A_STARTED)) {
	crm_trace("Action %.16llx (A_STARTED) %s", A_STARTED, text);
	}
	if (pcmk_is_set(action, A_HA_CONNECT)) {
	crm_trace("Action %.16llx (A_CONNECT) %s", A_HA_CONNECT, text);
	}
	if (pcmk_is_set(action, A_HA_DISCONNECT)) {
	crm_trace("Action %.16llx (A_DISCONNECT) %s", A_HA_DISCONNECT, text);
	}
	if (pcmk_is_set(action, A_LRM_CONNECT)) {
	crm_trace("Action %.16llx (A_LRM_CONNECT) %s", A_LRM_CONNECT, text);
	}
	if (pcmk_is_set(action, A_LRM_EVENT)) {
	crm_trace("Action %.16llx (A_LRM_EVENT) %s", A_LRM_EVENT, text);
	}
	if (pcmk_is_set(action, A_LRM_INVOKE)) {
	crm_trace("Action %.16llx (A_LRM_INVOKE) %s", A_LRM_INVOKE, text);
	}
	if (pcmk_is_set(action, A_LRM_DISCONNECT)) {
	crm_trace("Action %.16llx (A_LRM_DISCONNECT) %s", A_LRM_DISCONNECT, text);
	}
	if (pcmk_is_set(action, A_DC_TIMER_STOP)) {
	crm_trace("Action %.16llx (A_DC_TIMER_STOP) %s", A_DC_TIMER_STOP, text);
	}
	if (pcmk_is_set(action, A_DC_TIMER_START)) {
	crm_trace("Action %.16llx (A_DC_TIMER_START) %s", A_DC_TIMER_START, text);
	}
	if (pcmk_is_set(action, A_INTEGRATE_TIMER_START)) {
	crm_trace("Action %.16llx (A_INTEGRATE_TIMER_START) %s", A_INTEGRATE_TIMER_START, text);
	}
	if (pcmk_is_set(action, A_INTEGRATE_TIMER_STOP)) {
	crm_trace("Action %.16llx (A_INTEGRATE_TIMER_STOP) %s", A_INTEGRATE_TIMER_STOP, text);
	}
	if (pcmk_is_set(action, A_FINALIZE_TIMER_START)) {
	crm_trace("Action %.16llx (A_FINALIZE_TIMER_START) %s", A_FINALIZE_TIMER_START, text);
	}
	if (pcmk_is_set(action, A_FINALIZE_TIMER_STOP)) {
	crm_trace("Action %.16llx (A_FINALIZE_TIMER_STOP) %s", A_FINALIZE_TIMER_STOP, text);
	}
	if (pcmk_is_set(action, A_ELECTION_COUNT)) {
	crm_trace("Action %.16llx (A_ELECTION_COUNT) %s", A_ELECTION_COUNT, text);
	}
	if (pcmk_is_set(action, A_ELECTION_VOTE)) {
	crm_trace("Action %.16llx (A_ELECTION_VOTE) %s", A_ELECTION_VOTE, text);
	}
	if (pcmk_is_set(action, A_ELECTION_CHECK)) {
	crm_trace("Action %.16llx (A_ELECTION_CHECK) %s", A_ELECTION_CHECK, text);
	}
	if (pcmk_is_set(action, A_CL_JOIN_ANNOUNCE)) {
	crm_trace("Action %.16llx (A_CL_JOIN_ANNOUNCE) %s", A_CL_JOIN_ANNOUNCE, text);
	}
	if (pcmk_is_set(action, A_CL_JOIN_REQUEST)) {
	crm_trace("Action %.16llx (A_CL_JOIN_REQUEST) %s", A_CL_JOIN_REQUEST, text);
	}
	if (pcmk_is_set(action, A_CL_JOIN_RESULT)) {
	crm_trace("Action %.16llx (A_CL_JOIN_RESULT) %s", A_CL_JOIN_RESULT, text);
	}
	if (pcmk_is_set(action, A_DC_JOIN_OFFER_ALL)) {
	crm_trace("Action %.16llx (A_DC_JOIN_OFFER_ALL) %s", A_DC_JOIN_OFFER_ALL, text);
	}
	if (pcmk_is_set(action, A_DC_JOIN_OFFER_ONE)) {
	crm_trace("Action %.16llx (A_DC_JOIN_OFFER_ONE) %s", A_DC_JOIN_OFFER_ONE, text);
	}
	if (pcmk_is_set(action, A_DC_JOIN_PROCESS_REQ)) {
	crm_trace("Action %.16llx (A_DC_JOIN_PROCESS_REQ) %s", A_DC_JOIN_PROCESS_REQ, text);
	}
	if (pcmk_is_set(action, A_DC_JOIN_PROCESS_ACK)) {
	crm_trace("Action %.16llx (A_DC_JOIN_PROCESS_ACK) %s", A_DC_JOIN_PROCESS_ACK, text);
	}
	if (pcmk_is_set(action, A_DC_JOIN_FINALIZE)) {
	crm_trace("Action %.16llx (A_DC_JOIN_FINALIZE) %s", A_DC_JOIN_FINALIZE, text);
	}
	if (pcmk_is_set(action, A_MSG_PROCESS)) {
	crm_trace("Action %.16llx (A_MSG_PROCESS) %s", A_MSG_PROCESS, text);
	}
	if (pcmk_is_set(action, A_MSG_ROUTE)) {
	crm_trace("Action %.16llx (A_MSG_ROUTE) %s", A_MSG_ROUTE, text);
	}
	if (pcmk_is_set(action, A_RECOVER)) {
	crm_trace("Action %.16llx (A_RECOVER) %s", A_RECOVER, text);
	}
	if (pcmk_is_set(action, A_DC_RELEASE)) {
	crm_trace("Action %.16llx (A_DC_RELEASE) %s", A_DC_RELEASE, text);
	}
	if (pcmk_is_set(action, A_DC_RELEASED)) {
	crm_trace("Action %.16llx (A_DC_RELEASED) %s", A_DC_RELEASED, text);
	}
	if (pcmk_is_set(action, A_DC_TAKEOVER)) {
	crm_trace("Action %.16llx (A_DC_TAKEOVER) %s", A_DC_TAKEOVER, text);
	}
	if (pcmk_is_set(action, A_SHUTDOWN)) {
	crm_trace("Action %.16llx (A_SHUTDOWN) %s", A_SHUTDOWN, text);
	}
	if (pcmk_is_set(action, A_SHUTDOWN_REQ)) {
	crm_trace("Action %.16llx (A_SHUTDOWN_REQ) %s", A_SHUTDOWN_REQ, text);
	}
	if (pcmk_is_set(action, A_STOP)) {
	crm_trace("Action %.16llx (A_STOP ) %s", A_STOP, text);
	}
	if (pcmk_is_set(action, A_EXIT_0)) {
	crm_trace("Action %.16llx (A_EXIT_0) %s", A_EXIT_0, text);
	}
	if (pcmk_is_set(action, A_EXIT_1)) {
	crm_trace("Action %.16llx (A_EXIT_1) %s", A_EXIT_1, text);
	}
	if (pcmk_is_set(action, A_CIB_START)) {
	crm_trace("Action %.16llx (A_CIB_START) %s", A_CIB_START, text);
	}
	if (pcmk_is_set(action, A_CIB_STOP)) {
	crm_trace("Action %.16llx (A_CIB_STOP) %s", A_CIB_STOP, text);
	}
	if (pcmk_is_set(action, A_TE_INVOKE)) {
	crm_trace("Action %.16llx (A_TE_INVOKE) %s", A_TE_INVOKE, text);
	}
	if (pcmk_is_set(action, A_TE_START)) {
	crm_trace("Action %.16llx (A_TE_START) %s", A_TE_START, text);
	}
	if (pcmk_is_set(action, A_TE_STOP)) {
	crm_trace("Action %.16llx (A_TE_STOP) %s", A_TE_STOP, text);
	}
	if (pcmk_is_set(action, A_TE_CANCEL)) {
	crm_trace("Action %.16llx (A_TE_CANCEL) %s", A_TE_CANCEL, text);
	}
	if (pcmk_is_set(action, A_PE_INVOKE)) {
	crm_trace("Action %.16llx (A_PE_INVOKE) %s", A_PE_INVOKE, text);
	}
	if (pcmk_is_set(action, A_PE_START)) {
	crm_trace("Action %.16llx (A_PE_START) %s", A_PE_START, text);
	}
	if (pcmk_is_set(action, A_PE_STOP)) {
	crm_trace("Action %.16llx (A_PE_STOP) %s", A_PE_STOP, text);
	}
	if (pcmk_is_set(action, A_NODE_BLOCK)) {
	crm_trace("Action %.16llx (A_NODE_BLOCK) %s", A_NODE_BLOCK, text);
	}
	if (pcmk_is_set(action, A_UPDATE_NODESTATUS)) {
	crm_trace("Action %.16llx (A_UPDATE_NODESTATUS) %s", A_UPDATE_NODESTATUS, text);
	}
	if (pcmk_is_set(action, A_LOG)) {
	crm_trace("Action %.16llx (A_LOG ) %s", A_LOG, text);
	}
	if (pcmk_is_set(action, A_ERROR)) {
	crm_trace("Action %.16llx (A_ERROR ) %s", A_ERROR, text);
	}
	if (pcmk_is_set(action, A_WARN)) {
	crm_trace("Action %.16llx (A_WARN ) %s", A_WARN, text);
	}
	}

	gboolean
	update_dc(xmlNode * msg)
	{
	char *last_dc = fsa_our_dc;
	const char *dc_version = NULL;
	const char *welcome_from = NULL;

	if (msg != NULL) {
	gboolean invalid = FALSE;

	dc_version = crm_element_value(msg, F_CRM_VERSION);
	welcome_from = crm_element_value(msg, F_CRM_HOST_FROM);

	CRM_CHECK(dc_version != NULL, return FALSE);
	CRM_CHECK(welcome_from != NULL, return FALSE);

	if (AM_I_DC && !pcmk__str_eq(welcome_from, fsa_our_uname, pcmk__str_casei)) {
	invalid = TRUE;

	} else if (fsa_our_dc && !pcmk__str_eq(welcome_from, fsa_our_dc, pcmk__str_casei)) {
	invalid = TRUE;
	}

	if (invalid) {
	CRM_CHECK(fsa_our_dc != NULL, crm_err("We have no DC"));
	if (AM_I_DC) {
	crm_err("Not updating DC to %s (%s): we are also a DC", welcome_from, dc_version);
	} else {
	crm_warn("New DC %s is not %s", welcome_from, fsa_our_dc);
	}

	controld_set_fsa_action_flags(A_CL_JOIN_QUERY \| A_DC_TIMER_START);
	trigger_fsa();
	return FALSE;
	}
	}

	free(fsa_our_dc_version);
	fsa_our_dc_version = NULL;

	fsa_our_dc = NULL; /* Free'd as last_dc */

	if (welcome_from != NULL) {
	fsa_our_dc = strdup(welcome_from);
	}
	if (dc_version != NULL) {
	fsa_our_dc_version = strdup(dc_version);
	}

	if (pcmk__str_eq(fsa_our_dc, last_dc, pcmk__str_casei)) {
	/* do nothing */

	} else if (fsa_our_dc != NULL) {
	crm_node_t *dc_node = crm_get_peer(0, fsa_our_dc);

	crm_info("Set DC to %s (%s)", crm_str(fsa_our_dc), crm_str(fsa_our_dc_version));
	- crm_update_peer_expected(__func__, dc_node, CRMD_JOINSTATE_MEMBER);
	+ pcmk__update_peer_expected(__func__, dc_node, CRMD_JOINSTATE_MEMBER);

	} else if (last_dc != NULL) {
	crm_info("Unset DC. Was %s", crm_str(last_dc));
	}

	free(last_dc);
	return TRUE;
	}

	void crmd_peer_down(crm_node_t *peer, bool full)
	{
	if(full && peer->state == NULL) {
	- crm_update_peer_state(__func__, peer, CRM_NODE_LOST, 0);
	+ pcmk__update_peer_state(__func__, peer, CRM_NODE_LOST, 0);
	crm_update_peer_proc(__func__, peer, crm_proc_none, NULL);
	}
	crm_update_peer_join(__func__, peer, crm_join_none);
	- crm_update_peer_expected(__func__, peer, CRMD_JOINSTATE_DOWN);
	+ pcmk__update_peer_expected(__func__, peer, CRMD_JOINSTATE_DOWN);
	}

	#define MIN_CIB_OP_TIMEOUT (30)

	unsigned int
	cib_op_timeout()
	{
	static int env_timeout = -1;
	unsigned int calculated_timeout = 0;

	if (env_timeout == -1) {
	const char *env = getenv("PCMK_cib_timeout");

	if (env) {
	env_timeout = crm_parse_int(env, "0");
	}
	env_timeout = QB_MAX(env_timeout, MIN_CIB_OP_TIMEOUT);
	crm_trace("Minimum CIB op timeout: %us (environment: %s)",
	env_timeout, (env? env : "none"));
	}

	calculated_timeout = 1 + crm_active_peers();
	if (crm_remote_peer_cache) {
	calculated_timeout += g_hash_table_size(crm_remote_peer_cache);
	}
	calculated_timeout *= 10;

	calculated_timeout = QB_MAX(calculated_timeout, env_timeout);
	crm_trace("Calculated timeout: %us", calculated_timeout);

	if (fsa_cib_conn) {
	fsa_cib_conn->call_timeout = calculated_timeout;
	}
	return calculated_timeout;
	}

	/*!
	* \internal
	* \brief Check feature set compatibility of DC and joining node
	*
	* Return true if a joining node's CRM feature set is compatible with the
	* current DC's. The feature sets are compatible if they have the same major
	* version number, and the DC's minor version number is the same or older than
	* the joining node's. The minor-minor version is intended solely to allow
	* resource agents to detect feature support, and so is ignored.
	*
	* \param[in] dc_version DC's feature set
	* \param[in] join_version Joining node's version
	*/
	bool
	feature_set_compatible(const char dc_version, const char join_version)
	{
	char *dc_minor = NULL;
	char *join_minor = NULL;
	long dc_v = 0;
	long join_v = 0;

	// Get DC's major version
	errno = 0;
	dc_v = strtol(dc_version, &dc_minor, 10);
	if (errno) {
	return FALSE;
	}

	// Get joining node's major version
	errno = 0;
	join_v = strtol(join_version, &join_minor, 10);
	if (errno) {
	return FALSE;
	}

	// Major version component must be identical
	if (dc_v != join_v) {
	return FALSE;
	}

	// Get DC's minor version
	if (*dc_minor == '.') {
	++dc_minor;
	}
	errno = 0;
	dc_v = strtol(dc_minor, NULL, 10);
	if (errno) {
	return FALSE;
	}

	// Get joining node's minor version
	if (*join_minor == '.') {
	++join_minor;
	}
	errno = 0;
	join_v = strtol(join_minor, NULL, 10);
	if (errno) {
	return FALSE;
	}

	// DC's minor version must be the same or older
	return dc_v <= join_v;
	}

	const char *
	get_node_id(xmlNode *lrm_rsc_op)
	{
	xmlNode *node = lrm_rsc_op;

	while (node != NULL && !pcmk__str_eq(XML_CIB_TAG_STATE, TYPE(node), pcmk__str_casei)) {
	node = node->parent;
	}

	CRM_CHECK(node != NULL, return NULL);
	return ID(node);
	}
	diff --git a/daemons/fenced/fenced_commands.c b/daemons/fenced/fenced_commands.c
	index 3f04b8129e..2729d0d5d2 100644
	--- a/daemons/fenced/fenced_commands.c
	+++ b/daemons/fenced/fenced_commands.c
	@@ -1,2917 +1,2918 @@
	/*
	* Copyright 2009-2020 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <sys/param.h>
	#include <stdio.h>
	#include <sys/types.h>
	#include <sys/wait.h>
	#include <sys/stat.h>
	#include <unistd.h>
	#include <sys/utsname.h>

	#include <stdlib.h>
	#include <errno.h>
	#include <fcntl.h>
	#include <ctype.h>

	#include <crm/crm.h>
	#include <crm/msg_xml.h>
	#include <crm/common/ipc.h>
	#include <crm/common/ipc_internal.h>
	#include <crm/cluster/internal.h>
	#include <crm/common/mainloop.h>

	#include <crm/stonith-ng.h>
	#include <crm/fencing/internal.h>
	#include <crm/common/xml.h>

	#include <pacemaker-fenced.h>

	GHashTable *device_list = NULL;
	GHashTable *topology = NULL;
	GList *cmd_list = NULL;

	struct device_search_s {
	/* target of fence action */
	char *host;
	/* requested fence action */
	char *action;
	/* timeout to use if a device is queried dynamically for possible targets */
	int per_device_timeout;
	/* number of registered fencing devices at time of request */
	int replies_needed;
	/* number of device replies received so far */
	int replies_received;
	/* whether the target is eligible to perform requested action (or off) */
	bool allow_suicide;

	/* private data to pass to search callback function */
	void *user_data;
	/* function to call when all replies have been received */
	void (callback) (GList devices, void *user_data);
	/* devices capable of performing requested action (or off if remapping) */
	GListPtr capable;
	};

	static gboolean stonith_device_dispatch(gpointer user_data);
	static void st_child_done(GPid pid, int rc, const char *output, gpointer user_data);
	static void stonith_send_reply(xmlNode * reply, int call_options, const char *remote_peer,
	const char *client_id);

	static void search_devices_record_result(struct device_search_s search, const char device,
	gboolean can_fence);

	typedef struct async_command_s {

	int id;
	int pid;
	int fd_stdout;
	int options;
	int default_timeout; /* seconds */
	int timeout; /* seconds */

	int start_delay; /* seconds */
	int delay_id;

	char *op;
	char *origin;
	char *client;
	char *client_name;
	char *remote_op_id;

	char *victim;
	uint32_t victim_nodeid;
	char *action;
	char *device;
	char *mode;

	GListPtr device_list;
	GListPtr device_next;

	void *internal_user_data;
	void (done_cb) (GPid pid, int rc, const char output, gpointer user_data);
	guint timer_sigterm;
	guint timer_sigkill;
	/*! If the operation timed out, this is the last signal
	* we sent to the process to get it to terminate */
	int last_timeout_signo;

	stonith_device_t *active_on;
	stonith_device_t *activating_on;
	} async_command_t;

	static xmlNode stonith_construct_async_reply(async_command_t cmd, const char *output,
	xmlNode * data, int rc);

	static gboolean
	is_action_required(const char action, stonith_device_t device)
	{
	return device && device->automatic_unfencing && pcmk__str_eq(action, "on",
	pcmk__str_casei);
	}

	static int
	get_action_delay_max(stonith_device_t * device, const char * action)
	{
	const char *value = NULL;
	int delay_max = 0;

	if (!pcmk__strcase_any_of(action, "off", "reboot", NULL)) {
	return 0;
	}

	value = g_hash_table_lookup(device->params, PCMK_STONITH_DELAY_MAX);
	if (value) {
	delay_max = crm_parse_interval_spec(value) / 1000;
	}

	return delay_max;
	}

	static int
	get_action_delay_base(stonith_device_t * device, const char * action)
	{
	const char *value = NULL;
	int delay_base = 0;

	if (!pcmk__strcase_any_of(action, "off", "reboot", NULL)) {
	return 0;
	}

	value = g_hash_table_lookup(device->params, PCMK_STONITH_DELAY_BASE);
	if (value) {
	delay_base = crm_parse_interval_spec(value) / 1000;
	}

	return delay_base;
	}

	/*!
	* \internal
	* \brief Override STONITH timeout with pcmk_*_timeout if available
	*
	* \param[in] device STONITH device to use
	* \param[in] action STONITH action name
	* \param[in] default_timeout Timeout to use if device does not have
	* a pcmk_*_timeout parameter for action
	*
	* \return Value of pcmk_(action)_timeout if available, otherwise default_timeout
	* \note For consistency, it would be nice if reboot/off/on timeouts could be
	* set the same way as start/stop/monitor timeouts, i.e. with an
	* <operation> entry in the fencing resource configuration. However that
	* is insufficient because fencing devices may be registered directly via
	* the fencer's register_device() API instead of going through the CIB
	* (e.g. stonith_admin uses it for its -R option, and the executor uses it
	* to ensure a device is registered when a command is issued). As device
	* properties, pcmk_*_timeout parameters can be grabbed by the fencer when
	* the device is registered, whether by CIB change or API call.
	*/
	static int
	get_action_timeout(stonith_device_t * device, const char *action, int default_timeout)
	{
	if (action && device && device->params) {
	char buffer[64] = { 0, };
	const char *value = NULL;

	/* If "reboot" was requested but the device does not support it,
	* we will remap to "off", so check timeout for "off" instead
	*/
	if (pcmk__str_eq(action, "reboot", pcmk__str_casei)
	&& !pcmk_is_set(device->flags, st_device_supports_reboot)) {
	crm_trace("%s doesn't support reboot, using timeout for off instead",
	device->id);
	action = "off";
	}

	/* If the device config specified an action-specific timeout, use it */
	snprintf(buffer, sizeof(buffer), "pcmk_%s_timeout", action);
	value = g_hash_table_lookup(device->params, buffer);
	if (value) {
	return atoi(value);
	}
	}
	return default_timeout;
	}

	static void
	free_async_command(async_command_t * cmd)
	{
	if (!cmd) {
	return;
	}

	if (cmd->delay_id) {
	g_source_remove(cmd->delay_id);
	}

	cmd_list = g_list_remove(cmd_list, cmd);

	g_list_free_full(cmd->device_list, free);
	free(cmd->device);
	free(cmd->action);
	free(cmd->victim);
	free(cmd->remote_op_id);
	free(cmd->client);
	free(cmd->client_name);
	free(cmd->origin);
	free(cmd->mode);
	free(cmd->op);
	free(cmd);
	}

	static async_command_t *
	create_async_command(xmlNode * msg)
	{
	async_command_t *cmd = NULL;
	xmlNode *op = get_xpath_object("//@" F_STONITH_ACTION, msg, LOG_ERR);
	const char *action = crm_element_value(op, F_STONITH_ACTION);

	CRM_CHECK(action != NULL, crm_log_xml_warn(msg, "NoAction"); return NULL);

	crm_log_xml_trace(msg, "Command");
	cmd = calloc(1, sizeof(async_command_t));
	crm_element_value_int(msg, F_STONITH_CALLID, &(cmd->id));
	crm_element_value_int(msg, F_STONITH_CALLOPTS, &(cmd->options));
	crm_element_value_int(msg, F_STONITH_TIMEOUT, &(cmd->default_timeout));
	cmd->timeout = cmd->default_timeout;
	// Value -1 means disable any static/random fencing delays
	crm_element_value_int(msg, F_STONITH_DELAY, &(cmd->start_delay));

	cmd->origin = crm_element_value_copy(msg, F_ORIG);
	cmd->remote_op_id = crm_element_value_copy(msg, F_STONITH_REMOTE_OP_ID);
	cmd->client = crm_element_value_copy(msg, F_STONITH_CLIENTID);
	cmd->client_name = crm_element_value_copy(msg, F_STONITH_CLIENTNAME);
	cmd->op = crm_element_value_copy(msg, F_STONITH_OPERATION);
	cmd->action = strdup(action);
	cmd->victim = crm_element_value_copy(op, F_STONITH_TARGET);
	cmd->mode = crm_element_value_copy(op, F_STONITH_MODE);
	cmd->device = crm_element_value_copy(op, F_STONITH_DEVICE);

	CRM_CHECK(cmd->op != NULL, crm_log_xml_warn(msg, "NoOp"); free_async_command(cmd); return NULL);
	CRM_CHECK(cmd->client != NULL, crm_log_xml_warn(msg, "NoClient"));

	cmd->done_cb = st_child_done;
	cmd_list = g_list_append(cmd_list, cmd);
	return cmd;
	}

	static int
	get_action_limit(stonith_device_t * device)
	{
	const char *value = NULL;
	int action_limit = 1;

	value = g_hash_table_lookup(device->params, PCMK_STONITH_ACTION_LIMIT);
	if (value) {
	action_limit = crm_parse_int(value, "1");
	if (action_limit == 0) {
	/* pcmk_action_limit should not be 0. Enforce it to be 1. */
	action_limit = 1;
	}
	}

	return action_limit;
	}

	static int
	get_active_cmds(stonith_device_t * device)
	{
	int counter = 0;
	GListPtr gIter = NULL;
	GListPtr gIterNext = NULL;

	CRM_CHECK(device != NULL, return 0);

	for (gIter = cmd_list; gIter != NULL; gIter = gIterNext) {
	async_command_t *cmd = gIter->data;

	gIterNext = gIter->next;

	if (cmd->active_on == device) {
	counter++;
	}
	}

	return counter;
	}

	static void
	fork_cb(GPid pid, gpointer user_data)
	{
	async_command_t cmd = (async_command_t ) user_data;
	stonith_device_t * device =
	/* in case of a retry we've done the move from
	activating_on to active_on already
	*/
	cmd->activating_on?cmd->activating_on:cmd->active_on;

	CRM_ASSERT(device);
	crm_debug("Operation '%s' [%d]%s%s using %s now running with %ds timeout",
	cmd->action, pid,
	((cmd->victim == NULL)? "" : " targeting "),
	((cmd->victim == NULL)? "" : cmd->victim),
	device->id, cmd->timeout);
	cmd->active_on = device;
	cmd->activating_on = NULL;
	}

	static gboolean
	stonith_device_execute(stonith_device_t * device)
	{
	int exec_rc = 0;
	const char *action_str = NULL;
	const char *host_arg = NULL;
	async_command_t *cmd = NULL;
	stonith_action_t *action = NULL;
	int active_cmds = 0;
	int action_limit = 0;
	GListPtr gIter = NULL;
	GListPtr gIterNext = NULL;

	CRM_CHECK(device != NULL, return FALSE);

	active_cmds = get_active_cmds(device);
	action_limit = get_action_limit(device);
	if (action_limit > -1 && active_cmds >= action_limit) {
	crm_trace("%s is over its action limit of %d (%u active action%s)",
	device->id, action_limit, active_cmds,
	pcmk__plural_s(active_cmds));
	return TRUE;
	}

	for (gIter = device->pending_ops; gIter != NULL; gIter = gIterNext) {
	async_command_t *pending_op = gIter->data;

	gIterNext = gIter->next;

	if (pending_op && pending_op->delay_id) {
	crm_trace("Operation '%s'%s%s using %s was asked to run too early, "
	"waiting for start delay of %ds",
	pending_op->action,
	((pending_op->victim == NULL)? "" : " targeting "),
	((pending_op->victim == NULL)? "" : pending_op->victim),
	device->id, pending_op->start_delay);
	continue;
	}

	device->pending_ops = g_list_remove_link(device->pending_ops, gIter);
	g_list_free_1(gIter);

	cmd = pending_op;
	break;
	}

	if (cmd == NULL) {
	crm_trace("No actions using %s are needed", device->id);
	return TRUE;
	}

	if(pcmk__str_eq(device->agent, STONITH_WATCHDOG_AGENT, pcmk__str_casei)) {
	if(pcmk__str_eq(cmd->action, "reboot", pcmk__str_casei)) {
	pcmk__panic(__func__);
	goto done;

	} else if(pcmk__str_eq(cmd->action, "off", pcmk__str_casei)) {
	pcmk__panic(__func__);
	goto done;

	} else {
	crm_info("Faking success for %s watchdog operation", cmd->action);
	cmd->done_cb(0, 0, NULL, cmd);
	goto done;
	}
	}

	#if SUPPORT_CIBSECRETS
	if (pcmk__substitute_secrets(device->id, device->params) != pcmk_rc_ok) {
	/* replacing secrets failed! */
	if (pcmk__str_eq(cmd->action, "stop", pcmk__str_casei)) {
	/* don't fail on stop! */
	crm_info("Proceeding with stop operation for %s", device->id);

	} else {
	crm_err("Considering %s unconfigured: Failed to get secrets",
	device->id);
	exec_rc = PCMK_OCF_NOT_CONFIGURED;
	cmd->done_cb(0, exec_rc, NULL, cmd);
	goto done;
	}
	}
	#endif

	action_str = cmd->action;
	if (pcmk__str_eq(cmd->action, "reboot", pcmk__str_casei)
	&& !pcmk_is_set(device->flags, st_device_supports_reboot)) {

	crm_warn("Agent '%s' does not advertise support for 'reboot', performing 'off' action instead", device->agent);
	action_str = "off";
	}

	if (pcmk_is_set(device->flags, st_device_supports_parameter_port)) {
	host_arg = "port";

	} else if (pcmk_is_set(device->flags, st_device_supports_parameter_plug)) {
	host_arg = "plug";
	}

	action = stonith_action_create(device->agent,
	action_str,
	cmd->victim,
	cmd->victim_nodeid,
	cmd->timeout, device->params,
	device->aliases, host_arg);

	/* for async exec, exec_rc is negative for early error exit
	otherwise handling of success/errors is done via callbacks */
	cmd->activating_on = device;
	exec_rc = stonith_action_execute_async(action, (void *)cmd,
	cmd->done_cb, fork_cb);

	if (exec_rc < 0) {
	crm_warn("Operation '%s'%s%s using %s failed: %s " CRM_XS " rc=%d",
	cmd->action, cmd->victim ? " targeting " : "", cmd->victim ? cmd->victim : "",
	device->id, pcmk_strerror(exec_rc), exec_rc);
	cmd->activating_on = NULL;
	cmd->done_cb(0, exec_rc, NULL, cmd);
	}

	done:
	/* Device might get triggered to work by multiple fencing commands
	* simultaneously. Trigger the device again to make sure any
	* remaining concurrent commands get executed. */
	if (device->pending_ops) {
	mainloop_set_trigger(device->work);
	}
	return TRUE;
	}

	static gboolean
	stonith_device_dispatch(gpointer user_data)
	{
	return stonith_device_execute(user_data);
	}

	static gboolean
	start_delay_helper(gpointer data)
	{
	async_command_t *cmd = data;
	stonith_device_t *device = NULL;

	cmd->delay_id = 0;
	device = cmd->device ? g_hash_table_lookup(device_list, cmd->device) : NULL;

	if (device) {
	mainloop_set_trigger(device->work);
	}

	return FALSE;
	}

	static void
	schedule_stonith_command(async_command_t * cmd, stonith_device_t * device)
	{
	int delay_max = 0;
	int delay_base = 0;
	int requested_delay = cmd->start_delay;

	CRM_CHECK(cmd != NULL, return);
	CRM_CHECK(device != NULL, return);

	if (cmd->device) {
	free(cmd->device);
	}

	if (device->include_nodeid && cmd->victim) {
	crm_node_t *node = crm_get_peer(0, cmd->victim);

	cmd->victim_nodeid = node->id;
	}

	cmd->device = strdup(device->id);
	cmd->timeout = get_action_timeout(device, cmd->action, cmd->default_timeout);

	if (cmd->remote_op_id) {
	crm_debug("Scheduling '%s' action%s%s using %s for remote peer %s "
	"with op id %.8s and timeout %ds",
	cmd->action,
	cmd->victim ? " targeting " : "", cmd->victim ? cmd->victim : "",
	device->id, cmd->origin, cmd->remote_op_id, cmd->timeout);
	} else {
	crm_debug("Scheduling '%s' action%s%s using %s for %s with timeout %ds",
	cmd->action,
	cmd->victim ? " targeting " : "", cmd->victim ? cmd->victim : "",
	device->id, cmd->client, cmd->timeout);
	}

	device->pending_ops = g_list_append(device->pending_ops, cmd);
	mainloop_set_trigger(device->work);

	// Value -1 means disable any static/random fencing delays
	if (requested_delay < 0) {
	return;
	}

	delay_max = get_action_delay_max(device, cmd->action);
	delay_base = get_action_delay_base(device, cmd->action);
	if (delay_max == 0) {
	delay_max = delay_base;
	}
	if (delay_max < delay_base) {
	crm_warn(PCMK_STONITH_DELAY_BASE " (%ds) is larger than "
	PCMK_STONITH_DELAY_MAX " (%ds) for %s using %s "
	"(limiting to maximum delay)",
	delay_base, delay_max, cmd->action, device->id);
	delay_base = delay_max;
	}
	if (delay_max > 0) {
	// coverity[dont_call] We're not using rand() for security
	cmd->start_delay +=
	((delay_max != delay_base)?(rand() % (delay_max - delay_base)):0)
	+ delay_base;
	}

	if (cmd->start_delay > 0) {
	crm_notice("Delaying '%s' action%s%s using %s for %ds " CRM_XS
	" timeout=%ds requested_delay=%ds base=%ds max=%ds",
	cmd->action,
	cmd->victim ? " targeting " : "", cmd->victim ? cmd->victim : "",
	device->id, cmd->start_delay, cmd->timeout,
	requested_delay, delay_base, delay_max);
	cmd->delay_id =
	g_timeout_add_seconds(cmd->start_delay, start_delay_helper, cmd);
	}
	}

	static void
	free_device(gpointer data)
	{
	GListPtr gIter = NULL;
	stonith_device_t *device = data;

	g_hash_table_destroy(device->params);
	g_hash_table_destroy(device->aliases);

	for (gIter = device->pending_ops; gIter != NULL; gIter = gIter->next) {
	async_command_t *cmd = gIter->data;

	crm_warn("Removal of device '%s' purged operation '%s'", device->id, cmd->action);
	cmd->done_cb(0, -ENODEV, NULL, cmd);
	}
	g_list_free(device->pending_ops);

	g_list_free_full(device->targets, free);

	mainloop_destroy_trigger(device->work);

	free_xml(device->agent_metadata);
	free(device->namespace);
	free(device->on_target_actions);
	free(device->agent);
	free(device->id);
	free(device);
	}

	void free_device_list(void)
	{
	if (device_list != NULL) {
	g_hash_table_destroy(device_list);
	device_list = NULL;
	}
	}

	void
	init_device_list(void)
	{
	if (device_list == NULL) {
	device_list = g_hash_table_new_full(crm_str_hash, g_str_equal, NULL,
	free_device);
	}
	}

	static GHashTable *
	build_port_aliases(const char hostmap, GListPtr targets)
	{
	char *name = NULL;
	int last = 0, lpc = 0, max = 0, added = 0;
	GHashTable *aliases = crm_strcase_table_new();

	if (hostmap == NULL) {
	return aliases;
	}

	max = strlen(hostmap);
	for (; lpc <= max; lpc++) {
	switch (hostmap[lpc]) {
	/* Assignment chars */
	case '=':
	case ':':
	if (lpc > last) {
	free(name);
	name = calloc(1, 1 + lpc - last);
	memcpy(name, hostmap + last, lpc - last);
	}
	last = lpc + 1;
	break;

	/* Delimeter chars */
	/* case ',': Potentially used to specify multiple ports */
	case 0:
	case ';':
	case ' ':
	case '\t':
	if (name) {
	char *value = NULL;

	value = calloc(1, 1 + lpc - last);
	memcpy(value, hostmap + last, lpc - last);

	crm_debug("Adding alias '%s'='%s'", name, value);
	g_hash_table_replace(aliases, name, value);
	if (targets) {
	targets = g_list_append(targets, strdup(value));
	}
	value = NULL;
	name = NULL;
	added++;

	} else if (lpc > last) {
	crm_debug("Parse error at offset %d near '%s'", lpc - last, hostmap + last);
	}

	last = lpc + 1;
	break;
	}

	if (hostmap[lpc] == 0) {
	break;
	}
	}

	if (added == 0) {
	crm_info("No host mappings detected in '%s'", hostmap);
	}

	free(name);
	return aliases;
	}

	GHashTable *metadata_cache = NULL;

	void
	free_metadata_cache(void) {
	if (metadata_cache != NULL) {
	g_hash_table_destroy(metadata_cache);
	metadata_cache = NULL;
	}
	}

	static void
	init_metadata_cache(void) {
	if (metadata_cache == NULL) {
	metadata_cache = crm_str_table_new();
	}
	}

	static xmlNode *
	get_agent_metadata(const char *agent)
	{
	xmlNode *xml = NULL;
	char *buffer = NULL;

	init_metadata_cache();
	buffer = g_hash_table_lookup(metadata_cache, agent);
	if(pcmk__str_eq(agent, STONITH_WATCHDOG_AGENT, pcmk__str_casei)) {
	return NULL;

	} else if(buffer == NULL) {
	stonith_t *st = stonith_api_new();
	int rc;

	if (st == NULL) {
	crm_warn("Could not get agent meta-data: "
	"API memory allocation failed");
	return NULL;
	}
	rc = st->cmds->metadata(st, st_opt_sync_call, agent, NULL, &buffer, 10);
	stonith_api_delete(st);
	if (rc \|\| !buffer) {
	crm_err("Could not retrieve metadata for fencing agent %s", agent);
	return NULL;
	}
	g_hash_table_replace(metadata_cache, strdup(agent), buffer);
	}

	xml = string2xml(buffer);

	return xml;
	}

	static gboolean
	is_nodeid_required(xmlNode * xml)
	{
	xmlXPathObjectPtr xpath = NULL;

	if (stand_alone) {
	return FALSE;
	}

	if (!xml) {
	return FALSE;
	}

	xpath = xpath_search(xml, "//parameter[@name='nodeid']");
	if (numXpathResults(xpath) <= 0) {
	freeXpathObject(xpath);
	return FALSE;
	}

	freeXpathObject(xpath);
	return TRUE;
	}

	#define MAX_ACTION_LEN 256

	static char *
	add_action(char actions, const char action)
	{
	int offset = 0;

	if (actions == NULL) {
	actions = calloc(1, MAX_ACTION_LEN);
	} else {
	offset = strlen(actions);
	}

	if (offset > 0) {
	offset += snprintf(actions+offset, MAX_ACTION_LEN - offset, " ");
	}
	offset += snprintf(actions+offset, MAX_ACTION_LEN - offset, "%s", action);

	return actions;
	}

	static void
	read_action_metadata(stonith_device_t *device)
	{
	xmlXPathObjectPtr xpath = NULL;
	int max = 0;
	int lpc = 0;

	if (device->agent_metadata == NULL) {
	return;
	}

	xpath = xpath_search(device->agent_metadata, "//action");
	max = numXpathResults(xpath);

	if (max <= 0) {
	freeXpathObject(xpath);
	return;
	}

	for (lpc = 0; lpc < max; lpc++) {
	const char *on_target = NULL;
	const char *action = NULL;
	xmlNode *match = getXpathResult(xpath, lpc);

	CRM_LOG_ASSERT(match != NULL);
	if(match == NULL) { continue; };

	on_target = crm_element_value(match, "on_target");
	action = crm_element_value(match, "name");

	if(pcmk__str_eq(action, "list", pcmk__str_casei)) {
	stonith__set_device_flags(device->flags, device->id,
	st_device_supports_list);
	} else if(pcmk__str_eq(action, "status", pcmk__str_casei)) {
	stonith__set_device_flags(device->flags, device->id,
	st_device_supports_status);
	} else if(pcmk__str_eq(action, "reboot", pcmk__str_casei)) {
	stonith__set_device_flags(device->flags, device->id,
	st_device_supports_reboot);
	} else if (pcmk__str_eq(action, "on", pcmk__str_casei)) {
	/* "automatic" means the cluster will unfence node when it joins */
	const char *automatic = crm_element_value(match, "automatic");

	/* "required" is a deprecated synonym for "automatic" */
	const char *required = crm_element_value(match, "required");

	if (crm_is_true(automatic) \|\| crm_is_true(required)) {
	device->automatic_unfencing = TRUE;
	}
	}

	if (action && crm_is_true(on_target)) {
	device->on_target_actions = add_action(device->on_target_actions, action);
	}
	}

	freeXpathObject(xpath);
	}

	/*!
	* \internal
	* \brief Set a pcmk_*_action parameter if not already set
	*
	* \param[in,out] params Device parameters
	* \param[in] action Name of action
	* \param[in] value Value to use if action is not already set
	*/
	static void
	map_action(GHashTable params, const char action, const char *value)
	{
	char *key = crm_strdup_printf("pcmk_%s_action", action);

	if (g_hash_table_lookup(params, key)) {
	crm_warn("Ignoring %s='%s', see %s instead",
	STONITH_ATTR_ACTION_OP, value, key);
	free(key);
	} else {
	crm_warn("Mapping %s='%s' to %s='%s'",
	STONITH_ATTR_ACTION_OP, value, key, value);
	g_hash_table_insert(params, key, strdup(value));
	}
	}

	/*!
	* \internal
	* \brief Create device parameter table from XML
	*
	* \param[in] name Device name (used for logging only)
	* \param[in,out] params Device parameters
	*/
	static GHashTable *
	xml2device_params(const char name, xmlNode dev)
	{
	GHashTable *params = xml2list(dev);
	const char *value;

	/* Action should never be specified in the device configuration,
	* but we support it for users who are familiar with other software
	* that worked that way.
	*/
	value = g_hash_table_lookup(params, STONITH_ATTR_ACTION_OP);
	if (value != NULL) {
	crm_warn("%s has '%s' parameter, which should never be specified in configuration",
	name, STONITH_ATTR_ACTION_OP);

	if (*value == '\0') {
	crm_warn("Ignoring empty '%s' parameter", STONITH_ATTR_ACTION_OP);

	} else if (strcmp(value, "reboot") == 0) {
	crm_warn("Ignoring %s='reboot' (see stonith-action cluster property instead)",
	STONITH_ATTR_ACTION_OP);

	} else if (strcmp(value, "off") == 0) {
	map_action(params, "reboot", value);

	} else {
	map_action(params, "off", value);
	map_action(params, "reboot", value);
	}

	g_hash_table_remove(params, STONITH_ATTR_ACTION_OP);
	}

	return params;
	}

	static stonith_device_t *
	build_device_from_xml(xmlNode * msg)
	{
	const char *value;
	xmlNode *dev = get_xpath_object("//" F_STONITH_DEVICE, msg, LOG_ERR);
	stonith_device_t *device = NULL;
	char *agent = crm_element_value_copy(dev, "agent");

	CRM_CHECK(agent != NULL, return device);

	device = calloc(1, sizeof(stonith_device_t));

	CRM_CHECK(device != NULL, {free(agent); return device;});

	device->id = crm_element_value_copy(dev, XML_ATTR_ID);
	device->agent = agent;
	device->namespace = crm_element_value_copy(dev, "namespace");
	device->params = xml2device_params(device->id, dev);

	value = g_hash_table_lookup(device->params, PCMK_STONITH_HOST_LIST);
	if (value) {
	device->targets = stonith__parse_targets(value);
	}

	value = g_hash_table_lookup(device->params, PCMK_STONITH_HOST_MAP);
	device->aliases = build_port_aliases(value, &(device->targets));

	device->agent_metadata = get_agent_metadata(device->agent);
	if (device->agent_metadata) {
	read_action_metadata(device);
	stonith__device_parameter_flags(&(device->flags), device->id,
	device->agent_metadata);
	}

	value = g_hash_table_lookup(device->params, "nodeid");
	if (!value) {
	device->include_nodeid = is_nodeid_required(device->agent_metadata);
	}

	value = crm_element_value(dev, "rsc_provides");
	if (pcmk__str_eq(value, "unfencing", pcmk__str_casei)) {
	device->automatic_unfencing = TRUE;
	}

	if (is_action_required("on", device)) {
	crm_info("Fencing device '%s' requires unfencing", device->id);
	}

	if (device->on_target_actions) {
	crm_info("Fencing device '%s' requires actions (%s) to be executed "
	"on target", device->id, device->on_target_actions);
	}

	device->work = mainloop_add_trigger(G_PRIORITY_HIGH, stonith_device_dispatch, device);
	/* TODO: Hook up priority */

	return device;
	}

	static const char *
	target_list_type(stonith_device_t * dev)
	{
	const char *check_type = NULL;

	check_type = g_hash_table_lookup(dev->params, PCMK_STONITH_HOST_CHECK);

	if (check_type == NULL) {

	if (g_hash_table_lookup(dev->params, PCMK_STONITH_HOST_LIST)) {
	check_type = "static-list";
	} else if (g_hash_table_lookup(dev->params, PCMK_STONITH_HOST_MAP)) {
	check_type = "static-list";
	} else if (pcmk_is_set(dev->flags, st_device_supports_list)) {
	check_type = "dynamic-list";
	} else if (pcmk_is_set(dev->flags, st_device_supports_status)) {
	check_type = "status";
	} else {
	check_type = "none";
	}
	}

	return check_type;
	}

	static void
	schedule_internal_command(const char *origin,
	stonith_device_t * device,
	const char *action,
	const char *victim,
	int timeout,
	void *internal_user_data,
	void (done_cb) (GPid pid, int rc, const char output,
	gpointer user_data))
	{
	async_command_t *cmd = NULL;

	cmd = calloc(1, sizeof(async_command_t));

	cmd->id = -1;
	cmd->default_timeout = timeout ? timeout : 60;
	cmd->timeout = cmd->default_timeout;
	cmd->action = strdup(action);
	cmd->victim = victim ? strdup(victim) : NULL;
	cmd->device = strdup(device->id);
	cmd->origin = strdup(origin);
	cmd->client = strdup(crm_system_name);
	cmd->client_name = strdup(crm_system_name);

	cmd->internal_user_data = internal_user_data;
	cmd->done_cb = done_cb; /* cmd, not internal_user_data, is passed to 'done_cb' as the userdata */

	schedule_stonith_command(cmd, device);
	}

	gboolean
	string_in_list(GListPtr list, const char *item)
	{
	int lpc = 0;
	int max = g_list_length(list);

	for (lpc = 0; lpc < max; lpc++) {
	const char *value = g_list_nth_data(list, lpc);

	if (pcmk__str_eq(item, value, pcmk__str_casei)) {
	return TRUE;
	} else {
	crm_trace("%d: '%s' != '%s'", lpc, item, value);
	}
	}
	return FALSE;
	}

	static void
	status_search_cb(GPid pid, int rc, const char *output, gpointer user_data)
	{
	async_command_t *cmd = user_data;
	struct device_search_s *search = cmd->internal_user_data;
	stonith_device_t *dev = cmd->device ? g_hash_table_lookup(device_list, cmd->device) : NULL;
	gboolean can = FALSE;

	free_async_command(cmd);

	if (!dev) {
	search_devices_record_result(search, NULL, FALSE);
	return;
	}

	mainloop_set_trigger(dev->work);

	if (rc == 1 /* unknown */ ) {
	crm_trace("Host %s is not known by %s", search->host, dev->id);

	} else if (rc == 0 /* active / \|\| rc == 2 / inactive */ ) {
	crm_trace("Host %s is known by %s", search->host, dev->id);
	can = TRUE;

	} else {
	crm_notice("Unknown result when testing if %s can fence %s: rc=%d", dev->id, search->host,
	rc);
	}
	search_devices_record_result(search, dev->id, can);
	}

	static void
	dynamic_list_search_cb(GPid pid, int rc, const char *output, gpointer user_data)
	{
	async_command_t *cmd = user_data;
	struct device_search_s *search = cmd->internal_user_data;
	stonith_device_t *dev = cmd->device ? g_hash_table_lookup(device_list, cmd->device) : NULL;
	gboolean can_fence = FALSE;

	free_async_command(cmd);

	/* Host/alias must be in the list output to be eligible to be fenced
	*
	* Will cause problems if down'd nodes aren't listed or (for virtual nodes)
	* if the guest is still listed despite being moved to another machine
	*/
	if (!dev) {
	search_devices_record_result(search, NULL, FALSE);
	return;
	}

	mainloop_set_trigger(dev->work);

	/* If we successfully got the targets earlier, don't disable. */
	if (rc != 0 && !dev->targets) {
	crm_notice("Disabling port list queries for %s: %s "
	CRM_XS " rc=%d", dev->id, output, rc);
	/* Fall back to status */
	g_hash_table_replace(dev->params,
	strdup(PCMK_STONITH_HOST_CHECK), strdup("status"));

	g_list_free_full(dev->targets, free);
	dev->targets = NULL;
	} else if (!rc) {
	crm_info("Refreshing port list for %s", dev->id);
	g_list_free_full(dev->targets, free);
	dev->targets = stonith__parse_targets(output);
	dev->targets_age = time(NULL);
	}

	if (dev->targets) {
	const char *alias = g_hash_table_lookup(dev->aliases, search->host);

	if (!alias) {
	alias = search->host;
	}
	if (string_in_list(dev->targets, alias)) {
	can_fence = TRUE;
	}
	}
	search_devices_record_result(search, dev->id, can_fence);
	}

	/*!
	* \internal
	* \brief Returns true if any key in first is not in second or second has a different value for key
	*/
	static int
	device_params_diff(GHashTable first, GHashTable second) {
	char *key = NULL;
	char *value = NULL;
	GHashTableIter gIter;

	g_hash_table_iter_init(&gIter, first);
	while (g_hash_table_iter_next(&gIter, (void )&key, (void )&value)) {

	if(strstr(key, "CRM_meta") == key) {
	continue;
	} else if(strcmp(key, "crm_feature_set") == 0) {
	continue;
	} else {
	char *other_value = g_hash_table_lookup(second, key);

	if (!other_value \|\| !pcmk__str_eq(other_value, value, pcmk__str_casei)) {
	crm_trace("Different value for %s: %s != %s", key, other_value, value);
	return 1;
	}
	}
	}

	return 0;
	}

	/*!
	* \internal
	* \brief Checks to see if an identical device already exists in the device_list
	*/
	static stonith_device_t *
	device_has_duplicate(stonith_device_t * device)
	{
	stonith_device_t *dup = g_hash_table_lookup(device_list, device->id);

	if (!dup) {
	crm_trace("No match for %s", device->id);
	return NULL;

	} else if (!pcmk__str_eq(dup->agent, device->agent, pcmk__str_casei)) {
	crm_trace("Different agent: %s != %s", dup->agent, device->agent);
	return NULL;
	}

	/* Use calculate_operation_digest() here? */
	if (device_params_diff(device->params, dup->params) \|\|
	device_params_diff(dup->params, device->params)) {
	return NULL;
	}

	crm_trace("Match");
	return dup;
	}

	int
	stonith_device_register(xmlNode * msg, const char **desc, gboolean from_cib)
	{
	stonith_device_t *dup = NULL;
	stonith_device_t *device = build_device_from_xml(msg);
	guint ndevices = 0;

	CRM_CHECK(device != NULL, return -ENOMEM);

	dup = device_has_duplicate(device);
	if (dup) {
	ndevices = g_hash_table_size(device_list);
	crm_debug("Device '%s' already in device list (%d active device%s)",
	device->id, ndevices, pcmk__plural_s(ndevices));
	free_device(device);
	device = dup;

	} else {
	stonith_device_t *old = g_hash_table_lookup(device_list, device->id);

	if (from_cib && old && old->api_registered) {
	/* If the cib is writing over an entry that is shared with a stonith client,
	* copy any pending ops that currently exist on the old entry to the new one.
	* Otherwise the pending ops will be reported as failures
	*/
	crm_info("Overwriting existing entry for %s from CIB", device->id);
	device->pending_ops = old->pending_ops;
	device->api_registered = TRUE;
	old->pending_ops = NULL;
	if (device->pending_ops) {
	mainloop_set_trigger(device->work);
	}
	}
	g_hash_table_replace(device_list, device->id, device);

	ndevices = g_hash_table_size(device_list);
	crm_notice("Added '%s' to device list (%d active device%s)",
	device->id, ndevices, pcmk__plural_s(ndevices));
	}
	if (desc) {
	*desc = device->id;
	}

	if (from_cib) {
	device->cib_registered = TRUE;
	} else {
	device->api_registered = TRUE;
	}

	return pcmk_ok;
	}

	int
	stonith_device_remove(const char *id, gboolean from_cib)
	{
	stonith_device_t *device = g_hash_table_lookup(device_list, id);
	guint ndevices = 0;

	if (!device) {
	ndevices = g_hash_table_size(device_list);
	crm_info("Device '%s' not found (%d active device%s)",
	id, ndevices, pcmk__plural_s(ndevices));
	return pcmk_ok;
	}

	if (from_cib) {
	device->cib_registered = FALSE;
	} else {
	device->verified = FALSE;
	device->api_registered = FALSE;
	}

	if (!device->cib_registered && !device->api_registered) {
	g_hash_table_remove(device_list, id);
	ndevices = g_hash_table_size(device_list);
	crm_info("Removed '%s' from device list (%d active device%s)",
	id, ndevices, pcmk__plural_s(ndevices));
	} else {
	crm_trace("Not removing '%s' from device list (%d active) because "
	"still registered via:%s%s",
	id, g_hash_table_size(device_list),
	(device->cib_registered? " cib" : ""),
	(device->api_registered? " api" : ""));
	}
	return pcmk_ok;
	}

	/*!
	* \internal
	* \brief Return the number of stonith levels registered for a node
	*
	* \param[in] tp Node's topology table entry
	*
	* \return Number of non-NULL levels in topology entry
	* \note This function is used only for log messages.
	*/
	static int
	count_active_levels(stonith_topology_t * tp)
	{
	int lpc = 0;
	int count = 0;

	for (lpc = 0; lpc < ST_LEVEL_MAX; lpc++) {
	if (tp->levels[lpc] != NULL) {
	count++;
	}
	}
	return count;
	}

	static void
	free_topology_entry(gpointer data)
	{
	stonith_topology_t *tp = data;

	int lpc = 0;

	for (lpc = 0; lpc < ST_LEVEL_MAX; lpc++) {
	if (tp->levels[lpc] != NULL) {
	g_list_free_full(tp->levels[lpc], free);
	}
	}
	free(tp->target);
	free(tp->target_value);
	free(tp->target_pattern);
	free(tp->target_attribute);
	free(tp);
	}

	void
	free_topology_list(void)
	{
	if (topology != NULL) {
	g_hash_table_destroy(topology);
	topology = NULL;
	}
	}

	void
	init_topology_list(void)
	{
	if (topology == NULL) {
	topology = g_hash_table_new_full(crm_str_hash, g_str_equal, NULL,
	free_topology_entry);
	}
	}

	char stonith_level_key(xmlNode level, int mode)
	{
	if(mode == -1) {
	mode = stonith_level_kind(level);
	}

	switch(mode) {
	case 0:
	return crm_element_value_copy(level, XML_ATTR_STONITH_TARGET);
	case 1:
	return crm_element_value_copy(level, XML_ATTR_STONITH_TARGET_PATTERN);
	case 2:
	{
	const char *name = crm_element_value(level, XML_ATTR_STONITH_TARGET_ATTRIBUTE);
	const char *value = crm_element_value(level, XML_ATTR_STONITH_TARGET_VALUE);

	if(name && value) {
	return crm_strdup_printf("%s=%s", name, value);
	}
	}
	default:
	return crm_strdup_printf("Unknown-%d-%s", mode, ID(level));
	}
	}

	int stonith_level_kind(xmlNode * level)
	{
	int mode = 0;
	const char *target = crm_element_value(level, XML_ATTR_STONITH_TARGET);

	if(target == NULL) {
	mode++;
	target = crm_element_value(level, XML_ATTR_STONITH_TARGET_PATTERN);
	}

	if(stand_alone == FALSE && target == NULL) {

	mode++;

	if(crm_element_value(level, XML_ATTR_STONITH_TARGET_ATTRIBUTE) == NULL) {
	mode++;

	} else if(crm_element_value(level, XML_ATTR_STONITH_TARGET_VALUE) == NULL) {
	mode++;
	}
	}

	return mode;
	}

	static stonith_key_value_t *
	parse_device_list(const char *devices)
	{
	int lpc = 0;
	int max = 0;
	int last = 0;
	stonith_key_value_t *output = NULL;

	if (devices == NULL) {
	return output;
	}

	max = strlen(devices);
	for (lpc = 0; lpc <= max; lpc++) {
	if (devices[lpc] == ',' \|\| devices[lpc] == 0) {
	char *line = strndup(devices + last, lpc - last);

	output = stonith_key_value_add(output, NULL, line);
	free(line);

	last = lpc + 1;
	}
	}

	return output;
	}

	/*!
	* \internal
	* \brief Register a STONITH level for a target
	*
	* Given an XML request specifying the target name, level index, and device IDs
	* for the level, this will create an entry for the target in the global topology
	* table if one does not already exist, then append the specified device IDs to
	* the entry's device list for the specified level.
	*
	* \param[in] msg XML request for STONITH level registration
	* \param[out] desc If not NULL, will be set to string representation ("TARGET[LEVEL]")
	*
	* \return pcmk_ok on success, -EINVAL if XML does not specify valid level index
	*/
	int
	stonith_level_register(xmlNode msg, char *desc)
	{
	int id = 0;
	xmlNode *level;
	int mode;
	char *target;

	stonith_topology_t *tp;
	stonith_key_value_t *dIter = NULL;
	stonith_key_value_t *devices = NULL;

	/* Allow the XML here to point to the level tag directly, or wrapped in
	* another tag. If directly, don't search by xpath, because it might give
	* multiple hits (e.g. if the XML is the CIB).
	*/
	if (pcmk__str_eq(TYPE(msg), XML_TAG_FENCING_LEVEL, pcmk__str_casei)) {
	level = msg;
	} else {
	level = get_xpath_object("//" XML_TAG_FENCING_LEVEL, msg, LOG_ERR);
	}
	CRM_CHECK(level != NULL, return -EINVAL);

	mode = stonith_level_kind(level);
	target = stonith_level_key(level, mode);
	crm_element_value_int(level, XML_ATTR_STONITH_INDEX, &id);

	if (desc) {
	*desc = crm_strdup_printf("%s[%d]", target, id);
	}

	/* Sanity-check arguments */
	if (mode >= 3 \|\| (id <= 0) \|\| (id >= ST_LEVEL_MAX)) {
	crm_trace("Could not add %s[%d] (%d) to the topology (%d active entries)", target, id, mode, g_hash_table_size(topology));
	free(target);
	crm_log_xml_err(level, "Bad topology");
	return -EINVAL;
	}

	/* Find or create topology table entry */
	tp = g_hash_table_lookup(topology, target);
	if (tp == NULL) {
	tp = calloc(1, sizeof(stonith_topology_t));
	tp->kind = mode;
	tp->target = target;
	tp->target_value = crm_element_value_copy(level, XML_ATTR_STONITH_TARGET_VALUE);
	tp->target_pattern = crm_element_value_copy(level, XML_ATTR_STONITH_TARGET_PATTERN);
	tp->target_attribute = crm_element_value_copy(level, XML_ATTR_STONITH_TARGET_ATTRIBUTE);

	g_hash_table_replace(topology, tp->target, tp);
	crm_trace("Added %s (%d) to the topology (%d active entries)",
	target, mode, g_hash_table_size(topology));
	} else {
	free(target);
	}

	if (tp->levels[id] != NULL) {
	crm_info("Adding to the existing %s[%d] topology entry",
	tp->target, id);
	}

	devices = parse_device_list(crm_element_value(level, XML_ATTR_STONITH_DEVICES));
	for (dIter = devices; dIter; dIter = dIter->next) {
	const char *device = dIter->value;

	crm_trace("Adding device '%s' for %s[%d]", device, tp->target, id);
	tp->levels[id] = g_list_append(tp->levels[id], strdup(device));
	}
	stonith_key_value_freeall(devices, 1, 1);

	{
	int nlevels = count_active_levels(tp);

	crm_info("Target %s has %d active fencing level%s",
	tp->target, nlevels, pcmk__plural_s(nlevels));
	}
	return pcmk_ok;
	}

	int
	stonith_level_remove(xmlNode msg, char *desc)
	{
	int id = 0;
	stonith_topology_t *tp;
	char *target;

	/* Unlike additions, removal requests should always have one level tag */
	xmlNode *level = get_xpath_object("//" XML_TAG_FENCING_LEVEL, msg, LOG_ERR);

	CRM_CHECK(level != NULL, return -EINVAL);

	target = stonith_level_key(level, -1);
	crm_element_value_int(level, XML_ATTR_STONITH_INDEX, &id);
	if (desc) {
	*desc = crm_strdup_printf("%s[%d]", target, id);
	}

	/* Sanity-check arguments */
	if (id >= ST_LEVEL_MAX) {
	free(target);
	return -EINVAL;
	}

	tp = g_hash_table_lookup(topology, target);
	if (tp == NULL) {
	guint nentries = g_hash_table_size(topology);

	crm_info("No fencing topology found for %s (%d active %s)",
	target, nentries,
	pcmk__plural_alt(nentries, "entry", "entries"));

	} else if (id == 0 && g_hash_table_remove(topology, target)) {
	guint nentries = g_hash_table_size(topology);

	crm_info("Removed all fencing topology entries related to %s "
	"(%d active %s remaining)", target, nentries,
	pcmk__plural_alt(nentries, "entry", "entries"));

	} else if (id > 0 && tp->levels[id] != NULL) {
	guint nlevels;

	g_list_free_full(tp->levels[id], free);
	tp->levels[id] = NULL;

	nlevels = count_active_levels(tp);
	crm_info("Removed level %d from fencing topology for %s "
	"(%d active level%s remaining)",
	id, target, nlevels, pcmk__plural_s(nlevels));
	}

	free(target);
	return pcmk_ok;
	}

	/*!
	* \internal
	* \brief Schedule an (asynchronous) action directly on a stonith device
	*
	* Handle a STONITH_OP_EXEC API message by scheduling a requested agent action
	* directly on a specified device. Only list, monitor, and status actions are
	* expected to use this call, though it should work with any agent command.
	*
	* \param[in] msg API message XML with desired action
	* \param[out] output Unused
	*
	* \return -EINPROGRESS on success, -errno otherwise
	* \note If the action is monitor, the device must be registered via the API
	* (CIB registration is not sufficient), because monitor should not be
	* possible unless the device is "started" (API registered).
	*/
	static int
	stonith_device_action(xmlNode * msg, char **output)
	{
	xmlNode *dev = get_xpath_object("//" F_STONITH_DEVICE, msg, LOG_ERR);
	xmlNode *op = get_xpath_object("//@" F_STONITH_ACTION, msg, LOG_ERR);
	const char *id = crm_element_value(dev, F_STONITH_DEVICE);
	const char *action = crm_element_value(op, F_STONITH_ACTION);
	async_command_t *cmd = NULL;
	stonith_device_t *device = NULL;

	if ((id == NULL) \|\| (action == NULL)) {
	crm_info("Malformed API action request: device %s, action %s",
	(id? id : "not specified"),
	(action? action : "not specified"));
	return -EPROTO;
	}

	device = g_hash_table_lookup(device_list, id);
	if ((device == NULL)
	\|\| (!device->api_registered && !strcmp(action, "monitor"))) {

	// Monitors may run only on "started" (API-registered) devices
	crm_info("Ignoring API '%s' action request because device %s not found",
	action, id);
	return -ENODEV;
	}

	cmd = create_async_command(msg);
	if (cmd == NULL) {
	return -EPROTO;
	}

	schedule_stonith_command(cmd, device);
	return -EINPROGRESS;
	}

	static void
	search_devices_record_result(struct device_search_s search, const char device, gboolean can_fence)
	{
	search->replies_received++;

	if (can_fence && device) {
	search->capable = g_list_append(search->capable, strdup(device));
	}

	if (search->replies_needed == search->replies_received) {

	guint ndevices = g_list_length(search->capable);

	crm_debug("Search found %d device%s that can perform '%s' targeting %s",
	ndevices, pcmk__plural_s(ndevices),
	(search->action? search->action : "unknown action"),
	(search->host? search->host : "any node"));

	search->callback(search->capable, search->user_data);
	free(search->host);
	free(search->action);
	free(search);
	}
	}

	/*!
	* \internal
	* \brief Check whether the local host is allowed to execute a fencing action
	*
	* \param[in] device Fence device to check
	* \param[in] action Fence action to check
	* \param[in] target Hostname of fence target
	* \param[in] allow_suicide Whether self-fencing is allowed for this operation
	*
	* \return TRUE if local host is allowed to execute action, FALSE otherwise
	*/
	static gboolean
	localhost_is_eligible(const stonith_device_t device, const char action,
	const char *target, gboolean allow_suicide)
	{
	gboolean localhost_is_target = pcmk__str_eq(target, stonith_our_uname,
	pcmk__str_casei);

	if (device && action && device->on_target_actions
	&& strstr(device->on_target_actions, action)) {
	if (!localhost_is_target) {
	crm_trace("Operation '%s' using %s can only be executed for "
	"local host, not %s", action, device->id, target);
	return FALSE;
	}

	} else if (localhost_is_target && !allow_suicide) {
	crm_trace("'%s' operation does not support self-fencing", action);
	return FALSE;
	}
	return TRUE;
	}

	static void
	can_fence_host_with_device(stonith_device_t * dev, struct device_search_s *search)
	{
	gboolean can = FALSE;
	const char *check_type = NULL;
	const char *host = search->host;
	const char *alias = NULL;

	CRM_LOG_ASSERT(dev != NULL);

	if (dev == NULL) {
	goto search_report_results;
	} else if (host == NULL) {
	can = TRUE;
	goto search_report_results;
	}

	/* Short-circuit query if this host is not allowed to perform the action */
	if (pcmk__str_eq(search->action, "reboot", pcmk__str_casei)) {
	/* A "reboot" might get remapped to "off" then "on", so short-circuit
	* only if all three are disallowed. If only one or two are disallowed,
	* we'll report that with the results. We never allow suicide for
	* remapped "on" operations because the host is off at that point.
	*/
	if (!localhost_is_eligible(dev, "reboot", host, search->allow_suicide)
	&& !localhost_is_eligible(dev, "off", host, search->allow_suicide)
	&& !localhost_is_eligible(dev, "on", host, FALSE)) {
	goto search_report_results;
	}
	} else if (!localhost_is_eligible(dev, search->action, host,
	search->allow_suicide)) {
	goto search_report_results;
	}

	alias = g_hash_table_lookup(dev->aliases, host);
	if (alias == NULL) {
	alias = host;
	}

	check_type = target_list_type(dev);

	if (pcmk__str_eq(check_type, "none", pcmk__str_casei)) {
	can = TRUE;

	} else if (pcmk__str_eq(check_type, "static-list", pcmk__str_casei)) {

	/* Presence in the hostmap is sufficient
	* Only use if all hosts on which the device can be active can always fence all listed hosts
	*/

	if (string_in_list(dev->targets, host)) {
	can = TRUE;
	} else if (g_hash_table_lookup(dev->params, PCMK_STONITH_HOST_MAP)
	&& g_hash_table_lookup(dev->aliases, host)) {
	can = TRUE;
	}

	} else if (pcmk__str_eq(check_type, "dynamic-list", pcmk__str_casei)) {
	time_t now = time(NULL);

	if (dev->targets == NULL \|\| dev->targets_age + 60 < now) {
	crm_trace("Running '%s' to check whether %s is eligible to fence %s (%s)",
	check_type, dev->id, search->host, search->action);

	schedule_internal_command(__func__, dev, "list", NULL,
	search->per_device_timeout, search, dynamic_list_search_cb);

	/* we'll respond to this search request async in the cb */
	return;
	}

	if (string_in_list(dev->targets, alias)) {
	can = TRUE;
	}

	} else if (pcmk__str_eq(check_type, "status", pcmk__str_casei)) {
	crm_trace("Running '%s' to check whether %s is eligible to fence %s (%s)",
	check_type, dev->id, search->host, search->action);
	schedule_internal_command(__func__, dev, "status", search->host,
	search->per_device_timeout, search, status_search_cb);
	/* we'll respond to this search request async in the cb */
	return;
	} else {
	crm_err("Invalid value for " PCMK_STONITH_HOST_CHECK ": %s", check_type);
	check_type = "Invalid " PCMK_STONITH_HOST_CHECK;
	}

	if (pcmk__str_eq(host, alias, pcmk__str_casei)) {
	crm_notice("%s is%s eligible to fence (%s) %s: %s",
	dev->id, (can? "" : " not"), search->action, host,
	check_type);
	} else {
	crm_notice("%s is%s eligible to fence (%s) %s (aka. '%s'): %s",
	dev->id, (can? "" : " not"), search->action, host, alias,
	check_type);
	}

	search_report_results:
	search_devices_record_result(search, dev ? dev->id : NULL, can);
	}

	static void
	search_devices(gpointer key, gpointer value, gpointer user_data)
	{
	stonith_device_t *dev = value;
	struct device_search_s *search = user_data;

	can_fence_host_with_device(dev, search);
	}

	#define DEFAULT_QUERY_TIMEOUT 20
	static void
	get_capable_devices(const char host, const char action, int timeout, bool suicide, void *user_data,
	void (callback) (GList devices, void *user_data))
	{
	struct device_search_s *search;
	int per_device_timeout = DEFAULT_QUERY_TIMEOUT;
	int devices_needing_async_query = 0;
	char *key = NULL;
	const char *check_type = NULL;
	GHashTableIter gIter;
	stonith_device_t *device = NULL;
	guint ndevices = g_hash_table_size(device_list);

	if (ndevices == 0) {
	callback(NULL, user_data);
	return;
	}

	search = calloc(1, sizeof(struct device_search_s));
	if (!search) {
	callback(NULL, user_data);
	return;
	}

	g_hash_table_iter_init(&gIter, device_list);
	while (g_hash_table_iter_next(&gIter, (void )&key, (void )&device)) {
	check_type = target_list_type(device);
	if (pcmk__strcase_any_of(check_type, "status", "dynamic-list", NULL)) {
	devices_needing_async_query++;
	}
	}

	/* If we have devices that require an async event in order to know what
	* nodes they can fence, we have to give the events a timeout. The total
	* query timeout is divided among those events. */
	if (devices_needing_async_query) {
	per_device_timeout = timeout / devices_needing_async_query;
	if (!per_device_timeout) {
	crm_err("Fencing timeout %ds is too low; using %ds, "
	"but consider raising to at least %ds",
	timeout, DEFAULT_QUERY_TIMEOUT,
	DEFAULT_QUERY_TIMEOUT * devices_needing_async_query);
	per_device_timeout = DEFAULT_QUERY_TIMEOUT;
	} else if (per_device_timeout < DEFAULT_QUERY_TIMEOUT) {
	crm_notice("Fencing timeout %ds is low for the current "
	"configuration; consider raising to at least %ds",
	timeout, DEFAULT_QUERY_TIMEOUT * devices_needing_async_query);
	}
	}

	search->host = host ? strdup(host) : NULL;
	search->action = action ? strdup(action) : NULL;
	search->per_device_timeout = per_device_timeout;
	/* We are guaranteed this many replies. Even if a device gets
	* unregistered some how during the async search, we will get
	* the correct number of replies. */
	search->replies_needed = ndevices;
	search->allow_suicide = suicide;
	search->callback = callback;
	search->user_data = user_data;
	/* kick off the search */

	crm_debug("Searching %d device%s to see which can execute '%s' targeting %s",
	ndevices, pcmk__plural_s(ndevices),
	(search->action? search->action : "unknown action"),
	(search->host? search->host : "any node"));
	g_hash_table_foreach(device_list, search_devices, search);
	}

	struct st_query_data {
	xmlNode *reply;
	char *remote_peer;
	char *client_id;
	char *target;
	char *action;
	int call_options;
	};

	/*!
	* \internal
	* \brief Add action-specific attributes to query reply XML
	*
	* \param[in,out] xml XML to add attributes to
	* \param[in] action Fence action
	* \param[in] device Fence device
	*/
	static void
	add_action_specific_attributes(xmlNode xml, const char action,
	stonith_device_t *device)
	{
	int action_specific_timeout;
	int delay_max;
	int delay_base;

	CRM_CHECK(xml && action && device, return);

	if (is_action_required(action, device)) {
	crm_trace("Action '%s' is required using %s", action, device->id);
	crm_xml_add_int(xml, F_STONITH_DEVICE_REQUIRED, 1);
	}

	action_specific_timeout = get_action_timeout(device, action, 0);
	if (action_specific_timeout) {
	crm_trace("Action '%s' has timeout %dms using %s",
	action, action_specific_timeout, device->id);
	crm_xml_add_int(xml, F_STONITH_ACTION_TIMEOUT, action_specific_timeout);
	}

	delay_max = get_action_delay_max(device, action);
	if (delay_max > 0) {
	crm_trace("Action '%s' has maximum random delay %dms using %s",
	action, delay_max, device->id);
	crm_xml_add_int(xml, F_STONITH_DELAY_MAX, delay_max / 1000);
	}

	delay_base = get_action_delay_base(device, action);
	if (delay_base > 0) {
	crm_xml_add_int(xml, F_STONITH_DELAY_BASE, delay_base / 1000);
	}

	if ((delay_max > 0) && (delay_base == 0)) {
	crm_trace("Action '%s' has maximum random delay %dms using %s",
	action, delay_max, device->id);
	} else if ((delay_max == 0) && (delay_base > 0)) {
	crm_trace("Action '%s' has a static delay of %dms using %s",
	action, delay_base, device->id);
	} else if ((delay_max > 0) && (delay_base > 0)) {
	crm_trace("Action '%s' has a minimum delay of %dms and a randomly chosen "
	"maximum delay of %dms using %s",
	action, delay_base, delay_max, device->id);
	}
	}

	/*!
	* \internal
	* \brief Add "disallowed" attribute to query reply XML if appropriate
	*
	* \param[in,out] xml XML to add attribute to
	* \param[in] action Fence action
	* \param[in] device Fence device
	* \param[in] target Fence target
	* \param[in] allow_suicide Whether self-fencing is allowed
	*/
	static void
	add_disallowed(xmlNode xml, const char action, stonith_device_t *device,
	const char *target, gboolean allow_suicide)
	{
	if (!localhost_is_eligible(device, action, target, allow_suicide)) {
	crm_trace("Action '%s' using %s is disallowed for local host",
	action, device->id);
	crm_xml_add(xml, F_STONITH_ACTION_DISALLOWED, XML_BOOLEAN_TRUE);
	}
	}

	/*!
	* \internal
	* \brief Add child element with action-specific values to query reply XML
	*
	* \param[in,out] xml XML to add attribute to
	* \param[in] action Fence action
	* \param[in] device Fence device
	* \param[in] target Fence target
	* \param[in] allow_suicide Whether self-fencing is allowed
	*/
	static void
	add_action_reply(xmlNode xml, const char action, stonith_device_t *device,
	const char *target, gboolean allow_suicide)
	{
	xmlNode *child = create_xml_node(xml, F_STONITH_ACTION);

	crm_xml_add(child, XML_ATTR_ID, action);
	add_action_specific_attributes(child, action, device);
	add_disallowed(child, action, device, target, allow_suicide);
	}

	static void
	stonith_query_capable_device_cb(GList * devices, void *user_data)
	{
	struct st_query_data *query = user_data;
	int available_devices = 0;
	xmlNode *dev = NULL;
	xmlNode *list = NULL;
	GListPtr lpc = NULL;

	/* Pack the results into XML */
	list = create_xml_node(NULL, __func__);
	crm_xml_add(list, F_STONITH_TARGET, query->target);
	for (lpc = devices; lpc != NULL; lpc = lpc->next) {
	stonith_device_t *device = g_hash_table_lookup(device_list, lpc->data);
	const char *action = query->action;

	if (!device) {
	/* It is possible the device got unregistered while
	* determining who can fence the target */
	continue;
	}

	available_devices++;

	dev = create_xml_node(list, F_STONITH_DEVICE);
	crm_xml_add(dev, XML_ATTR_ID, device->id);
	crm_xml_add(dev, "namespace", device->namespace);
	crm_xml_add(dev, "agent", device->agent);
	crm_xml_add_int(dev, F_STONITH_DEVICE_VERIFIED, device->verified);

	/* If the originating fencer wants to reboot the node, and we have a
	* capable device that doesn't support "reboot", remap to "off" instead.
	*/
	if (!pcmk_is_set(device->flags, st_device_supports_reboot)
	&& pcmk__str_eq(query->action, "reboot", pcmk__str_casei)) {
	crm_trace("%s doesn't support reboot, using values for off instead",
	device->id);
	action = "off";
	}

	/* Add action-specific values if available */
	add_action_specific_attributes(dev, action, device);
	if (pcmk__str_eq(query->action, "reboot", pcmk__str_casei)) {
	/* A "reboot" might get remapped to "off" then "on", so after
	* sending the "reboot"-specific values in the main element, we add
	* sub-elements for "off" and "on" values.
	*
	* We short-circuited earlier if "reboot", "off" and "on" are all
	* disallowed for the local host. However if only one or two are
	* disallowed, we send back the results and mark which ones are
	* disallowed. If "reboot" is disallowed, this might cause problems
	* with older fencer versions, which won't check for it. Older
	* versions will ignore "off" and "on", so they are not a problem.
	*/
	add_disallowed(dev, action, device, query->target,
	pcmk_is_set(query->call_options, st_opt_allow_suicide));
	add_action_reply(dev, "off", device, query->target,
	pcmk_is_set(query->call_options, st_opt_allow_suicide));
	add_action_reply(dev, "on", device, query->target, FALSE);
	}

	/* A query without a target wants device parameters */
	if (query->target == NULL) {
	xmlNode *attrs = create_xml_node(dev, XML_TAG_ATTRS);

	g_hash_table_foreach(device->params, hash2field, attrs);
	}
	}

	crm_xml_add_int(list, F_STONITH_AVAILABLE_DEVICES, available_devices);
	if (query->target) {
	crm_debug("Found %d matching device%s for target '%s'",
	available_devices, pcmk__plural_s(available_devices),
	query->target);
	} else {
	crm_debug("%d device%s installed",
	available_devices, pcmk__plural_s(available_devices));
	}

	if (list != NULL) {
	crm_log_xml_trace(list, "Add query results");
	add_message_xml(query->reply, F_STONITH_CALLDATA, list);
	}
	stonith_send_reply(query->reply, query->call_options, query->remote_peer, query->client_id);

	free_xml(query->reply);
	free(query->remote_peer);
	free(query->client_id);
	free(query->target);
	free(query->action);
	free(query);
	free_xml(list);
	g_list_free_full(devices, free);
	}

	static void
	stonith_query(xmlNode * msg, const char remote_peer, const char client_id, int call_options)
	{
	struct st_query_data *query = NULL;
	const char *action = NULL;
	const char *target = NULL;
	int timeout = 0;
	xmlNode *dev = get_xpath_object("//@" F_STONITH_ACTION, msg, LOG_NEVER);

	crm_element_value_int(msg, F_STONITH_TIMEOUT, &timeout);
	if (dev) {
	const char *device = crm_element_value(dev, F_STONITH_DEVICE);

	target = crm_element_value(dev, F_STONITH_TARGET);
	action = crm_element_value(dev, F_STONITH_ACTION);
	if (device && pcmk__str_eq(device, "manual_ack", pcmk__str_casei)) {
	/* No query or reply necessary */
	return;
	}
	}

	crm_log_xml_debug(msg, "Query");
	query = calloc(1, sizeof(struct st_query_data));

	query->reply = stonith_construct_reply(msg, NULL, NULL, pcmk_ok);
	query->remote_peer = remote_peer ? strdup(remote_peer) : NULL;
	query->client_id = client_id ? strdup(client_id) : NULL;
	query->target = target ? strdup(target) : NULL;
	query->action = action ? strdup(action) : NULL;
	query->call_options = call_options;

	get_capable_devices(target, action, timeout,
	pcmk_is_set(call_options, st_opt_allow_suicide),
	query, stonith_query_capable_device_cb);
	}

	#define ST_LOG_OUTPUT_MAX 512
	static void
	log_operation(async_command_t * cmd, int rc, int pid, const char next, const char output, gboolean op_merged)
	{
	if (rc == 0) {
	next = NULL;
	}

	if (cmd->victim != NULL) {
	do_crm_log(((rc == 0)? LOG_NOTICE : LOG_ERR),
	"Operation '%s' [%d] (%scall %d from %s) targeting %s "
	"using %s returned %d (%s)%s%s",
	cmd->action, pid, (op_merged? "merged " : ""), cmd->id,
	cmd->client_name, cmd->victim,
	cmd->device, rc, pcmk_strerror(rc),
	(next? ", retrying with " : ""), (next ? next : ""));
	} else {
	do_crm_log_unlikely(((rc == 0)? LOG_DEBUG : LOG_NOTICE),
	"Operation '%s' [%d]%s using %s returned %d (%s)%s%s",
	cmd->action, pid, (op_merged? " (merged)" : ""),
	cmd->device, rc, pcmk_strerror(rc),
	(next? ", retrying with " : ""), (next ? next : ""));
	}

	if (output) {
	// Output may have multiple lines
	char *prefix = crm_strdup_printf("%s[%d]", cmd->device, pid);

	crm_log_output(rc == 0 ? LOG_DEBUG : LOG_WARNING, prefix, output);
	free(prefix);
	}
	}

	static void
	stonith_send_async_reply(async_command_t * cmd, const char *output, int rc, GPid pid, int options)
	{
	xmlNode *reply = NULL;
	gboolean bcast = FALSE;

	reply = stonith_construct_async_reply(cmd, output, NULL, rc);

	if (pcmk__str_eq(cmd->action, "metadata", pcmk__str_casei)) {
	/* Too verbose to log */
	crm_trace("Metadata query for %s", cmd->device);
	output = NULL;

	} else if (pcmk__str_any_of(cmd->action, "monitor", "list", "status", NULL)) {
	crm_trace("Never broadcast '%s' replies", cmd->action);

	} else if (!stand_alone && pcmk__str_eq(cmd->origin, cmd->victim, pcmk__str_casei) && !pcmk__str_eq(cmd->action, "on", pcmk__str_casei)) {
	crm_trace("Broadcast '%s' reply for %s", cmd->action, cmd->victim);
	crm_xml_add(reply, F_SUBTYPE, "broadcast");
	bcast = TRUE;
	}

	log_operation(cmd, rc, pid, NULL, output, (options & st_reply_opt_merged ? TRUE : FALSE));
	crm_log_xml_trace(reply, "Reply");

	if (options & st_reply_opt_merged) {
	crm_xml_add(reply, F_STONITH_MERGED, "true");
	}

	if (bcast) {
	crm_xml_add(reply, F_STONITH_OPERATION, T_STONITH_NOTIFY);
	send_cluster_message(NULL, crm_msg_stonith_ng, reply, FALSE);

	} else if (cmd->origin) {
	crm_trace("Directed reply to %s", cmd->origin);
	send_cluster_message(crm_get_peer(0, cmd->origin), crm_msg_stonith_ng, reply, FALSE);

	} else {
	crm_trace("Directed local %ssync reply to %s",
	(cmd->options & st_opt_sync_call) ? "" : "a-", cmd->client_name);
	do_local_reply(reply, cmd->client, cmd->options & st_opt_sync_call, FALSE);
	}

	if (stand_alone) {
	/* Do notification with a clean data object */
	xmlNode *notify_data = create_xml_node(NULL, T_STONITH_NOTIFY_FENCE);

	crm_xml_add_int(notify_data, F_STONITH_RC, rc);
	crm_xml_add(notify_data, F_STONITH_TARGET, cmd->victim);
	crm_xml_add(notify_data, F_STONITH_OPERATION, cmd->op);
	crm_xml_add(notify_data, F_STONITH_DELEGATE, "localhost");
	crm_xml_add(notify_data, F_STONITH_DEVICE, cmd->device);
	crm_xml_add(notify_data, F_STONITH_REMOTE_OP_ID, cmd->remote_op_id);
	crm_xml_add(notify_data, F_STONITH_ORIGIN, cmd->client);

	do_stonith_notify(0, T_STONITH_NOTIFY_FENCE, rc, notify_data);
	do_stonith_notify(0, T_STONITH_NOTIFY_HISTORY, 0, NULL);
	}

	free_xml(reply);
	}

	static void
	cancel_stonith_command(async_command_t * cmd)
	{
	stonith_device_t *device;

	CRM_CHECK(cmd != NULL, return);

	if (!cmd->device) {
	return;
	}

	device = g_hash_table_lookup(device_list, cmd->device);

	if (device) {
	crm_trace("Cancel scheduled '%s' action using %s",
	cmd->action, device->id);
	device->pending_ops = g_list_remove(device->pending_ops, cmd);
	}
	}

	static void
	st_child_done(GPid pid, int rc, const char *output, gpointer user_data)
	{
	stonith_device_t *device = NULL;
	stonith_device_t *next_device = NULL;
	async_command_t *cmd = user_data;

	GListPtr gIter = NULL;
	GListPtr gIterNext = NULL;

	CRM_CHECK(cmd != NULL, return);

	cmd->active_on = NULL;

	/* The device is ready to do something else now */
	device = g_hash_table_lookup(device_list, cmd->device);
	if (device) {
	if (!device->verified && (rc == pcmk_ok) &&
	(pcmk__strcase_any_of(cmd->action, "list", "monitor", "status", NULL))) {

	device->verified = TRUE;
	}

	mainloop_set_trigger(device->work);
	}

	crm_debug("Operation '%s' using %s returned %d (%d devices remaining)",
	cmd->action, cmd->device, rc, g_list_length(cmd->device_next));

	if (rc == 0) {
	GListPtr iter;
	/* see if there are any required devices left to execute for this op */
	for (iter = cmd->device_next; iter != NULL; iter = iter->next) {
	next_device = g_hash_table_lookup(device_list, iter->data);

	if (next_device != NULL && is_action_required(cmd->action, next_device)) {
	cmd->device_next = iter->next;
	break;
	}
	next_device = NULL;
	}

	} else if (rc != 0 && cmd->device_next && (is_action_required(cmd->action, device) == FALSE)) {
	/* if this device didn't work out, see if there are any others we can try.
	* if the failed device was 'required', we can't pick another device. */
	next_device = g_hash_table_lookup(device_list, cmd->device_next->data);
	cmd->device_next = cmd->device_next->next;
	}

	/* this operation requires more fencing, hooray! */
	if (next_device) {
	log_operation(cmd, rc, pid, next_device->id, output, FALSE);

	schedule_stonith_command(cmd, next_device);
	/* Prevent cmd from being freed */
	cmd = NULL;
	goto done;
	}

	stonith_send_async_reply(cmd, output, rc, pid, st_reply_opt_none);

	if (rc != 0) {
	goto done;
	}

	/* Check to see if any operations are scheduled to do the exact
	* same thing that just completed. If so, rather than
	* performing the same fencing operation twice, return the result
	* of this operation for all pending commands it matches. */
	for (gIter = cmd_list; gIter != NULL; gIter = gIterNext) {
	async_command_t *cmd_other = gIter->data;

	gIterNext = gIter->next;

	if (cmd == cmd_other) {
	continue;
	}

	/* A pending scheduled command matches the command that just finished if.
	* 1. The client connections are different.
	* 2. The node victim is the same.
	* 3. The fencing action is the same.
	* 4. The device scheduled to execute the action is the same.
	*/
	if (pcmk__str_eq(cmd->client, cmd_other->client, pcmk__str_casei) \|\|
	!pcmk__str_eq(cmd->victim, cmd_other->victim, pcmk__str_casei) \|\|
	!pcmk__str_eq(cmd->action, cmd_other->action, pcmk__str_casei) \|\|
	!pcmk__str_eq(cmd->device, cmd_other->device, pcmk__str_casei)) {

	continue;
	}

	/* Duplicate merging will do the right thing for either type of remapped
	* reboot. If the executing fencer remapped an unsupported reboot to
	* off, then cmd->action will be reboot and will be merged with any
	* other reboot requests. If the originating fencer remapped a
	* topology reboot to off then on, we will get here once with
	* cmd->action "off" and once with "on", and they will be merged
	* separately with similar requests.
	*/
	crm_notice("Merging fencing action '%s' targeting %s originating from "
	"client %s with identical fencing request from client %s",
	cmd_other->action, cmd_other->victim, cmd_other->client_name,
	cmd->client_name);

	cmd_list = g_list_remove_link(cmd_list, gIter);

	stonith_send_async_reply(cmd_other, output, rc, pid, st_reply_opt_merged);
	cancel_stonith_command(cmd_other);

	free_async_command(cmd_other);
	g_list_free_1(gIter);
	}

	done:
	free_async_command(cmd);
	}

	static gint
	sort_device_priority(gconstpointer a, gconstpointer b)
	{
	const stonith_device_t *dev_a = a;
	const stonith_device_t *dev_b = b;

	if (dev_a->priority > dev_b->priority) {
	return -1;
	} else if (dev_a->priority < dev_b->priority) {
	return 1;
	}
	return 0;
	}

	static void
	stonith_fence_get_devices_cb(GList * devices, void *user_data)
	{
	async_command_t *cmd = user_data;
	stonith_device_t *device = NULL;
	guint ndevices = g_list_length(devices);

	crm_info("Found %d matching device%s for target '%s'",
	ndevices, pcmk__plural_s(ndevices), cmd->victim);

	if (devices != NULL) {
	/* Order based on priority */
	devices = g_list_sort(devices, sort_device_priority);
	device = g_hash_table_lookup(device_list, devices->data);

	if (device) {
	cmd->device_list = devices;
	cmd->device_next = devices->next;
	devices = NULL; /* list owned by cmd now */
	}
	}

	/* we have a device, schedule it for fencing. */
	if (device) {
	schedule_stonith_command(cmd, device);
	/* in progress */
	return;
	}

	/* no device found! */
	stonith_send_async_reply(cmd, NULL, -ENODEV, 0, st_reply_opt_none);

	free_async_command(cmd);
	g_list_free_full(devices, free);
	}

	static int
	stonith_fence(xmlNode * msg)
	{
	const char *device_id = NULL;
	stonith_device_t *device = NULL;
	async_command_t *cmd = create_async_command(msg);
	xmlNode *dev = get_xpath_object("//@" F_STONITH_TARGET, msg, LOG_ERR);

	if (cmd == NULL) {
	return -EPROTO;
	}

	device_id = crm_element_value(dev, F_STONITH_DEVICE);
	if (device_id) {
	device = g_hash_table_lookup(device_list, device_id);
	if (device == NULL) {
	crm_err("Requested device '%s' is not available", device_id);
	return -ENODEV;
	}
	schedule_stonith_command(cmd, device);

	} else {
	const char *host = crm_element_value(dev, F_STONITH_TARGET);

	if (cmd->options & st_opt_cs_nodeid) {
	int nodeid = crm_atoi(host, NULL);
	- crm_node_t *node = crm_find_known_peer_full(nodeid, NULL, CRM_GET_PEER_ANY);
	+ crm_node_t *node = pcmk__search_known_node_cache(nodeid, NULL,
	+ CRM_GET_PEER_ANY);

	if (node) {
	host = node->uname;
	}
	}

	/* If we get to here, then self-fencing is implicitly allowed */
	get_capable_devices(host, cmd->action, cmd->default_timeout,
	TRUE, cmd, stonith_fence_get_devices_cb);
	}

	return -EINPROGRESS;
	}

	xmlNode *
	stonith_construct_reply(xmlNode * request, const char output, xmlNode data, int rc)
	{
	xmlNode *reply = NULL;

	reply = create_xml_node(NULL, T_STONITH_REPLY);

	crm_xml_add(reply, "st_origin", __func__);
	crm_xml_add(reply, F_TYPE, T_STONITH_NG);
	crm_xml_add(reply, "st_output", output);
	crm_xml_add_int(reply, F_STONITH_RC, rc);

	if (request == NULL) {
	/* Most likely, this is the result of a stonith operation that was
	* initiated before we came up. Unfortunately that means we lack enough
	* information to provide clients with a full result.
	*
	* @TODO Maybe synchronize this information at start-up?
	*/
	crm_warn("Missing request information for client notifications for "
	"operation with result %d (initiated before we came up?)", rc);

	} else {
	const char *name = NULL;
	const char *value = NULL;

	const char *names[] = {
	F_STONITH_OPERATION,
	F_STONITH_CALLID,
	F_STONITH_CLIENTID,
	F_STONITH_CLIENTNAME,
	F_STONITH_REMOTE_OP_ID,
	F_STONITH_CALLOPTS
	};

	crm_trace("Creating a result reply with%s reply output (rc=%d)",
	(data? "" : "out"), rc);
	for (int lpc = 0; lpc < DIMOF(names); lpc++) {
	name = names[lpc];
	value = crm_element_value(request, name);
	crm_xml_add(reply, name, value);
	}
	if (data != NULL) {
	add_message_xml(reply, F_STONITH_CALLDATA, data);
	}
	}
	return reply;
	}

	static xmlNode *
	stonith_construct_async_reply(async_command_t * cmd, const char output, xmlNode data, int rc)
	{
	xmlNode *reply = NULL;

	crm_trace("Creating a basic reply");
	reply = create_xml_node(NULL, T_STONITH_REPLY);

	crm_xml_add(reply, "st_origin", __func__);
	crm_xml_add(reply, F_TYPE, T_STONITH_NG);

	crm_xml_add(reply, F_STONITH_OPERATION, cmd->op);
	crm_xml_add(reply, F_STONITH_DEVICE, cmd->device);
	crm_xml_add(reply, F_STONITH_REMOTE_OP_ID, cmd->remote_op_id);
	crm_xml_add(reply, F_STONITH_CLIENTID, cmd->client);
	crm_xml_add(reply, F_STONITH_CLIENTNAME, cmd->client_name);
	crm_xml_add(reply, F_STONITH_TARGET, cmd->victim);
	crm_xml_add(reply, F_STONITH_ACTION, cmd->op);
	crm_xml_add(reply, F_STONITH_ORIGIN, cmd->origin);
	crm_xml_add_int(reply, F_STONITH_CALLID, cmd->id);
	crm_xml_add_int(reply, F_STONITH_CALLOPTS, cmd->options);

	crm_xml_add_int(reply, F_STONITH_RC, rc);

	crm_xml_add(reply, "st_output", output);

	if (data != NULL) {
	crm_info("Attaching reply output");
	add_message_xml(reply, F_STONITH_CALLDATA, data);
	}
	return reply;
	}

	bool fencing_peer_active(crm_node_t *peer)
	{
	if (peer == NULL) {
	return FALSE;
	} else if (peer->uname == NULL) {
	return FALSE;
	} else if (pcmk_is_set(peer->processes, crm_get_cluster_proc())) {
	return TRUE;
	}
	return FALSE;
	}

	/*!
	* \internal
	* \brief Determine if we need to use an alternate node to
	* fence the target. If so return that node's uname
	*
	* \retval NULL, no alternate host
	* \retval uname, uname of alternate host to use
	*/
	static const char *
	check_alternate_host(const char *target)
	{
	const char *alternate_host = NULL;

	crm_trace("Checking if we (%s) can fence %s", stonith_our_uname, target);
	if (find_topology_for_host(target) && pcmk__str_eq(target, stonith_our_uname, pcmk__str_casei)) {
	GHashTableIter gIter;
	crm_node_t *entry = NULL;

	g_hash_table_iter_init(&gIter, crm_peer_cache);
	while (g_hash_table_iter_next(&gIter, NULL, (void **)&entry)) {
	crm_trace("Checking for %s.%d != %s", entry->uname, entry->id, target);
	if (fencing_peer_active(entry)
	&& !pcmk__str_eq(entry->uname, target, pcmk__str_casei)) {
	alternate_host = entry->uname;
	break;
	}
	}
	if (alternate_host == NULL) {
	crm_err("No alternate host available to handle request "
	"for self-fencing with topology");
	g_hash_table_iter_init(&gIter, crm_peer_cache);
	while (g_hash_table_iter_next(&gIter, NULL, (void **)&entry)) {
	crm_notice("Peer[%d] %s", entry->id, entry->uname);
	}
	}
	}

	return alternate_host;
	}

	static void
	stonith_send_reply(xmlNode * reply, int call_options, const char *remote_peer,
	const char *client_id)
	{
	if (remote_peer) {
	send_cluster_message(crm_get_peer(0, remote_peer), crm_msg_stonith_ng, reply, FALSE);
	} else {
	do_local_reply(reply, client_id,
	pcmk_is_set(call_options, st_opt_sync_call),
	(remote_peer != NULL));
	}
	}

	static void
	remove_relay_op(xmlNode * request)
	{
	xmlNode *dev = get_xpath_object("//@" F_STONITH_ACTION, request, LOG_TRACE);
	const char *relay_op_id = NULL;
	const char *op_id = NULL;
	const char *client_name = NULL;
	const char *target = NULL;
	remote_fencing_op_t *relay_op = NULL;

	if (dev) {
	target = crm_element_value(dev, F_STONITH_TARGET);
	}

	relay_op_id = crm_element_value(request, F_STONITH_REMOTE_OP_ID_RELAY);
	op_id = crm_element_value(request, F_STONITH_REMOTE_OP_ID);
	client_name = crm_element_value(request, F_STONITH_CLIENTNAME);

	/* Delete RELAY operation. */
	if (relay_op_id && target && pcmk__str_eq(target, stonith_our_uname, pcmk__str_casei)) {
	relay_op = g_hash_table_lookup(stonith_remote_op_list, relay_op_id);

	if (relay_op) {
	GHashTableIter iter;
	remote_fencing_op_t *list_op = NULL;
	g_hash_table_iter_init(&iter, stonith_remote_op_list);

	/* If the operation to be deleted is registered as a duplicate, delete the registration. */
	while (g_hash_table_iter_next(&iter, NULL, (void **)&list_op)) {
	GListPtr dup_iter = NULL;
	if (list_op != relay_op) {
	for (dup_iter = list_op->duplicates; dup_iter != NULL; dup_iter = dup_iter->next) {
	remote_fencing_op_t *other = dup_iter->data;
	if (other == relay_op) {
	other->duplicates = g_list_remove(other->duplicates, relay_op);
	break;
	}
	}
	}
	}
	crm_debug("Deleting relay op %s ('%s' targeting %s for %s), "
	"replaced by op %s ('%s' targeting %s for %s)",
	relay_op->id, relay_op->action, relay_op->target,
	relay_op->client_name, op_id, relay_op->action, target,
	client_name);

	g_hash_table_remove(stonith_remote_op_list, relay_op_id);
	}
	}
	}

	static int
	handle_request(pcmk__client_t *client, uint32_t id, uint32_t flags,
	xmlNode request, const char remote_peer)
	{
	int call_options = 0;
	int rc = -EOPNOTSUPP;

	xmlNode *data = NULL;
	xmlNode *reply = NULL;

	char *output = NULL;
	const char *op = crm_element_value(request, F_STONITH_OPERATION);
	const char *client_id = crm_element_value(request, F_STONITH_CLIENTID);

	#if ENABLE_ACL
	/* IPC commands related to fencing configuration may be done only by
	* privileged users (i.e. root or hacluster) when ACLs are supported,
	* because all other users should go through the CIB to have ACLs applied.
	*
	* If no client was given, this is a peer request, which is always allowed.
	*/
	bool allowed = (client == NULL)
	\|\| pcmk_is_set(client->flags, pcmk__client_privileged);
	#else
	bool allowed = true;
	#endif

	crm_element_value_int(request, F_STONITH_CALLOPTS, &call_options);

	if (pcmk_is_set(call_options, st_opt_sync_call)) {
	CRM_ASSERT(client == NULL \|\| client->request_id == id);
	}

	if (pcmk__str_eq(op, CRM_OP_REGISTER, pcmk__str_none)) {
	xmlNode *reply = create_xml_node(NULL, "reply");

	CRM_ASSERT(client);
	crm_xml_add(reply, F_STONITH_OPERATION, CRM_OP_REGISTER);
	crm_xml_add(reply, F_STONITH_CLIENTID, client->id);
	pcmk__ipc_send_xml(client, id, reply, flags);
	client->request_id = 0;
	free_xml(reply);
	return 0;

	} else if (pcmk__str_eq(op, STONITH_OP_EXEC, pcmk__str_none)) {
	rc = stonith_device_action(request, &output);

	} else if (pcmk__str_eq(op, STONITH_OP_TIMEOUT_UPDATE, pcmk__str_none)) {
	const char *call_id = crm_element_value(request, F_STONITH_CALLID);
	const char *client_id = crm_element_value(request, F_STONITH_CLIENTID);
	int op_timeout = 0;

	crm_element_value_int(request, F_STONITH_TIMEOUT, &op_timeout);
	do_stonith_async_timeout_update(client_id, call_id, op_timeout);
	return 0;

	} else if (pcmk__str_eq(op, STONITH_OP_QUERY, pcmk__str_none)) {
	if (remote_peer) {
	create_remote_stonith_op(client_id, request, TRUE); /* Record it for the future notification */
	}

	/* Delete the DC node RELAY operation. */
	remove_relay_op(request);

	stonith_query(request, remote_peer, client_id, call_options);
	return 0;

	} else if (pcmk__str_eq(op, T_STONITH_NOTIFY, pcmk__str_none)) {
	const char *flag_name = NULL;

	CRM_ASSERT(client);
	flag_name = crm_element_value(request, F_STONITH_NOTIFY_ACTIVATE);
	if (flag_name) {
	crm_debug("Enabling %s callbacks for client %s",
	flag_name, pcmk__client_name(client));
	pcmk__set_client_flags(client, get_stonith_flag(flag_name));
	}

	flag_name = crm_element_value(request, F_STONITH_NOTIFY_DEACTIVATE);
	if (flag_name) {
	crm_debug("Disabling %s callbacks for client %s",
	flag_name, pcmk__client_name(client));
	pcmk__clear_client_flags(client, get_stonith_flag(flag_name));
	}

	pcmk__ipc_send_ack(client, id, flags, "ack", CRM_EX_OK);
	return 0;

	} else if (pcmk__str_eq(op, STONITH_OP_RELAY, pcmk__str_none)) {
	xmlNode *dev = get_xpath_object("//@" F_STONITH_TARGET, request, LOG_TRACE);

	crm_notice("Received forwarded fencing request from "
	"%s%s to fence (%s) peer %s",
	((client == NULL)? "peer" : "client"),
	((client == NULL)? remote_peer : pcmk__client_name(client)),
	crm_element_value(dev, F_STONITH_ACTION),
	crm_element_value(dev, F_STONITH_TARGET));

	if (initiate_remote_stonith_op(NULL, request, FALSE) != NULL) {
	rc = -EINPROGRESS;
	}

	} else if (pcmk__str_eq(op, STONITH_OP_FENCE, pcmk__str_none)) {

	if (remote_peer \|\| stand_alone) {
	rc = stonith_fence(request);

	} else if (call_options & st_opt_manual_ack) {
	remote_fencing_op_t *rop = NULL;
	xmlNode *dev = get_xpath_object("//@" F_STONITH_TARGET, request, LOG_TRACE);
	const char *target = crm_element_value(dev, F_STONITH_TARGET);

	crm_notice("Received manual confirmation that %s is fenced", target);
	rop = initiate_remote_stonith_op(client, request, TRUE);
	rc = stonith_manual_ack(request, rop);

	} else {
	const char *alternate_host = NULL;
	xmlNode *dev = get_xpath_object("//@" F_STONITH_TARGET, request, LOG_TRACE);
	const char *target = crm_element_value(dev, F_STONITH_TARGET);
	const char *action = crm_element_value(dev, F_STONITH_ACTION);
	const char *device = crm_element_value(dev, F_STONITH_DEVICE);

	if (client) {
	int tolerance = 0;

	crm_notice("Client %s wants to fence (%s) %s using %s",
	pcmk__client_name(client), action,
	target, (device? device : "any device"));

	crm_element_value_int(dev, F_STONITH_TOLERANCE, &tolerance);

	if (stonith_check_fence_tolerance(tolerance, target, action)) {
	rc = 0;
	goto done;
	}

	} else {
	crm_notice("Peer %s wants to fence (%s) '%s' with device '%s'",
	remote_peer, action, target, device ? device : "(any)");
	}

	alternate_host = check_alternate_host(target);

	if (alternate_host && client) {
	const char *client_id = NULL;
	remote_fencing_op_t *op = NULL;

	crm_notice("Forwarding self-fencing request to peer %s"
	"due to topology", alternate_host);

	if (client->id) {
	client_id = client->id;
	} else {
	client_id = crm_element_value(request, F_STONITH_CLIENTID);
	}

	/* Create an operation for RELAY and send the ID in the RELAY message. */
	/* When a QUERY response is received, delete the RELAY operation to avoid the existence of duplicate operations. */
	op = create_remote_stonith_op(client_id, request, FALSE);

	crm_xml_add(request, F_STONITH_OPERATION, STONITH_OP_RELAY);
	crm_xml_add(request, F_STONITH_CLIENTID, client->id);
	crm_xml_add(request, F_STONITH_REMOTE_OP_ID, op->id);
	send_cluster_message(crm_get_peer(0, alternate_host), crm_msg_stonith_ng, request,
	FALSE);
	rc = -EINPROGRESS;

	} else if (initiate_remote_stonith_op(client, request, FALSE) != NULL) {
	rc = -EINPROGRESS;
	}
	}

	} else if (pcmk__str_eq(op, STONITH_OP_FENCE_HISTORY, pcmk__str_none)) {
	rc = stonith_fence_history(request, &data, remote_peer, call_options);
	if (call_options & st_opt_discard_reply) {
	/* we don't expect answers to the broadcast
	* we might have sent out
	*/
	free_xml(data);
	return pcmk_ok;
	}

	} else if (pcmk__str_eq(op, STONITH_OP_DEVICE_ADD, pcmk__str_none)) {
	const char *device_id = NULL;

	if (allowed) {
	rc = stonith_device_register(request, &device_id, FALSE);
	} else {
	rc = -EACCES;
	}
	do_stonith_notify_device(call_options, op, rc, device_id);

	} else if (pcmk__str_eq(op, STONITH_OP_DEVICE_DEL, pcmk__str_none)) {
	xmlNode *dev = get_xpath_object("//" F_STONITH_DEVICE, request, LOG_ERR);
	const char *device_id = crm_element_value(dev, XML_ATTR_ID);

	if (allowed) {
	rc = stonith_device_remove(device_id, FALSE);
	} else {
	rc = -EACCES;
	}
	do_stonith_notify_device(call_options, op, rc, device_id);

	} else if (pcmk__str_eq(op, STONITH_OP_LEVEL_ADD, pcmk__str_none)) {
	char *device_id = NULL;

	if (allowed) {
	rc = stonith_level_register(request, &device_id);
	} else {
	rc = -EACCES;
	}
	do_stonith_notify_level(call_options, op, rc, device_id);
	free(device_id);

	} else if (pcmk__str_eq(op, STONITH_OP_LEVEL_DEL, pcmk__str_none)) {
	char *device_id = NULL;

	if (allowed) {
	rc = stonith_level_remove(request, &device_id);
	} else {
	rc = -EACCES;
	}
	do_stonith_notify_level(call_options, op, rc, device_id);

	} else if(pcmk__str_eq(op, CRM_OP_RM_NODE_CACHE, pcmk__str_casei)) {
	int node_id = 0;
	const char *name = NULL;

	crm_element_value_int(request, XML_ATTR_ID, &node_id);
	name = crm_element_value(request, XML_ATTR_UNAME);
	reap_crm_member(node_id, name);

	return pcmk_ok;

	} else {
	crm_err("Unknown IPC request %s from %s %s", op,
	((client == NULL)? "peer" : "client"),
	((client == NULL)? remote_peer : pcmk__client_name(client)));
	}

	done:

	if (rc == -EACCES) {
	crm_warn("Rejecting IPC request '%s' from unprivileged client %s",
	crm_str(op), pcmk__client_name(client));
	}

	/* Always reply unless the request is in process still.
	* If in progress, a reply will happen async after the request
	* processing is finished */
	if (rc != -EINPROGRESS) {
	crm_trace("Reply handling: %p %u %u %d %d %s", client, client?client->request_id:0,
	id, pcmk_is_set(call_options, st_opt_sync_call), call_options,
	crm_element_value(request, F_STONITH_CALLOPTS));

	if (pcmk_is_set(call_options, st_opt_sync_call)) {
	CRM_ASSERT(client == NULL \|\| client->request_id == id);
	}
	reply = stonith_construct_reply(request, output, data, rc);
	stonith_send_reply(reply, call_options, remote_peer, client_id);
	}

	free(output);
	free_xml(data);
	free_xml(reply);

	return rc;
	}

	static void
	handle_reply(pcmk__client_t client, xmlNode request, const char *remote_peer)
	{
	const char *op = crm_element_value(request, F_STONITH_OPERATION);

	if (pcmk__str_eq(op, STONITH_OP_QUERY, pcmk__str_none)) {
	process_remote_stonith_query(request);
	} else if (pcmk__str_eq(op, T_STONITH_NOTIFY, pcmk__str_none)) {
	process_remote_stonith_exec(request);
	} else if (pcmk__str_eq(op, STONITH_OP_FENCE, pcmk__str_none)) {
	/* Reply to a complex fencing op */
	process_remote_stonith_exec(request);
	} else {
	crm_err("Unknown %s reply from %s %s", op,
	((client == NULL)? "peer" : "client"),
	((client == NULL)? remote_peer : pcmk__client_name(client)));
	crm_log_xml_warn(request, "UnknownOp");
	}
	}

	void
	stonith_command(pcmk__client_t *client, uint32_t id, uint32_t flags,
	xmlNode request, const char remote_peer)
	{
	int call_options = 0;
	int rc = 0;
	gboolean is_reply = FALSE;

	/* Copy op for reporting. The original might get freed by handle_reply()
	* before we use it in crm_debug():
	* handle_reply()
	* \|- process_remote_stonith_exec()
	* \|-- remote_op_done()
	* \|--- handle_local_reply_and_notify()
	* \|---- crm_xml_add(...F_STONITH_OPERATION...)
	* \|--- free_xml(op->request)
	*/
	char *op = crm_element_value_copy(request, F_STONITH_OPERATION);

	if (get_xpath_object("//" T_STONITH_REPLY, request, LOG_NEVER)) {
	is_reply = TRUE;
	}

	crm_element_value_int(request, F_STONITH_CALLOPTS, &call_options);
	crm_debug("Processing %s%s %u from %s %s with call options 0x%08x",
	op, (is_reply? " reply" : ""), id,
	((client == NULL)? "peer" : "client"),
	((client == NULL)? remote_peer : pcmk__client_name(client)),
	call_options);

	if (pcmk_is_set(call_options, st_opt_sync_call)) {
	CRM_ASSERT(client == NULL \|\| client->request_id == id);
	}

	if (is_reply) {
	handle_reply(client, request, remote_peer);
	} else {
	rc = handle_request(client, id, flags, request, remote_peer);
	}

	crm_debug("Processed %s%s from %s %s: %s (rc=%d)",
	op, (is_reply? " reply" : ""),
	((client == NULL)? "peer" : "client"),
	((client == NULL)? remote_peer : pcmk__client_name(client)),
	((rc > 0)? "" : pcmk_strerror(rc)), rc);
	free(op);
	}
	diff --git a/daemons/fenced/fenced_history.c b/daemons/fenced/fenced_history.c
	index c052a2bbb7..1524fee5dc 100644
	--- a/daemons/fenced/fenced_history.c
	+++ b/daemons/fenced/fenced_history.c
	@@ -1,491 +1,492 @@
	/*
	* Copyright 2009-2020 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdio.h>
	#include <unistd.h>
	#include <stdlib.h>

	#include <crm/crm.h>
	#include <crm/msg_xml.h>
	#include <crm/common/ipc.h>
	#include <crm/common/ipc_internal.h>
	#include <crm/cluster/internal.h>

	#include <crm/stonith-ng.h>
	#include <crm/fencing/internal.h>
	#include <crm/common/xml.h>
	#include <crm/common/xml_internal.h>

	#include <pacemaker-fenced.h>

	#define MAX_STONITH_HISTORY 500

	/*!
	* \internal
	* \brief Send a broadcast to all nodes to trigger cleanup or
	* history synchronisation
	*
	* \param[in] history Optional history to be attached
	* \param[in] callopts We control cleanup via a flag in the callopts
	* \param[in] target Cleanup can be limited to certain fence-targets
	*/
	static void
	stonith_send_broadcast_history(xmlNode *history,
	int callopts,
	const char *target)
	{
	xmlNode *bcast = create_xml_node(NULL, "stonith_command");
	xmlNode *data = create_xml_node(NULL, __func__);

	if (target) {
	crm_xml_add(data, F_STONITH_TARGET, target);
	}
	crm_xml_add(bcast, F_TYPE, T_STONITH_NG);
	crm_xml_add(bcast, F_SUBTYPE, "broadcast");
	crm_xml_add(bcast, F_STONITH_OPERATION, STONITH_OP_FENCE_HISTORY);
	crm_xml_add_int(bcast, F_STONITH_CALLOPTS, callopts);
	if (history) {
	add_node_copy(data, history);
	}
	add_message_xml(bcast, F_STONITH_CALLDATA, data);
	send_cluster_message(NULL, crm_msg_stonith_ng, bcast, FALSE);

	free_xml(data);
	free_xml(bcast);
	}

	static gboolean
	stonith_remove_history_entry (gpointer key,
	gpointer value,
	gpointer user_data)
	{
	remote_fencing_op_t *op = value;
	const char target = (const char ) user_data;

	if ((op->state == st_failed) \|\| (op->state == st_done)) {
	if ((target) && (strcmp(op->target, target) != 0)) {
	return FALSE;
	}
	return TRUE;
	}

	return FALSE; /* don't clean pending operations */
	}

	/*!
	* \internal
	* \brief Send out a cleanup broadcast or do a local history-cleanup
	*
	* \param[in] target Cleanup can be limited to certain fence-targets
	* \param[in] broadcast Send out a cleanup broadcast
	*/
	static void
	stonith_fence_history_cleanup(const char *target,
	gboolean broadcast)
	{
	if (broadcast) {
	stonith_send_broadcast_history(NULL,
	st_opt_cleanup \| st_opt_discard_reply,
	target);
	/* we'll do the local clean when we receive back our own broadcast */
	} else if (stonith_remote_op_list) {
	g_hash_table_foreach_remove(stonith_remote_op_list,
	stonith_remove_history_entry,
	(gpointer) target);
	do_stonith_notify(0, T_STONITH_NOTIFY_HISTORY, 0, NULL);
	}
	}

	/* keeping the length of fence-history within bounds
	* =================================================
	*
	* If things are really running wild a lot of fencing-attempts
	* might fill up the hash-map, eventually using up a lot
	* of memory and creating huge history-sync messages.
	* Before the history being synced across nodes at least
	* the reboot of a cluster-node helped keeping the
	* history within bounds even though not in a reliable
	* manner.
	*
	* stonith_remote_op_list isn't sorted for time-stamps
	* thus it would be kind of expensive to delete e.g.
	* the oldest entry if it would grow past MAX_STONITH_HISTORY
	* entries.
	* It is more efficient to purge MAX_STONITH_HISTORY/2
	* entries whenever the list grows beyond MAX_STONITH_HISTORY.
	* (sort for age + purge the MAX_STONITH_HISTORY/2 oldest)
	* That done on a per-node-base might raise the
	* probability of large syncs to occur.
	* Things like introducing a broadcast to purge
	* MAX_STONITH_HISTORY/2 entries or not sync above a certain
	* threshold coming to mind ...
	* Simplest thing though is to purge the full history
	* throughout the cluster once MAX_STONITH_HISTORY is reached.
	* On the other hand this leads to purging the history in
	* situations where it would be handy to have it probably.
	*/


	static int
	op_time_sort(const void a_voidp, const void b_voidp)
	{
	const remote_fencing_op_t a = (const remote_fencing_op_t ) a_voidp;
	const remote_fencing_op_t b = (const remote_fencing_op_t ) b_voidp;
	gboolean a_pending = ((a)->state != st_failed) && ((a)->state != st_done);
	gboolean b_pending = ((b)->state != st_failed) && ((b)->state != st_done);

	if (a_pending && b_pending) {
	return 0;
	} else if (a_pending) {
	return -1;
	} else if (b_pending) {
	return 1;
	} else if ((b)->completed == (a)->completed) {
	return 0;
	} else if ((b)->completed > (a)->completed) {
	return 1;
	}

	return -1;
	}


	/*!
	* \internal
	* \brief Do a local history-trim to MAX_STONITH_HISTORY / 2 entries
	* once over MAX_STONITH_HISTORY
	*/
	void
	stonith_fence_history_trim(void)
	{
	guint num_ops;

	if (!stonith_remote_op_list) {
	return;
	}
	num_ops = g_hash_table_size(stonith_remote_op_list);
	if (num_ops > MAX_STONITH_HISTORY) {
	remote_fencing_op_t *ops[num_ops];
	remote_fencing_op_t *op = NULL;
	GHashTableIter iter;
	int i;

	crm_trace("Fencing History growing beyond limit of %d so purge "
	"half of failed/successful attempts", MAX_STONITH_HISTORY);

	/* write all ops into an array */
	i = 0;
	g_hash_table_iter_init(&iter, stonith_remote_op_list);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&op)) {
	ops[i++] = op;
	}
	/* run quicksort over the array so that we get pending ops
	* first and then sorted most recent to oldest
	*/
	qsort(ops, num_ops, sizeof(remote_fencing_op_t *), op_time_sort);
	/* purgest oldest half of the history entries */
	for (i = MAX_STONITH_HISTORY / 2; i < num_ops; i++) {
	/* keep pending ops even if they shouldn't fill more than
	* half of our buffer
	*/
	if ((ops[i]->state == st_failed) \|\| (ops[i]->state == st_done)) {
	g_hash_table_remove(stonith_remote_op_list, ops[i]->id);
	}
	}
	/* we've just purged valid data from the list so there is no need
	* to create a notification - if displayed it can stay
	*/
	}
	}

	/*!
	* \internal
	* \brief Convert xml fence-history to a hash-table like stonith_remote_op_list
	*
	* \param[in] history Fence-history in xml
	*
	* \return Fence-history as hash-table
	*/
	static GHashTable *
	stonith_xml_history_to_list(xmlNode *history)
	{
	xmlNode *xml_op = NULL;
	GHashTable *rv = NULL;

	init_stonith_remote_op_hash_table(&rv);

	CRM_LOG_ASSERT(rv != NULL);

	for (xml_op = pcmk__xml_first_child(history); xml_op != NULL;
	xml_op = pcmk__xml_next(xml_op)) {
	remote_fencing_op_t *op = NULL;
	char *id = crm_element_value_copy(xml_op, F_STONITH_REMOTE_OP_ID);
	int state;
	long long completed;

	if (!id) {
	crm_warn("Malformed fencing history received from peer");
	continue;
	}

	crm_trace("Attaching op %s to hashtable", id);

	op = calloc(1, sizeof(remote_fencing_op_t));

	op->id = id;
	op->target = crm_element_value_copy(xml_op, F_STONITH_TARGET);
	op->action = crm_element_value_copy(xml_op, F_STONITH_ACTION);
	op->originator = crm_element_value_copy(xml_op, F_STONITH_ORIGIN);
	op->delegate = crm_element_value_copy(xml_op, F_STONITH_DELEGATE);
	op->client_name = crm_element_value_copy(xml_op, F_STONITH_CLIENTNAME);
	crm_element_value_ll(xml_op, F_STONITH_DATE, &completed);
	op->completed = (time_t) completed;
	crm_element_value_int(xml_op, F_STONITH_STATE, &state);
	op->state = (enum op_state) state;

	g_hash_table_replace(rv, id, op);
	CRM_LOG_ASSERT(g_hash_table_lookup(rv, id) != NULL);
	}

	return rv;
	}

	/*!
	* \internal
	* \brief Craft xml difference between local fence-history and a history
	* coming from remote
	*
	* \param[in] remote_history Fence-history as hash-table (may be NULL)
	* \param[in] add_id If crafting the answer for an API
	* history-request there is no need for the id
	* \param[in] target Optionally limit to certain fence-target
	*
	* \return The fence-history as xml
	*/
	static xmlNode *
	stonith_local_history_diff(GHashTable *remote_history,
	gboolean add_id,
	const char *target)
	{
	xmlNode *history = NULL;
	int cnt = 0;

	if (stonith_remote_op_list) {
	GHashTableIter iter;
	remote_fencing_op_t *op = NULL;

	history = create_xml_node(NULL, F_STONITH_HISTORY_LIST);

	g_hash_table_iter_init(&iter, stonith_remote_op_list);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&op)) {
	xmlNode *entry = NULL;

	if (remote_history &&
	g_hash_table_lookup(remote_history, op->id)) {
	continue; /* skip entries broadcasted already */
	}

	if (!pcmk__str_eq(target, op->target, pcmk__str_null_matches)) {
	continue;
	}

	cnt++;
	crm_trace("Attaching op %s", op->id);
	entry = create_xml_node(history, STONITH_OP_EXEC);
	if (add_id) {
	crm_xml_add(entry, F_STONITH_REMOTE_OP_ID, op->id);
	}
	crm_xml_add(entry, F_STONITH_TARGET, op->target);
	crm_xml_add(entry, F_STONITH_ACTION, op->action);
	crm_xml_add(entry, F_STONITH_ORIGIN, op->originator);
	crm_xml_add(entry, F_STONITH_DELEGATE, op->delegate);
	crm_xml_add(entry, F_STONITH_CLIENTNAME, op->client_name);
	crm_xml_add_ll(entry, F_STONITH_DATE, op->completed);
	crm_xml_add_int(entry, F_STONITH_STATE, op->state);
	}
	}

	if (cnt == 0) {
	free_xml(history);
	return NULL;
	} else {
	return history;
	}
	}

	/*!
	* \internal
	* \brief Merge fence-history coming from remote into local history
	*
	* \param[in] history Hash-table holding remote history to be merged in
	*/
	static void
	stonith_merge_in_history_list(GHashTable *history)
	{
	GHashTableIter iter;
	remote_fencing_op_t *op = NULL;
	gboolean updated = FALSE;

	if (!history) {
	return;
	}

	init_stonith_remote_op_hash_table(&stonith_remote_op_list);

	g_hash_table_iter_init(&iter, history);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&op)) {
	remote_fencing_op_t *stored_op =
	g_hash_table_lookup(stonith_remote_op_list, op->id);

	if (stored_op) {
	continue; // Skip existent (@TODO state-merging might be desirable)
	}

	updated = TRUE;
	g_hash_table_iter_steal(&iter);

	if ((op->state != st_failed) &&
	(op->state != st_done) &&
	pcmk__str_eq(op->originator, stonith_our_uname, pcmk__str_casei)) {
	crm_warn("Failing pending operation %.8s originated by us but "
	"known only from peer history", op->id);
	op->state = st_failed;
	op->completed = time(NULL);
	/* use -EHOSTUNREACH to not introduce a new return-code that might
	trigger unexpected results at other places and to prevent
	remote_op_done from setting the delegate if not present
	*/
	stonith_bcast_result_to_peers(op, -EHOSTUNREACH, FALSE);
	}

	g_hash_table_insert(stonith_remote_op_list, op->id, op);
	/* we could trim the history here but if we bail
	* out after trim we might miss more recent entries
	* of those that might still be in the list
	* if we don't bail out trimming once is more
	* efficient and memory overhead is minimal as
	* we are just moving pointers from one hash to
	* another
	*/
	}
	stonith_fence_history_trim();
	if (updated) {
	do_stonith_notify(0, T_STONITH_NOTIFY_HISTORY, 0, NULL);
	}
	g_hash_table_destroy(history); /* remove what is left */
	}

	/*!
	* \internal
	* \brief Handle fence-history messages (either from API or coming in as
	* broadcasts
	*
	* \param[in] msg Request message
	* \param[in] output In case of a request from the API used to craft
	* a reply from
	* \param[in] remote_peer
	* \param[in] options call-options from the request
	*
	* \return always success as there is actully nothing that can go really wrong
	*/
	int
	stonith_fence_history(xmlNode msg, xmlNode *output,
	const char *remote_peer, int options)
	{
	int rc = 0;
	const char *target = NULL;
	xmlNode *dev = get_xpath_object("//@" F_STONITH_TARGET, msg, LOG_NEVER);
	xmlNode *out_history = NULL;

	if (dev) {
	target = crm_element_value(dev, F_STONITH_TARGET);
	if (target && (options & st_opt_cs_nodeid)) {
	int nodeid = crm_atoi(target, NULL);
	- crm_node_t *node = crm_find_known_peer_full(nodeid, NULL, CRM_GET_PEER_ANY);
	+ crm_node_t *node = pcmk__search_known_node_cache(nodeid, NULL,
	+ CRM_GET_PEER_ANY);

	if (node) {
	target = node->uname;
	}
	}
	}

	if (options & st_opt_cleanup) {
	crm_trace("Cleaning up operations on %s in %p", target,
	stonith_remote_op_list);

	stonith_fence_history_cleanup(target,
	crm_element_value(msg, F_STONITH_CALLID) != NULL);
	} else if (options & st_opt_broadcast) {
	/* there is no clear sign atm for when a history sync
	is done so send a notification for anything
	that smells like history-sync
	*/
	do_stonith_notify(0, T_STONITH_NOTIFY_HISTORY_SYNCED, 0, NULL);
	if (crm_element_value(msg, F_STONITH_CALLID)) {
	/* this is coming from the stonith-API
	*
	* craft a broadcast with node's history
	* so that every node can merge and broadcast
	* what it has on top
	*/
	out_history = stonith_local_history_diff(NULL, TRUE, NULL);
	crm_trace("Broadcasting history to peers");
	stonith_send_broadcast_history(out_history,
	st_opt_broadcast \| st_opt_discard_reply,
	NULL);
	} else if (remote_peer &&
	!pcmk__str_eq(remote_peer, stonith_our_uname, pcmk__str_casei)) {
	xmlNode *history = get_xpath_object("//" F_STONITH_HISTORY_LIST,
	msg, LOG_NEVER);
	GHashTable *received_history =
	history?stonith_xml_history_to_list(history):NULL;

	/* either a broadcast created directly upon stonith-API request
	* or a diff as response to such a thing
	*
	* in both cases it may have a history or not
	* if we have differential data
	* merge in what we've received and stop
	* otherwise broadcast what we have on top
	* marking as differential and merge in afterwards
	*/
	if (!history \|\|
	!crm_is_true(crm_element_value(history,
	F_STONITH_DIFFERENTIAL))) {
	out_history =
	stonith_local_history_diff(received_history, TRUE, NULL);
	if (out_history) {
	crm_trace("Broadcasting history-diff to peers");
	crm_xml_add(out_history, F_STONITH_DIFFERENTIAL,
	XML_BOOLEAN_TRUE);
	stonith_send_broadcast_history(out_history,
	st_opt_broadcast \| st_opt_discard_reply,
	NULL);
	} else {
	crm_trace("History-diff is empty - skip broadcast");
	}
	}
	stonith_merge_in_history_list(received_history);
	} else {
	crm_trace("Skipping history-query-broadcast (%s%s)"
	" we sent ourselves",
	remote_peer?"remote-peer=":"local-ipc",
	remote_peer?remote_peer:"");
	}
	} else {
	/* plain history request */
	crm_trace("Looking for operations on %s in %p", target,
	stonith_remote_op_list);
	*output = stonith_local_history_diff(NULL, FALSE, target);
	}
	free_xml(out_history);
	return rc;
	}
	diff --git a/daemons/fenced/fenced_remote.c b/daemons/fenced/fenced_remote.c
	index 7916fd2dd3..7f9e617177 100644
	--- a/daemons/fenced/fenced_remote.c
	+++ b/daemons/fenced/fenced_remote.c
	@@ -1,2173 +1,2174 @@
	/*
	* Copyright 2009-2020 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <sys/param.h>
	#include <stdio.h>
	#include <sys/types.h>
	#include <sys/wait.h>
	#include <sys/stat.h>
	#include <unistd.h>
	#include <sys/utsname.h>

	#include <stdlib.h>
	#include <errno.h>
	#include <fcntl.h>
	#include <ctype.h>
	#include <regex.h>

	#include <crm/crm.h>
	#include <crm/msg_xml.h>
	#include <crm/common/ipc.h>
	#include <crm/common/ipc_internal.h>
	#include <crm/cluster/internal.h>

	#include <crm/stonith-ng.h>
	#include <crm/fencing/internal.h>
	#include <crm/common/xml.h>
	#include <crm/common/xml_internal.h>

	#include <crm/common/util.h>
	#include <pacemaker-fenced.h>

	#define TIMEOUT_MULTIPLY_FACTOR 1.2

	/* When one fencer queries its peers for devices able to handle a fencing
	* request, each peer will reply with a list of such devices available to it.
	* Each reply will be parsed into a st_query_result_t, with each device's
	* information kept in a device_properties_t.
	*/

	typedef struct device_properties_s {
	/* Whether access to this device has been verified */
	gboolean verified;

	/* The remaining members are indexed by the operation's "phase" */

	/* Whether this device has been executed in each phase */
	gboolean executed[st_phase_max];
	/* Whether this device is disallowed from executing in each phase */
	gboolean disallowed[st_phase_max];
	/* Action-specific timeout for each phase */
	int custom_action_timeout[st_phase_max];
	/* Action-specific maximum random delay for each phase */
	int delay_max[st_phase_max];
	/* Action-specific base delay for each phase */
	int delay_base[st_phase_max];
	} device_properties_t;

	typedef struct st_query_result_s {
	/* Name of peer that sent this result */
	char *host;
	/* Only try peers for non-topology based operations once */
	gboolean tried;
	/* Number of entries in the devices table */
	int ndevices;
	/* Devices available to this host that are capable of fencing the target */
	GHashTable *devices;
	} st_query_result_t;

	GHashTable *stonith_remote_op_list = NULL;

	void call_remote_stonith(remote_fencing_op_t * op, st_query_result_t * peer, int rc);
	static void remote_op_done(remote_fencing_op_t * op, xmlNode * data, int rc, int dup);
	extern xmlNode stonith_create_op(int call_id, const char token, const char op, xmlNode data,
	int call_options);

	static void report_timeout_period(remote_fencing_op_t * op, int op_timeout);
	static int get_op_total_timeout(const remote_fencing_op_t *op,
	const st_query_result_t *chosen_peer);

	static gint
	sort_strings(gconstpointer a, gconstpointer b)
	{
	return strcmp(a, b);
	}

	static void
	free_remote_query(gpointer data)
	{
	if (data) {
	st_query_result_t *query = data;

	crm_trace("Free'ing query result from %s", query->host);
	g_hash_table_destroy(query->devices);
	free(query->host);
	free(query);
	}
	}

	void
	free_stonith_remote_op_list()
	{
	if (stonith_remote_op_list != NULL) {
	g_hash_table_destroy(stonith_remote_op_list);
	stonith_remote_op_list = NULL;
	}
	}

	struct peer_count_data {
	const remote_fencing_op_t *op;
	gboolean verified_only;
	int count;
	};

	/*!
	* \internal
	* \brief Increment a counter if a device has not been executed yet
	*
	* \param[in] key Device ID (ignored)
	* \param[in] value Device properties
	* \param[in] user_data Peer count data
	*/
	static void
	count_peer_device(gpointer key, gpointer value, gpointer user_data)
	{
	device_properties_t props = (device_properties_t)value;
	struct peer_count_data *data = user_data;

	if (!props->executed[data->op->phase]
	&& (!data->verified_only \|\| props->verified)) {
	++(data->count);
	}
	}

	/*!
	* \internal
	* \brief Check the number of available devices in a peer's query results
	*
	* \param[in] op Operation that results are for
	* \param[in] peer Peer to count
	* \param[in] verified_only Whether to count only verified devices
	*
	* \return Number of devices available to peer that were not already executed
	*/
	static int
	count_peer_devices(const remote_fencing_op_t op, const st_query_result_t peer,
	gboolean verified_only)
	{
	struct peer_count_data data;

	data.op = op;
	data.verified_only = verified_only;
	data.count = 0;
	if (peer) {
	g_hash_table_foreach(peer->devices, count_peer_device, &data);
	}
	return data.count;
	}

	/*!
	* \internal
	* \brief Search for a device in a query result
	*
	* \param[in] op Operation that result is for
	* \param[in] peer Query result for a peer
	* \param[in] device Device ID to search for
	*
	* \return Device properties if found, NULL otherwise
	*/
	static device_properties_t *
	find_peer_device(const remote_fencing_op_t op, const st_query_result_t peer,
	const char *device)
	{
	device_properties_t *props = g_hash_table_lookup(peer->devices, device);

	return (props && !props->executed[op->phase]
	&& !props->disallowed[op->phase])? props : NULL;
	}

	/*!
	* \internal
	* \brief Find a device in a peer's device list and mark it as executed
	*
	* \param[in] op Operation that peer result is for
	* \param[in,out] peer Peer with results to search
	* \param[in] device ID of device to mark as done
	* \param[in] verified_devices_only Only consider verified devices
	*
	* \return TRUE if device was found and marked, FALSE otherwise
	*/
	static gboolean
	grab_peer_device(const remote_fencing_op_t op, st_query_result_t peer,
	const char *device, gboolean verified_devices_only)
	{
	device_properties_t *props = find_peer_device(op, peer, device);

	if ((props == NULL) \|\| (verified_devices_only && !props->verified)) {
	return FALSE;
	}

	crm_trace("Removing %s from %s (%d remaining)",
	device, peer->host, count_peer_devices(op, peer, FALSE));
	props->executed[op->phase] = TRUE;
	return TRUE;
	}

	static void
	clear_remote_op_timers(remote_fencing_op_t * op)
	{
	if (op->query_timer) {
	g_source_remove(op->query_timer);
	op->query_timer = 0;
	}
	if (op->op_timer_total) {
	g_source_remove(op->op_timer_total);
	op->op_timer_total = 0;
	}
	if (op->op_timer_one) {
	g_source_remove(op->op_timer_one);
	op->op_timer_one = 0;
	}
	}

	static void
	free_remote_op(gpointer data)
	{
	remote_fencing_op_t *op = data;

	crm_log_xml_debug(op->request, "Destroying");

	clear_remote_op_timers(op);

	free(op->id);
	free(op->action);
	free(op->delegate);
	free(op->target);
	free(op->client_id);
	free(op->client_name);
	free(op->originator);

	if (op->query_results) {
	g_list_free_full(op->query_results, free_remote_query);
	}
	if (op->request) {
	free_xml(op->request);
	op->request = NULL;
	}
	if (op->devices_list) {
	g_list_free_full(op->devices_list, free);
	op->devices_list = NULL;
	}
	g_list_free_full(op->automatic_list, free);
	g_list_free(op->duplicates);
	free(op);
	}

	void
	init_stonith_remote_op_hash_table(GHashTable **table)
	{
	if (*table == NULL) {
	*table = g_hash_table_new_full(crm_str_hash, g_str_equal, NULL, free_remote_op);
	}
	}

	/*!
	* \internal
	* \brief Return an operation's originally requested action (before any remap)
	*
	* \param[in] op Operation to check
	*
	* \return Operation's original action
	*/
	static const char *
	op_requested_action(const remote_fencing_op_t *op)
	{
	return ((op->phase > st_phase_requested)? "reboot" : op->action);
	}

	/*!
	* \internal
	* \brief Remap a "reboot" operation to the "off" phase
	*
	* \param[in,out] op Operation to remap
	*/
	static void
	op_phase_off(remote_fencing_op_t *op)
	{
	crm_info("Remapping multiple-device reboot targeting %s to 'off' "
	CRM_XS " id=%.8s", op->target, op->id);
	op->phase = st_phase_off;

	/* Happily, "off" and "on" are shorter than "reboot", so we can reuse the
	* memory allocation at each phase.
	*/
	strcpy(op->action, "off");
	}

	/*!
	* \internal
	* \brief Advance a remapped reboot operation to the "on" phase
	*
	* \param[in,out] op Operation to remap
	*/
	static void
	op_phase_on(remote_fencing_op_t *op)
	{
	GListPtr iter = NULL;

	crm_info("Remapped 'off' targeting %s complete, "
	"remapping to 'on' for %s " CRM_XS " id=%.8s",
	op->target, op->client_name, op->id);
	op->phase = st_phase_on;
	strcpy(op->action, "on");

	/* Skip devices with automatic unfencing, because the cluster will handle it
	* when the node rejoins.
	*/
	for (iter = op->automatic_list; iter != NULL; iter = iter->next) {
	GListPtr match = g_list_find_custom(op->devices_list, iter->data,
	sort_strings);

	if (match) {
	op->devices_list = g_list_remove(op->devices_list, match->data);
	}
	}
	g_list_free_full(op->automatic_list, free);
	op->automatic_list = NULL;

	/* Rewind device list pointer */
	op->devices = op->devices_list;
	}

	/*!
	* \internal
	* \brief Reset a remapped reboot operation
	*
	* \param[in,out] op Operation to reset
	*/
	static void
	undo_op_remap(remote_fencing_op_t *op)
	{
	if (op->phase > 0) {
	crm_info("Undoing remap of reboot targeting %s for %s "
	CRM_XS " id=%.8s", op->target, op->client_name, op->id);
	op->phase = st_phase_requested;
	strcpy(op->action, "reboot");
	}
	}

	static xmlNode *
	create_op_done_notify(remote_fencing_op_t * op, int rc)
	{
	xmlNode *notify_data = create_xml_node(NULL, T_STONITH_NOTIFY_FENCE);

	crm_xml_add_int(notify_data, "state", op->state);
	crm_xml_add_int(notify_data, F_STONITH_RC, rc);
	crm_xml_add(notify_data, F_STONITH_TARGET, op->target);
	crm_xml_add(notify_data, F_STONITH_ACTION, op->action);
	crm_xml_add(notify_data, F_STONITH_DELEGATE, op->delegate);
	crm_xml_add(notify_data, F_STONITH_REMOTE_OP_ID, op->id);
	crm_xml_add(notify_data, F_STONITH_ORIGIN, op->originator);
	crm_xml_add(notify_data, F_STONITH_CLIENTID, op->client_id);
	crm_xml_add(notify_data, F_STONITH_CLIENTNAME, op->client_name);

	return notify_data;
	}

	void
	stonith_bcast_result_to_peers(remote_fencing_op_t * op, int rc, gboolean op_merged)
	{
	static int count = 0;
	xmlNode *bcast = create_xml_node(NULL, T_STONITH_REPLY);
	xmlNode *notify_data = create_op_done_notify(op, rc);

	count++;
	crm_trace("Broadcasting result to peers");
	crm_xml_add(bcast, F_TYPE, T_STONITH_NOTIFY);
	crm_xml_add(bcast, F_SUBTYPE, "broadcast");
	crm_xml_add(bcast, F_STONITH_OPERATION, T_STONITH_NOTIFY);
	crm_xml_add_int(bcast, "count", count);

	if (op_merged) {
	crm_xml_add(bcast, F_STONITH_MERGED, "true");
	}

	add_message_xml(bcast, F_STONITH_CALLDATA, notify_data);
	send_cluster_message(NULL, crm_msg_stonith_ng, bcast, FALSE);
	free_xml(notify_data);
	free_xml(bcast);

	return;
	}

	static void
	handle_local_reply_and_notify(remote_fencing_op_t * op, xmlNode * data, int rc)
	{
	xmlNode *notify_data = NULL;
	xmlNode *reply = NULL;

	if (op->notify_sent == TRUE) {
	/* nothing to do */
	return;
	}

	/* Do notification with a clean data object */
	notify_data = create_op_done_notify(op, rc);
	crm_xml_add_int(data, "state", op->state);
	crm_xml_add(data, F_STONITH_TARGET, op->target);
	crm_xml_add(data, F_STONITH_OPERATION, op->action);

	reply = stonith_construct_reply(op->request, NULL, data, rc);
	crm_xml_add(reply, F_STONITH_DELEGATE, op->delegate);

	/* Send fencing OP reply to local client that initiated fencing */
	do_local_reply(reply, op->client_id, op->call_options & st_opt_sync_call, FALSE);

	/* bcast to all local clients that the fencing operation happend */
	do_stonith_notify(0, T_STONITH_NOTIFY_FENCE, rc, notify_data);
	do_stonith_notify(0, T_STONITH_NOTIFY_HISTORY, 0, NULL);

	/* mark this op as having notify's already sent */
	op->notify_sent = TRUE;
	free_xml(reply);
	free_xml(notify_data);
	}

	static void
	handle_duplicates(remote_fencing_op_t * op, xmlNode * data, int rc)
	{
	GListPtr iter = NULL;

	for (iter = op->duplicates; iter != NULL; iter = iter->next) {
	remote_fencing_op_t *other = iter->data;

	if (other->state == st_duplicate) {
	other->state = op->state;
	crm_debug("Performing duplicate notification for %s@%s: %s "
	CRM_XS " id=%.8s",
	other->client_name, other->originator,
	pcmk_strerror(rc), other->id);
	remote_op_done(other, data, rc, TRUE);

	} else {
	// Possible if (for example) it timed out already
	crm_err("Skipping duplicate notification for %s@%s "
	CRM_XS " state=%s id=%.8s",
	other->client_name, other->originator,
	stonith_op_state_str(other->state), other->id);
	}
	}
	}

	/*!
	* \internal
	* \brief Finalize a remote operation.
	*
	* \description This function has two code paths.
	*
	* Path 1. This node is the owner of the operation and needs
	* to notify the cpg group via a broadcast as to the operation's
	* results.
	*
	* Path 2. The cpg broadcast is received. All nodes notify their local
	* stonith clients the operation results.
	*
	* So, The owner of the operation first notifies the cluster of the result,
	* and once that cpg notify is received back it notifies all the local clients.
	*
	* Nodes that are passive watchers of the operation will receive the
	* broadcast and only need to notify their local clients the operation finished.
	*
	* \param op, The fencing operation to finalize
	* \param data, The xml msg reply (if present) of the last delegated fencing
	* operation.
	* \param dup, Is this operation a duplicate, if so treat it a little differently
	* making sure the broadcast is not sent out.
	*/
	static void
	remote_op_done(remote_fencing_op_t * op, xmlNode * data, int rc, int dup)
	{
	int level = LOG_ERR;
	const char *subt = NULL;
	xmlNode *local_data = NULL;
	gboolean op_merged = FALSE;

	op->completed = time(NULL);
	clear_remote_op_timers(op);
	undo_op_remap(op);

	if (op->notify_sent == TRUE) {
	crm_err("Already sent notifications for '%s' targeting %s by %s for "
	"client %s@%s: %s " CRM_XS " rc=%d state=%s id=%.8s",
	op->action, op->target,
	(op->delegate? op->delegate : "unknown node"),
	op->client_name, op->originator, pcmk_strerror(rc),
	rc, stonith_op_state_str(op->state), op->id);
	goto remote_op_done_cleanup;
	}

	if (!op->delegate && data && rc != -ENODEV && rc != -EHOSTUNREACH) {
	xmlNode *ndata = get_xpath_object("//@" F_STONITH_DELEGATE, data,
	LOG_NEVER);
	if(ndata) {
	op->delegate = crm_element_value_copy(ndata, F_STONITH_DELEGATE);
	} else {
	op->delegate = crm_element_value_copy(data, F_ORIG);
	}
	}

	if (data == NULL) {
	data = create_xml_node(NULL, "remote-op");
	local_data = data;
	}

	if(dup) {
	op_merged = TRUE;
	} else if (crm_element_value(data, F_STONITH_MERGED)) {
	op_merged = TRUE;
	}

	/* Tell everyone the operation is done, we will continue
	* with doing the local notifications once we receive
	* the broadcast back. */
	subt = crm_element_value(data, F_SUBTYPE);
	if (dup == FALSE && !pcmk__str_eq(subt, "broadcast", pcmk__str_casei)) {
	/* Defer notification until the bcast message arrives */
	stonith_bcast_result_to_peers(op, rc, (op_merged? TRUE: FALSE));
	goto remote_op_done_cleanup;
	}

	if (rc == pcmk_ok \|\| dup) {
	level = LOG_NOTICE;
	} else if (!pcmk__str_eq(op->originator, stonith_our_uname, pcmk__str_casei)) {
	level = LOG_NOTICE;
	}

	do_crm_log(level, "Operation '%s'%s%s by %s for %s@%s%s: %s "
	CRM_XS " id=%.8s", op->action, (op->target? " targeting " : ""),
	(op->target? op->target : ""),
	(op->delegate? op->delegate : "unknown node"),
	op->client_name, op->originator,
	(op_merged? " (merged)" : ""), pcmk_strerror(rc), op->id);

	handle_local_reply_and_notify(op, data, rc);

	if (dup == FALSE) {
	handle_duplicates(op, data, rc);
	}

	/* Free non-essential parts of the record
	* Keep the record around so we can query the history
	*/
	if (op->query_results) {
	g_list_free_full(op->query_results, free_remote_query);
	op->query_results = NULL;
	}

	if (op->request) {
	free_xml(op->request);
	op->request = NULL;
	}

	remote_op_done_cleanup:
	free_xml(local_data);
	}

	static gboolean
	remote_op_watchdog_done(gpointer userdata)
	{
	remote_fencing_op_t *op = userdata;

	op->op_timer_one = 0;

	crm_notice("Self-fencing (%s) by %s for %s assumed complete "
	CRM_XS " id=%.8s",
	op->action, op->target, op->client_name, op->id);
	op->state = st_done;
	remote_op_done(op, NULL, pcmk_ok, FALSE);
	return FALSE;
	}

	static gboolean
	remote_op_timeout_one(gpointer userdata)
	{
	remote_fencing_op_t *op = userdata;

	op->op_timer_one = 0;

	crm_notice("Peer's '%s' action targeting %s for client %s timed out " CRM_XS
	" id=%.8s", op->action, op->target, op->client_name, op->id);
	call_remote_stonith(op, NULL, pcmk_ok);
	return FALSE;
	}

	static gboolean
	remote_op_timeout(gpointer userdata)
	{
	remote_fencing_op_t *op = userdata;

	op->op_timer_total = 0;

	if (op->state == st_done) {
	crm_debug("Action '%s' targeting %s for client %s already completed "
	CRM_XS " id=%.8s",
	op->action, op->target, op->client_name, op->id);
	return FALSE;
	}

	crm_debug("Action '%s' targeting %s for client %s timed out "
	CRM_XS " id=%.8s",
	op->action, op->target, op->client_name, op->id);

	if (op->phase == st_phase_on) {
	/* A remapped reboot operation timed out in the "on" phase, but the
	* "off" phase completed successfully, so quit trying any further
	* devices, and return success.
	*/
	remote_op_done(op, NULL, pcmk_ok, FALSE);
	return FALSE;
	}

	op->state = st_failed;

	remote_op_done(op, NULL, -ETIME, FALSE);

	return FALSE;
	}

	static gboolean
	remote_op_query_timeout(gpointer data)
	{
	remote_fencing_op_t *op = data;

	op->query_timer = 0;
	if (op->state == st_done) {
	crm_debug("Operation %.8s targeting %s already completed",
	op->id, op->target);
	} else if (op->state == st_exec) {
	crm_debug("Operation %.8s targeting %s already in progress",
	op->id, op->target);
	} else if (op->query_results) {
	crm_debug("Query %.8s targeting %s complete (state=%s)",
	op->id, op->target, stonith_op_state_str(op->state));
	call_remote_stonith(op, NULL, pcmk_ok);
	} else {
	crm_debug("Query %.8s targeting %s timed out (state=%s)",
	op->id, op->target, stonith_op_state_str(op->state));
	if (op->op_timer_total) {
	g_source_remove(op->op_timer_total);
	op->op_timer_total = 0;
	}
	remote_op_timeout(op);
	}

	return FALSE;
	}

	static gboolean
	topology_is_empty(stonith_topology_t *tp)
	{
	int i;

	if (tp == NULL) {
	return TRUE;
	}

	for (i = 0; i < ST_LEVEL_MAX; i++) {
	if (tp->levels[i] != NULL) {
	return FALSE;
	}
	}
	return TRUE;
	}

	/*!
	* \internal
	* \brief Add a device to an operation's automatic unfencing list
	*
	* \param[in,out] op Operation to modify
	* \param[in] device Device ID to add
	*/
	static void
	add_required_device(remote_fencing_op_t op, const char device)
	{
	GListPtr match = g_list_find_custom(op->automatic_list, device,
	sort_strings);

	if (!match) {
	op->automatic_list = g_list_prepend(op->automatic_list, strdup(device));
	}
	}

	/*!
	* \internal
	* \brief Remove a device from the automatic unfencing list
	*
	* \param[in,out] op Operation to modify
	* \param[in] device Device ID to remove
	*/
	static void
	remove_required_device(remote_fencing_op_t op, const char device)
	{
	GListPtr match = g_list_find_custom(op->automatic_list, device,
	sort_strings);

	if (match) {
	op->automatic_list = g_list_remove(op->automatic_list, match->data);
	}
	}

	/* deep copy the device list */
	static void
	set_op_device_list(remote_fencing_op_t * op, GListPtr devices)
	{
	GListPtr lpc = NULL;

	if (op->devices_list) {
	g_list_free_full(op->devices_list, free);
	op->devices_list = NULL;
	}
	for (lpc = devices; lpc != NULL; lpc = lpc->next) {
	op->devices_list = g_list_append(op->devices_list, strdup(lpc->data));
	}
	op->devices = op->devices_list;
	}

	/*!
	* \internal
	* \brief Check whether a node matches a topology target
	*
	* \param[in] tp Topology table entry to check
	* \param[in] node Name of node to check
	*
	* \return TRUE if node matches topology target
	*/
	static gboolean
	topology_matches(const stonith_topology_t tp, const char node)
	{
	regex_t r_patt;

	CRM_CHECK(node && tp && tp->target, return FALSE);
	switch(tp->kind) {
	case 2:
	/* This level targets by attribute, so tp->target is a NAME=VALUE pair
	* of a permanent attribute applied to targeted nodes. The test below
	* relies on the locally cached copy of the CIB, so if fencing needs to
	* be done before the initial CIB is received or after a malformed CIB
	* is received, then the topology will be unable to be used.
	*/
	if (node_has_attr(node, tp->target_attribute, tp->target_value)) {
	crm_notice("Matched %s with %s by attribute", node, tp->target);
	return TRUE;
	}
	break;
	case 1:
	/* This level targets by name, so tp->target is a regular expression
	* matching names of nodes to be targeted.
	*/

	if (regcomp(&r_patt, tp->target_pattern, REG_EXTENDED\|REG_NOSUB)) {
	crm_info("Bad regex '%s' for fencing level", tp->target);
	} else {
	int status = regexec(&r_patt, node, 0, NULL, 0);

	regfree(&r_patt);
	if (status == 0) {
	crm_notice("Matched %s with %s by name", node, tp->target);
	return TRUE;
	}
	}
	break;
	case 0:
	crm_trace("Testing %s against %s", node, tp->target);
	return pcmk__str_eq(tp->target, node, pcmk__str_casei);
	}
	crm_trace("No match for %s with %s", node, tp->target);
	return FALSE;
	}

	stonith_topology_t *
	find_topology_for_host(const char *host)
	{
	GHashTableIter tIter;
	stonith_topology_t *tp = g_hash_table_lookup(topology, host);

	if(tp != NULL) {
	crm_trace("Found %s for %s in %d entries", tp->target, host, g_hash_table_size(topology));
	return tp;
	}

	g_hash_table_iter_init(&tIter, topology);
	while (g_hash_table_iter_next(&tIter, NULL, (gpointer *) & tp)) {
	if (topology_matches(tp, host)) {
	crm_trace("Found %s for %s in %d entries", tp->target, host, g_hash_table_size(topology));
	return tp;
	}
	}

	crm_trace("No matches for %s in %d topology entries", host, g_hash_table_size(topology));
	return NULL;
	}

	/*!
	* \internal
	* \brief Set fencing operation's device list to target's next topology level
	*
	* \param[in,out] op Remote fencing operation to modify
	* \param[in] empty_ok If true, an operation without a target (i.e.
	* queries) or a target without a topology will get a
	* pcmk_rc_ok return value instead of ENODEV
	*
	* \return Standard Pacemaker return value
	*/
	static int
	advance_topology_level(remote_fencing_op_t *op, bool empty_ok)
	{
	stonith_topology_t *tp = NULL;

	if (op->target) {
	tp = find_topology_for_host(op->target);
	}
	if (topology_is_empty(tp)) {
	return empty_ok? pcmk_rc_ok : ENODEV;
	}

	CRM_ASSERT(tp->levels != NULL);

	stonith__set_call_options(op->call_options, op->id, st_opt_topology);

	/* This is a new level, so undo any remapping left over from previous */
	undo_op_remap(op);

	do {
	op->level++;

	} while (op->level < ST_LEVEL_MAX && tp->levels[op->level] == NULL);

	if (op->level < ST_LEVEL_MAX) {
	crm_trace("Attempting fencing level %d targeting %s (%d devices) "
	"for client %s@%s (id=%.8s)",
	op->level, op->target, g_list_length(tp->levels[op->level]),
	op->client_name, op->originator, op->id);
	set_op_device_list(op, tp->levels[op->level]);

	// The requested delay has been applied for the first fencing level
	if (op->level > 1 && op->delay > 0) {
	op->delay = 0;
	}

	if (g_list_next(op->devices_list) && pcmk__str_eq(op->action, "reboot", pcmk__str_casei)) {
	/* A reboot has been requested for a topology level with multiple
	* devices. Instead of rebooting the devices sequentially, we will
	* turn them all off, then turn them all on again. (Think about
	* switched power outlets for redundant power supplies.)
	*/
	op_phase_off(op);
	}
	return pcmk_rc_ok;
	}

	crm_notice("All fencing options targeting %s for client %s@%s failed "
	CRM_XS " id=%.8s",
	op->target, op->client_name, op->originator, op->id);
	return ENODEV;
	}

	/*!
	* \brief Check to see if this operation is a duplicate of another in flight
	* operation. If so merge this operation into the inflight operation, and mark
	* it as a duplicate.
	*/
	static void
	merge_duplicates(remote_fencing_op_t * op)
	{
	GHashTableIter iter;
	remote_fencing_op_t *other = NULL;

	time_t now = time(NULL);

	g_hash_table_iter_init(&iter, stonith_remote_op_list);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&other)) {
	const char *other_action = op_requested_action(other);

	if (!strcmp(op->id, other->id)) {
	continue; // Don't compare against self
	}
	if (other->state > st_exec) {
	crm_trace("%.8s not duplicate of %.8s: not in progress",
	op->id, other->id);
	continue;
	}
	if (!pcmk__str_eq(op->target, other->target, pcmk__str_casei)) {
	crm_trace("%.8s not duplicate of %.8s: node %s vs. %s",
	op->id, other->id, op->target, other->target);
	continue;
	}
	if (!pcmk__str_eq(op->action, other_action, pcmk__str_casei)) {
	crm_trace("%.8s not duplicate of %.8s: action %s vs. %s",
	op->id, other->id, op->action, other_action);
	continue;
	}
	if (pcmk__str_eq(op->client_name, other->client_name, pcmk__str_casei)) {
	crm_trace("%.8s not duplicate of %.8s: same client %s",
	op->id, other->id, op->client_name);
	continue;
	}
	if (pcmk__str_eq(other->target, other->originator, pcmk__str_casei)) {
	crm_trace("%.8s not duplicate of %.8s: suicide for %s",
	op->id, other->id, other->target);
	continue;
	}
	if (!fencing_peer_active(crm_get_peer(0, other->originator))) {
	crm_notice("Failing action '%s' targeting %s originating from "
	"client %s@%s: Originator is dead " CRM_XS " id=%.8s",
	other->action, other->target, other->client_name,
	other->originator, other->id);
	crm_trace("%.8s not duplicate of %.8s: originator dead",
	op->id, other->id);
	other->state = st_failed;
	continue;
	}
	if ((other->total_timeout > 0)
	&& (now > (other->total_timeout + other->created))) {
	crm_trace("%.8s not duplicate of %.8s: old (%ld vs. %ld + %d)",
	op->id, other->id, now, other->created,
	other->total_timeout);
	continue;
	}

	/* There is another in-flight request to fence the same host
	* Piggyback on that instead. If it fails, so do we.
	*/
	other->duplicates = g_list_append(other->duplicates, op);
	if (other->total_timeout == 0) {
	other->total_timeout = op->total_timeout =
	TIMEOUT_MULTIPLY_FACTOR * get_op_total_timeout(op, NULL);
	crm_trace("Best guess as to timeout used for %.8s: %d",
	other->id, other->total_timeout);
	}
	crm_notice("Merging fencing action '%s' targeting %s originating from "
	"client %s with identical request from %s@%s "
	CRM_XS " original=%.8s duplicate=%.8s total_timeout=%ds",
	op->action, op->target, op->client_name,
	other->client_name, other->originator,
	op->id, other->id, other->total_timeout);
	report_timeout_period(op, other->total_timeout);
	op->state = st_duplicate;
	}
	}

	static uint32_t fencing_active_peers(void)
	{
	uint32_t count = 0;
	crm_node_t *entry;
	GHashTableIter gIter;

	g_hash_table_iter_init(&gIter, crm_peer_cache);
	while (g_hash_table_iter_next(&gIter, NULL, (void **)&entry)) {
	if(fencing_peer_active(entry)) {
	count++;
	}
	}
	return count;
	}

	int
	stonith_manual_ack(xmlNode * msg, remote_fencing_op_t * op)
	{
	xmlNode *dev = get_xpath_object("//@" F_STONITH_TARGET, msg, LOG_ERR);

	op->state = st_done;
	op->completed = time(NULL);
	op->delegate = strdup("a human");

	crm_notice("Injecting manual confirmation that %s is safely off/down",
	crm_element_value(dev, F_STONITH_TARGET));

	remote_op_done(op, msg, pcmk_ok, FALSE);

	/* Replies are sent via done_cb->stonith_send_async_reply()->do_local_reply() */
	return -EINPROGRESS;
	}

	/*!
	* \internal
	* \brief Create a new remote stonith operation
	*
	* \param[in] client ID of local stonith client that initiated the operation
	* \param[in] request The request from the client that started the operation
	* \param[in] peer TRUE if this operation is owned by another stonith peer
	* (an operation owned by one peer is stored on all peers,
	* but only the owner executes it; all nodes get the results
	* once the owner finishes execution)
	*/
	void *
	create_remote_stonith_op(const char client, xmlNode request, gboolean peer)
	{
	remote_fencing_op_t *op = NULL;
	xmlNode *dev = get_xpath_object("//@" F_STONITH_TARGET, request, LOG_NEVER);
	int call_options = 0;
	const char *operation = NULL;

	init_stonith_remote_op_hash_table(&stonith_remote_op_list);

	/* If this operation is owned by another node, check to make
	* sure we haven't already created this operation. */
	if (peer && dev) {
	const char *op_id = crm_element_value(dev, F_STONITH_REMOTE_OP_ID);

	CRM_CHECK(op_id != NULL, return NULL);

	op = g_hash_table_lookup(stonith_remote_op_list, op_id);
	if (op) {
	crm_debug("Reusing existing remote fencing op %.8s for %s",
	op_id, ((client == NULL)? "unknown client" : client));
	return op;
	}
	}

	op = calloc(1, sizeof(remote_fencing_op_t));
	CRM_ASSERT(op != NULL);

	crm_element_value_int(request, F_STONITH_TIMEOUT, &(op->base_timeout));
	// Value -1 means disable any static/random fencing delays
	crm_element_value_int(request, F_STONITH_DELAY, &(op->delay));

	if (peer && dev) {
	op->id = crm_element_value_copy(dev, F_STONITH_REMOTE_OP_ID);
	} else {
	op->id = crm_generate_uuid();
	}

	g_hash_table_replace(stonith_remote_op_list, op->id, op);

	op->state = st_query;
	op->replies_expected = fencing_active_peers();
	op->action = crm_element_value_copy(dev, F_STONITH_ACTION);
	op->originator = crm_element_value_copy(dev, F_STONITH_ORIGIN);
	op->delegate = crm_element_value_copy(dev, F_STONITH_DELEGATE); /* May not be set */
	op->created = time(NULL);

	if (op->originator == NULL) {
	/* Local or relayed request */
	op->originator = strdup(stonith_our_uname);
	}

	CRM_LOG_ASSERT(client != NULL);
	if (client) {
	op->client_id = strdup(client);
	}


	/* For a RELAY operation, set fenced on the client. */
	operation = crm_element_value(request, F_STONITH_OPERATION);

	if (pcmk__str_eq(operation, STONITH_OP_RELAY, pcmk__str_none)) {
	op->client_name = crm_strdup_printf("%s.%lu", crm_system_name,
	(unsigned long) getpid());
	} else {
	op->client_name = crm_element_value_copy(request, F_STONITH_CLIENTNAME);
	}

	op->target = crm_element_value_copy(dev, F_STONITH_TARGET);
	op->request = copy_xml(request); /* TODO: Figure out how to avoid this */
	crm_element_value_int(request, F_STONITH_CALLOPTS, &call_options);
	op->call_options = call_options;

	crm_element_value_int(request, F_STONITH_CALLID, &(op->client_callid));

	crm_trace("%s new fencing op %s ('%s' targeting %s for client %s, "
	"base timeout %d, %u %s expected)",
	(peer && dev)? "Recorded" : "Generated", op->id, op->action,
	op->target, op->client_name, op->base_timeout,
	op->replies_expected,
	pcmk__plural_alt(op->replies_expected, "reply", "replies"));

	if (op->call_options & st_opt_cs_nodeid) {
	int nodeid = crm_atoi(op->target, NULL);
	- crm_node_t *node = crm_find_known_peer_full(nodeid, NULL, CRM_GET_PEER_ANY);
	+ crm_node_t *node = pcmk__search_known_node_cache(nodeid, NULL,
	+ CRM_GET_PEER_ANY);

	/* Ensure the conversion only happens once */
	stonith__clear_call_options(op->call_options, op->id, st_opt_cs_nodeid);

	if (node && node->uname) {
	free(op->target);
	op->target = strdup(node->uname);

	} else {
	crm_warn("Could not expand nodeid '%s' into a host name", op->target);
	}
	}

	/* check to see if this is a duplicate operation of another in-flight operation */
	merge_duplicates(op);

	if (op->state != st_duplicate) {
	/* kick history readers */
	do_stonith_notify(0, T_STONITH_NOTIFY_HISTORY, 0, NULL);
	}

	/* safe to trim as long as that doesn't touch pending ops */
	stonith_fence_history_trim();

	return op;
	}

	remote_fencing_op_t *
	initiate_remote_stonith_op(pcmk__client_t client, xmlNode request,
	gboolean manual_ack)
	{
	int query_timeout = 0;
	xmlNode *query = NULL;
	const char *client_id = NULL;
	remote_fencing_op_t *op = NULL;
	const char *relay_op_id = NULL;
	const char *operation = NULL;

	if (client) {
	client_id = client->id;
	} else {
	client_id = crm_element_value(request, F_STONITH_CLIENTID);
	}

	CRM_LOG_ASSERT(client_id != NULL);
	op = create_remote_stonith_op(client_id, request, FALSE);
	op->owner = TRUE;
	if (manual_ack) {
	crm_notice("Processing manual confirmation of fencing targeting %s "
	CRM_XS " id=%.8s", op->target, op->id);
	return op;
	}

	CRM_CHECK(op->action, return NULL);

	if (advance_topology_level(op, true) != pcmk_rc_ok) {
	op->state = st_failed;
	}

	switch (op->state) {
	case st_failed:
	crm_warn("Could not request peer fencing (%s) targeting %s "
	CRM_XS " id=%.8s", op->action, op->target, op->id);
	remote_op_done(op, NULL, -EINVAL, FALSE);
	return op;

	case st_duplicate:
	crm_info("Requesting peer fencing (%s) targeting %s (duplicate) "
	CRM_XS " id=%.8s", op->action, op->target, op->id);
	return op;

	default:
	crm_notice("Requesting peer fencing (%s) targeting %s "
	CRM_XS " id=%.8s state=%s base_timeout=%d",
	op->action, op->target, op->id,
	stonith_op_state_str(op->state), op->base_timeout);
	}

	query = stonith_create_op(op->client_callid, op->id, STONITH_OP_QUERY,
	NULL, op->call_options);

	crm_xml_add(query, F_STONITH_REMOTE_OP_ID, op->id);
	crm_xml_add(query, F_STONITH_TARGET, op->target);
	crm_xml_add(query, F_STONITH_ACTION, op_requested_action(op));
	crm_xml_add(query, F_STONITH_ORIGIN, op->originator);
	crm_xml_add(query, F_STONITH_CLIENTID, op->client_id);
	crm_xml_add(query, F_STONITH_CLIENTNAME, op->client_name);
	crm_xml_add_int(query, F_STONITH_TIMEOUT, op->base_timeout);

	/* In case of RELAY operation, RELAY information is added to the query to delete the original operation of RELAY. */
	operation = crm_element_value(request, F_STONITH_OPERATION);
	if (pcmk__str_eq(operation, STONITH_OP_RELAY, pcmk__str_none)) {
	relay_op_id = crm_element_value(request, F_STONITH_REMOTE_OP_ID);
	if (relay_op_id) {
	crm_xml_add(query, F_STONITH_REMOTE_OP_ID_RELAY, relay_op_id);
	}
	}

	send_cluster_message(NULL, crm_msg_stonith_ng, query, FALSE);
	free_xml(query);

	query_timeout = op->base_timeout * TIMEOUT_MULTIPLY_FACTOR;
	op->query_timer = g_timeout_add((1000 * query_timeout), remote_op_query_timeout, op);

	return op;
	}

	enum find_best_peer_options {
	/! Skip checking the target peer for capable fencing devices /
	FIND_PEER_SKIP_TARGET = 0x0001,
	/! Only check the target peer for capable fencing devices /
	FIND_PEER_TARGET_ONLY = 0x0002,
	/! Skip peers and devices that are not verified /
	FIND_PEER_VERIFIED_ONLY = 0x0004,
	};

	static st_query_result_t *
	find_best_peer(const char device, remote_fencing_op_t op, enum find_best_peer_options options)
	{
	GListPtr iter = NULL;
	gboolean verified_devices_only = (options & FIND_PEER_VERIFIED_ONLY) ? TRUE : FALSE;

	if (!device && pcmk_is_set(op->call_options, st_opt_topology)) {
	return NULL;
	}

	for (iter = op->query_results; iter != NULL; iter = iter->next) {
	st_query_result_t *peer = iter->data;

	crm_trace("Testing result from %s targeting %s with %d device%s: %d %x",
	peer->host, op->target, peer->ndevices,
	pcmk__plural_s(peer->ndevices), peer->tried, options);
	if ((options & FIND_PEER_SKIP_TARGET) && pcmk__str_eq(peer->host, op->target, pcmk__str_casei)) {
	continue;
	}
	if ((options & FIND_PEER_TARGET_ONLY) && !pcmk__str_eq(peer->host, op->target, pcmk__str_casei)) {
	continue;
	}

	if (pcmk_is_set(op->call_options, st_opt_topology)) {

	if (grab_peer_device(op, peer, device, verified_devices_only)) {
	return peer;
	}

	} else if ((peer->tried == FALSE)
	&& count_peer_devices(op, peer, verified_devices_only)) {

	/* No topology: Use the current best peer */
	crm_trace("Simple fencing");
	return peer;
	}
	}

	return NULL;
	}

	static st_query_result_t *
	stonith_choose_peer(remote_fencing_op_t * op)
	{
	const char *device = NULL;
	st_query_result_t *peer = NULL;
	uint32_t active = fencing_active_peers();

	do {
	if (op->devices) {
	device = op->devices->data;
	crm_trace("Checking for someone to fence (%s) %s using %s",
	op->action, op->target, device);
	} else {
	crm_trace("Checking for someone to fence (%s) %s",
	op->action, op->target);
	}

	/* Best choice is a peer other than the target with verified access */
	peer = find_best_peer(device, op, FIND_PEER_SKIP_TARGET\|FIND_PEER_VERIFIED_ONLY);
	if (peer) {
	crm_trace("Found verified peer %s for %s", peer->host, device?device:"<any>");
	return peer;
	}

	if(op->query_timer != 0 && op->replies < QB_MIN(op->replies_expected, active)) {
	crm_trace("Waiting before looking for unverified devices to fence %s", op->target);
	return NULL;
	}

	/* If no other peer has verified access, next best is unverified access */
	peer = find_best_peer(device, op, FIND_PEER_SKIP_TARGET);
	if (peer) {
	crm_trace("Found best unverified peer %s", peer->host);
	return peer;
	}

	/* If no other peer can do it, last option is self-fencing
	* (which is never allowed for the "on" phase of a remapped reboot)
	*/
	if (op->phase != st_phase_on) {
	peer = find_best_peer(device, op, FIND_PEER_TARGET_ONLY);
	if (peer) {
	crm_trace("%s will fence itself", peer->host);
	return peer;
	}
	}

	/* Try the next fencing level if there is one (unless we're in the "on"
	* phase of a remapped "reboot", because we ignore errors in that case)
	*/
	} while ((op->phase != st_phase_on)
	&& pcmk_is_set(op->call_options, st_opt_topology)
	&& (advance_topology_level(op, false) == pcmk_rc_ok));

	crm_notice("Couldn't find anyone to fence (%s) %s using %s",
	op->action, op->target, (device? device : "any device"));
	return NULL;
	}

	static int
	get_device_timeout(const remote_fencing_op_t op, const st_query_result_t peer,
	const char *device)
	{
	device_properties_t *props;

	if (!peer \|\| !device) {
	return op->base_timeout;
	}

	props = g_hash_table_lookup(peer->devices, device);
	if (!props) {
	return op->base_timeout;
	}

	return (props->custom_action_timeout[op->phase]?
	props->custom_action_timeout[op->phase] : op->base_timeout)
	+ props->delay_max[op->phase];
	}

	struct timeout_data {
	const remote_fencing_op_t *op;
	const st_query_result_t *peer;
	int total_timeout;
	};

	/*!
	* \internal
	* \brief Add timeout to a total if device has not been executed yet
	*
	* \param[in] key GHashTable key (device ID)
	* \param[in] value GHashTable value (device properties)
	* \param[in] user_data Timeout data
	*/
	static void
	add_device_timeout(gpointer key, gpointer value, gpointer user_data)
	{
	const char *device_id = key;
	device_properties_t *props = value;
	struct timeout_data *timeout = user_data;

	if (!props->executed[timeout->op->phase]
	&& !props->disallowed[timeout->op->phase]) {
	timeout->total_timeout += get_device_timeout(timeout->op,
	timeout->peer, device_id);
	}
	}

	static int
	get_peer_timeout(const remote_fencing_op_t op, const st_query_result_t peer)
	{
	struct timeout_data timeout;

	timeout.op = op;
	timeout.peer = peer;
	timeout.total_timeout = 0;

	g_hash_table_foreach(peer->devices, add_device_timeout, &timeout);

	return (timeout.total_timeout? timeout.total_timeout : op->base_timeout);
	}

	static int
	get_op_total_timeout(const remote_fencing_op_t *op,
	const st_query_result_t *chosen_peer)
	{
	int total_timeout = 0;
	stonith_topology_t *tp = find_topology_for_host(op->target);

	if (pcmk_is_set(op->call_options, st_opt_topology) && tp) {
	int i;
	GListPtr device_list = NULL;
	GListPtr iter = NULL;

	/* Yep, this looks scary, nested loops all over the place.
	* Here is what is going on.
	* Loop1: Iterate through fencing levels.
	* Loop2: If a fencing level has devices, loop through each device
	* Loop3: For each device in a fencing level, see what peer owns it
	* and what that peer has reported the timeout is for the device.
	*/
	for (i = 0; i < ST_LEVEL_MAX; i++) {
	if (!tp->levels[i]) {
	continue;
	}
	for (device_list = tp->levels[i]; device_list; device_list = device_list->next) {
	for (iter = op->query_results; iter != NULL; iter = iter->next) {
	const st_query_result_t *peer = iter->data;

	if (find_peer_device(op, peer, device_list->data)) {
	total_timeout += get_device_timeout(op, peer,
	device_list->data);
	break;
	}
	} /* End Loop3: match device with peer that owns device, find device's timeout period */
	} /* End Loop2: iterate through devices at a specific level */
	} /End Loop1: iterate through fencing levels /

	} else if (chosen_peer) {
	total_timeout = get_peer_timeout(op, chosen_peer);
	} else {
	total_timeout = op->base_timeout;
	}

	return total_timeout ? total_timeout : op->base_timeout;
	}

	static void
	report_timeout_period(remote_fencing_op_t * op, int op_timeout)
	{
	GListPtr iter = NULL;
	xmlNode *update = NULL;
	const char *client_node = NULL;
	const char *client_id = NULL;
	const char *call_id = NULL;

	if (op->call_options & st_opt_sync_call) {
	/* There is no reason to report the timeout for a synchronous call. It
	* is impossible to use the reported timeout to do anything when the client
	* is blocking for the response. This update is only important for
	* async calls that require a callback to report the results in. */
	return;
	} else if (!op->request) {
	return;
	}

	crm_trace("Reporting timeout for %s (id=%.8s)", op->client_name, op->id);
	client_node = crm_element_value(op->request, F_STONITH_CLIENTNODE);
	call_id = crm_element_value(op->request, F_STONITH_CALLID);
	client_id = crm_element_value(op->request, F_STONITH_CLIENTID);
	if (!client_node \|\| !call_id \|\| !client_id) {
	return;
	}

	if (pcmk__str_eq(client_node, stonith_our_uname, pcmk__str_casei)) {
	/* The client is connected to this node, send the update direclty to them */
	do_stonith_async_timeout_update(client_id, call_id, op_timeout);
	return;
	}

	/* The client is connected to another node, relay this update to them */
	update = stonith_create_op(op->client_callid, op->id, STONITH_OP_TIMEOUT_UPDATE, NULL, 0);
	crm_xml_add(update, F_STONITH_REMOTE_OP_ID, op->id);
	crm_xml_add(update, F_STONITH_CLIENTID, client_id);
	crm_xml_add(update, F_STONITH_CALLID, call_id);
	crm_xml_add_int(update, F_STONITH_TIMEOUT, op_timeout);

	send_cluster_message(crm_get_peer(0, client_node), crm_msg_stonith_ng, update, FALSE);

	free_xml(update);

	for (iter = op->duplicates; iter != NULL; iter = iter->next) {
	remote_fencing_op_t *dup = iter->data;

	crm_trace("Reporting timeout for duplicate %.8s to client %s",
	dup->id, dup->client_name);
	report_timeout_period(iter->data, op_timeout);
	}
	}

	/*!
	* \internal
	* \brief Advance an operation to the next device in its topology
	*
	* \param[in,out] op Operation to advance
	* \param[in] device ID of device just completed
	* \param[in] msg XML reply that contained device result (if available)
	* \param[in] rc Return code of device's execution
	*/
	static void
	advance_topology_device_in_level(remote_fencing_op_t op, const char device,
	xmlNode *msg, int rc)
	{
	/* Advance to the next device at this topology level, if any */
	if (op->devices) {
	op->devices = op->devices->next;
	}

	/* Handle automatic unfencing if an "on" action was requested */
	if ((op->phase == st_phase_requested) && pcmk__str_eq(op->action, "on", pcmk__str_casei)) {
	/* If the device we just executed was required, it's not anymore */
	remove_required_device(op, device);

	/* If there are no more devices at this topology level, run through any
	* remaining devices with automatic unfencing
	*/
	if (op->devices == NULL) {
	op->devices = op->automatic_list;
	}
	}

	if ((op->devices == NULL) && (op->phase == st_phase_off)) {
	/* We're done with this level and with required devices, but we had
	* remapped "reboot" to "off", so start over with "on". If any devices
	* need to be turned back on, op->devices will be non-NULL after this.
	*/
	op_phase_on(op);
	}

	if (op->devices) {
	/* Necessary devices remain, so execute the next one */
	crm_trace("Next targeting %s on behalf of %s@%s (rc was %d)",
	op->target, op->client_name, op->originator, rc);

	// The requested delay has been applied for the first device
	if (op->delay > 0) {
	op->delay = 0;
	}

	call_remote_stonith(op, NULL, pcmk_ok);
	} else {
	/* We're done with all devices and phases, so finalize operation */
	crm_trace("Marking complex fencing op targeting %s as complete",
	op->target);
	op->state = st_done;
	remote_op_done(op, msg, rc, FALSE);
	}
	}

	void
	call_remote_stonith(remote_fencing_op_t * op, st_query_result_t * peer, int rc)
	{
	const char *device = NULL;
	int timeout = op->base_timeout;

	crm_trace("Action %.8s targeting %s for %s is %s",
	op->id, op->target, op->client_name,
	stonith_op_state_str(op->state));
	if ((peer == NULL) && !pcmk_is_set(op->call_options, st_opt_topology)) {
	peer = stonith_choose_peer(op);
	}

	if (!op->op_timer_total) {
	int total_timeout = get_op_total_timeout(op, peer);

	op->total_timeout = TIMEOUT_MULTIPLY_FACTOR * total_timeout;
	op->op_timer_total = g_timeout_add(1000 * op->total_timeout, remote_op_timeout, op);
	report_timeout_period(op, op->total_timeout);
	crm_info("Total timeout set to %d for peer's fencing targeting %s for %s"
	CRM_XS "id=%.8s",
	total_timeout, op->target, op->client_name, op->id);
	}

	if (pcmk_is_set(op->call_options, st_opt_topology) && op->devices) {
	/* Ignore any peer preference, they might not have the device we need */
	/* When using topology, stonith_choose_peer() removes the device from
	* further consideration, so be sure to calculate timeout beforehand */
	peer = stonith_choose_peer(op);

	device = op->devices->data;
	timeout = get_device_timeout(op, peer, device);
	}

	if (peer) {
	int timeout_one = 0;
	xmlNode *remote_op = stonith_create_op(op->client_callid, op->id, STONITH_OP_FENCE, NULL, 0);

	crm_xml_add(remote_op, F_STONITH_REMOTE_OP_ID, op->id);
	crm_xml_add(remote_op, F_STONITH_TARGET, op->target);
	crm_xml_add(remote_op, F_STONITH_ACTION, op->action);
	crm_xml_add(remote_op, F_STONITH_ORIGIN, op->originator);
	crm_xml_add(remote_op, F_STONITH_CLIENTID, op->client_id);
	crm_xml_add(remote_op, F_STONITH_CLIENTNAME, op->client_name);
	crm_xml_add_int(remote_op, F_STONITH_TIMEOUT, timeout);
	crm_xml_add_int(remote_op, F_STONITH_CALLOPTS, op->call_options);
	crm_xml_add_int(remote_op, F_STONITH_DELAY, op->delay);

	if (device) {
	timeout_one = TIMEOUT_MULTIPLY_FACTOR *
	get_device_timeout(op, peer, device);
	crm_notice("Requesting that %s perform '%s' action targeting %s "
	"using %s " CRM_XS " for client %s (%ds)",
	peer->host, op->action, op->target, device,
	op->client_name, timeout_one);
	crm_xml_add(remote_op, F_STONITH_DEVICE, device);
	crm_xml_add(remote_op, F_STONITH_MODE, "slave");

	} else {
	timeout_one = TIMEOUT_MULTIPLY_FACTOR * get_peer_timeout(op, peer);
	crm_notice("Requesting that %s perform '%s' action targeting %s "
	CRM_XS " for client %s (%ds, %lds)",
	peer->host, op->action, op->target, op->client_name,
	timeout_one, stonith_watchdog_timeout_ms);
	crm_xml_add(remote_op, F_STONITH_MODE, "smart");
	}

	op->state = st_exec;
	if (op->op_timer_one) {
	g_source_remove(op->op_timer_one);
	}

	if(stonith_watchdog_timeout_ms > 0 && device && pcmk__str_eq(device, "watchdog", pcmk__str_casei)) {
	crm_notice("Waiting %lds for %s to self-fence (%s) for client %s "
	CRM_XS " id=%.8s", (stonith_watchdog_timeout_ms / 1000),
	op->target, op->action, op->client_name, op->id);
	op->op_timer_one = g_timeout_add(stonith_watchdog_timeout_ms, remote_op_watchdog_done, op);

	/* TODO check devices to verify watchdog will be in use */
	} else if(stonith_watchdog_timeout_ms > 0
	&& pcmk__str_eq(peer->host, op->target, pcmk__str_casei)
	&& !pcmk__str_eq(op->action, "on", pcmk__str_casei)) {
	crm_notice("Waiting %lds for %s to self-fence (%s) for client %s "
	CRM_XS " id=%.8s", (stonith_watchdog_timeout_ms / 1000),
	op->target, op->action, op->client_name, op->id);
	op->op_timer_one = g_timeout_add(stonith_watchdog_timeout_ms, remote_op_watchdog_done, op);

	} else {
	op->op_timer_one = g_timeout_add((1000 * timeout_one), remote_op_timeout_one, op);
	}


	send_cluster_message(crm_get_peer(0, peer->host), crm_msg_stonith_ng, remote_op, FALSE);
	peer->tried = TRUE;
	free_xml(remote_op);
	return;

	} else if (op->phase == st_phase_on) {
	/* A remapped "on" cannot be executed, but the node was already
	* turned off successfully, so ignore the error and continue.
	*/
	crm_warn("Ignoring %s 'on' failure (no capable peers) targeting %s "
	"after successful 'off'", device, op->target);
	advance_topology_device_in_level(op, device, NULL, pcmk_ok);
	return;

	} else if (op->owner == FALSE) {
	crm_err("Fencing (%s) targeting %s for client %s is not ours to control",
	op->action, op->target, op->client_name);

	} else if (op->query_timer == 0) {
	/* We've exhausted all available peers */
	crm_info("No remaining peers capable of fencing (%s) %s for client %s "
	CRM_XS " state=%s", op->action, op->target, op->client_name,
	stonith_op_state_str(op->state));
	CRM_LOG_ASSERT(op->state < st_done);
	remote_op_timeout(op);

	} else if(op->replies >= op->replies_expected \|\| op->replies >= fencing_active_peers()) {
	// int rc = -EHOSTUNREACH;

	/* if the operation never left the query state,
	* but we have all the expected replies, then no devices
	* are available to execute the fencing operation. */

	if(stonith_watchdog_timeout_ms && pcmk__str_eq(device, "watchdog", pcmk__str_null_matches \| pcmk__str_casei)) {
	crm_notice("Waiting %lds for %s to self-fence (%s) for client %s "
	CRM_XS " id=%.8s", (stonith_watchdog_timeout_ms / 1000),
	op->target, op->action, op->client_name, op->id);
	op->op_timer_one = g_timeout_add(stonith_watchdog_timeout_ms, remote_op_watchdog_done, op);
	return;
	}

	if (op->state == st_query) {
	crm_info("No peers (out of %d) have devices capable of fencing "
	"(%s) %s for client %s " CRM_XS " state=%s",
	op->replies, op->action, op->target, op->client_name,
	stonith_op_state_str(op->state));

	rc = -ENODEV;
	} else {
	if (pcmk_is_set(op->call_options, st_opt_topology)) {
	rc = -EHOSTUNREACH;
	}

	crm_info("No peers (out of %d) are capable of fencing (%s) %s "
	"for client %s " CRM_XS " state=%s",
	op->replies, op->action, op->target, op->client_name,
	stonith_op_state_str(op->state));
	}

	op->state = st_failed;
	remote_op_done(op, NULL, rc, FALSE);

	} else {
	crm_info("Waiting for additional peers capable of fencing (%s) %s%s%s "
	"for client %s " CRM_XS " id=%.8s",
	op->action, op->target, (device? " using " : ""),
	(device? device : ""), op->client_name, op->id);
	}
	}

	/*!
	* \internal
	* \brief Comparison function for sorting query results
	*
	* \param[in] a GList item to compare
	* \param[in] b GList item to compare
	*
	* \return Per the glib documentation, "a negative integer if the first value
	* comes before the second, 0 if they are equal, or a positive integer
	* if the first value comes after the second."
	*/
	static gint
	sort_peers(gconstpointer a, gconstpointer b)
	{
	const st_query_result_t *peer_a = a;
	const st_query_result_t *peer_b = b;

	return (peer_b->ndevices - peer_a->ndevices);
	}

	/*!
	* \internal
	* \brief Determine if all the devices in the topology are found or not
	*/
	static gboolean
	all_topology_devices_found(remote_fencing_op_t * op)
	{
	GListPtr device = NULL;
	GListPtr iter = NULL;
	device_properties_t *match = NULL;
	stonith_topology_t *tp = NULL;
	gboolean skip_target = FALSE;
	int i;

	tp = find_topology_for_host(op->target);
	if (!tp) {
	return FALSE;
	}
	if (pcmk__strcase_any_of(op->action, "off", "reboot", NULL)) {
	/* Don't count the devices on the target node if we are killing
	* the target node. */
	skip_target = TRUE;
	}

	for (i = 0; i < ST_LEVEL_MAX; i++) {
	for (device = tp->levels[i]; device; device = device->next) {
	match = NULL;
	for (iter = op->query_results; iter && !match; iter = iter->next) {
	st_query_result_t *peer = iter->data;

	if (skip_target && pcmk__str_eq(peer->host, op->target, pcmk__str_casei)) {
	continue;
	}
	match = find_peer_device(op, peer, device->data);
	}
	if (!match) {
	return FALSE;
	}
	}
	}

	return TRUE;
	}

	/*!
	* \internal
	* \brief Parse action-specific device properties from XML
	*
	* \param[in] msg XML element containing the properties
	* \param[in] peer Name of peer that sent XML (for logs)
	* \param[in] device Device ID (for logs)
	* \param[in] action Action the properties relate to (for logs)
	* \param[in] phase Phase the properties relate to
	* \param[in,out] props Device properties to update
	*/
	static void
	parse_action_specific(xmlNode xml, const char peer, const char *device,
	const char action, remote_fencing_op_t op,
	enum st_remap_phase phase, device_properties_t *props)
	{
	props->custom_action_timeout[phase] = 0;
	crm_element_value_int(xml, F_STONITH_ACTION_TIMEOUT,
	&props->custom_action_timeout[phase]);
	if (props->custom_action_timeout[phase]) {
	crm_trace("Peer %s with device %s returned %s action timeout %d",
	peer, device, action, props->custom_action_timeout[phase]);
	}

	props->delay_max[phase] = 0;
	crm_element_value_int(xml, F_STONITH_DELAY_MAX, &props->delay_max[phase]);
	if (props->delay_max[phase]) {
	crm_trace("Peer %s with device %s returned maximum of random delay %d for %s",
	peer, device, props->delay_max[phase], action);
	}

	props->delay_base[phase] = 0;
	crm_element_value_int(xml, F_STONITH_DELAY_BASE, &props->delay_base[phase]);
	if (props->delay_base[phase]) {
	crm_trace("Peer %s with device %s returned base delay %d for %s",
	peer, device, props->delay_base[phase], action);
	}

	/* Handle devices with automatic unfencing */
	if (pcmk__str_eq(action, "on", pcmk__str_casei)) {
	int required = 0;

	crm_element_value_int(xml, F_STONITH_DEVICE_REQUIRED, &required);
	if (required) {
	crm_trace("Peer %s requires device %s to execute for action %s",
	peer, device, action);
	add_required_device(op, device);
	}
	}

	/* If a reboot is remapped to off+on, it's possible that a node is allowed
	* to perform one action but not another.
	*/
	if (crm_is_true(crm_element_value(xml, F_STONITH_ACTION_DISALLOWED))) {
	props->disallowed[phase] = TRUE;
	crm_trace("Peer %s is disallowed from executing %s for device %s",
	peer, action, device);
	}
	}

	/*!
	* \internal
	* \brief Parse one device's properties from peer's XML query reply
	*
	* \param[in] xml XML node containing device properties
	* \param[in,out] op Operation that query and reply relate to
	* \param[in,out] result Peer's results
	* \param[in] device ID of device being parsed
	*/
	static void
	add_device_properties(xmlNode xml, remote_fencing_op_t op,
	st_query_result_t result, const char device)
	{
	xmlNode *child;
	int verified = 0;
	device_properties_t *props = calloc(1, sizeof(device_properties_t));

	/* Add a new entry to this result's devices list */
	CRM_ASSERT(props != NULL);
	g_hash_table_insert(result->devices, strdup(device), props);

	/* Peers with verified (monitored) access will be preferred */
	crm_element_value_int(xml, F_STONITH_DEVICE_VERIFIED, &verified);
	if (verified) {
	crm_trace("Peer %s has confirmed a verified device %s",
	result->host, device);
	props->verified = TRUE;
	}

	/* Parse action-specific device properties */
	parse_action_specific(xml, result->host, device, op_requested_action(op),
	op, st_phase_requested, props);
	for (child = pcmk__xml_first_child(xml); child != NULL;
	child = pcmk__xml_next(child)) {
	/* Replies for "reboot" operations will include the action-specific
	* values for "off" and "on" in child elements, just in case the reboot
	* winds up getting remapped.
	*/
	if (pcmk__str_eq(ID(child), "off", pcmk__str_casei)) {
	parse_action_specific(child, result->host, device, "off",
	op, st_phase_off, props);
	} else if (pcmk__str_eq(ID(child), "on", pcmk__str_casei)) {
	parse_action_specific(child, result->host, device, "on",
	op, st_phase_on, props);
	}
	}
	}

	/*!
	* \internal
	* \brief Parse a peer's XML query reply and add it to operation's results
	*
	* \param[in,out] op Operation that query and reply relate to
	* \param[in] host Name of peer that sent this reply
	* \param[in] ndevices Number of devices expected in reply
	* \param[in] xml XML node containing device list
	*
	* \return Newly allocated result structure with parsed reply
	*/
	static st_query_result_t *
	add_result(remote_fencing_op_t op, const char host, int ndevices, xmlNode *xml)
	{
	st_query_result_t *result = calloc(1, sizeof(st_query_result_t));
	xmlNode *child;

	// cppcheck seems not to understand the abort logic in CRM_CHECK
	// cppcheck-suppress memleak
	CRM_CHECK(result != NULL, return NULL);
	result->host = strdup(host);
	result->devices = crm_str_table_new();

	/* Each child element describes one capable device available to the peer */
	for (child = pcmk__xml_first_child(xml); child != NULL;
	child = pcmk__xml_next(child)) {
	const char *device = ID(child);

	if (device) {
	add_device_properties(child, op, result, device);
	}
	}

	result->ndevices = g_hash_table_size(result->devices);
	CRM_CHECK(ndevices == result->ndevices,
	crm_err("Query claimed to have %d device%s but %d found",
	ndevices, pcmk__plural_s(ndevices), result->ndevices));

	op->query_results = g_list_insert_sorted(op->query_results, result, sort_peers);
	return result;
	}

	/*!
	* \internal
	* \brief Handle a peer's reply to our fencing query
	*
	* Parse a query result from XML and store it in the remote operation
	* table, and when enough replies have been received, issue a fencing request.
	*
	* \param[in] msg XML reply received
	*
	* \return pcmk_ok on success, -errno on error
	*
	* \note See initiate_remote_stonith_op() for how the XML query was initially
	* formed, and stonith_query() for how the peer formed its XML reply.
	*/
	int
	process_remote_stonith_query(xmlNode * msg)
	{
	int ndevices = 0;
	gboolean host_is_target = FALSE;
	gboolean have_all_replies = FALSE;
	const char *id = NULL;
	const char *host = NULL;
	remote_fencing_op_t *op = NULL;
	st_query_result_t *result = NULL;
	uint32_t replies_expected;
	xmlNode *dev = get_xpath_object("//@" F_STONITH_REMOTE_OP_ID, msg, LOG_ERR);

	CRM_CHECK(dev != NULL, return -EPROTO);

	id = crm_element_value(dev, F_STONITH_REMOTE_OP_ID);
	CRM_CHECK(id != NULL, return -EPROTO);

	dev = get_xpath_object("//@" F_STONITH_AVAILABLE_DEVICES, msg, LOG_ERR);
	CRM_CHECK(dev != NULL, return -EPROTO);
	crm_element_value_int(dev, F_STONITH_AVAILABLE_DEVICES, &ndevices);

	op = g_hash_table_lookup(stonith_remote_op_list, id);
	if (op == NULL) {
	crm_debug("Received query reply for unknown or expired operation %s",
	id);
	return -EOPNOTSUPP;
	}

	replies_expected = fencing_active_peers();
	if (op->replies_expected < replies_expected) {
	replies_expected = op->replies_expected;
	}
	if ((++op->replies >= replies_expected) && (op->state == st_query)) {
	have_all_replies = TRUE;
	}
	host = crm_element_value(msg, F_ORIG);
	host_is_target = pcmk__str_eq(host, op->target, pcmk__str_casei);

	crm_info("Query result %d of %d from %s for %s/%s (%d device%s) %s",
	op->replies, replies_expected, host,
	op->target, op->action, ndevices, pcmk__plural_s(ndevices), id);
	if (ndevices > 0) {
	result = add_result(op, host, ndevices, dev);
	}

	if (pcmk_is_set(op->call_options, st_opt_topology)) {
	/* If we start the fencing before all the topology results are in,
	* it is possible fencing levels will be skipped because of the missing
	* query results. */
	if (op->state == st_query && all_topology_devices_found(op)) {
	/* All the query results are in for the topology, start the fencing ops. */
	crm_trace("All topology devices found");
	call_remote_stonith(op, result, pcmk_ok);

	} else if (have_all_replies) {
	crm_info("All topology query replies have arrived, continuing (%d expected/%d received) ",
	replies_expected, op->replies);
	call_remote_stonith(op, NULL, pcmk_ok);
	}

	} else if (op->state == st_query) {
	int nverified = count_peer_devices(op, result, TRUE);

	/* We have a result for a non-topology fencing op that looks promising,
	* go ahead and start fencing before query timeout */
	if (result && (host_is_target == FALSE) && nverified) {
	/* we have a verified device living on a peer that is not the target */
	crm_trace("Found %d verified device%s",
	nverified, pcmk__plural_s(nverified));
	call_remote_stonith(op, result, pcmk_ok);

	} else if (have_all_replies) {
	crm_info("All query replies have arrived, continuing (%d expected/%d received) ",
	replies_expected, op->replies);
	call_remote_stonith(op, NULL, pcmk_ok);

	} else {
	crm_trace("Waiting for more peer results before launching fencing operation");
	}

	} else if (result && (op->state == st_done)) {
	crm_info("Discarding query result from %s (%d device%s): "
	"Operation is %s", result->host,
	result->ndevices, pcmk__plural_s(result->ndevices),
	stonith_op_state_str(op->state));
	}

	return pcmk_ok;
	}

	/*!
	* \internal
	* \brief Handle a peer's reply to a fencing request
	*
	* Parse a fencing reply from XML, and either finalize the operation
	* or attempt another device as appropriate.
	*
	* \param[in] msg XML reply received
	*
	* \return pcmk_ok on success, -errno on error
	*/
	int
	process_remote_stonith_exec(xmlNode * msg)
	{
	int rc = 0;
	const char *id = NULL;
	const char *device = NULL;
	remote_fencing_op_t *op = NULL;
	xmlNode *dev = get_xpath_object("//@" F_STONITH_REMOTE_OP_ID, msg, LOG_ERR);

	CRM_CHECK(dev != NULL, return -EPROTO);

	id = crm_element_value(dev, F_STONITH_REMOTE_OP_ID);
	CRM_CHECK(id != NULL, return -EPROTO);

	dev = get_xpath_object("//@" F_STONITH_RC, msg, LOG_ERR);
	CRM_CHECK(dev != NULL, return -EPROTO);

	crm_element_value_int(dev, F_STONITH_RC, &rc);

	device = crm_element_value(dev, F_STONITH_DEVICE);

	if (stonith_remote_op_list) {
	op = g_hash_table_lookup(stonith_remote_op_list, id);
	}

	if (op == NULL && rc == pcmk_ok) {
	/* Record successful fencing operations */
	const char *client_id = crm_element_value(dev, F_STONITH_CLIENTID);

	op = create_remote_stonith_op(client_id, dev, TRUE);
	}

	if (op == NULL) {
	/* Could be for an event that began before we started */
	/* TODO: Record the op for later querying */
	crm_info("Received peer result of unknown or expired operation %s", id);
	return -EOPNOTSUPP;
	}

	if (op->devices && device && !pcmk__str_eq(op->devices->data, device, pcmk__str_casei)) {
	crm_err("Received outdated reply for device %s (instead of %s) to "
	"fence (%s) %s. Operation already timed out at peer level.",
	device, (const char *) op->devices->data, op->action, op->target);
	return rc;
	}

	if (pcmk__str_eq(crm_element_value(msg, F_SUBTYPE), "broadcast", pcmk__str_casei)) {
	crm_debug("Finalizing action '%s' targeting %s on behalf of %s@%s: %s "
	CRM_XS " rc=%d id=%.8s",
	op->action, op->target, op->client_name, op->originator,
	pcmk_strerror(rc), rc, op->id);
	if (rc == pcmk_ok) {
	op->state = st_done;
	} else {
	op->state = st_failed;
	}
	remote_op_done(op, msg, rc, FALSE);
	return pcmk_ok;
	} else if (!pcmk__str_eq(op->originator, stonith_our_uname, pcmk__str_casei)) {
	/* If this isn't a remote level broadcast, and we are not the
	* originator of the operation, we should not be receiving this msg. */
	crm_err("Received non-broadcast fencing result for operation %.8s "
	"we do not own (device %s targeting %s)",
	op->id, device, op->target);
	return rc;
	}

	if (pcmk_is_set(op->call_options, st_opt_topology)) {
	const char *device = crm_element_value(msg, F_STONITH_DEVICE);

	crm_notice("Action '%s' targeting %s using %s on behalf of %s@%s: %s "
	CRM_XS " rc=%d",
	op->action, op->target, device, op->client_name,
	op->originator, pcmk_strerror(rc), rc);

	/* We own the op, and it is complete. broadcast the result to all nodes
	* and notify our local clients. */
	if (op->state == st_done) {
	remote_op_done(op, msg, rc, FALSE);
	return rc;
	}

	if ((op->phase == 2) && (rc != pcmk_ok)) {
	/* A remapped "on" failed, but the node was already turned off
	* successfully, so ignore the error and continue.
	*/
	crm_warn("Ignoring %s 'on' failure (exit code %d) targeting %s "
	"after successful 'off'", device, rc, op->target);
	rc = pcmk_ok;
	}

	if (rc == pcmk_ok) {
	/* An operation completed successfully. Try another device if
	* necessary, otherwise mark the operation as done. */
	advance_topology_device_in_level(op, device, msg, rc);
	return rc;
	} else {
	/* This device failed, time to try another topology level. If no other
	* levels are available, mark this operation as failed and report results. */
	if (advance_topology_level(op, false) != pcmk_rc_ok) {
	op->state = st_failed;
	remote_op_done(op, msg, rc, FALSE);
	return rc;
	}
	}
	} else if (rc == pcmk_ok && op->devices == NULL) {
	crm_trace("All done for %s", op->target);

	op->state = st_done;
	remote_op_done(op, msg, rc, FALSE);
	return rc;
	} else if (rc == -ETIME && op->devices == NULL) {
	/* If the operation timed out don't bother retrying other peers. */
	op->state = st_failed;
	remote_op_done(op, msg, rc, FALSE);
	return rc;
	} else {
	/* fall-through and attempt other fencing action using another peer */
	}

	/* Retry on failure */
	crm_trace("Next for %s on behalf of %s@%s (rc was %d)", op->target, op->originator,
	op->client_name, rc);
	call_remote_stonith(op, NULL, rc);
	return rc;
	}

	gboolean
	stonith_check_fence_tolerance(int tolerance, const char target, const char action)
	{
	GHashTableIter iter;
	time_t now = time(NULL);
	remote_fencing_op_t *rop = NULL;

	if (tolerance <= 0 \|\| !stonith_remote_op_list \|\| target == NULL \|\|
	action == NULL) {
	return FALSE;
	}

	g_hash_table_iter_init(&iter, stonith_remote_op_list);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&rop)) {
	if (strcmp(rop->target, target) != 0) {
	continue;
	} else if (rop->state != st_done) {
	continue;
	/* We don't have to worry about remapped reboots here
	* because if state is done, any remapping has been undone
	*/
	} else if (strcmp(rop->action, action) != 0) {
	continue;
	} else if ((rop->completed + tolerance) < now) {
	continue;
	}

	crm_notice("Target %s was fenced (%s) less than %ds ago by %s on behalf of %s",
	target, action, tolerance, rop->delegate, rop->originator);
	return TRUE;
	}
	return FALSE;
	}
	diff --git a/daemons/fenced/pacemaker-fenced.c b/daemons/fenced/pacemaker-fenced.c
	index 96ad5a33ec..69c29a74bf 100644
	--- a/daemons/fenced/pacemaker-fenced.c
	+++ b/daemons/fenced/pacemaker-fenced.c
	@@ -1,1549 +1,1549 @@
	/*
	* Copyright 2009-2020 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <sys/param.h>
	#include <stdio.h>
	#include <sys/types.h>
	#include <sys/stat.h>
	#include <unistd.h>
	#include <sys/utsname.h>

	#include <stdlib.h>
	#include <errno.h>
	#include <fcntl.h>
	#include <inttypes.h> // PRIu32, PRIx32

	#include <crm/crm.h>
	#include <crm/msg_xml.h>
	#include <crm/common/ipc.h>
	#include <crm/common/ipc_internal.h>
	#include <crm/cluster/internal.h>

	#include <crm/stonith-ng.h>
	#include <crm/fencing/internal.h>
	#include <crm/common/xml.h>
	#include <crm/common/xml_internal.h>

	#include <crm/common/mainloop.h>

	#include <crm/cib/internal.h>
	#include <crm/pengine/status.h>
	#include <pacemaker-internal.h>

	#include <pacemaker-fenced.h>

	char *stonith_our_uname = NULL;
	char *stonith_our_uuid = NULL;
	long stonith_watchdog_timeout_ms = 0;

	static GMainLoop *mainloop = NULL;

	gboolean stand_alone = FALSE;
	static gboolean no_cib_connect = FALSE;
	static gboolean stonith_shutdown_flag = FALSE;

	static qb_ipcs_service_t *ipcs = NULL;
	static xmlNode *local_cib = NULL;
	static pe_working_set_t *fenced_data_set = NULL;

	static cib_t *cib_api = NULL;

	static void stonith_shutdown(int nsig);
	static void stonith_cleanup(void);

	static int32_t
	st_ipc_accept(qb_ipcs_connection_t * c, uid_t uid, gid_t gid)
	{
	if (stonith_shutdown_flag) {
	crm_info("Ignoring new client [%d] during shutdown",
	pcmk__client_pid(c));
	return -EPERM;
	}

	if (pcmk__new_client(c, uid, gid) == NULL) {
	return -EIO;
	}
	return 0;
	}

	/* Exit code means? */
	static int32_t
	st_ipc_dispatch(qb_ipcs_connection_t * qbc, void *data, size_t size)
	{
	uint32_t id = 0;
	uint32_t flags = 0;
	int call_options = 0;
	xmlNode *request = NULL;
	pcmk__client_t *c = pcmk__find_client(qbc);
	const char *op = NULL;

	if (c == NULL) {
	crm_info("Invalid client: %p", qbc);
	return 0;
	}

	request = pcmk__client_data2xml(c, data, &id, &flags);
	if (request == NULL) {
	pcmk__ipc_send_ack(c, id, flags, "nack", CRM_EX_PROTOCOL);
	return 0;
	}


	op = crm_element_value(request, F_CRM_TASK);
	if(pcmk__str_eq(op, CRM_OP_RM_NODE_CACHE, pcmk__str_casei)) {
	crm_xml_add(request, F_TYPE, T_STONITH_NG);
	crm_xml_add(request, F_STONITH_OPERATION, op);
	crm_xml_add(request, F_STONITH_CLIENTID, c->id);
	crm_xml_add(request, F_STONITH_CLIENTNAME, pcmk__client_name(c));
	crm_xml_add(request, F_STONITH_CLIENTNODE, stonith_our_uname);

	send_cluster_message(NULL, crm_msg_stonith_ng, request, FALSE);
	free_xml(request);
	return 0;
	}

	if (c->name == NULL) {
	const char *value = crm_element_value(request, F_STONITH_CLIENTNAME);

	if (value == NULL) {
	value = "unknown";
	}
	c->name = crm_strdup_printf("%s.%u", value, c->pid);
	}

	crm_element_value_int(request, F_STONITH_CALLOPTS, &call_options);
	crm_trace("Flags 0x%08" PRIx32 "/0x%08x for command %" PRIu32
	" from client %s", flags, call_options, id, pcmk__client_name(c));

	if (pcmk_is_set(call_options, st_opt_sync_call)) {
	CRM_ASSERT(flags & crm_ipc_client_response);
	CRM_LOG_ASSERT(c->request_id == 0); /* This means the client has two synchronous events in-flight */
	c->request_id = id; /* Reply only to the last one */
	}

	crm_xml_add(request, F_STONITH_CLIENTID, c->id);
	crm_xml_add(request, F_STONITH_CLIENTNAME, pcmk__client_name(c));
	crm_xml_add(request, F_STONITH_CLIENTNODE, stonith_our_uname);

	stonith_command(c, id, flags, request, NULL);

	free_xml(request);
	return 0;
	}

	/* Error code means? */
	static int32_t
	st_ipc_closed(qb_ipcs_connection_t * c)
	{
	pcmk__client_t *client = pcmk__find_client(c);

	if (client == NULL) {
	return 0;
	}

	crm_trace("Connection %p closed", c);
	pcmk__free_client(client);

	/* 0 means: yes, go ahead and destroy the connection */
	return 0;
	}

	static void
	st_ipc_destroy(qb_ipcs_connection_t * c)
	{
	crm_trace("Connection %p destroyed", c);
	st_ipc_closed(c);
	}

	static void
	stonith_peer_callback(xmlNode * msg, void *private_data)
	{
	const char *remote_peer = crm_element_value(msg, F_ORIG);
	const char *op = crm_element_value(msg, F_STONITH_OPERATION);

	if (pcmk__str_eq(op, "poke", pcmk__str_none)) {
	return;
	}

	crm_log_xml_trace(msg, "Peer[inbound]");
	stonith_command(NULL, 0, 0, msg, remote_peer);
	}

	#if SUPPORT_COROSYNC
	static void
	stonith_peer_ais_callback(cpg_handle_t handle,
	const struct cpg_name *groupName,
	uint32_t nodeid, uint32_t pid, void *msg, size_t msg_len)
	{
	uint32_t kind = 0;
	xmlNode *xml = NULL;
	const char *from = NULL;
	char *data = pcmk_message_common_cs(handle, nodeid, pid, msg, &kind, &from);

	if(data == NULL) {
	return;
	}
	if (kind == crm_class_cluster) {
	xml = string2xml(data);
	if (xml == NULL) {
	crm_err("Invalid XML: '%.120s'", data);
	free(data);
	return;
	}
	crm_xml_add(xml, F_ORIG, from);
	/* crm_xml_add_int(xml, F_SEQ, wrapper->id); */
	stonith_peer_callback(xml, NULL);
	}

	free_xml(xml);
	free(data);
	return;
	}

	static void
	stonith_peer_cs_destroy(gpointer user_data)
	{
	crm_crit("Lost connection to cluster layer, shutting down");
	stonith_shutdown(0);
	}
	#endif

	void
	do_local_reply(xmlNode * notify_src, const char *client_id, gboolean sync_reply, gboolean from_peer)
	{
	/* send callback to originating child */
	pcmk__client_t *client_obj = NULL;
	int local_rc = pcmk_rc_ok;

	crm_trace("Sending response");
	client_obj = pcmk__find_client_by_id(client_id);

	crm_trace("Sending callback to request originator");
	if (client_obj == NULL) {
	local_rc = EPROTO;
	crm_trace("No client to sent the response to. F_STONITH_CLIENTID not set.");

	} else {
	int rid = 0;

	if (sync_reply) {
	CRM_LOG_ASSERT(client_obj->request_id);

	rid = client_obj->request_id;
	client_obj->request_id = 0;

	crm_trace("Sending response %d to client %s%s",
	rid, pcmk__client_name(client_obj),
	(from_peer? " (originator of delegated request)" : ""));

	} else {
	crm_trace("Sending an event to client %s%s",
	pcmk__client_name(client_obj),
	(from_peer? " (originator of delegated request)" : ""));
	}

	local_rc = pcmk__ipc_send_xml(client_obj, rid, notify_src,
	(sync_reply? crm_ipc_flags_none
	: crm_ipc_server_event));
	}

	if ((local_rc != pcmk_rc_ok) && (client_obj != NULL)) {
	crm_warn("%s reply to client %s failed: %s",
	(sync_reply? "Synchronous" : "Asynchronous"),
	pcmk__client_name(client_obj), pcmk_rc_str(local_rc));
	}
	}

	uint64_t
	get_stonith_flag(const char *name)
	{
	if (pcmk__str_eq(name, T_STONITH_NOTIFY_FENCE, pcmk__str_casei)) {
	return st_callback_notify_fence;

	} else if (pcmk__str_eq(name, STONITH_OP_DEVICE_ADD, pcmk__str_casei)) {
	return st_callback_device_add;

	} else if (pcmk__str_eq(name, STONITH_OP_DEVICE_DEL, pcmk__str_casei)) {
	return st_callback_device_del;

	} else if (pcmk__str_eq(name, T_STONITH_NOTIFY_HISTORY, pcmk__str_casei)) {
	return st_callback_notify_history;

	} else if (pcmk__str_eq(name, T_STONITH_NOTIFY_HISTORY_SYNCED, pcmk__str_casei)) {
	return st_callback_notify_history_synced;

	}
	return st_callback_unknown;
	}

	static void
	stonith_notify_client(gpointer key, gpointer value, gpointer user_data)
	{

	xmlNode *update_msg = user_data;
	pcmk__client_t *client = value;
	const char *type = NULL;

	CRM_CHECK(client != NULL, return);
	CRM_CHECK(update_msg != NULL, return);

	type = crm_element_value(update_msg, F_SUBTYPE);
	CRM_CHECK(type != NULL, crm_log_xml_err(update_msg, "notify"); return);

	if (client->ipcs == NULL) {
	crm_trace("Skipping client with NULL channel");
	return;
	}

	if (pcmk_is_set(client->flags, get_stonith_flag(type))) {
	int rc = pcmk__ipc_send_xml(client, 0, update_msg,
	crm_ipc_server_event\|crm_ipc_server_error);

	if (rc != pcmk_rc_ok) {
	crm_warn("%s notification of client %s failed: %s "
	CRM_XS " id=%.8s rc=%d", type, pcmk__client_name(client),
	pcmk_rc_str(rc), client->id, rc);
	} else {
	crm_trace("Sent %s notification to client %s",
	type, pcmk__client_name(client));
	}
	}
	}

	void
	do_stonith_async_timeout_update(const char client_id, const char call_id, int timeout)
	{
	pcmk__client_t *client = NULL;
	xmlNode *notify_data = NULL;

	if (!timeout \|\| !call_id \|\| !client_id) {
	return;
	}

	client = pcmk__find_client_by_id(client_id);
	if (!client) {
	return;
	}

	notify_data = create_xml_node(NULL, T_STONITH_TIMEOUT_VALUE);
	crm_xml_add(notify_data, F_TYPE, T_STONITH_TIMEOUT_VALUE);
	crm_xml_add(notify_data, F_STONITH_CALLID, call_id);
	crm_xml_add_int(notify_data, F_STONITH_TIMEOUT, timeout);

	crm_trace("timeout update is %d for client %s and call id %s", timeout, client_id, call_id);

	if (client) {
	pcmk__ipc_send_xml(client, 0, notify_data, crm_ipc_server_event);
	}

	free_xml(notify_data);
	}

	void
	do_stonith_notify(int options, const char type, int result, xmlNode data)
	{
	/* TODO: Standardize the contents of data */
	xmlNode *update_msg = create_xml_node(NULL, "notify");

	CRM_CHECK(type != NULL,;);

	crm_xml_add(update_msg, F_TYPE, T_STONITH_NOTIFY);
	crm_xml_add(update_msg, F_SUBTYPE, type);
	crm_xml_add(update_msg, F_STONITH_OPERATION, type);
	crm_xml_add_int(update_msg, F_STONITH_RC, result);

	if (data != NULL) {
	add_message_xml(update_msg, F_STONITH_CALLDATA, data);
	}

	crm_trace("Notifying clients");
	pcmk__foreach_ipc_client(stonith_notify_client, update_msg);
	free_xml(update_msg);
	crm_trace("Notify complete");
	}

	static void
	do_stonith_notify_config(int options, const char *op, int rc,
	const char *desc, int active)
	{
	xmlNode *notify_data = create_xml_node(NULL, op);

	CRM_CHECK(notify_data != NULL, return);

	crm_xml_add(notify_data, F_STONITH_DEVICE, desc);
	crm_xml_add_int(notify_data, F_STONITH_ACTIVE, active);

	do_stonith_notify(options, op, rc, notify_data);
	free_xml(notify_data);
	}

	void
	do_stonith_notify_device(int options, const char op, int rc, const char desc)
	{
	do_stonith_notify_config(options, op, rc, desc, g_hash_table_size(device_list));
	}

	void
	do_stonith_notify_level(int options, const char op, int rc, const char desc)
	{
	do_stonith_notify_config(options, op, rc, desc, g_hash_table_size(topology));
	}

	static void
	topology_remove_helper(const char *node, int level)
	{
	int rc;
	char *desc = NULL;
	xmlNode *data = create_xml_node(NULL, XML_TAG_FENCING_LEVEL);

	crm_xml_add(data, F_STONITH_ORIGIN, __func__);
	crm_xml_add_int(data, XML_ATTR_STONITH_INDEX, level);
	crm_xml_add(data, XML_ATTR_STONITH_TARGET, node);

	rc = stonith_level_remove(data, &desc);
	do_stonith_notify_level(0, STONITH_OP_LEVEL_DEL, rc, desc);

	free_xml(data);
	free(desc);
	}

	static void
	remove_cib_device(xmlXPathObjectPtr xpathObj)
	{
	int max = numXpathResults(xpathObj), lpc = 0;

	for (lpc = 0; lpc < max; lpc++) {
	const char *rsc_id = NULL;
	const char *standard = NULL;
	xmlNode *match = getXpathResult(xpathObj, lpc);

	CRM_LOG_ASSERT(match != NULL);
	if(match != NULL) {
	standard = crm_element_value(match, XML_AGENT_ATTR_CLASS);
	}

	if (!pcmk__str_eq(standard, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_casei)) {
	continue;
	}

	rsc_id = crm_element_value(match, XML_ATTR_ID);

	stonith_device_remove(rsc_id, TRUE);
	}
	}

	static void
	handle_topology_change(xmlNode *match, bool remove)
	{
	int rc;
	char *desc = NULL;

	CRM_CHECK(match != NULL, return);
	crm_trace("Updating %s", ID(match));

	if(remove) {
	int index = 0;
	char *key = stonith_level_key(match, -1);

	crm_element_value_int(match, XML_ATTR_STONITH_INDEX, &index);
	topology_remove_helper(key, index);
	free(key);
	}

	rc = stonith_level_register(match, &desc);
	do_stonith_notify_level(0, STONITH_OP_LEVEL_ADD, rc, desc);

	free(desc);
	}

	static void
	remove_fencing_topology(xmlXPathObjectPtr xpathObj)
	{
	int max = numXpathResults(xpathObj), lpc = 0;

	for (lpc = 0; lpc < max; lpc++) {
	xmlNode *match = getXpathResult(xpathObj, lpc);

	CRM_LOG_ASSERT(match != NULL);
	if (match && crm_element_value(match, XML_DIFF_MARKER)) {
	/* Deletion */
	int index = 0;
	char *target = stonith_level_key(match, -1);

	crm_element_value_int(match, XML_ATTR_STONITH_INDEX, &index);
	if (target == NULL) {
	crm_err("Invalid fencing target in element %s", ID(match));

	} else if (index <= 0) {
	crm_err("Invalid level for %s in element %s", target, ID(match));

	} else {
	topology_remove_helper(target, index);
	}
	/* } else { Deal with modifications during the 'addition' stage */
	}
	}
	}

	static void
	register_fencing_topology(xmlXPathObjectPtr xpathObj)
	{
	int max = numXpathResults(xpathObj), lpc = 0;

	for (lpc = 0; lpc < max; lpc++) {
	xmlNode *match = getXpathResult(xpathObj, lpc);

	handle_topology_change(match, TRUE);
	}
	}

	/* Fencing
	<diff crm_feature_set="3.0.6">
	<diff-removed>
	<fencing-topology>
	<fencing-level id="f-p1.1" target="pcmk-1" index="1" devices="poison-pill" __crm_diff_marker__="removed:top"/>
	<fencing-level id="f-p1.2" target="pcmk-1" index="2" devices="power" __crm_diff_marker__="removed:top"/>
	<fencing-level devices="disk,network" id="f-p2.1"/>
	</fencing-topology>
	</diff-removed>
	<diff-added>
	<fencing-topology>
	<fencing-level id="f-p.1" target="pcmk-1" index="1" devices="poison-pill" __crm_diff_marker__="added:top"/>
	<fencing-level id="f-p2.1" target="pcmk-2" index="1" devices="disk,something"/>
	<fencing-level id="f-p3.1" target="pcmk-2" index="2" devices="power" __crm_diff_marker__="added:top"/>
	</fencing-topology>
	</diff-added>
	</diff>
	*/

	static void
	fencing_topology_init(void)
	{
	xmlXPathObjectPtr xpathObj = NULL;
	const char *xpath = "//" XML_TAG_FENCING_LEVEL;

	crm_trace("Full topology refresh");
	free_topology_list();
	init_topology_list();

	/* Grab everything */
	xpathObj = xpath_search(local_cib, xpath);
	register_fencing_topology(xpathObj);

	freeXpathObject(xpathObj);
	}

	#define rsc_name(x) x->clone_name?x->clone_name:x->id

	/*!
	* \internal
	* \brief Check whether our uname is in a resource's allowed node list
	*
	* \param[in] rsc Resource to check
	*
	* \return Pointer to node object if found, NULL otherwise
	*/
	static pe_node_t *
	our_node_allowed_for(pe_resource_t *rsc)
	{
	GHashTableIter iter;
	pe_node_t *node = NULL;

	if (rsc && stonith_our_uname) {
	g_hash_table_iter_init(&iter, rsc->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
	if (node && strcmp(node->details->uname, stonith_our_uname) == 0) {
	break;
	}
	node = NULL;
	}
	}
	return node;
	}

	/*!
	* \internal
	* \brief If a resource or any of its children are STONITH devices, update their
	* definitions given a cluster working set.
	*
	* \param[in] rsc Resource to check
	* \param[in] data_set Cluster working set with device information
	*/
	static void cib_device_update(pe_resource_t rsc, pe_working_set_t data_set)
	{
	pe_node_t *node = NULL;
	const char *value = NULL;
	const char *rclass = NULL;
	pe_node_t *parent = NULL;
	gboolean remove = TRUE;

	/* If this is a complex resource, check children rather than this resource itself.
	* TODO: Mark each installed device and remove if untouched when this process finishes.
	*/
	if(rsc->children) {
	GListPtr gIter = NULL;
	for (gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
	cib_device_update(gIter->data, data_set);
	if(pe_rsc_is_clone(rsc)) {
	crm_trace("Only processing one copy of the clone %s", rsc->id);
	break;
	}
	}
	return;
	}

	/* We only care about STONITH resources. */
	rclass = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
	if (!pcmk__str_eq(rclass, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_casei)) {
	return;
	}

	/* If this STONITH resource is disabled, just remove it. */
	if (pe__resource_is_disabled(rsc)) {
	crm_info("Device %s has been disabled", rsc->id);
	goto update_done;
	}

	/* Check whether our node is allowed for this resource (and its parent if in a group) */
	node = our_node_allowed_for(rsc);
	if (rsc->parent && (rsc->parent->variant == pe_group)) {
	parent = our_node_allowed_for(rsc->parent);
	}

	if(node == NULL) {
	/* Our node is disallowed, so remove the device */
	GHashTableIter iter;

	crm_info("Device %s has been disabled on %s: unknown", rsc->id, stonith_our_uname);
	g_hash_table_iter_init(&iter, rsc->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
	crm_trace("Available: %s = %d", node->details->uname, node->weight);
	}

	goto update_done;

	} else if(node->weight < 0 \|\| (parent && parent->weight < 0)) {
	/* Our node (or its group) is disallowed by score, so remove the device */
	char *score = score2char((node->weight < 0) ? node->weight : parent->weight);

	crm_info("Device %s has been disabled on %s: score=%s", rsc->id, stonith_our_uname, score);
	free(score);

	goto update_done;

	} else {
	/* Our node is allowed, so update the device information */
	int rc;
	xmlNode *data;
	GHashTableIter gIter;
	stonith_key_value_t *params = NULL;

	const char *name = NULL;
	const char *agent = crm_element_value(rsc->xml, XML_EXPR_ATTR_TYPE);
	const char *rsc_provides = NULL;

	crm_debug("Device %s is allowed on %s: score=%d", rsc->id, stonith_our_uname, node->weight);
	get_rsc_attributes(rsc->parameters, rsc, node, data_set);
	get_meta_attributes(rsc->meta, rsc, node, data_set);

	rsc_provides = g_hash_table_lookup(rsc->meta, PCMK_STONITH_PROVIDES);

	g_hash_table_iter_init(&gIter, rsc->parameters);
	while (g_hash_table_iter_next(&gIter, (gpointer ) & name, (gpointer ) & value)) {
	if (!name \|\| !value) {
	continue;
	}
	params = stonith_key_value_add(params, name, value);
	crm_trace(" %s=%s", name, value);
	}

	remove = FALSE;
	data = create_device_registration_xml(rsc_name(rsc), st_namespace_any,
	agent, params, rsc_provides);
	stonith_key_value_freeall(params, 1, 1);
	rc = stonith_device_register(data, NULL, TRUE);
	CRM_ASSERT(rc == pcmk_ok);
	free_xml(data);
	}

	update_done:

	if(remove && g_hash_table_lookup(device_list, rsc_name(rsc))) {
	stonith_device_remove(rsc_name(rsc), TRUE);
	}
	}

	/*!
	* \internal
	* \brief Update all STONITH device definitions based on current CIB
	*/
	static void
	cib_devices_update(void)
	{
	GListPtr gIter = NULL;

	crm_info("Updating devices to version %s.%s.%s",
	crm_element_value(local_cib, XML_ATTR_GENERATION_ADMIN),
	crm_element_value(local_cib, XML_ATTR_GENERATION),
	crm_element_value(local_cib, XML_ATTR_NUMUPDATES));

	CRM_ASSERT(fenced_data_set != NULL);
	fenced_data_set->input = local_cib;
	fenced_data_set->now = crm_time_new(NULL);
	fenced_data_set->localhost = stonith_our_uname;
	pe__set_working_set_flags(fenced_data_set, pe_flag_quick_location);

	cluster_status(fenced_data_set);
	pcmk__schedule_actions(fenced_data_set, NULL, NULL);

	for (gIter = fenced_data_set->resources; gIter != NULL; gIter = gIter->next) {
	cib_device_update(gIter->data, fenced_data_set);
	}
	fenced_data_set->input = NULL; // Wasn't a copy, so don't let API free it
	pe_reset_working_set(fenced_data_set);
	}

	static void
	update_cib_stonith_devices_v2(const char event, xmlNode msg)
	{
	xmlNode *change = NULL;
	char *reason = NULL;
	bool needs_update = FALSE;
	xmlNode *patchset = get_message_xml(msg, F_CIB_UPDATE_RESULT);

	for (change = pcmk__xml_first_child(patchset); change != NULL;
	change = pcmk__xml_next(change)) {
	const char *op = crm_element_value(change, XML_DIFF_OP);
	const char *xpath = crm_element_value(change, XML_DIFF_PATH);
	const char *shortpath = NULL;

	if ((op == NULL) \|\|
	(strcmp(op, "move") == 0) \|\|
	strstr(xpath, "/"XML_CIB_TAG_STATUS)) {
	continue;
	} else if (pcmk__str_eq(op, "delete", pcmk__str_casei) && strstr(xpath, "/"XML_CIB_TAG_RESOURCE)) {
	const char *rsc_id = NULL;
	char *search = NULL;
	char *mutable = NULL;

	if (strstr(xpath, XML_TAG_ATTR_SETS) \|\|
	strstr(xpath, XML_TAG_META_SETS)) {
	needs_update = TRUE;
	reason = strdup("(meta) attribute deleted from resource");
	break;
	}
	mutable = strdup(xpath);
	rsc_id = strstr(mutable, "primitive[@id=\'");
	if (rsc_id != NULL) {
	rsc_id += strlen("primitive[@id=\'");
	search = strchr(rsc_id, '\'');
	}
	if (search != NULL) {
	*search = 0;
	stonith_device_remove(rsc_id, TRUE);
	} else {
	crm_warn("Ignoring malformed CIB update (resource deletion)");
	}
	free(mutable);

	} else if (strstr(xpath, "/"XML_CIB_TAG_RESOURCES) \|\|
	strstr(xpath, "/"XML_CIB_TAG_CONSTRAINTS) \|\|
	strstr(xpath, "/"XML_CIB_TAG_RSCCONFIG)) {
	shortpath = strrchr(xpath, '/'); CRM_ASSERT(shortpath);
	reason = crm_strdup_printf("%s %s", op, shortpath+1);
	needs_update = TRUE;
	break;
	}
	}

	if(needs_update) {
	crm_info("Updating device list from CIB: %s", reason);
	cib_devices_update();
	} else {
	crm_trace("No updates for device list found in CIB");
	}
	free(reason);
	}


	static void
	update_cib_stonith_devices_v1(const char event, xmlNode msg)
	{
	const char *reason = "none";
	gboolean needs_update = FALSE;
	xmlXPathObjectPtr xpath_obj = NULL;

	/* process new constraints */
	xpath_obj = xpath_search(msg, "//" F_CIB_UPDATE_RESULT "//" XML_CONS_TAG_RSC_LOCATION);
	if (numXpathResults(xpath_obj) > 0) {
	int max = numXpathResults(xpath_obj), lpc = 0;

	/* Safest and simplest to always recompute */
	needs_update = TRUE;
	reason = "new location constraint";

	for (lpc = 0; lpc < max; lpc++) {
	xmlNode *match = getXpathResult(xpath_obj, lpc);

	crm_log_xml_trace(match, "new constraint");
	}
	}
	freeXpathObject(xpath_obj);

	/* process deletions */
	xpath_obj = xpath_search(msg, "//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_REMOVED "//" XML_CIB_TAG_RESOURCE);
	if (numXpathResults(xpath_obj) > 0) {
	remove_cib_device(xpath_obj);
	}
	freeXpathObject(xpath_obj);

	/* process additions */
	xpath_obj = xpath_search(msg, "//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_ADDED "//" XML_CIB_TAG_RESOURCE);
	if (numXpathResults(xpath_obj) > 0) {
	int max = numXpathResults(xpath_obj), lpc = 0;

	for (lpc = 0; lpc < max; lpc++) {
	const char *rsc_id = NULL;
	const char *standard = NULL;
	xmlNode *match = getXpathResult(xpath_obj, lpc);

	rsc_id = crm_element_value(match, XML_ATTR_ID);
	standard = crm_element_value(match, XML_AGENT_ATTR_CLASS);

	if (!pcmk__str_eq(standard, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_casei)) {
	continue;
	}

	crm_trace("Fencing resource %s was added or modified", rsc_id);
	reason = "new resource";
	needs_update = TRUE;
	}
	}
	freeXpathObject(xpath_obj);

	if(needs_update) {
	crm_info("Updating device list from CIB: %s", reason);
	cib_devices_update();
	}
	}

	static void
	update_cib_stonith_devices(const char event, xmlNode msg)
	{
	int format = 1;
	xmlNode *patchset = get_message_xml(msg, F_CIB_UPDATE_RESULT);

	CRM_ASSERT(patchset);
	crm_element_value_int(patchset, "format", &format);
	switch(format) {
	case 1:
	update_cib_stonith_devices_v1(event, msg);
	break;
	case 2:
	update_cib_stonith_devices_v2(event, msg);
	break;
	default:
	crm_warn("Unknown patch format: %d", format);
	}
	}

	/* Needs to hold node name + attribute name + attribute value + 75 */
	#define XPATH_MAX 512

	/*!
	* \internal
	* \brief Check whether a node has a specific attribute name/value
	*
	* \param[in] node Name of node to check
	* \param[in] name Name of an attribute to look for
	* \param[in] value The value the named attribute needs to be set to in order to be considered a match
	*
	* \return TRUE if the locally cached CIB has the specified node attribute
	*/
	gboolean
	node_has_attr(const char node, const char name, const char *value)
	{
	char xpath[XPATH_MAX];
	xmlNode *match;
	int n;

	CRM_CHECK(local_cib != NULL, return FALSE);

	/* Search for the node's attributes in the CIB. While the schema allows
	* multiple sets of instance attributes, and allows instance attributes to
	* use id-ref to reference values elsewhere, that is intended for resources,
	* so we ignore that here.
	*/
	n = snprintf(xpath, XPATH_MAX, "//" XML_CIB_TAG_NODES
	"/" XML_CIB_TAG_NODE "[@uname='%s']/" XML_TAG_ATTR_SETS
	"/" XML_CIB_TAG_NVPAIR "[@name='%s' and @value='%s']",
	node, name, value);
	match = get_xpath_object(xpath, local_cib, LOG_NEVER);

	CRM_CHECK(n < XPATH_MAX, return FALSE);
	return (match != NULL);
	}

	static void
	update_fencing_topology(const char event, xmlNode msg)
	{
	int format = 1;
	const char *xpath;
	xmlXPathObjectPtr xpathObj = NULL;
	xmlNode *patchset = get_message_xml(msg, F_CIB_UPDATE_RESULT);

	CRM_ASSERT(patchset);
	crm_element_value_int(patchset, "format", &format);

	if(format == 1) {
	/* Process deletions (only) */
	xpath = "//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_REMOVED "//" XML_TAG_FENCING_LEVEL;
	xpathObj = xpath_search(msg, xpath);

	remove_fencing_topology(xpathObj);
	freeXpathObject(xpathObj);

	/* Process additions and changes */
	xpath = "//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_ADDED "//" XML_TAG_FENCING_LEVEL;
	xpathObj = xpath_search(msg, xpath);

	register_fencing_topology(xpathObj);
	freeXpathObject(xpathObj);

	} else if(format == 2) {
	xmlNode *change = NULL;
	int add[] = { 0, 0, 0 };
	int del[] = { 0, 0, 0 };

	xml_patch_versions(patchset, add, del);

	for (change = pcmk__xml_first_child(patchset); change != NULL;
	change = pcmk__xml_next(change)) {
	const char *op = crm_element_value(change, XML_DIFF_OP);
	const char *xpath = crm_element_value(change, XML_DIFF_PATH);

	if(op == NULL) {
	continue;

	} else if(strstr(xpath, "/" XML_TAG_FENCING_LEVEL) != NULL) {
	/* Change to a specific entry */

	crm_trace("Handling %s operation %d.%d.%d for %s", op, add[0], add[1], add[2], xpath);
	if(strcmp(op, "move") == 0) {
	continue;

	} else if(strcmp(op, "create") == 0) {
	handle_topology_change(change->children, FALSE);

	} else if(strcmp(op, "modify") == 0) {
	xmlNode *match = first_named_child(change, XML_DIFF_RESULT);

	if(match) {
	handle_topology_change(match->children, TRUE);
	}

	} else if(strcmp(op, "delete") == 0) {
	/* Nuclear option, all we have is the path and an id... not enough to remove a specific entry */
	crm_info("Re-initializing fencing topology after %s operation %d.%d.%d for %s",
	op, add[0], add[1], add[2], xpath);
	fencing_topology_init();
	return;
	}

	} else if (strstr(xpath, "/" XML_TAG_FENCING_TOPOLOGY) != NULL) {
	/* Change to the topology in general */
	crm_info("Re-initializing fencing topology after top-level %s operation %d.%d.%d for %s",
	op, add[0], add[1], add[2], xpath);
	fencing_topology_init();
	return;

	} else if (strstr(xpath, "/" XML_CIB_TAG_CONFIGURATION)) {
	/* Changes to the whole config section, possibly including the topology as a whild */
	if(first_named_child(change, XML_TAG_FENCING_TOPOLOGY) == NULL) {
	crm_trace("Nothing for us in %s operation %d.%d.%d for %s.",
	op, add[0], add[1], add[2], xpath);

	} else if(strcmp(op, "delete") == 0 \|\| strcmp(op, "create") == 0) {
	crm_info("Re-initializing fencing topology after top-level %s operation %d.%d.%d for %s.",
	op, add[0], add[1], add[2], xpath);
	fencing_topology_init();
	return;
	}

	} else {
	crm_trace("Nothing for us in %s operation %d.%d.%d for %s",
	op, add[0], add[1], add[2], xpath);
	}
	}

	} else {
	crm_warn("Unknown patch format: %d", format);
	}
	}
	static bool have_cib_devices = FALSE;

	static void
	update_cib_cache_cb(const char event, xmlNode msg)
	{
	int rc = pcmk_ok;
	xmlNode *stonith_enabled_xml = NULL;
	xmlNode *stonith_watchdog_xml = NULL;
	const char *stonith_enabled_s = NULL;
	static gboolean stonith_enabled_saved = TRUE;

	if(!have_cib_devices) {
	crm_trace("Skipping updates until we get a full dump");
	return;

	} else if(msg == NULL) {
	crm_trace("Missing %s update", event);
	return;
	}

	/* Maintain a local copy of the CIB so that we have full access
	* to device definitions, location constraints, and node attributes
	*/
	if (local_cib != NULL) {
	int rc = pcmk_ok;
	xmlNode *patchset = NULL;

	crm_element_value_int(msg, F_CIB_RC, &rc);
	if (rc != pcmk_ok) {
	return;
	}

	patchset = get_message_xml(msg, F_CIB_UPDATE_RESULT);
	xml_log_patchset(LOG_TRACE, "Config update", patchset);
	rc = xml_apply_patchset(local_cib, patchset, TRUE);
	switch (rc) {
	case pcmk_ok:
	case -pcmk_err_old_data:
	break;
	case -pcmk_err_diff_resync:
	case -pcmk_err_diff_failed:
	crm_notice("[%s] Patch aborted: %s (%d)", event, pcmk_strerror(rc), rc);
	free_xml(local_cib);
	local_cib = NULL;
	break;
	default:
	crm_warn("[%s] ABORTED: %s (%d)", event, pcmk_strerror(rc), rc);
	free_xml(local_cib);
	local_cib = NULL;
	}
	}

	if (local_cib == NULL) {
	crm_trace("Re-requesting full CIB");
	rc = cib_api->cmds->query(cib_api, NULL, &local_cib, cib_scope_local \| cib_sync_call);
	if(rc != pcmk_ok) {
	crm_err("Couldn't retrieve the CIB: %s (%d)", pcmk_strerror(rc), rc);
	return;
	}
	CRM_ASSERT(local_cib != NULL);
	stonith_enabled_saved = FALSE; /* Trigger a full refresh below */
	}

	- crm_peer_caches_refresh(local_cib);
	+ pcmk__refresh_node_caches_from_cib(local_cib);

	stonith_enabled_xml = get_xpath_object("//nvpair[@name='stonith-enabled']",
	local_cib, LOG_NEVER);
	if (stonith_enabled_xml) {
	stonith_enabled_s = crm_element_value(stonith_enabled_xml, XML_NVPAIR_ATTR_VALUE);
	}

	if (stonith_enabled_s == NULL \|\| crm_is_true(stonith_enabled_s)) {
	long timeout_ms = 0;
	const char *value = NULL;

	stonith_watchdog_xml = get_xpath_object("//nvpair[@name='stonith-watchdog-timeout']",
	local_cib, LOG_NEVER);
	if (stonith_watchdog_xml) {
	value = crm_element_value(stonith_watchdog_xml, XML_NVPAIR_ATTR_VALUE);
	}

	if(value) {
	timeout_ms = crm_get_msec(value);
	}
	if (timeout_ms < 0) {
	timeout_ms = pcmk__auto_watchdog_timeout();
	}

	if(timeout_ms != stonith_watchdog_timeout_ms) {
	crm_notice("New watchdog timeout %lds (was %lds)", timeout_ms/1000, stonith_watchdog_timeout_ms/1000);
	stonith_watchdog_timeout_ms = timeout_ms;
	}

	} else {
	stonith_watchdog_timeout_ms = 0;
	}

	if (stonith_enabled_s && crm_is_true(stonith_enabled_s) == FALSE) {
	crm_trace("Ignoring CIB updates while fencing is disabled");
	stonith_enabled_saved = FALSE;
	return;

	} else if (stonith_enabled_saved == FALSE) {
	crm_info("Updating fencing device and topology lists "
	"now that fencing is enabled");
	stonith_enabled_saved = TRUE;
	fencing_topology_init();
	cib_devices_update();

	} else {
	update_fencing_topology(event, msg);
	update_cib_stonith_devices(event, msg);
	}
	}

	static void
	init_cib_cache_cb(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
	{
	crm_info("Updating device list from CIB");
	have_cib_devices = TRUE;
	local_cib = copy_xml(output);

	- crm_peer_caches_refresh(local_cib);
	+ pcmk__refresh_node_caches_from_cib(local_cib);

	fencing_topology_init();
	cib_devices_update();
	}

	static void
	stonith_shutdown(int nsig)
	{
	crm_info("Terminating with %d clients", pcmk__ipc_client_count());
	stonith_shutdown_flag = TRUE;
	if (mainloop != NULL && g_main_loop_is_running(mainloop)) {
	g_main_loop_quit(mainloop);
	} else {
	stonith_cleanup();
	crm_exit(CRM_EX_OK);
	}
	}

	static void
	cib_connection_destroy(gpointer user_data)
	{
	if (stonith_shutdown_flag) {
	crm_info("Connection to the CIB manager closed");
	return;
	} else {
	crm_crit("Lost connection to the CIB manager, shutting down");
	}
	if (cib_api) {
	cib_api->cmds->signoff(cib_api);
	}
	stonith_shutdown(0);
	}

	static void
	stonith_cleanup(void)
	{
	if (cib_api) {
	cib_api->cmds->del_notify_callback(cib_api, T_CIB_DIFF_NOTIFY, update_cib_cache_cb);
	cib_api->cmds->signoff(cib_api);
	}

	if (ipcs) {
	qb_ipcs_destroy(ipcs);
	}

	crm_peer_destroy();
	pcmk__client_cleanup();
	free_stonith_remote_op_list();
	free_topology_list();
	free_device_list();
	free_metadata_cache();

	free(stonith_our_uname);
	stonith_our_uname = NULL;

	free_xml(local_cib);
	local_cib = NULL;
	}

	static pcmk__cli_option_t long_options[] = {
	// long option, argument type, storage, short option, description, flags
	{
	"stand-alone", no_argument, 0, 's',
	NULL, pcmk__option_default
	},
	{
	"stand-alone-w-cpg", no_argument, 0, 'c',
	NULL, pcmk__option_default
	},
	{
	"logfile", required_argument, 0, 'l',
	NULL, pcmk__option_default
	},
	{
	"verbose", no_argument, 0, 'V',
	NULL, pcmk__option_default
	},
	{
	"version", no_argument, 0, '$',
	NULL, pcmk__option_default
	},
	{
	"help", no_argument, 0, '?',
	NULL, pcmk__option_default
	},
	{ 0, 0, 0, 0 }
	};

	static void
	setup_cib(void)
	{
	int rc, retries = 0;

	cib_api = cib_new();
	if (cib_api == NULL) {
	crm_err("No connection to the CIB manager");
	return;
	}

	do {
	sleep(retries);
	rc = cib_api->cmds->signon(cib_api, CRM_SYSTEM_STONITHD, cib_command);
	} while (rc == -ENOTCONN && ++retries < 5);

	if (rc != pcmk_ok) {
	crm_err("Could not connect to the CIB manager: %s (%d)", pcmk_strerror(rc), rc);

	} else if (pcmk_ok !=
	cib_api->cmds->add_notify_callback(cib_api, T_CIB_DIFF_NOTIFY, update_cib_cache_cb)) {
	crm_err("Could not set CIB notification callback");

	} else {
	rc = cib_api->cmds->query(cib_api, NULL, NULL, cib_scope_local);
	cib_api->cmds->register_callback(cib_api, rc, 120, FALSE, NULL, "init_cib_cache_cb",
	init_cib_cache_cb);
	cib_api->cmds->set_connection_dnotify(cib_api, cib_connection_destroy);
	crm_info("Watching for fencing topology changes");
	}
	}

	struct qb_ipcs_service_handlers ipc_callbacks = {
	.connection_accept = st_ipc_accept,
	.connection_created = NULL,
	.msg_process = st_ipc_dispatch,
	.connection_closed = st_ipc_closed,
	.connection_destroyed = st_ipc_destroy
	};

	/*!
	* \internal
	* \brief Callback for peer status changes
	*
	* \param[in] type What changed
	* \param[in] node What peer had the change
	* \param[in] data Previous value of what changed
	*/
	static void
	st_peer_update_callback(enum crm_status_type type, crm_node_t * node, const void *data)
	{
	if ((type != crm_status_processes)
	&& !pcmk_is_set(node->flags, crm_remote_node)) {
	/*
	* This is a hack until we can send to a nodeid and/or we fix node name lookups
	* These messages are ignored in stonith_peer_callback()
	*/
	xmlNode *query = create_xml_node(NULL, "stonith_command");

	crm_xml_add(query, F_XML_TAGNAME, "stonith_command");
	crm_xml_add(query, F_TYPE, T_STONITH_NG);
	crm_xml_add(query, F_STONITH_OPERATION, "poke");

	crm_debug("Broadcasting our uname because of node %u", node->id);
	send_cluster_message(NULL, crm_msg_stonith_ng, query, FALSE);

	free_xml(query);
	}
	}

	int
	main(int argc, char **argv)
	{
	int flag;
	int lpc = 0;
	int argerr = 0;
	int option_index = 0;
	crm_cluster_t cluster;
	const char *actions[] = { "reboot", "off", "on", "list", "monitor", "status" };
	crm_ipc_t *old_instance = NULL;

	crm_log_preinit(NULL, argc, argv);
	pcmk__set_cli_options(NULL, "[options]", long_options,
	"daemon for executing fencing devices in a "
	"Pacemaker cluster");

	while (1) {
	flag = pcmk__next_cli_option(argc, argv, &option_index, NULL);
	if (flag == -1) {
	break;
	}

	switch (flag) {
	case 'V':
	crm_bump_log_level(argc, argv);
	break;
	case 'l':
	crm_add_logfile(optarg);
	break;
	case 's':
	stand_alone = TRUE;
	break;
	case 'c':
	stand_alone = FALSE;
	no_cib_connect = TRUE;
	break;
	case '$':
	case '?':
	pcmk__cli_help(flag, CRM_EX_OK);
	break;
	default:
	++argerr;
	break;
	}
	}

	if (argc - optind == 1 && pcmk__str_eq("metadata", argv[optind], pcmk__str_casei)) {
	printf("<?xml version=\"1.0\"?><!DOCTYPE resource-agent SYSTEM \"ra-api-1.dtd\">\n");
	printf("<resource-agent name=\"pacemaker-fenced\">\n");
	printf(" <version>1.0</version>\n");
	printf(" <longdesc lang=\"en\">Instance attributes available for all \"stonith\"-class resources"
	" and used by Pacemaker's fence daemon, formerly known as stonithd</longdesc>\n");
	printf(" <shortdesc lang=\"en\">Instance attributes available for all \"stonith\"-class resources</shortdesc>\n");
	printf(" <parameters>\n");

	#if 0
	// priority is not implemented yet
	printf(" <parameter name=\"priority\" unique=\"0\">\n");
	printf(" <shortdesc lang=\"en\">Devices that are not in a topology "
	"are tried in order of highest to lowest integer priority</shortdesc>\n");
	printf(" <content type=\"integer\" default=\"0\"/>\n");
	printf(" </parameter>\n");
	#endif

	printf(" <parameter name=\"%s\" unique=\"0\">\n",
	PCMK_STONITH_HOST_ARGUMENT);
	printf
	(" <shortdesc lang=\"en\">Advanced use only: An alternate parameter to supply instead of 'port'</shortdesc>\n");
	printf
	(" <longdesc lang=\"en\">Some devices do not support the standard 'port' parameter or may provide additional ones.\n"
	"Use this to specify an alternate, device-specific, parameter that should indicate the machine to be fenced.\n"
	"A value of 'none' can be used to tell the cluster not to supply any additional parameters.\n"
	" </longdesc>\n");
	printf(" <content type=\"string\" default=\"port\"/>\n");
	printf(" </parameter>\n");

	printf(" <parameter name=\"%s\" unique=\"0\">\n",
	PCMK_STONITH_HOST_MAP);
	printf
	(" <shortdesc lang=\"en\">A mapping of host names to ports numbers for devices that do not support host names.</shortdesc>\n");
	printf
	(" <longdesc lang=\"en\">Eg. node1:1;node2:2,3 would tell the cluster to use port 1 for node1 and ports 2 and 3 for node2</longdesc>\n");
	printf(" <content type=\"string\" default=\"\"/>\n");
	printf(" </parameter>\n");

	printf(" <parameter name=\"%s\" unique=\"0\">\n",
	PCMK_STONITH_HOST_LIST);
	printf(" <shortdesc lang=\"en\">A list of machines controlled by "
	"this device (Optional unless %s=static-list).</shortdesc>\n",
	PCMK_STONITH_HOST_CHECK);
	printf(" <content type=\"string\" default=\"\"/>\n");
	printf(" </parameter>\n");

	printf(" <parameter name=\"%s\" unique=\"0\">\n",
	PCMK_STONITH_HOST_CHECK);
	printf
	(" <shortdesc lang=\"en\">How to determine which machines are controlled by the device.</shortdesc>\n");
	printf(" <longdesc lang=\"en\">Allowed values: dynamic-list "
	"(query the device via the 'list' command), static-list "
	"(check the " PCMK_STONITH_HOST_LIST " attribute), status "
	"(query the device via the 'status' command), none (assume "
	"every device can fence every machine)</longdesc>\n");
	printf(" <content type=\"string\" default=\"dynamic-list\"/>\n");
	printf(" </parameter>\n");

	printf(" <parameter name=\"%s\" unique=\"0\">\n",
	PCMK_STONITH_DELAY_MAX);
	printf(" <shortdesc lang=\"en\">Enable a delay of no more than the "
	"time specified before executing fencing actions. Pacemaker "
	"derives the overall delay by taking the value of "
	PCMK_STONITH_DELAY_BASE " and adding a random delay value such "
	"that the sum is kept below this maximum.</shortdesc>\n");
	printf(" <longdesc lang=\"en\">This prevents double fencing when "
	"using slow devices such as sbd.\nUse this to enable a random "
	"delay for fencing actions.\nThe overall delay is derived from "
	"this random delay value adding a static delay so that the sum "
	"is kept below the maximum delay.</longdesc>\n");
	printf(" <content type=\"time\" default=\"0s\"/>\n");
	printf(" </parameter>\n");

	printf(" <parameter name=\"%s\" unique=\"0\">\n",
	PCMK_STONITH_DELAY_BASE);
	printf(" <shortdesc lang=\"en\">Enable a base delay for "
	"fencing actions and specify base delay value.</shortdesc>\n");
	printf(" <longdesc lang=\"en\">This prevents double fencing when "
	"different delays are configured on the nodes.\nUse this to "
	"enable a static delay for fencing actions.\nThe overall delay "
	"is derived from a random delay value adding this static delay "
	"so that the sum is kept below the maximum delay.</longdesc>\n");
	printf(" <content type=\"time\" default=\"0s\"/>\n");
	printf(" </parameter>\n");

	printf(" <parameter name=\"%s\" unique=\"0\">\n",
	PCMK_STONITH_ACTION_LIMIT);
	printf
	(" <shortdesc lang=\"en\">The maximum number of actions can be performed in parallel on this device</shortdesc>\n");
	printf
	(" <longdesc lang=\"en\">Cluster property concurrent-fencing=true needs to be configured first.\n"
	"Then use this to specify the maximum number of actions can be performed in parallel on this device. -1 is unlimited.</longdesc>\n");
	printf(" <content type=\"integer\" default=\"1\"/>\n");
	printf(" </parameter>\n");


	for (lpc = 0; lpc < DIMOF(actions); lpc++) {
	printf(" <parameter name=\"pcmk_%s_action\" unique=\"0\">\n", actions[lpc]);
	printf
	(" <shortdesc lang=\"en\">Advanced use only: An alternate command to run instead of '%s'</shortdesc>\n",
	actions[lpc]);
	printf
	(" <longdesc lang=\"en\">Some devices do not support the standard commands or may provide additional ones.\n"
	"Use this to specify an alternate, device-specific, command that implements the '%s' action.</longdesc>\n",
	actions[lpc]);
	printf(" <content type=\"string\" default=\"%s\"/>\n", actions[lpc]);
	printf(" </parameter>\n");

	printf(" <parameter name=\"pcmk_%s_timeout\" unique=\"0\">\n", actions[lpc]);
	printf
	(" <shortdesc lang=\"en\">Advanced use only: Specify an alternate timeout to use for %s actions instead of stonith-timeout</shortdesc>\n",
	actions[lpc]);
	printf
	(" <longdesc lang=\"en\">Some devices need much more/less time to complete than normal.\n"
	"Use this to specify an alternate, device-specific, timeout for '%s' actions.</longdesc>\n",
	actions[lpc]);
	printf(" <content type=\"time\" default=\"60s\"/>\n");
	printf(" </parameter>\n");

	printf(" <parameter name=\"pcmk_%s_retries\" unique=\"0\">\n", actions[lpc]);
	printf
	(" <shortdesc lang=\"en\">Advanced use only: The maximum number of times to retry the '%s' command within the timeout period</shortdesc>\n",
	actions[lpc]);
	printf(" <longdesc lang=\"en\">Some devices do not support multiple connections."
	" Operations may 'fail' if the device is busy with another task so Pacemaker will automatically retry the operation, if there is time remaining."
	" Use this option to alter the number of times Pacemaker retries '%s' actions before giving up."
	"</longdesc>\n", actions[lpc]);
	printf(" <content type=\"integer\" default=\"2\"/>\n");
	printf(" </parameter>\n");
	}

	printf(" </parameters>\n");
	printf("</resource-agent>\n");
	return CRM_EX_OK;
	}

	if (optind != argc) {
	++argerr;
	}

	if (argerr) {
	pcmk__cli_help('?', CRM_EX_USAGE);
	}

	crm_log_init(NULL, LOG_INFO, TRUE, FALSE, argc, argv, FALSE);

	crm_notice("Starting Pacemaker fencer");

	old_instance = crm_ipc_new("stonith-ng", 0);
	if (crm_ipc_connect(old_instance)) {
	/* IPC end-point already up */
	crm_ipc_close(old_instance);
	crm_ipc_destroy(old_instance);
	crm_err("pacemaker-fenced is already active, aborting startup");
	crm_exit(CRM_EX_OK);
	} else {
	/* not up or not authentic, we'll proceed either way */
	crm_ipc_destroy(old_instance);
	old_instance = NULL;
	}

	mainloop_add_signal(SIGTERM, stonith_shutdown);

	crm_peer_init();

	fenced_data_set = pe_new_working_set();
	CRM_ASSERT(fenced_data_set != NULL);
	pe__set_working_set_flags(fenced_data_set,
	pe_flag_no_counts\|pe_flag_no_compat);

	if (stand_alone == FALSE) {

	if (is_corosync_cluster()) {
	#if SUPPORT_COROSYNC
	cluster.destroy = stonith_peer_cs_destroy;
	cluster.cpg.cpg_deliver_fn = stonith_peer_ais_callback;
	cluster.cpg.cpg_confchg_fn = pcmk_cpg_membership;
	#endif
	}

	crm_set_status_callback(&st_peer_update_callback);

	if (crm_cluster_connect(&cluster) == FALSE) {
	crm_crit("Cannot sign in to the cluster... terminating");
	crm_exit(CRM_EX_FATAL);
	}
	stonith_our_uname = cluster.uname;
	stonith_our_uuid = cluster.uuid;

	if (no_cib_connect == FALSE) {
	setup_cib();
	}

	} else {
	stonith_our_uname = strdup("localhost");
	}

	init_device_list();
	init_topology_list();

	if(stonith_watchdog_timeout_ms > 0) {
	int rc;
	xmlNode *xml;
	stonith_key_value_t *params = NULL;

	params = stonith_key_value_add(params, PCMK_STONITH_HOST_LIST,
	stonith_our_uname);

	xml = create_device_registration_xml("watchdog", st_namespace_internal,
	STONITH_WATCHDOG_AGENT, params,
	NULL);
	stonith_key_value_freeall(params, 1, 1);
	rc = stonith_device_register(xml, NULL, FALSE);
	free_xml(xml);
	if (rc != pcmk_ok) {
	crm_crit("Cannot register watchdog pseudo fence agent");
	crm_exit(CRM_EX_FATAL);
	}
	}

	pcmk__serve_fenced_ipc(&ipcs, &ipc_callbacks);

	/* Create the mainloop and run it... */
	mainloop = g_main_loop_new(NULL, FALSE);
	crm_notice("Pacemaker fencer successfully started and accepting connections");
	g_main_loop_run(mainloop);

	stonith_cleanup();
	pe_free_working_set(fenced_data_set);
	crm_exit(CRM_EX_OK);
	}
	diff --git a/daemons/pacemakerd/pacemakerd.c b/daemons/pacemakerd/pacemakerd.c
	index 0bdf53024a..509b0f8242 100644
	--- a/daemons/pacemakerd/pacemakerd.c
	+++ b/daemons/pacemakerd/pacemakerd.c
	@@ -1,1358 +1,1350 @@
	/*
	* Copyright 2010-2020 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>
	#include "pacemakerd.h"

	#include <pwd.h>
	#include <grp.h>
	#include <poll.h>
	#include <errno.h>
	#include <stdio.h>
	#include <stdbool.h>
	#include <sys/stat.h>
	#include <sys/types.h>
	#include <sys/time.h>
	#include <sys/resource.h>
	#include <sys/reboot.h>

	#include <crm/crm.h> /* indirectly: CRM_EX_* */
	#include <crm/cib/internal.h> /* cib_channel_ro */
	#include <crm/msg_xml.h>
	#include <crm/common/ipc_internal.h>
	#include <crm/common/mainloop.h>
	#include <crm/cluster/internal.h>
	#include <crm/cluster.h>

	#include <dirent.h>
	#include <ctype.h>

	static gboolean fatal_error = FALSE;
	static GMainLoop *mainloop = NULL;
	static bool global_keep_tracking = false;

	#define PCMK_PROCESS_CHECK_INTERVAL 5

	static crm_trigger_t *shutdown_trigger = NULL;
	static crm_trigger_t *startup_trigger = NULL;
	static const char *pid_file = PCMK_RUN_DIR "/pacemaker.pid";

	/* state we report when asked via pacemakerd-api status-ping */
	static const char *pacemakerd_state = XML_PING_ATTR_PACEMAKERDSTATE_INIT;
	static gboolean running_with_sbd = FALSE; /* local copy */
	/* When contacted via pacemakerd-api by a client having sbd in
	* the name we assume it is sbd-daemon which wants to know
	* if pacemakerd shutdown gracefully.
	* Thus when everything is shutdown properly pacemakerd
	* waits till it has reported the graceful completion of
	* shutdown to sbd and just when sbd-client closes the
	* connection we can assume that the report has arrived
	* properly so that pacemakerd can finally exit.
	* Following two variables are used to track that handshake.
	*/
	static unsigned int shutdown_complete_state_reported_to = 0;
	static gboolean shutdown_complete_state_reported_client_closed = FALSE;

	typedef struct pcmk_child_s {
	pid_t pid;
	- long flag;
	int start_seq;
	int respawn_count;
	gboolean respawn;
	const char *name;
	const char *uid;
	const char *command;
	const char endpoint; / IPC server name */

	gboolean active_before_startup;
	} pcmk_child_t;

	/* Index into the array below */
	#define PCMK_CHILD_CONTROLD 3

	static pcmk_child_t pcmk_children[] = {
	{
	- 0, crm_proc_none, 0, 0, FALSE, "none",
	- NULL, NULL
	+ 0, 0, 0, FALSE, "none", NULL, NULL, NULL
	},
	{
	- 0, crm_proc_execd, 3, 0, TRUE, "pacemaker-execd",
	- NULL, CRM_DAEMON_DIR "/pacemaker-execd",
	- CRM_SYSTEM_LRMD
	+ 0, 3, 0, TRUE, "pacemaker-execd", NULL,
	+ CRM_DAEMON_DIR "/pacemaker-execd", CRM_SYSTEM_LRMD
	},
	{
	- 0, crm_proc_based, 1, 0, TRUE, "pacemaker-based",
	- CRM_DAEMON_USER, CRM_DAEMON_DIR "/pacemaker-based",
	- PCMK__SERVER_BASED_RO
	+ 0, 1, 0, TRUE, "pacemaker-based", CRM_DAEMON_USER,
	+ CRM_DAEMON_DIR "/pacemaker-based", PCMK__SERVER_BASED_RO
	},
	{
	- 0, crm_proc_controld, 6, 0, TRUE, "pacemaker-controld",
	- CRM_DAEMON_USER, CRM_DAEMON_DIR "/pacemaker-controld",
	- CRM_SYSTEM_CRMD
	+ 0, 6, 0, TRUE, "pacemaker-controld", CRM_DAEMON_USER,
	+ CRM_DAEMON_DIR "/pacemaker-controld", CRM_SYSTEM_CRMD
	},
	{
	- 0, crm_proc_attrd, 4, 0, TRUE, "pacemaker-attrd",
	- CRM_DAEMON_USER, CRM_DAEMON_DIR "/pacemaker-attrd",
	- T_ATTRD
	+ 0, 4, 0, TRUE, "pacemaker-attrd", CRM_DAEMON_USER,
	+ CRM_DAEMON_DIR "/pacemaker-attrd", T_ATTRD
	},
	{
	- 0, crm_proc_schedulerd, 5, 0, TRUE, "pacemaker-schedulerd",
	- CRM_DAEMON_USER, CRM_DAEMON_DIR "/pacemaker-schedulerd",
	- CRM_SYSTEM_PENGINE
	+ 0, 5, 0, TRUE, "pacemaker-schedulerd", CRM_DAEMON_USER,
	+ CRM_DAEMON_DIR "/pacemaker-schedulerd", CRM_SYSTEM_PENGINE
	},
	{
	- 0, crm_proc_fenced, 2, 0, TRUE, "pacemaker-fenced",
	- NULL, CRM_DAEMON_DIR "/pacemaker-fenced",
	- "stonith-ng"
	+ 0, 2, 0, TRUE, "pacemaker-fenced", NULL,
	+ CRM_DAEMON_DIR "/pacemaker-fenced", "stonith-ng"
	},
	};

	static gboolean check_active_before_startup_processes(gpointer user_data);
	static int child_liveness(pcmk_child_t *child);
	static gboolean start_child(pcmk_child_t * child);

	static void
	pcmk_process_exit(pcmk_child_t * child)
	{
	child->pid = 0;
	child->active_before_startup = FALSE;

	child->respawn_count += 1;
	if (child->respawn_count > MAX_RESPAWN) {
	crm_err("Child respawn count exceeded by %s", child->name);
	child->respawn = FALSE;
	}

	if (shutdown_trigger) {
	/* resume step-wise shutdown (returned TRUE yields no parallelizing) */
	mainloop_set_trigger(shutdown_trigger);

	} else if (!child->respawn) {
	/* nothing to do */

	} else if (crm_is_true(getenv("PCMK_fail_fast"))) {
	crm_err("Rebooting system because of %s", child->name);
	pcmk__panic(__func__);

	} else if (child_liveness(child) == pcmk_rc_ok) {
	crm_warn("One-off suppressing strict respawning of a child process %s,"
	" appears alright per %s IPC end-point",
	child->name, child->endpoint);
	/* need to monitor how it evolves, and start new process if badly */
	child->active_before_startup = TRUE;
	if (!global_keep_tracking) {
	global_keep_tracking = true;
	g_timeout_add_seconds(PCMK_PROCESS_CHECK_INTERVAL,
	check_active_before_startup_processes, NULL);
	}

	} else {
	crm_notice("Respawning failed child process: %s", child->name);
	start_child(child);
	}
	}

	static void
	pcmk_child_exit(mainloop_child_t * p, pid_t pid, int core, int signo, int exitcode)
	{
	pcmk_child_t *child = mainloop_child_userdata(p);
	const char *name = mainloop_child_name(p);

	if (signo) {
	do_crm_log(((signo == SIGKILL)? LOG_WARNING : LOG_ERR),
	"%s[%d] terminated with signal %d (core=%d)",
	name, pid, signo, core);

	} else {
	switch(exitcode) {
	case CRM_EX_OK:
	crm_info("%s[%d] exited with status %d (%s)",
	name, pid, exitcode, crm_exit_str(exitcode));
	break;

	case CRM_EX_FATAL:
	crm_warn("Shutting cluster down because %s[%d] had fatal failure",
	name, pid);
	child->respawn = FALSE;
	fatal_error = TRUE;
	pcmk_shutdown(SIGTERM);
	break;

	case CRM_EX_PANIC:
	crm_emerg("%s[%d] instructed the machine to reset", name, pid);
	child->respawn = FALSE;
	fatal_error = TRUE;
	pcmk__panic(__func__);
	pcmk_shutdown(SIGTERM);
	break;

	default:
	crm_err("%s[%d] exited with status %d (%s)",
	name, pid, exitcode, crm_exit_str(exitcode));
	break;
	}
	}

	pcmk_process_exit(child);
	}

	static gboolean
	stop_child(pcmk_child_t * child, int signal)
	{
	if (signal == 0) {
	signal = SIGTERM;
	}

	/* why to skip PID of 1?
	- FreeBSD ~ how untrackable process behind IPC is masqueraded as
	- elsewhere: how "init" task is designated; in particular, in systemd
	arrangement of socket-based activation, this is pretty real */
	if (child->command == NULL \|\| child->pid == PCMK__SPECIAL_PID) {
	crm_debug("Nothing to do for child \"%s\" (process %lld)",
	child->name, (long long) PCMK__SPECIAL_PID_AS_0(child->pid));
	return TRUE;
	}

	if (child->pid <= 0) {
	crm_trace("Client %s not running", child->name);
	return TRUE;
	}

	errno = 0;
	if (kill(child->pid, signal) == 0) {
	crm_notice("Stopping %s "CRM_XS" sent signal %d to process %lld",
	child->name, signal, (long long) child->pid);

	} else {
	crm_err("Could not stop %s (process %lld) with signal %d: %s",
	child->name, (long long) child->pid, signal, strerror(errno));
	}

	return TRUE;
	}

	static char *opts_default[] = { NULL, NULL };
	static char *opts_vgrind[] = { NULL, NULL, NULL, NULL, NULL };

	/* TODO once libqb is taught to juggle with IPC end-points carried over as
	bare file descriptor (https://github.com/ClusterLabs/libqb/issues/325)
	it shall hand over these descriptors here if/once they are successfully
	pre-opened in (presumably) child_liveness(), to avoid any remaining
	room for races */
	static gboolean
	start_child(pcmk_child_t * child)
	{
	uid_t uid = 0;
	gid_t gid = 0;
	gboolean use_valgrind = FALSE;
	gboolean use_callgrind = FALSE;
	const char *env_valgrind = getenv("PCMK_valgrind_enabled");
	const char *env_callgrind = getenv("PCMK_callgrind_enabled");

	child->active_before_startup = FALSE;

	if (child->command == NULL) {
	crm_info("Nothing to do for child \"%s\"", child->name);
	return TRUE;
	}

	if (env_callgrind != NULL && crm_is_true(env_callgrind)) {
	use_callgrind = TRUE;
	use_valgrind = TRUE;

	} else if (env_callgrind != NULL && strstr(env_callgrind, child->name)) {
	use_callgrind = TRUE;
	use_valgrind = TRUE;

	} else if (env_valgrind != NULL && crm_is_true(env_valgrind)) {
	use_valgrind = TRUE;

	} else if (env_valgrind != NULL && strstr(env_valgrind, child->name)) {
	use_valgrind = TRUE;
	}

	if (use_valgrind && strlen(VALGRIND_BIN) == 0) {
	crm_warn("Cannot enable valgrind for %s:"
	" The location of the valgrind binary is unknown", child->name);
	use_valgrind = FALSE;
	}

	if (child->uid) {
	if (crm_user_lookup(child->uid, &uid, &gid) < 0) {
	crm_err("Invalid user (%s) for %s: not found", child->uid, child->name);
	return FALSE;
	}
	crm_info("Using uid=%u and group=%u for process %s", uid, gid, child->name);
	}

	child->pid = fork();
	CRM_ASSERT(child->pid != -1);

	if (child->pid > 0) {
	/* parent */
	mainloop_child_add(child->pid, 0, child->name, child, pcmk_child_exit);

	crm_info("Forked child %lld for process %s%s",
	(long long) child->pid, child->name,
	use_valgrind ? " (valgrind enabled: " VALGRIND_BIN ")" : "");
	return TRUE;

	} else {
	/* Start a new session */
	(void)setsid();

	/* Setup the two alternate arg arrays */
	opts_vgrind[0] = strdup(VALGRIND_BIN);
	if (use_callgrind) {
	opts_vgrind[1] = strdup("--tool=callgrind");
	opts_vgrind[2] = strdup("--callgrind-out-file=" CRM_STATE_DIR "/callgrind.out.%p");
	opts_vgrind[3] = strdup(child->command);
	opts_vgrind[4] = NULL;
	} else {
	opts_vgrind[1] = strdup(child->command);
	opts_vgrind[2] = NULL;
	opts_vgrind[3] = NULL;
	opts_vgrind[4] = NULL;
	}
	opts_default[0] = strdup(child->command);

	if(gid) {
	// Whether we need root group access to talk to cluster layer
	bool need_root_group = TRUE;

	if (is_corosync_cluster()) {
	/* Corosync clusters can drop root group access, because we set
	* uidgid.gid.${gid}=1 via CMAP, which allows these processes to
	* connect to corosync.
	*/
	need_root_group = FALSE;
	}

	// Drop root group access if not needed
	if (!need_root_group && (setgid(gid) < 0)) {
	crm_warn("Could not set group to %d: %s", gid, strerror(errno));
	}

	/* Initialize supplementary groups to only those always granted to
	* the user, plus haclient (so we can access IPC).
	*/
	if (initgroups(child->uid, gid) < 0) {
	crm_err("Cannot initialize groups for %s: %s (%d)", child->uid, pcmk_strerror(errno), errno);
	}
	}

	if (uid && setuid(uid) < 0) {
	crm_warn("Could not set user to %s (id %d): %s",
	child->uid, uid, strerror(errno));
	}

	pcmk__close_fds_in_child(true);

	pcmk__open_devnull(O_RDONLY); // stdin (fd 0)
	pcmk__open_devnull(O_WRONLY); // stdout (fd 1)
	pcmk__open_devnull(O_WRONLY); // stderr (fd 2)

	if (use_valgrind) {
	(void)execvp(VALGRIND_BIN, opts_vgrind);
	} else {
	(void)execvp(child->command, opts_default);
	}
	crm_crit("Could not execute %s: %s", child->command, strerror(errno));
	crm_exit(CRM_EX_FATAL);
	}
	return TRUE; /* never reached */
	}

	static gboolean
	escalate_shutdown(gpointer data)
	{

	pcmk_child_t *child = data;

	if (child->pid == PCMK__SPECIAL_PID) {
	pcmk_process_exit(child);

	} else if (child->pid != 0) {
	/* Use SIGSEGV instead of SIGKILL to create a core so we can see what it was up to */
	crm_err("Child %s not terminating in a timely manner, forcing", child->name);
	stop_child(child, SIGSEGV);
	}
	return FALSE;
	}

	#define SHUTDOWN_ESCALATION_PERIOD 180000 /* 3m */

	static gboolean
	pcmk_shutdown_worker(gpointer user_data)
	{
	static int phase = SIZEOF(pcmk_children);
	static time_t next_log = 0;

	int lpc = 0;

	if (phase == SIZEOF(pcmk_children)) {
	crm_notice("Shutting down Pacemaker");
	pacemakerd_state = XML_PING_ATTR_PACEMAKERDSTATE_SHUTTINGDOWN;
	}

	for (; phase > 0; phase--) {
	/* Don't stop anything with start_seq < 1 */

	for (lpc = SIZEOF(pcmk_children) - 1; lpc >= 0; lpc--) {
	pcmk_child_t *child = &(pcmk_children[lpc]);

	if (phase != child->start_seq) {
	continue;
	}

	if (child->pid != 0) {
	time_t now = time(NULL);

	if (child->respawn) {
	if (child->pid == PCMK__SPECIAL_PID) {
	crm_warn("The process behind %s IPC cannot be"
	" terminated, so either wait the graceful"
	" period of %ld s for its native termination"
	" if it vitally depends on some other daemons"
	" going down in a controlled way already,"
	" or locate and kill the correct %s process"
	" on your own; set PCMK_fail_fast=1 to avoid"
	" this altogether next time around",
	child->name, (long) SHUTDOWN_ESCALATION_PERIOD,
	child->command);
	}
	next_log = now + 30;
	child->respawn = FALSE;
	stop_child(child, SIGTERM);
	if (phase < pcmk_children[PCMK_CHILD_CONTROLD].start_seq) {
	g_timeout_add(SHUTDOWN_ESCALATION_PERIOD,
	escalate_shutdown, child);
	}

	} else if (now >= next_log) {
	next_log = now + 30;
	crm_notice("Still waiting for %s to terminate "
	CRM_XS " pid=%lld seq=%d",
	child->name, (long long) child->pid,
	child->start_seq);
	}
	return TRUE;
	}

	/* cleanup */
	crm_debug("%s confirmed stopped", child->name);
	child->pid = 0;
	}
	}

	crm_notice("Shutdown complete");
	pacemakerd_state = XML_PING_ATTR_PACEMAKERDSTATE_SHUTDOWNCOMPLETE;
	if (!fatal_error && running_with_sbd &&
	pcmk__get_sbd_sync_resource_startup() &&
	!shutdown_complete_state_reported_client_closed) {
	crm_notice("Waiting for SBD to pick up shutdown-complete-state.");
	return TRUE;
	}

	{
	const char *delay = pcmk__env_option("shutdown_delay");
	if(delay) {
	sync();
	sleep(crm_get_msec(delay) / 1000);
	}
	}

	g_main_loop_quit(mainloop);

	if (fatal_error) {
	crm_notice("Shutting down and staying down after fatal error");
	#ifdef SUPPORT_COROSYNC
	pcmkd_shutdown_corosync();
	#endif
	crm_exit(CRM_EX_FATAL);
	}

	return TRUE;
	}

	static void
	pcmk_ignore(int nsig)
	{
	crm_info("Ignoring signal %s (%d)", strsignal(nsig), nsig);
	}

	static void
	pcmk_sigquit(int nsig)
	{
	pcmk__panic(__func__);
	}

	void
	pcmk_shutdown(int nsig)
	{
	if (shutdown_trigger == NULL) {
	shutdown_trigger = mainloop_add_trigger(G_PRIORITY_HIGH, pcmk_shutdown_worker, NULL);
	}
	mainloop_set_trigger(shutdown_trigger);
	}

	static int32_t
	pcmk_ipc_accept(qb_ipcs_connection_t * c, uid_t uid, gid_t gid)
	{
	crm_trace("Connection %p", c);
	if (pcmk__new_client(c, uid, gid) == NULL) {
	return -EIO;
	}
	return 0;
	}

	static void
	pcmk_handle_ping_request(pcmk__client_t c, xmlNode msg, uint32_t id)
	{
	const char *value = NULL;
	xmlNode *ping = NULL;
	xmlNode *reply = NULL;
	time_t pinged = time(NULL);
	const char *from = crm_element_value(msg, F_CRM_SYS_FROM);

	/* Pinged for status */
	crm_trace("Pinged from %s.%s",
	crm_str(crm_element_value(msg, F_CRM_ORIGIN)),
	from?from:"unknown");
	ping = create_xml_node(NULL, XML_CRM_TAG_PING);
	value = crm_element_value(msg, F_CRM_SYS_TO);
	crm_xml_add(ping, XML_PING_ATTR_SYSFROM, value);
	crm_xml_add(ping, XML_PING_ATTR_PACEMAKERDSTATE, pacemakerd_state);
	crm_xml_add_ll(ping, XML_ATTR_TSTAMP, (long long) pinged);
	crm_xml_add(ping, XML_PING_ATTR_STATUS, "ok");
	reply = create_reply(msg, ping);
	free_xml(ping);
	if (reply) {
	if (pcmk__ipc_send_xml(c, id, reply, crm_ipc_server_event) !=
	pcmk_rc_ok) {
	crm_err("Failed sending ping reply to client %s",
	pcmk__client_name(c));
	}
	free_xml(reply);
	} else {
	crm_err("Failed building ping reply for client %s",
	pcmk__client_name(c));
	}
	/* just proceed state on sbd pinging us */
	if (from && strstr(from, "sbd")) {
	if (pcmk__str_eq(pacemakerd_state, XML_PING_ATTR_PACEMAKERDSTATE_SHUTDOWNCOMPLETE, pcmk__str_none)) {
	if (pcmk__get_sbd_sync_resource_startup()) {
	crm_notice("Shutdown-complete-state passed to SBD.");
	}
	shutdown_complete_state_reported_to = c->pid;
	} else if (pcmk__str_eq(pacemakerd_state, XML_PING_ATTR_PACEMAKERDSTATE_WAITPING, pcmk__str_none)) {
	crm_notice("Received startup-trigger from SBD.");
	pacemakerd_state = XML_PING_ATTR_PACEMAKERDSTATE_STARTINGDAEMONS;
	mainloop_set_trigger(startup_trigger);
	}
	}
	}

	/* Exit code means? */
	static int32_t
	pcmk_ipc_dispatch(qb_ipcs_connection_t * qbc, void *data, size_t size)
	{
	uint32_t id = 0;
	uint32_t flags = 0;
	const char *task = NULL;
	xmlNode *msg = NULL;
	pcmk__client_t *c = pcmk__find_client(qbc);

	CRM_CHECK(c != NULL, return 0);

	msg = pcmk__client_data2xml(c, data, &id, &flags);
	if (msg == NULL) {
	pcmk__ipc_send_ack(c, id, flags, "ack", CRM_EX_PROTOCOL);
	return 0;
	}

	task = crm_element_value(msg, F_CRM_TASK);
	if (pcmk__str_eq(task, CRM_OP_QUIT, pcmk__str_none)) {
	#if ENABLE_ACL
	/* Only allow privileged users (i.e. root or hacluster)
	* to shut down Pacemaker from the command line (or direct IPC).
	*
	* We only check when ACLs are enabled, because without them, any client
	* with IPC access could shut down Pacemaker via the CIB anyway.
	*/
	bool allowed = pcmk_is_set(c->flags, pcmk__client_privileged);
	#else
	bool allowed = true;
	#endif
	if (allowed) {
	crm_notice("Shutting down in response to IPC request %s from %s",
	crm_element_value(msg, F_CRM_REFERENCE),
	crm_element_value(msg, F_CRM_ORIGIN));
	pcmk__ipc_send_ack(c, id, flags, "ack", CRM_EX_OK);
	pcmk_shutdown(15);
	} else {
	crm_warn("Ignoring shutdown request from unprivileged client %s",
	pcmk__client_name(c));
	pcmk__ipc_send_ack(c, id, flags, "ack", CRM_EX_INSUFFICIENT_PRIV);
	}

	} else if (pcmk__str_eq(task, CRM_OP_RM_NODE_CACHE, pcmk__str_none)) {
	crm_trace("Ignoring request from client %s to purge node "
	"because peer cache is not used", pcmk__client_name(c));
	pcmk__ipc_send_ack(c, id, flags, "ack", CRM_EX_OK);

	} else if (pcmk__str_eq(task, CRM_OP_PING, pcmk__str_none)) {
	pcmk__ipc_send_ack(c, id, flags, "ack", CRM_EX_INDETERMINATE);
	pcmk_handle_ping_request(c, msg, id);

	} else {
	crm_debug("Unrecognized IPC command '%s' from client %s",
	crm_str(task), pcmk__client_name(c));
	pcmk__ipc_send_ack(c, id, flags, "ack", CRM_EX_INVALID_PARAM);
	}

	free_xml(msg);
	return 0;
	}

	/* Error code means? */
	static int32_t
	pcmk_ipc_closed(qb_ipcs_connection_t * c)
	{
	pcmk__client_t *client = pcmk__find_client(c);

	if (client == NULL) {
	return 0;
	}
	crm_trace("Connection %p", c);
	if (shutdown_complete_state_reported_to == client->pid) {
	shutdown_complete_state_reported_client_closed = TRUE;
	if (shutdown_trigger) {
	mainloop_set_trigger(shutdown_trigger);
	}
	}
	pcmk__free_client(client);
	return 0;
	}

	static void
	pcmk_ipc_destroy(qb_ipcs_connection_t * c)
	{
	crm_trace("Connection %p", c);
	pcmk_ipc_closed(c);
	}

	struct qb_ipcs_service_handlers mcp_ipc_callbacks = {
	.connection_accept = pcmk_ipc_accept,
	.connection_created = NULL,
	.msg_process = pcmk_ipc_dispatch,
	.connection_closed = pcmk_ipc_closed,
	.connection_destroyed = pcmk_ipc_destroy
	};

	static pcmk__cli_option_t long_options[] = {
	// long option, argument type, storage, short option, description, flags
	{
	"help", no_argument, NULL, '?',
	"\tThis text", pcmk__option_default
	},
	{
	"version", no_argument, NULL, '$',
	"\tVersion information", pcmk__option_default
	},
	{
	"verbose", no_argument, NULL, 'V',
	"\tIncrease debug output", pcmk__option_default
	},
	{
	"shutdown", no_argument, NULL, 'S',
	"\tInstruct Pacemaker to shutdown on this machine", pcmk__option_default
	},
	{
	"features", no_argument, NULL, 'F',
	"\tDisplay full version and list of features Pacemaker was built with",
	pcmk__option_default
	},
	{
	"-spacer-", no_argument, NULL, '-',
	"\nAdditional Options:", pcmk__option_default
	},
	{
	"foreground", no_argument, NULL, 'f',
	"\t(Ignored) Pacemaker always runs in the foreground",
	pcmk__option_default
	},
	{
	"pid-file", required_argument, NULL, 'p',
	"\t(Ignored) Daemon pid file location", pcmk__option_default
	},
	{
	"standby", no_argument, NULL, 's',
	"\tStart node in standby state", pcmk__option_default
	},
	{ 0, 0, 0, 0 }
	};

	static void
	mcp_chown(const char *path, uid_t uid, gid_t gid)
	{
	int rc = chown(path, uid, gid);

	if (rc < 0) {
	crm_warn("Cannot change the ownership of %s to user %s and gid %d: %s",
	path, CRM_DAEMON_USER, gid, pcmk_strerror(errno));
	}
	}

	/*!
	* \internal
	* \brief Check the liveness of the child based on IPC name and PID if tracked
	*
	* \param[inout] child Child tracked data
	*
	* \return Standard Pacemaker return code
	*
	* \note Return codes of particular interest include pcmk_rc_ipc_unresponsive
	* indicating that no trace of IPC liveness was detected,
	* pcmk_rc_ipc_unauthorized indicating that the IPC endpoint is blocked by
	* an unauthorized process, and pcmk_rc_ipc_pid_only indicating that
	* the child is up by PID but not IPC end-point (possibly starting).
	* \note This function doesn't modify any of \p child members but \c pid,
	* and is not actively toying with processes as such but invoking
	* \c stop_child in one particular case (there's for some reason
	* a different authentic holder of the IPC end-point).
	*/
	static int
	child_liveness(pcmk_child_t *child)
	{
	uid_t cl_uid = 0;
	gid_t cl_gid = 0;
	const uid_t root_uid = 0;
	const gid_t root_gid = 0;
	const uid_t *ref_uid;
	const gid_t *ref_gid;
	int rc = pcmk_rc_ipc_unresponsive;
	pid_t ipc_pid = 0;

	if (child->endpoint == NULL
	&& (child->pid <= 0 \|\| child->pid == PCMK__SPECIAL_PID)) {
	crm_err("Cannot track child %s for missing both API end-point and PID",
	child->name);
	rc = EINVAL; // Misuse of function when child is not trackable

	} else if (child->endpoint != NULL) {
	int legacy_rc = pcmk_ok;

	if (child->uid == NULL) {
	ref_uid = &root_uid;
	ref_gid = &root_gid;
	} else {
	ref_uid = &cl_uid;
	ref_gid = &cl_gid;
	legacy_rc = pcmk_daemon_user(&cl_uid, &cl_gid);
	}

	if (legacy_rc < 0) {
	rc = pcmk_legacy2rc(legacy_rc);
	crm_err("Could not find user and group IDs for user %s: %s "
	CRM_XS " rc=%d", CRM_DAEMON_USER, pcmk_rc_str(rc), rc);
	} else {
	rc = pcmk__ipc_is_authentic_process_active(child->endpoint,
	ref_uid, ref_gid,
	&ipc_pid);
	if ((rc == pcmk_rc_ok) \|\| (rc == pcmk_rc_ipc_unresponsive)) {
	if (child->pid <= 0) {
	/* If rc is pcmk_rc_ok, ipc_pid is nonzero and this
	* initializes a new child. If rc is
	* pcmk_rc_ipc_unresponsive, ipc_pid is zero, and we will
	* investigate further.
	*/
	child->pid = ipc_pid;
	} else if ((ipc_pid != 0) && (child->pid != ipc_pid)) {
	/* An unexpected (but authorized) process is responding to
	* IPC. Investigate further.
	*/
	rc = pcmk_rc_ipc_unresponsive;
	}
	}
	}
	}

	if (rc == pcmk_rc_ipc_unresponsive) {
	/* If we get here, a child without IPC is being tracked, no IPC liveness
	* has been detected, or IPC liveness has been detected with an
	* unexpected (but authorized) process. This is safe on FreeBSD since
	* the only change possible from a proper child's PID into "special" PID
	* of 1 behind more loosely related process.
	*/
	int ret = pcmk__pid_active(child->pid, child->name);

	if (ipc_pid && ((ret != pcmk_rc_ok)
	\|\| ipc_pid == PCMK__SPECIAL_PID
	\|\| (pcmk__pid_active(ipc_pid,
	child->name) == pcmk_rc_ok))) {
	/* An unexpected (but authorized) process was detected at the IPC
	* endpoint, and either it is active, or the child we're tracking is
	* not.
	*/

	if (ret == pcmk_rc_ok) {
	/* The child we're tracking is active. Kill it, and adopt the
	* detected process. This assumes that our children don't fork
	* (thus getting a different PID owning the IPC), but rather the
	* tracking got out of sync because of some means external to
	* Pacemaker, and adopting the detected process is better than
	* killing it and possibly having to spawn a new child.
	*/
	/* not possessing IPC, afterall (what about corosync CPG?) */
	stop_child(child, SIGKILL);
	}
	rc = pcmk_rc_ok;
	child->pid = ipc_pid;
	} else if (ret == pcmk_rc_ok) {
	// Our tracked child's PID was found active, but not its IPC
	rc = pcmk_rc_ipc_pid_only;
	} else if ((child->pid == 0) && (ret == EINVAL)) {
	// FreeBSD can return EINVAL
	rc = pcmk_rc_ipc_unresponsive;
	} else {
	switch (ret) {
	case EACCES:
	rc = pcmk_rc_ipc_unauthorized;
	break;
	case ESRCH:
	rc = pcmk_rc_ipc_unresponsive;
	break;
	default:
	rc = ret;
	break;
	}
	}
	}
	return rc;
	}

	static gboolean
	check_active_before_startup_processes(gpointer user_data)
	{
	int start_seq = 1, lpc = 0;
	static int max = SIZEOF(pcmk_children);
	gboolean keep_tracking = FALSE;

	for (start_seq = 1; start_seq < max; start_seq++) {
	for (lpc = 0; lpc < max; lpc++) {
	if (pcmk_children[lpc].active_before_startup == FALSE) {
	/* we are already tracking it as a child process. */
	continue;
	} else if (start_seq != pcmk_children[lpc].start_seq) {
	continue;
	} else {
	int rc = child_liveness(&pcmk_children[lpc]);

	switch (rc) {
	case pcmk_rc_ok:
	break;
	case pcmk_rc_ipc_unresponsive:
	case pcmk_rc_ipc_pid_only: // This case: it was previously OK
	if (pcmk_children[lpc].respawn == TRUE) {
	crm_err("%s[%lld] terminated%s", pcmk_children[lpc].name,
	(long long) PCMK__SPECIAL_PID_AS_0(pcmk_children[lpc].pid),
	(rc == pcmk_rc_ipc_pid_only)? " as IPC server" : "");
	} else {
	/* orderly shutdown */
	crm_notice("%s[%lld] terminated%s", pcmk_children[lpc].name,
	(long long) PCMK__SPECIAL_PID_AS_0(pcmk_children[lpc].pid),
	(rc == pcmk_rc_ipc_pid_only)? " as IPC server" : "");
	}
	pcmk_process_exit(&(pcmk_children[lpc]));
	continue;
	default:
	crm_exit(CRM_EX_FATAL);
	break; /* static analysis/noreturn */
	}
	}
	/* at least one of the processes found at startup
	* is still going, so keep this recurring timer around */
	keep_tracking = TRUE;
	}
	}

	global_keep_tracking = keep_tracking;
	return keep_tracking;
	}

	/*!
	* \internal
	* \brief Initial one-off check of the pre-existing "child" processes
	*
	* With "child" process, we mean the subdaemon that defines an API end-point
	* (all of them do as of the comment) -- the possible complement is skipped
	* as it is deemed it has no such shared resources to cause conflicts about,
	* hence it can presumably be started anew without hesitation.
	* If that won't hold true in the future, the concept of a shared resource
	* will have to be generalized beyond the API end-point.
	*
	* For boundary cases that the "child" is still starting (IPC end-point is yet
	* to be witnessed), or more rarely (practically FreeBSD only), when there's
	* a pre-existing "untrackable" authentic process, we give the situation some
	* time to possibly unfold in the right direction, meaning that said socket
	* will appear or the unattainable process will disappear per the observable
	* IPC, respectively.
	*
	* \return Standard Pacemaker return code
	*
	* \note Since this gets run at the very start, \c respawn_count fields
	* for particular children get temporarily overloaded with "rounds
	* of waiting" tracking, restored once we are about to finish with
	* success (i.e. returning value >=0) and will remain unrestored
	* otherwise. One way to suppress liveness detection logic for
	* particular child is to set the said value to a negative number.
	*/
	#define WAIT_TRIES 4 /* together with interleaved sleeps, worst case ~ 1s */
	static int
	find_and_track_existing_processes(void)
	{
	bool tracking = false;
	bool wait_in_progress;
	int rc;
	size_t i, rounds;

	for (rounds = 1; rounds <= WAIT_TRIES; rounds++) {
	wait_in_progress = false;
	for (i = 0; i < SIZEOF(pcmk_children); i++) {

	if ((pcmk_children[i].endpoint == NULL)
	\|\| (pcmk_children[i].respawn_count < 0)) {
	continue;
	}

	rc = child_liveness(&pcmk_children[i]);
	if (rc == pcmk_rc_ipc_unresponsive) {
	/* As a speculation, don't give up if there are more rounds to
	* come for other reasons, but don't artificially wait just
	* because of this, since we would preferably start ASAP.
	*/
	continue;
	}

	pcmk_children[i].respawn_count = rounds;
	switch (rc) {
	case pcmk_rc_ok:
	if (pcmk_children[i].pid == PCMK__SPECIAL_PID) {
	if (crm_is_true(getenv("PCMK_fail_fast"))) {
	crm_crit("Cannot reliably track pre-existing"
	" authentic process behind %s IPC on this"
	" platform and PCMK_fail_fast requested",
	pcmk_children[i].endpoint);
	return EOPNOTSUPP;
	} else if (pcmk_children[i].respawn_count == WAIT_TRIES) {
	crm_notice("Assuming pre-existing authentic, though"
	" on this platform untrackable, process"
	" behind %s IPC is stable (was in %d"
	" previous samples) so rather than"
	" bailing out (PCMK_fail_fast not"
	" requested), we just switch to a less"
	" optimal IPC liveness monitoring"
	" (not very suitable for heavy load)",
	pcmk_children[i].name, WAIT_TRIES - 1);
	crm_warn("The process behind %s IPC cannot be"
	" terminated, so the overall shutdown"
	" will get delayed implicitly (%ld s),"
	" which serves as a graceful period for"
	" its native termination if it vitally"
	" depends on some other daemons going"
	" down in a controlled way already",
	pcmk_children[i].name,
	(long) SHUTDOWN_ESCALATION_PERIOD);
	} else {
	wait_in_progress = true;
	crm_warn("Cannot reliably track pre-existing"
	" authentic process behind %s IPC on this"
	" platform, can still disappear in %d"
	" attempt(s)", pcmk_children[i].endpoint,
	WAIT_TRIES - pcmk_children[i].respawn_count);
	continue;
	}
	}
	crm_notice("Tracking existing %s process (pid=%lld)",
	pcmk_children[i].name,
	(long long) PCMK__SPECIAL_PID_AS_0(
	pcmk_children[i].pid));
	pcmk_children[i].respawn_count = -1; /* 0~keep watching */
	pcmk_children[i].active_before_startup = TRUE;
	tracking = true;
	break;
	case pcmk_rc_ipc_pid_only:
	if (pcmk_children[i].respawn_count == WAIT_TRIES) {
	crm_crit("%s IPC end-point for existing authentic"
	" process %lld did not (re)appear",
	pcmk_children[i].endpoint,
	(long long) PCMK__SPECIAL_PID_AS_0(
	pcmk_children[i].pid));
	return rc;
	}
	wait_in_progress = true;
	crm_warn("Cannot find %s IPC end-point for existing"
	" authentic process %lld, can still (re)appear"
	" in %d attempts (?)",
	pcmk_children[i].endpoint,
	(long long) PCMK__SPECIAL_PID_AS_0(
	pcmk_children[i].pid),
	WAIT_TRIES - pcmk_children[i].respawn_count);
	continue;
	default:
	crm_crit("Checked liveness of %s: %s " CRM_XS " rc=%d",
	pcmk_children[i].name, pcmk_rc_str(rc), rc);
	return rc;
	}
	}
	if (!wait_in_progress) {
	break;
	}
	(void) poll(NULL, 0, 250); /* a bit for changes to possibly happen */
	}
	for (i = 0; i < SIZEOF(pcmk_children); i++) {
	pcmk_children[i].respawn_count = 0; /* restore pristine state */
	}

	if (tracking) {
	g_timeout_add_seconds(PCMK_PROCESS_CHECK_INTERVAL,
	check_active_before_startup_processes, NULL);
	}
	return pcmk_rc_ok;
	}

	static gboolean
	init_children_processes(void *user_data)
	{
	int start_seq = 1, lpc = 0;
	static int max = SIZEOF(pcmk_children);

	/* start any children that have not been detected */
	for (start_seq = 1; start_seq < max; start_seq++) {
	/* don't start anything with start_seq < 1 */
	for (lpc = 0; lpc < max; lpc++) {
	if (pcmk_children[lpc].pid != 0) {
	/* we are already tracking it */
	continue;
	}

	if (start_seq == pcmk_children[lpc].start_seq) {
	start_child(&(pcmk_children[lpc]));
	}
	}
	}

	/* From this point on, any daemons being started will be due to
	* respawning rather than node start.
	*
	* This may be useful for the daemons to know
	*/
	setenv("PCMK_respawned", "true", 1);
	pacemakerd_state = XML_PING_ATTR_PACEMAKERDSTATE_RUNNING;
	return TRUE;
	}

	static void
	remove_core_file_limit(void)
	{
	struct rlimit cores;
	int rc = getrlimit(RLIMIT_CORE, &cores);

	if (rc < 0) {
	crm_warn("Cannot determine current maximum core file size: %s",
	strerror(errno));
	return;
	}

	if ((cores.rlim_max == 0) && (geteuid() == 0)) {
	cores.rlim_max = RLIM_INFINITY;
	} else {
	crm_info("Maximum core file size is %llu bytes",
	(unsigned long long) cores.rlim_max);
	}
	cores.rlim_cur = cores.rlim_max;

	rc = setrlimit(RLIMIT_CORE, &cores);
	if (rc < 0) {
	crm_warn("Cannot raise system limit on core file size "
	"(consider doing so manually)");
	}
	}

	static crm_exit_t
	request_shutdown(crm_ipc_t *ipc)
	{
	xmlNode *request = NULL;
	xmlNode *reply = NULL;
	int rc = 0;
	crm_exit_t status = CRM_EX_OK;

	request = create_request(CRM_OP_QUIT, NULL, NULL, CRM_SYSTEM_MCP,
	CRM_SYSTEM_MCP, NULL);
	if (request == NULL) {
	crm_err("Unable to create shutdown request"); // Probably memory error
	status = CRM_EX_TEMPFAIL;
	goto done;
	}

	crm_notice("Requesting shutdown of existing Pacemaker instance");
	rc = crm_ipc_send(ipc, request, crm_ipc_client_response, 0, &reply);
	if (rc < 0) {
	crm_err("Could not send shutdown request");
	status = crm_errno2exit(rc);
	goto done;
	}

	if ((rc == 0) \|\| (reply == NULL)) {
	crm_err("Unrecognized response to shutdown request");
	status = CRM_EX_PROTOCOL;
	goto done;
	}

	if ((crm_element_value_int(reply, "status", &rc) == 0)
	&& (rc != CRM_EX_OK)) {
	crm_err("Shutdown request failed: %s", crm_exit_str(rc));
	status = rc;
	goto done;
	}

	// Wait for pacemakerd to shut down IPC (with 30-minute timeout)
	status = CRM_EX_TIMEOUT;
	for (int i = 0; i < 900; ++i) {
	if (!crm_ipc_connected(ipc)) {
	status = CRM_EX_OK;
	break;
	}
	sleep(2);
	}

	done:
	free_xml(request);
	crm_ipc_close(ipc);
	crm_ipc_destroy(ipc);
	return status;
	}

	int
	main(int argc, char **argv)
	{
	int flag;
	int argerr = 0;

	int option_index = 0;
	bool old_instance_connected = false;
	gboolean shutdown = FALSE;

	uid_t pcmk_uid = 0;
	gid_t pcmk_gid = 0;
	crm_ipc_t *old_instance = NULL;
	qb_ipcs_service_t *ipcs = NULL;

	crm_log_preinit(NULL, argc, argv);
	pcmk__set_cli_options(NULL, "[options]", long_options,
	"primary Pacemaker daemon that launches and "
	"monitors all subsidiary Pacemaker daemons");
	mainloop_add_signal(SIGHUP, pcmk_ignore);
	mainloop_add_signal(SIGQUIT, pcmk_sigquit);

	while (1) {
	flag = pcmk__next_cli_option(argc, argv, &option_index, NULL);
	if (flag == -1)
	break;

	switch (flag) {
	case 'V':
	crm_bump_log_level(argc, argv);
	break;
	case 'f':
	/* Legacy */
	break;
	case 'p':
	pid_file = optarg;
	break;
	case 's':
	pcmk__set_env_option("node_start_state", "standby");
	break;
	case '$':
	case '?':
	pcmk__cli_help(flag, CRM_EX_OK);
	break;
	case 'S':
	shutdown = TRUE;
	break;
	case 'F':
	printf("Pacemaker %s (Build: %s)\n Supporting v%s: %s\n", PACEMAKER_VERSION, BUILD_VERSION,
	CRM_FEATURE_SET, CRM_FEATURES);
	crm_exit(CRM_EX_OK);
	default:
	printf("Argument code 0%o (%c) is not (?yet?) supported\n", flag, flag);
	++argerr;
	break;
	}
	}

	if (optind < argc) {
	printf("non-option ARGV-elements: ");
	while (optind < argc)
	printf("%s ", argv[optind++]);
	printf("\n");
	}
	if (argerr) {
	pcmk__cli_help('?', CRM_EX_USAGE);
	}


	setenv("LC_ALL", "C", 1);

	pcmk__set_env_option("mcp", "true");

	crm_log_init(NULL, LOG_INFO, TRUE, FALSE, argc, argv, FALSE);

	crm_debug("Checking for existing Pacemaker instance");
	old_instance = crm_ipc_new(CRM_SYSTEM_MCP, 0);
	old_instance_connected = crm_ipc_connect(old_instance);

	if (shutdown) {
	if (old_instance_connected) {
	crm_exit(request_shutdown(old_instance));
	} else {
	crm_err("Could not request shutdown of existing "
	"Pacemaker instance: %s", strerror(errno));
	crm_ipc_close(old_instance);
	crm_ipc_destroy(old_instance);
	crm_exit(CRM_EX_DISCONNECT);
	}

	} else if (old_instance_connected) {
	crm_ipc_close(old_instance);
	crm_ipc_destroy(old_instance);
	crm_err("Aborting start-up because active Pacemaker instance found");
	crm_exit(CRM_EX_FATAL);
	}

	crm_ipc_close(old_instance);
	crm_ipc_destroy(old_instance);

	#ifdef SUPPORT_COROSYNC
	if (mcp_read_config() == FALSE) {
	crm_exit(CRM_EX_UNAVAILABLE);
	}
	#endif

	// OCF shell functions and cluster-glue need facility under different name
	{
	const char *facility = pcmk__env_option("logfacility");

	if (facility && !pcmk__str_eq(facility, "none", pcmk__str_casei)) {
	setenv("HA_LOGFACILITY", facility, 1);
	}
	}

	crm_notice("Starting Pacemaker %s "CRM_XS" build=%s features:%s",
	PACEMAKER_VERSION, BUILD_VERSION, CRM_FEATURES);
	mainloop = g_main_loop_new(NULL, FALSE);

	remove_core_file_limit();

	if (pcmk_daemon_user(&pcmk_uid, &pcmk_gid) < 0) {
	crm_err("Cluster user %s does not exist, aborting Pacemaker startup", CRM_DAEMON_USER);
	crm_exit(CRM_EX_NOUSER);
	}

	// Used by some resource agents
	if ((mkdir(CRM_STATE_DIR, 0750) < 0) && (errno != EEXIST)) {
	crm_warn("Could not create " CRM_STATE_DIR ": %s", pcmk_strerror(errno));
	} else {
	mcp_chown(CRM_STATE_DIR, pcmk_uid, pcmk_gid);
	}

	/* Used to store core/blackbox/scheduler/cib files in */
	crm_build_path(CRM_PACEMAKER_DIR, 0750);
	mcp_chown(CRM_PACEMAKER_DIR, pcmk_uid, pcmk_gid);

	/* Used to store core files in */
	crm_build_path(CRM_CORE_DIR, 0750);
	mcp_chown(CRM_CORE_DIR, pcmk_uid, pcmk_gid);

	/* Used to store blackbox dumps in */
	crm_build_path(CRM_BLACKBOX_DIR, 0750);
	mcp_chown(CRM_BLACKBOX_DIR, pcmk_uid, pcmk_gid);

	// Used to store scheduler inputs in
	crm_build_path(PE_STATE_DIR, 0750);
	mcp_chown(PE_STATE_DIR, pcmk_uid, pcmk_gid);

	/* Used to store the cluster configuration */
	crm_build_path(CRM_CONFIG_DIR, 0750);
	mcp_chown(CRM_CONFIG_DIR, pcmk_uid, pcmk_gid);

	// Don't build CRM_RSCTMP_DIR, pacemaker-execd will do it

	ipcs = mainloop_add_ipc_server(CRM_SYSTEM_MCP, QB_IPC_NATIVE, &mcp_ipc_callbacks);
	if (ipcs == NULL) {
	crm_err("Couldn't start IPC server");
	crm_exit(CRM_EX_OSERR);
	}

	#ifdef SUPPORT_COROSYNC
	/* Allows us to block shutdown */
	if (!cluster_connect_cfg()) {
	crm_exit(CRM_EX_PROTOCOL);
	}
	#endif

	if (pcmk__locate_sbd() > 0) {
	setenv("PCMK_watchdog", "true", 1);
	running_with_sbd = TRUE;
	} else {
	setenv("PCMK_watchdog", "false", 1);
	}

	switch (find_and_track_existing_processes()) {
	case pcmk_rc_ok:
	break;
	case pcmk_rc_ipc_unauthorized:
	crm_exit(CRM_EX_CANTCREAT);
	default:
	crm_exit(CRM_EX_FATAL);
	};

	mainloop_add_signal(SIGTERM, pcmk_shutdown);
	mainloop_add_signal(SIGINT, pcmk_shutdown);

	if ((running_with_sbd) && pcmk__get_sbd_sync_resource_startup()) {
	crm_notice("Waiting for startup-trigger from SBD.");
	pacemakerd_state = XML_PING_ATTR_PACEMAKERDSTATE_WAITPING;
	startup_trigger = mainloop_add_trigger(G_PRIORITY_HIGH, init_children_processes, NULL);
	} else {
	if (running_with_sbd) {
	crm_warn("Enabling SBD_SYNC_RESOURCE_STARTUP would (if supported "
	"by your SBD version) improve reliability of "
	"interworking between SBD & pacemaker.");
	}
	pacemakerd_state = XML_PING_ATTR_PACEMAKERDSTATE_STARTINGDAEMONS;
	init_children_processes(NULL);
	}

	crm_notice("Pacemaker daemon successfully started and accepting connections");
	g_main_loop_run(mainloop);

	if (ipcs) {
	crm_trace("Closing IPC server");
	mainloop_del_ipc_server(ipcs);
	ipcs = NULL;
	}

	g_main_loop_unref(mainloop);
	#ifdef SUPPORT_COROSYNC
	cluster_disconnect_cfg();
	#endif
	crm_exit(CRM_EX_OK);
	}
	diff --git a/daemons/pacemakerd/pcmkd_corosync.c b/daemons/pacemakerd/pcmkd_corosync.c
	index 7d00c3e64b..06a435a394 100644
	--- a/daemons/pacemakerd/pcmkd_corosync.c
	+++ b/daemons/pacemakerd/pcmkd_corosync.c
	@@ -1,306 +1,306 @@
	/*
	* Copyright 2010-2020 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>
	#include "pacemakerd.h"

	#include <sys/utsname.h>
	#include <sys/stat.h> /* for calls to stat() */
	#include <libgen.h> /* For basename() and dirname() */

	#include <sys/types.h>
	#include <pwd.h> /* For getpwname() */

	#include <corosync/hdb.h>
	#include <corosync/cfg.h>
	#include <corosync/cpg.h>
	#include <corosync/cmap.h>

	#include <crm/cluster/internal.h>
	#include <crm/common/ipc.h> /* for crm_ipc_is_authentic_process */
	#include <crm/common/mainloop.h>

	#include <crm/common/ipc_internal.h> /* PCMK__SPECIAL_PID* */

	static corosync_cfg_handle_t cfg_handle = 0;

	/* =::=::=::= CFG - Shutdown stuff =::=::=::= */

	static void
	cfg_shutdown_callback(corosync_cfg_handle_t h, corosync_cfg_shutdown_flags_t flags)
	{
	crm_info("Corosync wants to shut down: %s",
	(flags == COROSYNC_CFG_SHUTDOWN_FLAG_IMMEDIATE) ? "immediate" :
	(flags == COROSYNC_CFG_SHUTDOWN_FLAG_REGARDLESS) ? "forced" : "optional");

	/* Never allow corosync to shut down while we're running */
	corosync_cfg_replyto_shutdown(h, COROSYNC_CFG_SHUTDOWN_FLAG_NO);
	}

	static corosync_cfg_callbacks_t cfg_callbacks = {
	.corosync_cfg_shutdown_callback = cfg_shutdown_callback,
	};

	static int
	pcmk_cfg_dispatch(gpointer user_data)
	{
	corosync_cfg_handle_t handle = (corosync_cfg_handle_t ) user_data;
	cs_error_t rc = corosync_cfg_dispatch(*handle, CS_DISPATCH_ALL);

	if (rc != CS_OK) {
	return -1;
	}
	return 0;
	}

	static void
	cfg_connection_destroy(gpointer user_data)
	{
	crm_err("Lost connection to cluster layer");
	corosync_cfg_finalize(cfg_handle);
	cfg_handle = 0;
	pcmk_shutdown(SIGTERM);
	}

	gboolean
	cluster_disconnect_cfg(void)
	{
	if (cfg_handle) {
	corosync_cfg_finalize(cfg_handle);
	cfg_handle = 0;
	}

	pcmk_shutdown(SIGTERM);
	return TRUE;
	}

	#define cs_repeat(counter, max, code) do { \
	code; \
	if(rc == CS_ERR_TRY_AGAIN \|\| rc == CS_ERR_QUEUE_FULL) { \
	counter++; \
	crm_debug("Retrying Corosync operation after %ds", counter); \
	sleep(counter); \
	} else { \
	break; \
	} \
	} while(counter < max)

	gboolean
	cluster_connect_cfg(void)
	{
	cs_error_t rc;
	int fd = -1, retries = 0, rv;
	uid_t found_uid = 0;
	gid_t found_gid = 0;
	pid_t found_pid = 0;
	uint32_t nodeid;

	static struct mainloop_fd_callbacks cfg_fd_callbacks = {
	.dispatch = pcmk_cfg_dispatch,
	.destroy = cfg_connection_destroy,
	};

	cs_repeat(retries, 30, rc = corosync_cfg_initialize(&cfg_handle, &cfg_callbacks));

	if (rc != CS_OK) {
	crm_crit("Could not connect to Corosync CFG: %s " CRM_XS " rc=%d",
	cs_strerror(rc), rc);
	return FALSE;
	}

	rc = corosync_cfg_fd_get(cfg_handle, &fd);
	if (rc != CS_OK) {
	crm_crit("Could not get Corosync CFG descriptor: %s " CRM_XS " rc=%d",
	cs_strerror(rc), rc);
	goto bail;
	}

	/* CFG provider run as root (in given user namespace, anyway)? */
	if (!(rv = crm_ipc_is_authentic_process(fd, (uid_t) 0,(gid_t) 0, &found_pid,
	&found_uid, &found_gid))) {
	crm_crit("Rejecting Corosync CFG provider because process %lld "
	"is running as uid %lld gid %lld, not root",
	(long long) PCMK__SPECIAL_PID_AS_0(found_pid),
	(long long) found_uid, (long long) found_gid);
	goto bail;
	} else if (rv < 0) {
	crm_crit("Could not authenticate Corosync CFG provider: %s "
	CRM_XS " rc=%d", strerror(-rv), -rv);
	goto bail;
	}

	retries = 0;
	cs_repeat(retries, 30, rc = corosync_cfg_local_get(cfg_handle, &nodeid));
	if (rc != CS_OK) {
	crm_crit("Could not get local node ID from Corosync: %s "
	CRM_XS " rc=%d", cs_strerror(rc), rc);
	goto bail;
	}
	crm_debug("Corosync reports local node ID is %lu", (unsigned long) nodeid);

	mainloop_add_fd("corosync-cfg", G_PRIORITY_DEFAULT, fd, &cfg_handle, &cfg_fd_callbacks);
	return TRUE;

	bail:
	corosync_cfg_finalize(cfg_handle);
	return FALSE;
	}

	void
	pcmkd_shutdown_corosync(void)
	{
	cs_error_t rc;

	if (cfg_handle == 0) {
	crm_warn("Unable to shut down Corosync: No connection");
	return;
	}
	crm_info("Asking Corosync to shut down");
	rc = corosync_cfg_try_shutdown(cfg_handle,
	COROSYNC_CFG_SHUTDOWN_FLAG_IMMEDIATE);
	if (rc == CS_OK) {
	corosync_cfg_finalize(cfg_handle);
	cfg_handle = 0;
	} else {
	crm_warn("Corosync shutdown failed: %s " CRM_XS " rc=%d",
	cs_strerror(rc), rc);
	}
	}


	/* =::=::=::= Configuration =::=::=::= */
	static int
	get_config_opt(uint64_t unused, cmap_handle_t object_handle, const char key, char *value,
	const char *fallback)
	{
	int rc = 0, retries = 0;

	cs_repeat(retries, 5, rc = cmap_get_string(object_handle, key, value));
	if (rc != CS_OK) {
	crm_trace("Search for %s failed %d, defaulting to %s", key, rc, fallback);
	if (fallback) {
	*value = strdup(fallback);
	} else {
	*value = NULL;
	}
	}
	crm_trace("%s: %s", key, *value);
	return rc;
	}

	gboolean
	mcp_read_config(void)
	{
	cs_error_t rc = CS_OK;
	int retries = 0;
	cmap_handle_t local_handle;
	uint64_t config = 0;
	int fd = -1;
	uid_t found_uid = 0;
	gid_t found_gid = 0;
	pid_t found_pid = 0;
	int rv;
	enum cluster_type_e stack;

	// There can be only one possibility
	do {
	rc = cmap_initialize(&local_handle);
	if (rc != CS_OK) {
	retries++;
	crm_info("Could not connect to Corosync CMAP: %s (retrying in %ds) "
	CRM_XS " rc=%d", cs_strerror(rc), retries, rc);
	sleep(retries);

	} else {
	break;
	}

	} while (retries < 5);

	if (rc != CS_OK) {
	crm_crit("Could not connect to Corosync CMAP: %s "
	CRM_XS " rc=%d", cs_strerror(rc), rc);
	return FALSE;
	}

	rc = cmap_fd_get(local_handle, &fd);
	if (rc != CS_OK) {
	crm_crit("Could not get Corosync CMAP descriptor: %s " CRM_XS " rc=%d",
	cs_strerror(rc), rc);
	cmap_finalize(local_handle);
	return FALSE;
	}

	/* CMAP provider run as root (in given user namespace, anyway)? */
	if (!(rv = crm_ipc_is_authentic_process(fd, (uid_t) 0,(gid_t) 0, &found_pid,
	&found_uid, &found_gid))) {
	crm_crit("Rejecting Corosync CMAP provider because process %lld "
	"is running as uid %lld gid %lld, not root",
	(long long) PCMK__SPECIAL_PID_AS_0(found_pid),
	(long long) found_uid, (long long) found_gid);
	cmap_finalize(local_handle);
	return FALSE;
	} else if (rv < 0) {
	crm_crit("Could not authenticate Corosync CMAP provider: %s "
	CRM_XS " rc=%d", strerror(-rv), -rv);
	cmap_finalize(local_handle);
	return FALSE;
	}

	stack = get_cluster_type();
	if (stack != pcmk_cluster_corosync) {
	crm_crit("Expected corosync stack but detected %s " CRM_XS " stack=%d",
	name_for_cluster_type(stack), stack);
	return FALSE;
	}

	crm_info("Reading configuration for %s stack",
	name_for_cluster_type(stack));
	pcmk__set_env_option("cluster_type", "corosync");
	pcmk__set_env_option("quorum_type", "corosync");

	// If debug logging is not configured, check whether corosync has it
	if (pcmk__env_option("debug") == NULL) {
	char *debug_enabled = NULL;

	get_config_opt(config, local_handle, "logging.debug", &debug_enabled, "off");

	if (crm_is_true(debug_enabled)) {
	pcmk__set_env_option("debug", "1");
	if (get_crm_log_level() < LOG_DEBUG) {
	set_crm_log_level(LOG_DEBUG);
	}

	} else {
	pcmk__set_env_option("debug", "0");
	}

	free(debug_enabled);
	}

	if(local_handle){
	gid_t gid = 0;
	if (pcmk_daemon_user(NULL, &gid) < 0) {
	crm_warn("Could not authorize group with Corosync " CRM_XS
	" No group found for user %s", CRM_DAEMON_USER);

	} else {
	char key[PATH_MAX];
	snprintf(key, PATH_MAX, "uidgid.gid.%u", gid);
	rc = cmap_set_uint8(local_handle, key, 1);
	if (rc != CS_OK) {
	crm_warn("Could not authorize group with Corosync: %s " CRM_XS
	- " group=%u rc=%d", ais_error2text(rc), gid, rc);
	+ " group=%u rc=%d", pcmk__cs_err_str(rc), gid, rc);
	}
	}
	}
	cmap_finalize(local_handle);

	return TRUE;
	}
	diff --git a/include/crm/cluster.h b/include/crm/cluster.h
	index a9e3dc363f..27f2bf603d 100644
	--- a/include/crm/cluster.h
	+++ b/include/crm/cluster.h
	@@ -1,214 +1,228 @@
	/*
	- * Copyright 2004-2018 the Pacemaker project contributors
	+ * Copyright 2004-2020 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#ifndef CRM_COMMON_CLUSTER__H
	# define CRM_COMMON_CLUSTER__H

	#ifdef __cplusplus
	extern "C" {
	#endif

	# include <stdint.h> // uint32_t, uint64_t
	# include <glib.h> // gboolean, GHashTable
	# include <libxml/tree.h> // xmlNode
	# include <crm/common/xml.h>
	# include <crm/common/util.h>

	# if SUPPORT_COROSYNC
	# include <corosync/cpg.h>
	# endif

	extern gboolean crm_have_quorum;
	extern GHashTable *crm_peer_cache;
	extern GHashTable *crm_remote_peer_cache;
	extern unsigned long long crm_peer_seq;

	-/* INDENT-OFF */
	#define CRM_NODE_LOST "lost"
	#define CRM_NODE_MEMBER "member"

	-enum crm_join_phase
	-{
	+enum crm_join_phase {
	crm_join_nack = -1,
	crm_join_none = 0,
	crm_join_welcomed = 1,
	crm_join_integrated = 2,
	crm_join_finalized = 3,
	crm_join_confirmed = 4,
	};

	-enum crm_node_flags
	-{
	+enum crm_node_flags {
	/* node is not a cluster node and should not be considered for cluster membership */
	crm_remote_node = 0x0001,

	/* node's cache entry is dirty */
	crm_node_dirty = 0x0010,
	};
	-/* INDENT-ON */

	typedef struct crm_peer_node_s {
	char *uname; // Node name as known to cluster
	char *uuid; // Node UUID to ensure uniqueness
	char *state; // @TODO change to enum
	uint64_t flags; // Bitmask of crm_node_flags
	uint64_t last_seen; // Only needed by cluster nodes
	uint32_t processes; // @TODO most not needed, merge into flags

	+ /* @TODO When we can break public API compatibility, we can make the rest of
	+ * these members separate structs and use void *cluster_data and
	+ * void *user_data here instead, to abstract the cluster layer further.
	+ */
	+
	// Currently only needed by corosync stack
	uint32_t id; // Node ID
	time_t when_lost; // When CPG membership was last lost

	// Only used by controller
	enum crm_join_phase join;
	char *expected;
	} crm_node_t;

	void crm_peer_init(void);
	void crm_peer_destroy(void);

	typedef struct crm_cluster_s {
	char *uuid;
	char *uname;
	uint32_t nodeid;

	void (*destroy) (gpointer);

	# if SUPPORT_COROSYNC
	+ /* @TODO When we can break public API compatibility, make these members a
	+ * separate struct and use void *cluster_data here instead, to abstract the
	+ * cluster layer further.
	+ */
	struct cpg_name group;
	cpg_callbacks_t cpg;
	cpg_handle_t cpg_handle;
	# endif

	} crm_cluster_t;

	-gboolean crm_cluster_connect(crm_cluster_t * cluster);
	-void crm_cluster_disconnect(crm_cluster_t * cluster);
	+gboolean crm_cluster_connect(crm_cluster_t *cluster);
	+void crm_cluster_disconnect(crm_cluster_t *cluster);

	-/* INDENT-OFF */
	enum crm_ais_msg_class {
	crm_class_cluster = 0,
	};

	enum crm_ais_msg_types {
	crm_msg_none = 0,
	crm_msg_ais = 1,
	crm_msg_lrmd = 2,
	crm_msg_cib = 3,
	crm_msg_crmd = 4,
	crm_msg_attrd = 5,
	crm_msg_stonithd = 6,
	crm_msg_te = 7,
	crm_msg_pe = 8,
	crm_msg_stonith_ng = 9,
	};

	/* used with crm_get_peer_full */
	enum crm_get_peer_flags {
	CRM_GET_PEER_CLUSTER = 0x0001,
	CRM_GET_PEER_REMOTE = 0x0002,
	CRM_GET_PEER_ANY = CRM_GET_PEER_CLUSTER\|CRM_GET_PEER_REMOTE,
	};
	-/* INDENT-ON */
	-
	-gboolean send_cluster_message(crm_node_t * node, enum crm_ais_msg_types service,
	- xmlNode * data, gboolean ordered);

	+gboolean send_cluster_message(crm_node_t *node, enum crm_ais_msg_types service,
	+ xmlNode *data, gboolean ordered);

	int crm_remote_peer_cache_size(void);

	/* Initialize and refresh the remote peer cache from a cib config */
	void crm_remote_peer_cache_refresh(xmlNode *cib);
	crm_node_t crm_remote_peer_get(const char node_name);
	void crm_remote_peer_cache_remove(const char *node_name);

	/* allows filtering of remote and cluster nodes using crm_get_peer_flags */
	crm_node_t crm_get_peer_full(unsigned int id, const char uname, int flags);

	/* only searches cluster nodes */
	crm_node_t crm_get_peer(unsigned int id, const char uname);

	guint crm_active_peers(void);
	gboolean crm_is_peer_active(const crm_node_t * node);
	guint reap_crm_member(uint32_t id, const char *name);
	-int crm_terminate_member(int nodeid, const char uname, void unused);
	-int crm_terminate_member_no_mainloop(int nodeid, const char uname, int connection);

	# if SUPPORT_COROSYNC
	uint32_t get_local_nodeid(cpg_handle_t handle);

	gboolean cluster_connect_cpg(crm_cluster_t *cluster);
	void cluster_disconnect_cpg(crm_cluster_t * cluster);

	void pcmk_cpg_membership(cpg_handle_t handle,
	const struct cpg_name *groupName,
	const struct cpg_address *member_list, size_t member_list_entries,
	const struct cpg_address *left_list, size_t left_list_entries,
	const struct cpg_address *joined_list, size_t joined_list_entries);
	gboolean crm_is_corosync_peer_active(const crm_node_t * node);
	gboolean send_cluster_text(enum crm_ais_msg_class msg_class, const char *data,
	gboolean local, crm_node_t * node,
	enum crm_ais_msg_types dest);
	char pcmk_message_common_cs(cpg_handle_t handle, uint32_t nodeid, uint32_t pid, void msg,
	uint32_t kind, const char *from);
	# endif

	const char crm_peer_uuid(crm_node_t node);
	const char crm_peer_uname(const char uuid);
	void set_uuid(xmlNode xml, const char attr, crm_node_t *node);

	enum crm_status_type {
	crm_status_uname,
	crm_status_nstate,
	crm_status_processes,
	};

	enum crm_ais_msg_types text2msg_type(const char *text);
	void crm_set_status_callback(void (dispatch) (enum crm_status_type, crm_node_t , const void *));
	void crm_set_autoreap(gboolean autoreap);

	-/* INDENT-OFF */
	-enum cluster_type_e
	-{
	+enum cluster_type_e {
	pcmk_cluster_unknown = 0x0001,
	pcmk_cluster_invalid = 0x0002,
	// 0x0004 was heartbeat
	// 0x0010 was corosync 1 with plugin
	pcmk_cluster_corosync = 0x0020,
	// 0x0040 was corosync 1 with CMAN
	};
	-/* INDENT-ON */

	enum cluster_type_e get_cluster_type(void);
	const char *name_for_cluster_type(enum cluster_type_e type);

	gboolean is_corosync_cluster(void);

	const char *get_local_node_name(void);
	char *get_node_name(uint32_t nodeid);

	+/*!
	+ * \brief Get log-friendly string equivalent of a join phase
	+ *
	+ * \param[in] phase Join phase
	+ *
	+ * \return Log-friendly string equivalent of \p phase
	+ */
	static inline const char *
	crm_join_phase_str(enum crm_join_phase phase)
	{
	switch (phase) {
	case crm_join_nack: return "nack";
	case crm_join_none: return "none";
	case crm_join_welcomed: return "welcomed";
	case crm_join_integrated: return "integrated";
	case crm_join_finalized: return "finalized";
	case crm_join_confirmed: return "confirmed";
	}
	return "invalid";
	}

	+#ifndef PCMK__NO_COMPAT
	+/* Everything here is deprecated and kept only for public API backward
	+ * compatibility. It will be moved to compatibility.h in a future release.
	+ */
	+
	+int crm_terminate_member(int nodeid, const char uname, void unused);
	+int crm_terminate_member_no_mainloop(int nodeid, const char *uname,
	+ int *connection);
	+#endif
	+
	#ifdef __cplusplus
	}
	#endif

	#endif
	diff --git a/include/crm/cluster/internal.h b/include/crm/cluster/internal.h
	index 881d015d4a..bf75b8cc87 100644
	--- a/include/crm/cluster/internal.h
	+++ b/include/crm/cluster/internal.h
	@@ -1,364 +1,122 @@
	/*
	* Copyright 2004-2020 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#ifndef CRM_CLUSTER_INTERNAL__H
	# define CRM_CLUSTER_INTERNAL__H

	+# include <stdint.h> // uint32_t, uint64_t
	# include <crm/cluster.h>

	-#define pcmk__set_peer_flags(peer, flags_to_set) do { \
	- (peer)->flags = pcmk__set_flags_as(__func__, __LINE__, LOG_TRACE, \
	- "Peer", (peer)->uname, \
	- (peer)->flags, (flags_to_set), \
	- #flags_to_set); \
	- } while (0)
	-
	-#define pcmk__clear_peer_flags(peer, flags_to_clear) do { \
	- (peer)->flags = pcmk__clear_flags_as(__func__, __LINE__, \
	- LOG_TRACE, \
	- "Peer", (peer)->uname, \
	- (peer)->flags, (flags_to_clear), \
	- #flags_to_clear); \
	- } while (0)
	-
	-typedef struct crm_ais_host_s AIS_Host;
	-typedef struct crm_ais_msg_s AIS_Message;
	-
	-struct crm_ais_host_s {
	- uint32_t id;
	- uint32_t pid;
	- gboolean local;
	- enum crm_ais_msg_types type;
	- uint32_t size;
	- char uname[MAX_NAME];
	-
	-} __attribute__ ((packed));
	-
	-#if SUPPORT_COROSYNC
	-# include <qb/qbipc_common.h>
	-# include <corosync/corotypes.h>
	-typedef struct qb_ipc_response_header cs_ipc_header_response_t;
	-#else
	-typedef struct {
	- int size __attribute__ ((aligned(8)));
	- int id __attribute__ ((aligned(8)));
	- int error __attribute__ ((aligned(8)));
	-} __attribute__ ((aligned(8))) cs_ipc_header_response_t;
	-#endif
	-
	-struct crm_ais_msg_s {
	- cs_ipc_header_response_t header __attribute__ ((aligned(8)));
	- uint32_t id;
	- gboolean is_compressed;
	-
	- AIS_Host host;
	- AIS_Host sender;
	-
	- uint32_t size;
	- uint32_t compressed_size;
	- /* 584 bytes */
	- char data[0];
	-
	-} __attribute__ ((packed));
	-
	/* INDENT-OFF */
	enum crm_proc_flag {
	crm_proc_none = 0x00000001,

	// Cluster layers
	crm_proc_cpg = 0x04000000,

	// Daemons
	crm_proc_execd = 0x00000010,
	crm_proc_based = 0x00000100,
	crm_proc_controld = 0x00000200,
	crm_proc_attrd = 0x00001000,
	crm_proc_schedulerd = 0x00010000,
	crm_proc_fenced = 0x00100000,
	};
	/* INDENT-ON */

	/*!
	* \internal
	* \brief Return the process bit corresponding to the current cluster stack
	*
	* \return Process flag if detectable, otherwise 0
	*/
	static inline uint32_t
	crm_get_cluster_proc(void)
	{
	switch (get_cluster_type()) {
	case pcmk_cluster_corosync:
	return crm_proc_cpg;

	default:
	break;
	}
	return crm_proc_none;
	}

	-static inline const char *
	-peer2text(enum crm_proc_flag proc)
	-{
	- const char *text = "unknown";
	-
	- switch (proc) {
	- case crm_proc_none:
	- text = "none";
	- break;
	- case crm_proc_based:
	- text = "pacemaker-based";
	- break;
	- case crm_proc_controld:
	- text = "pacemaker-controld";
	- break;
	- case crm_proc_schedulerd:
	- text = "pacemaker-schedulerd";
	- break;
	- case crm_proc_execd:
	- text = "pacemaker-execd";
	- break;
	- case crm_proc_attrd:
	- text = "pacemaker-attrd";
	- break;
	- case crm_proc_fenced:
	- text = "pacemaker-fenced";
	- break;
	- case crm_proc_cpg:
	- text = "corosync-cpg";
	- break;
	- }
	- return text;
	-}
	-
	-static inline const char *
	-ais_dest(const AIS_Host *host)
	-{
	- if (host->local) {
	- return "local";
	- } else if (host->size > 0) {
	- return host->uname;
	- } else {
	- return "<all>";
	- }
	-}
	-
	-# define ais_data_len(msg) (msg->is_compressed?msg->compressed_size:msg->size)
	-
	-/*
	-typedef enum {
	- CS_OK = 1,
	- CS_ERR_LIBRARY = 2,
	- CS_ERR_VERSION = 3,
	- CS_ERR_INIT = 4,
	- CS_ERR_TIMEOUT = 5,
	- CS_ERR_TRY_AGAIN = 6,
	- CS_ERR_INVALID_PARAM = 7,
	- CS_ERR_NO_MEMORY = 8,
	- CS_ERR_BAD_HANDLE = 9,
	- CS_ERR_BUSY = 10,
	- CS_ERR_ACCESS = 11,
	- CS_ERR_NOT_EXIST = 12,
	- CS_ERR_NAME_TOO_LONG = 13,
	- CS_ERR_EXIST = 14,
	- CS_ERR_NO_SPACE = 15,
	- CS_ERR_INTERRUPT = 16,
	- CS_ERR_NAME_NOT_FOUND = 17,
	- CS_ERR_NO_RESOURCES = 18,
	- CS_ERR_NOT_SUPPORTED = 19,
	- CS_ERR_BAD_OPERATION = 20,
	- CS_ERR_FAILED_OPERATION = 21,
	- CS_ERR_MESSAGE_ERROR = 22,
	- CS_ERR_QUEUE_FULL = 23,
	- CS_ERR_QUEUE_NOT_AVAILABLE = 24,
	- CS_ERR_BAD_FLAGS = 25,
	- CS_ERR_TOO_BIG = 26,
	- CS_ERR_NO_SECTIONS = 27,
	- CS_ERR_CONTEXT_NOT_FOUND = 28,
	- CS_ERR_TOO_MANY_GROUPS = 30,
	- CS_ERR_SECURITY = 100
	-} cs_error_t;
	+/*!
	+ * \internal
	+ * \brief Get log-friendly string description of a Corosync return code
	+ *
	+ * \param[in] error Corosync return code
	+ *
	+ * \return Log-friendly string description corresponding to \p error
	*/
	static inline const char *
	-ais_error2text(int error)
	+pcmk__cs_err_str(int error)
	{
	- const char *text = "unknown";
	-
	# if SUPPORT_COROSYNC
	switch (error) {
	- case CS_OK:
	- text = "OK";
	- break;
	- case CS_ERR_LIBRARY:
	- text = "Library error";
	- break;
	- case CS_ERR_VERSION:
	- text = "Version error";
	- break;
	- case CS_ERR_INIT:
	- text = "Initialization error";
	- break;
	- case CS_ERR_TIMEOUT:
	- text = "Timeout";
	- break;
	- case CS_ERR_TRY_AGAIN:
	- text = "Try again";
	- break;
	- case CS_ERR_INVALID_PARAM:
	- text = "Invalid parameter";
	- break;
	- case CS_ERR_NO_MEMORY:
	- text = "No memory";
	- break;
	- case CS_ERR_BAD_HANDLE:
	- text = "Bad handle";
	- break;
	- case CS_ERR_BUSY:
	- text = "Busy";
	- break;
	- case CS_ERR_ACCESS:
	- text = "Access error";
	- break;
	- case CS_ERR_NOT_EXIST:
	- text = "Doesn't exist";
	- break;
	- case CS_ERR_NAME_TOO_LONG:
	- text = "Name too long";
	- break;
	- case CS_ERR_EXIST:
	- text = "Exists";
	- break;
	- case CS_ERR_NO_SPACE:
	- text = "No space";
	- break;
	- case CS_ERR_INTERRUPT:
	- text = "Interrupt";
	- break;
	- case CS_ERR_NAME_NOT_FOUND:
	- text = "Name not found";
	- break;
	- case CS_ERR_NO_RESOURCES:
	- text = "No resources";
	- break;
	- case CS_ERR_NOT_SUPPORTED:
	- text = "Not supported";
	- break;
	- case CS_ERR_BAD_OPERATION:
	- text = "Bad operation";
	- break;
	- case CS_ERR_FAILED_OPERATION:
	- text = "Failed operation";
	- break;
	- case CS_ERR_MESSAGE_ERROR:
	- text = "Message error";
	- break;
	- case CS_ERR_QUEUE_FULL:
	- text = "Queue full";
	- break;
	- case CS_ERR_QUEUE_NOT_AVAILABLE:
	- text = "Queue not available";
	- break;
	- case CS_ERR_BAD_FLAGS:
	- text = "Bad flags";
	- break;
	- case CS_ERR_TOO_BIG:
	- text = "Too big";
	- break;
	- case CS_ERR_NO_SECTIONS:
	- text = "No sections";
	- break;
	+ case CS_OK: return "OK";
	+ case CS_ERR_LIBRARY: return "Library error";
	+ case CS_ERR_VERSION: return "Version error";
	+ case CS_ERR_INIT: return "Initialization error";
	+ case CS_ERR_TIMEOUT: return "Timeout";
	+ case CS_ERR_TRY_AGAIN: return "Try again";
	+ case CS_ERR_INVALID_PARAM: return "Invalid parameter";
	+ case CS_ERR_NO_MEMORY: return "No memory";
	+ case CS_ERR_BAD_HANDLE: return "Bad handle";
	+ case CS_ERR_BUSY: return "Busy";
	+ case CS_ERR_ACCESS: return "Access error";
	+ case CS_ERR_NOT_EXIST: return "Doesn't exist";
	+ case CS_ERR_NAME_TOO_LONG: return "Name too long";
	+ case CS_ERR_EXIST: return "Exists";
	+ case CS_ERR_NO_SPACE: return "No space";
	+ case CS_ERR_INTERRUPT: return "Interrupt";
	+ case CS_ERR_NAME_NOT_FOUND: return "Name not found";
	+ case CS_ERR_NO_RESOURCES: return "No resources";
	+ case CS_ERR_NOT_SUPPORTED: return "Not supported";
	+ case CS_ERR_BAD_OPERATION: return "Bad operation";
	+ case CS_ERR_FAILED_OPERATION: return "Failed operation";
	+ case CS_ERR_MESSAGE_ERROR: return "Message error";
	+ case CS_ERR_QUEUE_FULL: return "Queue full";
	+ case CS_ERR_QUEUE_NOT_AVAILABLE: return "Queue not available";
	+ case CS_ERR_BAD_FLAGS: return "Bad flags";
	+ case CS_ERR_TOO_BIG: return "Too big";
	+ case CS_ERR_NO_SECTIONS: return "No sections";
	}
	# endif
	- return text;
	-}
	-
	-static inline const char *
	-msg_type2text(enum crm_ais_msg_types type)
	-{
	- const char *text = "unknown";
	-
	- switch (type) {
	- case crm_msg_none:
	- text = "unknown";
	- break;
	- case crm_msg_ais:
	- text = "ais";
	- break;
	- case crm_msg_cib:
	- text = "cib";
	- break;
	- case crm_msg_crmd:
	- text = "crmd";
	- break;
	- case crm_msg_pe:
	- text = "pengine";
	- break;
	- case crm_msg_te:
	- text = "tengine";
	- break;
	- case crm_msg_lrmd:
	- text = "lrmd";
	- break;
	- case crm_msg_attrd:
	- text = "attrd";
	- break;
	- case crm_msg_stonithd:
	- text = "stonithd";
	- break;
	- case crm_msg_stonith_ng:
	- text = "stonith-ng";
	- break;
	- }
	- return text;
	+ return "Corosync error";
	}

	-gboolean check_message_sanity(const AIS_Message * msg, const char *data);
	-
	# if SUPPORT_COROSYNC
	-
	-gboolean send_cpg_iov(struct iovec * iov);
	-
	-char get_corosync_uuid(crm_node_t peer);
	-char corosync_node_name(uint64_t /cmap_handle_t */ cmap_handle, uint32_t nodeid);
	-char *corosync_cluster_name(void);
	-int corosync_cmap_has_config(const char *prefix);
	-
	-gboolean corosync_initialize_nodelist(void cluster, gboolean force_member, xmlNode xml_parent);
	-
	-gboolean send_cluster_message_cs(xmlNode * msg, gboolean local,
	- crm_node_t * node, enum crm_ais_msg_types dest);
	-
	-enum cluster_type_e find_corosync_variant(void);
	-
	-void terminate_cs_connection(crm_cluster_t * cluster);
	-gboolean init_cs_connection(crm_cluster_t * cluster);
	-gboolean init_cs_connection_once(crm_cluster_t * cluster);
	+char *pcmk__corosync_cluster_name(void);
	+bool pcmk__corosync_add_nodes(xmlNode *xml_parent);
	# endif

	crm_node_t crm_update_peer_proc(const char source, crm_node_t * peer,
	uint32_t flag, const char *status);
	-crm_node_t crm_update_peer_state(const char source, crm_node_t * node,
	- const char *state, uint64_t membership);
	-
	-void crm_update_peer_uname(crm_node_t node, const char uname);
	-void crm_update_peer_expected(const char source, crm_node_t node, const char *expected);
	-void crm_reap_unseen_nodes(uint64_t ring_id);
	-
	-gboolean cluster_connect_quorum(gboolean(*dispatch) (unsigned long long, gboolean),
	- void (*destroy) (gpointer));
	-
	-gboolean node_name_is_valid(const char key, const char name);
	-
	-crm_node_t * crm_find_peer_full(unsigned int id, const char *uname, int flags);
	-crm_node_t * crm_find_peer(unsigned int id, const char *uname);
	-
	-void crm_peer_caches_refresh(xmlNode *cib);
	-crm_node_t crm_find_known_peer_full(unsigned int id, const char uname, int flags);
	+crm_node_t pcmk__update_peer_state(const char source, crm_node_t *node,
	+ const char *state, uint64_t membership);
	+
	+void pcmk__update_peer_expected(const char source, crm_node_t node,
	+ const char *expected);
	+void pcmk__reap_unseen_nodes(uint64_t ring_id);
	+
	+void pcmk__corosync_quorum_connect(gboolean (*dispatch)(unsigned long long,
	+ gboolean),
	+ void (*destroy) (gpointer));
	+crm_node_t pcmk__search_node_caches(unsigned int id, const char uname,
	+ uint32_t flags);
	+crm_node_t pcmk__search_cluster_node_cache(unsigned int id, const char uname);
	+
	+void pcmk__refresh_node_caches_from_cib(xmlNode *cib);
	+crm_node_t pcmk__search_known_node_cache(unsigned int id, const char uname,
	+ uint32_t flags);

	#endif
	diff --git a/lib/cluster/Makefile.am b/lib/cluster/Makefile.am
	index a167083da5..9433570fc7 100644
	--- a/lib/cluster/Makefile.am
	+++ b/lib/cluster/Makefile.am
	@@ -1,27 +1,29 @@
	#
	# Copyright 2004-2018 the Pacemaker project contributors
	#
	# The version control history for this file may have further details.
	#
	# This source code is licensed under the GNU General Public License version 2
	# or later (GPLv2+) WITHOUT ANY WARRANTY.
	#
	include $(top_srcdir)/mk/common.mk

	+noinst_HEADERS = crmcluster_private.h
	+
	## libraries
	lib_LTLIBRARIES = libcrmcluster.la

	libcrmcluster_la_LDFLAGS = -version-info 29:5:0

	libcrmcluster_la_CFLAGS = $(CFLAGS_HARDENED_LIB)
	libcrmcluster_la_LDFLAGS += $(LDFLAGS_HARDENED_LIB)

	libcrmcluster_la_LIBADD = $(top_builddir)/lib/common/libcrmcommon.la $(top_builddir)/lib/fencing/libstonithd.la $(CLUSTERLIBS)

	libcrmcluster_la_SOURCES = election.c cluster.c membership.c
	if BUILD_CS_SUPPORT
	libcrmcluster_la_SOURCES += cpg.c corosync.c
	endif

	clean-generic:
	rm -f .log .debug .xml ~
	diff --git a/lib/cluster/cluster.c b/lib/cluster/cluster.c
	index 9e2971cdf1..36039dd56f 100644
	--- a/lib/cluster/cluster.c
	+++ b/lib/cluster/cluster.c
	@@ -1,354 +1,377 @@
	/*
	* Copyright 2004-2020 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>
	#include <dlfcn.h>

	#include <stdio.h>
	#include <unistd.h>
	#include <string.h>
	#include <stdlib.h>
	#include <time.h>
	#include <sys/param.h>
	#include <sys/types.h>

	#include <crm/crm.h>
	#include <crm/msg_xml.h>

	#include <crm/common/ipc.h>
	#include <crm/cluster/internal.h>
	+#include "crmcluster_private.h"

	CRM_TRACE_INIT_DATA(cluster);

	+/*!
	+ * \brief Get (and set if needed) a node's UUID
	+ *
	+ * \param[in] peer Node to check
	+ *
	+ * \return Node UUID of \p peer, or NULL if unknown
	+ */
	const char *
	crm_peer_uuid(crm_node_t *peer)
	{
	char *uuid = NULL;
	- enum cluster_type_e type = get_cluster_type();

	// Check simple cases first, to avoid any calls that might block
	- if(peer == NULL) {
	+ if (peer == NULL) {
	return NULL;
	-
	- } else if (peer->uuid) {
	+ }
	+ if (peer->uuid != NULL) {
	return peer->uuid;
	}

	- switch (type) {
	+ switch (get_cluster_type()) {
	case pcmk_cluster_corosync:
	#if SUPPORT_COROSYNC
	- uuid = get_corosync_uuid(peer);
	+ uuid = pcmk__corosync_uuid(peer);
	#endif
	break;

	case pcmk_cluster_unknown:
	case pcmk_cluster_invalid:
	crm_err("Unsupported cluster type");
	break;
	}

	peer->uuid = uuid;
	return peer->uuid;
	}

	+/*!
	+ * \brief Connect to the cluster layer
	+ *
	+ * \param[in] Initialized cluster object to connect
	+ *
	+ * \return TRUE on success, otherwise FALSE
	+ */
	gboolean
	-crm_cluster_connect(crm_cluster_t * cluster)
	+crm_cluster_connect(crm_cluster_t *cluster)
	{
	enum cluster_type_e type = get_cluster_type();

	- crm_notice("Connecting to cluster infrastructure: %s",
	+ crm_notice("Connecting to %s cluster infrastructure",
	name_for_cluster_type(type));
	switch (type) {
	case pcmk_cluster_corosync:
	#if SUPPORT_COROSYNC
	if (is_corosync_cluster()) {
	crm_peer_init();
	- return init_cs_connection(cluster);
	+ return pcmk__corosync_connect(cluster);
	}
	#endif
	break;
	default:
	break;
	}
	return FALSE;
	}

	+/*!
	+ * \brief Disconnect from the cluster layer
	+ *
	+ * \param[in] cluster Cluster object to disconnect
	+ */
	void
	-crm_cluster_disconnect(crm_cluster_t * cluster)
	+crm_cluster_disconnect(crm_cluster_t *cluster)
	{
	enum cluster_type_e type = get_cluster_type();

	- crm_info("Disconnecting from cluster infrastructure: %s",
	+ crm_info("Disconnecting from %s cluster infrastructure",
	name_for_cluster_type(type));
	switch (type) {
	case pcmk_cluster_corosync:
	#if SUPPORT_COROSYNC
	if (is_corosync_cluster()) {
	crm_peer_destroy();
	- terminate_cs_connection(cluster);
	+ pcmk__corosync_disconnect(cluster);
	}
	#endif
	break;
	default:
	break;
	}
	}

	+/*!
	+ * \brief Send an XML message via the cluster messaging layer
	+ *
	+ * \param[in] node Cluster node to send message to
	+ * \param[in] service Message type to use in message host info
	+ * \param[in] data XML message to send
	+ * \param[in] ordered Ignored for currently supported messaging layers
	+ *
	+ * \return TRUE on success, otherwise FALSE
	+ */
	gboolean
	-send_cluster_message(crm_node_t * node, enum crm_ais_msg_types service, xmlNode * data,
	- gboolean ordered)
	+send_cluster_message(crm_node_t *node, enum crm_ais_msg_types service,
	+ xmlNode *data, gboolean ordered)
	{
	switch (get_cluster_type()) {
	case pcmk_cluster_corosync:
	#if SUPPORT_COROSYNC
	- return send_cluster_message_cs(data, FALSE, node, service);
	+ return pcmk__cpg_send_xml(data, node, service);
	#endif
	break;
	default:
	break;
	}
	return FALSE;
	}

	+/*!
	+ * \brief Get the local node's name
	+ *
	+ * \return Local node's name
	+ * \note This will fatally exit if local node name cannot be known.
	+ */
	const char *
	get_local_node_name(void)
	{
	static char *name = NULL;

	- if(name) {
	- return name;
	+ if (name == NULL) {
	+ name = get_node_name(0);
	}
	- name = get_node_name(0);
	return name;
	}

	+/*!
	+ * \brief Get the node name corresponding to a cluster node ID
	+ *
	+ * \param[in] nodeid Node ID to check (or 0 for local node)
	+ *
	+ * \return Node name corresponding to \p nodeid
	+ * \note This will fatally exit if \p nodeid is 0 and local node name cannot be
	+ * known.
	+ */
	char *
	get_node_name(uint32_t nodeid)
	{
	char *name = NULL;
	- enum cluster_type_e stack;
	+ enum cluster_type_e stack = get_cluster_type();

	- stack = get_cluster_type();
	switch (stack) {
	# if SUPPORT_COROSYNC
	case pcmk_cluster_corosync:
	- name = corosync_node_name(0, nodeid);
	+ name = pcmk__corosync_name(0, nodeid);
	break;
	# endif

	default:
	crm_err("Unknown cluster type: %s (%d)", name_for_cluster_type(stack), stack);
	}

	- if(name == NULL && nodeid == 0) {
	+ if ((name == NULL) && (nodeid == 0)) {
	name = pcmk_hostname();
	if (name == NULL) {
	+ // @TODO Maybe let the caller decide what to do
	crm_err("Could not obtain the local %s node name",
	name_for_cluster_type(stack));
	crm_exit(CRM_EX_FATAL);
	}
	crm_notice("Defaulting to uname -n for the local %s node name",
	name_for_cluster_type(stack));
	}

	if (name == NULL) {
	- crm_notice("Could not obtain a node name for %s nodeid %u",
	+ crm_notice("Could not obtain a node name for %s node with id %u",
	name_for_cluster_type(stack), nodeid);
	}
	return name;
	}

	/*!
	* \brief Get the node name corresponding to a node UUID
	*
	* \param[in] uuid UUID of desired node
	*
	* \return name of desired node
	*
	* \note This relies on the remote peer cache being populated with all
	* remote nodes in the cluster, so callers should maintain that cache.
	*/
	const char *
	crm_peer_uname(const char *uuid)
	{
	GHashTableIter iter;
	crm_node_t *node = NULL;

	CRM_CHECK(uuid != NULL, return NULL);

	/* remote nodes have the same uname and uuid */
	if (g_hash_table_lookup(crm_remote_peer_cache, uuid)) {
	return uuid;
	}

	/* avoid blocking calls where possible */
	g_hash_table_iter_init(&iter, crm_peer_cache);
	while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &node)) {
	- if(node->uuid && strcasecmp(node->uuid, uuid) == 0) {
	- if(node->uname) {
	+ if (pcmk__str_eq(node->uuid, uuid, pcmk__str_casei)) {
	+ if (node->uname != NULL) {
	return node->uname;
	}
	break;
	}
	}
	node = NULL;

	#if SUPPORT_COROSYNC
	if (is_corosync_cluster()) {
	uint32_t id = (uint32_t) crm_parse_ll(uuid, NULL);

	if (id != 0) {
	- node = crm_find_peer(id, NULL);
	+ node = pcmk__search_cluster_node_cache(id, NULL);
	} else {
	crm_err("Invalid node id: %s", uuid);
	}

	- if (node) {
	- crm_info("Setting uuid for node %s[%u] to '%s'", node->uname, node->id, uuid);
	+ if (node != NULL) {
	+ crm_info("Setting uuid for node %s[%u] to %s",
	+ node->uname, node->id, uuid);
	node->uuid = strdup(uuid);
	- if(node->uname) {
	- return node->uname;
	- }
	+ return node->uname;
	}
	return NULL;
	}
	#endif

	return NULL;
	}

	+/*!
	+ * \brief Add a node's UUID as an XML attribute
	+ *
	+ * \param[in,out] xml XML element to add UUID to
	+ * \param[in] attr XML attribute name to set
	+ * \param[in] node Node whose UUID should be used as attribute value
	+ */
	void
	set_uuid(xmlNode xml, const char attr, crm_node_t *node)
	{
	- const char *uuid_calc = crm_peer_uuid(node);
	-
	- crm_xml_add(xml, attr, uuid_calc);
	- return;
	+ crm_xml_add(xml, attr, crm_peer_uuid(node));
	}

	+/*!
	+ * \brief Get a log-friendly string equivalent of a cluster type
	+ *
	+ * \param[in] type Cluster type
	+ *
	+ * \return Log-friendly string corresponding to \p type
	+ */
	const char *
	name_for_cluster_type(enum cluster_type_e type)
	{
	switch (type) {
	case pcmk_cluster_corosync:
	return "corosync";
	case pcmk_cluster_unknown:
	return "unknown";
	case pcmk_cluster_invalid:
	return "invalid";
	}
	crm_err("Invalid cluster type: %d", type);
	return "invalid";
	}

	-/* Do not expose these two */
	-int set_cluster_type(enum cluster_type_e type);
	-static enum cluster_type_e cluster_type = pcmk_cluster_unknown;
	-
	-int
	-set_cluster_type(enum cluster_type_e type)
	-{
	- if (cluster_type == pcmk_cluster_unknown) {
	- crm_info("Cluster type set to: %s", name_for_cluster_type(type));
	- cluster_type = type;
	- return 0;
	-
	- } else if (cluster_type == type) {
	- return 0;
	-
	- } else if (pcmk_cluster_unknown == type) {
	- cluster_type = type;
	- return 0;
	- }
	-
	- crm_err("Cluster type already set to %s, ignoring %s",
	- name_for_cluster_type(cluster_type), name_for_cluster_type(type));
	- return -1;
	-}
	+/*!
	+ * \brief Get (and validate) the local cluster type
	+ *
	+ * \return Local cluster type
	+ * \note This will fatally exit if the local cluster type is invalid.
	+ */
	enum cluster_type_e
	get_cluster_type(void)
	{
	- bool detected = FALSE;
	+ bool detected = false;
	const char *cluster = NULL;
	+ static enum cluster_type_e cluster_type = pcmk_cluster_unknown;

	/* Return the previous calculation, if any */
	if (cluster_type != pcmk_cluster_unknown) {
	return cluster_type;
	}

	cluster = pcmk__env_option("cluster_type");

	#if SUPPORT_COROSYNC
	/* If nothing is defined in the environment, try corosync (if supported) */
	- if(cluster == NULL) {
	+ if (cluster == NULL) {
	crm_debug("Testing with Corosync");
	- cluster_type = find_corosync_variant();
	+ cluster_type = pcmk__corosync_detect();
	if (cluster_type != pcmk_cluster_unknown) {
	- detected = TRUE;
	+ detected = true;
	goto done;
	}
	}
	#endif

	/* Something was defined in the environment, test it against what we support */
	- crm_info("Verifying cluster type: '%s'", cluster?cluster:"-unspecified-");
	+ crm_info("Verifying cluster type: '%s'",
	+ ((cluster == NULL)? "-unspecified-" : cluster));
	if (cluster == NULL) {

	#if SUPPORT_COROSYNC
	} else if (pcmk__str_eq(cluster, "corosync", pcmk__str_casei)) {
	cluster_type = pcmk_cluster_corosync;
	#endif

	} else {
	cluster_type = pcmk_cluster_invalid;
	goto done; /* Keep the compiler happy when no stacks are supported */
	}

	done:
	if (cluster_type == pcmk_cluster_unknown) {
	crm_notice("Could not determine the current cluster type");

	} else if (cluster_type == pcmk_cluster_invalid) {
	crm_notice("This installation does not support the '%s' cluster infrastructure: terminating.",
	cluster);
	crm_exit(CRM_EX_FATAL);

	} else {
	- crm_info("%s an active '%s' cluster", detected?"Detected":"Assuming", name_for_cluster_type(cluster_type));
	+ crm_info("%s an active '%s' cluster",
	+ (detected? "Detected" : "Assuming"),
	+ name_for_cluster_type(cluster_type));
	}

	return cluster_type;
	}

	+/*!
	+ * \brief Check whether the local cluster is a Corosync cluster
	+ *
	+ * \return TRUE if the local cluster is a Corosync cluster, otherwise FALSE
	+ */
	gboolean
	is_corosync_cluster(void)
	{
	return get_cluster_type() == pcmk_cluster_corosync;
	}
	-
	-gboolean
	-node_name_is_valid(const char key, const char name)
	-{
	- int octet;
	-
	- if (name == NULL) {
	- crm_trace("%s is empty", key);
	- return FALSE;
	-
	- } else if (sscanf(name, "%d.%d.%d.%d", &octet, &octet, &octet, &octet) == 4) {
	- crm_trace("%s contains an ipv4 address, ignoring: %s", key, name);
	- return FALSE;
	-
	- } else if (strstr(name, ":") != NULL) {
	- crm_trace("%s contains an ipv6 address, ignoring: %s", key, name);
	- return FALSE;
	- }
	- crm_trace("%s is valid", key);
	- return TRUE;
	-}
	diff --git a/lib/cluster/corosync.c b/lib/cluster/corosync.c
	index 0484691772..99bdecbe16 100644
	--- a/lib/cluster/corosync.c
	+++ b/lib/cluster/corosync.c
	@@ -1,778 +1,798 @@
	/*
	- * Copyright 2004-2019 the Pacemaker project contributors
	+ * Copyright 2004-2020 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <sys/socket.h>
	#include <netinet/in.h>
	#include <arpa/inet.h>
	#include <netdb.h>
	#include <inttypes.h> /* U64T ~ PRIu64 */

	#include <bzlib.h>

	#include <crm/common/ipc.h>
	#include <crm/cluster/internal.h>
	#include <crm/common/mainloop.h>
	#include <sys/utsname.h>

	#include <qb/qbipcc.h>
	#include <qb/qbutil.h>

	#include <corosync/corodefs.h>
	#include <corosync/corotypes.h>
	#include <corosync/hdb.h>
	#include <corosync/cfg.h>
	#include <corosync/cmap.h>
	#include <corosync/quorum.h>

	#include <crm/msg_xml.h>

	#include <crm/common/ipc_internal.h> /* PCMK__SPECIAL_PID* */
	+#include "crmcluster_private.h"

	-quorum_handle_t pcmk_quorum_handle = 0;
	+static quorum_handle_t pcmk_quorum_handle = 0;

	-gboolean(*quorum_app_callback) (unsigned long long seq, gboolean quorate) = NULL;
	+static gboolean (*quorum_app_callback)(unsigned long long seq,
	+ gboolean quorate) = NULL;

	+/*!
	+ * \internal
	+ * \brief Get the Corosync UUID associated with a Pacemaker node
	+ *
	+ * \param[in] node Pacemaker node
	+ *
	+ * \return Newly allocated string with node's Corosync UUID, or NULL if unknown
	+ * \note It is the caller's responsibility to free the result with free().
	+ */
	char *
	-get_corosync_uuid(crm_node_t *node)
	+pcmk__corosync_uuid(crm_node_t *node)
	{
	- if (node && is_corosync_cluster()) {
	+ if ((node != NULL) && is_corosync_cluster()) {
	if (node->id > 0) {
	return crm_strdup_printf("%u", node->id);
	} else {
	crm_info("Node %s is not yet known by corosync", node->uname);
	}
	}
	return NULL;
	}

	+static bool
	+node_name_is_valid(const char key, const char name)
	+{
	+ int octet;
	+
	+ if (name == NULL) {
	+ crm_trace("%s is empty", key);
	+ return false;
	+
	+ } else if (sscanf(name, "%d.%d.%d.%d", &octet, &octet, &octet, &octet) == 4) {
	+ crm_trace("%s contains an IPv4 address (%s), ignoring", key, name);
	+ return false;
	+
	+ } else if (strstr(name, ":") != NULL) {
	+ crm_trace("%s contains an IPv6 address (%s), ignoring", key, name);
	+ return false;
	+ }
	+ crm_trace("'%s: %s' is valid", key, name);
	+ return true;
	+}
	+
	/*
	- * CFG functionality stolen from node_name() in corosync-quorumtool.c
	- * This resolves the first address assigned to a node and returns the name or IP address.
	+ * \internal
	+ * \brief Get Corosync node name corresponding to a node ID
	+ *
	+ * \param[in] cmap_handle Connection to Corosync CMAP
	+ * \param[in] nodeid Node ID to check
	+ *
	+ * \return Newly allocated string with name or (if no name) IP address
	+ * associated with first address assigned to a Corosync node ID (or NULL
	+ * if unknown)
	+ * \note It is the caller's responsibility to free the result with free().
	*/
	char *
	-corosync_node_name(uint64_t /cmap_handle_t / cmap_handle, uint32_t nodeid)
	+pcmk__corosync_name(uint64_t /cmap_handle_t / cmap_handle, uint32_t nodeid)
	{
	+ // Originally based on corosync-quorumtool.c:node_name()
	+
	int lpc = 0;
	cs_error_t rc = CS_OK;
	int retries = 0;
	char *name = NULL;
	cmap_handle_t local_handle = 0;
	int fd = -1;
	uid_t found_uid = 0;
	gid_t found_gid = 0;
	pid_t found_pid = 0;
	int rv;

	if (nodeid == 0) {
	nodeid = get_local_nodeid(0);
	}

	if (cmap_handle == 0 && local_handle == 0) {
	retries = 0;
	crm_trace("Initializing CMAP connection");
	do {
	rc = cmap_initialize(&local_handle);
	if (rc != CS_OK) {
	retries++;
	crm_debug("API connection setup failed: %s. Retrying in %ds", cs_strerror(rc),
	retries);
	sleep(retries);
	}

	} while (retries < 5 && rc != CS_OK);

	if (rc != CS_OK) {
	crm_warn("Could not connect to Cluster Configuration Database API, error %s",
	cs_strerror(rc));
	local_handle = 0;
	}
	}

	if (cmap_handle == 0) {
	cmap_handle = local_handle;

	rc = cmap_fd_get(cmap_handle, &fd);
	if (rc != CS_OK) {
	crm_err("Could not obtain the CMAP API connection: %s (%d)",
	cs_strerror(rc), rc);
	goto bail;
	}

	/* CMAP provider run as root (in given user namespace, anyway)? */
	if (!(rv = crm_ipc_is_authentic_process(fd, (uid_t) 0,(gid_t) 0, &found_pid,
	&found_uid, &found_gid))) {
	crm_err("CMAP provider is not authentic:"
	" process %lld (uid: %lld, gid: %lld)",
	(long long) PCMK__SPECIAL_PID_AS_0(found_pid),
	(long long) found_uid, (long long) found_gid);
	goto bail;
	} else if (rv < 0) {
	crm_err("Could not verify authenticity of CMAP provider: %s (%d)",
	strerror(-rv), -rv);
	goto bail;
	}
	}

	while (name == NULL && cmap_handle != 0) {
	uint32_t id = 0;
	char *key = NULL;

	key = crm_strdup_printf("nodelist.node.%d.nodeid", lpc);
	rc = cmap_get_uint32(cmap_handle, key, &id);
	crm_trace("Checking %u vs %u from %s", nodeid, id, key);
	free(key);

	if (rc != CS_OK) {
	break;
	}

	if (nodeid == id) {
	- crm_trace("Searching for node name for %u in nodelist.node.%d %s", nodeid, lpc, name);
	+ crm_trace("Searching for node name for %u in nodelist.node.%d %s",
	+ nodeid, lpc, crm_str(name));
	if (name == NULL) {
	key = crm_strdup_printf("nodelist.node.%d.name", lpc);
	cmap_get_string(cmap_handle, key, &name);
	- crm_trace("%s = %s", key, name);
	+ crm_trace("%s = %s", key, crm_str(name));
	free(key);
	}
	if (name == NULL) {
	key = crm_strdup_printf("nodelist.node.%d.ring0_addr", lpc);
	cmap_get_string(cmap_handle, key, &name);
	- crm_trace("%s = %s", key, name);
	+ crm_trace("%s = %s", key, crm_str(name));

	- if (node_name_is_valid(key, name) == FALSE) {
	+ if (!node_name_is_valid(key, name)) {
	free(name);
	name = NULL;
	}
	free(key);
	}
	break;
	}

	lpc++;
	}

	bail:
	if(local_handle) {
	cmap_finalize(local_handle);
	}

	if (name == NULL) {
	crm_info("Unable to get node name for nodeid %u", nodeid);
	}
	return name;
	}

	+/*!
	+ * \internal
	+ * \brief Disconnect from Corosync cluster
	+ *
	+ * \param[in] cluster Cluster connection to disconnect
	+ */
	void
	-terminate_cs_connection(crm_cluster_t *cluster)
	+pcmk__corosync_disconnect(crm_cluster_t *cluster)
	{
	cluster_disconnect_cpg(cluster);
	if (pcmk_quorum_handle) {
	quorum_finalize(pcmk_quorum_handle);
	pcmk_quorum_handle = 0;
	}
	crm_notice("Disconnected from Corosync");
	}

	+/*!
	+ * \internal
	+ * \brief Dispatch function for quorum connection file descriptor
	+ *
	+ * \param[in] user_data Ignored
	+ *
	+ * \return 0 on success, -1 on error (per mainloop_io_t interface)
	+ */
	static int
	-pcmk_quorum_dispatch(gpointer user_data)
	+quorum_dispatch_cb(gpointer user_data)
	{
	- int rc = 0;
	+ int rc = quorum_dispatch(pcmk_quorum_handle, CS_DISPATCH_ALL);

	- rc = quorum_dispatch(pcmk_quorum_handle, CS_DISPATCH_ALL);
	if (rc < 0) {
	crm_err("Connection to the Quorum API failed: %d", rc);
	quorum_finalize(pcmk_quorum_handle);
	pcmk_quorum_handle = 0;
	return -1;
	}
	return 0;
	}

	+/*!
	+ * \internal
	+ * \brief Notification callback for Corosync quorum connection
	+ *
	+ * \param[in] handle Corosync quorum connection
	+ * \param[in] quorate Whether cluster is quorate
	+ * \param[in] ring_id Corosync ring ID
	+ * \param[in] view_list_entries Number of entries in \p view_list
	+ * \param[in] view_list Corosync node IDs in membership
	+ */
	static void
	-pcmk_quorum_notification(quorum_handle_t handle,
	- uint32_t quorate,
	- uint64_t ring_id, uint32_t view_list_entries, uint32_t * view_list)
	+quorum_notification_cb(quorum_handle_t handle, uint32_t quorate,
	+ uint64_t ring_id, uint32_t view_list_entries,
	+ uint32_t *view_list)
	{
	int i;
	GHashTableIter iter;
	crm_node_t *node = NULL;
	static gboolean init_phase = TRUE;

	if (quorate != crm_have_quorum) {
	if (quorate) {
	crm_notice("Quorum acquired " CRM_XS " membership=%" U64T " members=%lu",
	ring_id, (long unsigned int)view_list_entries);
	} else {
	crm_warn("Quorum lost " CRM_XS " membership=%" U64T " members=%lu",
	ring_id, (long unsigned int)view_list_entries);
	}
	crm_have_quorum = quorate;

	} else {
	crm_info("Quorum %s " CRM_XS " membership=%" U64T " members=%lu",
	(quorate? "retained" : "still lost"), ring_id,
	(long unsigned int)view_list_entries);
	}

	if (view_list_entries == 0 && init_phase) {
	crm_info("Corosync membership is still forming, ignoring");
	return;
	}

	init_phase = FALSE;

	/* Reset last_seen for all cached nodes so we can tell which ones aren't
	* in the view list */
	g_hash_table_iter_init(&iter, crm_peer_cache);
	while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &node)) {
	node->last_seen = 0;
	}

	/* Update the peer cache for each node in view list */
	for (i = 0; i < view_list_entries; i++) {
	uint32_t id = view_list[i];

	crm_debug("Member[%d] %u ", i, id);

	/* Get this node's peer cache entry (adding one if not already there) */
	node = crm_get_peer(id, NULL);
	if (node->uname == NULL) {
	- char *name = corosync_node_name(0, id);
	+ char *name = pcmk__corosync_name(0, id);

	crm_info("Obtaining name for new node %u", id);
	node = crm_get_peer(id, name);
	free(name);
	}

	/* Update the node state (including updating last_seen to ring_id) */
	- crm_update_peer_state(__func__, node, CRM_NODE_MEMBER, ring_id);
	+ pcmk__update_peer_state(__func__, node, CRM_NODE_MEMBER, ring_id);
	}

	/* Remove any peer cache entries we didn't update */
	- crm_reap_unseen_nodes(ring_id);
	+ pcmk__reap_unseen_nodes(ring_id);

	if (quorum_app_callback) {
	quorum_app_callback(ring_id, quorate);
	}
	}

	quorum_callbacks_t quorum_callbacks = {
	- .quorum_notify_fn = pcmk_quorum_notification,
	+ .quorum_notify_fn = quorum_notification_cb,
	};

	-gboolean
	-cluster_connect_quorum(gboolean(*dispatch) (unsigned long long, gboolean),
	- void (*destroy) (gpointer))
	+/*!
	+ * \internal
	+ * \brief Connect to Corosync quorum service
	+ *
	+ * \param[in] dispatch Connection dispatch callback
	+ * \param[in] destroy Connection destroy callback
	+ */
	+void
	+pcmk__corosync_quorum_connect(gboolean (*dispatch)(unsigned long long,
	+ gboolean),
	+ void (*destroy)(gpointer))
	{
	cs_error_t rc;
	int fd = 0;
	int quorate = 0;
	uint32_t quorum_type = 0;
	struct mainloop_fd_callbacks quorum_fd_callbacks;
	uid_t found_uid = 0;
	gid_t found_gid = 0;
	pid_t found_pid = 0;
	int rv;

	- quorum_fd_callbacks.dispatch = pcmk_quorum_dispatch;
	+ quorum_fd_callbacks.dispatch = quorum_dispatch_cb;
	quorum_fd_callbacks.destroy = destroy;

	crm_debug("Configuring Pacemaker to obtain quorum from Corosync");

	rc = quorum_initialize(&pcmk_quorum_handle, &quorum_callbacks, &quorum_type);
	if (rc != CS_OK) {
	crm_err("Could not connect to the Quorum API: %s (%d)",
	cs_strerror(rc), rc);
	goto bail;

	} else if (quorum_type != QUORUM_SET) {
	crm_err("Corosync quorum is not configured");
	goto bail;
	}

	rc = quorum_fd_get(pcmk_quorum_handle, &fd);
	if (rc != CS_OK) {
	crm_err("Could not obtain the Quorum API connection: %s (%d)",
	strerror(rc), rc);
	goto bail;
	}

	/* Quorum provider run as root (in given user namespace, anyway)? */
	if (!(rv = crm_ipc_is_authentic_process(fd, (uid_t) 0,(gid_t) 0, &found_pid,
	&found_uid, &found_gid))) {
	crm_err("Quorum provider is not authentic:"
	" process %lld (uid: %lld, gid: %lld)",
	(long long) PCMK__SPECIAL_PID_AS_0(found_pid),
	(long long) found_uid, (long long) found_gid);
	rc = CS_ERR_ACCESS;
	goto bail;
	} else if (rv < 0) {
	crm_err("Could not verify authenticity of Quorum provider: %s (%d)",
	strerror(-rv), -rv);
	rc = CS_ERR_ACCESS;
	goto bail;
	}

	rc = quorum_getquorate(pcmk_quorum_handle, &quorate);
	if (rc != CS_OK) {
	crm_err("Could not obtain the current Quorum API state: %d", rc);
	goto bail;
	}

	if (quorate) {
	crm_notice("Quorum acquired");
	} else {
	crm_warn("Quorum lost");
	}
	quorum_app_callback = dispatch;
	crm_have_quorum = quorate;

	rc = quorum_trackstart(pcmk_quorum_handle, CS_TRACK_CHANGES \| CS_TRACK_CURRENT);
	if (rc != CS_OK) {
	crm_err("Could not setup Quorum API notifications: %d", rc);
	goto bail;
	}

	mainloop_add_fd("quorum", G_PRIORITY_HIGH, fd, dispatch, &quorum_fd_callbacks);

	- corosync_initialize_nodelist(NULL, FALSE, NULL);
	+ pcmk__corosync_add_nodes(NULL);

	bail:
	if (rc != CS_OK) {
	quorum_finalize(pcmk_quorum_handle);
	- return FALSE;
	}
	- return TRUE;
	-}
	-
	-gboolean
	-init_cs_connection(crm_cluster_t * cluster)
	-{
	- int retries = 0;
	-
	- while (retries < 5) {
	- int rc = init_cs_connection_once(cluster);
	-
	- retries++;
	-
	- switch (rc) {
	- case CS_OK:
	- return TRUE;
	- case CS_ERR_TRY_AGAIN:
	- case CS_ERR_QUEUE_FULL:
	- sleep(retries);
	- break;
	- default:
	- return FALSE;
	- }
	- }
	-
	- crm_err("Could not connect to corosync after %d retries", retries);
	- return FALSE;
	}

	+/*!
	+ * \internal
	+ * \brief Connect to Corosync cluster layer
	+ *
	+ * \param[in] cluster Initialized cluster object to connect
	+ */
	gboolean
	-init_cs_connection_once(crm_cluster_t * cluster)
	+pcmk__corosync_connect(crm_cluster_t *cluster)
	{
	crm_node_t *peer = NULL;
	enum cluster_type_e stack = get_cluster_type();

	crm_peer_init();

	- /* Here we just initialize comms */
	if (stack != pcmk_cluster_corosync) {
	- crm_err("Invalid cluster type: %s (%d)", name_for_cluster_type(stack), stack);
	+ crm_err("Invalid cluster type: %s " CRM_XS " stack=%d",
	+ name_for_cluster_type(stack), stack);
	return FALSE;
	}

	- if (cluster_connect_cpg(cluster) == FALSE) {
	+ if (!cluster_connect_cpg(cluster)) {
	+ // Error message was logged by cluster_connect_cpg()
	return FALSE;
	}
	- crm_info("Connection to '%s': established", name_for_cluster_type(stack));
	+ crm_info("Connection to %s established", name_for_cluster_type(stack));

	cluster->nodeid = get_local_nodeid(0);
	- if(cluster->nodeid == 0) {
	- crm_err("Could not establish local nodeid");
	+ if (cluster->nodeid == 0) {
	+ crm_err("Could not determine local node ID");
	return FALSE;
	}

	cluster->uname = get_node_name(0);
	- if(cluster->uname == NULL) {
	- crm_err("Could not establish local node name");
	+ if (cluster->uname == NULL) {
	+ crm_err("Could not determine local node name");
	return FALSE;
	}

	- /* Ensure the local node always exists */
	+ // Ensure local node always exists in peer cache
	peer = crm_get_peer(cluster->nodeid, cluster->uname);
	- cluster->uuid = get_corosync_uuid(peer);
	+ cluster->uuid = pcmk__corosync_uuid(peer);

	return TRUE;
	}

	-gboolean
	-check_message_sanity(const AIS_Message * msg, const char *data)
	-{
	- gboolean sane = TRUE;
	- int dest = msg->host.type;
	- int tmp_size = msg->header.size - sizeof(AIS_Message);
	-
	- if (sane && msg->header.size == 0) {
	- crm_warn("Message with no size");
	- sane = FALSE;
	- }
	-
	- if (sane && msg->header.error != CS_OK) {
	- crm_warn("Message header contains an error: %d", msg->header.error);
	- sane = FALSE;
	- }
	-
	- if (sane && ais_data_len(msg) != tmp_size) {
	- crm_warn("Message payload size is incorrect: expected %d, got %d", ais_data_len(msg),
	- tmp_size);
	- sane = TRUE;
	- }
	-
	- if (sane && ais_data_len(msg) == 0) {
	- crm_warn("Message with no payload");
	- sane = FALSE;
	- }
	-
	- if (sane && data && msg->is_compressed == FALSE) {
	- int str_size = strlen(data) + 1;
	-
	- if (ais_data_len(msg) != str_size) {
	- int lpc = 0;
	-
	- crm_warn("Message payload is corrupted: expected %d bytes, got %d",
	- ais_data_len(msg), str_size);
	- sane = FALSE;
	- for (lpc = (str_size - 10); lpc < msg->size; lpc++) {
	- if (lpc < 0) {
	- lpc = 0;
	- }
	- crm_debug("bad_data[%d]: %d / '%c'", lpc, data[lpc], data[lpc]);
	- }
	- }
	- }
	-
	- if (sane == FALSE) {
	- crm_err("Invalid message %d: (dest=%s:%s, from=%s:%s.%u, compressed=%d, size=%d, total=%d)",
	- msg->id, ais_dest(&(msg->host)), msg_type2text(dest),
	- ais_dest(&(msg->sender)), msg_type2text(msg->sender.type),
	- msg->sender.pid, msg->is_compressed, ais_data_len(msg), msg->header.size);
	-
	- } else {
	- crm_trace
	- ("Verified message %d: (dest=%s:%s, from=%s:%s.%u, compressed=%d, size=%d, total=%d)",
	- msg->id, ais_dest(&(msg->host)), msg_type2text(dest), ais_dest(&(msg->sender)),
	- msg_type2text(msg->sender.type), msg->sender.pid, msg->is_compressed,
	- ais_data_len(msg), msg->header.size);
	- }
	-
	- return sane;
	-}
	-
	+/*!
	+ * \internal
	+ * \brief Check whether a Corosync cluster is active
	+ *
	+ * \return pcmk_cluster_corosync if Corosync is found, else pcmk_cluster_unknown
	+ */
	enum cluster_type_e
	-find_corosync_variant(void)
	+pcmk__corosync_detect(void)
	{
	int rc = CS_OK;
	cmap_handle_t handle;

	rc = cmap_initialize(&handle);

	switch(rc) {
	case CS_OK:
	break;
	case CS_ERR_SECURITY:
	crm_debug("Failed to initialize the cmap API: Permission denied (%d)", rc);
	/* It's there, we just can't talk to it.
	* Good enough for us to identify as 'corosync'
	*/
	return pcmk_cluster_corosync;

	default:
	crm_info("Failed to initialize the cmap API: %s (%d)",
	- ais_error2text(rc), rc);
	+ pcmk__cs_err_str(rc), rc);
	return pcmk_cluster_unknown;
	}

	cmap_finalize(handle);
	return pcmk_cluster_corosync;
	}

	+/*!
	+ * \brief Check whether a Corosync cluster peer is active
	+ *
	+ * \param[in] node Node to check
	+ *
	+ * \return TRUE if \p node is an active Corosync peer, otherwise FALSE
	+ */
	gboolean
	-crm_is_corosync_peer_active(const crm_node_t * node)
	+crm_is_corosync_peer_active(const crm_node_t *node)
	{
	if (node == NULL) {
	- crm_trace("NULL");
	+ crm_trace("Corosync peer inactive: NULL");
	return FALSE;

	} else if (!pcmk__str_eq(node->state, CRM_NODE_MEMBER, pcmk__str_casei)) {
	- crm_trace("%s: state=%s", node->uname, node->state);
	+ crm_trace("Corosync peer %s inactive: state=%s",
	+ node->uname, node->state);
	return FALSE;

	- } else if ((node->processes & crm_proc_cpg) == 0) {
	- crm_trace("%s: processes=%.16x", node->uname, node->processes);
	+ } else if (!pcmk_is_set(node->processes, crm_proc_cpg)) {
	+ crm_trace("Corosync peer %s inactive: processes=%.16x",
	+ node->uname, node->processes);
	return FALSE;
	}
	return TRUE;
	}

	-gboolean
	-corosync_initialize_nodelist(void cluster, gboolean force_member, xmlNode xml_parent)
	+/*!
	+ * \internal
	+ * \brief Load Corosync node list (via CMAP) into peer cache and optionally XML
	+ *
	+ * \param[in] xml_parent If not NULL, add a <node> entry to this for each node
	+ *
	+ * \return true if any nodes were found, false otherwise
	+ */
	+bool
	+pcmk__corosync_add_nodes(xmlNode *xml_parent)
	{
	int lpc = 0;
	cs_error_t rc = CS_OK;
	int retries = 0;
	- gboolean any = FALSE;
	+ bool any = false;
	cmap_handle_t cmap_handle;
	int fd = -1;
	uid_t found_uid = 0;
	gid_t found_gid = 0;
	pid_t found_pid = 0;
	int rv;

	do {
	rc = cmap_initialize(&cmap_handle);
	if (rc != CS_OK) {
	retries++;
	crm_debug("API connection setup failed: %s. Retrying in %ds", cs_strerror(rc),
	retries);
	sleep(retries);
	}

	} while (retries < 5 && rc != CS_OK);

	if (rc != CS_OK) {
	crm_warn("Could not connect to Cluster Configuration Database API, error %d", rc);
	- return FALSE;
	+ return false;
	}

	rc = cmap_fd_get(cmap_handle, &fd);
	if (rc != CS_OK) {
	crm_err("Could not obtain the CMAP API connection: %s (%d)",
	cs_strerror(rc), rc);
	goto bail;
	}

	/* CMAP provider run as root (in given user namespace, anyway)? */
	if (!(rv = crm_ipc_is_authentic_process(fd, (uid_t) 0,(gid_t) 0, &found_pid,
	&found_uid, &found_gid))) {
	crm_err("CMAP provider is not authentic:"
	" process %lld (uid: %lld, gid: %lld)",
	(long long) PCMK__SPECIAL_PID_AS_0(found_pid),
	(long long) found_uid, (long long) found_gid);
	goto bail;
	} else if (rv < 0) {
	crm_err("Could not verify authenticity of CMAP provider: %s (%d)",
	strerror(-rv), -rv);
	goto bail;
	}

	crm_peer_init();
	crm_trace("Initializing corosync nodelist");
	for (lpc = 0; TRUE; lpc++) {
	uint32_t nodeid = 0;
	char *name = NULL;
	char *key = NULL;

	key = crm_strdup_printf("nodelist.node.%d.nodeid", lpc);
	rc = cmap_get_uint32(cmap_handle, key, &nodeid);
	free(key);

	if (rc != CS_OK) {
	break;
	}

	- name = corosync_node_name(cmap_handle, nodeid);
	+ name = pcmk__corosync_name(cmap_handle, nodeid);
	if (name != NULL) {
	GHashTableIter iter;
	crm_node_t *node = NULL;

	g_hash_table_iter_init(&iter, crm_peer_cache);
	while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &node)) {
	if(node && node->uname && strcasecmp(node->uname, name) == 0) {
	if (node->id && node->id != nodeid) {
	crm_crit("Nodes %u and %u share the same name '%s': shutting down", node->id,
	nodeid, name);
	crm_exit(CRM_EX_FATAL);
	}
	}
	}
	}

	if (nodeid > 0 \|\| name != NULL) {
	crm_trace("Initializing node[%d] %u = %s", lpc, nodeid, name);
	crm_get_peer(nodeid, name);
	}

	if (nodeid > 0 && name != NULL) {
	- any = TRUE;
	+ any = true;

	if (xml_parent) {
	xmlNode *node = create_xml_node(xml_parent, XML_CIB_TAG_NODE);

	crm_xml_set_id(node, "%u", nodeid);
	crm_xml_add(node, XML_ATTR_UNAME, name);
	- if (force_member) {
	- crm_xml_add(node, XML_ATTR_TYPE, CRM_NODE_MEMBER);
	- }
	}
	}

	free(name);
	}
	bail:
	cmap_finalize(cmap_handle);
	return any;
	}

	+/*!
	+ * \internal
	+ * \brief Get cluster name from Corosync configuration (via CMAP)
	+ *
	+ * \return Newly allocated string with cluster name if configured, or NULL
	+ */
	char *
	-corosync_cluster_name(void)
	+pcmk__corosync_cluster_name(void)
	{
	cmap_handle_t handle;
	char *cluster_name = NULL;
	cs_error_t rc = CS_OK;
	int fd = -1;
	uid_t found_uid = 0;
	gid_t found_gid = 0;
	pid_t found_pid = 0;
	int rv;

	rc = cmap_initialize(&handle);
	if (rc != CS_OK) {
	crm_info("Failed to initialize the cmap API: %s (%d)",
	cs_strerror(rc), rc);
	return NULL;
	}

	rc = cmap_fd_get(handle, &fd);
	if (rc != CS_OK) {
	crm_err("Could not obtain the CMAP API connection: %s (%d)",
	cs_strerror(rc), rc);
	goto bail;
	}

	/* CMAP provider run as root (in given user namespace, anyway)? */
	if (!(rv = crm_ipc_is_authentic_process(fd, (uid_t) 0,(gid_t) 0, &found_pid,
	&found_uid, &found_gid))) {
	crm_err("CMAP provider is not authentic:"
	" process %lld (uid: %lld, gid: %lld)",
	(long long) PCMK__SPECIAL_PID_AS_0(found_pid),
	(long long) found_uid, (long long) found_gid);
	goto bail;
	} else if (rv < 0) {
	crm_err("Could not verify authenticity of CMAP provider: %s (%d)",
	strerror(-rv), -rv);
	goto bail;
	}

	rc = cmap_get_string(handle, "totem.cluster_name", &cluster_name);
	if (rc != CS_OK) {
	crm_info("Cannot get totem.cluster_name: %s (%d)", cs_strerror(rc), rc);

	} else {
	crm_debug("cmap totem.cluster_name = '%s'", cluster_name);
	}

	bail:
	cmap_finalize(handle);
	return cluster_name;
	}

	/*!
	* \internal
	- * \brief Check whether Corosync has any configuration keys with given prefix
	- *
	- * \param[in] prefix Configuration key prefix
	+ * \brief Check (via CMAP) whether Corosync configuration has a node list
	*
	- * \return -1 on error, 0 if no such key present, 1 if any such key present
	- * \note This function always returns the result of the first time it was
	- * called, even if later called with a different section name (which
	- * obviously should never be done), and even if the corosync configuration
	- * has since been reloaded.
	+ * \return true if Corosync has node list, otherwise false
	*/
	-int
	-corosync_cmap_has_config(const char *prefix)
	+bool
	+pcmk__corosync_has_nodelist(void)
	{
	- cs_error_t rc = CS_OK;
	+ cs_error_t cs_rc = CS_OK;
	int retries = 0;
	- static int found = -1;
	cmap_handle_t cmap_handle;
	cmap_iter_handle_t iter_handle;
	char key_name[CMAP_KEYNAME_MAXLEN + 1];
	int fd = -1;
	uid_t found_uid = 0;
	gid_t found_gid = 0;
	pid_t found_pid = 0;
	- int rv;
	+ int rc = pcmk_ok;

	- if(found != -1) {
	- return found;
	+ static bool got_result = false;
	+ static bool result = false;
	+
	+ if (got_result) {
	+ return result;
	}

	+ // Connect to CMAP
	do {
	- rc = cmap_initialize(&cmap_handle);
	- if (rc != CS_OK) {
	+ cs_rc = cmap_initialize(&cmap_handle);
	+ if (cs_rc != CS_OK) {
	retries++;
	- crm_debug("API connection setup failed: %s. Retrying in %ds", cs_strerror(rc),
	- retries);
	+ crm_debug("CMAP connection failed: %s (rc=%d, retrying in %ds)",
	+ cs_strerror(cs_rc), cs_rc, retries);
	sleep(retries);
	}
	-
	- } while (retries < 5 && rc != CS_OK);
	-
	- if (rc != CS_OK) {
	- crm_warn("Could not connect to Cluster Configuration Database API: %s (rc=%d)",
	- cs_strerror(rc), rc);
	- return -1;
	- }
	-
	- rc = cmap_fd_get(cmap_handle, &fd);
	- if (rc != CS_OK) {
	- crm_err("Could not obtain the CMAP API connection: %s (%d)",
	- cs_strerror(rc), rc);
	+ } while ((retries < 5) && (cs_rc != CS_OK));
	+ if (cs_rc != CS_OK) {
	+ crm_warn("Assuming Corosync does not have node list: "
	+ "CMAP connection failed (%s) " CRM_XS " rc=%d",
	+ cs_strerror(cs_rc), cs_rc);
	+ return false;
	+ }
	+
	+ // Get CMAP connection file descriptor
	+ cs_rc = cmap_fd_get(cmap_handle, &fd);
	+ if (cs_rc != CS_OK) {
	+ crm_warn("Assuming Corosync does not have node list: "
	+ "CMAP unusable (%s) " CRM_XS " rc=%d",
	+ cs_strerror(cs_rc), cs_rc);
	goto bail;
	}

	- /* CMAP provider run as root (in given user namespace, anyway)? */
	- if (!(rv = crm_ipc_is_authentic_process(fd, (uid_t) 0,(gid_t) 0, &found_pid,
	- &found_uid, &found_gid))) {
	- crm_err("CMAP provider is not authentic:"
	- " process %lld (uid: %lld, gid: %lld)",
	- (long long) PCMK__SPECIAL_PID_AS_0(found_pid),
	- (long long) found_uid, (long long) found_gid);
	+ // Check whether CMAP connection is authentic (i.e. provided by root)
	+ rc = crm_ipc_is_authentic_process(fd, (uid_t) 0, (gid_t) 0,
	+ &found_pid, &found_uid, &found_gid);
	+ if (rc == 0) {
	+ crm_warn("Assuming Corosync does not have node list: "
	+ "CMAP provider is inauthentic "
	+ CRM_XS " pid=%lld uid=%lld gid=%lld",
	+ (long long) PCMK__SPECIAL_PID_AS_0(found_pid),
	+ (long long) found_uid, (long long) found_gid);
	goto bail;
	- } else if (rv < 0) {
	- crm_err("Could not verify authenticity of CMAP provider: %s (%d)",
	- strerror(-rv), -rv);
	+ } else if (rc < 0) {
	+ crm_warn("Assuming Corosync does not have node list: "
	+ "Could not verify CMAP authenticity (%s) " CRM_XS " rc=%d",
	+ pcmk_strerror(rc), rc);
	goto bail;
	}

	- rc = cmap_iter_init(cmap_handle, prefix, &iter_handle);
	- if (rc != CS_OK) {
	- crm_warn("Failed to initialize iteration for corosync cmap '%s': %s (rc=%d)",
	- prefix, cs_strerror(rc), rc);
	+ // Check whether nodelist section is presetn
	+ cs_rc = cmap_iter_init(cmap_handle, "nodelist", &iter_handle);
	+ if (cs_rc != CS_OK) {
	+ crm_warn("Assuming Corosync does not have node list: "
	+ "CMAP not readable (%s) " CRM_XS " rc=%d",
	+ cs_strerror(cs_rc), cs_rc);
	goto bail;
	}

	- found = 0;
	- if (cmap_iter_next(cmap_handle, iter_handle, key_name, NULL, NULL) == CS_OK) {
	- crm_trace("'%s' is configured in corosync cmap: %s", prefix, key_name);
	- found = 1;
	+ cs_rc = cmap_iter_next(cmap_handle, iter_handle, key_name, NULL, NULL);
	+ if (cs_rc == CS_OK) {
	+ result = true;
	}
	+
	cmap_iter_finalize(cmap_handle, iter_handle);
	+ got_result = true;
	+ crm_debug("Corosync %s node list", (result? "has" : "does not have"));

	bail:
	cmap_finalize(cmap_handle);
	-
	- return found;
	+ return result;
	}
	diff --git a/lib/cluster/cpg.c b/lib/cluster/cpg.c
	index b8f184f6c6..f9429b6b56 100644
	--- a/lib/cluster/cpg.c
	+++ b/lib/cluster/cpg.c
	@@ -1,813 +1,1066 @@
	/*
	* Copyright 2004-2020 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>
	#include <bzlib.h>
	#include <sys/socket.h>
	#include <netinet/in.h>
	#include <arpa/inet.h>
	#include <netdb.h>

	#include <crm/common/ipc.h>
	#include <crm/cluster/internal.h>
	#include <crm/common/mainloop.h>
	#include <sys/utsname.h>

	+#include <qb/qbipc_common.h>
	#include <qb/qbipcc.h>
	#include <qb/qbutil.h>

	#include <corosync/corodefs.h>
	#include <corosync/corotypes.h>
	#include <corosync/hdb.h>
	#include <corosync/cpg.h>

	#include <crm/msg_xml.h>

	#include <crm/common/ipc_internal.h> /* PCMK__SPECIAL_PID* */
	+#include "crmcluster_private.h"

	-cpg_handle_t pcmk_cpg_handle = 0; /* TODO: Remove, use cluster.cpg_handle */
	-
	-static bool cpg_evicted = FALSE;
	-gboolean(pcmk_cpg_dispatch_fn) (int kind, const char from, const char *data) = NULL;
	-
	-#define cs_repeat(counter, max, code) do { \
	- code; \
	- if(rc == CS_ERR_TRY_AGAIN \|\| rc == CS_ERR_QUEUE_FULL) { \
	- counter++; \
	- crm_debug("Retrying operation after %ds", counter); \
	- sleep(counter); \
	- } else { \
	- break; \
	- } \
	- } while(counter < max)
	-
	+/* @TODO Once we can update the public API to require crm_cluster_t* in more
	+ * functions, we can ditch this in favor of cluster->cpg_handle.
	+ */
	+static cpg_handle_t pcmk_cpg_handle = 0;
	+
	+// @TODO These could be moved to crm_cluster_t* at that time as well
	+static bool cpg_evicted = false;
	+static GList *cs_message_queue = NULL;
	+static int cs_message_timer = 0;
	+
	+struct pcmk__cpg_host_s {
	+ uint32_t id;
	+ uint32_t pid;
	+ gboolean local;
	+ enum crm_ais_msg_types type;
	+ uint32_t size;
	+ char uname[MAX_NAME];
	+} __attribute__ ((packed));
	+
	+typedef struct pcmk__cpg_host_s pcmk__cpg_host_t;
	+
	+struct pcmk__cpg_msg_s {
	+ struct qb_ipc_response_header header __attribute__ ((aligned(8)));
	+ uint32_t id;
	+ gboolean is_compressed;
	+
	+ pcmk__cpg_host_t host;
	+ pcmk__cpg_host_t sender;
	+
	+ uint32_t size;
	+ uint32_t compressed_size;
	+ /* 584 bytes */
	+ char data[0];
	+
	+} __attribute__ ((packed));
	+
	+typedef struct pcmk__cpg_msg_s pcmk__cpg_msg_t;
	+
	+static void crm_cs_flush(gpointer data);
	+
	+#define msg_data_len(msg) (msg->is_compressed?msg->compressed_size:msg->size)
	+
	+#define cs_repeat(rc, counter, max, code) do { \
	+ rc = code; \
	+ if ((rc == CS_ERR_TRY_AGAIN) \|\| (rc == CS_ERR_QUEUE_FULL)) { \
	+ counter++; \
	+ crm_debug("Retrying operation after %ds", counter); \
	+ sleep(counter); \
	+ } else { \
	+ break; \
	+ } \
	+ } while (counter < max)
	+
	+/*!
	+ * \brief Disconnect from Corosync CPG
	+ *
	+ * \param[in] Cluster to disconnect
	+ */
	void
	cluster_disconnect_cpg(crm_cluster_t *cluster)
	{
	pcmk_cpg_handle = 0;
	if (cluster->cpg_handle) {
	crm_trace("Disconnecting CPG");
	cpg_leave(cluster->cpg_handle, &cluster->group);
	cpg_finalize(cluster->cpg_handle);
	cluster->cpg_handle = 0;

	} else {
	crm_info("No CPG connection");
	}
	}

	-uint32_t get_local_nodeid(cpg_handle_t handle)
	+/*!
	+ * \brief Get the local Corosync node ID (via CPG)
	+ *
	+ * \param[in] handle CPG connection to use (or 0 to use new connection)
	+ *
	+ * \return Corosync ID of local node (or 0 if not known)
	+ */
	+uint32_t
	+get_local_nodeid(cpg_handle_t handle)
	{
	cs_error_t rc = CS_OK;
	int retries = 0;
	static uint32_t local_nodeid = 0;
	cpg_handle_t local_handle = handle;
	cpg_callbacks_t cb = { };
	int fd = -1;
	uid_t found_uid = 0;
	gid_t found_gid = 0;
	pid_t found_pid = 0;
	int rv;

	if(local_nodeid != 0) {
	return local_nodeid;
	}

	if(handle == 0) {
	crm_trace("Creating connection");
	- cs_repeat(retries, 5, rc = cpg_initialize(&local_handle, &cb));
	+ cs_repeat(rc, retries, 5, cpg_initialize(&local_handle, &cb));
	if (rc != CS_OK) {
	crm_err("Could not connect to the CPG API: %s (%d)",
	cs_strerror(rc), rc);
	return 0;
	}

	rc = cpg_fd_get(local_handle, &fd);
	if (rc != CS_OK) {
	crm_err("Could not obtain the CPG API connection: %s (%d)",
	cs_strerror(rc), rc);
	goto bail;
	}

	/* CPG provider run as root (in given user namespace, anyway)? */
	if (!(rv = crm_ipc_is_authentic_process(fd, (uid_t) 0,(gid_t) 0, &found_pid,
	&found_uid, &found_gid))) {
	crm_err("CPG provider is not authentic:"
	" process %lld (uid: %lld, gid: %lld)",
	(long long) PCMK__SPECIAL_PID_AS_0(found_pid),
	(long long) found_uid, (long long) found_gid);
	goto bail;
	} else if (rv < 0) {
	crm_err("Could not verify authenticity of CPG provider: %s (%d)",
	strerror(-rv), -rv);
	goto bail;
	}
	}

	if (rc == CS_OK) {
	retries = 0;
	crm_trace("Performing lookup");
	- cs_repeat(retries, 5, rc = cpg_local_get(local_handle, &local_nodeid));
	+ cs_repeat(rc, retries, 5, cpg_local_get(local_handle, &local_nodeid));
	}

	if (rc != CS_OK) {
	- crm_err("Could not get local node id from the CPG API: %s (%d)", ais_error2text(rc), rc);
	+ crm_err("Could not get local node id from the CPG API: %s (%d)",
	+ pcmk__cs_err_str(rc), rc);
	}

	bail:
	if(handle == 0) {
	crm_trace("Closing connection");
	cpg_finalize(local_handle);
	}
	crm_debug("Local nodeid is %u", local_nodeid);
	return local_nodeid;
	}

	-
	-GListPtr cs_message_queue = NULL;
	-int cs_message_timer = 0;
	-
	-static ssize_t crm_cs_flush(gpointer data);
	-
	+/*!
	+ * \internal
	+ * \brief Callback function for Corosync message queue timer
	+ *
	+ * \param[in] data CPG handle
	+ *
	+ * \return FALSE (to indicate to glib that timer should not be removed)
	+ */
	static gboolean
	crm_cs_flush_cb(gpointer data)
	{
	cs_message_timer = 0;
	crm_cs_flush(data);
	return FALSE;
	}

	+// Send no more than this many CPG messages in one flush
	#define CS_SEND_MAX 200
	-static ssize_t
	+
	+/*!
	+ * \internal
	+ * \brief Send messages in Corosync CPG message queue
	+ *
	+ * \param[in] data CPG handle
	+ */
	+static void
	crm_cs_flush(gpointer data)
	{
	- int sent = 0;
	- ssize_t rc = 0;
	- int queue_len = 0;
	- static unsigned int last_sent = 0;
	- cpg_handle_t handle = (cpg_handle_t )data;
	+ unsigned int sent = 0;
	+ guint queue_len = 0;
	+ cs_error_t rc = 0;
	+ cpg_handle_t handle = (cpg_handle_t ) data;

	if (*handle == 0) {
	crm_trace("Connection is dead");
	- return pcmk_ok;
	+ return;
	}

	queue_len = g_list_length(cs_message_queue);
	- if ((queue_len % 1000) == 0 && queue_len > 1) {
	+ if (((queue_len % 1000) == 0) && (queue_len > 1)) {
	crm_err("CPG queue has grown to %d", queue_len);

	} else if (queue_len == CS_SEND_MAX) {
	crm_warn("CPG queue has grown to %d", queue_len);
	}

	- if (cs_message_timer) {
	+ if (cs_message_timer != 0) {
	/* There is already a timer, wait until it goes off */
	crm_trace("Timer active %d", cs_message_timer);
	- return pcmk_ok;
	+ return;
	}

	- while (cs_message_queue && sent < CS_SEND_MAX) {
	+ while ((cs_message_queue != NULL) && (sent < CS_SEND_MAX)) {
	struct iovec *iov = cs_message_queue->data;

	- errno = 0;
	rc = cpg_mcast_joined(*handle, CPG_TYPE_AGREED, iov, 1);
	-
	if (rc != CS_OK) {
	break;
	}

	sent++;
	- last_sent++;
	crm_trace("CPG message sent, size=%llu",
	(unsigned long long) iov->iov_len);

	cs_message_queue = g_list_remove(cs_message_queue, iov);
	free(iov->iov_base);
	free(iov);
	}

	queue_len -= sent;
	- if (sent > 1 \|\| cs_message_queue) {
	- crm_info("Sent %d CPG messages (%d remaining, last=%u): %s (%lld)",
	- sent, queue_len, last_sent, ais_error2text(rc),
	- (long long) rc);
	+ if ((sent > 1) \|\| (cs_message_queue != NULL)) {
	+ crm_info("Sent %u CPG messages (%d remaining): %s (%d)",
	+ sent, queue_len, pcmk__cs_err_str(rc), (int) rc);
	} else {
	- crm_trace("Sent %d CPG messages (%d remaining, last=%u): %s (%lld)",
	- sent, queue_len, last_sent, ais_error2text(rc),
	- (long long) rc);
	+ crm_trace("Sent %u CPG messages (%d remaining): %s (%d)",
	+ sent, queue_len, pcmk__cs_err_str(rc), (int) rc);
	}

	if (cs_message_queue) {
	uint32_t delay_ms = 100;
	- if(rc != CS_OK) {
	+ if (rc != CS_OK) {
	/* Proportionally more if sending failed but cap at 1s */
	delay_ms = QB_MIN(1000, CS_SEND_MAX + (10 * queue_len));
	}
	cs_message_timer = g_timeout_add(delay_ms, crm_cs_flush_cb, data);
	}
	-
	- return rc;
	-}
	-
	-gboolean
	-send_cpg_iov(struct iovec * iov)
	-{
	- static unsigned int queued = 0;
	-
	- queued++;
	- crm_trace("Queueing CPG message %u (%llu bytes)",
	- queued, (unsigned long long) iov->iov_len);
	- cs_message_queue = g_list_append(cs_message_queue, iov);
	- crm_cs_flush(&pcmk_cpg_handle);
	- return TRUE;
	}

	+/*!
	+ * \internal
	+ * \brief Dispatch function for CPG handle
	+ *
	+ * \param[in] user_data Cluster object
	+ *
	+ * \return 0 on success, -1 on error (per mainloop_io_t interface)
	+ */
	static int
	pcmk_cpg_dispatch(gpointer user_data)
	{
	- int rc = 0;
	- crm_cluster_t cluster = (crm_cluster_t) user_data;
	+ cs_error_t rc = CS_OK;
	+ crm_cluster_t cluster = (crm_cluster_t ) user_data;

	rc = cpg_dispatch(cluster->cpg_handle, CS_DISPATCH_ONE);
	if (rc != CS_OK) {
	- crm_err("Connection to the CPG API failed: %s (%d)", ais_error2text(rc), rc);
	+ crm_err("Connection to the CPG API failed: %s (%d)",
	+ pcmk__cs_err_str(rc), rc);
	cpg_finalize(cluster->cpg_handle);
	cluster->cpg_handle = 0;
	return -1;

	- } else if(cpg_evicted) {
	+ } else if (cpg_evicted) {
	crm_err("Evicted from CPG membership");
	return -1;
	}
	return 0;
	}

	+static inline const char *
	+ais_dest(const pcmk__cpg_host_t *host)
	+{
	+ if (host->local) {
	+ return "local";
	+ } else if (host->size > 0) {
	+ return host->uname;
	+ } else {
	+ return "<all>";
	+ }
	+}
	+
	+static inline const char *
	+msg_type2text(enum crm_ais_msg_types type)
	+{
	+ const char *text = "unknown";
	+
	+ switch (type) {
	+ case crm_msg_none:
	+ text = "unknown";
	+ break;
	+ case crm_msg_ais:
	+ text = "ais";
	+ break;
	+ case crm_msg_cib:
	+ text = "cib";
	+ break;
	+ case crm_msg_crmd:
	+ text = "crmd";
	+ break;
	+ case crm_msg_pe:
	+ text = "pengine";
	+ break;
	+ case crm_msg_te:
	+ text = "tengine";
	+ break;
	+ case crm_msg_lrmd:
	+ text = "lrmd";
	+ break;
	+ case crm_msg_attrd:
	+ text = "attrd";
	+ break;
	+ case crm_msg_stonithd:
	+ text = "stonithd";
	+ break;
	+ case crm_msg_stonith_ng:
	+ text = "stonith-ng";
	+ break;
	+ }
	+ return text;
	+}
	+
	+/*!
	+ * \internal
	+ * \brief Check whether a Corosync CPG message is valid
	+ *
	+ * \param[in] msg Corosync CPG message to check
	+ *
	+ * \return true if \p msg is valid, otherwise false
	+ */
	+static bool
	+check_message_sanity(const pcmk__cpg_msg_t *msg)
	+{
	+ gboolean sane = TRUE;
	+ int dest = msg->host.type;
	+ int tmp_size = msg->header.size - sizeof(pcmk__cpg_msg_t);
	+
	+ if (sane && msg->header.size == 0) {
	+ crm_warn("Message with no size");
	+ sane = FALSE;
	+ }
	+
	+ if (sane && msg->header.error != CS_OK) {
	+ crm_warn("Message header contains an error: %d", msg->header.error);
	+ sane = FALSE;
	+ }
	+
	+ if (sane && msg_data_len(msg) != tmp_size) {
	+ crm_warn("Message payload size is incorrect: expected %d, got %d", msg_data_len(msg),
	+ tmp_size);
	+ sane = FALSE;
	+ }
	+
	+ if (sane && msg_data_len(msg) == 0) {
	+ crm_warn("Message with no payload");
	+ sane = FALSE;
	+ }
	+
	+ if (sane && !msg->is_compressed && (msg->size > 0)) {
	+ size_t str_size = strlen(msg->data) + 1;
	+
	+ if (msg->size != str_size) {
	+ crm_warn("Message payload is corrupted: expected %llu bytes, got %llu",
	+ (unsigned long long) msg->size,
	+ (unsigned long long) str_size);
	+ sane = FALSE;
	+ }
	+ }
	+
	+ if (sane == FALSE) {
	+ crm_err("Invalid message %d: (dest=%s:%s, from=%s:%s.%u, compressed=%d, size=%d, total=%d)",
	+ msg->id, ais_dest(&(msg->host)), msg_type2text(dest),
	+ ais_dest(&(msg->sender)), msg_type2text(msg->sender.type),
	+ msg->sender.pid, msg->is_compressed, msg_data_len(msg), msg->header.size);
	+
	+ } else {
	+ crm_trace
	+ ("Verified message %d: (dest=%s:%s, from=%s:%s.%u, compressed=%d, size=%d, total=%d)",
	+ msg->id, ais_dest(&(msg->host)), msg_type2text(dest), ais_dest(&(msg->sender)),
	+ msg_type2text(msg->sender.type), msg->sender.pid, msg->is_compressed,
	+ msg_data_len(msg), msg->header.size);
	+ }
	+
	+ return sane;
	+}
	+
	+/*!
	+ * \brief Extract text data from a Corosync CPG message
	+ *
	+ * \param[in] handle CPG connection (to get local node ID if not yet known)
	+ * \param[in] nodeid Corosync ID of node that sent message
	+ * \param[in] pid Process ID of message sender (for logging only)
	+ * \param[in] content CPG message
	+ * \param[out] kind If not NULL, will be set to CPG header ID
	+ * (which should be an enum crm_ais_msg_class value,
	+ * currently always crm_class_cluster)
	+ * \param[out] from If not NULL, will be set to sender uname
	+ * (valid for the lifetime of \p content)
	+ *
	+ * \return Newly allocated string with message data
	+ * \note It is the caller's responsibility to free the return value with free().
	+ */
	char *
	pcmk_message_common_cs(cpg_handle_t handle, uint32_t nodeid, uint32_t pid, void *content,
	uint32_t kind, const char *from)
	{
	char *data = NULL;
	- AIS_Message msg = (AIS_Message ) content;
	+ pcmk__cpg_msg_t msg = (pcmk__cpg_msg_t ) content;

	if(handle) {
	// Do filtering and field massaging
	uint32_t local_nodeid = get_local_nodeid(handle);
	const char *local_name = get_local_node_name();

	if (msg->sender.id > 0 && msg->sender.id != nodeid) {
	crm_err("Nodeid mismatch from %d.%d: claimed nodeid=%u", nodeid, pid, msg->sender.id);
	return NULL;

	} else if (msg->host.id != 0 && (local_nodeid != msg->host.id)) {
	/* Not for us */
	crm_trace("Not for us: %u != %u", msg->host.id, local_nodeid);
	return NULL;
	} else if (msg->host.size != 0 && !pcmk__str_eq(msg->host.uname, local_name, pcmk__str_casei)) {
	/* Not for us */
	crm_trace("Not for us: %s != %s", msg->host.uname, local_name);
	return NULL;
	}

	msg->sender.id = nodeid;
	if (msg->sender.size == 0) {
	crm_node_t *peer = crm_get_peer(nodeid, NULL);

	if (peer == NULL) {
	crm_err("Peer with nodeid=%u is unknown", nodeid);

	} else if (peer->uname == NULL) {
	crm_err("No uname for peer with nodeid=%u", nodeid);

	} else {
	crm_notice("Fixing uname for peer with nodeid=%u", nodeid);
	msg->sender.size = strlen(peer->uname);
	memset(msg->sender.uname, 0, MAX_NAME);
	memcpy(msg->sender.uname, peer->uname, msg->sender.size);
	}
	}
	}

	crm_trace("Got new%s message (size=%d, %d, %d)",
	msg->is_compressed ? " compressed" : "",
	- ais_data_len(msg), msg->size, msg->compressed_size);
	+ msg_data_len(msg), msg->size, msg->compressed_size);

	if (kind != NULL) {
	*kind = msg->header.id;
	}
	if (from != NULL) {
	*from = msg->sender.uname;
	}

	if (msg->is_compressed && msg->size > 0) {
	int rc = BZ_OK;
	char *uncompressed = NULL;
	unsigned int new_size = msg->size + 1;

	- if (check_message_sanity(msg, NULL) == FALSE) {
	+ if (!check_message_sanity(msg)) {
	goto badmsg;
	}

	crm_trace("Decompressing message data");
	uncompressed = calloc(1, new_size);
	rc = BZ2_bzBuffToBuffDecompress(uncompressed, &new_size, msg->data, msg->compressed_size, 1, 0);

	if (rc != BZ_OK) {
	crm_err("Decompression failed: %s " CRM_XS " bzerror=%d",
	bz2_strerror(rc), rc);
	free(uncompressed);
	goto badmsg;
	}

	CRM_ASSERT(rc == BZ_OK);
	CRM_ASSERT(new_size == msg->size);

	data = uncompressed;

	- } else if (check_message_sanity(msg, data) == FALSE) {
	+ } else if (!check_message_sanity(msg)) {
	goto badmsg;

	- } else if (pcmk__str_eq("identify", data, pcmk__str_casei)) {
	- char *pid_s = pcmk__getpid_s();
	-
	- send_cluster_text(crm_class_cluster, pid_s, TRUE, NULL, crm_msg_ais);
	- free(pid_s);
	- return NULL;
	-
	} else {
	data = strdup(msg->data);
	}

	// Is this necessary?
	crm_get_peer(msg->sender.id, msg->sender.uname);

	crm_trace("Payload: %.200s", data);
	return data;

	badmsg:
	crm_err("Invalid message (id=%d, dest=%s:%s, from=%s:%s.%d):"
	" min=%d, total=%d, size=%d, bz2_size=%d",
	msg->id, ais_dest(&(msg->host)), msg_type2text(msg->host.type),
	ais_dest(&(msg->sender)), msg_type2text(msg->sender.type),
	- msg->sender.pid, (int)sizeof(AIS_Message),
	+ msg->sender.pid, (int)sizeof(pcmk__cpg_msg_t),
	msg->header.size, msg->size, msg->compressed_size);

	free(data);
	return NULL;
	}

	-static int cmp_member_list_nodeid(const void *first,
	- const void *second)
	+/*!
	+ * \internal
	+ * \brief Compare cpg_address objects by node ID
	+ *
	+ * \param[in] first First cpg_address structure to compare
	+ * \param[in] second Second cpg_address structure to compare
	+ *
	+ * \return Negative number if first's node ID is lower,
	+ * positive number if first's node ID is greater,
	+ * or 0 if both node IDs are equal
	+ */
	+static int
	+cmp_member_list_nodeid(const void first, const void second)
	{
	const struct cpg_address const a = ((const struct cpg_address **) first),
	const b = ((const struct cpg_address **) second);
	if (a->nodeid < b->nodeid) {
	return -1;
	} else if (a->nodeid > b->nodeid) {
	return 1;
	}
	/* don't bother with "reason" nor "pid" */
	return 0;
	}

	+/*!
	+ * \internal
	+ * \brief Get a readable string equivalent of a cpg_reason_t value
	+ *
	+ * \param[in] reason CPG reason value
	+ *
	+ * \return Readable string suitable for logging
	+ */
	static const char *
	cpgreason2str(cpg_reason_t reason)
	{
	switch (reason) {
	case CPG_REASON_JOIN: return " via cpg_join";
	case CPG_REASON_LEAVE: return " via cpg_leave";
	case CPG_REASON_NODEDOWN: return " via cluster exit";
	case CPG_REASON_NODEUP: return " via cluster join";
	case CPG_REASON_PROCDOWN: return " for unknown reason";
	default: break;
	}
	return "";
	}

	+/*!
	+ * \internal
	+ * \brief Get a log-friendly node name
	+ *
	+ * \param[in] peer Node to check
	+ *
	+ * \return Node's uname, or readable string if not known
	+ */
	static inline const char *
	peer_name(crm_node_t *peer)
	{
	if (peer == NULL) {
	return "unknown node";
	} else if (peer->uname == NULL) {
	return "peer node";
	} else {
	return peer->uname;
	}
	}

	+/*!
	+ * \brief Handle a CPG configuration change event
	+ *
	+ * \param[in] handle CPG connection
	+ * \param[in] cpg_name CPG group name
	+ * \param[in] member_list List of current CPG members
	+ * \param[in] member_list_entries Number of entries in \p member_list
	+ * \param[in] left_list List of CPG members that left
	+ * \param[in] left_list_entries Number of entries in \p left_list
	+ * \param[in] joined_list List of CPG members that joined
	+ * \param[in] joined_list_entries Number of entries in \p joined_list
	+ */
	void
	pcmk_cpg_membership(cpg_handle_t handle,
	const struct cpg_name *groupName,
	const struct cpg_address *member_list, size_t member_list_entries,
	const struct cpg_address *left_list, size_t left_list_entries,
	const struct cpg_address *joined_list, size_t joined_list_entries)
	{
	int i;
	gboolean found = FALSE;
	static int counter = 0;
	uint32_t local_nodeid = get_local_nodeid(handle);
	const struct cpg_address key, *sorted;

	sorted = malloc(member_list_entries * sizeof(const struct cpg_address *));
	CRM_ASSERT(sorted != NULL);

	for (size_t iter = 0; iter < member_list_entries; iter++) {
	sorted[iter] = member_list + iter;
	}
	/* so that the cross-matching multiply-subscribed nodes is then cheap */
	qsort(sorted, member_list_entries, sizeof(const struct cpg_address *),
	cmp_member_list_nodeid);

	for (i = 0; i < left_list_entries; i++) {
	- crm_node_t *peer = crm_find_peer(left_list[i].nodeid, NULL);
	+ crm_node_t *peer = pcmk__search_cluster_node_cache(left_list[i].nodeid,
	+ NULL);
	const struct cpg_address **rival = NULL;

	/* in CPG world, NODE:PROCESS-IN-MEMBERSHIP-OF-G is an 1:N relation
	and not playing by this rule may go wild in case of multiple
	residual instances of the same pacemaker daemon at the same node
	-- we must ensure that the possible local rival(s) won't make us
	cry out and bail (e.g. when they quit themselves), since all the
	surrounding logic denies this simple fact that the full membership
	is discriminated also per the PID of the process beside mere node
	ID (and implicitly, group ID); practically, this will be sound in
	terms of not preventing progress, since all the CPG joiners are
	also API end-point carriers, and that's what matters locally
	(who's the winner);
	remotely, we will just compare leave_list and member_list and if
	the left process has its node retained in member_list (under some
	other PID, anyway) we will just ignore it as well
	XXX: long-term fix is to establish in-out PID-aware tracking? */
	if (peer) {
	key = &left_list[i];
	rival = bsearch(&key, sorted, member_list_entries,
	sizeof(const struct cpg_address *),
	cmp_member_list_nodeid);
	}

	if (rival == NULL) {
	crm_info("Group %s event %d: %s (node %u pid %u) left%s",
	groupName->value, counter, peer_name(peer),
	left_list[i].nodeid, left_list[i].pid,
	cpgreason2str(left_list[i].reason));
	if (peer) {
	crm_update_peer_proc(__func__, peer, crm_proc_cpg,
	OFFLINESTATUS);
	}
	} else if (left_list[i].nodeid == local_nodeid) {
	crm_warn("Group %s event %d: duplicate local pid %u left%s",
	groupName->value, counter,
	left_list[i].pid, cpgreason2str(left_list[i].reason));
	} else {
	crm_warn("Group %s event %d: "
	"%s (node %u) duplicate pid %u left%s (%u remains)",
	groupName->value, counter, peer_name(peer),
	left_list[i].nodeid, left_list[i].pid,
	cpgreason2str(left_list[i].reason), (*rival)->pid);
	}
	}
	free(sorted);
	sorted = NULL;

	for (i = 0; i < joined_list_entries; i++) {
	crm_info("Group %s event %d: node %u pid %u joined%s",
	groupName->value, counter, joined_list[i].nodeid,
	joined_list[i].pid, cpgreason2str(joined_list[i].reason));
	}

	for (i = 0; i < member_list_entries; i++) {
	crm_node_t *peer = crm_get_peer(member_list[i].nodeid, NULL);

	if (member_list[i].nodeid == local_nodeid
	&& member_list[i].pid != getpid()) {
	/* see the note above */
	crm_warn("Group %s event %d: detected duplicate local pid %u",
	groupName->value, counter, member_list[i].pid);
	continue;
	}
	crm_info("Group %s event %d: %s (node %u pid %u) is member",
	groupName->value, counter, peer_name(peer),
	member_list[i].nodeid, member_list[i].pid);

	/* If the caller left auto-reaping enabled, this will also update the
	* state to member.
	*/
	peer = crm_update_peer_proc(__func__, peer, crm_proc_cpg,
	ONLINESTATUS);

	if (peer && peer->state && strcmp(peer->state, CRM_NODE_MEMBER)) {
	/* The node is a CPG member, but we currently think it's not a
	* cluster member. This is possible only if auto-reaping was
	* disabled. The node may be joining, and we happened to get the CPG
	* notification before the quorum notification; or the node may have
	* just died, and we are processing its final messages; or a bug
	* has affected the peer cache.
	*/
	time_t now = time(NULL);

	if (peer->when_lost == 0) {
	// Track when we first got into this contradictory state
	peer->when_lost = now;

	} else if (now > (peer->when_lost + 60)) {
	// If it persists for more than a minute, update the state
	crm_warn("Node %u is member of group %s but was believed offline",
	member_list[i].nodeid, groupName->value);
	- crm_update_peer_state(__func__, peer, CRM_NODE_MEMBER, 0);
	+ pcmk__update_peer_state(__func__, peer, CRM_NODE_MEMBER, 0);
	}
	}

	if (local_nodeid == member_list[i].nodeid) {
	found = TRUE;
	}
	}

	if (!found) {
	crm_err("Local node was evicted from group %s", groupName->value);
	- cpg_evicted = TRUE;
	+ cpg_evicted = true;
	}

	counter++;
	}

	+/*!
	+ * \brief Connect to Corosync CPG
	+ *
	+ * \param[in] cluster Cluster object
	+ *
	+ * \return TRUE on success, otherwise FALSE
	+ */
	gboolean
	cluster_connect_cpg(crm_cluster_t *cluster)
	{
	cs_error_t rc;
	int fd = -1;
	int retries = 0;
	uint32_t id = 0;
	crm_node_t *peer = NULL;
	cpg_handle_t handle = 0;
	const char *message_name = pcmk__message_name(crm_system_name);
	uid_t found_uid = 0;
	gid_t found_gid = 0;
	pid_t found_pid = 0;
	int rv;

	struct mainloop_fd_callbacks cpg_fd_callbacks = {
	.dispatch = pcmk_cpg_dispatch,
	.destroy = cluster->destroy,
	};

	cpg_callbacks_t cpg_callbacks = {
	.cpg_deliver_fn = cluster->cpg.cpg_deliver_fn,
	.cpg_confchg_fn = cluster->cpg.cpg_confchg_fn,
	/* .cpg_deliver_fn = pcmk_cpg_deliver, */
	/* .cpg_confchg_fn = pcmk_cpg_membership, */
	};

	- cpg_evicted = FALSE;
	+ cpg_evicted = false;
	cluster->group.length = 0;
	cluster->group.value[0] = 0;

	/* group.value is char[128] */
	strncpy(cluster->group.value, message_name, 127);
	cluster->group.value[127] = 0;
	cluster->group.length = 1 + QB_MIN(127, strlen(cluster->group.value));

	- cs_repeat(retries, 30, rc = cpg_initialize(&handle, &cpg_callbacks));
	+ cs_repeat(rc, retries, 30, cpg_initialize(&handle, &cpg_callbacks));
	if (rc != CS_OK) {
	crm_err("Could not connect to the CPG API: %s (%d)",
	cs_strerror(rc), rc);
	goto bail;
	}

	rc = cpg_fd_get(handle, &fd);
	if (rc != CS_OK) {
	crm_err("Could not obtain the CPG API connection: %s (%d)",
	cs_strerror(rc), rc);
	goto bail;
	}

	/* CPG provider run as root (in given user namespace, anyway)? */
	if (!(rv = crm_ipc_is_authentic_process(fd, (uid_t) 0,(gid_t) 0, &found_pid,
	&found_uid, &found_gid))) {
	crm_err("CPG provider is not authentic:"
	" process %lld (uid: %lld, gid: %lld)",
	(long long) PCMK__SPECIAL_PID_AS_0(found_pid),
	(long long) found_uid, (long long) found_gid);
	rc = CS_ERR_ACCESS;
	goto bail;
	} else if (rv < 0) {
	crm_err("Could not verify authenticity of CPG provider: %s (%d)",
	strerror(-rv), -rv);
	rc = CS_ERR_ACCESS;
	goto bail;
	}

	id = get_local_nodeid(handle);
	if (id == 0) {
	crm_err("Could not get local node id from the CPG API");
	goto bail;

	}
	cluster->nodeid = id;

	retries = 0;
	- cs_repeat(retries, 30, rc = cpg_join(handle, &cluster->group));
	+ cs_repeat(rc, retries, 30, cpg_join(handle, &cluster->group));
	if (rc != CS_OK) {
	crm_err("Could not join the CPG group '%s': %d", message_name, rc);
	goto bail;
	}

	pcmk_cpg_handle = handle;
	cluster->cpg_handle = handle;
	mainloop_add_fd("corosync-cpg", G_PRIORITY_MEDIUM, fd, cluster, &cpg_fd_callbacks);

	bail:
	if (rc != CS_OK) {
	cpg_finalize(handle);
	return FALSE;
	}

	peer = crm_get_peer(id, NULL);
	crm_update_peer_proc(__func__, peer, crm_proc_cpg, ONLINESTATUS);
	return TRUE;
	}

	+/*!
	+ * \internal
	+ * \brief Send an XML message via Corosync CPG
	+ *
	+ * \param[in] msg XML message to send
	+ * \param[in] node Cluster node to send message to
	+ * \param[in] dest Type of message to send
	+ *
	+ * \return TRUE on success, otherwise FALSE
	+ */
	gboolean
	-send_cluster_message_cs(xmlNode * msg, gboolean local, crm_node_t * node, enum crm_ais_msg_types dest)
	+pcmk__cpg_send_xml(xmlNode msg, crm_node_t node, enum crm_ais_msg_types dest)
	{
	gboolean rc = TRUE;
	char *data = NULL;

	data = dump_xml_unformatted(msg);
	- rc = send_cluster_text(crm_class_cluster, data, local, node, dest);
	+ rc = send_cluster_text(crm_class_cluster, data, FALSE, node, dest);
	free(data);
	return rc;
	}

	+/*!
	+ * \internal
	+ * \brief Send string data via Corosync CPG
	+ *
	+ * \param[in] msg_class Message class (to set as CPG header ID)
	+ * \param[in] data Data to send
	+ * \param[in] local What to set as host "local" value (which is never used)
	+ * \param[in] node Cluster node to send message to
	+ * \param[in] dest Type of message to send
	+ *
	+ * \return TRUE on success, otherwise FALSE
	+ */
	gboolean
	send_cluster_text(enum crm_ais_msg_class msg_class, const char *data,
	gboolean local, crm_node_t *node, enum crm_ais_msg_types dest)
	{
	static int msg_id = 0;
	static int local_pid = 0;
	static int local_name_len = 0;
	static const char *local_name = NULL;

	char *target = NULL;
	struct iovec *iov;
	- AIS_Message *msg = NULL;
	+ pcmk__cpg_msg_t *msg = NULL;
	enum crm_ais_msg_types sender = text2msg_type(crm_system_name);

	switch (msg_class) {
	case crm_class_cluster:
	break;
	default:
	crm_err("Invalid message class: %d", msg_class);
	return FALSE;
	}

	CRM_CHECK(dest != crm_msg_ais, return FALSE);

	- if(local_name == NULL) {
	+ if (local_name == NULL) {
	local_name = get_local_node_name();
	}
	- if(local_name_len == 0 && local_name) {
	+ if ((local_name_len == 0) && (local_name != NULL)) {
	local_name_len = strlen(local_name);
	}

	if (data == NULL) {
	data = "";
	}

	if (local_pid == 0) {
	local_pid = getpid();
	}

	if (sender == crm_msg_none) {
	sender = local_pid;
	}

	- msg = calloc(1, sizeof(AIS_Message));
	+ msg = calloc(1, sizeof(pcmk__cpg_msg_t));

	msg_id++;
	msg->id = msg_id;
	msg->header.id = msg_class;
	msg->header.error = CS_OK;

	msg->host.type = dest;
	msg->host.local = local;

	if (node) {
	if (node->uname) {
	target = strdup(node->uname);
	msg->host.size = strlen(node->uname);
	memset(msg->host.uname, 0, MAX_NAME);
	memcpy(msg->host.uname, node->uname, msg->host.size);
	} else {
	target = crm_strdup_printf("%u", node->id);
	}
	msg->host.id = node->id;
	} else {
	target = strdup("all");
	}

	msg->sender.id = 0;
	msg->sender.type = sender;
	msg->sender.pid = local_pid;
	msg->sender.size = local_name_len;
	memset(msg->sender.uname, 0, MAX_NAME);
	- if(local_name && msg->sender.size) {
	+ if ((local_name != NULL) && (msg->sender.size != 0)) {
	memcpy(msg->sender.uname, local_name, msg->sender.size);
	}

	msg->size = 1 + strlen(data);
	- msg->header.size = sizeof(AIS_Message) + msg->size;
	+ msg->header.size = sizeof(pcmk__cpg_msg_t) + msg->size;

	if (msg->size < CRM_BZ2_THRESHOLD) {
	msg = pcmk__realloc(msg, msg->header.size);
	memcpy(msg->data, data, msg->size);

	} else {
	char *compressed = NULL;
	unsigned int new_size = 0;
	char *uncompressed = strdup(data);

	if (pcmk__compress(uncompressed, (unsigned int) msg->size, 0,
	&compressed, &new_size) == pcmk_rc_ok) {

	- msg->header.size = sizeof(AIS_Message) + new_size;
	+ msg->header.size = sizeof(pcmk__cpg_msg_t) + new_size;
	msg = pcmk__realloc(msg, msg->header.size);
	memcpy(msg->data, compressed, new_size);

	msg->is_compressed = TRUE;
	msg->compressed_size = new_size;

	} else {
	// cppcheck seems not to understand the abort logic in pcmk__realloc
	// cppcheck-suppress memleak
	msg = pcmk__realloc(msg, msg->header.size);
	memcpy(msg->data, data, msg->size);
	}

	free(uncompressed);
	free(compressed);
	}

	iov = calloc(1, sizeof(struct iovec));
	iov->iov_base = msg;
	iov->iov_len = msg->header.size;

	if (msg->compressed_size) {
	crm_trace("Queueing CPG message %u to %s (%llu bytes, %d bytes compressed payload): %.200s",
	msg->id, target, (unsigned long long) iov->iov_len,
	msg->compressed_size, data);
	} else {
	crm_trace("Queueing CPG message %u to %s (%llu bytes, %d bytes payload): %.200s",
	msg->id, target, (unsigned long long) iov->iov_len,
	msg->size, data);
	}
	free(target);

	- send_cpg_iov(iov);
	+ cs_message_queue = g_list_append(cs_message_queue, iov);
	+ crm_cs_flush(&pcmk_cpg_handle);

	return TRUE;
	}

	+/*!
	+ * \brief Get the message type equivalent of a string
	+ *
	+ * \param[in] text String of message type
	+ *
	+ * \return Message type equivalent of \p text
	+ */
	enum crm_ais_msg_types
	text2msg_type(const char *text)
	{
	int type = crm_msg_none;

	CRM_CHECK(text != NULL, return type);
	text = pcmk__message_name(text);
	if (pcmk__str_eq(text, "ais", pcmk__str_casei)) {
	type = crm_msg_ais;
	} else if (pcmk__str_eq(text, CRM_SYSTEM_CIB, pcmk__str_casei)) {
	type = crm_msg_cib;
	} else if (pcmk__strcase_any_of(text, CRM_SYSTEM_CRMD, CRM_SYSTEM_DC, NULL)) {
	type = crm_msg_crmd;
	} else if (pcmk__str_eq(text, CRM_SYSTEM_TENGINE, pcmk__str_casei)) {
	type = crm_msg_te;
	} else if (pcmk__str_eq(text, CRM_SYSTEM_PENGINE, pcmk__str_casei)) {
	type = crm_msg_pe;
	} else if (pcmk__str_eq(text, CRM_SYSTEM_LRMD, pcmk__str_casei)) {
	type = crm_msg_lrmd;
	} else if (pcmk__str_eq(text, CRM_SYSTEM_STONITHD, pcmk__str_casei)) {
	type = crm_msg_stonithd;
	} else if (pcmk__str_eq(text, "stonith-ng", pcmk__str_casei)) {
	type = crm_msg_stonith_ng;
	} else if (pcmk__str_eq(text, "attrd", pcmk__str_casei)) {
	type = crm_msg_attrd;

	} else {
	/* This will normally be a transient client rather than
	* a cluster daemon. Set the type to the pid of the client
	*/
	int scan_rc = sscanf(text, "%d", &type);

	if (scan_rc != 1 \|\| type <= crm_msg_stonith_ng) {
	/* Ensure it's sane */
	type = crm_msg_none;
	}
	}
	return type;
	}
	diff --git a/lib/cluster/crmcluster_private.h b/lib/cluster/crmcluster_private.h
	new file mode 100644
	index 0000000000..bb5ea7a586
	--- /dev/null
	+++ b/lib/cluster/crmcluster_private.h
	@@ -0,0 +1,47 @@
	+/*
	+ * Copyright 2020 the Pacemaker project contributors
	+ *
	+ * The version control history for this file may have further details.
	+ *
	+ * This source code is licensed under the GNU Lesser General Public License
	+ * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	+ */
	+
	+#ifndef PCMK__CRMCLUSTER_PRIVATE__H
	+# define PCMK__CRMCLUSTER_PRIVATE__H
	+
	+/* This header is for the sole use of libcrmcluster, so that functions can be
	+ * declared with G_GNUC_INTERNAL for efficiency.
	+ */
	+
	+#include <stdint.h> // uint32_t, uint64_t
	+
	+#include <glib.h> // G_GNUC_INTERNAL, gboolean
	+#include <libxml/tree.h> // xmlNode
	+
	+#include <crm/cluster.h> // cluster_type_e, crm_node_t
	+
	+G_GNUC_INTERNAL
	+enum cluster_type_e pcmk__corosync_detect(void);
	+
	+G_GNUC_INTERNAL
	+bool pcmk__corosync_has_nodelist(void);
	+
	+G_GNUC_INTERNAL
	+char pcmk__corosync_uuid(crm_node_t peer);
	+
	+G_GNUC_INTERNAL
	+char pcmk__corosync_name(uint64_t /cmap_handle_t */ cmap_handle,
	+ uint32_t nodeid);
	+
	+G_GNUC_INTERNAL
	+gboolean pcmk__corosync_connect(crm_cluster_t *cluster);
	+
	+G_GNUC_INTERNAL
	+void pcmk__corosync_disconnect(crm_cluster_t *cluster);
	+
	+G_GNUC_INTERNAL
	+gboolean pcmk__cpg_send_xml(xmlNode msg, crm_node_t node,
	+ enum crm_ais_msg_types dest);
	+
	+#endif // PCMK__CRMCLUSTER_PRIVATE__H
	diff --git a/lib/cluster/election.c b/lib/cluster/election.c
	index f7cd14d88b..a20d14dc24 100644
	--- a/lib/cluster/election.c
	+++ b/lib/cluster/election.c
	@@ -1,719 +1,725 @@
	/*
	- * Copyright 2004-2019 Andrew Beekhof <andrew@beekhof.net>
	+ * Copyright 2004-2020 the Pacemaker project contributors
	+ *
	+ * The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <sys/time.h>
	#include <sys/resource.h>

	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>

	#include <crm/common/mainloop.h>
	#include <crm/cluster/internal.h>
	#include <crm/cluster/election.h>
	#include <crm/crm.h>

	#define STORM_INTERVAL 2 /* in seconds */

	struct election_s {
	enum election_result state;
	guint count; // How many times local node has voted
	char *name; // Descriptive name for this election
	char *uname; // Local node's name
	GSourceFunc cb; // Function to call if election is won
	GHashTable *voted; // Key = node name, value = how node voted
	mainloop_timer_t *timeout; // When to abort if all votes not received
	int election_wins; // Track wins, for storm detection
	bool wrote_blackbox; // Write a storm blackbox at most once
	time_t expires; // When storm detection period ends
	time_t last_election_loss; // When dampening period ends
	};

	-static void election_complete(election_t *e)
	+static void
	+election_complete(election_t *e)
	{
	e->state = election_won;
	-
	- if(e->cb) {
	+ if (e->cb != NULL) {
	e->cb(e);
	}
	-
	election_reset(e);
	}

	-static gboolean election_timer_cb(gpointer user_data)
	+static gboolean
	+election_timer_cb(gpointer user_data)
	{
	election_t *e = user_data;

	crm_info("%s timed out, declaring local node as winner", e->name);
	election_complete(e);
	return FALSE;
	}

	+/*!
	+ * \brief Get current state of an election
	+ *
	+ * \param[in] e Election object
	+ *
	+ * \return Current state of \e
	+ */
	enum election_result
	election_state(election_t *e)
	{
	- if(e) {
	- return e->state;
	- }
	- return election_error;
	+ return (e == NULL)? election_error : e->state;
	}

	/*!
	* \brief Create a new election object
	*
	* Every node that wishes to participate in an election must create an election
	* object. Typically, this should be done once, at start-up. A caller should
	* only create a single election object.
	*
	* \param[in] name Label for election (for logging)
	* \param[in] uname Local node's name
	* \param[in] period_ms How long to wait for all peers to vote
	* \param[in] cb Function to call if local node wins election
	*
	* \return Newly allocated election object on success, NULL on error
	* \note The caller is responsible for freeing the returned value using
	* election_fini().
	*/
	election_t *
	election_init(const char name, const char uname, guint period_ms, GSourceFunc cb)
	{
	election_t *e = NULL;

	static guint count = 0;

	CRM_CHECK(uname != NULL, return NULL);

	e = calloc(1, sizeof(election_t));
	if (e == NULL) {
	crm_perror(LOG_CRIT, "Cannot create election");
	return NULL;
	}

	e->uname = strdup(uname);
	if (e->uname == NULL) {
	crm_perror(LOG_CRIT, "Cannot create election");
	free(e);
	return NULL;
	}

	e->name = name? crm_strdup_printf("election-%s", name)
	: crm_strdup_printf("election-%u", count++);
	e->cb = cb;
	e->timeout = mainloop_timer_add(e->name, period_ms, FALSE,
	election_timer_cb, e);
	crm_trace("Created %s", e->name);
	return e;
	}

	/*!
	* \brief Disregard any previous vote by specified peer
	*
	* This discards any recorded vote from a specified peer. Election users should
	* call this whenever a voting peer becomes inactive.
	*
	* \param[in] e Election object
	* \param[in] uname Name of peer to disregard
	*/
	void
	election_remove(election_t e, const char uname)
	{
	- if(e && uname && e->voted) {
	+ if ((e != NULL) && (uname != NULL) && (e->voted != NULL)) {
	crm_trace("Discarding %s (no-)vote from lost peer %s", e->name, uname);
	g_hash_table_remove(e->voted, uname);
	}
	}

	/*!
	* \brief Stop election timer and disregard all votes
	*
	* \param[in] e Election object
	*/
	void
	election_reset(election_t *e)
	{
	if (e != NULL) {
	crm_trace("Resetting election %s", e->name);
	mainloop_timer_stop(e->timeout);
	if (e->voted) {
	crm_trace("Destroying voted cache with %d members", g_hash_table_size(e->voted));
	g_hash_table_destroy(e->voted);
	e->voted = NULL;
	}
	}
	}

	/*!
	* \brief Free an election object
	*
	* Free all memory associated with an election object, stopping its
	* election timer (if running).
	*
	* \param[in] e Election object
	*/
	void
	election_fini(election_t *e)
	{
	- if(e) {
	+ if (e != NULL) {
	election_reset(e);
	crm_trace("Destroying %s", e->name);
	mainloop_timer_del(e->timeout);
	free(e->uname);
	free(e->name);
	free(e);
	}
	}

	static void
	election_timeout_start(election_t *e)
	{
	- if(e) {
	+ if (e != NULL) {
	mainloop_timer_start(e->timeout);
	}
	}

	/*!
	* \brief Stop an election's timer, if running
	*
	* \param[in] e Election object
	*/
	void
	election_timeout_stop(election_t *e)
	{
	- if(e) {
	+ if (e != NULL) {
	mainloop_timer_stop(e->timeout);
	}
	}

	/*!
	* \brief Change an election's timeout (restarting timer if running)
	*
	* \param[in] e Election object
	* \param[in] period New timeout
	*/
	void
	election_timeout_set_period(election_t *e, guint period)
	{
	- if(e) {
	+ if (e != NULL) {
	mainloop_timer_set_period(e->timeout, period);
	} else {
	crm_err("No election defined");
	}
	}

	static int
	-crm_uptime(struct timeval *output)
	+get_uptime(struct timeval *output)
	{
	static time_t expires = 0;
	static struct rusage info;

	time_t tm_now = time(NULL);

	if (expires < tm_now) {
	int rc = 0;

	info.ru_utime.tv_sec = 0;
	info.ru_utime.tv_usec = 0;
	rc = getrusage(RUSAGE_SELF, &info);

	output->tv_sec = 0;
	output->tv_usec = 0;

	if (rc < 0) {
	crm_perror(LOG_ERR, "Could not calculate the current uptime");
	expires = 0;
	return -1;
	}

	crm_debug("Current CPU usage is: %lds, %ldus", (long)info.ru_utime.tv_sec,
	(long)info.ru_utime.tv_usec);
	}

	expires = tm_now + STORM_INTERVAL; /* N seconds after the last _access_ */
	output->tv_sec = info.ru_utime.tv_sec;
	output->tv_usec = info.ru_utime.tv_usec;

	return 1;
	}

	static int
	-crm_compare_age(struct timeval your_age)
	+compare_age(struct timeval your_age)
	{
	struct timeval our_age;

	- crm_uptime(&our_age); /* If an error occurred, our_age will be compared as {0,0} */
	+ get_uptime(&our_age); /* If an error occurred, our_age will be compared as {0,0} */

	if (our_age.tv_sec > your_age.tv_sec) {
	crm_debug("Win: %ld vs %ld (seconds)", (long)our_age.tv_sec, (long)your_age.tv_sec);
	return 1;
	} else if (our_age.tv_sec < your_age.tv_sec) {
	crm_debug("Lose: %ld vs %ld (seconds)", (long)our_age.tv_sec, (long)your_age.tv_sec);
	return -1;
	} else if (our_age.tv_usec > your_age.tv_usec) {
	crm_debug("Win: %ld.%06ld vs %ld.%06ld (usec)",
	(long)our_age.tv_sec, (long)our_age.tv_usec, (long)your_age.tv_sec, (long)your_age.tv_usec);
	return 1;
	} else if (our_age.tv_usec < your_age.tv_usec) {
	crm_debug("Lose: %ld.%06ld vs %ld.%06ld (usec)",
	(long)our_age.tv_sec, (long)our_age.tv_usec, (long)your_age.tv_sec, (long)your_age.tv_usec);
	return -1;
	}

	return 0;
	}

	/*!
	* \brief Start a new election by offering local node's candidacy
	*
	* Broadcast a "vote" election message containing the local node's ID,
	* (incremented) election counter, and uptime, and start the election timer.
	*
	* \param[in] e Election object
	* \note Any nodes agreeing to the candidacy will send a "no-vote" reply, and if
	* all active peers do so, or if the election times out, the local node
	* wins the election. (If we lose to any peer vote, we will stop the
	* timer, so a timeout means we did not lose -- either some peer did not
	* vote, or we did not call election_check() in time.)
	*/
	void
	election_vote(election_t *e)
	{
	struct timeval age;
	xmlNode *vote = NULL;
	crm_node_t *our_node;

	if (e == NULL) {
	crm_trace("Election vote requested, but no election available");
	return;
	}

	our_node = crm_get_peer(0, e->uname);
	if ((our_node == NULL) \|\| (crm_is_peer_active(our_node) == FALSE)) {
	crm_trace("Cannot vote in %s yet: local node not connected to cluster",
	e->name);
	return;
	}

	election_reset(e);
	e->state = election_in_progress;
	vote = create_request(CRM_OP_VOTE, NULL, NULL, CRM_SYSTEM_CRMD, CRM_SYSTEM_CRMD, NULL);

	e->count++;
	crm_xml_add(vote, F_CRM_ELECTION_OWNER, our_node->uuid);
	crm_xml_add_int(vote, F_CRM_ELECTION_ID, e->count);

	- crm_uptime(&age);
	+ get_uptime(&age);
	crm_xml_add_timeval(vote, F_CRM_ELECTION_AGE_S, F_CRM_ELECTION_AGE_US, &age);

	send_cluster_message(NULL, crm_msg_crmd, vote, TRUE);
	free_xml(vote);

	crm_debug("Started %s round %d", e->name, e->count);
	election_timeout_start(e);
	return;
	}

	/*!
	* \brief Check whether local node has won an election
	*
	* If all known peers have sent no-vote messages, stop the election timer, set
	* the election state to won, and call any registered win callback.
	*
	* \param[in] e Election object
	*
	* \return TRUE if local node has won, FALSE otherwise
	* \note If all known peers have sent no-vote messages, but the election owner
	* does not call this function, the election will not be won (and the
	* callback will not be called) until the election times out.
	* \note This should be called when election_count_vote() returns
	* \c election_in_progress.
	*/
	bool
	election_check(election_t *e)
	{
	int voted_size = 0;
	int num_members = 0;

	- if(e == NULL) {
	+ if (e == NULL) {
	crm_trace("Election check requested, but no election available");
	return FALSE;
	}
	if (e->voted == NULL) {
	crm_trace("%s check requested, but no votes received yet", e->name);
	return FALSE;
	}

	voted_size = g_hash_table_size(e->voted);
	num_members = crm_active_peers();

	/* in the case of #voted > #members, it is better to
	* wait for the timeout and give the cluster time to
	* stabilize
	*/
	if (voted_size >= num_members) {
	/* we won and everyone has voted */
	election_timeout_stop(e);
	if (voted_size > num_members) {
	GHashTableIter gIter;
	const crm_node_t *node;
	char *key = NULL;

	crm_warn("Received too many votes in %s", e->name);
	g_hash_table_iter_init(&gIter, crm_peer_cache);
	while (g_hash_table_iter_next(&gIter, NULL, (gpointer *) & node)) {
	if (crm_is_peer_active(node)) {
	crm_warn("* expected vote: %s", node->uname);
	}
	}

	g_hash_table_iter_init(&gIter, e->voted);
	while (g_hash_table_iter_next(&gIter, (gpointer *) & key, NULL)) {
	crm_warn("* actual vote: %s", key);
	}

	}

	crm_info("%s won by local node", e->name);
	election_complete(e);
	return TRUE;

	} else {
	crm_debug("%s still waiting on %d of %d votes",
	e->name, num_members - voted_size, num_members);
	}

	return FALSE;
	}

	#define LOSS_DAMPEN 2 /* in seconds */

	struct vote {
	const char *op;
	const char *from;
	const char *version;
	const char *election_owner;
	int election_id;
	struct timeval age;
	};

	/*!
	* \brief Unpack an election message
	*
	* \param[in] e Election object
	* \param[in] message Election message XML
	* \param[out] vote Parsed fields from message
	*
	* \return TRUE if election message and election are valid, FALSE otherwise
	* \note The parsed struct's pointer members are valid only for the lifetime of
	* the message argument.
	*/
	static bool
	parse_election_message(election_t e, xmlNode message, struct vote *vote)
	{
	CRM_CHECK(message && vote, return FALSE);

	vote->election_id = -1;
	vote->age.tv_sec = -1;
	vote->age.tv_usec = -1;

	vote->op = crm_element_value(message, F_CRM_TASK);
	vote->from = crm_element_value(message, F_CRM_HOST_FROM);
	vote->version = crm_element_value(message, F_CRM_VERSION);
	vote->election_owner = crm_element_value(message, F_CRM_ELECTION_OWNER);

	crm_element_value_int(message, F_CRM_ELECTION_ID, &(vote->election_id));

	if ((vote->op == NULL) \|\| (vote->from == NULL) \|\| (vote->version == NULL)
	\|\| (vote->election_owner == NULL) \|\| (vote->election_id < 0)) {

	crm_warn("Invalid %s message from %s in %s ",
	(vote->op? vote->op : "election"),
	(vote->from? vote->from : "unspecified node"),
	(e? e->name : "election"));
	return FALSE;
	}

	// Op-specific validation

	if (pcmk__str_eq(vote->op, CRM_OP_VOTE, pcmk__str_none)) {
	// Only vote ops have uptime
	crm_element_value_timeval(message, F_CRM_ELECTION_AGE_S,
	F_CRM_ELECTION_AGE_US, &(vote->age));
	if ((vote->age.tv_sec < 0) \|\| (vote->age.tv_usec < 0)) {
	crm_warn("Cannot count %s %s from %s because it is missing uptime",
	(e? e->name : "election"), vote->op, vote->from);
	return FALSE;
	}

	} else if (!pcmk__str_eq(vote->op, CRM_OP_NOVOTE, pcmk__str_none)) {
	crm_info("Cannot process %s message from %s because %s is not a known election op",
	(e? e->name : "election"), vote->from, vote->op);
	return FALSE;
	}

	// Election validation

	if (e == NULL) {
	crm_info("Cannot count %s from %s because no election available",
	vote->op, vote->from);
	return FALSE;
	}

	/* If the membership cache is NULL, we REALLY shouldn't be voting --
	* the question is how we managed to get here.
	*/
	if (crm_peer_cache == NULL) {
	crm_info("Cannot count %s %s from %s because no peer information available",
	e->name, vote->op, vote->from);
	return FALSE;
	}
	return TRUE;
	}

	static void
	record_vote(election_t e, struct vote vote)
	{
	char *voter_copy = NULL;
	char *vote_copy = NULL;

	CRM_ASSERT(e && vote && vote->from && vote->op);
	if (e->voted == NULL) {
	e->voted = crm_str_table_new();
	}

	voter_copy = strdup(vote->from);
	vote_copy = strdup(vote->op);
	CRM_ASSERT(voter_copy && vote_copy);

	g_hash_table_replace(e->voted, voter_copy, vote_copy);
	}

	static void
	send_no_vote(crm_node_t peer, struct vote vote)
	{
	// @TODO probably shouldn't hardcode CRM_SYSTEM_CRMD and crm_msg_crmd

	xmlNode *novote = create_request(CRM_OP_NOVOTE, NULL, vote->from,
	CRM_SYSTEM_CRMD, CRM_SYSTEM_CRMD, NULL);

	crm_xml_add(novote, F_CRM_ELECTION_OWNER, vote->election_owner);
	crm_xml_add_int(novote, F_CRM_ELECTION_ID, vote->election_id);

	send_cluster_message(peer, crm_msg_crmd, novote, TRUE);
	free_xml(novote);
	}

	/*!
	* \brief Process an election message (vote or no-vote) from a peer
	*
	* \param[in] e Election object
	* \param[in] vote Election message XML from peer
	* \param[in] can_win Whether to consider the local node eligible for winning
	*
	* \return Election state after new vote is considered
	* \note If the peer message is a vote, and we prefer the peer to win, this will
	* send a no-vote reply to the peer.
	* \note The situations "we lost to this vote" from "this is a late no-vote
	* after we've already lost" both return election_lost. If a caller needs
	* to distinguish them, it should save the current state before calling
	* this function, and then compare the result.
	*/
	enum election_result
	election_count_vote(election_t e, xmlNode message, bool can_win)
	{
	int log_level = LOG_INFO;
	gboolean done = FALSE;
	gboolean we_lose = FALSE;
	const char *reason = "unknown";
	bool we_are_owner = FALSE;
	crm_node_t our_node = NULL, your_node = NULL;
	time_t tm_now = time(NULL);
	struct vote vote;

	CRM_CHECK(message != NULL, return election_error);
	if (parse_election_message(e, message, &vote) == FALSE) {
	return election_error;
	}

	your_node = crm_get_peer(0, vote.from);
	our_node = crm_get_peer(0, e->uname);
	we_are_owner = (our_node != NULL)
	&& pcmk__str_eq(our_node->uuid, vote.election_owner,
	pcmk__str_none);

	- if(can_win == FALSE) {
	+ if (!can_win) {
	reason = "Not eligible";
	we_lose = TRUE;

	} else if (our_node == NULL \|\| crm_is_peer_active(our_node) == FALSE) {
	reason = "We are not part of the cluster";
	log_level = LOG_ERR;
	we_lose = TRUE;

	} else if (we_are_owner && (vote.election_id != e->count)) {
	log_level = LOG_TRACE;
	reason = "Superseded";
	done = TRUE;

	} else if (your_node == NULL \|\| crm_is_peer_active(your_node) == FALSE) {
	/* Possibly we cached the message in the FSA queue at a point that it wasn't */
	reason = "Peer is not part of our cluster";
	log_level = LOG_WARNING;
	done = TRUE;

	} else if (pcmk__str_eq(vote.op, CRM_OP_NOVOTE, pcmk__str_none)
	\|\| pcmk__str_eq(vote.from, e->uname, pcmk__str_none)) {
	/* Receiving our own broadcast vote, or a no-vote from peer, is a vote
	* for us to win
	*/
	if (!we_are_owner) {
	crm_warn("Cannot count %s round %d %s from %s because we are not election owner (%s)",
	e->name, vote.election_id, vote.op, vote.from,
	vote.election_owner);
	return election_error;
	}
	if (e->state != election_in_progress) {
	// Should only happen if we already lost
	crm_debug("Not counting %s round %d %s from %s because no election in progress",
	e->name, vote.election_id, vote.op, vote.from);
	return e->state;
	}
	record_vote(e, &vote);
	reason = "Recorded";
	done = TRUE;

	} else {
	// A peer vote requires a comparison to determine which node is better
	- int age_result = crm_compare_age(vote.age);
	+ int age_result = compare_age(vote.age);
	int version_result = compare_version(vote.version, CRM_FEATURE_SET);

	if (version_result < 0) {
	reason = "Version";
	we_lose = TRUE;

	} else if (version_result > 0) {
	reason = "Version";

	} else if (age_result < 0) {
	reason = "Uptime";
	we_lose = TRUE;

	} else if (age_result > 0) {
	reason = "Uptime";

	} else if (strcasecmp(e->uname, vote.from) > 0) {
	reason = "Host name";
	we_lose = TRUE;

	} else {
	reason = "Host name";
	}
	}

	if (e->expires < tm_now) {
	e->election_wins = 0;
	e->expires = tm_now + STORM_INTERVAL;

	} else if (done == FALSE && we_lose == FALSE) {
	int peers = 1 + g_hash_table_size(crm_peer_cache);

	/* If every node has to vote down every other node, thats N*(N-1) total elections
	* Allow some leeway before _really_ complaining
	*/
	e->election_wins++;
	if (e->election_wins > (peers * peers)) {
	crm_warn("%s election storm detected: %d wins in %d seconds",
	e->name, e->election_wins, STORM_INTERVAL);
	e->election_wins = 0;
	e->expires = tm_now + STORM_INTERVAL;
	if (e->wrote_blackbox == FALSE) {
	/* It's questionable whether a black box (from every node in the
	* cluster) would be truly helpful in diagnosing an election
	* storm. It's also highly doubtful a production environment
	* would get multiple election storms from distinct causes, so
	* saving one blackbox per process lifetime should be
	* sufficient. Alternatives would be to save a timestamp of the
	* last blackbox write instead of a boolean, and write a new one
	* if some amount of time has passed; or to save a storm count,
	* write a blackbox on every Nth occurrence.
	*/
	crm_write_blackbox(0, NULL);
	e->wrote_blackbox = TRUE;
	}
	}
	}

	if (done) {
	do_crm_log(log_level + 1,
	"Processed %s round %d %s (current round %d) from %s (%s)",
	e->name, vote.election_id, vote.op, e->count, vote.from,
	reason);
	return e->state;

	} else if (we_lose == FALSE) {
	/* We track the time of the last election loss to implement an election
	* dampening period, reducing the likelihood of an election storm. If
	* this node has lost within the dampening period, don't start a new
	* election, even if we win against a peer's vote -- the peer we lost to
	* should win again.
	*
	* @TODO This has a problem case: if an election winner immediately
	* leaves the cluster, and a new election is immediately called, all
	* nodes could lose, with no new winner elected. The ideal solution
	* would be to tie the election structure with the peer caches, which
	* would allow us to clear the dampening when the previous winner
	* leaves (and would allow other improvements as well).
	*/
	if ((e->last_election_loss == 0)
	\|\| ((tm_now - e->last_election_loss) > (time_t) LOSS_DAMPEN)) {

	do_crm_log(log_level, "%s round %d (owner node ID %s) pass: %s from %s (%s)",
	e->name, vote.election_id, vote.election_owner, vote.op,
	vote.from, reason);

	e->last_election_loss = 0;
	election_timeout_stop(e);

	/* Start a new election by voting down this, and other, peers */
	e->state = election_start;
	return e->state;
	} else {
	char *loss_time = ctime(&e->last_election_loss);

	if (loss_time) {
	// Show only HH:MM:SS
	loss_time += 11;
	loss_time[8] = '\0';
	}
	crm_info("Ignoring %s round %d (owner node ID %s) pass vs %s because we lost less than %ds ago at %s",
	e->name, vote.election_id, vote.election_owner, vote.from,
	LOSS_DAMPEN, (loss_time? loss_time : "unknown"));
	}
	}

	e->last_election_loss = tm_now;

	do_crm_log(log_level, "%s round %d (owner node ID %s) lost: %s from %s (%s)",
	e->name, vote.election_id, vote.election_owner, vote.op,
	vote.from, reason);

	election_reset(e);
	send_no_vote(your_node, &vote);
	e->state = election_lost;
	return e->state;
	}

	/*!
	* \brief Reset any election dampening currently in effect
	*
	* \param[in] e Election object to clear
	*/
	void
	election_clear_dampening(election_t *e)
	{
	e->last_election_loss = 0;
	}
	diff --git a/lib/cluster/membership.c b/lib/cluster/membership.c
	index 9c981b2bea..8bf5764efa 100644
	--- a/lib/cluster/membership.c
	+++ b/lib/cluster/membership.c
	@@ -1,1191 +1,1333 @@
	/*
	* Copyright 2004-2020 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#ifndef _GNU_SOURCE
	# define _GNU_SOURCE
	#endif

	#include <sys/param.h>
	#include <sys/types.h>
	#include <stdio.h>
	#include <unistd.h>
	#include <string.h>
	#include <glib.h>
	#include <crm/common/ipc.h>
	#include <crm/cluster/internal.h>
	#include <crm/msg_xml.h>
	#include <crm/stonith-ng.h>
	+#include "crmcluster_private.h"

	/* The peer cache remembers cluster nodes that have been seen.
	* This is managed mostly automatically by libcluster, based on
	* cluster membership events.
	*
	* Because cluster nodes can have conflicting names or UUIDs,
	* the hash table key is a uniquely generated ID.
	*/
	GHashTable *crm_peer_cache = NULL;

	/*
	* The remote peer cache tracks pacemaker_remote nodes. While the
	* value has the same type as the peer cache's, it is tracked separately for
	* three reasons: pacemaker_remote nodes can't have conflicting names or UUIDs,
	* so the name (which is also the UUID) is used as the hash table key; there
	* is no equivalent of membership events, so management is not automatic; and
	* most users of the peer cache need to exclude pacemaker_remote nodes.
	*
	* That said, using a single cache would be more logical and less error-prone,
	* so it would be a good idea to merge them one day.
	*
	* libcluster provides two avenues for populating the cache:
	* crm_remote_peer_get() and crm_remote_peer_cache_remove() directly manage it,
	* while crm_remote_peer_cache_refresh() populates it via the CIB.
	*/
	GHashTable *crm_remote_peer_cache = NULL;

	-GHashTable *crm_known_peer_cache = NULL;
	+/*
	+ * The known node cache tracks cluster and remote nodes that have been seen in
	+ * the CIB. It is useful mainly when a caller needs to know about a node that
	+ * may no longer be in the membership, but doesn't want to add the node to the
	+ * main peer cache tables.
	+ */
	+static GHashTable *known_node_cache = NULL;

	unsigned long long crm_peer_seq = 0;
	gboolean crm_have_quorum = FALSE;
	static gboolean crm_autoreap = TRUE;

	+// Flag setting and clearing for crm_node_t:flags
	+
	+#define set_peer_flags(peer, flags_to_set) do { \
	+ (peer)->flags = pcmk__set_flags_as(__func__, __LINE__, LOG_TRACE, \
	+ "Peer", (peer)->uname, \
	+ (peer)->flags, (flags_to_set), \
	+ #flags_to_set); \
	+ } while (0)
	+
	+#define clear_peer_flags(peer, flags_to_clear) do { \
	+ (peer)->flags = pcmk__clear_flags_as(__func__, __LINE__, \
	+ LOG_TRACE, \
	+ "Peer", (peer)->uname, \
	+ (peer)->flags, (flags_to_clear), \
	+ #flags_to_clear); \
	+ } while (0)
	+
	+static void update_peer_uname(crm_node_t node, const char uname);
	+
	int
	crm_remote_peer_cache_size(void)
	{
	if (crm_remote_peer_cache == NULL) {
	return 0;
	}
	return g_hash_table_size(crm_remote_peer_cache);
	}

	/*!
	* \brief Get a remote node peer cache entry, creating it if necessary
	*
	* \param[in] node_name Name of remote node
	*
	* \return Cache entry for node on success, NULL (and set errno) otherwise
	*
	* \note When creating a new entry, this will leave the node state undetermined,
	- * so the caller should also call crm_update_peer_state() if the state is
	- * known.
	+ * so the caller should also call pcmk__update_peer_state() if the state
	+ * is known.
	*/
	crm_node_t *
	crm_remote_peer_get(const char *node_name)
	{
	crm_node_t *node;

	if (node_name == NULL) {
	errno = -EINVAL;
	return NULL;
	}

	/* Return existing cache entry if one exists */
	node = g_hash_table_lookup(crm_remote_peer_cache, node_name);
	if (node) {
	return node;
	}

	/* Allocate a new entry */
	node = calloc(1, sizeof(crm_node_t));
	if (node == NULL) {
	return NULL;
	}

	/* Populate the essential information */
	- pcmk__set_peer_flags(node, crm_remote_node);
	+ set_peer_flags(node, crm_remote_node);
	node->uuid = strdup(node_name);
	if (node->uuid == NULL) {
	free(node);
	errno = -ENOMEM;
	return NULL;
	}

	/* Add the new entry to the cache */
	g_hash_table_replace(crm_remote_peer_cache, node->uuid, node);
	crm_trace("added %s to remote cache", node_name);

	/* Update the entry's uname, ensuring peer status callbacks are called */
	- crm_update_peer_uname(node, node_name);
	+ update_peer_uname(node, node_name);
	return node;
	}

	void
	crm_remote_peer_cache_remove(const char *node_name)
	{
	if (g_hash_table_remove(crm_remote_peer_cache, node_name)) {
	crm_trace("removed %s from remote peer cache", node_name);
	}
	}

	/*!
	* \internal
	* \brief Return node status based on a CIB status entry
	*
	* \param[in] node_state XML of node state
	*
	* \return CRM_NODE_LOST if XML_NODE_IN_CLUSTER is false in node_state,
	* CRM_NODE_MEMBER otherwise
	* \note Unlike most boolean XML attributes, this one defaults to true, for
	* backward compatibility with older controllers that don't set it.
	*/
	static const char *
	remote_state_from_cib(xmlNode *node_state)
	{
	const char *status;

	status = crm_element_value(node_state, XML_NODE_IN_CLUSTER);
	if (status && !crm_is_true(status)) {
	status = CRM_NODE_LOST;
	} else {
	status = CRM_NODE_MEMBER;
	}
	return status;
	}

	/* user data for looping through remote node xpath searches */
	struct refresh_data {
	const char field; / XML attribute to check for node name */
	gboolean has_state; /* whether to update node state based on XML */
	};

	/*!
	* \internal
	* \brief Process one pacemaker_remote node xpath search result
	*
	* \param[in] result XML search result
	* \param[in] user_data what to look for in the XML
	*/
	static void
	remote_cache_refresh_helper(xmlNode result, void user_data)
	{
	struct refresh_data *data = user_data;
	const char *remote = crm_element_value(result, data->field);
	const char *state = NULL;
	crm_node_t *node;

	CRM_CHECK(remote != NULL, return);

	/* Determine node's state, if the result has it */
	if (data->has_state) {
	state = remote_state_from_cib(result);
	}

	/* Check whether cache already has entry for node */
	node = g_hash_table_lookup(crm_remote_peer_cache, remote);

	if (node == NULL) {
	/* Node is not in cache, so add a new entry for it */
	node = crm_remote_peer_get(remote);
	CRM_ASSERT(node);
	if (state) {
	- crm_update_peer_state(__func__, node, state, 0);
	+ pcmk__update_peer_state(__func__, node, state, 0);
	}

	} else if (pcmk_is_set(node->flags, crm_node_dirty)) {
	/* Node is in cache and hasn't been updated already, so mark it clean */
	- pcmk__clear_peer_flags(node, crm_node_dirty);
	+ clear_peer_flags(node, crm_node_dirty);
	if (state) {
	- crm_update_peer_state(__func__, node, state, 0);
	+ pcmk__update_peer_state(__func__, node, state, 0);
	}
	}
	}

	static void
	mark_dirty(gpointer key, gpointer value, gpointer user_data)
	{
	- pcmk__set_peer_flags((crm_node_t *) value, crm_node_dirty);
	+ set_peer_flags((crm_node_t *) value, crm_node_dirty);
	}

	static gboolean
	is_dirty(gpointer key, gpointer value, gpointer user_data)
	{
	return pcmk_is_set(((crm_node_t*)value)->flags, crm_node_dirty);
	}

	/* search string to find CIB resources entries for guest nodes */
	#define XPATH_GUEST_NODE_CONFIG \
	"//" XML_TAG_CIB "//" XML_CIB_TAG_CONFIGURATION "//" XML_CIB_TAG_RESOURCE \
	"//" XML_TAG_META_SETS "//" XML_CIB_TAG_NVPAIR \
	"[@name='" XML_RSC_ATTR_REMOTE_NODE "']"

	/* search string to find CIB resources entries for remote nodes */
	#define XPATH_REMOTE_NODE_CONFIG \
	"//" XML_TAG_CIB "//" XML_CIB_TAG_CONFIGURATION "//" XML_CIB_TAG_RESOURCE \
	"[@type='remote'][@provider='pacemaker']"

	/* search string to find CIB node status entries for pacemaker_remote nodes */
	#define XPATH_REMOTE_NODE_STATUS \
	"//" XML_TAG_CIB "//" XML_CIB_TAG_STATUS "//" XML_CIB_TAG_STATE \
	"[@" XML_NODE_IS_REMOTE "='true']"

	/*!
	* \brief Repopulate the remote peer cache based on CIB XML
	*
	* \param[in] xmlNode CIB XML to parse
	*/
	void
	crm_remote_peer_cache_refresh(xmlNode *cib)
	{
	struct refresh_data data;

	crm_peer_init();

	/* First, we mark all existing cache entries as dirty,
	* so that later we can remove any that weren't in the CIB.
	* We don't empty the cache, because we need to detect changes in state.
	*/
	g_hash_table_foreach(crm_remote_peer_cache, mark_dirty, NULL);

	/* Look for guest nodes and remote nodes in the status section */
	data.field = "id";
	data.has_state = TRUE;
	crm_foreach_xpath_result(cib, XPATH_REMOTE_NODE_STATUS,
	remote_cache_refresh_helper, &data);

	/* Look for guest nodes and remote nodes in the configuration section,
	* because they may have just been added and not have a status entry yet.
	* In that case, the cached node state will be left NULL, so that the
	* peer status callback isn't called until we're sure the node started
	* successfully.
	*/
	data.field = "value";
	data.has_state = FALSE;
	crm_foreach_xpath_result(cib, XPATH_GUEST_NODE_CONFIG,
	remote_cache_refresh_helper, &data);
	data.field = "id";
	data.has_state = FALSE;
	crm_foreach_xpath_result(cib, XPATH_REMOTE_NODE_CONFIG,
	remote_cache_refresh_helper, &data);

	/* Remove all old cache entries that weren't seen in the CIB */
	g_hash_table_foreach_remove(crm_remote_peer_cache, is_dirty, NULL);
	}

	gboolean
	crm_is_peer_active(const crm_node_t * node)
	{
	if(node == NULL) {
	return FALSE;
	}

	if (pcmk_is_set(node->flags, crm_remote_node)) {
	/* remote nodes are never considered active members. This
	* guarantees they will never be considered for DC membership.*/
	return FALSE;
	}
	#if SUPPORT_COROSYNC
	if (is_corosync_cluster()) {
	return crm_is_corosync_peer_active(node);
	}
	#endif
	crm_err("Unhandled cluster type: %s", name_for_cluster_type(get_cluster_type()));
	return FALSE;
	}

	static gboolean
	crm_reap_dead_member(gpointer key, gpointer value, gpointer user_data)
	{
	crm_node_t *node = value;
	crm_node_t *search = user_data;

	if (search == NULL) {
	return FALSE;

	} else if (search->id && node->id != search->id) {
	return FALSE;

	} else if (search->id == 0 && !pcmk__str_eq(node->uname, search->uname, pcmk__str_casei)) {
	return FALSE;

	} else if (crm_is_peer_active(value) == FALSE) {
	crm_info("Removing node with name %s and id %u from membership cache",
	(node->uname? node->uname : "unknown"), node->id);
	return TRUE;
	}
	return FALSE;
	}

	/*!
	* \brief Remove all peer cache entries matching a node ID and/or uname
	*
	* \param[in] id ID of node to remove (or 0 to ignore)
	* \param[in] name Uname of node to remove (or NULL to ignore)
	*
	* \return Number of cache entries removed
	*/
	guint
	reap_crm_member(uint32_t id, const char *name)
	{
	int matches = 0;
	crm_node_t search;

	if (crm_peer_cache == NULL) {
	crm_trace("Membership cache not initialized, ignoring purge request");
	return 0;
	}

	search.id = id;
	search.uname = name ? strdup(name) : NULL;
	matches = g_hash_table_foreach_remove(crm_peer_cache, crm_reap_dead_member, &search);
	if(matches) {
	crm_notice("Purged %d peer%s with id=%u%s%s from the membership cache",
	matches, pcmk__plural_s(matches), search.id,
	(search.uname? " and/or uname=" : ""),
	(search.uname? search.uname : ""));

	} else {
	crm_info("No peers with id=%u%s%s to purge from the membership cache",
	search.id, (search.uname? " and/or uname=" : ""),
	(search.uname? search.uname : ""));
	}

	free(search.uname);
	return matches;
	}

	static void
	-crm_count_peer(gpointer key, gpointer value, gpointer user_data)
	+count_peer(gpointer key, gpointer value, gpointer user_data)
	{
	guint *count = user_data;
	crm_node_t *node = value;

	if (crm_is_peer_active(node)) {
	count = count + 1;
	}
	}

	guint
	crm_active_peers(void)
	{
	guint count = 0;

	if (crm_peer_cache) {
	- g_hash_table_foreach(crm_peer_cache, crm_count_peer, &count);
	+ g_hash_table_foreach(crm_peer_cache, count_peer, &count);
	}
	return count;
	}

	static void
	destroy_crm_node(gpointer data)
	{
	crm_node_t *node = data;

	crm_trace("Destroying entry for node %u: %s", node->id, node->uname);

	free(node->uname);
	free(node->state);
	free(node->uuid);
	free(node->expected);
	free(node);
	}

	void
	crm_peer_init(void)
	{
	if (crm_peer_cache == NULL) {
	crm_peer_cache = g_hash_table_new_full(crm_strcase_hash, crm_strcase_equal, free, destroy_crm_node);
	}

	if (crm_remote_peer_cache == NULL) {
	crm_remote_peer_cache = g_hash_table_new_full(crm_strcase_hash, crm_strcase_equal, NULL, destroy_crm_node);
	}

	- if (crm_known_peer_cache == NULL) {
	- crm_known_peer_cache = g_hash_table_new_full(crm_strcase_hash, crm_strcase_equal, free, destroy_crm_node);
	+ if (known_node_cache == NULL) {
	+ known_node_cache = g_hash_table_new_full(crm_strcase_hash,
	+ crm_strcase_equal, free,
	+ destroy_crm_node);
	}
	}

	void
	crm_peer_destroy(void)
	{
	if (crm_peer_cache != NULL) {
	crm_trace("Destroying peer cache with %d members", g_hash_table_size(crm_peer_cache));
	g_hash_table_destroy(crm_peer_cache);
	crm_peer_cache = NULL;
	}

	if (crm_remote_peer_cache != NULL) {
	crm_trace("Destroying remote peer cache with %d members", g_hash_table_size(crm_remote_peer_cache));
	g_hash_table_destroy(crm_remote_peer_cache);
	crm_remote_peer_cache = NULL;
	}

	- if (crm_known_peer_cache != NULL) {
	- crm_trace("Destroying known peer cache with %d members", g_hash_table_size(crm_known_peer_cache));
	- g_hash_table_destroy(crm_known_peer_cache);
	- crm_known_peer_cache = NULL;
	+ if (known_node_cache != NULL) {
	+ crm_trace("Destroying known node cache with %d members",
	+ g_hash_table_size(known_node_cache));
	+ g_hash_table_destroy(known_node_cache);
	+ known_node_cache = NULL;
	}

	}

	-void (crm_status_callback) (enum crm_status_type, crm_node_t , const void *) = NULL;
	+static void (peer_status_callback)(enum crm_status_type, crm_node_t ,
	+ const void *) = NULL;

	/*!
	* \brief Set a client function that will be called after peer status changes
	*
	* \param[in] dispatch Pointer to function to use as callback
	*
	* \note Previously, client callbacks were responsible for peer cache
	* management. This is no longer the case, and client callbacks should do
	* only client-specific handling. Callbacks MUST NOT add or remove entries
	* in the peer caches.
	*/
	void
	crm_set_status_callback(void (dispatch) (enum crm_status_type, crm_node_t , const void *))
	{
	- crm_status_callback = dispatch;
	+ peer_status_callback = dispatch;
	}

	/*!
	* \brief Tell the library whether to automatically reap lost nodes
	*
	* If TRUE (the default), calling crm_update_peer_proc() will also update the
	- * peer state to CRM_NODE_MEMBER or CRM_NODE_LOST, and crm_update_peer_state()
	+ * peer state to CRM_NODE_MEMBER or CRM_NODE_LOST, and pcmk__update_peer_state()
	* will reap peers whose state changes to anything other than CRM_NODE_MEMBER.
	* Callers should leave this enabled unless they plan to manage the cache
	* separately on their own.
	*
	* \param[in] autoreap TRUE to enable automatic reaping, FALSE to disable
	*/
	void
	crm_set_autoreap(gboolean autoreap)
	{
	crm_autoreap = autoreap;
	}

	-static void crm_dump_peer_hash(int level, const char *caller)
	+static void
	+dump_peer_hash(int level, const char *caller)
	{
	GHashTableIter iter;
	const char *id = NULL;
	crm_node_t *node = NULL;

	g_hash_table_iter_init(&iter, crm_peer_cache);
	while (g_hash_table_iter_next(&iter, (gpointer ) &id, (gpointer ) &node)) {
	do_crm_log(level, "%s: Node %u/%s = %p - %s", caller, node->id, node->uname, node, id);
	}
	}

	-static gboolean crm_hash_find_by_data(gpointer key, gpointer value, gpointer user_data)
	+static gboolean
	+hash_find_by_data(gpointer key, gpointer value, gpointer user_data)
	{
	- if(value == user_data) {
	- return TRUE;
	- }
	- return FALSE;
	+ return value == user_data;
	}

	+/*!
	+ * \internal
	+ * \brief Search caches for a node (cluster or Pacemaker Remote)
	+ *
	+ * \param[in] id If not 0, cluster node ID to search for
	+ * \param[in] uname If not NULL, node name to search for
	+ * \param[in] flags Bitmask of enum crm_get_peer_flags
	+ *
	+ * \return Node cache entry if found, otherwise NULL
	+ */
	crm_node_t *
	-crm_find_peer_full(unsigned int id, const char *uname, int flags)
	+pcmk__search_node_caches(unsigned int id, const char *uname, uint32_t flags)
	{
	crm_node_t *node = NULL;

	CRM_ASSERT(id > 0 \|\| uname != NULL);

	crm_peer_init();

	- if ((uname != NULL) && (flags & CRM_GET_PEER_REMOTE)) {
	+ if ((uname != NULL) && pcmk_is_set(flags, CRM_GET_PEER_REMOTE)) {
	node = g_hash_table_lookup(crm_remote_peer_cache, uname);
	}

	- if (node == NULL && (flags & CRM_GET_PEER_CLUSTER)) {
	- node = crm_find_peer(id, uname);
	+ if ((node == NULL) && pcmk_is_set(flags, CRM_GET_PEER_CLUSTER)) {
	+ node = pcmk__search_cluster_node_cache(id, uname);
	}
	return node;
	}

	+/*!
	+ * \brief Get a node cache entry (cluster or Pacemaker Remote)
	+ *
	+ * \param[in] id If not 0, cluster node ID to search for
	+ * \param[in] uname If not NULL, node name to search for
	+ * \param[in] flags Bitmask of enum crm_get_peer_flags
	+ *
	+ * \return (Possibly newly created) node cache entry
	+ */
	crm_node_t *
	crm_get_peer_full(unsigned int id, const char *uname, int flags)
	{
	crm_node_t *node = NULL;

	CRM_ASSERT(id > 0 \|\| uname != NULL);

	crm_peer_init();

	- if (flags & CRM_GET_PEER_REMOTE) {
	+ if (pcmk_is_set(flags, CRM_GET_PEER_REMOTE)) {
	node = g_hash_table_lookup(crm_remote_peer_cache, uname);
	}

	- if (node == NULL && (flags & CRM_GET_PEER_CLUSTER)) {
	+ if ((node == NULL) && pcmk_is_set(flags, CRM_GET_PEER_CLUSTER)) {
	node = crm_get_peer(id, uname);
	}
	return node;
	}

	+/*!
	+ * \internal
	+ * \brief Search cluster node cache
	+ *
	+ * \param[in] id If not 0, cluster node ID to search for
	+ * \param[in] uname If not NULL, node name to search for
	+ *
	+ * \return Cluster node cache entry if found, otherwise NULL
	+ */
	crm_node_t *
	-crm_find_peer(unsigned int id, const char *uname)
	+pcmk__search_cluster_node_cache(unsigned int id, const char *uname)
	{
	GHashTableIter iter;
	crm_node_t *node = NULL;
	crm_node_t *by_id = NULL;
	crm_node_t *by_name = NULL;

	CRM_ASSERT(id > 0 \|\| uname != NULL);

	crm_peer_init();

	if (uname != NULL) {
	g_hash_table_iter_init(&iter, crm_peer_cache);
	while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &node)) {
	if(node->uname && strcasecmp(node->uname, uname) == 0) {
	crm_trace("Name match: %s = %p", node->uname, node);
	by_name = node;
	break;
	}
	}
	}

	if (id > 0) {
	g_hash_table_iter_init(&iter, crm_peer_cache);
	while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &node)) {
	if(node->id == id) {
	crm_trace("ID match: %u = %p", node->id, node);
	by_id = node;
	break;
	}
	}
	}

	node = by_id; /* Good default */
	if(by_id == by_name) {
	/* Nothing to do if they match (both NULL counts) */
	crm_trace("Consistent: %p for %u/%s", by_id, id, uname);

	} else if(by_id == NULL && by_name) {
	crm_trace("Only one: %p for %u/%s", by_name, id, uname);

	if(id && by_name->id) {
	- crm_dump_peer_hash(LOG_WARNING, __func__);
	+ dump_peer_hash(LOG_WARNING, __func__);
	crm_crit("Node %u and %u share the same name '%s'",
	id, by_name->id, uname);
	node = NULL; /* Create a new one */

	} else {
	node = by_name;
	}

	} else if(by_name == NULL && by_id) {
	crm_trace("Only one: %p for %u/%s", by_id, id, uname);

	if(uname && by_id->uname) {
	- crm_dump_peer_hash(LOG_WARNING, __func__);
	+ dump_peer_hash(LOG_WARNING, __func__);
	crm_crit("Node '%s' and '%s' share the same cluster nodeid %u: assuming '%s' is correct",
	uname, by_id->uname, id, uname);
	}

	} else if(uname && by_id->uname) {
	if(pcmk__str_eq(uname, by_id->uname, pcmk__str_casei)) {
	crm_notice("Node '%s' has changed its ID from %u to %u", by_id->uname, by_name->id, by_id->id);
	- g_hash_table_foreach_remove(crm_peer_cache, crm_hash_find_by_data, by_name);
	+ g_hash_table_foreach_remove(crm_peer_cache, hash_find_by_data, by_name);

	} else {
	crm_warn("Node '%s' and '%s' share the same cluster nodeid: %u %s", by_id->uname, by_name->uname, id, uname);
	- crm_dump_peer_hash(LOG_INFO, __func__);
	+ dump_peer_hash(LOG_INFO, __func__);
	crm_abort(__FILE__, __func__, __LINE__, "member weirdness", TRUE,
	TRUE);
	}

	} else if(id && by_name->id) {
	crm_warn("Node %u and %u share the same name: '%s'", by_id->id, by_name->id, uname);

	} else {
	/* Simple merge */

	/* Only corosync-based clusters use node IDs. The functions that call
	- * crm_update_peer_state() and crm_update_peer_proc() only know nodeid,
	- * so 'by_id' is authoritative when merging.
	+ * pcmk__update_peer_state() and crm_update_peer_proc() only know
	+ * nodeid, so 'by_id' is authoritative when merging.
	*/
	- crm_dump_peer_hash(LOG_DEBUG, __func__);
	+ dump_peer_hash(LOG_DEBUG, __func__);

	crm_info("Merging %p into %p", by_name, by_id);
	- g_hash_table_foreach_remove(crm_peer_cache, crm_hash_find_by_data, by_name);
	+ g_hash_table_foreach_remove(crm_peer_cache, hash_find_by_data, by_name);
	}

	return node;
	}

	#if SUPPORT_COROSYNC
	static guint
	-crm_remove_conflicting_peer(crm_node_t *node)
	+remove_conflicting_peer(crm_node_t *node)
	{
	int matches = 0;
	GHashTableIter iter;
	crm_node_t *existing_node = NULL;

	if (node->id == 0 \|\| node->uname == NULL) {
	return 0;
	}

	- if (corosync_cmap_has_config("nodelist") != 0) {
	+ if (!pcmk__corosync_has_nodelist()) {
	return 0;
	}

	g_hash_table_iter_init(&iter, crm_peer_cache);
	while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &existing_node)) {
	if (existing_node->id > 0
	&& existing_node->id != node->id
	&& existing_node->uname != NULL
	&& strcasecmp(existing_node->uname, node->uname) == 0) {

	if (crm_is_peer_active(existing_node)) {
	continue;
	}

	crm_warn("Removing cached offline node %u/%s which has conflicting uname with %u",
	existing_node->id, existing_node->uname, node->id);

	g_hash_table_iter_remove(&iter);
	matches++;
	}
	}

	return matches;
	}
	#endif

	+/*!
	+ * \brief Get a cluster node cache entry
	+ *
	+ * \param[in] id If not 0, cluster node ID to search for
	+ * \param[in] uname If not NULL, node name to search for
	+ *
	+ * \return (Possibly newly created) cluster node cache entry
	+ */
	/* coverity[-alloc] Memory is referenced in one or both hashtables */
	crm_node_t *
	crm_get_peer(unsigned int id, const char *uname)
	{
	crm_node_t *node = NULL;
	char *uname_lookup = NULL;

	CRM_ASSERT(id > 0 \|\| uname != NULL);

	crm_peer_init();

	- node = crm_find_peer(id, uname);
	+ node = pcmk__search_cluster_node_cache(id, uname);

	/* if uname wasn't provided, and find_peer did not turn up a uname based on id.
	* we need to do a lookup of the node name using the id in the cluster membership. */
	if ((node == NULL \|\| node->uname == NULL) && (uname == NULL)) {
	uname_lookup = get_node_name(id);
	}

	if (uname_lookup) {
	uname = uname_lookup;
	crm_trace("Inferred a name of '%s' for node %u", uname, id);

	/* try to turn up the node one more time now that we know the uname. */
	if (node == NULL) {
	- node = crm_find_peer(id, uname);
	+ node = pcmk__search_cluster_node_cache(id, uname);
	}
	}


	if (node == NULL) {
	char *uniqueid = crm_generate_uuid();

	node = calloc(1, sizeof(crm_node_t));
	CRM_ASSERT(node);

	crm_info("Created entry %s/%p for node %s/%u (%d total)",
	uniqueid, node, uname, id, 1 + g_hash_table_size(crm_peer_cache));
	g_hash_table_replace(crm_peer_cache, uniqueid, node);
	}

	if(id > 0 && uname && (node->id == 0 \|\| node->uname == NULL)) {
	crm_info("Node %u is now known as %s", id, uname);
	}

	if(id > 0 && node->id == 0) {
	node->id = id;
	}

	if (uname && (node->uname == NULL)) {
	- crm_update_peer_uname(node, uname);
	+ update_peer_uname(node, uname);
	}

	if(node->uuid == NULL) {
	const char *uuid = crm_peer_uuid(node);

	if (uuid) {
	crm_info("Node %u has uuid %s", id, uuid);

	} else {
	crm_info("Cannot obtain a UUID for node %u/%s", id, node->uname);
	}
	}

	free(uname_lookup);

	return node;
	}

	/*!
	* \internal
	* \brief Update a node's uname
	*
	* \param[in] node Node object to update
	* \param[in] uname New name to set
	*
	* \note This function should not be called within a peer cache iteration,
	* because in some cases it can remove conflicting cache entries,
	* which would invalidate the iterator.
	*/
	-void
	-crm_update_peer_uname(crm_node_t node, const char uname)
	+static void
	+update_peer_uname(crm_node_t node, const char uname)
	{
	CRM_CHECK(uname != NULL,
	crm_err("Bug: can't update node name without name"); return);
	CRM_CHECK(node != NULL,
	crm_err("Bug: can't update node name to %s without node", uname);
	return);

	if (pcmk__str_eq(uname, node->uname, pcmk__str_casei)) {
	crm_debug("Node uname '%s' did not change", uname);
	return;
	}

	for (const char c = uname; c; ++c) {
	if ((c >= 'A') && (c <= 'Z')) {
	crm_warn("Node names with capitals are discouraged, consider changing '%s'",
	uname);
	break;
	}
	}

	free(node->uname);
	node->uname = strdup(uname);
	CRM_ASSERT(node->uname != NULL);

	- if (crm_status_callback) {
	- crm_status_callback(crm_status_uname, node, NULL);
	+ if (peer_status_callback != NULL) {
	+ peer_status_callback(crm_status_uname, node, NULL);
	}

	#if SUPPORT_COROSYNC
	if (is_corosync_cluster() && !pcmk_is_set(node->flags, crm_remote_node)) {
	- crm_remove_conflicting_peer(node);
	+ remove_conflicting_peer(node);
	}
	#endif
	}

	+/*!
	+ * \internal
	+ * \brief Get log-friendly string equivalent of a process flag
	+ *
	+ * \param[in] proc Process flag
	+ *
	+ * \return Log-friendly string equivalent of \p proc
	+ */
	+static inline const char *
	+proc2text(enum crm_proc_flag proc)
	+{
	+ const char *text = "unknown";
	+
	+ switch (proc) {
	+ case crm_proc_none:
	+ text = "none";
	+ break;
	+ case crm_proc_based:
	+ text = "pacemaker-based";
	+ break;
	+ case crm_proc_controld:
	+ text = "pacemaker-controld";
	+ break;
	+ case crm_proc_schedulerd:
	+ text = "pacemaker-schedulerd";
	+ break;
	+ case crm_proc_execd:
	+ text = "pacemaker-execd";
	+ break;
	+ case crm_proc_attrd:
	+ text = "pacemaker-attrd";
	+ break;
	+ case crm_proc_fenced:
	+ text = "pacemaker-fenced";
	+ break;
	+ case crm_proc_cpg:
	+ text = "corosync-cpg";
	+ break;
	+ }
	+ return text;
	+}
	+
	/*!
	* \internal
	* \brief Update a node's process information (and potentially state)
	*
	* \param[in] source Caller's function name (for log messages)
	* \param[in] node Node object to update
	* \param[in] flag Bitmask of new process information
	* \param[in] status node status (online, offline, etc.)
	*
	* \return NULL if any node was reaped from peer caches, value of node otherwise
	*
	* \note If this function returns NULL, the supplied node object was likely
	* freed and should not be used again. This function should not be
	* called within a cache iteration if reaping is possible, otherwise
	* reaping could invalidate the iterator.
	*/
	crm_node_t *
	crm_update_peer_proc(const char source, crm_node_t node, uint32_t flag, const char *status)
	{
	uint32_t last = 0;
	gboolean changed = FALSE;

	CRM_CHECK(node != NULL, crm_err("%s: Could not set %s to %s for NULL",
	- source, peer2text(flag), status); return NULL);
	+ source, proc2text(flag), status);
	+ return NULL);

	/* Pacemaker doesn't spawn processes on remote nodes */
	if (pcmk_is_set(node->flags, crm_remote_node)) {
	return node;
	}

	last = node->processes;
	if (status == NULL) {
	node->processes = flag;
	if (node->processes != last) {
	changed = TRUE;
	}

	} else if (pcmk__str_eq(status, ONLINESTATUS, pcmk__str_casei)) {
	if ((node->processes & flag) != flag) {
	node->processes = pcmk__set_flags_as(__func__, __LINE__,
	LOG_TRACE, "Peer process",
	node->uname, node->processes,
	flag, "processes");
	changed = TRUE;
	}

	} else if (node->processes & flag) {
	node->processes = pcmk__clear_flags_as(__func__, __LINE__,
	LOG_TRACE, "Peer process",
	node->uname, node->processes,
	flag, "processes");
	changed = TRUE;
	}

	if (changed) {
	if (status == NULL && flag <= crm_proc_none) {
	crm_info("%s: Node %s[%u] - all processes are now offline", source, node->uname,
	node->id);
	} else {
	crm_info("%s: Node %s[%u] - %s is now %s", source, node->uname, node->id,
	- peer2text(flag), status);
	+ proc2text(flag), status);
	}

	/* Call the client callback first, then update the peer state,
	* in case the node will be reaped
	*/
	- if (crm_status_callback) {
	- crm_status_callback(crm_status_processes, node, &last);
	+ if (peer_status_callback != NULL) {
	+ peer_status_callback(crm_status_processes, node, &last);
	}

	/* The client callback shouldn't touch the peer caches,
	* but as a safety net, bail if the peer cache was destroyed.
	*/
	if (crm_peer_cache == NULL) {
	return NULL;
	}

	if (crm_autoreap) {
	const char *peer_state = NULL;

	if (pcmk_is_set(node->processes, crm_get_cluster_proc())) {
	peer_state = CRM_NODE_MEMBER;
	} else {
	peer_state = CRM_NODE_LOST;
	}
	- node = crm_update_peer_state(__func__, node, peer_state, 0);
	+ node = pcmk__update_peer_state(__func__, node, peer_state, 0);
	}
	} else {
	crm_trace("%s: Node %s[%u] - %s is unchanged (%s)", source, node->uname, node->id,
	- peer2text(flag), status);
	+ proc2text(flag), status);
	}
	return node;
	}

	+/*!
	+ * \internal
	+ * \brief Update a cluster node cache entry's expected join state
	+ *
	+ * \param[in] source Caller's function name (for logging)
	+ * \param[in,out] node Node to update
	+ * \param[in] expected Node's new join state
	+ */
	void
	-crm_update_peer_expected(const char source, crm_node_t node, const char *expected)
	+pcmk__update_peer_expected(const char source, crm_node_t node,
	+ const char *expected)
	{
	char *last = NULL;
	gboolean changed = FALSE;

	CRM_CHECK(node != NULL, crm_err("%s: Could not set 'expected' to %s", source, expected);
	return);

	/* Remote nodes don't participate in joins */
	if (pcmk_is_set(node->flags, crm_remote_node)) {
	return;
	}

	last = node->expected;
	if (expected != NULL && !pcmk__str_eq(node->expected, expected, pcmk__str_casei)) {
	node->expected = strdup(expected);
	changed = TRUE;
	}

	if (changed) {
	crm_info("%s: Node %s[%u] - expected state is now %s (was %s)", source, node->uname, node->id,
	expected, last);
	free(last);
	} else {
	crm_trace("%s: Node %s[%u] - expected state is unchanged (%s)", source, node->uname,
	node->id, expected);
	}
	}

	/*!
	* \internal
	* \brief Update a node's state and membership information
	*
	* \param[in] source Caller's function name (for log messages)
	* \param[in] node Node object to update
	* \param[in] state Node's new state
	* \param[in] membership Node's new membership ID
	* \param[in] iter If not NULL, pointer to node's peer cache iterator
	*
	* \return NULL if any node was reaped, value of node otherwise
	*
	* \note If this function returns NULL, the supplied node object was likely
	* freed and should not be used again. This function may be called from
	* within a peer cache iteration if the iterator is supplied.
	*/
	static crm_node_t *
	-crm_update_peer_state_iter(const char source, crm_node_t node, const char state, uint64_t membership, GHashTableIter iter)
	+update_peer_state_iter(const char source, crm_node_t node, const char *state,
	+ uint64_t membership, GHashTableIter *iter)
	{
	gboolean is_member;

	CRM_CHECK(node != NULL,
	crm_err("Could not set state for unknown host to %s"
	CRM_XS " source=%s", state, source);
	return NULL);

	is_member = pcmk__str_eq(state, CRM_NODE_MEMBER, pcmk__str_casei);
	if (is_member) {
	node->when_lost = 0;
	if (membership) {
	node->last_seen = membership;
	}
	}

	if (state && !pcmk__str_eq(node->state, state, pcmk__str_casei)) {
	char *last = node->state;

	node->state = strdup(state);
	crm_notice("Node %s state is now %s " CRM_XS
	" nodeid=%u previous=%s source=%s", node->uname, state,
	node->id, (last? last : "unknown"), source);
	- if (crm_status_callback) {
	- crm_status_callback(crm_status_nstate, node, last);
	+ if (peer_status_callback != NULL) {
	+ peer_status_callback(crm_status_nstate, node, last);
	}
	free(last);

	if (crm_autoreap && !is_member
	&& !pcmk_is_set(node->flags, crm_remote_node)) {
	/* We only autoreap from the peer cache, not the remote peer cache,
	* because the latter should be managed only by
	* crm_remote_peer_cache_refresh().
	*/
	if(iter) {
	crm_notice("Purged 1 peer with id=%u and/or uname=%s from the membership cache", node->id, node->uname);
	g_hash_table_iter_remove(iter);

	} else {
	reap_crm_member(node->id, node->uname);
	}
	node = NULL;
	}

	} else {
	crm_trace("Node %s state is unchanged (%s) " CRM_XS
	" nodeid=%u source=%s", node->uname, state, node->id, source);
	}
	return node;
	}

	/*!
	* \brief Update a node's state and membership information
	*
	* \param[in] source Caller's function name (for log messages)
	* \param[in] node Node object to update
	* \param[in] state Node's new state
	* \param[in] membership Node's new membership ID
	*
	* \return NULL if any node was reaped, value of node otherwise
	*
	* \note If this function returns NULL, the supplied node object was likely
	* freed and should not be used again. This function should not be
	* called within a cache iteration if reaping is possible,
	* otherwise reaping could invalidate the iterator.
	*/
	crm_node_t *
	-crm_update_peer_state(const char source, crm_node_t node, const char *state, uint64_t membership)
	+pcmk__update_peer_state(const char source, crm_node_t node,
	+ const char *state, uint64_t membership)
	{
	- return crm_update_peer_state_iter(source, node, state, membership, NULL);
	+ return update_peer_state_iter(source, node, state, membership, NULL);
	}

	/*!
	* \internal
	* \brief Reap all nodes from cache whose membership information does not match
	*
	* \param[in] membership Membership ID of nodes to keep
	*/
	void
	-crm_reap_unseen_nodes(uint64_t membership)
	+pcmk__reap_unseen_nodes(uint64_t membership)
	{
	GHashTableIter iter;
	crm_node_t *node = NULL;

	crm_trace("Reaping unseen nodes...");
	g_hash_table_iter_init(&iter, crm_peer_cache);
	while (g_hash_table_iter_next(&iter, NULL, (gpointer *)&node)) {
	if (node->last_seen != membership) {
	if (node->state) {
	/*
	- * Calling crm_update_peer_state_iter() allows us to
	+ * Calling update_peer_state_iter() allows us to
	* remove the node from crm_peer_cache without
	* invalidating our iterator
	*/
	- crm_update_peer_state_iter(__func__, node, CRM_NODE_LOST,
	+ update_peer_state_iter(__func__, node, CRM_NODE_LOST,
	membership, &iter);

	} else {
	crm_info("State of node %s[%u] is still unknown",
	node->uname, node->id);
	}
	}
	}
	}

	-int
	-crm_terminate_member(int nodeid, const char uname, void unused)
	-{
	- /* Always use the synchronous, non-mainloop version */
	- return stonith_api_kick(nodeid, uname, 120, TRUE);
	-}
	-
	-int
	-crm_terminate_member_no_mainloop(int nodeid, const char uname, int connection)
	-{
	- return stonith_api_kick(nodeid, uname, 120, TRUE);
	-}
	-
	static crm_node_t *
	-crm_find_known_peer(const char id, const char uname)
	+find_known_node(const char id, const char uname)
	{
	GHashTableIter iter;
	crm_node_t *node = NULL;
	crm_node_t *by_id = NULL;
	crm_node_t *by_name = NULL;

	if (uname) {
	- g_hash_table_iter_init(&iter, crm_known_peer_cache);
	+ g_hash_table_iter_init(&iter, known_node_cache);
	while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &node)) {
	if (node->uname && strcasecmp(node->uname, uname) == 0) {
	crm_trace("Name match: %s = %p", node->uname, node);
	by_name = node;
	break;
	}
	}
	}

	if (id) {
	- g_hash_table_iter_init(&iter, crm_known_peer_cache);
	+ g_hash_table_iter_init(&iter, known_node_cache);
	while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &node)) {
	if(strcasecmp(node->uuid, id) == 0) {
	crm_trace("ID match: %s= %p", id, node);
	by_id = node;
	break;
	}
	}
	}

	node = by_id; /* Good default */
	if (by_id == by_name) {
	/* Nothing to do if they match (both NULL counts) */
	crm_trace("Consistent: %p for %s/%s", by_id, id, uname);

	} else if (by_id == NULL && by_name) {
	crm_trace("Only one: %p for %s/%s", by_name, id, uname);

	if (id) {
	node = NULL;

	} else {
	node = by_name;
	}

	} else if (by_name == NULL && by_id) {
	crm_trace("Only one: %p for %s/%s", by_id, id, uname);

	if (uname) {
	node = NULL;
	}

	} else if (uname && by_id->uname
	&& pcmk__str_eq(uname, by_id->uname, pcmk__str_casei)) {
	/* Multiple nodes have the same uname in the CIB.
	* Return by_id. */

	} else if (id && by_name->uuid
	&& pcmk__str_eq(id, by_name->uuid, pcmk__str_casei)) {
	/* Multiple nodes have the same id in the CIB.
	* Return by_name. */
	node = by_name;

	} else {
	node = NULL;
	}

	if (node == NULL) {
	crm_debug("Couldn't find node%s%s%s%s",
	id? " " : "",
	id? id : "",
	uname? " with name " : "",
	uname? uname : "");
	}

	return node;
	}

	static void
	-known_peer_cache_refresh_helper(xmlNode xml_node, void user_data)
	+known_node_cache_refresh_helper(xmlNode xml_node, void user_data)
	{
	const char *id = crm_element_value(xml_node, XML_ATTR_ID);
	const char *uname = crm_element_value(xml_node, XML_ATTR_UNAME);
	crm_node_t * node = NULL;

	CRM_CHECK(id != NULL && uname !=NULL, return);
	- node = crm_find_known_peer(id, uname);
	+ node = find_known_node(id, uname);

	if (node == NULL) {
	char *uniqueid = crm_generate_uuid();

	node = calloc(1, sizeof(crm_node_t));
	CRM_ASSERT(node != NULL);

	node->uname = strdup(uname);
	CRM_ASSERT(node->uname != NULL);

	node->uuid = strdup(id);
	CRM_ASSERT(node->uuid != NULL);

	- g_hash_table_replace(crm_known_peer_cache, uniqueid, node);
	+ g_hash_table_replace(known_node_cache, uniqueid, node);

	} else if (pcmk_is_set(node->flags, crm_node_dirty)) {
	if (!pcmk__str_eq(uname, node->uname, pcmk__str_casei)) {
	free(node->uname);
	node->uname = strdup(uname);
	CRM_ASSERT(node->uname != NULL);
	}

	/* Node is in cache and hasn't been updated already, so mark it clean */
	- pcmk__clear_peer_flags(node, crm_node_dirty);
	+ clear_peer_flags(node, crm_node_dirty);
	}

	}

	#define XPATH_MEMBER_NODE_CONFIG \
	"//" XML_TAG_CIB "/" XML_CIB_TAG_CONFIGURATION "/" XML_CIB_TAG_NODES \
	"/" XML_CIB_TAG_NODE "[not(@type) or @type='member']"

	static void
	-crm_known_peer_cache_refresh(xmlNode *cib)
	+refresh_known_node_cache(xmlNode *cib)
	{
	crm_peer_init();

	- g_hash_table_foreach(crm_known_peer_cache, mark_dirty, NULL);
	+ g_hash_table_foreach(known_node_cache, mark_dirty, NULL);

	crm_foreach_xpath_result(cib, XPATH_MEMBER_NODE_CONFIG,
	- known_peer_cache_refresh_helper, NULL);
	+ known_node_cache_refresh_helper, NULL);

	/* Remove all old cache entries that weren't seen in the CIB */
	- g_hash_table_foreach_remove(crm_known_peer_cache, is_dirty, NULL);
	+ g_hash_table_foreach_remove(known_node_cache, is_dirty, NULL);
	}

	void
	-crm_peer_caches_refresh(xmlNode *cib)
	+pcmk__refresh_node_caches_from_cib(xmlNode *cib)
	{
	crm_remote_peer_cache_refresh(cib);
	- crm_known_peer_cache_refresh(cib);
	+ refresh_known_node_cache(cib);
	}

	+/*!
	+ * \internal
	+ * \brief Search known node cache
	+ *
	+ * \param[in] id If not 0, cluster node ID to search for
	+ * \param[in] uname If not NULL, node name to search for
	+ * \param[in] flags Bitmask of enum crm_get_peer_flags
	+ *
	+ * \return Known node cache entry if found, otherwise NULL
	+ */
	crm_node_t *
	-crm_find_known_peer_full(unsigned int id, const char *uname, int flags)
	+pcmk__search_known_node_cache(unsigned int id, const char *uname,
	+ uint32_t flags)
	{
	crm_node_t *node = NULL;
	char *id_str = NULL;

	CRM_ASSERT(id > 0 \|\| uname != NULL);

	- node = crm_find_peer_full(id, uname, flags);
	+ node = pcmk__search_node_caches(id, uname, flags);

	if (node \|\| !(flags & CRM_GET_PEER_CLUSTER)) {
	return node;
	}

	if (id > 0) {
	id_str = crm_strdup_printf("%u", id);
	}

	- node = crm_find_known_peer(id_str, uname);
	+ node = find_known_node(id_str, uname);

	free(id_str);
	return node;
	}
	+
	+
	+// Deprecated functions kept only for backward API compatibility
	+
	+int crm_terminate_member(int nodeid, const char uname, void unused);
	+int crm_terminate_member_no_mainloop(int nodeid, const char *uname,
	+ int *connection);
	+/*!
	+ * \deprecated Use stonith_api_kick() from libstonithd instead
	+ */
	+int
	+crm_terminate_member(int nodeid, const char uname, void unused)
	+{
	+ /* Always use the synchronous, non-mainloop version */
	+ return stonith_api_kick(nodeid, uname, 120, TRUE);
	+}
	+
	+/*!
	+ * \deprecated Use stonith_api_kick() from libstonithd instead
	+ */
	+int
	+crm_terminate_member_no_mainloop(int nodeid, const char uname, int connection)
	+{
	+ return stonith_api_kick(nodeid, uname, 120, TRUE);
	+}

File Metadata

Mime Type: text/x-diff
Expires: Mon, Apr 21, 3:23 PM (1 d, 19 h)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 1664943
Default Alt Text: (899 KB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions