No OneTemporary
Actions

Size

463 KB

Referenced Files

None

Subscribers

None

View Options

This file is larger than 256 KB, so syntax highlighting was skipped.

	diff --git a/cts/lab/CIB.py b/cts/lab/CIB.py
	index 3011993b80..75b168a0b8 100644
	--- a/cts/lab/CIB.py
	+++ b/cts/lab/CIB.py
	@@ -1,518 +1,518 @@
	""" CIB generator for Pacemaker's Cluster Test Suite (CTS)
	"""

	__copyright__ = "Copyright 2008-2021 the Pacemaker project contributors"
	__license__ = "GNU General Public License version 2 or later (GPLv2+) WITHOUT ANY WARRANTY"

	import os
	import warnings
	import tempfile

	from cts.CTSvars import *


	class CibBase(object):
	def __init__(self, Factory, tag, _id, **kwargs):
	self.tag = tag
	self.name = _id
	self.kwargs = kwargs
	self.children = []
	self.Factory = Factory

	def __repr__(self):
	return "%s-%s" % (self.tag, self.name)

	def add_child(self, child):
	self.children.append(child)

	def __setitem__(self, key, value):
	if value:
	self.kwargs[key] = value
	else:
	self.kwargs.pop(key, None)

	from cts.cib_xml import *


	class ConfigBase(object):
	cts_cib = None
	version = "unknown"
	Factory = None

	def __init__(self, CM, factory, tmpfile=None):
	self.CM = CM
	self.Factory = factory

	if not tmpfile:
	warnings.filterwarnings("ignore")
	f=tempfile.NamedTemporaryFile(delete=True)
	f.close()
	tmpfile = f.name
	warnings.resetwarnings()

	self.Factory.tmpfile = tmpfile

	def version(self):
	return self.version

	def NextIP(self):
	ip = self.CM.Env["IPBase"]
	if ":" in ip:
	(prefix, sep, suffix) = ip.rpartition(":")
	suffix = str(hex(int(suffix, 16)+1)).lstrip("0x")
	else:
	(prefix, sep, suffix) = ip.rpartition(".")
	suffix = str(int(suffix)+1)

	ip = prefix + sep + suffix
	self.CM.Env["IPBase"] = ip
	return ip.strip()


	class CIB12(ConfigBase):
	version = "pacemaker-1.2"
	counter = 1

	def _show(self, command=""):
	output = ""
	(rc, result) = self.Factory.rsh(self.Factory.target, "HOME=/root CIB_file="+self.Factory.tmpfile+" cibadmin -Ql "+command, None, )
	for line in result:
	output += line
	self.Factory.debug("Generated Config: "+line)
	return output

	def NewIP(self, name=None, standard="ocf"):
	if self.CM.Env["IPagent"] == "IPaddr2":
	ip = self.NextIP()
	if not name:
	if ":" in ip:
	(prefix, sep, suffix) = ip.rpartition(":")
	name = "r"+suffix
	else:
	name = "r"+ip

	r = Resource(self.Factory, name, self.CM.Env["IPagent"], standard)
	r["ip"] = ip

	if ":" in ip:
	r["cidr_netmask"] = "64"
	r["nic"] = "eth0"
	else:
	r["cidr_netmask"] = "32"

	else:
	if not name:
	name = "r%s%d" % (self.CM.Env["IPagent"], self.counter)
	self.counter = self.counter + 1
	r = Resource(self.Factory, name, self.CM.Env["IPagent"], standard)

	r.add_op("monitor", "5s")
	return r

	def get_node_id(self, node_name):
	""" Check the cluster configuration for a node ID. """

	# We can't account for every possible configuration,
	# so we only return a node ID if:
	# * The node is specified in /etc/corosync/corosync.conf
	# with "ring0_addr:" equal to node_name and "nodeid:"
	# explicitly specified.
	# In all other cases, we return 0.
	node_id = 0

	# awkward command: use } as record separator
	# so each corosync.conf "object" is one record;
	# match the "node {" record that has "ring0_addr: node_name";
	# then print the substring of that record after "nodeid:"
	(rc, output) = self.Factory.rsh(self.Factory.target,
	r"""awk -v RS="}" """
	r"""'/^(\snodelist\s{)?\snode\s{.(ring0_addr\|name):\s%s(\s+\|$)/"""
	r"""{gsub(/.nodeid:\s/,"");gsub(/\s+.*$/,"");print}'%s"""
	% (node_name, CTSvars.COROSYNC_CONF), None)

	if rc == 0 and len(output) == 1:
	try:
	node_id = int(output[0])
	except ValueError:
	node_id = 0

	return node_id

	def install(self, target):
	old = self.Factory.tmpfile

	# Force a rebuild
	self.cts_cib = None

	self.Factory.tmpfile = CTSvars.CRM_CONFIG_DIR+"/cib.xml"
	self.contents(target)
	self.Factory.rsh(self.Factory.target, "chown "+CTSvars.CRM_DAEMON_USER+" "+self.Factory.tmpfile)

	self.Factory.tmpfile = old

	def contents(self, target=None):
	# fencing resource
	if self.cts_cib:
	return self.cts_cib

	if target:
	self.Factory.target = target

	self.Factory.rsh(self.Factory.target, "HOME=/root cibadmin --empty %s > %s" % (self.version, self.Factory.tmpfile))
	self.num_nodes = len(self.CM.Env["nodes"])

	no_quorum = "stop"
	if self.num_nodes < 3:
	no_quorum = "ignore"
	self.Factory.log("Cluster only has %d nodes, configuring: no-quorum-policy=ignore" % self.num_nodes)

	# We don't need a nodes section unless we add attributes
	stn = None

	# Fencing resource
	# Define first so that the shell doesn't reject every update
	if self.CM.Env["DoFencing"]:

	# Define the "real" fencing device
	st = Resource(self.Factory, "Fencing", ""+self.CM.Env["stonith-type"], "stonith")

	# Set a threshold for unreliable stonith devices such as the vmware one
	st.add_meta("migration-threshold", "5")
	st.add_op("monitor", "120s", timeout="120s")
	st.add_op("stop", "0", timeout="60s")
	st.add_op("start", "0", timeout="60s")

	# For remote node tests, a cluster node is stopped and brought back up
	# as a remote node with the name "remote-OLDNAME". To allow fencing
	# devices to fence these nodes, create a list of all possible node names.
	all_node_names = [ prefix+n for n in self.CM.Env["nodes"] for prefix in ('', 'remote-') ]

	# Add all parameters specified by user
	entries = self.CM.Env["stonith-params"].split(',')
	for entry in entries:
	try:
	(name, value) = entry.split('=', 1)
	except ValueError:
	print("Warning: skipping invalid fencing parameter: %s" % entry)
	continue

	# Allow user to specify "all" as the node list, and expand it here
	if name in [ "hostlist", "pcmk_host_list" ] and value == "all":
	value = ' '.join(all_node_names)

	st[name] = value

	st.commit()

	# Test advanced fencing logic
	if True:
	stf_nodes = []
	stt_nodes = []
	attr_nodes = {}

	# Create the levels
	stl = FencingTopology(self.Factory)
	for node in self.CM.Env["nodes"]:
	# Remote node tests will rename the node
	remote_node = "remote-" + node

	# Randomly assign node to a fencing method
	ftype = self.CM.Env.RandomGen.choice(["levels-and", "levels-or ", "broadcast "])

	# For levels-and, randomly choose targeting by node name or attribute
	by = ""
	if ftype == "levels-and":
	node_id = self.get_node_id(node)
	if node_id == 0 or self.CM.Env.RandomGen.choice([True, False]):
	by = " (by name)"
	else:
	attr_nodes[node] = node_id
	by = " (by attribute)"

	self.CM.log(" - Using %s fencing for node: %s%s" % (ftype, node, by))

	if ftype == "levels-and":
	# If targeting by name, add a topology level for this node
	if node not in attr_nodes:
	stl.level(1, node, "FencingPass,Fencing")

	# Always target remote nodes by name, otherwise we would need to add
	# an attribute to the remote node only during remote tests (we don't
	# want nonexistent remote nodes showing up in the non-remote tests).
	# That complexity is not worth the effort.
	stl.level(1, remote_node, "FencingPass,Fencing")

	# Add the node (and its remote equivalent) to the list of levels-and nodes.
	stt_nodes.extend([node, remote_node])

	elif ftype == "levels-or ":
	for n in [ node, remote_node ]:
	stl.level(1, n, "FencingFail")
	stl.level(2, n, "Fencing")
	stf_nodes.extend([node, remote_node])

	# If any levels-and nodes were targeted by attribute,
	# create the attributes and a level for the attribute.
	if attr_nodes:
	stn = Nodes(self.Factory)
	for (node_name, node_id) in list(attr_nodes.items()):
	stn.add_node(node_name, node_id, { "cts-fencing" : "levels-and" })
	stl.level(1, None, "FencingPass,Fencing", "cts-fencing", "levels-and")

	# Create a Dummy agent that always passes for levels-and
	if len(stt_nodes):
	stt = Resource(self.Factory, "FencingPass", "fence_dummy", "stonith")
	stt["pcmk_host_list"] = " ".join(stt_nodes)
	# Wait this many seconds before doing anything, handy for letting disks get flushed too
	stt["random_sleep_range"] = "30"
	stt["mode"] = "pass"
	stt.commit()

	# Create a Dummy agent that always fails for levels-or
	if len(stf_nodes):
	stf = Resource(self.Factory, "FencingFail", "fence_dummy", "stonith")
	stf["pcmk_host_list"] = " ".join(stf_nodes)
	# Wait this many seconds before doing anything, handy for letting disks get flushed too
	stf["random_sleep_range"] = "30"
	stf["mode"] = "fail"
	stf.commit()

	# Now commit the levels themselves
	stl.commit()

	o = Option(self.Factory)
	o["stonith-enabled"] = self.CM.Env["DoFencing"]
	o["start-failure-is-fatal"] = "false"
	o["pe-input-series-max"] = "5000"
	o["shutdown-escalation"] = "5min"
	o["batch-limit"] = "10"
	o["dc-deadtime"] = "5s"
	o["no-quorum-policy"] = no_quorum

	if self.CM.Env["DoBSC"] == 1:
	o["ident-string"] = "Linux-HA TEST configuration file - REMOVEME!!"

	o.commit()

	o = OpDefaults(self.Factory)
	o["timeout"] = "90s"
	o.commit()

	# Commit the nodes section if we defined one
	if stn is not None:
	stn.commit()

	# Add an alerts section if possible
	if self.Factory.rsh.exists_on_all(self.CM.Env["notification-agent"], self.CM.Env["nodes"]):
	alerts = Alerts(self.Factory)
	alerts.add_alert(self.CM.Env["notification-agent"],
	self.CM.Env["notification-recipient"])
	alerts.commit()

	# Add resources?
	if self.CM.Env["CIBResource"] == 1:
	self.add_resources()

	if self.CM.cluster_monitor == 1:
	mon = Resource(self.Factory, "cluster_mon", "ocf", "ClusterMon", "pacemaker")
	mon.add_op("start", "0", requires="nothing")
	mon.add_op("monitor", "5s", requires="nothing")
	mon["update"] = "10"
	mon["extra_options"] = "-r -n"
	mon["user"] = "abeekhof"
	mon["htmlfile"] = "/suse/abeekhof/Export/cluster.html"
	mon.commit()

	#self._create('''location prefer-dc cluster_mon rule -INFINITY: \#is_dc eq false''')

	# generate cib
	self.cts_cib = self._show()

	if self.Factory.tmpfile != CTSvars.CRM_CONFIG_DIR+"/cib.xml":
	self.Factory.rsh(self.Factory.target, "rm -f "+self.Factory.tmpfile)

	return self.cts_cib

	def add_resources(self):
	# Per-node resources
	for node in self.CM.Env["nodes"]:
	name = "rsc_"+node
	r = self.NewIP(name)
	r.prefer(node, "100")
	r.commit()

	# Migrator
	# Make this slightly sticky (since we have no other location constraints) to avoid relocation during Reattach
	m = Resource(self.Factory, "migrator","Dummy", "ocf", "pacemaker")
	m["passwd"] = "whatever"
	m.add_meta("resource-stickiness","1")
	m.add_meta("allow-migrate", "1")
	m.add_op("monitor", "P10S")
	m.commit()

	# Ping the test exerciser
	p = Resource(self.Factory, "ping-1","ping", "ocf", "pacemaker")
	p.add_op("monitor", "60s")
	p["host_list"] = self.CM.Env["cts-exerciser"]
	p["name"] = "connected"
	p["debug"] = "true"

	c = Clone(self.Factory, "Connectivity", p)
	c["globally-unique"] = "false"
	c.commit()

	# promotable clone resource
	s = Resource(self.Factory, "stateful-1", "Stateful", "ocf", "pacemaker")
	s.add_op("monitor", "15s", timeout="60s")
	s.add_op("monitor", "16s", timeout="60s", role="Promoted")
	ms = Clone(self.Factory, "promotable-1", s)
	ms["promotable"] = "true"
	ms["clone-max"] = self.num_nodes
	ms["clone-node-max"] = 1
	ms["promoted-max"] = 1
	ms["promoted-node-max"] = 1

	# Require connectivity to run the promotable clone
	r = Rule(self.Factory, "connected", "-INFINITY", op="or")
	r.add_child(Expression(self.Factory, "m1-connected-1", "connected", "lt", "1"))
	r.add_child(Expression(self.Factory, "m1-connected-2", "connected", "not_defined", None))
	ms.prefer("connected", rule=r)

	ms.commit()

	# Group Resource
	g = Group(self.Factory, "group-1")
	g.add_child(self.NewIP())

	if self.CM.Env["have_systemd"]:
	sysd = Resource(self.Factory, "petulant",
	"pacemaker-cts-dummyd@10", "service")
	sysd.add_op("monitor", "P10S")
	g.add_child(sysd)
	else:
	g.add_child(self.NewIP())

	g.add_child(self.NewIP())

	# Make group depend on the promotable clone
	g.after("promotable-1", first="promote", then="start")
	g.colocate("promotable-1", "INFINITY", withrole="Promoted")

	g.commit()

	# LSB resource
	lsb = Resource(self.Factory, "lsb-dummy", "LSBDummy", "lsb")
	lsb.add_op("monitor", "5s")

	# LSB with group
	lsb.after("group-1")
	lsb.colocate("group-1")

	lsb.commit()


	class CIB20(CIB12):
	version = "pacemaker-2.5"

	class CIB30(CIB12):
	version = "pacemaker-3.7"

	#class HASI(CIB10):
	# def add_resources(self):
	# # DLM resource
	# self._create('''primitive dlm ocf:pacemaker:controld op monitor interval=120s''')
	# self._create('''clone dlm-clone dlm meta globally-unique=false interleave=true''')

	# O2CB resource
	# self._create('''primitive o2cb ocf:ocfs2:o2cb op monitor interval=120s''')
	# self._create('''clone o2cb-clone o2cb meta globally-unique=false interleave=true''')
	# self._create('''colocation o2cb-with-dlm INFINITY: o2cb-clone dlm-clone''')
	# self._create('''order start-o2cb-after-dlm mandatory: dlm-clone o2cb-clone''')


	class ConfigFactory(object):
	def __init__(self, CM):
	self.CM = CM
	self.rsh = self.CM.rsh
	self.register("pacemaker12", CIB12, CM, self)
	self.register("pacemaker20", CIB20, CM, self)
	self.register("pacemaker30", CIB30, CM, self)
	# self.register("hae", HASI, CM, self)
	if self.CM.Env["ListTests"] == 0:
	self.target = self.CM.Env["nodes"][0]
	self.tmpfile = None

	def log(self, args):
	self.CM.log("cib: %s" % args)

	def debug(self, args):
	self.CM.debug("cib: %s" % args)

	def register(self, methodName, constructor, args, *kargs):
	"""register a constructor"""
	_args = [constructor]
	_args.extend(args)
	setattr(self, methodName, ConfigFactoryItem(_args, *kargs))

	def unregister(self, methodName):
	"""unregister a constructor"""
	delattr(self, methodName)

	def createConfig(self, name="pacemaker-1.0"):
	if name == "pacemaker-1.0":
	name = "pacemaker10";
	elif name == "pacemaker-1.2":
	name = "pacemaker12";
	elif name == "pacemaker-2.0":
	name = "pacemaker20";
	elif name.startswith("pacemaker-3."):
	name = "pacemaker30";
	elif name == "hasi":
	name = "hae";

	if hasattr(self, name):
	return getattr(self, name)()
	else:
	self.CM.log("Configuration variant '%s' is unknown. Defaulting to latest config" % name)

	return self.pacemaker30()


	class ConfigFactoryItem(object):
	def __init__(self, function, args, *kargs):
	self._function = function
	self._args = args
	self._kargs = kargs

	def __call__(self, args, *kargs):
	"""call function"""
	_args = list(self._args)
	_args.extend(args)
	_kargs = self._kargs.copy()
	_kargs.update(kargs)
	return self._function(_args,*_kargs)

	if __name__ == '__main__':
	""" Unit test (pass cluster node names as command line arguments) """

	import cts.CTS
	import cts.CM_corosync
	import sys

	if len(sys.argv) < 2:
	print("Usage: %s <node> ..." % sys.argv[0])
	sys.exit(1)

	args = [
	"--nodes", " ".join(sys.argv[1:]),
	"--clobber-cib",
	"--populate-resources",
	"--stack", "corosync",
	"--test-ip-base", "fe80::1234:56:7890:1000",
	"--stonith", "rhcs",
	]
	env = CTS.CtsLab(args)
	- cm = CM_corosync.crm_corosync(env)
	+ cm = CM_corosync.crm_corosync()
	CibFactory = ConfigFactory(cm)
	cib = CibFactory.createConfig("pacemaker-3.0")
	print(cib.contents())
	diff --git a/cts/lab/CM_corosync.py b/cts/lab/CM_corosync.py
	index 0e55ad0c8d..38194452ee 100644
	--- a/cts/lab/CM_corosync.py
	+++ b/cts/lab/CM_corosync.py
	@@ -1,60 +1,60 @@
	""" Corosync-specific class for Pacemaker's Cluster Test Suite (CTS)
	"""

	__copyright__ = "Copyright 2007-2021 the Pacemaker project contributors"
	__license__ = "GNU General Public License version 2 or later (GPLv2+) WITHOUT ANY WARRANTY"

	from cts.CTSvars import *
	from cts.ClusterManager import ClusterManager
	from cts.CTS import Process
	from cts.patterns import PatternSelector

	class crm_corosync(ClusterManager):
	'''
	Corosync version 2 cluster manager class
	'''
	- def __init__(self, Environment, randseed=None, name=None):
	+ def __init__(self, name=None):
	if not name: name="crm-corosync"
	- ClusterManager.__init__(self, Environment, randseed=randseed)
	+ ClusterManager.__init__(self)

	self.fullcomplist = {}
	self.templates = PatternSelector(self.name)

	def Components(self):
	complist = []
	if not len(list(self.fullcomplist.keys())):
	for c in [ "pacemaker-based", "pacemaker-controld", "pacemaker-attrd", "pacemaker-execd", "pacemaker-fenced" ]:
	self.fullcomplist[c] = Process(
	self, c,
	pats = self.templates.get_component(self.name, c),
	badnews_ignore = self.templates.get_component(self.name, "%s-ignore" % c),
	common_ignore = self.templates.get_component(self.name, "common-ignore"))

	# the scheduler uses dc_pats instead of pats
	self.fullcomplist["pacemaker-schedulerd"] = Process(
	self, "pacemaker-schedulerd",
	dc_pats = self.templates.get_component(self.name, "pacemaker-schedulerd"),
	badnews_ignore = self.templates.get_component(self.name, "pacemaker-schedulerd-ignore"),
	common_ignore = self.templates.get_component(self.name, "common-ignore"))

	# add (or replace) extra components
	self.fullcomplist["corosync"] = Process(
	self, "corosync",
	pats = self.templates.get_component(self.name, "corosync"),
	badnews_ignore = self.templates.get_component(self.name, "corosync-ignore"),
	common_ignore = self.templates.get_component(self.name, "common-ignore")
	)

	# Processes running under valgrind can't be shot with "killall -9 processname",
	# so don't include them in the returned list
	vgrind = self.Env["valgrind-procs"].split()
	for key in list(self.fullcomplist.keys()):
	if self.Env["valgrind-tests"]:
	if key in vgrind:
	self.log("Filtering %s from the component list as it is being profiled by valgrind" % key)
	continue
	if key == "pacemaker-fenced" and not self.Env["DoFencing"]:
	continue
	complist.append(self.fullcomplist[key])

	return complist
	diff --git a/cts/lab/CTS.py b/cts/lab/CTS.py
	index 34f5a686e3..9df3184090 100644
	--- a/cts/lab/CTS.py
	+++ b/cts/lab/CTS.py
	@@ -1,185 +1,185 @@
	""" Main classes for Pacemaker's Cluster Test Suite (CTS)
	"""

	__copyright__ = "Copyright 2000-2021 the Pacemaker project contributors"
	__license__ = "GNU General Public License version 2 or later (GPLv2+) WITHOUT ANY WARRANTY"

	import sys
	import time
	import traceback

	from cts.logging import LogFactory
	-from cts.remote import RemoteFactory, input_wrapper
	+from cts.remote import RemoteFactory
	from cts.environment import EnvFactory

	class CtsLab(object):
	'''This class defines the Lab Environment for the Cluster Test System.
	It defines those things which are expected to change from test
	environment to test environment for the same cluster manager.

	It is where you define the set of nodes that are in your test lab
	what kind of reset mechanism you use, etc.

	At this point in time, it is the intent of this class to model static
	configuration and/or environmental data about the environment which
	doesn't change as the tests proceed.

	Well-known names (keys) are an important concept in this class.
	The HasMinimalKeys member function knows the minimal set of
	well-known names for the class.

	The following names are standard (well-known) at this time:

	nodes An array of the nodes in the cluster
	reset A ResetMechanism object
	logger An array of objects that log strings...
	CMclass The type of ClusterManager we are running
	(This is a class object, not a class instance)
	RandSeed Random seed. It is a triple of bytes. (optional)

	The CTS code ignores names it doesn't know about/need.
	The individual tests have access to this information, and it is
	perfectly acceptable to provide hints, tweaks, fine-tuning
	directions or other information to the tests through this mechanism.
	'''

	def __init__(self, args=None):
	self.Env = EnvFactory().getInstance(args)
	self.Scenario = None
	self.logger = LogFactory()
	self.rsh = RemoteFactory().getInstance()

	def dump(self):
	self.Env.dump()

	def has_key(self, key):
	return key in list(self.Env.keys())

	def __getitem__(self, key):
	return self.Env[key]

	def __setitem__(self, key, value):
	self.Env[key] = value

	def run(self, Scenario, Iterations):
	if not Scenario:
	self.logger.log("No scenario was defined")
	return 1

	self.logger.log("Cluster nodes: ")
	for node in self.Env["nodes"]:
	self.logger.log(" * %s" % (node))

	if not Scenario.SetUp():
	return 1

	try :
	Scenario.run(Iterations)
	except :
	self.logger.log("Exception by %s" % sys.exc_info()[0])
	self.logger.traceback(traceback)

	Scenario.summarize()
	Scenario.TearDown()
	return 1

	#ClusterManager.oprofileSave(Iterations)
	Scenario.TearDown()

	Scenario.summarize()
	if Scenario.Stats["failure"] > 0:
	return Scenario.Stats["failure"]

	elif Scenario.Stats["success"] != Iterations:
	self.logger.log("No failure count but success != requested iterations")
	return 1

	return 0

	def __CheckNode(self, node):
	"Raise a ValueError if the given node isn't valid"

	if not self.IsValidNode(node):
	raise ValueError("Invalid node [%s] in CheckNode" % node)

	class NodeStatus(object):
	def __init__(self, env):
	self.Env = env

	def IsNodeBooted(self, node):
	'''Return TRUE if the given node is booted (responds to pings)'''
	if self.Env["docker"]:
	return RemoteFactory().getInstance()("localhost", "docker inspect --format {{.State.Running}} %s \| grep -q true" % node, silent=True) == 0

	return RemoteFactory().getInstance()("localhost", "ping -nq -c1 -w1 %s" % node, silent=True) == 0

	def IsSshdUp(self, node):
	rc = RemoteFactory().getInstance()(node, "true", silent=True)
	return rc == 0

	def WaitForNodeToComeUp(self, node, Timeout=300):
	'''Return TRUE when given node comes up, or None/FALSE if timeout'''
	timeout = Timeout
	anytimeouts = 0
	while timeout > 0:
	if self.IsNodeBooted(node) and self.IsSshdUp(node):
	if anytimeouts:
	# Fudge to wait for the system to finish coming up
	time.sleep(30)
	LogFactory().debug("Node %s now up" % node)
	return 1

	time.sleep(30)
	if (not anytimeouts):
	LogFactory().debug("Waiting for node %s to come up" % node)

	anytimeouts = 1
	timeout = timeout - 1

	LogFactory().log("%s did not come up within %d tries" % (node, Timeout))
	if self.Env["continue"] == 1:
	answer = "Y"
	else:
	try:
	- answer = input_wrapper('Continue? [nY]')
	+ answer = input('Continue? [nY]')
	except EOFError as e:
	answer = "n"
	if answer and answer == "n":
	raise ValueError("%s did not come up within %d tries" % (node, Timeout))

	def WaitForAllNodesToComeUp(self, nodes, timeout=300):
	'''Return TRUE when all nodes come up, or FALSE if timeout'''

	for node in nodes:
	if not self.WaitForNodeToComeUp(node, timeout):
	return None
	return 1


	class Component(object):
	def kill(self, node):
	None


	class Process(Component):
	def __init__(self, cm, name, process=None, dc_only=0, pats=[], dc_pats=[], badnews_ignore=[], common_ignore=[], triggersreboot=0):
	self.name = str(name)
	self.dc_only = dc_only
	self.pats = pats
	self.dc_pats = dc_pats
	self.CM = cm
	self.badnews_ignore = badnews_ignore
	self.badnews_ignore.extend(common_ignore)
	self.triggersreboot = triggersreboot

	if process:
	self.proc = str(process)
	else:
	self.proc = str(name)
	self.KillCmd = "killall -9 " + self.proc

	def kill(self, node):
	if self.CM.rsh(node, self.KillCmd) != 0:
	self.CM.log ("ERROR: Kill %s failed on node %s" % (self.name,node))
	return None
	return 1
	diff --git a/cts/lab/CTSaudits.py b/cts/lab/CTSaudits.py
	index 9a8d64cfe9..3b7f6f0dfd 100755
	--- a/cts/lab/CTSaudits.py
	+++ b/cts/lab/CTSaudits.py
	@@ -1,862 +1,861 @@
	""" Auditing classes for Pacemaker's Cluster Test Suite (CTS)
	"""

	-__copyright__ = "Copyright 2000-2020 the Pacemaker project contributors"
	+__copyright__ = "Copyright 2000-2021 the Pacemaker project contributors"
	__license__ = "GNU General Public License version 2 or later (GPLv2+) WITHOUT ANY WARRANTY"

	import time, re, uuid
	from cts.watcher import LogWatcher
	-from cts.remote import input_wrapper
	from cts.CTSvars import *

	class ClusterAudit(object):

	def __init__(self, cm):
	self.CM = cm

	def __call__(self):
	raise ValueError("Abstract Class member (__call__)")

	def is_applicable(self):
	'''Return TRUE if we are applicable in the current test configuration'''
	raise ValueError("Abstract Class member (is_applicable)")
	return 1

	def log(self, args):
	self.CM.log("audit: %s" % args)

	def debug(self, args):
	self.CM.debug("audit: %s" % args)

	def name(self):
	raise ValueError("Abstract Class member (name)")

	AllAuditClasses = [ ]


	class LogAudit(ClusterAudit):

	def name(self):
	return "LogAudit"

	def __init__(self, cm):
	self.CM = cm
	self.kinds = [ "combined syslog", "journal", "remote" ]

	def RestartClusterLogging(self, nodes=None):
	if not nodes:
	nodes = self.CM.Env["nodes"]

	self.CM.debug("Restarting logging on: %s" % repr(nodes))

	for node in nodes:
	if self.CM.Env["have_systemd"]:
	if self.CM.rsh(node, "systemctl stop systemd-journald.socket") != 0:
	self.CM.log ("ERROR: Cannot stop 'systemd-journald' on %s" % node)
	if self.CM.rsh(node, "systemctl start systemd-journald.service") != 0:
	self.CM.log ("ERROR: Cannot start 'systemd-journald' on %s" % node)

	if self.CM.rsh(node, "service %s restart" % self.CM.Env["syslogd"]) != 0:
	self.CM.log ("ERROR: Cannot restart '%s' on %s" % (self.CM.Env["syslogd"], node))

	def TestLogging(self):
	patterns = []
	prefix = "Test message from"
	suffix = str(uuid.uuid4())
	watch = {}

	for node in self.CM.Env["nodes"]:
	# Look for the node name in two places to make sure
	# that syslog is logging with the correct hostname
	m = re.search("^([^.]+).*", node)
	if m:
	simple = m.group(1)
	else:
	simple = node
	patterns.append("%s.*%s %s %s" % (simple, prefix, node, suffix))

	watch_pref = self.CM.Env["LogWatcher"]
	if watch_pref == "any":
	for k in self.kinds:
	watch[k] = LogWatcher(self.CM.Env["LogFileName"], patterns, "LogAudit", 5, silent=True, hosts=self.CM.Env["nodes"], kind=k)
	watch[k].setwatch()
	else:
	k = watch_pref
	watch[k] = LogWatcher(self.CM.Env["LogFileName"], patterns, "LogAudit", 5, silent=True, hosts=self.CM.Env["nodes"], kind=k)
	watch[k].setwatch()

	if watch_pref == "any": self.CM.log("Writing log with key: %s" % (suffix))
	for node in self.CM.Env["nodes"]:
	cmd = "logger -p %s.info %s %s %s" % (self.CM.Env["SyslogFacility"], prefix, node, suffix)
	if self.CM.rsh(node, cmd, synchronous=0, silent=True) != 0:
	self.CM.log ("ERROR: Cannot execute remote command [%s] on %s" % (cmd, node))

	for k in self.kinds:
	if k in watch:
	w = watch[k]
	if watch_pref == "any": self.CM.log("Testing for %s logs" % (k))
	w.lookforall(silent=True)
	if not w.unmatched:
	if watch_pref == "any":
	self.CM.log ("Continuing with %s-based log reader" % (w.kind))
	self.CM.Env["LogWatcher"] = w.kind
	return 1

	for k in list(watch.keys()):
	w = watch[k]
	if w.unmatched:
	for regex in w.unmatched:
	self.CM.log ("Test message [%s] not found in %s logs." % (regex, w.kind))

	return 0

	def __call__(self):
	max = 3
	attempt = 0

	self.CM.ns.WaitForAllNodesToComeUp(self.CM.Env["nodes"])
	while attempt <= max and self.TestLogging() == 0:
	attempt = attempt + 1
	self.RestartClusterLogging()
	time.sleep(60*attempt)

	if attempt > max:
	self.CM.log("ERROR: Cluster logging unrecoverable.")
	return 0

	return 1

	def is_applicable(self):
	if self.CM.Env["DoBSC"]:
	return 0
	if self.CM.Env["LogAuditDisabled"]:
	return 0
	return 1


	class DiskAudit(ClusterAudit):

	def name(self):
	return "DiskspaceAudit"

	def __init__(self, cm):
	self.CM = cm

	def __call__(self):
	result = 1
	# @TODO Use directory of PCMK_logfile if set on host
	dfcmd = "df -BM " + CTSvars.CRM_LOG_DIR + " \| tail -1 \| awk '{print $(NF-1)\" \"$(NF-2)}' \| tr -d 'M%'"

	self.CM.ns.WaitForAllNodesToComeUp(self.CM.Env["nodes"])
	for node in self.CM.Env["nodes"]:
	dfout = self.CM.rsh(node, dfcmd, 1)
	if not dfout:
	self.CM.log ("ERROR: Cannot execute remote df command [%s] on %s" % (dfcmd, node))
	else:
	try:
	(used, remain) = dfout.split()
	used_percent = int(used)
	remaining_mb = int(remain)
	except (ValueError, TypeError):
	self.CM.log("Warning: df output '%s' from %s was invalid [%s, %s]"
	% (dfout, node, used, remain))
	else:
	if remaining_mb < 10 or used_percent > 95:
	self.CM.log("CRIT: Out of log disk space on %s (%d%% / %dMB)"
	% (node, used_percent, remaining_mb))
	result = None
	if self.CM.Env["continue"] == 1:
	answer = "Y"
	else:
	try:
	- answer = input_wrapper('Continue? [nY]')
	+ answer = input('Continue? [nY]')
	except EOFError as e:
	answer = "n"

	if answer and answer == "n":
	raise ValueError("Disk full on %s" % (node))

	elif remaining_mb < 100 or used_percent > 90:
	self.CM.log("WARN: Low on log disk space (%dMB) on %s" % (remaining_mb, node))
	return result

	def is_applicable(self):
	if self.CM.Env["DoBSC"]:
	return 0
	return 1


	class FileAudit(ClusterAudit):

	def name(self):
	return "FileAudit"

	def __init__(self, cm):
	self.CM = cm
	self.known = []

	def __call__(self):
	result = 1

	self.CM.ns.WaitForAllNodesToComeUp(self.CM.Env["nodes"])
	for node in self.CM.Env["nodes"]:

	(rc, lsout) = self.CM.rsh(node, "ls -al /var/lib/pacemaker/cores/* \| grep core.[0-9]", None)
	for line in lsout:
	line = line.strip()
	if line not in self.known:
	result = 0
	self.known.append(line)
	self.CM.log("Warning: Pacemaker core file on %s: %s" % (node, line))

	(rc, lsout) = self.CM.rsh(node, "ls -al /var/lib/corosync \| grep core.[0-9]", None)
	for line in lsout:
	line = line.strip()
	if line not in self.known:
	result = 0
	self.known.append(line)
	self.CM.log("Warning: Corosync core file on %s: %s" % (node, line))

	if node in self.CM.ShouldBeStatus and self.CM.ShouldBeStatus[node] == "down":
	clean = 0
	(rc, lsout) = self.CM.rsh(node, "ls -al /dev/shm \| grep qb-", None)
	for line in lsout:
	result = 0
	clean = 1
	self.CM.log("Warning: Stale IPC file on %s: %s" % (node, line))

	if clean:
	(rc, lsout) = self.CM.rsh(node, "ps axf \| grep -e pacemaker -e corosync", None)
	for line in lsout:
	self.CM.debug("ps[%s]: %s" % (node, line))

	self.CM.rsh(node, "rm -rf /dev/shm/qb-*")

	else:
	self.CM.debug("Skipping %s" % node)

	return result

	def is_applicable(self):
	return 1


	class AuditResource(object):
	def __init__(self, cm, line):
	fields = line.split()
	self.CM = cm
	self.line = line
	self.type = fields[1]
	self.id = fields[2]
	self.clone_id = fields[3]
	self.parent = fields[4]
	self.rprovider = fields[5]
	self.rclass = fields[6]
	self.rtype = fields[7]
	self.host = fields[8]
	self.needs_quorum = fields[9]
	self.flags = int(fields[10])
	self.flags_s = fields[11]

	if self.parent == "NA":
	self.parent = None

	def unique(self):
	if self.flags & int("0x00000020", 16):
	return 1
	return 0

	def orphan(self):
	if self.flags & int("0x00000001", 16):
	return 1
	return 0

	def managed(self):
	if self.flags & int("0x00000002", 16):
	return 1
	return 0


	class AuditConstraint(object):
	def __init__(self, cm, line):
	fields = line.split()
	self.CM = cm
	self.line = line
	self.type = fields[1]
	self.id = fields[2]
	self.rsc = fields[3]
	self.target = fields[4]
	self.score = fields[5]
	self.rsc_role = fields[6]
	self.target_role = fields[7]

	if self.rsc_role == "NA":
	self.rsc_role = None
	if self.target_role == "NA":
	self.target_role = None


	class PrimitiveAudit(ClusterAudit):
	def name(self):
	return "PrimitiveAudit"

	def __init__(self, cm):
	self.CM = cm

	def doResourceAudit(self, resource, quorum):
	rc = 1
	active = self.CM.ResourceLocation(resource.id)

	if len(active) == 1:
	if quorum:
	self.debug("Resource %s active on %s" % (resource.id, repr(active)))

	elif resource.needs_quorum == 1:
	self.CM.log("Resource %s active without quorum: %s"
	% (resource.id, repr(active)))
	rc = 0

	elif not resource.managed():
	self.CM.log("Resource %s not managed. Active on %s"
	% (resource.id, repr(active)))

	elif not resource.unique():
	# TODO: Figure out a clever way to actually audit these resource types
	if len(active) > 1:
	self.debug("Non-unique resource %s is active on: %s"
	% (resource.id, repr(active)))
	else:
	self.debug("Non-unique resource %s is not active" % resource.id)

	elif len(active) > 1:
	self.CM.log("Resource %s is active multiple times: %s"
	% (resource.id, repr(active)))
	rc = 0

	elif resource.orphan():
	self.debug("Resource %s is an inactive orphan" % resource.id)

	elif len(self.inactive_nodes) == 0:
	self.CM.log("WARN: Resource %s not served anywhere" % resource.id)
	rc = 0

	elif self.CM.Env["warn-inactive"] == 1:
	if quorum or not resource.needs_quorum:
	self.CM.log("WARN: Resource %s not served anywhere (Inactive nodes: %s)"
	% (resource.id, repr(self.inactive_nodes)))
	else:
	self.debug("Resource %s not served anywhere (Inactive nodes: %s)"
	% (resource.id, repr(self.inactive_nodes)))

	elif quorum or not resource.needs_quorum:
	self.debug("Resource %s not served anywhere (Inactive nodes: %s)"
	% (resource.id, repr(self.inactive_nodes)))

	return rc

	def setup(self):
	self.target = None
	self.resources = []
	self.constraints = []
	self.active_nodes = []
	self.inactive_nodes = []

	for node in self.CM.Env["nodes"]:
	if self.CM.ShouldBeStatus[node] == "up":
	self.active_nodes.append(node)
	else:
	self.inactive_nodes.append(node)

	for node in self.CM.Env["nodes"]:
	if self.target == None and self.CM.ShouldBeStatus[node] == "up":
	self.target = node

	if not self.target:
	# TODO: In Pacemaker 1.0 clusters we'll be able to run crm_resource
	# with CIB_file=/path/to/cib.xml even when the cluster isn't running
	self.debug("No nodes active - skipping %s" % self.name())
	return 0

	(rc, lines) = self.CM.rsh(self.target, "crm_resource -c", None)

	for line in lines:
	if re.search("^Resource", line):
	self.resources.append(AuditResource(self.CM, line))
	elif re.search("^Constraint", line):
	self.constraints.append(AuditConstraint(self.CM, line))
	else:
	self.CM.log("Unknown entry: %s" % line);

	return 1

	def __call__(self):
	rc = 1

	if not self.setup():
	return 1

	quorum = self.CM.HasQuorum(None)
	for resource in self.resources:
	if resource.type == "primitive":
	if self.doResourceAudit(resource, quorum) == 0:
	rc = 0
	return rc

	def is_applicable(self):
	# @TODO Due to long-ago refactoring, this name test would never match,
	# so this audit (and those derived from it) would never run.
	# Uncommenting the next lines fixes the name test, but that then
	# exposes pre-existing bugs that need to be fixed.
	#if self.CM["Name"] == "crm-corosync":
	# return 1
	return 0


	class GroupAudit(PrimitiveAudit):
	def name(self):
	return "GroupAudit"

	def __call__(self):
	rc = 1
	if not self.setup():
	return 1

	for group in self.resources:
	if group.type == "group":
	first_match = 1
	group_location = None
	for child in self.resources:
	if child.parent == group.id:
	nodes = self.CM.ResourceLocation(child.id)

	if first_match and len(nodes) > 0:
	group_location = nodes[0]

	first_match = 0

	if len(nodes) > 1:
	rc = 0
	self.CM.log("Child %s of %s is active more than once: %s"
	% (child.id, group.id, repr(nodes)))

	elif len(nodes) == 0:
	# Groups are allowed to be partially active
	# However we do need to make sure later children aren't running
	group_location = None
	self.debug("Child %s of %s is stopped" % (child.id, group.id))

	elif nodes[0] != group_location:
	rc = 0
	self.CM.log("Child %s of %s is active on the wrong node (%s) expected %s"
	% (child.id, group.id, nodes[0], group_location))
	else:
	self.debug("Child %s of %s is active on %s" % (child.id, group.id, nodes[0]))

	return rc


	class CloneAudit(PrimitiveAudit):
	def name(self):
	return "CloneAudit"

	def __call__(self):
	rc = 1
	if not self.setup():
	return 1

	for clone in self.resources:
	if clone.type == "clone":
	for child in self.resources:
	if child.parent == clone.id and child.type == "primitive":
	self.debug("Checking child %s of %s..." % (child.id, clone.id))
	# Check max and node_max
	# Obtain with:
	# crm_resource -g clone_max --meta -r child.id
	# crm_resource -g clone_node_max --meta -r child.id

	return rc


	class ColocationAudit(PrimitiveAudit):
	def name(self):
	return "ColocationAudit"

	def crm_location(self, resource):
	(rc, lines) = self.CM.rsh(self.target, "crm_resource -W -r %s -Q"%resource, None)
	hosts = []
	if rc == 0:
	for line in lines:
	fields = line.split()
	hosts.append(fields[0])

	return hosts

	def __call__(self):
	rc = 1
	if not self.setup():
	return 1

	for coloc in self.constraints:
	if coloc.type == "rsc_colocation":
	source = self.crm_location(coloc.rsc)
	target = self.crm_location(coloc.target)
	if len(source) == 0:
	self.debug("Colocation audit (%s): %s not running" % (coloc.id, coloc.rsc))
	else:
	for node in source:
	if not node in target:
	rc = 0
	self.CM.log("Colocation audit (%s): %s running on %s (not in %s)"
	% (coloc.id, coloc.rsc, node, repr(target)))
	else:
	self.debug("Colocation audit (%s): %s running on %s (in %s)"
	% (coloc.id, coloc.rsc, node, repr(target)))

	return rc


	class ControllerStateAudit(ClusterAudit):
	def __init__(self, cm):
	self.CM = cm
	self.Stats = {"calls":0
	, "success":0
	, "failure":0
	, "skipped":0
	, "auditfail":0}

	def has_key(self, key):
	return key in self.Stats

	def __setitem__(self, key, value):
	self.Stats[key] = value

	def __getitem__(self, key):
	return self.Stats[key]

	def incr(self, name):
	'''Increment (or initialize) the value associated with the given name'''
	if not name in self.Stats:
	self.Stats[name] = 0
	self.Stats[name] = self.Stats[name]+1

	def __call__(self):
	passed = 1
	up_are_down = 0
	down_are_up = 0
	unstable_list = []

	for node in self.CM.Env["nodes"]:
	should_be = self.CM.ShouldBeStatus[node]
	rc = self.CM.test_node_CM(node)
	if rc > 0:
	if should_be == "down":
	down_are_up = down_are_up + 1
	if rc == 1:
	unstable_list.append(node)
	elif should_be == "up":
	up_are_down = up_are_down + 1

	if len(unstable_list) > 0:
	passed = 0
	self.CM.log("Cluster is not stable: %d (of %d): %s"
	% (len(unstable_list), self.CM.upcount(), repr(unstable_list)))

	if up_are_down > 0:
	passed = 0
	self.CM.log("%d (of %d) nodes expected to be up were down."
	% (up_are_down, len(self.CM.Env["nodes"])))

	if down_are_up > 0:
	passed = 0
	self.CM.log("%d (of %d) nodes expected to be down were up."
	% (down_are_up, len(self.CM.Env["nodes"])))

	return passed

	def name(self):
	return "ControllerStateAudit"

	def is_applicable(self):
	# @TODO Due to long-ago refactoring, this name test would never match,
	# so this audit (and those derived from it) would never run.
	# Uncommenting the next lines fixes the name test, but that then
	# exposes pre-existing bugs that need to be fixed.
	#if self.CM["Name"] == "crm-corosync":
	# return 1
	return 0


	class CIBAudit(ClusterAudit):
	def __init__(self, cm):
	self.CM = cm
	self.Stats = {"calls":0
	, "success":0
	, "failure":0
	, "skipped":0
	, "auditfail":0}

	def has_key(self, key):
	return key in self.Stats

	def __setitem__(self, key, value):
	self.Stats[key] = value

	def __getitem__(self, key):
	return self.Stats[key]

	def incr(self, name):
	'''Increment (or initialize) the value associated with the given name'''
	if not name in self.Stats:
	self.Stats[name] = 0
	self.Stats[name] = self.Stats[name]+1

	def __call__(self):
	passed = 1
	ccm_partitions = self.CM.find_partitions()

	if len(ccm_partitions) == 0:
	self.debug("\tNo partitions to audit")
	return 1

	for partition in ccm_partitions:
	self.debug("\tAuditing CIB consistency for: %s" % partition)
	partition_passed = 0
	if self.audit_cib_contents(partition) == 0:
	passed = 0

	return passed

	def audit_cib_contents(self, hostlist):
	passed = 1
	node0 = None
	node0_xml = None

	partition_hosts = hostlist.split()
	for node in partition_hosts:
	node_xml = self.store_remote_cib(node, node0)

	if node_xml == None:
	self.CM.log("Could not perform audit: No configuration from %s" % node)
	passed = 0

	elif node0 == None:
	node0 = node
	node0_xml = node_xml

	elif node0_xml == None:
	self.CM.log("Could not perform audit: No configuration from %s" % node0)
	passed = 0

	else:
	(rc, result) = self.CM.rsh(
	node0, "crm_diff -VV -cf --new %s --original %s" % (node_xml, node0_xml), None)

	if rc != 0:
	self.CM.log("Diff between %s and %s failed: %d" % (node0_xml, node_xml, rc))
	passed = 0

	for line in result:
	if not re.search("<diff/>", line):
	passed = 0
	self.debug("CibDiff[%s-%s]: %s" % (node0, node, line))
	else:
	self.debug("CibDiff[%s-%s] Ignoring: %s" % (node0, node, line))

	# self.CM.rsh(node0, "rm -f %s" % node_xml)
	# self.CM.rsh(node0, "rm -f %s" % node0_xml)
	return passed

	def store_remote_cib(self, node, target):
	combined = ""
	filename = "/tmp/ctsaudit.%s.xml" % node

	if not target:
	target = node

	(rc, lines) = self.CM.rsh(node, self.CM["CibQuery"], None)
	if rc != 0:
	self.CM.log("Could not retrieve configuration")
	return None

	self.CM.rsh("localhost", "rm -f %s" % filename)
	for line in lines:
	self.CM.rsh("localhost", "echo \'%s\' >> %s" % (line[:-1], filename), silent=True)

	if self.CM.rsh.cp(filename, "root@%s:%s" % (target, filename), silent=True) != 0:
	self.CM.log("Could not store configuration")
	return None
	return filename

	def name(self):
	return "CibAudit"

	def is_applicable(self):
	# @TODO Due to long-ago refactoring, this name test would never match,
	# so this audit (and those derived from it) would never run.
	# Uncommenting the next lines fixes the name test, but that then
	# exposes pre-existing bugs that need to be fixed.
	#if self.CM["Name"] == "crm-corosync":
	# return 1
	return 0


	class PartitionAudit(ClusterAudit):
	def __init__(self, cm):
	self.CM = cm
	self.Stats = {"calls":0
	, "success":0
	, "failure":0
	, "skipped":0
	, "auditfail":0}
	self.NodeEpoch = {}
	self.NodeState = {}
	self.NodeQuorum = {}

	def has_key(self, key):
	return key in self.Stats

	def __setitem__(self, key, value):
	self.Stats[key] = value

	def __getitem__(self, key):
	return self.Stats[key]

	def incr(self, name):
	'''Increment (or initialize) the value associated with the given name'''
	if not name in self.Stats:
	self.Stats[name] = 0
	self.Stats[name] = self.Stats[name]+1

	def __call__(self):
	passed = 1
	ccm_partitions = self.CM.find_partitions()

	if ccm_partitions == None or len(ccm_partitions) == 0:
	return 1

	self.CM.cluster_stable(double_check=True)

	if len(ccm_partitions) != self.CM.partitions_expected:
	self.CM.log("ERROR: %d cluster partitions detected:" % len(ccm_partitions))
	passed = 0
	for partition in ccm_partitions:
	self.CM.log("\t %s" % partition)

	for partition in ccm_partitions:
	partition_passed = 0
	if self.audit_partition(partition) == 0:
	passed = 0

	return passed

	def trim_string(self, avalue):
	if not avalue:
	return None
	if len(avalue) > 1:
	return avalue[:-1]

	def trim2int(self, avalue):
	if not avalue:
	return None
	if len(avalue) > 1:
	return int(avalue[:-1])

	def audit_partition(self, partition):
	passed = 1
	dc_found = []
	dc_allowed_list = []
	lowest_epoch = None
	node_list = partition.split()

	self.debug("Auditing partition: %s" % (partition))
	for node in node_list:
	if self.CM.ShouldBeStatus[node] != "up":
	self.CM.log("Warn: Node %s appeared out of nowhere" % (node))
	self.CM.ShouldBeStatus[node] = "up"
	# not in itself a reason to fail the audit (not what we're
	# checking for in this audit)

	self.NodeState[node] = self.CM.rsh(node, self.CM["StatusCmd"] % node, 1)
	self.NodeEpoch[node] = self.CM.rsh(node, self.CM["EpochCmd"], 1)
	self.NodeQuorum[node] = self.CM.rsh(node, self.CM["QuorumCmd"], 1)

	self.debug("Node %s: %s - %s - %s." % (node, self.NodeState[node], self.NodeEpoch[node], self.NodeQuorum[node]))
	self.NodeState[node] = self.trim_string(self.NodeState[node])
	self.NodeEpoch[node] = self.trim2int(self.NodeEpoch[node])
	self.NodeQuorum[node] = self.trim_string(self.NodeQuorum[node])

	if not self.NodeEpoch[node]:
	self.CM.log("Warn: Node %s dissappeared: cant determin epoch" % (node))
	self.CM.ShouldBeStatus[node] = "down"
	# not in itself a reason to fail the audit (not what we're
	# checking for in this audit)
	elif lowest_epoch == None or self.NodeEpoch[node] < lowest_epoch:
	lowest_epoch = self.NodeEpoch[node]

	if not lowest_epoch:
	self.CM.log("Lowest epoch not determined in %s" % (partition))
	passed = 0

	for node in node_list:
	if self.CM.ShouldBeStatus[node] == "up":
	if self.CM.is_node_dc(node, self.NodeState[node]):
	dc_found.append(node)
	if self.NodeEpoch[node] == lowest_epoch:
	self.debug("%s: OK" % node)
	elif not self.NodeEpoch[node]:
	self.debug("Check on %s ignored: no node epoch" % node)
	elif not lowest_epoch:
	self.debug("Check on %s ignored: no lowest epoch" % node)
	else:
	self.CM.log("DC %s is not the oldest node (%d vs. %d)"
	% (node, self.NodeEpoch[node], lowest_epoch))
	passed = 0

	if len(dc_found) == 0:
	self.CM.log("DC not found on any of the %d allowed nodes: %s (of %s)"
	% (len(dc_allowed_list), str(dc_allowed_list), str(node_list)))

	elif len(dc_found) > 1:
	self.CM.log("%d DCs (%s) found in cluster partition: %s"
	% (len(dc_found), str(dc_found), str(node_list)))
	passed = 0

	if passed == 0:
	for node in node_list:
	if self.CM.ShouldBeStatus[node] == "up":
	self.CM.log("epoch %s : %s"
	% (self.NodeEpoch[node], self.NodeState[node]))

	return passed

	def name(self):
	return "PartitionAudit"

	def is_applicable(self):
	# @TODO Due to long-ago refactoring, this name test would never match,
	# so this audit (and those derived from it) would never run.
	# Uncommenting the next lines fixes the name test, but that then
	# exposes pre-existing bugs that need to be fixed.
	#if self.CM["Name"] == "crm-corosync":
	# return 1
	return 0

	AllAuditClasses.append(DiskAudit)
	AllAuditClasses.append(FileAudit)
	AllAuditClasses.append(LogAudit)
	AllAuditClasses.append(ControllerStateAudit)
	AllAuditClasses.append(PartitionAudit)
	AllAuditClasses.append(PrimitiveAudit)
	AllAuditClasses.append(GroupAudit)
	AllAuditClasses.append(CloneAudit)
	AllAuditClasses.append(ColocationAudit)
	AllAuditClasses.append(CIBAudit)


	def AuditList(cm):
	result = []
	for auditclass in AllAuditClasses:
	a = auditclass(cm)
	if a.is_applicable():
	result.append(a)
	return result
	diff --git a/cts/lab/CTSlab.py.in b/cts/lab/CTSlab.py.in
	index 7bca3bd204..ac7f25620e 100644
	--- a/cts/lab/CTSlab.py.in
	+++ b/cts/lab/CTSlab.py.in
	@@ -1,128 +1,128 @@
	#!@PYTHON@
	""" Command-line interface to Pacemaker's Cluster Test Suite (CTS)
	"""

	__copyright__ = "Copyright 2001-2021 the Pacemaker project contributors"
	__license__ = "GNU General Public License version 2 or later (GPLv2+) WITHOUT ANY WARRANTY"

	import sys, signal, os

	pdir = os.path.dirname(sys.path[0])
	sys.path.insert(0, pdir) # So that things work from the source directory

	try:
	from cts.CTS import *
	from cts.CTSvars import *
	from cts.CM_corosync import *
	from cts.CTSaudits import AuditList
	from cts.CTStests import TestList
	from cts.CTSscenarios import *
	from cts.logging import LogFactory
	except ImportError as e:
	sys.stderr.write("abort: %s\n" % e)
	sys.stderr.write("check your install and PYTHONPATH; couldn't find cts libraries in:\n%s\n" %
	' '.join(sys.path))
	sys.exit(1)

	# These are globals so they can be used by the signal handler.
	scenario = None
	LogFactory().add_stderr()


	def sig_handler(signum, frame) :
	LogFactory().log("Interrupted by signal %d"%signum)
	if scenario: scenario.summarize()
	if signum == 15 :
	if scenario: scenario.TearDown()
	sys.exit(1)


	if __name__ == '__main__':

	Environment = CtsLab(sys.argv[1:])
	NumIter = Environment["iterations"]
	if NumIter is None:
	NumIter = 1
	Tests = []

	# Set the signal handler
	signal.signal(15, sig_handler)
	signal.signal(10, sig_handler)

	# Create the Cluster Manager object
	cm = None
	if Environment["Stack"] == "corosync 2+":
	- cm = crm_corosync(Environment)
	+ cm = crm_corosync()

	else:
	LogFactory().log("Unknown stack: "+Environment["stack"])
	sys.exit(1)

	if Environment["TruncateLog"] == 1:
	if Environment["OutputFile"] is None:
	LogFactory().log("Ignoring truncate request because no output file specified")
	else:
	LogFactory().log("Truncating %s" % Environment["OutputFile"])
	with open(Environment["OutputFile"], "w") as outputfile:
	outputfile.truncate(0)

	Audits = AuditList(cm)

	if Environment["ListTests"] == 1:
	Tests = TestList(cm, Audits)
	LogFactory().log("Total %d tests"%len(Tests))
	for test in Tests :
	LogFactory().log(str(test.name));
	sys.exit(0)

	elif len(Environment["tests"]) == 0:
	Tests = TestList(cm, Audits)

	else:
	Chosen = Environment["tests"]
	for TestCase in Chosen:
	match = None

	for test in TestList(cm, Audits):
	if test.name == TestCase:
	match = test

	if not match:
	LogFactory().log("--choose: No applicable/valid tests chosen")
	sys.exit(1)
	else:
	Tests.append(match)

	# Scenario selection
	if Environment["scenario"] == "basic-sanity":
	scenario = RandomTests(cm, [ BasicSanityCheck(Environment) ], Audits, Tests)

	elif Environment["scenario"] == "all-once":
	NumIter = len(Tests)
	scenario = AllOnce(
	cm, [ BootCluster(Environment), PacketLoss(Environment) ], Audits, Tests)
	elif Environment["scenario"] == "sequence":
	scenario = Sequence(
	cm, [ BootCluster(Environment), PacketLoss(Environment) ], Audits, Tests)
	elif Environment["scenario"] == "boot":
	scenario = Boot(cm, [ LeaveBooted(Environment)], Audits, [])
	else:
	scenario = RandomTests(
	cm, [ BootCluster(Environment), PacketLoss(Environment) ], Audits, Tests)

	LogFactory().log(">>>>>>>>>>>>>>>> BEGINNING " + repr(NumIter) + " TESTS ")
	LogFactory().log("Stack: %s (%s)" % (Environment["Stack"], Environment["Name"]))
	LogFactory().log("Schema: %s" % Environment["Schema"])
	LogFactory().log("Scenario: %s" % scenario.__doc__)
	LogFactory().log("CTS Exerciser: %s" % Environment["cts-exerciser"])
	LogFactory().log("CTS Logfile: %s" % Environment["OutputFile"])
	LogFactory().log("Random Seed: %s" % Environment["RandSeed"])
	LogFactory().log("Syslog variant: %s" % Environment["syslogd"].strip())
	LogFactory().log("System log files: %s" % Environment["LogFileName"])
	if Environment.has_key("IPBase"):
	LogFactory().log("Base IP for resources: %s" % Environment["IPBase"])
	LogFactory().log("Cluster starts at boot: %d" % Environment["at-boot"])

	Environment.dump()
	rc = Environment.run(scenario, NumIter)
	sys.exit(rc)
	diff --git a/cts/lab/CTSscenarios.py b/cts/lab/CTSscenarios.py
	index 750b0b11c6..a40cceba79 100644
	--- a/cts/lab/CTSscenarios.py
	+++ b/cts/lab/CTSscenarios.py
	@@ -1,602 +1,601 @@
	""" Test scenario classes for Pacemaker's Cluster Test Suite (CTS)
	"""

	__copyright__ = "Copyright 2000-2021 the Pacemaker project contributors"
	__license__ = "GNU General Public License version 2 or later (GPLv2+) WITHOUT ANY WARRANTY"

	import os
	import re
	import sys
	import time

	from cts.CTStests import CTSTest
	from cts.CTSaudits import ClusterAudit
	from cts.watcher import LogWatcher
	-from cts.remote import input_wrapper

	class ScenarioComponent(object):

	def __init__(self, Env):
	self.Env = Env

	def IsApplicable(self):
	'''Return TRUE if the current ScenarioComponent is applicable
	in the given LabEnvironment given to the constructor.
	'''

	raise ValueError("Abstract Class member (IsApplicable)")

	def SetUp(self, CM):
	'''Set up the given ScenarioComponent'''
	raise ValueError("Abstract Class member (Setup)")

	def TearDown(self, CM):
	'''Tear down (undo) the given ScenarioComponent'''
	raise ValueError("Abstract Class member (Setup)")


	class Scenario(object):
	(
	'''The basic idea of a scenario is that of an ordered list of
	ScenarioComponent objects. Each ScenarioComponent is SetUp() in turn,
	and then after the tests have been run, they are torn down using TearDown()
	(in reverse order).

	A Scenario is applicable to a particular cluster manager iff each
	ScenarioComponent is applicable.

	A partially set up scenario is torn down if it fails during setup.
	''')

	def __init__(self, ClusterManager, Components, Audits, Tests):

	"Initialize the Scenario from the list of ScenarioComponents"

	self.ClusterManager = ClusterManager
	self.Components = Components
	self.Audits = Audits
	self.Tests = Tests

	self.BadNews = None
	self.TestSets = []
	self.Stats = {"success":0, "failure":0, "BadNews":0, "skipped":0}
	self.Sets = []

	#self.ns=CTS.NodeStatus(self.Env)

	for comp in Components:
	if not issubclass(comp.__class__, ScenarioComponent):
	raise ValueError("Init value must be subclass of ScenarioComponent")

	for audit in Audits:
	if not issubclass(audit.__class__, ClusterAudit):
	raise ValueError("Init value must be subclass of ClusterAudit")

	for test in Tests:
	if not issubclass(test.__class__, CTSTest):
	raise ValueError("Init value must be a subclass of CTSTest")

	def IsApplicable(self):
	(
	'''A Scenario IsApplicable() iff each of its ScenarioComponents IsApplicable()
	'''
	)

	for comp in self.Components:
	if not comp.IsApplicable():
	return None
	return 1

	def SetUp(self):
	'''Set up the Scenario. Return TRUE on success.'''

	self.ClusterManager.prepare()
	self.audit() # Also detects remote/local log config
	self.ClusterManager.StatsMark(0)
	self.ClusterManager.ns.WaitForAllNodesToComeUp(self.ClusterManager.Env["nodes"])

	self.audit()
	self.ClusterManager.install_support()

	self.BadNews = LogWatcher(self.ClusterManager.Env["LogFileName"],
	self.ClusterManager.templates.get_patterns(
	self.ClusterManager.Env["Name"], "BadNews"), "BadNews", 0,
	kind=self.ClusterManager.Env["LogWatcher"],
	hosts=self.ClusterManager.Env["nodes"])
	self.BadNews.setwatch() # Call after we've figured out what type of log watching to do in LogAudit

	j = 0
	while j < len(self.Components):
	if not self.Components[j].SetUp(self.ClusterManager):
	# OOPS! We failed. Tear partial setups down.
	self.audit()
	self.ClusterManager.log("Tearing down partial setup")
	self.TearDown(j)
	return None
	j = j + 1

	self.audit()
	return 1

	def TearDown(self, max=None):

	'''Tear Down the Scenario - in reverse order.'''

	if max == None:
	max = len(self.Components)-1
	j = max
	while j >= 0:
	self.Components[j].TearDown(self.ClusterManager)
	j = j - 1

	self.audit()
	self.ClusterManager.StatsExtract()
	self.ClusterManager.install_support("uninstall")

	def incr(self, name):
	'''Increment (or initialize) the value associated with the given name'''
	if not name in self.Stats:
	self.Stats[name] = 0
	self.Stats[name] = self.Stats[name]+1

	def run(self, Iterations):
	self.ClusterManager.oprofileStart()
	try:
	self.run_loop(Iterations)
	self.ClusterManager.oprofileStop()
	except:
	self.ClusterManager.oprofileStop()
	raise

	def run_loop(self, Iterations):
	raise ValueError("Abstract Class member (run_loop)")

	def run_test(self, test, testcount):
	nodechoice = self.ClusterManager.Env.RandomNode()

	ret = 1
	where = ""
	did_run = 0

	self.ClusterManager.StatsMark(testcount)
	self.ClusterManager.instance_errorstoignore_clear()
	self.ClusterManager.log(("Running test %s" % test.name).ljust(35) + (" (%s) " % nodechoice).ljust(15) + "[" + ("%d" % testcount).rjust(3) + "]")

	starttime = test.set_timer()
	if not test.setup(nodechoice):
	self.ClusterManager.log("Setup failed")
	ret = 0

	elif not test.canrunnow(nodechoice):
	self.ClusterManager.log("Skipped")
	test.skipped()

	else:
	did_run = 1
	ret = test(nodechoice)

	if not test.teardown(nodechoice):
	self.ClusterManager.log("Teardown failed")
	if self.ClusterManager.Env["continue"] == 1:
	answer = "Y"
	else:
	try:
	- answer = input_wrapper('Continue? [nY]')
	+ answer = input('Continue? [nY]')
	except EOFError as e:
	answer = "n"
	if answer and answer == "n":
	raise ValueError("Teardown of %s on %s failed" % (test.name, nodechoice))
	ret = 0

	stoptime = time.time()
	self.ClusterManager.oprofileSave(testcount)

	elapsed_time = stoptime - starttime
	test_time = stoptime - test.get_timer()
	if not test["min_time"]:
	test["elapsed_time"] = elapsed_time
	test["min_time"] = test_time
	test["max_time"] = test_time
	else:
	test["elapsed_time"] = test["elapsed_time"] + elapsed_time
	if test_time < test["min_time"]:
	test["min_time"] = test_time
	if test_time > test["max_time"]:
	test["max_time"] = test_time

	if ret:
	self.incr("success")
	test.log_timer()
	else:
	self.incr("failure")
	self.ClusterManager.statall()
	did_run = 1 # Force the test count to be incremented anyway so test extraction works

	self.audit(test.errorstoignore())
	return did_run

	def summarize(self):
	self.ClusterManager.log("****************")
	self.ClusterManager.log("Overall Results:" + repr(self.Stats))
	self.ClusterManager.log("****************")

	stat_filter = {
	"calls":0,
	"failure":0,
	"skipped":0,
	"auditfail":0,
	}
	self.ClusterManager.log("Test Summary")
	for test in self.Tests:
	for key in list(stat_filter.keys()):
	stat_filter[key] = test.Stats[key]
	self.ClusterManager.log(("Test %s: "%test.name).ljust(25) + " %s"%repr(stat_filter))

	self.ClusterManager.debug("Detailed Results")
	for test in self.Tests:
	self.ClusterManager.debug(("Test %s: "%test.name).ljust(25) + " %s"%repr(test.Stats))

	self.ClusterManager.log("<<<<<<<<<<<<<<<< TESTS COMPLETED")

	def audit(self, LocalIgnore=[]):
	errcount = 0
	ignorelist = []
	ignorelist.append("CTS:")
	ignorelist.extend(LocalIgnore)
	ignorelist.extend(self.ClusterManager.errorstoignore())
	ignorelist.extend(self.ClusterManager.instance_errorstoignore())

	# This makes sure everything is stabilized before starting...
	failed = 0
	for audit in self.Audits:
	if not audit():
	self.ClusterManager.log("Audit " + audit.name() + " FAILED.")
	failed += 1
	else:
	self.ClusterManager.debug("Audit " + audit.name() + " passed.")

	while errcount < 1000:
	match = None
	if self.BadNews:
	match = self.BadNews.look(0)

	if match:
	add_err = 1
	for ignore in ignorelist:
	if add_err == 1 and re.search(ignore, match):
	add_err = 0
	if add_err == 1:
	self.ClusterManager.log("BadNews: " + match)
	self.incr("BadNews")
	errcount = errcount + 1
	else:
	break
	else:
	if self.ClusterManager.Env["continue"] == 1:
	answer = "Y"
	else:
	try:
	- answer = input_wrapper('Big problems. Continue? [nY]')
	+ answer = input('Big problems. Continue? [nY]')
	except EOFError as e:
	answer = "n"
	if answer and answer == "n":
	self.ClusterManager.log("Shutting down.")
	self.summarize()
	self.TearDown()
	raise ValueError("Looks like we hit a BadNews jackpot!")

	if self.BadNews:
	self.BadNews.end()
	return failed


	class AllOnce(Scenario):
	'''Every Test Once''' # Accessable as __doc__
	def run_loop(self, Iterations):
	testcount = 1
	for test in self.Tests:
	self.run_test(test, testcount)
	testcount += 1


	class RandomTests(Scenario):
	'''Random Test Execution'''
	def run_loop(self, Iterations):
	testcount = 1
	while testcount <= Iterations:
	test = self.ClusterManager.Env.RandomGen.choice(self.Tests)
	self.run_test(test, testcount)
	testcount += 1


	class BasicSanity(Scenario):
	'''Basic Cluster Sanity'''
	def run_loop(self, Iterations):
	testcount = 1
	while testcount <= Iterations:
	test = self.Environment.RandomGen.choice(self.Tests)
	self.run_test(test, testcount)
	testcount += 1


	class Sequence(Scenario):
	'''Named Tests in Sequence'''
	def run_loop(self, Iterations):
	testcount = 1
	while testcount <= Iterations:
	for test in self.Tests:
	self.run_test(test, testcount)
	testcount += 1


	class Boot(Scenario):
	'''Start the Cluster'''
	def run_loop(self, Iterations):
	testcount = 0


	class BootCluster(ScenarioComponent):
	(
	'''BootCluster is the most basic of ScenarioComponents.
	This ScenarioComponent simply starts the cluster manager on all the nodes.
	It is fairly robust as it waits for all nodes to come up before starting
	as they might have been rebooted or crashed for some reason beforehand.
	''')
	def __init__(self, Env):
	pass

	def IsApplicable(self):
	'''BootCluster is so generic it is always Applicable'''
	return 1

	def SetUp(self, CM):
	'''Basic Cluster Manager startup. Start everything'''

	CM.prepare()

	# Clear out the cobwebs ;-)
	CM.stopall(verbose=True, force=True)

	# Now start the Cluster Manager on all the nodes.
	CM.log("Starting Cluster Manager on all nodes.")
	return CM.startall(verbose=True, quick=True)

	def TearDown(self, CM, force=False):
	'''Set up the given ScenarioComponent'''

	# Stop the cluster manager everywhere

	CM.log("Stopping Cluster Manager on all nodes")
	return CM.stopall(verbose=True, force=force)


	class LeaveBooted(BootCluster):
	def TearDown(self, CM):
	'''Set up the given ScenarioComponent'''

	# Stop the cluster manager everywhere

	CM.log("Leaving Cluster running on all nodes")
	return 1


	class PingFest(ScenarioComponent):
	(
	'''PingFest does a flood ping to each node in the cluster from the test machine.

	If the LabEnvironment Parameter PingSize is set, it will be used as the size
	of ping packet requested (via the -s option). If it is not set, it defaults
	to 1024 bytes.

	According to the manual page for ping:
	Outputs packets as fast as they come back or one hundred times per
	second, whichever is more. For every ECHO_REQUEST sent a period ``.''
	is printed, while for every ECHO_REPLY received a backspace is printed.
	This provides a rapid display of how many packets are being dropped.
	Only the super-user may use this option. This can be very hard on a net-
	work and should be used with caution.
	''' )

	def __init__(self, Env):
	self.Env = Env

	def IsApplicable(self):
	'''PingFests are always applicable ;-)
	'''

	return 1

	def SetUp(self, CM):
	'''Start the PingFest!'''

	self.PingSize = 1024
	if "PingSize" in list(CM.Env.keys()):
	self.PingSize = CM.Env["PingSize"]

	CM.log("Starting %d byte flood pings" % self.PingSize)

	self.PingPids = []
	for node in CM.Env["nodes"]:
	self.PingPids.append(self._pingchild(node))

	CM.log("Ping PIDs: " + repr(self.PingPids))
	return 1

	def TearDown(self, CM):
	'''Stop it right now! My ears are pinging!!'''

	for pid in self.PingPids:
	if pid != None:
	CM.log("Stopping ping process %d" % pid)
	os.kill(pid, signal.SIGKILL)

	def _pingchild(self, node):

	Args = ["ping", "-qfn", "-s", str(self.PingSize), node]

	sys.stdin.flush()
	sys.stdout.flush()
	sys.stderr.flush()
	pid = os.fork()

	if pid < 0:
	self.Env.log("Cannot fork ping child")
	return None
	if pid > 0:
	return pid

	# Otherwise, we're the child process.

	os.execvp("ping", Args)
	self.Env.log("Cannot execvp ping: " + repr(Args))
	sys.exit(1)


	class PacketLoss(ScenarioComponent):
	(
	'''
	It would be useful to do some testing of CTS with a modest amount of packet loss
	enabled - so we could see that everything runs like it should with a certain
	amount of packet loss present.
	''')

	def IsApplicable(self):
	'''always Applicable'''
	return 1

	def SetUp(self, CM):
	'''Reduce the reliability of communications'''
	if float(CM.Env["XmitLoss"]) == 0 and float(CM.Env["RecvLoss"]) == 0 :
	return 1

	for node in CM.Env["nodes"]:
	CM.reducecomm_node(node)

	CM.log("Reduce the reliability of communications")

	return 1

	def TearDown(self, CM):
	'''Fix the reliability of communications'''

	if float(CM.Env["XmitLoss"]) == 0 and float(CM.Env["RecvLoss"]) == 0 :
	return 1

	for node in CM.Env["nodes"]:
	CM.unisolate_node(node)

	CM.log("Fix the reliability of communications")


	class BasicSanityCheck(ScenarioComponent):
	(
	'''
	''')

	def IsApplicable(self):
	return self.Env["DoBSC"]

	def SetUp(self, CM):

	CM.prepare()

	# Clear out the cobwebs
	self.TearDown(CM)

	# Now start the Cluster Manager on all the nodes.
	CM.log("Starting Cluster Manager on BSC node(s).")
	return CM.startall()

	def TearDown(self, CM):
	CM.log("Stopping Cluster Manager on BSC node(s).")
	return CM.stopall()


	class Benchmark(ScenarioComponent):
	(
	'''
	''')

	def IsApplicable(self):
	return self.Env["benchmark"]

	def SetUp(self, CM):

	CM.prepare()

	# Clear out the cobwebs
	self.TearDown(CM, force=True)

	# Now start the Cluster Manager on all the nodes.
	CM.log("Starting Cluster Manager on all node(s).")
	return CM.startall()

	def TearDown(self, CM):
	CM.log("Stopping Cluster Manager on all node(s).")
	return CM.stopall()


	class RollingUpgrade(ScenarioComponent):
	(
	'''
	Test a rolling upgrade between two versions of the stack
	''')

	def __init__(self, Env):
	self.Env = Env

	def IsApplicable(self):
	if not self.Env["rpm-dir"]:
	return None
	if not self.Env["current-version"]:
	return None
	if not self.Env["previous-version"]:
	return None

	return 1

	def install(self, node, version):

	target_dir = "/tmp/rpm-%s" % version
	src_dir = "%s/%s" % (self.CM.Env["rpm-dir"], version)

	rc = self.CM.rsh(node, "mkdir -p %s" % target_dir)
	rc = self.CM.cp("%s/*.rpm %s:%s" % (src_dir, node, target_dir))
	rc = self.CM.rsh(node, "rpm -Uvh --force %s/*.rpm" % (target_dir))

	return self.success()

	def upgrade(self, node):
	return self.install(node, self.CM.Env["current-version"])

	def downgrade(self, node):
	return self.install(node, self.CM.Env["previous-version"])

	def SetUp(self, CM):
	print(repr(self)+"prepare")
	CM.prepare()

	# Clear out the cobwebs
	CM.stopall(force=True)

	CM.log("Downgrading all nodes to %s." % self.Env["previous-version"])

	for node in self.Env["nodes"]:
	if not self.downgrade(node):
	CM.log("Couldn't downgrade %s" % node)
	return None

	return 1

	def TearDown(self, CM):
	# Stop everything
	CM.log("Stopping Cluster Manager on Upgrade nodes.")
	CM.stopall()

	CM.log("Upgrading all nodes to %s." % self.Env["current-version"])
	for node in self.Env["nodes"]:
	if not self.upgrade(node):
	CM.log("Couldn't upgrade %s" % node)
	return None

	return 1
	diff --git a/cts/lab/CTStests.py b/cts/lab/CTStests.py
	index 649489cf6f..62c832eb45 100644
	--- a/cts/lab/CTStests.py
	+++ b/cts/lab/CTStests.py
	@@ -1,3142 +1,3142 @@
	""" Test-specific classes for Pacemaker's Cluster Test Suite (CTS)
	"""

	__copyright__ = "Copyright 2000-2021 the Pacemaker project contributors"
	__license__ = "GNU General Public License version 2 or later (GPLv2+) WITHOUT ANY WARRANTY"

	#
	# SPECIAL NOTE:
	#
	# Tests may NOT implement any cluster-manager-specific code in them.
	# EXTEND the ClusterManager object to provide the base capabilities
	# the test needs if you need to do something that the current CM classes
	# do not. Otherwise you screw up the whole point of the object structure
	# in CTS.
	#
	# Thank you.
	#

	import os
	import re
	import time
	import subprocess
	import tempfile

	from stat import *
	from cts import CTS
	from cts.CTSaudits import *
	from cts.CTSvars import *
	from cts.patterns import PatternSelector
	from cts.logging import LogFactory
	-from cts.remote import RemoteFactory, input_wrapper
	+from cts.remote import RemoteFactory
	from cts.watcher import LogWatcher
	from cts.environment import EnvFactory

	AllTestClasses = [ ]


	class CTSTest(object):
	'''
	A Cluster test.
	We implement the basic set of properties and behaviors for a generic
	cluster test.

	Cluster tests track their own statistics.
	We keep each of the kinds of counts we track as separate {name,value}
	pairs.
	'''

	def __init__(self, cm):
	#self.name="the unnamed test"
	self.Stats = {"calls":0
	, "success":0
	, "failure":0
	, "skipped":0
	, "auditfail":0}

	# if not issubclass(cm.__class__, ClusterManager):
	# raise ValueError("Must be a ClusterManager object")
	self.CM = cm
	self.Env = EnvFactory().getInstance()
	self.rsh = RemoteFactory().getInstance()
	self.logger = LogFactory()
	self.templates = PatternSelector(cm["Name"])
	self.Audits = []
	self.timeout = 120
	self.passed = 1
	self.is_loop = 0
	self.is_unsafe = 0
	self.is_docker_unsafe = 0
	self.is_experimental = 0
	self.is_container = 0
	self.is_valgrind = 0
	self.benchmark = 0 # which tests to benchmark
	self.timer = {} # timers

	def log(self, args):
	self.logger.log(args)

	def debug(self, args):
	self.logger.debug(args)

	def has_key(self, key):
	return key in self.Stats

	def __setitem__(self, key, value):
	self.Stats[key] = value

	def __getitem__(self, key):
	if str(key) == "0":
	raise ValueError("Bad call to 'foo in X', should reference 'foo in X.Stats' instead")

	if key in self.Stats:
	return self.Stats[key]
	return None

	def log_mark(self, msg):
	self.debug("MARK: test %s %s %d" % (self.name,msg,time.time()))
	return

	def get_timer(self,key = "test"):
	try: return self.timer[key]
	except: return 0

	def set_timer(self,key = "test"):
	self.timer[key] = time.time()
	return self.timer[key]

	def log_timer(self,key = "test"):
	elapsed = 0
	if key in self.timer:
	elapsed = time.time() - self.timer[key]
	s = key == "test" and self.name or "%s:%s" % (self.name,key)
	self.debug("%s runtime: %.2f" % (s, elapsed))
	del self.timer[key]
	return elapsed

	def incr(self, name):
	'''Increment (or initialize) the value associated with the given name'''
	if not name in self.Stats:
	self.Stats[name] = 0
	self.Stats[name] = self.Stats[name]+1

	# Reset the test passed boolean
	if name == "calls":
	self.passed = 1

	def failure(self, reason="none"):
	'''Increment the failure count'''
	self.passed = 0
	self.incr("failure")
	self.logger.log(("Test %s" % self.name).ljust(35) + " FAILED: %s" % reason)
	return None

	def success(self):
	'''Increment the success count'''
	self.incr("success")
	return 1

	def skipped(self):
	'''Increment the skipped count'''
	self.incr("skipped")
	return 1

	def __call__(self, node):
	'''Perform the given test'''
	raise ValueError("Abstract Class member (__call__)")
	self.incr("calls")
	return self.failure()

	def audit(self):
	passed = 1
	if len(self.Audits) > 0:
	for audit in self.Audits:
	if not audit():
	self.logger.log("Internal %s Audit %s FAILED." % (self.name, audit.name()))
	self.incr("auditfail")
	passed = 0
	return passed

	def setup(self, node):
	'''Setup the given test'''
	return self.success()

	def teardown(self, node):
	'''Tear down the given test'''
	return self.success()

	def create_watch(self, patterns, timeout, name=None):
	if not name:
	name = self.name
	return LogWatcher(self.Env["LogFileName"], patterns, name, timeout, kind=self.Env["LogWatcher"], hosts=self.Env["nodes"])

	def local_badnews(self, prefix, watch, local_ignore=[]):
	errcount = 0
	if not prefix:
	prefix = "LocalBadNews:"

	ignorelist = []
	ignorelist.append(" CTS: ")
	ignorelist.append(prefix)
	ignorelist.extend(local_ignore)

	while errcount < 100:
	match = watch.look(0)
	if match:
	add_err = 1
	for ignore in ignorelist:
	if add_err == 1 and re.search(ignore, match):
	add_err = 0
	if add_err == 1:
	self.logger.log(prefix + " " + match)
	errcount = errcount + 1
	else:
	break
	else:
	self.logger.log("Too many errors!")

	watch.end()
	return errcount

	def is_applicable(self):
	return self.is_applicable_common()

	def is_applicable_common(self):
	'''Return TRUE if we are applicable in the current test configuration'''
	#raise ValueError("Abstract Class member (is_applicable)")

	if self.is_loop and not self.Env["loop-tests"]:
	return 0
	elif self.is_unsafe and not self.Env["unsafe-tests"]:
	return 0
	elif self.is_valgrind and not self.Env["valgrind-tests"]:
	return 0
	elif self.is_experimental and not self.Env["experimental-tests"]:
	return 0
	elif self.is_docker_unsafe and self.Env["docker"]:
	return 0
	elif self.is_container and not self.Env["container-tests"]:
	return 0
	elif self.Env["benchmark"] and self.benchmark == 0:
	return 0

	return 1

	def find_ocfs2_resources(self, node):
	self.r_o2cb = None
	self.r_ocfs2 = []

	(rc, lines) = self.rsh(node, "crm_resource -c", None)
	for line in lines:
	if re.search("^Resource", line):
	r = AuditResource(self.CM, line)
	if r.rtype == "o2cb" and r.parent != "NA":
	self.debug("Found o2cb: %s" % self.r_o2cb)
	self.r_o2cb = r.parent
	if re.search("^Constraint", line):
	c = AuditConstraint(self.CM, line)
	if c.type == "rsc_colocation" and c.target == self.r_o2cb:
	self.r_ocfs2.append(c.rsc)

	self.debug("Found ocfs2 filesystems: %s" % repr(self.r_ocfs2))
	return len(self.r_ocfs2)

	def canrunnow(self, node):
	'''Return TRUE if we can meaningfully run right now'''
	return 1

	def errorstoignore(self):
	'''Return list of errors which are 'normal' and should be ignored'''
	return []


	class StopTest(CTSTest):
	'''Stop (deactivate) the cluster manager on a node'''
	def __init__(self, cm):
	CTSTest.__init__(self, cm)
	self.name = "Stop"

	def __call__(self, node):
	'''Perform the 'stop' test. '''
	self.incr("calls")
	if self.CM.ShouldBeStatus[node] != "up":
	return self.skipped()

	patterns = []
	# Technically we should always be able to notice ourselves stopping
	patterns.append(self.templates["Pat:We_stopped"] % node)

	# Any active node needs to notice this one left
	# (note that this won't work if we have multiple partitions)
	for other in self.Env["nodes"]:
	if self.CM.ShouldBeStatus[other] == "up" and other != node:
	patterns.append(self.templates["Pat:They_stopped"] %(other, self.CM.key_for_node(node)))
	#self.debug("Checking %s will notice %s left"%(other, node))

	watch = self.create_watch(patterns, self.Env["DeadTime"])
	watch.setwatch()

	if node == self.CM.OurNode:
	self.incr("us")
	else:
	if self.CM.upcount() <= 1:
	self.incr("all")
	else:
	self.incr("them")

	self.CM.StopaCM(node)
	watch_result = watch.lookforall()

	failreason = None
	UnmatchedList = "\|\|"
	if watch.unmatched:
	(rc, output) = self.rsh(node, "/bin/ps axf", None)
	for line in output:
	self.debug(line)

	(rc, output) = self.rsh(node, "/usr/sbin/dlm_tool dump 2>/dev/null", None)
	for line in output:
	self.debug(line)

	for regex in watch.unmatched:
	self.logger.log ("ERROR: Shutdown pattern not found: %s" % (regex))
	UnmatchedList += regex + "\|\|";
	failreason = "Missing shutdown pattern"

	self.CM.cluster_stable(self.Env["DeadTime"])

	if not watch.unmatched or self.CM.upcount() == 0:
	return self.success()

	if len(watch.unmatched) >= self.CM.upcount():
	return self.failure("no match against (%s)" % UnmatchedList)

	if failreason == None:
	return self.success()
	else:
	return self.failure(failreason)
	#
	# We don't register StopTest because it's better when called by
	# another test...
	#


	class StartTest(CTSTest):
	'''Start (activate) the cluster manager on a node'''
	def __init__(self, cm, debug=None):
	CTSTest.__init__(self,cm)
	self.name = "start"
	self.debug = debug

	def __call__(self, node):
	'''Perform the 'start' test. '''
	self.incr("calls")

	if self.CM.upcount() == 0:
	self.incr("us")
	else:
	self.incr("them")

	if self.CM.ShouldBeStatus[node] != "down":
	return self.skipped()
	elif self.CM.StartaCM(node):
	return self.success()
	else:
	return self.failure("Startup %s on node %s failed"
	% (self.Env["Name"], node))

	#
	# We don't register StartTest because it's better when called by
	# another test...
	#


	class FlipTest(CTSTest):
	'''If it's running, stop it. If it's stopped start it.
	Overthrow the status quo...
	'''
	def __init__(self, cm):
	CTSTest.__init__(self,cm)
	self.name = "Flip"
	self.start = StartTest(cm)
	self.stop = StopTest(cm)

	def __call__(self, node):
	'''Perform the 'Flip' test. '''
	self.incr("calls")
	if self.CM.ShouldBeStatus[node] == "up":
	self.incr("stopped")
	ret = self.stop(node)
	type = "up->down"
	# Give the cluster time to recognize it's gone...
	time.sleep(self.Env["StableTime"])
	elif self.CM.ShouldBeStatus[node] == "down":
	self.incr("started")
	ret = self.start(node)
	type = "down->up"
	else:
	return self.skipped()

	self.incr(type)
	if ret:
	return self.success()
	else:
	return self.failure("%s failure" % type)

	# Register FlipTest as a good test to run
	AllTestClasses.append(FlipTest)


	class RestartTest(CTSTest):
	'''Stop and restart a node'''
	def __init__(self, cm):
	CTSTest.__init__(self,cm)
	self.name = "Restart"
	self.start = StartTest(cm)
	self.stop = StopTest(cm)
	self.benchmark = 1

	def __call__(self, node):
	'''Perform the 'restart' test. '''
	self.incr("calls")

	self.incr("node:" + node)

	ret1 = 1
	if self.CM.StataCM(node):
	self.incr("WasStopped")
	if not self.start(node):
	return self.failure("start (setup) failure: "+node)

	self.set_timer()
	if not self.stop(node):
	return self.failure("stop failure: "+node)
	if not self.start(node):
	return self.failure("start failure: "+node)
	return self.success()

	# Register RestartTest as a good test to run
	AllTestClasses.append(RestartTest)


	class StonithdTest(CTSTest):
	def __init__(self, cm):
	CTSTest.__init__(self, cm)
	self.name = "Stonithd"
	self.startall = SimulStartLite(cm)
	self.benchmark = 1

	def __call__(self, node):
	self.incr("calls")
	if len(self.Env["nodes"]) < 2:
	return self.skipped()

	ret = self.startall(None)
	if not ret:
	return self.failure("Setup failed")

	is_dc = self.CM.is_node_dc(node)

	watchpats = []
	watchpats.append(self.templates["Pat:FenceOpOK"] % node)
	watchpats.append(self.templates["Pat:NodeFenced"] % node)

	if self.Env["at-boot"] == 0:
	self.debug("Expecting %s to stay down" % node)
	self.CM.ShouldBeStatus[node] = "down"
	else:
	self.debug("Expecting %s to come up again %d" % (node, self.Env["at-boot"]))
	watchpats.append("%s.* S_STARTING -> S_PENDING" % node)
	watchpats.append("%s.* S_PENDING -> S_NOT_DC" % node)

	watch = self.create_watch(watchpats, 30 + self.Env["DeadTime"] + self.Env["StableTime"] + self.Env["StartTime"])
	watch.setwatch()

	origin = self.Env.RandomGen.choice(self.Env["nodes"])

	rc = self.rsh(origin, "stonith_admin --reboot %s -VVVVVV" % node)

	if rc == 194:
	# 194 - 256 = -62 = Timer expired
	#
	# Look for the patterns, usually this means the required
	# device was running on the node to be fenced - or that
	# the required devices were in the process of being loaded
	# and/or moved
	#
	# Effectively the node committed suicide so there will be
	# no confirmation, but pacemaker should be watching and
	# fence the node again

	self.logger.log("Fencing command on %s to fence %s timed out" % (origin, node))

	elif origin != node and rc != 0:
	self.debug("Waiting for the cluster to recover")
	self.CM.cluster_stable()

	self.debug("Waiting for fenced node to come back up")
	self.CM.ns.WaitForAllNodesToComeUp(self.Env["nodes"], 600)

	self.logger.log("Fencing command on %s failed to fence %s (rc=%d)" % (origin, node, rc))

	elif origin == node and rc != 255:
	# 255 == broken pipe, ie. the node was fenced as expected
	self.logger.log("Locally originated fencing returned %d" % rc)

	self.set_timer("fence")
	matched = watch.lookforall()
	self.log_timer("fence")
	self.set_timer("reform")
	if watch.unmatched:
	self.logger.log("Patterns not found: " + repr(watch.unmatched))

	self.debug("Waiting for the cluster to recover")
	self.CM.cluster_stable()

	self.debug("Waiting for fenced node to come back up")
	self.CM.ns.WaitForAllNodesToComeUp(self.Env["nodes"], 600)

	self.debug("Waiting for the cluster to re-stabilize with all nodes")
	is_stable = self.CM.cluster_stable(self.Env["StartTime"])

	if not matched:
	return self.failure("Didn't find all expected patterns")
	elif not is_stable:
	return self.failure("Cluster did not become stable")

	self.log_timer("reform")
	return self.success()

	def errorstoignore(self):
	return [
	self.templates["Pat:Fencing_start"] % ".*",
	self.templates["Pat:Fencing_ok"] % ".*",
	self.templates["Pat:Fencing_active"],
	r"error.: Operation 'reboot' targeting . by .* for stonith_admin.*: Timer expired",
	]

	def is_applicable(self):
	if not self.is_applicable_common():
	return 0

	if "DoFencing" in list(self.Env.keys()):
	return self.Env["DoFencing"]

	return 1

	AllTestClasses.append(StonithdTest)


	class StartOnebyOne(CTSTest):
	'''Start all the nodes ~ one by one'''
	def __init__(self, cm):
	CTSTest.__init__(self,cm)
	self.name = "StartOnebyOne"
	self.stopall = SimulStopLite(cm)
	self.start = StartTest(cm)
	self.ns = CTS.NodeStatus(cm.Env)

	def __call__(self, dummy):
	'''Perform the 'StartOnebyOne' test. '''
	self.incr("calls")

	# We ignore the "node" parameter...

	# Shut down all the nodes...
	ret = self.stopall(None)
	if not ret:
	return self.failure("Test setup failed")

	failed = []
	self.set_timer()
	for node in self.Env["nodes"]:
	if not self.start(node):
	failed.append(node)

	if len(failed) > 0:
	return self.failure("Some node failed to start: " + repr(failed))

	return self.success()

	# Register StartOnebyOne as a good test to run
	AllTestClasses.append(StartOnebyOne)


	class SimulStart(CTSTest):
	'''Start all the nodes ~ simultaneously'''
	def __init__(self, cm):
	CTSTest.__init__(self,cm)
	self.name = "SimulStart"
	self.stopall = SimulStopLite(cm)
	self.startall = SimulStartLite(cm)

	def __call__(self, dummy):
	'''Perform the 'SimulStart' test. '''
	self.incr("calls")

	# We ignore the "node" parameter...

	# Shut down all the nodes...
	ret = self.stopall(None)
	if not ret:
	return self.failure("Setup failed")

	if not self.startall(None):
	return self.failure("Startall failed")

	return self.success()

	# Register SimulStart as a good test to run
	AllTestClasses.append(SimulStart)


	class SimulStop(CTSTest):
	'''Stop all the nodes ~ simultaneously'''
	def __init__(self, cm):
	CTSTest.__init__(self,cm)
	self.name = "SimulStop"
	self.startall = SimulStartLite(cm)
	self.stopall = SimulStopLite(cm)

	def __call__(self, dummy):
	'''Perform the 'SimulStop' test. '''
	self.incr("calls")

	# We ignore the "node" parameter...

	# Start up all the nodes...
	ret = self.startall(None)
	if not ret:
	return self.failure("Setup failed")

	if not self.stopall(None):
	return self.failure("Stopall failed")

	return self.success()

	# Register SimulStop as a good test to run
	AllTestClasses.append(SimulStop)


	class StopOnebyOne(CTSTest):
	'''Stop all the nodes in order'''
	def __init__(self, cm):
	CTSTest.__init__(self,cm)
	self.name = "StopOnebyOne"
	self.startall = SimulStartLite(cm)
	self.stop = StopTest(cm)

	def __call__(self, dummy):
	'''Perform the 'StopOnebyOne' test. '''
	self.incr("calls")

	# We ignore the "node" parameter...

	# Start up all the nodes...
	ret = self.startall(None)
	if not ret:
	return self.failure("Setup failed")

	failed = []
	self.set_timer()
	for node in self.Env["nodes"]:
	if not self.stop(node):
	failed.append(node)

	if len(failed) > 0:
	return self.failure("Some node failed to stop: " + repr(failed))

	return self.success()

	# Register StopOnebyOne as a good test to run
	AllTestClasses.append(StopOnebyOne)


	class RestartOnebyOne(CTSTest):
	'''Restart all the nodes in order'''
	def __init__(self, cm):
	CTSTest.__init__(self,cm)
	self.name = "RestartOnebyOne"
	self.startall = SimulStartLite(cm)

	def __call__(self, dummy):
	'''Perform the 'RestartOnebyOne' test. '''
	self.incr("calls")

	# We ignore the "node" parameter...

	# Start up all the nodes...
	ret = self.startall(None)
	if not ret:
	return self.failure("Setup failed")

	did_fail = []
	self.set_timer()
	self.restart = RestartTest(self.CM)
	for node in self.Env["nodes"]:
	if not self.restart(node):
	did_fail.append(node)

	if did_fail:
	return self.failure("Could not restart %d nodes: %s"
	% (len(did_fail), repr(did_fail)))
	return self.success()

	# Register StopOnebyOne as a good test to run
	AllTestClasses.append(RestartOnebyOne)


	class PartialStart(CTSTest):
	'''Start a node - but tell it to stop before it finishes starting up'''
	def __init__(self, cm):
	CTSTest.__init__(self,cm)
	self.name = "PartialStart"
	self.startall = SimulStartLite(cm)
	self.stopall = SimulStopLite(cm)
	self.stop = StopTest(cm)
	#self.is_unsafe = 1

	def __call__(self, node):
	'''Perform the 'PartialStart' test. '''
	self.incr("calls")

	ret = self.stopall(None)
	if not ret:
	return self.failure("Setup failed")

	watchpats = []
	watchpats.append("pacemaker-controld.Connecting to . cluster infrastructure")
	watch = self.create_watch(watchpats, self.Env["DeadTime"]+10)
	watch.setwatch()

	self.CM.StartaCMnoBlock(node)
	ret = watch.lookforall()
	if not ret:
	self.logger.log("Patterns not found: " + repr(watch.unmatched))
	return self.failure("Setup of %s failed" % node)

	ret = self.stop(node)
	if not ret:
	return self.failure("%s did not stop in time" % node)

	return self.success()

	def errorstoignore(self):
	'''Return list of errors which should be ignored'''

	# We might do some fencing in the 2-node case if we make it up far enough
	return [
	r"Executing reboot fencing operation",
	r"Requesting fencing $[^)]+$ of node ",
	]

	# Register StopOnebyOne as a good test to run
	AllTestClasses.append(PartialStart)


	class StandbyTest(CTSTest):
	def __init__(self, cm):
	CTSTest.__init__(self,cm)
	self.name = "Standby"
	self.benchmark = 1

	self.start = StartTest(cm)
	self.startall = SimulStartLite(cm)

	# make sure the node is active
	# set the node to standby mode
	# check resources, none resource should be running on the node
	# set the node to active mode
	# check resouces, resources should have been migrated back (SHOULD THEY?)

	def __call__(self, node):

	self.incr("calls")
	ret = self.startall(None)
	if not ret:
	return self.failure("Start all nodes failed")

	self.debug("Make sure node %s is active" % node)
	if self.CM.StandbyStatus(node) != "off":
	if not self.CM.SetStandbyMode(node, "off"):
	return self.failure("can't set node %s to active mode" % node)

	self.CM.cluster_stable()

	status = self.CM.StandbyStatus(node)
	if status != "off":
	return self.failure("standby status of %s is [%s] but we expect [off]" % (node, status))

	self.debug("Getting resources running on node %s" % node)
	rsc_on_node = self.CM.active_resources(node)

	watchpats = []
	watchpats.append(r"State transition .* -> S_POLICY_ENGINE")
	watch = self.create_watch(watchpats, self.Env["DeadTime"]+10)
	watch.setwatch()

	self.debug("Setting node %s to standby mode" % node)
	if not self.CM.SetStandbyMode(node, "on"):
	return self.failure("can't set node %s to standby mode" % node)

	self.set_timer("on")

	ret = watch.lookforall()
	if not ret:
	self.logger.log("Patterns not found: " + repr(watch.unmatched))
	self.CM.SetStandbyMode(node, "off")
	return self.failure("cluster didn't react to standby change on %s" % node)

	self.CM.cluster_stable()

	status = self.CM.StandbyStatus(node)
	if status != "on":
	return self.failure("standby status of %s is [%s] but we expect [on]" % (node, status))
	self.log_timer("on")

	self.debug("Checking resources")
	bad_run = self.CM.active_resources(node)
	if len(bad_run) > 0:
	rc = self.failure("%s set to standby, %s is still running on it" % (node, repr(bad_run)))
	self.debug("Setting node %s to active mode" % node)
	self.CM.SetStandbyMode(node, "off")
	return rc

	self.debug("Setting node %s to active mode" % node)
	if not self.CM.SetStandbyMode(node, "off"):
	return self.failure("can't set node %s to active mode" % node)

	self.set_timer("off")
	self.CM.cluster_stable()

	status = self.CM.StandbyStatus(node)
	if status != "off":
	return self.failure("standby status of %s is [%s] but we expect [off]" % (node, status))
	self.log_timer("off")

	return self.success()

	AllTestClasses.append(StandbyTest)


	class ValgrindTest(CTSTest):
	'''Check for memory leaks'''
	def __init__(self, cm):
	CTSTest.__init__(self,cm)
	self.name = "Valgrind"
	self.stopall = SimulStopLite(cm)
	self.startall = SimulStartLite(cm)
	self.is_valgrind = 1
	self.is_loop = 1

	def setup(self, node):
	self.incr("calls")

	ret = self.stopall(None)
	if not ret:
	return self.failure("Stop all nodes failed")

	# @TODO Edit /etc/sysconfig/pacemaker on all nodes to enable valgrind,
	# and clear any valgrind logs from previous runs. For now, we rely on
	# the user to do this manually.

	ret = self.startall(None)
	if not ret:
	return self.failure("Start all nodes failed")

	return self.success()

	def teardown(self, node):
	# Return all nodes to normal
	# @TODO Edit /etc/sysconfig/pacemaker on all nodes to disable valgrind
	ret = self.stopall(None)
	if not ret:
	return self.failure("Stop all nodes failed")

	return self.success()

	def find_leaks(self):
	# Check for leaks
	# (no longer used but kept in case feature is restored)
	leaked = []
	self.stop = StopTest(self.CM)

	for node in self.Env["nodes"]:
	rc = self.stop(node)
	if not rc:
	self.failure("Couldn't shut down %s" % node)

	rc = self.rsh(node, "grep -e indirectly.lost:.[1-9] -e definitely.lost:.[1-9] -e (ERROR\|error).SUMMARY:.[1-9].*errors %s" % self.logger.logPat, 0)
	if rc != 1:
	leaked.append(node)
	self.failure("Valgrind errors detected on %s" % node)
	(rc, output) = self.rsh(node, "grep -e lost: -e SUMMARY: %s" % self.logger.logPat, None)
	for line in output:
	self.logger.log(line)
	(rc, output) = self.rsh(node, "cat %s" % self.logger.logPat, None)
	for line in output:
	self.debug(line)

	self.rsh(node, "rm -f %s" % self.logger.logPat, None)
	return leaked

	def __call__(self, node):
	#leaked = self.find_leaks()
	#if len(leaked) > 0:
	# return self.failure("Nodes %s leaked" % repr(leaked))

	return self.success()

	def errorstoignore(self):
	'''Return list of errors which should be ignored'''
	return [
	r"pacemaker-based.: \\\\\\\\\\\\\",
	r"pacemaker-based.: . avoid confusing Valgrind",
	r"HA_VALGRIND_ENABLED",
	]


	class StandbyLoopTest(ValgrindTest):
	'''Check for memory leaks by putting a node in and out of standby for an hour'''
	# @TODO This is not a useful test for memory leaks
	def __init__(self, cm):
	ValgrindTest.__init__(self,cm)
	self.name = "StandbyLoop"

	def __call__(self, node):

	lpc = 0
	delay = 2
	failed = 0
	done = time.time() + self.Env["loop-minutes"] * 60
	while time.time() <= done and not failed:
	lpc = lpc + 1

	time.sleep(delay)
	if not self.CM.SetStandbyMode(node, "on"):
	self.failure("can't set node %s to standby mode" % node)
	failed = lpc

	time.sleep(delay)
	if not self.CM.SetStandbyMode(node, "off"):
	self.failure("can't set node %s to active mode" % node)
	failed = lpc

	leaked = self.find_leaks()
	if failed:
	return self.failure("Iteration %d failed" % failed)
	elif len(leaked) > 0:
	return self.failure("Nodes %s leaked" % repr(leaked))

	return self.success()

	#AllTestClasses.append(StandbyLoopTest)


	class BandwidthTest(CTSTest):
	# Tests should not be cluster-manager-specific
	# If you need to find out cluster manager configuration to do this, then
	# it should be added to the generic cluster manager API.
	'''Test the bandwidth which the cluster uses'''
	def __init__(self, cm):
	CTSTest.__init__(self, cm)
	self.name = "Bandwidth"
	self.start = StartTest(cm)
	self.__setitem__("min",0)
	self.__setitem__("max",0)
	self.__setitem__("totalbandwidth",0)
	(handle, self.tempfile) = tempfile.mkstemp(".cts")
	os.close(handle)
	self.startall = SimulStartLite(cm)

	def __call__(self, node):
	'''Perform the Bandwidth test'''
	self.incr("calls")

	if self.CM.upcount() < 1:
	return self.skipped()

	Path = self.CM.InternalCommConfig()
	if "ip" not in Path["mediatype"]:
	return self.skipped()

	port = Path["port"][0]
	port = int(port)

	ret = self.startall(None)
	if not ret:
	return self.failure("Test setup failed")
	time.sleep(5) # We get extra messages right after startup.

	fstmpfile = "/var/run/band_estimate"
	dumpcmd = "tcpdump -p -n -c 102 -i any udp port %d > %s 2>&1" \
	% (port, fstmpfile)

	rc = self.rsh(node, dumpcmd)
	if rc == 0:
	farfile = "root@%s:%s" % (node, fstmpfile)
	self.rsh.cp(farfile, self.tempfile)
	Bandwidth = self.countbandwidth(self.tempfile)
	if not Bandwidth:
	self.logger.log("Could not compute bandwidth.")
	return self.success()
	intband = int(Bandwidth + 0.5)
	self.logger.log("...bandwidth: %d bits/sec" % intband)
	self.Stats["totalbandwidth"] = self.Stats["totalbandwidth"] + Bandwidth
	if self.Stats["min"] == 0:
	self.Stats["min"] = Bandwidth
	if Bandwidth > self.Stats["max"]:
	self.Stats["max"] = Bandwidth
	if Bandwidth < self.Stats["min"]:
	self.Stats["min"] = Bandwidth
	self.rsh(node, "rm -f %s" % fstmpfile)
	os.unlink(self.tempfile)
	return self.success()
	else:
	return self.failure("no response from tcpdump command [%d]!" % rc)

	def countbandwidth(self, file):
	fp = open(file, "r")
	fp.seek(0)
	count = 0
	sum = 0
	while 1:
	line = fp.readline()
	if not line:
	return None
	if re.search("udp",line) or re.search("UDP,", line):
	count = count + 1
	linesplit = line.split(" ")
	for j in range(len(linesplit)-1):
	if linesplit[j] == "udp": break
	if linesplit[j] == "length:": break

	try:
	sum = sum + int(linesplit[j+1])
	except ValueError:
	self.logger.log("Invalid tcpdump line: %s" % line)
	return None
	T1 = linesplit[0]
	timesplit = T1.split(":")
	time2split = timesplit[2].split(".")
	time1 = (int(timesplit[0])60+int(timesplit[1]))60+int(time2split[0])+int(time2split[1])*0.000001
	break

	while count < 100:
	line = fp.readline()
	if not line:
	return None
	if re.search("udp",line) or re.search("UDP,", line):
	count = count+1
	linessplit = line.split(" ")
	for j in range(len(linessplit)-1):
	if linessplit[j] == "udp": break
	if linessplit[j] == "length:": break
	try:
	sum = int(linessplit[j+1]) + sum
	except ValueError:
	self.logger.log("Invalid tcpdump line: %s" % line)
	return None

	T2 = linessplit[0]
	timesplit = T2.split(":")
	time2split = timesplit[2].split(".")
	time2 = (int(timesplit[0])60+int(timesplit[1]))60+int(time2split[0])+int(time2split[1])*0.000001
	time = time2-time1
	if (time <= 0):
	return 0
	return int((sum*8)/time)

	def is_applicable(self):
	'''BandwidthTest never applicable'''
	return 0

	AllTestClasses.append(BandwidthTest)


	###################################################################
	class MaintenanceMode(CTSTest):
	###################################################################
	def __init__(self, cm):
	CTSTest.__init__(self,cm)
	self.name = "MaintenanceMode"
	self.start = StartTest(cm)
	self.startall = SimulStartLite(cm)
	self.max = 30
	#self.is_unsafe = 1
	self.benchmark = 1
	self.action = "asyncmon"
	self.interval = 0
	self.rid = "maintenanceDummy"

	def toggleMaintenanceMode(self, node, action):
	pats = []
	pats.append(self.templates["Pat:DC_IDLE"])

	# fail the resource right after turning Maintenance mode on
	# verify it is not recovered until maintenance mode is turned off
	if action == "On":
	pats.append(self.templates["Pat:RscOpFail"] % (self.action, self.rid))
	else:
	pats.append(self.templates["Pat:RscOpOK"] % ("stop", self.rid))
	pats.append(self.templates["Pat:RscOpOK"] % ("start", self.rid))

	watch = self.create_watch(pats, 60)
	watch.setwatch()

	self.debug("Turning maintenance mode %s" % action)
	self.rsh(node, self.templates["MaintenanceMode%s" % (action)])
	if (action == "On"):
	self.rsh(node, "crm_resource -V -F -r %s -H %s &>/dev/null" % (self.rid, node))

	self.set_timer("recover%s" % (action))
	watch.lookforall()
	self.log_timer("recover%s" % (action))
	if watch.unmatched:
	self.debug("Failed to find patterns when turning maintenance mode %s" % action)
	return repr(watch.unmatched)

	return ""

	def insertMaintenanceDummy(self, node):
	pats = []
	pats.append(("%s.*" % node) + (self.templates["Pat:RscOpOK"] % ("start", self.rid)))

	watch = self.create_watch(pats, 60)
	watch.setwatch()

	self.CM.AddDummyRsc(node, self.rid)

	self.set_timer("addDummy")
	watch.lookforall()
	self.log_timer("addDummy")

	if watch.unmatched:
	self.debug("Failed to find patterns when adding maintenance dummy resource")
	return repr(watch.unmatched)
	return ""

	def removeMaintenanceDummy(self, node):
	pats = []
	pats.append(self.templates["Pat:RscOpOK"] % ("stop", self.rid))

	watch = self.create_watch(pats, 60)
	watch.setwatch()
	self.CM.RemoveDummyRsc(node, self.rid)

	self.set_timer("removeDummy")
	watch.lookforall()
	self.log_timer("removeDummy")

	if watch.unmatched:
	self.debug("Failed to find patterns when removing maintenance dummy resource")
	return repr(watch.unmatched)
	return ""

	def managedRscList(self, node):
	rscList = []
	(rc, lines) = self.rsh(node, "crm_resource -c", None)
	for line in lines:
	if re.search("^Resource", line):
	tmp = AuditResource(self.CM, line)
	if tmp.managed():
	rscList.append(tmp.id)

	return rscList

	def verifyResources(self, node, rscList, managed):
	managedList = list(rscList)
	managed_str = "managed"
	if not managed:
	managed_str = "unmanaged"

	(rc, lines) = self.rsh(node, "crm_resource -c", None)
	for line in lines:
	if re.search("^Resource", line):
	tmp = AuditResource(self.CM, line)
	if managed and not tmp.managed():
	continue
	elif not managed and tmp.managed():
	continue
	elif managedList.count(tmp.id):
	managedList.remove(tmp.id)

	if len(managedList) == 0:
	self.debug("Found all %s resources on %s" % (managed_str, node))
	return True

	self.logger.log("Could not find all %s resources on %s. %s" % (managed_str, node, managedList))
	return False

	def __call__(self, node):
	'''Perform the 'MaintenanceMode' test. '''
	self.incr("calls")
	verify_managed = False
	verify_unmanaged = False
	failPat = ""

	ret = self.startall(None)
	if not ret:
	return self.failure("Setup failed")

	# get a list of all the managed resources. We use this list
	# after enabling maintenance mode to verify all managed resources
	# become un-managed. After maintenance mode is turned off, we use
	# this list to verify all the resources become managed again.
	managedResources = self.managedRscList(node)
	if len(managedResources) == 0:
	self.logger.log("No managed resources on %s" % node)
	return self.skipped()

	# insert a fake resource we can fail during maintenance mode
	# so we can verify recovery does not take place until after maintenance
	# mode is disabled.
	failPat = failPat + self.insertMaintenanceDummy(node)

	# toggle maintenance mode ON, then fail dummy resource.
	failPat = failPat + self.toggleMaintenanceMode(node, "On")

	# verify all the resources are now unmanaged
	if self.verifyResources(node, managedResources, False):
	verify_unmanaged = True

	# Toggle maintenance mode OFF, verify dummy is recovered.
	failPat = failPat + self.toggleMaintenanceMode(node, "Off")

	# verify all the resources are now managed again
	if self.verifyResources(node, managedResources, True):
	verify_managed = True

	# Remove our maintenance dummy resource.
	failPat = failPat + self.removeMaintenanceDummy(node)

	self.CM.cluster_stable()

	if failPat != "":
	return self.failure("Unmatched patterns: %s" % (failPat))
	elif verify_unmanaged is False:
	return self.failure("Failed to verify resources became unmanaged during maintenance mode")
	elif verify_managed is False:
	return self.failure("Failed to verify resources switched back to managed after disabling maintenance mode")

	return self.success()

	def errorstoignore(self):
	'''Return list of errors which should be ignored'''
	return [
	r"Updating failcount for %s" % self.rid,
	r"schedulerd.: Recover %s\s$.*$" % self.rid,
	r"Unknown operation: fail",
	self.templates["Pat:RscOpOK"] % (self.action, self.rid),
	r"(ERROR\|error).: Action %s_%s_%d . initiated outside of a transition" % (self.rid, self.action, self.interval),
	]

	AllTestClasses.append(MaintenanceMode)


	class ResourceRecover(CTSTest):
	def __init__(self, cm):
	CTSTest.__init__(self,cm)
	self.name = "ResourceRecover"
	self.start = StartTest(cm)
	self.startall = SimulStartLite(cm)
	self.max = 30
	self.rid = None
	self.rid_alt = None
	#self.is_unsafe = 1
	self.benchmark = 1

	# these are the values used for the new LRM API call
	self.action = "asyncmon"
	self.interval = 0

	def __call__(self, node):
	'''Perform the 'ResourceRecover' test. '''
	self.incr("calls")

	ret = self.startall(None)
	if not ret:
	return self.failure("Setup failed")

	resourcelist = self.CM.active_resources(node)
	# if there are no resourcelist, return directly
	if len(resourcelist) == 0:
	self.logger.log("No active resources on %s" % node)
	return self.skipped()

	self.rid = self.Env.RandomGen.choice(resourcelist)
	self.rid_alt = self.rid

	rsc = None
	(rc, lines) = self.rsh(node, "crm_resource -c", None)
	for line in lines:
	if re.search("^Resource", line):
	tmp = AuditResource(self.CM, line)
	if tmp.id == self.rid:
	rsc = tmp
	# Handle anonymous clones that get renamed
	self.rid = rsc.clone_id
	break

	if not rsc:
	return self.failure("Could not find %s in the resource list" % self.rid)

	self.debug("Shooting %s aka. %s" % (rsc.clone_id, rsc.id))

	pats = []
	pats.append(self.templates["Pat:CloneOpFail"] % (self.action, rsc.id, rsc.clone_id))

	if rsc.managed():
	pats.append(self.templates["Pat:RscOpOK"] % ("stop", self.rid))
	if rsc.unique():
	pats.append(self.templates["Pat:RscOpOK"] % ("start", self.rid))
	else:
	# Anonymous clones may get restarted with a different clone number
	pats.append(self.templates["Pat:RscOpOK"] % ("start", ".*"))

	watch = self.create_watch(pats, 60)
	watch.setwatch()

	self.rsh(node, "crm_resource -V -F -r %s -H %s &>/dev/null" % (self.rid, node))

	self.set_timer("recover")
	watch.lookforall()
	self.log_timer("recover")

	self.CM.cluster_stable()
	recovered = self.CM.ResourceLocation(self.rid)

	if watch.unmatched:
	return self.failure("Patterns not found: %s" % repr(watch.unmatched))

	elif rsc.unique() and len(recovered) > 1:
	return self.failure("%s is now active on more than one node: %s"%(self.rid, repr(recovered)))

	elif len(recovered) > 0:
	self.debug("%s is running on: %s" % (self.rid, repr(recovered)))

	elif rsc.managed():
	return self.failure("%s was not recovered and is inactive" % self.rid)

	return self.success()

	def errorstoignore(self):
	'''Return list of errors which should be ignored'''
	return [
	r"Updating failcount for %s" % self.rid,
	r"schedulerd.: Recover (%s\|%s)\s$.*$" % (self.rid, self.rid_alt),
	r"Unknown operation: fail",
	self.templates["Pat:RscOpOK"] % (self.action, self.rid),
	r"(ERROR\|error).: Action %s_%s_%d . initiated outside of a transition" % (self.rid, self.action, self.interval),
	]

	AllTestClasses.append(ResourceRecover)


	class ComponentFail(CTSTest):
	def __init__(self, cm):
	CTSTest.__init__(self,cm)
	self.name = "ComponentFail"
	# TODO make this work correctly in docker.
	self.is_docker_unsafe = 1
	self.startall = SimulStartLite(cm)
	self.complist = cm.Components()
	self.patterns = []
	self.okerrpatterns = []
	self.is_unsafe = 1

	def __call__(self, node):
	'''Perform the 'ComponentFail' test. '''
	self.incr("calls")
	self.patterns = []
	self.okerrpatterns = []

	# start all nodes
	ret = self.startall(None)
	if not ret:
	return self.failure("Setup failed")

	if not self.CM.cluster_stable(self.Env["StableTime"]):
	return self.failure("Setup failed - unstable")

	node_is_dc = self.CM.is_node_dc(node, None)

	# select a component to kill
	chosen = self.Env.RandomGen.choice(self.complist)
	while chosen.dc_only == 1 and node_is_dc == 0:
	chosen = self.Env.RandomGen.choice(self.complist)

	self.debug("...component %s (dc=%d,boot=%d)" % (chosen.name, node_is_dc,chosen.triggersreboot))
	self.incr(chosen.name)

	if chosen.name != "corosync":
	self.patterns.append(self.templates["Pat:ChildKilled"] %(node, chosen.name))
	self.patterns.append(self.templates["Pat:ChildRespawn"] %(node, chosen.name))

	self.patterns.extend(chosen.pats)
	if node_is_dc:
	self.patterns.extend(chosen.dc_pats)

	# @TODO this should be a flag in the Component
	if chosen.name in [ "corosync", "pacemaker-based", "pacemaker-fenced" ]:
	# Ignore actions for fence devices if fencer will respawn
	# (their registration will be lost, and probes will fail)
	self.okerrpatterns = [ self.templates["Pat:Fencing_active"] ]
	(rc, lines) = self.rsh(node, "crm_resource -c", None)
	for line in lines:
	if re.search("^Resource", line):
	r = AuditResource(self.CM, line)
	if r.rclass == "stonith":
	self.okerrpatterns.append(self.templates["Pat:Fencing_recover"] % r.id)
	self.okerrpatterns.append(self.templates["Pat:Fencing_probe"] % r.id)

	# supply a copy so self.patterns doesn't end up empty
	tmpPats = []
	tmpPats.extend(self.patterns)
	self.patterns.extend(chosen.badnews_ignore)

	# Look for STONITH ops, depending on Env["at-boot"] we might need to change the nodes status
	stonithPats = []
	stonithPats.append(self.templates["Pat:Fencing_ok"] % node)
	stonith = self.create_watch(stonithPats, 0)
	stonith.setwatch()

	# set the watch for stable
	watch = self.create_watch(
	tmpPats, self.Env["DeadTime"] + self.Env["StableTime"] + self.Env["StartTime"])
	watch.setwatch()

	# kill the component
	chosen.kill(node)

	self.debug("Waiting for the cluster to recover")
	self.CM.cluster_stable()

	self.debug("Waiting for any fenced node to come back up")
	self.CM.ns.WaitForAllNodesToComeUp(self.Env["nodes"], 600)

	self.debug("Waiting for the cluster to re-stabilize with all nodes")
	self.CM.cluster_stable(self.Env["StartTime"])

	self.debug("Checking if %s was shot" % node)
	shot = stonith.look(60)
	if shot:
	self.debug("Found: " + repr(shot))
	self.okerrpatterns.append(self.templates["Pat:Fencing_start"] % node)

	if self.Env["at-boot"] == 0:
	self.CM.ShouldBeStatus[node] = "down"

	# If fencing occurred, chances are many (if not all) the expected logs
	# will not be sent - or will be lost when the node reboots
	return self.success()

	# check for logs indicating a graceful recovery
	matched = watch.lookforall(allow_multiple_matches=1)
	if watch.unmatched:
	self.logger.log("Patterns not found: " + repr(watch.unmatched))

	self.debug("Waiting for the cluster to re-stabilize with all nodes")
	is_stable = self.CM.cluster_stable(self.Env["StartTime"])

	if not matched:
	return self.failure("Didn't find all expected %s patterns" % chosen.name)
	elif not is_stable:
	return self.failure("Cluster did not become stable after killing %s" % chosen.name)

	return self.success()

	def errorstoignore(self):
	'''Return list of errors which should be ignored'''
	# Note that okerrpatterns refers to the last time we ran this test
	# The good news is that this works fine for us...
	self.okerrpatterns.extend(self.patterns)
	return self.okerrpatterns

	AllTestClasses.append(ComponentFail)


	class SplitBrainTest(CTSTest):
	'''It is used to test split-brain. when the path between the two nodes break
	check the two nodes both take over the resource'''
	def __init__(self,cm):
	CTSTest.__init__(self,cm)
	self.name = "SplitBrain"
	self.start = StartTest(cm)
	self.startall = SimulStartLite(cm)
	self.is_experimental = 1

	def isolate_partition(self, partition):
	other_nodes = []
	other_nodes.extend(self.Env["nodes"])

	for node in partition:
	try:
	other_nodes.remove(node)
	except ValueError:
	self.logger.log("Node "+node+" not in " + repr(self.Env["nodes"]) + " from " +repr(partition))

	if len(other_nodes) == 0:
	return 1

	self.debug("Creating partition: " + repr(partition))
	self.debug("Everyone else: " + repr(other_nodes))

	for node in partition:
	if not self.CM.isolate_node(node, other_nodes):
	self.logger.log("Could not isolate %s" % node)
	return 0

	return 1

	def heal_partition(self, partition):
	other_nodes = []
	other_nodes.extend(self.Env["nodes"])

	for node in partition:
	try:
	other_nodes.remove(node)
	except ValueError:
	self.logger.log("Node "+node+" not in " + repr(self.Env["nodes"]))

	if len(other_nodes) == 0:
	return 1

	self.debug("Healing partition: " + repr(partition))
	self.debug("Everyone else: " + repr(other_nodes))

	for node in partition:
	self.CM.unisolate_node(node, other_nodes)

	def __call__(self, node):
	'''Perform split-brain test'''
	self.incr("calls")
	self.passed = 1
	partitions = {}

	ret = self.startall(None)
	if not ret:
	return self.failure("Setup failed")

	while 1:
	# Retry until we get multiple partitions
	partitions = {}
	p_max = len(self.Env["nodes"])
	for node in self.Env["nodes"]:
	p = self.Env.RandomGen.randint(1, p_max)
	if not p in partitions:
	partitions[p] = []
	partitions[p].append(node)
	p_max = len(list(partitions.keys()))
	if p_max > 1:
	break
	# else, try again

	self.debug("Created %d partitions" % p_max)
	for key in list(partitions.keys()):
	self.debug("Partition["+str(key)+"]:\t"+repr(partitions[key]))

	# Disabling STONITH to reduce test complexity for now
	self.rsh(node, "crm_attribute -V -n stonith-enabled -v false")

	for key in list(partitions.keys()):
	self.isolate_partition(partitions[key])

	count = 30
	while count > 0:
	if len(self.CM.find_partitions()) != p_max:
	time.sleep(10)
	else:
	break
	else:
	self.failure("Expected partitions were not created")

	# Target number of partitions formed - wait for stability
	if not self.CM.cluster_stable():
	self.failure("Partitioned cluster not stable")

	# Now audit the cluster state
	self.CM.partitions_expected = p_max
	if not self.audit():
	self.failure("Audits failed")
	self.CM.partitions_expected = 1

	# And heal them again
	for key in list(partitions.keys()):
	self.heal_partition(partitions[key])

	# Wait for a single partition to form
	count = 30
	while count > 0:
	if len(self.CM.find_partitions()) != 1:
	time.sleep(10)
	count -= 1
	else:
	break
	else:
	self.failure("Cluster did not reform")

	# Wait for it to have the right number of members
	count = 30
	while count > 0:
	members = []

	partitions = self.CM.find_partitions()
	if len(partitions) > 0:
	members = partitions[0].split()

	if len(members) != len(self.Env["nodes"]):
	time.sleep(10)
	count -= 1
	else:
	break
	else:
	self.failure("Cluster did not completely reform")

	# Wait up to 20 minutes - the delay is more preferable than
	# trying to continue with in a messed up state
	if not self.CM.cluster_stable(1200):
	self.failure("Reformed cluster not stable")
	if self.Env["continue"] == 1:
	answer = "Y"
	else:
	try:
	- answer = input_wrapper('Continue? [nY]')
	+ answer = input('Continue? [nY]')
	except EOFError as e:
	answer = "n"
	if answer and answer == "n":
	raise ValueError("Reformed cluster not stable")

	# Turn fencing back on
	if self.Env["DoFencing"]:
	self.rsh(node, "crm_attribute -V -D -n stonith-enabled")

	self.CM.cluster_stable()

	if self.passed:
	return self.success()
	return self.failure("See previous errors")

	def errorstoignore(self):
	'''Return list of errors which are 'normal' and should be ignored'''
	return [
	r"Another DC detected:",
	r"(ERROR\|error).: .Application of an update diff failed",
	r"pacemaker-controld.:.not in our membership list",
	r"CRIT:.node.returning after partition",
	]

	def is_applicable(self):
	if not self.is_applicable_common():
	return 0
	return len(self.Env["nodes"]) > 2

	AllTestClasses.append(SplitBrainTest)


	class Reattach(CTSTest):
	def __init__(self, cm):
	CTSTest.__init__(self,cm)
	self.name = "Reattach"
	self.startall = SimulStartLite(cm)
	self.restart1 = RestartTest(cm)
	self.stopall = SimulStopLite(cm)
	self.is_unsafe = 0 # Handled by canrunnow()

	def _is_managed(self, node):
	is_managed = self.rsh(node, "crm_attribute -t rsc_defaults -n is-managed -q -G -d true", 1)
	is_managed = is_managed[:-1] # Strip off the newline
	return is_managed == "true"

	def _set_unmanaged(self, node):
	self.debug("Disable resource management")
	self.rsh(node, "crm_attribute -t rsc_defaults -n is-managed -v false")

	def _set_managed(self, node):
	self.debug("Re-enable resource management")
	self.rsh(node, "crm_attribute -t rsc_defaults -n is-managed -D")

	def setup(self, node):
	attempt = 0
	if not self.startall(None):
	return None

	# Make sure we are really _really_ stable and that all
	# resources, including those that depend on transient node
	# attributes, are started
	while not self.CM.cluster_stable(double_check=True):
	if attempt < 5:
	attempt += 1
	self.debug("Not stable yet, re-testing")
	else:
	self.logger.log("Cluster is not stable")
	return None

	return 1

	def teardown(self, node):

	# Make sure 'node' is up
	start = StartTest(self.CM)
	start(node)

	if not self._is_managed(node):
	self.logger.log("Attempting to re-enable resource management on %s" % node)
	self._set_managed(node)
	self.CM.cluster_stable()
	if not self._is_managed(node):
	self.logger.log("Could not re-enable resource management")
	return 0

	return 1

	def canrunnow(self, node):
	'''Return TRUE if we can meaningfully run right now'''
	if self.find_ocfs2_resources(node):
	self.logger.log("Detach/Reattach scenarios are not possible with OCFS2 services present")
	return 0
	return 1

	def __call__(self, node):
	self.incr("calls")

	pats = []
	# Conveniently, the scheduler will display this message when disabling
	# management, even if fencing is not enabled, so we can rely on it.
	managed = self.create_watch(["Delaying fencing operations"], 60)
	managed.setwatch()

	self._set_unmanaged(node)

	if not managed.lookforall():
	self.logger.log("Patterns not found: " + repr(managed.unmatched))
	return self.failure("Resource management not disabled")

	pats = []
	pats.append(self.templates["Pat:RscOpOK"] % ("start", ".*"))
	pats.append(self.templates["Pat:RscOpOK"] % ("stop", ".*"))
	pats.append(self.templates["Pat:RscOpOK"] % ("promote", ".*"))
	pats.append(self.templates["Pat:RscOpOK"] % ("demote", ".*"))
	pats.append(self.templates["Pat:RscOpOK"] % ("migrate", ".*"))

	watch = self.create_watch(pats, 60, "ShutdownActivity")
	watch.setwatch()

	self.debug("Shutting down the cluster")
	ret = self.stopall(None)
	if not ret:
	self._set_managed(node)
	return self.failure("Couldn't shut down the cluster")

	self.debug("Bringing the cluster back up")
	ret = self.startall(None)
	time.sleep(5) # allow ping to update the CIB
	if not ret:
	self._set_managed(node)
	return self.failure("Couldn't restart the cluster")

	if self.local_badnews("ResourceActivity:", watch):
	self._set_managed(node)
	return self.failure("Resources stopped or started during cluster restart")

	watch = self.create_watch(pats, 60, "StartupActivity")
	watch.setwatch()

	# Re-enable resource management (and verify it happened).
	self._set_managed(node)
	self.CM.cluster_stable()
	if not self._is_managed(node):
	return self.failure("Could not re-enable resource management")

	# Ignore actions for STONITH resources
	ignore = []
	(rc, lines) = self.rsh(node, "crm_resource -c", None)
	for line in lines:
	if re.search("^Resource", line):
	r = AuditResource(self.CM, line)
	if r.rclass == "stonith":

	self.debug("Ignoring start actions for %s" % r.id)
	ignore.append(self.templates["Pat:RscOpOK"] % ("start", r.id))

	if self.local_badnews("ResourceActivity:", watch, ignore):
	return self.failure("Resources stopped or started after resource management was re-enabled")

	return ret

	def errorstoignore(self):
	'''Return list of errors which should be ignored'''
	return [
	r"resource( was\|s were) active at shutdown",
	]

	def is_applicable(self):
	return 1

	AllTestClasses.append(Reattach)


	class SpecialTest1(CTSTest):
	'''Set up a custom test to cause quorum failure issues for Andrew'''
	def __init__(self, cm):
	CTSTest.__init__(self,cm)
	self.name = "SpecialTest1"
	self.startall = SimulStartLite(cm)
	self.restart1 = RestartTest(cm)
	self.stopall = SimulStopLite(cm)

	def __call__(self, node):
	'''Perform the 'SpecialTest1' test for Andrew. '''
	self.incr("calls")

	# Shut down all the nodes...
	ret = self.stopall(None)
	if not ret:
	return self.failure("Could not stop all nodes")

	# Test config recovery when the other nodes come up
	self.rsh(node, "rm -f "+CTSvars.CRM_CONFIG_DIR+"/cib*")

	# Start the selected node
	ret = self.restart1(node)
	if not ret:
	return self.failure("Could not start "+node)

	# Start all remaining nodes
	ret = self.startall(None)
	if not ret:
	return self.failure("Could not start the remaining nodes")

	return self.success()

	def errorstoignore(self):
	'''Return list of errors which should be ignored'''
	# Errors that occur as a result of the CIB being wiped
	return [
	r"error.*: v1 patchset error, patch failed to apply: Application of an update diff failed",
	r"error.*: Resource start-up disabled since no STONITH resources have been defined",
	r"error.*: Either configure some or disable STONITH with the stonith-enabled option",
	r"error.*: NOTE: Clusters with shared data need STONITH to ensure data integrity",
	]

	AllTestClasses.append(SpecialTest1)


	class HAETest(CTSTest):
	'''Set up a custom test to cause quorum failure issues for Andrew'''
	def __init__(self, cm):
	CTSTest.__init__(self,cm)
	self.name = "HAETest"
	self.stopall = SimulStopLite(cm)
	self.startall = SimulStartLite(cm)
	self.is_loop = 1

	def setup(self, node):
	# Start all remaining nodes
	ret = self.startall(None)
	if not ret:
	return self.failure("Couldn't start all nodes")
	return self.success()

	def teardown(self, node):
	# Stop everything
	ret = self.stopall(None)
	if not ret:
	return self.failure("Couldn't stop all nodes")
	return self.success()

	def wait_on_state(self, node, resource, expected_clones, attempts=240):
	while attempts > 0:
	active = 0
	(rc, lines) = self.rsh(node, "crm_resource -r %s -W -Q" % resource, stdout=None)

	# Hack until crm_resource does the right thing
	if rc == 0 and lines:
	active = len(lines)

	if len(lines) == expected_clones:
	return 1

	elif rc == 1:
	self.debug("Resource %s is still inactive" % resource)

	elif rc == 234:
	self.logger.log("Unknown resource %s" % resource)
	return 0

	elif rc == 246:
	self.logger.log("Cluster is inactive")
	return 0

	elif rc != 0:
	self.logger.log("Call to crm_resource failed, rc=%d" % rc)
	return 0

	else:
	self.debug("Resource %s is active on %d times instead of %d" % (resource, active, expected_clones))

	attempts -= 1
	time.sleep(1)

	return 0

	def find_dlm(self, node):
	self.r_dlm = None

	(rc, lines) = self.rsh(node, "crm_resource -c", None)
	for line in lines:
	if re.search("^Resource", line):
	r = AuditResource(self.CM, line)
	if r.rtype == "controld" and r.parent != "NA":
	self.debug("Found dlm: %s" % self.r_dlm)
	self.r_dlm = r.parent
	return 1
	return 0

	def find_hae_resources(self, node):
	self.r_dlm = None
	self.r_o2cb = None
	self.r_ocfs2 = []

	if self.find_dlm(node):
	self.find_ocfs2_resources(node)

	def is_applicable(self):
	if not self.is_applicable_common():
	return 0
	if self.Env["Schema"] == "hae":
	return 1
	return None


	class HAERoleTest(HAETest):
	def __init__(self, cm):
	'''Lars' mount/unmount test for the HA extension. '''
	HAETest.__init__(self,cm)
	self.name = "HAERoleTest"

	def change_state(self, node, resource, target):
	rc = self.rsh(node, "crm_resource -V -r %s -p target-role -v %s --meta" % (resource, target))
	return rc

	def __call__(self, node):
	self.incr("calls")
	lpc = 0
	failed = 0
	delay = 2
	done = time.time() + self.Env["loop-minutes"]*60
	self.find_hae_resources(node)

	clone_max = len(self.Env["nodes"])
	while time.time() <= done and not failed:
	lpc = lpc + 1

	self.change_state(node, self.r_dlm, "Stopped")
	if not self.wait_on_state(node, self.r_dlm, 0):
	self.failure("%s did not go down correctly" % self.r_dlm)
	failed = lpc

	self.change_state(node, self.r_dlm, "Started")
	if not self.wait_on_state(node, self.r_dlm, clone_max):
	self.failure("%s did not come up correctly" % self.r_dlm)
	failed = lpc

	if not self.wait_on_state(node, self.r_o2cb, clone_max):
	self.failure("%s did not come up correctly" % self.r_o2cb)
	failed = lpc

	for fs in self.r_ocfs2:
	if not self.wait_on_state(node, fs, clone_max):
	self.failure("%s did not come up correctly" % fs)
	failed = lpc

	if failed:
	return self.failure("iteration %d failed" % failed)
	return self.success()

	AllTestClasses.append(HAERoleTest)


	class HAEStandbyTest(HAETest):
	'''Set up a custom test to cause quorum failure issues for Andrew'''
	def __init__(self, cm):
	HAETest.__init__(self,cm)
	self.name = "HAEStandbyTest"

	def change_state(self, node, resource, target):
	rc = self.rsh(node, "crm_standby -V -l reboot -v %s" % (target))
	return rc

	def __call__(self, node):
	self.incr("calls")

	lpc = 0
	failed = 0
	done = time.time() + self.Env["loop-minutes"]*60
	self.find_hae_resources(node)

	clone_max = len(self.Env["nodes"])
	while time.time() <= done and not failed:
	lpc = lpc + 1

	self.change_state(node, self.r_dlm, "true")
	if not self.wait_on_state(node, self.r_dlm, clone_max-1):
	self.failure("%s did not go down correctly" % self.r_dlm)
	failed = lpc

	self.change_state(node, self.r_dlm, "false")
	if not self.wait_on_state(node, self.r_dlm, clone_max):
	self.failure("%s did not come up correctly" % self.r_dlm)
	failed = lpc

	if not self.wait_on_state(node, self.r_o2cb, clone_max):
	self.failure("%s did not come up correctly" % self.r_o2cb)
	failed = lpc

	for fs in self.r_ocfs2:
	if not self.wait_on_state(node, fs, clone_max):
	self.failure("%s did not come up correctly" % fs)
	failed = lpc

	if failed:
	return self.failure("iteration %d failed" % failed)
	return self.success()

	AllTestClasses.append(HAEStandbyTest)


	class NearQuorumPointTest(CTSTest):
	'''
	This test brings larger clusters near the quorum point (50%).
	In addition, it will test doing starts and stops at the same time.

	Here is how I think it should work:
	- loop over the nodes and decide randomly which will be up and which
	will be down Use a 50% probability for each of up/down.
	- figure out what to do to get into that state from the current state
	- in parallel, bring up those going up and bring those going down.
	'''

	def __init__(self, cm):
	CTSTest.__init__(self,cm)
	self.name = "NearQuorumPoint"

	def __call__(self, dummy):
	'''Perform the 'NearQuorumPoint' test. '''
	self.incr("calls")
	startset = []
	stopset = []

	stonith = self.CM.prepare_fencing_watcher("NearQuorumPoint")
	#decide what to do with each node
	for node in self.Env["nodes"]:
	action = self.Env.RandomGen.choice(["start","stop"])
	#action = self.Env.RandomGen.choice(["start","stop","no change"])
	if action == "start" :
	startset.append(node)
	elif action == "stop" :
	stopset.append(node)

	self.debug("start nodes:" + repr(startset))
	self.debug("stop nodes:" + repr(stopset))

	#add search patterns
	watchpats = [ ]
	for node in stopset:
	if self.CM.ShouldBeStatus[node] == "up":
	watchpats.append(self.templates["Pat:We_stopped"] % node)

	for node in startset:
	if self.CM.ShouldBeStatus[node] == "down":
	#watchpats.append(self.templates["Pat:NonDC_started"] % node)
	watchpats.append(self.templates["Pat:Local_started"] % node)
	else:
	for stopping in stopset:
	if self.CM.ShouldBeStatus[stopping] == "up":
	watchpats.append(self.templates["Pat:They_stopped"] % (node, self.CM.key_for_node(stopping)))

	if len(watchpats) == 0:
	return self.skipped()

	if len(startset) != 0:
	watchpats.append(self.templates["Pat:DC_IDLE"])

	watch = self.create_watch(watchpats, self.Env["DeadTime"]+10)

	watch.setwatch()

	#begin actions
	for node in stopset:
	if self.CM.ShouldBeStatus[node] == "up":
	self.CM.StopaCMnoBlock(node)

	for node in startset:
	if self.CM.ShouldBeStatus[node] == "down":
	self.CM.StartaCMnoBlock(node)

	#get the result
	if watch.lookforall():
	self.CM.cluster_stable()
	self.CM.fencing_cleanup("NearQuorumPoint", stonith)
	return self.success()

	self.logger.log("Warn: Patterns not found: " + repr(watch.unmatched))

	#get the "bad" nodes
	upnodes = []
	for node in stopset:
	if self.CM.StataCM(node) == 1:
	upnodes.append(node)

	downnodes = []
	for node in startset:
	if self.CM.StataCM(node) == 0:
	downnodes.append(node)

	self.CM.fencing_cleanup("NearQuorumPoint", stonith)
	if upnodes == [] and downnodes == []:
	self.CM.cluster_stable()

	# Make sure they're completely down with no residule
	for node in stopset:
	self.rsh(node, self.templates["StopCmd"])

	return self.success()

	if len(upnodes) > 0:
	self.logger.log("Warn: Unstoppable nodes: " + repr(upnodes))

	if len(downnodes) > 0:
	self.logger.log("Warn: Unstartable nodes: " + repr(downnodes))

	return self.failure()

	def is_applicable(self):
	return 1

	AllTestClasses.append(NearQuorumPointTest)


	class RollingUpgradeTest(CTSTest):
	'''Perform a rolling upgrade of the cluster'''
	def __init__(self, cm):
	CTSTest.__init__(self,cm)
	self.name = "RollingUpgrade"
	self.start = StartTest(cm)
	self.stop = StopTest(cm)
	self.stopall = SimulStopLite(cm)
	self.startall = SimulStartLite(cm)

	def setup(self, node):
	# Start all remaining nodes
	ret = self.stopall(None)
	if not ret:
	return self.failure("Couldn't stop all nodes")

	for node in self.Env["nodes"]:
	if not self.downgrade(node, None):
	return self.failure("Couldn't downgrade %s" % node)

	ret = self.startall(None)
	if not ret:
	return self.failure("Couldn't start all nodes")
	return self.success()

	def teardown(self, node):
	# Stop everything
	ret = self.stopall(None)
	if not ret:
	return self.failure("Couldn't stop all nodes")

	for node in self.Env["nodes"]:
	if not self.upgrade(node, None):
	return self.failure("Couldn't upgrade %s" % node)

	return self.success()

	def install(self, node, version, start=1, flags="--force"):

	target_dir = "/tmp/rpm-%s" % version
	src_dir = "%s/%s" % (self.Env["rpm-dir"], version)

	self.logger.log("Installing %s on %s with %s" % (version, node, flags))
	if not self.stop(node):
	return self.failure("stop failure: "+node)

	rc = self.rsh(node, "mkdir -p %s" % target_dir)
	rc = self.rsh(node, "rm -f %s/*.rpm" % target_dir)
	(rc, lines) = self.rsh(node, "ls -1 %s/*.rpm" % src_dir, None)
	for line in lines:
	line = line[:-1]
	rc = self.rsh.cp("%s" % (line), "%s:%s/" % (node, target_dir))
	rc = self.rsh(node, "rpm -Uvh %s %s/*.rpm" % (flags, target_dir))

	if start and not self.start(node):
	return self.failure("start failure: "+node)

	return self.success()

	def upgrade(self, node, start=1):
	return self.install(node, self.Env["current-version"], start)

	def downgrade(self, node, start=1):
	return self.install(node, self.Env["previous-version"], start, "--force --nodeps")

	def __call__(self, node):
	'''Perform the 'Rolling Upgrade' test. '''
	self.incr("calls")

	for node in self.Env["nodes"]:
	if self.upgrade(node):
	return self.failure("Couldn't upgrade %s" % node)

	self.CM.cluster_stable()

	return self.success()

	def is_applicable(self):
	if not self.is_applicable_common():
	return None

	if not "rpm-dir" in list(self.Env.keys()):
	return None
	if not "current-version" in list(self.Env.keys()):
	return None
	if not "previous-version" in list(self.Env.keys()):
	return None

	return 1

	# Register RestartTest as a good test to run
	AllTestClasses.append(RollingUpgradeTest)


	class BSC_AddResource(CTSTest):
	'''Add a resource to the cluster'''
	def __init__(self, cm):
	CTSTest.__init__(self, cm)
	self.name = "AddResource"
	self.resource_offset = 0
	self.cib_cmd = """cibadmin -C -o %s -X '%s' """

	def __call__(self, node):
	self.incr("calls")
	self.resource_offset = self.resource_offset + 1

	r_id = "bsc-rsc-%s-%d" % (node, self.resource_offset)
	start_pat = "pacemaker-controld.%s_start_0.confirmed.*ok"

	patterns = []
	patterns.append(start_pat % r_id)

	watch = self.create_watch(patterns, self.Env["DeadTime"])
	watch.setwatch()

	ip = self.NextIP()
	if not self.make_ip_resource(node, r_id, "ocf", "IPaddr", ip):
	return self.failure("Make resource %s failed" % r_id)

	failed = 0
	watch_result = watch.lookforall()
	if watch.unmatched:
	for regex in watch.unmatched:
	self.logger.log ("Warn: Pattern not found: %s" % (regex))
	failed = 1

	if failed:
	return self.failure("Resource pattern(s) not found")

	if not self.CM.cluster_stable(self.Env["DeadTime"]):
	return self.failure("Unstable cluster")

	return self.success()

	def NextIP(self):
	ip = self.Env["IPBase"]
	if ":" in ip:
	fields = ip.rpartition(":")
	fields[2] = str(hex(int(fields[2], 16)+1))
	print(str(hex(int(f[2], 16)+1)))
	else:
	fields = ip.rpartition('.')
	fields[2] = str(int(fields[2])+1)

	ip = fields[0] + fields[1] + fields[3];
	self.Env["IPBase"] = ip
	return ip.strip()

	def make_ip_resource(self, node, id, rclass, type, ip):
	self.logger.log("Creating %s:%s:%s (%s) on %s" % (rclass,type,id,ip,node))
	rsc_xml="""
	<primitive id="%s" class="%s" type="%s" provider="heartbeat">
	<instance_attributes id="%s"><attributes>
	<nvpair id="%s" name="ip" value="%s"/>
	</attributes></instance_attributes>
	</primitive>""" % (id, rclass, type, id, id, ip)

	node_constraint = """
	<rsc_location id="run_%s" rsc="%s">
	<rule id="pref_run_%s" score="100">
	<expression id="%s_loc_expr" attribute="#uname" operation="eq" value="%s"/>
	</rule>
	</rsc_location>""" % (id, id, id, id, node)

	rc = 0
	(rc, lines) = self.rsh(node, self.cib_cmd % ("constraints", node_constraint), None)
	if rc != 0:
	self.logger.log("Constraint creation failed: %d" % rc)
	return None

	(rc, lines) = self.rsh(node, self.cib_cmd % ("resources", rsc_xml), None)
	if rc != 0:
	self.logger.log("Resource creation failed: %d" % rc)
	return None

	return 1

	def is_applicable(self):
	if self.Env["DoBSC"]:
	return 1
	return None

	AllTestClasses.append(BSC_AddResource)


	class SimulStopLite(CTSTest):
	'''Stop any active nodes ~ simultaneously'''
	def __init__(self, cm):
	CTSTest.__init__(self,cm)
	self.name = "SimulStopLite"

	def __call__(self, dummy):
	'''Perform the 'SimulStopLite' setup work. '''
	self.incr("calls")

	self.debug("Setup: " + self.name)

	# We ignore the "node" parameter...
	watchpats = [ ]

	for node in self.Env["nodes"]:
	if self.CM.ShouldBeStatus[node] == "up":
	self.incr("WasStarted")
	watchpats.append(self.templates["Pat:We_stopped"] % node)

	if len(watchpats) == 0:
	return self.success()

	# Stop all the nodes - at about the same time...
	watch = self.create_watch(watchpats, self.Env["DeadTime"]+10)

	watch.setwatch()
	self.set_timer()
	for node in self.Env["nodes"]:
	if self.CM.ShouldBeStatus[node] == "up":
	self.CM.StopaCMnoBlock(node)
	if watch.lookforall():
	# Make sure they're completely down with no residule
	for node in self.Env["nodes"]:
	self.rsh(node, self.templates["StopCmd"])

	return self.success()

	did_fail = 0
	up_nodes = []
	for node in self.Env["nodes"]:
	if self.CM.StataCM(node) == 1:
	did_fail = 1
	up_nodes.append(node)

	if did_fail:
	return self.failure("Active nodes exist: " + repr(up_nodes))

	self.logger.log("Warn: All nodes stopped but CTS didn't detect: "
	+ repr(watch.unmatched))

	return self.failure("Missing log message: "+repr(watch.unmatched))

	def is_applicable(self):
	'''SimulStopLite is a setup test and never applicable'''
	return 0


	class SimulStartLite(CTSTest):
	'''Start any stopped nodes ~ simultaneously'''
	def __init__(self, cm):
	CTSTest.__init__(self,cm)
	self.name = "SimulStartLite"

	def __call__(self, dummy):
	'''Perform the 'SimulStartList' setup work. '''
	self.incr("calls")
	self.debug("Setup: " + self.name)

	# We ignore the "node" parameter...
	node_list = []
	for node in self.Env["nodes"]:
	if self.CM.ShouldBeStatus[node] == "down":
	self.incr("WasStopped")
	node_list.append(node)

	self.set_timer()
	while len(node_list) > 0:
	# Repeat until all nodes come up
	watchpats = [ ]

	uppat = self.templates["Pat:NonDC_started"]
	if self.CM.upcount() == 0:
	uppat = self.templates["Pat:Local_started"]

	watchpats.append(self.templates["Pat:DC_IDLE"])
	for node in node_list:
	watchpats.append(uppat % node)
	watchpats.append(self.templates["Pat:InfraUp"] % node)
	watchpats.append(self.templates["Pat:PacemakerUp"] % node)

	# Start all the nodes - at about the same time...
	watch = self.create_watch(watchpats, self.Env["DeadTime"]+10)
	watch.setwatch()

	stonith = self.CM.prepare_fencing_watcher(self.name)

	for node in node_list:
	self.CM.StartaCMnoBlock(node)

	watch.lookforall()

	node_list = self.CM.fencing_cleanup(self.name, stonith)

	if node_list == None:
	return self.failure("Cluster did not stabilize")

	# Remove node_list messages from watch.unmatched
	for node in node_list:
	self.logger.debug("Dealing with stonith operations for %s" % repr(node_list))
	if watch.unmatched:
	try:
	watch.unmatched.remove(uppat % node)
	except:
	self.debug("Already matched: %s" % (uppat % node))
	try:
	watch.unmatched.remove(self.templates["Pat:InfraUp"] % node)
	except:
	self.debug("Already matched: %s" % (self.templates["Pat:InfraUp"] % node))
	try:
	watch.unmatched.remove(self.templates["Pat:PacemakerUp"] % node)
	except:
	self.debug("Already matched: %s" % (self.templates["Pat:PacemakerUp"] % node))

	if watch.unmatched:
	for regex in watch.unmatched:
	self.logger.log ("Warn: Startup pattern not found: %s" %(regex))

	if not self.CM.cluster_stable():
	return self.failure("Cluster did not stabilize")

	did_fail = 0
	unstable = []
	for node in self.Env["nodes"]:
	if self.CM.StataCM(node) == 0:
	did_fail = 1
	unstable.append(node)

	if did_fail:
	return self.failure("Unstarted nodes exist: " + repr(unstable))

	unstable = []
	for node in self.Env["nodes"]:
	if not self.CM.node_stable(node):
	did_fail = 1
	unstable.append(node)

	if did_fail:
	return self.failure("Unstable cluster nodes exist: " + repr(unstable))

	return self.success()

	def is_applicable(self):
	'''SimulStartLite is a setup test and never applicable'''
	return 0


	def TestList(cm, audits):
	result = []
	for testclass in AllTestClasses:
	bound_test = testclass(cm)
	if bound_test.is_applicable():
	bound_test.Audits = audits
	result.append(bound_test)
	return result


	class RemoteLXC(CTSTest):
	def __init__(self, cm):
	CTSTest.__init__(self,cm)
	self.name = "RemoteLXC"
	self.start = StartTest(cm)
	self.startall = SimulStartLite(cm)
	self.num_containers = 2
	self.is_container = 1
	self.is_docker_unsafe = 1
	self.failed = 0
	self.fail_string = ""

	def start_lxc_simple(self, node):

	# restore any artifacts laying around from a previous test.
	self.rsh(node, "/usr/share/pacemaker/tests/cts/lxc_autogen.sh -s -R &>/dev/null")

	# generate the containers, put them in the config, add some resources to them
	pats = [ ]
	watch = self.create_watch(pats, 120)
	watch.setwatch()
	pats.append(self.templates["Pat:RscOpOK"] % ("start", "lxc1"))
	pats.append(self.templates["Pat:RscOpOK"] % ("start", "lxc2"))
	pats.append(self.templates["Pat:RscOpOK"] % ("start", "lxc-ms"))
	pats.append(self.templates["Pat:RscOpOK"] % ("promote", "lxc-ms"))

	self.rsh(node, "/usr/share/pacemaker/tests/cts/lxc_autogen.sh -g -a -m -s -c %d &>/dev/null" % self.num_containers)
	self.set_timer("remoteSimpleInit")
	watch.lookforall()
	self.log_timer("remoteSimpleInit")
	if watch.unmatched:
	self.fail_string = "Unmatched patterns: %s" % (repr(watch.unmatched))
	self.failed = 1

	def cleanup_lxc_simple(self, node):

	pats = [ ]
	# if the test failed, attempt to clean up the cib and libvirt environment
	# as best as possible
	if self.failed == 1:
	# restore libvirt and cib
	self.rsh(node, "/usr/share/pacemaker/tests/cts/lxc_autogen.sh -s -R &>/dev/null")
	return

	watch = self.create_watch(pats, 120)
	watch.setwatch()

	pats.append(self.templates["Pat:RscOpOK"] % ("stop", "container1"))
	pats.append(self.templates["Pat:RscOpOK"] % ("stop", "container2"))

	self.rsh(node, "/usr/share/pacemaker/tests/cts/lxc_autogen.sh -p &>/dev/null")
	self.set_timer("remoteSimpleCleanup")
	watch.lookforall()
	self.log_timer("remoteSimpleCleanup")

	if watch.unmatched:
	self.fail_string = "Unmatched patterns: %s" % (repr(watch.unmatched))
	self.failed = 1

	# cleanup libvirt
	self.rsh(node, "/usr/share/pacemaker/tests/cts/lxc_autogen.sh -s -R &>/dev/null")

	def __call__(self, node):
	'''Perform the 'RemoteLXC' test. '''
	self.incr("calls")

	ret = self.startall(None)
	if not ret:
	return self.failure("Setup failed, start all nodes failed.")

	rc = self.rsh(node, "/usr/share/pacemaker/tests/cts/lxc_autogen.sh -v &>/dev/null")
	if rc == 1:
	self.log("Environment test for lxc support failed.")
	return self.skipped()

	self.start_lxc_simple(node)
	self.cleanup_lxc_simple(node)

	self.debug("Waiting for the cluster to recover")
	self.CM.cluster_stable()

	if self.failed == 1:
	return self.failure(self.fail_string)

	return self.success()

	def errorstoignore(self):
	'''Return list of errors which should be ignored'''
	return [
	r"Updating failcount for ping",
	r"schedulerd.: Recover (ping\|lxc-ms\|container)\s$.*$",
	# The orphaned lxc-ms resource causes an expected transition error
	# that is a result of the scheduler not having knowledge that the
	# promotable resource used to be a clone. As a result, it looks like that
	# resource is running in multiple locations when it shouldn't... But in
	# this instance we know why this error is occurring and that it is expected.
	r"Calculated [Tt]ransition .*pe-error",
	r"Resource lxc-ms .* is active on 2 nodes attempting recovery",
	r"Unknown operation: fail",
	r"VirtualDomain.*ERROR: Unable to determine emulator",
	]

	AllTestClasses.append(RemoteLXC)


	class RemoteDriver(CTSTest):

	def __init__(self, cm):
	CTSTest.__init__(self,cm)
	self.name = self.__class__.__name__
	self.is_docker_unsafe = 1
	self.start = StartTest(cm)
	self.startall = SimulStartLite(cm)
	self.stop = StopTest(cm)
	self.remote_rsc = "remote-rsc"
	self.cib_cmd = """cibadmin -C -o %s -X '%s' """
	self.reset()

	def reset(self):
	self.pcmk_started = 0
	self.failed = False
	self.fail_string = ""
	self.remote_node_added = 0
	self.remote_rsc_added = 0
	self.remote_use_reconnect_interval = self.Env.RandomGen.choice([True,False])

	def fail(self, msg):
	""" Mark test as failed. """

	self.failed = True

	# Always log the failure.
	self.logger.log(msg)

	# Use first failure as test status, as it's likely to be most useful.
	if not self.fail_string:
	self.fail_string = msg

	def get_othernode(self, node):
	for othernode in self.Env["nodes"]:
	if othernode == node:
	# we don't want to try and use the cib that we just shutdown.
	# find a cluster node that is not our soon to be remote-node.
	continue
	else:
	return othernode

	def del_rsc(self, node, rsc):
	othernode = self.get_othernode(node)
	rc = self.rsh(othernode, "crm_resource -D -r %s -t primitive" % (rsc))
	if rc != 0:
	self.fail("Removal of resource '%s' failed" % rsc)

	def add_rsc(self, node, rsc_xml):
	othernode = self.get_othernode(node)
	rc = self.rsh(othernode, self.cib_cmd % ("resources", rsc_xml))
	if rc != 0:
	self.fail("resource creation failed")

	def add_primitive_rsc(self, node):
	rsc_xml = """
	<primitive class="ocf" id="%(node)s" provider="heartbeat" type="Dummy">
	<meta_attributes id="%(node)s-meta_attributes"/>
	<operations>
	<op id="%(node)s-monitor-interval-20s" interval="20s" name="monitor"/>
	</operations>
	</primitive>""" % { "node": self.remote_rsc }
	self.add_rsc(node, rsc_xml)
	if not self.failed:
	self.remote_rsc_added = 1

	def add_connection_rsc(self, node):
	rsc_xml = """
	<primitive class="ocf" id="%(node)s" provider="pacemaker" type="remote">
	<instance_attributes id="%(node)s-instance_attributes">
	<nvpair id="%(node)s-instance_attributes-server" name="server" value="%(server)s"/>
	""" % { "node": self.remote_node, "server": node }

	if self.remote_use_reconnect_interval:
	# Set reconnect interval on resource
	rsc_xml = rsc_xml + """
	<nvpair id="%s-instance_attributes-reconnect_interval" name="reconnect_interval" value="60s"/>
	""" % (self.remote_node)

	rsc_xml = rsc_xml + """
	</instance_attributes>
	<operations>
	<op id="%(node)s-start" name="start" interval="0" timeout="120s"/>
	<op id="%(node)s-monitor-20s" name="monitor" interval="20s" timeout="45s"/>
	</operations>
	</primitive>
	""" % { "node": self.remote_node }

	self.add_rsc(node, rsc_xml)
	if not self.failed:
	self.remote_node_added = 1

	def disable_services(self, node):
	self.corosync_enabled = self.Env.service_is_enabled(node, "corosync")
	if self.corosync_enabled:
	self.Env.disable_service(node, "corosync")

	self.pacemaker_enabled = self.Env.service_is_enabled(node, "pacemaker")
	if self.pacemaker_enabled:
	self.Env.disable_service(node, "pacemaker")

	def restore_services(self, node):
	if self.corosync_enabled:
	self.Env.enable_service(node, "corosync")

	if self.pacemaker_enabled:
	self.Env.enable_service(node, "pacemaker")

	def stop_pcmk_remote(self, node):
	# disable pcmk remote
	for i in range(10):
	rc = self.rsh(node, "service pacemaker_remote stop")
	if rc != 0:
	time.sleep(6)
	else:
	break

	def start_pcmk_remote(self, node):
	for i in range(10):
	rc = self.rsh(node, "service pacemaker_remote start")
	if rc != 0:
	time.sleep(6)
	else:
	self.pcmk_started = 1
	break

	def freeze_pcmk_remote(self, node):
	""" Simulate a Pacemaker Remote daemon failure. """

	# We freeze the process.
	self.rsh(node, "killall -STOP pacemaker-remoted")

	def resume_pcmk_remote(self, node):
	# We resume the process.
	self.rsh(node, "killall -CONT pacemaker-remoted")

	def start_metal(self, node):
	# Cluster nodes are reused as remote nodes in remote tests. If cluster
	# services were enabled at boot, in case the remote node got fenced, the
	# cluster node would join instead of the expected remote one. Meanwhile
	# pacemaker_remote would not be able to start. Depending on the chances,
	# the situations might not be able to be orchestrated gracefully any more.
	#
	# Temporarily disable any enabled cluster serivces.
	self.disable_services(node)

	pcmk_started = 0

	# make sure the resource doesn't already exist for some reason
	self.rsh(node, "crm_resource -D -r %s -t primitive" % (self.remote_rsc))
	self.rsh(node, "crm_resource -D -r %s -t primitive" % (self.remote_node))

	if not self.stop(node):
	self.fail("Failed to shutdown cluster node %s" % node)
	return

	self.start_pcmk_remote(node)

	if self.pcmk_started == 0:
	self.fail("Failed to start pacemaker_remote on node %s" % node)
	return

	# Convert node to baremetal now that it has shutdown the cluster stack
	pats = [ ]
	watch = self.create_watch(pats, 120)
	watch.setwatch()
	pats.append(self.templates["Pat:RscOpOK"] % ("start", self.remote_node))
	pats.append(self.templates["Pat:DC_IDLE"])

	self.add_connection_rsc(node)

	self.set_timer("remoteMetalInit")
	watch.lookforall()
	self.log_timer("remoteMetalInit")
	if watch.unmatched:
	self.fail("Unmatched patterns: %s" % watch.unmatched)

	def migrate_connection(self, node):
	if self.failed:
	return

	pats = [ ]
	pats.append(self.templates["Pat:RscOpOK"] % ("migrate_to", self.remote_node))
	pats.append(self.templates["Pat:RscOpOK"] % ("migrate_from", self.remote_node))
	pats.append(self.templates["Pat:DC_IDLE"])
	watch = self.create_watch(pats, 120)
	watch.setwatch()

	(rc, lines) = self.rsh(node, "crm_resource -M -r %s" % (self.remote_node), None)
	if rc != 0:
	self.fail("failed to move remote node connection resource")
	return

	self.set_timer("remoteMetalMigrate")
	watch.lookforall()
	self.log_timer("remoteMetalMigrate")

	if watch.unmatched:
	self.fail("Unmatched patterns: %s" % watch.unmatched)
	return

	def fail_rsc(self, node):
	if self.failed:
	return

	watchpats = [ ]
	watchpats.append(self.templates["Pat:RscRemoteOpOK"] % ("stop", self.remote_rsc, self.remote_node))
	watchpats.append(self.templates["Pat:RscRemoteOpOK"] % ("start", self.remote_rsc, self.remote_node))
	watchpats.append(self.templates["Pat:DC_IDLE"])

	watch = self.create_watch(watchpats, 120)
	watch.setwatch()

	self.debug("causing dummy rsc to fail.")

	rc = self.rsh(node, "rm -f /var/run/resource-agents/Dummy*")

	self.set_timer("remoteRscFail")
	watch.lookforall()
	self.log_timer("remoteRscFail")
	if watch.unmatched:
	self.fail("Unmatched patterns during rsc fail: %s" % watch.unmatched)

	def fail_connection(self, node):
	if self.failed:
	return

	watchpats = [ ]
	watchpats.append(self.templates["Pat:FenceOpOK"] % self.remote_node)
	watchpats.append(self.templates["Pat:NodeFenced"] % self.remote_node)

	watch = self.create_watch(watchpats, 120)
	watch.setwatch()

	# freeze the pcmk remote daemon. this will result in fencing
	self.debug("Force stopped active remote node")
	self.freeze_pcmk_remote(node)

	self.debug("Waiting for remote node to be fenced.")
	self.set_timer("remoteMetalFence")
	watch.lookforall()
	self.log_timer("remoteMetalFence")
	if watch.unmatched:
	self.fail("Unmatched patterns: %s" % watch.unmatched)
	return

	self.debug("Waiting for the remote node to come back up")
	self.CM.ns.WaitForNodeToComeUp(node, 120);

	pats = [ ]
	watch = self.create_watch(pats, 240)
	watch.setwatch()
	pats.append(self.templates["Pat:RscOpOK"] % ("start", self.remote_node))
	if self.remote_rsc_added == 1:
	pats.append(self.templates["Pat:RscRemoteOpOK"] % ("start", self.remote_rsc, self.remote_node))

	# start the remote node again watch it integrate back into cluster.
	self.start_pcmk_remote(node)
	if self.pcmk_started == 0:
	self.fail("Failed to start pacemaker_remote on node %s" % node)
	return

	self.debug("Waiting for remote node to rejoin cluster after being fenced.")
	self.set_timer("remoteMetalRestart")
	watch.lookforall()
	self.log_timer("remoteMetalRestart")
	if watch.unmatched:
	self.fail("Unmatched patterns: %s" % watch.unmatched)
	return

	def add_dummy_rsc(self, node):
	if self.failed:
	return

	# verify we can put a resource on the remote node
	pats = [ ]
	watch = self.create_watch(pats, 120)
	watch.setwatch()
	pats.append(self.templates["Pat:RscRemoteOpOK"] % ("start", self.remote_rsc, self.remote_node))
	pats.append(self.templates["Pat:DC_IDLE"])

	# Add a resource that must live on remote-node
	self.add_primitive_rsc(node)

	# force that rsc to prefer the remote node.
	(rc, line) = self.CM.rsh(node, "crm_resource -M -r %s -N %s -f" % (self.remote_rsc, self.remote_node), None)
	if rc != 0:
	self.fail("Failed to place remote resource on remote node.")
	return

	self.set_timer("remoteMetalRsc")
	watch.lookforall()
	self.log_timer("remoteMetalRsc")
	if watch.unmatched:
	self.fail("Unmatched patterns: %s" % watch.unmatched)

	def test_attributes(self, node):
	if self.failed:
	return

	# This verifies permanent attributes can be set on a remote-node. It also
	# verifies the remote-node can edit its own cib node section remotely.
	(rc, line) = self.CM.rsh(node, "crm_attribute -l forever -n testattr -v testval -N %s" % (self.remote_node), None)
	if rc != 0:
	self.fail("Failed to set remote-node attribute. rc:%s output:%s" % (rc, line))
	return

	(rc, line) = self.CM.rsh(node, "crm_attribute -l forever -n testattr -q -N %s" % (self.remote_node), None)
	if rc != 0:
	self.fail("Failed to get remote-node attribute")
	return

	(rc, line) = self.CM.rsh(node, "crm_attribute -l forever -n testattr -D -N %s" % (self.remote_node), None)
	if rc != 0:
	self.fail("Failed to delete remote-node attribute")
	return

	def cleanup_metal(self, node):
	self.restore_services(node)

	if self.pcmk_started == 0:
	return

	pats = [ ]

	watch = self.create_watch(pats, 120)
	watch.setwatch()

	if self.remote_rsc_added == 1:
	pats.append(self.templates["Pat:RscOpOK"] % ("stop", self.remote_rsc))
	if self.remote_node_added == 1:
	pats.append(self.templates["Pat:RscOpOK"] % ("stop", self.remote_node))

	self.set_timer("remoteMetalCleanup")

	self.resume_pcmk_remote(node)

	if self.remote_rsc_added == 1:

	# Remove dummy resource added for remote node tests
	self.debug("Cleaning up dummy rsc put on remote node")
	self.rsh(self.get_othernode(node), "crm_resource -U -r %s" % self.remote_rsc)
	self.del_rsc(node, self.remote_rsc)

	if self.remote_node_added == 1:

	# Remove remote node's connection resource
	self.debug("Cleaning up remote node connection resource")
	self.rsh(self.get_othernode(node), "crm_resource -U -r %s" % (self.remote_node))
	self.del_rsc(node, self.remote_node)

	watch.lookforall()
	self.log_timer("remoteMetalCleanup")

	if watch.unmatched:
	self.fail("Unmatched patterns: %s" % watch.unmatched)

	self.stop_pcmk_remote(node)

	self.debug("Waiting for the cluster to recover")
	self.CM.cluster_stable()

	if self.remote_node_added == 1:
	# Remove remote node itself
	self.debug("Cleaning up node entry for remote node")
	self.rsh(self.get_othernode(node), "crm_node --force --remove %s" % self.remote_node)

	def setup_env(self, node):

	self.remote_node = "remote-%s" % (node)

	# we are assuming if all nodes have a key, that it is
	# the right key... If any node doesn't have a remote
	# key, we regenerate it everywhere.
	if self.rsh.exists_on_all("/etc/pacemaker/authkey", self.Env["nodes"]):
	return

	# create key locally
	(handle, keyfile) = tempfile.mkstemp(".cts")
	os.close(handle)
	subprocess.check_call(["dd", "if=/dev/urandom", "of=%s" % keyfile, "bs=4096", "count=1"],
	stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)

	# sync key throughout the cluster
	for node in self.Env["nodes"]:
	self.rsh(node, "mkdir -p --mode=0750 /etc/pacemaker")
	self.rsh.cp(keyfile, "root@%s:/etc/pacemaker/authkey" % node)
	self.rsh(node, "chgrp haclient /etc/pacemaker /etc/pacemaker/authkey")
	self.rsh(node, "chmod 0640 /etc/pacemaker/authkey")
	os.unlink(keyfile)

	def is_applicable(self):
	if not self.is_applicable_common():
	return False

	for node in self.Env["nodes"]:
	rc = self.rsh(node, "which pacemaker-remoted >/dev/null 2>&1")
	if rc != 0:
	return False
	return True

	def start_new_test(self, node):
	self.incr("calls")
	self.reset()

	ret = self.startall(None)
	if not ret:
	return self.failure("setup failed: could not start all nodes")

	self.setup_env(node)
	self.start_metal(node)
	self.add_dummy_rsc(node)
	return True

	def __call__(self, node):
	return self.failure("This base class is not meant to be called directly.")

	def errorstoignore(self):
	'''Return list of errors which should be ignored'''
	return [ r"""is running on remote.*which isn't allowed""",
	r"""Connection terminated""",
	r"""Could not send remote""",
	]

	# RemoteDriver is just a base class for other tests, so it is not added to AllTestClasses


	class RemoteBasic(RemoteDriver):

	def __call__(self, node):
	'''Perform the 'RemoteBaremetal' test. '''

	if not self.start_new_test(node):
	return self.failure(self.fail_string)

	self.test_attributes(node)
	self.cleanup_metal(node)

	self.debug("Waiting for the cluster to recover")
	self.CM.cluster_stable()
	if self.failed:
	return self.failure(self.fail_string)

	return self.success()

	AllTestClasses.append(RemoteBasic)

	class RemoteStonithd(RemoteDriver):

	def __call__(self, node):
	'''Perform the 'RemoteStonithd' test. '''

	if not self.start_new_test(node):
	return self.failure(self.fail_string)

	self.fail_connection(node)
	self.cleanup_metal(node)

	self.debug("Waiting for the cluster to recover")
	self.CM.cluster_stable()
	if self.failed:
	return self.failure(self.fail_string)

	return self.success()

	def is_applicable(self):
	if not RemoteDriver.is_applicable(self):
	return False

	if "DoFencing" in list(self.Env.keys()):
	return self.Env["DoFencing"]

	return True

	def errorstoignore(self):
	ignore_pats = [
	r"Lost connection to Pacemaker Remote node",
	r"Software caused connection abort",
	r"pacemaker-controld.:\s+error.: Operation remote-.*_monitor",
	r"pacemaker-controld.:\s+error.: Result of monitor operation for remote-.*",
	r"schedulerd.:\s+Recover remote-.\s$.$",
	r"error: Result of monitor operation for .* on remote-.*: No executor connection",
	]

	ignore_pats.extend(RemoteDriver.errorstoignore(self))
	return ignore_pats

	AllTestClasses.append(RemoteStonithd)


	class RemoteMigrate(RemoteDriver):

	def __call__(self, node):
	'''Perform the 'RemoteMigrate' test. '''

	if not self.start_new_test(node):
	return self.failure(self.fail_string)

	self.migrate_connection(node)
	self.cleanup_metal(node)

	self.debug("Waiting for the cluster to recover")
	self.CM.cluster_stable()
	if self.failed:
	return self.failure(self.fail_string)

	return self.success()

	def is_applicable(self):
	if not RemoteDriver.is_applicable(self):
	return 0
	# This test requires at least three nodes: one to convert to a
	# remote node, one to host the connection originally, and one
	# to migrate the connection to.
	if len(self.Env["nodes"]) < 3:
	return 0
	return 1

	AllTestClasses.append(RemoteMigrate)


	class RemoteRscFailure(RemoteDriver):

	def __call__(self, node):
	'''Perform the 'RemoteRscFailure' test. '''

	if not self.start_new_test(node):
	return self.failure(self.fail_string)

	# This is an important step. We are migrating the connection
	# before failing the resource. This verifies that the migration
	# has properly maintained control over the remote-node.
	self.migrate_connection(node)

	self.fail_rsc(node)
	self.cleanup_metal(node)

	self.debug("Waiting for the cluster to recover")
	self.CM.cluster_stable()
	if self.failed:
	return self.failure(self.fail_string)

	return self.success()

	def errorstoignore(self):
	ignore_pats = [
	r"schedulerd.: Recover remote-rsc\s$.*$",
	r"Dummy.*: No process state file found",
	]

	ignore_pats.extend(RemoteDriver.errorstoignore(self))
	return ignore_pats

	def is_applicable(self):
	if not RemoteDriver.is_applicable(self):
	return 0
	# This test requires at least three nodes: one to convert to a
	# remote node, one to host the connection originally, and one
	# to migrate the connection to.
	if len(self.Env["nodes"]) < 3:
	return 0
	return 1

	AllTestClasses.append(RemoteRscFailure)

	# vim:ts=4:sw=4:et:
	diff --git a/cts/lab/ClusterManager.py b/cts/lab/ClusterManager.py
	index 0dfb95ce29..8daffaebb1 100644
	--- a/cts/lab/ClusterManager.py
	+++ b/cts/lab/ClusterManager.py
	@@ -1,1067 +1,1067 @@
	""" ClusterManager class for Pacemaker's Cluster Test Suite (CTS)
	"""

	__copyright__ = """Copyright 2000-2021 the Pacemaker project contributors.
	Certain portions by Huang Zhen <zhenhltc@cn.ibm.com> are copyright 2004
	International Business Machines. The version control history for this file
	may have further details."""
	__license__ = "GNU General Public License version 2 or later (GPLv2+) WITHOUT ANY WARRANTY"

	import os
	import re
	import time

	from collections import UserDict

	from cts.CTSvars import *
	from cts.CIB import ConfigFactory
	from cts.CTS import NodeStatus, Process
	from cts.CTStests import AuditResource
	from cts.logging import LogFactory
	from cts.watcher import LogWatcher
	from cts.remote import RemoteFactory
	from cts.environment import EnvFactory
	from cts.patterns import PatternSelector

	has_log_stats = {}
	log_stats_bin = CTSvars.CRM_DAEMON_DIR + "/cts_log_stats.sh"
	log_stats = """
	#!%s
	# Tool for generating system load reports while CTS runs

	trap "" 1

	f=$1; shift
	action=$1; shift
	base=`basename $0`

	if [ ! -e $f ]; then
	echo "Time, Load 1, Load 5, Load 15, Test Marker" > $f
	fi

	function killpid() {
	if [ -e $f.pid ]; then
	kill -9 `cat $f.pid`
	rm -f $f.pid
	fi
	}

	function status() {
	if [ -e $f.pid ]; then
	kill -0 `cat $f.pid`
	return $?
	else
	return 1
	fi
	}

	function start() {
	# Is it already running?
	if
	status
	then
	return
	fi

	echo Active as $$
	echo $$ > $f.pid

	while [ 1 = 1 ]; do
	uptime \| sed s/up.*:/,/ \| tr '\\n' ',' >> $f
	#top -b -c -n1 \| grep -e usr/libexec/pacemaker \| grep -v -e grep -e python \| head -n 1 \| sed s@/usr/libexec/pacemaker/@@ \| awk '{print " 0, "$9", "$10", "$12}' \| tr '\\n' ',' >> $f
	echo 0 >> $f
	sleep 5
	done
	}

	case $action in
	start)
	start
	;;
	start-bg\|bg)
	# Use c --ssh -- ./stats.sh file start-bg
	nohup $0 $f start >/dev/null 2>&1 </dev/null &
	;;
	stop)
	killpid
	;;
	delete)
	killpid
	rm -f $f
	;;
	mark)
	uptime \| sed s/up.*:/,/ \| tr '\\n' ',' >> $f
	echo " $*" >> $f
	start
	;;
	*)
	echo "Unknown action: $action."
	;;
	esac
	""" % (CTSvars.BASH_PATH)

	class ClusterManager(UserDict):
	'''The Cluster Manager class.
	This is an subclass of the Python dictionary class.
	(this is because it contains lots of {name,value} pairs,
	not because it's behavior is that terribly similar to a
	dictionary in other ways.)

	This is an abstract class which class implements high-level
	operations on the cluster and/or its cluster managers.
	Actual cluster managers classes are subclassed from this type.

	One of the things we do is track the state we think every node should
	be in.
	'''

	def __InitialConditions(self):
	#if os.geteuid() != 0:
	# raise ValueError("Must Be Root!")
	None

	def _finalConditions(self):
	for key in list(self.keys()):
	if self[key] == None:
	raise ValueError("Improper derivation: self[" + key + "] must be overridden by subclass.")

	- def __init__(self, Environment, randseed=None):
	+ def __init__(self):
	self.Env = EnvFactory().getInstance()
	self.templates = PatternSelector(self.Env["Name"])
	self.__InitialConditions()
	self.logger = LogFactory()
	self.TestLoggingLevel=0
	self.data = {}
	self.name = self.Env["Name"]

	self.rsh = RemoteFactory().getInstance()
	self.ShouldBeStatus={}
	self.ns = NodeStatus(self.Env)
	self.OurNode = os.uname()[1].lower()
	self.__instance_errorstoignore = []

	self.fastfail = 0
	self.cib_installed = 0
	self.config = None
	self.cluster_monitor = 0
	self.use_short_names = 1

	if self.Env["DoBSC"]:
	del self.templates["Pat:They_stopped"]

	self._finalConditions()

	self.check_transitions = 0
	self.check_elections = 0
	self.CIBsync = {}
	self.CibFactory = ConfigFactory(self)
	self.cib = self.CibFactory.createConfig(self.Env["Schema"])

	def __getitem__(self, key):
	if key == "Name":
	return self.name

	print("FIXME: Getting %s from %s" % (key, repr(self)))
	if key in self.data:
	return self.data[key]

	return self.templates.get_patterns(self.Env["Name"], key)

	def __setitem__(self, key, value):
	print("FIXME: Setting %s=%s on %s" % (key, value, repr(self)))
	self.data[key] = value

	def key_for_node(self, node):
	return node

	def instance_errorstoignore_clear(self):
	'''Allows the test scenario to reset instance errors to ignore on each iteration.'''
	self.__instance_errorstoignore = []

	def instance_errorstoignore(self):
	'''Return list of errors which are 'normal' for a specific test instance'''
	return self.__instance_errorstoignore

	def log(self, args):
	self.logger.log(args)

	def debug(self, args):
	self.logger.debug(args)

	def upcount(self):
	'''How many nodes are up?'''
	count = 0
	for node in self.Env["nodes"]:
	if self.ShouldBeStatus[node] == "up":
	count = count + 1
	return count

	def install_support(self, command="install"):
	for node in self.Env["nodes"]:
	self.rsh(node, CTSvars.CRM_DAEMON_DIR + "/cts-support " + command)

	def prepare_fencing_watcher(self, name):
	# If we don't have quorum now but get it as a result of starting this node,
	# then a bunch of nodes might get fenced
	upnode = None
	if self.HasQuorum(None):
	self.debug("Have quorum")
	return None

	if not self.templates["Pat:Fencing_start"]:
	print("No start pattern")
	return None

	if not self.templates["Pat:Fencing_ok"]:
	print("No ok pattern")
	return None

	stonith = None
	stonithPats = []
	for peer in self.Env["nodes"]:
	if self.ShouldBeStatus[peer] != "up":
	stonithPats.append(self.templates["Pat:Fencing_ok"] % peer)
	stonithPats.append(self.templates["Pat:Fencing_start"] % peer)

	stonith = LogWatcher(self.Env["LogFileName"], stonithPats, "StartupFencing", 0, hosts=self.Env["nodes"], kind=self.Env["LogWatcher"])
	stonith.setwatch()
	return stonith

	def fencing_cleanup(self, node, stonith):
	peer_list = []
	peer_state = {}

	self.debug("Looking for nodes that were fenced as a result of %s starting" % node)

	# If we just started a node, we may now have quorum (and permission to fence)
	if not stonith:
	self.debug("Nothing to do")
	return peer_list

	q = self.HasQuorum(None)
	if not q and len(self.Env["nodes"]) > 2:
	# We didn't gain quorum - we shouldn't have shot anyone
	self.debug("Quorum: %d Len: %d" % (q, len(self.Env["nodes"])))
	return peer_list

	for n in self.Env["nodes"]:
	peer_state[n] = "unknown"

	# Now see if any states need to be updated
	self.debug("looking for: " + repr(stonith.regexes))
	shot = stonith.look(0)
	while shot:
	line = repr(shot)
	self.debug("Found: " + line)
	del stonith.regexes[stonith.whichmatch]

	# Extract node name
	for n in self.Env["nodes"]:
	if re.search(self.templates["Pat:Fencing_ok"] % n, shot):
	peer = n
	peer_state[peer] = "complete"
	self.__instance_errorstoignore.append(self.templates["Pat:Fencing_ok"] % peer)

	elif peer_state[n] != "complete" and re.search(self.templates["Pat:Fencing_start"] % n, shot):
	# TODO: Correctly detect multiple fencing operations for the same host
	peer = n
	peer_state[peer] = "in-progress"
	self.__instance_errorstoignore.append(self.templates["Pat:Fencing_start"] % peer)

	if not peer:
	self.logger.log("ERROR: Unknown stonith match: %s" % line)

	elif not peer in peer_list:
	self.debug("Found peer: " + peer)
	peer_list.append(peer)

	# Get the next one
	shot = stonith.look(60)

	for peer in peer_list:

	self.debug(" Peer %s was fenced as a result of %s starting: %s" % (peer, node, peer_state[peer]))
	if self.Env["at-boot"]:
	self.ShouldBeStatus[peer] = "up"
	else:
	self.ShouldBeStatus[peer] = "down"

	if peer_state[peer] == "in-progress":
	# Wait for any in-progress operations to complete
	shot = stonith.look(60)
	while len(stonith.regexes) and shot:
	line = repr(shot)
	self.debug("Found: " + line)
	del stonith.regexes[stonith.whichmatch]
	shot = stonith.look(60)

	# Now make sure the node is alive too
	self.ns.WaitForNodeToComeUp(peer, self.Env["DeadTime"])

	# Poll until it comes up
	if self.Env["at-boot"]:
	if not self.StataCM(peer):
	time.sleep(self.Env["StartTime"])

	if not self.StataCM(peer):
	self.logger.log("ERROR: Peer %s failed to restart after being fenced" % peer)
	return None

	return peer_list

	def StartaCM(self, node, verbose=False):

	'''Start up the cluster manager on a given node'''
	if verbose: self.logger.log("Starting %s on node %s" % (self.templates["Name"], node))
	else: self.debug("Starting %s on node %s" % (self.templates["Name"], node))
	ret = 1

	if not node in self.ShouldBeStatus:
	self.ShouldBeStatus[node] = "down"

	if self.ShouldBeStatus[node] != "down":
	return 1

	patterns = []
	# Technically we should always be able to notice ourselves starting
	patterns.append(self.templates["Pat:Local_started"] % node)
	if self.upcount() == 0:
	patterns.append(self.templates["Pat:DC_started"] % node)
	else:
	patterns.append(self.templates["Pat:NonDC_started"] % node)

	watch = LogWatcher(
	self.Env["LogFileName"], patterns, "StartaCM", self.Env["StartTime"]+10, hosts=self.Env["nodes"], kind=self.Env["LogWatcher"])

	self.install_config(node)

	self.ShouldBeStatus[node] = "any"
	if self.StataCM(node) and self.cluster_stable(self.Env["DeadTime"]):
	self.logger.log ("%s was already started" % (node))
	return 1

	stonith = self.prepare_fencing_watcher(node)
	watch.setwatch()

	if self.rsh(node, self.templates["StartCmd"]) != 0:
	self.logger.log ("Warn: Start command failed on node %s" % (node))
	self.fencing_cleanup(node, stonith)
	return None

	self.ShouldBeStatus[node] = "up"
	watch_result = watch.lookforall()

	if watch.unmatched:
	for regex in watch.unmatched:
	self.logger.log ("Warn: Startup pattern not found: %s" % (regex))

	if watch_result and self.cluster_stable(self.Env["DeadTime"]):
	#self.debug("Found match: "+ repr(watch_result))
	self.fencing_cleanup(node, stonith)
	return 1

	elif self.StataCM(node) and self.cluster_stable(self.Env["DeadTime"]):
	self.fencing_cleanup(node, stonith)
	return 1

	self.logger.log ("Warn: Start failed for node %s" % (node))
	return None

	def StartaCMnoBlock(self, node, verbose=False):

	'''Start up the cluster manager on a given node with none-block mode'''

	if verbose: self.logger.log("Starting %s on node %s" % (self["Name"], node))
	else: self.debug("Starting %s on node %s" % (self["Name"], node))

	self.install_config(node)
	self.rsh(node, self.templates["StartCmd"], synchronous=0)
	self.ShouldBeStatus[node] = "up"
	return 1

	def StopaCM(self, node, verbose=False, force=False):

	'''Stop the cluster manager on a given node'''

	if verbose: self.logger.log("Stopping %s on node %s" % (self["Name"], node))
	else: self.debug("Stopping %s on node %s" % (self["Name"], node))

	if self.ShouldBeStatus[node] != "up" and force == False:
	return 1

	if self.rsh(node, self.templates["StopCmd"]) == 0:
	# Make sure we can continue even if corosync leaks
	# fdata-* is the old name
	#self.rsh(node, "rm -rf /dev/shm/qb-* /dev/shm/fdata-*")
	self.ShouldBeStatus[node] = "down"
	self.cluster_stable(self.Env["DeadTime"])
	return 1
	else:
	self.logger.log ("ERROR: Could not stop %s on node %s" % (self["Name"], node))

	return None

	def StopaCMnoBlock(self, node):

	'''Stop the cluster manager on a given node with none-block mode'''

	self.debug("Stopping %s on node %s" % (self["Name"], node))

	self.rsh(node, self.templates["StopCmd"], synchronous=0)
	self.ShouldBeStatus[node] = "down"
	return 1

	def RereadCM(self, node):

	'''Force the cluster manager on a given node to reread its config
	This may be a no-op on certain cluster managers.
	'''
	rc=self.rsh(node, self.templates["RereadCmd"])
	if rc == 0:
	return 1
	else:
	self.logger.log ("Could not force %s on node %s to reread its config"
	% (self["Name"], node))
	return None

	def startall(self, nodelist=None, verbose=False, quick=False):

	'''Start the cluster manager on every node in the cluster.
	We can do it on a subset of the cluster if nodelist is not None.
	'''
	map = {}
	if not nodelist:
	nodelist = self.Env["nodes"]

	for node in nodelist:
	if self.ShouldBeStatus[node] == "down":
	self.ns.WaitForAllNodesToComeUp(nodelist, 300)

	if not quick:
	# This is used for "basic sanity checks", so only start one node ...
	if not self.StartaCM(node, verbose=verbose):
	return 0
	return 1

	# Approximation of SimulStartList for --boot
	watchpats = [ ]
	watchpats.append(self.templates["Pat:DC_IDLE"])
	for node in nodelist:
	watchpats.append(self.templates["Pat:InfraUp"] % node)
	watchpats.append(self.templates["Pat:PacemakerUp"] % node)
	watchpats.append(self.templates["Pat:Local_started"] % node)
	watchpats.append(self.templates["Pat:They_up"] % (nodelist[0], node))

	# Start all the nodes - at about the same time...
	watch = LogWatcher(self.Env["LogFileName"], watchpats, "fast-start", self.Env["DeadTime"]+10, hosts=self.Env["nodes"], kind=self.Env["LogWatcher"])
	watch.setwatch()

	if not self.StartaCM(nodelist[0], verbose=verbose):
	return 0
	for node in nodelist:
	self.StartaCMnoBlock(node, verbose=verbose)

	watch.lookforall()
	if watch.unmatched:
	for regex in watch.unmatched:
	self.logger.log ("Warn: Startup pattern not found: %s" % (regex))

	if not self.cluster_stable():
	self.logger.log("Cluster did not stabilize")
	return 0

	return 1

	def stopall(self, nodelist=None, verbose=False, force=False):

	'''Stop the cluster managers on every node in the cluster.
	We can do it on a subset of the cluster if nodelist is not None.
	'''

	ret = 1
	map = {}
	if not nodelist:
	nodelist = self.Env["nodes"]
	for node in self.Env["nodes"]:
	if self.ShouldBeStatus[node] == "up" or force == True:
	if not self.StopaCM(node, verbose=verbose, force=force):
	ret = 0
	return ret

	def rereadall(self, nodelist=None):

	'''Force the cluster managers on every node in the cluster
	to reread their config files. We can do it on a subset of the
	cluster if nodelist is not None.
	'''

	map = {}
	if not nodelist:
	nodelist = self.Env["nodes"]
	for node in self.Env["nodes"]:
	if self.ShouldBeStatus[node] == "up":
	self.RereadCM(node)

	def statall(self, nodelist=None):

	'''Return the status of the cluster managers in the cluster.
	We can do it on a subset of the cluster if nodelist is not None.
	'''

	result = {}
	if not nodelist:
	nodelist = self.Env["nodes"]
	for node in nodelist:
	if self.StataCM(node):
	result[node] = "up"
	else:
	result[node] = "down"
	return result

	def isolate_node(self, target, nodes=None):
	'''isolate the communication between the nodes'''
	if not nodes:
	nodes = self.Env["nodes"]

	for node in nodes:
	if node != target:
	rc = self.rsh(target, self.templates["BreakCommCmd"] % self.key_for_node(node))
	if rc != 0:
	self.logger.log("Could not break the communication between %s and %s: %d" % (target, node, rc))
	return None
	else:
	self.debug("Communication cut between %s and %s" % (target, node))
	return 1

	def unisolate_node(self, target, nodes=None):
	'''fix the communication between the nodes'''
	if not nodes:
	nodes = self.Env["nodes"]

	for node in nodes:
	if node != target:
	restored = 0

	# Limit the amount of time we have asynchronous connectivity for
	# Restore both sides as simultaneously as possible
	self.rsh(target, self.templates["FixCommCmd"] % self.key_for_node(node), synchronous=0)
	self.rsh(node, self.templates["FixCommCmd"] % self.key_for_node(target), synchronous=0)
	self.debug("Communication restored between %s and %s" % (target, node))

	def reducecomm_node(self,node):
	'''reduce the communication between the nodes'''
	rc = self.rsh(node, self.templates["ReduceCommCmd"]%(self.Env["XmitLoss"],self.Env["RecvLoss"]))
	if rc == 0:
	return 1
	else:
	self.logger.log("Could not reduce the communication between the nodes from node: %s" % node)
	return None

	def restorecomm_node(self,node):
	'''restore the saved communication between the nodes'''
	rc = 0
	if float(self.Env["XmitLoss"]) != 0 or float(self.Env["RecvLoss"]) != 0 :
	rc = self.rsh(node, self.templates["RestoreCommCmd"]);
	if rc == 0:
	return 1
	else:
	self.logger.log("Could not restore the communication between the nodes from node: %s" % node)
	return None

	def oprofileStart(self, node=None):
	if not node:
	for n in self.Env["oprofile"]:
	self.oprofileStart(n)

	elif node in self.Env["oprofile"]:
	self.debug("Enabling oprofile on %s" % node)
	self.rsh(node, "opcontrol --init")
	self.rsh(node, "opcontrol --setup --no-vmlinux --separate=lib --callgraph=20 --image=all")
	self.rsh(node, "opcontrol --start")
	self.rsh(node, "opcontrol --reset")

	def oprofileSave(self, test, node=None):
	if not node:
	for n in self.Env["oprofile"]:
	self.oprofileSave(test, n)

	elif node in self.Env["oprofile"]:
	self.rsh(node, "opcontrol --dump")
	self.rsh(node, "opcontrol --save=cts.%d" % test)
	# Read back with: opreport -l session:cts.0 image:<directory>/c*
	if None:
	self.rsh(node, "opcontrol --reset")
	else:
	self.oprofileStop(node)
	self.oprofileStart(node)

	def oprofileStop(self, node=None):
	if not node:
	for n in self.Env["oprofile"]:
	self.oprofileStop(n)

	elif node in self.Env["oprofile"]:
	self.debug("Stopping oprofile on %s" % node)
	self.rsh(node, "opcontrol --reset")
	self.rsh(node, "opcontrol --shutdown 2>&1 > /dev/null")


	def StatsExtract(self):
	if not self.Env["stats"]:
	return

	for host in self.Env["nodes"]:
	log_stats_file = "%s/cts-stats.csv" % CTSvars.CRM_DAEMON_DIR
	if host in has_log_stats:
	self.rsh(host, '''bash %s %s stop''' % (log_stats_bin, log_stats_file))
	(rc, lines) = self.rsh(host, '''cat %s''' % log_stats_file, stdout=2)
	self.rsh(host, '''bash %s %s delete''' % (log_stats_bin, log_stats_file))

	fname = "cts-stats-%d-nodes-%s.csv" % (len(self.Env["nodes"]), host)
	print("Extracted stats: %s" % fname)
	fd = open(fname, "a")
	fd.writelines(lines)
	fd.close()

	def StatsMark(self, testnum):
	'''Mark the test number in the stats log'''

	global has_log_stats
	if not self.Env["stats"]:
	return

	for host in self.Env["nodes"]:
	log_stats_file = "%s/cts-stats.csv" % CTSvars.CRM_DAEMON_DIR
	if not host in has_log_stats:

	global log_stats
	global log_stats_bin
	script=log_stats
	#script = re.sub("\\\\", "\\\\", script)
	script = re.sub('\"', '\\\"', script)
	script = re.sub("'", "\'", script)
	script = re.sub("`", "\`", script)
	script = re.sub("\$", "\\\$", script)

	self.debug("Installing %s on %s" % (log_stats_bin, host))
	self.rsh(host, '''echo "%s" > %s''' % (script, log_stats_bin), silent=True)
	self.rsh(host, '''bash %s %s delete''' % (log_stats_bin, log_stats_file))
	has_log_stats[host] = 1

	# Now mark it
	self.rsh(host, '''bash %s %s mark %s''' % (log_stats_bin, log_stats_file, testnum), synchronous=0)

	def errorstoignore(self):
	# At some point implement a more elegant solution that
	# also produces a report at the end
	""" Return a list of known error messages that should be ignored """
	return PatternSelector().get_patterns(self.name, "BadNewsIgnore")

	def install_config(self, node):
	if not self.ns.WaitForNodeToComeUp(node):
	self.log("Node %s is not up." % node)
	return None

	if not node in self.CIBsync and self.Env["ClobberCIB"] == 1:
	self.CIBsync[node] = 1
	self.rsh(node, "rm -f "+CTSvars.CRM_CONFIG_DIR+"/cib*")

	# Only install the CIB on the first node, all the other ones will pick it up from there
	if self.cib_installed == 1:
	return None

	self.cib_installed = 1
	if self.Env["CIBfilename"] == None:
	self.log("Installing Generated CIB on node %s" % (node))
	self.cib.install(node)

	else:
	self.log("Installing CIB (%s) on node %s" % (self.Env["CIBfilename"], node))
	if 0 != self.rsh.cp(self.Env["CIBfilename"], "root@" + (self.templates["CIBfile"] % node)):
	raise ValueError("Can not scp file to %s %d"%(node))

	self.rsh(node, "chown "+CTSvars.CRM_DAEMON_USER+" "+CTSvars.CRM_CONFIG_DIR+"/cib.xml")

	def prepare(self):
	'''Finish the Initialization process. Prepare to test...'''

	self.partitions_expected = 1
	for node in self.Env["nodes"]:
	self.ShouldBeStatus[node] = ""
	if self.Env["experimental-tests"]:
	self.unisolate_node(node)
	self.StataCM(node)

	def test_node_CM(self, node):
	'''Report the status of the cluster manager on a given node'''

	watchpats = [ ]
	watchpats.append("Current ping state: (S_IDLE\|S_NOT_DC)")
	watchpats.append(self.templates["Pat:NonDC_started"] % node)
	watchpats.append(self.templates["Pat:DC_started"] % node)
	idle_watch = LogWatcher(self.Env["LogFileName"], watchpats, "ClusterIdle", hosts=[node], kind=self.Env["LogWatcher"])
	idle_watch.setwatch()

	out = self.rsh(node, self.templates["StatusCmd"]%node, 1)
	self.debug("Node %s status: '%s'" %(node, out))

	if not out or (out.find('ok') < 0):
	if self.ShouldBeStatus[node] == "up":
	self.log(
	"Node status for %s is %s but we think it should be %s"
	% (node, "down", self.ShouldBeStatus[node]))
	self.ShouldBeStatus[node] = "down"
	return 0

	if self.ShouldBeStatus[node] == "down":
	self.log(
	"Node status for %s is %s but we think it should be %s: %s"
	% (node, "up", self.ShouldBeStatus[node], out))

	self.ShouldBeStatus[node] = "up"

	# check the output first - because syslog-ng loses messages
	if out.find('S_NOT_DC') != -1:
	# Up and stable
	return 2
	if out.find('S_IDLE') != -1:
	# Up and stable
	return 2

	# fall back to syslog-ng and wait
	if not idle_watch.look():
	# just up
	self.debug("Warn: Node %s is unstable: %s" % (node, out))
	return 1

	# Up and stable
	return 2

	# Is the node up or is the node down
	def StataCM(self, node):
	'''Report the status of the cluster manager on a given node'''

	if self.test_node_CM(node) > 0:
	return 1
	return None

	# Being up and being stable is not the same question...
	def node_stable(self, node):
	'''Report the status of the cluster manager on a given node'''

	if self.test_node_CM(node) == 2:
	return 1
	self.log("Warn: Node %s not stable" % (node))
	return None

	def partition_stable(self, nodes, timeout=None):
	watchpats = [ ]
	watchpats.append("Current ping state: S_IDLE")
	watchpats.append(self.templates["Pat:DC_IDLE"])
	self.debug("Waiting for cluster stability...")

	if timeout == None:
	timeout = self.Env["DeadTime"]

	if len(nodes) < 3:
	self.debug("Cluster is inactive")
	return 1

	idle_watch = LogWatcher(self.Env["LogFileName"], watchpats, "ClusterStable", timeout, hosts=nodes.split(), kind=self.Env["LogWatcher"])
	idle_watch.setwatch()

	for node in nodes.split():
	# have each node dump its current state
	self.rsh(node, self.templates["StatusCmd"] % node, 1)

	ret = idle_watch.look()
	while ret:
	self.debug(ret)
	for node in nodes.split():
	if re.search(node, ret):
	return 1
	ret = idle_watch.look()

	self.debug("Warn: Partition %s not IDLE after %ds" % (repr(nodes), timeout))
	return None

	def cluster_stable(self, timeout=None, double_check=False):
	partitions = self.find_partitions()

	for partition in partitions:
	if not self.partition_stable(partition, timeout):
	return None

	if double_check:
	# Make sure we are really stable and that all resources,
	# including those that depend on transient node attributes,
	# are started if they were going to be
	time.sleep(5)
	for partition in partitions:
	if not self.partition_stable(partition, timeout):
	return None

	return 1

	def is_node_dc(self, node, status_line=None):
	rc = 0

	if not status_line:
	status_line = self.rsh(node, self.templates["StatusCmd"]%node, 1)

	if not status_line:
	rc = 0
	elif status_line.find('S_IDLE') != -1:
	rc = 1
	elif status_line.find('S_INTEGRATION') != -1:
	rc = 1
	elif status_line.find('S_FINALIZE_JOIN') != -1:
	rc = 1
	elif status_line.find('S_POLICY_ENGINE') != -1:
	rc = 1
	elif status_line.find('S_TRANSITION_ENGINE') != -1:
	rc = 1

	return rc

	def active_resources(self, node):
	(rc, output) = self.rsh(node, """crm_resource -c""", None)
	resources = []
	for line in output:
	if re.search("^Resource", line):
	tmp = AuditResource(self, line)
	if tmp.type == "primitive" and tmp.host == node:
	resources.append(tmp.id)
	return resources

	def ResourceLocation(self, rid):
	ResourceNodes = []
	for node in self.Env["nodes"]:
	if self.ShouldBeStatus[node] == "up":

	cmd = self.templates["RscRunning"] % (rid)
	(rc, lines) = self.rsh(node, cmd, None)

	if rc == 127:
	self.log("Command '%s' failed. Binary or pacemaker-cts package not installed?" % cmd)
	for line in lines:
	self.log("Output: "+line)
	elif rc == 0:
	ResourceNodes.append(node)

	return ResourceNodes

	def find_partitions(self):
	ccm_partitions = []

	for node in self.Env["nodes"]:
	if self.ShouldBeStatus[node] == "up":
	partition = self.rsh(node, self.templates["PartitionCmd"], 1)

	if not partition:
	self.log("no partition details for %s" % node)
	elif len(partition) > 2:
	nodes = partition.split()
	nodes.sort()
	partition = ' '.join(nodes)

	found = 0
	for a_partition in ccm_partitions:
	if partition == a_partition:
	found = 1
	if found == 0:
	self.debug("Adding partition from %s: %s" % (node, partition))
	ccm_partitions.append(partition)
	else:
	self.debug("Partition '%s' from %s is consistent with existing entries" % (partition, node))

	else:
	self.log("bad partition details for %s" % node)
	else:
	self.debug("Node %s is down... skipping" % node)

	self.debug("Found partitions: %s" % repr(ccm_partitions) )
	return ccm_partitions

	def HasQuorum(self, node_list):
	# If we are auditing a partition, then one side will
	# have quorum and the other not.
	# So the caller needs to tell us which we are checking
	# If no value for node_list is specified... assume all nodes
	if not node_list:
	node_list = self.Env["nodes"]

	for node in node_list:
	if self.ShouldBeStatus[node] == "up":
	quorum = self.rsh(node, self.templates["QuorumCmd"], 1)
	if quorum.find("1") != -1:
	return 1
	elif quorum.find("0") != -1:
	return 0
	else:
	self.debug("WARN: Unexpected quorum test result from " + node + ":" + quorum)

	return 0

	def Components(self):
	complist = []
	common_ignore = [
	"Pending action:",
	"(ERROR\|error): crm_log_message_adv:",
	"(ERROR\|error): MSG: No message to dump",
	"pending LRM operations at shutdown",
	"Lost connection to the CIB manager",
	"Connection to the CIB terminated...",
	"Sending message to the CIB manager FAILED",
	"Action A_RECOVER .* not supported",
	"(ERROR\|error): stonithd_op_result_ready: not signed on",
	"pingd.*(ERROR\|error): send_update: Could not send update",
	"send_ipc_message: IPC Channel to .* is not connected",
	"unconfirmed_actions: Waiting on .* unconfirmed actions",
	"cib_native_msgready: Message pending on command channel",
	r": Performing A_EXIT_1 - forcefully exiting ",
	r"Resource .* was active at shutdown. You may ignore this error if it is unmanaged.",
	]

	stonith_ignore = [
	r"Updating failcount for child_DoFencing",
	r"error.*: Fencer connection failed $will retry$",
	"pacemaker-execd.*(ERROR\|error): stonithd_receive_ops_result failed.",
	]

	stonith_ignore.extend(common_ignore)

	ccm = Process(self, "ccm", triggersreboot=self.fastfail, pats = [
	"State transition .* S_RECOVERY",
	"pacemaker-controld.Action A_RECOVER . not supported",
	r"pacemaker-controld.: Input I_TERMINATE .from do_recover",
	r"pacemaker-controld.*: Could not recover from internal error",
	"pacemaker-controld.I_ERROR.crmd_cib_connection_destroy",
	# these status numbers are likely wrong now
	r"pacemaker-controld.*exited with status 2",
	r"attrd.*exited with status 1",
	r"cib.*exited with status 2",

	# Not if it was fenced
	# "A new node joined the cluster",

	# "WARN: determine_online_status: Node .* is unclean",
	# "Scheduling Node .* for STONITH",
	# "Executing .* fencing operation",
	# "tengine_stonith_callback: .*result=0",
	# "Processing I_NODE_JOIN:.* cause=C_HA_MESSAGE",
	# "State transition S_.* -> S_INTEGRATION.*input=I_NODE_JOIN",
	"State transition S_STARTING -> S_PENDING",
	], badnews_ignore = common_ignore)

	based = Process(self, "pacemaker-based", triggersreboot=self.fastfail, pats = [
	"State transition .* S_RECOVERY",
	"Lost connection to the CIB manager",
	"Connection to the CIB manager terminated",
	r"pacemaker-controld.: Input I_TERMINATE .from do_recover",
	"pacemaker-controld.I_ERROR.crmd_cib_connection_destroy",
	r"pacemaker-controld.*: Could not recover from internal error",
	# these status numbers are likely wrong now
	r"pacemaker-controld.*exited with status 2",
	r"attrd.*exited with status 1",
	], badnews_ignore = common_ignore)

	execd = Process(self, "pacemaker-execd", triggersreboot=self.fastfail, pats = [
	"State transition .* S_RECOVERY",
	"LRM Connection failed",
	"pacemaker-controld.I_ERROR.lrm_connection_destroy",
	"State transition S_STARTING -> S_PENDING",
	r"pacemaker-controld.: Input I_TERMINATE .from do_recover",
	r"pacemaker-controld.*: Could not recover from internal error",
	# this status number is likely wrong now
	r"pacemaker-controld.*exited with status 2",
	], badnews_ignore = common_ignore)

	controld = Process(self, "pacemaker-controld", triggersreboot=self.fastfail,
	pats = [
	# "WARN: determine_online_status: Node .* is unclean",
	# "Scheduling Node .* for STONITH",
	# "Executing .* fencing operation",
	# "tengine_stonith_callback: .*result=0",
	"State transition .* S_IDLE",
	"State transition S_STARTING -> S_PENDING",
	], badnews_ignore = common_ignore)

	schedulerd = Process(self, "pacemaker-schedulerd", triggersreboot=self.fastfail, pats = [
	"State transition .* S_RECOVERY",
	r"pacemaker-controld.: Input I_TERMINATE .from do_recover",
	r"pacemaker-controld.*: Could not recover from internal error",
	r"pacemaker-controld.CRIT.: Connection to the scheduler failed",
	"pacemaker-controld.I_ERROR.save_cib_contents",
	# this status number is likely wrong now
	r"pacemaker-controld.*exited with status 2",
	], badnews_ignore = common_ignore, dc_only=1)

	if self.Env["DoFencing"] == 1 :
	complist.append(Process(self, "stoniths", triggersreboot=self.fastfail, dc_pats = [
	r"pacemaker-controld.CRIT.: Fencing daemon connection failed",
	"Attempting connection to fencing daemon",
	], badnews_ignore = stonith_ignore))

	if self.fastfail == 0:
	ccm.pats.extend([
	# these status numbers are likely wrong now
	r"attrd.*exited with status 1",
	r"pacemaker-(based\|controld).*exited with status 2",
	])
	based.pats.extend([
	# these status numbers are likely wrong now
	r"attrd.*exited with status 1",
	r"pacemaker-controld.*exited with status 2",
	])
	execd.pats.extend([
	# these status numbers are likely wrong now
	r"pacemaker-controld.*exited with status 2",
	])

	complist.append(ccm)
	complist.append(based)
	complist.append(execd)
	complist.append(controld)
	complist.append(schedulerd)

	return complist

	def StandbyStatus(self, node):
	out=self.rsh(node, self.templates["StandbyQueryCmd"] % node, 1)
	if not out:
	return "off"
	out = out[:-1]
	self.debug("Standby result: "+out)
	return out

	# status == "on" : Enter Standby mode
	# status == "off": Enter Active mode
	def SetStandbyMode(self, node, status):
	current_status = self.StandbyStatus(node)
	cmd = self.templates["StandbyCmd"] % (node, status)
	ret = self.rsh(node, cmd)
	return True

	def AddDummyRsc(self, node, rid):
	rsc_xml = """ '<resources>
	<primitive class=\"ocf\" id=\"%s\" provider=\"pacemaker\" type=\"Dummy\">
	<operations>
	<op id=\"%s-interval-10s\" interval=\"10s\" name=\"monitor\"/
	</operations>
	</primitive>
	</resources>'""" % (rid, rid)
	constraint_xml = """ '<constraints>
	<rsc_location id=\"location-%s-%s\" node=\"%s\" rsc=\"%s\" score=\"INFINITY\"/>
	</constraints>'
	""" % (rid, node, node, rid)

	self.rsh(node, self.templates['CibAddXml'] % (rsc_xml))
	self.rsh(node, self.templates['CibAddXml'] % (constraint_xml))

	def RemoveDummyRsc(self, node, rid):
	constraint = "\"//rsc_location[@rsc='%s']\"" % (rid)
	rsc = "\"//primitive[@id='%s']\"" % (rid)

	self.rsh(node, self.templates['CibDelXpath'] % constraint)
	self.rsh(node, self.templates['CibDelXpath'] % rsc)
	diff --git a/cts/lab/remote.py b/cts/lab/remote.py
	index 35c741c1e0..96ecdacc3f 100644
	--- a/cts/lab/remote.py
	+++ b/cts/lab/remote.py
	@@ -1,292 +1,284 @@
	""" Remote command runner for Pacemaker's Cluster Test Suite (CTS)
	"""

	__copyright__ = "Copyright 2014-2020 the Pacemaker project contributors"
	__license__ = "GNU General Public License version 2 or later (GPLv2+) WITHOUT ANY WARRANTY"

	import re
	import os
	import sys

	from subprocess import Popen,PIPE
	from threading import Thread

	pdir=os.path.dirname(sys.path[0])
	sys.path.insert(0, pdir) # So that things work from the source directory

	from cts.CTSvars import *
	from cts.logging import *

	trace_rsh=None
	trace_lw=None

	def convert2string(lines):
	- if sys.version_info < (3, ):
	- return lines
	-
	if isinstance(lines, bytes):
	return lines.decode("utf-8")
	elif isinstance(lines, list):
	aList = []
	for line in lines:
	if isinstance(line, bytes):
	line = line.decode("utf-8")
	aList.append(line)
	return aList
	return lines

	-def input_wrapper(data):
	- if sys.version_info > (3,):
	- return input(data)
	- return raw_input(data)
	-
	class AsyncWaitProc(Thread):
	def __init__(self, proc, node, command, completionDelegate=None):
	self.proc = proc
	self.node = node
	self.command = command
	self.logger = LogFactory()
	self.delegate = completionDelegate;
	Thread.__init__(self)

	def run(self):
	outLines = None
	errLines = None
	self.logger.debug("cmd: async: target=%s, pid=%d: %s" % (self.node, self.proc.pid, self.command))

	self.proc.wait()
	self.logger.debug("cmd: pid %d returned %d" % (self.proc.pid, self.proc.returncode))

	if self.proc.stderr:
	errLines = self.proc.stderr.readlines()
	self.proc.stderr.close()
	for line in errLines:
	self.logger.debug("cmd: stderr[%d]: %s" % (self.proc.pid, line))

	errLines = convert2string(errLines)

	if self.proc.stdout:
	outLines = self.proc.stdout.readlines()
	self.proc.stdout.close()

	outLines = convert2string(outLines)

	# for line in outLines:
	# self.logger.debug("cmd: stdout[%d]: %s" % (self.proc.pid, line))

	if self.delegate:
	self.delegate.async_complete(self.proc.pid, self.proc.returncode, outLines, errLines)

	class AsyncRemoteCmd(Thread):
	def __init__(self, node, command, completionDelegate=None):
	self.proc = None
	self.node = node
	self.command = command
	self.logger = LogFactory()
	self.delegate = completionDelegate;
	Thread.__init__(self)

	def run(self):
	outLines = None
	errLines = None

	self.proc = Popen(self.command, stdout = PIPE, stderr = PIPE, close_fds = True, shell = True)

	self.logger.debug("cmd: async: target=%s, pid=%d: %s" % (self.node, self.proc.pid, self.command))
	self.proc.wait()
	self.logger.debug("cmd: pid %d returned %d to %s" % (self.proc.pid, self.proc.returncode, repr(self.delegate)))

	if self.proc.stderr:
	errLines = self.proc.stderr.readlines()
	self.proc.stderr.close()
	for line in errLines:
	self.logger.debug("cmd: stderr[%d]: %s" % (self.proc.pid, line))
	errLines = convert2string(errLines)

	if self.proc.stdout:
	outLines = self.proc.stdout.readlines()
	self.proc.stdout.close()
	outLines = convert2string(outLines)

	# for line in outLines:
	# self.logger.log("cmd: stdout[%d]: %s" % (self.proc.pid, line))

	if self.delegate:
	self.delegate.async_complete(self.proc.pid, self.proc.returncode, outLines, errLines)

	class RemotePrimitives(object):
	def __init__(self, Command=None, CpCommand=None):
	if CpCommand:
	self.CpCommand = CpCommand
	else:
	# -B: batch mode, -q: no stats (quiet)
	self.CpCommand = "scp -B -q"

	if Command:
	self.Command = Command
	else:
	# -n: no stdin, -x: no X11,
	# -o ServerAliveInterval=5 disconnect after 3*5s if the server stops responding
	self.Command = "ssh -l root -n -x -o ServerAliveInterval=5 -o ConnectTimeout=10 -o TCPKeepAlive=yes -o ServerAliveCountMax=3 "

	class RemoteExec(object):
	'''This is an abstract remote execution class. It runs a command on another
	machine - somehow. The somehow is up to us. This particular
	class uses ssh.
	Most of the work is done by fork/exec of ssh or scp.
	'''

	def __init__(self, rsh, silent=False):
	self.rsh = rsh
	self.silent = silent
	self.logger = LogFactory()

	if trace_rsh:
	self.silent = False

	self.OurNode=os.uname()[1].lower()

	def _fixcmd(self, cmd):
	return re.sub("\'", "'\\''", cmd)

	def _cmd(self, *args):

	'''Compute the string that will run the given command on the
	given remote system'''

	args= args[0]
	sysname = args[0]
	command = args[1]

	#print("sysname: %s, us: %s" % (sysname, self.OurNode))
	if sysname == None or sysname.lower() == self.OurNode or sysname == "localhost":
	ret = command
	else:
	ret = self.rsh.Command + " " + sysname + " '" + self._fixcmd(command) + "'"
	#print ("About to run %s\n" % ret)
	return ret

	def log(self, args):
	if not self.silent:
	self.logger.log(args)

	def debug(self, args):
	if not self.silent:
	self.logger.debug(args)

	def call_async(self, node, command, completionDelegate=None):
	#if completionDelegate: print("Waiting for %d on %s: %s" % (proc.pid, node, command))
	aproc = AsyncRemoteCmd(node, self._cmd([node, command]), completionDelegate=completionDelegate)
	aproc.start()
	return aproc


	def __call__(self, node, command, stdout=0, synchronous=1, silent=False, blocking=True, completionDelegate=None):
	'''Run the given command on the given remote system
	If you call this class like a function, this is the function that gets
	called. It just runs it roughly as though it were a system() call
	on the remote machine. The first argument is name of the machine to
	run it on.
	'''

	if trace_rsh:
	silent = False

	rc = 0
	result = None
	proc = Popen(self._cmd([node, command]),
	stdout = PIPE, stderr = PIPE, close_fds = True, shell = True)

	#if completionDelegate: print("Waiting for %d on %s: %s" % (proc.pid, node, command))
	if not synchronous and proc.pid > 0 and not self.silent:
	aproc = AsyncWaitProc(proc, node, command, completionDelegate=completionDelegate)
	aproc.start()
	return 0

	#if not blocking:
	# import fcntl
	# fcntl.fcntl(proc.stdout.fileno(), fcntl.F_SETFL, os.O_NONBLOCK)

	if proc.stdout:
	if stdout == 1:
	result = proc.stdout.readline()
	else:
	result = proc.stdout.readlines()
	proc.stdout.close()
	else:
	self.log("No stdout stream")

	rc = proc.wait()

	if not silent: self.debug("cmd: target=%s, rc=%d: %s" % (node, rc, command))

	result = convert2string(result)

	if proc.stderr:
	errors = proc.stderr.readlines()
	proc.stderr.close()

	if stdout == 1:
	return result

	if completionDelegate:
	completionDelegate.async_complete(proc.pid, proc.returncode, result, errors)

	if not silent:
	for err in errors:
	self.debug("cmd: stderr: %s" % err)

	if stdout == 0:
	if not silent and result:
	for line in result:
	self.debug("cmd: stdout: %s" % line)
	return rc

	return (rc, result)

	def cp(self, source, target, silent=False):
	'''Perform a remote copy'''
	cpstring = self.rsh.CpCommand + " \'" + source + "\'" + " \'" + target + "\'"
	rc = os.system(cpstring)
	if trace_rsh:
	silent = False
	if not silent: self.debug("cmd: rc=%d: %s" % (rc, cpstring))

	return rc

	def exists_on_all(self, filename, hosts, test="r"):
	""" Return True if specified file exists on all specified hosts. """

	for host in hosts:
	rc = self(host, "test -%s %s" % (test, filename))
	if rc != 0:
	return False
	return True


	class RemoteFactory(object):
	# Class variables
	rsh = RemotePrimitives()
	instance = None

	def getInstance(self):
	if not RemoteFactory.instance:
	RemoteFactory.instance = RemoteExec(RemoteFactory.rsh, False)
	return RemoteFactory.instance

	def new(self, silent=False):
	return RemoteExec(RemoteFactory.rsh, silent)

	def enable_docker(self):
	print("Using DOCKER backend for connections to cluster nodes")

	RemoteFactory.rsh.Command = "/usr/libexec/phd/docker/phd_docker_remote_cmd "
	RemoteFactory.rsh.CpCommand = "/usr/libexec/phd/docker/phd_docker_cp"

	def enable_qarsh(self):
	# http://nstraz.wordpress.com/2008/12/03/introducing-qarsh/
	print("Using QARSH for connections to cluster nodes")

	RemoteFactory.rsh.Command = "qarsh -t 300 -l root"
	RemoteFactory.rsh.CpCommand = "qacp -q"

	diff --git a/daemons/based/based_remote.c b/daemons/based/based_remote.c
	index 7a3f5dc52b..6917f7bbd0 100644
	--- a/daemons/based/based_remote.c
	+++ b/daemons/based/based_remote.c
	@@ -1,686 +1,685 @@
	/*
	* Copyright 2004-2021 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>
	#include <crm/crm.h>

	#include <sys/param.h>
	#include <stdio.h>
	#include <sys/types.h>
	#include <sys/stat.h>
	#include <unistd.h>
	#include <inttypes.h> // PRIx64
	#include <sys/socket.h>
	#include <arpa/inet.h>

	#include <netinet/ip.h>

	#include <stdlib.h>
	#include <errno.h>
	#include <glib.h>

	#include <crm/msg_xml.h>
	#include <crm/common/ipc.h>
	#include <crm/common/ipc_internal.h>
	#include <crm/common/xml.h>
	#include <crm/common/remote_internal.h>
	#include <crm/cib/internal.h>

	#include "pacemaker-based.h"

	/* #undef HAVE_PAM_PAM_APPL_H */
	/* #undef HAVE_GNUTLS_GNUTLS_H */

	#ifdef HAVE_GNUTLS_GNUTLS_H
	-# undef KEYFILE
	# include <gnutls/gnutls.h>
	#endif

	#include <pwd.h>
	#include <grp.h>
	#if HAVE_SECURITY_PAM_APPL_H
	# include <security/pam_appl.h>
	# define HAVE_PAM 1
	#else
	# if HAVE_PAM_PAM_APPL_H
	# include <pam/pam_appl.h>
	# define HAVE_PAM 1
	# endif
	#endif

	extern int remote_tls_fd;
	extern gboolean cib_shutdown_flag;

	int init_remote_listener(int port, gboolean encrypted);
	void cib_remote_connection_destroy(gpointer user_data);

	#ifdef HAVE_GNUTLS_GNUTLS_H
	gnutls_dh_params_t dh_params;
	gnutls_anon_server_credentials_t anon_cred_s;
	static void
	debug_log(int level, const char *str)
	{
	fputs(str, stderr);
	}
	#endif

	#define REMOTE_AUTH_TIMEOUT 10000

	int num_clients;
	int authenticate_user(const char user, const char passwd);
	static int cib_remote_listen(gpointer data);
	static int cib_remote_msg(gpointer data);

	static void
	remote_connection_destroy(gpointer user_data)
	{
	crm_info("No longer listening for remote connections");
	return;
	}

	int
	init_remote_listener(int port, gboolean encrypted)
	{
	int rc;
	int *ssock = NULL;
	struct sockaddr_in saddr;
	int optval;

	static struct mainloop_fd_callbacks remote_listen_fd_callbacks = {
	.dispatch = cib_remote_listen,
	.destroy = remote_connection_destroy,
	};

	if (port <= 0) {
	/* don't start it */
	return 0;
	}

	if (encrypted) {
	#ifndef HAVE_GNUTLS_GNUTLS_H
	crm_warn("TLS support is not available");
	return 0;
	#else
	crm_notice("Starting TLS listener on port %d", port);
	crm_gnutls_global_init();
	/* gnutls_global_set_log_level (10); */
	gnutls_global_set_log_function(debug_log);
	if (pcmk__init_tls_dh(&dh_params) != pcmk_rc_ok) {
	return -1;
	}
	gnutls_anon_allocate_server_credentials(&anon_cred_s);
	gnutls_anon_set_server_dh_params(anon_cred_s, dh_params);
	#endif
	} else {
	crm_warn("Starting plain-text listener on port %d", port);
	}
	#ifndef HAVE_PAM
	crm_warn("PAM is _not_ enabled!");
	#endif

	/* create server socket */
	ssock = malloc(sizeof(int));
	if(ssock == NULL) {
	crm_perror(LOG_ERR, "Listener socket allocation failed");
	return -1;
	}

	*ssock = socket(AF_INET, SOCK_STREAM, 0);
	if (*ssock == -1) {
	crm_perror(LOG_ERR, "Listener socket creation failed");
	free(ssock);
	return -1;
	}

	/* reuse address */
	optval = 1;
	rc = setsockopt(*ssock, SOL_SOCKET, SO_REUSEADDR, &optval, sizeof(optval));
	if (rc < 0) {
	crm_perror(LOG_WARNING,
	"Local address reuse not allowed on listener socket");
	}

	/* bind server socket */
	memset(&saddr, '\0', sizeof(saddr));
	saddr.sin_family = AF_INET;
	saddr.sin_addr.s_addr = INADDR_ANY;
	saddr.sin_port = htons(port);
	if (bind(ssock, (struct sockaddr )&saddr, sizeof(saddr)) == -1) {
	crm_perror(LOG_ERR, "Cannot bind to listener socket");
	close(*ssock);
	free(ssock);
	return -2;
	}
	if (listen(*ssock, 10) == -1) {
	crm_perror(LOG_ERR, "Cannot listen on socket");
	close(*ssock);
	free(ssock);
	return -3;
	}

	mainloop_add_fd("cib-remote", G_PRIORITY_DEFAULT, *ssock, ssock, &remote_listen_fd_callbacks);
	crm_debug("Started listener on port %d", port);

	return *ssock;
	}

	static int
	check_group_membership(const char usr, const char grp)
	{
	int index = 0;
	struct passwd *pwd = NULL;
	struct group *group = NULL;

	CRM_CHECK(usr != NULL, return FALSE);
	CRM_CHECK(grp != NULL, return FALSE);

	pwd = getpwnam(usr);
	if (pwd == NULL) {
	crm_err("No user named '%s' exists!", usr);
	return FALSE;
	}

	group = getgrgid(pwd->pw_gid);
	if (group != NULL && pcmk__str_eq(grp, group->gr_name, pcmk__str_none)) {
	return TRUE;
	}

	group = getgrnam(grp);
	if (group == NULL) {
	crm_err("No group named '%s' exists!", grp);
	return FALSE;
	}

	while (TRUE) {
	char *member = group->gr_mem[index++];

	if (member == NULL) {
	break;

	} else if (pcmk__str_eq(usr, member, pcmk__str_none)) {
	return TRUE;
	}
	};

	return FALSE;
	}

	static gboolean
	cib_remote_auth(xmlNode * login)
	{
	const char *user = NULL;
	const char *pass = NULL;
	const char *tmp = NULL;

	crm_log_xml_info(login, "Login: ");
	if (login == NULL) {
	return FALSE;
	}

	tmp = crm_element_name(login);
	if (!pcmk__str_eq(tmp, "cib_command", pcmk__str_casei)) {
	crm_err("Wrong tag: %s", tmp);
	return FALSE;
	}

	tmp = crm_element_value(login, "op");
	if (!pcmk__str_eq(tmp, "authenticate", pcmk__str_casei)) {
	crm_err("Wrong operation: %s", tmp);
	return FALSE;
	}

	user = crm_element_value(login, "user");
	pass = crm_element_value(login, "password");

	if (!user \|\| !pass) {
	crm_err("missing auth credentials");
	return FALSE;
	}

	/* Non-root daemons can only validate the password of the
	* user they're running as
	*/
	if (check_group_membership(user, CRM_DAEMON_GROUP) == FALSE) {
	crm_err("User is not a member of the required group");
	return FALSE;

	} else if (authenticate_user(user, pass) == FALSE) {
	crm_err("PAM auth failed");
	return FALSE;
	}

	return TRUE;
	}

	static gboolean
	remote_auth_timeout_cb(gpointer data)
	{
	pcmk__client_t *client = data;

	client->remote->auth_timeout = 0;

	if (client->remote->authenticated == TRUE) {
	return FALSE;
	}

	mainloop_del_fd(client->remote->source);
	crm_err("Remote client authentication timed out");

	return FALSE;
	}

	static int
	cib_remote_listen(gpointer data)
	{
	int csock = 0;
	unsigned laddr;
	struct sockaddr_storage addr;
	char ipstr[INET6_ADDRSTRLEN];
	int ssock = (int )data;
	int rc;

	pcmk__client_t *new_client = NULL;

	static struct mainloop_fd_callbacks remote_client_fd_callbacks = {
	.dispatch = cib_remote_msg,
	.destroy = cib_remote_connection_destroy,
	};

	/* accept the connection */
	laddr = sizeof(addr);
	memset(&addr, 0, sizeof(addr));
	csock = accept(ssock, (struct sockaddr *)&addr, &laddr);
	if (csock == -1) {
	crm_perror(LOG_ERR, "Could not accept socket connection");
	return TRUE;
	}

	pcmk__sockaddr2str(&addr, ipstr);
	crm_debug("New %s connection from %s",
	((ssock == remote_tls_fd)? "secure" : "clear-text"), ipstr);

	rc = pcmk__set_nonblocking(csock);
	if (rc != pcmk_rc_ok) {
	crm_err("Could not set socket non-blocking: %s " CRM_XS " rc=%d",
	pcmk_rc_str(rc), rc);
	close(csock);
	return TRUE;
	}

	num_clients++;

	new_client = pcmk__new_unauth_client(NULL);
	new_client->remote = calloc(1, sizeof(pcmk__remote_t));

	if (ssock == remote_tls_fd) {
	#ifdef HAVE_GNUTLS_GNUTLS_H
	pcmk__set_client_flags(new_client, pcmk__client_tls);

	/* create gnutls session for the server socket */
	new_client->remote->tls_session = pcmk__new_tls_session(csock,
	GNUTLS_SERVER,
	GNUTLS_CRD_ANON,
	anon_cred_s);
	if (new_client->remote->tls_session == NULL) {
	close(csock);
	return TRUE;
	}
	#endif
	} else {
	pcmk__set_client_flags(new_client, pcmk__client_tcp);
	new_client->remote->tcp_socket = csock;
	}

	// Require the client to authenticate within this time
	new_client->remote->auth_timeout = g_timeout_add(REMOTE_AUTH_TIMEOUT,
	remote_auth_timeout_cb,
	new_client);
	crm_info("Remote CIB client pending authentication "
	CRM_XS " %p id: %s", new_client, new_client->id);

	new_client->remote->source =
	mainloop_add_fd("cib-remote-client", G_PRIORITY_DEFAULT, csock, new_client,
	&remote_client_fd_callbacks);

	return TRUE;
	}

	void
	cib_remote_connection_destroy(gpointer user_data)
	{
	pcmk__client_t *client = user_data;
	int csock = 0;

	if (client == NULL) {
	return;
	}

	crm_trace("Cleaning up after client %s disconnect",
	pcmk__client_name(client));

	num_clients--;
	crm_trace("Num unfree'd clients: %d", num_clients);

	switch (PCMK__CLIENT_TYPE(client)) {
	case pcmk__client_tcp:
	csock = client->remote->tcp_socket;
	break;
	#ifdef HAVE_GNUTLS_GNUTLS_H
	case pcmk__client_tls:
	if (client->remote->tls_session) {
	void sock_ptr = gnutls_transport_get_ptr(client->remote->tls_session);

	csock = GPOINTER_TO_INT(sock_ptr);
	if (client->remote->tls_handshake_complete) {
	gnutls_bye(*client->remote->tls_session, GNUTLS_SHUT_WR);
	}
	gnutls_deinit(*client->remote->tls_session);
	gnutls_free(client->remote->tls_session);
	client->remote->tls_session = NULL;
	}
	break;
	#endif
	default:
	crm_warn("Unknown transport for client %s "
	CRM_XS " flags=0x%016" PRIx64,
	pcmk__client_name(client), client->flags);
	}

	if (csock > 0) {
	close(csock);
	}

	pcmk__free_client(client);

	crm_trace("Freed the cib client");

	if (cib_shutdown_flag) {
	cib_shutdown(0);
	}
	return;
	}

	static void
	cib_handle_remote_msg(pcmk__client_t client, xmlNode command)
	{
	const char *value = NULL;

	value = crm_element_name(command);
	if (!pcmk__str_eq(value, "cib_command", pcmk__str_casei)) {
	crm_log_xml_trace(command, "Bad command: ");
	return;
	}

	if (client->name == NULL) {
	value = crm_element_value(command, F_CLIENTNAME);
	if (value == NULL) {
	client->name = strdup(client->id);
	} else {
	client->name = strdup(value);
	}
	}

	if (client->userdata == NULL) {
	value = crm_element_value(command, F_CIB_CALLBACK_TOKEN);
	if (value != NULL) {
	client->userdata = strdup(value);
	crm_trace("Callback channel for %s is %s", client->id, (char*)client->userdata);

	} else {
	client->userdata = strdup(client->id);
	}
	}

	/* unset dangerous options */
	xml_remove_prop(command, F_ORIG);
	xml_remove_prop(command, F_CIB_HOST);
	xml_remove_prop(command, F_CIB_GLOBAL_UPDATE);

	crm_xml_add(command, F_TYPE, T_CIB);
	crm_xml_add(command, F_CIB_CLIENTID, client->id);
	crm_xml_add(command, F_CIB_CLIENTNAME, client->name);
	crm_xml_add(command, F_CIB_USER, client->user);

	if (crm_element_value(command, F_CIB_CALLID) == NULL) {
	char *call_uuid = crm_generate_uuid();

	/* fix the command */
	crm_xml_add(command, F_CIB_CALLID, call_uuid);
	free(call_uuid);
	}

	if (crm_element_value(command, F_CIB_CALLOPTS) == NULL) {
	crm_xml_add_int(command, F_CIB_CALLOPTS, 0);
	}

	crm_log_xml_trace(command, "Remote command: ");
	cib_common_callback_worker(0, 0, command, client, TRUE);
	}

	static int
	cib_remote_msg(gpointer data)
	{
	xmlNode *command = NULL;
	pcmk__client_t *client = data;
	int rc;
	int timeout = client->remote->authenticated ? -1 : 1000;

	crm_trace("Remote %s message received for client %s",
	pcmk__client_type_str(PCMK__CLIENT_TYPE(client)),
	pcmk__client_name(client));

	#ifdef HAVE_GNUTLS_GNUTLS_H
	if ((PCMK__CLIENT_TYPE(client) == pcmk__client_tls)
	&& !(client->remote->tls_handshake_complete)) {

	int rc = pcmk__read_handshake_data(client);

	if (rc == EAGAIN) {
	/* No more data is available at the moment. Just return for now;
	* we'll get invoked again once the client sends more.
	*/
	return 0;
	} else if (rc != pcmk_rc_ok) {
	return -1;
	}

	crm_debug("TLS handshake with remote CIB client completed");
	client->remote->tls_handshake_complete = TRUE;
	if (client->remote->auth_timeout) {
	g_source_remove(client->remote->auth_timeout);
	}

	// Require the client to authenticate within this time
	client->remote->auth_timeout = g_timeout_add(REMOTE_AUTH_TIMEOUT,
	remote_auth_timeout_cb,
	client);
	return 0;
	}
	#endif

	rc = pcmk__read_remote_message(client->remote, timeout);

	/* must pass auth before we will process anything else */
	if (client->remote->authenticated == FALSE) {
	xmlNode *reg;
	const char *user = NULL;

	command = pcmk__remote_message_xml(client->remote);
	if (cib_remote_auth(command) == FALSE) {
	free_xml(command);
	return -1;
	}

	crm_notice("Remote CIB client connection accepted");
	client->remote->authenticated = TRUE;
	g_source_remove(client->remote->auth_timeout);
	client->remote->auth_timeout = 0;
	client->name = crm_element_value_copy(command, "name");

	user = crm_element_value(command, "user");
	if (user) {
	client->user = strdup(user);
	}

	/* send ACK */
	reg = create_xml_node(NULL, "cib_result");
	crm_xml_add(reg, F_CIB_OPERATION, CRM_OP_REGISTER);
	crm_xml_add(reg, F_CIB_CLIENTID, client->id);
	pcmk__remote_send_xml(client->remote, reg);
	free_xml(reg);
	free_xml(command);
	}

	command = pcmk__remote_message_xml(client->remote);
	while (command) {
	crm_trace("Remote client message received");
	cib_handle_remote_msg(client, command);
	free_xml(command);
	command = pcmk__remote_message_xml(client->remote);
	}

	if (rc == ENOTCONN) {
	crm_trace("Remote CIB client disconnected while reading from it");
	return -1;
	}

	return 0;
	}

	#ifdef HAVE_PAM
	static int
	construct_pam_passwd(int num_msg, const struct pam_message **msg,
	struct pam_response *response, void data)
	{
	int count = 0;
	struct pam_response *reply;
	char string = (char )data;

	CRM_CHECK(data, return PAM_CONV_ERR);
	CRM_CHECK(num_msg == 1, return PAM_CONV_ERR); /* We only want to handle one message */

	reply = calloc(1, sizeof(struct pam_response));
	CRM_ASSERT(reply != NULL);

	for (count = 0; count < num_msg; ++count) {
	switch (msg[count]->msg_style) {
	case PAM_TEXT_INFO:
	crm_info("PAM: %s", msg[count]->msg);
	break;
	case PAM_PROMPT_ECHO_OFF:
	case PAM_PROMPT_ECHO_ON:
	reply[count].resp_retcode = 0;
	reply[count].resp = string; /* We already made a copy */
	case PAM_ERROR_MSG:
	/* In theory we'd want to print this, but then
	* we see the password prompt in the logs
	*/
	/* crm_err("PAM error: %s", msg[count]->msg); */
	break;
	default:
	crm_err("Unhandled conversation type: %d", msg[count]->msg_style);
	goto bail;
	}
	}

	*response = reply;
	reply = NULL;

	return PAM_SUCCESS;

	bail:
	for (count = 0; count < num_msg; ++count) {
	if (reply[count].resp != NULL) {
	switch (msg[count]->msg_style) {
	case PAM_PROMPT_ECHO_ON:
	case PAM_PROMPT_ECHO_OFF:
	/* Erase the data - it contained a password */
	while (*(reply[count].resp)) {
	*(reply[count].resp)++ = '\0';
	}
	free(reply[count].resp);
	break;
	}
	reply[count].resp = NULL;
	}
	}
	free(reply);
	reply = NULL;

	return PAM_CONV_ERR;
	}
	#endif

	int
	authenticate_user(const char user, const char passwd)
	{
	#ifndef HAVE_PAM
	gboolean pass = TRUE;
	#else
	int rc = 0;
	gboolean pass = FALSE;
	const void *p_user = NULL;

	struct pam_conv p_conv;
	struct pam_handle *pam_h = NULL;
	static const char *pam_name = NULL;

	if (pam_name == NULL) {
	pam_name = getenv("CIB_pam_service");
	}
	if (pam_name == NULL) {
	pam_name = "login";
	}

	p_conv.conv = construct_pam_passwd;
	p_conv.appdata_ptr = strdup(passwd);

	rc = pam_start(pam_name, user, &p_conv, &pam_h);
	if (rc != PAM_SUCCESS) {
	crm_err("Could not initialize PAM: %s (%d)", pam_strerror(pam_h, rc), rc);
	goto bail;
	}

	rc = pam_authenticate(pam_h, 0);
	if (rc != PAM_SUCCESS) {
	crm_err("Authentication failed for %s: %s (%d)", user, pam_strerror(pam_h, rc), rc);
	goto bail;
	}

	/* Make sure we authenticated the user we wanted to authenticate.
	* Since we also run as non-root, it might be worth pre-checking
	* the user has the same EID as us, since that the only user we
	* can authenticate.
	*/
	rc = pam_get_item(pam_h, PAM_USER, &p_user);
	if (rc != PAM_SUCCESS) {
	crm_err("Internal PAM error: %s (%d)", pam_strerror(pam_h, rc), rc);
	goto bail;

	} else if (p_user == NULL) {
	crm_err("Unknown user authenticated.");
	goto bail;

	} else if (!pcmk__str_eq(p_user, user, pcmk__str_casei)) {
	crm_err("User mismatch: %s vs. %s.", (const char )p_user, (const char )user);
	goto bail;
	}

	rc = pam_acct_mgmt(pam_h, 0);
	if (rc != PAM_SUCCESS) {
	crm_err("Access denied: %s (%d)", pam_strerror(pam_h, rc), rc);
	goto bail;
	}
	pass = TRUE;

	bail:
	pam_end(pam_h, rc);
	#endif
	return pass;
	}
	diff --git a/daemons/based/pacemaker-based.h b/daemons/based/pacemaker-based.h
	index 421a7b9783..fb8c99d37b 100644
	--- a/daemons/based/pacemaker-based.h
	+++ b/daemons/based/pacemaker-based.h
	@@ -1,159 +1,158 @@
	/*
	- * Copyright 2004-2020 the Pacemaker project contributors
	+ * Copyright 2004-2021 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#ifndef PACEMAKER_BASED__H
	# define PACEMAKER_BASED__H

	#include <stdio.h>
	#include <sys/types.h>
	#include <unistd.h>
	#include <stdlib.h>
	#include <glib.h>
	#include <errno.h>
	#include <fcntl.h>

	#include <crm/crm.h>
	#include <crm/cib.h>
	#include <crm/common/xml.h>
	#include <crm/cluster.h>
	#include <crm/common/ipc_internal.h>
	#include <crm/common/mainloop.h>
	#include <crm/cib/internal.h>

	#ifdef HAVE_GNUTLS_GNUTLS_H
	-# undef KEYFILE
	# include <gnutls/gnutls.h>
	#endif

	// CIB-specific client flags
	enum cib_client_flags {
	// Notifications
	cib_notify_pre = (UINT64_C(1) << 0),
	cib_notify_post = (UINT64_C(1) << 1),
	cib_notify_replace = (UINT64_C(1) << 2),
	cib_notify_confirm = (UINT64_C(1) << 3),
	cib_notify_diff = (UINT64_C(1) << 4),

	// Whether client is another cluster daemon
	cib_is_daemon = (UINT64_C(1) << 12),
	};

	typedef struct cib_operation_s {
	const char *operation;
	gboolean modifies_cib;
	gboolean needs_privileges;
	gboolean needs_quorum;
	int (prepare) (xmlNode , xmlNode , const char );
	int (cleanup) (int, xmlNode , xmlNode *);
	int (fn) (const char , int, const char , xmlNode ,
	xmlNode , xmlNode , xmlNode , xmlNode );
	} cib_operation_t;

	extern gboolean cib_is_master;
	extern GHashTable *peer_hash;
	extern GHashTable *config_hash;
	extern xmlNode *the_cib;
	extern crm_trigger_t *cib_writer;
	extern gboolean cib_writes_enabled;

	extern GMainLoop *mainloop;
	extern crm_cluster_t crm_cluster;
	extern GHashTable *local_notify_queue;
	extern gboolean legacy_mode;
	extern gboolean stand_alone;
	extern gboolean cib_shutdown_flag;
	extern const char *cib_root;
	extern char *cib_our_uname;
	extern int cib_status;
	extern FILE *msg_cib_strm;

	extern struct qb_ipcs_service_handlers ipc_ro_callbacks;
	extern struct qb_ipcs_service_handlers ipc_rw_callbacks;
	extern qb_ipcs_service_t *ipcs_ro;
	extern qb_ipcs_service_t *ipcs_rw;
	extern qb_ipcs_service_t *ipcs_shm;

	void cib_peer_callback(xmlNode msg, void private_data);
	void cib_common_callback_worker(uint32_t id, uint32_t flags,
	xmlNode op_request, pcmk__client_t cib_client,
	gboolean privileged);
	void cib_shutdown(int nsig);
	void initiate_exit(void);
	void terminate_cib(const char *caller, int fast);
	gboolean cib_legacy_mode(void);

	gboolean uninitializeCib(void);
	gboolean verifyCibXml(xmlNode *cib);
	xmlNode readCibXml(char buffer);
	xmlNode readCibXmlFile(const char dir, const char *file,
	gboolean discard_status);
	int activateCibXml(xmlNode doc, gboolean to_disk, const char op);

	xmlNode createCibRequest(gboolean isLocal, const char operation,
	const char section, const char verbose,
	xmlNode *data);
	int cib_process_shutdown_req(const char op, int options, const char section,
	xmlNode req, xmlNode input,
	xmlNode existing_cib, xmlNode *result_cib,
	xmlNode **answer);
	int cib_process_default(const char op, int options, const char section,
	xmlNode req, xmlNode input, xmlNode *existing_cib,
	xmlNode result_cib, xmlNode answer);
	int cib_process_ping(const char op, int options, const char section,
	xmlNode req, xmlNode input, xmlNode *existing_cib,
	xmlNode result_cib, xmlNode answer);
	int cib_process_readwrite(const char op, int options, const char section,
	xmlNode req, xmlNode input, xmlNode *existing_cib,
	xmlNode result_cib, xmlNode answer);
	int cib_process_replace_svr(const char op, int options, const char section,
	xmlNode req, xmlNode input, xmlNode *existing_cib,
	xmlNode result_cib, xmlNode answer);
	int cib_server_process_diff(const char op, int options, const char section,
	xmlNode req, xmlNode input, xmlNode *existing_cib,
	xmlNode result_cib, xmlNode answer);
	int cib_process_sync(const char op, int options, const char section,
	xmlNode req, xmlNode input, xmlNode *existing_cib,
	xmlNode result_cib, xmlNode answer);
	int cib_process_sync_one(const char op, int options, const char section,
	xmlNode req, xmlNode input, xmlNode *existing_cib,
	xmlNode result_cib, xmlNode answer);
	int cib_process_delete_absolute(const char *op, int options,
	const char section, xmlNode req,
	xmlNode input, xmlNode existing_cib,
	xmlNode result_cib, xmlNode answer);
	int cib_process_upgrade_server(const char op, int options, const char section,
	xmlNode req, xmlNode input,
	xmlNode existing_cib, xmlNode *result_cib,
	xmlNode **answer);
	void send_sync_request(const char *host);

	xmlNode cib_msg_copy(xmlNode msg, gboolean with_data);
	xmlNode cib_construct_reply(xmlNode request, xmlNode *output, int rc);
	int cib_get_operation_id(const char op, int operation);
	cib_op_t *cib_op_func(int call_type);
	gboolean cib_op_modifies(int call_type);
	int cib_op_prepare(int call_type, xmlNode request, xmlNode *input,
	const char **section);
	int cib_op_cleanup(int call_type, int options, xmlNode **input,
	xmlNode **output);
	int cib_op_can_run(int call_type, int call_options, gboolean privileged,
	gboolean global_update);
	void cib_diff_notify(int options, const char client, const char call_id,
	const char op, xmlNode update, int result,
	xmlNode *old_cib);
	void cib_replace_notify(const char origin, xmlNode update, int result,
	xmlNode *diff, int change_section);

	static inline const char *
	cib_config_lookup(const char *opt)
	{
	return g_hash_table_lookup(config_hash, opt);
	}

	#endif // PACEMAKER_BASED__H
	diff --git a/daemons/execd/pacemaker-execd.h b/daemons/execd/pacemaker-execd.h
	index 3aa9e3aacc..51ef8d22e6 100644
	--- a/daemons/execd/pacemaker-execd.h
	+++ b/daemons/execd/pacemaker-execd.h
	@@ -1,103 +1,102 @@
	/*
	* Copyright 2012-2021 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#ifndef PACEMAKER_EXECD__H
	# define PACEMAKER_EXECD__H

	# include <glib.h>
	# include <crm/common/ipc_internal.h>
	# include <crm/lrmd.h>
	# include <crm/stonith-ng.h>

	# ifdef HAVE_GNUTLS_GNUTLS_H
	-# undef KEYFILE
	# include <gnutls/gnutls.h>
	# endif

	extern GHashTable *rsc_list;

	typedef struct lrmd_rsc_s {
	char *rsc_id;
	char *class;
	char *provider;
	char *type;

	int call_opts;

	/* NEVER dereference this pointer,
	* It simply exists as a switch to let us know
	* when the currently active operation has completed */
	void *active;

	/* Operations in this list
	* have not been executed yet. */
	GList *pending_ops;
	/* Operations in this list are recurring operations
	* that have been handed off from the pending ops list. */
	GList *recurring_ops;

	int st_probe_rc; // What value should be returned for a probe if stonith

	crm_trigger_t *work;
	} lrmd_rsc_t;

	# ifdef HAVE_GNUTLS_GNUTLS_H
	// in remoted_tls.c
	int lrmd_init_remote_tls_server(void);
	void lrmd_tls_server_destroy(void);
	# endif

	int lrmd_server_send_reply(pcmk__client_t client, uint32_t id, xmlNode reply);

	int lrmd_server_send_notify(pcmk__client_t client, xmlNode msg);

	void notify_of_new_client(pcmk__client_t *new_client);

	void process_lrmd_message(pcmk__client_t *client, uint32_t id,
	xmlNode *request);

	void free_rsc(gpointer data);

	void handle_shutdown_ack(void);

	void handle_shutdown_nack(void);

	void lrmd_client_destroy(pcmk__client_t *client);

	void client_disconnect_cleanup(const char *client_id);

	/*!
	* \brief Don't worry about freeing this connection. It is
	* taken care of after mainloop exits by the main() function.
	*/
	stonith_t *get_stonith_connection(void);

	/*!
	* \brief This is a callback that tells the lrmd
	* the current stonith connection has gone away. This allows
	* us to timeout any pending stonith commands
	*/
	void stonith_connection_failed(void);

	#ifdef PCMK__COMPILE_REMOTE
	void ipc_proxy_init(void);
	void ipc_proxy_cleanup(void);
	void ipc_proxy_add_provider(pcmk__client_t *client);
	void ipc_proxy_remove_provider(pcmk__client_t *client);
	void ipc_proxy_forward_client(pcmk__client_t client, xmlNode xml);
	pcmk__client_t *ipc_proxy_get_provider(void);
	int ipc_proxy_shutdown_req(pcmk__client_t *ipc_proxy);
	void remoted_spawn_pidone(int argc, char argv, char envp);
	#endif

	int process_lrmd_alert_exec(pcmk__client_t *client, uint32_t id,
	xmlNode *request);
	void lrmd_drain_alerts(GMainLoop *mloop);

	#endif // PACEMAKER_EXECD__H
	diff --git a/daemons/fenced/fenced_history.c b/daemons/fenced/fenced_history.c
	index bc159383c2..a9c57dc862 100644
	--- a/daemons/fenced/fenced_history.c
	+++ b/daemons/fenced/fenced_history.c
	@@ -1,525 +1,528 @@
	/*
	* Copyright 2009-2021 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdio.h>
	#include <unistd.h>
	#include <stdlib.h>

	#include <crm/crm.h>
	#include <crm/msg_xml.h>
	#include <crm/common/ipc.h>
	#include <crm/common/ipc_internal.h>
	#include <crm/cluster/internal.h>

	#include <crm/stonith-ng.h>
	#include <crm/fencing/internal.h>
	#include <crm/common/xml.h>
	#include <crm/common/xml_internal.h>

	#include <pacemaker-fenced.h>

	#define MAX_STONITH_HISTORY 500

	/*!
	* \internal
	* \brief Send a broadcast to all nodes to trigger cleanup or
	* history synchronisation
	*
	* \param[in] history Optional history to be attached
	* \param[in] callopts We control cleanup via a flag in the callopts
	* \param[in] target Cleanup can be limited to certain fence-targets
	*/
	static void
	stonith_send_broadcast_history(xmlNode *history,
	int callopts,
	const char *target)
	{
	xmlNode *bcast = create_xml_node(NULL, "stonith_command");
	xmlNode *data = create_xml_node(NULL, __func__);

	if (target) {
	crm_xml_add(data, F_STONITH_TARGET, target);
	}
	crm_xml_add(bcast, F_TYPE, T_STONITH_NG);
	crm_xml_add(bcast, F_SUBTYPE, "broadcast");
	crm_xml_add(bcast, F_STONITH_OPERATION, STONITH_OP_FENCE_HISTORY);
	crm_xml_add_int(bcast, F_STONITH_CALLOPTS, callopts);
	if (history) {
	add_node_copy(data, history);
	}
	add_message_xml(bcast, F_STONITH_CALLDATA, data);
	send_cluster_message(NULL, crm_msg_stonith_ng, bcast, FALSE);

	free_xml(data);
	free_xml(bcast);
	}

	static gboolean
	stonith_remove_history_entry (gpointer key,
	gpointer value,
	gpointer user_data)
	{
	remote_fencing_op_t *op = value;
	const char target = (const char ) user_data;

	if ((op->state == st_failed) \|\| (op->state == st_done)) {
	if ((target) && (strcmp(op->target, target) != 0)) {
	return FALSE;
	}
	return TRUE;
	}

	return FALSE; /* don't clean pending operations */
	}

	/*!
	* \internal
	* \brief Send out a cleanup broadcast or do a local history-cleanup
	*
	* \param[in] target Cleanup can be limited to certain fence-targets
	* \param[in] broadcast Send out a cleanup broadcast
	*/
	static void
	stonith_fence_history_cleanup(const char *target,
	gboolean broadcast)
	{
	if (broadcast) {
	stonith_send_broadcast_history(NULL,
	st_opt_cleanup \| st_opt_discard_reply,
	target);
	/* we'll do the local clean when we receive back our own broadcast */
	} else if (stonith_remote_op_list) {
	g_hash_table_foreach_remove(stonith_remote_op_list,
	stonith_remove_history_entry,
	(gpointer) target);
	do_stonith_notify(T_STONITH_NOTIFY_HISTORY, pcmk_ok, NULL);
	}
	}

	/* keeping the length of fence-history within bounds
	* =================================================
	*
	* If things are really running wild a lot of fencing-attempts
	* might fill up the hash-map, eventually using up a lot
	* of memory and creating huge history-sync messages.
	* Before the history being synced across nodes at least
	* the reboot of a cluster-node helped keeping the
	* history within bounds even though not in a reliable
	* manner.
	*
	* stonith_remote_op_list isn't sorted for time-stamps
	* thus it would be kind of expensive to delete e.g.
	* the oldest entry if it would grow past MAX_STONITH_HISTORY
	* entries.
	* It is more efficient to purge MAX_STONITH_HISTORY/2
	* entries whenever the list grows beyond MAX_STONITH_HISTORY.
	* (sort for age + purge the MAX_STONITH_HISTORY/2 oldest)
	* That done on a per-node-base might raise the
	* probability of large syncs to occur.
	* Things like introducing a broadcast to purge
	* MAX_STONITH_HISTORY/2 entries or not sync above a certain
	* threshold coming to mind ...
	* Simplest thing though is to purge the full history
	* throughout the cluster once MAX_STONITH_HISTORY is reached.
	* On the other hand this leads to purging the history in
	* situations where it would be handy to have it probably.
	*/


	static int
	op_time_sort(const void a_voidp, const void b_voidp)
	{
	const remote_fencing_op_t a = (const remote_fencing_op_t ) a_voidp;
	const remote_fencing_op_t b = (const remote_fencing_op_t ) b_voidp;
	gboolean a_pending = ((a)->state != st_failed) && ((a)->state != st_done);
	gboolean b_pending = ((b)->state != st_failed) && ((b)->state != st_done);

	if (a_pending && b_pending) {
	return 0;
	} else if (a_pending) {
	return -1;
	} else if (b_pending) {
	return 1;
	} else if ((b)->completed == (a)->completed) {
	if ((b)->completed_nsec > (a)->completed_nsec) {
	return 1;
	} else if ((b)->completed_nsec == (a)->completed_nsec) {
	return 0;
	}
	} else if ((b)->completed > (a)->completed) {
	return 1;
	}

	return -1;
	}


	/*!
	* \internal
	* \brief Do a local history-trim to MAX_STONITH_HISTORY / 2 entries
	* once over MAX_STONITH_HISTORY
	*/
	void
	stonith_fence_history_trim(void)
	{
	guint num_ops;

	if (!stonith_remote_op_list) {
	return;
	}
	num_ops = g_hash_table_size(stonith_remote_op_list);
	if (num_ops > MAX_STONITH_HISTORY) {
	remote_fencing_op_t *ops[num_ops];
	remote_fencing_op_t *op = NULL;
	GHashTableIter iter;
	int i;

	crm_trace("Fencing History growing beyond limit of %d so purge "
	"half of failed/successful attempts", MAX_STONITH_HISTORY);

	/* write all ops into an array */
	i = 0;
	g_hash_table_iter_init(&iter, stonith_remote_op_list);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&op)) {
	ops[i++] = op;
	}
	/* run quicksort over the array so that we get pending ops
	* first and then sorted most recent to oldest
	*/
	qsort(ops, num_ops, sizeof(remote_fencing_op_t *), op_time_sort);
	/* purgest oldest half of the history entries */
	for (i = MAX_STONITH_HISTORY / 2; i < num_ops; i++) {
	/* keep pending ops even if they shouldn't fill more than
	* half of our buffer
	*/
	if ((ops[i]->state == st_failed) \|\| (ops[i]->state == st_done)) {
	g_hash_table_remove(stonith_remote_op_list, ops[i]->id);
	}
	}
	/* we've just purged valid data from the list so there is no need
	* to create a notification - if displayed it can stay
	*/
	}
	}

	/*!
	* \internal
	* \brief Convert xml fence-history to a hash-table like stonith_remote_op_list
	*
	* \param[in] history Fence-history in xml
	*
	* \return Fence-history as hash-table
	*/
	static GHashTable *
	stonith_xml_history_to_list(xmlNode *history)
	{
	xmlNode *xml_op = NULL;
	GHashTable *rv = NULL;

	init_stonith_remote_op_hash_table(&rv);

	CRM_LOG_ASSERT(rv != NULL);

	for (xml_op = pcmk__xml_first_child(history); xml_op != NULL;
	xml_op = pcmk__xml_next(xml_op)) {
	remote_fencing_op_t *op = NULL;
	char *id = crm_element_value_copy(xml_op, F_STONITH_REMOTE_OP_ID);
	int state;
	long long completed;
	long long completed_nsec = 0L;

	if (!id) {
	crm_warn("Malformed fencing history received from peer");
	continue;
	}

	crm_trace("Attaching op %s to hashtable", id);

	op = calloc(1, sizeof(remote_fencing_op_t));

	op->id = id;
	op->target = crm_element_value_copy(xml_op, F_STONITH_TARGET);
	op->action = crm_element_value_copy(xml_op, F_STONITH_ACTION);
	op->originator = crm_element_value_copy(xml_op, F_STONITH_ORIGIN);
	op->delegate = crm_element_value_copy(xml_op, F_STONITH_DELEGATE);
	op->client_name = crm_element_value_copy(xml_op, F_STONITH_CLIENTNAME);
	crm_element_value_ll(xml_op, F_STONITH_DATE, &completed);
	op->completed = (time_t) completed;
	crm_element_value_ll(xml_op, F_STONITH_DATE_NSEC, &completed_nsec);
	op->completed_nsec = completed_nsec;
	crm_element_value_int(xml_op, F_STONITH_STATE, &state);
	op->state = (enum op_state) state;

	g_hash_table_replace(rv, id, op);
	CRM_LOG_ASSERT(g_hash_table_lookup(rv, id) != NULL);
	}

	return rv;
	}

	/*!
	* \internal
	* \brief Craft xml difference between local fence-history and a history
	* coming from remote, and merge the remote history into the local
	*
	* \param[in] remote_history Fence-history as hash-table (may be NULL)
	* \param[in] add_id If crafting the answer for an API
	* history-request there is no need for the id
	* \param[in] target Optionally limit to certain fence-target
	*
	* \return The fence-history as xml
	*/
	static xmlNode *
	stonith_local_history_diff_and_merge(GHashTable *remote_history,
	gboolean add_id,
	const char *target)
	{
	xmlNode *history = NULL;
	GHashTableIter iter;
	remote_fencing_op_t *op = NULL;
	gboolean updated = FALSE;
	int cnt = 0;

	if (stonith_remote_op_list) {
	char *id = NULL;

	history = create_xml_node(NULL, F_STONITH_HISTORY_LIST);

	g_hash_table_iter_init(&iter, stonith_remote_op_list);
	while (g_hash_table_iter_next(&iter, (void )&id, (void )&op)) {
	xmlNode *entry = NULL;

	if (remote_history) {
	remote_fencing_op_t *remote_op =
	g_hash_table_lookup(remote_history, op->id);

	if (remote_op) {
	if (stonith__op_state_pending(op->state)
	&& !stonith__op_state_pending(remote_op->state)) {

	crm_debug("Updating outdated pending operation %.8s "
	"(state=%s) according to the one (state=%s) from "
	"remote peer history",
	op->id, stonith_op_state_str(op->state),
	stonith_op_state_str(remote_op->state));

	g_hash_table_steal(remote_history, op->id);
	op->id = remote_op->id;
	remote_op->id = id;
	g_hash_table_iter_replace(&iter, remote_op);

	updated = TRUE;
	continue; /* skip outdated entries */

	} else if (!stonith__op_state_pending(op->state)
	&& stonith__op_state_pending(remote_op->state)) {

	crm_debug("Broadcasting operation %.8s (state=%s) to "
	"update the outdated pending one "
	"(state=%s) in remote peer history",
	op->id, stonith_op_state_str(op->state),
	stonith_op_state_str(remote_op->state));

	g_hash_table_remove(remote_history, op->id);

	} else {
	g_hash_table_remove(remote_history, op->id);
	continue; /* skip entries broadcasted already */
	}
	}
	}

	if (!pcmk__str_eq(target, op->target, pcmk__str_null_matches)) {
	continue;
	}

	cnt++;
	crm_trace("Attaching op %s", op->id);
	entry = create_xml_node(history, STONITH_OP_EXEC);
	if (add_id) {
	crm_xml_add(entry, F_STONITH_REMOTE_OP_ID, op->id);
	}
	crm_xml_add(entry, F_STONITH_TARGET, op->target);
	crm_xml_add(entry, F_STONITH_ACTION, op->action);
	crm_xml_add(entry, F_STONITH_ORIGIN, op->originator);
	crm_xml_add(entry, F_STONITH_DELEGATE, op->delegate);
	crm_xml_add(entry, F_STONITH_CLIENTNAME, op->client_name);
	crm_xml_add_ll(entry, F_STONITH_DATE, op->completed);
	crm_xml_add_ll(entry, F_STONITH_DATE_NSEC, op->completed_nsec);
	crm_xml_add_int(entry, F_STONITH_STATE, op->state);
	}
	}

	if (remote_history) {
	init_stonith_remote_op_hash_table(&stonith_remote_op_list);

	updated \|= g_hash_table_size(remote_history);

	g_hash_table_iter_init(&iter, remote_history);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&op)) {

	if (stonith__op_state_pending(op->state) &&
	pcmk__str_eq(op->originator, stonith_our_uname, pcmk__str_casei)) {
	crm_warn("Failing pending operation %.8s originated by us but "
	"known only from peer history", op->id);
	op->state = st_failed;
	set_fencing_completed(op);
	/* use -EHOSTUNREACH to not introduce a new return-code that might
	trigger unexpected results at other places and to prevent
	remote_op_done from setting the delegate if not present
	*/
	stonith_bcast_result_to_peers(op, -EHOSTUNREACH, FALSE);
	}

	g_hash_table_iter_steal(&iter);
	g_hash_table_replace(stonith_remote_op_list, op->id, op);
	/* we could trim the history here but if we bail
	* out after trim we might miss more recent entries
	* of those that might still be in the list
	* if we don't bail out trimming once is more
	* efficient and memory overhead is minimal as
	* we are just moving pointers from one hash to
	* another
	*/
	}

	g_hash_table_destroy(remote_history); /* remove what is left */
	}

	if (updated) {
	stonith_fence_history_trim();
	do_stonith_notify(T_STONITH_NOTIFY_HISTORY, pcmk_ok, NULL);
	}

	if (cnt == 0) {
	free_xml(history);
	return NULL;
	} else {
	return history;
	}
	}

	/*!
	* \internal
	* \brief Craft xml from the local fence-history
	*
	* \param[in] add_id If crafting the answer for an API
	* history-request there is no need for the id
	* \param[in] target Optionally limit to certain fence-target
	*
	* \return The fence-history as xml
	*/
	static xmlNode *
	stonith_local_history(gboolean add_id, const char *target)
	{
	return stonith_local_history_diff_and_merge(NULL, add_id, target);
	}

	/*!
	* \internal
	* \brief Handle fence-history messages (either from API or coming in as
	* broadcasts
	*
	* \param[in] msg Request message
	* \param[in] output In case of a request from the API used to craft
	* a reply from
	* \param[in] remote_peer
	* \param[in] options call-options from the request
	*/
	void
	stonith_fence_history(xmlNode msg, xmlNode *output,
	const char *remote_peer, int options)
	{
	const char *target = NULL;
	xmlNode *dev = get_xpath_object("//@" F_STONITH_TARGET, msg, LOG_NEVER);
	xmlNode *out_history = NULL;

	if (dev) {
	target = crm_element_value(dev, F_STONITH_TARGET);
	if (target && (options & st_opt_cs_nodeid)) {
	int nodeid;
	crm_node_t *node;

	pcmk__scan_min_int(target, &nodeid, 0);
	node = pcmk__search_known_node_cache(nodeid, NULL, CRM_GET_PEER_ANY);
	if (node) {
	target = node->uname;
	}
	}
	}

	if (options & st_opt_cleanup) {
	crm_trace("Cleaning up operations on %s in %p", target,
	stonith_remote_op_list);

	stonith_fence_history_cleanup(target,
	crm_element_value(msg, F_STONITH_CALLID) != NULL);
	} else if (options & st_opt_broadcast) {
	/* there is no clear sign atm for when a history sync
	is done so send a notification for anything
	that smells like history-sync
	*/
	do_stonith_notify(T_STONITH_NOTIFY_HISTORY_SYNCED, pcmk_ok, NULL);
	if (crm_element_value(msg, F_STONITH_CALLID)) {
	/* this is coming from the stonith-API
	*
	* craft a broadcast with node's history
	* so that every node can merge and broadcast
	* what it has on top
	*/
	out_history = stonith_local_history(TRUE, NULL);
	crm_trace("Broadcasting history to peers");
	stonith_send_broadcast_history(out_history,
	st_opt_broadcast \| st_opt_discard_reply,
	NULL);
	} else if (remote_peer &&
	!pcmk__str_eq(remote_peer, stonith_our_uname, pcmk__str_casei)) {
	xmlNode *history = get_xpath_object("//" F_STONITH_HISTORY_LIST,
	msg, LOG_NEVER);
	- GHashTable *received_history =
	- history?stonith_xml_history_to_list(history):NULL;

	/* either a broadcast created directly upon stonith-API request
	* or a diff as response to such a thing
	*
	* in both cases it may have a history or not
	* if we have differential data
	* merge in what we've received and stop
	* otherwise broadcast what we have on top
	* marking as differential and merge in afterwards
	*/
	if (!history \|\| !pcmk__xe_attr_is_true(history, F_STONITH_DIFFERENTIAL)) {
	+ GHashTable *received_history = NULL;
	+
	+ if (history != NULL) {
	+ received_history = stonith_xml_history_to_list(history);
	+ }
	out_history =
	stonith_local_history_diff_and_merge(received_history, TRUE, NULL);
	if (out_history) {
	crm_trace("Broadcasting history-diff to peers");
	pcmk__xe_set_bool_attr(out_history, F_STONITH_DIFFERENTIAL, true);
	stonith_send_broadcast_history(out_history,
	st_opt_broadcast \| st_opt_discard_reply,
	NULL);
	} else {
	crm_trace("History-diff is empty - skip broadcast");
	}
	}
	} else {
	crm_trace("Skipping history-query-broadcast (%s%s)"
	" we sent ourselves",
	remote_peer?"remote-peer=":"local-ipc",
	remote_peer?remote_peer:"");
	}
	} else {
	/* plain history request */
	crm_trace("Looking for operations on %s in %p", target,
	stonith_remote_op_list);
	*output = stonith_local_history(FALSE, target);
	}
	free_xml(out_history);
	}
	diff --git a/daemons/pacemakerd/pcmkd_corosync.c b/daemons/pacemakerd/pcmkd_corosync.c
	index 49b206ed82..7990bc43c5 100644
	--- a/daemons/pacemakerd/pcmkd_corosync.c
	+++ b/daemons/pacemakerd/pcmkd_corosync.c
	@@ -1,374 +1,374 @@
	/*
	* Copyright 2010-2021 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>
	#include "pacemakerd.h"

	#include <sys/utsname.h>
	#include <sys/stat.h> /* for calls to stat() */
	#include <libgen.h> /* For basename() and dirname() */

	#include <sys/types.h>
	#include <pwd.h> /* For getpwname() */

	#include <corosync/hdb.h>
	#include <corosync/cfg.h>
	#include <corosync/cpg.h>
	#include <corosync/cmap.h>

	#include <crm/cluster/internal.h>
	#include <crm/common/ipc.h> /* for crm_ipc_is_authentic_process */
	#include <crm/common/mainloop.h>

	#include <crm/common/ipc_internal.h> /* PCMK__SPECIAL_PID* */

	static corosync_cfg_handle_t cfg_handle = 0;
	static mainloop_timer_t *reconnect_timer = NULL;

	/* =::=::=::= CFG - Shutdown stuff =::=::=::= */

	static void
	cfg_shutdown_callback(corosync_cfg_handle_t h, corosync_cfg_shutdown_flags_t flags)
	{
	crm_info("Corosync wants to shut down: %s",
	(flags == COROSYNC_CFG_SHUTDOWN_FLAG_IMMEDIATE) ? "immediate" :
	(flags == COROSYNC_CFG_SHUTDOWN_FLAG_REGARDLESS) ? "forced" : "optional");

	/* Never allow corosync to shut down while we're running */
	corosync_cfg_replyto_shutdown(h, COROSYNC_CFG_SHUTDOWN_FLAG_NO);
	}

	static corosync_cfg_callbacks_t cfg_callbacks = {
	.corosync_cfg_shutdown_callback = cfg_shutdown_callback,
	};

	static int
	pcmk_cfg_dispatch(gpointer user_data)
	{
	corosync_cfg_handle_t handle = (corosync_cfg_handle_t ) user_data;
	cs_error_t rc = corosync_cfg_dispatch(*handle, CS_DISPATCH_ALL);

	if (rc != CS_OK) {
	return -1;
	}
	return 0;
	}

	static void
	close_cfg(void)
	{
	if (cfg_handle != 0) {
	#ifdef HAVE_COROSYNC_CFG_TRACKSTART
	/* Ideally, we would call corosync_cfg_trackstop(cfg_handle) here, but a
	* bug in corosync 3.1.1 and 3.1.2 makes it hang forever. Thankfully,
	* it's not necessary since we exit immediately after this.
	*/
	#endif
	corosync_cfg_finalize(cfg_handle);
	cfg_handle = 0;
	}
	}

	static gboolean
	cluster_reconnect_cb(gpointer data)
	{
	if (cluster_connect_cfg()) {
	mainloop_timer_del(reconnect_timer);
	reconnect_timer = NULL;
	crm_notice("Cluster reconnect succeeded");
	mcp_read_config();
	restart_cluster_subdaemons();
	return G_SOURCE_REMOVE;
	} else {
	- crm_info("Cluster reconnect failed"
	+ crm_info("Cluster reconnect failed "
	"(connection will be reattempted once per second)");
	}
	/*
	* In theory this will continue forever. In practice the CIB connection from
	* attrd will timeout and shut down Pacemaker when it gets bored.
	*/
	return G_SOURCE_CONTINUE;
	}


	static void
	cfg_connection_destroy(gpointer user_data)
	{
	crm_warn("Lost connection to cluster layer "
	"(connection will be reattempted once per second)");
	corosync_cfg_finalize(cfg_handle);
	cfg_handle = 0;
	reconnect_timer = mainloop_timer_add("corosync reconnect", 1000, TRUE, cluster_reconnect_cb, NULL);
	mainloop_timer_start(reconnect_timer);
	}

	void
	cluster_disconnect_cfg(void)
	{
	close_cfg();
	if (reconnect_timer != NULL) {
	/* The mainloop should be gone by this point, so this isn't necessary,
	* but cleaning up memory should make valgrind happier.
	*/
	mainloop_timer_del(reconnect_timer);
	reconnect_timer = NULL;
	}
	}

	#define cs_repeat(counter, max, code) do { \
	code; \
	if(rc == CS_ERR_TRY_AGAIN \|\| rc == CS_ERR_QUEUE_FULL) { \
	counter++; \
	crm_debug("Retrying Corosync operation after %ds", counter); \
	sleep(counter); \
	} else { \
	break; \
	} \
	} while(counter < max)

	gboolean
	cluster_connect_cfg(void)
	{
	cs_error_t rc;
	int fd = -1, retries = 0, rv;
	uid_t found_uid = 0;
	gid_t found_gid = 0;
	pid_t found_pid = 0;
	uint32_t nodeid;

	static struct mainloop_fd_callbacks cfg_fd_callbacks = {
	.dispatch = pcmk_cfg_dispatch,
	.destroy = cfg_connection_destroy,
	};

	cs_repeat(retries, 30, rc = corosync_cfg_initialize(&cfg_handle, &cfg_callbacks));

	if (rc != CS_OK) {
	crm_crit("Could not connect to Corosync CFG: %s " CRM_XS " rc=%d",
	cs_strerror(rc), rc);
	return FALSE;
	}

	rc = corosync_cfg_fd_get(cfg_handle, &fd);
	if (rc != CS_OK) {
	crm_crit("Could not get Corosync CFG descriptor: %s " CRM_XS " rc=%d",
	cs_strerror(rc), rc);
	goto bail;
	}

	/* CFG provider run as root (in given user namespace, anyway)? */
	if (!(rv = crm_ipc_is_authentic_process(fd, (uid_t) 0,(gid_t) 0, &found_pid,
	&found_uid, &found_gid))) {
	crm_crit("Rejecting Corosync CFG provider because process %lld "
	"is running as uid %lld gid %lld, not root",
	(long long) PCMK__SPECIAL_PID_AS_0(found_pid),
	(long long) found_uid, (long long) found_gid);
	goto bail;
	} else if (rv < 0) {
	crm_crit("Could not authenticate Corosync CFG provider: %s "
	CRM_XS " rc=%d", strerror(-rv), -rv);
	goto bail;
	}

	retries = 0;
	cs_repeat(retries, 30, rc = corosync_cfg_local_get(cfg_handle, &nodeid));
	if (rc != CS_OK) {
	crm_crit("Could not get local node ID from Corosync: %s "
	CRM_XS " rc=%d", cs_strerror(rc), rc);
	goto bail;
	}
	crm_debug("Corosync reports local node ID is %lu", (unsigned long) nodeid);

	#ifdef HAVE_COROSYNC_CFG_TRACKSTART
	rc = corosync_cfg_trackstart(cfg_handle, 0);
	if (rc != CS_OK) {
	crm_crit("Could not enable Corosync CFG shutdown tracker: %s " CRM_XS " rc=%d",
	cs_strerror(rc), rc);
	goto bail;
	}
	#endif

	mainloop_add_fd("corosync-cfg", G_PRIORITY_DEFAULT, fd, &cfg_handle, &cfg_fd_callbacks);
	return TRUE;

	bail:
	corosync_cfg_finalize(cfg_handle);
	return FALSE;
	}

	void
	pcmkd_shutdown_corosync(void)
	{
	cs_error_t rc;

	if (cfg_handle == 0) {
	crm_warn("Unable to shut down Corosync: No connection");
	return;
	}
	crm_info("Asking Corosync to shut down");
	rc = corosync_cfg_try_shutdown(cfg_handle,
	COROSYNC_CFG_SHUTDOWN_FLAG_IMMEDIATE);
	if (rc == CS_OK) {
	close_cfg();
	} else {
	crm_warn("Corosync shutdown failed: %s " CRM_XS " rc=%d",
	cs_strerror(rc), rc);
	}
	}

	bool
	pcmkd_corosync_connected(void)
	{
	cpg_handle_t local_handle = 0;
	cpg_model_v1_data_t cpg_model_info = {CPG_MODEL_V1, NULL, NULL, NULL, 0};
	int fd = -1;

	if (cpg_model_initialize(&local_handle, CPG_MODEL_V1, (cpg_model_data_t *) &cpg_model_info, NULL) != CS_OK) {
	return false;
	}

	if (cpg_fd_get(local_handle, &fd) != CS_OK) {
	return false;
	}

	cpg_finalize(local_handle);

	return true;
	}

	/* =::=::=::= Configuration =::=::=::= */
	static int
	get_config_opt(uint64_t unused, cmap_handle_t object_handle, const char key, char *value,
	const char *fallback)
	{
	int rc = 0, retries = 0;

	cs_repeat(retries, 5, rc = cmap_get_string(object_handle, key, value));
	if (rc != CS_OK) {
	crm_trace("Search for %s failed %d, defaulting to %s", key, rc, fallback);
	if (fallback) {
	*value = strdup(fallback);
	} else {
	*value = NULL;
	}
	}
	crm_trace("%s: %s", key, *value);
	return rc;
	}

	gboolean
	mcp_read_config(void)
	{
	cs_error_t rc = CS_OK;
	int retries = 0;
	cmap_handle_t local_handle;
	uint64_t config = 0;
	int fd = -1;
	uid_t found_uid = 0;
	gid_t found_gid = 0;
	pid_t found_pid = 0;
	int rv;
	enum cluster_type_e stack;

	// There can be only one possibility
	do {
	rc = pcmk__init_cmap(&local_handle);
	if (rc != CS_OK) {
	retries++;
	crm_info("Could not connect to Corosync CMAP: %s (retrying in %ds) "
	CRM_XS " rc=%d", cs_strerror(rc), retries, rc);
	sleep(retries);

	} else {
	break;
	}

	} while (retries < 5);

	if (rc != CS_OK) {
	crm_crit("Could not connect to Corosync CMAP: %s "
	CRM_XS " rc=%d", cs_strerror(rc), rc);
	return FALSE;
	}

	rc = cmap_fd_get(local_handle, &fd);
	if (rc != CS_OK) {
	crm_crit("Could not get Corosync CMAP descriptor: %s " CRM_XS " rc=%d",
	cs_strerror(rc), rc);
	cmap_finalize(local_handle);
	return FALSE;
	}

	/* CMAP provider run as root (in given user namespace, anyway)? */
	if (!(rv = crm_ipc_is_authentic_process(fd, (uid_t) 0,(gid_t) 0, &found_pid,
	&found_uid, &found_gid))) {
	crm_crit("Rejecting Corosync CMAP provider because process %lld "
	"is running as uid %lld gid %lld, not root",
	(long long) PCMK__SPECIAL_PID_AS_0(found_pid),
	(long long) found_uid, (long long) found_gid);
	cmap_finalize(local_handle);
	return FALSE;
	} else if (rv < 0) {
	crm_crit("Could not authenticate Corosync CMAP provider: %s "
	CRM_XS " rc=%d", strerror(-rv), -rv);
	cmap_finalize(local_handle);
	return FALSE;
	}

	stack = get_cluster_type();
	if (stack != pcmk_cluster_corosync) {
	crm_crit("Expected Corosync cluster layer but detected %s "
	CRM_XS " stack=%d", name_for_cluster_type(stack), stack);
	return FALSE;
	}

	crm_info("Reading configuration for %s stack",
	name_for_cluster_type(stack));
	pcmk__set_env_option(PCMK__ENV_CLUSTER_TYPE, "corosync");
	pcmk__set_env_option(PCMK__ENV_QUORUM_TYPE, "corosync");

	// If debug logging is not configured, check whether corosync has it
	if (pcmk__env_option(PCMK__ENV_DEBUG) == NULL) {
	char *debug_enabled = NULL;

	get_config_opt(config, local_handle, "logging.debug", &debug_enabled, "off");

	if (crm_is_true(debug_enabled)) {
	pcmk__set_env_option(PCMK__ENV_DEBUG, "1");
	if (get_crm_log_level() < LOG_DEBUG) {
	set_crm_log_level(LOG_DEBUG);
	}

	} else {
	pcmk__set_env_option(PCMK__ENV_DEBUG, "0");
	}

	free(debug_enabled);
	}

	if(local_handle){
	gid_t gid = 0;
	if (pcmk_daemon_user(NULL, &gid) < 0) {
	crm_warn("Could not authorize group with Corosync " CRM_XS
	" No group found for user %s", CRM_DAEMON_USER);

	} else {
	char key[PATH_MAX];
	snprintf(key, PATH_MAX, "uidgid.gid.%u", gid);
	rc = cmap_set_uint8(local_handle, key, 1);
	if (rc != CS_OK) {
	crm_warn("Could not authorize group with Corosync: %s " CRM_XS
	" group=%u rc=%d", pcmk__cs_err_str(rc), gid, rc);
	}
	}
	}
	cmap_finalize(local_handle);

	return TRUE;
	}
	diff --git a/include/pcmki/pcmki_scheduler.h b/include/pcmki/pcmki_scheduler.h
	index 336a5778b4..bcf2717731 100644
	--- a/include/pcmki/pcmki_scheduler.h
	+++ b/include/pcmki/pcmki_scheduler.h
	@@ -1,111 +1,104 @@
	/*
	* Copyright 2014-2021 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#ifndef PENGINE__H
	# define PENGINE__H

	typedef struct rsc_ticket_s rsc_ticket_t;
	-typedef struct lrm_agent_s lrm_agent_t;

	# include <glib.h>
	# include <crm/crm.h>
	# include <crm/common/iso8601.h>
	# include <crm/pengine/rules.h>
	# include <crm/pengine/common.h>
	# include <crm/pengine/status.h>

	# include <crm/pengine/complex.h>

	-enum pe_stop_fail {
	- pesf_block,
	- pesf_stonith,
	- pesf_ignore
	-};
	-
	enum pe_weights {
	pe_weights_none = 0x0,
	pe_weights_init = 0x1,
	pe_weights_forward = 0x4,
	pe_weights_positive = 0x8,
	pe_weights_rollback = 0x10,
	};

	typedef struct {
	const char *id;
	const char *node_attribute;
	pe_resource_t *dependent; // The resource being colocated
	pe_resource_t *primary; // The resource the dependent is colocated with

	int dependent_role; // Colocation applies only if dependent has this role
	int primary_role; // Colocation applies only if primary has this role

	int score;
	bool influence; // Whether dependent influences active primary placement
	} pcmk__colocation_t;

	enum loss_ticket_policy_e {
	loss_ticket_stop,
	loss_ticket_demote,
	loss_ticket_fence,
	loss_ticket_freeze
	};

	struct rsc_ticket_s {
	const char *id;
	pe_resource_t *rsc_lh;
	pe_ticket_t *ticket;
	enum loss_ticket_policy_e loss_policy;

	int role_lh;
	};

	extern gboolean stage0(pe_working_set_t * data_set);
	extern gboolean probe_resources(pe_working_set_t * data_set);
	extern gboolean stage2(pe_working_set_t * data_set);
	extern gboolean stage4(pe_working_set_t * data_set);
	extern gboolean stage5(pe_working_set_t * data_set);
	extern gboolean stage6(pe_working_set_t * data_set);
	extern gboolean stage8(pe_working_set_t * data_set);

	void pcmk__unpack_constraints(pe_working_set_t *data_set);

	extern void graph_element_from_action(pe_action_t * action, pe_working_set_t * data_set);
	extern void add_maintenance_update(pe_working_set_t *data_set);
	xmlNode pcmk__schedule_actions(pe_working_set_t data_set, xmlNode *xml_input,
	crm_time_t *now);

	extern const char *transition_idle_timeout;

	/*!
	* \internal
	* \brief Check whether colocation's left-hand preferences should be considered
	*
	* \param[in] colocation Colocation constraint
	* \param[in] rsc Right-hand instance (normally this will be
	* colocation->primary, which NULL will be treated as,
	* but for clones or bundles with multiple instances
	* this can be a particular instance)
	*
	* \return true if colocation influence should be effective, otherwise false
	*/
	static inline bool
	pcmk__colocation_has_influence(const pcmk__colocation_t *colocation,
	const pe_resource_t *rsc)
	{
	if (rsc == NULL) {
	rsc = colocation->primary;
	}

	/* The left hand of a colocation influences the right hand's location
	* if the influence option is true, or the right hand is not yet active.
	*/
	return colocation->influence \|\| (rsc->running_on == NULL);
	}

	#endif
	diff --git a/lib/cib/cib_remote.c b/lib/cib/cib_remote.c
	index 2576fcc654..61273e69b0 100644
	--- a/lib/cib/cib_remote.c
	+++ b/lib/cib/cib_remote.c
	@@ -1,635 +1,633 @@
	/*
	* Copyright 2008-2021 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <unistd.h>
	#include <stdlib.h>
	#include <stdio.h>
	#include <stdarg.h>
	#include <string.h>
	#include <netdb.h>
	#include <termios.h>
	#include <sys/socket.h>

	#include <glib.h>

	#include <crm/crm.h>
	#include <crm/cib/internal.h>
	#include <crm/msg_xml.h>
	#include <crm/common/ipc_internal.h>
	#include <crm/common/mainloop.h>
	#include <crm/common/remote_internal.h>
	#include <crm/common/output_internal.h>

	#ifdef HAVE_GNUTLS_GNUTLS_H
	-# undef KEYFILE
	-# include <gnutls/gnutls.h>
	-gnutls_anon_client_credentials_t anon_cred_c;

	-#define TLS_HANDSHAKE_TIMEOUT_MS 5000
	+# include <gnutls/gnutls.h>

	-const int kx_prio[] = {
	- GNUTLS_KX_ANON_DH,
	- 0
	-};
	+# define TLS_HANDSHAKE_TIMEOUT_MS 5000

	+static gnutls_anon_client_credentials_t anon_cred_c;
	static gboolean remote_gnutls_credentials_init = FALSE;
	+
	#else
	+
	typedef void gnutls_session_t;
	-#endif
	+
	+#endif // HAVE_GNUTLS_GNUTLS_H

	#include <arpa/inet.h>

	#define DH_BITS 1024

	typedef struct cib_remote_opaque_s {
	int flags;
	int socket;
	int port;
	char *server;
	char *user;
	char *passwd;
	gboolean encrypted;
	pcmk__remote_t command;
	pcmk__remote_t callback;
	pcmk__output_t *out;

	} cib_remote_opaque_t;

	void cib_remote_connection_destroy(gpointer user_data);
	int cib_remote_callback_dispatch(gpointer user_data);
	int cib_remote_command_dispatch(gpointer user_data);
	int cib_remote_signon(cib_t * cib, const char *name, enum cib_conn_type type);
	int cib_remote_signoff(cib_t * cib);
	int cib_remote_free(cib_t * cib);

	int cib_remote_perform_op(cib_t * cib, const char op, const char host, const char *section,
	xmlNode * data, xmlNode ** output_data, int call_options,
	const char *name);

	static int
	cib_remote_inputfd(cib_t * cib)
	{
	cib_remote_opaque_t *private = cib->variant_opaque;

	return private->callback.tcp_socket;
	}

	static int
	cib_remote_set_connection_dnotify(cib_t * cib, void (*dnotify) (gpointer user_data))
	{
	return -EPROTONOSUPPORT;
	}

	static int
	cib_remote_register_notification(cib_t * cib, const char *callback, int enabled)
	{
	xmlNode *notify_msg = create_xml_node(NULL, "cib_command");
	cib_remote_opaque_t *private = cib->variant_opaque;

	crm_xml_add(notify_msg, F_CIB_OPERATION, T_CIB_NOTIFY);
	crm_xml_add(notify_msg, F_CIB_NOTIFY_TYPE, callback);
	crm_xml_add_int(notify_msg, F_CIB_NOTIFY_ACTIVATE, enabled);
	pcmk__remote_send_xml(&private->callback, notify_msg);
	free_xml(notify_msg);
	return pcmk_ok;
	}

	cib_t *
	cib_remote_new(const char server, const char user, const char *passwd, int port,
	gboolean encrypted)
	{
	cib_remote_opaque_t *private = NULL;
	cib_t *cib = cib_new_variant();

	if (cib == NULL) {
	return NULL;
	}

	private = calloc(1, sizeof(cib_remote_opaque_t));

	if (private == NULL) {
	free(cib);
	return NULL;
	}

	cib->variant = cib_remote;
	cib->variant_opaque = private;

	if (server) {
	private->server = strdup(server);
	}

	if (user) {
	private->user = strdup(user);
	}

	if (passwd) {
	private->passwd = strdup(passwd);
	}

	private->port = port;
	private->encrypted = encrypted;

	/* assign variant specific ops */
	cib->delegate_fn = cib_remote_perform_op;
	cib->cmds->signon = cib_remote_signon;
	cib->cmds->signoff = cib_remote_signoff;
	cib->cmds->free = cib_remote_free;
	cib->cmds->inputfd = cib_remote_inputfd;

	cib->cmds->register_notification = cib_remote_register_notification;
	cib->cmds->set_connection_dnotify = cib_remote_set_connection_dnotify;

	return cib;
	}

	static int
	cib_tls_close(cib_t * cib)
	{
	cib_remote_opaque_t *private = cib->variant_opaque;

	#ifdef HAVE_GNUTLS_GNUTLS_H
	if (private->encrypted) {
	if (private->command.tls_session) {
	gnutls_bye(*(private->command.tls_session), GNUTLS_SHUT_RDWR);
	gnutls_deinit(*(private->command.tls_session));
	gnutls_free(private->command.tls_session);
	}

	if (private->callback.tls_session) {
	gnutls_bye(*(private->callback.tls_session), GNUTLS_SHUT_RDWR);
	gnutls_deinit(*(private->callback.tls_session));
	gnutls_free(private->callback.tls_session);
	}
	private->command.tls_session = NULL;
	private->callback.tls_session = NULL;
	if (remote_gnutls_credentials_init) {
	gnutls_anon_free_client_credentials(anon_cred_c);
	gnutls_global_deinit();
	remote_gnutls_credentials_init = FALSE;
	}
	}
	#endif

	if (private->command.tcp_socket) {
	shutdown(private->command.tcp_socket, SHUT_RDWR); /* no more receptions */
	close(private->command.tcp_socket);
	}
	if (private->callback.tcp_socket) {
	shutdown(private->callback.tcp_socket, SHUT_RDWR); /* no more receptions */
	close(private->callback.tcp_socket);
	}
	private->command.tcp_socket = 0;
	private->callback.tcp_socket = 0;

	free(private->command.buffer);
	free(private->callback.buffer);
	private->command.buffer = NULL;
	private->callback.buffer = NULL;

	return 0;
	}

	static int
	cib_tls_signon(cib_t cib, pcmk__remote_t connection, gboolean event_channel)
	{
	cib_remote_opaque_t *private = cib->variant_opaque;
	int rc;

	xmlNode *answer = NULL;
	xmlNode *login = NULL;

	static struct mainloop_fd_callbacks cib_fd_callbacks = { 0, };

	cib_fd_callbacks.dispatch =
	event_channel ? cib_remote_callback_dispatch : cib_remote_command_dispatch;
	cib_fd_callbacks.destroy = cib_remote_connection_destroy;

	connection->tcp_socket = -1;
	#ifdef HAVE_GNUTLS_GNUTLS_H
	connection->tls_session = NULL;
	#endif
	rc = pcmk__connect_remote(private->server, private->port, 0, NULL,
	&(connection->tcp_socket), NULL, NULL);
	if (rc != pcmk_rc_ok) {
	crm_info("Remote connection to %s:%d failed: %s " CRM_XS " rc=%d",
	private->server, private->port, pcmk_rc_str(rc), rc);
	return -ENOTCONN;
	}

	if (private->encrypted) {
	/* initialize GnuTls lib */
	#ifdef HAVE_GNUTLS_GNUTLS_H
	if (remote_gnutls_credentials_init == FALSE) {
	crm_gnutls_global_init();
	gnutls_anon_allocate_client_credentials(&anon_cred_c);
	remote_gnutls_credentials_init = TRUE;
	}

	/* bind the socket to GnuTls lib */
	connection->tls_session = pcmk__new_tls_session(connection->tcp_socket,
	GNUTLS_CLIENT,
	GNUTLS_CRD_ANON,
	anon_cred_c);
	if (connection->tls_session == NULL) {
	cib_tls_close(cib);
	return -1;
	}

	if (pcmk__tls_client_handshake(connection, TLS_HANDSHAKE_TIMEOUT_MS)
	!= pcmk_rc_ok) {
	crm_err("Session creation for %s:%d failed", private->server, private->port);

	gnutls_deinit(*connection->tls_session);
	gnutls_free(connection->tls_session);
	connection->tls_session = NULL;
	cib_tls_close(cib);
	return -1;
	}
	#else
	return -EPROTONOSUPPORT;
	#endif
	}

	/* login to server */
	login = create_xml_node(NULL, "cib_command");
	crm_xml_add(login, "op", "authenticate");
	crm_xml_add(login, "user", private->user);
	crm_xml_add(login, "password", private->passwd);
	crm_xml_add(login, "hidden", "password");

	pcmk__remote_send_xml(connection, login);
	free_xml(login);

	rc = pcmk_ok;
	if (pcmk__read_remote_message(connection, -1) == ENOTCONN) {
	rc = -ENOTCONN;
	}

	answer = pcmk__remote_message_xml(connection);

	crm_log_xml_trace(answer, "Reply");
	if (answer == NULL) {
	rc = -EPROTO;

	} else {
	/* grab the token */
	const char *msg_type = crm_element_value(answer, F_CIB_OPERATION);
	const char *tmp_ticket = crm_element_value(answer, F_CIB_CLIENTID);

	if (!pcmk__str_eq(msg_type, CRM_OP_REGISTER, pcmk__str_casei)) {
	crm_err("Invalid registration message: %s", msg_type);
	rc = -EPROTO;

	} else if (tmp_ticket == NULL) {
	rc = -EPROTO;

	} else {
	connection->token = strdup(tmp_ticket);
	}
	}
	free_xml(answer);
	answer = NULL;

	if (rc != 0) {
	cib_tls_close(cib);
	return rc;
	}

	crm_trace("remote client connection established");
	connection->source = mainloop_add_fd("cib-remote", G_PRIORITY_HIGH,
	connection->tcp_socket, cib,
	&cib_fd_callbacks);
	return rc;
	}

	void
	cib_remote_connection_destroy(gpointer user_data)
	{
	crm_err("Connection destroyed");
	#ifdef HAVE_GNUTLS_GNUTLS_H
	cib_tls_close(user_data);
	#endif
	return;
	}

	int
	cib_remote_command_dispatch(gpointer user_data)
	{
	int rc;
	cib_t *cib = user_data;
	cib_remote_opaque_t *private = cib->variant_opaque;

	rc = pcmk__read_remote_message(&private->command, -1);

	free(private->command.buffer);
	private->command.buffer = NULL;
	crm_err("received late reply for remote cib connection, discarding");

	if (rc == ENOTCONN) {
	return -1;
	}
	return 0;
	}

	int
	cib_remote_callback_dispatch(gpointer user_data)
	{
	int rc;
	cib_t *cib = user_data;
	cib_remote_opaque_t *private = cib->variant_opaque;

	xmlNode *msg = NULL;

	crm_info("Message on callback channel");

	rc = pcmk__read_remote_message(&private->callback, -1);

	msg = pcmk__remote_message_xml(&private->callback);
	while (msg) {
	const char *type = crm_element_value(msg, F_TYPE);

	crm_trace("Activating %s callbacks...", type);

	if (pcmk__str_eq(type, T_CIB, pcmk__str_casei)) {
	cib_native_callback(cib, msg, 0, 0);

	} else if (pcmk__str_eq(type, T_CIB_NOTIFY, pcmk__str_casei)) {
	g_list_foreach(cib->notify_list, cib_native_notify, msg);

	} else {
	crm_err("Unknown message type: %s", type);
	}

	free_xml(msg);
	msg = pcmk__remote_message_xml(&private->callback);
	}

	if (rc == ENOTCONN) {
	return -1;
	}

	return 0;
	}

	int
	cib_remote_signon(cib_t * cib, const char *name, enum cib_conn_type type)
	{
	int rc = pcmk_ok;
	cib_remote_opaque_t *private = cib->variant_opaque;

	if (private->passwd == NULL) {
	if (private->out == NULL) {
	/* If no pcmk__output_t is set, just assume that a text prompt
	* is good enough.
	*/
	pcmk__text_prompt("Password", false, &(private->passwd));
	} else {
	private->out->prompt("Password", false, &(private->passwd));
	}
	}

	if (private->server == NULL \|\| private->user == NULL) {
	rc = -EINVAL;
	}

	if (rc == pcmk_ok) {
	rc = cib_tls_signon(cib, &(private->command), FALSE);
	}

	if (rc == pcmk_ok) {
	rc = cib_tls_signon(cib, &(private->callback), TRUE);
	}

	if (rc == pcmk_ok) {
	xmlNode *hello =
	cib_create_op(0, private->callback.token, CRM_OP_REGISTER, NULL, NULL, NULL, 0, NULL);
	crm_xml_add(hello, F_CIB_CLIENTNAME, name);
	pcmk__remote_send_xml(&private->command, hello);
	free_xml(hello);
	}

	if (rc == pcmk_ok) {
	crm_info("Opened connection to %s:%d for %s",
	private->server, private->port, name);
	cib->state = cib_connected_command;
	cib->type = cib_command;

	} else {
	crm_info("Connection to %s:%d for %s failed: %s\n",
	private->server, private->port, name, pcmk_strerror(rc));
	}

	return rc;
	}

	int
	cib_remote_signoff(cib_t * cib)
	{
	int rc = pcmk_ok;

	/* cib_remote_opaque_t private = cib->variant_opaque; /

	crm_debug("Disconnecting from the CIB manager");
	#ifdef HAVE_GNUTLS_GNUTLS_H
	cib_tls_close(cib);
	#endif

	cib->state = cib_disconnected;
	cib->type = cib_no_connection;

	return rc;
	}

	int
	cib_remote_free(cib_t * cib)
	{
	int rc = pcmk_ok;

	crm_warn("Freeing CIB");
	if (cib->state != cib_disconnected) {
	rc = cib_remote_signoff(cib);
	if (rc == pcmk_ok) {
	cib_remote_opaque_t *private = cib->variant_opaque;

	free(private->server);
	free(private->user);
	free(private->passwd);
	free(cib->cmds);
	free(private);
	free(cib);
	}
	}

	return rc;
	}

	int
	cib_remote_perform_op(cib_t * cib, const char op, const char host, const char *section,
	xmlNode * data, xmlNode ** output_data, int call_options, const char *name)
	{
	int rc;
	int remaining_time = 0;
	time_t start_time;

	xmlNode *op_msg = NULL;
	xmlNode *op_reply = NULL;

	cib_remote_opaque_t *private = cib->variant_opaque;

	if (cib->state == cib_disconnected) {
	return -ENOTCONN;
	}

	if (output_data != NULL) {
	*output_data = NULL;
	}

	if (op == NULL) {
	crm_err("No operation specified");
	return -EINVAL;
	}

	cib->call_id++;
	if (cib->call_id < 1) {
	cib->call_id = 1;
	}

	op_msg =
	cib_create_op(cib->call_id, private->callback.token, op, host, section, data, call_options,
	NULL);
	if (op_msg == NULL) {
	return -EPROTO;
	}

	crm_trace("Sending %s message to the CIB manager", op);
	if (!(call_options & cib_sync_call)) {
	pcmk__remote_send_xml(&private->callback, op_msg);
	} else {
	pcmk__remote_send_xml(&private->command, op_msg);
	}
	free_xml(op_msg);

	if ((call_options & cib_discard_reply)) {
	crm_trace("Discarding reply");
	return pcmk_ok;

	} else if (!(call_options & cib_sync_call)) {
	return cib->call_id;
	}

	crm_trace("Waiting for a synchronous reply");

	start_time = time(NULL);
	remaining_time = cib->call_timeout ? cib->call_timeout : 60;

	rc = pcmk_rc_ok;
	while (remaining_time > 0 && (rc != ENOTCONN)) {
	int reply_id = -1;
	int msg_id = cib->call_id;

	rc = pcmk__read_remote_message(&private->command,
	remaining_time * 1000);
	op_reply = pcmk__remote_message_xml(&private->command);

	if (!op_reply) {
	break;
	}

	crm_element_value_int(op_reply, F_CIB_CALLID, &reply_id);

	if (reply_id == msg_id) {
	break;

	} else if (reply_id < msg_id) {
	crm_debug("Received old reply: %d (wanted %d)", reply_id, msg_id);
	crm_log_xml_trace(op_reply, "Old reply");

	} else if ((reply_id - 10000) > msg_id) {
	/* wrap-around case */
	crm_debug("Received old reply: %d (wanted %d)", reply_id, msg_id);
	crm_log_xml_trace(op_reply, "Old reply");
	} else {
	crm_err("Received a __future__ reply:" " %d (wanted %d)", reply_id, msg_id);
	}

	free_xml(op_reply);
	op_reply = NULL;

	/* wasn't the right reply, try and read some more */
	remaining_time = time(NULL) - start_time;
	}

	/* if(IPC_ISRCONN(native->command_channel) == FALSE) { */
	/* crm_err("The CIB manager disconnected: %d", */
	/* native->command_channel->ch_status); */
	/* cib->state = cib_disconnected; */
	/* } */

	if (rc == ENOTCONN) {
	crm_err("Disconnected while waiting for reply.");
	return -ENOTCONN;
	} else if (op_reply == NULL) {
	crm_err("No reply message - empty");
	return -ENOMSG;
	}

	crm_trace("Synchronous reply received");

	/* Start processing the reply... */
	if (crm_element_value_int(op_reply, F_CIB_RC, &rc) != 0) {
	rc = -EPROTO;
	}

	if (rc == -pcmk_err_diff_resync) {
	/* This is an internal value that clients do not and should not care about */
	rc = pcmk_ok;
	}

	if (rc == pcmk_ok \|\| rc == -EPERM) {
	crm_log_xml_debug(op_reply, "passed");

	} else {
	/* } else if(rc == -ETIME) { */
	crm_err("Call failed: %s", pcmk_strerror(rc));
	crm_log_xml_warn(op_reply, "failed");
	}

	if (output_data == NULL) {
	/* do nothing more */

	} else if (!(call_options & cib_discard_reply)) {
	xmlNode *tmp = get_message_xml(op_reply, F_CIB_CALLDATA);

	if (tmp == NULL) {
	crm_trace("No output in reply to \"%s\" command %d", op, cib->call_id - 1);
	} else {
	*output_data = copy_xml(tmp);
	}
	}

	free_xml(op_reply);

	return rc;
	}

	void
	cib__set_output(cib_t cib, pcmk__output_t out)
	{
	cib_remote_opaque_t *private;

	if (cib->variant != cib_remote) {
	return;
	}

	private = cib->variant_opaque;
	private->out = out;
	}
	diff --git a/lib/common/acl.c b/lib/common/acl.c
	index 90253b056b..4dfe7e97cb 100644
	--- a/lib/common/acl.c
	+++ b/lib/common/acl.c
	@@ -1,783 +1,782 @@
	/*
	* Copyright 2004-2021 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdio.h>
	#include <sys/types.h>
	#include <pwd.h>
	#include <string.h>
	#include <stdlib.h>
	#include <stdarg.h>

	#include <libxml/tree.h>

	#include <crm/crm.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>
	#include <crm/common/xml_internal.h>
	#include "crmcommon_private.h"

	#define MAX_XPATH_LEN 4096

	typedef struct xml_acl_s {
	enum xml_private_flags mode;
	char *xpath;
	} xml_acl_t;

	static void
	free_acl(void *data)
	{
	if (data) {
	xml_acl_t *acl = data;

	free(acl->xpath);
	free(acl);
	}
	}

	void
	pcmk__free_acls(GList *acls)
	{
	g_list_free_full(acls, free_acl);
	}

	static GList *
	create_acl(xmlNode xml, GList acls, enum xml_private_flags mode)
	{
	xml_acl_t *acl = NULL;

	const char *tag = crm_element_value(xml, XML_ACL_ATTR_TAG);
	const char *ref = crm_element_value(xml, XML_ACL_ATTR_REF);
	const char *xpath = crm_element_value(xml, XML_ACL_ATTR_XPATH);
	const char *attr = crm_element_value(xml, XML_ACL_ATTR_ATTRIBUTE);

	if (tag == NULL) {
	// @COMPAT rolling upgrades <=1.1.11
	tag = crm_element_value(xml, XML_ACL_ATTR_TAGv1);
	}
	if (ref == NULL) {
	// @COMPAT rolling upgrades <=1.1.11
	ref = crm_element_value(xml, XML_ACL_ATTR_REFv1);
	}

	if ((tag == NULL) && (ref == NULL) && (xpath == NULL)) {
	// Schema should prevent this, but to be safe ...
	crm_trace("Ignoring ACL <%s> element without selection criteria",
	crm_element_name(xml));
	return NULL;
	}

	acl = calloc(1, sizeof (xml_acl_t));
	CRM_ASSERT(acl != NULL);

	acl->mode = mode;
	if (xpath) {
	acl->xpath = strdup(xpath);
	CRM_ASSERT(acl->xpath != NULL);
	crm_trace("Unpacked ACL <%s> element using xpath: %s",
	crm_element_name(xml), acl->xpath);

	} else {
	int offset = 0;
	char buffer[MAX_XPATH_LEN];

	if (tag) {
	offset += snprintf(buffer + offset, MAX_XPATH_LEN - offset,
	"//%s", tag);
	} else {
	offset += snprintf(buffer + offset, MAX_XPATH_LEN - offset,
	"//*");
	}

	if (ref \|\| attr) {
	offset += snprintf(buffer + offset, MAX_XPATH_LEN - offset,
	"[");
	}

	if (ref) {
	offset += snprintf(buffer + offset, MAX_XPATH_LEN - offset,
	"@id='%s'", ref);
	}

	// NOTE: schema currently does not allow this
	if (ref && attr) {
	offset += snprintf(buffer + offset, MAX_XPATH_LEN - offset,
	" and ");
	}

	if (attr) {
	offset += snprintf(buffer + offset, MAX_XPATH_LEN - offset,
	"@%s", attr);
	}

	if (ref \|\| attr) {
	offset += snprintf(buffer + offset, MAX_XPATH_LEN - offset,
	"]");
	}

	CRM_LOG_ASSERT(offset > 0);
	acl->xpath = strdup(buffer);
	CRM_ASSERT(acl->xpath != NULL);

	crm_trace("Unpacked ACL <%s> element as xpath: %s",
	crm_element_name(xml), acl->xpath);
	}

	return g_list_append(acls, acl);
	}

	/*!
	* \internal
	* \brief Unpack a user, group, or role subtree of the ACLs section
	*
	* \param[in] acl_top XML of entire ACLs section
	* \param[in] acl_entry XML of ACL element being unpacked
	* \param[in,out] acls List of ACLs unpacked so far
	*
	* \return New head of (possibly modified) acls
	*/
	static GList *
	parse_acl_entry(xmlNode acl_top, xmlNode acl_entry, GList *acls)
	{
	xmlNode *child = NULL;

	for (child = pcmk__xe_first_child(acl_entry); child;
	child = pcmk__xe_next(child)) {
	const char *tag = crm_element_name(child);
	const char *kind = crm_element_value(child, XML_ACL_ATTR_KIND);

	if (strcmp(XML_ACL_TAG_PERMISSION, tag) == 0){
	CRM_ASSERT(kind != NULL);
	crm_trace("Unpacking ACL <%s> element of kind '%s'", tag, kind);
	tag = kind;
	} else {
	crm_trace("Unpacking ACL <%s> element", tag);
	}

	if (strcmp(XML_ACL_TAG_ROLE_REF, tag) == 0
	\|\| strcmp(XML_ACL_TAG_ROLE_REFv1, tag) == 0) {
	const char *ref_role = crm_element_value(child, XML_ATTR_ID);

	if (ref_role) {
	xmlNode *role = NULL;

	for (role = pcmk__xe_first_child(acl_top); role;
	role = pcmk__xe_next(role)) {
	if (!strcmp(XML_ACL_TAG_ROLE, (const char *) role->name)) {
	const char *role_id = crm_element_value(role,
	XML_ATTR_ID);

	if (role_id && strcmp(ref_role, role_id) == 0) {
	crm_trace("Unpacking referenced role '%s' in ACL <%s> element",
	role_id, crm_element_name(acl_entry));
	acls = parse_acl_entry(acl_top, role, acls);
	break;
	}
	}
	}
	}

	} else if (strcmp(XML_ACL_TAG_READ, tag) == 0) {
	acls = create_acl(child, acls, pcmk__xf_acl_read);

	} else if (strcmp(XML_ACL_TAG_WRITE, tag) == 0) {
	acls = create_acl(child, acls, pcmk__xf_acl_write);

	} else if (strcmp(XML_ACL_TAG_DENY, tag) == 0) {
	acls = create_acl(child, acls, pcmk__xf_acl_deny);

	} else {
	crm_warn("Ignoring unknown ACL %s '%s'",
	(kind? "kind" : "element"), tag);
	}
	}

	return acls;
	}

	/*
	<acls>
	<acl_target id="l33t-haxor"><role id="auto-l33t-haxor"/></acl_target>
	<acl_role id="auto-l33t-haxor">
	<acl_permission id="crook-nothing" kind="deny" xpath="/cib"/>
	</acl_role>
	<acl_target id="niceguy">
	<role id="observer"/>
	</acl_target>
	<acl_role id="observer">
	<acl_permission id="observer-read-1" kind="read" xpath="/cib"/>
	<acl_permission id="observer-write-1" kind="write" xpath="//nvpair[@name='stonith-enabled']"/>
	<acl_permission id="observer-write-2" kind="write" xpath="//nvpair[@name='target-role']"/>
	</acl_role>
	<acl_target id="badidea"><role id="auto-badidea"/></acl_target>
	<acl_role id="auto-badidea">
	<acl_permission id="badidea-resources" kind="read" xpath="//meta_attributes"/>
	<acl_permission id="badidea-resources-2" kind="deny" reference="dummy-meta_attributes"/>
	</acl_role>
	</acls>
	*/

	static const char *
	acl_to_text(enum xml_private_flags flags)
	{
	if (pcmk_is_set(flags, pcmk__xf_acl_deny)) {
	return "deny";

	} else if (pcmk_any_flags_set(flags, pcmk__xf_acl_write\|pcmk__xf_acl_create)) {
	return "read/write";

	} else if (pcmk_is_set(flags, pcmk__xf_acl_read)) {
	return "read";
	}
	return "none";
	}

	void
	pcmk__apply_acl(xmlNode *xml)
	{
	GList *aIter = NULL;
	xml_private_t *p = xml->doc->_private;
	xmlXPathObjectPtr xpathObj = NULL;

	if (!xml_acl_enabled(xml)) {
	crm_trace("Skipping ACLs for user '%s' because not enabled for this XML",
	p->user);
	return;
	}

	for (aIter = p->acls; aIter != NULL; aIter = aIter->next) {
	int max = 0, lpc = 0;
	xml_acl_t *acl = aIter->data;

	xpathObj = xpath_search(xml, acl->xpath);
	max = numXpathResults(xpathObj);

	for (lpc = 0; lpc < max; lpc++) {
	xmlNode *match = getXpathResult(xpathObj, lpc);
	char *path = xml_get_path(match);

	p = match->_private;
	crm_trace("Applying %s ACL to %s matched by %s",
	acl_to_text(acl->mode), path, acl->xpath);
	pcmk__set_xml_flags(p, acl->mode);
	free(path);
	}
	crm_trace("Applied %s ACL %s (%d match%s)",
	acl_to_text(acl->mode), acl->xpath, max,
	((max == 1)? "" : "es"));
	freeXpathObject(xpathObj);
	}
	}

	/*!
	* \internal
	* \brief Unpack ACLs for a given user
	*
	* \param[in] source XML with ACL definitions
	* \param[in,out] target XML that ACLs will be applied to
	* \param[in] user Username whose ACLs need to be unpacked
	*/
	void
	pcmk__unpack_acl(xmlNode source, xmlNode target, const char *user)
	{
	xml_private_t *p = NULL;

	if ((target == NULL) \|\| (target->doc == NULL)
	\|\| (target->doc->_private == NULL)) {
	return;
	}

	p = target->doc->_private;
	if (!pcmk_acl_required(user)) {
	crm_trace("Not unpacking ACLs because not required for user '%s'",
	user);

	} else if (p->acls == NULL) {
	xmlNode *acls = get_xpath_object("//" XML_CIB_TAG_ACLS,
	source, LOG_NEVER);

	free(p->user);
	p->user = strdup(user);

	if (acls) {
	xmlNode *child = NULL;

	for (child = pcmk__xe_first_child(acls); child;
	child = pcmk__xe_next(child)) {
	const char *tag = crm_element_name(child);

	if (!strcmp(tag, XML_ACL_TAG_USER)
	\|\| !strcmp(tag, XML_ACL_TAG_USERv1)) {
	const char *id = crm_element_value(child, XML_ATTR_ID);

	if (id && strcmp(id, user) == 0) {
	crm_debug("Unpacking ACLs for user '%s'", id);
	p->acls = parse_acl_entry(acls, child, p->acls);
	}
	}
	}
	}
	}
	}

	static inline bool
	test_acl_mode(enum xml_private_flags allowed, enum xml_private_flags requested)
	{
	if (pcmk_is_set(allowed, pcmk__xf_acl_deny)) {
	return false;

	} else if (pcmk_all_flags_set(allowed, requested)) {
	return true;

	} else if (pcmk_is_set(requested, pcmk__xf_acl_read)
	&& pcmk_is_set(allowed, pcmk__xf_acl_write)) {
	return true;

	} else if (pcmk_is_set(requested, pcmk__xf_acl_create)
	&& pcmk_any_flags_set(allowed, pcmk__xf_acl_write\|pcmk__xf_created)) {
	return true;
	}
	return false;
	}

	static bool
	purge_xml_attributes(xmlNode *xml)
	{
	xmlNode *child = NULL;
	xmlAttr *xIter = NULL;
	bool readable_children = false;
	xml_private_t *p = xml->_private;

	if (test_acl_mode(p->flags, pcmk__xf_acl_read)) {
	crm_trace("%s[@id=%s] is readable", crm_element_name(xml), ID(xml));
	return true;
	}

	xIter = xml->properties;
	while (xIter != NULL) {
	xmlAttr *tmp = xIter;
	const char prop_name = (const char )xIter->name;

	xIter = xIter->next;
	if (strcmp(prop_name, XML_ATTR_ID) == 0) {
	continue;
	}

	xmlUnsetProp(xml, tmp->name);
	}

	child = pcmk__xml_first_child(xml);
	while ( child != NULL ) {
	xmlNode *tmp = child;

	child = pcmk__xml_next(child);
	readable_children \|= purge_xml_attributes(tmp);
	}

	if (!readable_children) {
	free_xml(xml); /* Nothing readable under here, purge completely */
	}
	return readable_children;
	}

	/*!
	- * \internal
	* \brief Copy ACL-allowed portions of specified XML
	*
	* \param[in] user Username whose ACLs should be used
	* \param[in] acl_source XML containing ACLs
	* \param[in] xml XML to be copied
	* \param[out] result Copy of XML portions readable via ACLs
	*
	* \return true if xml exists and ACLs are required for user, false otherwise
	* \note If this returns true, caller should use \p result rather than \p xml
	*/
	bool
	xml_acl_filtered_copy(const char user, xmlNode acl_source, xmlNode *xml,
	xmlNode **result)
	{
	GList *aIter = NULL;
	xmlNode *target = NULL;
	xml_private_t *doc = NULL;

	*result = NULL;
	if ((xml == NULL) \|\| !pcmk_acl_required(user)) {
	crm_trace("Not filtering XML because ACLs not required for user '%s'",
	user);
	return false;
	}

	crm_trace("Filtering XML copy using user '%s' ACLs", user);
	target = copy_xml(xml);
	if (target == NULL) {
	return true;
	}

	pcmk__unpack_acl(acl_source, target, user);
	pcmk__set_xml_doc_flag(target, pcmk__xf_acl_enabled);
	pcmk__apply_acl(target);

	doc = target->doc->_private;
	for(aIter = doc->acls; aIter != NULL && target; aIter = aIter->next) {
	int max = 0;
	xml_acl_t *acl = aIter->data;

	if (acl->mode != pcmk__xf_acl_deny) {
	/* Nothing to do */

	} else if (acl->xpath) {
	int lpc = 0;
	xmlXPathObjectPtr xpathObj = xpath_search(target, acl->xpath);

	max = numXpathResults(xpathObj);
	for(lpc = 0; lpc < max; lpc++) {
	xmlNode *match = getXpathResult(xpathObj, lpc);

	if (!purge_xml_attributes(match) && (match == target)) {
	crm_trace("ACLs deny user '%s' access to entire XML document",
	user);
	freeXpathObject(xpathObj);
	return true;
	}
	}
	crm_trace("ACLs deny user '%s' access to %s (%d %s)",
	user, acl->xpath, max,
	pcmk__plural_alt(max, "match", "matches"));
	freeXpathObject(xpathObj);
	}
	}

	if (!purge_xml_attributes(target)) {
	crm_trace("ACLs deny user '%s' access to entire XML document", user);
	return true;
	}

	if (doc->acls) {
	g_list_free_full(doc->acls, free_acl);
	doc->acls = NULL;

	} else {
	crm_trace("User '%s' without ACLs denied access to entire XML document",
	user);
	free_xml(target);
	target = NULL;
	}

	if (target) {
	*result = target;
	}

	return true;
	}

	/*!
	* \internal
	* \brief Check whether creation of an XML element is implicitly allowed
	*
	* Check whether XML is a "scaffolding" element whose creation is implicitly
	* allowed regardless of ACLs (that is, it is not in the ACL section and has
	* no attributes other than "id").
	*
	* \param[in] xml XML element to check
	*
	* \return true if XML element is implicitly allowed, false otherwise
	*/
	static bool
	implicitly_allowed(xmlNode *xml)
	{
	char *path = NULL;

	for (xmlAttr *prop = xml->properties; prop != NULL; prop = prop->next) {
	if (strcmp((const char *) prop->name, XML_ATTR_ID) != 0) {
	return false;
	}
	}

	path = xml_get_path(xml);
	if (strstr(path, "/" XML_CIB_TAG_ACLS "/") != NULL) {
	free(path);
	return false;
	}
	free(path);

	return true;
	}

	#define display_id(xml) (ID(xml)? ID(xml) : "<unset>")

	/*!
	* \internal
	* \brief Drop XML nodes created in violation of ACLs
	*
	* Given an XML element, free all of its descendent nodes created in violation
	* of ACLs, with the exception of allowing "scaffolding" elements (i.e. those
	* that aren't in the ACL section and don't have any attributes other than
	* "id").
	*
	* \param[in,out] xml XML to check
	* \param[in] check_top Whether to apply checks to argument itself
	* (if true, xml might get freed)
	*/
	void
	pcmk__apply_creation_acl(xmlNode *xml, bool check_top)
	{
	xml_private_t *p = xml->_private;

	if (pcmk_is_set(p->flags, pcmk__xf_created)) {
	if (implicitly_allowed(xml)) {
	crm_trace("Creation of <%s> scaffolding with id=\"%s\""
	" is implicitly allowed",
	crm_element_name(xml), display_id(xml));

	} else if (pcmk__check_acl(xml, NULL, pcmk__xf_acl_write)) {
	crm_trace("ACLs allow creation of <%s> with id=\"%s\"",
	crm_element_name(xml), display_id(xml));

	} else if (check_top) {
	crm_trace("ACLs disallow creation of <%s> with id=\"%s\"",
	crm_element_name(xml), display_id(xml));
	pcmk_free_xml_subtree(xml);
	return;

	} else {
	crm_notice("ACLs would disallow creation of %s<%s> with id=\"%s\" ",
	((xml == xmlDocGetRootElement(xml->doc))? "root element " : ""),
	crm_element_name(xml), display_id(xml));
	}
	}

	for (xmlNode *cIter = pcmk__xml_first_child(xml); cIter != NULL; ) {
	xmlNode *child = cIter;
	cIter = pcmk__xml_next(cIter); /* In case it is free'd */
	pcmk__apply_creation_acl(child, true);
	}
	}

	bool
	xml_acl_denied(xmlNode *xml)
	{
	if (xml && xml->doc && xml->doc->_private){
	xml_private_t *p = xml->doc->_private;

	return pcmk_is_set(p->flags, pcmk__xf_acl_denied);
	}
	return false;
	}

	void
	xml_acl_disable(xmlNode *xml)
	{
	if (xml_acl_enabled(xml)) {
	xml_private_t *p = xml->doc->_private;

	/* Catch anything that was created but shouldn't have been */
	pcmk__apply_acl(xml);
	pcmk__apply_creation_acl(xml, false);
	pcmk__clear_xml_flags(p, pcmk__xf_acl_enabled);
	}
	}

	bool
	xml_acl_enabled(xmlNode *xml)
	{
	if (xml && xml->doc && xml->doc->_private){
	xml_private_t *p = xml->doc->_private;

	return pcmk_is_set(p->flags, pcmk__xf_acl_enabled);
	}
	return false;
	}

	bool
	pcmk__check_acl(xmlNode xml, const char name, enum xml_private_flags mode)
	{
	CRM_ASSERT(xml);
	CRM_ASSERT(xml->doc);
	CRM_ASSERT(xml->doc->_private);

	if (pcmk__tracking_xml_changes(xml, false) && xml_acl_enabled(xml)) {
	int offset = 0;
	xmlNode *parent = xml;
	char buffer[MAX_XPATH_LEN];
	xml_private_t *docp = xml->doc->_private;

	offset = pcmk__element_xpath(NULL, xml, buffer, offset,
	sizeof(buffer));
	if (name) {
	offset += snprintf(buffer + offset, MAX_XPATH_LEN - offset,
	"[@%s]", name);
	}
	CRM_LOG_ASSERT(offset > 0);

	if (docp->acls == NULL) {
	crm_trace("User '%s' without ACLs denied %s access to %s",
	docp->user, acl_to_text(mode), buffer);
	pcmk__set_xml_doc_flag(xml, pcmk__xf_acl_denied);
	return false;
	}

	/* Walk the tree upwards looking for xml_acl_* flags
	* - Creating an attribute requires write permissions for the node
	* - Creating a child requires write permissions for the parent
	*/

	if (name) {
	xmlAttr *attr = xmlHasProp(xml, (pcmkXmlStr) name);

	if (attr && mode == pcmk__xf_acl_create) {
	mode = pcmk__xf_acl_write;
	}
	}

	while (parent && parent->_private) {
	xml_private_t *p = parent->_private;
	if (test_acl_mode(p->flags, mode)) {
	return true;

	} else if (pcmk_is_set(p->flags, pcmk__xf_acl_deny)) {
	crm_trace("%sACL denies user '%s' %s access to %s",
	(parent != xml) ? "Parent " : "", docp->user,
	acl_to_text(mode), buffer);
	pcmk__set_xml_doc_flag(xml, pcmk__xf_acl_denied);
	return false;
	}
	parent = parent->parent;
	}

	crm_trace("Default ACL denies user '%s' %s access to %s",
	docp->user, acl_to_text(mode), buffer);
	pcmk__set_xml_doc_flag(xml, pcmk__xf_acl_denied);
	return false;
	}

	return true;
	}

	/*!
	* \brief Check whether ACLs are required for a given user
	*
	* \param[in] User name to check
	*
	* \return true if the user requires ACLs, false otherwise
	*/
	bool
	pcmk_acl_required(const char *user)
	{
	if (pcmk__str_empty(user)) {
	crm_trace("ACLs not required because no user set");
	return false;

	} else if (!strcmp(user, CRM_DAEMON_USER) \|\| !strcmp(user, "root")) {
	crm_trace("ACLs not required for privileged user %s", user);
	return false;
	}
	crm_trace("ACLs required for %s", user);
	return true;
	}

	char *
	pcmk__uid2username(uid_t uid)
	{
	struct passwd *pwent = getpwuid(uid);

	if (pwent == NULL) {
	crm_perror(LOG_INFO, "Cannot get user details for user ID %d", uid);
	return NULL;
	}
	return strdup(pwent->pw_name);
	}

	/*!
	* \internal
	* \brief Set the ACL user field properly on an XML request
	*
	* Multiple user names are potentially involved in an XML request: the effective
	* user of the current process; the user name known from an IPC client
	* connection; and the user name obtained from the request itself, whether by
	* the current standard XML attribute name or an older legacy attribute name.
	* This function chooses the appropriate one that should be used for ACLs, sets
	* it in the request (using the standard attribute name, and the legacy name if
	* given), and returns it.
	*
	* \param[in,out] request XML request to update
	* \param[in] field Alternate name for ACL user name XML attribute
	* \param[in] peer_user User name as known from IPC connection
	*
	* \return ACL user name actually used
	*/
	const char *
	pcmk__update_acl_user(xmlNode request, const char field,
	const char *peer_user)
	{
	static const char *effective_user = NULL;
	const char *requested_user = NULL;
	const char *user = NULL;

	if (effective_user == NULL) {
	effective_user = pcmk__uid2username(geteuid());
	if (effective_user == NULL) {
	effective_user = strdup("#unprivileged");
	CRM_CHECK(effective_user != NULL, return NULL);
	crm_err("Unable to determine effective user, assuming unprivileged for ACLs");
	}
	}

	requested_user = crm_element_value(request, XML_ACL_TAG_USER);
	if (requested_user == NULL) {
	/* @COMPAT rolling upgrades <=1.1.11
	*
	* field is checked for backward compatibility with older versions that
	* did not use XML_ACL_TAG_USER.
	*/
	requested_user = crm_element_value(request, field);
	}

	if (!pcmk__is_privileged(effective_user)) {
	/* We're not running as a privileged user, set or overwrite any existing
	* value for $XML_ACL_TAG_USER
	*/
	user = effective_user;

	} else if (peer_user == NULL && requested_user == NULL) {
	/* No user known or requested, use 'effective_user' and make sure one is
	* set for the request
	*/
	user = effective_user;

	} else if (peer_user == NULL) {
	/* No user known, trusting 'requested_user' */
	user = requested_user;

	} else if (!pcmk__is_privileged(peer_user)) {
	/* The peer is not a privileged user, set or overwrite any existing
	* value for $XML_ACL_TAG_USER
	*/
	user = peer_user;

	} else if (requested_user == NULL) {
	/* Even if we're privileged, make sure there is always a value set */
	user = peer_user;

	} else {
	/* Legal delegation to 'requested_user' */
	user = requested_user;
	}

	// This requires pointer comparison, not string comparison
	if (user != crm_element_value(request, XML_ACL_TAG_USER)) {
	crm_xml_add(request, XML_ACL_TAG_USER, user);
	}

	if (field != NULL && user != crm_element_value(request, field)) {
	crm_xml_add(request, field, user);
	}

	return requested_user;
	}
	diff --git a/lib/lrmd/lrmd_client.c b/lib/lrmd/lrmd_client.c
	index 81faefdbec..ee31bb5ae9 100644
	--- a/lib/lrmd/lrmd_client.c
	+++ b/lib/lrmd/lrmd_client.c
	@@ -1,2399 +1,2398 @@
	/*
	* Copyright 2012-2021 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <unistd.h>
	#include <stdlib.h>
	#include <stdio.h>
	#include <stdint.h> // uint32_t, uint64_t
	#include <stdarg.h>
	#include <string.h>
	#include <ctype.h>
	#include <errno.h>

	#include <sys/types.h>
	#include <sys/wait.h>

	#include <glib.h>
	#include <dirent.h>

	#include <crm/crm.h>
	#include <crm/lrmd.h>
	#include <crm/lrmd_internal.h>
	#include <crm/services.h>
	#include <crm/services_internal.h>
	#include <crm/common/mainloop.h>
	#include <crm/common/ipc_internal.h>
	#include <crm/common/remote_internal.h>
	#include <crm/msg_xml.h>

	#include <crm/stonith-ng.h>
	#include <crm/fencing/internal.h>

	#ifdef HAVE_GNUTLS_GNUTLS_H
	-# undef KEYFILE
	# include <gnutls/gnutls.h>
	#endif

	#include <sys/socket.h>
	#include <netinet/in.h>
	#include <netinet/ip.h>
	#include <arpa/inet.h>
	#include <netdb.h>

	#define MAX_TLS_RECV_WAIT 10000

	CRM_TRACE_INIT_DATA(lrmd);

	static int lrmd_api_disconnect(lrmd_t * lrmd);
	static int lrmd_api_is_connected(lrmd_t * lrmd);

	/* IPC proxy functions */
	int lrmd_internal_proxy_send(lrmd_t * lrmd, xmlNode *msg);
	static void lrmd_internal_proxy_dispatch(lrmd_t lrmd, xmlNode msg);
	void lrmd_internal_set_proxy_callback(lrmd_t * lrmd, void userdata, void (callback)(lrmd_t lrmd, void userdata, xmlNode *msg));

	#ifdef HAVE_GNUTLS_GNUTLS_H
	# define LRMD_CLIENT_HANDSHAKE_TIMEOUT 5000 /* 5 seconds */
	gnutls_psk_client_credentials_t psk_cred_s;
	static void lrmd_tls_disconnect(lrmd_t * lrmd);
	static int global_remote_msg_id = 0;
	static void lrmd_tls_connection_destroy(gpointer userdata);
	#endif

	typedef struct lrmd_private_s {
	uint64_t type;
	char *token;
	mainloop_io_t *source;

	/* IPC parameters */
	crm_ipc_t *ipc;

	pcmk__remote_t *remote;

	/* Extra TLS parameters */
	char *remote_nodename;
	#ifdef HAVE_GNUTLS_GNUTLS_H
	char *server;
	int port;
	gnutls_psk_client_credentials_t psk_cred_c;

	/* while the async connection is occurring, this is the id
	* of the connection timeout timer. */
	int async_timer;
	int sock;
	/* since tls requires a round trip across the network for a
	* request/reply, there are times where we just want to be able
	* to send a request from the client and not wait around (or even care
	* about) what the reply is. */
	int expected_late_replies;
	GList *pending_notify;
	crm_trigger_t *process_notify;
	#endif

	lrmd_event_callback callback;

	/* Internal IPC proxy msg passing for remote guests */
	void (proxy_callback)(lrmd_t lrmd, void userdata, xmlNode msg);
	void *proxy_callback_userdata;
	char *peer_version;
	} lrmd_private_t;

	static lrmd_list_t *
	lrmd_list_add(lrmd_list_t * head, const char *value)
	{
	lrmd_list_t p, end;

	p = calloc(1, sizeof(lrmd_list_t));
	p->val = strdup(value);

	end = head;
	while (end && end->next) {
	end = end->next;
	}

	if (end) {
	end->next = p;
	} else {
	head = p;
	}

	return head;
	}

	void
	lrmd_list_freeall(lrmd_list_t * head)
	{
	lrmd_list_t *p;

	while (head) {
	char val = (char )head->val;

	p = head->next;
	free(val);
	free(head);
	head = p;
	}
	}

	lrmd_key_value_t *
	lrmd_key_value_add(lrmd_key_value_t * head, const char key, const char value)
	{
	lrmd_key_value_t p, end;

	p = calloc(1, sizeof(lrmd_key_value_t));
	p->key = strdup(key);
	p->value = strdup(value);

	end = head;
	while (end && end->next) {
	end = end->next;
	}

	if (end) {
	end->next = p;
	} else {
	head = p;
	}

	return head;
	}

	void
	lrmd_key_value_freeall(lrmd_key_value_t * head)
	{
	lrmd_key_value_t *p;

	while (head) {
	p = head->next;
	free(head->key);
	free(head->value);
	free(head);
	head = p;
	}
	}

	/*!
	* Create a new lrmd_event_data_t object
	*
	* \param[in] rsc_id ID of resource involved in event
	* \param[in] task Action name
	* \param[in] interval_ms Action interval
	*
	* \return Newly allocated and initialized lrmd_event_data_t
	* \note This functions asserts on memory errors, so the return value is
	* guaranteed to be non-NULL. The caller is responsible for freeing the
	* result with lrmd_free_event().
	*/
	lrmd_event_data_t *
	lrmd_new_event(const char rsc_id, const char task, guint interval_ms)
	{
	lrmd_event_data_t *event = calloc(1, sizeof(lrmd_event_data_t));

	CRM_ASSERT(event != NULL);
	if (rsc_id != NULL) {
	event->rsc_id = strdup(rsc_id);
	CRM_ASSERT(event->rsc_id != NULL);
	}
	if (task != NULL) {
	event->op_type = strdup(task);
	CRM_ASSERT(event->op_type != NULL);
	}
	event->interval_ms = interval_ms;
	return event;
	}

	lrmd_event_data_t *
	lrmd_copy_event(lrmd_event_data_t * event)
	{
	lrmd_event_data_t *copy = NULL;

	copy = calloc(1, sizeof(lrmd_event_data_t));

	/* This will get all the int values.
	* we just have to be careful not to leave any
	* dangling pointers to strings. */
	memcpy(copy, event, sizeof(lrmd_event_data_t));

	copy->rsc_id = event->rsc_id ? strdup(event->rsc_id) : NULL;
	copy->op_type = event->op_type ? strdup(event->op_type) : NULL;
	copy->user_data = event->user_data ? strdup(event->user_data) : NULL;
	copy->output = event->output ? strdup(event->output) : NULL;
	copy->exit_reason = event->exit_reason ? strdup(event->exit_reason) : NULL;
	copy->remote_nodename = event->remote_nodename ? strdup(event->remote_nodename) : NULL;
	copy->params = pcmk__str_table_dup(event->params);

	return copy;
	}

	/*!
	* \brief Free an executor event
	*
	* \param[in] Executor event object to free
	*/
	void
	lrmd_free_event(lrmd_event_data_t *event)
	{
	if (event == NULL) {
	return;
	}
	// @TODO Why are these const char *?
	free((void *) event->rsc_id);
	free((void *) event->op_type);
	free((void *) event->user_data);
	free((void *) event->remote_nodename);
	lrmd__reset_result(event);
	if (event->params != NULL) {
	g_hash_table_destroy(event->params);
	}
	free(event);
	}

	static void
	lrmd_dispatch_internal(lrmd_t * lrmd, xmlNode * msg)
	{
	const char *type;
	const char *proxy_session = crm_element_value(msg, F_LRMD_IPC_SESSION);
	lrmd_private_t *native = lrmd->lrmd_private;
	lrmd_event_data_t event = { 0, };

	if (proxy_session != NULL) {
	/* this is proxy business */
	lrmd_internal_proxy_dispatch(lrmd, msg);
	return;
	} else if (!native->callback) {
	/* no callback set */
	crm_trace("notify event received but client has not set callback");
	return;
	}

	event.remote_nodename = native->remote_nodename;
	type = crm_element_value(msg, F_LRMD_OPERATION);
	crm_element_value_int(msg, F_LRMD_CALLID, &event.call_id);
	event.rsc_id = crm_element_value(msg, F_LRMD_RSC_ID);

	if (pcmk__str_eq(type, LRMD_OP_RSC_REG, pcmk__str_none)) {
	event.type = lrmd_event_register;
	} else if (pcmk__str_eq(type, LRMD_OP_RSC_UNREG, pcmk__str_none)) {
	event.type = lrmd_event_unregister;
	} else if (pcmk__str_eq(type, LRMD_OP_RSC_EXEC, pcmk__str_none)) {
	time_t epoch = 0;

	crm_element_value_int(msg, F_LRMD_TIMEOUT, &event.timeout);
	crm_element_value_ms(msg, F_LRMD_RSC_INTERVAL, &event.interval_ms);
	crm_element_value_int(msg, F_LRMD_RSC_START_DELAY, &event.start_delay);
	crm_element_value_int(msg, F_LRMD_EXEC_RC, (int *)&event.rc);
	crm_element_value_int(msg, F_LRMD_OP_STATUS, &event.op_status);
	crm_element_value_int(msg, F_LRMD_RSC_DELETED, &event.rsc_deleted);

	crm_element_value_epoch(msg, F_LRMD_RSC_RUN_TIME, &epoch);
	event.t_run = (unsigned int) epoch;

	crm_element_value_epoch(msg, F_LRMD_RSC_RCCHANGE_TIME, &epoch);
	event.t_rcchange = (unsigned int) epoch;

	crm_element_value_int(msg, F_LRMD_RSC_EXEC_TIME, (int *)&event.exec_time);
	crm_element_value_int(msg, F_LRMD_RSC_QUEUE_TIME, (int *)&event.queue_time);

	event.op_type = crm_element_value(msg, F_LRMD_RSC_ACTION);
	event.user_data = crm_element_value(msg, F_LRMD_RSC_USERDATA_STR);
	event.type = lrmd_event_exec_complete;

	// No need to duplicate the memory, so don't use setter functions
	event.output = crm_element_value(msg, F_LRMD_RSC_OUTPUT);
	event.exit_reason = crm_element_value(msg, F_LRMD_RSC_EXIT_REASON);

	event.params = xml2list(msg);
	} else if (pcmk__str_eq(type, LRMD_OP_NEW_CLIENT, pcmk__str_none)) {
	event.type = lrmd_event_new_client;
	} else if (pcmk__str_eq(type, LRMD_OP_POKE, pcmk__str_none)) {
	event.type = lrmd_event_poke;
	} else {
	return;
	}

	crm_trace("op %s notify event received", type);
	native->callback(&event);

	if (event.params) {
	g_hash_table_destroy(event.params);
	}
	}

	// \return Always 0, to indicate that IPC mainloop source should be kept
	static int
	lrmd_ipc_dispatch(const char *buffer, ssize_t length, gpointer userdata)
	{
	lrmd_t *lrmd = userdata;
	lrmd_private_t *native = lrmd->lrmd_private;

	if (native->callback != NULL) {
	xmlNode *msg = string2xml(buffer);

	lrmd_dispatch_internal(lrmd, msg);
	free_xml(msg);
	}
	return 0;
	}

	#ifdef HAVE_GNUTLS_GNUTLS_H
	static void
	lrmd_free_xml(gpointer userdata)
	{
	free_xml((xmlNode *) userdata);
	}

	static bool
	remote_executor_connected(lrmd_t * lrmd)
	{
	lrmd_private_t *native = lrmd->lrmd_private;

	return (native->remote->tls_session != NULL);
	}

	/*!
	* \internal
	* \brief TLS dispatch function (for both trigger and file descriptor sources)
	*
	* \param[in] userdata API connection
	*
	* \return Always return a nonnegative value, which as a file descriptor
	* dispatch function means keep the mainloop source, and as a
	* trigger dispatch function, 0 means remove the trigger from the
	* mainloop while 1 means keep it (and job completed)
	*/
	static int
	lrmd_tls_dispatch(gpointer userdata)
	{
	lrmd_t *lrmd = userdata;
	lrmd_private_t *native = lrmd->lrmd_private;
	xmlNode *xml = NULL;
	int rc = pcmk_rc_ok;

	if (!remote_executor_connected(lrmd)) {
	crm_trace("TLS dispatch triggered after disconnect");
	return 0;
	}

	crm_trace("TLS dispatch triggered");

	/* First check if there are any pending notifies to process that came
	* while we were waiting for replies earlier. */
	if (native->pending_notify) {
	GList *iter = NULL;

	crm_trace("Processing pending notifies");
	for (iter = native->pending_notify; iter; iter = iter->next) {
	lrmd_dispatch_internal(lrmd, iter->data);
	}
	g_list_free_full(native->pending_notify, lrmd_free_xml);
	native->pending_notify = NULL;
	}

	/* Next read the current buffer and see if there are any messages to handle. */
	switch (pcmk__remote_ready(native->remote, 0)) {
	case pcmk_rc_ok:
	rc = pcmk__read_remote_message(native->remote, -1);
	xml = pcmk__remote_message_xml(native->remote);
	break;
	case ETIME:
	// Nothing to read, check if a full message is already in buffer
	xml = pcmk__remote_message_xml(native->remote);
	break;
	default:
	rc = ENOTCONN;
	break;
	}
	while (xml) {
	const char *msg_type = crm_element_value(xml, F_LRMD_REMOTE_MSG_TYPE);
	if (pcmk__str_eq(msg_type, "notify", pcmk__str_casei)) {
	lrmd_dispatch_internal(lrmd, xml);
	} else if (pcmk__str_eq(msg_type, "reply", pcmk__str_casei)) {
	if (native->expected_late_replies > 0) {
	native->expected_late_replies--;
	} else {
	int reply_id = 0;
	crm_element_value_int(xml, F_LRMD_CALLID, &reply_id);
	/* if this happens, we want to know about it */
	crm_err("Got outdated Pacemaker Remote reply %d", reply_id);
	}
	}
	free_xml(xml);
	xml = pcmk__remote_message_xml(native->remote);
	}

	if (rc == ENOTCONN) {
	crm_info("Lost %s executor connection while reading data",
	(native->remote_nodename? native->remote_nodename : "local"));
	lrmd_tls_disconnect(lrmd);
	return 0;
	}
	return 1;
	}
	#endif

	/* Not used with mainloop */
	int
	lrmd_poll(lrmd_t * lrmd, int timeout)
	{
	lrmd_private_t *native = lrmd->lrmd_private;

	switch (native->type) {
	case pcmk__client_ipc:
	return crm_ipc_ready(native->ipc);

	#ifdef HAVE_GNUTLS_GNUTLS_H
	case pcmk__client_tls:
	if (native->pending_notify) {
	return 1;
	} else {
	int rc = pcmk__remote_ready(native->remote, 0);

	switch (rc) {
	case pcmk_rc_ok:
	return 1;
	case ETIME:
	return 0;
	default:
	return pcmk_rc2legacy(rc);
	}
	}
	#endif
	default:
	crm_err("Unsupported executor connection type (bug?): %d",
	native->type);
	return -EPROTONOSUPPORT;
	}
	}

	/* Not used with mainloop */
	bool
	lrmd_dispatch(lrmd_t * lrmd)
	{
	lrmd_private_t *private = NULL;

	CRM_ASSERT(lrmd != NULL);

	private = lrmd->lrmd_private;
	switch (private->type) {
	case pcmk__client_ipc:
	while (crm_ipc_ready(private->ipc)) {
	if (crm_ipc_read(private->ipc) > 0) {
	const char *msg = crm_ipc_buffer(private->ipc);

	lrmd_ipc_dispatch(msg, strlen(msg), lrmd);
	}
	}
	break;
	#ifdef HAVE_GNUTLS_GNUTLS_H
	case pcmk__client_tls:
	lrmd_tls_dispatch(lrmd);
	break;
	#endif
	default:
	crm_err("Unsupported executor connection type (bug?): %d",
	private->type);
	}

	if (lrmd_api_is_connected(lrmd) == FALSE) {
	crm_err("Connection closed");
	return FALSE;
	}

	return TRUE;
	}

	static xmlNode *
	lrmd_create_op(const char token, const char op, xmlNode *data, int timeout,
	enum lrmd_call_options options)
	{
	xmlNode *op_msg = create_xml_node(NULL, "lrmd_command");

	CRM_CHECK(op_msg != NULL, return NULL);
	CRM_CHECK(token != NULL, return NULL);

	crm_xml_add(op_msg, F_XML_TAGNAME, "lrmd_command");
	crm_xml_add(op_msg, F_TYPE, T_LRMD);
	crm_xml_add(op_msg, F_LRMD_CALLBACK_TOKEN, token);
	crm_xml_add(op_msg, F_LRMD_OPERATION, op);
	crm_xml_add_int(op_msg, F_LRMD_TIMEOUT, timeout);
	crm_xml_add_int(op_msg, F_LRMD_CALLOPTS, options);

	if (data != NULL) {
	add_message_xml(op_msg, F_LRMD_CALLDATA, data);
	}

	crm_trace("Created executor %s command with call options %.8lx (%d)",
	op, (long)options, options);
	return op_msg;
	}

	static void
	lrmd_ipc_connection_destroy(gpointer userdata)
	{
	lrmd_t *lrmd = userdata;
	lrmd_private_t *native = lrmd->lrmd_private;

	crm_info("IPC connection destroyed");

	/* Prevent these from being cleaned up in lrmd_api_disconnect() */
	native->ipc = NULL;
	native->source = NULL;

	if (native->callback) {
	lrmd_event_data_t event = { 0, };
	event.type = lrmd_event_disconnect;
	event.remote_nodename = native->remote_nodename;
	native->callback(&event);
	}
	}

	#ifdef HAVE_GNUTLS_GNUTLS_H
	static void
	lrmd_tls_connection_destroy(gpointer userdata)
	{
	lrmd_t *lrmd = userdata;
	lrmd_private_t *native = lrmd->lrmd_private;

	crm_info("TLS connection destroyed");

	if (native->remote->tls_session) {
	gnutls_bye(*native->remote->tls_session, GNUTLS_SHUT_RDWR);
	gnutls_deinit(*native->remote->tls_session);
	gnutls_free(native->remote->tls_session);
	}
	if (native->psk_cred_c) {
	gnutls_psk_free_client_credentials(native->psk_cred_c);
	}
	if (native->sock) {
	close(native->sock);
	}
	if (native->process_notify) {
	mainloop_destroy_trigger(native->process_notify);
	native->process_notify = NULL;
	}
	if (native->pending_notify) {
	g_list_free_full(native->pending_notify, lrmd_free_xml);
	native->pending_notify = NULL;
	}

	free(native->remote->buffer);
	native->remote->buffer = NULL;
	native->source = 0;
	native->sock = 0;
	native->psk_cred_c = NULL;
	native->remote->tls_session = NULL;
	native->sock = 0;

	if (native->callback) {
	lrmd_event_data_t event = { 0, };
	event.remote_nodename = native->remote_nodename;
	event.type = lrmd_event_disconnect;
	native->callback(&event);
	}
	return;
	}

	// \return Standard Pacemaker return code
	int
	lrmd__remote_send_xml(pcmk__remote_t session, xmlNode msg, uint32_t id,
	const char *msg_type)
	{
	crm_xml_add_int(msg, F_LRMD_REMOTE_MSG_ID, id);
	crm_xml_add(msg, F_LRMD_REMOTE_MSG_TYPE, msg_type);
	return pcmk__remote_send_xml(session, msg);
	}

	// \return Standard Pacemaker return code
	static int
	read_remote_reply(lrmd_t *lrmd, int total_timeout, int expected_reply_id,
	xmlNode **reply)
	{
	lrmd_private_t *native = lrmd->lrmd_private;
	time_t start = time(NULL);
	const char *msg_type = NULL;
	int reply_id = 0;
	int remaining_timeout = 0;
	int rc = pcmk_rc_ok;

	/* A timeout of 0 here makes no sense. We have to wait a period of time
	* for the response to come back. If -1 or 0, default to 10 seconds. */
	if (total_timeout <= 0 \|\| total_timeout > MAX_TLS_RECV_WAIT) {
	total_timeout = MAX_TLS_RECV_WAIT;
	}

	for (reply = NULL; reply == NULL; ) {

	*reply = pcmk__remote_message_xml(native->remote);
	if (*reply == NULL) {
	/* read some more off the tls buffer if we still have time left. */
	if (remaining_timeout) {
	remaining_timeout = total_timeout - ((time(NULL) - start) * 1000);
	} else {
	remaining_timeout = total_timeout;
	}
	if (remaining_timeout <= 0) {
	return ETIME;
	}

	rc = pcmk__read_remote_message(native->remote, remaining_timeout);
	if (rc != pcmk_rc_ok) {
	return rc;
	}

	*reply = pcmk__remote_message_xml(native->remote);
	if (*reply == NULL) {
	return ENOMSG;
	}
	}

	crm_element_value_int(*reply, F_LRMD_REMOTE_MSG_ID, &reply_id);
	msg_type = crm_element_value(*reply, F_LRMD_REMOTE_MSG_TYPE);

	if (!msg_type) {
	crm_err("Empty msg type received while waiting for reply");
	free_xml(*reply);
	*reply = NULL;
	} else if (pcmk__str_eq(msg_type, "notify", pcmk__str_casei)) {
	/* got a notify while waiting for reply, trigger the notify to be processed later */
	crm_info("queueing notify");
	native->pending_notify = g_list_append(native->pending_notify, *reply);
	if (native->process_notify) {
	crm_info("notify trigger set.");
	mainloop_set_trigger(native->process_notify);
	}
	*reply = NULL;
	} else if (!pcmk__str_eq(msg_type, "reply", pcmk__str_casei)) {
	/* msg isn't a reply, make some noise */
	crm_err("Expected a reply, got %s", msg_type);
	free_xml(*reply);
	*reply = NULL;
	} else if (reply_id != expected_reply_id) {
	if (native->expected_late_replies > 0) {
	native->expected_late_replies--;
	} else {
	crm_err("Got outdated reply, expected id %d got id %d", expected_reply_id, reply_id);
	}
	free_xml(*reply);
	*reply = NULL;
	}
	}

	if (native->remote->buffer && native->process_notify) {
	mainloop_set_trigger(native->process_notify);
	}

	return rc;
	}

	// \return Standard Pacemaker return code
	static int
	send_remote_message(lrmd_t lrmd, xmlNode msg)
	{
	int rc = pcmk_rc_ok;
	lrmd_private_t *native = lrmd->lrmd_private;

	global_remote_msg_id++;
	if (global_remote_msg_id <= 0) {
	global_remote_msg_id = 1;
	}

	rc = lrmd__remote_send_xml(native->remote, msg, global_remote_msg_id,
	"request");
	if (rc != pcmk_rc_ok) {
	crm_err("Disconnecting because TLS message could not be sent to "
	"Pacemaker Remote: %s", pcmk_rc_str(rc));
	lrmd_tls_disconnect(lrmd);
	}
	return rc;
	}

	static int
	lrmd_tls_send_recv(lrmd_t * lrmd, xmlNode * msg, int timeout, xmlNode ** reply)
	{
	int rc = 0;
	xmlNode *xml = NULL;

	if (!remote_executor_connected(lrmd)) {
	return -ENOTCONN;
	}

	rc = send_remote_message(lrmd, msg);
	if (rc != pcmk_rc_ok) {
	return pcmk_rc2legacy(rc);
	}

	rc = read_remote_reply(lrmd, timeout, global_remote_msg_id, &xml);
	if (rc != pcmk_rc_ok) {
	crm_err("Disconnecting remote after request %d reply not received: %s "
	CRM_XS " rc=%d timeout=%dms",
	global_remote_msg_id, pcmk_rc_str(rc), rc, timeout);
	lrmd_tls_disconnect(lrmd);
	}

	if (reply) {
	*reply = xml;
	} else {
	free_xml(xml);
	}

	return pcmk_rc2legacy(rc);
	}
	#endif

	static int
	lrmd_send_xml(lrmd_t * lrmd, xmlNode * msg, int timeout, xmlNode ** reply)
	{
	int rc = pcmk_ok;
	lrmd_private_t *native = lrmd->lrmd_private;

	switch (native->type) {
	case pcmk__client_ipc:
	rc = crm_ipc_send(native->ipc, msg, crm_ipc_client_response, timeout, reply);
	break;
	#ifdef HAVE_GNUTLS_GNUTLS_H
	case pcmk__client_tls:
	rc = lrmd_tls_send_recv(lrmd, msg, timeout, reply);
	break;
	#endif
	default:
	crm_err("Unsupported executor connection type (bug?): %d",
	native->type);
	rc = -EPROTONOSUPPORT;
	}

	return rc;
	}

	static int
	lrmd_send_xml_no_reply(lrmd_t * lrmd, xmlNode * msg)
	{
	int rc = pcmk_ok;
	lrmd_private_t *native = lrmd->lrmd_private;

	switch (native->type) {
	case pcmk__client_ipc:
	rc = crm_ipc_send(native->ipc, msg, crm_ipc_flags_none, 0, NULL);
	break;
	#ifdef HAVE_GNUTLS_GNUTLS_H
	case pcmk__client_tls:
	rc = send_remote_message(lrmd, msg);
	if (rc == pcmk_rc_ok) {
	/* we don't want to wait around for the reply, but
	* since the request/reply protocol needs to behave the same
	* as libqb, a reply will eventually come later anyway. */
	native->expected_late_replies++;
	}
	rc = pcmk_rc2legacy(rc);
	break;
	#endif
	default:
	crm_err("Unsupported executor connection type (bug?): %d",
	native->type);
	rc = -EPROTONOSUPPORT;
	}

	return rc;
	}

	static int
	lrmd_api_is_connected(lrmd_t * lrmd)
	{
	lrmd_private_t *native = lrmd->lrmd_private;

	switch (native->type) {
	case pcmk__client_ipc:
	return crm_ipc_connected(native->ipc);
	#ifdef HAVE_GNUTLS_GNUTLS_H
	case pcmk__client_tls:
	return remote_executor_connected(lrmd);
	#endif
	default:
	crm_err("Unsupported executor connection type (bug?): %d",
	native->type);
	return 0;
	}
	}

	/*!
	* \internal
	* \brief Send a prepared API command to the executor
	*
	* \param[in] lrmd Existing connection to the executor
	* \param[in] op Name of API command to send
	* \param[in] data Command data XML to add to the sent command
	* \param[out] output_data If expecting a reply, it will be stored here
	* \param[in] timeout Timeout in milliseconds (if 0, defaults to
	* a sensible value per the type of connection,
	* standard vs. pacemaker remote);
	* also propagated to the command XML
	* \param[in] call_options Call options to pass to server when sending
	* \param[in] expect_reply If TRUE, wait for a reply from the server;
	* must be TRUE for IPC (as opposed to TLS) clients
	*
	* \return pcmk_ok on success, -errno on error
	*/
	static int
	lrmd_send_command(lrmd_t lrmd, const char op, xmlNode *data,
	xmlNode **output_data, int timeout,
	enum lrmd_call_options options, gboolean expect_reply)
	{
	int rc = pcmk_ok;
	lrmd_private_t *native = lrmd->lrmd_private;
	xmlNode *op_msg = NULL;
	xmlNode *op_reply = NULL;

	if (!lrmd_api_is_connected(lrmd)) {
	return -ENOTCONN;
	}

	if (op == NULL) {
	crm_err("No operation specified");
	return -EINVAL;
	}

	CRM_CHECK(native->token != NULL,;
	);
	crm_trace("Sending %s op to executor", op);

	op_msg = lrmd_create_op(native->token, op, data, timeout, options);

	if (op_msg == NULL) {
	return -EINVAL;
	}

	if (expect_reply) {
	rc = lrmd_send_xml(lrmd, op_msg, timeout, &op_reply);
	} else {
	rc = lrmd_send_xml_no_reply(lrmd, op_msg);
	goto done;
	}

	if (rc < 0) {
	crm_perror(LOG_ERR, "Couldn't perform %s operation (timeout=%d): %d", op, timeout, rc);
	goto done;

	} else if(op_reply == NULL) {
	rc = -ENOMSG;
	goto done;
	}

	rc = pcmk_ok;
	crm_trace("%s op reply received", op);
	if (crm_element_value_int(op_reply, F_LRMD_RC, &rc) != 0) {
	rc = -ENOMSG;
	goto done;
	}

	crm_log_xml_trace(op_reply, "Reply");

	if (output_data) {
	*output_data = op_reply;
	op_reply = NULL; /* Prevent subsequent free */
	}

	done:
	if (lrmd_api_is_connected(lrmd) == FALSE) {
	crm_err("Executor disconnected");
	}

	free_xml(op_msg);
	free_xml(op_reply);
	return rc;
	}

	static int
	lrmd_api_poke_connection(lrmd_t * lrmd)
	{
	int rc;
	lrmd_private_t *native = lrmd->lrmd_private;
	xmlNode *data = create_xml_node(NULL, F_LRMD_RSC);

	crm_xml_add(data, F_LRMD_ORIGIN, __func__);
	rc = lrmd_send_command(lrmd, LRMD_OP_POKE, data, NULL, 0, 0,
	(native->type == pcmk__client_ipc));
	free_xml(data);

	return rc < 0 ? rc : pcmk_ok;
	}

	// \return Standard Pacemaker return code
	int
	lrmd__validate_remote_settings(lrmd_t lrmd, GHashTable hash)
	{
	int rc = pcmk_rc_ok;
	const char *value;
	lrmd_private_t *native = lrmd->lrmd_private;
	xmlNode *data = create_xml_node(NULL, F_LRMD_OPERATION);

	crm_xml_add(data, F_LRMD_ORIGIN, __func__);

	value = g_hash_table_lookup(hash, "stonith-watchdog-timeout");
	if ((value) &&
	(stonith__watchdog_fencing_enabled_for_node(native->remote_nodename))) {
	crm_xml_add(data, F_LRMD_WATCHDOG, value);
	}

	rc = lrmd_send_command(lrmd, LRMD_OP_CHECK, data, NULL, 0, 0,
	(native->type == pcmk__client_ipc));
	free_xml(data);
	return (rc < 0)? pcmk_legacy2rc(rc) : pcmk_rc_ok;
	}

	static int
	lrmd_handshake(lrmd_t * lrmd, const char *name)
	{
	int rc = pcmk_ok;
	lrmd_private_t *native = lrmd->lrmd_private;
	xmlNode *reply = NULL;
	xmlNode *hello = create_xml_node(NULL, "lrmd_command");

	crm_xml_add(hello, F_TYPE, T_LRMD);
	crm_xml_add(hello, F_LRMD_OPERATION, CRM_OP_REGISTER);
	crm_xml_add(hello, F_LRMD_CLIENTNAME, name);
	crm_xml_add(hello, F_LRMD_PROTOCOL_VERSION, LRMD_PROTOCOL_VERSION);

	/* advertise that we are a proxy provider */
	if (native->proxy_callback) {
	pcmk__xe_set_bool_attr(hello, F_LRMD_IS_IPC_PROVIDER, true);
	}

	rc = lrmd_send_xml(lrmd, hello, -1, &reply);

	if (rc < 0) {
	crm_perror(LOG_DEBUG, "Couldn't complete registration with the executor API: %d", rc);
	rc = -ECOMM;
	} else if (reply == NULL) {
	crm_err("Did not receive registration reply");
	rc = -EPROTO;
	} else {
	const char *version = crm_element_value(reply, F_LRMD_PROTOCOL_VERSION);
	const char *msg_type = crm_element_value(reply, F_LRMD_OPERATION);
	const char *tmp_ticket = crm_element_value(reply, F_LRMD_CLIENTID);

	crm_element_value_int(reply, F_LRMD_RC, &rc);

	if (rc == -EPROTO) {
	crm_err("Executor protocol version mismatch between client (%s) and server (%s)",
	LRMD_PROTOCOL_VERSION, version);
	crm_log_xml_err(reply, "Protocol Error");

	} else if (!pcmk__str_eq(msg_type, CRM_OP_REGISTER, pcmk__str_casei)) {
	crm_err("Invalid registration message: %s", msg_type);
	crm_log_xml_err(reply, "Bad reply");
	rc = -EPROTO;
	} else if (tmp_ticket == NULL) {
	crm_err("No registration token provided");
	crm_log_xml_err(reply, "Bad reply");
	rc = -EPROTO;
	} else {
	crm_trace("Obtained registration token: %s", tmp_ticket);
	native->token = strdup(tmp_ticket);
	native->peer_version = strdup(version?version:"1.0"); /* Included since 1.1 */
	rc = pcmk_ok;
	}
	}

	free_xml(reply);
	free_xml(hello);

	if (rc != pcmk_ok) {
	lrmd_api_disconnect(lrmd);
	}
	return rc;
	}

	static int
	lrmd_ipc_connect(lrmd_t * lrmd, int *fd)
	{
	int rc = pcmk_ok;
	lrmd_private_t *native = lrmd->lrmd_private;

	struct ipc_client_callbacks lrmd_callbacks = {
	.dispatch = lrmd_ipc_dispatch,
	.destroy = lrmd_ipc_connection_destroy
	};

	crm_info("Connecting to executor");

	if (fd) {
	/* No mainloop */
	native->ipc = crm_ipc_new(CRM_SYSTEM_LRMD, 0);
	if (native->ipc && crm_ipc_connect(native->ipc)) {
	*fd = crm_ipc_get_fd(native->ipc);
	} else if (native->ipc) {
	crm_perror(LOG_ERR, "Connection to executor failed");
	rc = -ENOTCONN;
	}
	} else {
	native->source = mainloop_add_ipc_client(CRM_SYSTEM_LRMD, G_PRIORITY_HIGH, 0, lrmd, &lrmd_callbacks);
	native->ipc = mainloop_get_ipc_client(native->source);
	}

	if (native->ipc == NULL) {
	crm_debug("Could not connect to the executor API");
	rc = -ENOTCONN;
	}

	return rc;
	}

	#ifdef HAVE_GNUTLS_GNUTLS_H
	static void
	copy_gnutls_datum(gnutls_datum_t dest, gnutls_datum_t source)
	{
	CRM_ASSERT((dest != NULL) && (source != NULL) && (source->data != NULL));

	dest->data = gnutls_malloc(source->size);
	CRM_ASSERT(dest->data);

	memcpy(dest->data, source->data, source->size);
	dest->size = source->size;
	}

	static void
	clear_gnutls_datum(gnutls_datum_t *datum)
	{
	gnutls_free(datum->data);
	datum->data = NULL;
	datum->size = 0;
	}

	#define KEY_READ_LEN 256 // Chunk size for reading key from file

	// \return Standard Pacemaker return code
	static int
	read_gnutls_key(const char location, gnutls_datum_t key)
	{
	FILE *stream = NULL;
	size_t buf_len = KEY_READ_LEN;

	if ((location == NULL) \|\| (key == NULL)) {
	return EINVAL;
	}

	stream = fopen(location, "r");
	if (stream == NULL) {
	return errno;
	}

	key->data = gnutls_malloc(buf_len);
	key->size = 0;
	while (!feof(stream)) {
	int next = fgetc(stream);

	if (next == EOF) {
	if (!feof(stream)) {
	crm_warn("Pacemaker Remote key read was partially successful "
	"(copy in memory may be corrupted)");
	}
	break;
	}
	if (key->size == buf_len) {
	buf_len = key->size + KEY_READ_LEN;
	key->data = gnutls_realloc(key->data, buf_len);
	CRM_ASSERT(key->data);
	}
	key->data[key->size++] = (unsigned char) next;
	}
	fclose(stream);

	if (key->size == 0) {
	clear_gnutls_datum(key);
	return ENOKEY;
	}
	return pcmk_rc_ok;
	}

	// Cache the most recently used Pacemaker Remote authentication key

	struct key_cache_s {
	time_t updated; // When cached key was read (valid for 1 minute)
	const char *location; // Where cached key was read from
	gnutls_datum_t key; // Cached key
	};

	static bool
	key_is_cached(struct key_cache_s *key_cache)
	{
	return key_cache->updated != 0;
	}

	static bool
	key_cache_expired(struct key_cache_s *key_cache)
	{
	return (time(NULL) - key_cache->updated) >= 60;
	}

	static void
	clear_key_cache(struct key_cache_s *key_cache)
	{
	clear_gnutls_datum(&(key_cache->key));
	if ((key_cache->updated != 0) \|\| (key_cache->location != NULL)) {
	key_cache->updated = 0;
	key_cache->location = NULL;
	crm_debug("Cleared Pacemaker Remote key cache");
	}
	}

	static void
	get_cached_key(struct key_cache_s key_cache, gnutls_datum_t key)
	{
	copy_gnutls_datum(key, &(key_cache->key));
	crm_debug("Using cached Pacemaker Remote key from %s",
	crm_str(key_cache->location));
	}

	static void
	cache_key(struct key_cache_s key_cache, gnutls_datum_t key,
	const char *location)
	{
	key_cache->updated = time(NULL);
	key_cache->location = location;
	copy_gnutls_datum(&(key_cache->key), key);
	crm_debug("Using (and cacheing) Pacemaker Remote key from %s",
	crm_str(location));
	}

	/*!
	* \internal
	* \brief Get Pacemaker Remote authentication key from file or cache
	*
	* \param[in] location Path to key file to try (this memory must
	* persist across all calls of this function)
	* \param[out] key Key from location or cache
	*
	* \return Standard Pacemaker return code
	*/
	static int
	get_remote_key(const char location, gnutls_datum_t key)
	{
	static struct key_cache_s key_cache = { 0, };
	int rc = pcmk_rc_ok;

	if ((location == NULL) \|\| (key == NULL)) {
	return EINVAL;
	}

	if (key_is_cached(&key_cache)) {
	if (key_cache_expired(&key_cache)) {
	clear_key_cache(&key_cache);
	} else {
	get_cached_key(&key_cache, key);
	return pcmk_rc_ok;
	}
	}

	rc = read_gnutls_key(location, key);
	if (rc != pcmk_rc_ok) {
	return rc;
	}
	cache_key(&key_cache, key, location);
	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief Initialize the Pacemaker Remote authentication key
	*
	* Try loading the Pacemaker Remote authentication key from cache if available,
	* otherwise from these locations, in order of preference: the value of the
	* PCMK_authkey_location environment variable, if set; the Pacemaker default key
	* file location; or (for historical reasons) /etc/corosync/authkey.
	*
	* \param[out] key Where to store key
	*
	* \return Standard Pacemaker return code
	*/
	int
	lrmd__init_remote_key(gnutls_datum_t *key)
	{
	static const char *env_location = NULL;
	static bool need_env = true;

	int env_rc = pcmk_rc_ok;
	int default_rc = pcmk_rc_ok;
	int alt_rc = pcmk_rc_ok;

	bool env_is_default = false;
	bool env_is_fallback = false;

	if (need_env) {
	env_location = getenv("PCMK_authkey_location");
	need_env = false;
	}

	// Try location in environment variable, if set
	if (env_location != NULL) {
	env_rc = get_remote_key(env_location, key);
	if (env_rc == pcmk_rc_ok) {
	return pcmk_rc_ok;
	}

	env_is_default = !strcmp(env_location, DEFAULT_REMOTE_KEY_LOCATION);
	env_is_fallback = !strcmp(env_location, ALT_REMOTE_KEY_LOCATION);

	/* @TODO It would be more secure to fail, rather than fall back to the
	* default, if an explicitly set key location is not readable, and it
	* would be better to never use the Corosync location as a fallback.
	* However, that would break any deployments currently working with the
	* fallbacks.
	*/
	}

	// Try default location, if environment wasn't explicitly set to it
	if (env_is_default) {
	default_rc = env_rc;
	} else {
	default_rc = get_remote_key(DEFAULT_REMOTE_KEY_LOCATION, key);
	}

	// Try fallback location, if environment wasn't set to it and default failed
	if (env_is_fallback) {
	alt_rc = env_rc;
	} else if (default_rc != pcmk_rc_ok) {
	alt_rc = get_remote_key(ALT_REMOTE_KEY_LOCATION, key);
	}

	// We have all results, so log and return

	if ((env_rc != pcmk_rc_ok) && (default_rc != pcmk_rc_ok)
	&& (alt_rc != pcmk_rc_ok)) { // Environment set, everything failed

	crm_warn("Could not read Pacemaker Remote key from %s (%s%s%s%s%s): %s",
	env_location,
	env_is_default? "" : "or default location ",
	env_is_default? "" : DEFAULT_REMOTE_KEY_LOCATION,
	!env_is_default && !env_is_fallback? " " : "",
	env_is_fallback? "" : "or fallback location ",
	env_is_fallback? "" : ALT_REMOTE_KEY_LOCATION,
	pcmk_rc_str(env_rc));
	return ENOKEY;
	}

	if (env_rc != pcmk_rc_ok) { // Environment set but failed, using a default
	crm_warn("Could not read Pacemaker Remote key from %s "
	"(using %s location %s instead): %s",
	env_location,
	(default_rc == pcmk_rc_ok)? "default" : "fallback",
	(default_rc == pcmk_rc_ok)? DEFAULT_REMOTE_KEY_LOCATION : ALT_REMOTE_KEY_LOCATION,
	pcmk_rc_str(env_rc));
	return pcmk_rc_ok;
	}

	if ((default_rc != pcmk_rc_ok) && (alt_rc != pcmk_rc_ok)) {
	// Environment unset, defaults failed
	crm_warn("Could not read Pacemaker Remote key from default location %s"
	" (or fallback location %s): %s",
	DEFAULT_REMOTE_KEY_LOCATION, ALT_REMOTE_KEY_LOCATION,
	pcmk_rc_str(default_rc));
	return ENOKEY;
	}

	return pcmk_rc_ok; // Environment variable unset, a default worked
	}

	static void
	lrmd_gnutls_global_init(void)
	{
	static int gnutls_init = 0;

	if (!gnutls_init) {
	crm_gnutls_global_init();
	}
	gnutls_init = 1;
	}
	#endif

	static void
	report_async_connection_result(lrmd_t * lrmd, int rc)
	{
	lrmd_private_t *native = lrmd->lrmd_private;

	if (native->callback) {
	lrmd_event_data_t event = { 0, };
	event.type = lrmd_event_connect;
	event.remote_nodename = native->remote_nodename;
	event.connection_rc = rc;
	native->callback(&event);
	}
	}

	#ifdef HAVE_GNUTLS_GNUTLS_H
	static inline int
	lrmd__tls_client_handshake(pcmk__remote_t *remote)
	{
	return pcmk__tls_client_handshake(remote, LRMD_CLIENT_HANDSHAKE_TIMEOUT);
	}

	/*!
	* \internal
	* \brief Add trigger and file descriptor mainloop sources for TLS
	*
	* \param[in] lrmd API connection with established TLS session
	* \param[in] do_handshake Whether to perform executor handshake
	*
	* \return Standard Pacemaker return code
	*/
	static int
	add_tls_to_mainloop(lrmd_t *lrmd, bool do_handshake)
	{
	lrmd_private_t *native = lrmd->lrmd_private;
	int rc = pcmk_rc_ok;

	char *name = crm_strdup_printf("pacemaker-remote-%s:%d",
	native->server, native->port);

	struct mainloop_fd_callbacks tls_fd_callbacks = {
	.dispatch = lrmd_tls_dispatch,
	.destroy = lrmd_tls_connection_destroy,
	};

	native->process_notify = mainloop_add_trigger(G_PRIORITY_HIGH,
	lrmd_tls_dispatch, lrmd);
	native->source = mainloop_add_fd(name, G_PRIORITY_HIGH, native->sock, lrmd,
	&tls_fd_callbacks);

	/* Async connections lose the client name provided by the API caller, so we
	* have to use our generated name here to perform the executor handshake.
	*
	* @TODO Keep track of the caller-provided name. Perhaps we should be using
	* that name in this function instead of generating one anyway.
	*/
	if (do_handshake) {
	rc = lrmd_handshake(lrmd, name);
	rc = pcmk_legacy2rc(rc);
	}
	free(name);
	return rc;
	}

	static void
	lrmd_tcp_connect_cb(void *userdata, int rc, int sock)
	{
	lrmd_t *lrmd = userdata;
	lrmd_private_t *native = lrmd->lrmd_private;
	gnutls_datum_t psk_key = { NULL, 0 };

	native->async_timer = 0;

	if (rc != pcmk_rc_ok) {
	lrmd_tls_connection_destroy(lrmd);
	crm_info("Could not connect to Pacemaker Remote at %s:%d: %s "
	CRM_XS " rc=%d",
	native->server, native->port, pcmk_rc_str(rc), rc);
	report_async_connection_result(lrmd, pcmk_rc2legacy(rc));
	return;
	}

	/* The TCP connection was successful, so establish the TLS connection.
	* @TODO make this async to avoid blocking code in client
	*/

	native->sock = sock;

	rc = lrmd__init_remote_key(&psk_key);
	if (rc != pcmk_rc_ok) {
	crm_info("Could not connect to Pacemaker Remote at %s:%d: %s "
	CRM_XS " rc=%d",
	native->server, native->port, pcmk_rc_str(rc), rc);
	lrmd_tls_connection_destroy(lrmd);
	report_async_connection_result(lrmd, pcmk_rc2legacy(rc));
	return;
	}

	gnutls_psk_allocate_client_credentials(&native->psk_cred_c);
	gnutls_psk_set_client_credentials(native->psk_cred_c, DEFAULT_REMOTE_USERNAME, &psk_key, GNUTLS_PSK_KEY_RAW);
	gnutls_free(psk_key.data);

	native->remote->tls_session = pcmk__new_tls_session(sock, GNUTLS_CLIENT,
	GNUTLS_CRD_PSK,
	native->psk_cred_c);
	if (native->remote->tls_session == NULL) {
	lrmd_tls_connection_destroy(lrmd);
	report_async_connection_result(lrmd, -EPROTO);
	return;
	}

	if (lrmd__tls_client_handshake(native->remote) != pcmk_rc_ok) {
	crm_warn("Disconnecting after TLS handshake with Pacemaker Remote server %s:%d failed",
	native->server, native->port);
	gnutls_deinit(*native->remote->tls_session);
	gnutls_free(native->remote->tls_session);
	native->remote->tls_session = NULL;
	lrmd_tls_connection_destroy(lrmd);
	report_async_connection_result(lrmd, -EKEYREJECTED);
	return;
	}

	crm_info("TLS connection to Pacemaker Remote server %s:%d succeeded",
	native->server, native->port);
	rc = add_tls_to_mainloop(lrmd, true);
	report_async_connection_result(lrmd, pcmk_rc2legacy(rc));
	}

	static int
	lrmd_tls_connect_async(lrmd_t * lrmd, int timeout /ms / )
	{
	int rc;
	int timer_id = 0;
	lrmd_private_t *native = lrmd->lrmd_private;

	lrmd_gnutls_global_init();
	native->sock = -1;
	rc = pcmk__connect_remote(native->server, native->port, timeout, &timer_id,
	&(native->sock), lrmd, lrmd_tcp_connect_cb);
	if (rc != pcmk_rc_ok) {
	crm_warn("Pacemaker Remote connection to %s:%d failed: %s "
	CRM_XS " rc=%d",
	native->server, native->port, pcmk_rc_str(rc), rc);
	return pcmk_rc2legacy(rc);
	}
	native->async_timer = timer_id;
	return pcmk_ok;
	}

	static int
	lrmd_tls_connect(lrmd_t * lrmd, int *fd)
	{
	int rc;

	lrmd_private_t *native = lrmd->lrmd_private;
	gnutls_datum_t psk_key = { NULL, 0 };

	lrmd_gnutls_global_init();

	native->sock = -1;
	rc = pcmk__connect_remote(native->server, native->port, 0, NULL,
	&(native->sock), NULL, NULL);
	if (rc != pcmk_rc_ok) {
	crm_warn("Pacemaker Remote connection to %s:%d failed: %s "
	CRM_XS " rc=%d",
	native->server, native->port, pcmk_rc_str(rc), rc);
	lrmd_tls_connection_destroy(lrmd);
	return -ENOTCONN;
	}

	rc = lrmd__init_remote_key(&psk_key);
	if (rc != pcmk_rc_ok) {
	lrmd_tls_connection_destroy(lrmd);
	return pcmk_rc2legacy(rc);
	}

	gnutls_psk_allocate_client_credentials(&native->psk_cred_c);
	gnutls_psk_set_client_credentials(native->psk_cred_c, DEFAULT_REMOTE_USERNAME, &psk_key, GNUTLS_PSK_KEY_RAW);
	gnutls_free(psk_key.data);

	native->remote->tls_session = pcmk__new_tls_session(native->sock, GNUTLS_CLIENT,
	GNUTLS_CRD_PSK,
	native->psk_cred_c);
	if (native->remote->tls_session == NULL) {
	lrmd_tls_connection_destroy(lrmd);
	return -EPROTO;
	}

	if (lrmd__tls_client_handshake(native->remote) != pcmk_rc_ok) {
	crm_err("Session creation for %s:%d failed", native->server, native->port);
	gnutls_deinit(*native->remote->tls_session);
	gnutls_free(native->remote->tls_session);
	native->remote->tls_session = NULL;
	lrmd_tls_connection_destroy(lrmd);
	return -EKEYREJECTED;
	}

	crm_info("Client TLS connection established with Pacemaker Remote server %s:%d", native->server,
	native->port);

	if (fd) {
	*fd = native->sock;
	} else {
	add_tls_to_mainloop(lrmd, false);
	}
	return pcmk_ok;
	}
	#endif

	static int
	lrmd_api_connect(lrmd_t * lrmd, const char name, int fd)
	{
	int rc = -ENOTCONN;
	lrmd_private_t *native = lrmd->lrmd_private;

	switch (native->type) {
	case pcmk__client_ipc:
	rc = lrmd_ipc_connect(lrmd, fd);
	break;
	#ifdef HAVE_GNUTLS_GNUTLS_H
	case pcmk__client_tls:
	rc = lrmd_tls_connect(lrmd, fd);
	break;
	#endif
	default:
	crm_err("Unsupported executor connection type (bug?): %d",
	native->type);
	rc = -EPROTONOSUPPORT;
	}

	if (rc == pcmk_ok) {
	rc = lrmd_handshake(lrmd, name);
	}

	return rc;
	}

	static int
	lrmd_api_connect_async(lrmd_t * lrmd, const char *name, int timeout)
	{
	int rc = pcmk_ok;
	lrmd_private_t *native = lrmd->lrmd_private;

	CRM_CHECK(native && native->callback, return -EINVAL);

	switch (native->type) {
	case pcmk__client_ipc:
	/* fake async connection with ipc. it should be fast
	* enough that we gain very little from async */
	rc = lrmd_api_connect(lrmd, name, NULL);
	if (!rc) {
	report_async_connection_result(lrmd, rc);
	}
	break;
	#ifdef HAVE_GNUTLS_GNUTLS_H
	case pcmk__client_tls:
	rc = lrmd_tls_connect_async(lrmd, timeout);
	if (rc) {
	/* connection failed, report rc now */
	report_async_connection_result(lrmd, rc);
	}
	break;
	#endif
	default:
	crm_err("Unsupported executor connection type (bug?): %d",
	native->type);
	rc = -EPROTONOSUPPORT;
	}

	return rc;
	}

	static void
	lrmd_ipc_disconnect(lrmd_t * lrmd)
	{
	lrmd_private_t *native = lrmd->lrmd_private;

	if (native->source != NULL) {
	/* Attached to mainloop */
	mainloop_del_ipc_client(native->source);
	native->source = NULL;
	native->ipc = NULL;

	} else if (native->ipc) {
	/* Not attached to mainloop */
	crm_ipc_t *ipc = native->ipc;

	native->ipc = NULL;
	crm_ipc_close(ipc);
	crm_ipc_destroy(ipc);
	}
	}

	#ifdef HAVE_GNUTLS_GNUTLS_H
	static void
	lrmd_tls_disconnect(lrmd_t * lrmd)
	{
	lrmd_private_t *native = lrmd->lrmd_private;

	if (native->remote->tls_session) {
	gnutls_bye(*native->remote->tls_session, GNUTLS_SHUT_RDWR);
	gnutls_deinit(*native->remote->tls_session);
	gnutls_free(native->remote->tls_session);
	native->remote->tls_session = 0;
	}

	if (native->async_timer) {
	g_source_remove(native->async_timer);
	native->async_timer = 0;
	}

	if (native->source != NULL) {
	/* Attached to mainloop */
	mainloop_del_ipc_client(native->source);
	native->source = NULL;

	} else if (native->sock) {
	close(native->sock);
	native->sock = 0;
	}

	if (native->pending_notify) {
	g_list_free_full(native->pending_notify, lrmd_free_xml);
	native->pending_notify = NULL;
	}
	}
	#endif

	static int
	lrmd_api_disconnect(lrmd_t * lrmd)
	{
	lrmd_private_t *native = lrmd->lrmd_private;
	int rc = pcmk_ok;

	crm_info("Disconnecting %s %s executor connection",
	pcmk__client_type_str(native->type),
	(native->remote_nodename? native->remote_nodename : "local"));
	switch (native->type) {
	case pcmk__client_ipc:
	lrmd_ipc_disconnect(lrmd);
	break;
	#ifdef HAVE_GNUTLS_GNUTLS_H
	case pcmk__client_tls:
	lrmd_tls_disconnect(lrmd);
	break;
	#endif
	default:
	crm_err("Unsupported executor connection type (bug?): %d",
	native->type);
	rc = -EPROTONOSUPPORT;
	}

	free(native->token);
	native->token = NULL;

	free(native->peer_version);
	native->peer_version = NULL;
	return rc;
	}

	static int
	lrmd_api_register_rsc(lrmd_t * lrmd,
	const char *rsc_id,
	const char *class,
	const char provider, const char type, enum lrmd_call_options options)
	{
	int rc = pcmk_ok;
	xmlNode *data = NULL;

	if (!class \|\| !type \|\| !rsc_id) {
	return -EINVAL;
	}
	if (pcmk_is_set(pcmk_get_ra_caps(class), pcmk_ra_cap_provider)
	&& (provider == NULL)) {
	return -EINVAL;
	}

	data = create_xml_node(NULL, F_LRMD_RSC);

	crm_xml_add(data, F_LRMD_ORIGIN, __func__);
	crm_xml_add(data, F_LRMD_RSC_ID, rsc_id);
	crm_xml_add(data, F_LRMD_CLASS, class);
	crm_xml_add(data, F_LRMD_PROVIDER, provider);
	crm_xml_add(data, F_LRMD_TYPE, type);
	rc = lrmd_send_command(lrmd, LRMD_OP_RSC_REG, data, NULL, 0, options, TRUE);
	free_xml(data);

	return rc;
	}

	static int
	lrmd_api_unregister_rsc(lrmd_t * lrmd, const char *rsc_id, enum lrmd_call_options options)
	{
	int rc = pcmk_ok;
	xmlNode *data = create_xml_node(NULL, F_LRMD_RSC);

	crm_xml_add(data, F_LRMD_ORIGIN, __func__);
	crm_xml_add(data, F_LRMD_RSC_ID, rsc_id);
	rc = lrmd_send_command(lrmd, LRMD_OP_RSC_UNREG, data, NULL, 0, options, TRUE);
	free_xml(data);

	return rc;
	}

	lrmd_rsc_info_t *
	lrmd_new_rsc_info(const char rsc_id, const char standard,
	const char provider, const char type)
	{
	lrmd_rsc_info_t *rsc_info = calloc(1, sizeof(lrmd_rsc_info_t));

	CRM_ASSERT(rsc_info);
	if (rsc_id) {
	rsc_info->id = strdup(rsc_id);
	CRM_ASSERT(rsc_info->id);
	}
	if (standard) {
	rsc_info->standard = strdup(standard);
	CRM_ASSERT(rsc_info->standard);
	}
	if (provider) {
	rsc_info->provider = strdup(provider);
	CRM_ASSERT(rsc_info->provider);
	}
	if (type) {
	rsc_info->type = strdup(type);
	CRM_ASSERT(rsc_info->type);
	}
	return rsc_info;
	}

	lrmd_rsc_info_t *
	lrmd_copy_rsc_info(lrmd_rsc_info_t * rsc_info)
	{
	return lrmd_new_rsc_info(rsc_info->id, rsc_info->standard,
	rsc_info->provider, rsc_info->type);
	}

	void
	lrmd_free_rsc_info(lrmd_rsc_info_t * rsc_info)
	{
	if (!rsc_info) {
	return;
	}
	free(rsc_info->id);
	free(rsc_info->type);
	free(rsc_info->standard);
	free(rsc_info->provider);
	free(rsc_info);
	}

	static lrmd_rsc_info_t *
	lrmd_api_get_rsc_info(lrmd_t * lrmd, const char *rsc_id, enum lrmd_call_options options)
	{
	lrmd_rsc_info_t *rsc_info = NULL;
	xmlNode *data = create_xml_node(NULL, F_LRMD_RSC);
	xmlNode *output = NULL;
	const char *class = NULL;
	const char *provider = NULL;
	const char *type = NULL;

	crm_xml_add(data, F_LRMD_ORIGIN, __func__);
	crm_xml_add(data, F_LRMD_RSC_ID, rsc_id);
	lrmd_send_command(lrmd, LRMD_OP_RSC_INFO, data, &output, 0, options, TRUE);
	free_xml(data);

	if (!output) {
	return NULL;
	}

	class = crm_element_value(output, F_LRMD_CLASS);
	provider = crm_element_value(output, F_LRMD_PROVIDER);
	type = crm_element_value(output, F_LRMD_TYPE);

	if (!class \|\| !type) {
	free_xml(output);
	return NULL;
	} else if (pcmk_is_set(pcmk_get_ra_caps(class), pcmk_ra_cap_provider)
	&& !provider) {
	free_xml(output);
	return NULL;
	}

	rsc_info = lrmd_new_rsc_info(rsc_id, class, provider, type);
	free_xml(output);
	return rsc_info;
	}

	void
	lrmd_free_op_info(lrmd_op_info_t *op_info)
	{
	if (op_info) {
	free(op_info->rsc_id);
	free(op_info->action);
	free(op_info->interval_ms_s);
	free(op_info->timeout_ms_s);
	free(op_info);
	}
	}

	static int
	lrmd_api_get_recurring_ops(lrmd_t lrmd, const char rsc_id, int timeout_ms,
	enum lrmd_call_options options, GList **output)
	{
	xmlNode *data = NULL;
	xmlNode *output_xml = NULL;
	int rc = pcmk_ok;

	if (output == NULL) {
	return -EINVAL;
	}
	*output = NULL;

	// Send request
	if (rsc_id) {
	data = create_xml_node(NULL, F_LRMD_RSC);
	crm_xml_add(data, F_LRMD_ORIGIN, __func__);
	crm_xml_add(data, F_LRMD_RSC_ID, rsc_id);
	}
	rc = lrmd_send_command(lrmd, LRMD_OP_GET_RECURRING, data, &output_xml,
	timeout_ms, options, TRUE);
	if (data) {
	free_xml(data);
	}

	// Process reply
	if ((rc != pcmk_ok) \|\| (output_xml == NULL)) {
	return rc;
	}
	for (xmlNode *rsc_xml = first_named_child(output_xml, F_LRMD_RSC);
	(rsc_xml != NULL) && (rc == pcmk_ok);
	rsc_xml = crm_next_same_xml(rsc_xml)) {

	rsc_id = crm_element_value(rsc_xml, F_LRMD_RSC_ID);
	if (rsc_id == NULL) {
	crm_err("Could not parse recurring operation information from executor");
	continue;
	}
	for (xmlNode *op_xml = first_named_child(rsc_xml, T_LRMD_RSC_OP);
	op_xml != NULL; op_xml = crm_next_same_xml(op_xml)) {

	lrmd_op_info_t *op_info = calloc(1, sizeof(lrmd_op_info_t));

	if (op_info == NULL) {
	rc = -ENOMEM;
	break;
	}
	op_info->rsc_id = strdup(rsc_id);
	op_info->action = crm_element_value_copy(op_xml, F_LRMD_RSC_ACTION);
	op_info->interval_ms_s = crm_element_value_copy(op_xml,
	F_LRMD_RSC_INTERVAL);
	op_info->timeout_ms_s = crm_element_value_copy(op_xml,
	F_LRMD_TIMEOUT);
	output = g_list_prepend(output, op_info);
	}
	}
	free_xml(output_xml);
	return rc;
	}


	static void
	lrmd_api_set_callback(lrmd_t * lrmd, lrmd_event_callback callback)
	{
	lrmd_private_t *native = lrmd->lrmd_private;

	native->callback = callback;
	}

	void
	lrmd_internal_set_proxy_callback(lrmd_t * lrmd, void userdata, void (callback)(lrmd_t lrmd, void userdata, xmlNode *msg))
	{
	lrmd_private_t *native = lrmd->lrmd_private;

	native->proxy_callback = callback;
	native->proxy_callback_userdata = userdata;
	}

	void
	lrmd_internal_proxy_dispatch(lrmd_t lrmd, xmlNode msg)
	{
	lrmd_private_t *native = lrmd->lrmd_private;

	if (native->proxy_callback) {
	crm_log_xml_trace(msg, "PROXY_INBOUND");
	native->proxy_callback(lrmd, native->proxy_callback_userdata, msg);
	}
	}

	int
	lrmd_internal_proxy_send(lrmd_t * lrmd, xmlNode *msg)
	{
	if (lrmd == NULL) {
	return -ENOTCONN;
	}
	crm_xml_add(msg, F_LRMD_OPERATION, CRM_OP_IPC_FWD);

	crm_log_xml_trace(msg, "PROXY_OUTBOUND");
	return lrmd_send_xml_no_reply(lrmd, msg);
	}

	static int
	stonith_get_metadata(const char provider, const char type, char **output)
	{
	int rc = pcmk_ok;
	stonith_t *stonith_api = stonith_api_new();

	if (stonith_api == NULL) {
	crm_err("Could not get fence agent meta-data: API memory allocation failed");
	return -ENOMEM;
	}

	rc = stonith_api->cmds->metadata(stonith_api, st_opt_sync_call, type,
	provider, output, 0);
	if ((rc == pcmk_ok) && (*output == NULL)) {
	rc = -EIO;
	}
	stonith_api->cmds->free(stonith_api);
	return rc;
	}

	static int
	lrmd_api_get_metadata(lrmd_t lrmd, const char standard, const char *provider,
	const char type, char *output,
	enum lrmd_call_options options)
	{
	return lrmd->cmds->get_metadata_params(lrmd, standard, provider, type,
	output, options, NULL);
	}

	static int
	lrmd_api_get_metadata_params(lrmd_t lrmd, const char standard,
	const char provider, const char type,
	char **output, enum lrmd_call_options options,
	lrmd_key_value_t *params)
	{
	svc_action_t *action = NULL;
	GHashTable *params_table = NULL;

	if (!standard \|\| !type) {
	lrmd_key_value_freeall(params);
	return -EINVAL;
	}

	if (pcmk__str_eq(standard, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_casei)) {
	lrmd_key_value_freeall(params);
	return stonith_get_metadata(provider, type, output);
	}

	params_table = pcmk__strkey_table(free, free);
	for (const lrmd_key_value_t *param = params; param; param = param->next) {
	g_hash_table_insert(params_table, strdup(param->key), strdup(param->value));
	}
	action = services__create_resource_action(type, standard, provider, type,
	CRMD_ACTION_METADATA, 0,
	CRMD_METADATA_CALL_TIMEOUT,
	params_table, 0);
	lrmd_key_value_freeall(params);

	if (action == NULL) {
	return -ENOMEM;
	}
	if (action->rc != PCMK_OCF_UNKNOWN) {
	services_action_free(action);
	return -EINVAL;
	}

	if (!services_action_sync(action)) {
	crm_err("Failed to retrieve meta-data for %s:%s:%s",
	standard, provider, type);
	services_action_free(action);
	return -EIO;
	}

	if (!action->stdout_data) {
	crm_err("Failed to receive meta-data for %s:%s:%s",
	standard, provider, type);
	services_action_free(action);
	return -EIO;
	}

	*output = strdup(action->stdout_data);
	services_action_free(action);

	return pcmk_ok;
	}

	static int
	lrmd_api_exec(lrmd_t lrmd, const char rsc_id, const char *action,
	const char *userdata, guint interval_ms,
	int timeout, /* ms */
	int start_delay, /* ms */
	enum lrmd_call_options options, lrmd_key_value_t * params)
	{
	int rc = pcmk_ok;
	xmlNode *data = create_xml_node(NULL, F_LRMD_RSC);
	xmlNode *args = create_xml_node(data, XML_TAG_ATTRS);
	lrmd_key_value_t *tmp = NULL;

	crm_xml_add(data, F_LRMD_ORIGIN, __func__);
	crm_xml_add(data, F_LRMD_RSC_ID, rsc_id);
	crm_xml_add(data, F_LRMD_RSC_ACTION, action);
	crm_xml_add(data, F_LRMD_RSC_USERDATA_STR, userdata);
	crm_xml_add_ms(data, F_LRMD_RSC_INTERVAL, interval_ms);
	crm_xml_add_int(data, F_LRMD_TIMEOUT, timeout);
	crm_xml_add_int(data, F_LRMD_RSC_START_DELAY, start_delay);

	for (tmp = params; tmp; tmp = tmp->next) {
	hash2smartfield((gpointer) tmp->key, (gpointer) tmp->value, args);
	}

	rc = lrmd_send_command(lrmd, LRMD_OP_RSC_EXEC, data, NULL, timeout, options, TRUE);
	free_xml(data);

	lrmd_key_value_freeall(params);
	return rc;
	}

	/* timeout is in ms */
	static int
	lrmd_api_exec_alert(lrmd_t lrmd, const char alert_id, const char *alert_path,
	int timeout, lrmd_key_value_t *params)
	{
	int rc = pcmk_ok;
	xmlNode *data = create_xml_node(NULL, F_LRMD_ALERT);
	xmlNode *args = create_xml_node(data, XML_TAG_ATTRS);
	lrmd_key_value_t *tmp = NULL;

	crm_xml_add(data, F_LRMD_ORIGIN, __func__);
	crm_xml_add(data, F_LRMD_ALERT_ID, alert_id);
	crm_xml_add(data, F_LRMD_ALERT_PATH, alert_path);
	crm_xml_add_int(data, F_LRMD_TIMEOUT, timeout);

	for (tmp = params; tmp; tmp = tmp->next) {
	hash2smartfield((gpointer) tmp->key, (gpointer) tmp->value, args);
	}

	rc = lrmd_send_command(lrmd, LRMD_OP_ALERT_EXEC, data, NULL, timeout,
	lrmd_opt_notify_orig_only, TRUE);
	free_xml(data);

	lrmd_key_value_freeall(params);
	return rc;
	}

	static int
	lrmd_api_cancel(lrmd_t lrmd, const char rsc_id, const char *action,
	guint interval_ms)
	{
	int rc = pcmk_ok;
	xmlNode *data = create_xml_node(NULL, F_LRMD_RSC);

	crm_xml_add(data, F_LRMD_ORIGIN, __func__);
	crm_xml_add(data, F_LRMD_RSC_ACTION, action);
	crm_xml_add(data, F_LRMD_RSC_ID, rsc_id);
	crm_xml_add_ms(data, F_LRMD_RSC_INTERVAL, interval_ms);
	rc = lrmd_send_command(lrmd, LRMD_OP_RSC_CANCEL, data, NULL, 0, 0, TRUE);
	free_xml(data);
	return rc;
	}

	static int
	list_stonith_agents(lrmd_list_t ** resources)
	{
	int rc = 0;
	stonith_t *stonith_api = stonith_api_new();
	stonith_key_value_t *stonith_resources = NULL;
	stonith_key_value_t *dIter = NULL;

	if (stonith_api == NULL) {
	crm_err("Could not list fence agents: API memory allocation failed");
	return -ENOMEM;
	}
	stonith_api->cmds->list_agents(stonith_api, st_opt_sync_call, NULL,
	&stonith_resources, 0);
	stonith_api->cmds->free(stonith_api);

	for (dIter = stonith_resources; dIter; dIter = dIter->next) {
	rc++;
	if (resources) {
	resources = lrmd_list_add(resources, dIter->value);
	}
	}

	stonith_key_value_freeall(stonith_resources, 1, 0);
	return rc;
	}

	static int
	lrmd_api_list_agents(lrmd_t * lrmd, lrmd_list_t ** resources, const char *class,
	const char *provider)
	{
	int rc = 0;
	int stonith_count = 0; // Initially, whether to include stonith devices

	if (pcmk__str_eq(class, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_casei)) {
	stonith_count = 1;

	} else {
	GList *gIter = NULL;
	GList *agents = resources_list_agents(class, provider);

	for (gIter = agents; gIter != NULL; gIter = gIter->next) {
	resources = lrmd_list_add(resources, (const char *)gIter->data);
	rc++;
	}
	g_list_free_full(agents, free);

	if (!class) {
	stonith_count = 1;
	}
	}

	if (stonith_count) {
	// Now, if stonith devices are included, how many there are
	stonith_count = list_stonith_agents(resources);
	if (stonith_count > 0) {
	rc += stonith_count;
	}
	}
	if (rc == 0) {
	crm_notice("No agents found for class %s", class);
	rc = -EPROTONOSUPPORT;
	}
	return rc;
	}

	static bool
	does_provider_have_agent(const char agent, const char provider, const char *class)
	{
	bool found = false;
	GList *agents = NULL;
	GList *gIter2 = NULL;

	agents = resources_list_agents(class, provider);
	for (gIter2 = agents; gIter2 != NULL; gIter2 = gIter2->next) {
	if (pcmk__str_eq(agent, gIter2->data, pcmk__str_casei)) {
	found = true;
	}
	}
	g_list_free_full(agents, free);
	return found;
	}

	static int
	lrmd_api_list_ocf_providers(lrmd_t * lrmd, const char agent, lrmd_list_t * providers)
	{
	int rc = pcmk_ok;
	char *provider = NULL;
	GList *ocf_providers = NULL;
	GList *gIter = NULL;

	ocf_providers = resources_list_providers(PCMK_RESOURCE_CLASS_OCF);

	for (gIter = ocf_providers; gIter != NULL; gIter = gIter->next) {
	provider = gIter->data;
	if (!agent \|\| does_provider_have_agent(agent, provider,
	PCMK_RESOURCE_CLASS_OCF)) {
	providers = lrmd_list_add(providers, (const char *)gIter->data);
	rc++;
	}
	}

	g_list_free_full(ocf_providers, free);
	return rc;
	}

	static int
	lrmd_api_list_standards(lrmd_t * lrmd, lrmd_list_t ** supported)
	{
	int rc = 0;
	GList *standards = NULL;
	GList *gIter = NULL;

	standards = resources_list_standards();

	for (gIter = standards; gIter != NULL; gIter = gIter->next) {
	supported = lrmd_list_add(supported, (const char *)gIter->data);
	rc++;
	}

	if (list_stonith_agents(NULL) > 0) {
	supported = lrmd_list_add(supported, PCMK_RESOURCE_CLASS_STONITH);
	rc++;
	}

	g_list_free_full(standards, free);
	return rc;
	}

	/*!
	* \internal
	* \brief Create an executor API object
	*
	* \param[out] api Will be set to newly created API object (it is the
	* caller's responsibility to free this value with
	* lrmd_api_delete() if this function succeeds)
	* \param[in] nodename If the object will be used for a remote connection,
	* the node name to use in cluster for remote executor
	* \param[in] server If the object will be used for a remote connection,
	* the resolvable host name to connect to
	* \param[in] port If the object will be used for a remote connection,
	* port number on \p server to connect to
	*
	* \return Standard Pacemaker return code
	* \note If the caller leaves one of \p nodename or \p server NULL, the other's
	* value will be used for both. If the caller leaves both NULL, an API
	* object will be created for a local executor connection.
	*/
	int
	lrmd__new(lrmd_t *api, const char nodename, const char *server, int port)
	{
	lrmd_private_t *pvt = NULL;

	if (api == NULL) {
	return EINVAL;
	}
	*api = NULL;

	// Allocate all memory needed

	*api = calloc(1, sizeof(lrmd_t));
	if (*api == NULL) {
	return ENOMEM;
	}

	pvt = calloc(1, sizeof(lrmd_private_t));
	if (pvt == NULL) {
	lrmd_api_delete(*api);
	*api = NULL;
	return ENOMEM;
	}
	(*api)->lrmd_private = pvt;

	// @TODO Do we need to do this for local connections?
	pvt->remote = calloc(1, sizeof(pcmk__remote_t));

	(*api)->cmds = calloc(1, sizeof(lrmd_api_operations_t));

	if ((pvt->remote == NULL) \|\| ((*api)->cmds == NULL)) {
	lrmd_api_delete(*api);
	*api = NULL;
	return ENOMEM;
	}

	// Set methods
	(*api)->cmds->connect = lrmd_api_connect;
	(*api)->cmds->connect_async = lrmd_api_connect_async;
	(*api)->cmds->is_connected = lrmd_api_is_connected;
	(*api)->cmds->poke_connection = lrmd_api_poke_connection;
	(*api)->cmds->disconnect = lrmd_api_disconnect;
	(*api)->cmds->register_rsc = lrmd_api_register_rsc;
	(*api)->cmds->unregister_rsc = lrmd_api_unregister_rsc;
	(*api)->cmds->get_rsc_info = lrmd_api_get_rsc_info;
	(*api)->cmds->get_recurring_ops = lrmd_api_get_recurring_ops;
	(*api)->cmds->set_callback = lrmd_api_set_callback;
	(*api)->cmds->get_metadata = lrmd_api_get_metadata;
	(*api)->cmds->exec = lrmd_api_exec;
	(*api)->cmds->cancel = lrmd_api_cancel;
	(*api)->cmds->list_agents = lrmd_api_list_agents;
	(*api)->cmds->list_ocf_providers = lrmd_api_list_ocf_providers;
	(*api)->cmds->list_standards = lrmd_api_list_standards;
	(*api)->cmds->exec_alert = lrmd_api_exec_alert;
	(*api)->cmds->get_metadata_params = lrmd_api_get_metadata_params;

	if ((nodename == NULL) && (server == NULL)) {
	pvt->type = pcmk__client_ipc;
	} else {
	#ifdef HAVE_GNUTLS_GNUTLS_H
	if (nodename == NULL) {
	nodename = server;
	} else if (server == NULL) {
	server = nodename;
	}
	pvt->type = pcmk__client_tls;
	pvt->remote_nodename = strdup(nodename);
	pvt->server = strdup(server);
	if ((pvt->remote_nodename == NULL) \|\| (pvt->server == NULL)) {
	lrmd_api_delete(*api);
	*api = NULL;
	return ENOMEM;
	}
	pvt->port = port;
	if (pvt->port == 0) {
	pvt->port = crm_default_remote_port();
	}
	#else
	crm_err("Cannot communicate with Pacemaker Remote "
	"because GnuTLS is not enabled for this build");
	lrmd_api_delete(*api);
	*api = NULL;
	return EOPNOTSUPP;
	#endif
	}
	return pcmk_rc_ok;
	}

	lrmd_t *
	lrmd_api_new(void)
	{
	lrmd_t *api = NULL;

	CRM_ASSERT(lrmd__new(&api, NULL, NULL, 0) == pcmk_rc_ok);
	return api;
	}

	lrmd_t *
	lrmd_remote_api_new(const char nodename, const char server, int port)
	{
	lrmd_t *api = NULL;

	CRM_ASSERT(lrmd__new(&api, nodename, server, port) == pcmk_rc_ok);
	return api;
	}

	void
	lrmd_api_delete(lrmd_t * lrmd)
	{
	if (lrmd == NULL) {
	return;
	}
	if (lrmd->cmds != NULL) { // Never NULL, but make static analysis happy
	if (lrmd->cmds->disconnect != NULL) { // Also never really NULL
	lrmd->cmds->disconnect(lrmd); // No-op if already disconnected
	}
	free(lrmd->cmds);
	}
	if (lrmd->lrmd_private != NULL) {
	lrmd_private_t *native = lrmd->lrmd_private;

	#ifdef HAVE_GNUTLS_GNUTLS_H
	free(native->server);
	#endif
	free(native->remote_nodename);
	free(native->remote);
	free(native->token);
	free(native->peer_version);
	free(lrmd->lrmd_private);
	}
	free(lrmd);
	}

	/*!
	* \internal
	* \brief Set the result of an executor event
	*
	* \param[in,out] event Executor event to set
	* \param[in] rc OCF exit status of event
	* \param[in] op_status Executor status of event
	* \param[in] exit_reason Human-friendly description of event
	*/
	void
	lrmd__set_result(lrmd_event_data_t *event, enum ocf_exitcode rc, int op_status,
	const char *exit_reason)
	{
	if (event == NULL) {
	return;
	}

	event->rc = rc;
	event->op_status = op_status;

	if (!pcmk__str_eq(event->exit_reason, exit_reason, pcmk__str_none)) {
	free((void *) event->exit_reason);
	event->exit_reason = (exit_reason == NULL)? NULL : strdup(exit_reason);
	}
	}

	/*!
	* \internal
	* \brief Clear an executor event's exit reason, output, and error output
	*
	* \param[in] event Executor event to reset
	*/
	void
	lrmd__reset_result(lrmd_event_data_t *event)
	{
	if (event == NULL) {
	return;
	}

	free((void *) event->exit_reason);
	event->exit_reason = NULL;

	free((void *) event->output);
	event->output = NULL;
	}
	diff --git a/lib/pengine/native.c b/lib/pengine/native.c
	index 1198fd6a75..36121c527f 100644
	--- a/lib/pengine/native.c
	+++ b/lib/pengine/native.c
	@@ -1,1369 +1,1366 @@
	/*
	* Copyright 2004-2021 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <crm/common/output.h>
	#include <crm/pengine/rules.h>
	#include <crm/pengine/status.h>
	#include <crm/pengine/complex.h>
	#include <crm/pengine/internal.h>
	#include <crm/msg_xml.h>
	#include <pe_status_private.h>

	#define VARIANT_NATIVE 1
	#include "./variant.h"

	#ifdef PCMK__COMPAT_2_0
	#define PROVIDER_SEP "::"
	#else
	#define PROVIDER_SEP ":"
	#endif

	/*!
	* \internal
	* \brief Check whether a resource is active on multiple nodes
	*/
	static bool
	is_multiply_active(pe_resource_t *rsc)
	{
	unsigned int count = 0;

	if (rsc->variant == pe_native) {
	pe__find_active_requires(rsc, &count);
	}
	return count > 1;
	}

	static void
	native_priority_to_node(pe_resource_t * rsc, pe_node_t * node, gboolean failed)
	{
	int priority = 0;

	if ((rsc->priority == 0) \|\| (failed == TRUE)) {
	return;
	}

	if (rsc->role == RSC_ROLE_PROMOTED) {
	// Promoted instance takes base priority + 1
	priority = rsc->priority + 1;

	} else {
	priority = rsc->priority;
	}

	node->details->priority += priority;
	pe_rsc_trace(rsc, "Node '%s' now has priority %d with %s'%s' (priority: %d%s)",
	node->details->uname, node->details->priority,
	(rsc->role == RSC_ROLE_PROMOTED)? "promoted " : "",
	rsc->id, rsc->priority,
	(rsc->role == RSC_ROLE_PROMOTED)? " + 1" : "");

	/* Priority of a resource running on a guest node is added to the cluster
	* node as well. */
	if (node->details->remote_rsc
	&& node->details->remote_rsc->container) {
	GList *gIter = node->details->remote_rsc->container->running_on;

	for (; gIter != NULL; gIter = gIter->next) {
	pe_node_t *a_node = gIter->data;

	a_node->details->priority += priority;
	pe_rsc_trace(rsc, "Node '%s' now has priority %d with %s'%s' (priority: %d%s) "
	"from guest node '%s'",
	a_node->details->uname, a_node->details->priority,
	(rsc->role == RSC_ROLE_PROMOTED)? "promoted " : "",
	rsc->id, rsc->priority,
	(rsc->role == RSC_ROLE_PROMOTED)? " + 1" : "",
	node->details->uname);
	}
	}
	}

	void
	native_add_running(pe_resource_t * rsc, pe_node_t * node, pe_working_set_t * data_set, gboolean failed)
	{
	GList *gIter = rsc->running_on;

	CRM_CHECK(node != NULL, return);
	for (; gIter != NULL; gIter = gIter->next) {
	pe_node_t a_node = (pe_node_t ) gIter->data;

	CRM_CHECK(a_node != NULL, return);
	if (pcmk__str_eq(a_node->details->id, node->details->id, pcmk__str_casei)) {
	return;
	}
	}

	pe_rsc_trace(rsc, "Adding %s to %s %s", rsc->id, node->details->uname,
	pcmk_is_set(rsc->flags, pe_rsc_managed)? "" : "(unmanaged)");

	rsc->running_on = g_list_append(rsc->running_on, node);
	if (rsc->variant == pe_native) {
	node->details->running_rsc = g_list_append(node->details->running_rsc, rsc);

	native_priority_to_node(rsc, node, failed);
	}

	if (rsc->variant == pe_native && node->details->maintenance) {
	pe__clear_resource_flags(rsc, pe_rsc_managed);
	}

	if (!pcmk_is_set(rsc->flags, pe_rsc_managed)) {
	pe_resource_t *p = rsc->parent;

	pe_rsc_info(rsc, "resource %s isn't managed", rsc->id);
	resource_location(rsc, node, INFINITY, "not_managed_default", data_set);

	while(p && node->details->online) {
	/* add without the additional location constraint */
	p->running_on = g_list_append(p->running_on, node);
	p = p->parent;
	}
	return;
	}

	if (is_multiply_active(rsc)) {
	switch (rsc->recovery_type) {
	case recovery_stop_only:
	{
	GHashTableIter gIter;
	pe_node_t *local_node = NULL;

	/* make sure it doesn't come up again */
	if (rsc->allowed_nodes != NULL) {
	g_hash_table_destroy(rsc->allowed_nodes);
	}
	rsc->allowed_nodes = pe__node_list2table(data_set->nodes);
	g_hash_table_iter_init(&gIter, rsc->allowed_nodes);
	while (g_hash_table_iter_next(&gIter, NULL, (void **)&local_node)) {
	local_node->weight = -INFINITY;
	}
	}
	break;
	case recovery_stop_start:
	break;
	case recovery_block:
	pe__clear_resource_flags(rsc, pe_rsc_managed);
	pe__set_resource_flags(rsc, pe_rsc_block);

	/* If the resource belongs to a group or bundle configured with
	* multiple-active=block, block the entire entity.
	*/
	if (rsc->parent
	&& (rsc->parent->variant == pe_group \|\| rsc->parent->variant == pe_container)
	&& rsc->parent->recovery_type == recovery_block) {
	GList *gIter = rsc->parent->children;

	for (; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child = (pe_resource_t ) gIter->data;

	pe__clear_resource_flags(child, pe_rsc_managed);
	pe__set_resource_flags(child, pe_rsc_block);
	}
	}
	break;
	}
	crm_debug("%s is active on multiple nodes including %s: %s",
	rsc->id, node->details->uname,
	recovery2text(rsc->recovery_type));

	} else {
	pe_rsc_trace(rsc, "Resource %s is active on: %s", rsc->id, node->details->uname);
	}

	if (rsc->parent != NULL) {
	native_add_running(rsc->parent, node, data_set, FALSE);
	}
	}

	static void
	recursive_clear_unique(pe_resource_t *rsc, gpointer user_data)
	{
	pe__clear_resource_flags(rsc, pe_rsc_unique);
	add_hash_param(rsc->meta, XML_RSC_ATTR_UNIQUE, XML_BOOLEAN_FALSE);
	g_list_foreach(rsc->children, (GFunc) recursive_clear_unique, NULL);
	}

	gboolean
	native_unpack(pe_resource_t * rsc, pe_working_set_t * data_set)
	{
	pe_resource_t *parent = uber_parent(rsc);
	native_variant_data_t *native_data = NULL;
	const char *standard = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
	uint32_t ra_caps = pcmk_get_ra_caps(standard);

	pe_rsc_trace(rsc, "Processing resource %s...", rsc->id);

	native_data = calloc(1, sizeof(native_variant_data_t));
	rsc->variant_opaque = native_data;

	// Only some agent standards support unique and promotable clones
	if (!pcmk_is_set(ra_caps, pcmk_ra_cap_unique)
	&& pcmk_is_set(rsc->flags, pe_rsc_unique) && pe_rsc_is_clone(parent)) {

	/* @COMPAT We should probably reject this situation as an error (as we
	* do for promotable below) rather than warn and convert, but that would
	* be a backward-incompatible change that we should probably do with a
	* transform at a schema major version bump.
	*/
	pe__force_anon(standard, parent, rsc->id, data_set);

	/* Clear globally-unique on the parent and all its descendents unpacked
	* so far (clearing the parent should make any future children unpacking
	* correct). We have to clear this resource explicitly because it isn't
	* hooked into the parent's children yet.
	*/
	recursive_clear_unique(parent, NULL);
	recursive_clear_unique(rsc, NULL);
	}
	if (!pcmk_is_set(ra_caps, pcmk_ra_cap_promotable)
	&& pcmk_is_set(parent->flags, pe_rsc_promotable)) {

	pe_err("Resource %s is of type %s and therefore "
	"cannot be used as a promotable clone resource",
	rsc->id, standard);
	return FALSE;
	}
	return TRUE;
	}

	static bool
	rsc_is_on_node(pe_resource_t rsc, const pe_node_t node, int flags)
	{
	pe_rsc_trace(rsc, "Checking whether %s is on %s",
	rsc->id, node->details->uname);

	if (pcmk_is_set(flags, pe_find_current) && rsc->running_on) {

	for (GList *iter = rsc->running_on; iter; iter = iter->next) {
	pe_node_t loc = (pe_node_t ) iter->data;

	if (loc->details == node->details) {
	return TRUE;
	}
	}

	} else if (pcmk_is_set(flags, pe_find_inactive)
	&& (rsc->running_on == NULL)) {
	return TRUE;

	} else if (!pcmk_is_set(flags, pe_find_current) && rsc->allocated_to
	&& (rsc->allocated_to->details == node->details)) {
	return TRUE;
	}
	return FALSE;
	}

	pe_resource_t *
	native_find_rsc(pe_resource_t * rsc, const char id, const pe_node_t on_node,
	int flags)
	{
	bool match = FALSE;
	pe_resource_t *result = NULL;

	CRM_CHECK(id && rsc && rsc->id, return NULL);

	if (flags & pe_find_clone) {
	const char *rid = ID(rsc->xml);

	if (!pe_rsc_is_clone(uber_parent(rsc))) {
	match = FALSE;

	} else if (!strcmp(id, rsc->id) \|\| pcmk__str_eq(id, rid, pcmk__str_casei)) {
	match = TRUE;
	}

	} else if (!strcmp(id, rsc->id)) {
	match = TRUE;

	} else if (pcmk_is_set(flags, pe_find_renamed)
	&& rsc->clone_name && strcmp(rsc->clone_name, id) == 0) {
	match = TRUE;

	} else if (pcmk_is_set(flags, pe_find_any)
	\|\| (pcmk_is_set(flags, pe_find_anon)
	&& !pcmk_is_set(rsc->flags, pe_rsc_unique))) {
	match = pe_base_name_eq(rsc, id);
	}

	if (match && on_node) {
	bool match_node = rsc_is_on_node(rsc, on_node, flags);

	if (match_node == FALSE) {
	match = FALSE;
	}
	}

	if (match) {
	return rsc;
	}

	for (GList *gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child = (pe_resource_t ) gIter->data;

	result = rsc->fns->find_rsc(child, id, on_node, flags);
	if (result) {
	return result;
	}
	}
	return NULL;
	}

	// create is ignored
	char *
	native_parameter(pe_resource_t * rsc, pe_node_t * node, gboolean create, const char *name,
	pe_working_set_t * data_set)
	{
	char *value_copy = NULL;
	const char *value = NULL;
	GHashTable *params = NULL;

	CRM_CHECK(rsc != NULL, return NULL);
	CRM_CHECK(name != NULL && strlen(name) != 0, return NULL);

	pe_rsc_trace(rsc, "Looking up %s in %s", name, rsc->id);
	params = pe_rsc_params(rsc, node, data_set);
	value = g_hash_table_lookup(params, name);
	if (value == NULL) {
	/* try meta attributes instead */
	value = g_hash_table_lookup(rsc->meta, name);
	}
	if (value != NULL) {
	value_copy = strdup(value);
	}
	return value_copy;
	}

	gboolean
	native_active(pe_resource_t * rsc, gboolean all)
	{
	for (GList *gIter = rsc->running_on; gIter != NULL; gIter = gIter->next) {
	pe_node_t a_node = (pe_node_t ) gIter->data;

	if (a_node->details->unclean) {
	pe_rsc_trace(rsc, "Resource %s: node %s is unclean",
	rsc->id, a_node->details->uname);
	return TRUE;
	} else if (a_node->details->online == FALSE && pcmk_is_set(rsc->flags, pe_rsc_managed)) {
	pe_rsc_trace(rsc, "Resource %s: node %s is offline",
	rsc->id, a_node->details->uname);
	} else {
	pe_rsc_trace(rsc, "Resource %s active on %s",
	rsc->id, a_node->details->uname);
	return TRUE;
	}
	}
	return FALSE;
	}

	struct print_data_s {
	long options;
	void *print_data;
	};

	static const char *
	native_pending_state(pe_resource_t * rsc)
	{
	const char *pending_state = NULL;

	if (pcmk__str_eq(rsc->pending_task, CRMD_ACTION_START, pcmk__str_casei)) {
	pending_state = "Starting";

	} else if (pcmk__str_eq(rsc->pending_task, CRMD_ACTION_STOP, pcmk__str_casei)) {
	pending_state = "Stopping";

	} else if (pcmk__str_eq(rsc->pending_task, CRMD_ACTION_MIGRATE, pcmk__str_casei)) {
	pending_state = "Migrating";

	} else if (pcmk__str_eq(rsc->pending_task, CRMD_ACTION_MIGRATED, pcmk__str_casei)) {
	/* Work might be done in here. */
	pending_state = "Migrating";

	} else if (pcmk__str_eq(rsc->pending_task, CRMD_ACTION_PROMOTE, pcmk__str_casei)) {
	pending_state = "Promoting";

	} else if (pcmk__str_eq(rsc->pending_task, CRMD_ACTION_DEMOTE, pcmk__str_casei)) {
	pending_state = "Demoting";
	}

	return pending_state;
	}

	static const char *
	native_pending_task(pe_resource_t * rsc)
	{
	const char *pending_task = NULL;

	if (pcmk__str_eq(rsc->pending_task, CRMD_ACTION_STATUS, pcmk__str_casei)) {
	pending_task = "Monitoring";

	/* Pending probes are not printed, even if pending
	* operations are requested. If someone ever requests that
	* behavior, uncomment this and the corresponding part of
	* unpack.c:unpack_rsc_op().
	*/
	/*
	} else if (pcmk__str_eq(rsc->pending_task, "probe", pcmk__str_casei)) {
	pending_task = "Checking";
	*/
	}

	return pending_task;
	}

	static enum rsc_role_e
	native_displayable_role(pe_resource_t *rsc)
	{
	enum rsc_role_e role = rsc->role;

	if ((role == RSC_ROLE_STARTED)
	&& pcmk_is_set(uber_parent(rsc)->flags, pe_rsc_promotable)) {

	role = RSC_ROLE_UNPROMOTED;
	}
	return role;
	}

	static const char *
	native_displayable_state(pe_resource_t *rsc, bool print_pending)
	{
	const char *rsc_state = NULL;

	if (print_pending) {
	rsc_state = native_pending_state(rsc);
	}
	if (rsc_state == NULL) {
	rsc_state = role2text(native_displayable_role(rsc));
	}
	return rsc_state;
	}

	static void
	native_print_xml(pe_resource_t * rsc, const char pre_text, long options, void print_data)
	{
	const char *class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
	const char *prov = crm_element_value(rsc->xml, XML_AGENT_ATTR_PROVIDER);
	const char *rsc_state = native_displayable_state(rsc, pcmk_is_set(options, pe_print_pending));
	const char *target_role = NULL;

	/* resource information. */
	status_print("%s<resource ", pre_text);
	status_print("id=\"%s\" ", rsc_printable_id(rsc));
	status_print("resource_agent=\"%s%s%s:%s\" ", class,
	((prov == NULL)? "" : PROVIDER_SEP),
	((prov == NULL)? "" : prov),
	crm_element_value(rsc->xml, XML_ATTR_TYPE));

	status_print("role=\"%s\" ", rsc_state);
	if (rsc->meta) {
	target_role = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_TARGET_ROLE);
	}
	if (target_role) {
	status_print("target_role=\"%s\" ", target_role);
	}
	status_print("active=\"%s\" ", pcmk__btoa(rsc->fns->active(rsc, TRUE)));
	status_print("orphaned=\"%s\" ", pe__rsc_bool_str(rsc, pe_rsc_orphan));
	status_print("blocked=\"%s\" ", pe__rsc_bool_str(rsc, pe_rsc_block));
	status_print("managed=\"%s\" ", pe__rsc_bool_str(rsc, pe_rsc_managed));
	status_print("failed=\"%s\" ", pe__rsc_bool_str(rsc, pe_rsc_failed));
	status_print("failure_ignored=\"%s\" ",
	pe__rsc_bool_str(rsc, pe_rsc_failure_ignored));
	status_print("nodes_running_on=\"%d\" ", g_list_length(rsc->running_on));

	if (options & pe_print_pending) {
	const char *pending_task = native_pending_task(rsc);

	if (pending_task) {
	status_print("pending=\"%s\" ", pending_task);
	}
	}

	/* print out the nodes this resource is running on */
	if (options & pe_print_rsconly) {
	status_print("/>\n");
	/* do nothing */
	} else if (rsc->running_on != NULL) {
	GList *gIter = rsc->running_on;

	status_print(">\n");
	for (; gIter != NULL; gIter = gIter->next) {
	pe_node_t node = (pe_node_t ) gIter->data;

	status_print("%s <node name=\"%s\" id=\"%s\" cached=\"%s\"/>\n", pre_text,
	node->details->uname, node->details->id,
	pcmk__btoa(node->details->online == FALSE));
	}
	status_print("%s</resource>\n", pre_text);
	} else {
	status_print("/>\n");
	}
	}

	// Append a flag to resource description string's flags list
	static bool
	add_output_flag(GString s, const char flag_desc, bool have_flags)
	{
	g_string_append(s, (have_flags? ", " : " ("));
	g_string_append(s, flag_desc);
	return true;
	}

	// Append a node name to resource description string's node list
	static bool
	add_output_node(GString s, const char node, bool have_nodes)
	{
	g_string_append(s, (have_nodes? " " : " [ "));
	g_string_append(s, node);
	return true;
	}

	/*!
	* \internal
	* \brief Create a string description of a resource
	*
	* \param[in] rsc Resource to describe
	* \param[in] name Desired identifier for the resource
	* \param[in] node If not NULL, node that resource is "on"
	* \param[in] show_opts Bitmask of pcmk_show_opt_e.
	* \param[in] target_role Resource's target role
	* \param[in] show_nodes Whether to display nodes when multiply active
	*
	* \return Newly allocated string description of resource
	* \note Caller must free the result with g_free().
	*/
	gchar *
	pcmk__native_output_string(pe_resource_t rsc, const char name, pe_node_t *node,
	unsigned long show_opts, const char *target_role, bool show_nodes)
	{
	const char *class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
	const char *provider = NULL;
	const char *kind = crm_element_value(rsc->xml, XML_ATTR_TYPE);
	gchar *retval = NULL;
	GString *outstr = NULL;
	bool have_flags = false;

	if (rsc->variant != pe_native) {
	return NULL;
	}

	CRM_CHECK(name != NULL, name = "unknown");
	CRM_CHECK(kind != NULL, kind = "unknown");
	CRM_CHECK(class != NULL, class = "unknown");

	if (pcmk_is_set(pcmk_get_ra_caps(class), pcmk_ra_cap_provider)) {
	provider = crm_element_value(rsc->xml, XML_AGENT_ATTR_PROVIDER);
	}

	if ((node == NULL) && (rsc->lock_node != NULL)) {
	node = rsc->lock_node;
	}
	if (pcmk_is_set(show_opts, pcmk_show_rsc_only)
	\|\| pcmk__list_of_multiple(rsc->running_on)) {
	node = NULL;
	}

	// We need a string of at least this size
	outstr = g_string_sized_new(strlen(name) + strlen(class) + strlen(kind)
	+ (provider? (strlen(provider) + 2) : 0)
	+ (node? strlen(node->details->uname) + 1 : 0)
	+ 11);

	// Resource name and agent
	g_string_printf(outstr, "%s\t(%s%s%s:%s):\t", name, class,
	((provider == NULL)? "" : PROVIDER_SEP),
	((provider == NULL)? "" : provider), kind);

	// State on node
	if (pcmk_is_set(rsc->flags, pe_rsc_orphan)) {
	g_string_append(outstr, " ORPHANED");
	}
	if (pcmk_is_set(rsc->flags, pe_rsc_failed)) {
	enum rsc_role_e role = native_displayable_role(rsc);

	if (role > RSC_ROLE_UNPROMOTED) {
	g_string_append_printf(outstr, " FAILED %s", role2text(role));
	} else {
	g_string_append(outstr, " FAILED");
	}
	} else {
	g_string_append_printf(outstr, " %s", native_displayable_state(rsc, pcmk_is_set(show_opts, pcmk_show_pending)));
	}
	if (node) {
	g_string_append_printf(outstr, " %s", node->details->uname);
	}

	// Flags, as: (<flag> [...])
	if (node && !(node->details->online) && node->details->unclean) {
	have_flags = add_output_flag(outstr, "UNCLEAN", have_flags);
	}
	if (node && (node == rsc->lock_node)) {
	have_flags = add_output_flag(outstr, "LOCKED", have_flags);
	}
	if (pcmk_is_set(show_opts, pcmk_show_pending)) {
	const char *pending_task = native_pending_task(rsc);

	if (pending_task) {
	have_flags = add_output_flag(outstr, pending_task, have_flags);
	}
	}
	if (target_role) {
	enum rsc_role_e target_role_e = text2role(target_role);

	/* Only show target role if it limits our abilities (i.e. ignore
	* Started, as it is the default anyways, and doesn't prevent the
	* resource from becoming promoted).
	*/
	if (target_role_e == RSC_ROLE_STOPPED) {
	have_flags = add_output_flag(outstr, "disabled", have_flags);

	} else if (pcmk_is_set(uber_parent(rsc)->flags, pe_rsc_promotable)
	&& target_role_e == RSC_ROLE_UNPROMOTED) {
	have_flags = add_output_flag(outstr, "target-role:", have_flags);
	g_string_append(outstr, target_role);
	}
	}
	if (pcmk_is_set(rsc->flags, pe_rsc_block)) {
	have_flags = add_output_flag(outstr, "blocked", have_flags);
	} else if (!pcmk_is_set(rsc->flags, pe_rsc_managed)) {
	have_flags = add_output_flag(outstr, "unmanaged", have_flags);
	}
	if (pcmk_is_set(rsc->flags, pe_rsc_failure_ignored)) {
	have_flags = add_output_flag(outstr, "failure ignored", have_flags);
	}
	if (have_flags) {
	g_string_append(outstr, ")");
	}

	// User-supplied description
	if (pcmk_is_set(show_opts, pcmk_show_rsc_only)
	\|\| pcmk__list_of_multiple(rsc->running_on)) {
	const char *desc = crm_element_value(rsc->xml, XML_ATTR_DESC);

	if (desc) {
	g_string_append_printf(outstr, " %s", desc);
	}
	}

	if (show_nodes && !pcmk_is_set(show_opts, pcmk_show_rsc_only)
	&& pcmk__list_of_multiple(rsc->running_on)) {
	bool have_nodes = false;

	for (GList *iter = rsc->running_on; iter != NULL; iter = iter->next) {
	pe_node_t n = (pe_node_t ) iter->data;

	have_nodes = add_output_node(outstr, n->details->uname, have_nodes);
	}
	if (have_nodes) {
	g_string_append(outstr, " ]");
	}
	}

	retval = outstr->str;
	g_string_free(outstr, FALSE);
	return retval;
	}

	int
	pe__common_output_html(pcmk__output_t out, pe_resource_t rsc,
	const char name, pe_node_t node, unsigned int show_opts)
	{
	const char *kind = crm_element_value(rsc->xml, XML_ATTR_TYPE);
	const char *target_role = NULL;

	xmlNodePtr list_node = NULL;
	const char *cl = NULL;

	CRM_ASSERT(rsc->variant == pe_native);
	CRM_ASSERT(kind != NULL);

	if (rsc->meta) {
	const char *is_internal = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_INTERNAL_RSC);

	if (crm_is_true(is_internal)
	&& !pcmk_is_set(show_opts, pcmk_show_implicit_rscs)) {

	crm_trace("skipping print of internal resource %s", rsc->id);
	return pcmk_rc_no_output;
	}
	target_role = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_TARGET_ROLE);
	}

	if (!pcmk_is_set(rsc->flags, pe_rsc_managed)) {
	cl = "rsc-managed";

	} else if (pcmk_is_set(rsc->flags, pe_rsc_failed)) {
	cl = "rsc-failed";

	} else if (rsc->variant == pe_native && (rsc->running_on == NULL)) {
	cl = "rsc-failed";

	} else if (pcmk__list_of_multiple(rsc->running_on)) {
	cl = "rsc-multiple";

	} else if (pcmk_is_set(rsc->flags, pe_rsc_failure_ignored)) {
	cl = "rsc-failure-ignored";

	} else {
	cl = "rsc-ok";
	}

	{
	gchar *s = pcmk__native_output_string(rsc, name, node, show_opts,
	target_role, true);

	list_node = pcmk__output_create_html_node(out, "li", NULL, NULL, NULL);
	pcmk_create_html_node(list_node, "span", NULL, cl, s);
	g_free(s);
	}

	return pcmk_rc_ok;
	}

	int
	pe__common_output_text(pcmk__output_t out, pe_resource_t rsc,
	const char name, pe_node_t node, unsigned int show_opts)
	{
	const char *target_role = NULL;

	CRM_ASSERT(rsc->variant == pe_native);

	if (rsc->meta) {
	const char *is_internal = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_INTERNAL_RSC);

	if (crm_is_true(is_internal)
	&& !pcmk_is_set(show_opts, pcmk_show_implicit_rscs)) {

	crm_trace("skipping print of internal resource %s", rsc->id);
	return pcmk_rc_no_output;
	}
	target_role = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_TARGET_ROLE);
	}

	{
	gchar *s = pcmk__native_output_string(rsc, name, node, show_opts,
	target_role, true);

	out->list_item(out, NULL, "%s", s);
	g_free(s);
	}

	return pcmk_rc_ok;
	}

	void
	common_print(pe_resource_t * rsc, const char pre_text, const char name, pe_node_t node, long options, void print_data)
	{
	const char *target_role = NULL;

	CRM_ASSERT(rsc->variant == pe_native);

	if (rsc->meta) {
	const char *is_internal = g_hash_table_lookup(rsc->meta,
	XML_RSC_ATTR_INTERNAL_RSC);

	if (crm_is_true(is_internal)
	&& !pcmk_is_set(options, pe_print_implicit)) {

	crm_trace("skipping print of internal resource %s", rsc->id);
	return;
	}
	target_role = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_TARGET_ROLE);
	}

	if (options & pe_print_xml) {
	native_print_xml(rsc, pre_text, options, print_data);
	return;
	}

	if ((pre_text == NULL) && (options & pe_print_printf)) {
	pre_text = " ";
	}

	if (options & pe_print_html) {
	if (!pcmk_is_set(rsc->flags, pe_rsc_managed)) {
	status_print("<font color=\"yellow\">");

	} else if (pcmk_is_set(rsc->flags, pe_rsc_failed)) {
	status_print("<font color=\"red\">");

	} else if (rsc->running_on == NULL) {
	status_print("<font color=\"red\">");

	} else if (pcmk__list_of_multiple(rsc->running_on)) {
	status_print("<font color=\"orange\">");

	} else if (pcmk_is_set(rsc->flags, pe_rsc_failure_ignored)) {
	status_print("<font color=\"yellow\">");

	} else {
	status_print("<font color=\"green\">");
	}
	}

	{
	gchar *resource_s = pcmk__native_output_string(rsc, name, node, options,
	target_role, false);
	status_print("%s%s", (pre_text? pre_text : ""), resource_s);
	g_free(resource_s);
	}

	if (pcmk_is_set(options, pe_print_html)) {
	status_print(" </font> ");
	}

	if (!pcmk_is_set(options, pe_print_rsconly)
	&& pcmk__list_of_multiple(rsc->running_on)) {

	GList *gIter = rsc->running_on;
	int counter = 0;

	if (options & pe_print_html) {
	status_print("<ul>\n");
	} else if ((options & pe_print_printf)
	\|\| (options & pe_print_ncurses)) {
	status_print("[");
	}

	for (; gIter != NULL; gIter = gIter->next) {
	pe_node_t n = (pe_node_t ) gIter->data;

	counter++;

	if (options & pe_print_html) {
	status_print("<li>\n%s", n->details->uname);

	} else if ((options & pe_print_printf)
	\|\| (options & pe_print_ncurses)) {
	status_print(" %s", n->details->uname);

	} else if ((options & pe_print_log)) {
	status_print("\t%d : %s", counter, n->details->uname);

	} else {
	status_print("%s", n->details->uname);
	}
	if (options & pe_print_html) {
	status_print("</li>\n");

	}
	}

	if (options & pe_print_html) {
	status_print("</ul>\n");
	} else if ((options & pe_print_printf)
	\|\| (options & pe_print_ncurses)) {
	status_print(" ]");
	}
	}

	if (options & pe_print_html) {
	status_print("<br/>\n");
	} else if (options & pe_print_suppres_nl) {
	/* nothing */
	} else if ((options & pe_print_printf) \|\| (options & pe_print_ncurses)) {
	status_print("\n");
	}
	}

	void
	native_print(pe_resource_t * rsc, const char pre_text, long options, void print_data)
	{
	pe_node_t *node = NULL;

	CRM_ASSERT(rsc->variant == pe_native);
	if (options & pe_print_xml) {
	native_print_xml(rsc, pre_text, options, print_data);
	return;
	}

	node = pe__current_node(rsc);

	if (node == NULL) {
	// This is set only if a non-probe action is pending on this node
	node = rsc->pending_node;
	}

	common_print(rsc, pre_text, rsc_printable_id(rsc), node, options, print_data);
	}

	PCMK__OUTPUT_ARGS("primitive", "unsigned int", "pe_resource_t ", "GList ", "GList *")
	int
	pe__resource_xml(pcmk__output_t *out, va_list args)
	{
	unsigned int show_opts = va_arg(args, unsigned int);
	pe_resource_t rsc = va_arg(args, pe_resource_t );
	GList only_node G_GNUC_UNUSED = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );

	bool print_pending = pcmk_is_set(show_opts, pcmk_show_pending);
	const char *class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
	const char *prov = crm_element_value(rsc->xml, XML_AGENT_ATTR_PROVIDER);
	const char *rsc_state = native_displayable_state(rsc, print_pending);

	char ra_name[LINE_MAX];
	char *nodes_running_on = NULL;
	- char *priority = NULL;
	int rc = pcmk_rc_no_output;
	const char *target_role = NULL;

	if (rsc->meta != NULL) {
	target_role = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_TARGET_ROLE);
	}

	CRM_ASSERT(rsc->variant == pe_native);

	if (rsc->fns->is_filtered(rsc, only_rsc, TRUE)) {
	return pcmk_rc_no_output;
	}

	/* resource information. */
	snprintf(ra_name, LINE_MAX, "%s%s%s:%s", class,
	((prov == NULL)? "" : PROVIDER_SEP), ((prov == NULL)? "" : prov),
	crm_element_value(rsc->xml, XML_ATTR_TYPE));

	nodes_running_on = pcmk__itoa(g_list_length(rsc->running_on));
	- priority = pcmk__ftoa(rsc->priority);

	rc = pe__name_and_nvpairs_xml(out, true, "resource", 12,
	"id", rsc_printable_id(rsc),
	"resource_agent", ra_name,
	"role", rsc_state,
	"target_role", target_role,
	"active", pcmk__btoa(rsc->fns->active(rsc, TRUE)),
	"orphaned", pe__rsc_bool_str(rsc, pe_rsc_orphan),
	"blocked", pe__rsc_bool_str(rsc, pe_rsc_block),
	"managed", pe__rsc_bool_str(rsc, pe_rsc_managed),
	"failed", pe__rsc_bool_str(rsc, pe_rsc_failed),
	"failure_ignored", pe__rsc_bool_str(rsc, pe_rsc_failure_ignored),
	"nodes_running_on", nodes_running_on,
	"pending", (print_pending? native_pending_task(rsc) : NULL));
	- free(priority);
	free(nodes_running_on);

	CRM_ASSERT(rc == pcmk_rc_ok);

	if (rsc->running_on != NULL) {
	GList *gIter = rsc->running_on;

	for (; gIter != NULL; gIter = gIter->next) {
	pe_node_t node = (pe_node_t ) gIter->data;

	rc = pe__name_and_nvpairs_xml(out, false, "node", 3,
	"name", node->details->uname,
	"id", node->details->id,
	"cached", pcmk__btoa(node->details->online));
	CRM_ASSERT(rc == pcmk_rc_ok);
	}
	}

	pcmk__output_xml_pop_parent(out);
	return rc;
	}

	PCMK__OUTPUT_ARGS("primitive", "unsigned int", "pe_resource_t ", "GList ", "GList *")
	int
	pe__resource_html(pcmk__output_t *out, va_list args)
	{
	unsigned int show_opts = va_arg(args, unsigned int);
	pe_resource_t rsc = va_arg(args, pe_resource_t );
	GList only_node G_GNUC_UNUSED = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );

	pe_node_t *node = pe__current_node(rsc);

	if (rsc->fns->is_filtered(rsc, only_rsc, TRUE)) {
	return pcmk_rc_no_output;
	}

	CRM_ASSERT(rsc->variant == pe_native);

	if (node == NULL) {
	// This is set only if a non-probe action is pending on this node
	node = rsc->pending_node;
	}
	return pe__common_output_html(out, rsc, rsc_printable_id(rsc), node, show_opts);
	}

	PCMK__OUTPUT_ARGS("primitive", "unsigned int", "pe_resource_t ", "GList ", "GList *")
	int
	pe__resource_text(pcmk__output_t *out, va_list args)
	{
	unsigned int show_opts = va_arg(args, unsigned int);
	pe_resource_t rsc = va_arg(args, pe_resource_t );
	GList only_node G_GNUC_UNUSED = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );

	pe_node_t *node = pe__current_node(rsc);

	CRM_ASSERT(rsc->variant == pe_native);

	if (rsc->fns->is_filtered(rsc, only_rsc, TRUE)) {
	return pcmk_rc_no_output;
	}

	if (node == NULL) {
	// This is set only if a non-probe action is pending on this node
	node = rsc->pending_node;
	}
	return pe__common_output_text(out, rsc, rsc_printable_id(rsc), node, show_opts);
	}

	void
	native_free(pe_resource_t * rsc)
	{
	pe_rsc_trace(rsc, "Freeing resource action list (not the data)");
	common_free(rsc);
	}

	enum rsc_role_e
	native_resource_state(const pe_resource_t * rsc, gboolean current)
	{
	enum rsc_role_e role = rsc->next_role;

	if (current) {
	role = rsc->role;
	}
	pe_rsc_trace(rsc, "%s state: %s", rsc->id, role2text(role));
	return role;
	}

	/*!
	* \internal
	* \brief List nodes where a resource (or any of its children) is
	*
	* \param[in] rsc Resource to check
	* \param[out] list List to add result to
	* \param[in] current 0 = where allocated, 1 = where running,
	* 2 = where running or pending
	*
	* \return If list contains only one node, that node, or NULL otherwise
	*/
	pe_node_t *
	native_location(const pe_resource_t rsc, GList *list, int current)
	{
	pe_node_t *one = NULL;
	GList *result = NULL;

	if (rsc->children) {
	GList *gIter = rsc->children;

	for (; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child = (pe_resource_t ) gIter->data;

	child->fns->location(child, &result, current);
	}

	} else if (current) {

	if (rsc->running_on) {
	result = g_list_copy(rsc->running_on);
	}
	if ((current == 2) && rsc->pending_node
	&& !pe_find_node_id(result, rsc->pending_node->details->id)) {
	result = g_list_append(result, rsc->pending_node);
	}

	} else if (current == FALSE && rsc->allocated_to) {
	result = g_list_append(NULL, rsc->allocated_to);
	}

	if (result && (result->next == NULL)) {
	one = result->data;
	}

	if (list) {
	GList *gIter = result;

	for (; gIter != NULL; gIter = gIter->next) {
	pe_node_t node = (pe_node_t ) gIter->data;

	if (list == NULL \|\| pe_find_node_id(list, node->details->id) == NULL) {
	list = g_list_append(list, node);
	}
	}
	}

	g_list_free(result);
	return one;
	}

	static void
	get_rscs_brief(GList rsc_list, GHashTable rsc_table, GHashTable * active_table)
	{
	GList *gIter = rsc_list;

	for (; gIter != NULL; gIter = gIter->next) {
	pe_resource_t rsc = (pe_resource_t ) gIter->data;

	const char *class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
	const char *kind = crm_element_value(rsc->xml, XML_ATTR_TYPE);

	int offset = 0;
	char buffer[LINE_MAX];

	int *rsc_counter = NULL;
	int *active_counter = NULL;

	if (rsc->variant != pe_native) {
	continue;
	}

	offset += snprintf(buffer + offset, LINE_MAX - offset, "%s", class);
	if (pcmk_is_set(pcmk_get_ra_caps(class), pcmk_ra_cap_provider)) {
	const char *prov = crm_element_value(rsc->xml, XML_AGENT_ATTR_PROVIDER);

	if (prov != NULL) {
	offset += snprintf(buffer + offset, LINE_MAX - offset,
	PROVIDER_SEP "%s", prov);
	}
	}
	offset += snprintf(buffer + offset, LINE_MAX - offset, ":%s", kind);
	CRM_LOG_ASSERT(offset > 0);

	if (rsc_table) {
	rsc_counter = g_hash_table_lookup(rsc_table, buffer);
	if (rsc_counter == NULL) {
	rsc_counter = calloc(1, sizeof(int));
	*rsc_counter = 0;
	g_hash_table_insert(rsc_table, strdup(buffer), rsc_counter);
	}
	(*rsc_counter)++;
	}

	if (active_table) {
	GList *gIter2 = rsc->running_on;

	for (; gIter2 != NULL; gIter2 = gIter2->next) {
	pe_node_t node = (pe_node_t ) gIter2->data;
	GHashTable *node_table = NULL;

	if (node->details->unclean == FALSE && node->details->online == FALSE &&
	pcmk_is_set(rsc->flags, pe_rsc_managed)) {
	continue;
	}

	node_table = g_hash_table_lookup(active_table, node->details->uname);
	if (node_table == NULL) {
	node_table = pcmk__strkey_table(free, free);
	g_hash_table_insert(active_table, strdup(node->details->uname), node_table);
	}

	active_counter = g_hash_table_lookup(node_table, buffer);
	if (active_counter == NULL) {
	active_counter = calloc(1, sizeof(int));
	*active_counter = 0;
	g_hash_table_insert(node_table, strdup(buffer), active_counter);
	}
	(*active_counter)++;
	}
	}
	}
	}

	static void
	destroy_node_table(gpointer data)
	{
	GHashTable *node_table = data;

	if (node_table) {
	g_hash_table_destroy(node_table);
	}
	}

	void
	print_rscs_brief(GList rsc_list, const char pre_text, long options,
	void *print_data, gboolean print_all)
	{
	GHashTable *rsc_table = pcmk__strkey_table(free, free);
	GHashTable *active_table = pcmk__strkey_table(free, destroy_node_table);
	GHashTableIter hash_iter;
	char *type = NULL;
	int *rsc_counter = NULL;

	get_rscs_brief(rsc_list, rsc_table, active_table);

	g_hash_table_iter_init(&hash_iter, rsc_table);
	while (g_hash_table_iter_next(&hash_iter, (gpointer )&type, (gpointer )&rsc_counter)) {
	GHashTableIter hash_iter2;
	char *node_name = NULL;
	GHashTable *node_table = NULL;
	int active_counter_all = 0;

	g_hash_table_iter_init(&hash_iter2, active_table);
	while (g_hash_table_iter_next(&hash_iter2, (gpointer )&node_name, (gpointer )&node_table)) {
	int *active_counter = g_hash_table_lookup(node_table, type);

	if (active_counter == NULL \|\| *active_counter == 0) {
	continue;

	} else {
	active_counter_all += *active_counter;
	}

	if (options & pe_print_rsconly) {
	node_name = NULL;
	}

	if (options & pe_print_html) {
	status_print("<li>\n");
	}

	if (print_all) {
	status_print("%s%d/%d\t(%s):\tActive %s\n", pre_text ? pre_text : "",
	active_counter ? *active_counter : 0,
	rsc_counter ? *rsc_counter : 0, type,
	active_counter && (*active_counter > 0) && node_name ? node_name : "");
	} else {
	status_print("%s%d\t(%s):\tActive %s\n", pre_text ? pre_text : "",
	active_counter ? *active_counter : 0, type,
	active_counter && (*active_counter > 0) && node_name ? node_name : "");
	}

	if (options & pe_print_html) {
	status_print("</li>\n");
	}
	}

	if (print_all && active_counter_all == 0) {
	if (options & pe_print_html) {
	status_print("<li>\n");
	}

	status_print("%s%d/%d\t(%s):\tActive\n", pre_text ? pre_text : "",
	active_counter_all,
	rsc_counter ? *rsc_counter : 0, type);

	if (options & pe_print_html) {
	status_print("</li>\n");
	}
	}
	}

	if (rsc_table) {
	g_hash_table_destroy(rsc_table);
	rsc_table = NULL;
	}
	if (active_table) {
	g_hash_table_destroy(active_table);
	active_table = NULL;
	}
	}

	int
	pe__rscs_brief_output(pcmk__output_t out, GList rsc_list, unsigned int show_opts)
	{
	GHashTable *rsc_table = pcmk__strkey_table(free, free);
	GHashTable *active_table = pcmk__strkey_table(free, destroy_node_table);
	GList *sorted_rscs;
	int rc = pcmk_rc_no_output;

	get_rscs_brief(rsc_list, rsc_table, active_table);

	/* Make a list of the rsc_table keys so that it can be sorted. This is to make sure
	* output order stays consistent between systems.
	*/
	sorted_rscs = g_hash_table_get_keys(rsc_table);
	sorted_rscs = g_list_sort(sorted_rscs, (GCompareFunc) strcmp);

	for (GList *gIter = sorted_rscs; gIter; gIter = gIter->next) {
	char type = (char ) gIter->data;
	int *rsc_counter = g_hash_table_lookup(rsc_table, type);

	GList *sorted_nodes = NULL;
	int active_counter_all = 0;

	/* Also make a list of the active_table keys so it can be sorted. If there's
	* more than one instance of a type of resource running, we need the nodes to
	* be sorted to make sure output order stays consistent between systems.
	*/
	sorted_nodes = g_hash_table_get_keys(active_table);
	sorted_nodes = g_list_sort(sorted_nodes, (GCompareFunc) pcmk__numeric_strcasecmp);

	for (GList *gIter2 = sorted_nodes; gIter2; gIter2 = gIter2->next) {
	char node_name = (char ) gIter2->data;
	GHashTable *node_table = g_hash_table_lookup(active_table, node_name);
	int *active_counter = NULL;

	if (node_table == NULL) {
	continue;
	}

	active_counter = g_hash_table_lookup(node_table, type);

	if (active_counter == NULL \|\| *active_counter == 0) {
	continue;

	} else {
	active_counter_all += *active_counter;
	}

	if (pcmk_is_set(show_opts, pcmk_show_rsc_only)) {
	node_name = NULL;
	}

	if (pcmk_is_set(show_opts, pcmk_show_inactive_rscs)) {
	out->list_item(out, NULL, "%d/%d\t(%s):\tActive %s",
	*active_counter,
	rsc_counter ? *rsc_counter : 0, type,
	(*active_counter > 0) && node_name ? node_name : "");
	} else {
	out->list_item(out, NULL, "%d\t(%s):\tActive %s",
	*active_counter, type,
	(*active_counter > 0) && node_name ? node_name : "");
	}

	rc = pcmk_rc_ok;
	}

	if (pcmk_is_set(show_opts, pcmk_show_inactive_rscs) && active_counter_all == 0) {
	out->list_item(out, NULL, "%d/%d\t(%s):\tActive",
	active_counter_all,
	rsc_counter ? *rsc_counter : 0, type);
	rc = pcmk_rc_ok;
	}

	if (sorted_nodes) {
	g_list_free(sorted_nodes);
	}
	}

	if (rsc_table) {
	g_hash_table_destroy(rsc_table);
	rsc_table = NULL;
	}
	if (active_table) {
	g_hash_table_destroy(active_table);
	active_table = NULL;
	}
	if (sorted_rscs) {
	g_list_free(sorted_rscs);
	}

	return rc;
	}

	gboolean
	pe__native_is_filtered(pe_resource_t rsc, GList only_rsc, gboolean check_parent)
	{
	if (pcmk__str_in_list(rsc_printable_id(rsc), only_rsc, pcmk__str_star_matches) \|\|
	pcmk__str_in_list(rsc->id, only_rsc, pcmk__str_star_matches)) {
	return FALSE;
	} else if (check_parent && rsc->parent) {
	pe_resource_t *up = uber_parent(rsc);

	if (pe_rsc_is_bundled(rsc)) {
	return up->parent->fns->is_filtered(up->parent, only_rsc, FALSE);
	} else {
	return up->fns->is_filtered(up, only_rsc, FALSE);
	}
	}

	return TRUE;
	}

File Metadata

Mime Type: text/x-diff
Expires: Sat, Jan 25, 12:09 PM (19 h, 50 m)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 1322507
Default Alt Text: (463 KB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions