Page MenuHomeClusterLabs Projects
Files F4624317

No OneTemporary
Actions

View Options

This file is larger than 256 KB, so syntax highlighting was skipped.
diff --git a/cts/lab/CTStests.py b/cts/lab/CTStests.py
index 8fdabb11bf..16e31a4338 100644
--- a/cts/lab/CTStests.py
+++ b/cts/lab/CTStests.py
@@ -1,3135 +1,3135 @@
""" Test-specific classes for Pacemaker's Cluster Test Suite (CTS)
"""
-__copyright__ = "Copyright 2000-2021 the Pacemaker project contributors"
+__copyright__ = "Copyright 2000-2022 the Pacemaker project contributors"
__license__ = "GNU General Public License version 2 or later (GPLv2+) WITHOUT ANY WARRANTY"
#
# SPECIAL NOTE:
#
# Tests may NOT implement any cluster-manager-specific code in them.
# EXTEND the ClusterManager object to provide the base capabilities
# the test needs if you need to do something that the current CM classes
# do not. Otherwise you screw up the whole point of the object structure
# in CTS.
#
# Thank you.
#
import os
import re
import time
import subprocess
import tempfile
from stat import *
from cts import CTS
from cts.CTSaudits import *
from cts.CTSvars import *
from cts.patterns import PatternSelector
from cts.logging import LogFactory
from cts.remote import RemoteFactory
from cts.watcher import LogWatcher
from cts.environment import EnvFactory
AllTestClasses = [ ]
class CTSTest(object):
'''
A Cluster test.
We implement the basic set of properties and behaviors for a generic
cluster test.
Cluster tests track their own statistics.
We keep each of the kinds of counts we track as separate {name,value}
pairs.
'''
def __init__(self, cm):
#self.name="the unnamed test"
self.Stats = {"calls":0
, "success":0
, "failure":0
, "skipped":0
, "auditfail":0}
# if not issubclass(cm.__class__, ClusterManager):
# raise ValueError("Must be a ClusterManager object")
self.CM = cm
self.Env = EnvFactory().getInstance()
self.rsh = RemoteFactory().getInstance()
self.logger = LogFactory()
self.templates = PatternSelector(cm["Name"])
self.Audits = []
self.timeout = 120
self.passed = 1
self.is_loop = 0
self.is_unsafe = 0
self.is_experimental = 0
self.is_container = 0
self.is_valgrind = 0
self.benchmark = 0 # which tests to benchmark
self.timer = {} # timers
def log(self, args):
self.logger.log(args)
def debug(self, args):
self.logger.debug(args)
def has_key(self, key):
return key in self.Stats
def __setitem__(self, key, value):
self.Stats[key] = value
def __getitem__(self, key):
if str(key) == "0":
raise ValueError("Bad call to 'foo in X', should reference 'foo in X.Stats' instead")
if key in self.Stats:
return self.Stats[key]
return None
def log_mark(self, msg):
self.debug("MARK: test %s %s %d" % (self.name,msg,time.time()))
return
def get_timer(self,key = "test"):
try: return self.timer[key]
except: return 0
def set_timer(self,key = "test"):
self.timer[key] = time.time()
return self.timer[key]
def log_timer(self,key = "test"):
elapsed = 0
if key in self.timer:
elapsed = time.time() - self.timer[key]
s = key == "test" and self.name or "%s:%s" % (self.name,key)
self.debug("%s runtime: %.2f" % (s, elapsed))
del self.timer[key]
return elapsed
def incr(self, name):
'''Increment (or initialize) the value associated with the given name'''
if not name in self.Stats:
self.Stats[name] = 0
self.Stats[name] = self.Stats[name]+1
# Reset the test passed boolean
if name == "calls":
self.passed = 1
def failure(self, reason="none"):
'''Increment the failure count'''
self.passed = 0
self.incr("failure")
self.logger.log(("Test %s" % self.name).ljust(35) + " FAILED: %s" % reason)
return None
def success(self):
'''Increment the success count'''
self.incr("success")
return 1
def skipped(self):
'''Increment the skipped count'''
self.incr("skipped")
return 1
def __call__(self, node):
'''Perform the given test'''
raise ValueError("Abstract Class member (__call__)")
self.incr("calls")
return self.failure()
def audit(self):
passed = 1
if len(self.Audits) > 0:
for audit in self.Audits:
if not audit():
self.logger.log("Internal %s Audit %s FAILED." % (self.name, audit.name()))
self.incr("auditfail")
passed = 0
return passed
def setup(self, node):
'''Setup the given test'''
return self.success()
def teardown(self, node):
'''Tear down the given test'''
return self.success()
def create_watch(self, patterns, timeout, name=None):
if not name:
name = self.name
return LogWatcher(self.Env["LogFileName"], patterns, name, timeout, kind=self.Env["LogWatcher"], hosts=self.Env["nodes"])
def local_badnews(self, prefix, watch, local_ignore=[]):
errcount = 0
if not prefix:
prefix = "LocalBadNews:"
ignorelist = []
ignorelist.append(" CTS: ")
ignorelist.append(prefix)
ignorelist.extend(local_ignore)
while errcount < 100:
match = watch.look(0)
if match:
add_err = 1
for ignore in ignorelist:
if add_err == 1 and re.search(ignore, match):
add_err = 0
if add_err == 1:
self.logger.log(prefix + " " + match)
errcount = errcount + 1
else:
break
else:
self.logger.log("Too many errors!")
watch.end()
return errcount
def is_applicable(self):
return self.is_applicable_common()
def is_applicable_common(self):
'''Return TRUE if we are applicable in the current test configuration'''
#raise ValueError("Abstract Class member (is_applicable)")
if self.is_loop and not self.Env["loop-tests"]:
return 0
elif self.is_unsafe and not self.Env["unsafe-tests"]:
return 0
elif self.is_valgrind and not self.Env["valgrind-tests"]:
return 0
elif self.is_experimental and not self.Env["experimental-tests"]:
return 0
elif self.is_container and not self.Env["container-tests"]:
return 0
elif self.Env["benchmark"] and self.benchmark == 0:
return 0
return 1
def find_ocfs2_resources(self, node):
self.r_o2cb = None
self.r_ocfs2 = []
(rc, lines) = self.rsh(node, "crm_resource -c", None)
for line in lines:
if re.search("^Resource", line):
r = AuditResource(self.CM, line)
if r.rtype == "o2cb" and r.parent != "NA":
self.debug("Found o2cb: %s" % self.r_o2cb)
self.r_o2cb = r.parent
if re.search("^Constraint", line):
c = AuditConstraint(self.CM, line)
if c.type == "rsc_colocation" and c.target == self.r_o2cb:
self.r_ocfs2.append(c.rsc)
self.debug("Found ocfs2 filesystems: %s" % repr(self.r_ocfs2))
return len(self.r_ocfs2)
def canrunnow(self, node):
'''Return TRUE if we can meaningfully run right now'''
return 1
def errorstoignore(self):
'''Return list of errors which are 'normal' and should be ignored'''
return []
class StopTest(CTSTest):
'''Stop (deactivate) the cluster manager on a node'''
def __init__(self, cm):
CTSTest.__init__(self, cm)
self.name = "Stop"
def __call__(self, node):
'''Perform the 'stop' test. '''
self.incr("calls")
if self.CM.ShouldBeStatus[node] != "up":
return self.skipped()
patterns = []
# Technically we should always be able to notice ourselves stopping
patterns.append(self.templates["Pat:We_stopped"] % node)
# Any active node needs to notice this one left
# (note that this won't work if we have multiple partitions)
for other in self.Env["nodes"]:
if self.CM.ShouldBeStatus[other] == "up" and other != node:
patterns.append(self.templates["Pat:They_stopped"] %(other, self.CM.key_for_node(node)))
#self.debug("Checking %s will notice %s left"%(other, node))
watch = self.create_watch(patterns, self.Env["DeadTime"])
watch.setwatch()
if node == self.CM.OurNode:
self.incr("us")
else:
if self.CM.upcount() <= 1:
self.incr("all")
else:
self.incr("them")
self.CM.StopaCM(node)
watch_result = watch.lookforall()
failreason = None
UnmatchedList = "||"
if watch.unmatched:
(rc, output) = self.rsh(node, "/bin/ps axf", None)
for line in output:
self.debug(line)
(rc, output) = self.rsh(node, "/usr/sbin/dlm_tool dump 2>/dev/null", None)
for line in output:
self.debug(line)
for regex in watch.unmatched:
self.logger.log ("ERROR: Shutdown pattern not found: %s" % (regex))
UnmatchedList += regex + "||";
failreason = "Missing shutdown pattern"
self.CM.cluster_stable(self.Env["DeadTime"])
if not watch.unmatched or self.CM.upcount() == 0:
return self.success()
if len(watch.unmatched) >= self.CM.upcount():
return self.failure("no match against (%s)" % UnmatchedList)
if failreason == None:
return self.success()
else:
return self.failure(failreason)
#
# We don't register StopTest because it's better when called by
# another test...
#
class StartTest(CTSTest):
'''Start (activate) the cluster manager on a node'''
def __init__(self, cm, debug=None):
CTSTest.__init__(self,cm)
self.name = "start"
self.debug = debug
def __call__(self, node):
'''Perform the 'start' test. '''
self.incr("calls")
if self.CM.upcount() == 0:
self.incr("us")
else:
self.incr("them")
if self.CM.ShouldBeStatus[node] != "down":
return self.skipped()
elif self.CM.StartaCM(node):
return self.success()
else:
return self.failure("Startup %s on node %s failed"
% (self.Env["Name"], node))
#
# We don't register StartTest because it's better when called by
# another test...
#
class FlipTest(CTSTest):
'''If it's running, stop it. If it's stopped start it.
Overthrow the status quo...
'''
def __init__(self, cm):
CTSTest.__init__(self,cm)
self.name = "Flip"
self.start = StartTest(cm)
self.stop = StopTest(cm)
def __call__(self, node):
'''Perform the 'Flip' test. '''
self.incr("calls")
if self.CM.ShouldBeStatus[node] == "up":
self.incr("stopped")
ret = self.stop(node)
type = "up->down"
# Give the cluster time to recognize it's gone...
time.sleep(self.Env["StableTime"])
elif self.CM.ShouldBeStatus[node] == "down":
self.incr("started")
ret = self.start(node)
type = "down->up"
else:
return self.skipped()
self.incr(type)
if ret:
return self.success()
else:
return self.failure("%s failure" % type)
# Register FlipTest as a good test to run
AllTestClasses.append(FlipTest)
class RestartTest(CTSTest):
'''Stop and restart a node'''
def __init__(self, cm):
CTSTest.__init__(self,cm)
self.name = "Restart"
self.start = StartTest(cm)
self.stop = StopTest(cm)
self.benchmark = 1
def __call__(self, node):
'''Perform the 'restart' test. '''
self.incr("calls")
self.incr("node:" + node)
ret1 = 1
if self.CM.StataCM(node):
self.incr("WasStopped")
if not self.start(node):
return self.failure("start (setup) failure: "+node)
self.set_timer()
if not self.stop(node):
return self.failure("stop failure: "+node)
if not self.start(node):
return self.failure("start failure: "+node)
return self.success()
# Register RestartTest as a good test to run
AllTestClasses.append(RestartTest)
class StonithdTest(CTSTest):
def __init__(self, cm):
CTSTest.__init__(self, cm)
self.name = "Stonithd"
self.startall = SimulStartLite(cm)
self.benchmark = 1
def __call__(self, node):
self.incr("calls")
if len(self.Env["nodes"]) < 2:
return self.skipped()
ret = self.startall(None)
if not ret:
return self.failure("Setup failed")
is_dc = self.CM.is_node_dc(node)
watchpats = []
watchpats.append(self.templates["Pat:Fencing_ok"] % node)
watchpats.append(self.templates["Pat:NodeFenced"] % node)
if self.Env["at-boot"] == 0:
self.debug("Expecting %s to stay down" % node)
self.CM.ShouldBeStatus[node] = "down"
else:
self.debug("Expecting %s to come up again %d" % (node, self.Env["at-boot"]))
watchpats.append("%s.* S_STARTING -> S_PENDING" % node)
watchpats.append("%s.* S_PENDING -> S_NOT_DC" % node)
watch = self.create_watch(watchpats, 30 + self.Env["DeadTime"] + self.Env["StableTime"] + self.Env["StartTime"])
watch.setwatch()
origin = self.Env.RandomGen.choice(self.Env["nodes"])
rc = self.rsh(origin, "stonith_admin --reboot %s -VVVVVV" % node)
if rc == 194:
# 194 - 256 = -62 = Timer expired
#
# Look for the patterns, usually this means the required
# device was running on the node to be fenced - or that
# the required devices were in the process of being loaded
# and/or moved
#
# Effectively the node committed suicide so there will be
# no confirmation, but pacemaker should be watching and
# fence the node again
self.logger.log("Fencing command on %s to fence %s timed out" % (origin, node))
elif origin != node and rc != 0:
self.debug("Waiting for the cluster to recover")
self.CM.cluster_stable()
self.debug("Waiting for fenced node to come back up")
self.CM.ns.WaitForAllNodesToComeUp(self.Env["nodes"], 600)
self.logger.log("Fencing command on %s failed to fence %s (rc=%d)" % (origin, node, rc))
elif origin == node and rc != 255:
# 255 == broken pipe, ie. the node was fenced as expected
self.logger.log("Locally originated fencing returned %d" % rc)
self.set_timer("fence")
matched = watch.lookforall()
self.log_timer("fence")
self.set_timer("reform")
if watch.unmatched:
self.logger.log("Patterns not found: " + repr(watch.unmatched))
self.debug("Waiting for the cluster to recover")
self.CM.cluster_stable()
self.debug("Waiting for fenced node to come back up")
self.CM.ns.WaitForAllNodesToComeUp(self.Env["nodes"], 600)
self.debug("Waiting for the cluster to re-stabilize with all nodes")
is_stable = self.CM.cluster_stable(self.Env["StartTime"])
if not matched:
return self.failure("Didn't find all expected patterns")
elif not is_stable:
return self.failure("Cluster did not become stable")
self.log_timer("reform")
return self.success()
def errorstoignore(self):
return [
self.templates["Pat:Fencing_start"] % ".*",
self.templates["Pat:Fencing_ok"] % ".*",
self.templates["Pat:Fencing_active"],
r"error.*: Operation 'reboot' targeting .* by .* for stonith_admin.*: Timer expired",
]
def is_applicable(self):
if not self.is_applicable_common():
return 0
if "DoFencing" in list(self.Env.keys()):
return self.Env["DoFencing"]
return 1
AllTestClasses.append(StonithdTest)
class StartOnebyOne(CTSTest):
'''Start all the nodes ~ one by one'''
def __init__(self, cm):
CTSTest.__init__(self,cm)
self.name = "StartOnebyOne"
self.stopall = SimulStopLite(cm)
self.start = StartTest(cm)
self.ns = CTS.NodeStatus(cm.Env)
def __call__(self, dummy):
'''Perform the 'StartOnebyOne' test. '''
self.incr("calls")
# We ignore the "node" parameter...
# Shut down all the nodes...
ret = self.stopall(None)
if not ret:
return self.failure("Test setup failed")
failed = []
self.set_timer()
for node in self.Env["nodes"]:
if not self.start(node):
failed.append(node)
if len(failed) > 0:
return self.failure("Some node failed to start: " + repr(failed))
return self.success()
# Register StartOnebyOne as a good test to run
AllTestClasses.append(StartOnebyOne)
class SimulStart(CTSTest):
'''Start all the nodes ~ simultaneously'''
def __init__(self, cm):
CTSTest.__init__(self,cm)
self.name = "SimulStart"
self.stopall = SimulStopLite(cm)
self.startall = SimulStartLite(cm)
def __call__(self, dummy):
'''Perform the 'SimulStart' test. '''
self.incr("calls")
# We ignore the "node" parameter...
# Shut down all the nodes...
ret = self.stopall(None)
if not ret:
return self.failure("Setup failed")
if not self.startall(None):
return self.failure("Startall failed")
return self.success()
# Register SimulStart as a good test to run
AllTestClasses.append(SimulStart)
class SimulStop(CTSTest):
'''Stop all the nodes ~ simultaneously'''
def __init__(self, cm):
CTSTest.__init__(self,cm)
self.name = "SimulStop"
self.startall = SimulStartLite(cm)
self.stopall = SimulStopLite(cm)
def __call__(self, dummy):
'''Perform the 'SimulStop' test. '''
self.incr("calls")
# We ignore the "node" parameter...
# Start up all the nodes...
ret = self.startall(None)
if not ret:
return self.failure("Setup failed")
if not self.stopall(None):
return self.failure("Stopall failed")
return self.success()
# Register SimulStop as a good test to run
AllTestClasses.append(SimulStop)
class StopOnebyOne(CTSTest):
'''Stop all the nodes in order'''
def __init__(self, cm):
CTSTest.__init__(self,cm)
self.name = "StopOnebyOne"
self.startall = SimulStartLite(cm)
self.stop = StopTest(cm)
def __call__(self, dummy):
'''Perform the 'StopOnebyOne' test. '''
self.incr("calls")
# We ignore the "node" parameter...
# Start up all the nodes...
ret = self.startall(None)
if not ret:
return self.failure("Setup failed")
failed = []
self.set_timer()
for node in self.Env["nodes"]:
if not self.stop(node):
failed.append(node)
if len(failed) > 0:
return self.failure("Some node failed to stop: " + repr(failed))
return self.success()
# Register StopOnebyOne as a good test to run
AllTestClasses.append(StopOnebyOne)
class RestartOnebyOne(CTSTest):
'''Restart all the nodes in order'''
def __init__(self, cm):
CTSTest.__init__(self,cm)
self.name = "RestartOnebyOne"
self.startall = SimulStartLite(cm)
def __call__(self, dummy):
'''Perform the 'RestartOnebyOne' test. '''
self.incr("calls")
# We ignore the "node" parameter...
# Start up all the nodes...
ret = self.startall(None)
if not ret:
return self.failure("Setup failed")
did_fail = []
self.set_timer()
self.restart = RestartTest(self.CM)
for node in self.Env["nodes"]:
if not self.restart(node):
did_fail.append(node)
if did_fail:
return self.failure("Could not restart %d nodes: %s"
% (len(did_fail), repr(did_fail)))
return self.success()
# Register StopOnebyOne as a good test to run
AllTestClasses.append(RestartOnebyOne)
class PartialStart(CTSTest):
'''Start a node - but tell it to stop before it finishes starting up'''
def __init__(self, cm):
CTSTest.__init__(self,cm)
self.name = "PartialStart"
self.startall = SimulStartLite(cm)
self.stopall = SimulStopLite(cm)
self.stop = StopTest(cm)
#self.is_unsafe = 1
def __call__(self, node):
'''Perform the 'PartialStart' test. '''
self.incr("calls")
ret = self.stopall(None)
if not ret:
return self.failure("Setup failed")
watchpats = []
watchpats.append("pacemaker-controld.*Connecting to .* cluster infrastructure")
watch = self.create_watch(watchpats, self.Env["DeadTime"]+10)
watch.setwatch()
self.CM.StartaCMnoBlock(node)
ret = watch.lookforall()
if not ret:
self.logger.log("Patterns not found: " + repr(watch.unmatched))
return self.failure("Setup of %s failed" % node)
ret = self.stop(node)
if not ret:
return self.failure("%s did not stop in time" % node)
return self.success()
def errorstoignore(self):
'''Return list of errors which should be ignored'''
# We might do some fencing in the 2-node case if we make it up far enough
return [
r"Executing reboot fencing operation",
r"Requesting fencing \([^)]+\) of node ",
]
# Register StopOnebyOne as a good test to run
AllTestClasses.append(PartialStart)
class StandbyTest(CTSTest):
def __init__(self, cm):
CTSTest.__init__(self,cm)
self.name = "Standby"
self.benchmark = 1
self.start = StartTest(cm)
self.startall = SimulStartLite(cm)
# make sure the node is active
# set the node to standby mode
# check resources, none resource should be running on the node
# set the node to active mode
# check resouces, resources should have been migrated back (SHOULD THEY?)
def __call__(self, node):
self.incr("calls")
ret = self.startall(None)
if not ret:
return self.failure("Start all nodes failed")
self.debug("Make sure node %s is active" % node)
if self.CM.StandbyStatus(node) != "off":
if not self.CM.SetStandbyMode(node, "off"):
return self.failure("can't set node %s to active mode" % node)
self.CM.cluster_stable()
status = self.CM.StandbyStatus(node)
if status != "off":
return self.failure("standby status of %s is [%s] but we expect [off]" % (node, status))
self.debug("Getting resources running on node %s" % node)
rsc_on_node = self.CM.active_resources(node)
watchpats = []
watchpats.append(r"State transition .* -> S_POLICY_ENGINE")
watch = self.create_watch(watchpats, self.Env["DeadTime"]+10)
watch.setwatch()
self.debug("Setting node %s to standby mode" % node)
if not self.CM.SetStandbyMode(node, "on"):
return self.failure("can't set node %s to standby mode" % node)
self.set_timer("on")
ret = watch.lookforall()
if not ret:
self.logger.log("Patterns not found: " + repr(watch.unmatched))
self.CM.SetStandbyMode(node, "off")
return self.failure("cluster didn't react to standby change on %s" % node)
self.CM.cluster_stable()
status = self.CM.StandbyStatus(node)
if status != "on":
return self.failure("standby status of %s is [%s] but we expect [on]" % (node, status))
self.log_timer("on")
self.debug("Checking resources")
bad_run = self.CM.active_resources(node)
if len(bad_run) > 0:
rc = self.failure("%s set to standby, %s is still running on it" % (node, repr(bad_run)))
self.debug("Setting node %s to active mode" % node)
self.CM.SetStandbyMode(node, "off")
return rc
self.debug("Setting node %s to active mode" % node)
if not self.CM.SetStandbyMode(node, "off"):
return self.failure("can't set node %s to active mode" % node)
self.set_timer("off")
self.CM.cluster_stable()
status = self.CM.StandbyStatus(node)
if status != "off":
return self.failure("standby status of %s is [%s] but we expect [off]" % (node, status))
self.log_timer("off")
return self.success()
AllTestClasses.append(StandbyTest)
class ValgrindTest(CTSTest):
'''Check for memory leaks'''
def __init__(self, cm):
CTSTest.__init__(self,cm)
self.name = "Valgrind"
self.stopall = SimulStopLite(cm)
self.startall = SimulStartLite(cm)
self.is_valgrind = 1
self.is_loop = 1
def setup(self, node):
self.incr("calls")
ret = self.stopall(None)
if not ret:
return self.failure("Stop all nodes failed")
# @TODO Edit /etc/sysconfig/pacemaker on all nodes to enable valgrind,
# and clear any valgrind logs from previous runs. For now, we rely on
# the user to do this manually.
ret = self.startall(None)
if not ret:
return self.failure("Start all nodes failed")
return self.success()
def teardown(self, node):
# Return all nodes to normal
# @TODO Edit /etc/sysconfig/pacemaker on all nodes to disable valgrind
ret = self.stopall(None)
if not ret:
return self.failure("Stop all nodes failed")
return self.success()
def find_leaks(self):
# Check for leaks
# (no longer used but kept in case feature is restored)
leaked = []
self.stop = StopTest(self.CM)
for node in self.Env["nodes"]:
rc = self.stop(node)
if not rc:
self.failure("Couldn't shut down %s" % node)
rc = self.rsh(node, "grep -e indirectly.*lost:.*[1-9] -e definitely.*lost:.*[1-9] -e (ERROR|error).*SUMMARY:.*[1-9].*errors %s" % self.logger.logPat, 0)
if rc != 1:
leaked.append(node)
self.failure("Valgrind errors detected on %s" % node)
(rc, output) = self.rsh(node, "grep -e lost: -e SUMMARY: %s" % self.logger.logPat, None)
for line in output:
self.logger.log(line)
(rc, output) = self.rsh(node, "cat %s" % self.logger.logPat, None)
for line in output:
self.debug(line)
self.rsh(node, "rm -f %s" % self.logger.logPat, None)
return leaked
def __call__(self, node):
#leaked = self.find_leaks()
#if len(leaked) > 0:
# return self.failure("Nodes %s leaked" % repr(leaked))
return self.success()
def errorstoignore(self):
'''Return list of errors which should be ignored'''
return [
r"pacemaker-based.*: \*\*\*\*\*\*\*\*\*\*\*\*\*",
r"pacemaker-based.*: .* avoid confusing Valgrind",
r"HA_VALGRIND_ENABLED",
]
class StandbyLoopTest(ValgrindTest):
'''Check for memory leaks by putting a node in and out of standby for an hour'''
# @TODO This is not a useful test for memory leaks
def __init__(self, cm):
ValgrindTest.__init__(self,cm)
self.name = "StandbyLoop"
def __call__(self, node):
lpc = 0
delay = 2
failed = 0
done = time.time() + self.Env["loop-minutes"] * 60
while time.time() <= done and not failed:
lpc = lpc + 1
time.sleep(delay)
if not self.CM.SetStandbyMode(node, "on"):
self.failure("can't set node %s to standby mode" % node)
failed = lpc
time.sleep(delay)
if not self.CM.SetStandbyMode(node, "off"):
self.failure("can't set node %s to active mode" % node)
failed = lpc
leaked = self.find_leaks()
if failed:
return self.failure("Iteration %d failed" % failed)
elif len(leaked) > 0:
return self.failure("Nodes %s leaked" % repr(leaked))
return self.success()
#AllTestClasses.append(StandbyLoopTest)
class BandwidthTest(CTSTest):
# Tests should not be cluster-manager-specific
# If you need to find out cluster manager configuration to do this, then
# it should be added to the generic cluster manager API.
'''Test the bandwidth which the cluster uses'''
def __init__(self, cm):
CTSTest.__init__(self, cm)
self.name = "Bandwidth"
self.start = StartTest(cm)
self.__setitem__("min",0)
self.__setitem__("max",0)
self.__setitem__("totalbandwidth",0)
(handle, self.tempfile) = tempfile.mkstemp(".cts")
os.close(handle)
self.startall = SimulStartLite(cm)
def __call__(self, node):
'''Perform the Bandwidth test'''
self.incr("calls")
if self.CM.upcount() < 1:
return self.skipped()
Path = self.CM.InternalCommConfig()
if "ip" not in Path["mediatype"]:
return self.skipped()
port = Path["port"][0]
port = int(port)
ret = self.startall(None)
if not ret:
return self.failure("Test setup failed")
time.sleep(5) # We get extra messages right after startup.
fstmpfile = "/var/run/band_estimate"
dumpcmd = "tcpdump -p -n -c 102 -i any udp port %d > %s 2>&1" \
% (port, fstmpfile)
rc = self.rsh(node, dumpcmd)
if rc == 0:
farfile = "root@%s:%s" % (node, fstmpfile)
self.rsh.cp(farfile, self.tempfile)
Bandwidth = self.countbandwidth(self.tempfile)
if not Bandwidth:
self.logger.log("Could not compute bandwidth.")
return self.success()
intband = int(Bandwidth + 0.5)
self.logger.log("...bandwidth: %d bits/sec" % intband)
self.Stats["totalbandwidth"] = self.Stats["totalbandwidth"] + Bandwidth
if self.Stats["min"] == 0:
self.Stats["min"] = Bandwidth
if Bandwidth > self.Stats["max"]:
self.Stats["max"] = Bandwidth
if Bandwidth < self.Stats["min"]:
self.Stats["min"] = Bandwidth
self.rsh(node, "rm -f %s" % fstmpfile)
os.unlink(self.tempfile)
return self.success()
else:
return self.failure("no response from tcpdump command [%d]!" % rc)
def countbandwidth(self, file):
fp = open(file, "r")
fp.seek(0)
count = 0
sum = 0
while 1:
line = fp.readline()
if not line:
return None
if re.search("udp",line) or re.search("UDP,", line):
count = count + 1
linesplit = line.split(" ")
for j in range(len(linesplit)-1):
if linesplit[j] == "udp": break
if linesplit[j] == "length:": break
try:
sum = sum + int(linesplit[j+1])
except ValueError:
self.logger.log("Invalid tcpdump line: %s" % line)
return None
T1 = linesplit[0]
timesplit = T1.split(":")
time2split = timesplit[2].split(".")
time1 = (int(timesplit[0])*60+int(timesplit[1]))*60+int(time2split[0])+int(time2split[1])*0.000001
break
while count < 100:
line = fp.readline()
if not line:
return None
if re.search("udp",line) or re.search("UDP,", line):
count = count+1
linessplit = line.split(" ")
for j in range(len(linessplit)-1):
if linessplit[j] == "udp": break
if linessplit[j] == "length:": break
try:
sum = int(linessplit[j+1]) + sum
except ValueError:
self.logger.log("Invalid tcpdump line: %s" % line)
return None
T2 = linessplit[0]
timesplit = T2.split(":")
time2split = timesplit[2].split(".")
time2 = (int(timesplit[0])*60+int(timesplit[1]))*60+int(time2split[0])+int(time2split[1])*0.000001
time = time2-time1
if (time <= 0):
return 0
return int((sum*8)/time)
def is_applicable(self):
'''BandwidthTest never applicable'''
return 0
AllTestClasses.append(BandwidthTest)
###################################################################
class MaintenanceMode(CTSTest):
###################################################################
def __init__(self, cm):
CTSTest.__init__(self,cm)
self.name = "MaintenanceMode"
self.start = StartTest(cm)
self.startall = SimulStartLite(cm)
self.max = 30
#self.is_unsafe = 1
self.benchmark = 1
self.action = "asyncmon"
self.interval = 0
self.rid = "maintenanceDummy"
def toggleMaintenanceMode(self, node, action):
pats = []
pats.append(self.templates["Pat:DC_IDLE"])
# fail the resource right after turning Maintenance mode on
# verify it is not recovered until maintenance mode is turned off
if action == "On":
pats.append(self.templates["Pat:RscOpFail"] % (self.action, self.rid))
else:
pats.append(self.templates["Pat:RscOpOK"] % ("stop", self.rid))
pats.append(self.templates["Pat:RscOpOK"] % ("start", self.rid))
watch = self.create_watch(pats, 60)
watch.setwatch()
self.debug("Turning maintenance mode %s" % action)
self.rsh(node, self.templates["MaintenanceMode%s" % (action)])
if (action == "On"):
self.rsh(node, "crm_resource -V -F -r %s -H %s &>/dev/null" % (self.rid, node))
self.set_timer("recover%s" % (action))
watch.lookforall()
self.log_timer("recover%s" % (action))
if watch.unmatched:
self.debug("Failed to find patterns when turning maintenance mode %s" % action)
return repr(watch.unmatched)
return ""
def insertMaintenanceDummy(self, node):
pats = []
pats.append(("%s.*" % node) + (self.templates["Pat:RscOpOK"] % ("start", self.rid)))
watch = self.create_watch(pats, 60)
watch.setwatch()
self.CM.AddDummyRsc(node, self.rid)
self.set_timer("addDummy")
watch.lookforall()
self.log_timer("addDummy")
if watch.unmatched:
self.debug("Failed to find patterns when adding maintenance dummy resource")
return repr(watch.unmatched)
return ""
def removeMaintenanceDummy(self, node):
pats = []
pats.append(self.templates["Pat:RscOpOK"] % ("stop", self.rid))
watch = self.create_watch(pats, 60)
watch.setwatch()
self.CM.RemoveDummyRsc(node, self.rid)
self.set_timer("removeDummy")
watch.lookforall()
self.log_timer("removeDummy")
if watch.unmatched:
self.debug("Failed to find patterns when removing maintenance dummy resource")
return repr(watch.unmatched)
return ""
def managedRscList(self, node):
rscList = []
(rc, lines) = self.rsh(node, "crm_resource -c", None)
for line in lines:
if re.search("^Resource", line):
tmp = AuditResource(self.CM, line)
if tmp.managed():
rscList.append(tmp.id)
return rscList
def verifyResources(self, node, rscList, managed):
managedList = list(rscList)
managed_str = "managed"
if not managed:
managed_str = "unmanaged"
(rc, lines) = self.rsh(node, "crm_resource -c", None)
for line in lines:
if re.search("^Resource", line):
tmp = AuditResource(self.CM, line)
if managed and not tmp.managed():
continue
elif not managed and tmp.managed():
continue
elif managedList.count(tmp.id):
managedList.remove(tmp.id)
if len(managedList) == 0:
self.debug("Found all %s resources on %s" % (managed_str, node))
return True
self.logger.log("Could not find all %s resources on %s. %s" % (managed_str, node, managedList))
return False
def __call__(self, node):
'''Perform the 'MaintenanceMode' test. '''
self.incr("calls")
verify_managed = False
verify_unmanaged = False
failPat = ""
ret = self.startall(None)
if not ret:
return self.failure("Setup failed")
# get a list of all the managed resources. We use this list
# after enabling maintenance mode to verify all managed resources
# become un-managed. After maintenance mode is turned off, we use
# this list to verify all the resources become managed again.
managedResources = self.managedRscList(node)
if len(managedResources) == 0:
self.logger.log("No managed resources on %s" % node)
return self.skipped()
# insert a fake resource we can fail during maintenance mode
# so we can verify recovery does not take place until after maintenance
# mode is disabled.
failPat = failPat + self.insertMaintenanceDummy(node)
# toggle maintenance mode ON, then fail dummy resource.
failPat = failPat + self.toggleMaintenanceMode(node, "On")
# verify all the resources are now unmanaged
if self.verifyResources(node, managedResources, False):
verify_unmanaged = True
# Toggle maintenance mode OFF, verify dummy is recovered.
failPat = failPat + self.toggleMaintenanceMode(node, "Off")
# verify all the resources are now managed again
if self.verifyResources(node, managedResources, True):
verify_managed = True
# Remove our maintenance dummy resource.
failPat = failPat + self.removeMaintenanceDummy(node)
self.CM.cluster_stable()
if failPat != "":
return self.failure("Unmatched patterns: %s" % (failPat))
elif verify_unmanaged is False:
return self.failure("Failed to verify resources became unmanaged during maintenance mode")
elif verify_managed is False:
return self.failure("Failed to verify resources switched back to managed after disabling maintenance mode")
return self.success()
def errorstoignore(self):
'''Return list of errors which should be ignored'''
return [
r"Updating failcount for %s" % self.rid,
- r"schedulerd.*: Recover %s\s*\(.*\)" % self.rid,
+ r"schedulerd.*: Recover\s+%s\s+\(.*\)" % self.rid,
r"Unknown operation: fail",
self.templates["Pat:RscOpOK"] % (self.action, self.rid),
r"(ERROR|error).*: Action %s_%s_%d .* initiated outside of a transition" % (self.rid, self.action, self.interval),
]
AllTestClasses.append(MaintenanceMode)
class ResourceRecover(CTSTest):
def __init__(self, cm):
CTSTest.__init__(self,cm)
self.name = "ResourceRecover"
self.start = StartTest(cm)
self.startall = SimulStartLite(cm)
self.max = 30
self.rid = None
self.rid_alt = None
#self.is_unsafe = 1
self.benchmark = 1
# these are the values used for the new LRM API call
self.action = "asyncmon"
self.interval = 0
def __call__(self, node):
'''Perform the 'ResourceRecover' test. '''
self.incr("calls")
ret = self.startall(None)
if not ret:
return self.failure("Setup failed")
resourcelist = self.CM.active_resources(node)
# if there are no resourcelist, return directly
if len(resourcelist) == 0:
self.logger.log("No active resources on %s" % node)
return self.skipped()
self.rid = self.Env.RandomGen.choice(resourcelist)
self.rid_alt = self.rid
rsc = None
(rc, lines) = self.rsh(node, "crm_resource -c", None)
for line in lines:
if re.search("^Resource", line):
tmp = AuditResource(self.CM, line)
if tmp.id == self.rid:
rsc = tmp
# Handle anonymous clones that get renamed
self.rid = rsc.clone_id
break
if not rsc:
return self.failure("Could not find %s in the resource list" % self.rid)
self.debug("Shooting %s aka. %s" % (rsc.clone_id, rsc.id))
pats = []
pats.append(self.templates["Pat:CloneOpFail"] % (self.action, rsc.id, rsc.clone_id))
if rsc.managed():
pats.append(self.templates["Pat:RscOpOK"] % ("stop", self.rid))
if rsc.unique():
pats.append(self.templates["Pat:RscOpOK"] % ("start", self.rid))
else:
# Anonymous clones may get restarted with a different clone number
pats.append(self.templates["Pat:RscOpOK"] % ("start", ".*"))
watch = self.create_watch(pats, 60)
watch.setwatch()
self.rsh(node, "crm_resource -V -F -r %s -H %s &>/dev/null" % (self.rid, node))
self.set_timer("recover")
watch.lookforall()
self.log_timer("recover")
self.CM.cluster_stable()
recovered = self.CM.ResourceLocation(self.rid)
if watch.unmatched:
return self.failure("Patterns not found: %s" % repr(watch.unmatched))
elif rsc.unique() and len(recovered) > 1:
return self.failure("%s is now active on more than one node: %s"%(self.rid, repr(recovered)))
elif len(recovered) > 0:
self.debug("%s is running on: %s" % (self.rid, repr(recovered)))
elif rsc.managed():
return self.failure("%s was not recovered and is inactive" % self.rid)
return self.success()
def errorstoignore(self):
'''Return list of errors which should be ignored'''
return [
r"Updating failcount for %s" % self.rid,
- r"schedulerd.*: Recover (%s|%s)\s*\(.*\)" % (self.rid, self.rid_alt),
+ r"schedulerd.*: Recover\s+(%s|%s)\s+\(.*\)" % (self.rid, self.rid_alt),
r"Unknown operation: fail",
self.templates["Pat:RscOpOK"] % (self.action, self.rid),
r"(ERROR|error).*: Action %s_%s_%d .* initiated outside of a transition" % (self.rid, self.action, self.interval),
]
AllTestClasses.append(ResourceRecover)
class ComponentFail(CTSTest):
def __init__(self, cm):
CTSTest.__init__(self,cm)
self.name = "ComponentFail"
self.startall = SimulStartLite(cm)
self.complist = cm.Components()
self.patterns = []
self.okerrpatterns = []
self.is_unsafe = 1
def __call__(self, node):
'''Perform the 'ComponentFail' test. '''
self.incr("calls")
self.patterns = []
self.okerrpatterns = []
# start all nodes
ret = self.startall(None)
if not ret:
return self.failure("Setup failed")
if not self.CM.cluster_stable(self.Env["StableTime"]):
return self.failure("Setup failed - unstable")
node_is_dc = self.CM.is_node_dc(node, None)
# select a component to kill
chosen = self.Env.RandomGen.choice(self.complist)
while chosen.dc_only == 1 and node_is_dc == 0:
chosen = self.Env.RandomGen.choice(self.complist)
self.debug("...component %s (dc=%d,boot=%d)" % (chosen.name, node_is_dc,chosen.triggersreboot))
self.incr(chosen.name)
if chosen.name != "corosync":
self.patterns.append(self.templates["Pat:ChildKilled"] %(node, chosen.name))
self.patterns.append(self.templates["Pat:ChildRespawn"] %(node, chosen.name))
self.patterns.extend(chosen.pats)
if node_is_dc:
self.patterns.extend(chosen.dc_pats)
# @TODO this should be a flag in the Component
if chosen.name in [ "corosync", "pacemaker-based", "pacemaker-fenced" ]:
# Ignore actions for fence devices if fencer will respawn
# (their registration will be lost, and probes will fail)
self.okerrpatterns = [ self.templates["Pat:Fencing_active"] ]
(rc, lines) = self.rsh(node, "crm_resource -c", None)
for line in lines:
if re.search("^Resource", line):
r = AuditResource(self.CM, line)
if r.rclass == "stonith":
self.okerrpatterns.append(self.templates["Pat:Fencing_recover"] % r.id)
self.okerrpatterns.append(self.templates["Pat:Fencing_probe"] % r.id)
# supply a copy so self.patterns doesn't end up empty
tmpPats = []
tmpPats.extend(self.patterns)
self.patterns.extend(chosen.badnews_ignore)
# Look for STONITH ops, depending on Env["at-boot"] we might need to change the nodes status
stonithPats = []
stonithPats.append(self.templates["Pat:Fencing_ok"] % node)
stonith = self.create_watch(stonithPats, 0)
stonith.setwatch()
# set the watch for stable
watch = self.create_watch(
tmpPats, self.Env["DeadTime"] + self.Env["StableTime"] + self.Env["StartTime"])
watch.setwatch()
# kill the component
chosen.kill(node)
self.debug("Waiting for the cluster to recover")
self.CM.cluster_stable()
self.debug("Waiting for any fenced node to come back up")
self.CM.ns.WaitForAllNodesToComeUp(self.Env["nodes"], 600)
self.debug("Waiting for the cluster to re-stabilize with all nodes")
self.CM.cluster_stable(self.Env["StartTime"])
self.debug("Checking if %s was shot" % node)
shot = stonith.look(60)
if shot:
self.debug("Found: " + repr(shot))
self.okerrpatterns.append(self.templates["Pat:Fencing_start"] % node)
if self.Env["at-boot"] == 0:
self.CM.ShouldBeStatus[node] = "down"
# If fencing occurred, chances are many (if not all) the expected logs
# will not be sent - or will be lost when the node reboots
return self.success()
# check for logs indicating a graceful recovery
matched = watch.lookforall(allow_multiple_matches=1)
if watch.unmatched:
self.logger.log("Patterns not found: " + repr(watch.unmatched))
self.debug("Waiting for the cluster to re-stabilize with all nodes")
is_stable = self.CM.cluster_stable(self.Env["StartTime"])
if not matched:
return self.failure("Didn't find all expected %s patterns" % chosen.name)
elif not is_stable:
return self.failure("Cluster did not become stable after killing %s" % chosen.name)
return self.success()
def errorstoignore(self):
'''Return list of errors which should be ignored'''
# Note that okerrpatterns refers to the last time we ran this test
# The good news is that this works fine for us...
self.okerrpatterns.extend(self.patterns)
return self.okerrpatterns
AllTestClasses.append(ComponentFail)
class SplitBrainTest(CTSTest):
'''It is used to test split-brain. when the path between the two nodes break
check the two nodes both take over the resource'''
def __init__(self,cm):
CTSTest.__init__(self,cm)
self.name = "SplitBrain"
self.start = StartTest(cm)
self.startall = SimulStartLite(cm)
self.is_experimental = 1
def isolate_partition(self, partition):
other_nodes = []
other_nodes.extend(self.Env["nodes"])
for node in partition:
try:
other_nodes.remove(node)
except ValueError:
self.logger.log("Node "+node+" not in " + repr(self.Env["nodes"]) + " from " +repr(partition))
if len(other_nodes) == 0:
return 1
self.debug("Creating partition: " + repr(partition))
self.debug("Everyone else: " + repr(other_nodes))
for node in partition:
if not self.CM.isolate_node(node, other_nodes):
self.logger.log("Could not isolate %s" % node)
return 0
return 1
def heal_partition(self, partition):
other_nodes = []
other_nodes.extend(self.Env["nodes"])
for node in partition:
try:
other_nodes.remove(node)
except ValueError:
self.logger.log("Node "+node+" not in " + repr(self.Env["nodes"]))
if len(other_nodes) == 0:
return 1
self.debug("Healing partition: " + repr(partition))
self.debug("Everyone else: " + repr(other_nodes))
for node in partition:
self.CM.unisolate_node(node, other_nodes)
def __call__(self, node):
'''Perform split-brain test'''
self.incr("calls")
self.passed = 1
partitions = {}
ret = self.startall(None)
if not ret:
return self.failure("Setup failed")
while 1:
# Retry until we get multiple partitions
partitions = {}
p_max = len(self.Env["nodes"])
for node in self.Env["nodes"]:
p = self.Env.RandomGen.randint(1, p_max)
if not p in partitions:
partitions[p] = []
partitions[p].append(node)
p_max = len(list(partitions.keys()))
if p_max > 1:
break
# else, try again
self.debug("Created %d partitions" % p_max)
for key in list(partitions.keys()):
self.debug("Partition["+str(key)+"]:\t"+repr(partitions[key]))
# Disabling STONITH to reduce test complexity for now
self.rsh(node, "crm_attribute -V -n stonith-enabled -v false")
for key in list(partitions.keys()):
self.isolate_partition(partitions[key])
count = 30
while count > 0:
if len(self.CM.find_partitions()) != p_max:
time.sleep(10)
else:
break
else:
self.failure("Expected partitions were not created")
# Target number of partitions formed - wait for stability
if not self.CM.cluster_stable():
self.failure("Partitioned cluster not stable")
# Now audit the cluster state
self.CM.partitions_expected = p_max
if not self.audit():
self.failure("Audits failed")
self.CM.partitions_expected = 1
# And heal them again
for key in list(partitions.keys()):
self.heal_partition(partitions[key])
# Wait for a single partition to form
count = 30
while count > 0:
if len(self.CM.find_partitions()) != 1:
time.sleep(10)
count -= 1
else:
break
else:
self.failure("Cluster did not reform")
# Wait for it to have the right number of members
count = 30
while count > 0:
members = []
partitions = self.CM.find_partitions()
if len(partitions) > 0:
members = partitions[0].split()
if len(members) != len(self.Env["nodes"]):
time.sleep(10)
count -= 1
else:
break
else:
self.failure("Cluster did not completely reform")
# Wait up to 20 minutes - the delay is more preferable than
# trying to continue with in a messed up state
if not self.CM.cluster_stable(1200):
self.failure("Reformed cluster not stable")
if self.Env["continue"] == 1:
answer = "Y"
else:
try:
answer = input('Continue? [nY]')
except EOFError as e:
answer = "n"
if answer and answer == "n":
raise ValueError("Reformed cluster not stable")
# Turn fencing back on
if self.Env["DoFencing"]:
self.rsh(node, "crm_attribute -V -D -n stonith-enabled")
self.CM.cluster_stable()
if self.passed:
return self.success()
return self.failure("See previous errors")
def errorstoignore(self):
'''Return list of errors which are 'normal' and should be ignored'''
return [
r"Another DC detected:",
r"(ERROR|error).*: .*Application of an update diff failed",
r"pacemaker-controld.*:.*not in our membership list",
r"CRIT:.*node.*returning after partition",
]
def is_applicable(self):
if not self.is_applicable_common():
return 0
return len(self.Env["nodes"]) > 2
AllTestClasses.append(SplitBrainTest)
class Reattach(CTSTest):
def __init__(self, cm):
CTSTest.__init__(self,cm)
self.name = "Reattach"
self.startall = SimulStartLite(cm)
self.restart1 = RestartTest(cm)
self.stopall = SimulStopLite(cm)
self.is_unsafe = 0 # Handled by canrunnow()
def _is_managed(self, node):
is_managed = self.rsh(node, "crm_attribute -t rsc_defaults -n is-managed -q -G -d true", 1)
is_managed = is_managed[:-1] # Strip off the newline
return is_managed == "true"
def _set_unmanaged(self, node):
self.debug("Disable resource management")
self.rsh(node, "crm_attribute -t rsc_defaults -n is-managed -v false")
def _set_managed(self, node):
self.debug("Re-enable resource management")
self.rsh(node, "crm_attribute -t rsc_defaults -n is-managed -D")
def setup(self, node):
attempt = 0
if not self.startall(None):
return None
# Make sure we are really _really_ stable and that all
# resources, including those that depend on transient node
# attributes, are started
while not self.CM.cluster_stable(double_check=True):
if attempt < 5:
attempt += 1
self.debug("Not stable yet, re-testing")
else:
self.logger.log("Cluster is not stable")
return None
return 1
def teardown(self, node):
# Make sure 'node' is up
start = StartTest(self.CM)
start(node)
if not self._is_managed(node):
self.logger.log("Attempting to re-enable resource management on %s" % node)
self._set_managed(node)
self.CM.cluster_stable()
if not self._is_managed(node):
self.logger.log("Could not re-enable resource management")
return 0
return 1
def canrunnow(self, node):
'''Return TRUE if we can meaningfully run right now'''
if self.find_ocfs2_resources(node):
self.logger.log("Detach/Reattach scenarios are not possible with OCFS2 services present")
return 0
return 1
def __call__(self, node):
self.incr("calls")
pats = []
# Conveniently, the scheduler will display this message when disabling
# management, even if fencing is not enabled, so we can rely on it.
managed = self.create_watch(["No fencing will be done"], 60)
managed.setwatch()
self._set_unmanaged(node)
if not managed.lookforall():
self.logger.log("Patterns not found: " + repr(managed.unmatched))
return self.failure("Resource management not disabled")
pats = []
pats.append(self.templates["Pat:RscOpOK"] % ("start", ".*"))
pats.append(self.templates["Pat:RscOpOK"] % ("stop", ".*"))
pats.append(self.templates["Pat:RscOpOK"] % ("promote", ".*"))
pats.append(self.templates["Pat:RscOpOK"] % ("demote", ".*"))
pats.append(self.templates["Pat:RscOpOK"] % ("migrate", ".*"))
watch = self.create_watch(pats, 60, "ShutdownActivity")
watch.setwatch()
self.debug("Shutting down the cluster")
ret = self.stopall(None)
if not ret:
self._set_managed(node)
return self.failure("Couldn't shut down the cluster")
self.debug("Bringing the cluster back up")
ret = self.startall(None)
time.sleep(5) # allow ping to update the CIB
if not ret:
self._set_managed(node)
return self.failure("Couldn't restart the cluster")
if self.local_badnews("ResourceActivity:", watch):
self._set_managed(node)
return self.failure("Resources stopped or started during cluster restart")
watch = self.create_watch(pats, 60, "StartupActivity")
watch.setwatch()
# Re-enable resource management (and verify it happened).
self._set_managed(node)
self.CM.cluster_stable()
if not self._is_managed(node):
return self.failure("Could not re-enable resource management")
# Ignore actions for STONITH resources
ignore = []
(rc, lines) = self.rsh(node, "crm_resource -c", None)
for line in lines:
if re.search("^Resource", line):
r = AuditResource(self.CM, line)
if r.rclass == "stonith":
self.debug("Ignoring start actions for %s" % r.id)
ignore.append(self.templates["Pat:RscOpOK"] % ("start", r.id))
if self.local_badnews("ResourceActivity:", watch, ignore):
return self.failure("Resources stopped or started after resource management was re-enabled")
return ret
def errorstoignore(self):
'''Return list of errors which should be ignored'''
return [
r"resource( was|s were) active at shutdown",
]
def is_applicable(self):
return 1
AllTestClasses.append(Reattach)
class SpecialTest1(CTSTest):
'''Set up a custom test to cause quorum failure issues for Andrew'''
def __init__(self, cm):
CTSTest.__init__(self,cm)
self.name = "SpecialTest1"
self.startall = SimulStartLite(cm)
self.restart1 = RestartTest(cm)
self.stopall = SimulStopLite(cm)
def __call__(self, node):
'''Perform the 'SpecialTest1' test for Andrew. '''
self.incr("calls")
# Shut down all the nodes...
ret = self.stopall(None)
if not ret:
return self.failure("Could not stop all nodes")
# Test config recovery when the other nodes come up
self.rsh(node, "rm -f "+CTSvars.CRM_CONFIG_DIR+"/cib*")
# Start the selected node
ret = self.restart1(node)
if not ret:
return self.failure("Could not start "+node)
# Start all remaining nodes
ret = self.startall(None)
if not ret:
return self.failure("Could not start the remaining nodes")
return self.success()
def errorstoignore(self):
'''Return list of errors which should be ignored'''
# Errors that occur as a result of the CIB being wiped
return [
r"error.*: v1 patchset error, patch failed to apply: Application of an update diff failed",
r"error.*: Resource start-up disabled since no STONITH resources have been defined",
r"error.*: Either configure some or disable STONITH with the stonith-enabled option",
r"error.*: NOTE: Clusters with shared data need STONITH to ensure data integrity",
]
AllTestClasses.append(SpecialTest1)
class HAETest(CTSTest):
'''Set up a custom test to cause quorum failure issues for Andrew'''
def __init__(self, cm):
CTSTest.__init__(self,cm)
self.name = "HAETest"
self.stopall = SimulStopLite(cm)
self.startall = SimulStartLite(cm)
self.is_loop = 1
def setup(self, node):
# Start all remaining nodes
ret = self.startall(None)
if not ret:
return self.failure("Couldn't start all nodes")
return self.success()
def teardown(self, node):
# Stop everything
ret = self.stopall(None)
if not ret:
return self.failure("Couldn't stop all nodes")
return self.success()
def wait_on_state(self, node, resource, expected_clones, attempts=240):
while attempts > 0:
active = 0
(rc, lines) = self.rsh(node, "crm_resource -r %s -W -Q" % resource, stdout=None)
# Hack until crm_resource does the right thing
if rc == 0 and lines:
active = len(lines)
if len(lines) == expected_clones:
return 1
elif rc == 1:
self.debug("Resource %s is still inactive" % resource)
elif rc == 234:
self.logger.log("Unknown resource %s" % resource)
return 0
elif rc == 246:
self.logger.log("Cluster is inactive")
return 0
elif rc != 0:
self.logger.log("Call to crm_resource failed, rc=%d" % rc)
return 0
else:
self.debug("Resource %s is active on %d times instead of %d" % (resource, active, expected_clones))
attempts -= 1
time.sleep(1)
return 0
def find_dlm(self, node):
self.r_dlm = None
(rc, lines) = self.rsh(node, "crm_resource -c", None)
for line in lines:
if re.search("^Resource", line):
r = AuditResource(self.CM, line)
if r.rtype == "controld" and r.parent != "NA":
self.debug("Found dlm: %s" % self.r_dlm)
self.r_dlm = r.parent
return 1
return 0
def find_hae_resources(self, node):
self.r_dlm = None
self.r_o2cb = None
self.r_ocfs2 = []
if self.find_dlm(node):
self.find_ocfs2_resources(node)
def is_applicable(self):
if not self.is_applicable_common():
return 0
if self.Env["Schema"] == "hae":
return 1
return None
class HAERoleTest(HAETest):
def __init__(self, cm):
'''Lars' mount/unmount test for the HA extension. '''
HAETest.__init__(self,cm)
self.name = "HAERoleTest"
def change_state(self, node, resource, target):
rc = self.rsh(node, "crm_resource -V -r %s -p target-role -v %s --meta" % (resource, target))
return rc
def __call__(self, node):
self.incr("calls")
lpc = 0
failed = 0
delay = 2
done = time.time() + self.Env["loop-minutes"]*60
self.find_hae_resources(node)
clone_max = len(self.Env["nodes"])
while time.time() <= done and not failed:
lpc = lpc + 1
self.change_state(node, self.r_dlm, "Stopped")
if not self.wait_on_state(node, self.r_dlm, 0):
self.failure("%s did not go down correctly" % self.r_dlm)
failed = lpc
self.change_state(node, self.r_dlm, "Started")
if not self.wait_on_state(node, self.r_dlm, clone_max):
self.failure("%s did not come up correctly" % self.r_dlm)
failed = lpc
if not self.wait_on_state(node, self.r_o2cb, clone_max):
self.failure("%s did not come up correctly" % self.r_o2cb)
failed = lpc
for fs in self.r_ocfs2:
if not self.wait_on_state(node, fs, clone_max):
self.failure("%s did not come up correctly" % fs)
failed = lpc
if failed:
return self.failure("iteration %d failed" % failed)
return self.success()
AllTestClasses.append(HAERoleTest)
class HAEStandbyTest(HAETest):
'''Set up a custom test to cause quorum failure issues for Andrew'''
def __init__(self, cm):
HAETest.__init__(self,cm)
self.name = "HAEStandbyTest"
def change_state(self, node, resource, target):
rc = self.rsh(node, "crm_standby -V -l reboot -v %s" % (target))
return rc
def __call__(self, node):
self.incr("calls")
lpc = 0
failed = 0
done = time.time() + self.Env["loop-minutes"]*60
self.find_hae_resources(node)
clone_max = len(self.Env["nodes"])
while time.time() <= done and not failed:
lpc = lpc + 1
self.change_state(node, self.r_dlm, "true")
if not self.wait_on_state(node, self.r_dlm, clone_max-1):
self.failure("%s did not go down correctly" % self.r_dlm)
failed = lpc
self.change_state(node, self.r_dlm, "false")
if not self.wait_on_state(node, self.r_dlm, clone_max):
self.failure("%s did not come up correctly" % self.r_dlm)
failed = lpc
if not self.wait_on_state(node, self.r_o2cb, clone_max):
self.failure("%s did not come up correctly" % self.r_o2cb)
failed = lpc
for fs in self.r_ocfs2:
if not self.wait_on_state(node, fs, clone_max):
self.failure("%s did not come up correctly" % fs)
failed = lpc
if failed:
return self.failure("iteration %d failed" % failed)
return self.success()
AllTestClasses.append(HAEStandbyTest)
class NearQuorumPointTest(CTSTest):
'''
This test brings larger clusters near the quorum point (50%).
In addition, it will test doing starts and stops at the same time.
Here is how I think it should work:
- loop over the nodes and decide randomly which will be up and which
will be down Use a 50% probability for each of up/down.
- figure out what to do to get into that state from the current state
- in parallel, bring up those going up and bring those going down.
'''
def __init__(self, cm):
CTSTest.__init__(self,cm)
self.name = "NearQuorumPoint"
def __call__(self, dummy):
'''Perform the 'NearQuorumPoint' test. '''
self.incr("calls")
startset = []
stopset = []
stonith = self.CM.prepare_fencing_watcher("NearQuorumPoint")
#decide what to do with each node
for node in self.Env["nodes"]:
action = self.Env.RandomGen.choice(["start","stop"])
#action = self.Env.RandomGen.choice(["start","stop","no change"])
if action == "start" :
startset.append(node)
elif action == "stop" :
stopset.append(node)
self.debug("start nodes:" + repr(startset))
self.debug("stop nodes:" + repr(stopset))
#add search patterns
watchpats = [ ]
for node in stopset:
if self.CM.ShouldBeStatus[node] == "up":
watchpats.append(self.templates["Pat:We_stopped"] % node)
for node in startset:
if self.CM.ShouldBeStatus[node] == "down":
#watchpats.append(self.templates["Pat:NonDC_started"] % node)
watchpats.append(self.templates["Pat:Local_started"] % node)
else:
for stopping in stopset:
if self.CM.ShouldBeStatus[stopping] == "up":
watchpats.append(self.templates["Pat:They_stopped"] % (node, self.CM.key_for_node(stopping)))
if len(watchpats) == 0:
return self.skipped()
if len(startset) != 0:
watchpats.append(self.templates["Pat:DC_IDLE"])
watch = self.create_watch(watchpats, self.Env["DeadTime"]+10)
watch.setwatch()
#begin actions
for node in stopset:
if self.CM.ShouldBeStatus[node] == "up":
self.CM.StopaCMnoBlock(node)
for node in startset:
if self.CM.ShouldBeStatus[node] == "down":
self.CM.StartaCMnoBlock(node)
#get the result
if watch.lookforall():
self.CM.cluster_stable()
self.CM.fencing_cleanup("NearQuorumPoint", stonith)
return self.success()
self.logger.log("Warn: Patterns not found: " + repr(watch.unmatched))
#get the "bad" nodes
upnodes = []
for node in stopset:
if self.CM.StataCM(node) == 1:
upnodes.append(node)
downnodes = []
for node in startset:
if self.CM.StataCM(node) == 0:
downnodes.append(node)
self.CM.fencing_cleanup("NearQuorumPoint", stonith)
if upnodes == [] and downnodes == []:
self.CM.cluster_stable()
# Make sure they're completely down with no residule
for node in stopset:
self.rsh(node, self.templates["StopCmd"])
return self.success()
if len(upnodes) > 0:
self.logger.log("Warn: Unstoppable nodes: " + repr(upnodes))
if len(downnodes) > 0:
self.logger.log("Warn: Unstartable nodes: " + repr(downnodes))
return self.failure()
def is_applicable(self):
return 1
AllTestClasses.append(NearQuorumPointTest)
class RollingUpgradeTest(CTSTest):
'''Perform a rolling upgrade of the cluster'''
def __init__(self, cm):
CTSTest.__init__(self,cm)
self.name = "RollingUpgrade"
self.start = StartTest(cm)
self.stop = StopTest(cm)
self.stopall = SimulStopLite(cm)
self.startall = SimulStartLite(cm)
def setup(self, node):
# Start all remaining nodes
ret = self.stopall(None)
if not ret:
return self.failure("Couldn't stop all nodes")
for node in self.Env["nodes"]:
if not self.downgrade(node, None):
return self.failure("Couldn't downgrade %s" % node)
ret = self.startall(None)
if not ret:
return self.failure("Couldn't start all nodes")
return self.success()
def teardown(self, node):
# Stop everything
ret = self.stopall(None)
if not ret:
return self.failure("Couldn't stop all nodes")
for node in self.Env["nodes"]:
if not self.upgrade(node, None):
return self.failure("Couldn't upgrade %s" % node)
return self.success()
def install(self, node, version, start=1, flags="--force"):
target_dir = "/tmp/rpm-%s" % version
src_dir = "%s/%s" % (self.Env["rpm-dir"], version)
self.logger.log("Installing %s on %s with %s" % (version, node, flags))
if not self.stop(node):
return self.failure("stop failure: "+node)
rc = self.rsh(node, "mkdir -p %s" % target_dir)
rc = self.rsh(node, "rm -f %s/*.rpm" % target_dir)
(rc, lines) = self.rsh(node, "ls -1 %s/*.rpm" % src_dir, None)
for line in lines:
line = line[:-1]
rc = self.rsh.cp("%s" % (line), "%s:%s/" % (node, target_dir))
rc = self.rsh(node, "rpm -Uvh %s %s/*.rpm" % (flags, target_dir))
if start and not self.start(node):
return self.failure("start failure: "+node)
return self.success()
def upgrade(self, node, start=1):
return self.install(node, self.Env["current-version"], start)
def downgrade(self, node, start=1):
return self.install(node, self.Env["previous-version"], start, "--force --nodeps")
def __call__(self, node):
'''Perform the 'Rolling Upgrade' test. '''
self.incr("calls")
for node in self.Env["nodes"]:
if self.upgrade(node):
return self.failure("Couldn't upgrade %s" % node)
self.CM.cluster_stable()
return self.success()
def is_applicable(self):
if not self.is_applicable_common():
return None
if not "rpm-dir" in list(self.Env.keys()):
return None
if not "current-version" in list(self.Env.keys()):
return None
if not "previous-version" in list(self.Env.keys()):
return None
return 1
# Register RestartTest as a good test to run
AllTestClasses.append(RollingUpgradeTest)
class BSC_AddResource(CTSTest):
'''Add a resource to the cluster'''
def __init__(self, cm):
CTSTest.__init__(self, cm)
self.name = "AddResource"
self.resource_offset = 0
self.cib_cmd = """cibadmin -C -o %s -X '%s' """
def __call__(self, node):
self.incr("calls")
self.resource_offset = self.resource_offset + 1
r_id = "bsc-rsc-%s-%d" % (node, self.resource_offset)
start_pat = "pacemaker-controld.*%s_start_0.*confirmed.*ok"
patterns = []
patterns.append(start_pat % r_id)
watch = self.create_watch(patterns, self.Env["DeadTime"])
watch.setwatch()
ip = self.NextIP()
if not self.make_ip_resource(node, r_id, "ocf", "IPaddr", ip):
return self.failure("Make resource %s failed" % r_id)
failed = 0
watch_result = watch.lookforall()
if watch.unmatched:
for regex in watch.unmatched:
self.logger.log ("Warn: Pattern not found: %s" % (regex))
failed = 1
if failed:
return self.failure("Resource pattern(s) not found")
if not self.CM.cluster_stable(self.Env["DeadTime"]):
return self.failure("Unstable cluster")
return self.success()
def NextIP(self):
ip = self.Env["IPBase"]
if ":" in ip:
fields = ip.rpartition(":")
fields[2] = str(hex(int(fields[2], 16)+1))
print(str(hex(int(f[2], 16)+1)))
else:
fields = ip.rpartition('.')
fields[2] = str(int(fields[2])+1)
ip = fields[0] + fields[1] + fields[3];
self.Env["IPBase"] = ip
return ip.strip()
def make_ip_resource(self, node, id, rclass, type, ip):
self.logger.log("Creating %s:%s:%s (%s) on %s" % (rclass,type,id,ip,node))
rsc_xml="""
<primitive id="%s" class="%s" type="%s" provider="heartbeat">
<instance_attributes id="%s"><attributes>
<nvpair id="%s" name="ip" value="%s"/>
</attributes></instance_attributes>
</primitive>""" % (id, rclass, type, id, id, ip)
node_constraint = """
<rsc_location id="run_%s" rsc="%s">
<rule id="pref_run_%s" score="100">
<expression id="%s_loc_expr" attribute="#uname" operation="eq" value="%s"/>
</rule>
</rsc_location>""" % (id, id, id, id, node)
rc = 0
(rc, lines) = self.rsh(node, self.cib_cmd % ("constraints", node_constraint), None)
if rc != 0:
self.logger.log("Constraint creation failed: %d" % rc)
return None
(rc, lines) = self.rsh(node, self.cib_cmd % ("resources", rsc_xml), None)
if rc != 0:
self.logger.log("Resource creation failed: %d" % rc)
return None
return 1
def is_applicable(self):
if self.Env["DoBSC"]:
return 1
return None
AllTestClasses.append(BSC_AddResource)
class SimulStopLite(CTSTest):
'''Stop any active nodes ~ simultaneously'''
def __init__(self, cm):
CTSTest.__init__(self,cm)
self.name = "SimulStopLite"
def __call__(self, dummy):
'''Perform the 'SimulStopLite' setup work. '''
self.incr("calls")
self.debug("Setup: " + self.name)
# We ignore the "node" parameter...
watchpats = [ ]
for node in self.Env["nodes"]:
if self.CM.ShouldBeStatus[node] == "up":
self.incr("WasStarted")
watchpats.append(self.templates["Pat:We_stopped"] % node)
if len(watchpats) == 0:
return self.success()
# Stop all the nodes - at about the same time...
watch = self.create_watch(watchpats, self.Env["DeadTime"]+10)
watch.setwatch()
self.set_timer()
for node in self.Env["nodes"]:
if self.CM.ShouldBeStatus[node] == "up":
self.CM.StopaCMnoBlock(node)
if watch.lookforall():
# Make sure they're completely down with no residule
for node in self.Env["nodes"]:
self.rsh(node, self.templates["StopCmd"])
return self.success()
did_fail = 0
up_nodes = []
for node in self.Env["nodes"]:
if self.CM.StataCM(node) == 1:
did_fail = 1
up_nodes.append(node)
if did_fail:
return self.failure("Active nodes exist: " + repr(up_nodes))
self.logger.log("Warn: All nodes stopped but CTS didn't detect: "
+ repr(watch.unmatched))
return self.failure("Missing log message: "+repr(watch.unmatched))
def is_applicable(self):
'''SimulStopLite is a setup test and never applicable'''
return 0
class SimulStartLite(CTSTest):
'''Start any stopped nodes ~ simultaneously'''
def __init__(self, cm):
CTSTest.__init__(self,cm)
self.name = "SimulStartLite"
def __call__(self, dummy):
'''Perform the 'SimulStartList' setup work. '''
self.incr("calls")
self.debug("Setup: " + self.name)
# We ignore the "node" parameter...
node_list = []
for node in self.Env["nodes"]:
if self.CM.ShouldBeStatus[node] == "down":
self.incr("WasStopped")
node_list.append(node)
self.set_timer()
while len(node_list) > 0:
# Repeat until all nodes come up
watchpats = [ ]
uppat = self.templates["Pat:NonDC_started"]
if self.CM.upcount() == 0:
uppat = self.templates["Pat:Local_started"]
watchpats.append(self.templates["Pat:DC_IDLE"])
for node in node_list:
watchpats.append(uppat % node)
watchpats.append(self.templates["Pat:InfraUp"] % node)
watchpats.append(self.templates["Pat:PacemakerUp"] % node)
# Start all the nodes - at about the same time...
watch = self.create_watch(watchpats, self.Env["DeadTime"]+10)
watch.setwatch()
stonith = self.CM.prepare_fencing_watcher(self.name)
for node in node_list:
self.CM.StartaCMnoBlock(node)
watch.lookforall()
node_list = self.CM.fencing_cleanup(self.name, stonith)
if node_list == None:
return self.failure("Cluster did not stabilize")
# Remove node_list messages from watch.unmatched
for node in node_list:
self.logger.debug("Dealing with stonith operations for %s" % repr(node_list))
if watch.unmatched:
try:
watch.unmatched.remove(uppat % node)
except:
self.debug("Already matched: %s" % (uppat % node))
try:
watch.unmatched.remove(self.templates["Pat:InfraUp"] % node)
except:
self.debug("Already matched: %s" % (self.templates["Pat:InfraUp"] % node))
try:
watch.unmatched.remove(self.templates["Pat:PacemakerUp"] % node)
except:
self.debug("Already matched: %s" % (self.templates["Pat:PacemakerUp"] % node))
if watch.unmatched:
for regex in watch.unmatched:
self.logger.log ("Warn: Startup pattern not found: %s" %(regex))
if not self.CM.cluster_stable():
return self.failure("Cluster did not stabilize")
did_fail = 0
unstable = []
for node in self.Env["nodes"]:
if self.CM.StataCM(node) == 0:
did_fail = 1
unstable.append(node)
if did_fail:
return self.failure("Unstarted nodes exist: " + repr(unstable))
unstable = []
for node in self.Env["nodes"]:
if not self.CM.node_stable(node):
did_fail = 1
unstable.append(node)
if did_fail:
return self.failure("Unstable cluster nodes exist: " + repr(unstable))
return self.success()
def is_applicable(self):
'''SimulStartLite is a setup test and never applicable'''
return 0
def TestList(cm, audits):
result = []
for testclass in AllTestClasses:
bound_test = testclass(cm)
if bound_test.is_applicable():
bound_test.Audits = audits
result.append(bound_test)
return result
class RemoteLXC(CTSTest):
def __init__(self, cm):
CTSTest.__init__(self,cm)
self.name = "RemoteLXC"
self.start = StartTest(cm)
self.startall = SimulStartLite(cm)
self.num_containers = 2
self.is_container = 1
self.failed = 0
self.fail_string = ""
def start_lxc_simple(self, node):
# restore any artifacts laying around from a previous test.
self.rsh(node, "/usr/share/pacemaker/tests/cts/lxc_autogen.sh -s -R &>/dev/null")
# generate the containers, put them in the config, add some resources to them
pats = [ ]
watch = self.create_watch(pats, 120)
watch.setwatch()
pats.append(self.templates["Pat:RscOpOK"] % ("start", "lxc1"))
pats.append(self.templates["Pat:RscOpOK"] % ("start", "lxc2"))
pats.append(self.templates["Pat:RscOpOK"] % ("start", "lxc-ms"))
pats.append(self.templates["Pat:RscOpOK"] % ("promote", "lxc-ms"))
self.rsh(node, "/usr/share/pacemaker/tests/cts/lxc_autogen.sh -g -a -m -s -c %d &>/dev/null" % self.num_containers)
self.set_timer("remoteSimpleInit")
watch.lookforall()
self.log_timer("remoteSimpleInit")
if watch.unmatched:
self.fail_string = "Unmatched patterns: %s" % (repr(watch.unmatched))
self.failed = 1
def cleanup_lxc_simple(self, node):
pats = [ ]
# if the test failed, attempt to clean up the cib and libvirt environment
# as best as possible
if self.failed == 1:
# restore libvirt and cib
self.rsh(node, "/usr/share/pacemaker/tests/cts/lxc_autogen.sh -s -R &>/dev/null")
return
watch = self.create_watch(pats, 120)
watch.setwatch()
pats.append(self.templates["Pat:RscOpOK"] % ("stop", "container1"))
pats.append(self.templates["Pat:RscOpOK"] % ("stop", "container2"))
self.rsh(node, "/usr/share/pacemaker/tests/cts/lxc_autogen.sh -p &>/dev/null")
self.set_timer("remoteSimpleCleanup")
watch.lookforall()
self.log_timer("remoteSimpleCleanup")
if watch.unmatched:
self.fail_string = "Unmatched patterns: %s" % (repr(watch.unmatched))
self.failed = 1
# cleanup libvirt
self.rsh(node, "/usr/share/pacemaker/tests/cts/lxc_autogen.sh -s -R &>/dev/null")
def __call__(self, node):
'''Perform the 'RemoteLXC' test. '''
self.incr("calls")
ret = self.startall(None)
if not ret:
return self.failure("Setup failed, start all nodes failed.")
rc = self.rsh(node, "/usr/share/pacemaker/tests/cts/lxc_autogen.sh -v &>/dev/null")
if rc == 1:
self.log("Environment test for lxc support failed.")
return self.skipped()
self.start_lxc_simple(node)
self.cleanup_lxc_simple(node)
self.debug("Waiting for the cluster to recover")
self.CM.cluster_stable()
if self.failed == 1:
return self.failure(self.fail_string)
return self.success()
def errorstoignore(self):
'''Return list of errors which should be ignored'''
return [
r"Updating failcount for ping",
- r"schedulerd.*: Recover (ping|lxc-ms|container)\s*\(.*\)",
+ r"schedulerd.*: Recover\s+(ping|lxc-ms|container)\s+\(.*\)",
# The orphaned lxc-ms resource causes an expected transition error
# that is a result of the scheduler not having knowledge that the
# promotable resource used to be a clone. As a result, it looks like that
# resource is running in multiple locations when it shouldn't... But in
# this instance we know why this error is occurring and that it is expected.
r"Calculated [Tt]ransition .*pe-error",
r"Resource lxc-ms .* is active on 2 nodes attempting recovery",
r"Unknown operation: fail",
r"VirtualDomain.*ERROR: Unable to determine emulator",
]
AllTestClasses.append(RemoteLXC)
class RemoteDriver(CTSTest):
def __init__(self, cm):
CTSTest.__init__(self,cm)
self.name = self.__class__.__name__
self.start = StartTest(cm)
self.startall = SimulStartLite(cm)
self.stop = StopTest(cm)
self.remote_rsc = "remote-rsc"
self.cib_cmd = """cibadmin -C -o %s -X '%s' """
self.reset()
def reset(self):
self.pcmk_started = 0
self.failed = False
self.fail_string = ""
self.remote_node_added = 0
self.remote_rsc_added = 0
self.remote_use_reconnect_interval = self.Env.RandomGen.choice([True,False])
def fail(self, msg):
""" Mark test as failed. """
self.failed = True
# Always log the failure.
self.logger.log(msg)
# Use first failure as test status, as it's likely to be most useful.
if not self.fail_string:
self.fail_string = msg
def get_othernode(self, node):
for othernode in self.Env["nodes"]:
if othernode == node:
# we don't want to try and use the cib that we just shutdown.
# find a cluster node that is not our soon to be remote-node.
continue
else:
return othernode
def del_rsc(self, node, rsc):
othernode = self.get_othernode(node)
rc = self.rsh(othernode, "crm_resource -D -r %s -t primitive" % (rsc))
if rc != 0:
self.fail("Removal of resource '%s' failed" % rsc)
def add_rsc(self, node, rsc_xml):
othernode = self.get_othernode(node)
rc = self.rsh(othernode, self.cib_cmd % ("resources", rsc_xml))
if rc != 0:
self.fail("resource creation failed")
def add_primitive_rsc(self, node):
rsc_xml = """
<primitive class="ocf" id="%(node)s" provider="heartbeat" type="Dummy">
<meta_attributes id="%(node)s-meta_attributes"/>
<operations>
<op id="%(node)s-monitor-interval-20s" interval="20s" name="monitor"/>
</operations>
</primitive>""" % { "node": self.remote_rsc }
self.add_rsc(node, rsc_xml)
if not self.failed:
self.remote_rsc_added = 1
def add_connection_rsc(self, node):
rsc_xml = """
<primitive class="ocf" id="%(node)s" provider="pacemaker" type="remote">
<instance_attributes id="%(node)s-instance_attributes">
<nvpair id="%(node)s-instance_attributes-server" name="server" value="%(server)s"/>
""" % { "node": self.remote_node, "server": node }
if self.remote_use_reconnect_interval:
# Set reconnect interval on resource
rsc_xml = rsc_xml + """
<nvpair id="%s-instance_attributes-reconnect_interval" name="reconnect_interval" value="60s"/>
""" % (self.remote_node)
rsc_xml = rsc_xml + """
</instance_attributes>
<operations>
<op id="%(node)s-start" name="start" interval="0" timeout="120s"/>
<op id="%(node)s-monitor-20s" name="monitor" interval="20s" timeout="45s"/>
</operations>
</primitive>
""" % { "node": self.remote_node }
self.add_rsc(node, rsc_xml)
if not self.failed:
self.remote_node_added = 1
def disable_services(self, node):
self.corosync_enabled = self.Env.service_is_enabled(node, "corosync")
if self.corosync_enabled:
self.Env.disable_service(node, "corosync")
self.pacemaker_enabled = self.Env.service_is_enabled(node, "pacemaker")
if self.pacemaker_enabled:
self.Env.disable_service(node, "pacemaker")
def restore_services(self, node):
if self.corosync_enabled:
self.Env.enable_service(node, "corosync")
if self.pacemaker_enabled:
self.Env.enable_service(node, "pacemaker")
def stop_pcmk_remote(self, node):
# disable pcmk remote
for i in range(10):
rc = self.rsh(node, "service pacemaker_remote stop")
if rc != 0:
time.sleep(6)
else:
break
def start_pcmk_remote(self, node):
for i in range(10):
rc = self.rsh(node, "service pacemaker_remote start")
if rc != 0:
time.sleep(6)
else:
self.pcmk_started = 1
break
def freeze_pcmk_remote(self, node):
""" Simulate a Pacemaker Remote daemon failure. """
# We freeze the process.
self.rsh(node, "killall -STOP pacemaker-remoted")
def resume_pcmk_remote(self, node):
# We resume the process.
self.rsh(node, "killall -CONT pacemaker-remoted")
def start_metal(self, node):
# Cluster nodes are reused as remote nodes in remote tests. If cluster
# services were enabled at boot, in case the remote node got fenced, the
# cluster node would join instead of the expected remote one. Meanwhile
# pacemaker_remote would not be able to start. Depending on the chances,
# the situations might not be able to be orchestrated gracefully any more.
#
# Temporarily disable any enabled cluster serivces.
self.disable_services(node)
pcmk_started = 0
# make sure the resource doesn't already exist for some reason
self.rsh(node, "crm_resource -D -r %s -t primitive" % (self.remote_rsc))
self.rsh(node, "crm_resource -D -r %s -t primitive" % (self.remote_node))
if not self.stop(node):
self.fail("Failed to shutdown cluster node %s" % node)
return
self.start_pcmk_remote(node)
if self.pcmk_started == 0:
self.fail("Failed to start pacemaker_remote on node %s" % node)
return
# Convert node to baremetal now that it has shutdown the cluster stack
pats = [ ]
watch = self.create_watch(pats, 120)
watch.setwatch()
pats.append(self.templates["Pat:RscOpOK"] % ("start", self.remote_node))
pats.append(self.templates["Pat:DC_IDLE"])
self.add_connection_rsc(node)
self.set_timer("remoteMetalInit")
watch.lookforall()
self.log_timer("remoteMetalInit")
if watch.unmatched:
self.fail("Unmatched patterns: %s" % watch.unmatched)
def migrate_connection(self, node):
if self.failed:
return
pats = [ ]
pats.append(self.templates["Pat:RscOpOK"] % ("migrate_to", self.remote_node))
pats.append(self.templates["Pat:RscOpOK"] % ("migrate_from", self.remote_node))
pats.append(self.templates["Pat:DC_IDLE"])
watch = self.create_watch(pats, 120)
watch.setwatch()
(rc, lines) = self.rsh(node, "crm_resource -M -r %s" % (self.remote_node), None)
if rc != 0:
self.fail("failed to move remote node connection resource")
return
self.set_timer("remoteMetalMigrate")
watch.lookforall()
self.log_timer("remoteMetalMigrate")
if watch.unmatched:
self.fail("Unmatched patterns: %s" % watch.unmatched)
return
def fail_rsc(self, node):
if self.failed:
return
watchpats = [ ]
watchpats.append(self.templates["Pat:RscRemoteOpOK"] % ("stop", self.remote_rsc, self.remote_node))
watchpats.append(self.templates["Pat:RscRemoteOpOK"] % ("start", self.remote_rsc, self.remote_node))
watchpats.append(self.templates["Pat:DC_IDLE"])
watch = self.create_watch(watchpats, 120)
watch.setwatch()
self.debug("causing dummy rsc to fail.")
rc = self.rsh(node, "rm -f /var/run/resource-agents/Dummy*")
self.set_timer("remoteRscFail")
watch.lookforall()
self.log_timer("remoteRscFail")
if watch.unmatched:
self.fail("Unmatched patterns during rsc fail: %s" % watch.unmatched)
def fail_connection(self, node):
if self.failed:
return
watchpats = [ ]
watchpats.append(self.templates["Pat:Fencing_ok"] % self.remote_node)
watchpats.append(self.templates["Pat:NodeFenced"] % self.remote_node)
watch = self.create_watch(watchpats, 120)
watch.setwatch()
# freeze the pcmk remote daemon. this will result in fencing
self.debug("Force stopped active remote node")
self.freeze_pcmk_remote(node)
self.debug("Waiting for remote node to be fenced.")
self.set_timer("remoteMetalFence")
watch.lookforall()
self.log_timer("remoteMetalFence")
if watch.unmatched:
self.fail("Unmatched patterns: %s" % watch.unmatched)
return
self.debug("Waiting for the remote node to come back up")
self.CM.ns.WaitForNodeToComeUp(node, 120);
pats = [ ]
watch = self.create_watch(pats, 240)
watch.setwatch()
pats.append(self.templates["Pat:RscOpOK"] % ("start", self.remote_node))
if self.remote_rsc_added == 1:
pats.append(self.templates["Pat:RscRemoteOpOK"] % ("start", self.remote_rsc, self.remote_node))
# start the remote node again watch it integrate back into cluster.
self.start_pcmk_remote(node)
if self.pcmk_started == 0:
self.fail("Failed to start pacemaker_remote on node %s" % node)
return
self.debug("Waiting for remote node to rejoin cluster after being fenced.")
self.set_timer("remoteMetalRestart")
watch.lookforall()
self.log_timer("remoteMetalRestart")
if watch.unmatched:
self.fail("Unmatched patterns: %s" % watch.unmatched)
return
def add_dummy_rsc(self, node):
if self.failed:
return
# verify we can put a resource on the remote node
pats = [ ]
watch = self.create_watch(pats, 120)
watch.setwatch()
pats.append(self.templates["Pat:RscRemoteOpOK"] % ("start", self.remote_rsc, self.remote_node))
pats.append(self.templates["Pat:DC_IDLE"])
# Add a resource that must live on remote-node
self.add_primitive_rsc(node)
# force that rsc to prefer the remote node.
(rc, line) = self.CM.rsh(node, "crm_resource -M -r %s -N %s -f" % (self.remote_rsc, self.remote_node), None)
if rc != 0:
self.fail("Failed to place remote resource on remote node.")
return
self.set_timer("remoteMetalRsc")
watch.lookforall()
self.log_timer("remoteMetalRsc")
if watch.unmatched:
self.fail("Unmatched patterns: %s" % watch.unmatched)
def test_attributes(self, node):
if self.failed:
return
# This verifies permanent attributes can be set on a remote-node. It also
# verifies the remote-node can edit its own cib node section remotely.
(rc, line) = self.CM.rsh(node, "crm_attribute -l forever -n testattr -v testval -N %s" % (self.remote_node), None)
if rc != 0:
self.fail("Failed to set remote-node attribute. rc:%s output:%s" % (rc, line))
return
(rc, line) = self.CM.rsh(node, "crm_attribute -l forever -n testattr -q -N %s" % (self.remote_node), None)
if rc != 0:
self.fail("Failed to get remote-node attribute")
return
(rc, line) = self.CM.rsh(node, "crm_attribute -l forever -n testattr -D -N %s" % (self.remote_node), None)
if rc != 0:
self.fail("Failed to delete remote-node attribute")
return
def cleanup_metal(self, node):
self.restore_services(node)
if self.pcmk_started == 0:
return
pats = [ ]
watch = self.create_watch(pats, 120)
watch.setwatch()
if self.remote_rsc_added == 1:
pats.append(self.templates["Pat:RscOpOK"] % ("stop", self.remote_rsc))
if self.remote_node_added == 1:
pats.append(self.templates["Pat:RscOpOK"] % ("stop", self.remote_node))
self.set_timer("remoteMetalCleanup")
self.resume_pcmk_remote(node)
if self.remote_rsc_added == 1:
# Remove dummy resource added for remote node tests
self.debug("Cleaning up dummy rsc put on remote node")
self.rsh(self.get_othernode(node), "crm_resource -U -r %s" % self.remote_rsc)
self.del_rsc(node, self.remote_rsc)
if self.remote_node_added == 1:
# Remove remote node's connection resource
self.debug("Cleaning up remote node connection resource")
self.rsh(self.get_othernode(node), "crm_resource -U -r %s" % (self.remote_node))
self.del_rsc(node, self.remote_node)
watch.lookforall()
self.log_timer("remoteMetalCleanup")
if watch.unmatched:
self.fail("Unmatched patterns: %s" % watch.unmatched)
self.stop_pcmk_remote(node)
self.debug("Waiting for the cluster to recover")
self.CM.cluster_stable()
if self.remote_node_added == 1:
# Remove remote node itself
self.debug("Cleaning up node entry for remote node")
self.rsh(self.get_othernode(node), "crm_node --force --remove %s" % self.remote_node)
def setup_env(self, node):
self.remote_node = "remote-%s" % (node)
# we are assuming if all nodes have a key, that it is
# the right key... If any node doesn't have a remote
# key, we regenerate it everywhere.
if self.rsh.exists_on_all("/etc/pacemaker/authkey", self.Env["nodes"]):
return
# create key locally
(handle, keyfile) = tempfile.mkstemp(".cts")
os.close(handle)
subprocess.check_call(["dd", "if=/dev/urandom", "of=%s" % keyfile, "bs=4096", "count=1"],
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
# sync key throughout the cluster
for node in self.Env["nodes"]:
self.rsh(node, "mkdir -p --mode=0750 /etc/pacemaker")
self.rsh.cp(keyfile, "root@%s:/etc/pacemaker/authkey" % node)
self.rsh(node, "chgrp haclient /etc/pacemaker /etc/pacemaker/authkey")
self.rsh(node, "chmod 0640 /etc/pacemaker/authkey")
os.unlink(keyfile)
def is_applicable(self):
if not self.is_applicable_common():
return False
for node in self.Env["nodes"]:
rc = self.rsh(node, "which pacemaker-remoted >/dev/null 2>&1")
if rc != 0:
return False
return True
def start_new_test(self, node):
self.incr("calls")
self.reset()
ret = self.startall(None)
if not ret:
return self.failure("setup failed: could not start all nodes")
self.setup_env(node)
self.start_metal(node)
self.add_dummy_rsc(node)
return True
def __call__(self, node):
return self.failure("This base class is not meant to be called directly.")
def errorstoignore(self):
'''Return list of errors which should be ignored'''
return [ r"""is running on remote.*which isn't allowed""",
r"""Connection terminated""",
r"""Could not send remote""",
]
# RemoteDriver is just a base class for other tests, so it is not added to AllTestClasses
class RemoteBasic(RemoteDriver):
def __call__(self, node):
'''Perform the 'RemoteBaremetal' test. '''
if not self.start_new_test(node):
return self.failure(self.fail_string)
self.test_attributes(node)
self.cleanup_metal(node)
self.debug("Waiting for the cluster to recover")
self.CM.cluster_stable()
if self.failed:
return self.failure(self.fail_string)
return self.success()
AllTestClasses.append(RemoteBasic)
class RemoteStonithd(RemoteDriver):
def __call__(self, node):
'''Perform the 'RemoteStonithd' test. '''
if not self.start_new_test(node):
return self.failure(self.fail_string)
self.fail_connection(node)
self.cleanup_metal(node)
self.debug("Waiting for the cluster to recover")
self.CM.cluster_stable()
if self.failed:
return self.failure(self.fail_string)
return self.success()
def is_applicable(self):
if not RemoteDriver.is_applicable(self):
return False
if "DoFencing" in list(self.Env.keys()):
return self.Env["DoFencing"]
return True
def errorstoignore(self):
ignore_pats = [
r"Lost connection to Pacemaker Remote node",
r"Software caused connection abort",
r"pacemaker-controld.*:\s+error.*: Operation remote-.*_monitor",
r"pacemaker-controld.*:\s+error.*: Result of monitor operation for remote-.*",
- r"schedulerd.*:\s+Recover remote-.*\s*\(.*\)",
+ r"schedulerd.*:\s+Recover\s+remote-.*\s+\(.*\)",
r"error: Result of monitor operation for .* on remote-.*: Internal communication failure",
]
ignore_pats.extend(RemoteDriver.errorstoignore(self))
return ignore_pats
AllTestClasses.append(RemoteStonithd)
class RemoteMigrate(RemoteDriver):
def __call__(self, node):
'''Perform the 'RemoteMigrate' test. '''
if not self.start_new_test(node):
return self.failure(self.fail_string)
self.migrate_connection(node)
self.cleanup_metal(node)
self.debug("Waiting for the cluster to recover")
self.CM.cluster_stable()
if self.failed:
return self.failure(self.fail_string)
return self.success()
def is_applicable(self):
if not RemoteDriver.is_applicable(self):
return 0
# This test requires at least three nodes: one to convert to a
# remote node, one to host the connection originally, and one
# to migrate the connection to.
if len(self.Env["nodes"]) < 3:
return 0
return 1
AllTestClasses.append(RemoteMigrate)
class RemoteRscFailure(RemoteDriver):
def __call__(self, node):
'''Perform the 'RemoteRscFailure' test. '''
if not self.start_new_test(node):
return self.failure(self.fail_string)
# This is an important step. We are migrating the connection
# before failing the resource. This verifies that the migration
# has properly maintained control over the remote-node.
self.migrate_connection(node)
self.fail_rsc(node)
self.cleanup_metal(node)
self.debug("Waiting for the cluster to recover")
self.CM.cluster_stable()
if self.failed:
return self.failure(self.fail_string)
return self.success()
def errorstoignore(self):
ignore_pats = [
- r"schedulerd.*: Recover remote-rsc\s*\(.*\)",
+ r"schedulerd.*: Recover\s+remote-rsc\s+\(.*\)",
r"Dummy.*: No process state file found",
]
ignore_pats.extend(RemoteDriver.errorstoignore(self))
return ignore_pats
def is_applicable(self):
if not RemoteDriver.is_applicable(self):
return 0
# This test requires at least three nodes: one to convert to a
# remote node, one to host the connection originally, and one
# to migrate the connection to.
if len(self.Env["nodes"]) < 3:
return 0
return 1
AllTestClasses.append(RemoteRscFailure)
# vim:ts=4:sw=4:et:
diff --git a/cts/lab/patterns.py b/cts/lab/patterns.py
index 47d9a0ab93..adebdb5108 100644
--- a/cts/lab/patterns.py
+++ b/cts/lab/patterns.py
@@ -1,397 +1,397 @@
""" Pattern-holding classes for Pacemaker's Cluster Test Suite (CTS)
"""
__copyright__ = "Copyright 2008-2022 the Pacemaker project contributors"
__license__ = "GNU General Public License version 2 or later (GPLv2+) WITHOUT ANY WARRANTY"
import sys, os
from cts.CTSvars import *
patternvariants = {}
class BasePatterns(object):
def __init__(self, name):
self.name = name
patternvariants[name] = self
self.ignore = [
"avoid confusing Valgrind",
# Logging bug in some versions of libvirtd
r"libvirtd.*: internal error: Failed to parse PCI config address",
# pcs can log this when node is fenced, but fencing is OK in some
# tests (and we will catch it in pacemaker logs when not OK)
r"pcs.daemon:No response from: .* request: get_configs, error:",
]
self.BadNews = []
self.components = {}
self.commands = {
"StatusCmd" : "crmadmin -t 60 -S %s 2>/dev/null",
"CibQuery" : "cibadmin -Ql",
"CibAddXml" : "cibadmin --modify -c --xml-text %s",
"CibDelXpath" : "cibadmin --delete --xpath %s",
# 300,000 == 5 minutes
"RscRunning" : CTSvars.CRM_DAEMON_DIR + "/cts-exec-helper -R -r %s",
"CIBfile" : "%s:"+CTSvars.CRM_CONFIG_DIR+"/cib.xml",
"TmpDir" : "/tmp",
"BreakCommCmd" : "iptables -A INPUT -s %s -j DROP >/dev/null 2>&1",
"FixCommCmd" : "iptables -D INPUT -s %s -j DROP >/dev/null 2>&1",
# tc qdisc add dev lo root handle 1: cbq avpkt 1000 bandwidth 1000mbit
# tc class add dev lo parent 1: classid 1:1 cbq rate "$RATE"kbps allot 17000 prio 5 bounded isolated
# tc filter add dev lo parent 1: protocol ip prio 16 u32 match ip dst 127.0.0.1 match ip sport $PORT 0xFFFF flowid 1:1
# tc qdisc add dev lo parent 1: netem delay "$LATENCY"msec "$(($LATENCY/4))"msec 10% 2> /dev/null > /dev/null
"ReduceCommCmd" : "",
"RestoreCommCmd" : "tc qdisc del dev lo root",
"MaintenanceModeOn" : "cibadmin --modify -c --xml-text '<cluster_property_set id=\"cib-bootstrap-options\"><nvpair id=\"cts-maintenance-mode-setting\" name=\"maintenance-mode\" value=\"true\"/></cluster_property_set>'",
"MaintenanceModeOff" : "cibadmin --delete --xpath \"//nvpair[@name='maintenance-mode']\"",
"StandbyCmd" : "crm_attribute -Vq -U %s -n standby -l forever -v %s 2>/dev/null",
"StandbyQueryCmd" : "crm_attribute -qG -U %s -n standby -l forever -d off 2>/dev/null",
}
self.search = {
"Pat:DC_IDLE" : "pacemaker-controld.*State transition.*-> S_IDLE",
# This won't work if we have multiple partitions
"Pat:Local_started" : "%s\W.*controller successfully started",
"Pat:NonDC_started" : r"%s\W.*State transition.*-> S_NOT_DC",
"Pat:DC_started" : r"%s\W.*State transition.*-> S_IDLE",
"Pat:We_stopped" : "%s\W.*OVERRIDE THIS PATTERN",
"Pat:They_stopped" : "%s\W.*LOST:.* %s ",
"Pat:They_dead" : "node %s.*: is dead",
"Pat:They_up" : "%s %s\W.*OVERRIDE THIS PATTERN",
"Pat:TransitionComplete" : "Transition status: Complete: complete",
"Pat:Fencing_start" : r"Requesting peer fencing .* targeting %s",
"Pat:Fencing_ok" : r"pacemaker-fenced.*:\s*Operation .* targeting %s by .* for .*@.*: OK",
- "Pat:Fencing_recover" : r"pacemaker-schedulerd.*: Recover %s",
+ "Pat:Fencing_recover" : r"pacemaker-schedulerd.*: Recover\s+%s",
"Pat:Fencing_active" : r"stonith resource .* is active on 2 nodes (attempting recovery)",
"Pat:Fencing_probe" : r"pacemaker-controld.* Result of probe operation for %s on .*: Error",
"Pat:RscOpOK" : r"pacemaker-controld.*:\s+Result of %s operation for %s.*: (0 \()?ok",
"Pat:RscOpFail" : r"pacemaker-schedulerd.*:.*Unexpected result .* recorded for %s of %s ",
"Pat:CloneOpFail" : r"pacemaker-schedulerd.*:.*Unexpected result .* recorded for %s of (%s|%s) ",
"Pat:RscRemoteOpOK" : r"pacemaker-controld.*:\s+Result of %s operation for %s on %s: (0 \()?ok",
"Pat:NodeFenced" : r"pacemaker-controld.*:\s* Peer %s was terminated \(.*\) by .* on behalf of .*: OK",
}
def get_component(self, key):
if key in self.components:
return self.components[key]
print("Unknown component '%s' for %s" % (key, self.name))
return []
def get_patterns(self, key):
if key == "BadNews":
return self.BadNews
elif key == "BadNewsIgnore":
return self.ignore
elif key == "Commands":
return self.commands
elif key == "Search":
return self.search
elif key == "Components":
return self.components
def __getitem__(self, key):
if key == "Name":
return self.name
elif key in self.commands:
return self.commands[key]
elif key in self.search:
return self.search[key]
else:
print("Unknown template '%s' for %s" % (key, self.name))
return None
class crm_corosync(BasePatterns):
'''
Patterns for Corosync version 2 cluster manager class
'''
def __init__(self, name):
BasePatterns.__init__(self, name)
self.commands.update({
"StartCmd" : "service corosync start && service pacemaker start",
"StopCmd" : "service pacemaker stop; [ ! -e /usr/sbin/pacemaker-remoted ] || service pacemaker_remote stop; service corosync stop",
"EpochCmd" : "crm_node -e",
"QuorumCmd" : "crm_node -q",
"PartitionCmd" : "crm_node -p",
})
self.search.update({
# Close enough ... "Corosync Cluster Engine exiting normally" isn't
# printed reliably.
"Pat:We_stopped" : "%s\W.*Unloading all Corosync service engines",
"Pat:They_stopped" : "%s\W.*pacemaker-controld.*Node %s(\[|\s).*state is now lost",
"Pat:They_dead" : "pacemaker-controld.*Node %s(\[|\s).*state is now lost",
"Pat:They_up" : "\W%s\W.*pacemaker-controld.*Node %s state is now member",
"Pat:ChildExit" : r"\[[0-9]+\] exited with status [0-9]+ \(",
# "with signal 9" == pcmk_child_exit(), "$" == check_active_before_startup_processes()
"Pat:ChildKilled" : r"%s\W.*pacemakerd.*%s\[[0-9]+\] terminated( with signal 9|$)",
"Pat:ChildRespawn" : "%s\W.*pacemakerd.*Respawning %s subdaemon after unexpected exit",
"Pat:InfraUp" : "%s\W.*corosync.*Initializing transport",
"Pat:PacemakerUp" : "%s\W.*pacemakerd.*Starting Pacemaker",
})
self.ignore = self.ignore + [
r"crm_mon:",
r"crmadmin:",
r"update_trace_data",
r"async_notify:.*strange, client not found",
r"Parse error: Ignoring unknown option .*nodename",
r"error.*: Operation 'reboot' .* using FencingFail returned ",
r"getinfo response error: 1$",
r"sbd.* error: inquisitor_child: DEBUG MODE IS ACTIVE",
r"sbd.* pcmk:\s*error:.*Connection to cib_ro.* (failed|closed)",
]
self.BadNews = [
r"[^(]error:",
r"crit:",
r"ERROR:",
r"CRIT:",
r"Shutting down...NOW",
r"Timer I_TERMINATE just popped",
r"input=I_ERROR",
r"input=I_FAIL",
r"input=I_INTEGRATED cause=C_TIMER_POPPED",
r"input=I_FINALIZED cause=C_TIMER_POPPED",
r"input=I_ERROR",
r"(pacemakerd|pacemaker-execd|pacemaker-controld):.*, exiting",
r"schedulerd.*Attempting recovery of resource",
r"is taking more than 2x its timeout",
r"Confirm not received from",
r"Welcome reply not received from",
r"Attempting to schedule .* after a stop",
r"Resource .* was active at shutdown",
r"duplicate entries for call_id",
r"Search terminated:",
r":global_timer_callback",
r"Faking parameter digest creation",
r"Parameters to .* action changed:",
r"Parameters to .* changed",
r"pacemakerd.*\[[0-9]+\] terminated( with signal| as IPC server|$)",
- r"pacemaker-schedulerd.*Recover .*\(.* -\> .*\)",
+ r"pacemaker-schedulerd.*Recover\s+.*\(.* -\> .*\)",
r"rsyslogd.* imuxsock lost .* messages from pid .* due to rate-limiting",
r"Peer is not part of our cluster",
r"We appear to be in an election loop",
r"Unknown node -> we will not deliver message",
r"(Blackbox dump requested|Problem detected)",
r"pacemakerd.*Could not connect to Cluster Configuration Database API",
r"Receiving messages from a node we think is dead",
r"share the same cluster nodeid",
r"share the same name",
#r"crm_ipc_send:.*Request .* failed",
#r"crm_ipc_send:.*Sending to .* is disabled until pending reply is received",
# Not inherently bad, but worth tracking
#r"No need to invoke the TE",
#r"ping.*: DEBUG: Updated connected = 0",
#r"Digest mis-match:",
r"pacemaker-controld:.*Transition failed: terminated",
r"Local CIB .* differs from .*:",
r"warn.*:\s*Continuing but .* will NOT be used",
r"warn.*:\s*Cluster configuration file .* is corrupt",
#r"Executing .* fencing operation",
r"Election storm",
r"stalled the FSA with pending inputs",
]
self.components["common-ignore"] = [
r"Pending action:",
r"resource( was|s were) active at shutdown",
r"pending LRM operations at shutdown",
r"Lost connection to the CIB manager",
r"pacemaker-controld.*:\s*Action A_RECOVER .* not supported",
r"pacemaker-controld.*:\s*Performing A_EXIT_1 - forcefully exiting ",
r".*:\s*Requesting fencing \([^)]+\) of node ",
r"(Blackbox dump requested|Problem detected)",
]
self.components["corosync-ignore"] = [
r"Could not connect to Corosync CFG: CS_ERR_LIBRARY",
r"error:.*Connection to the CPG API failed: Library error",
r"\[[0-9]+\] exited with status [0-9]+ \(",
r"\[[0-9]+\] terminated with signal 15",
r"pacemaker-based.*error:.*Corosync connection lost",
r"pacemaker-fenced.*error:.*Corosync connection terminated",
r"pacemaker-controld.*State transition .* S_RECOVERY",
r"pacemaker-controld.*error:.*Input (I_ERROR|I_TERMINATE ) .*received in state",
r"pacemaker-controld.*error:.*Could not recover from internal error",
r"error:.*Connection to cib_(shm|rw).* (failed|closed)",
r"error:.*cib_(shm|rw) IPC provider disconnected while waiting",
r"error:.*Connection to (fencer|stonith-ng).* (closed|failed|lost)",
r"crit: Fencing daemon connection failed",
# This is overbroad, but we don't have a way to say that only
# certain transition errors are acceptable (if the fencer respawns,
# fence devices may appear multiply active). We have to rely on
# other causes of a transition error logging their own error
# message, which is the usual practice.
r"pacemaker-schedulerd.* Calculated transition .*/pe-error",
]
self.components["corosync"] = [
# We expect each daemon to lose its cluster connection.
# However, if the CIB manager loses its connection first,
# it's possible for another daemon to lose that connection and
# exit before losing the cluster connection.
r"pacemakerd.*:\s*warning:.*Lost connection to cluster layer",
r"pacemaker-attrd.*:\s*(crit|error):.*Lost connection to (cluster layer|the CIB manager)",
r"pacemaker-based.*:\s*(crit|error):.*Lost connection to cluster layer",
r"pacemaker-controld.*:\s*(crit|error):.*Lost connection to (cluster layer|the CIB manager)",
r"pacemaker-fenced.*:\s*(crit|error):.*Lost connection to (cluster layer|the CIB manager)",
r"schedulerd.*Scheduling node .* for fencing",
r"pacemaker-controld.*:\s*Peer .* was terminated \(.*\) by .* on behalf of .*:\s*OK",
]
self.components["pacemaker-based"] = [
r"pacemakerd.* pacemaker-attrd\[[0-9]+\] exited with status 102",
r"pacemakerd.* pacemaker-controld\[[0-9]+\] exited with status 1",
r"pacemakerd.* Respawning pacemaker-attrd subdaemon after unexpected exit",
r"pacemakerd.* Respawning pacemaker-based subdaemon after unexpected exit",
r"pacemakerd.* Respawning pacemaker-controld subdaemon after unexpected exit",
r"pacemakerd.* Respawning pacemaker-fenced subdaemon after unexpected exit",
r"pacemaker-.* Connection to cib_.* (failed|closed)",
r"pacemaker-attrd.*:.*Lost connection to the CIB manager",
r"pacemaker-controld.*:.*Lost connection to the CIB manager",
r"pacemaker-controld.*I_ERROR.*crmd_cib_connection_destroy",
r"pacemaker-controld.* State transition .* S_RECOVERY",
r"pacemaker-controld.*: Input I_TERMINATE .*from do_recover",
r"pacemaker-controld.*Could not recover from internal error",
]
self.components["pacemaker-based-ignore"] = [
r"pacemaker-execd.*Connection to (fencer|stonith-ng).* (closed|failed|lost)",
- r"pacemaker-controld.*:\s+Result of .* operation for Fencing.*Error (Lost connection to fencer)",
+ r"pacemaker-controld.*:\s+Result of .* operation for Fencing.*Error \(Lost connection to fencer\)",
r"pacemaker-controld.*:Could not connect to attrd: Connection refused",
# This is overbroad, but we don't have a way to say that only
# certain transition errors are acceptable (if the fencer respawns,
# fence devices may appear multiply active). We have to rely on
# other causes of a transition error logging their own error
# message, which is the usual practice.
r"pacemaker-schedulerd.* Calculated transition .*/pe-error",
]
self.components["pacemaker-execd"] = [
r"pacemaker-controld.*Connection to executor failed",
r"pacemaker-controld.*I_ERROR.*lrm_connection_destroy",
r"pacemaker-controld.*State transition .* S_RECOVERY",
r"pacemaker-controld.*: Input I_TERMINATE .*from do_recover",
r"pacemaker-controld.*Could not recover from internal error",
r"pacemakerd.*pacemaker-controld\[[0-9]+\] exited with status 1",
r"pacemakerd.* Respawning pacemaker-execd subdaemon after unexpected exit",
r"pacemakerd.* Respawning pacemaker-controld subdaemon after unexpected exit",
]
self.components["pacemaker-execd-ignore"] = [
r"pacemaker-(attrd|controld).*Connection to lrmd.* (failed|closed)",
r"pacemaker-(attrd|controld).*Could not execute alert",
]
self.components["pacemaker-controld"] = [
r"State transition .* -> S_IDLE",
]
self.components["pacemaker-controld-ignore"] = []
self.components["pacemaker-attrd"] = []
self.components["pacemaker-attrd-ignore"] = []
self.components["pacemaker-schedulerd"] = [
"State transition .* S_RECOVERY",
r"pacemakerd.* Respawning pacemaker-controld subdaemon after unexpected exit",
r"pacemaker-controld\[[0-9]+\] exited with status 1 \(",
r"Connection to the scheduler failed",
"pacemaker-controld.*I_ERROR.*save_cib_contents",
r"pacemaker-controld.*: Input I_TERMINATE .*from do_recover",
"pacemaker-controld.*Could not recover from internal error",
]
self.components["pacemaker-schedulerd-ignore"] = [
r"Connection to pengine.* (failed|closed)",
]
self.components["pacemaker-fenced"] = [
r"error:.*Connection to (fencer|stonith-ng).* (closed|failed|lost)",
r"Fencing daemon connection failed",
r"pacemaker-controld.*Fencer successfully connected",
]
self.components["pacemaker-fenced-ignore"] = [
r"(error|warning):.*Connection to (fencer|stonith-ng).* (closed|failed|lost)",
r"crit:.*Fencing daemon connection failed",
r"error:.*Fencer connection failed \(will retry\)",
- r"pacemaker-controld.*:\s+Result of .* operation for Fencing.*Error (Lost connection to fencer)",
+ r"pacemaker-controld.*:\s+Result of .* operation for Fencing.*Error \(Lost connection to fencer\)",
# This is overbroad, but we don't have a way to say that only
# certain transition errors are acceptable (if the fencer respawns,
# fence devices may appear multiply active). We have to rely on
# other causes of a transition error logging their own error
# message, which is the usual practice.
r"pacemaker-schedulerd.* Calculated transition .*/pe-error",
]
self.components["pacemaker-fenced-ignore"].extend(self.components["common-ignore"])
class PatternSelector(object):
def __init__(self, name=None):
self.name = name
self.base = BasePatterns("crm-base")
if not name:
crm_corosync("crm-corosync")
elif name == "crm-corosync":
crm_corosync(name)
def get_variant(self, variant):
if variant in patternvariants:
return patternvariants[variant]
print("defaulting to crm-base for %s" % variant)
return self.base
def get_patterns(self, variant, kind):
return self.get_variant(variant).get_patterns(kind)
def get_template(self, variant, key):
v = self.get_variant(variant)
return v[key]
def get_component(self, variant, kind):
return self.get_variant(variant).get_component(kind)
def __getitem__(self, key):
return self.get_template(self.name, key)
# python cts/CTSpatt.py -k crm-corosync -t StartCmd
if __name__ == '__main__':
pdir=os.path.dirname(sys.path[0])
sys.path.insert(0, pdir) # So that things work from the source directory
kind=None
template=None
skipthis=None
args=sys.argv[1:]
for i in range(0, len(args)):
if skipthis:
skipthis=None
continue
elif args[i] == "-k" or args[i] == "--kind":
skipthis=1
kind = args[i+1]
elif args[i] == "-t" or args[i] == "--template":
skipthis=1
template = args[i+1]
else:
print("Illegal argument " + args[i])
print(PatternSelector(kind)[template])
diff --git a/daemons/controld/controld_execd.c b/daemons/controld/controld_execd.c
index 11c6a411f1..809e5257c7 100644
--- a/daemons/controld/controld_execd.c
+++ b/daemons/controld/controld_execd.c
@@ -1,2945 +1,2984 @@
/*
* Copyright 2004-2022 the Pacemaker project contributors
*
* The version control history for this file may have further details.
*
* This source code is licensed under the GNU General Public License version 2
* or later (GPLv2+) WITHOUT ANY WARRANTY.
*/
#include <crm_internal.h>
#include <regex.h>
#include <sys/param.h>
#include <sys/types.h>
#include <sys/wait.h>
#include <crm/crm.h>
#include <crm/lrmd.h> // lrmd_event_data_t, lrmd_rsc_info_t, etc.
#include <crm/services.h>
#include <crm/msg_xml.h>
#include <crm/common/xml.h>
#include <crm/pengine/rules.h>
#include <crm/lrmd_internal.h>
#include <pacemaker-internal.h>
#include <pacemaker-controld.h>
#define START_DELAY_THRESHOLD 5 * 60 * 1000
#define MAX_LRM_REG_FAILS 30
struct delete_event_s {
int rc;
const char *rsc;
lrm_state_t *lrm_state;
};
extern pcmk__output_t *logger_out;
static gboolean is_rsc_active(lrm_state_t * lrm_state, const char *rsc_id);
static gboolean build_active_RAs(lrm_state_t * lrm_state, xmlNode * rsc_list);
static gboolean stop_recurring_actions(gpointer key, gpointer value, gpointer user_data);
static lrmd_event_data_t *construct_op(const lrm_state_t *lrm_state,
const xmlNode *rsc_op,
const char *rsc_id,
const char *operation);
static void do_lrm_rsc_op(lrm_state_t *lrm_state, lrmd_rsc_info_t *rsc,
- const char *operation, xmlNode *msg);
+ xmlNode *msg, struct ra_metadata_s *md);
static gboolean lrm_state_verify_stopped(lrm_state_t * lrm_state, enum crmd_fsa_state cur_state,
int log_level);
static int do_update_resource(const char *node_name, lrmd_rsc_info_t *rsc,
lrmd_event_data_t *op, time_t lock_time);
static void
lrm_connection_destroy(void)
{
if (pcmk_is_set(fsa_input_register, R_LRM_CONNECTED)) {
crm_crit("Connection to executor failed");
register_fsa_input(C_FSA_INTERNAL, I_ERROR, NULL);
controld_clear_fsa_input_flags(R_LRM_CONNECTED);
} else {
crm_info("Disconnected from executor");
}
}
static char *
make_stop_id(const char *rsc, int call_id)
{
return crm_strdup_printf("%s:%d", rsc, call_id);
}
static void
copy_instance_keys(gpointer key, gpointer value, gpointer user_data)
{
if (strstr(key, CRM_META "_") == NULL) {
g_hash_table_replace(user_data, strdup((const char *)key), strdup((const char *)value));
}
}
static void
copy_meta_keys(gpointer key, gpointer value, gpointer user_data)
{
if (strstr(key, CRM_META "_") != NULL) {
g_hash_table_replace(user_data, strdup((const char *)key), strdup((const char *)value));
}
}
/*!
* \internal
* \brief Remove a recurring operation from a resource's history
*
* \param[in,out] history Resource history to modify
* \param[in] op Operation to remove
*
* \return TRUE if the operation was found and removed, FALSE otherwise
*/
static gboolean
history_remove_recurring_op(rsc_history_t *history, const lrmd_event_data_t *op)
{
GList *iter;
for (iter = history->recurring_op_list; iter != NULL; iter = iter->next) {
lrmd_event_data_t *existing = iter->data;
if ((op->interval_ms == existing->interval_ms)
&& pcmk__str_eq(op->rsc_id, existing->rsc_id, pcmk__str_none)
&& pcmk__str_eq(op->op_type, existing->op_type, pcmk__str_casei)) {
history->recurring_op_list = g_list_delete_link(history->recurring_op_list, iter);
lrmd_free_event(existing);
return TRUE;
}
}
return FALSE;
}
/*!
* \internal
* \brief Free all recurring operations in resource history
*
* \param[in,out] history Resource history to modify
*/
static void
history_free_recurring_ops(rsc_history_t *history)
{
GList *iter;
for (iter = history->recurring_op_list; iter != NULL; iter = iter->next) {
lrmd_free_event(iter->data);
}
g_list_free(history->recurring_op_list);
history->recurring_op_list = NULL;
}
/*!
* \internal
* \brief Free resource history
*
* \param[in,out] history Resource history to free
*/
void
history_free(gpointer data)
{
rsc_history_t *history = (rsc_history_t*)data;
if (history->stop_params) {
g_hash_table_destroy(history->stop_params);
}
/* Don't need to free history->rsc.id because it's set to history->id */
free(history->rsc.type);
free(history->rsc.standard);
free(history->rsc.provider);
lrmd_free_event(history->failed);
lrmd_free_event(history->last);
free(history->id);
history_free_recurring_ops(history);
free(history);
}
static void
update_history_cache(lrm_state_t * lrm_state, lrmd_rsc_info_t * rsc, lrmd_event_data_t * op)
{
int target_rc = 0;
rsc_history_t *entry = NULL;
if (op->rsc_deleted) {
crm_debug("Purged history for '%s' after %s", op->rsc_id, op->op_type);
controld_delete_resource_history(op->rsc_id, lrm_state->node_name,
NULL, crmd_cib_smart_opt());
return;
}
if (pcmk__str_eq(op->op_type, RSC_NOTIFY, pcmk__str_casei)) {
return;
}
crm_debug("Updating history for '%s' with %s op", op->rsc_id, op->op_type);
entry = g_hash_table_lookup(lrm_state->resource_history, op->rsc_id);
if (entry == NULL && rsc) {
entry = calloc(1, sizeof(rsc_history_t));
entry->id = strdup(op->rsc_id);
g_hash_table_insert(lrm_state->resource_history, entry->id, entry);
entry->rsc.id = entry->id;
entry->rsc.type = strdup(rsc->type);
entry->rsc.standard = strdup(rsc->standard);
pcmk__str_update(&entry->rsc.provider, rsc->provider);
} else if (entry == NULL) {
crm_info("Resource %s no longer exists, not updating cache", op->rsc_id);
return;
}
entry->last_callid = op->call_id;
target_rc = rsc_op_expected_rc(op);
if (op->op_status == PCMK_EXEC_CANCELLED) {
if (op->interval_ms > 0) {
crm_trace("Removing cancelled recurring op: " PCMK__OP_FMT,
op->rsc_id, op->op_type, op->interval_ms);
history_remove_recurring_op(entry, op);
return;
} else {
crm_trace("Skipping " PCMK__OP_FMT " rc=%d, status=%d",
op->rsc_id, op->op_type, op->interval_ms, op->rc,
op->op_status);
}
} else if (did_rsc_op_fail(op, target_rc)) {
/* Store failed monitors here, otherwise the block below will cause them
* to be forgotten when a stop happens.
*/
if (entry->failed) {
lrmd_free_event(entry->failed);
}
entry->failed = lrmd_copy_event(op);
} else if (op->interval_ms == 0) {
if (entry->last) {
lrmd_free_event(entry->last);
}
entry->last = lrmd_copy_event(op);
if (op->params && pcmk__strcase_any_of(op->op_type, CRMD_ACTION_START,
CRMD_ACTION_RELOAD,
CRMD_ACTION_RELOAD_AGENT,
CRMD_ACTION_STATUS, NULL)) {
if (entry->stop_params) {
g_hash_table_destroy(entry->stop_params);
}
entry->stop_params = pcmk__strkey_table(free, free);
g_hash_table_foreach(op->params, copy_instance_keys, entry->stop_params);
}
}
if (op->interval_ms > 0) {
/* Ensure there are no duplicates */
history_remove_recurring_op(entry, op);
crm_trace("Adding recurring op: " PCMK__OP_FMT,
op->rsc_id, op->op_type, op->interval_ms);
entry->recurring_op_list = g_list_prepend(entry->recurring_op_list, lrmd_copy_event(op));
} else if (entry->recurring_op_list && !pcmk__str_eq(op->op_type, RSC_STATUS, pcmk__str_casei)) {
crm_trace("Dropping %d recurring ops because of: " PCMK__OP_FMT,
g_list_length(entry->recurring_op_list), op->rsc_id,
op->op_type, op->interval_ms);
history_free_recurring_ops(entry);
}
}
/*!
* \internal
* \brief Send a direct OK ack for a resource task
*
* \param[in] lrm_state LRM connection
* \param[in] input Input message being ack'ed
* \param[in] rsc_id ID of affected resource
* \param[in] rsc Affected resource (if available)
* \param[in] task Operation task being ack'ed
* \param[in] ack_host Name of host to send ack to
* \param[in] ack_sys IPC system name to ack
*/
static void
send_task_ok_ack(const lrm_state_t *lrm_state, const ha_msg_input_t *input,
const char *rsc_id, const lrmd_rsc_info_t *rsc,
const char *task, const char *ack_host, const char *ack_sys)
{
lrmd_event_data_t *op = construct_op(lrm_state, input->xml, rsc_id, task);
lrmd__set_result(op, PCMK_OCF_OK, PCMK_EXEC_DONE, NULL);
controld_ack_event_directly(ack_host, ack_sys, rsc, op, rsc_id);
lrmd_free_event(op);
}
static inline const char *
op_node_name(lrmd_event_data_t *op)
{
return op->remote_nodename? op->remote_nodename : fsa_our_uname;
}
void
lrm_op_callback(lrmd_event_data_t * op)
{
CRM_CHECK(op != NULL, return);
switch (op->type) {
case lrmd_event_disconnect:
if (op->remote_nodename == NULL) {
/* If this is the local executor IPC connection, set the right
* bits in the controller when the connection goes down.
*/
lrm_connection_destroy();
}
break;
case lrmd_event_exec_complete:
{
lrm_state_t *lrm_state = lrm_state_find(op_node_name(op));
CRM_ASSERT(lrm_state != NULL);
process_lrm_event(lrm_state, op, NULL, NULL);
}
break;
default:
break;
}
}
static void
try_local_executor_connect(long long action, fsa_data_t *msg_data,
lrm_state_t *lrm_state)
{
int rc = pcmk_rc_ok;
crm_debug("Connecting to the local executor");
// If we can connect, great
rc = controld_connect_local_executor(lrm_state);
if (rc == pcmk_rc_ok) {
controld_set_fsa_input_flags(R_LRM_CONNECTED);
crm_info("Connection to the local executor established");
return;
}
// Otherwise, if we can try again, set a timer to do so
if (lrm_state->num_lrm_register_fails < MAX_LRM_REG_FAILS) {
crm_warn("Failed to connect to the local executor %d time%s "
"(%d max): %s", lrm_state->num_lrm_register_fails,
pcmk__plural_s(lrm_state->num_lrm_register_fails),
MAX_LRM_REG_FAILS, pcmk_rc_str(rc));
controld_start_timer(wait_timer);
crmd_fsa_stall(FALSE);
return;
}
// Otherwise give up
crm_err("Failed to connect to the executor the max allowed "
"%d time%s: %s", lrm_state->num_lrm_register_fails,
pcmk__plural_s(lrm_state->num_lrm_register_fails),
pcmk_rc_str(rc));
register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
}
/* A_LRM_CONNECT */
void
do_lrm_control(long long action,
enum crmd_fsa_cause cause,
enum crmd_fsa_state cur_state,
enum crmd_fsa_input current_input, fsa_data_t * msg_data)
{
/* This only pertains to local executor connections. Remote connections are
* handled as resources within the scheduler. Connecting and disconnecting
* from remote executor instances is handled differently.
*/
lrm_state_t *lrm_state = NULL;
if(fsa_our_uname == NULL) {
return; /* Nothing to do */
}
lrm_state = lrm_state_find_or_create(fsa_our_uname);
if (lrm_state == NULL) {
register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
return;
}
if (action & A_LRM_DISCONNECT) {
if (lrm_state_verify_stopped(lrm_state, cur_state, LOG_INFO) == FALSE) {
if (action == A_LRM_DISCONNECT) {
crmd_fsa_stall(FALSE);
return;
}
}
controld_clear_fsa_input_flags(R_LRM_CONNECTED);
crm_info("Disconnecting from the executor");
lrm_state_disconnect(lrm_state);
lrm_state_reset_tables(lrm_state, FALSE);
crm_notice("Disconnected from the executor");
}
if (action & A_LRM_CONNECT) {
try_local_executor_connect(action, msg_data, lrm_state);
}
if (action & ~(A_LRM_CONNECT | A_LRM_DISCONNECT)) {
crm_err("Unexpected action %s in %s", fsa_action2string(action),
__func__);
}
}
static gboolean
lrm_state_verify_stopped(lrm_state_t * lrm_state, enum crmd_fsa_state cur_state, int log_level)
{
int counter = 0;
gboolean rc = TRUE;
const char *when = "lrm disconnect";
GHashTableIter gIter;
const char *key = NULL;
rsc_history_t *entry = NULL;
active_op_t *pending = NULL;
crm_debug("Checking for active resources before exit");
if (cur_state == S_TERMINATE) {
log_level = LOG_ERR;
when = "shutdown";
} else if (pcmk_is_set(fsa_input_register, R_SHUTDOWN)) {
when = "shutdown... waiting";
}
if (lrm_state->pending_ops && lrm_state_is_connected(lrm_state) == TRUE) {
guint removed = g_hash_table_foreach_remove(
lrm_state->pending_ops, stop_recurring_actions, lrm_state);
guint nremaining = g_hash_table_size(lrm_state->pending_ops);
if (removed || nremaining) {
crm_notice("Stopped %u recurring operation%s at %s (%u remaining)",
removed, pcmk__plural_s(removed), when, nremaining);
}
}
if (lrm_state->pending_ops) {
g_hash_table_iter_init(&gIter, lrm_state->pending_ops);
while (g_hash_table_iter_next(&gIter, NULL, (void **)&pending)) {
/* Ignore recurring actions in the shutdown calculations */
if (pending->interval_ms == 0) {
counter++;
}
}
}
if (counter > 0) {
do_crm_log(log_level, "%d pending executor operation%s at %s",
counter, pcmk__plural_s(counter), when);
if ((cur_state == S_TERMINATE)
|| !pcmk_is_set(fsa_input_register, R_SENT_RSC_STOP)) {
g_hash_table_iter_init(&gIter, lrm_state->pending_ops);
while (g_hash_table_iter_next(&gIter, (gpointer*)&key, (gpointer*)&pending)) {
do_crm_log(log_level, "Pending action: %s (%s)", key, pending->op_key);
}
} else {
rc = FALSE;
}
return rc;
}
if (lrm_state->resource_history == NULL) {
return rc;
}
if (pcmk_is_set(fsa_input_register, R_SHUTDOWN)) {
/* At this point we're not waiting, we're just shutting down */
when = "shutdown";
}
counter = 0;
g_hash_table_iter_init(&gIter, lrm_state->resource_history);
while (g_hash_table_iter_next(&gIter, NULL, (gpointer*)&entry)) {
if (is_rsc_active(lrm_state, entry->id) == FALSE) {
continue;
}
counter++;
if (log_level == LOG_ERR) {
crm_info("Found %s active at %s", entry->id, when);
} else {
crm_trace("Found %s active at %s", entry->id, when);
}
if (lrm_state->pending_ops) {
GHashTableIter hIter;
g_hash_table_iter_init(&hIter, lrm_state->pending_ops);
while (g_hash_table_iter_next(&hIter, (gpointer*)&key, (gpointer*)&pending)) {
if (pcmk__str_eq(entry->id, pending->rsc_id, pcmk__str_none)) {
crm_notice("%sction %s (%s) incomplete at %s",
pending->interval_ms == 0 ? "A" : "Recurring a",
key, pending->op_key, when);
}
}
}
}
if (counter) {
crm_err("%d resource%s active at %s",
counter, (counter == 1)? " was" : "s were", when);
}
return rc;
}
/*!
* \internal
* \brief Build XML and string of parameters meeting some criteria, for digest
*
* \param[in] op Executor event with parameter table to use
* \param[in] metadata Parsed meta-data for executed resource agent
* \param[in] param_type Flag used for selection criteria
* \param[out] result Will be set to newly created XML with selected
* parameters as attributes
*
* \return Newly allocated space-separated string of parameter names
* \note Selection criteria varies by param_type: for the restart digest, we
* want parameters that are *not* marked reloadable (OCF 1.1) or that
* *are* marked unique (pre-1.1), for both string and XML results; for the
* secure digest, we want parameters that *are* marked private for the
* string, but parameters that are *not* marked private for the XML.
* \note It is the caller's responsibility to free the string return value with
* free() and the XML result with free_xml().
*/
static char *
build_parameter_list(const lrmd_event_data_t *op,
const struct ra_metadata_s *metadata,
enum ra_param_flags_e param_type, xmlNode **result)
{
char *list = NULL;
size_t len = 0;
*result = create_xml_node(NULL, XML_TAG_PARAMS);
/* Consider all parameters only except private ones to be consistent with
* what scheduler does with calculate_secure_digest().
*/
if (param_type == ra_param_private
&& compare_version(fsa_our_dc_version, "3.16.0") >= 0) {
g_hash_table_foreach(op->params, hash2field, *result);
pcmk__filter_op_for_digest(*result);
}
for (GList *iter = metadata->ra_params; iter != NULL; iter = iter->next) {
struct ra_param_s *param = (struct ra_param_s *) iter->data;
bool accept_for_list = false;
bool accept_for_xml = false;
switch (param_type) {
case ra_param_reloadable:
accept_for_list = !pcmk_is_set(param->rap_flags, param_type);
accept_for_xml = accept_for_list;
break;
case ra_param_unique:
accept_for_list = pcmk_is_set(param->rap_flags, param_type);
accept_for_xml = accept_for_list;
break;
case ra_param_private:
accept_for_list = pcmk_is_set(param->rap_flags, param_type);
accept_for_xml = !accept_for_list;
break;
}
if (accept_for_list) {
crm_trace("Attr %s is %s", param->rap_name, ra_param_flag2text(param_type));
if (list == NULL) {
// We will later search for " WORD ", so start list with a space
pcmk__add_word(&list, &len, " ");
}
pcmk__add_word(&list, &len, param->rap_name);
} else {
crm_trace("Rejecting %s for %s", param->rap_name, ra_param_flag2text(param_type));
}
if (accept_for_xml) {
const char *v = g_hash_table_lookup(op->params, param->rap_name);
if (v != NULL) {
crm_trace("Adding attr %s=%s to the xml result", param->rap_name, v);
crm_xml_add(*result, param->rap_name, v);
}
} else {
crm_trace("Removing attr %s from the xml result", param->rap_name);
xml_remove_prop(*result, param->rap_name);
}
}
if (list != NULL) {
// We will later search for " WORD ", so end list with a space
pcmk__add_word(&list, &len, " ");
}
return list;
}
static void
append_restart_list(lrmd_event_data_t *op, struct ra_metadata_s *metadata,
xmlNode *update, const char *version)
{
char *list = NULL;
char *digest = NULL;
xmlNode *restart = NULL;
CRM_LOG_ASSERT(op->params != NULL);
if (op->interval_ms > 0) {
/* monitors are not reloadable */
return;
}
if (pcmk_is_set(metadata->ra_flags, ra_supports_reload_agent)) {
// Add parameters not marked reloadable to the "op-force-restart" list
list = build_parameter_list(op, metadata, ra_param_reloadable,
&restart);
} else if (pcmk_is_set(metadata->ra_flags, ra_supports_legacy_reload)) {
/* @COMPAT pre-OCF-1.1 resource agents
*
* Before OCF 1.1, Pacemaker abused "unique=0" to indicate
* reloadability. Add any parameters with unique="1" to the
* "op-force-restart" list.
*/
list = build_parameter_list(op, metadata, ra_param_unique, &restart);
} else {
// Resource does not support agent reloads
return;
}
digest = calculate_operation_digest(restart, version);
/* Add "op-force-restart" and "op-restart-digest" to indicate the resource supports reload,
* no matter if it actually supports any parameters with unique="1"). */
crm_xml_add(update, XML_LRM_ATTR_OP_RESTART, list? list: "");
crm_xml_add(update, XML_LRM_ATTR_RESTART_DIGEST, digest);
crm_trace("%s: %s, %s", op->rsc_id, digest, list);
crm_log_xml_trace(restart, "restart digest source");
free_xml(restart);
free(digest);
free(list);
}
static void
append_secure_list(lrmd_event_data_t *op, struct ra_metadata_s *metadata,
xmlNode *update, const char *version)
{
char *list = NULL;
char *digest = NULL;
xmlNode *secure = NULL;
CRM_LOG_ASSERT(op->params != NULL);
/*
* To keep XML_LRM_ATTR_OP_SECURE short, we want it to contain the
* secure parameters but XML_LRM_ATTR_SECURE_DIGEST to be based on
* the insecure ones
*/
list = build_parameter_list(op, metadata, ra_param_private, &secure);
if (list != NULL) {
digest = calculate_operation_digest(secure, version);
crm_xml_add(update, XML_LRM_ATTR_OP_SECURE, list);
crm_xml_add(update, XML_LRM_ATTR_SECURE_DIGEST, digest);
crm_trace("%s: %s, %s", op->rsc_id, digest, list);
crm_log_xml_trace(secure, "secure digest source");
} else {
crm_trace("%s: no secure parameters", op->rsc_id);
}
free_xml(secure);
free(digest);
free(list);
}
static gboolean
build_operation_update(xmlNode * parent, const lrmd_rsc_info_t *rsc,
lrmd_event_data_t *op, const char *node_name,
const char *src)
{
int target_rc = 0;
xmlNode *xml_op = NULL;
struct ra_metadata_s *metadata = NULL;
const char *caller_version = NULL;
lrm_state_t *lrm_state = NULL;
- uint32_t metadata_source = controld_metadata_from_agent;
if (op == NULL) {
return FALSE;
}
target_rc = rsc_op_expected_rc(op);
- /* there is a small risk in formerly mixed clusters that it will
- * be sub-optimal.
- *
- * however with our upgrade policy, the update we send should
- * still be completely supported anyway
- */
caller_version = g_hash_table_lookup(op->params, XML_ATTR_CRM_VERSION);
- CRM_LOG_ASSERT(caller_version != NULL);
-
- if(caller_version == NULL) {
- caller_version = CRM_FEATURE_SET;
- }
+ CRM_CHECK(caller_version != NULL, caller_version = CRM_FEATURE_SET);
xml_op = pcmk__create_history_xml(parent, op, caller_version, target_rc,
fsa_our_uname, src);
if (xml_op == NULL) {
return TRUE;
}
if ((rsc == NULL) || (op->params == NULL)
|| !crm_op_needs_metadata(rsc->standard, op->op_type)) {
crm_trace("No digests needed for %s action on %s (params=%p rsc=%p)",
op->op_type, op->rsc_id, op->params, rsc);
return TRUE;
}
lrm_state = lrm_state_find(node_name);
if (lrm_state == NULL) {
crm_warn("Cannot calculate digests for operation " PCMK__OP_FMT
" because we have no connection to executor for %s",
op->rsc_id, op->op_type, op->interval_ms, node_name);
return TRUE;
}
- /* Getting meta-data from cache is OK unless this is a successful start
- * action -- always refresh from the agent for those, in case the
- * resource agent was updated.
+ /* Ideally the metadata is cached, and the agent is just a fallback.
*
- * @TODO Only refresh the meta-data after starts if the agent actually
- * changed (using something like inotify, or a hash or modification time of
- * the agent executable).
+ * @TODO Go through all callers and ensure they get metadata asynchronously
+ * first.
*/
- if ((op->op_status != PCMK_EXEC_DONE) || (op->rc != target_rc)
- || !pcmk__str_eq(op->op_type, CRMD_ACTION_START, pcmk__str_none)) {
- metadata_source |= controld_metadata_from_cache;
- }
- metadata = controld_get_rsc_metadata(lrm_state, rsc, metadata_source);
+ metadata = controld_get_rsc_metadata(lrm_state, rsc,
+ controld_metadata_from_agent
+ |controld_metadata_from_cache);
if (metadata == NULL) {
return TRUE;
}
#if ENABLE_VERSIONED_ATTRS
crm_xml_add(xml_op, XML_ATTR_RA_VERSION, metadata->ra_version);
#endif
crm_trace("Including additional digests for %s:%s:%s",
rsc->standard, rsc->provider, rsc->type);
append_restart_list(op, metadata, xml_op, caller_version);
append_secure_list(op, metadata, xml_op, caller_version);
return TRUE;
}
static gboolean
is_rsc_active(lrm_state_t * lrm_state, const char *rsc_id)
{
rsc_history_t *entry = NULL;
entry = g_hash_table_lookup(lrm_state->resource_history, rsc_id);
if (entry == NULL || entry->last == NULL) {
return FALSE;
}
crm_trace("Processing %s: %s.%d=%d", rsc_id, entry->last->op_type,
entry->last->interval_ms, entry->last->rc);
if (entry->last->rc == PCMK_OCF_OK && pcmk__str_eq(entry->last->op_type, CRMD_ACTION_STOP, pcmk__str_casei)) {
return FALSE;
} else if (entry->last->rc == PCMK_OCF_OK
&& pcmk__str_eq(entry->last->op_type, CRMD_ACTION_MIGRATE, pcmk__str_casei)) {
// A stricter check is too complex ... leave that to the scheduler
return FALSE;
} else if (entry->last->rc == PCMK_OCF_NOT_RUNNING) {
return FALSE;
} else if ((entry->last->interval_ms == 0)
&& (entry->last->rc == PCMK_OCF_NOT_CONFIGURED)) {
/* Badly configured resources can't be reliably stopped */
return FALSE;
}
return TRUE;
}
static gboolean
build_active_RAs(lrm_state_t * lrm_state, xmlNode * rsc_list)
{
GHashTableIter iter;
rsc_history_t *entry = NULL;
g_hash_table_iter_init(&iter, lrm_state->resource_history);
while (g_hash_table_iter_next(&iter, NULL, (void **)&entry)) {
GList *gIter = NULL;
xmlNode *xml_rsc = create_xml_node(rsc_list, XML_LRM_TAG_RESOURCE);
crm_xml_add(xml_rsc, XML_ATTR_ID, entry->id);
crm_xml_add(xml_rsc, XML_ATTR_TYPE, entry->rsc.type);
crm_xml_add(xml_rsc, XML_AGENT_ATTR_CLASS, entry->rsc.standard);
crm_xml_add(xml_rsc, XML_AGENT_ATTR_PROVIDER, entry->rsc.provider);
if (entry->last && entry->last->params) {
const char *container = g_hash_table_lookup(entry->last->params, CRM_META"_"XML_RSC_ATTR_CONTAINER);
if (container) {
crm_trace("Resource %s is a part of container resource %s", entry->id, container);
crm_xml_add(xml_rsc, XML_RSC_ATTR_CONTAINER, container);
}
}
build_operation_update(xml_rsc, &(entry->rsc), entry->failed, lrm_state->node_name,
__func__);
build_operation_update(xml_rsc, &(entry->rsc), entry->last, lrm_state->node_name,
__func__);
for (gIter = entry->recurring_op_list; gIter != NULL; gIter = gIter->next) {
build_operation_update(xml_rsc, &(entry->rsc), gIter->data, lrm_state->node_name,
__func__);
}
}
return FALSE;
}
-static xmlNode *
-do_lrm_query_internal(lrm_state_t *lrm_state, int update_flags)
+xmlNode *
+controld_query_executor_state(void)
{
xmlNode *xml_state = NULL;
xmlNode *xml_data = NULL;
xmlNode *rsc_list = NULL;
crm_node_t *peer = NULL;
+ lrm_state_t *lrm_state = lrm_state_find(fsa_our_uname);
+
+ if (!lrm_state) {
+ crm_err("Could not find executor state for node %s", fsa_our_uname);
+ return NULL;
+ }
peer = crm_get_peer_full(0, lrm_state->node_name, CRM_GET_PEER_ANY);
CRM_CHECK(peer != NULL, return NULL);
- xml_state = create_node_state_update(peer, update_flags, NULL,
- __func__);
+ xml_state = create_node_state_update(peer,
+ node_update_cluster|node_update_peer,
+ NULL, __func__);
if (xml_state == NULL) {
return NULL;
}
xml_data = create_xml_node(xml_state, XML_CIB_TAG_LRM);
crm_xml_add(xml_data, XML_ATTR_ID, peer->uuid);
rsc_list = create_xml_node(xml_data, XML_LRM_TAG_RESOURCES);
/* Build a list of active (not always running) resources */
build_active_RAs(lrm_state, rsc_list);
crm_log_xml_trace(xml_state, "Current executor state");
return xml_state;
}
-xmlNode *
-controld_query_executor_state(const char *node_name)
-{
- lrm_state_t *lrm_state = lrm_state_find(node_name);
-
- if (!lrm_state) {
- crm_err("Could not find executor state for node %s", node_name);
- return NULL;
- }
- return do_lrm_query_internal(lrm_state,
- node_update_cluster|node_update_peer);
-}
-
/*!
* \internal
* \brief Map standard Pacemaker return code to operation status and OCF code
*
* \param[out] event Executor event whose status and return code should be set
* \param[in] rc Standard Pacemaker return code
*/
void
controld_rc2event(lrmd_event_data_t *event, int rc)
{
/* This is called for cleanup requests from controller peers/clients, not
* for resource actions, so no exit reason is needed.
*/
switch (rc) {
case pcmk_rc_ok:
lrmd__set_result(event, PCMK_OCF_OK, PCMK_EXEC_DONE, NULL);
break;
case EACCES:
lrmd__set_result(event, PCMK_OCF_INSUFFICIENT_PRIV,
PCMK_EXEC_ERROR, NULL);
break;
default:
lrmd__set_result(event, PCMK_OCF_UNKNOWN_ERROR, PCMK_EXEC_ERROR,
NULL);
break;
}
}
/*!
* \internal
* \brief Trigger a new transition after CIB status was deleted
*
* If a CIB status delete was not expected (as part of the transition graph),
* trigger a new transition by updating the (arbitrary) "last-lrm-refresh"
* cluster property.
*
* \param[in] from_sys IPC name that requested the delete
* \param[in] rsc_id Resource whose status was deleted (for logging only)
*/
void
controld_trigger_delete_refresh(const char *from_sys, const char *rsc_id)
{
if (!pcmk__str_eq(from_sys, CRM_SYSTEM_TENGINE, pcmk__str_casei)) {
char *now_s = crm_strdup_printf("%lld", (long long) time(NULL));
crm_debug("Triggering a refresh after %s cleaned %s", from_sys, rsc_id);
cib__update_node_attr(logger_out, fsa_cib_conn, cib_none, XML_CIB_TAG_CRMCONFIG,
NULL, NULL, NULL, NULL, "last-lrm-refresh", now_s, NULL, NULL);
free(now_s);
}
}
static void
notify_deleted(lrm_state_t * lrm_state, ha_msg_input_t * input, const char *rsc_id, int rc)
{
lrmd_event_data_t *op = NULL;
const char *from_sys = crm_element_value(input->msg, F_CRM_SYS_FROM);
const char *from_host = crm_element_value(input->msg, F_CRM_HOST_FROM);
crm_info("Notifying %s on %s that %s was%s deleted",
from_sys, (from_host? from_host : "localhost"), rsc_id,
((rc == pcmk_ok)? "" : " not"));
op = construct_op(lrm_state, input->xml, rsc_id, CRMD_ACTION_DELETE);
controld_rc2event(op, pcmk_legacy2rc(rc));
controld_ack_event_directly(from_host, from_sys, NULL, op, rsc_id);
lrmd_free_event(op);
controld_trigger_delete_refresh(from_sys, rsc_id);
}
static gboolean
lrm_remove_deleted_rsc(gpointer key, gpointer value, gpointer user_data)
{
struct delete_event_s *event = user_data;
struct pending_deletion_op_s *op = value;
if (pcmk__str_eq(event->rsc, op->rsc, pcmk__str_none)) {
notify_deleted(event->lrm_state, op->input, event->rsc, event->rc);
return TRUE;
}
return FALSE;
}
static gboolean
lrm_remove_deleted_op(gpointer key, gpointer value, gpointer user_data)
{
const char *rsc = user_data;
active_op_t *pending = value;
if (pcmk__str_eq(rsc, pending->rsc_id, pcmk__str_none)) {
crm_info("Removing op %s:%d for deleted resource %s",
pending->op_key, pending->call_id, rsc);
return TRUE;
}
return FALSE;
}
static void
delete_rsc_entry(lrm_state_t * lrm_state, ha_msg_input_t * input, const char *rsc_id,
GHashTableIter * rsc_gIter, int rc, const char *user_name)
{
struct delete_event_s event;
CRM_CHECK(rsc_id != NULL, return);
if (rc == pcmk_ok) {
char *rsc_id_copy = strdup(rsc_id);
if (rsc_gIter) {
g_hash_table_iter_remove(rsc_gIter);
} else {
g_hash_table_remove(lrm_state->resource_history, rsc_id_copy);
}
controld_delete_resource_history(rsc_id_copy, lrm_state->node_name,
user_name, crmd_cib_smart_opt());
g_hash_table_foreach_remove(lrm_state->pending_ops, lrm_remove_deleted_op, rsc_id_copy);
free(rsc_id_copy);
}
if (input) {
notify_deleted(lrm_state, input, rsc_id, rc);
}
event.rc = rc;
event.rsc = rsc_id;
event.lrm_state = lrm_state;
g_hash_table_foreach_remove(lrm_state->deletion_ops, lrm_remove_deleted_rsc, &event);
}
/*!
* \internal
* \brief Erase an LRM history entry from the CIB, given the operation data
*
* \param[in] op Operation whose history should be deleted
*/
static void
erase_lrm_history_by_op(const lrmd_event_data_t *op)
{
xmlNode *xml_top = NULL;
CRM_CHECK(op != NULL, return);
xml_top = create_xml_node(NULL, XML_LRM_TAG_RSC_OP);
crm_xml_add_int(xml_top, XML_LRM_ATTR_CALLID, op->call_id);
crm_xml_add(xml_top, XML_ATTR_TRANSITION_KEY, op->user_data);
if (op->interval_ms > 0) {
char *op_id = pcmk__op_key(op->rsc_id, op->op_type, op->interval_ms);
/* Avoid deleting last_failure too (if it was a result of this recurring op failing) */
crm_xml_add(xml_top, XML_ATTR_ID, op_id);
free(op_id);
}
crm_debug("Erasing resource operation history for " PCMK__OP_FMT " (call=%d)",
op->rsc_id, op->op_type, op->interval_ms, op->call_id);
fsa_cib_conn->cmds->remove(fsa_cib_conn, XML_CIB_TAG_STATUS, xml_top,
cib_quorum_override);
crm_log_xml_trace(xml_top, "op:cancel");
free_xml(xml_top);
}
/* Define xpath to find LRM resource history entry by node and resource */
#define XPATH_HISTORY \
"/" XML_TAG_CIB "/" XML_CIB_TAG_STATUS \
"/" XML_CIB_TAG_STATE "[@" XML_ATTR_UNAME "='%s']" \
"/" XML_CIB_TAG_LRM "/" XML_LRM_TAG_RESOURCES \
"/" XML_LRM_TAG_RESOURCE "[@" XML_ATTR_ID "='%s']" \
"/" XML_LRM_TAG_RSC_OP
/* ... and also by operation key */
#define XPATH_HISTORY_ID XPATH_HISTORY \
"[@" XML_ATTR_ID "='%s']"
/* ... and also by operation key and operation call ID */
#define XPATH_HISTORY_CALL XPATH_HISTORY \
"[@" XML_ATTR_ID "='%s' and @" XML_LRM_ATTR_CALLID "='%d']"
/* ... and also by operation key and original operation key */
#define XPATH_HISTORY_ORIG XPATH_HISTORY \
"[@" XML_ATTR_ID "='%s' and @" XML_LRM_ATTR_TASK_KEY "='%s']"
/*!
* \internal
* \brief Erase an LRM history entry from the CIB, given operation identifiers
*
* \param[in] lrm_state LRM state of the node to clear history for
* \param[in] rsc_id Name of resource to clear history for
* \param[in] key Operation key of operation to clear history for
* \param[in] orig_op If specified, delete only if it has this original op
* \param[in] call_id If specified, delete entry only if it has this call ID
*/
static void
erase_lrm_history_by_id(const lrm_state_t *lrm_state, const char *rsc_id,
const char *key, const char *orig_op, int call_id)
{
char *op_xpath = NULL;
CRM_CHECK((rsc_id != NULL) && (key != NULL), return);
if (call_id > 0) {
op_xpath = crm_strdup_printf(XPATH_HISTORY_CALL,
lrm_state->node_name, rsc_id, key,
call_id);
} else if (orig_op) {
op_xpath = crm_strdup_printf(XPATH_HISTORY_ORIG,
lrm_state->node_name, rsc_id, key,
orig_op);
} else {
op_xpath = crm_strdup_printf(XPATH_HISTORY_ID,
lrm_state->node_name, rsc_id, key);
}
crm_debug("Erasing resource operation history for %s on %s (call=%d)",
key, rsc_id, call_id);
fsa_cib_conn->cmds->remove(fsa_cib_conn, op_xpath, NULL,
cib_quorum_override | cib_xpath);
free(op_xpath);
}
static inline gboolean
last_failed_matches_op(rsc_history_t *entry, const char *op, guint interval_ms)
{
if (entry == NULL) {
return FALSE;
}
if (op == NULL) {
return TRUE;
}
return (pcmk__str_eq(op, entry->failed->op_type, pcmk__str_casei)
&& (interval_ms == entry->failed->interval_ms));
}
/*!
* \internal
* \brief Clear a resource's last failure
*
* Erase a resource's last failure on a particular node from both the
* LRM resource history in the CIB, and the resource history remembered
* for the LRM state.
*
* \param[in] rsc_id Resource name
* \param[in] node_name Node name
* \param[in] operation If specified, only clear if matching this operation
* \param[in] interval_ms If operation is specified, it has this interval
*/
void
lrm_clear_last_failure(const char *rsc_id, const char *node_name,
const char *operation, guint interval_ms)
{
char *op_key = NULL;
char *orig_op_key = NULL;
lrm_state_t *lrm_state = NULL;
lrm_state = lrm_state_find(node_name);
if (lrm_state == NULL) {
return;
}
/* Erase from CIB */
op_key = pcmk__op_key(rsc_id, "last_failure", 0);
if (operation) {
orig_op_key = pcmk__op_key(rsc_id, operation, interval_ms);
}
erase_lrm_history_by_id(lrm_state, rsc_id, op_key, orig_op_key, 0);
free(op_key);
free(orig_op_key);
/* Remove from memory */
if (lrm_state->resource_history) {
rsc_history_t *entry = g_hash_table_lookup(lrm_state->resource_history,
rsc_id);
if (last_failed_matches_op(entry, operation, interval_ms)) {
lrmd_free_event(entry->failed);
entry->failed = NULL;
}
}
}
/* Returns: gboolean - cancellation is in progress */
static gboolean
cancel_op(lrm_state_t * lrm_state, const char *rsc_id, const char *key, int op, gboolean remove)
{
int rc = pcmk_ok;
char *local_key = NULL;
active_op_t *pending = NULL;
CRM_CHECK(op != 0, return FALSE);
CRM_CHECK(rsc_id != NULL, return FALSE);
if (key == NULL) {
local_key = make_stop_id(rsc_id, op);
key = local_key;
}
pending = g_hash_table_lookup(lrm_state->pending_ops, key);
if (pending) {
if (remove && !pcmk_is_set(pending->flags, active_op_remove)) {
controld_set_active_op_flags(pending, active_op_remove);
crm_debug("Scheduling %s for removal", key);
}
if (pcmk_is_set(pending->flags, active_op_cancelled)) {
crm_debug("Operation %s already cancelled", key);
free(local_key);
return FALSE;
}
controld_set_active_op_flags(pending, active_op_cancelled);
} else {
crm_info("No pending op found for %s", key);
free(local_key);
return FALSE;
}
crm_debug("Cancelling op %d for %s (%s)", op, rsc_id, key);
rc = lrm_state_cancel(lrm_state, pending->rsc_id, pending->op_type,
pending->interval_ms);
if (rc == pcmk_ok) {
crm_debug("Op %d for %s (%s): cancelled", op, rsc_id, key);
free(local_key);
return TRUE;
}
crm_debug("Op %d for %s (%s): Nothing to cancel", op, rsc_id, key);
/* The caller needs to make sure the entry is
* removed from the pending_ops list
*
* Usually by returning TRUE inside the worker function
* supplied to g_hash_table_foreach_remove()
*
* Not removing the entry from pending_ops will block
* the node from shutting down
*/
free(local_key);
return FALSE;
}
struct cancel_data {
gboolean done;
gboolean remove;
const char *key;
lrmd_rsc_info_t *rsc;
lrm_state_t *lrm_state;
};
static gboolean
cancel_action_by_key(gpointer key, gpointer value, gpointer user_data)
{
gboolean remove = FALSE;
struct cancel_data *data = user_data;
active_op_t *op = value;
if (pcmk__str_eq(op->op_key, data->key, pcmk__str_none)) {
data->done = TRUE;
remove = !cancel_op(data->lrm_state, data->rsc->id, key, op->call_id, data->remove);
}
return remove;
}
static gboolean
cancel_op_key(lrm_state_t * lrm_state, lrmd_rsc_info_t * rsc, const char *key, gboolean remove)
{
guint removed = 0;
struct cancel_data data;
CRM_CHECK(rsc != NULL, return FALSE);
CRM_CHECK(key != NULL, return FALSE);
data.key = key;
data.rsc = rsc;
data.done = FALSE;
data.remove = remove;
data.lrm_state = lrm_state;
removed = g_hash_table_foreach_remove(lrm_state->pending_ops, cancel_action_by_key, &data);
crm_trace("Removed %u op cache entries, new size: %u",
removed, g_hash_table_size(lrm_state->pending_ops));
return data.done;
}
/*!
* \internal
* \brief Retrieve resource information from LRM
*
* \param[in,out] lrm_state Executor connection state to use
* \param[in] rsc_xml XML containing resource configuration
* \param[in] do_create If true, register resource if not already
* \param[out] rsc_info Where to store information obtained from executor
*
* \retval pcmk_ok Success (and rsc_info holds newly allocated result)
* \retval -EINVAL Required information is missing from arguments
* \retval -ENOTCONN No active connection to LRM
* \retval -ENODEV Resource not found
* \retval -errno Error communicating with executor when registering resource
*
* \note Caller is responsible for freeing result on success.
*/
static int
get_lrm_resource(lrm_state_t *lrm_state, const xmlNode *rsc_xml,
gboolean do_create, lrmd_rsc_info_t **rsc_info)
{
const char *id = ID(rsc_xml);
CRM_CHECK(lrm_state && rsc_xml && rsc_info, return -EINVAL);
CRM_CHECK(id, return -EINVAL);
if (lrm_state_is_connected(lrm_state) == FALSE) {
return -ENOTCONN;
}
crm_trace("Retrieving resource information for %s from the executor", id);
*rsc_info = lrm_state_get_rsc_info(lrm_state, id, 0);
// If resource isn't known by ID, try clone name, if provided
if (!*rsc_info) {
const char *long_id = crm_element_value(rsc_xml, XML_ATTR_ID_LONG);
if (long_id) {
*rsc_info = lrm_state_get_rsc_info(lrm_state, long_id, 0);
}
}
if ((*rsc_info == NULL) && do_create) {
const char *class = crm_element_value(rsc_xml, XML_AGENT_ATTR_CLASS);
const char *provider = crm_element_value(rsc_xml, XML_AGENT_ATTR_PROVIDER);
const char *type = crm_element_value(rsc_xml, XML_ATTR_TYPE);
int rc;
crm_trace("Registering resource %s with the executor", id);
rc = lrm_state_register_rsc(lrm_state, id, class, provider, type,
lrmd_opt_drop_recurring);
if (rc != pcmk_ok) {
fsa_data_t *msg_data = NULL;
crm_err("Could not register resource %s with the executor on %s: %s "
CRM_XS " rc=%d",
id, lrm_state->node_name, pcmk_strerror(rc), rc);
/* Register this as an internal error if this involves the local
* executor. Otherwise, we're likely dealing with an unresponsive
* remote node, which is not an FSA failure.
*/
if (lrm_state_is_local(lrm_state) == TRUE) {
register_fsa_error(C_FSA_INTERNAL, I_FAIL, NULL);
}
return rc;
}
*rsc_info = lrm_state_get_rsc_info(lrm_state, id, 0);
}
return *rsc_info? pcmk_ok : -ENODEV;
}
static void
delete_resource(lrm_state_t * lrm_state,
const char *id,
lrmd_rsc_info_t * rsc,
GHashTableIter * gIter,
const char *sys,
const char *user,
ha_msg_input_t * request,
gboolean unregister)
{
int rc = pcmk_ok;
crm_info("Removing resource %s from executor for %s%s%s",
id, sys, (user? " as " : ""), (user? user : ""));
if (rsc && unregister) {
rc = lrm_state_unregister_rsc(lrm_state, id, 0);
}
if (rc == pcmk_ok) {
crm_trace("Resource %s deleted from executor", id);
} else if (rc == -EINPROGRESS) {
crm_info("Deletion of resource '%s' from executor is pending", id);
if (request) {
struct pending_deletion_op_s *op = NULL;
char *ref = crm_element_value_copy(request->msg, XML_ATTR_REFERENCE);
op = calloc(1, sizeof(struct pending_deletion_op_s));
op->rsc = strdup(rsc->id);
op->input = copy_ha_msg_input(request);
g_hash_table_insert(lrm_state->deletion_ops, ref, op);
}
return;
} else {
crm_warn("Could not delete '%s' from executor for %s%s%s: %s "
CRM_XS " rc=%d", id, sys, (user? " as " : ""),
(user? user : ""), pcmk_strerror(rc), rc);
}
delete_rsc_entry(lrm_state, request, id, gIter, rc, user);
}
static int
get_fake_call_id(lrm_state_t *lrm_state, const char *rsc_id)
{
int call_id = 999999999;
rsc_history_t *entry = NULL;
if(lrm_state) {
entry = g_hash_table_lookup(lrm_state->resource_history, rsc_id);
}
/* Make sure the call id is greater than the last successful operation,
* otherwise the failure will not result in a possible recovery of the resource
* as it could appear the failure occurred before the successful start */
if (entry) {
call_id = entry->last_callid + 1;
}
if (call_id < 0) {
call_id = 1;
}
return call_id;
}
static void
fake_op_status(lrm_state_t *lrm_state, lrmd_event_data_t *op, int op_status,
enum ocf_exitcode op_exitcode, const char *exit_reason)
{
op->call_id = get_fake_call_id(lrm_state, op->rsc_id);
op->t_run = time(NULL);
op->t_rcchange = op->t_run;
lrmd__set_result(op, op_exitcode, op_status, exit_reason);
}
static void
force_reprobe(lrm_state_t *lrm_state, const char *from_sys,
const char *from_host, const char *user_name,
gboolean is_remote_node)
{
GHashTableIter gIter;
rsc_history_t *entry = NULL;
crm_info("Clearing resource history on node %s", lrm_state->node_name);
g_hash_table_iter_init(&gIter, lrm_state->resource_history);
while (g_hash_table_iter_next(&gIter, NULL, (void **)&entry)) {
/* only unregister the resource during a reprobe if it is not a remote connection
* resource. otherwise unregistering the connection will terminate remote-node
* membership */
gboolean unregister = TRUE;
if (is_remote_lrmd_ra(NULL, NULL, entry->id)) {
lrm_state_t *remote_lrm_state = lrm_state_find(entry->id);
if (remote_lrm_state) {
/* when forcing a reprobe, make sure to clear remote node before
* clearing the remote node's connection resource */
force_reprobe(remote_lrm_state, from_sys, from_host, user_name, TRUE);
}
unregister = FALSE;
}
delete_resource(lrm_state, entry->id, &entry->rsc, &gIter, from_sys,
user_name, NULL, unregister);
}
/* Now delete the copy in the CIB */
controld_delete_node_state(lrm_state->node_name, controld_section_lrm,
cib_scope_local);
// @COMPAT DCs < 1.1.14 need this deleted (in case it was explicitly false)
update_attrd(lrm_state->node_name, CRM_OP_PROBED, NULL, user_name, is_remote_node);
}
/*!
* \internal
* \brief Fail a requested action without actually executing it
*
* For an action that can't be executed, process it similarly to an actual
* execution result, with specified error status (except for notify actions,
* which will always be treated as successful).
*
* \param[in,out] lrm_state Executor connection that action is for
* \param[in] action Action XML from request
* \param[in] rc Desired return code to use
* \param[in] op_status Desired operation status to use
* \param[in] exit_reason Human-friendly detail, if error
*/
static void
synthesize_lrmd_failure(lrm_state_t *lrm_state, const xmlNode *action,
int op_status, enum ocf_exitcode rc,
const char *exit_reason)
{
lrmd_event_data_t *op = NULL;
const char *operation = crm_element_value(action, XML_LRM_ATTR_TASK);
const char *target_node = crm_element_value(action, XML_LRM_ATTR_TARGET);
xmlNode *xml_rsc = find_xml_node(action, XML_CIB_TAG_RESOURCE, TRUE);
if ((xml_rsc == NULL) || (ID(xml_rsc) == NULL)) {
/* @TODO Should we do something else, like direct ack? */
crm_info("Can't fake %s failure (%d) on %s without resource configuration",
crm_element_value(action, XML_LRM_ATTR_TASK_KEY), rc,
target_node);
return;
} else if(operation == NULL) {
/* This probably came from crm_resource -C, nothing to do */
crm_info("Can't fake %s failure (%d) on %s without operation",
ID(xml_rsc), rc, target_node);
return;
}
op = construct_op(lrm_state, action, ID(xml_rsc), operation);
if (pcmk__str_eq(operation, RSC_NOTIFY, pcmk__str_casei)) { // Notifications can't fail
fake_op_status(lrm_state, op, PCMK_EXEC_DONE, PCMK_OCF_OK, NULL);
} else {
fake_op_status(lrm_state, op, op_status, rc, exit_reason);
}
crm_info("Faking " PCMK__OP_FMT " result (%d) on %s",
op->rsc_id, op->op_type, op->interval_ms, op->rc, target_node);
// Process the result as if it came from the LRM
process_lrm_event(lrm_state, op, NULL, action);
lrmd_free_event(op);
}
/*!
* \internal
* \brief Get target of an LRM operation
*
* \param[in] xml LRM operation data XML
*
* \return LRM operation target node name (local node or Pacemaker Remote node)
*/
static const char *
lrm_op_target(const xmlNode *xml)
{
const char *target = NULL;
if (xml) {
target = crm_element_value(xml, XML_LRM_ATTR_TARGET);
}
if (target == NULL) {
target = fsa_our_uname;
}
return target;
}
static void
fail_lrm_resource(xmlNode *xml, lrm_state_t *lrm_state, const char *user_name,
const char *from_host, const char *from_sys)
{
lrmd_event_data_t *op = NULL;
lrmd_rsc_info_t *rsc = NULL;
xmlNode *xml_rsc = find_xml_node(xml, XML_CIB_TAG_RESOURCE, TRUE);
CRM_CHECK(xml_rsc != NULL, return);
/* The executor simply executes operations and reports the results, without
* any concept of success or failure, so to fail a resource, we must fake
* what a failure looks like.
*
* To do this, we create a fake executor operation event for the resource,
* and pass that event to the executor client callback so it will be
* processed as if it came from the executor.
*/
op = construct_op(lrm_state, xml, ID(xml_rsc), "asyncmon");
free((char*) op->user_data);
op->user_data = NULL;
op->interval_ms = 0;
if (user_name && !pcmk__is_privileged(user_name)) {
crm_err("%s does not have permission to fail %s", user_name, ID(xml_rsc));
fake_op_status(lrm_state, op, PCMK_EXEC_ERROR,
PCMK_OCF_INSUFFICIENT_PRIV,
"Unprivileged user cannot fail resources");
controld_ack_event_directly(from_host, from_sys, NULL, op, ID(xml_rsc));
lrmd_free_event(op);
return;
}
if (get_lrm_resource(lrm_state, xml_rsc, TRUE, &rsc) == pcmk_ok) {
crm_info("Failing resource %s...", rsc->id);
fake_op_status(lrm_state, op, PCMK_EXEC_DONE, PCMK_OCF_UNKNOWN_ERROR,
"Simulated failure");
process_lrm_event(lrm_state, op, NULL, xml);
op->rc = PCMK_OCF_OK; // The request to fail the resource succeeded
lrmd_free_rsc_info(rsc);
} else {
crm_info("Cannot find/create resource in order to fail it...");
crm_log_xml_warn(xml, "bad input");
fake_op_status(lrm_state, op, PCMK_EXEC_ERROR, PCMK_OCF_UNKNOWN_ERROR,
"Cannot fail unknown resource");
}
controld_ack_event_directly(from_host, from_sys, NULL, op, ID(xml_rsc));
lrmd_free_event(op);
}
-static void
-handle_refresh_op(lrm_state_t *lrm_state, const char *user_name,
- const char *from_host, const char *from_sys)
-{
- int rc = pcmk_ok;
- xmlNode *fragment = do_lrm_query_internal(lrm_state, node_update_all);
-
- fsa_cib_update(XML_CIB_TAG_STATUS, fragment, cib_quorum_override, rc, user_name);
- crm_info("Forced a local resource history refresh: call=%d", rc);
-
- if (!pcmk__str_eq(CRM_SYSTEM_CRMD, from_sys, pcmk__str_casei)) {
- xmlNode *reply = create_request(CRM_OP_INVOKE_LRM, fragment, from_host,
- from_sys, CRM_SYSTEM_LRMD,
- fsa_our_uuid);
-
- crm_debug("ACK'ing refresh from %s (%s)", from_sys, from_host);
-
- if (relay_message(reply, TRUE) == FALSE) {
- crm_log_xml_err(reply, "Unable to route reply");
- }
- free_xml(reply);
- }
-
- free_xml(fragment);
-}
-
-static void
-handle_query_op(xmlNode *msg, lrm_state_t *lrm_state)
-{
- xmlNode *data = do_lrm_query_internal(lrm_state, node_update_all);
- xmlNode *reply = create_reply(msg, data);
-
- if (relay_message(reply, TRUE) == FALSE) {
- crm_err("Unable to route reply");
- crm_log_xml_err(reply, "reply");
- }
- free_xml(reply);
- free_xml(data);
-}
-
static void
handle_reprobe_op(lrm_state_t *lrm_state, const char *from_sys,
const char *from_host, const char *user_name,
gboolean is_remote_node)
{
crm_notice("Forcing the status of all resources to be redetected");
force_reprobe(lrm_state, from_sys, from_host, user_name, is_remote_node);
if (!pcmk__strcase_any_of(from_sys, CRM_SYSTEM_PENGINE, CRM_SYSTEM_TENGINE, NULL)) {
xmlNode *reply = create_request(CRM_OP_INVOKE_LRM, NULL, from_host,
from_sys, CRM_SYSTEM_LRMD,
fsa_our_uuid);
crm_debug("ACK'ing re-probe from %s (%s)", from_sys, from_host);
if (relay_message(reply, TRUE) == FALSE) {
crm_log_xml_err(reply, "Unable to route reply");
}
free_xml(reply);
}
}
static bool do_lrm_cancel(ha_msg_input_t *input, lrm_state_t *lrm_state,
lrmd_rsc_info_t *rsc, const char *from_host, const char *from_sys)
{
char *op_key = NULL;
char *meta_key = NULL;
int call = 0;
const char *call_id = NULL;
const char *op_task = NULL;
guint interval_ms = 0;
gboolean in_progress = FALSE;
xmlNode *params = find_xml_node(input->xml, XML_TAG_ATTRS, TRUE);
CRM_CHECK(params != NULL, return FALSE);
meta_key = crm_meta_name(XML_LRM_ATTR_TASK);
op_task = crm_element_value(params, meta_key);
free(meta_key);
CRM_CHECK(op_task != NULL, return FALSE);
meta_key = crm_meta_name(XML_LRM_ATTR_INTERVAL_MS);
if (crm_element_value_ms(params, meta_key, &interval_ms) != pcmk_ok) {
free(meta_key);
return FALSE;
}
free(meta_key);
op_key = pcmk__op_key(rsc->id, op_task, interval_ms);
meta_key = crm_meta_name(XML_LRM_ATTR_CALLID);
call_id = crm_element_value(params, meta_key);
free(meta_key);
crm_debug("Scheduler requested op %s (call=%s) be cancelled",
op_key, (call_id? call_id : "NA"));
pcmk__scan_min_int(call_id, &call, 0);
if (call == 0) {
// Normal case when the scheduler cancels a recurring op
in_progress = cancel_op_key(lrm_state, rsc, op_key, TRUE);
} else {
// Normal case when the scheduler cancels an orphan op
in_progress = cancel_op(lrm_state, rsc->id, NULL, call, TRUE);
}
// Acknowledge cancellation operation if for a remote connection resource
if (!in_progress || is_remote_lrmd_ra(NULL, NULL, rsc->id)) {
char *op_id = make_stop_id(rsc->id, call);
if (is_remote_lrmd_ra(NULL, NULL, rsc->id) == FALSE) {
crm_info("Nothing known about operation %d for %s", call, op_key);
}
erase_lrm_history_by_id(lrm_state, rsc->id, op_key, NULL, call);
send_task_ok_ack(lrm_state, input, rsc->id, rsc, op_task,
from_host, from_sys);
/* needed at least for cancellation of a remote operation */
g_hash_table_remove(lrm_state->pending_ops, op_id);
free(op_id);
} else {
/* No ack is needed since abcdaa8, but peers with older versions
* in a rolling upgrade need one. We didn't bump the feature set
* at that commit, so we can only compare against the previous
* CRM version (3.0.8). If any peers have feature set 3.0.9 but
* not abcdaa8, they will time out waiting for the ack (no
* released versions of Pacemaker are affected).
*/
const char *peer_version = crm_element_value(params, XML_ATTR_CRM_VERSION);
if (compare_version(peer_version, "3.0.8") <= 0) {
crm_info("Sending compatibility ack for %s cancellation to %s (CRM version %s)",
op_key, from_host, peer_version);
send_task_ok_ack(lrm_state, input, rsc->id, rsc, op_task,
from_host, from_sys);
}
}
free(op_key);
return TRUE;
}
static void
do_lrm_delete(ha_msg_input_t *input, lrm_state_t *lrm_state,
lrmd_rsc_info_t *rsc, const char *from_sys, const char *from_host,
bool crm_rsc_delete, const char *user_name)
{
gboolean unregister = TRUE;
int cib_rc = controld_delete_resource_history(rsc->id, lrm_state->node_name,
user_name,
cib_dryrun|cib_sync_call);
if (cib_rc != pcmk_rc_ok) {
lrmd_event_data_t *op = NULL;
op = construct_op(lrm_state, input->xml, rsc->id, CRMD_ACTION_DELETE);
/* These are resource clean-ups, not actions, so no exit reason is
* needed.
*/
lrmd__set_result(op, pcmk_rc2ocf(cib_rc), PCMK_EXEC_ERROR, NULL);
controld_ack_event_directly(from_host, from_sys, NULL, op, rsc->id);
lrmd_free_event(op);
return;
}
if (crm_rsc_delete && is_remote_lrmd_ra(NULL, NULL, rsc->id)) {
unregister = FALSE;
}
delete_resource(lrm_state, rsc->id, rsc, NULL, from_sys,
user_name, input, unregister);
}
+// User data for asynchronous metadata execution
+struct metadata_cb_data {
+ lrmd_rsc_info_t *rsc; // Copy of resource information
+ xmlNode *input_xml; // Copy of FSA input XML
+};
+
+static struct metadata_cb_data *
+new_metadata_cb_data(lrmd_rsc_info_t *rsc, xmlNode *input_xml)
+{
+ struct metadata_cb_data *data = NULL;
+
+ data = calloc(1, sizeof(struct metadata_cb_data));
+ CRM_ASSERT(data != NULL);
+ data->input_xml = copy_xml(input_xml);
+ data->rsc = lrmd_copy_rsc_info(rsc);
+ return data;
+}
+
+static void
+free_metadata_cb_data(struct metadata_cb_data *data)
+{
+ lrmd_free_rsc_info(data->rsc);
+ free_xml(data->input_xml);
+ free(data);
+}
+
+/*!
+ * \internal
+ * \brief Execute an action after metadata has been retrieved
+ *
+ * \param[in] pid Ignored
+ * \param[in] result Result of metadata action
+ * \param[in] user_data Metadata callback data
+ */
+static void
+metadata_complete(int pid, const pcmk__action_result_t *result, void *user_data)
+{
+ struct metadata_cb_data *data = (struct metadata_cb_data *) user_data;
+
+ struct ra_metadata_s *md = NULL;
+ lrm_state_t *lrm_state = lrm_state_find(lrm_op_target(data->input_xml));
+
+ if ((lrm_state != NULL) && pcmk__result_ok(result)) {
+ md = controld_cache_metadata(lrm_state->metadata_cache, data->rsc,
+ result->action_stdout);
+ }
+ do_lrm_rsc_op(lrm_state, data->rsc, data->input_xml, md);
+ free_metadata_cb_data(data);
+}
+
/* A_LRM_INVOKE */
void
do_lrm_invoke(long long action,
enum crmd_fsa_cause cause,
enum crmd_fsa_state cur_state,
enum crmd_fsa_input current_input, fsa_data_t * msg_data)
{
lrm_state_t *lrm_state = NULL;
const char *crm_op = NULL;
const char *from_sys = NULL;
const char *from_host = NULL;
const char *operation = NULL;
ha_msg_input_t *input = fsa_typed_data(fsa_dt_ha_msg);
const char *user_name = NULL;
const char *target_node = NULL;
gboolean is_remote_node = FALSE;
bool crm_rsc_delete = FALSE;
target_node = lrm_op_target(input->xml);
is_remote_node = !pcmk__str_eq(target_node, fsa_our_uname,
pcmk__str_casei);
lrm_state = lrm_state_find(target_node);
if ((lrm_state == NULL) && is_remote_node) {
crm_err("Failing action because local node has never had connection to remote node %s",
target_node);
synthesize_lrmd_failure(NULL, input->xml, PCMK_EXEC_NOT_CONNECTED,
PCMK_OCF_UNKNOWN_ERROR,
"Local node has no connection to remote");
return;
}
CRM_ASSERT(lrm_state != NULL);
user_name = pcmk__update_acl_user(input->msg, F_CRM_USER, NULL);
crm_op = crm_element_value(input->msg, F_CRM_TASK);
from_sys = crm_element_value(input->msg, F_CRM_SYS_FROM);
if (!pcmk__str_eq(from_sys, CRM_SYSTEM_TENGINE, pcmk__str_none)) {
from_host = crm_element_value(input->msg, F_CRM_HOST_FROM);
}
if (pcmk__str_eq(crm_op, CRM_OP_LRM_DELETE, pcmk__str_none)) {
if (!pcmk__str_eq(from_sys, CRM_SYSTEM_TENGINE, pcmk__str_none)) {
crm_rsc_delete = TRUE; // from crm_resource
}
operation = CRMD_ACTION_DELETE;
} else if (input->xml != NULL) {
operation = crm_element_value(input->xml, XML_LRM_ATTR_TASK);
}
CRM_CHECK(!pcmk__str_empty(crm_op) || !pcmk__str_empty(operation), return);
crm_trace("'%s' execution request from %s as %s user",
pcmk__s(crm_op, operation),
pcmk__s(from_sys, "unknown subsystem"),
pcmk__s(user_name, "current"));
if (pcmk__str_eq(crm_op, CRM_OP_LRM_FAIL, pcmk__str_none)) {
fail_lrm_resource(input->xml, lrm_state, user_name, from_host,
from_sys);
} else if (pcmk__str_eq(crm_op, CRM_OP_LRM_REFRESH, pcmk__str_none)) {
- handle_refresh_op(lrm_state, user_name, from_host, from_sys);
-
- } else if (pcmk__str_eq(crm_op, CRM_OP_LRM_QUERY, pcmk__str_none)) {
- handle_query_op(input->msg, lrm_state);
+ /* @COMPAT This can only be sent by crm_resource --refresh on a
+ * Pacemaker Remote node running Pacemaker 1.1.9, which is extremely
+ * unlikely. It previously would cause the controller to re-write its
+ * resource history to the CIB. Just ignore it.
+ */
+ crm_notice("Ignoring refresh request from Pacemaker Remote 1.1.9 node");
// @COMPAT DCs <1.1.14 in a rolling upgrade might schedule this op
} else if (pcmk__str_eq(operation, CRM_OP_PROBED, pcmk__str_none)) {
update_attrd(lrm_state->node_name, CRM_OP_PROBED, XML_BOOLEAN_TRUE,
user_name, is_remote_node);
} else if (pcmk__str_eq(crm_op, CRM_OP_REPROBE, pcmk__str_none)
|| pcmk__str_eq(operation, CRM_OP_REPROBE, pcmk__str_none)) {
handle_reprobe_op(lrm_state, from_sys, from_host, user_name,
is_remote_node);
} else if (operation != NULL) {
lrmd_rsc_info_t *rsc = NULL;
xmlNode *xml_rsc = find_xml_node(input->xml, XML_CIB_TAG_RESOURCE, TRUE);
gboolean create_rsc = !pcmk__str_eq(operation, CRMD_ACTION_DELETE,
pcmk__str_none);
int rc;
// We can't return anything meaningful without a resource ID
CRM_CHECK(xml_rsc && ID(xml_rsc), return);
rc = get_lrm_resource(lrm_state, xml_rsc, create_rsc, &rsc);
if (rc == -ENOTCONN) {
synthesize_lrmd_failure(lrm_state, input->xml,
PCMK_EXEC_NOT_CONNECTED,
PCMK_OCF_UNKNOWN_ERROR,
"Not connected to remote executor");
return;
} else if ((rc < 0) && !create_rsc) {
/* Delete of malformed or nonexistent resource
* (deleting something that does not exist is a success)
*/
crm_notice("Not registering resource '%s' for a %s event "
CRM_XS " get-rc=%d (%s) transition-key=%s",
ID(xml_rsc), operation,
rc, pcmk_strerror(rc), ID(input->xml));
delete_rsc_entry(lrm_state, input, ID(xml_rsc), NULL, pcmk_ok,
user_name);
return;
} else if (rc == -EINVAL) {
// Resource operation on malformed resource
crm_err("Invalid resource definition for %s", ID(xml_rsc));
crm_log_xml_warn(input->msg, "invalid resource");
synthesize_lrmd_failure(lrm_state, input->xml, PCMK_EXEC_ERROR,
PCMK_OCF_NOT_CONFIGURED, // fatal error
"Invalid resource definition");
return;
} else if (rc < 0) {
// Error communicating with the executor
crm_err("Could not register resource '%s' with executor: %s "
CRM_XS " rc=%d",
ID(xml_rsc), pcmk_strerror(rc), rc);
crm_log_xml_warn(input->msg, "failed registration");
synthesize_lrmd_failure(lrm_state, input->xml, PCMK_EXEC_ERROR,
PCMK_OCF_INVALID_PARAM, // hard error
"Could not register resource with executor");
return;
}
if (pcmk__str_eq(operation, CRMD_ACTION_CANCEL, pcmk__str_none)) {
if (!do_lrm_cancel(input, lrm_state, rsc, from_host, from_sys)) {
crm_log_xml_warn(input->xml, "Bad command");
}
} else if (pcmk__str_eq(operation, CRMD_ACTION_DELETE, pcmk__str_none)) {
do_lrm_delete(input, lrm_state, rsc, from_sys, from_host,
crm_rsc_delete, user_name);
- } else if (pcmk__str_any_of(operation, CRMD_ACTION_RELOAD,
- CRMD_ACTION_RELOAD_AGENT, NULL)) {
- /* Pre-2.1.0 DCs will schedule reload actions only, and 2.1.0+ DCs
- * will schedule reload-agent actions only. In either case, we need
- * to map that to whatever the resource agent actually supports.
- * Default to the OCF 1.1 name.
- */
+ } else {
struct ra_metadata_s *md = NULL;
- const char *reload_name = CRMD_ACTION_RELOAD_AGENT;
- md = controld_get_rsc_metadata(lrm_state, rsc,
- controld_metadata_from_cache);
- if ((md != NULL)
- && pcmk_is_set(md->ra_flags, ra_supports_legacy_reload)) {
- reload_name = CRMD_ACTION_RELOAD;
+ /* Getting metadata from cache is OK except for start actions --
+ * always refresh from the agent for those, in case the resource
+ * agent was updated.
+ *
+ * @TODO Only refresh metadata for starts if the agent actually
+ * changed (using something like inotify, or a hash or modification
+ * time of the agent executable).
+ */
+ if (strcmp(operation, CRMD_ACTION_START) != 0) {
+ md = controld_get_rsc_metadata(lrm_state, rsc,
+ controld_metadata_from_cache);
}
- do_lrm_rsc_op(lrm_state, rsc, reload_name, input->xml);
- } else {
- do_lrm_rsc_op(lrm_state, rsc, operation, input->xml);
+ if ((md == NULL) && crm_op_needs_metadata(rsc->standard,
+ operation)) {
+ /* Most likely, we'll need the agent metadata to record the
+ * pending operation and the operation result. Get it now rather
+ * than wait until then, so the metadata action doesn't eat into
+ * the real action's timeout.
+ *
+ * @TODO Metadata is retrieved via direct execution of the
+ * agent, which has a couple of related issues: the executor
+ * should execute agents, not the controller; and metadata for
+ * Pacemaker Remote nodes should be collected on those nodes,
+ * not locally.
+ */
+ struct metadata_cb_data *data = NULL;
+
+ data = new_metadata_cb_data(rsc, input->xml);
+ crm_info("Retrieving metadata for %s (%s%s%s:%s) asynchronously",
+ rsc->id, rsc->standard,
+ ((rsc->provider == NULL)? "" : ":"),
+ ((rsc->provider == NULL)? "" : rsc->provider),
+ rsc->type);
+ (void) lrmd__metadata_async(rsc, metadata_complete,
+ (void *) data);
+ } else {
+ do_lrm_rsc_op(lrm_state, rsc, input->xml, md);
+ }
}
lrmd_free_rsc_info(rsc);
} else {
crm_err("Invalid execution request: unknown command '%s' (bug?)",
crm_op);
register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
}
}
#if ENABLE_VERSIONED_ATTRS
static void
resolve_versioned_parameters(lrm_state_t *lrm_state, const char *rsc_id,
const xmlNode *rsc_op, GHashTable *params)
{
/* Resource info *should* already be cached, so we don't get
* executor call */
lrmd_rsc_info_t *rsc = lrm_state_get_rsc_info(lrm_state, rsc_id, 0);
struct ra_metadata_s *metadata;
metadata = controld_get_rsc_metadata(lrm_state, rsc,
controld_metadata_from_cache);
if (metadata) {
xmlNode *versioned_attrs = NULL;
GHashTable *hash = NULL;
char *key = NULL;
char *value = NULL;
GHashTableIter iter;
versioned_attrs = first_named_child(rsc_op, XML_TAG_OP_VER_ATTRS);
hash = pe_unpack_versioned_parameters(versioned_attrs, metadata->ra_version);
g_hash_table_iter_init(&iter, hash);
while (g_hash_table_iter_next(&iter, (gpointer *) &key, (gpointer *) &value)) {
g_hash_table_iter_steal(&iter);
g_hash_table_replace(params, key, value);
}
g_hash_table_destroy(hash);
versioned_attrs = first_named_child(rsc_op, XML_TAG_OP_VER_META);
hash = pe_unpack_versioned_parameters(versioned_attrs, metadata->ra_version);
g_hash_table_iter_init(&iter, hash);
while (g_hash_table_iter_next(&iter, (gpointer *) &key, (gpointer *) &value)) {
g_hash_table_replace(params, crm_meta_name(key), strdup(value));
if (pcmk__str_eq(key, XML_ATTR_TIMEOUT, pcmk__str_casei)) {
pcmk__scan_min_int(value, &op->timeout, 0);
} else if (pcmk__str_eq(key, XML_OP_ATTR_START_DELAY, pcmk__str_casei)) {
pcmk__scan_min_int(value, &op->start_delay, 0);
}
}
g_hash_table_destroy(hash);
versioned_attrs = first_named_child(rsc_op, XML_TAG_RSC_VER_ATTRS);
hash = pe_unpack_versioned_parameters(versioned_attrs, metadata->ra_version);
g_hash_table_iter_init(&iter, hash);
while (g_hash_table_iter_next(&iter, (gpointer *) &key, (gpointer *) &value)) {
g_hash_table_iter_steal(&iter);
g_hash_table_replace(params, key, value);
}
g_hash_table_destroy(hash);
}
lrmd_free_rsc_info(rsc);
}
#endif
static lrmd_event_data_t *
construct_op(const lrm_state_t *lrm_state, const xmlNode *rsc_op,
const char *rsc_id, const char *operation)
{
lrmd_event_data_t *op = NULL;
const char *op_delay = NULL;
const char *op_timeout = NULL;
GHashTable *params = NULL;
xmlNode *primitive = NULL;
const char *class = NULL;
const char *transition = NULL;
CRM_ASSERT(rsc_id && operation);
op = lrmd_new_event(rsc_id, operation, 0);
op->type = lrmd_event_exec_complete;
op->timeout = 0;
op->start_delay = 0;
lrmd__set_result(op, PCMK_OCF_UNKNOWN, PCMK_EXEC_PENDING, NULL);
if (rsc_op == NULL) {
CRM_LOG_ASSERT(pcmk__str_eq(CRMD_ACTION_STOP, operation, pcmk__str_casei));
op->user_data = NULL;
/* the stop_all_resources() case
* by definition there is no DC (or they'd be shutting
* us down).
* So we should put our version here.
*/
op->params = pcmk__strkey_table(free, free);
g_hash_table_insert(op->params, strdup(XML_ATTR_CRM_VERSION), strdup(CRM_FEATURE_SET));
crm_trace("Constructed %s op for %s", operation, rsc_id);
return op;
}
params = xml2list(rsc_op);
g_hash_table_remove(params, CRM_META "_op_target_rc");
op_delay = crm_meta_value(params, XML_OP_ATTR_START_DELAY);
pcmk__scan_min_int(op_delay, &op->start_delay, 0);
op_timeout = crm_meta_value(params, XML_ATTR_TIMEOUT);
pcmk__scan_min_int(op_timeout, &op->timeout, 0);
if (pcmk__guint_from_hash(params, CRM_META "_" XML_LRM_ATTR_INTERVAL_MS, 0,
&(op->interval_ms)) != pcmk_rc_ok) {
op->interval_ms = 0;
}
/* Use pcmk_monitor_timeout instead of meta timeout for stonith
recurring monitor, if set */
primitive = find_xml_node(rsc_op, XML_CIB_TAG_RESOURCE, FALSE);
class = crm_element_value(primitive, XML_AGENT_ATTR_CLASS);
if (pcmk_is_set(pcmk_get_ra_caps(class), pcmk_ra_cap_fence_params)
&& pcmk__str_eq(operation, CRMD_ACTION_STATUS, pcmk__str_casei)
&& (op->interval_ms > 0)) {
op_timeout = g_hash_table_lookup(params, "pcmk_monitor_timeout");
if (op_timeout != NULL) {
op->timeout = crm_get_msec(op_timeout);
}
}
#if ENABLE_VERSIONED_ATTRS
if (lrm_state && !is_remote_lrmd_ra(NULL, NULL, rsc_id)
&& !pcmk__strcase_any_of(op_type, CRMD_ACTION_METADATA, CRMD_ACTION_DELETE,
NULL)) {
resolve_versioned_parameters(lrm_state, rsc_id, rsc_op, params);
}
#endif
if (!pcmk__str_eq(operation, RSC_STOP, pcmk__str_casei)) {
op->params = params;
} else {
rsc_history_t *entry = NULL;
if (lrm_state) {
entry = g_hash_table_lookup(lrm_state->resource_history, rsc_id);
}
/* If we do not have stop parameters cached, use
* whatever we are given */
if (!entry || !entry->stop_params) {
op->params = params;
} else {
/* Copy the cached parameter list so that we stop the resource
* with the old attributes, not the new ones */
op->params = pcmk__strkey_table(free, free);
g_hash_table_foreach(params, copy_meta_keys, op->params);
g_hash_table_foreach(entry->stop_params, copy_instance_keys, op->params);
g_hash_table_destroy(params);
params = NULL;
}
}
/* sanity */
if (op->timeout <= 0) {
op->timeout = op->interval_ms;
}
if (op->start_delay < 0) {
op->start_delay = 0;
}
transition = crm_element_value(rsc_op, XML_ATTR_TRANSITION_KEY);
CRM_CHECK(transition != NULL, return op);
op->user_data = strdup(transition);
if (op->interval_ms != 0) {
if (pcmk__strcase_any_of(operation, CRMD_ACTION_START, CRMD_ACTION_STOP, NULL)) {
crm_err("Start and Stop actions cannot have an interval: %u",
op->interval_ms);
op->interval_ms = 0;
}
}
crm_trace("Constructed %s op for %s: interval=%u",
operation, rsc_id, op->interval_ms);
return op;
}
/*!
* \internal
* \brief Send a (synthesized) event result
*
* Reply with a synthesized event result directly, as opposed to going through
* the executor.
*
* \param[in] to_host Host to send result to
* \param[in] to_sys IPC name to send result (NULL for transition engine)
* \param[in] rsc Type information about resource the result is for
* \param[in,out] op Event with result to send
* \param[in] rsc_id ID of resource the result is for
*/
void
controld_ack_event_directly(const char *to_host, const char *to_sys,
const lrmd_rsc_info_t *rsc, lrmd_event_data_t *op,
const char *rsc_id)
{
xmlNode *reply = NULL;
xmlNode *update, *iter;
crm_node_t *peer = NULL;
CRM_CHECK(op != NULL, return);
if (op->rsc_id == NULL) {
CRM_ASSERT(rsc_id != NULL);
op->rsc_id = strdup(rsc_id);
}
if (to_sys == NULL) {
to_sys = CRM_SYSTEM_TENGINE;
}
peer = crm_get_peer(0, fsa_our_uname);
update = create_node_state_update(peer, node_update_none, NULL,
__func__);
iter = create_xml_node(update, XML_CIB_TAG_LRM);
crm_xml_add(iter, XML_ATTR_ID, fsa_our_uuid);
iter = create_xml_node(iter, XML_LRM_TAG_RESOURCES);
iter = create_xml_node(iter, XML_LRM_TAG_RESOURCE);
crm_xml_add(iter, XML_ATTR_ID, op->rsc_id);
build_operation_update(iter, rsc, op, fsa_our_uname, __func__);
reply = create_request(CRM_OP_INVOKE_LRM, update, to_host, to_sys, CRM_SYSTEM_LRMD, NULL);
crm_log_xml_trace(update, "[direct ACK]");
crm_debug("ACK'ing resource op " PCMK__OP_FMT " from %s: %s",
op->rsc_id, op->op_type, op->interval_ms, op->user_data,
crm_element_value(reply, XML_ATTR_REFERENCE));
if (relay_message(reply, TRUE) == FALSE) {
crm_log_xml_err(reply, "Unable to route reply");
}
free_xml(update);
free_xml(reply);
}
gboolean
verify_stopped(enum crmd_fsa_state cur_state, int log_level)
{
gboolean res = TRUE;
GList *lrm_state_list = lrm_state_get_list();
GList *state_entry;
for (state_entry = lrm_state_list; state_entry != NULL; state_entry = state_entry->next) {
lrm_state_t *lrm_state = state_entry->data;
if (!lrm_state_verify_stopped(lrm_state, cur_state, log_level)) {
/* keep iterating through all even when false is returned */
res = FALSE;
}
}
controld_set_fsa_input_flags(R_SENT_RSC_STOP);
g_list_free(lrm_state_list); lrm_state_list = NULL;
return res;
}
struct stop_recurring_action_s {
lrmd_rsc_info_t *rsc;
lrm_state_t *lrm_state;
};
static gboolean
stop_recurring_action_by_rsc(gpointer key, gpointer value, gpointer user_data)
{
gboolean remove = FALSE;
struct stop_recurring_action_s *event = user_data;
active_op_t *op = value;
if ((op->interval_ms != 0)
&& pcmk__str_eq(op->rsc_id, event->rsc->id, pcmk__str_none)) {
crm_debug("Cancelling op %d for %s (%s)", op->call_id, op->rsc_id, (char*)key);
remove = !cancel_op(event->lrm_state, event->rsc->id, key, op->call_id, FALSE);
}
return remove;
}
static gboolean
stop_recurring_actions(gpointer key, gpointer value, gpointer user_data)
{
gboolean remove = FALSE;
lrm_state_t *lrm_state = user_data;
active_op_t *op = value;
if (op->interval_ms != 0) {
crm_info("Cancelling op %d for %s (%s)", op->call_id, op->rsc_id,
(const char *) key);
remove = !cancel_op(lrm_state, op->rsc_id, key, op->call_id, FALSE);
}
return remove;
}
static void
record_pending_op(const char *node_name, lrmd_rsc_info_t *rsc, lrmd_event_data_t *op)
{
const char *record_pending = NULL;
CRM_CHECK(node_name != NULL, return);
CRM_CHECK(rsc != NULL, return);
CRM_CHECK(op != NULL, return);
// Never record certain operation types as pending
if ((op->op_type == NULL) || (op->params == NULL)
|| !controld_action_is_recordable(op->op_type)) {
return;
}
// defaults to true
record_pending = crm_meta_value(op->params, XML_OP_ATTR_PENDING);
if (record_pending && !crm_is_true(record_pending)) {
return;
}
op->call_id = -1;
lrmd__set_result(op, PCMK_OCF_UNKNOWN, PCMK_EXEC_PENDING, NULL);
op->t_run = time(NULL);
op->t_rcchange = op->t_run;
/* write a "pending" entry to the CIB, inhibit notification */
crm_debug("Recording pending op " PCMK__OP_FMT " on %s in the CIB",
op->rsc_id, op->op_type, op->interval_ms, node_name);
do_update_resource(node_name, rsc, op, 0);
}
static void
-do_lrm_rsc_op(lrm_state_t *lrm_state, lrmd_rsc_info_t *rsc,
- const char *operation, xmlNode *msg)
+do_lrm_rsc_op(lrm_state_t *lrm_state, lrmd_rsc_info_t *rsc, xmlNode *msg,
+ struct ra_metadata_s *md)
{
int rc;
int call_id = 0;
char *op_id = NULL;
lrmd_event_data_t *op = NULL;
lrmd_key_value_t *params = NULL;
fsa_data_t *msg_data = NULL;
const char *transition = NULL;
+ const char *operation = NULL;
gboolean stop_recurring = FALSE;
const char *nack_reason = NULL;
- CRM_CHECK(rsc != NULL, return);
- CRM_CHECK(operation != NULL, return);
+ CRM_CHECK((rsc != NULL) && (msg != NULL), return);
- if (msg != NULL) {
- transition = crm_element_value(msg, XML_ATTR_TRANSITION_KEY);
- if (transition == NULL) {
- crm_log_xml_err(msg, "Missing transition number");
+ operation = crm_element_value(msg, XML_LRM_ATTR_TASK);
+ CRM_CHECK(!pcmk__str_empty(operation), return);
+
+ transition = crm_element_value(msg, XML_ATTR_TRANSITION_KEY);
+ if (pcmk__str_empty(transition)) {
+ crm_log_xml_err(msg, "Missing transition number");
+ }
+
+ if (lrm_state == NULL) {
+ // This shouldn't be possible, but provide a failsafe just in case
+ crm_err("Cannot execute %s of %s: No executor connection "
+ CRM_XS " transition_key=%s",
+ operation, rsc->id, pcmk__s(transition, ""));
+ synthesize_lrmd_failure(NULL, msg, PCMK_EXEC_INVALID,
+ PCMK_OCF_UNKNOWN_ERROR,
+ "No executor connection");
+ return;
+ }
+
+ if (pcmk__str_any_of(operation, CRMD_ACTION_RELOAD,
+ CRMD_ACTION_RELOAD_AGENT, NULL)) {
+ /* Pre-2.1.0 DCs will schedule reload actions only, and 2.1.0+ DCs
+ * will schedule reload-agent actions only. In either case, we need
+ * to map that to whatever the resource agent actually supports.
+ * Default to the OCF 1.1 name.
+ */
+ if ((md != NULL)
+ && pcmk_is_set(md->ra_flags, ra_supports_legacy_reload)) {
+ operation = CRMD_ACTION_RELOAD;
+ } else {
+ operation = CRMD_ACTION_RELOAD_AGENT;
}
}
op = construct_op(lrm_state, msg, rsc->id, operation);
CRM_CHECK(op != NULL, return);
if (is_remote_lrmd_ra(NULL, NULL, rsc->id)
&& (op->interval_ms == 0)
&& strcmp(operation, CRMD_ACTION_MIGRATE) == 0) {
/* pcmk remote connections are a special use case.
* We never ever want to stop monitoring a connection resource until
* the entire migration has completed. If the connection is unexpectedly
* severed, even during a migration, this is an event we must detect.*/
stop_recurring = FALSE;
} else if ((op->interval_ms == 0)
&& strcmp(operation, CRMD_ACTION_STATUS) != 0
&& strcmp(operation, CRMD_ACTION_NOTIFY) != 0) {
/* stop any previous monitor operations before changing the resource state */
stop_recurring = TRUE;
}
if (stop_recurring == TRUE) {
guint removed = 0;
struct stop_recurring_action_s data;
data.rsc = rsc;
data.lrm_state = lrm_state;
removed = g_hash_table_foreach_remove(
lrm_state->pending_ops, stop_recurring_action_by_rsc, &data);
if (removed) {
crm_debug("Stopped %u recurring operation%s in preparation for "
PCMK__OP_FMT, removed, pcmk__plural_s(removed),
rsc->id, operation, op->interval_ms);
}
}
/* now do the op */
crm_notice("Requesting local execution of %s operation for %s on %s "
CRM_XS " transition_key=%s op_key=" PCMK__OP_FMT,
crm_action_str(op->op_type, op->interval_ms), rsc->id, lrm_state->node_name,
- transition, rsc->id, operation, op->interval_ms);
+ pcmk__s(transition, ""), rsc->id, operation, op->interval_ms);
if (pcmk_is_set(fsa_input_register, R_SHUTDOWN)
&& pcmk__str_eq(operation, RSC_START, pcmk__str_casei)) {
register_fsa_input(C_SHUTDOWN, I_SHUTDOWN, NULL);
nack_reason = "Not attempting start due to shutdown in progress";
} else if (fsa_state != S_NOT_DC
&& fsa_state != S_POLICY_ENGINE /* Recalculating */
&& fsa_state != S_TRANSITION_ENGINE
&& !pcmk__str_eq(operation, CRMD_ACTION_STOP, pcmk__str_casei)) {
nack_reason = "Controller cannot attempt actions at this time";
}
if (nack_reason != NULL) {
crm_notice("Discarding attempt to perform action %s on %s in state %s (shutdown=%s)",
operation, rsc->id, fsa_state2string(fsa_state),
pcmk__btoa(pcmk_is_set(fsa_input_register, R_SHUTDOWN)));
lrmd__set_result(op, PCMK_OCF_UNKNOWN_ERROR, PCMK_EXEC_INVALID,
nack_reason);
controld_ack_event_directly(NULL, NULL, rsc, op, rsc->id);
lrmd_free_event(op);
free(op_id);
return;
}
record_pending_op(lrm_state->node_name, rsc, op);
op_id = pcmk__op_key(rsc->id, op->op_type, op->interval_ms);
if (op->interval_ms > 0) {
/* cancel it so we can then restart it without conflict */
cancel_op_key(lrm_state, rsc, op_id, FALSE);
}
if (op->params) {
char *key = NULL;
char *value = NULL;
GHashTableIter iter;
g_hash_table_iter_init(&iter, op->params);
while (g_hash_table_iter_next(&iter, (gpointer *) & key, (gpointer *) & value)) {
params = lrmd_key_value_add(params, key, value);
}
}
rc = controld_execute_resource_agent(lrm_state, rsc->id, op->op_type,
op->user_data, op->interval_ms,
op->timeout, op->start_delay, params,
&call_id);
if (rc == pcmk_rc_ok) {
/* record all operations so we can wait
* for them to complete during shutdown
*/
char *call_id_s = make_stop_id(rsc->id, call_id);
active_op_t *pending = NULL;
pending = calloc(1, sizeof(active_op_t));
crm_trace("Recording pending op: %d - %s %s", call_id, op_id, call_id_s);
pending->call_id = call_id;
pending->interval_ms = op->interval_ms;
pending->op_type = strdup(operation);
pending->op_key = strdup(op_id);
pending->rsc_id = strdup(rsc->id);
pending->start_time = time(NULL);
pcmk__str_update(&pending->user_data, op->user_data);
if (crm_element_value_epoch(msg, XML_CONFIG_ATTR_SHUTDOWN_LOCK,
&(pending->lock_time)) != pcmk_ok) {
pending->lock_time = 0;
}
g_hash_table_replace(lrm_state->pending_ops, call_id_s, pending);
if ((op->interval_ms > 0)
&& (op->start_delay > START_DELAY_THRESHOLD)) {
int target_rc = PCMK_OCF_OK;
crm_info("Faking confirmation of %s: execution postponed for over 5 minutes", op_id);
decode_transition_key(op->user_data, NULL, NULL, NULL, &target_rc);
lrmd__set_result(op, target_rc, PCMK_EXEC_DONE, NULL);
controld_ack_event_directly(NULL, NULL, rsc, op, rsc->id);
}
pending->params = op->params;
op->params = NULL;
} else if (lrm_state_is_local(lrm_state)) {
crm_err("Could not initiate %s action for resource %s locally: %s "
CRM_XS " rc=%d", operation, rsc->id, pcmk_rc_str(rc), rc);
fake_op_status(lrm_state, op, PCMK_EXEC_NOT_CONNECTED,
PCMK_OCF_UNKNOWN_ERROR, pcmk_rc_str(rc));
process_lrm_event(lrm_state, op, NULL, NULL);
register_fsa_error(C_FSA_INTERNAL, I_FAIL, NULL);
} else {
crm_err("Could not initiate %s action for resource %s remotely on %s: "
"%s " CRM_XS " rc=%d",
operation, rsc->id, lrm_state->node_name, pcmk_rc_str(rc), rc);
fake_op_status(lrm_state, op, PCMK_EXEC_NOT_CONNECTED,
PCMK_OCF_UNKNOWN_ERROR, pcmk_rc_str(rc));
process_lrm_event(lrm_state, op, NULL, NULL);
}
free(op_id);
lrmd_free_event(op);
}
int last_resource_update = 0;
static void
cib_rsc_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
{
switch (rc) {
case pcmk_ok:
case -pcmk_err_diff_failed:
case -pcmk_err_diff_resync:
crm_trace("Resource update %d complete: rc=%d", call_id, rc);
break;
default:
crm_warn("Resource update %d failed: (rc=%d) %s", call_id, rc, pcmk_strerror(rc));
}
if (call_id == last_resource_update) {
last_resource_update = 0;
trigger_fsa();
}
}
/* Only successful stops, and probes that found the resource inactive, get locks
* recorded in the history. This ensures the resource stays locked to the node
* until it is active there again after the node comes back up.
*/
static bool
should_preserve_lock(lrmd_event_data_t *op)
{
if (!controld_shutdown_lock_enabled) {
return false;
}
if (!strcmp(op->op_type, RSC_STOP) && (op->rc == PCMK_OCF_OK)) {
return true;
}
if (!strcmp(op->op_type, RSC_STATUS) && (op->rc == PCMK_OCF_NOT_RUNNING)) {
return true;
}
return false;
}
static int
do_update_resource(const char *node_name, lrmd_rsc_info_t *rsc,
lrmd_event_data_t *op, time_t lock_time)
{
/*
<status>
<nodes_status id=uname>
<lrm>
<lrm_resources>
<lrm_resource id=...>
</...>
*/
int rc = pcmk_ok;
xmlNode *update, *iter = NULL;
int call_opt = crmd_cib_smart_opt();
const char *uuid = NULL;
CRM_CHECK(op != NULL, return 0);
iter = create_xml_node(iter, XML_CIB_TAG_STATUS);
update = iter;
iter = create_xml_node(iter, XML_CIB_TAG_STATE);
if (pcmk__str_eq(node_name, fsa_our_uname, pcmk__str_casei)) {
uuid = fsa_our_uuid;
} else {
/* remote nodes uuid and uname are equal */
uuid = node_name;
pcmk__xe_set_bool_attr(iter, XML_NODE_IS_REMOTE, true);
}
CRM_LOG_ASSERT(uuid != NULL);
if(uuid == NULL) {
rc = -EINVAL;
goto done;
}
crm_xml_add(iter, XML_ATTR_UUID, uuid);
crm_xml_add(iter, XML_ATTR_UNAME, node_name);
crm_xml_add(iter, XML_ATTR_ORIGIN, __func__);
iter = create_xml_node(iter, XML_CIB_TAG_LRM);
crm_xml_add(iter, XML_ATTR_ID, uuid);
iter = create_xml_node(iter, XML_LRM_TAG_RESOURCES);
iter = create_xml_node(iter, XML_LRM_TAG_RESOURCE);
crm_xml_add(iter, XML_ATTR_ID, op->rsc_id);
build_operation_update(iter, rsc, op, node_name, __func__);
if (rsc) {
const char *container = NULL;
crm_xml_add(iter, XML_ATTR_TYPE, rsc->type);
crm_xml_add(iter, XML_AGENT_ATTR_CLASS, rsc->standard);
crm_xml_add(iter, XML_AGENT_ATTR_PROVIDER, rsc->provider);
if (lock_time != 0) {
/* Actions on a locked resource should either preserve the lock by
* recording it with the action result, or clear it.
*/
if (!should_preserve_lock(op)) {
lock_time = 0;
}
crm_xml_add_ll(iter, XML_CONFIG_ATTR_SHUTDOWN_LOCK,
(long long) lock_time);
}
if (op->params) {
container = g_hash_table_lookup(op->params, CRM_META"_"XML_RSC_ATTR_CONTAINER);
}
if (container) {
crm_trace("Resource %s is a part of container resource %s", op->rsc_id, container);
crm_xml_add(iter, XML_RSC_ATTR_CONTAINER, container);
}
} else {
crm_warn("Resource %s no longer exists in the executor", op->rsc_id);
controld_ack_event_directly(NULL, NULL, rsc, op, op->rsc_id);
goto cleanup;
}
crm_log_xml_trace(update, __func__);
/* make it an asynchronous call and be done with it
*
* Best case:
* the resource state will be discovered during
* the next signup or election.
*
* Bad case:
* we are shutting down and there is no DC at the time,
* but then why were we shutting down then anyway?
* (probably because of an internal error)
*
* Worst case:
* we get shot for having resources "running" that really weren't
*
* the alternative however means blocking here for too long, which
* isn't acceptable
*/
fsa_cib_update(XML_CIB_TAG_STATUS, update, call_opt, rc, NULL);
if (rc > 0) {
last_resource_update = rc;
}
done:
/* the return code is a call number, not an error code */
crm_trace("Sent resource state update message: %d for %s=%u on %s",
rc, op->op_type, op->interval_ms, op->rsc_id);
fsa_register_cib_callback(rc, FALSE, NULL, cib_rsc_callback);
cleanup:
free_xml(update);
return rc;
}
void
do_lrm_event(long long action,
enum crmd_fsa_cause cause,
enum crmd_fsa_state cur_state, enum crmd_fsa_input cur_input, fsa_data_t * msg_data)
{
CRM_CHECK(FALSE, return);
}
static char *
unescape_newlines(const char *string)
{
char *pch = NULL;
char *ret = NULL;
static const char *escaped_newline = "\\n";
if (!string) {
return NULL;
}
ret = strdup(string);
pch = strstr(ret, escaped_newline);
while (pch != NULL) {
/* Replace newline escape pattern with actual newline (and a space so we
* don't have to shuffle the rest of the buffer)
*/
pch[0] = '\n';
pch[1] = ' ';
pch = strstr(pch, escaped_newline);
}
return ret;
}
static bool
did_lrm_rsc_op_fail(lrm_state_t *lrm_state, const char * rsc_id,
const char * op_type, guint interval_ms)
{
rsc_history_t *entry = NULL;
CRM_CHECK(lrm_state != NULL, return FALSE);
CRM_CHECK(rsc_id != NULL, return FALSE);
CRM_CHECK(op_type != NULL, return FALSE);
entry = g_hash_table_lookup(lrm_state->resource_history, rsc_id);
if (entry == NULL || entry->failed == NULL) {
return FALSE;
}
if (pcmk__str_eq(entry->failed->rsc_id, rsc_id, pcmk__str_none)
&& pcmk__str_eq(entry->failed->op_type, op_type, pcmk__str_casei)
&& entry->failed->interval_ms == interval_ms) {
return TRUE;
}
return FALSE;
}
/*!
* \internal
* \brief Log the result of an executor action (actual or synthesized)
*
* \param[in] op Executor action to log result for
* \param[in] op_key Operation key for action
* \param[in] node_name Name of node action was performed on, if known
* \param[in] update_id Call ID for CIB update (or 0 if none)
* \param[in] confirmed Whether to log that graph action was confirmed
*/
static void
log_executor_event(const lrmd_event_data_t *op, const char *op_key,
const char *node_name, int update_id, gboolean confirmed)
{
int log_level = LOG_ERR;
GString *str = g_string_sized_new(100); // reasonable starting size
g_string_printf(str, "Result of %s operation for %s",
crm_action_str(op->op_type, op->interval_ms), op->rsc_id);
if (node_name != NULL) {
g_string_append_printf(str, " on %s", node_name);
}
switch (op->op_status) {
case PCMK_EXEC_DONE:
log_level = LOG_NOTICE;
g_string_append_printf(str, ": %s",
services_ocf_exitcode_str(op->rc));
break;
case PCMK_EXEC_TIMEOUT:
g_string_append_printf(str, ": %s after %s",
pcmk_exec_status_str(op->op_status),
pcmk__readable_interval(op->timeout));
break;
case PCMK_EXEC_CANCELLED:
log_level = LOG_INFO;
// Fall through
default:
g_string_append_printf(str, ": %s",
pcmk_exec_status_str(op->op_status));
}
if ((op->exit_reason != NULL)
&& ((op->op_status != PCMK_EXEC_DONE) || (op->rc != PCMK_OCF_OK))) {
g_string_append_printf(str, " (%s)", op->exit_reason);
}
g_string_append(str, " " CRM_XS);
if (update_id != 0) {
g_string_append_printf(str, " CIB update %d,", update_id);
}
g_string_append_printf(str, " graph action %sconfirmed; call=%d key=%s",
(confirmed? "" : "un"), op->call_id, op_key);
if (op->op_status == PCMK_EXEC_DONE) {
g_string_append_printf(str, " rc=%d", op->rc);
}
do_crm_log(log_level, "%s", str->str);
g_string_free(str, TRUE);
/* The services library has already logged the output at info or debug
* level, so just raise to notice if it looks like a failure.
*/
if ((op->output != NULL) && (op->rc != PCMK_OCF_OK)) {
char *prefix = crm_strdup_printf(PCMK__OP_FMT "@%s output",
op->rsc_id, op->op_type,
op->interval_ms, node_name);
crm_log_output(LOG_NOTICE, prefix, op->output);
free(prefix);
}
}
void
process_lrm_event(lrm_state_t *lrm_state, lrmd_event_data_t *op,
active_op_t *pending, const xmlNode *action_xml)
{
char *op_id = NULL;
char *op_key = NULL;
int update_id = 0;
gboolean remove = FALSE;
gboolean removed = FALSE;
bool need_direct_ack = FALSE;
lrmd_rsc_info_t *rsc = NULL;
const char *node_name = NULL;
CRM_CHECK(op != NULL, return);
CRM_CHECK(op->rsc_id != NULL, return);
// Remap new status codes for older DCs
if (compare_version(fsa_our_dc_version, "3.2.0") < 0) {
switch (op->op_status) {
case PCMK_EXEC_NOT_CONNECTED:
lrmd__set_result(op, PCMK_OCF_CONNECTION_DIED,
PCMK_EXEC_ERROR, op->exit_reason);
break;
case PCMK_EXEC_INVALID:
lrmd__set_result(op, CRM_DIRECT_NACK_RC, PCMK_EXEC_ERROR,
op->exit_reason);
break;
default:
break;
}
}
op_id = make_stop_id(op->rsc_id, op->call_id);
op_key = pcmk__op_key(op->rsc_id, op->op_type, op->interval_ms);
// Get resource info if available (from executor state or action XML)
if (lrm_state) {
rsc = lrm_state_get_rsc_info(lrm_state, op->rsc_id, 0);
}
if ((rsc == NULL) && action_xml) {
xmlNode *xml = find_xml_node(action_xml, XML_CIB_TAG_RESOURCE, TRUE);
const char *standard = crm_element_value(xml, XML_AGENT_ATTR_CLASS);
const char *provider = crm_element_value(xml, XML_AGENT_ATTR_PROVIDER);
const char *type = crm_element_value(xml, XML_ATTR_TYPE);
if (standard && type) {
crm_info("%s agent information not cached, using %s%s%s:%s from action XML",
op->rsc_id, standard,
(provider? ":" : ""), (provider? provider : ""), type);
rsc = lrmd_new_rsc_info(op->rsc_id, standard, provider, type);
} else {
crm_err("Can't process %s result because %s agent information not cached or in XML",
op_key, op->rsc_id);
}
}
// Get node name if available (from executor state or action XML)
if (lrm_state) {
node_name = lrm_state->node_name;
} else if (action_xml) {
node_name = crm_element_value(action_xml, XML_LRM_ATTR_TARGET);
}
if(pending == NULL) {
remove = TRUE;
if (lrm_state) {
pending = g_hash_table_lookup(lrm_state->pending_ops, op_id);
}
}
if (op->op_status == PCMK_EXEC_ERROR) {
switch(op->rc) {
case PCMK_OCF_NOT_RUNNING:
case PCMK_OCF_RUNNING_PROMOTED:
case PCMK_OCF_DEGRADED:
case PCMK_OCF_DEGRADED_PROMOTED:
// Leave it to the TE/scheduler to decide if this is an error
op->op_status = PCMK_EXEC_DONE;
break;
default:
/* Nothing to do */
break;
}
}
if (op->op_status != PCMK_EXEC_CANCELLED) {
/* We might not record the result, so directly acknowledge it to the
* originator instead, so it doesn't time out waiting for the result
* (especially important if part of a transition).
*/
need_direct_ack = TRUE;
if (controld_action_is_recordable(op->op_type)) {
if (node_name && rsc) {
// We should record the result, and happily, we can
update_id = do_update_resource(node_name, rsc, op,
pending? pending->lock_time : 0);
need_direct_ack = FALSE;
} else if (op->rsc_deleted) {
/* We shouldn't record the result (likely the resource was
* refreshed, cleaned, or removed while this operation was
* in flight).
*/
crm_notice("Not recording %s result in CIB because "
"resource information was removed since it was initiated",
op_key);
} else {
/* This shouldn't be possible; the executor didn't consider the
* resource deleted, but we couldn't find resource or node
* information.
*/
crm_err("Unable to record %s result in CIB: %s", op_key,
(node_name? "No resource information" : "No node name"));
}
}
} else if (op->interval_ms == 0) {
/* A non-recurring operation was cancelled. Most likely, the
* never-initiated action was removed from the executor's pending
* operations list upon resource removal.
*/
need_direct_ack = TRUE;
} else if (pending == NULL) {
/* This recurring operation was cancelled, but was not pending. No
* transition actions are waiting on it, nothing needs to be done.
*/
} else if (op->user_data == NULL) {
/* This recurring operation was cancelled and pending, but we don't
* have a transition key. This should never happen.
*/
crm_err("Recurring operation %s was cancelled without transition information",
op_key);
} else if (pcmk_is_set(pending->flags, active_op_remove)) {
/* This recurring operation was cancelled (by us) and pending, and we
* have been waiting for it to finish.
*/
if (lrm_state) {
erase_lrm_history_by_op(op);
}
/* If the recurring operation had failed, the lrm_rsc_op is recorded as
* "last_failure" which won't get erased from the cib given the logic on
* purpose in erase_lrm_history_by_op(). So that the cancel action won't
* have a chance to get confirmed by DC with process_op_deletion().
* Cluster transition would get stuck waiting for the remaining action
* timer to time out.
*
* Directly acknowledge the cancel operation in this case.
*/
if (did_lrm_rsc_op_fail(lrm_state, pending->rsc_id,
pending->op_type, pending->interval_ms)) {
need_direct_ack = TRUE;
}
} else if (op->rsc_deleted) {
/* This recurring operation was cancelled (but not by us, and the
* executor does not have resource information, likely due to resource
* cleanup, refresh, or removal) and pending.
*/
crm_debug("Recurring op %s was cancelled due to resource deletion",
op_key);
need_direct_ack = TRUE;
} else {
/* This recurring operation was cancelled (but not by us, likely by the
* executor before stopping the resource) and pending. We don't need to
* do anything special.
*/
}
if (need_direct_ack) {
controld_ack_event_directly(NULL, NULL, NULL, op, op->rsc_id);
}
if(remove == FALSE) {
/* The caller will do this afterwards, but keep the logging consistent */
removed = TRUE;
} else if (lrm_state && ((op->interval_ms == 0)
|| (op->op_status == PCMK_EXEC_CANCELLED))) {
gboolean found = g_hash_table_remove(lrm_state->pending_ops, op_id);
if (op->interval_ms != 0) {
removed = TRUE;
} else if (found) {
removed = TRUE;
crm_trace("Op %s (call=%d, stop-id=%s, remaining=%u): Confirmed",
op_key, op->call_id, op_id,
g_hash_table_size(lrm_state->pending_ops));
}
}
log_executor_event(op, op_key, node_name, update_id, removed);
if (lrm_state) {
if (!pcmk__str_eq(op->op_type, RSC_METADATA, pcmk__str_casei)) {
crmd_alert_resource_op(lrm_state->node_name, op);
} else if (rsc && (op->rc == PCMK_OCF_OK)) {
char *metadata = unescape_newlines(op->output);
- metadata_cache_update(lrm_state->metadata_cache, rsc, metadata);
+ controld_cache_metadata(lrm_state->metadata_cache, rsc, metadata);
free(metadata);
}
}
if (op->rsc_deleted) {
crm_info("Deletion of resource '%s' complete after %s", op->rsc_id, op_key);
if (lrm_state) {
delete_rsc_entry(lrm_state, NULL, op->rsc_id, NULL, pcmk_ok, NULL);
}
}
/* If a shutdown was escalated while operations were pending,
* then the FSA will be stalled right now... allow it to continue
*/
mainloop_set_trigger(fsa_source);
if (lrm_state && rsc) {
update_history_cache(lrm_state, rsc, op);
}
lrmd_free_rsc_info(rsc);
free(op_key);
free(op_id);
}
diff --git a/daemons/controld/controld_fsa.h b/daemons/controld/controld_fsa.h
index 296232f23d..d137310ce2 100644
--- a/daemons/controld/controld_fsa.h
+++ b/daemons/controld/controld_fsa.h
@@ -1,718 +1,718 @@
/*
- * Copyright 2004-2021 the Pacemaker project contributors
+ * Copyright 2004-2022 the Pacemaker project contributors
*
* The version control history for this file may have further details.
*
* This source code is licensed under the GNU Lesser General Public License
* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
*/
#ifndef CRMD_FSA__H
# define CRMD_FSA__H
# include <crm/crm.h>
# include <crm/cib.h>
# include <crm/common/xml.h>
# include <crm/common/mainloop.h>
# include <crm/cluster.h>
# include <crm/cluster/election_internal.h>
# include <crm/common/ipc_internal.h>
/*! States the controller can be in */
enum crmd_fsa_state {
S_IDLE = 0, /* Nothing happening */
S_ELECTION, /* Take part in the election algorithm as
* described below
*/
S_INTEGRATION, /* integrate that status of new nodes (which is
* all of them if we have just been elected DC)
* to form a complete and up-to-date picture of
* the CIB
*/
S_FINALIZE_JOIN, /* integrate that status of new nodes (which is
* all of them if we have just been elected DC)
* to form a complete and up-to-date picture of
* the CIB
*/
S_NOT_DC, /* we are in non-DC mode */
S_POLICY_ENGINE, /* Determine next stable state of the cluster */
S_RECOVERY, /* Something bad happened, check everything is ok
* before continuing and attempt to recover if
* required
*/
S_RELEASE_DC, /* we were the DC, but now we arent anymore,
* possibly by our own request, and we should
* release all unnecessary sub-systems, finish
* any pending actions, do general cleanup and
* unset anything that makes us think we are
* special :)
*/
S_STARTING, /* we are just starting out */
S_PENDING, /* we are not a full/active member yet */
S_STOPPING, /* We are in the final stages of shutting down */
S_TERMINATE, /* We are going to shutdown, this is the equiv of
* "Sending TERM signal to all processes" in Linux
* and in worst case scenarios could be considered
* a self STONITH
*/
S_TRANSITION_ENGINE, /* Attempt to make the calculated next stable
* state of the cluster a reality
*/
S_HALT, /* Freeze - don't do anything
* Something bad happened that needs the admin to fix
* Wait for I_ELECTION
*/
/* ----------- Last input found in table is above ---------- */
S_ILLEGAL /* This is an illegal FSA state */
/* (must be last) */
};
# define MAXSTATE S_ILLEGAL
/*
Once we start and do some basic sanity checks, we go into the
S_NOT_DC state and await instructions from the DC or input from
the cluster layer which indicates the election algorithm needs to run.
If the election algorithm is triggered, we enter the S_ELECTION state
from where we can either go back to the S_NOT_DC state or progress
to the S_INTEGRATION state (or S_RELEASE_DC if we used to be the DC
but aren't anymore). See the libcrmcluster API documentation for more
information about the election algorithm.
Once the election is complete, if we are the DC, we enter the
S_INTEGRATION state which is a DC-in-waiting style state. We are
the DC, but we shouldn't do anything yet because we may not have an
up-to-date picture of the cluster. There may of course be times
when this fails, so we should go back to the S_RECOVERY stage and
check everything is ok. We may also end up here if a new node came
online, since each node is authoritative about itself, and we would want
to incorporate its information into the CIB.
Once we have the latest CIB, we then enter the S_POLICY_ENGINE state
where invoke the scheduler. It is possible that between
invoking the scheduler and receiving an answer, that we receive
more input. In this case, we would discard the orginal result and
invoke it again.
Once we are satisfied with the output from the scheduler, we
enter S_TRANSITION_ENGINE and feed the scheduler's output to the
Transition Engine who attempts to make the scheduler's
calculation a reality. If the transition completes successfully,
we enter S_IDLE, otherwise we go back to S_POLICY_ENGINE with the
current unstable state and try again.
Of course, we may be asked to shutdown at any time, however we must
progress to S_NOT_DC before doing so. Once we have handed over DC
duties to another node, we can then shut down like everyone else,
that is, by asking the DC for permission and waiting for it to take all
our resources away.
The case where we are the DC and the only node in the cluster is a
special case and handled as an escalation which takes us to
S_SHUTDOWN. Similarly, if any other point in the shutdown
fails or stalls, this is escalated and we end up in S_TERMINATE.
At any point, the controller can relay messages for its subsystems,
but outbound messages (from subsystems) should probably be blocked
until S_INTEGRATION (for the DC) or the join protocol has
completed (for non-DC controllers).
*/
/*======================================
*
* Inputs/Events/Stimuli to be given to the finite state machine
*
* Some of these a true events, and others are synthesised based on
* the "register" (see below) and the contents or source of messages.
*
* The machine keeps processing until receiving I_NULL
*
*======================================*/
enum crmd_fsa_input {
/* 0 */
I_NULL, /* Nothing happened */
/* 1 */
I_CIB_OP, /* An update to the CIB occurred */
I_CIB_UPDATE, /* An update to the CIB occurred */
I_DC_TIMEOUT, /* We have lost communication with the DC */
I_ELECTION, /* Someone started an election */
I_PE_CALC, /* The scheduler needs to be invoked */
I_RELEASE_DC, /* The election completed and we were not
* elected, but we were the DC beforehand
*/
I_ELECTION_DC, /* The election completed and we were (re-)elected
* DC
*/
I_ERROR, /* Something bad happened (more serious than
* I_FAIL) and may not have been due to the action
* being performed. For example, we may have lost
* our connection to the CIB.
*/
/* 9 */
I_FAIL, /* The action failed to complete successfully */
I_INTEGRATED,
I_FINALIZED,
I_NODE_JOIN, /* A node has entered the cluster */
I_NOT_DC, /* We are not and were not the DC before or after
* the current operation or state
*/
I_RECOVERED, /* The recovery process completed successfully */
I_RELEASE_FAIL, /* We could not give up DC status for some reason
*/
I_RELEASE_SUCCESS, /* We are no longer the DC */
I_RESTART, /* The current set of actions needs to be
* restarted
*/
I_TE_SUCCESS, /* Some non-resource, non-cluster-layer action
* is required of us, e.g. ping
*/
/* 20 */
I_ROUTER, /* Do our job as router and forward this to the
* right place
*/
I_SHUTDOWN, /* We are asking to shutdown */
I_STOP, /* We have been told to shutdown */
I_TERMINATE, /* Actually exit */
I_STARTUP,
I_PE_SUCCESS, /* The action completed successfully */
I_JOIN_OFFER, /* The DC is offering membership */
I_JOIN_REQUEST, /* The client is requesting membership */
I_JOIN_RESULT, /* If not the DC: The result of a join request
* Else: A client is responding with its local state info
*/
I_WAIT_FOR_EVENT, /* we may be waiting for an async task to "happen"
* and until it does, we can't do anything else
*/
I_DC_HEARTBEAT, /* The DC is telling us that it is alive and well */
I_LRM_EVENT,
/* 30 */
I_PENDING,
I_HALT,
/* ------------ Last input found in table is above ----------- */
I_ILLEGAL /* This is an illegal value for an FSA input */
/* (must be last) */
};
# define MAXINPUT I_ILLEGAL
# define I_MESSAGE I_ROUTER
/*======================================
*
* actions
*
* Some of the actions below will always occur together for now, but this may
* not always be the case, so they are split up so that they can easily be
* called independently in the future, if necessary.
*
* For example, separating A_LRM_CONNECT from A_STARTUP might be useful
* if we ever try to recover from a faulty or disconnected executor.
*
*======================================*/
/* Don't do anything */
# define A_NOTHING 0x0000000000000000ULL
/* -- Startup actions -- */
/* Hook to perform any actions (other than connecting to other daemons)
* that might be needed as part of the startup.
*/
# define A_STARTUP 0x0000000000000001ULL
/* Hook to perform any actions that might be needed as part
* after startup is successful.
*/
# define A_STARTED 0x0000000000000002ULL
/* Connect to cluster layer */
# define A_HA_CONNECT 0x0000000000000004ULL
# define A_HA_DISCONNECT 0x0000000000000008ULL
# define A_INTEGRATE_TIMER_START 0x0000000000000010ULL
# define A_INTEGRATE_TIMER_STOP 0x0000000000000020ULL
# define A_FINALIZE_TIMER_START 0x0000000000000040ULL
# define A_FINALIZE_TIMER_STOP 0x0000000000000080ULL
/* -- Election actions -- */
# define A_DC_TIMER_START 0x0000000000000100ULL
# define A_DC_TIMER_STOP 0x0000000000000200ULL
# define A_ELECTION_COUNT 0x0000000000000400ULL
# define A_ELECTION_VOTE 0x0000000000000800ULL
# define A_ELECTION_START 0x0000000000001000ULL
/* -- Message processing -- */
/* Process the queue of requests */
# define A_MSG_PROCESS 0x0000000000002000ULL
/* Send the message to the correct recipient */
# define A_MSG_ROUTE 0x0000000000004000ULL
/* Send a welcome message to new node(s) */
# define A_DC_JOIN_OFFER_ONE 0x0000000000008000ULL
/* -- Server Join protocol actions -- */
/* Send a welcome message to all nodes */
# define A_DC_JOIN_OFFER_ALL 0x0000000000010000ULL
/* Process the remote node's ack of our join message */
# define A_DC_JOIN_PROCESS_REQ 0x0000000000020000ULL
/* Send out the results of the Join phase */
# define A_DC_JOIN_FINALIZE 0x0000000000040000ULL
/* Send out the results of the Join phase */
# define A_DC_JOIN_PROCESS_ACK 0x0000000000080000ULL
/* -- Client Join protocol actions -- */
# define A_CL_JOIN_QUERY 0x0000000000100000ULL
# define A_CL_JOIN_ANNOUNCE 0x0000000000200000ULL
/* Request membership to the DC list */
# define A_CL_JOIN_REQUEST 0x0000000000400000ULL
/* Did the DC accept or reject the request */
# define A_CL_JOIN_RESULT 0x0000000000800000ULL
/* -- Recovery, DC start/stop -- */
/* Something bad happened, try to recover */
# define A_RECOVER 0x0000000001000000ULL
/* Hook to perform any actions (apart from starting, the TE, scheduler,
* and gathering the latest CIB) that might be necessary before
* giving up the responsibilities of being the DC.
*/
# define A_DC_RELEASE 0x0000000002000000ULL
/* */
# define A_DC_RELEASED 0x0000000004000000ULL
/* Hook to perform any actions (apart from starting, the TE, scheduler,
* and gathering the latest CIB) that might be necessary before
* taking over the responsibilities of being the DC.
*/
# define A_DC_TAKEOVER 0x0000000008000000ULL
/* -- Shutdown actions -- */
# define A_SHUTDOWN 0x0000000010000000ULL
# define A_STOP 0x0000000020000000ULL
# define A_EXIT_0 0x0000000040000000ULL
# define A_EXIT_1 0x0000000080000000ULL
# define A_SHUTDOWN_REQ 0x0000000100000000ULL
# define A_ELECTION_CHECK 0x0000000200000000ULL
# define A_DC_JOIN_FINAL 0x0000000400000000ULL
/* -- CIB actions -- */
# define A_CIB_START 0x0000020000000000ULL
# define A_CIB_STOP 0x0000040000000000ULL
/* -- Transition Engine actions -- */
/* Attempt to reach the newly calculated cluster state. This is
* only called once per transition (except if it is asked to
* stop the transition or start a new one).
* Once given a cluster state to reach, the TE will determine
* tasks that can be performed in parallel, execute them, wait
* for replies and then determine the next set until the new
* state is reached or no further tasks can be taken.
*/
# define A_TE_INVOKE 0x0000100000000000ULL
# define A_TE_START 0x0000200000000000ULL
# define A_TE_STOP 0x0000400000000000ULL
# define A_TE_CANCEL 0x0000800000000000ULL
# define A_TE_HALT 0x0001000000000000ULL
/* -- Scheduler actions -- */
/* Calculate the next state for the cluster. This is only
* invoked once per needed calculation.
*/
# define A_PE_INVOKE 0x0002000000000000ULL
# define A_PE_START 0x0004000000000000ULL
# define A_PE_STOP 0x0008000000000000ULL
/* -- Misc actions -- */
/* Add a system generate "block" so that resources arent moved
* to or are activly moved away from the affected node. This
* way we can return quickly even if busy with other things.
*/
# define A_NODE_BLOCK 0x0010000000000000ULL
/* Update our information in the local CIB */
# define A_UPDATE_NODESTATUS 0x0020000000000000ULL
# define A_READCONFIG 0x0080000000000000ULL
/* -- LRM Actions -- */
/* Connect to pacemaker-execd */
# define A_LRM_CONNECT 0x0100000000000000ULL
/* Disconnect from pacemaker-execd */
# define A_LRM_DISCONNECT 0x0200000000000000ULL
# define A_LRM_INVOKE 0x0400000000000000ULL
# define A_LRM_EVENT 0x0800000000000000ULL
/* -- Logging actions -- */
# define A_LOG 0x1000000000000000ULL
# define A_ERROR 0x2000000000000000ULL
# define A_WARN 0x4000000000000000ULL
# define O_EXIT (A_SHUTDOWN|A_STOP|A_LRM_DISCONNECT|A_HA_DISCONNECT|A_EXIT_0|A_CIB_STOP)
# define O_RELEASE (A_DC_TIMER_STOP|A_DC_RELEASE|A_PE_STOP|A_TE_STOP|A_DC_RELEASED)
# define O_PE_RESTART (A_PE_START|A_PE_STOP)
# define O_TE_RESTART (A_TE_START|A_TE_STOP)
# define O_CIB_RESTART (A_CIB_START|A_CIB_STOP)
# define O_LRM_RECONNECT (A_LRM_CONNECT|A_LRM_DISCONNECT)
# define O_DC_TIMER_RESTART (A_DC_TIMER_STOP|A_DC_TIMER_START)
/*======================================
*
* "register" contents
*
* Things we may want to remember regardless of which state we are in.
*
* These also count as inputs for synthesizing I_*
*
*======================================*/
# define R_THE_DC 0x00000001ULL
/* Are we the DC? */
# define R_STARTING 0x00000002ULL
/* Are we starting up? */
# define R_SHUTDOWN 0x00000004ULL
/* Are we trying to shut down? */
# define R_STAYDOWN 0x00000008ULL
/* Should we restart? */
# define R_JOIN_OK 0x00000010ULL /* Have we completed the join process */
# define R_READ_CONFIG 0x00000040ULL
# define R_INVOKE_PE 0x00000080ULL // Should the scheduler be invoked?
# define R_CIB_CONNECTED 0x00000100ULL
/* Is the CIB connected? */
# define R_PE_CONNECTED 0x00000200ULL // Is the scheduler connected?
# define R_TE_CONNECTED 0x00000400ULL
/* Is the Transition Engine connected? */
# define R_LRM_CONNECTED 0x00000800ULL // Is pacemaker-execd connected?
# define R_CIB_REQUIRED 0x00001000ULL
/* Is the CIB required? */
# define R_PE_REQUIRED 0x00002000ULL // Is the scheduler required?
# define R_TE_REQUIRED 0x00004000ULL
/* Is the Transition Engine required? */
# define R_ST_REQUIRED 0x00008000ULL
/* Is the Stonith daemon required? */
# define R_CIB_DONE 0x00010000ULL
/* Have we calculated the CIB? */
# define R_HAVE_CIB 0x00020000ULL /* Do we have an up-to-date CIB */
# define R_CIB_ASKED 0x00040000ULL /* Have we asked for an up-to-date CIB */
# define R_MEMBERSHIP 0x00100000ULL /* Have we got cluster layer data yet */
# define R_PEER_DATA 0x00200000ULL /* Have we got T_CL_STATUS data yet */
# define R_HA_DISCONNECTED 0x00400000ULL /* did we sign out of our own accord */
# define R_REQ_PEND 0x01000000ULL
/* Are there Requests waiting for
processing? */
# define R_PE_PEND 0x02000000ULL // Are we awaiting reply from scheduler?
# define R_TE_PEND 0x04000000ULL
/* Has the TE been invoked and we're
awaiting completion? */
# define R_RESP_PEND 0x08000000ULL
/* Do we have clients waiting on a
response? if so perhaps we shouldn't
stop yet */
# define R_IN_TRANSITION 0x10000000ULL
/* */
# define R_SENT_RSC_STOP 0x20000000ULL /* Have we sent a stop action to all
* resources in preparation for
* shutting down */
# define R_IN_RECOVERY 0x80000000ULL
#define CRM_DIRECT_NACK_RC (99) // Deprecated (see PCMK_EXEC_INVALID)
enum crmd_fsa_cause {
C_UNKNOWN = 0,
C_STARTUP,
C_IPC_MESSAGE,
C_HA_MESSAGE,
C_CRMD_STATUS_CALLBACK,
C_LRM_OP_CALLBACK,
C_TIMER_POPPED,
C_SHUTDOWN,
C_FSA_INTERNAL,
};
enum fsa_data_type {
fsa_dt_none,
fsa_dt_ha_msg,
fsa_dt_xml,
fsa_dt_lrm,
};
typedef struct fsa_data_s fsa_data_t;
struct fsa_data_s {
int id;
enum crmd_fsa_input fsa_input;
enum crmd_fsa_cause fsa_cause;
uint64_t actions;
const char *origin;
void *data;
enum fsa_data_type data_type;
};
/* Global FSA stuff */
extern gboolean do_fsa_stall;
extern enum crmd_fsa_state fsa_state;
extern uint64_t fsa_input_register;
extern uint64_t fsa_actions;
#define controld_set_fsa_input_flags(flags_to_set) do { \
fsa_input_register = pcmk__set_flags_as(__func__, __LINE__, \
LOG_TRACE, \
"FSA input", "controller", \
fsa_input_register, \
(flags_to_set), \
#flags_to_set); \
} while (0)
#define controld_clear_fsa_input_flags(flags_to_clear) do { \
fsa_input_register = pcmk__clear_flags_as(__func__, __LINE__, \
LOG_TRACE, \
"FSA input", "controller",\
fsa_input_register, \
(flags_to_clear), \
#flags_to_clear); \
} while (0)
#define controld_set_fsa_action_flags(flags_to_set) do { \
fsa_actions = pcmk__set_flags_as(__func__, __LINE__, \
LOG_DEBUG, \
"FSA action", "controller", \
fsa_actions, (flags_to_set), \
#flags_to_set); \
} while (0)
#define controld_clear_fsa_action_flags(flags_to_clear) do { \
fsa_actions = pcmk__clear_flags_as(__func__, __LINE__, \
LOG_DEBUG, \
"FSA action", "controller", \
fsa_actions, (flags_to_clear), \
#flags_to_clear); \
} while (0)
extern cib_t *fsa_cib_conn;
extern char *fsa_our_uname;
extern char *fsa_our_uuid;
extern char *fsa_pe_ref; // Last invocation of the scheduler
extern char *fsa_our_dc;
extern char *fsa_our_dc_version;
extern GList *fsa_message_queue;
extern char *fsa_cluster_name;
extern crm_trigger_t *fsa_source;
extern crm_trigger_t *config_read;
extern unsigned long long saved_ccm_membership_id;
extern gboolean ever_had_quorum;
// These should be moved elsewhere
void do_update_cib_nodes(gboolean overwrite, const char *caller);
int crmd_cib_smart_opt(void);
-xmlNode *controld_query_executor_state(const char *node_name);
+xmlNode *controld_query_executor_state(void);
const char *fsa_input2string(enum crmd_fsa_input input);
const char *fsa_state2string(enum crmd_fsa_state state);
const char *fsa_cause2string(enum crmd_fsa_cause cause);
const char *fsa_action2string(long long action);
enum crmd_fsa_state s_crmd_fsa(enum crmd_fsa_cause cause);
# define AM_I_DC pcmk_is_set(fsa_input_register, R_THE_DC)
# define AM_I_OPERATIONAL !pcmk_is_set(fsa_input_register, R_STARTING)
# define trigger_fsa() do { \
if (fsa_source != NULL) { \
crm_trace("Triggering FSA"); \
mainloop_set_trigger(fsa_source); \
} \
} while(0)
/* A_READCONFIG */
void do_read_config(long long action, enum crmd_fsa_cause cause,
enum crmd_fsa_state cur_state,
enum crmd_fsa_input current_input, fsa_data_t *msg_data);
/* A_PE_INVOKE */
void do_pe_invoke(long long action, enum crmd_fsa_cause cause,
enum crmd_fsa_state cur_state,
enum crmd_fsa_input current_input, fsa_data_t *msg_data);
/* A_LOG */
void do_log(long long action, enum crmd_fsa_cause cause,
enum crmd_fsa_state cur_state,
enum crmd_fsa_input cur_input, fsa_data_t *msg_data);
/* A_STARTUP */
void do_startup(long long action, enum crmd_fsa_cause cause,
enum crmd_fsa_state cur_state,
enum crmd_fsa_input cur_input, fsa_data_t *msg_data);
/* A_CIB_START, STOP, RESTART */
void do_cib_control(long long action, enum crmd_fsa_cause cause,
enum crmd_fsa_state cur_state,
enum crmd_fsa_input cur_input, fsa_data_t *msg_data);
/* A_HA_CONNECT */
void do_ha_control(long long action, enum crmd_fsa_cause cause,
enum crmd_fsa_state cur_state,
enum crmd_fsa_input cur_input, fsa_data_t *msg_data);
/* A_LRM_CONNECT */
void do_lrm_control(long long action, enum crmd_fsa_cause cause,
enum crmd_fsa_state cur_state,
enum crmd_fsa_input cur_input, fsa_data_t *msg_data);
/* A_PE_START, STOP, RESTART */
void do_pe_control(long long action, enum crmd_fsa_cause cause,
enum crmd_fsa_state cur_state,
enum crmd_fsa_input cur_input, fsa_data_t *msg_data);
/* A_TE_START, STOP, RESTART */
void do_te_control(long long action, enum crmd_fsa_cause cause,
enum crmd_fsa_state cur_state,
enum crmd_fsa_input cur_input, fsa_data_t *msg_data);
/* A_STARTED */
void do_started(long long action, enum crmd_fsa_cause cause,
enum crmd_fsa_state cur_state,
enum crmd_fsa_input cur_input, fsa_data_t *msg_data);
/* A_MSG_ROUTE */
void do_msg_route(long long action, enum crmd_fsa_cause cause,
enum crmd_fsa_state cur_state,
enum crmd_fsa_input cur_input, fsa_data_t *msg_data);
/* A_RECOVER */
void do_recover(long long action, enum crmd_fsa_cause cause,
enum crmd_fsa_state cur_state,
enum crmd_fsa_input cur_input, fsa_data_t *msg_data);
/* A_ELECTION_VOTE */
void do_election_vote(long long action, enum crmd_fsa_cause cause,
enum crmd_fsa_state cur_state,
enum crmd_fsa_input cur_input, fsa_data_t *msg_data);
/* A_ELECTION_COUNT */
void do_election_count_vote(long long action, enum crmd_fsa_cause cause,
enum crmd_fsa_state cur_state,
enum crmd_fsa_input cur_input,
fsa_data_t *msg_data);
/* A_ELECTION_CHECK */
void do_election_check(long long action, enum crmd_fsa_cause cause,
enum crmd_fsa_state cur_state,
enum crmd_fsa_input cur_input, fsa_data_t *msg_data);
/* A_DC_TIMER_STOP */
void do_timer_control(long long action, enum crmd_fsa_cause cause,
enum crmd_fsa_state cur_state,
enum crmd_fsa_input cur_input, fsa_data_t *msg_data);
/* A_DC_TAKEOVER */
void do_dc_takeover(long long action, enum crmd_fsa_cause cause,
enum crmd_fsa_state cur_state,
enum crmd_fsa_input cur_input, fsa_data_t *msg_data);
/* A_DC_RELEASE */
void do_dc_release(long long action, enum crmd_fsa_cause cause,
enum crmd_fsa_state cur_state,
enum crmd_fsa_input cur_input, fsa_data_t *msg_data);
/* A_DC_JOIN_OFFER_ALL */
void do_dc_join_offer_all(long long action, enum crmd_fsa_cause cause,
enum crmd_fsa_state cur_state,
enum crmd_fsa_input cur_input, fsa_data_t *msg_data);
/* A_DC_JOIN_OFFER_ONE */
void do_dc_join_offer_one(long long action, enum crmd_fsa_cause cause,
enum crmd_fsa_state cur_state,
enum crmd_fsa_input cur_input, fsa_data_t *msg_data);
/* A_DC_JOIN_ACK */
void do_dc_join_ack(long long action, enum crmd_fsa_cause cause,
enum crmd_fsa_state cur_state,
enum crmd_fsa_input cur_input, fsa_data_t *msg_data);
/* A_DC_JOIN_REQ */
void do_dc_join_filter_offer(long long action, enum crmd_fsa_cause cause,
enum crmd_fsa_state cur_state,
enum crmd_fsa_input cur_input,
fsa_data_t *msg_data);
/* A_DC_JOIN_FINALIZE */
void do_dc_join_finalize(long long action, enum crmd_fsa_cause cause,
enum crmd_fsa_state cur_state,
enum crmd_fsa_input cur_input, fsa_data_t *msg_data);
/* A_CL_JOIN_QUERY */
/* is there a DC out there? */
void do_cl_join_query(long long action, enum crmd_fsa_cause cause,
enum crmd_fsa_state cur_state,
enum crmd_fsa_input current_input, fsa_data_t *msg_data);
/* A_CL_JOIN_ANNOUNCE */
void do_cl_join_announce(long long action, enum crmd_fsa_cause cause,
enum crmd_fsa_state cur_state,
enum crmd_fsa_input current_input, fsa_data_t *msg_data);
/* A_CL_JOIN_REQUEST */
void do_cl_join_offer_respond(long long action, enum crmd_fsa_cause cause,
enum crmd_fsa_state cur_state,
enum crmd_fsa_input current_input,
fsa_data_t *msg_data);
/* A_CL_JOIN_RESULT */
void do_cl_join_finalize_respond(long long action, enum crmd_fsa_cause cause,
enum crmd_fsa_state cur_state,
enum crmd_fsa_input current_input,
fsa_data_t *msg_data);
/* A_LRM_INVOKE */
void do_lrm_invoke(long long action, enum crmd_fsa_cause cause,
enum crmd_fsa_state cur_state,
enum crmd_fsa_input cur_input, fsa_data_t *msg_data);
/* A_LRM_EVENT */
void do_lrm_event(long long action, enum crmd_fsa_cause cause,
enum crmd_fsa_state cur_state,
enum crmd_fsa_input cur_input, fsa_data_t *msg_data);
/* A_TE_INVOKE, A_TE_CANCEL */
void do_te_invoke(long long action, enum crmd_fsa_cause cause,
enum crmd_fsa_state cur_state,
enum crmd_fsa_input cur_input, fsa_data_t *msg_data);
/* A_SHUTDOWN_REQ */
void do_shutdown_req(long long action, enum crmd_fsa_cause cause,
enum crmd_fsa_state cur_state,
enum crmd_fsa_input cur_input, fsa_data_t *msg_data);
/* A_SHUTDOWN */
void do_shutdown(long long action, enum crmd_fsa_cause cause,
enum crmd_fsa_state cur_state,
enum crmd_fsa_input cur_input, fsa_data_t *msg_data);
/* A_STOP */
void do_stop(long long action, enum crmd_fsa_cause cause,
enum crmd_fsa_state cur_state,
enum crmd_fsa_input cur_input, fsa_data_t *msg_data);
/* A_EXIT_0, A_EXIT_1 */
void do_exit(long long action, enum crmd_fsa_cause cause,
enum crmd_fsa_state cur_state,
enum crmd_fsa_input cur_input, fsa_data_t *msg_data);
/* A_DC_JOIN_FINAL */
void do_dc_join_final(long long action, enum crmd_fsa_cause cause,
enum crmd_fsa_state cur_state,
enum crmd_fsa_input current_input, fsa_data_t *msg_data);
#endif
diff --git a/daemons/controld/controld_join_client.c b/daemons/controld/controld_join_client.c
index bd3f7eafce..f1123143ff 100644
--- a/daemons/controld/controld_join_client.c
+++ b/daemons/controld/controld_join_client.c
@@ -1,319 +1,319 @@
/*
* Copyright 2004-2022 the Pacemaker project contributors
*
* The version control history for this file may have further details.
*
* This source code is licensed under the GNU General Public License version 2
* or later (GPLv2+) WITHOUT ANY WARRANTY.
*/
#include <crm_internal.h>
#include <crm/crm.h>
#include <crm/cib.h>
#include <crm/msg_xml.h>
#include <crm/common/xml.h>
#include <pacemaker-controld.h>
extern pcmk__output_t *logger_out;
int reannounce_count = 0;
void join_query_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data);
extern ha_msg_input_t *copy_ha_msg_input(ha_msg_input_t * orig);
/*!
* \internal
* \brief Remember if DC is shutting down as we join
*
* If we're joining while the current DC is shutting down, update its expected
* state, so we don't fence it if we become the new DC. (We weren't a peer
* when it broadcast its shutdown request.)
*
* \param[in] msg A join message from the DC
*/
static void
update_dc_expected(const xmlNode *msg)
{
if (fsa_our_dc && pcmk__xe_attr_is_true(msg, F_CRM_DC_LEAVING)) {
crm_node_t *dc_node = crm_get_peer(0, fsa_our_dc);
pcmk__update_peer_expected(__func__, dc_node, CRMD_JOINSTATE_DOWN);
}
}
/* A_CL_JOIN_QUERY */
/* is there a DC out there? */
void
do_cl_join_query(long long action,
enum crmd_fsa_cause cause,
enum crmd_fsa_state cur_state,
enum crmd_fsa_input current_input, fsa_data_t * msg_data)
{
xmlNode *req = create_request(CRM_OP_JOIN_ANNOUNCE, NULL, NULL,
CRM_SYSTEM_DC, CRM_SYSTEM_CRMD, NULL);
sleep(1); // Give the cluster layer time to propagate to the DC
update_dc(NULL); /* Unset any existing value so that the result is not discarded */
crm_debug("Querying for a DC");
send_cluster_message(NULL, crm_msg_crmd, req, FALSE);
free_xml(req);
}
/* A_CL_JOIN_ANNOUNCE */
/* this is kind of a workaround for the fact that we may not be around or
* are otherwise unable to reply when the DC sends out A_DC_JOIN_OFFER_ALL
*/
void
do_cl_join_announce(long long action,
enum crmd_fsa_cause cause,
enum crmd_fsa_state cur_state,
enum crmd_fsa_input current_input, fsa_data_t * msg_data)
{
/* don't announce if we're in one of these states */
if (cur_state != S_PENDING) {
crm_warn("Not announcing cluster join because in state %s",
fsa_state2string(cur_state));
return;
}
if (AM_I_OPERATIONAL) {
/* send as a broadcast */
xmlNode *req = create_request(CRM_OP_JOIN_ANNOUNCE, NULL, NULL,
CRM_SYSTEM_DC, CRM_SYSTEM_CRMD, NULL);
crm_debug("Announcing availability");
update_dc(NULL);
send_cluster_message(NULL, crm_msg_crmd, req, FALSE);
free_xml(req);
} else {
/* Delay announce until we have finished local startup */
crm_warn("Delaying announce of cluster join until local startup is complete");
return;
}
}
static int query_call_id = 0;
/* A_CL_JOIN_REQUEST */
/* aka. accept the welcome offer */
void
do_cl_join_offer_respond(long long action,
enum crmd_fsa_cause cause,
enum crmd_fsa_state cur_state,
enum crmd_fsa_input current_input, fsa_data_t * msg_data)
{
ha_msg_input_t *input = fsa_typed_data(fsa_dt_ha_msg);
const char *welcome_from;
const char *join_id;
CRM_CHECK(input != NULL, return);
#if 0
if (we are sick) {
log error;
/* save the request for later? */
return;
}
#endif
welcome_from = crm_element_value(input->msg, F_CRM_HOST_FROM);
join_id = crm_element_value(input->msg, F_CRM_JOIN_ID);
crm_trace("Accepting cluster join offer from node %s "CRM_XS" join-%s",
welcome_from, crm_element_value(input->msg, F_CRM_JOIN_ID));
/* we only ever want the last one */
if (query_call_id > 0) {
crm_trace("Cancelling previous join query: %d", query_call_id);
remove_cib_op_callback(query_call_id, FALSE);
query_call_id = 0;
}
if (update_dc(input->msg) == FALSE) {
crm_warn("Discarding cluster join offer from node %s (expected %s)",
welcome_from, fsa_our_dc);
return;
}
update_dc_expected(input->msg);
query_call_id =
fsa_cib_conn->cmds->query(fsa_cib_conn, NULL, NULL, cib_scope_local | cib_no_children);
fsa_register_cib_callback(query_call_id, FALSE, strdup(join_id), join_query_callback);
crm_trace("Registered join query callback: %d", query_call_id);
controld_set_fsa_action_flags(A_DC_TIMER_STOP);
trigger_fsa();
}
void
join_query_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
{
char *join_id = user_data;
xmlNode *generation = create_xml_node(NULL, XML_CIB_TAG_GENERATION_TUPPLE);
CRM_LOG_ASSERT(join_id != NULL);
if (query_call_id != call_id) {
crm_trace("Query %d superseded", call_id);
goto done;
}
query_call_id = 0;
if(rc != pcmk_ok || output == NULL) {
crm_err("Could not retrieve version details for join-%s: %s (%d)",
join_id, pcmk_strerror(rc), rc);
register_fsa_error_adv(C_FSA_INTERNAL, I_ERROR, NULL, NULL, __func__);
} else if (fsa_our_dc == NULL) {
crm_debug("Membership is in flux, not continuing join-%s", join_id);
} else {
xmlNode *reply = NULL;
crm_debug("Respond to join offer join-%s from %s", join_id, fsa_our_dc);
copy_in_properties(generation, output);
reply = create_request(CRM_OP_JOIN_REQUEST, generation, fsa_our_dc,
CRM_SYSTEM_DC, CRM_SYSTEM_CRMD, NULL);
crm_xml_add(reply, F_CRM_JOIN_ID, join_id);
crm_xml_add(reply, XML_ATTR_CRM_VERSION, CRM_FEATURE_SET);
send_cluster_message(crm_get_peer(0, fsa_our_dc), crm_msg_crmd, reply, TRUE);
free_xml(reply);
}
done:
free_xml(generation);
}
static void
set_join_state(const char * start_state)
{
if (pcmk__str_eq(start_state, "standby", pcmk__str_casei)) {
crm_notice("Forcing node %s to join in %s state per configured environment",
fsa_our_uname, start_state);
cib__update_node_attr(logger_out, fsa_cib_conn, cib_sync_call,
XML_CIB_TAG_NODES, fsa_our_uuid, NULL, NULL,
NULL, "standby", "on", NULL, NULL);
} else if (pcmk__str_eq(start_state, "online", pcmk__str_casei)) {
crm_notice("Forcing node %s to join in %s state per configured environment",
fsa_our_uname, start_state);
cib__update_node_attr(logger_out, fsa_cib_conn, cib_sync_call,
XML_CIB_TAG_NODES, fsa_our_uuid, NULL, NULL,
NULL, "standby", "off", NULL, NULL);
} else if (pcmk__str_eq(start_state, "default", pcmk__str_casei)) {
crm_debug("Not forcing a starting state on node %s", fsa_our_uname);
} else {
crm_warn("Unrecognized start state '%s', using 'default' (%s)",
start_state, fsa_our_uname);
}
}
/* A_CL_JOIN_RESULT */
/* aka. this is notification that we have (or have not) been accepted */
void
do_cl_join_finalize_respond(long long action,
enum crmd_fsa_cause cause,
enum crmd_fsa_state cur_state,
enum crmd_fsa_input current_input, fsa_data_t * msg_data)
{
xmlNode *tmp1 = NULL;
gboolean was_nack = TRUE;
static gboolean first_join = TRUE;
ha_msg_input_t *input = fsa_typed_data(fsa_dt_ha_msg);
const char *start_state = pcmk__env_option(PCMK__ENV_NODE_START_STATE);
int join_id = -1;
const char *op = crm_element_value(input->msg, F_CRM_TASK);
const char *welcome_from = crm_element_value(input->msg, F_CRM_HOST_FROM);
if (!pcmk__str_eq(op, CRM_OP_JOIN_ACKNAK, pcmk__str_casei)) {
crm_trace("Ignoring op=%s message", op);
return;
}
/* calculate if it was an ack or a nack */
if (pcmk__xe_attr_is_true(input->msg, CRM_OP_JOIN_ACKNAK)) {
was_nack = FALSE;
}
crm_element_value_int(input->msg, F_CRM_JOIN_ID, &join_id);
if (was_nack) {
crm_err("Shutting down because cluster join with leader %s failed "
CRM_XS" join-%d NACK'd", welcome_from, join_id);
register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
return;
}
if (AM_I_DC == FALSE && pcmk__str_eq(welcome_from, fsa_our_uname, pcmk__str_casei)) {
crm_warn("Discarding our own welcome - we're no longer the DC");
return;
}
if (update_dc(input->msg) == FALSE) {
crm_warn("Discarding %s from node %s (expected from %s)",
op, welcome_from, fsa_our_dc);
return;
}
update_dc_expected(input->msg);
/* record the node's feature set as a transient attribute */
update_attrd(fsa_our_uname, CRM_ATTR_FEATURE_SET, CRM_FEATURE_SET, NULL,
FALSE);
/* send our status section to the DC */
- tmp1 = controld_query_executor_state(fsa_our_uname);
+ tmp1 = controld_query_executor_state();
if (tmp1 != NULL) {
xmlNode *reply = create_request(CRM_OP_JOIN_CONFIRM, tmp1, fsa_our_dc,
CRM_SYSTEM_DC, CRM_SYSTEM_CRMD, NULL);
crm_xml_add_int(reply, F_CRM_JOIN_ID, join_id);
crm_debug("Confirming join-%d: sending local operation history to %s",
join_id, fsa_our_dc);
/*
* If this is the node's first join since the controller started on it,
* set its initial state (standby or member) according to the user's
* preference.
*
* We do not clear the LRM history here. Even if the DC failed to do it
* when we last left, removing them here creates a race condition if the
* controller is being recovered. Instead of a list of active resources
* from the executor, we may end up with a blank status section. If we
* are _NOT_ lucky, we will probe for the "wrong" instance of anonymous
* clones and end up with multiple active instances on the machine.
*/
if (first_join && !pcmk_is_set(fsa_input_register, R_SHUTDOWN)) {
first_join = FALSE;
if (start_state) {
set_join_state(start_state);
}
}
send_cluster_message(crm_get_peer(0, fsa_our_dc), crm_msg_crmd, reply, TRUE);
free_xml(reply);
if (AM_I_DC == FALSE) {
register_fsa_input_adv(cause, I_NOT_DC, NULL, A_NOTHING, TRUE,
__func__);
}
free_xml(tmp1);
} else {
crm_err("Could not confirm join-%d with %s: Local operation history failed",
join_id, fsa_our_dc);
register_fsa_error(C_FSA_INTERNAL, I_FAIL, NULL);
}
}
diff --git a/daemons/controld/controld_join_dc.c b/daemons/controld/controld_join_dc.c
index c86abc8a32..a426e959b3 100644
--- a/daemons/controld/controld_join_dc.c
+++ b/daemons/controld/controld_join_dc.c
@@ -1,775 +1,775 @@
/*
* Copyright 2004-2022 the Pacemaker project contributors
*
* The version control history for this file may have further details.
*
* This source code is licensed under the GNU General Public License version 2
* or later (GPLv2+) WITHOUT ANY WARRANTY.
*/
#include <crm_internal.h>
#include <crm/crm.h>
#include <crm/msg_xml.h>
#include <crm/common/xml.h>
#include <crm/cluster.h>
#include <pacemaker-controld.h>
char *max_epoch = NULL;
char *max_generation_from = NULL;
xmlNode *max_generation_xml = NULL;
void finalize_join_for(gpointer key, gpointer value, gpointer user_data);
void finalize_sync_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data);
gboolean check_join_state(enum crmd_fsa_state cur_state, const char *source);
/* Numeric counter used to identify join rounds (an unsigned int would be
* appropriate, except we get and set it in XML as int)
*/
static int current_join_id = 0;
unsigned long long saved_ccm_membership_id = 0;
void
crm_update_peer_join(const char *source, crm_node_t * node, enum crm_join_phase phase)
{
enum crm_join_phase last = 0;
CRM_CHECK(node != NULL, return);
/* Remote nodes do not participate in joins */
if (pcmk_is_set(node->flags, crm_remote_node)) {
return;
}
last = node->join;
if(phase == last) {
crm_trace("Node %s join-%d phase is still %s "
CRM_XS " nodeid=%u source=%s",
node->uname, current_join_id, crm_join_phase_str(last),
node->id, source);
} else if ((phase <= crm_join_none) || (phase == (last + 1))) {
node->join = phase;
crm_trace("Node %s join-%d phase is now %s (was %s) "
CRM_XS " nodeid=%u source=%s",
node->uname, current_join_id, crm_join_phase_str(phase),
crm_join_phase_str(last), node->id, source);
} else {
crm_warn("Rejecting join-%d phase update for node %s because "
"can't go from %s to %s " CRM_XS " nodeid=%u source=%s",
current_join_id, node->uname, crm_join_phase_str(last),
crm_join_phase_str(phase), node->id, source);
}
}
static void
start_join_round(void)
{
GHashTableIter iter;
crm_node_t *peer = NULL;
crm_debug("Starting new join round join-%d", current_join_id);
g_hash_table_iter_init(&iter, crm_peer_cache);
while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &peer)) {
crm_update_peer_join(__func__, peer, crm_join_none);
}
if (max_generation_from != NULL) {
free(max_generation_from);
max_generation_from = NULL;
}
if (max_generation_xml != NULL) {
free_xml(max_generation_xml);
max_generation_xml = NULL;
}
controld_clear_fsa_input_flags(R_HAVE_CIB|R_CIB_ASKED);
}
/*!
* \internal
* \brief Create a join message from the DC
*
* \param[in] join_op Join operation name
* \param[in] host_to Recipient of message
*/
static xmlNode *
create_dc_message(const char *join_op, const char *host_to)
{
xmlNode *msg = create_request(join_op, NULL, host_to, CRM_SYSTEM_CRMD,
CRM_SYSTEM_DC, NULL);
/* Identify which election this is a part of */
crm_xml_add_int(msg, F_CRM_JOIN_ID, current_join_id);
/* Add a field specifying whether the DC is shutting down. This keeps the
* joining node from fencing the old DC if it becomes the new DC.
*/
pcmk__xe_set_bool_attr(msg, F_CRM_DC_LEAVING,
pcmk_is_set(fsa_input_register, R_SHUTDOWN));
return msg;
}
static void
join_make_offer(gpointer key, gpointer value, gpointer user_data)
{
xmlNode *offer = NULL;
crm_node_t *member = (crm_node_t *)value;
CRM_ASSERT(member != NULL);
if (crm_is_peer_active(member) == FALSE) {
crm_info("Not making join-%d offer to inactive node %s",
current_join_id,
(member->uname? member->uname : "with unknown name"));
if(member->expected == NULL && pcmk__str_eq(member->state, CRM_NODE_LOST, pcmk__str_casei)) {
/* You would think this unsafe, but in fact this plus an
* active resource is what causes it to be fenced.
*
* Yes, this does mean that any node that dies at the same
* time as the old DC and is not running resource (still)
* won't be fenced.
*
* I'm not happy about this either.
*/
pcmk__update_peer_expected(__func__, member, CRMD_JOINSTATE_DOWN);
}
return;
}
if (member->uname == NULL) {
crm_info("Not making join-%d offer to node uuid %s with unknown name",
current_join_id, member->uuid);
return;
}
if (saved_ccm_membership_id != crm_peer_seq) {
saved_ccm_membership_id = crm_peer_seq;
crm_info("Making join-%d offers based on membership event %llu",
current_join_id, crm_peer_seq);
}
if(user_data && member->join > crm_join_none) {
crm_info("Not making join-%d offer to already known node %s (%s)",
current_join_id, member->uname,
crm_join_phase_str(member->join));
return;
}
crm_update_peer_join(__func__, (crm_node_t*)member, crm_join_none);
offer = create_dc_message(CRM_OP_JOIN_OFFER, member->uname);
// Advertise our feature set so the joining node can bail if not compatible
crm_xml_add(offer, XML_ATTR_CRM_VERSION, CRM_FEATURE_SET);
crm_info("Sending join-%d offer to %s", current_join_id, member->uname);
send_cluster_message(member, crm_msg_crmd, offer, TRUE);
free_xml(offer);
crm_update_peer_join(__func__, member, crm_join_welcomed);
}
/* A_DC_JOIN_OFFER_ALL */
void
do_dc_join_offer_all(long long action,
enum crmd_fsa_cause cause,
enum crmd_fsa_state cur_state,
enum crmd_fsa_input current_input, fsa_data_t * msg_data)
{
int count;
/* Reset everyone's status back to down or in_ccm in the CIB.
* Any nodes that are active in the CIB but not in the cluster membership
* will be seen as offline by the scheduler anyway.
*/
current_join_id++;
start_join_round();
// do_update_cib_nodes(TRUE, __func__);
update_dc(NULL);
if (cause == C_HA_MESSAGE && current_input == I_NODE_JOIN) {
crm_info("A new node joined the cluster");
}
g_hash_table_foreach(crm_peer_cache, join_make_offer, NULL);
count = crmd_join_phase_count(crm_join_welcomed);
crm_info("Waiting on join-%d requests from %d outstanding node%s",
current_join_id, count, pcmk__plural_s(count));
// Don't waste time by invoking the scheduler yet
}
/* A_DC_JOIN_OFFER_ONE */
void
do_dc_join_offer_one(long long action,
enum crmd_fsa_cause cause,
enum crmd_fsa_state cur_state,
enum crmd_fsa_input current_input, fsa_data_t * msg_data)
{
crm_node_t *member;
ha_msg_input_t *welcome = NULL;
int count;
const char *join_to = NULL;
if (msg_data->data == NULL) {
crm_info("Making join-%d offers to any unconfirmed nodes "
"because an unknown node joined", current_join_id);
g_hash_table_foreach(crm_peer_cache, join_make_offer, &member);
check_join_state(cur_state, __func__);
return;
}
welcome = fsa_typed_data(fsa_dt_ha_msg);
if (welcome == NULL) {
// fsa_typed_data() already logged an error
return;
}
join_to = crm_element_value(welcome->msg, F_CRM_HOST_FROM);
if (join_to == NULL) {
crm_err("Can't make join-%d offer to unknown node", current_join_id);
return;
}
member = crm_get_peer(0, join_to);
/* It is possible that a node will have been sick or starting up when the
* original offer was made. However, it will either re-announce itself in
* due course, or we can re-store the original offer on the client.
*/
crm_update_peer_join(__func__, member, crm_join_none);
join_make_offer(NULL, member, NULL);
/* If the offer isn't to the local node, make an offer to the local node as
* well, to ensure the correct value for max_generation_from.
*/
if (strcmp(join_to, fsa_our_uname) != 0) {
member = crm_get_peer(0, fsa_our_uname);
join_make_offer(NULL, member, NULL);
}
/* This was a genuine join request; cancel any existing transition and
* invoke the scheduler.
*/
abort_transition(INFINITY, pcmk__graph_restart, "Node join", NULL);
count = crmd_join_phase_count(crm_join_welcomed);
crm_info("Waiting on join-%d requests from %d outstanding node%s",
current_join_id, count, pcmk__plural_s(count));
// Don't waste time by invoking the scheduler yet
}
static int
compare_int_fields(xmlNode * left, xmlNode * right, const char *field)
{
const char *elem_l = crm_element_value(left, field);
const char *elem_r = crm_element_value(right, field);
long long int_elem_l;
long long int_elem_r;
pcmk__scan_ll(elem_l, &int_elem_l, -1LL);
pcmk__scan_ll(elem_r, &int_elem_r, -1LL);
if (int_elem_l < int_elem_r) {
return -1;
} else if (int_elem_l > int_elem_r) {
return 1;
}
return 0;
}
/* A_DC_JOIN_PROCESS_REQ */
void
do_dc_join_filter_offer(long long action,
enum crmd_fsa_cause cause,
enum crmd_fsa_state cur_state,
enum crmd_fsa_input current_input, fsa_data_t * msg_data)
{
xmlNode *generation = NULL;
int cmp = 0;
int join_id = -1;
int count = 0;
gboolean ack_nack_bool = TRUE;
ha_msg_input_t *join_ack = fsa_typed_data(fsa_dt_ha_msg);
const char *join_from = crm_element_value(join_ack->msg, F_CRM_HOST_FROM);
const char *ref = crm_element_value(join_ack->msg, F_CRM_REFERENCE);
const char *join_version = crm_element_value(join_ack->msg,
XML_ATTR_CRM_VERSION);
crm_node_t *join_node = NULL;
if (join_from == NULL) {
crm_err("Ignoring invalid join request without node name");
return;
}
join_node = crm_get_peer(0, join_from);
crm_element_value_int(join_ack->msg, F_CRM_JOIN_ID, &join_id);
if (join_id != current_join_id) {
crm_debug("Ignoring join-%d request from %s because we are on join-%d",
join_id, join_from, current_join_id);
check_join_state(cur_state, __func__);
return;
}
generation = join_ack->xml;
if (max_generation_xml != NULL && generation != NULL) {
int lpc = 0;
const char *attributes[] = {
XML_ATTR_GENERATION_ADMIN,
XML_ATTR_GENERATION,
XML_ATTR_NUMUPDATES,
};
for (lpc = 0; cmp == 0 && lpc < PCMK__NELEM(attributes); lpc++) {
cmp = compare_int_fields(max_generation_xml, generation, attributes[lpc]);
}
}
if (ref == NULL) {
ref = "none"; // for logging only
}
if (crm_is_peer_active(join_node) == FALSE) {
crm_err("Rejecting join-%d request from inactive node %s "
CRM_XS " ref=%s", join_id, join_from, ref);
ack_nack_bool = FALSE;
} else if (generation == NULL) {
crm_err("Rejecting invalid join-%d request from node %s "
"missing CIB generation " CRM_XS " ref=%s",
join_id, join_from, ref);
ack_nack_bool = FALSE;
} else if ((join_version == NULL)
|| !feature_set_compatible(CRM_FEATURE_SET, join_version)) {
crm_err("Rejecting join-%d request from node %s because feature set %s"
" is incompatible with ours (%s) " CRM_XS " ref=%s",
join_id, join_from, (join_version? join_version : "pre-3.1.0"),
CRM_FEATURE_SET, ref);
ack_nack_bool = FALSE;
} else if (max_generation_xml == NULL) {
crm_debug("Accepting join-%d request from %s "
"(with first CIB generation) " CRM_XS " ref=%s",
join_id, join_from, ref);
max_generation_xml = copy_xml(generation);
max_generation_from = strdup(join_from);
} else if (cmp < 0 || (cmp == 0 && pcmk__str_eq(join_from, fsa_our_uname, pcmk__str_casei))) {
crm_debug("Accepting join-%d request from %s (with better "
"CIB generation than current best from %s) " CRM_XS " ref=%s",
join_id, join_from, max_generation_from, ref);
crm_log_xml_debug(max_generation_xml, "Old max generation");
crm_log_xml_debug(generation, "New max generation");
pcmk__str_update(&max_generation_from, join_from);
free_xml(max_generation_xml);
max_generation_xml = copy_xml(join_ack->xml);
} else {
crm_debug("Accepting join-%d request from %s " CRM_XS " ref=%s",
join_id, join_from, ref);
}
if (ack_nack_bool == FALSE) {
crm_update_peer_join(__func__, join_node, crm_join_nack);
pcmk__update_peer_expected(__func__, join_node, CRMD_JOINSTATE_NACK);
} else {
crm_update_peer_join(__func__, join_node, crm_join_integrated);
pcmk__update_peer_expected(__func__, join_node, CRMD_JOINSTATE_MEMBER);
}
count = crmd_join_phase_count(crm_join_integrated);
crm_debug("%d node%s currently integrated in join-%d",
count, pcmk__plural_s(count), join_id);
if (check_join_state(cur_state, __func__) == FALSE) {
// Don't waste time by invoking the scheduler yet
count = crmd_join_phase_count(crm_join_welcomed);
crm_debug("Waiting on join-%d requests from %d outstanding node%s",
join_id, count, pcmk__plural_s(count));
}
}
/* A_DC_JOIN_FINALIZE */
void
do_dc_join_finalize(long long action,
enum crmd_fsa_cause cause,
enum crmd_fsa_state cur_state,
enum crmd_fsa_input current_input, fsa_data_t * msg_data)
{
char *sync_from = NULL;
int rc = pcmk_ok;
int count_welcomed = crmd_join_phase_count(crm_join_welcomed);
int count_integrated = crmd_join_phase_count(crm_join_integrated);
/* This we can do straight away and avoid clients timing us out
* while we compute the latest CIB
*/
if (count_welcomed != 0) {
crm_debug("Waiting on join-%d requests from %d outstanding node%s "
"before finalizing join", current_join_id, count_welcomed,
pcmk__plural_s(count_welcomed));
crmd_join_phase_log(LOG_DEBUG);
/* crmd_fsa_stall(FALSE); Needed? */
return;
} else if (count_integrated == 0) {
crm_debug("Finalization not needed for join-%d at the current time",
current_join_id);
crmd_join_phase_log(LOG_DEBUG);
check_join_state(fsa_state, __func__);
return;
}
controld_clear_fsa_input_flags(R_HAVE_CIB);
if (pcmk__str_eq(max_generation_from, fsa_our_uname, pcmk__str_null_matches | pcmk__str_casei)) {
controld_set_fsa_input_flags(R_HAVE_CIB);
}
if (pcmk_is_set(fsa_input_register, R_IN_TRANSITION)) {
crm_warn("Delaying join-%d finalization while transition in progress",
current_join_id);
crmd_join_phase_log(LOG_DEBUG);
crmd_fsa_stall(FALSE);
return;
}
if (max_generation_from && !pcmk_is_set(fsa_input_register, R_HAVE_CIB)) {
/* ask for the agreed best CIB */
sync_from = strdup(max_generation_from);
controld_set_fsa_input_flags(R_CIB_ASKED);
crm_notice("Finalizing join-%d for %d node%s (sync'ing CIB from %s)",
current_join_id, count_integrated,
pcmk__plural_s(count_integrated), sync_from);
crm_log_xml_notice(max_generation_xml, "Requested CIB version");
} else {
/* Send _our_ CIB out to everyone */
sync_from = strdup(fsa_our_uname);
crm_debug("Finalizing join-%d for %d node%s (sync'ing from local CIB)",
current_join_id, count_integrated,
pcmk__plural_s(count_integrated));
crm_log_xml_debug(max_generation_xml, "Requested CIB version");
}
crmd_join_phase_log(LOG_DEBUG);
rc = fsa_cib_conn->cmds->sync_from(fsa_cib_conn, sync_from, NULL, cib_quorum_override);
fsa_register_cib_callback(rc, FALSE, sync_from, finalize_sync_callback);
}
void
finalize_sync_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
{
CRM_LOG_ASSERT(-EPERM != rc);
controld_clear_fsa_input_flags(R_CIB_ASKED);
if (rc != pcmk_ok) {
do_crm_log(((rc == -pcmk_err_old_data)? LOG_WARNING : LOG_ERR),
"Could not sync CIB from %s in join-%d: %s",
(char *) user_data, current_join_id, pcmk_strerror(rc));
/* restart the whole join process */
register_fsa_error_adv(C_FSA_INTERNAL, I_ELECTION_DC, NULL, NULL,
__func__);
} else if (!AM_I_DC) {
crm_debug("Sync'ed CIB for join-%d but no longer DC", current_join_id);
} else if (fsa_state != S_FINALIZE_JOIN) {
crm_debug("Sync'ed CIB for join-%d but no longer in S_FINALIZE_JOIN (%s)",
current_join_id, fsa_state2string(fsa_state));
} else {
controld_set_fsa_input_flags(R_HAVE_CIB);
controld_clear_fsa_input_flags(R_CIB_ASKED);
/* make sure dc_uuid is re-set to us */
if (check_join_state(fsa_state, __func__) == FALSE) {
int count_integrated = crmd_join_phase_count(crm_join_integrated);
crm_debug("Notifying %d node%s of join-%d results",
count_integrated, pcmk__plural_s(count_integrated),
current_join_id);
g_hash_table_foreach(crm_peer_cache, finalize_join_for, NULL);
}
}
}
static void
join_update_complete_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
{
fsa_data_t *msg_data = NULL;
if (rc == pcmk_ok) {
crm_debug("join-%d node history update (via CIB call %d) complete",
current_join_id, call_id);
check_join_state(fsa_state, __func__);
} else {
crm_err("join-%d node history update (via CIB call %d) failed: %s "
"(next transition may determine resource status incorrectly)",
current_join_id, call_id, pcmk_strerror(rc));
crm_log_xml_debug(msg, "failed");
register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
}
}
/* A_DC_JOIN_PROCESS_ACK */
void
do_dc_join_ack(long long action,
enum crmd_fsa_cause cause,
enum crmd_fsa_state cur_state,
enum crmd_fsa_input current_input, fsa_data_t * msg_data)
{
int join_id = -1;
int call_id = 0;
ha_msg_input_t *join_ack = fsa_typed_data(fsa_dt_ha_msg);
enum controld_section_e section = controld_section_lrm;
const char *op = crm_element_value(join_ack->msg, F_CRM_TASK);
const char *join_from = crm_element_value(join_ack->msg, F_CRM_HOST_FROM);
crm_node_t *peer = NULL;
// Sanity checks
if (join_from == NULL) {
crm_warn("Ignoring message received without node identification");
return;
}
if (op == NULL) {
crm_warn("Ignoring message received from %s without task", join_from);
return;
}
if (strcmp(op, CRM_OP_JOIN_CONFIRM)) {
crm_debug("Ignoring '%s' message from %s while waiting for '%s'",
op, join_from, CRM_OP_JOIN_CONFIRM);
return;
}
if (crm_element_value_int(join_ack->msg, F_CRM_JOIN_ID, &join_id) != 0) {
crm_warn("Ignoring join confirmation from %s without valid join ID",
join_from);
return;
}
peer = crm_get_peer(0, join_from);
if (peer->join != crm_join_finalized) {
crm_info("Ignoring out-of-sequence join-%d confirmation from %s "
"(currently %s not %s)",
join_id, join_from, crm_join_phase_str(peer->join),
crm_join_phase_str(crm_join_finalized));
return;
}
if (join_id != current_join_id) {
crm_err("Rejecting join-%d confirmation from %s "
"because currently on join-%d",
join_id, join_from, current_join_id);
crm_update_peer_join(__func__, peer, crm_join_nack);
return;
}
crm_update_peer_join(__func__, peer, crm_join_confirmed);
/* Update CIB with node's current executor state. A new transition will be
* triggered later, when the CIB notifies us of the change.
*/
if (controld_shutdown_lock_enabled) {
section = controld_section_lrm_unlocked;
}
controld_delete_node_state(join_from, section, cib_scope_local);
if (pcmk__str_eq(join_from, fsa_our_uname, pcmk__str_casei)) {
- xmlNode *now_dc_lrmd_state = controld_query_executor_state(fsa_our_uname);
+ xmlNode *now_dc_lrmd_state = controld_query_executor_state();
if (now_dc_lrmd_state != NULL) {
fsa_cib_update(XML_CIB_TAG_STATUS, now_dc_lrmd_state,
cib_scope_local | cib_quorum_override | cib_can_create, call_id, NULL);
free_xml(now_dc_lrmd_state);
crm_debug("Updating local node history for join-%d "
"from query result (via CIB call %d)", join_id, call_id);
} else {
fsa_cib_update(XML_CIB_TAG_STATUS, join_ack->xml,
cib_scope_local | cib_quorum_override | cib_can_create, call_id, NULL);
crm_warn("Updating local node history from join-%d confirmation "
"because query failed (via CIB call %d)", join_id, call_id);
}
} else {
fsa_cib_update(XML_CIB_TAG_STATUS, join_ack->xml,
cib_scope_local | cib_quorum_override | cib_can_create, call_id, NULL);
crm_debug("Updating node history for %s from join-%d confirmation "
"(via CIB call %d)", join_from, join_id, call_id);
}
fsa_register_cib_callback(call_id, FALSE, NULL, join_update_complete_callback);
}
void
finalize_join_for(gpointer key, gpointer value, gpointer user_data)
{
xmlNode *acknak = NULL;
xmlNode *tmp1 = NULL;
crm_node_t *join_node = value;
const char *join_to = join_node->uname;
if(join_node->join != crm_join_integrated) {
crm_trace("Not updating non-integrated node %s (%s) for join-%d",
join_to, crm_join_phase_str(join_node->join),
current_join_id);
return;
}
crm_trace("Updating node state for %s", join_to);
tmp1 = create_xml_node(NULL, XML_CIB_TAG_NODE);
set_uuid(tmp1, XML_ATTR_UUID, join_node);
crm_xml_add(tmp1, XML_ATTR_UNAME, join_to);
fsa_cib_anon_update(XML_CIB_TAG_NODES, tmp1);
free_xml(tmp1);
join_node = crm_get_peer(0, join_to);
if (crm_is_peer_active(join_node) == FALSE) {
/*
* NACK'ing nodes that the membership layer doesn't know about yet
* simply creates more churn
*
* Better to leave them waiting and let the join restart when
* the new membership event comes in
*
* All other NACKs (due to versions etc) should still be processed
*/
pcmk__update_peer_expected(__func__, join_node, CRMD_JOINSTATE_PENDING);
return;
}
// Acknowledge node's join request
crm_debug("Acknowledging join-%d request from %s",
current_join_id, join_to);
acknak = create_dc_message(CRM_OP_JOIN_ACKNAK, join_to);
pcmk__xe_set_bool_attr(acknak, CRM_OP_JOIN_ACKNAK, true);
crm_update_peer_join(__func__, join_node, crm_join_finalized);
pcmk__update_peer_expected(__func__, join_node, CRMD_JOINSTATE_MEMBER);
send_cluster_message(crm_get_peer(0, join_to), crm_msg_crmd, acknak, TRUE);
free_xml(acknak);
return;
}
gboolean
check_join_state(enum crmd_fsa_state cur_state, const char *source)
{
static unsigned long long highest_seq = 0;
if (saved_ccm_membership_id != crm_peer_seq) {
crm_debug("join-%d: Membership changed from %llu to %llu "
CRM_XS " highest=%llu state=%s for=%s",
current_join_id, saved_ccm_membership_id, crm_peer_seq, highest_seq,
fsa_state2string(cur_state), source);
if(highest_seq < crm_peer_seq) {
/* Don't spam the FSA with duplicates */
highest_seq = crm_peer_seq;
register_fsa_input_before(C_FSA_INTERNAL, I_NODE_JOIN, NULL);
}
} else if (cur_state == S_INTEGRATION) {
if (crmd_join_phase_count(crm_join_welcomed) == 0) {
int count = crmd_join_phase_count(crm_join_integrated);
crm_debug("join-%d: Integration of %d peer%s complete "
CRM_XS " state=%s for=%s",
current_join_id, count, pcmk__plural_s(count),
fsa_state2string(cur_state), source);
register_fsa_input_before(C_FSA_INTERNAL, I_INTEGRATED, NULL);
return TRUE;
}
} else if (cur_state == S_FINALIZE_JOIN) {
if (!pcmk_is_set(fsa_input_register, R_HAVE_CIB)) {
crm_debug("join-%d: Delaying finalization until we have CIB "
CRM_XS " state=%s for=%s",
current_join_id, fsa_state2string(cur_state), source);
return TRUE;
} else if (crmd_join_phase_count(crm_join_welcomed) != 0) {
int count = crmd_join_phase_count(crm_join_welcomed);
crm_debug("join-%d: Still waiting on %d welcomed node%s "
CRM_XS " state=%s for=%s",
current_join_id, count, pcmk__plural_s(count),
fsa_state2string(cur_state), source);
crmd_join_phase_log(LOG_DEBUG);
} else if (crmd_join_phase_count(crm_join_integrated) != 0) {
int count = crmd_join_phase_count(crm_join_integrated);
crm_debug("join-%d: Still waiting on %d integrated node%s "
CRM_XS " state=%s for=%s",
current_join_id, count, pcmk__plural_s(count),
fsa_state2string(cur_state), source);
crmd_join_phase_log(LOG_DEBUG);
} else if (crmd_join_phase_count(crm_join_finalized) != 0) {
int count = crmd_join_phase_count(crm_join_finalized);
crm_debug("join-%d: Still waiting on %d finalized node%s "
CRM_XS " state=%s for=%s",
current_join_id, count, pcmk__plural_s(count),
fsa_state2string(cur_state), source);
crmd_join_phase_log(LOG_DEBUG);
} else {
crm_debug("join-%d: Complete " CRM_XS " state=%s for=%s",
current_join_id, fsa_state2string(cur_state), source);
register_fsa_input_later(C_FSA_INTERNAL, I_FINALIZED, NULL);
return TRUE;
}
}
return FALSE;
}
void
do_dc_join_final(long long action,
enum crmd_fsa_cause cause,
enum crmd_fsa_state cur_state,
enum crmd_fsa_input current_input, fsa_data_t * msg_data)
{
crm_debug("Ensuring DC, quorum and node attributes are up-to-date");
crm_update_quorum(crm_have_quorum, TRUE);
}
int crmd_join_phase_count(enum crm_join_phase phase)
{
int count = 0;
crm_node_t *peer;
GHashTableIter iter;
g_hash_table_iter_init(&iter, crm_peer_cache);
while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &peer)) {
if(peer->join == phase) {
count++;
}
}
return count;
}
void crmd_join_phase_log(int level)
{
crm_node_t *peer;
GHashTableIter iter;
g_hash_table_iter_init(&iter, crm_peer_cache);
while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &peer)) {
do_crm_log(level, "join-%d: %s=%s", current_join_id, peer->uname,
crm_join_phase_str(peer->join));
}
}
diff --git a/daemons/controld/controld_messages.c b/daemons/controld/controld_messages.c
index 4f1af80dc8..c7d4ed7708 100644
--- a/daemons/controld/controld_messages.c
+++ b/daemons/controld/controld_messages.c
@@ -1,1290 +1,1290 @@
/*
* Copyright 2004-2022 the Pacemaker project contributors
*
* The version control history for this file may have further details.
*
* This source code is licensed under the GNU General Public License version 2
* or later (GPLv2+) WITHOUT ANY WARRANTY.
*/
#include <crm_internal.h>
#include <sys/param.h>
#include <string.h>
#include <time.h>
#include <crm/crm.h>
#include <crm/msg_xml.h>
#include <crm/common/xml.h>
#include <crm/cluster/internal.h>
#include <crm/cib.h>
#include <crm/common/ipc_internal.h>
#include <pacemaker-controld.h>
GList *fsa_message_queue = NULL;
extern void crm_shutdown(int nsig);
static enum crmd_fsa_input handle_message(xmlNode *msg,
enum crmd_fsa_cause cause);
static void handle_response(xmlNode *stored_msg);
static enum crmd_fsa_input handle_request(xmlNode *stored_msg,
enum crmd_fsa_cause cause);
static enum crmd_fsa_input handle_shutdown_request(xmlNode *stored_msg);
static void send_msg_via_ipc(xmlNode * msg, const char *sys);
/* debug only, can wrap all it likes */
int last_data_id = 0;
void
register_fsa_error_adv(enum crmd_fsa_cause cause, enum crmd_fsa_input input,
fsa_data_t * cur_data, void *new_data, const char *raised_from)
{
/* save the current actions if any */
if (fsa_actions != A_NOTHING) {
register_fsa_input_adv(cur_data ? cur_data->fsa_cause : C_FSA_INTERNAL,
I_NULL, cur_data ? cur_data->data : NULL,
fsa_actions, TRUE, __func__);
}
/* reset the action list */
crm_info("Resetting the current action list");
fsa_dump_actions(fsa_actions, "Drop");
fsa_actions = A_NOTHING;
/* register the error */
register_fsa_input_adv(cause, input, new_data, A_NOTHING, TRUE, raised_from);
}
int
register_fsa_input_adv(enum crmd_fsa_cause cause, enum crmd_fsa_input input,
void *data, uint64_t with_actions,
gboolean prepend, const char *raised_from)
{
unsigned old_len = g_list_length(fsa_message_queue);
fsa_data_t *fsa_data = NULL;
if (raised_from == NULL) {
raised_from = "<unknown>";
}
if (input == I_NULL && with_actions == A_NOTHING /* && data == NULL */ ) {
/* no point doing anything */
crm_err("Cannot add entry to queue: no input and no action");
return 0;
}
if (input == I_WAIT_FOR_EVENT) {
do_fsa_stall = TRUE;
crm_debug("Stalling the FSA pending further input: source=%s cause=%s data=%p queue=%d",
raised_from, fsa_cause2string(cause), data, old_len);
if (old_len > 0) {
fsa_dump_queue(LOG_TRACE);
prepend = FALSE;
}
if (data == NULL) {
controld_set_fsa_action_flags(with_actions);
fsa_dump_actions(with_actions, "Restored");
return 0;
}
/* Store everything in the new event and reset fsa_actions */
with_actions |= fsa_actions;
fsa_actions = A_NOTHING;
}
last_data_id++;
crm_trace("%s %s FSA input %d (%s) due to %s, %s data",
raised_from, (prepend? "prepended" : "appended"), last_data_id,
fsa_input2string(input), fsa_cause2string(cause),
(data? "with" : "without"));
fsa_data = calloc(1, sizeof(fsa_data_t));
fsa_data->id = last_data_id;
fsa_data->fsa_input = input;
fsa_data->fsa_cause = cause;
fsa_data->origin = raised_from;
fsa_data->data = NULL;
fsa_data->data_type = fsa_dt_none;
fsa_data->actions = with_actions;
if (with_actions != A_NOTHING) {
crm_trace("Adding actions %.16llx to input",
(unsigned long long) with_actions);
}
if (data != NULL) {
switch (cause) {
case C_FSA_INTERNAL:
case C_CRMD_STATUS_CALLBACK:
case C_IPC_MESSAGE:
case C_HA_MESSAGE:
CRM_CHECK(((ha_msg_input_t *) data)->msg != NULL,
crm_err("Bogus data from %s", raised_from));
crm_trace("Copying %s data from %s as cluster message data",
fsa_cause2string(cause), raised_from);
fsa_data->data = copy_ha_msg_input(data);
fsa_data->data_type = fsa_dt_ha_msg;
break;
case C_LRM_OP_CALLBACK:
crm_trace("Copying %s data from %s as lrmd_event_data_t",
fsa_cause2string(cause), raised_from);
fsa_data->data = lrmd_copy_event((lrmd_event_data_t *) data);
fsa_data->data_type = fsa_dt_lrm;
break;
case C_TIMER_POPPED:
case C_SHUTDOWN:
case C_UNKNOWN:
case C_STARTUP:
crm_crit("Copying %s data (from %s) is not yet implemented",
fsa_cause2string(cause), raised_from);
crmd_exit(CRM_EX_SOFTWARE);
break;
}
}
/* make sure to free it properly later */
if (prepend) {
fsa_message_queue = g_list_prepend(fsa_message_queue, fsa_data);
} else {
fsa_message_queue = g_list_append(fsa_message_queue, fsa_data);
}
crm_trace("FSA message queue length is %d",
g_list_length(fsa_message_queue));
/* fsa_dump_queue(LOG_TRACE); */
if (old_len == g_list_length(fsa_message_queue)) {
crm_err("Couldn't add message to the queue");
}
if (fsa_source && input != I_WAIT_FOR_EVENT) {
crm_trace("Triggering FSA");
mainloop_set_trigger(fsa_source);
}
return last_data_id;
}
void
fsa_dump_queue(int log_level)
{
int offset = 0;
GList *lpc = NULL;
for (lpc = fsa_message_queue; lpc != NULL; lpc = lpc->next) {
fsa_data_t *data = (fsa_data_t *) lpc->data;
do_crm_log_unlikely(log_level,
"queue[%d.%d]: input %s raised by %s(%p.%d)\t(cause=%s)",
offset++, data->id, fsa_input2string(data->fsa_input),
data->origin, data->data, data->data_type,
fsa_cause2string(data->fsa_cause));
}
}
ha_msg_input_t *
copy_ha_msg_input(ha_msg_input_t * orig)
{
ha_msg_input_t *copy = calloc(1, sizeof(ha_msg_input_t));
CRM_ASSERT(copy != NULL);
copy->msg = (orig && orig->msg)? copy_xml(orig->msg) : NULL;
copy->xml = get_message_xml(copy->msg, F_CRM_DATA);
return copy;
}
void
delete_fsa_input(fsa_data_t * fsa_data)
{
lrmd_event_data_t *op = NULL;
xmlNode *foo = NULL;
if (fsa_data == NULL) {
return;
}
crm_trace("About to free %s data", fsa_cause2string(fsa_data->fsa_cause));
if (fsa_data->data != NULL) {
switch (fsa_data->data_type) {
case fsa_dt_ha_msg:
delete_ha_msg_input(fsa_data->data);
break;
case fsa_dt_xml:
foo = fsa_data->data;
free_xml(foo);
break;
case fsa_dt_lrm:
op = (lrmd_event_data_t *) fsa_data->data;
lrmd_free_event(op);
break;
case fsa_dt_none:
if (fsa_data->data != NULL) {
crm_err("Don't know how to free %s data from %s",
fsa_cause2string(fsa_data->fsa_cause), fsa_data->origin);
crmd_exit(CRM_EX_SOFTWARE);
}
break;
}
crm_trace("%s data freed", fsa_cause2string(fsa_data->fsa_cause));
}
free(fsa_data);
}
/* returns the next message */
fsa_data_t *
get_message(void)
{
fsa_data_t *message = g_list_nth_data(fsa_message_queue, 0);
fsa_message_queue = g_list_remove(fsa_message_queue, message);
crm_trace("Processing input %d", message->id);
return message;
}
void *
fsa_typed_data_adv(fsa_data_t * fsa_data, enum fsa_data_type a_type, const char *caller)
{
void *ret_val = NULL;
if (fsa_data == NULL) {
crm_err("%s: No FSA data available", caller);
} else if (fsa_data->data == NULL) {
crm_err("%s: No message data available. Origin: %s", caller, fsa_data->origin);
} else if (fsa_data->data_type != a_type) {
crm_crit("%s: Message data was the wrong type! %d vs. requested=%d. Origin: %s",
caller, fsa_data->data_type, a_type, fsa_data->origin);
CRM_ASSERT(fsa_data->data_type == a_type);
} else {
ret_val = fsa_data->data;
}
return ret_val;
}
/* A_MSG_ROUTE */
void
do_msg_route(long long action,
enum crmd_fsa_cause cause,
enum crmd_fsa_state cur_state,
enum crmd_fsa_input current_input, fsa_data_t * msg_data)
{
ha_msg_input_t *input = fsa_typed_data(fsa_dt_ha_msg);
route_message(msg_data->fsa_cause, input->msg);
}
void
route_message(enum crmd_fsa_cause cause, xmlNode * input)
{
ha_msg_input_t fsa_input;
enum crmd_fsa_input result = I_NULL;
fsa_input.msg = input;
CRM_CHECK(cause == C_IPC_MESSAGE || cause == C_HA_MESSAGE, return);
/* try passing the buck first */
if (relay_message(input, cause == C_IPC_MESSAGE)) {
return;
}
/* handle locally */
result = handle_message(input, cause);
/* done or process later? */
switch (result) {
case I_NULL:
case I_CIB_OP:
case I_ROUTER:
case I_NODE_JOIN:
case I_JOIN_REQUEST:
case I_JOIN_RESULT:
break;
default:
/* Defering local processing of message */
register_fsa_input_later(cause, result, &fsa_input);
return;
}
if (result != I_NULL) {
/* add to the front of the queue */
register_fsa_input(cause, result, &fsa_input);
}
}
gboolean
relay_message(xmlNode * msg, gboolean originated_locally)
{
int dest = 1;
bool is_for_dc = false;
bool is_for_dcib = false;
bool is_for_te = false;
bool is_for_crm = false;
bool is_for_cib = false;
bool is_local = false;
const char *host_to = crm_element_value(msg, F_CRM_HOST_TO);
const char *sys_to = crm_element_value(msg, F_CRM_SYS_TO);
const char *sys_from = crm_element_value(msg, F_CRM_SYS_FROM);
const char *type = crm_element_value(msg, F_TYPE);
const char *task = crm_element_value(msg, F_CRM_TASK);
const char *ref = crm_element_value(msg, XML_ATTR_REFERENCE);
if (ref == NULL) {
ref = "without reference ID";
}
if (msg == NULL) {
crm_warn("Cannot route empty message");
return TRUE;
} else if (pcmk__str_eq(task, CRM_OP_HELLO, pcmk__str_casei)) {
crm_trace("No routing needed for hello message %s", ref);
return TRUE;
} else if (!pcmk__str_eq(type, T_CRM, pcmk__str_casei)) {
crm_warn("Received invalid message %s: type '%s' not '" T_CRM "'",
ref, pcmk__s(type, ""));
crm_log_xml_warn(msg, "[bad message type]");
return TRUE;
} else if (sys_to == NULL) {
crm_warn("Received invalid message %s: no subsystem", ref);
crm_log_xml_warn(msg, "[no subsystem]");
return TRUE;
}
is_for_dc = (strcasecmp(CRM_SYSTEM_DC, sys_to) == 0);
is_for_dcib = (strcasecmp(CRM_SYSTEM_DCIB, sys_to) == 0);
is_for_te = (strcasecmp(CRM_SYSTEM_TENGINE, sys_to) == 0);
is_for_cib = (strcasecmp(CRM_SYSTEM_CIB, sys_to) == 0);
is_for_crm = (strcasecmp(CRM_SYSTEM_CRMD, sys_to) == 0);
is_local = false;
if (pcmk__str_empty(host_to)) {
if (is_for_dc || is_for_te) {
is_local = false;
} else if (is_for_crm) {
if (pcmk__strcase_any_of(task, CRM_OP_NODE_INFO,
PCMK__CONTROLD_CMD_NODES, NULL)) {
/* Node info requests do not specify a host, which is normally
* treated as "all hosts", because the whole point is that the
* client may not know the local node name. Always handle these
* requests locally.
*/
is_local = true;
} else {
is_local = !originated_locally;
}
} else {
is_local = true;
}
} else if (pcmk__str_eq(fsa_our_uname, host_to, pcmk__str_casei)) {
is_local = true;
} else if (is_for_crm && pcmk__str_eq(task, CRM_OP_LRM_DELETE, pcmk__str_casei)) {
xmlNode *msg_data = get_message_xml(msg, F_CRM_DATA);
const char *mode = crm_element_value(msg_data, PCMK__XA_MODE);
if (pcmk__str_eq(mode, XML_TAG_CIB, pcmk__str_casei)) {
// Local delete of an offline node's resource history
is_local = true;
}
}
if (is_for_dc || is_for_dcib || is_for_te) {
if (AM_I_DC && is_for_te) {
crm_trace("Route message %s locally as transition request", ref);
send_msg_via_ipc(msg, sys_to);
} else if (AM_I_DC) {
crm_trace("Route message %s locally as DC request", ref);
return FALSE; // More to be done by caller
} else if (originated_locally && !pcmk__strcase_any_of(sys_from, CRM_SYSTEM_PENGINE,
CRM_SYSTEM_TENGINE, NULL)) {
#if SUPPORT_COROSYNC
if (is_corosync_cluster()) {
dest = text2msg_type(sys_to);
}
#endif
crm_trace("Relay message %s to DC", ref);
send_cluster_message(host_to ? crm_get_peer(0, host_to) : NULL, dest, msg, TRUE);
} else {
/* Neither the TE nor the scheduler should be sending messages
* to DCs on other nodes. By definition, if we are no longer the DC,
* then the scheduler's or TE's data should be discarded.
*/
crm_trace("Discard message %s because we are not DC", ref);
}
} else if (is_local && (is_for_crm || is_for_cib)) {
crm_trace("Route message %s locally as controller request", ref);
return FALSE; // More to be done by caller
} else if (is_local) {
crm_trace("Relay message %s locally to %s",
ref, (sys_to? sys_to : "unknown client"));
crm_log_xml_trace(msg, "[IPC relay]");
send_msg_via_ipc(msg, sys_to);
} else {
crm_node_t *node_to = NULL;
#if SUPPORT_COROSYNC
if (is_corosync_cluster()) {
dest = text2msg_type(sys_to);
if (dest == crm_msg_none || dest > crm_msg_stonith_ng) {
dest = crm_msg_crmd;
}
}
#endif
if (host_to) {
node_to = pcmk__search_cluster_node_cache(0, host_to);
if (node_to == NULL) {
crm_warn("Cannot route message %s: Unknown node %s",
ref, host_to);
return TRUE;
}
crm_trace("Relay message %s to %s",
ref, (node_to->uname? node_to->uname : "peer"));
} else {
crm_trace("Broadcast message %s to all peers", ref);
}
send_cluster_message(host_to ? node_to : NULL, dest, msg, TRUE);
}
return TRUE; // No further processing of message is needed
}
// Return true if field contains a positive integer
static bool
authorize_version(xmlNode *message_data, const char *field,
const char *client_name, const char *ref, const char *uuid)
{
const char *version = crm_element_value(message_data, field);
long long version_num;
if ((pcmk__scan_ll(version, &version_num, -1LL) != pcmk_rc_ok)
|| (version_num < 0LL)) {
crm_warn("Rejected IPC hello from %s: '%s' is not a valid protocol %s "
CRM_XS " ref=%s uuid=%s",
client_name, ((version == NULL)? "" : version),
field, (ref? ref : "none"), uuid);
return false;
}
return true;
}
/*!
* \internal
* \brief Check whether a client IPC message is acceptable
*
* If a given client IPC message is a hello, "authorize" it by ensuring it has
* valid information such as a protocol version, and return false indicating
* that nothing further needs to be done with the message. If the message is not
* a hello, just return true to indicate it needs further processing.
*
* \param[in] client_msg XML of IPC message
* \param[in,out] curr_client If IPC is not proxied, client that sent message
* \param[in] proxy_session If IPC is proxied, the session ID
*
* \return true if message needs further processing, false if it doesn't
*/
bool
controld_authorize_ipc_message(const xmlNode *client_msg, pcmk__client_t *curr_client,
const char *proxy_session)
{
xmlNode *message_data = NULL;
const char *client_name = NULL;
const char *op = crm_element_value(client_msg, F_CRM_TASK);
const char *ref = crm_element_value(client_msg, XML_ATTR_REFERENCE);
const char *uuid = (curr_client? curr_client->id : proxy_session);
if (uuid == NULL) {
crm_warn("IPC message from client rejected: No client identifier "
CRM_XS " ref=%s", (ref? ref : "none"));
goto rejected;
}
if (!pcmk__str_eq(CRM_OP_HELLO, op, pcmk__str_casei)) {
// Only hello messages need to be authorized
return true;
}
message_data = get_message_xml(client_msg, F_CRM_DATA);
client_name = crm_element_value(message_data, "client_name");
if (pcmk__str_empty(client_name)) {
crm_warn("IPC hello from client rejected: No client name",
CRM_XS " ref=%s uuid=%s", (ref? ref : "none"), uuid);
goto rejected;
}
if (!authorize_version(message_data, "major_version", client_name, ref,
uuid)) {
goto rejected;
}
if (!authorize_version(message_data, "minor_version", client_name, ref,
uuid)) {
goto rejected;
}
crm_trace("Validated IPC hello from client %s", client_name);
if (curr_client) {
curr_client->userdata = strdup(client_name);
}
mainloop_set_trigger(fsa_source);
return false;
rejected:
if (curr_client) {
qb_ipcs_disconnect(curr_client->ipcs);
}
return false;
}
static enum crmd_fsa_input
handle_message(xmlNode *msg, enum crmd_fsa_cause cause)
{
const char *type = NULL;
CRM_CHECK(msg != NULL, return I_NULL);
type = crm_element_value(msg, F_CRM_MSG_TYPE);
if (pcmk__str_eq(type, XML_ATTR_REQUEST, pcmk__str_none)) {
return handle_request(msg, cause);
} else if (pcmk__str_eq(type, XML_ATTR_RESPONSE, pcmk__str_none)) {
handle_response(msg);
return I_NULL;
}
crm_err("Unknown message type: %s", type);
return I_NULL;
}
static enum crmd_fsa_input
handle_failcount_op(xmlNode * stored_msg)
{
const char *rsc = NULL;
const char *uname = NULL;
const char *op = NULL;
char *interval_spec = NULL;
guint interval_ms = 0;
gboolean is_remote_node = FALSE;
xmlNode *xml_op = get_message_xml(stored_msg, F_CRM_DATA);
if (xml_op) {
xmlNode *xml_rsc = first_named_child(xml_op, XML_CIB_TAG_RESOURCE);
xmlNode *xml_attrs = first_named_child(xml_op, XML_TAG_ATTRS);
if (xml_rsc) {
rsc = ID(xml_rsc);
}
if (xml_attrs) {
op = crm_element_value(xml_attrs,
CRM_META "_" XML_RSC_ATTR_CLEAR_OP);
crm_element_value_ms(xml_attrs,
CRM_META "_" XML_RSC_ATTR_CLEAR_INTERVAL,
&interval_ms);
}
}
uname = crm_element_value(xml_op, XML_LRM_ATTR_TARGET);
if ((rsc == NULL) || (uname == NULL)) {
crm_log_xml_warn(stored_msg, "invalid failcount op");
return I_NULL;
}
if (crm_element_value(xml_op, XML_LRM_ATTR_ROUTER_NODE)) {
is_remote_node = TRUE;
}
if (interval_ms) {
interval_spec = crm_strdup_printf("%ums", interval_ms);
}
update_attrd_clear_failures(uname, rsc, op, interval_spec, is_remote_node);
free(interval_spec);
lrm_clear_last_failure(rsc, uname, op, interval_ms);
return I_NULL;
}
static enum crmd_fsa_input
handle_lrm_delete(xmlNode *stored_msg)
{
const char *mode = NULL;
xmlNode *msg_data = get_message_xml(stored_msg, F_CRM_DATA);
CRM_CHECK(msg_data != NULL, return I_NULL);
/* CRM_OP_LRM_DELETE has two distinct modes. The default behavior is to
* relay the operation to the affected node, which will unregister the
* resource from the local executor, clear the resource's history from the
* CIB, and do some bookkeeping in the controller.
*
* However, if the affected node is offline, the client will specify
* mode="cib" which means the controller receiving the operation should
* clear the resource's history from the CIB and nothing else. This is used
* to clear shutdown locks.
*/
mode = crm_element_value(msg_data, PCMK__XA_MODE);
if ((mode == NULL) || strcmp(mode, XML_TAG_CIB)) {
// Relay to affected node
crm_xml_add(stored_msg, F_CRM_SYS_TO, CRM_SYSTEM_LRMD);
return I_ROUTER;
} else {
// Delete CIB history locally (compare with do_lrm_delete())
const char *from_sys = NULL;
const char *user_name = NULL;
const char *rsc_id = NULL;
const char *node = NULL;
xmlNode *rsc_xml = NULL;
int rc = pcmk_rc_ok;
rsc_xml = first_named_child(msg_data, XML_CIB_TAG_RESOURCE);
CRM_CHECK(rsc_xml != NULL, return I_NULL);
rsc_id = ID(rsc_xml);
from_sys = crm_element_value(stored_msg, F_CRM_SYS_FROM);
node = crm_element_value(msg_data, XML_LRM_ATTR_TARGET);
user_name = pcmk__update_acl_user(stored_msg, F_CRM_USER, NULL);
crm_debug("Handling " CRM_OP_LRM_DELETE " for %s on %s locally%s%s "
"(clearing CIB resource history only)", rsc_id, node,
(user_name? " for user " : ""), (user_name? user_name : ""));
rc = controld_delete_resource_history(rsc_id, node, user_name,
cib_dryrun|cib_sync_call);
if (rc == pcmk_rc_ok) {
rc = controld_delete_resource_history(rsc_id, node, user_name,
crmd_cib_smart_opt());
}
//Notify client and tengine.(Only notify tengine if mode = "cib" and CRM_OP_LRM_DELETE.)
if (from_sys) {
lrmd_event_data_t *op = NULL;
const char *from_host = crm_element_value(stored_msg,
F_CRM_HOST_FROM);
const char *transition;
if (strcmp(from_sys, CRM_SYSTEM_TENGINE)) {
transition = crm_element_value(msg_data,
XML_ATTR_TRANSITION_KEY);
} else {
transition = crm_element_value(stored_msg,
XML_ATTR_TRANSITION_KEY);
}
crm_info("Notifying %s on %s that %s was%s deleted",
from_sys, (from_host? from_host : "local node"), rsc_id,
((rc == pcmk_rc_ok)? "" : " not"));
op = lrmd_new_event(rsc_id, CRMD_ACTION_DELETE, 0);
op->type = lrmd_event_exec_complete;
op->user_data = strdup(transition? transition : FAKE_TE_ID);
op->params = pcmk__strkey_table(free, free);
g_hash_table_insert(op->params, strdup(XML_ATTR_CRM_VERSION),
strdup(CRM_FEATURE_SET));
controld_rc2event(op, rc);
controld_ack_event_directly(from_host, from_sys, NULL, op, rsc_id);
lrmd_free_event(op);
controld_trigger_delete_refresh(from_sys, rsc_id);
}
return I_NULL;
}
}
/*!
* \brief Handle a CRM_OP_REMOTE_STATE message by updating remote peer cache
*
* \param[in] msg Message XML
*
* \return Next FSA input
*/
static enum crmd_fsa_input
handle_remote_state(const xmlNode *msg)
{
const char *remote_uname = ID(msg);
crm_node_t *remote_peer;
bool remote_is_up = false;
int rc = pcmk_rc_ok;
rc = pcmk__xe_get_bool_attr(msg, XML_NODE_IN_CLUSTER, &remote_is_up);
CRM_CHECK(remote_uname && rc == pcmk_rc_ok, return I_NULL);
remote_peer = crm_remote_peer_get(remote_uname);
CRM_CHECK(remote_peer, return I_NULL);
pcmk__update_peer_state(__func__, remote_peer,
remote_is_up ? CRM_NODE_MEMBER : CRM_NODE_LOST,
0);
return I_NULL;
}
/*!
* \brief Handle a CRM_OP_PING message
*
* \param[in] msg Message XML
*
* \return Next FSA input
*/
static enum crmd_fsa_input
handle_ping(const xmlNode *msg)
{
const char *value = NULL;
xmlNode *ping = NULL;
xmlNode *reply = NULL;
// Build reply
ping = create_xml_node(NULL, XML_CRM_TAG_PING);
value = crm_element_value(msg, F_CRM_SYS_TO);
crm_xml_add(ping, XML_PING_ATTR_SYSFROM, value);
// Add controller state
value = fsa_state2string(fsa_state);
crm_xml_add(ping, XML_PING_ATTR_CRMDSTATE, value);
crm_notice("Current ping state: %s", value); // CTS needs this
// Add controller health
// @TODO maybe do some checks to determine meaningful status
crm_xml_add(ping, XML_PING_ATTR_STATUS, "ok");
// Send reply
reply = create_reply(msg, ping);
free_xml(ping);
if (reply != NULL) {
(void) relay_message(reply, TRUE);
free_xml(reply);
}
// Nothing further to do
return I_NULL;
}
/*!
* \brief Handle a PCMK__CONTROLD_CMD_NODES message
*
* \param[in] request Message XML
*
* \return Next FSA input
*/
static enum crmd_fsa_input
handle_node_list(const xmlNode *request)
{
GHashTableIter iter;
crm_node_t *node = NULL;
xmlNode *reply = NULL;
xmlNode *reply_data = NULL;
// Create message data for reply
reply_data = create_xml_node(NULL, XML_CIB_TAG_NODES);
g_hash_table_iter_init(&iter, crm_peer_cache);
while (g_hash_table_iter_next(&iter, NULL, (gpointer *) & node)) {
xmlNode *xml = create_xml_node(reply_data, XML_CIB_TAG_NODE);
crm_xml_add_ll(xml, XML_ATTR_ID, (long long) node->id); // uint32_t
crm_xml_add(xml, XML_ATTR_UNAME, node->uname);
crm_xml_add(xml, XML_NODE_IN_CLUSTER, node->state);
}
// Create and send reply
reply = create_reply(request, reply_data);
free_xml(reply_data);
if (reply) {
(void) relay_message(reply, TRUE);
free_xml(reply);
}
// Nothing further to do
return I_NULL;
}
/*!
* \brief Handle a CRM_OP_NODE_INFO request
*
* \param[in] msg Message XML
*
* \return Next FSA input
*/
static enum crmd_fsa_input
handle_node_info_request(const xmlNode *msg)
{
const char *value = NULL;
crm_node_t *node = NULL;
int node_id = 0;
xmlNode *reply = NULL;
xmlNode *reply_data = NULL;
// Build reply
reply_data = create_xml_node(NULL, XML_CIB_TAG_NODE);
crm_xml_add(reply_data, XML_PING_ATTR_SYSFROM, CRM_SYSTEM_CRMD);
// Add whether current partition has quorum
pcmk__xe_set_bool_attr(reply_data, XML_ATTR_HAVE_QUORUM, fsa_has_quorum);
// Check whether client requested node info by ID and/or name
crm_element_value_int(msg, XML_ATTR_ID, &node_id);
if (node_id < 0) {
node_id = 0;
}
value = crm_element_value(msg, XML_ATTR_UNAME);
// Default to local node if none given
if ((node_id == 0) && (value == NULL)) {
value = fsa_our_uname;
}
node = pcmk__search_node_caches(node_id, value, CRM_GET_PEER_ANY);
if (node) {
crm_xml_add_int(reply_data, XML_ATTR_ID, node->id);
crm_xml_add(reply_data, XML_ATTR_UUID, node->uuid);
crm_xml_add(reply_data, XML_ATTR_UNAME, node->uname);
crm_xml_add(reply_data, XML_NODE_IS_PEER, node->state);
pcmk__xe_set_bool_attr(reply_data, XML_NODE_IS_REMOTE,
pcmk_is_set(node->flags, crm_remote_node));
}
// Send reply
reply = create_reply(msg, reply_data);
free_xml(reply_data);
if (reply != NULL) {
(void) relay_message(reply, TRUE);
free_xml(reply);
}
// Nothing further to do
return I_NULL;
}
static void
verify_feature_set(xmlNode *msg)
{
const char *dc_version = crm_element_value(msg, XML_ATTR_CRM_VERSION);
if (dc_version == NULL) {
/* All we really know is that the DC feature set is older than 3.1.0,
* but that's also all that really matters.
*/
dc_version = "3.0.14";
}
if (feature_set_compatible(dc_version, CRM_FEATURE_SET)) {
crm_trace("Local feature set (%s) is compatible with DC's (%s)",
CRM_FEATURE_SET, dc_version);
} else {
crm_err("Local feature set (%s) is incompatible with DC's (%s)",
CRM_FEATURE_SET, dc_version);
// Nothing is likely to improve without administrator involvement
controld_set_fsa_input_flags(R_STAYDOWN);
crmd_exit(CRM_EX_FATAL);
}
}
// DC gets own shutdown all-clear
static enum crmd_fsa_input
handle_shutdown_self_ack(xmlNode *stored_msg)
{
const char *host_from = crm_element_value(stored_msg, F_CRM_HOST_FROM);
if (pcmk_is_set(fsa_input_register, R_SHUTDOWN)) {
// The expected case -- we initiated own shutdown sequence
crm_info("Shutting down controller");
return I_STOP;
}
if (pcmk__str_eq(host_from, fsa_our_dc, pcmk__str_casei)) {
// Must be logic error -- DC confirming its own unrequested shutdown
crm_err("Shutting down controller immediately due to "
"unexpected shutdown confirmation");
return I_TERMINATE;
}
if (fsa_state != S_STOPPING) {
// Shouldn't happen -- non-DC confirming unrequested shutdown
crm_err("Starting new DC election because %s is "
"confirming shutdown we did not request",
(host_from? host_from : "another node"));
return I_ELECTION;
}
// Shouldn't happen, but we are already stopping anyway
crm_debug("Ignoring unexpected shutdown confirmation from %s",
(host_from? host_from : "another node"));
return I_NULL;
}
// Non-DC gets shutdown all-clear from DC
static enum crmd_fsa_input
handle_shutdown_ack(xmlNode *stored_msg)
{
const char *host_from = crm_element_value(stored_msg, F_CRM_HOST_FROM);
if (host_from == NULL) {
crm_warn("Ignoring shutdown request without origin specified");
return I_NULL;
}
if ((fsa_our_dc == NULL) || (strcmp(host_from, fsa_our_dc) == 0)) {
if (pcmk_is_set(fsa_input_register, R_SHUTDOWN)) {
crm_info("Shutting down controller after confirmation from %s",
host_from);
} else {
crm_err("Shutting down controller after unexpected "
"shutdown request from %s", host_from);
controld_set_fsa_input_flags(R_STAYDOWN);
}
return I_STOP;
}
crm_warn("Ignoring shutdown request from %s because DC is %s",
host_from, fsa_our_dc);
return I_NULL;
}
static enum crmd_fsa_input
handle_request(xmlNode *stored_msg, enum crmd_fsa_cause cause)
{
xmlNode *msg = NULL;
const char *op = crm_element_value(stored_msg, F_CRM_TASK);
/* Optimize this for the DC - it has the most to do */
if (op == NULL) {
crm_log_xml_warn(stored_msg, "[request without " F_CRM_TASK "]");
return I_NULL;
}
if (strcmp(op, CRM_OP_SHUTDOWN_REQ) == 0) {
const char *from = crm_element_value(stored_msg, F_CRM_HOST_FROM);
crm_node_t *node = pcmk__search_cluster_node_cache(0, from);
pcmk__update_peer_expected(__func__, node, CRMD_JOINSTATE_DOWN);
if(AM_I_DC == FALSE) {
return I_NULL; /* Done */
}
}
/*========== DC-Only Actions ==========*/
if (AM_I_DC) {
if (strcmp(op, CRM_OP_JOIN_ANNOUNCE) == 0) {
return I_NODE_JOIN;
} else if (strcmp(op, CRM_OP_JOIN_REQUEST) == 0) {
return I_JOIN_REQUEST;
} else if (strcmp(op, CRM_OP_JOIN_CONFIRM) == 0) {
return I_JOIN_RESULT;
} else if (strcmp(op, CRM_OP_SHUTDOWN) == 0) {
return handle_shutdown_self_ack(stored_msg);
} else if (strcmp(op, CRM_OP_SHUTDOWN_REQ) == 0) {
// Another controller wants to shut down its node
return handle_shutdown_request(stored_msg);
} else if (strcmp(op, CRM_OP_REMOTE_STATE) == 0) {
/* a remote connection host is letting us know the node state */
return handle_remote_state(stored_msg);
}
}
/*========== common actions ==========*/
if (strcmp(op, CRM_OP_NOVOTE) == 0) {
ha_msg_input_t fsa_input;
fsa_input.msg = stored_msg;
register_fsa_input_adv(C_HA_MESSAGE, I_NULL, &fsa_input,
A_ELECTION_COUNT | A_ELECTION_CHECK, FALSE,
__func__);
} else if (strcmp(op, CRM_OP_THROTTLE) == 0) {
throttle_update(stored_msg);
if (AM_I_DC && transition_graph != NULL) {
if (!transition_graph->complete) {
crm_debug("The throttle changed. Trigger a graph.");
trigger_graph();
}
}
return I_NULL;
} else if (strcmp(op, CRM_OP_CLEAR_FAILCOUNT) == 0) {
return handle_failcount_op(stored_msg);
} else if (strcmp(op, CRM_OP_VOTE) == 0) {
/* count the vote and decide what to do after that */
ha_msg_input_t fsa_input;
fsa_input.msg = stored_msg;
register_fsa_input_adv(C_HA_MESSAGE, I_NULL, &fsa_input,
A_ELECTION_COUNT | A_ELECTION_CHECK, FALSE,
__func__);
/* Sometimes we _must_ go into S_ELECTION */
if (fsa_state == S_HALT) {
crm_debug("Forcing an election from S_HALT");
return I_ELECTION;
#if 0
} else if (AM_I_DC) {
/* This is the old way of doing things but what is gained? */
return I_ELECTION;
#endif
}
} else if (strcmp(op, CRM_OP_JOIN_OFFER) == 0) {
verify_feature_set(stored_msg);
crm_debug("Raising I_JOIN_OFFER: join-%s", crm_element_value(stored_msg, F_CRM_JOIN_ID));
return I_JOIN_OFFER;
} else if (strcmp(op, CRM_OP_JOIN_ACKNAK) == 0) {
crm_debug("Raising I_JOIN_RESULT: join-%s", crm_element_value(stored_msg, F_CRM_JOIN_ID));
return I_JOIN_RESULT;
} else if (strcmp(op, CRM_OP_LRM_DELETE) == 0) {
return handle_lrm_delete(stored_msg);
} else if ((strcmp(op, CRM_OP_LRM_FAIL) == 0)
- || (strcmp(op, CRM_OP_LRM_REFRESH) == 0)
+ || (strcmp(op, CRM_OP_LRM_REFRESH) == 0) // @COMPAT
|| (strcmp(op, CRM_OP_REPROBE) == 0)) {
crm_xml_add(stored_msg, F_CRM_SYS_TO, CRM_SYSTEM_LRMD);
return I_ROUTER;
} else if (strcmp(op, CRM_OP_NOOP) == 0) {
return I_NULL;
} else if (strcmp(op, CRM_OP_LOCAL_SHUTDOWN) == 0) {
crm_shutdown(SIGTERM);
/*return I_SHUTDOWN; */
return I_NULL;
} else if (strcmp(op, CRM_OP_PING) == 0) {
return handle_ping(stored_msg);
} else if (strcmp(op, CRM_OP_NODE_INFO) == 0) {
return handle_node_info_request(stored_msg);
} else if (strcmp(op, CRM_OP_RM_NODE_CACHE) == 0) {
int id = 0;
const char *name = NULL;
crm_element_value_int(stored_msg, XML_ATTR_ID, &id);
name = crm_element_value(stored_msg, XML_ATTR_UNAME);
if(cause == C_IPC_MESSAGE) {
msg = create_request(CRM_OP_RM_NODE_CACHE, NULL, NULL, CRM_SYSTEM_CRMD, CRM_SYSTEM_CRMD, NULL);
if (send_cluster_message(NULL, crm_msg_crmd, msg, TRUE) == FALSE) {
crm_err("Could not instruct peers to remove references to node %s/%u", name, id);
} else {
crm_notice("Instructing peers to remove references to node %s/%u", name, id);
}
free_xml(msg);
} else {
reap_crm_member(id, name);
/* If we're forgetting this node, also forget any failures to fence
* it, so we don't carry that over to any node added later with the
* same name.
*/
st_fail_count_reset(name);
}
} else if (strcmp(op, CRM_OP_MAINTENANCE_NODES) == 0) {
xmlNode *xml = get_message_xml(stored_msg, F_CRM_DATA);
remote_ra_process_maintenance_nodes(xml);
} else if (strcmp(op, PCMK__CONTROLD_CMD_NODES) == 0) {
return handle_node_list(stored_msg);
/*========== (NOT_DC)-Only Actions ==========*/
} else if (!AM_I_DC) {
if (strcmp(op, CRM_OP_SHUTDOWN) == 0) {
return handle_shutdown_ack(stored_msg);
}
} else {
crm_err("Unexpected request (%s) sent to %s", op, AM_I_DC ? "the DC" : "non-DC node");
crm_log_xml_err(stored_msg, "Unexpected");
}
return I_NULL;
}
static void
handle_response(xmlNode *stored_msg)
{
const char *op = crm_element_value(stored_msg, F_CRM_TASK);
if (op == NULL) {
crm_log_xml_err(stored_msg, "Bad message");
} else if (AM_I_DC && strcmp(op, CRM_OP_PECALC) == 0) {
// Check whether scheduler answer been superseded by subsequent request
const char *msg_ref = crm_element_value(stored_msg, XML_ATTR_REFERENCE);
if (msg_ref == NULL) {
crm_err("%s - Ignoring calculation with no reference", op);
} else if (pcmk__str_eq(msg_ref, fsa_pe_ref, pcmk__str_casei)) {
ha_msg_input_t fsa_input;
controld_stop_sched_timer();
fsa_input.msg = stored_msg;
register_fsa_input_later(C_IPC_MESSAGE, I_PE_SUCCESS, &fsa_input);
} else {
crm_info("%s calculation %s is obsolete", op, msg_ref);
}
} else if (strcmp(op, CRM_OP_VOTE) == 0
|| strcmp(op, CRM_OP_SHUTDOWN_REQ) == 0 || strcmp(op, CRM_OP_SHUTDOWN) == 0) {
} else {
const char *host_from = crm_element_value(stored_msg, F_CRM_HOST_FROM);
crm_err("Unexpected response (op=%s, src=%s) sent to the %s",
op, host_from, AM_I_DC ? "DC" : "controller");
}
}
static enum crmd_fsa_input
handle_shutdown_request(xmlNode * stored_msg)
{
/* handle here to avoid potential version issues
* where the shutdown message/procedure may have
* been changed in later versions.
*
* This way the DC is always in control of the shutdown
*/
char *now_s = NULL;
const char *host_from = crm_element_value(stored_msg, F_CRM_HOST_FROM);
if (host_from == NULL) {
/* we're shutting down and the DC */
host_from = fsa_our_uname;
}
crm_info("Creating shutdown request for %s (state=%s)", host_from, fsa_state2string(fsa_state));
crm_log_xml_trace(stored_msg, "message");
now_s = pcmk__ttoa(time(NULL));
update_attrd(host_from, XML_CIB_ATTR_SHUTDOWN, now_s, NULL, FALSE);
free(now_s);
/* will be picked up by the TE as long as its running */
return I_NULL;
}
static void
send_msg_via_ipc(xmlNode * msg, const char *sys)
{
pcmk__client_t *client_channel = NULL;
CRM_CHECK(sys != NULL, return);
client_channel = pcmk__find_client_by_id(sys);
if (crm_element_value(msg, F_CRM_HOST_FROM) == NULL) {
crm_xml_add(msg, F_CRM_HOST_FROM, fsa_our_uname);
}
if (client_channel != NULL) {
/* Transient clients such as crmadmin */
pcmk__ipc_send_xml(client_channel, 0, msg, crm_ipc_server_event);
} else if (pcmk__str_eq(sys, CRM_SYSTEM_TENGINE, pcmk__str_none)) {
xmlNode *data = get_message_xml(msg, F_CRM_DATA);
process_te_message(msg, data);
} else if (pcmk__str_eq(sys, CRM_SYSTEM_LRMD, pcmk__str_none)) {
fsa_data_t fsa_data;
ha_msg_input_t fsa_input;
fsa_input.msg = msg;
fsa_input.xml = get_message_xml(msg, F_CRM_DATA);
fsa_data.id = 0;
fsa_data.actions = 0;
fsa_data.data = &fsa_input;
fsa_data.fsa_input = I_MESSAGE;
fsa_data.fsa_cause = C_IPC_MESSAGE;
fsa_data.origin = __func__;
fsa_data.data_type = fsa_dt_ha_msg;
do_lrm_invoke(A_LRM_INVOKE, C_IPC_MESSAGE, fsa_state, I_MESSAGE, &fsa_data);
} else if (crmd_is_proxy_session(sys)) {
crmd_proxy_send(sys, msg);
} else {
crm_info("Received invalid request: unknown subsystem '%s'", sys);
}
}
void
delete_ha_msg_input(ha_msg_input_t * orig)
{
if (orig == NULL) {
return;
}
free_xml(orig->msg);
free(orig);
}
/*!
* \internal
* \brief Notify the DC of a remote node state change
*
* \param[in] node_name Node's name
* \param[in] node_up TRUE if node is up, FALSE if down
*/
void
send_remote_state_message(const char *node_name, gboolean node_up)
{
/* If we don't have a DC, or the message fails, we have a failsafe:
* the DC will eventually pick up the change via the CIB node state.
* The message allows it to happen sooner if possible.
*/
if (fsa_our_dc) {
xmlNode *msg = create_request(CRM_OP_REMOTE_STATE, NULL, fsa_our_dc,
CRM_SYSTEM_DC, CRM_SYSTEM_CRMD, NULL);
crm_info("Notifying DC %s of Pacemaker Remote node %s %s",
fsa_our_dc, node_name, (node_up? "coming up" : "going down"));
crm_xml_add(msg, XML_ATTR_ID, node_name);
pcmk__xe_set_bool_attr(msg, XML_NODE_IN_CLUSTER, node_up);
send_cluster_message(crm_get_peer(0, fsa_our_dc), crm_msg_crmd, msg,
TRUE);
free_xml(msg);
} else {
crm_debug("No DC to notify of Pacemaker Remote node %s %s",
node_name, (node_up? "coming up" : "going down"));
}
}
diff --git a/daemons/controld/controld_metadata.c b/daemons/controld/controld_metadata.c
index ed558ec086..d7c8ec0b27 100644
--- a/daemons/controld/controld_metadata.c
+++ b/daemons/controld/controld_metadata.c
@@ -1,389 +1,398 @@
/*
* Copyright 2017-2022 the Pacemaker project contributors
*
* The version control history for this file may have further details.
*
* This source code is licensed under the GNU General Public License version 2
* or later (GPLv2+) WITHOUT ANY WARRANTY.
*/
#include <crm_internal.h>
#include <stdio.h>
#include <glib.h>
#include <regex.h>
#include <crm/crm.h>
#include <crm/lrmd.h>
#include <pacemaker-controld.h>
#if ENABLE_VERSIONED_ATTRS
static regex_t *version_format_regex = NULL;
#endif
static void
ra_param_free(void *param)
{
if (param) {
struct ra_param_s *p = (struct ra_param_s *) param;
if (p->rap_name) {
free(p->rap_name);
}
free(param);
}
}
static void
metadata_free(void *metadata)
{
if (metadata) {
struct ra_metadata_s *md = (struct ra_metadata_s *) metadata;
if (md->ra_version) {
free(md->ra_version);
}
g_list_free_full(md->ra_params, ra_param_free);
free(metadata);
}
}
GHashTable *
metadata_cache_new(void)
{
return pcmk__strkey_table(free, metadata_free);
}
void
metadata_cache_free(GHashTable *mdc)
{
if (mdc) {
crm_trace("Destroying metadata cache with %d members", g_hash_table_size(mdc));
g_hash_table_destroy(mdc);
}
}
void
metadata_cache_reset(GHashTable *mdc)
{
if (mdc) {
crm_trace("Resetting metadata cache with %d members",
g_hash_table_size(mdc));
g_hash_table_remove_all(mdc);
}
}
#if ENABLE_VERSIONED_ATTRS
static gboolean
valid_version_format(const char *version)
{
if (version == NULL) {
return FALSE;
}
if (version_format_regex == NULL) {
/* The OCF standard allows free-form versioning, but for our purposes of
* versioned resource and operation attributes, we constrain it to
* dot-separated numbers. Agents are still free to use other schemes,
* but we can't determine attributes based on them.
*/
const char *regex_string = "^[[:digit:]]+([.][[:digit:]]+)*$";
version_format_regex = calloc(1, sizeof(regex_t));
regcomp(version_format_regex, regex_string, REG_EXTENDED | REG_NOSUB);
/* If our regex doesn't compile, it's a bug on our side, so CRM_CHECK()
* will give us a core dump to catch it. Pretend the version is OK
* because we don't want our mistake to break versioned attributes
* (which should only ever happen in a development branch anyway).
*/
CRM_CHECK(version_format_regex != NULL, return TRUE);
}
return regexec(version_format_regex, version, 0, NULL, 0) == 0;
}
#endif
void
metadata_cache_fini(void)
{
#if ENABLE_VERSIONED_ATTRS
if (version_format_regex) {
regfree(version_format_regex);
free(version_format_regex);
version_format_regex = NULL;
}
#endif
}
#if ENABLE_VERSIONED_ATTRS
static char *
ra_version_from_xml(xmlNode *metadata_xml, const lrmd_rsc_info_t *rsc)
{
const char *version = crm_element_value(metadata_xml, XML_ATTR_VERSION);
if (version == NULL) {
crm_debug("Metadata for %s:%s:%s does not specify a version",
rsc->standard, rsc->provider, rsc->type);
version = PCMK_DEFAULT_AGENT_VERSION;
} else if (!valid_version_format(version)) {
crm_notice("%s:%s:%s metadata version has unrecognized format",
rsc->standard, rsc->provider, rsc->type);
version = PCMK_DEFAULT_AGENT_VERSION;
} else {
crm_debug("Metadata for %s:%s:%s has version %s",
rsc->standard, rsc->provider, rsc->type, version);
}
return strdup(version);
}
#endif
static struct ra_param_s *
ra_param_from_xml(xmlNode *param_xml)
{
const char *param_name = crm_element_value(param_xml, "name");
struct ra_param_s *p;
p = calloc(1, sizeof(struct ra_param_s));
if (p == NULL) {
- crm_crit("Could not allocate memory for resource metadata");
return NULL;
}
p->rap_name = strdup(param_name);
if (p->rap_name == NULL) {
- crm_crit("Could not allocate memory for resource metadata");
free(p);
return NULL;
}
if (pcmk__xe_attr_is_true(param_xml, "reloadable")) {
controld_set_ra_param_flags(p, ra_param_reloadable);
}
if (pcmk__xe_attr_is_true(param_xml, "unique")) {
controld_set_ra_param_flags(p, ra_param_unique);
}
if (pcmk__xe_attr_is_true(param_xml, "private")) {
controld_set_ra_param_flags(p, ra_param_private);
}
return p;
}
static void
log_ra_ocf_version(const char *ra_key, const char *ra_ocf_version)
{
if (pcmk__str_empty(ra_ocf_version)) {
crm_warn("%s does not advertise OCF version supported", ra_key);
} else if (compare_version(ra_ocf_version, "2") >= 0) {
crm_warn("%s supports OCF version %s (this Pacemaker version supports "
PCMK_OCF_VERSION " and might not work properly with agent)",
ra_key, ra_ocf_version);
} else if (compare_version(ra_ocf_version, PCMK_OCF_VERSION) > 0) {
crm_info("%s supports OCF version %s (this Pacemaker version supports "
PCMK_OCF_VERSION " and might not use all agent features)",
ra_key, ra_ocf_version);
} else {
crm_debug("%s supports OCF version %s", ra_key, ra_ocf_version);
}
}
struct ra_metadata_s *
-metadata_cache_update(GHashTable *mdc, const lrmd_rsc_info_t *rsc,
- const char *metadata_str)
+controld_cache_metadata(GHashTable *mdc, const lrmd_rsc_info_t *rsc,
+ const char *metadata_str)
{
char *key = NULL;
+ const char *reason = NULL;
xmlNode *metadata = NULL;
xmlNode *match = NULL;
struct ra_metadata_s *md = NULL;
bool any_private_params = false;
bool ocf1_1 = false;
CRM_CHECK(mdc && rsc && metadata_str, return NULL);
key = crm_generate_ra_key(rsc->standard, rsc->provider, rsc->type);
if (!key) {
- crm_crit("Invalid resource agent standard or type provided");
+ reason = "Invalid resource agent standard or type";
goto err;
}
metadata = string2xml(metadata_str);
if (!metadata) {
- crm_err("Metadata for %s:%s:%s is not valid XML",
- rsc->standard, rsc->provider, rsc->type);
+ reason = "Metadata is not valid XML";
goto err;
}
md = calloc(1, sizeof(struct ra_metadata_s));
if (md == NULL) {
- crm_crit("Could not allocate memory for resource metadata");
+ reason = "Could not allocate memory";
goto err;
}
#if ENABLE_VERSIONED_ATTRS
md->ra_version = ra_version_from_xml(metadata, rsc);
#endif
if (strcmp(rsc->standard, PCMK_RESOURCE_CLASS_OCF) == 0) {
xmlChar *content = NULL;
xmlNode *version_element = first_named_child(metadata, "version");
if (version_element != NULL) {
content = xmlNodeGetContent(version_element);
}
log_ra_ocf_version(key, (const char *) content);
if (content != NULL) {
ocf1_1 = (compare_version((const char *) content, "1.1") >= 0);
xmlFree(content);
}
}
// Check supported actions
match = first_named_child(metadata, "actions");
for (match = first_named_child(match, "action"); match != NULL;
match = crm_next_same_xml(match)) {
const char *action_name = crm_element_value(match, "name");
if (pcmk__str_eq(action_name, CRMD_ACTION_RELOAD_AGENT,
pcmk__str_none)) {
if (ocf1_1) {
controld_set_ra_flags(md, key, ra_supports_reload_agent);
} else {
crm_notice("reload-agent action will not be used with %s "
"because it does not support OCF 1.1 or later", key);
}
} else if (!ocf1_1 && pcmk__str_eq(action_name, CRMD_ACTION_RELOAD,
pcmk__str_casei)) {
controld_set_ra_flags(md, key, ra_supports_legacy_reload);
}
}
// Build a parameter list
match = first_named_child(metadata, "parameters");
for (match = first_named_child(match, "parameter"); match != NULL;
match = crm_next_same_xml(match)) {
const char *param_name = crm_element_value(match, "name");
if (param_name == NULL) {
crm_warn("Metadata for %s:%s:%s has parameter without a name",
rsc->standard, rsc->provider, rsc->type);
} else {
struct ra_param_s *p = ra_param_from_xml(match);
if (p == NULL) {
+ reason = "Could not allocate memory";
goto err;
}
if (pcmk_is_set(p->rap_flags, ra_param_private)) {
any_private_params = true;
}
md->ra_params = g_list_prepend(md->ra_params, p);
}
}
/* Newer resource agents support the "private" parameter attribute to
* indicate sensitive parameters. For backward compatibility with older
* agents, implicitly treat a few common names as private when the agent
* doesn't specify any explicitly.
*/
if (!any_private_params) {
for (GList *iter = md->ra_params; iter != NULL; iter = iter->next) {
struct ra_param_s *p = iter->data;
if (pcmk__str_any_of(p->rap_name, "password", "passwd", "user",
NULL)) {
controld_set_ra_param_flags(p, ra_param_private);
}
}
}
g_hash_table_replace(mdc, key, md);
free_xml(metadata);
return md;
err:
+ crm_warn("Unable to update metadata for %s (%s%s%s:%s): %s",
+ rsc->id, rsc->standard, ((rsc->provider == NULL)? "" : ":"),
+ pcmk__s(rsc->provider, ""), rsc->type, reason);
free(key);
free_xml(metadata);
metadata_free(md);
return NULL;
}
/*!
* \internal
* \brief Get meta-data for a resource
*
* \param[in,out] lrm_state Use meta-data cache from this executor connection
* \param[in] rsc Resource to get meta-data for
* \param[in] source Allowed meta-data sources (bitmask of
* enum controld_metadata_source_e values)
*
* \return Meta-data cache entry for given resource, or NULL if not available
*/
struct ra_metadata_s *
controld_get_rsc_metadata(lrm_state_t *lrm_state, const lrmd_rsc_info_t *rsc,
uint32_t source)
{
struct ra_metadata_s *metadata = NULL;
char *metadata_str = NULL;
char *key = NULL;
int rc = pcmk_ok;
CRM_CHECK((lrm_state != NULL) && (rsc != NULL), return NULL);
if (pcmk_is_set(source, controld_metadata_from_cache)) {
key = crm_generate_ra_key(rsc->standard, rsc->provider, rsc->type);
if (key != NULL) {
metadata = g_hash_table_lookup(lrm_state->metadata_cache, key);
free(key);
}
if (metadata != NULL) {
+ crm_debug("Retrieved metadata for %s (%s%s%s:%s) from cache",
+ rsc->id, rsc->standard,
+ ((rsc->provider == NULL)? "" : ":"),
+ ((rsc->provider == NULL)? "" : rsc->provider),
+ rsc->type);
return metadata;
}
}
if (!pcmk_is_set(source, controld_metadata_from_agent)) {
return NULL;
}
- /* For now, we always collect resource agent meta-data via a local,
- * synchronous, direct execution of the agent. This has multiple issues:
- * the executor should execute agents, not the controller; meta-data for
- * Pacemaker Remote nodes should be collected on those nodes, not
- * locally; and the meta-data call shouldn't eat into the timeout of the
- * real action being performed.
+ /* For most actions, metadata was cached asynchronously before action
+ * execution (via metadata_complete()).
*
- * These issues are planned to be addressed by having the scheduler
- * schedule a meta-data cache check at the beginning of each transition.
- * Once that is working, this block will only be a fallback in case the
- * initial collection fails.
+ * However if that failed, and for other actions, retrieve the metadata now
+ * via a local, synchronous, direct execution of the agent.
+ *
+ * This has multiple issues, which is why this is just a fallback: the
+ * executor should execute agents, not the controller; metadata for
+ * Pacemaker Remote nodes should be collected on those nodes, not locally;
+ * the metadata call shouldn't eat into the timeout of the real action being
+ * performed; and the synchronous call blocks the controller (which also
+ * means that if the metadata action tries to contact the controller,
+ * everything will hang until the timeout).
*/
+ crm_debug("Retrieving metadata for %s (%s%s%s:%s) synchronously",
+ rsc->id, rsc->standard,
+ ((rsc->provider == NULL)? "" : ":"),
+ ((rsc->provider == NULL)? "" : rsc->provider),
+ rsc->type);
rc = lrm_state_get_metadata(lrm_state, rsc->standard, rsc->provider,
rsc->type, &metadata_str, 0);
if (rc != pcmk_ok) {
crm_warn("Failed to get metadata for %s (%s%s%s:%s): %s",
rsc->id, rsc->standard,
((rsc->provider == NULL)? "" : ":"),
((rsc->provider == NULL)? "" : rsc->provider),
rsc->type, pcmk_strerror(rc));
return NULL;
}
- metadata = metadata_cache_update(lrm_state->metadata_cache, rsc,
- metadata_str);
+ metadata = controld_cache_metadata(lrm_state->metadata_cache, rsc,
+ metadata_str);
free(metadata_str);
- if (metadata == NULL) {
- crm_warn("Failed to update metadata for %s (%s%s%s:%s)",
- rsc->id, rsc->standard, ((rsc->provider == NULL)? "" : ":"),
- ((rsc->provider == NULL)? "" : rsc->provider), rsc->type);
- }
return metadata;
}
diff --git a/daemons/controld/controld_metadata.h b/daemons/controld/controld_metadata.h
index 33e4d06a0a..f1f9a382f2 100644
--- a/daemons/controld/controld_metadata.h
+++ b/daemons/controld/controld_metadata.h
@@ -1,98 +1,98 @@
/*
* Copyright 2017-2022 the Pacemaker project contributors
*
* The version control history for this file may have further details.
*
* This source code is licensed under the GNU General Public License version 2
* or later (GPLv2+) WITHOUT ANY WARRANTY.
*/
#ifndef CRMD_METADATA_H
#define CRMD_METADATA_H
#include <stdint.h> // uint32_t
#include <glib.h> // GList, GHashTable
#include "controld_lrm.h" // lrm_state_t, lrm_rsc_info_t
/*
* @COMPAT pre-OCF-1.1 resource agents
*
* Pacemaker previously used the "reload" action to reload agent parameters,
* but most agents used it to reload the service configuration. Pacemaker also
* misused the OCF 1.0 "unique" parameter attribute to indicate reloadability.
*
* OCF 1.1 created the "reload-agent" action and "reloadable" parameter
* attribute for the Pacemaker usage.
*
* Pacemaker now supports the OCF 1.1 usage. The old usage is now deprecated,
* but will be supported if the agent does not claim OCF 1.1 or later
* compliance and does not advertise the reload-agent action.
*/
enum ra_flags_e {
ra_supports_legacy_reload = (1 << 0),
ra_supports_reload_agent = (1 << 1),
};
enum ra_param_flags_e {
ra_param_unique = (1 << 0),
ra_param_private = (1 << 1),
ra_param_reloadable = (1 << 2),
};
// Allowed sources of resource agent meta-data when requesting it
enum controld_metadata_source_e {
controld_metadata_from_cache = (1 << 0),
controld_metadata_from_agent = (1 << 1),
};
struct ra_param_s {
char *rap_name;
uint32_t rap_flags; // bitmask of ra_param_flags_s
};
struct ra_metadata_s {
char *ra_version;
GList *ra_params; // ra_param_s
uint32_t ra_flags; // bitmask of ra_flags_e
};
#define controld_set_ra_flags(ra_md, ra_key, flags_to_set) do { \
(ra_md)->ra_flags = pcmk__set_flags_as(__func__, __LINE__, \
LOG_TRACE, "Resource agent", ra_key, \
(ra_md)->ra_flags, (flags_to_set), #flags_to_set); \
} while (0)
#define controld_set_ra_param_flags(ra_param, flags_to_set) do { \
(ra_param)->rap_flags = pcmk__set_flags_as(__func__, __LINE__, \
LOG_TRACE, "Resource agent parameter", (ra_param)->rap_name, \
(ra_param)->rap_flags, (flags_to_set), #flags_to_set); \
} while (0)
GHashTable *metadata_cache_new(void);
void metadata_cache_free(GHashTable *mdc);
void metadata_cache_reset(GHashTable *mdc);
void metadata_cache_fini(void);
-struct ra_metadata_s *metadata_cache_update(GHashTable *mdc,
- const lrmd_rsc_info_t *rsc,
- const char *metadata_str);
+struct ra_metadata_s *controld_cache_metadata(GHashTable *mdc,
+ const lrmd_rsc_info_t *rsc,
+ const char *metadata_str);
struct ra_metadata_s *controld_get_rsc_metadata(lrm_state_t *lrm_state,
const lrmd_rsc_info_t *rsc,
uint32_t source);
static inline const char *
ra_param_flag2text(enum ra_param_flags_e flag)
{
switch (flag) {
case ra_param_reloadable:
return "reloadable";
case ra_param_unique:
return "unique";
case ra_param_private:
return "private";
default:
return "unknown";
}
}
#endif
diff --git a/daemons/fenced/fenced_commands.c b/daemons/fenced/fenced_commands.c
index 251725fe4d..817a3411f8 100644
--- a/daemons/fenced/fenced_commands.c
+++ b/daemons/fenced/fenced_commands.c
@@ -1,3614 +1,3614 @@
/*
* Copyright 2009-2022 the Pacemaker project contributors
*
* The version control history for this file may have further details.
*
* This source code is licensed under the GNU General Public License version 2
* or later (GPLv2+) WITHOUT ANY WARRANTY.
*/
#include <crm_internal.h>
#include <sys/param.h>
#include <stdio.h>
#include <sys/types.h>
#include <sys/wait.h>
#include <sys/stat.h>
#include <unistd.h>
#include <sys/utsname.h>
#include <stdlib.h>
#include <errno.h>
#include <fcntl.h>
#include <ctype.h>
#include <crm/crm.h>
#include <crm/msg_xml.h>
#include <crm/common/ipc.h>
#include <crm/common/ipc_internal.h>
#include <crm/cluster/internal.h>
#include <crm/common/mainloop.h>
#include <crm/stonith-ng.h>
#include <crm/fencing/internal.h>
#include <crm/common/xml.h>
#include <pacemaker-fenced.h>
GHashTable *device_list = NULL;
GHashTable *topology = NULL;
static GList *cmd_list = NULL;
static GHashTable *fenced_handlers = NULL;
struct device_search_s {
/* target of fence action */
char *host;
/* requested fence action */
char *action;
/* timeout to use if a device is queried dynamically for possible targets */
int per_device_timeout;
/* number of registered fencing devices at time of request */
int replies_needed;
/* number of device replies received so far */
int replies_received;
/* whether the target is eligible to perform requested action (or off) */
bool allow_suicide;
/* private data to pass to search callback function */
void *user_data;
/* function to call when all replies have been received */
void (*callback) (GList * devices, void *user_data);
/* devices capable of performing requested action (or off if remapping) */
GList *capable;
/* Whether to perform searches that support the action */
uint32_t support_action_only;
};
static gboolean stonith_device_dispatch(gpointer user_data);
static void st_child_done(int pid, const pcmk__action_result_t *result,
void *user_data);
static void stonith_send_reply(xmlNode * reply, int call_options, const char *remote_peer,
pcmk__client_t *client);
static void search_devices_record_result(struct device_search_s *search, const char *device,
gboolean can_fence);
static int get_agent_metadata(const char *agent, xmlNode **metadata);
static void read_action_metadata(stonith_device_t *device);
static enum fenced_target_by unpack_level_kind(const xmlNode *level);
typedef struct async_command_s {
int id;
int pid;
int fd_stdout;
int options;
int default_timeout; /* seconds */
int timeout; /* seconds */
int start_delay; // seconds (-1 means disable static/random fencing delays)
int delay_id;
char *op;
char *origin;
char *client;
char *client_name;
char *remote_op_id;
char *target;
uint32_t target_nodeid;
char *action;
char *device;
GList *device_list;
GList *next_device_iter; // device_list entry for next device to execute
void *internal_user_data;
void (*done_cb) (int pid, const pcmk__action_result_t *result,
void *user_data);
guint timer_sigterm;
guint timer_sigkill;
/*! If the operation timed out, this is the last signal
* we sent to the process to get it to terminate */
int last_timeout_signo;
stonith_device_t *active_on;
stonith_device_t *activating_on;
} async_command_t;
static xmlNode *construct_async_reply(const async_command_t *cmd,
const pcmk__action_result_t *result);
static gboolean
is_action_required(const char *action, const stonith_device_t *device)
{
return device && device->automatic_unfencing && pcmk__str_eq(action, "on",
pcmk__str_casei);
}
static int
get_action_delay_max(const stonith_device_t *device, const char *action)
{
const char *value = NULL;
int delay_max = 0;
if (!pcmk__is_fencing_action(action)) {
return 0;
}
value = g_hash_table_lookup(device->params, PCMK_STONITH_DELAY_MAX);
if (value) {
delay_max = crm_parse_interval_spec(value) / 1000;
}
return delay_max;
}
static int
get_action_delay_base(const stonith_device_t *device, const char *action,
const char *target)
{
char *hash_value = NULL;
int delay_base = 0;
if (!pcmk__is_fencing_action(action)) {
return 0;
}
hash_value = g_hash_table_lookup(device->params, PCMK_STONITH_DELAY_BASE);
if (hash_value) {
char *value = strdup(hash_value);
char *valptr = value;
CRM_ASSERT(value != NULL);
if (target != NULL) {
for (char *val = strtok(value, "; \t"); val != NULL; val = strtok(NULL, "; \t")) {
char *mapval = strchr(val, ':');
if (mapval == NULL || mapval[1] == 0) {
crm_err("pcmk_delay_base: empty value in mapping", val);
continue;
}
if (mapval != val && strncasecmp(target, val, (size_t)(mapval - val)) == 0) {
value = mapval + 1;
crm_debug("pcmk_delay_base mapped to %s for %s",
value, target);
break;
}
}
}
if (strchr(value, ':') == 0) {
delay_base = crm_parse_interval_spec(value) / 1000;
}
free(valptr);
}
return delay_base;
}
/*!
* \internal
* \brief Override STONITH timeout with pcmk_*_timeout if available
*
* \param[in] device STONITH device to use
* \param[in] action STONITH action name
* \param[in] default_timeout Timeout to use if device does not have
* a pcmk_*_timeout parameter for action
*
* \return Value of pcmk_(action)_timeout if available, otherwise default_timeout
* \note For consistency, it would be nice if reboot/off/on timeouts could be
* set the same way as start/stop/monitor timeouts, i.e. with an
* <operation> entry in the fencing resource configuration. However that
* is insufficient because fencing devices may be registered directly via
* the fencer's register_device() API instead of going through the CIB
* (e.g. stonith_admin uses it for its -R option, and the executor uses it
* to ensure a device is registered when a command is issued). As device
* properties, pcmk_*_timeout parameters can be grabbed by the fencer when
* the device is registered, whether by CIB change or API call.
*/
static int
get_action_timeout(const stonith_device_t *device, const char *action,
int default_timeout)
{
if (action && device && device->params) {
char buffer[64] = { 0, };
const char *value = NULL;
/* If "reboot" was requested but the device does not support it,
* we will remap to "off", so check timeout for "off" instead
*/
if (pcmk__str_eq(action, "reboot", pcmk__str_casei)
&& !pcmk_is_set(device->flags, st_device_supports_reboot)) {
crm_trace("%s doesn't support reboot, using timeout for off instead",
device->id);
action = "off";
}
/* If the device config specified an action-specific timeout, use it */
snprintf(buffer, sizeof(buffer), "pcmk_%s_timeout", action);
value = g_hash_table_lookup(device->params, buffer);
if (value) {
return atoi(value);
}
}
return default_timeout;
}
/*!
* \internal
* \brief Get the currently executing device for a fencing operation
*
* \param[in] cmd Fencing operation to check
*
* \return Currently executing device for \p cmd if any, otherwise NULL
*/
static stonith_device_t *
cmd_device(const async_command_t *cmd)
{
if ((cmd == NULL) || (cmd->device == NULL) || (device_list == NULL)) {
return NULL;
}
return g_hash_table_lookup(device_list, cmd->device);
}
static void
free_async_command(async_command_t * cmd)
{
if (!cmd) {
return;
}
if (cmd->delay_id) {
g_source_remove(cmd->delay_id);
}
cmd_list = g_list_remove(cmd_list, cmd);
g_list_free_full(cmd->device_list, free);
free(cmd->device);
free(cmd->action);
free(cmd->target);
free(cmd->remote_op_id);
free(cmd->client);
free(cmd->client_name);
free(cmd->origin);
free(cmd->op);
free(cmd);
}
/*!
* \internal
* \brief Create a new asynchronous fencing operation from request XML
*
* \param[in] msg Fencing request XML (from IPC or CPG)
*
* \return Newly allocated fencing operation on success, otherwise NULL
*
* \note This asserts on memory errors, so a NULL return indicates an
* unparseable message.
*/
static async_command_t *
create_async_command(xmlNode *msg)
{
xmlNode *op = NULL;
async_command_t *cmd = NULL;
if (msg == NULL) {
return NULL;
}
op = get_xpath_object("//@" F_STONITH_ACTION, msg, LOG_ERR);
if (op == NULL) {
return NULL;
}
cmd = calloc(1, sizeof(async_command_t));
CRM_ASSERT(cmd != NULL);
// All messages must include these
cmd->action = crm_element_value_copy(op, F_STONITH_ACTION);
cmd->op = crm_element_value_copy(msg, F_STONITH_OPERATION);
cmd->client = crm_element_value_copy(msg, F_STONITH_CLIENTID);
if ((cmd->action == NULL) || (cmd->op == NULL) || (cmd->client == NULL)) {
free_async_command(cmd);
return NULL;
}
crm_element_value_int(msg, F_STONITH_CALLID, &(cmd->id));
crm_element_value_int(msg, F_STONITH_CALLOPTS, &(cmd->options));
crm_element_value_int(msg, F_STONITH_DELAY, &(cmd->start_delay));
crm_element_value_int(msg, F_STONITH_TIMEOUT, &(cmd->default_timeout));
cmd->timeout = cmd->default_timeout;
cmd->origin = crm_element_value_copy(msg, F_ORIG);
cmd->remote_op_id = crm_element_value_copy(msg, F_STONITH_REMOTE_OP_ID);
cmd->client_name = crm_element_value_copy(msg, F_STONITH_CLIENTNAME);
cmd->target = crm_element_value_copy(op, F_STONITH_TARGET);
cmd->device = crm_element_value_copy(op, F_STONITH_DEVICE);
cmd->done_cb = st_child_done;
// Track in global command list
cmd_list = g_list_append(cmd_list, cmd);
return cmd;
}
static int
get_action_limit(stonith_device_t * device)
{
const char *value = NULL;
int action_limit = 1;
value = g_hash_table_lookup(device->params, PCMK_STONITH_ACTION_LIMIT);
if ((value == NULL)
|| (pcmk__scan_min_int(value, &action_limit, INT_MIN) != pcmk_rc_ok)
|| (action_limit == 0)) {
action_limit = 1;
}
return action_limit;
}
static int
get_active_cmds(stonith_device_t * device)
{
int counter = 0;
GList *gIter = NULL;
GList *gIterNext = NULL;
CRM_CHECK(device != NULL, return 0);
for (gIter = cmd_list; gIter != NULL; gIter = gIterNext) {
async_command_t *cmd = gIter->data;
gIterNext = gIter->next;
if (cmd->active_on == device) {
counter++;
}
}
return counter;
}
static void
fork_cb(int pid, void *user_data)
{
async_command_t *cmd = (async_command_t *) user_data;
stonith_device_t * device =
/* in case of a retry we've done the move from
activating_on to active_on already
*/
cmd->activating_on?cmd->activating_on:cmd->active_on;
CRM_ASSERT(device);
crm_debug("Operation '%s' [%d]%s%s using %s now running with %ds timeout",
cmd->action, pid,
((cmd->target == NULL)? "" : " targeting "),
pcmk__s(cmd->target, ""), device->id, cmd->timeout);
cmd->active_on = device;
cmd->activating_on = NULL;
}
static int
get_agent_metadata_cb(gpointer data) {
stonith_device_t *device = data;
guint period_ms;
switch (get_agent_metadata(device->agent, &device->agent_metadata)) {
case pcmk_rc_ok:
if (device->agent_metadata) {
read_action_metadata(device);
stonith__device_parameter_flags(&(device->flags), device->id,
device->agent_metadata);
}
return G_SOURCE_REMOVE;
case EAGAIN:
period_ms = pcmk__mainloop_timer_get_period(device->timer);
if (period_ms < 160 * 1000) {
mainloop_timer_set_period(device->timer, 2 * period_ms);
}
return G_SOURCE_CONTINUE;
default:
return G_SOURCE_REMOVE;
}
}
/*!
* \internal
* \brief Call a command's action callback for an internal (not library) result
*
* \param[in,out] cmd Command to report result for
* \param[in] execution_status Execution status to use for result
* \param[in] exit_status Exit status to use for result
* \param[in] exit_reason Exit reason to use for result
*/
static void
report_internal_result(async_command_t *cmd, int exit_status,
int execution_status, const char *exit_reason)
{
pcmk__action_result_t result = PCMK__UNKNOWN_RESULT;
pcmk__set_result(&result, exit_status, execution_status, exit_reason);
cmd->done_cb(0, &result, cmd);
pcmk__reset_result(&result);
}
static gboolean
stonith_device_execute(stonith_device_t * device)
{
int exec_rc = 0;
const char *action_str = NULL;
const char *host_arg = NULL;
async_command_t *cmd = NULL;
stonith_action_t *action = NULL;
int active_cmds = 0;
int action_limit = 0;
GList *gIter = NULL;
GList *gIterNext = NULL;
CRM_CHECK(device != NULL, return FALSE);
active_cmds = get_active_cmds(device);
action_limit = get_action_limit(device);
if (action_limit > -1 && active_cmds >= action_limit) {
crm_trace("%s is over its action limit of %d (%u active action%s)",
device->id, action_limit, active_cmds,
pcmk__plural_s(active_cmds));
return TRUE;
}
for (gIter = device->pending_ops; gIter != NULL; gIter = gIterNext) {
async_command_t *pending_op = gIter->data;
gIterNext = gIter->next;
if (pending_op && pending_op->delay_id) {
crm_trace("Operation '%s'%s%s using %s was asked to run too early, "
"waiting for start delay of %ds",
pending_op->action,
((pending_op->target == NULL)? "" : " targeting "),
pcmk__s(pending_op->target, ""),
device->id, pending_op->start_delay);
continue;
}
device->pending_ops = g_list_remove_link(device->pending_ops, gIter);
g_list_free_1(gIter);
cmd = pending_op;
break;
}
if (cmd == NULL) {
crm_trace("No actions using %s are needed", device->id);
return TRUE;
}
if (pcmk__str_any_of(device->agent, STONITH_WATCHDOG_AGENT,
STONITH_WATCHDOG_AGENT_INTERNAL, NULL)) {
if (pcmk__is_fencing_action(cmd->action)) {
if (node_does_watchdog_fencing(stonith_our_uname)) {
pcmk__panic(__func__);
goto done;
}
} else {
crm_info("Faking success for %s watchdog operation", cmd->action);
report_internal_result(cmd, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
goto done;
}
}
#if SUPPORT_CIBSECRETS
exec_rc = pcmk__substitute_secrets(device->id, device->params);
if (exec_rc != pcmk_rc_ok) {
if (pcmk__str_eq(cmd->action, "stop", pcmk__str_casei)) {
crm_info("Proceeding with stop operation for %s "
"despite being unable to load CIB secrets (%s)",
device->id, pcmk_rc_str(exec_rc));
} else {
crm_err("Considering %s unconfigured "
"because unable to load CIB secrets: %s",
device->id, pcmk_rc_str(exec_rc));
report_internal_result(cmd, CRM_EX_ERROR, PCMK_EXEC_NO_SECRETS,
"Failed to get CIB secrets");
goto done;
}
}
#endif
action_str = cmd->action;
if (pcmk__str_eq(cmd->action, "reboot", pcmk__str_casei)
&& !pcmk_is_set(device->flags, st_device_supports_reboot)) {
crm_notice("Remapping 'reboot' action%s%s using %s to 'off' "
"because agent '%s' does not support reboot",
((cmd->target == NULL)? "" : " targeting "),
pcmk__s(cmd->target, ""), device->id, device->agent);
action_str = "off";
}
if (pcmk_is_set(device->flags, st_device_supports_parameter_port)) {
host_arg = "port";
} else if (pcmk_is_set(device->flags, st_device_supports_parameter_plug)) {
host_arg = "plug";
}
action = stonith__action_create(device->agent, action_str, cmd->target,
cmd->target_nodeid, cmd->timeout,
device->params, device->aliases, host_arg);
/* for async exec, exec_rc is negative for early error exit
otherwise handling of success/errors is done via callbacks */
cmd->activating_on = device;
- exec_rc = stonith_action_execute_async(action, (void *)cmd,
- cmd->done_cb, fork_cb);
+ exec_rc = stonith__execute_async(action, (void *)cmd, cmd->done_cb,
+ fork_cb);
if (exec_rc < 0) {
cmd->activating_on = NULL;
cmd->done_cb(0, stonith__action_result(action), cmd);
stonith__destroy_action(action);
}
done:
/* Device might get triggered to work by multiple fencing commands
* simultaneously. Trigger the device again to make sure any
* remaining concurrent commands get executed. */
if (device->pending_ops) {
mainloop_set_trigger(device->work);
}
return TRUE;
}
static gboolean
stonith_device_dispatch(gpointer user_data)
{
return stonith_device_execute(user_data);
}
static gboolean
start_delay_helper(gpointer data)
{
async_command_t *cmd = data;
stonith_device_t *device = cmd_device(cmd);
cmd->delay_id = 0;
if (device) {
mainloop_set_trigger(device->work);
}
return FALSE;
}
static void
schedule_stonith_command(async_command_t * cmd, stonith_device_t * device)
{
int delay_max = 0;
int delay_base = 0;
int requested_delay = cmd->start_delay;
CRM_CHECK(cmd != NULL, return);
CRM_CHECK(device != NULL, return);
if (cmd->device) {
free(cmd->device);
}
if (device->include_nodeid && (cmd->target != NULL)) {
crm_node_t *node = crm_get_peer(0, cmd->target);
cmd->target_nodeid = node->id;
}
cmd->device = strdup(device->id);
cmd->timeout = get_action_timeout(device, cmd->action, cmd->default_timeout);
if (cmd->remote_op_id) {
crm_debug("Scheduling '%s' action%s%s using %s for remote peer %s "
"with op id %.8s and timeout %ds",
cmd->action,
(cmd->target == NULL)? "" : " targeting ",
pcmk__s(cmd->target, ""),
device->id, cmd->origin, cmd->remote_op_id, cmd->timeout);
} else {
crm_debug("Scheduling '%s' action%s%s using %s for %s with timeout %ds",
cmd->action,
(cmd->target == NULL)? "" : " targeting ",
pcmk__s(cmd->target, ""),
device->id, cmd->client, cmd->timeout);
}
device->pending_ops = g_list_append(device->pending_ops, cmd);
mainloop_set_trigger(device->work);
// Value -1 means disable any static/random fencing delays
if (requested_delay < 0) {
return;
}
delay_max = get_action_delay_max(device, cmd->action);
delay_base = get_action_delay_base(device, cmd->action, cmd->target);
if (delay_max == 0) {
delay_max = delay_base;
}
if (delay_max < delay_base) {
crm_warn(PCMK_STONITH_DELAY_BASE " (%ds) is larger than "
PCMK_STONITH_DELAY_MAX " (%ds) for %s using %s "
"(limiting to maximum delay)",
delay_base, delay_max, cmd->action, device->id);
delay_base = delay_max;
}
if (delay_max > 0) {
// coverity[dont_call] We're not using rand() for security
cmd->start_delay +=
((delay_max != delay_base)?(rand() % (delay_max - delay_base)):0)
+ delay_base;
}
if (cmd->start_delay > 0) {
crm_notice("Delaying '%s' action%s%s using %s for %ds " CRM_XS
" timeout=%ds requested_delay=%ds base=%ds max=%ds",
cmd->action,
(cmd->target == NULL)? "" : " targeting ",
pcmk__s(cmd->target, ""),
device->id, cmd->start_delay, cmd->timeout,
requested_delay, delay_base, delay_max);
cmd->delay_id =
g_timeout_add_seconds(cmd->start_delay, start_delay_helper, cmd);
}
}
static void
free_device(gpointer data)
{
GList *gIter = NULL;
stonith_device_t *device = data;
g_hash_table_destroy(device->params);
g_hash_table_destroy(device->aliases);
for (gIter = device->pending_ops; gIter != NULL; gIter = gIter->next) {
async_command_t *cmd = gIter->data;
crm_warn("Removal of device '%s' purged operation '%s'", device->id, cmd->action);
report_internal_result(cmd, CRM_EX_ERROR, PCMK_EXEC_NO_FENCE_DEVICE,
"Device was removed before action could be executed");
}
g_list_free(device->pending_ops);
g_list_free_full(device->targets, free);
if (device->timer) {
mainloop_timer_stop(device->timer);
mainloop_timer_del(device->timer);
}
mainloop_destroy_trigger(device->work);
free_xml(device->agent_metadata);
free(device->namespace);
free(device->on_target_actions);
free(device->agent);
free(device->id);
free(device);
}
void free_device_list(void)
{
if (device_list != NULL) {
g_hash_table_destroy(device_list);
device_list = NULL;
}
}
void
init_device_list(void)
{
if (device_list == NULL) {
device_list = pcmk__strkey_table(NULL, free_device);
}
}
static GHashTable *
build_port_aliases(const char *hostmap, GList ** targets)
{
char *name = NULL;
int last = 0, lpc = 0, max = 0, added = 0;
GHashTable *aliases = pcmk__strikey_table(free, free);
if (hostmap == NULL) {
return aliases;
}
max = strlen(hostmap);
for (; lpc <= max; lpc++) {
switch (hostmap[lpc]) {
/* Skip escaped chars */
case '\\':
lpc++;
break;
/* Assignment chars */
case '=':
case ':':
if (lpc > last) {
free(name);
name = calloc(1, 1 + lpc - last);
memcpy(name, hostmap + last, lpc - last);
}
last = lpc + 1;
break;
/* Delimeter chars */
/* case ',': Potentially used to specify multiple ports */
case 0:
case ';':
case ' ':
case '\t':
if (name) {
char *value = NULL;
int k = 0;
value = calloc(1, 1 + lpc - last);
memcpy(value, hostmap + last, lpc - last);
for (int i = 0; value[i] != '\0'; i++) {
if (value[i] != '\\') {
value[k++] = value[i];
}
}
value[k] = '\0';
crm_debug("Adding alias '%s'='%s'", name, value);
g_hash_table_replace(aliases, name, value);
if (targets) {
*targets = g_list_append(*targets, strdup(value));
}
value = NULL;
name = NULL;
added++;
} else if (lpc > last) {
crm_debug("Parse error at offset %d near '%s'", lpc - last, hostmap + last);
}
last = lpc + 1;
break;
}
if (hostmap[lpc] == 0) {
break;
}
}
if (added == 0) {
crm_info("No host mappings detected in '%s'", hostmap);
}
free(name);
return aliases;
}
GHashTable *metadata_cache = NULL;
void
free_metadata_cache(void) {
if (metadata_cache != NULL) {
g_hash_table_destroy(metadata_cache);
metadata_cache = NULL;
}
}
static void
init_metadata_cache(void) {
if (metadata_cache == NULL) {
metadata_cache = pcmk__strkey_table(free, free);
}
}
int
get_agent_metadata(const char *agent, xmlNode ** metadata)
{
char *buffer = NULL;
if (metadata == NULL) {
return EINVAL;
}
*metadata = NULL;
if (pcmk__str_eq(agent, STONITH_WATCHDOG_AGENT_INTERNAL, pcmk__str_none)) {
return pcmk_rc_ok;
}
init_metadata_cache();
buffer = g_hash_table_lookup(metadata_cache, agent);
if (buffer == NULL) {
stonith_t *st = stonith_api_new();
int rc;
if (st == NULL) {
crm_warn("Could not get agent meta-data: "
"API memory allocation failed");
return EAGAIN;
}
rc = st->cmds->metadata(st, st_opt_sync_call, agent,
NULL, &buffer, 10);
stonith_api_delete(st);
if (rc || !buffer) {
crm_err("Could not retrieve metadata for fencing agent %s", agent);
return EAGAIN;
}
g_hash_table_replace(metadata_cache, strdup(agent), buffer);
}
*metadata = string2xml(buffer);
return pcmk_rc_ok;
}
static gboolean
is_nodeid_required(xmlNode * xml)
{
xmlXPathObjectPtr xpath = NULL;
if (stand_alone) {
return FALSE;
}
if (!xml) {
return FALSE;
}
xpath = xpath_search(xml, "//parameter[@name='nodeid']");
if (numXpathResults(xpath) <= 0) {
freeXpathObject(xpath);
return FALSE;
}
freeXpathObject(xpath);
return TRUE;
}
#define MAX_ACTION_LEN 256
static char *
add_action(char *actions, const char *action)
{
int offset = 0;
if (actions == NULL) {
actions = calloc(1, MAX_ACTION_LEN);
} else {
offset = strlen(actions);
}
if (offset > 0) {
offset += snprintf(actions+offset, MAX_ACTION_LEN - offset, " ");
}
offset += snprintf(actions+offset, MAX_ACTION_LEN - offset, "%s", action);
return actions;
}
static void
read_action_metadata(stonith_device_t *device)
{
xmlXPathObjectPtr xpath = NULL;
int max = 0;
int lpc = 0;
if (device->agent_metadata == NULL) {
return;
}
xpath = xpath_search(device->agent_metadata, "//action");
max = numXpathResults(xpath);
if (max <= 0) {
freeXpathObject(xpath);
return;
}
for (lpc = 0; lpc < max; lpc++) {
const char *action = NULL;
xmlNode *match = getXpathResult(xpath, lpc);
CRM_LOG_ASSERT(match != NULL);
if(match == NULL) { continue; };
action = crm_element_value(match, "name");
if(pcmk__str_eq(action, "list", pcmk__str_casei)) {
stonith__set_device_flags(device->flags, device->id,
st_device_supports_list);
} else if(pcmk__str_eq(action, "status", pcmk__str_casei)) {
stonith__set_device_flags(device->flags, device->id,
st_device_supports_status);
} else if(pcmk__str_eq(action, "reboot", pcmk__str_casei)) {
stonith__set_device_flags(device->flags, device->id,
st_device_supports_reboot);
} else if (pcmk__str_eq(action, "on", pcmk__str_casei)) {
/* "automatic" means the cluster will unfence node when it joins */
/* "required" is a deprecated synonym for "automatic" */
if (pcmk__xe_attr_is_true(match, "automatic") || pcmk__xe_attr_is_true(match, "required")) {
device->automatic_unfencing = TRUE;
}
stonith__set_device_flags(device->flags, device->id,
st_device_supports_on);
}
if (action && pcmk__xe_attr_is_true(match, "on_target")) {
device->on_target_actions = add_action(device->on_target_actions, action);
}
}
freeXpathObject(xpath);
}
/*!
* \internal
* \brief Set a pcmk_*_action parameter if not already set
*
* \param[in,out] params Device parameters
* \param[in] action Name of action
* \param[in] value Value to use if action is not already set
*/
static void
map_action(GHashTable *params, const char *action, const char *value)
{
char *key = crm_strdup_printf("pcmk_%s_action", action);
if (g_hash_table_lookup(params, key)) {
crm_warn("Ignoring %s='%s', see %s instead",
STONITH_ATTR_ACTION_OP, value, key);
free(key);
} else {
crm_warn("Mapping %s='%s' to %s='%s'",
STONITH_ATTR_ACTION_OP, value, key, value);
g_hash_table_insert(params, key, strdup(value));
}
}
/*!
* \internal
* \brief Create device parameter table from XML
*
* \param[in] name Device name (used for logging only)
* \param[in] dev XML containing device parameters
*/
static GHashTable *
xml2device_params(const char *name, const xmlNode *dev)
{
GHashTable *params = xml2list(dev);
const char *value;
/* Action should never be specified in the device configuration,
* but we support it for users who are familiar with other software
* that worked that way.
*/
value = g_hash_table_lookup(params, STONITH_ATTR_ACTION_OP);
if (value != NULL) {
crm_warn("%s has '%s' parameter, which should never be specified in configuration",
name, STONITH_ATTR_ACTION_OP);
if (*value == '\0') {
crm_warn("Ignoring empty '%s' parameter", STONITH_ATTR_ACTION_OP);
} else if (strcmp(value, "reboot") == 0) {
crm_warn("Ignoring %s='reboot' (see stonith-action cluster property instead)",
STONITH_ATTR_ACTION_OP);
} else if (strcmp(value, "off") == 0) {
map_action(params, "reboot", value);
} else {
map_action(params, "off", value);
map_action(params, "reboot", value);
}
g_hash_table_remove(params, STONITH_ATTR_ACTION_OP);
}
return params;
}
static const char *
target_list_type(stonith_device_t * dev)
{
const char *check_type = NULL;
check_type = g_hash_table_lookup(dev->params, PCMK_STONITH_HOST_CHECK);
if (check_type == NULL) {
if (g_hash_table_lookup(dev->params, PCMK_STONITH_HOST_LIST)) {
check_type = "static-list";
} else if (g_hash_table_lookup(dev->params, PCMK_STONITH_HOST_MAP)) {
check_type = "static-list";
} else if (pcmk_is_set(dev->flags, st_device_supports_list)) {
check_type = "dynamic-list";
} else if (pcmk_is_set(dev->flags, st_device_supports_status)) {
check_type = "status";
} else {
check_type = PCMK__VALUE_NONE;
}
}
return check_type;
}
static stonith_device_t *
build_device_from_xml(xmlNode *dev)
{
const char *value;
stonith_device_t *device = NULL;
char *agent = crm_element_value_copy(dev, "agent");
CRM_CHECK(agent != NULL, return device);
device = calloc(1, sizeof(stonith_device_t));
CRM_CHECK(device != NULL, {free(agent); return device;});
device->id = crm_element_value_copy(dev, XML_ATTR_ID);
device->agent = agent;
device->namespace = crm_element_value_copy(dev, "namespace");
device->params = xml2device_params(device->id, dev);
value = g_hash_table_lookup(device->params, PCMK_STONITH_HOST_LIST);
if (value) {
device->targets = stonith__parse_targets(value);
}
value = g_hash_table_lookup(device->params, PCMK_STONITH_HOST_MAP);
device->aliases = build_port_aliases(value, &(device->targets));
value = target_list_type(device);
if (!pcmk__str_eq(value, "static-list", pcmk__str_casei) && device->targets) {
/* Other than "static-list", dev-> targets is unnecessary. */
g_list_free_full(device->targets, free);
device->targets = NULL;
}
switch (get_agent_metadata(device->agent, &device->agent_metadata)) {
case pcmk_rc_ok:
if (device->agent_metadata) {
read_action_metadata(device);
stonith__device_parameter_flags(&(device->flags), device->id,
device->agent_metadata);
}
break;
case EAGAIN:
if (device->timer == NULL) {
device->timer = mainloop_timer_add("get_agent_metadata", 10 * 1000,
TRUE, get_agent_metadata_cb, device);
}
if (!mainloop_timer_running(device->timer)) {
mainloop_timer_start(device->timer);
}
break;
default:
break;
}
value = g_hash_table_lookup(device->params, "nodeid");
if (!value) {
device->include_nodeid = is_nodeid_required(device->agent_metadata);
}
value = crm_element_value(dev, "rsc_provides");
if (pcmk__str_eq(value, PCMK__VALUE_UNFENCING, pcmk__str_casei)) {
device->automatic_unfencing = TRUE;
}
if (is_action_required("on", device)) {
crm_info("Fencing device '%s' requires unfencing", device->id);
}
if (device->on_target_actions) {
crm_info("Fencing device '%s' requires actions (%s) to be executed "
"on target", device->id, device->on_target_actions);
}
device->work = mainloop_add_trigger(G_PRIORITY_HIGH, stonith_device_dispatch, device);
/* TODO: Hook up priority */
return device;
}
static void
schedule_internal_command(const char *origin,
stonith_device_t * device,
const char *action,
const char *target,
int timeout,
void *internal_user_data,
void (*done_cb) (int pid,
const pcmk__action_result_t *result,
void *user_data))
{
async_command_t *cmd = NULL;
cmd = calloc(1, sizeof(async_command_t));
cmd->id = -1;
cmd->default_timeout = timeout ? timeout : 60;
cmd->timeout = cmd->default_timeout;
cmd->action = strdup(action);
pcmk__str_update(&cmd->target, target);
cmd->device = strdup(device->id);
cmd->origin = strdup(origin);
cmd->client = strdup(crm_system_name);
cmd->client_name = strdup(crm_system_name);
cmd->internal_user_data = internal_user_data;
cmd->done_cb = done_cb; /* cmd, not internal_user_data, is passed to 'done_cb' as the userdata */
schedule_stonith_command(cmd, device);
}
// Fence agent status commands use custom exit status codes
enum fence_status_code {
fence_status_invalid = -1,
fence_status_active = 0,
fence_status_unknown = 1,
fence_status_inactive = 2,
};
static void
status_search_cb(int pid, const pcmk__action_result_t *result, void *user_data)
{
async_command_t *cmd = user_data;
struct device_search_s *search = cmd->internal_user_data;
stonith_device_t *dev = cmd_device(cmd);
gboolean can = FALSE;
free_async_command(cmd);
if (!dev) {
search_devices_record_result(search, NULL, FALSE);
return;
}
mainloop_set_trigger(dev->work);
if (result->execution_status != PCMK_EXEC_DONE) {
crm_warn("Assuming %s cannot fence %s "
"because status could not be executed: %s%s%s%s",
dev->id, search->host,
pcmk_exec_status_str(result->execution_status),
((result->exit_reason == NULL)? "" : " ("),
((result->exit_reason == NULL)? "" : result->exit_reason),
((result->exit_reason == NULL)? "" : ")"));
search_devices_record_result(search, dev->id, FALSE);
return;
}
switch (result->exit_status) {
case fence_status_unknown:
crm_trace("%s reported it cannot fence %s", dev->id, search->host);
break;
case fence_status_active:
case fence_status_inactive:
crm_trace("%s reported it can fence %s", dev->id, search->host);
can = TRUE;
break;
default:
crm_warn("Assuming %s cannot fence %s "
"(status returned unknown code %d)",
dev->id, search->host, result->exit_status);
break;
}
search_devices_record_result(search, dev->id, can);
}
static void
dynamic_list_search_cb(int pid, const pcmk__action_result_t *result,
void *user_data)
{
async_command_t *cmd = user_data;
struct device_search_s *search = cmd->internal_user_data;
stonith_device_t *dev = cmd_device(cmd);
gboolean can_fence = FALSE;
free_async_command(cmd);
/* Host/alias must be in the list output to be eligible to be fenced
*
* Will cause problems if down'd nodes aren't listed or (for virtual nodes)
* if the guest is still listed despite being moved to another machine
*/
if (!dev) {
search_devices_record_result(search, NULL, FALSE);
return;
}
mainloop_set_trigger(dev->work);
if (pcmk__result_ok(result)) {
crm_info("Refreshing target list for %s", dev->id);
g_list_free_full(dev->targets, free);
dev->targets = stonith__parse_targets(result->action_stdout);
dev->targets_age = time(NULL);
} else if (dev->targets != NULL) {
if (result->execution_status == PCMK_EXEC_DONE) {
crm_info("Reusing most recent target list for %s "
"because list returned error code %d",
dev->id, result->exit_status);
} else {
crm_info("Reusing most recent target list for %s "
"because list could not be executed: %s%s%s%s",
dev->id, pcmk_exec_status_str(result->execution_status),
((result->exit_reason == NULL)? "" : " ("),
((result->exit_reason == NULL)? "" : result->exit_reason),
((result->exit_reason == NULL)? "" : ")"));
}
} else { // We have never successfully executed list
if (result->execution_status == PCMK_EXEC_DONE) {
crm_warn("Assuming %s cannot fence %s "
"because list returned error code %d",
dev->id, search->host, result->exit_status);
} else {
crm_warn("Assuming %s cannot fence %s "
"because list could not be executed: %s%s%s%s",
dev->id, search->host,
pcmk_exec_status_str(result->execution_status),
((result->exit_reason == NULL)? "" : " ("),
((result->exit_reason == NULL)? "" : result->exit_reason),
((result->exit_reason == NULL)? "" : ")"));
}
/* Fall back to pcmk_host_check="status" if the user didn't explicitly
* specify "dynamic-list".
*/
if (g_hash_table_lookup(dev->params, PCMK_STONITH_HOST_CHECK) == NULL) {
crm_notice("Switching to pcmk_host_check='status' for %s", dev->id);
g_hash_table_replace(dev->params, strdup(PCMK_STONITH_HOST_CHECK),
strdup("status"));
}
}
if (dev->targets) {
const char *alias = g_hash_table_lookup(dev->aliases, search->host);
if (!alias) {
alias = search->host;
}
if (pcmk__str_in_list(alias, dev->targets, pcmk__str_casei)) {
can_fence = TRUE;
}
}
search_devices_record_result(search, dev->id, can_fence);
}
/*!
* \internal
* \brief Returns true if any key in first is not in second or second has a different value for key
*/
static int
device_params_diff(GHashTable *first, GHashTable *second) {
char *key = NULL;
char *value = NULL;
GHashTableIter gIter;
g_hash_table_iter_init(&gIter, first);
while (g_hash_table_iter_next(&gIter, (void **)&key, (void **)&value)) {
if(strstr(key, "CRM_meta") == key) {
continue;
} else if(strcmp(key, "crm_feature_set") == 0) {
continue;
} else {
char *other_value = g_hash_table_lookup(second, key);
if (!other_value || !pcmk__str_eq(other_value, value, pcmk__str_casei)) {
crm_trace("Different value for %s: %s != %s", key, other_value, value);
return 1;
}
}
}
return 0;
}
/*!
* \internal
* \brief Checks to see if an identical device already exists in the device_list
*/
static stonith_device_t *
device_has_duplicate(const stonith_device_t *device)
{
stonith_device_t *dup = g_hash_table_lookup(device_list, device->id);
if (!dup) {
crm_trace("No match for %s", device->id);
return NULL;
} else if (!pcmk__str_eq(dup->agent, device->agent, pcmk__str_casei)) {
crm_trace("Different agent: %s != %s", dup->agent, device->agent);
return NULL;
}
/* Use calculate_operation_digest() here? */
if (device_params_diff(device->params, dup->params) ||
device_params_diff(dup->params, device->params)) {
return NULL;
}
crm_trace("Match");
return dup;
}
int
stonith_device_register(xmlNode *dev, gboolean from_cib)
{
stonith_device_t *dup = NULL;
stonith_device_t *device = build_device_from_xml(dev);
guint ndevices = 0;
int rv = pcmk_ok;
CRM_CHECK(device != NULL, return -ENOMEM);
/* do we have a watchdog-device? */
if (pcmk__str_eq(device->id, STONITH_WATCHDOG_ID, pcmk__str_none) ||
pcmk__str_any_of(device->agent, STONITH_WATCHDOG_AGENT,
STONITH_WATCHDOG_AGENT_INTERNAL, NULL)) do {
if (stonith_watchdog_timeout_ms <= 0) {
crm_err("Ignoring watchdog fence device without "
"stonith-watchdog-timeout set.");
rv = -ENODEV;
/* fall through to cleanup & return */
} else if (!pcmk__str_any_of(device->agent, STONITH_WATCHDOG_AGENT,
STONITH_WATCHDOG_AGENT_INTERNAL, NULL)) {
crm_err("Ignoring watchdog fence device with unknown "
"agent '%s' unequal '" STONITH_WATCHDOG_AGENT "'.",
device->agent?device->agent:"");
rv = -ENODEV;
/* fall through to cleanup & return */
} else if (!pcmk__str_eq(device->id, STONITH_WATCHDOG_ID,
pcmk__str_none)) {
crm_err("Ignoring watchdog fence device "
"named %s !='"STONITH_WATCHDOG_ID"'.",
device->id?device->id:"");
rv = -ENODEV;
/* fall through to cleanup & return */
} else {
if (pcmk__str_eq(device->agent, STONITH_WATCHDOG_AGENT,
pcmk__str_none)) {
/* this either has an empty list or the targets
configured for watchdog-fencing
*/
g_list_free_full(stonith_watchdog_targets, free);
stonith_watchdog_targets = device->targets;
device->targets = NULL;
}
if (node_does_watchdog_fencing(stonith_our_uname)) {
g_list_free_full(device->targets, free);
device->targets = stonith__parse_targets(stonith_our_uname);
g_hash_table_replace(device->params,
strdup(PCMK_STONITH_HOST_LIST),
strdup(stonith_our_uname));
/* proceed as with any other stonith-device */
break;
}
crm_debug("Skip registration of watchdog fence device on node not in host-list.");
/* cleanup and fall through to more cleanup and return */
device->targets = NULL;
stonith_device_remove(device->id, from_cib);
}
free_device(device);
return rv;
} while (0);
dup = device_has_duplicate(device);
if (dup) {
ndevices = g_hash_table_size(device_list);
crm_debug("Device '%s' already in device list (%d active device%s)",
device->id, ndevices, pcmk__plural_s(ndevices));
free_device(device);
device = dup;
dup = g_hash_table_lookup(device_list, device->id);
dup->dirty = FALSE;
} else {
stonith_device_t *old = g_hash_table_lookup(device_list, device->id);
if (from_cib && old && old->api_registered) {
/* If the cib is writing over an entry that is shared with a stonith client,
* copy any pending ops that currently exist on the old entry to the new one.
* Otherwise the pending ops will be reported as failures
*/
crm_info("Overwriting existing entry for %s from CIB", device->id);
device->pending_ops = old->pending_ops;
device->api_registered = TRUE;
old->pending_ops = NULL;
if (device->pending_ops) {
mainloop_set_trigger(device->work);
}
}
g_hash_table_replace(device_list, device->id, device);
ndevices = g_hash_table_size(device_list);
crm_notice("Added '%s' to device list (%d active device%s)",
device->id, ndevices, pcmk__plural_s(ndevices));
}
if (from_cib) {
device->cib_registered = TRUE;
} else {
device->api_registered = TRUE;
}
return pcmk_ok;
}
void
stonith_device_remove(const char *id, bool from_cib)
{
stonith_device_t *device = g_hash_table_lookup(device_list, id);
guint ndevices = 0;
if (!device) {
ndevices = g_hash_table_size(device_list);
crm_info("Device '%s' not found (%d active device%s)",
id, ndevices, pcmk__plural_s(ndevices));
return;
}
if (from_cib) {
device->cib_registered = FALSE;
} else {
device->verified = FALSE;
device->api_registered = FALSE;
}
if (!device->cib_registered && !device->api_registered) {
g_hash_table_remove(device_list, id);
ndevices = g_hash_table_size(device_list);
crm_info("Removed '%s' from device list (%d active device%s)",
id, ndevices, pcmk__plural_s(ndevices));
} else {
crm_trace("Not removing '%s' from device list (%d active) because "
"still registered via:%s%s",
id, g_hash_table_size(device_list),
(device->cib_registered? " cib" : ""),
(device->api_registered? " api" : ""));
}
}
/*!
* \internal
* \brief Return the number of stonith levels registered for a node
*
* \param[in] tp Node's topology table entry
*
* \return Number of non-NULL levels in topology entry
* \note This function is used only for log messages.
*/
static int
count_active_levels(const stonith_topology_t *tp)
{
int lpc = 0;
int count = 0;
for (lpc = 0; lpc < ST_LEVEL_MAX; lpc++) {
if (tp->levels[lpc] != NULL) {
count++;
}
}
return count;
}
static void
free_topology_entry(gpointer data)
{
stonith_topology_t *tp = data;
int lpc = 0;
for (lpc = 0; lpc < ST_LEVEL_MAX; lpc++) {
if (tp->levels[lpc] != NULL) {
g_list_free_full(tp->levels[lpc], free);
}
}
free(tp->target);
free(tp->target_value);
free(tp->target_pattern);
free(tp->target_attribute);
free(tp);
}
void
free_topology_list(void)
{
if (topology != NULL) {
g_hash_table_destroy(topology);
topology = NULL;
}
}
void
init_topology_list(void)
{
if (topology == NULL) {
topology = pcmk__strkey_table(NULL, free_topology_entry);
}
}
char *
stonith_level_key(const xmlNode *level, enum fenced_target_by mode)
{
if (mode == fenced_target_by_unknown) {
mode = unpack_level_kind(level);
}
switch (mode) {
case fenced_target_by_name:
return crm_element_value_copy(level, XML_ATTR_STONITH_TARGET);
case fenced_target_by_pattern:
return crm_element_value_copy(level, XML_ATTR_STONITH_TARGET_PATTERN);
case fenced_target_by_attribute:
return crm_strdup_printf("%s=%s",
crm_element_value(level, XML_ATTR_STONITH_TARGET_ATTRIBUTE),
crm_element_value(level, XML_ATTR_STONITH_TARGET_VALUE));
default:
return crm_strdup_printf("unknown-%s", ID(level));
}
}
/*!
* \internal
* \brief Parse target identification from topology level XML
*
* \param[in] level Topology level XML to parse
*
* \return How to identify target of \p level
*/
static int
unpack_level_kind(const xmlNode *level)
{
if (crm_element_value(level, XML_ATTR_STONITH_TARGET) != NULL) {
return fenced_target_by_name;
}
if (crm_element_value(level, XML_ATTR_STONITH_TARGET_PATTERN) != NULL) {
return fenced_target_by_pattern;
}
if (!stand_alone /* if standalone, there's no attribute manager */
&& (crm_element_value(level, XML_ATTR_STONITH_TARGET_ATTRIBUTE) != NULL)
&& (crm_element_value(level, XML_ATTR_STONITH_TARGET_VALUE) != NULL)) {
return fenced_target_by_attribute;
}
return fenced_target_by_unknown;
}
static stonith_key_value_t *
parse_device_list(const char *devices)
{
int lpc = 0;
int max = 0;
int last = 0;
stonith_key_value_t *output = NULL;
if (devices == NULL) {
return output;
}
max = strlen(devices);
for (lpc = 0; lpc <= max; lpc++) {
if (devices[lpc] == ',' || devices[lpc] == 0) {
char *line = strndup(devices + last, lpc - last);
output = stonith_key_value_add(output, NULL, line);
free(line);
last = lpc + 1;
}
}
return output;
}
/*!
* \internal
* \brief Unpack essential information from topology request XML
*
* \param[in] xml Request XML to search
* \param[out] mode If not NULL, where to store level kind
* \param[out] target If not NULL, where to store representation of target
* \param[out] id If not NULL, where to store level number
* \param[out] desc If not NULL, where to store log-friendly level description
*
* \return Topology level XML from within \p xml, or NULL if not found
* \note The caller is responsible for freeing \p *target and \p *desc if set.
*/
static xmlNode *
unpack_level_request(xmlNode *xml, enum fenced_target_by *mode, char **target,
int *id, char **desc)
{
enum fenced_target_by local_mode = fenced_target_by_unknown;
char *local_target = NULL;
int local_id = 0;
/* The level element can be the top element or lower. If top level, don't
* search by xpath, because it might give multiple hits if the XML is the
* CIB.
*/
if ((xml != NULL)
&& !pcmk__str_eq(TYPE(xml), XML_TAG_FENCING_LEVEL, pcmk__str_none)) {
xml = get_xpath_object("//" XML_TAG_FENCING_LEVEL, xml, LOG_WARNING);
}
if (xml == NULL) {
if (desc != NULL) {
*desc = crm_strdup_printf("missing");
}
} else {
local_mode = unpack_level_kind(xml);
local_target = stonith_level_key(xml, local_mode);
crm_element_value_int(xml, XML_ATTR_STONITH_INDEX, &local_id);
if (desc != NULL) {
*desc = crm_strdup_printf("%s[%d]", local_target, local_id);
}
}
if (mode != NULL) {
*mode = local_mode;
}
if (id != NULL) {
*id = local_id;
}
if (target != NULL) {
*target = local_target;
} else {
free(local_target);
}
return xml;
}
/*!
* \internal
* \brief Register a fencing topology level for a target
*
* Given an XML request specifying the target name, level index, and device IDs
* for the level, this will create an entry for the target in the global topology
* table if one does not already exist, then append the specified device IDs to
* the entry's device list for the specified level.
*
* \param[in] msg XML request for STONITH level registration
* \param[out] desc If not NULL, set to string representation "TARGET[LEVEL]"
* \param[out] result Where to set result of registration
*/
void
fenced_register_level(xmlNode *msg, char **desc, pcmk__action_result_t *result)
{
int id = 0;
xmlNode *level;
enum fenced_target_by mode;
char *target;
stonith_topology_t *tp;
stonith_key_value_t *dIter = NULL;
stonith_key_value_t *devices = NULL;
CRM_CHECK((msg != NULL) && (result != NULL), return);
level = unpack_level_request(msg, &mode, &target, &id, desc);
if (level == NULL) {
fenced_set_protocol_error(result);
return;
}
// Ensure an ID was given (even the client API adds an ID)
if (pcmk__str_empty(ID(level))) {
crm_warn("Ignoring registration for topology level without ID");
free(target);
crm_log_xml_trace(level, "Bad level");
pcmk__format_result(result, CRM_EX_INVALID_PARAM, PCMK_EXEC_INVALID,
"Topology level is invalid without ID");
return;
}
// Ensure a valid target was specified
if (mode == fenced_target_by_unknown) {
crm_warn("Ignoring registration for topology level '%s' "
"without valid target", ID(level));
free(target);
crm_log_xml_trace(level, "Bad level");
pcmk__format_result(result, CRM_EX_INVALID_PARAM, PCMK_EXEC_INVALID,
"Invalid target for topology level '%s'",
ID(level));
return;
}
// Ensure level ID is in allowed range
if ((id <= 0) || (id >= ST_LEVEL_MAX)) {
crm_warn("Ignoring topology registration for %s with invalid level %d",
target, id);
free(target);
crm_log_xml_trace(level, "Bad level");
pcmk__format_result(result, CRM_EX_INVALID_PARAM, PCMK_EXEC_INVALID,
"Invalid level number '%s' for topology level '%s'",
pcmk__s(crm_element_value(level,
XML_ATTR_STONITH_INDEX),
""),
ID(level));
return;
}
/* Find or create topology table entry */
tp = g_hash_table_lookup(topology, target);
if (tp == NULL) {
tp = calloc(1, sizeof(stonith_topology_t));
if (tp == NULL) {
pcmk__set_result(result, CRM_EX_ERROR, PCMK_EXEC_ERROR,
strerror(ENOMEM));
free(target);
return;
}
tp->kind = mode;
tp->target = target;
tp->target_value = crm_element_value_copy(level, XML_ATTR_STONITH_TARGET_VALUE);
tp->target_pattern = crm_element_value_copy(level, XML_ATTR_STONITH_TARGET_PATTERN);
tp->target_attribute = crm_element_value_copy(level, XML_ATTR_STONITH_TARGET_ATTRIBUTE);
g_hash_table_replace(topology, tp->target, tp);
crm_trace("Added %s (%d) to the topology (%d active entries)",
target, (int) mode, g_hash_table_size(topology));
} else {
free(target);
}
if (tp->levels[id] != NULL) {
crm_info("Adding to the existing %s[%d] topology entry",
tp->target, id);
}
devices = parse_device_list(crm_element_value(level, XML_ATTR_STONITH_DEVICES));
for (dIter = devices; dIter; dIter = dIter->next) {
const char *device = dIter->value;
crm_trace("Adding device '%s' for %s[%d]", device, tp->target, id);
tp->levels[id] = g_list_append(tp->levels[id], strdup(device));
}
stonith_key_value_freeall(devices, 1, 1);
{
int nlevels = count_active_levels(tp);
crm_info("Target %s has %d active fencing level%s",
tp->target, nlevels, pcmk__plural_s(nlevels));
}
pcmk__set_result(result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
}
/*!
* \internal
* \brief Unregister a fencing topology level for a target
*
* Given an XML request specifying the target name and level index (or 0 for all
* levels), this will remove any corresponding entry for the target from the
* global topology table.
*
* \param[in] msg XML request for STONITH level registration
* \param[out] desc If not NULL, set to string representation "TARGET[LEVEL]"
* \param[out] result Where to set result of unregistration
*/
void
fenced_unregister_level(xmlNode *msg, char **desc,
pcmk__action_result_t *result)
{
int id = -1;
stonith_topology_t *tp;
char *target;
xmlNode *level = NULL;
CRM_CHECK(result != NULL, return);
level = unpack_level_request(msg, NULL, &target, &id, desc);
if (level == NULL) {
fenced_set_protocol_error(result);
return;
}
// Ensure level ID is in allowed range
if ((id < 0) || (id >= ST_LEVEL_MAX)) {
crm_warn("Ignoring topology unregistration for %s with invalid level %d",
target, id);
free(target);
crm_log_xml_trace(level, "Bad level");
pcmk__format_result(result, CRM_EX_INVALID_PARAM, PCMK_EXEC_INVALID,
"Invalid level number '%s' for topology level %s",
pcmk__s(crm_element_value(level,
XML_ATTR_STONITH_INDEX),
"<null>"),
// Client API doesn't add ID to unregistration XML
pcmk__s(ID(level), ""));
return;
}
tp = g_hash_table_lookup(topology, target);
if (tp == NULL) {
guint nentries = g_hash_table_size(topology);
crm_info("No fencing topology found for %s (%d active %s)",
target, nentries,
pcmk__plural_alt(nentries, "entry", "entries"));
} else if (id == 0 && g_hash_table_remove(topology, target)) {
guint nentries = g_hash_table_size(topology);
crm_info("Removed all fencing topology entries related to %s "
"(%d active %s remaining)", target, nentries,
pcmk__plural_alt(nentries, "entry", "entries"));
} else if (tp->levels[id] != NULL) {
guint nlevels;
g_list_free_full(tp->levels[id], free);
tp->levels[id] = NULL;
nlevels = count_active_levels(tp);
crm_info("Removed level %d from fencing topology for %s "
"(%d active level%s remaining)",
id, target, nlevels, pcmk__plural_s(nlevels));
}
free(target);
pcmk__set_result(result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
}
static char *
list_to_string(GList *list, const char *delim, gboolean terminate_with_delim)
{
int max = g_list_length(list);
size_t delim_len = delim?strlen(delim):0;
size_t alloc_size = 1 + (max?((max-1+(terminate_with_delim?1:0))*delim_len):0);
char *rv;
GList *gIter;
for (gIter = list; gIter != NULL; gIter = gIter->next) {
const char *value = (const char *) gIter->data;
alloc_size += strlen(value);
}
rv = calloc(alloc_size, sizeof(char));
if (rv) {
char *pos = rv;
const char *lead_delim = "";
for (gIter = list; gIter != NULL; gIter = gIter->next) {
const char *value = (const char *) gIter->data;
pos = &pos[sprintf(pos, "%s%s", lead_delim, value)];
lead_delim = delim;
}
if (max && terminate_with_delim) {
sprintf(pos, "%s", delim);
}
}
return rv;
}
/*!
* \internal
* \brief Execute a fence agent action directly (and asynchronously)
*
* Handle a STONITH_OP_EXEC API message by scheduling a requested agent action
* directly on a specified device. Only list, monitor, and status actions are
* expected to use this call, though it should work with any agent command.
*
* \param[in] msg Request XML specifying action
* \param[out] result Where to store result of action
*
* \note If the action is monitor, the device must be registered via the API
* (CIB registration is not sufficient), because monitor should not be
* possible unless the device is "started" (API registered).
*/
static void
execute_agent_action(xmlNode *msg, pcmk__action_result_t *result)
{
xmlNode *dev = get_xpath_object("//" F_STONITH_DEVICE, msg, LOG_ERR);
xmlNode *op = get_xpath_object("//@" F_STONITH_ACTION, msg, LOG_ERR);
const char *id = crm_element_value(dev, F_STONITH_DEVICE);
const char *action = crm_element_value(op, F_STONITH_ACTION);
async_command_t *cmd = NULL;
stonith_device_t *device = NULL;
if ((id == NULL) || (action == NULL)) {
crm_info("Malformed API action request: device %s, action %s",
(id? id : "not specified"),
(action? action : "not specified"));
fenced_set_protocol_error(result);
return;
}
if (pcmk__str_eq(id, STONITH_WATCHDOG_ID, pcmk__str_none)) {
// Watchdog agent actions are implemented internally
if (stonith_watchdog_timeout_ms <= 0) {
pcmk__set_result(result, CRM_EX_ERROR, PCMK_EXEC_NO_FENCE_DEVICE,
"Watchdog fence device not configured");
return;
} else if (pcmk__str_eq(action, "list", pcmk__str_casei)) {
pcmk__set_result(result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
pcmk__set_result_output(result,
list_to_string(stonith_watchdog_targets,
"\n", TRUE),
NULL);
return;
} else if (pcmk__str_eq(action, "monitor", pcmk__str_casei)) {
pcmk__set_result(result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
return;
}
}
device = g_hash_table_lookup(device_list, id);
if (device == NULL) {
crm_info("Ignoring API '%s' action request because device %s not found",
action, id);
pcmk__format_result(result, CRM_EX_ERROR, PCMK_EXEC_NO_FENCE_DEVICE,
"'%s' not found", id);
return;
} else if (!device->api_registered && !strcmp(action, "monitor")) {
// Monitors may run only on "started" (API-registered) devices
crm_info("Ignoring API '%s' action request because device %s not active",
action, id);
pcmk__format_result(result, CRM_EX_ERROR, PCMK_EXEC_NO_FENCE_DEVICE,
"'%s' not active", id);
return;
}
cmd = create_async_command(msg);
if (cmd == NULL) {
crm_log_xml_warn(msg, "invalid");
fenced_set_protocol_error(result);
return;
}
schedule_stonith_command(cmd, device);
pcmk__set_result(result, CRM_EX_OK, PCMK_EXEC_PENDING, NULL);
}
static void
search_devices_record_result(struct device_search_s *search, const char *device, gboolean can_fence)
{
search->replies_received++;
if (can_fence && device) {
if (search->support_action_only != st_device_supports_none) {
stonith_device_t *dev = g_hash_table_lookup(device_list, device);
if (dev && !pcmk_is_set(dev->flags, search->support_action_only)) {
return;
}
}
search->capable = g_list_append(search->capable, strdup(device));
}
if (search->replies_needed == search->replies_received) {
guint ndevices = g_list_length(search->capable);
crm_debug("Search found %d device%s that can perform '%s' targeting %s",
ndevices, pcmk__plural_s(ndevices),
(search->action? search->action : "unknown action"),
(search->host? search->host : "any node"));
search->callback(search->capable, search->user_data);
free(search->host);
free(search->action);
free(search);
}
}
/*!
* \internal
* \brief Check whether the local host is allowed to execute a fencing action
*
* \param[in] device Fence device to check
* \param[in] action Fence action to check
* \param[in] target Hostname of fence target
* \param[in] allow_suicide Whether self-fencing is allowed for this operation
*
* \return TRUE if local host is allowed to execute action, FALSE otherwise
*/
static gboolean
localhost_is_eligible(const stonith_device_t *device, const char *action,
const char *target, gboolean allow_suicide)
{
gboolean localhost_is_target = pcmk__str_eq(target, stonith_our_uname,
pcmk__str_casei);
if (device && action && device->on_target_actions
&& strstr(device->on_target_actions, action)) {
if (!localhost_is_target) {
crm_trace("Operation '%s' using %s can only be executed for "
"local host, not %s", action, device->id, target);
return FALSE;
}
} else if (localhost_is_target && !allow_suicide) {
crm_trace("'%s' operation does not support self-fencing", action);
return FALSE;
}
return TRUE;
}
static void
can_fence_host_with_device(stonith_device_t * dev, struct device_search_s *search)
{
gboolean can = FALSE;
const char *check_type = NULL;
const char *host = search->host;
const char *alias = NULL;
CRM_LOG_ASSERT(dev != NULL);
if (dev == NULL) {
goto search_report_results;
} else if (host == NULL) {
can = TRUE;
goto search_report_results;
}
/* Short-circuit query if this host is not allowed to perform the action */
if (pcmk__str_eq(search->action, "reboot", pcmk__str_casei)) {
/* A "reboot" *might* get remapped to "off" then "on", so short-circuit
* only if all three are disallowed. If only one or two are disallowed,
* we'll report that with the results. We never allow suicide for
* remapped "on" operations because the host is off at that point.
*/
if (!localhost_is_eligible(dev, "reboot", host, search->allow_suicide)
&& !localhost_is_eligible(dev, "off", host, search->allow_suicide)
&& !localhost_is_eligible(dev, "on", host, FALSE)) {
goto search_report_results;
}
} else if (!localhost_is_eligible(dev, search->action, host,
search->allow_suicide)) {
goto search_report_results;
}
alias = g_hash_table_lookup(dev->aliases, host);
if (alias == NULL) {
alias = host;
}
check_type = target_list_type(dev);
if (pcmk__str_eq(check_type, PCMK__VALUE_NONE, pcmk__str_casei)) {
can = TRUE;
} else if (pcmk__str_eq(check_type, "static-list", pcmk__str_casei)) {
/* Presence in the hostmap is sufficient
* Only use if all hosts on which the device can be active can always fence all listed hosts
*/
if (pcmk__str_in_list(host, dev->targets, pcmk__str_casei)) {
can = TRUE;
} else if (g_hash_table_lookup(dev->params, PCMK_STONITH_HOST_MAP)
&& g_hash_table_lookup(dev->aliases, host)) {
can = TRUE;
}
} else if (pcmk__str_eq(check_type, "dynamic-list", pcmk__str_casei)) {
time_t now = time(NULL);
if (dev->targets == NULL || dev->targets_age + 60 < now) {
int device_timeout = get_action_timeout(dev, "list", search->per_device_timeout);
if (device_timeout > search->per_device_timeout) {
crm_notice("Since the pcmk_list_timeout(%ds) parameter of %s is larger than stonith-timeout(%ds), timeout may occur",
device_timeout, dev->id, search->per_device_timeout);
}
crm_trace("Running '%s' to check whether %s is eligible to fence %s (%s)",
check_type, dev->id, search->host, search->action);
schedule_internal_command(__func__, dev, "list", NULL,
search->per_device_timeout, search, dynamic_list_search_cb);
/* we'll respond to this search request async in the cb */
return;
}
if (pcmk__str_in_list(alias, dev->targets, pcmk__str_casei)) {
can = TRUE;
}
} else if (pcmk__str_eq(check_type, "status", pcmk__str_casei)) {
int device_timeout = get_action_timeout(dev, check_type, search->per_device_timeout);
if (device_timeout > search->per_device_timeout) {
crm_notice("Since the pcmk_status_timeout(%ds) parameter of %s is larger than stonith-timeout(%ds), timeout may occur",
device_timeout, dev->id, search->per_device_timeout);
}
crm_trace("Running '%s' to check whether %s is eligible to fence %s (%s)",
check_type, dev->id, search->host, search->action);
schedule_internal_command(__func__, dev, "status", search->host,
search->per_device_timeout, search, status_search_cb);
/* we'll respond to this search request async in the cb */
return;
} else {
crm_err("Invalid value for " PCMK_STONITH_HOST_CHECK ": %s", check_type);
check_type = "Invalid " PCMK_STONITH_HOST_CHECK;
}
if (pcmk__str_eq(host, alias, pcmk__str_casei)) {
crm_info("%s is%s eligible to fence (%s) %s: %s",
dev->id, (can? "" : " not"), search->action, host, check_type);
} else {
crm_info("%s is%s eligible to fence (%s) %s (aka. '%s'): %s",
dev->id, (can? "" : " not"), search->action, host, alias,
check_type);
}
search_report_results:
search_devices_record_result(search, dev ? dev->id : NULL, can);
}
static void
search_devices(gpointer key, gpointer value, gpointer user_data)
{
stonith_device_t *dev = value;
struct device_search_s *search = user_data;
can_fence_host_with_device(dev, search);
}
#define DEFAULT_QUERY_TIMEOUT 20
static void
get_capable_devices(const char *host, const char *action, int timeout, bool suicide, void *user_data,
void (*callback) (GList * devices, void *user_data), uint32_t support_action_only)
{
struct device_search_s *search;
guint ndevices = g_hash_table_size(device_list);
if (ndevices == 0) {
callback(NULL, user_data);
return;
}
search = calloc(1, sizeof(struct device_search_s));
if (!search) {
crm_crit("Cannot search for capable fence devices: %s",
strerror(ENOMEM));
callback(NULL, user_data);
return;
}
pcmk__str_update(&search->host, host);
pcmk__str_update(&search->action, action);
search->per_device_timeout = timeout;
search->allow_suicide = suicide;
search->callback = callback;
search->user_data = user_data;
search->support_action_only = support_action_only;
/* We are guaranteed this many replies, even if a device is
* unregistered while the search is in progress.
*/
search->replies_needed = ndevices;
crm_debug("Searching %d device%s to see which can execute '%s' targeting %s",
ndevices, pcmk__plural_s(ndevices),
(search->action? search->action : "unknown action"),
(search->host? search->host : "any node"));
g_hash_table_foreach(device_list, search_devices, search);
}
struct st_query_data {
xmlNode *reply;
char *remote_peer;
char *client_id;
char *target;
char *action;
int call_options;
};
/*!
* \internal
* \brief Add action-specific attributes to query reply XML
*
* \param[in,out] xml XML to add attributes to
* \param[in] action Fence action
* \param[in] device Fence device
* \param[in] target Fence target
*/
static void
add_action_specific_attributes(xmlNode *xml, const char *action,
const stonith_device_t *device,
const char *target)
{
int action_specific_timeout;
int delay_max;
int delay_base;
CRM_CHECK(xml && action && device, return);
if (is_action_required(action, device)) {
crm_trace("Action '%s' is required using %s", action, device->id);
crm_xml_add_int(xml, F_STONITH_DEVICE_REQUIRED, 1);
}
action_specific_timeout = get_action_timeout(device, action, 0);
if (action_specific_timeout) {
crm_trace("Action '%s' has timeout %dms using %s",
action, action_specific_timeout, device->id);
crm_xml_add_int(xml, F_STONITH_ACTION_TIMEOUT, action_specific_timeout);
}
delay_max = get_action_delay_max(device, action);
if (delay_max > 0) {
crm_trace("Action '%s' has maximum random delay %dms using %s",
action, delay_max, device->id);
crm_xml_add_int(xml, F_STONITH_DELAY_MAX, delay_max / 1000);
}
delay_base = get_action_delay_base(device, action, target);
if (delay_base > 0) {
crm_xml_add_int(xml, F_STONITH_DELAY_BASE, delay_base / 1000);
}
if ((delay_max > 0) && (delay_base == 0)) {
crm_trace("Action '%s' has maximum random delay %dms using %s",
action, delay_max, device->id);
} else if ((delay_max == 0) && (delay_base > 0)) {
crm_trace("Action '%s' has a static delay of %dms using %s",
action, delay_base, device->id);
} else if ((delay_max > 0) && (delay_base > 0)) {
crm_trace("Action '%s' has a minimum delay of %dms and a randomly chosen "
"maximum delay of %dms using %s",
action, delay_base, delay_max, device->id);
}
}
/*!
* \internal
* \brief Add "disallowed" attribute to query reply XML if appropriate
*
* \param[in,out] xml XML to add attribute to
* \param[in] action Fence action
* \param[in] device Fence device
* \param[in] target Fence target
* \param[in] allow_suicide Whether self-fencing is allowed
*/
static void
add_disallowed(xmlNode *xml, const char *action, const stonith_device_t *device,
const char *target, gboolean allow_suicide)
{
if (!localhost_is_eligible(device, action, target, allow_suicide)) {
crm_trace("Action '%s' using %s is disallowed for local host",
action, device->id);
pcmk__xe_set_bool_attr(xml, F_STONITH_ACTION_DISALLOWED, true);
}
}
/*!
* \internal
* \brief Add child element with action-specific values to query reply XML
*
* \param[in,out] xml XML to add attribute to
* \param[in] action Fence action
* \param[in] device Fence device
* \param[in] target Fence target
* \param[in] allow_suicide Whether self-fencing is allowed
*/
static void
add_action_reply(xmlNode *xml, const char *action,
const stonith_device_t *device, const char *target,
gboolean allow_suicide)
{
xmlNode *child = create_xml_node(xml, F_STONITH_ACTION);
crm_xml_add(child, XML_ATTR_ID, action);
add_action_specific_attributes(child, action, device, target);
add_disallowed(child, action, device, target, allow_suicide);
}
static void
stonith_query_capable_device_cb(GList * devices, void *user_data)
{
struct st_query_data *query = user_data;
int available_devices = 0;
xmlNode *dev = NULL;
xmlNode *list = NULL;
GList *lpc = NULL;
pcmk__client_t *client = NULL;
if (query->client_id != NULL) {
client = pcmk__find_client_by_id(query->client_id);
if ((client == NULL) && (query->remote_peer == NULL)) {
crm_trace("Skipping reply to %s: no longer a client",
query->client_id);
goto done;
}
}
/* Pack the results into XML */
list = create_xml_node(NULL, __func__);
crm_xml_add(list, F_STONITH_TARGET, query->target);
for (lpc = devices; lpc != NULL; lpc = lpc->next) {
stonith_device_t *device = g_hash_table_lookup(device_list, lpc->data);
const char *action = query->action;
if (!device) {
/* It is possible the device got unregistered while
* determining who can fence the target */
continue;
}
available_devices++;
dev = create_xml_node(list, F_STONITH_DEVICE);
crm_xml_add(dev, XML_ATTR_ID, device->id);
crm_xml_add(dev, "namespace", device->namespace);
crm_xml_add(dev, "agent", device->agent);
crm_xml_add_int(dev, F_STONITH_DEVICE_VERIFIED, device->verified);
crm_xml_add_int(dev, F_STONITH_DEVICE_SUPPORT_FLAGS, device->flags);
/* If the originating fencer wants to reboot the node, and we have a
* capable device that doesn't support "reboot", remap to "off" instead.
*/
if (!pcmk_is_set(device->flags, st_device_supports_reboot)
&& pcmk__str_eq(query->action, "reboot", pcmk__str_casei)) {
crm_trace("%s doesn't support reboot, using values for off instead",
device->id);
action = "off";
}
/* Add action-specific values if available */
add_action_specific_attributes(dev, action, device, query->target);
if (pcmk__str_eq(query->action, "reboot", pcmk__str_casei)) {
/* A "reboot" *might* get remapped to "off" then "on", so after
* sending the "reboot"-specific values in the main element, we add
* sub-elements for "off" and "on" values.
*
* We short-circuited earlier if "reboot", "off" and "on" are all
* disallowed for the local host. However if only one or two are
* disallowed, we send back the results and mark which ones are
* disallowed. If "reboot" is disallowed, this might cause problems
* with older fencer versions, which won't check for it. Older
* versions will ignore "off" and "on", so they are not a problem.
*/
add_disallowed(dev, action, device, query->target,
pcmk_is_set(query->call_options, st_opt_allow_suicide));
add_action_reply(dev, "off", device, query->target,
pcmk_is_set(query->call_options, st_opt_allow_suicide));
add_action_reply(dev, "on", device, query->target, FALSE);
}
/* A query without a target wants device parameters */
if (query->target == NULL) {
xmlNode *attrs = create_xml_node(dev, XML_TAG_ATTRS);
g_hash_table_foreach(device->params, hash2field, attrs);
}
}
crm_xml_add_int(list, F_STONITH_AVAILABLE_DEVICES, available_devices);
if (query->target) {
crm_debug("Found %d matching device%s for target '%s'",
available_devices, pcmk__plural_s(available_devices),
query->target);
} else {
crm_debug("%d device%s installed",
available_devices, pcmk__plural_s(available_devices));
}
if (list != NULL) {
crm_log_xml_trace(list, "Add query results");
add_message_xml(query->reply, F_STONITH_CALLDATA, list);
}
stonith_send_reply(query->reply, query->call_options, query->remote_peer,
client);
done:
free_xml(query->reply);
free(query->remote_peer);
free(query->client_id);
free(query->target);
free(query->action);
free(query);
free_xml(list);
g_list_free_full(devices, free);
}
/*!
* \internal
* \brief Log the result of an asynchronous command
*
* \param[in] cmd Command the result is for
* \param[in] result Result of command
* \param[in] pid Process ID of command, if available
* \param[in] next Alternate device that will be tried if command failed
* \param[in] op_merged Whether this command was merged with an earlier one
*/
static void
log_async_result(const async_command_t *cmd,
const pcmk__action_result_t *result,
int pid, const char *next, bool op_merged)
{
int log_level = LOG_ERR;
int output_log_level = LOG_NEVER;
guint devices_remaining = g_list_length(cmd->next_device_iter);
GString *msg = g_string_sized_new(80); // Reasonable starting size
// Choose log levels appropriately if we have a result
if (pcmk__result_ok(result)) {
log_level = (cmd->target == NULL)? LOG_DEBUG : LOG_NOTICE;
if ((result->action_stdout != NULL)
&& !pcmk__str_eq(cmd->action, "metadata", pcmk__str_casei)) {
output_log_level = LOG_DEBUG;
}
next = NULL;
} else {
log_level = (cmd->target == NULL)? LOG_NOTICE : LOG_ERR;
if ((result->action_stdout != NULL)
&& !pcmk__str_eq(cmd->action, "metadata", pcmk__str_casei)) {
output_log_level = LOG_WARNING;
}
}
// Build the log message piece by piece
g_string_printf(msg, "Operation '%s' ", cmd->action);
if (pid != 0) {
g_string_append_printf(msg, "[%d] ", pid);
}
if (cmd->target != NULL) {
g_string_append_printf(msg, "targeting %s ", cmd->target);
}
if (cmd->device != NULL) {
g_string_append_printf(msg, "using %s ", cmd->device);
}
// Add exit status or execution status as appropriate
if (result->execution_status == PCMK_EXEC_DONE) {
g_string_append_printf(msg, "returned %d", result->exit_status);
} else {
g_string_append_printf(msg, "could not be executed: %s",
pcmk_exec_status_str(result->execution_status));
}
// Add exit reason and next device if appropriate
if (result->exit_reason != NULL) {
g_string_append_printf(msg, " (%s)", result->exit_reason);
}
if (next != NULL) {
g_string_append_printf(msg, ", retrying with %s", next);
}
if (devices_remaining > 0) {
g_string_append_printf(msg, " (%u device%s remaining)",
(unsigned int) devices_remaining,
pcmk__plural_s(devices_remaining));
}
g_string_append_printf(msg, " " CRM_XS " %scall %d from %s",
(op_merged? "merged " : ""), cmd->id,
cmd->client_name);
// Log the result
do_crm_log(log_level, "%s", msg->str);
g_string_free(msg, TRUE);
// Log the output (which may have multiple lines), if appropriate
if (output_log_level != LOG_NEVER) {
char *prefix = crm_strdup_printf("%s[%d]", cmd->device, pid);
crm_log_output(output_log_level, prefix, result->action_stdout);
free(prefix);
}
}
/*!
* \internal
* \brief Reply to requester after asynchronous command completion
*
* \param[in] cmd Command that completed
* \param[in] result Result of command
* \param[in] pid Process ID of command, if available
* \param[in] merged If true, command was merged with another, not executed
*/
static void
send_async_reply(const async_command_t *cmd, const pcmk__action_result_t *result,
int pid, bool merged)
{
xmlNode *reply = NULL;
pcmk__client_t *client = NULL;
CRM_CHECK((cmd != NULL) && (result != NULL), return);
log_async_result(cmd, result, pid, NULL, merged);
if (cmd->client != NULL) {
client = pcmk__find_client_by_id(cmd->client);
if ((client == NULL) && (cmd->origin == NULL)) {
crm_trace("Skipping reply to %s: no longer a client", cmd->client);
return;
}
}
reply = construct_async_reply(cmd, result);
if (merged) {
pcmk__xe_set_bool_attr(reply, F_STONITH_MERGED, true);
}
if (!stand_alone && pcmk__is_fencing_action(cmd->action)
&& pcmk__str_eq(cmd->origin, cmd->target, pcmk__str_casei)) {
/* The target was also the originator, so broadcast the result on its
* behalf (since it will be unable to).
*/
crm_trace("Broadcast '%s' result for %s (target was also originator)",
cmd->action, cmd->target);
crm_xml_add(reply, F_SUBTYPE, "broadcast");
crm_xml_add(reply, F_STONITH_OPERATION, T_STONITH_NOTIFY);
send_cluster_message(NULL, crm_msg_stonith_ng, reply, FALSE);
} else {
// Reply only to the originator
stonith_send_reply(reply, cmd->options, cmd->origin, client);
}
crm_log_xml_trace(reply, "Reply");
free_xml(reply);
if (stand_alone) {
/* Do notification with a clean data object */
xmlNode *notify_data = create_xml_node(NULL, T_STONITH_NOTIFY_FENCE);
stonith__xe_set_result(notify_data, result);
crm_xml_add(notify_data, F_STONITH_TARGET, cmd->target);
crm_xml_add(notify_data, F_STONITH_OPERATION, cmd->op);
crm_xml_add(notify_data, F_STONITH_DELEGATE, "localhost");
crm_xml_add(notify_data, F_STONITH_DEVICE, cmd->device);
crm_xml_add(notify_data, F_STONITH_REMOTE_OP_ID, cmd->remote_op_id);
crm_xml_add(notify_data, F_STONITH_ORIGIN, cmd->client);
fenced_send_notification(T_STONITH_NOTIFY_FENCE, result, notify_data);
fenced_send_notification(T_STONITH_NOTIFY_HISTORY, NULL, NULL);
}
}
static void
cancel_stonith_command(async_command_t * cmd)
{
stonith_device_t *device = cmd_device(cmd);
if (device) {
crm_trace("Cancel scheduled '%s' action using %s",
cmd->action, device->id);
device->pending_ops = g_list_remove(device->pending_ops, cmd);
}
}
/*!
* \internal
* \brief Cancel and reply to any duplicates of a just-completed operation
*
* Check whether any fencing operations are scheduled to do the same thing as
* one that just succeeded. If so, rather than performing the same operation
* twice, return the result of this operation for all matching pending commands.
*
* \param[in,out] cmd Fencing operation that just succeeded
* \param[in] result Result of \p cmd
* \param[in] pid If nonzero, process ID of agent invocation (for logs)
*
* \note Duplicate merging will do the right thing for either type of remapped
* reboot. If the executing fencer remapped an unsupported reboot to off,
* then cmd->action will be "reboot" and will be merged with any other
* reboot requests. If the originating fencer remapped a topology reboot
* to off then on, we will get here once with cmd->action "off" and once
* with "on", and they will be merged separately with similar requests.
*/
static void
reply_to_duplicates(async_command_t *cmd, const pcmk__action_result_t *result,
int pid)
{
GList *next = NULL;
for (GList *iter = cmd_list; iter != NULL; iter = next) {
async_command_t *cmd_other = iter->data;
next = iter->next; // We might delete this entry, so grab next now
if (cmd == cmd_other) {
continue;
}
/* A pending operation matches if:
* 1. The client connections are different.
* 2. The target is the same.
* 3. The fencing action is the same.
* 4. The device scheduled to execute the action is the same.
*/
if (pcmk__str_eq(cmd->client, cmd_other->client, pcmk__str_casei) ||
!pcmk__str_eq(cmd->target, cmd_other->target, pcmk__str_casei) ||
!pcmk__str_eq(cmd->action, cmd_other->action, pcmk__str_casei) ||
!pcmk__str_eq(cmd->device, cmd_other->device, pcmk__str_casei)) {
continue;
}
crm_notice("Merging fencing action '%s'%s%s originating from "
"client %s with identical fencing request from client %s",
cmd_other->action,
(cmd_other->target == NULL)? "" : " targeting ",
pcmk__s(cmd_other->target, ""), cmd_other->client_name,
cmd->client_name);
// Stop tracking the duplicate, send its result, and cancel it
cmd_list = g_list_remove_link(cmd_list, iter);
send_async_reply(cmd_other, result, pid, true);
cancel_stonith_command(cmd_other);
free_async_command(cmd_other);
g_list_free_1(iter);
}
}
/*!
* \internal
* \brief Return the next required device (if any) for an operation
*
* \param[in,out] cmd Fencing operation that just succeeded
*
* \return Next device required for action if any, otherwise NULL
*/
static stonith_device_t *
next_required_device(async_command_t *cmd)
{
for (GList *iter = cmd->next_device_iter; iter != NULL; iter = iter->next) {
stonith_device_t *next_device = g_hash_table_lookup(device_list,
iter->data);
if (is_action_required(cmd->action, next_device)) {
/* This is only called for successful actions, so it's OK to skip
* non-required devices.
*/
cmd->next_device_iter = iter->next;
return next_device;
}
}
return NULL;
}
static void
st_child_done(int pid, const pcmk__action_result_t *result, void *user_data)
{
async_command_t *cmd = user_data;
stonith_device_t *device = NULL;
stonith_device_t *next_device = NULL;
CRM_CHECK(cmd != NULL, return);
device = cmd_device(cmd);
cmd->active_on = NULL;
/* The device is ready to do something else now */
if (device) {
if (!device->verified && pcmk__result_ok(result) &&
(pcmk__strcase_any_of(cmd->action, "list", "monitor", "status", NULL))) {
device->verified = TRUE;
}
mainloop_set_trigger(device->work);
}
if (pcmk__result_ok(result)) {
next_device = next_required_device(cmd);
} else if ((cmd->next_device_iter != NULL)
&& !is_action_required(cmd->action, device)) {
/* if this device didn't work out, see if there are any others we can try.
* if the failed device was 'required', we can't pick another device. */
next_device = g_hash_table_lookup(device_list,
cmd->next_device_iter->data);
cmd->next_device_iter = cmd->next_device_iter->next;
}
if (next_device == NULL) {
send_async_reply(cmd, result, pid, false);
if (pcmk__result_ok(result)) {
reply_to_duplicates(cmd, result, pid);
}
free_async_command(cmd);
} else { // This operation requires more fencing
log_async_result(cmd, result, pid, next_device->id, false);
schedule_stonith_command(cmd, next_device);
}
}
static gint
sort_device_priority(gconstpointer a, gconstpointer b)
{
const stonith_device_t *dev_a = a;
const stonith_device_t *dev_b = b;
if (dev_a->priority > dev_b->priority) {
return -1;
} else if (dev_a->priority < dev_b->priority) {
return 1;
}
return 0;
}
static void
stonith_fence_get_devices_cb(GList * devices, void *user_data)
{
async_command_t *cmd = user_data;
stonith_device_t *device = NULL;
guint ndevices = g_list_length(devices);
crm_info("Found %d matching device%s for target '%s'",
ndevices, pcmk__plural_s(ndevices), cmd->target);
if (devices != NULL) {
/* Order based on priority */
devices = g_list_sort(devices, sort_device_priority);
device = g_hash_table_lookup(device_list, devices->data);
}
if (device == NULL) { // No device found
pcmk__action_result_t result = PCMK__UNKNOWN_RESULT;
pcmk__format_result(&result, CRM_EX_ERROR, PCMK_EXEC_NO_FENCE_DEVICE,
"No device configured for target '%s'",
cmd->target);
send_async_reply(cmd, &result, 0, false);
pcmk__reset_result(&result);
free_async_command(cmd);
g_list_free_full(devices, free);
} else { // Device found, schedule it for fencing
cmd->device_list = devices;
cmd->next_device_iter = devices->next;
schedule_stonith_command(cmd, device);
}
}
/*!
* \internal
* \brief Execute a fence action via the local node
*
* \param[in] msg Fencing request
* \param[out] result Where to store result of fence action
*/
static void
fence_locally(xmlNode *msg, pcmk__action_result_t *result)
{
const char *device_id = NULL;
stonith_device_t *device = NULL;
async_command_t *cmd = NULL;
xmlNode *dev = NULL;
CRM_CHECK((msg != NULL) && (result != NULL), return);
dev = get_xpath_object("//@" F_STONITH_TARGET, msg, LOG_ERR);
cmd = create_async_command(msg);
if (cmd == NULL) {
crm_log_xml_warn(msg, "invalid");
fenced_set_protocol_error(result);
return;
}
device_id = crm_element_value(dev, F_STONITH_DEVICE);
if (device_id != NULL) {
device = g_hash_table_lookup(device_list, device_id);
if (device == NULL) {
crm_err("Requested device '%s' is not available", device_id);
pcmk__format_result(result, CRM_EX_ERROR, PCMK_EXEC_NO_FENCE_DEVICE,
"Requested device '%s' not found", device_id);
return;
}
schedule_stonith_command(cmd, device);
} else {
const char *host = crm_element_value(dev, F_STONITH_TARGET);
if (pcmk_is_set(cmd->options, st_opt_cs_nodeid)) {
int nodeid = 0;
crm_node_t *node = NULL;
pcmk__scan_min_int(host, &nodeid, 0);
node = pcmk__search_known_node_cache(nodeid, NULL, CRM_GET_PEER_ANY);
if (node != NULL) {
host = node->uname;
}
}
/* If we get to here, then self-fencing is implicitly allowed */
get_capable_devices(host, cmd->action, cmd->default_timeout,
TRUE, cmd, stonith_fence_get_devices_cb,
pcmk__str_eq(cmd->action, "on", pcmk__str_casei)? st_device_supports_on: st_device_supports_none);
}
pcmk__set_result(result, CRM_EX_OK, PCMK_EXEC_PENDING, NULL);
}
/*!
* \internal
* \brief Build an XML reply for a fencing operation
*
* \param[in] request Request that reply is for
* \param[in] data If not NULL, add to reply as call data
* \param[in] result Full result of fencing operation
*
* \return Newly created XML reply
* \note The caller is responsible for freeing the result.
* \note This has some overlap with construct_async_reply(), but that copies
* values from an async_command_t, whereas this one copies them from the
* request.
*/
xmlNode *
fenced_construct_reply(const xmlNode *request, xmlNode *data,
const pcmk__action_result_t *result)
{
xmlNode *reply = NULL;
reply = create_xml_node(NULL, T_STONITH_REPLY);
crm_xml_add(reply, "st_origin", __func__);
crm_xml_add(reply, F_TYPE, T_STONITH_NG);
stonith__xe_set_result(reply, result);
if (request == NULL) {
/* Most likely, this is the result of a stonith operation that was
* initiated before we came up. Unfortunately that means we lack enough
* information to provide clients with a full result.
*
* @TODO Maybe synchronize this information at start-up?
*/
crm_warn("Missing request information for client notifications for "
"operation with result '%s' (initiated before we came up?)",
pcmk_exec_status_str(result->execution_status));
} else {
const char *name = NULL;
const char *value = NULL;
// Attributes to copy from request to reply
const char *names[] = {
F_STONITH_OPERATION,
F_STONITH_CALLID,
F_STONITH_CLIENTID,
F_STONITH_CLIENTNAME,
F_STONITH_REMOTE_OP_ID,
F_STONITH_CALLOPTS
};
for (int lpc = 0; lpc < PCMK__NELEM(names); lpc++) {
name = names[lpc];
value = crm_element_value(request, name);
crm_xml_add(reply, name, value);
}
if (data != NULL) {
add_message_xml(reply, F_STONITH_CALLDATA, data);
}
}
return reply;
}
/*!
* \internal
* \brief Build an XML reply to an asynchronous fencing command
*
* \param[in] cmd Fencing command that reply is for
* \param[in] result Command result
*/
static xmlNode *
construct_async_reply(const async_command_t *cmd,
const pcmk__action_result_t *result)
{
xmlNode *reply = create_xml_node(NULL, T_STONITH_REPLY);
crm_xml_add(reply, "st_origin", __func__);
crm_xml_add(reply, F_TYPE, T_STONITH_NG);
crm_xml_add(reply, F_STONITH_OPERATION, cmd->op);
crm_xml_add(reply, F_STONITH_DEVICE, cmd->device);
crm_xml_add(reply, F_STONITH_REMOTE_OP_ID, cmd->remote_op_id);
crm_xml_add(reply, F_STONITH_CLIENTID, cmd->client);
crm_xml_add(reply, F_STONITH_CLIENTNAME, cmd->client_name);
crm_xml_add(reply, F_STONITH_TARGET, cmd->target);
crm_xml_add(reply, F_STONITH_ACTION, cmd->op);
crm_xml_add(reply, F_STONITH_ORIGIN, cmd->origin);
crm_xml_add_int(reply, F_STONITH_CALLID, cmd->id);
crm_xml_add_int(reply, F_STONITH_CALLOPTS, cmd->options);
stonith__xe_set_result(reply, result);
return reply;
}
bool fencing_peer_active(crm_node_t *peer)
{
if (peer == NULL) {
return FALSE;
} else if (peer->uname == NULL) {
return FALSE;
} else if (pcmk_is_set(peer->processes, crm_get_cluster_proc())) {
return TRUE;
}
return FALSE;
}
void
set_fencing_completed(remote_fencing_op_t *op)
{
struct timespec tv;
qb_util_timespec_from_epoch_get(&tv);
op->completed = tv.tv_sec;
op->completed_nsec = tv.tv_nsec;
}
/*!
* \internal
* \brief Look for alternate node needed if local node shouldn't fence target
*
* \param[in] target Node that must be fenced
*
* \return Name of an alternate node that should fence \p target if any,
* or NULL otherwise
*/
static const char *
check_alternate_host(const char *target)
{
if (pcmk__str_eq(target, stonith_our_uname, pcmk__str_casei)) {
GHashTableIter gIter;
crm_node_t *entry = NULL;
g_hash_table_iter_init(&gIter, crm_peer_cache);
while (g_hash_table_iter_next(&gIter, NULL, (void **)&entry)) {
if (fencing_peer_active(entry)
&& !pcmk__str_eq(entry->uname, target, pcmk__str_casei)) {
crm_notice("Forwarding self-fencing request to %s",
entry->uname);
return entry->uname;
}
}
crm_warn("Will handle own fencing because no peer can");
}
return NULL;
}
/*!
* \internal
* \brief Send a reply to a CPG peer or IPC client
*
* \param[in] reply XML reply to send
* \param[in] call_options Send synchronously if st_opt_sync_call is set
* \param[in] remote_peer If not NULL, name of peer node to send CPG reply
* \param[in,out] client If not NULL, client to send IPC reply
*/
static void
stonith_send_reply(xmlNode *reply, int call_options, const char *remote_peer,
pcmk__client_t *client)
{
CRM_CHECK((reply != NULL) && ((remote_peer != NULL) || (client != NULL)),
return);
if (remote_peer == NULL) {
do_local_reply(reply, client, call_options);
} else {
send_cluster_message(crm_get_peer(0, remote_peer), crm_msg_stonith_ng,
reply, FALSE);
}
}
static void
remove_relay_op(xmlNode * request)
{
xmlNode *dev = get_xpath_object("//@" F_STONITH_ACTION, request, LOG_TRACE);
const char *relay_op_id = NULL;
const char *op_id = NULL;
const char *client_name = NULL;
const char *target = NULL;
remote_fencing_op_t *relay_op = NULL;
if (dev) {
target = crm_element_value(dev, F_STONITH_TARGET);
}
relay_op_id = crm_element_value(request, F_STONITH_REMOTE_OP_ID_RELAY);
op_id = crm_element_value(request, F_STONITH_REMOTE_OP_ID);
client_name = crm_element_value(request, F_STONITH_CLIENTNAME);
/* Delete RELAY operation. */
if (relay_op_id && target && pcmk__str_eq(target, stonith_our_uname, pcmk__str_casei)) {
relay_op = g_hash_table_lookup(stonith_remote_op_list, relay_op_id);
if (relay_op) {
GHashTableIter iter;
remote_fencing_op_t *list_op = NULL;
g_hash_table_iter_init(&iter, stonith_remote_op_list);
/* If the operation to be deleted is registered as a duplicate, delete the registration. */
while (g_hash_table_iter_next(&iter, NULL, (void **)&list_op)) {
GList *dup_iter = NULL;
if (list_op != relay_op) {
for (dup_iter = list_op->duplicates; dup_iter != NULL; dup_iter = dup_iter->next) {
remote_fencing_op_t *other = dup_iter->data;
if (other == relay_op) {
other->duplicates = g_list_remove(other->duplicates, relay_op);
break;
}
}
}
}
crm_debug("Deleting relay op %s ('%s'%s%s for %s), "
"replaced by op %s ('%s'%s%s for %s)",
relay_op->id, relay_op->action,
(relay_op->target == NULL)? "" : " targeting ",
pcmk__s(relay_op->target, ""),
relay_op->client_name, op_id, relay_op->action,
(target == NULL)? "" : " targeting ", pcmk__s(target, ""),
client_name);
g_hash_table_remove(stonith_remote_op_list, relay_op_id);
}
}
}
/*!
* \internal
* \brief Check whether an API request was sent by a privileged user
*
* API commands related to fencing configuration may be done only by privileged
* IPC users (i.e. root or hacluster), because all other users should go through
* the CIB to have ACLs applied. If no client was given, this is a peer request,
* which is always allowed.
*
* \param[in] c IPC client that sent request (or NULL if sent by CPG peer)
* \param[in] op Requested API operation (for logging only)
*
* \return true if sender is peer or privileged client, otherwise false
*/
static inline bool
is_privileged(const pcmk__client_t *c, const char *op)
{
if ((c == NULL) || pcmk_is_set(c->flags, pcmk__client_privileged)) {
return true;
} else {
crm_warn("Rejecting IPC request '%s' from unprivileged client %s",
pcmk__s(op, ""), pcmk__client_name(c));
return false;
}
}
// CRM_OP_REGISTER
static xmlNode *
handle_register_request(pcmk__request_t *request)
{
xmlNode *reply = create_xml_node(NULL, "reply");
CRM_ASSERT(request->ipc_client != NULL);
crm_xml_add(reply, F_STONITH_OPERATION, CRM_OP_REGISTER);
crm_xml_add(reply, F_STONITH_CLIENTID, request->ipc_client->id);
pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
pcmk__set_request_flags(request, pcmk__request_reuse_options);
return reply;
}
// STONITH_OP_EXEC
static xmlNode *
handle_agent_request(pcmk__request_t *request)
{
execute_agent_action(request->xml, &request->result);
if (request->result.execution_status == PCMK_EXEC_PENDING) {
return NULL;
}
return fenced_construct_reply(request->xml, NULL, &request->result);
}
// STONITH_OP_TIMEOUT_UPDATE
static xmlNode *
handle_update_timeout_request(pcmk__request_t *request)
{
const char *call_id = crm_element_value(request->xml, F_STONITH_CALLID);
const char *client_id = crm_element_value(request->xml, F_STONITH_CLIENTID);
int op_timeout = 0;
crm_element_value_int(request->xml, F_STONITH_TIMEOUT, &op_timeout);
do_stonith_async_timeout_update(client_id, call_id, op_timeout);
pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
return NULL;
}
// STONITH_OP_QUERY
static xmlNode *
handle_query_request(pcmk__request_t *request)
{
int timeout = 0;
xmlNode *dev = NULL;
const char *action = NULL;
const char *target = NULL;
const char *client_id = crm_element_value(request->xml, F_STONITH_CLIENTID);
struct st_query_data *query = NULL;
if (request->peer != NULL) {
// Record it for the future notification
create_remote_stonith_op(client_id, request->xml, TRUE);
}
/* Delete the DC node RELAY operation. */
remove_relay_op(request->xml);
pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
dev = get_xpath_object("//@" F_STONITH_ACTION, request->xml, LOG_NEVER);
if (dev != NULL) {
const char *device = crm_element_value(dev, F_STONITH_DEVICE);
if (pcmk__str_eq(device, "manual_ack", pcmk__str_casei)) {
return NULL; // No query or reply necessary
}
target = crm_element_value(dev, F_STONITH_TARGET);
action = crm_element_value(dev, F_STONITH_ACTION);
}
crm_log_xml_trace(request->xml, "Query");
query = calloc(1, sizeof(struct st_query_data));
CRM_ASSERT(query != NULL);
query->reply = fenced_construct_reply(request->xml, NULL, &request->result);
pcmk__str_update(&query->remote_peer, request->peer);
pcmk__str_update(&query->client_id, client_id);
pcmk__str_update(&query->target, target);
pcmk__str_update(&query->action, action);
query->call_options = request->call_options;
crm_element_value_int(request->xml, F_STONITH_TIMEOUT, &timeout);
get_capable_devices(target, action, timeout,
pcmk_is_set(query->call_options, st_opt_allow_suicide),
query, stonith_query_capable_device_cb, st_device_supports_none);
return NULL;
}
// T_STONITH_NOTIFY
static xmlNode *
handle_notify_request(pcmk__request_t *request)
{
const char *flag_name = NULL;
CRM_ASSERT(request->ipc_client != NULL);
flag_name = crm_element_value(request->xml, F_STONITH_NOTIFY_ACTIVATE);
if (flag_name != NULL) {
crm_debug("Enabling %s callbacks for client %s",
flag_name, pcmk__request_origin(request));
pcmk__set_client_flags(request->ipc_client, get_stonith_flag(flag_name));
}
flag_name = crm_element_value(request->xml, F_STONITH_NOTIFY_DEACTIVATE);
if (flag_name != NULL) {
crm_debug("Disabling %s callbacks for client %s",
flag_name, pcmk__request_origin(request));
pcmk__clear_client_flags(request->ipc_client,
get_stonith_flag(flag_name));
}
pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
pcmk__set_request_flags(request, pcmk__request_reuse_options);
return pcmk__ipc_create_ack(request->ipc_flags, "ack", NULL, CRM_EX_OK);
}
// STONITH_OP_RELAY
static xmlNode *
handle_relay_request(pcmk__request_t *request)
{
xmlNode *dev = get_xpath_object("//@" F_STONITH_TARGET, request->xml,
LOG_TRACE);
crm_notice("Received forwarded fencing request from "
"%s %s to fence (%s) peer %s",
pcmk__request_origin_type(request),
pcmk__request_origin(request),
crm_element_value(dev, F_STONITH_ACTION),
crm_element_value(dev, F_STONITH_TARGET));
if (initiate_remote_stonith_op(NULL, request->xml, FALSE) == NULL) {
fenced_set_protocol_error(&request->result);
return fenced_construct_reply(request->xml, NULL, &request->result);
}
pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_PENDING, NULL);
return NULL;
}
// STONITH_OP_FENCE
static xmlNode *
handle_fence_request(pcmk__request_t *request)
{
if ((request->peer != NULL) || stand_alone) {
fence_locally(request->xml, &request->result);
} else if (pcmk_is_set(request->call_options, st_opt_manual_ack)) {
switch (fenced_handle_manual_confirmation(request->ipc_client,
request->xml)) {
case pcmk_rc_ok:
pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE,
NULL);
break;
case EINPROGRESS:
pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_PENDING,
NULL);
break;
default:
fenced_set_protocol_error(&request->result);
break;
}
} else {
const char *alternate_host = NULL;
xmlNode *dev = get_xpath_object("//@" F_STONITH_TARGET, request->xml,
LOG_TRACE);
const char *target = crm_element_value(dev, F_STONITH_TARGET);
const char *action = crm_element_value(dev, F_STONITH_ACTION);
const char *device = crm_element_value(dev, F_STONITH_DEVICE);
if (request->ipc_client != NULL) {
int tolerance = 0;
crm_notice("Client %s wants to fence (%s) %s using %s",
pcmk__request_origin(request), action,
target, (device? device : "any device"));
crm_element_value_int(dev, F_STONITH_TOLERANCE, &tolerance);
if (stonith_check_fence_tolerance(tolerance, target, action)) {
pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE,
NULL);
return fenced_construct_reply(request->xml, NULL,
&request->result);
}
alternate_host = check_alternate_host(target);
} else {
crm_notice("Peer %s wants to fence (%s) '%s' with device '%s'",
request->peer, action, target,
(device == NULL)? "(any)" : device);
}
if (alternate_host != NULL) {
const char *client_id = NULL;
remote_fencing_op_t *op = NULL;
if (request->ipc_client->id == 0) {
client_id = crm_element_value(request->xml, F_STONITH_CLIENTID);
} else {
client_id = request->ipc_client->id;
}
/* Create a duplicate fencing operation to relay with the client ID.
* When a query response is received, this operation should be
* deleted to avoid keeping the duplicate around.
*/
op = create_remote_stonith_op(client_id, request->xml, FALSE);
crm_xml_add(request->xml, F_STONITH_OPERATION, STONITH_OP_RELAY);
crm_xml_add(request->xml, F_STONITH_CLIENTID,
request->ipc_client->id);
crm_xml_add(request->xml, F_STONITH_REMOTE_OP_ID, op->id);
send_cluster_message(crm_get_peer(0, alternate_host),
crm_msg_stonith_ng, request->xml, FALSE);
pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_PENDING,
NULL);
} else if (initiate_remote_stonith_op(request->ipc_client, request->xml,
FALSE) == NULL) {
fenced_set_protocol_error(&request->result);
} else {
pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_PENDING,
NULL);
}
}
if (request->result.execution_status == PCMK_EXEC_PENDING) {
return NULL;
}
return fenced_construct_reply(request->xml, NULL, &request->result);
}
// STONITH_OP_FENCE_HISTORY
static xmlNode *
handle_history_request(pcmk__request_t *request)
{
xmlNode *reply = NULL;
xmlNode *data = NULL;
stonith_fence_history(request->xml, &data, request->peer,
request->call_options);
pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
if (!pcmk_is_set(request->call_options, st_opt_discard_reply)) {
/* When the local node broadcasts its history, it sets
* st_opt_discard_reply and doesn't need a reply.
*/
reply = fenced_construct_reply(request->xml, data, &request->result);
}
free_xml(data);
return reply;
}
// STONITH_OP_DEVICE_ADD
static xmlNode *
handle_device_add_request(pcmk__request_t *request)
{
const char *op = crm_element_value(request->xml, F_STONITH_OPERATION);
xmlNode *dev = get_xpath_object("//" F_STONITH_DEVICE, request->xml,
LOG_ERR);
if (is_privileged(request->ipc_client, op)) {
int rc = stonith_device_register(dev, FALSE);
pcmk__set_result(&request->result,
((rc == pcmk_ok)? CRM_EX_OK : CRM_EX_ERROR),
stonith__legacy2status(rc),
((rc == pcmk_ok)? NULL : pcmk_strerror(rc)));
} else {
pcmk__set_result(&request->result, CRM_EX_INSUFFICIENT_PRIV,
PCMK_EXEC_INVALID,
"Unprivileged users must register device via CIB");
}
fenced_send_device_notification(op, &request->result,
(dev == NULL)? NULL : ID(dev));
return fenced_construct_reply(request->xml, NULL, &request->result);
}
// STONITH_OP_DEVICE_DEL
static xmlNode *
handle_device_delete_request(pcmk__request_t *request)
{
xmlNode *dev = get_xpath_object("//" F_STONITH_DEVICE, request->xml,
LOG_ERR);
const char *device_id = crm_element_value(dev, XML_ATTR_ID);
const char *op = crm_element_value(request->xml, F_STONITH_OPERATION);
if (is_privileged(request->ipc_client, op)) {
stonith_device_remove(device_id, false);
pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
} else {
pcmk__set_result(&request->result, CRM_EX_INSUFFICIENT_PRIV,
PCMK_EXEC_INVALID,
"Unprivileged users must delete device via CIB");
}
fenced_send_device_notification(op, &request->result, device_id);
return fenced_construct_reply(request->xml, NULL, &request->result);
}
// STONITH_OP_LEVEL_ADD
static xmlNode *
handle_level_add_request(pcmk__request_t *request)
{
char *desc = NULL;
const char *op = crm_element_value(request->xml, F_STONITH_OPERATION);
if (is_privileged(request->ipc_client, op)) {
fenced_register_level(request->xml, &desc, &request->result);
} else {
unpack_level_request(request->xml, NULL, NULL, NULL, &desc);
pcmk__set_result(&request->result, CRM_EX_INSUFFICIENT_PRIV,
PCMK_EXEC_INVALID,
"Unprivileged users must add level via CIB");
}
fenced_send_level_notification(op, &request->result, desc);
free(desc);
return fenced_construct_reply(request->xml, NULL, &request->result);
}
// STONITH_OP_LEVEL_DEL
static xmlNode *
handle_level_delete_request(pcmk__request_t *request)
{
char *desc = NULL;
const char *op = crm_element_value(request->xml, F_STONITH_OPERATION);
if (is_privileged(request->ipc_client, op)) {
fenced_unregister_level(request->xml, &desc, &request->result);
} else {
unpack_level_request(request->xml, NULL, NULL, NULL, &desc);
pcmk__set_result(&request->result, CRM_EX_INSUFFICIENT_PRIV,
PCMK_EXEC_INVALID,
"Unprivileged users must delete level via CIB");
}
fenced_send_level_notification(op, &request->result, desc);
free(desc);
return fenced_construct_reply(request->xml, NULL, &request->result);
}
// CRM_OP_RM_NODE_CACHE
static xmlNode *
handle_cache_request(pcmk__request_t *request)
{
int node_id = 0;
const char *name = NULL;
crm_element_value_int(request->xml, XML_ATTR_ID, &node_id);
name = crm_element_value(request->xml, XML_ATTR_UNAME);
reap_crm_member(node_id, name);
pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
return NULL;
}
static xmlNode *
handle_unknown_request(pcmk__request_t *request)
{
crm_err("Unknown IPC request %s from %s %s",
request->op, pcmk__request_origin_type(request),
pcmk__request_origin(request));
pcmk__format_result(&request->result, CRM_EX_PROTOCOL, PCMK_EXEC_INVALID,
"Unknown IPC request type '%s' (bug?)", request->op);
return fenced_construct_reply(request->xml, NULL, &request->result);
}
static void
fenced_register_handlers(void)
{
pcmk__server_command_t handlers[] = {
{ CRM_OP_REGISTER, handle_register_request },
{ STONITH_OP_EXEC, handle_agent_request },
{ STONITH_OP_TIMEOUT_UPDATE, handle_update_timeout_request },
{ STONITH_OP_QUERY, handle_query_request },
{ T_STONITH_NOTIFY, handle_notify_request },
{ STONITH_OP_RELAY, handle_relay_request },
{ STONITH_OP_FENCE, handle_fence_request },
{ STONITH_OP_FENCE_HISTORY, handle_history_request },
{ STONITH_OP_DEVICE_ADD, handle_device_add_request },
{ STONITH_OP_DEVICE_DEL, handle_device_delete_request },
{ STONITH_OP_LEVEL_ADD, handle_level_add_request },
{ STONITH_OP_LEVEL_DEL, handle_level_delete_request },
{ CRM_OP_RM_NODE_CACHE, handle_cache_request },
{ NULL, handle_unknown_request },
};
fenced_handlers = pcmk__register_handlers(handlers);
}
void
fenced_unregister_handlers(void)
{
if (fenced_handlers != NULL) {
g_hash_table_destroy(fenced_handlers);
fenced_handlers = NULL;
}
}
static void
handle_request(pcmk__request_t *request)
{
xmlNode *reply = NULL;
const char *reason = NULL;
if (fenced_handlers == NULL) {
fenced_register_handlers();
}
reply = pcmk__process_request(request, fenced_handlers);
if (reply != NULL) {
if (pcmk_is_set(request->flags, pcmk__request_reuse_options)
&& (request->ipc_client != NULL)) {
/* Certain IPC-only commands must reuse the call options from the
* original request rather than the ones set by stonith_send_reply()
* -> do_local_reply().
*/
pcmk__ipc_send_xml(request->ipc_client, request->ipc_id, reply,
request->ipc_flags);
request->ipc_client->request_id = 0;
} else {
stonith_send_reply(reply, request->call_options,
request->peer, request->ipc_client);
}
free_xml(reply);
}
reason = request->result.exit_reason;
crm_debug("Processed %s request from %s %s: %s%s%s%s",
request->op, pcmk__request_origin_type(request),
pcmk__request_origin(request),
pcmk_exec_status_str(request->result.execution_status),
(reason == NULL)? "" : " (",
(reason == NULL)? "" : reason,
(reason == NULL)? "" : ")");
}
static void
handle_reply(pcmk__client_t *client, xmlNode *request, const char *remote_peer)
{
// Copy, because request might be freed before we want to log this
char *op = crm_element_value_copy(request, F_STONITH_OPERATION);
if (pcmk__str_eq(op, STONITH_OP_QUERY, pcmk__str_none)) {
process_remote_stonith_query(request);
} else if (pcmk__str_any_of(op, T_STONITH_NOTIFY, STONITH_OP_FENCE, NULL)) {
fenced_process_fencing_reply(request);
} else {
crm_err("Ignoring unknown %s reply from %s %s",
pcmk__s(op, "untyped"), ((client == NULL)? "peer" : "client"),
((client == NULL)? remote_peer : pcmk__client_name(client)));
crm_log_xml_warn(request, "UnknownOp");
free(op);
return;
}
crm_debug("Processed %s reply from %s %s",
op, ((client == NULL)? "peer" : "client"),
((client == NULL)? remote_peer : pcmk__client_name(client)));
free(op);
}
/*!
* \internal
* \brief Handle a message from an IPC client or CPG peer
*
* \param[in,out] client If not NULL, IPC client that sent message
* \param[in] id If from IPC client, IPC message ID
* \param[in] flags Message flags
* \param[in,out] message Message XML
* \param[in] remote_peer If not NULL, CPG peer that sent message
*/
void
stonith_command(pcmk__client_t *client, uint32_t id, uint32_t flags,
xmlNode *message, const char *remote_peer)
{
int call_options = st_opt_none;
bool is_reply = false;
CRM_CHECK(message != NULL, return);
if (get_xpath_object("//" T_STONITH_REPLY, message, LOG_NEVER) != NULL) {
is_reply = true;
}
crm_element_value_int(message, F_STONITH_CALLOPTS, &call_options);
crm_debug("Processing %ssynchronous %s %s %u from %s %s",
pcmk_is_set(call_options, st_opt_sync_call)? "" : "a",
crm_element_value(message, F_STONITH_OPERATION),
(is_reply? "reply" : "request"), id,
((client == NULL)? "peer" : "client"),
((client == NULL)? remote_peer : pcmk__client_name(client)));
if (pcmk_is_set(call_options, st_opt_sync_call)) {
CRM_ASSERT(client == NULL || client->request_id == id);
}
if (is_reply) {
handle_reply(client, message, remote_peer);
} else {
pcmk__request_t request = {
.ipc_client = client,
.ipc_id = id,
.ipc_flags = flags,
.peer = remote_peer,
.xml = message,
.call_options = call_options,
.result = PCMK__UNKNOWN_RESULT,
};
request.op = crm_element_value_copy(request.xml, F_STONITH_OPERATION);
CRM_CHECK(request.op != NULL, return);
if (pcmk_is_set(request.call_options, st_opt_sync_call)) {
pcmk__set_request_flags(&request, pcmk__request_sync);
}
handle_request(&request);
pcmk__reset_request(&request);
}
}
diff --git a/include/crm/crm.h b/include/crm/crm.h
index 087541616c..d7dc3c8455 100644
--- a/include/crm/crm.h
+++ b/include/crm/crm.h
@@ -1,240 +1,239 @@
/*
* Copyright 2004-2022 the Pacemaker project contributors
*
* The version control history for this file may have further details.
*
* This source code is licensed under the GNU Lesser General Public License
* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
*/
#ifndef PCMK__CRM_CRM__H
# define PCMK__CRM_CRM__H
# include <crm_config.h>
# include <stdlib.h>
# include <glib.h>
# include <stdbool.h>
# include <string.h>
# include <libxml/tree.h>
#ifdef __cplusplus
extern "C" {
#endif
/**
* \file
* \brief A dumping ground
* \ingroup core
*/
#ifndef PCMK_ALLOW_DEPRECATED
/*!
* \brief Allow use of deprecated Pacemaker APIs
*
* By default, external code using Pacemaker headers is allowed to use
* deprecated Pacemaker APIs. If PCMK_ALLOW_DEPRECATED is defined to 0 before
* including any Pacemaker headers, deprecated APIs will be unusable. It is
* strongly recommended to leave this unchanged for production and release
* builds, to avoid breakage when users upgrade to new Pacemaker releases that
* deprecate more APIs. This should be defined to 0 only for development and
* testing builds when desiring to check for usage of currently deprecated APIs.
*/
#define PCMK_ALLOW_DEPRECATED 1
#endif
/*!
* The CRM feature set assists with compatibility in mixed-version clusters.
* The major version number increases when nodes with different versions
* would not work (rolling upgrades are not allowed). The minor version
* number increases when mixed-version clusters are allowed only during
* rolling upgrades (a node with the oldest feature set will be elected DC). The
* minor-minor version number is ignored, but allows resource agents to detect
* cluster support for various features.
*
* The feature set also affects the processing of old saved CIBs (such as for
* many scheduler regression tests).
*
* Particular feature points currently tested by Pacemaker code:
*
* >2.1: Operation updates include timing data
* >=3.0.5: XML v2 digests are created
* >=3.0.8: Peers do not need acks for cancellations
* >=3.0.9: DC will send its own shutdown request to all peers
* XML v2 patchsets are created by default
* >=3.0.13: Fail counts include operation name and interval
* >=3.2.0: DC supports PCMK_EXEC_INVALID and PCMK_EXEC_NOT_CONNECTED
*/
# define CRM_FEATURE_SET "3.16.1"
/* Pacemaker's CPG protocols use fixed-width binary fields for the sender and
* recipient of a CPG message. This imposes an arbitrary limit on cluster node
* names.
*/
//! \brief Maximum length of a Corosync cluster node name (in bytes)
#define MAX_NAME 256
# define CRM_META "CRM_meta"
extern char *crm_system_name;
/* *INDENT-OFF* */
// How we represent "infinite" scores
# define CRM_SCORE_INFINITY 1000000
# define CRM_INFINITY_S "INFINITY"
# define CRM_PLUS_INFINITY_S "+" CRM_INFINITY_S
# define CRM_MINUS_INFINITY_S "-" CRM_INFINITY_S
/* @COMPAT API < 2.0.0 Deprecated "infinity" aliases
*
* INFINITY might be defined elsewhere (e.g. math.h), so undefine it first.
* This, of course, complicates any attempt to use the other definition in any
* code that includes this header.
*/
# undef INFINITY
# define INFINITY_S "INFINITY"
# define MINUS_INFINITY_S "-INFINITY"
# define INFINITY 1000000
/* Sub-systems */
# define CRM_SYSTEM_DC "dc"
#define CRM_SYSTEM_DCIB "dcib" // Primary instance of CIB manager
# define CRM_SYSTEM_CIB "cib"
# define CRM_SYSTEM_CRMD "crmd"
# define CRM_SYSTEM_LRMD "lrmd"
# define CRM_SYSTEM_PENGINE "pengine"
# define CRM_SYSTEM_TENGINE "tengine"
# define CRM_SYSTEM_STONITHD "stonithd"
# define CRM_SYSTEM_MCP "pacemakerd"
// Names of internally generated node attributes
# define CRM_ATTR_UNAME "#uname"
# define CRM_ATTR_ID "#id"
# define CRM_ATTR_KIND "#kind"
# define CRM_ATTR_ROLE "#role"
# define CRM_ATTR_IS_DC "#is_dc"
# define CRM_ATTR_CLUSTER_NAME "#cluster-name"
# define CRM_ATTR_SITE_NAME "#site-name"
# define CRM_ATTR_UNFENCED "#node-unfenced"
# define CRM_ATTR_DIGESTS_ALL "#digests-all"
# define CRM_ATTR_DIGESTS_SECURE "#digests-secure"
# define CRM_ATTR_RA_VERSION "#ra-version"
# define CRM_ATTR_PROTOCOL "#attrd-protocol"
# define CRM_ATTR_FEATURE_SET "#feature-set"
/* Valid operations */
# define CRM_OP_NOOP "noop"
# define CRM_OP_JOIN_ANNOUNCE "join_announce"
# define CRM_OP_JOIN_OFFER "join_offer"
# define CRM_OP_JOIN_REQUEST "join_request"
# define CRM_OP_JOIN_ACKNAK "join_ack_nack"
# define CRM_OP_JOIN_CONFIRM "join_confirm"
# define CRM_OP_PING "ping"
# define CRM_OP_NODE_INFO "node-info"
# define CRM_OP_THROTTLE "throttle"
# define CRM_OP_VOTE "vote"
# define CRM_OP_NOVOTE "no-vote"
# define CRM_OP_HELLO "hello"
# define CRM_OP_PECALC "pe_calc"
# define CRM_OP_QUIT "quit"
# define CRM_OP_LOCAL_SHUTDOWN "start_shutdown"
# define CRM_OP_SHUTDOWN_REQ "req_shutdown"
# define CRM_OP_SHUTDOWN "do_shutdown"
# define CRM_OP_FENCE "stonith"
# define CRM_OP_REGISTER "register"
# define CRM_OP_IPC_FWD "ipc_fwd"
# define CRM_OP_INVOKE_LRM "lrm_invoke"
-# define CRM_OP_LRM_REFRESH "lrm_refresh" /* Deprecated */
-# define CRM_OP_LRM_QUERY "lrm_query"
+# define CRM_OP_LRM_REFRESH "lrm_refresh" //!< Deprecated since 1.1.10
# define CRM_OP_LRM_DELETE "lrm_delete"
# define CRM_OP_LRM_FAIL "lrm_fail"
# define CRM_OP_PROBED "probe_complete"
# define CRM_OP_REPROBE "probe_again"
# define CRM_OP_CLEAR_FAILCOUNT "clear_failcount"
# define CRM_OP_REMOTE_STATE "remote_state"
# define CRM_OP_RELAXED_SET "one-or-more"
# define CRM_OP_RELAXED_CLONE "clone-one-or-more"
# define CRM_OP_RM_NODE_CACHE "rm_node_cache"
# define CRM_OP_MAINTENANCE_NODES "maintenance_nodes"
/* Possible cluster membership states */
# define CRMD_JOINSTATE_DOWN "down"
# define CRMD_JOINSTATE_PENDING "pending"
# define CRMD_JOINSTATE_MEMBER "member"
# define CRMD_JOINSTATE_NACK "banned"
# define CRMD_ACTION_DELETE "delete"
# define CRMD_ACTION_CANCEL "cancel"
# define CRMD_ACTION_RELOAD "reload"
# define CRMD_ACTION_RELOAD_AGENT "reload-agent"
# define CRMD_ACTION_MIGRATE "migrate_to"
# define CRMD_ACTION_MIGRATED "migrate_from"
# define CRMD_ACTION_START "start"
# define CRMD_ACTION_STARTED "running"
# define CRMD_ACTION_STOP "stop"
# define CRMD_ACTION_STOPPED "stopped"
# define CRMD_ACTION_PROMOTE "promote"
# define CRMD_ACTION_PROMOTED "promoted"
# define CRMD_ACTION_DEMOTE "demote"
# define CRMD_ACTION_DEMOTED "demoted"
# define CRMD_ACTION_NOTIFY "notify"
# define CRMD_ACTION_NOTIFIED "notified"
# define CRMD_ACTION_STATUS "monitor"
# define CRMD_ACTION_METADATA "meta-data"
# define CRMD_METADATA_CALL_TIMEOUT 30000
/* short names */
# define RSC_DELETE CRMD_ACTION_DELETE
# define RSC_CANCEL CRMD_ACTION_CANCEL
# define RSC_MIGRATE CRMD_ACTION_MIGRATE
# define RSC_MIGRATED CRMD_ACTION_MIGRATED
# define RSC_START CRMD_ACTION_START
# define RSC_STARTED CRMD_ACTION_STARTED
# define RSC_STOP CRMD_ACTION_STOP
# define RSC_STOPPED CRMD_ACTION_STOPPED
# define RSC_PROMOTE CRMD_ACTION_PROMOTE
# define RSC_PROMOTED CRMD_ACTION_PROMOTED
# define RSC_DEMOTE CRMD_ACTION_DEMOTE
# define RSC_DEMOTED CRMD_ACTION_DEMOTED
# define RSC_NOTIFY CRMD_ACTION_NOTIFY
# define RSC_NOTIFIED CRMD_ACTION_NOTIFIED
# define RSC_STATUS CRMD_ACTION_STATUS
# define RSC_METADATA CRMD_ACTION_METADATA
/* *INDENT-ON* */
# include <crm/common/cib.h>
# include <crm/common/logging.h>
# include <crm/common/util.h>
static inline const char *
crm_action_str(const char *task, guint interval_ms) {
if ((task != NULL) && (interval_ms == 0)
&& (strcasecmp(task, RSC_STATUS) == 0)) {
return "probe";
}
return task;
}
#if !defined(PCMK_ALLOW_DEPRECATED) || (PCMK_ALLOW_DEPRECATED == 1)
#include <crm/crm_compat.h>
#endif
#ifdef __cplusplus
}
#endif
#endif
diff --git a/include/crm/crm_compat.h b/include/crm/crm_compat.h
index 3b35a5e0cb..8a4b368d52 100644
--- a/include/crm/crm_compat.h
+++ b/include/crm/crm_compat.h
@@ -1,55 +1,58 @@
/*
- * Copyright 2004-2021 the Pacemaker project contributors
+ * Copyright 2004-2022 the Pacemaker project contributors
*
* The version control history for this file may have further details.
*
* This source code is licensed under the GNU Lesser General Public License
* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
*/
#ifndef PCMK__CRM_CRM_COMPAT__H
# define PCMK__CRM_CRM_COMPAT__H
#include <glib.h>
#ifdef __cplusplus
extern "C" {
#endif
/**
* \file
* \brief Deprecated Pacemaker utilities
* \ingroup core
* \deprecated Do not include this header directly. The utilities in this
* header, and the header itself, will be removed in a future
* release.
*/
//! \deprecated Use '\0' instead
#define EOS '\0'
//! \deprecated This defined constant will be removed in a future release
#define MAX_IPC_DELAY 120
+//! \deprecated This defined constant will be removed in a future release
+#define CRM_OP_LRM_QUERY "lrm_query"
+
//!@{
//! \deprecated This macro will be removed in a future release
# ifndef __GNUC__
# define __builtin_expect(expr, result) (expr)
# endif
#define __likely(expr) __builtin_expect(expr, 1)
#define __unlikely(expr) __builtin_expect(expr, 0)
// This ends the doxygen deprecation comment
//!@}
//! \deprecated Use GList * instead
typedef GList *GListPtr;
#ifdef __cplusplus
}
#endif
#endif // PCMK__CRM_CRM_COMPAT__H
diff --git a/include/crm/fencing/internal.h b/include/crm/fencing/internal.h
index c19995b3fb..9d931ae2d3 100644
--- a/include/crm/fencing/internal.h
+++ b/include/crm/fencing/internal.h
@@ -1,218 +1,222 @@
/*
* Copyright 2011-2022 the Pacemaker project contributors
*
* The version control history for this file may have further details.
*
* This source code is licensed under the GNU Lesser General Public License
* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
*/
#ifndef STONITH_NG_INTERNAL__H
# define STONITH_NG_INTERNAL__H
# include <glib.h>
# include <crm/common/ipc.h>
# include <crm/common/xml.h>
# include <crm/common/output_internal.h>
# include <crm/stonith-ng.h>
enum st_device_flags
{
st_device_supports_none = 0x0000,
st_device_supports_list = 0x0001,
st_device_supports_status = 0x0002,
st_device_supports_reboot = 0x0004,
st_device_supports_parameter_plug = 0x0008,
st_device_supports_parameter_port = 0x0010,
st_device_supports_on = 0x0020,
};
#define stonith__set_device_flags(device_flags, device_id, flags_to_set) do { \
device_flags = pcmk__set_flags_as(__func__, __LINE__, LOG_TRACE, \
"Fence device", device_id, \
(device_flags), (flags_to_set), \
#flags_to_set); \
} while (0)
#define stonith__set_call_options(st_call_opts, call_for, flags_to_set) do { \
st_call_opts = pcmk__set_flags_as(__func__, __LINE__, LOG_TRACE, \
"Fencer call", (call_for), \
(st_call_opts), (flags_to_set), \
#flags_to_set); \
} while (0)
#define stonith__clear_call_options(st_call_opts, call_for, flags_to_clear) do { \
st_call_opts = pcmk__clear_flags_as(__func__, __LINE__, LOG_TRACE, \
"Fencer call", (call_for), \
(st_call_opts), (flags_to_clear), \
#flags_to_clear); \
} while (0)
struct stonith_action_s;
typedef struct stonith_action_s stonith_action_t;
stonith_action_t *stonith__action_create(const char *agent,
- const char *_action,
+ const char *action_name,
const char *target,
uint32_t target_nodeid,
- int timeout,
+ int timeout_sec,
GHashTable *device_args,
GHashTable *port_map,
const char *host_arg);
void stonith__destroy_action(stonith_action_t *action);
pcmk__action_result_t *stonith__action_result(stonith_action_t *action);
int stonith__result2rc(const pcmk__action_result_t *result);
void stonith__xe_set_result(xmlNode *xml, const pcmk__action_result_t *result);
void stonith__xe_get_result(xmlNode *xml, pcmk__action_result_t *result);
xmlNode *stonith__find_xe_with_result(xmlNode *xml);
-int
-stonith_action_execute_async(stonith_action_t * action,
- void *userdata,
- void (*done) (int pid,
- const pcmk__action_result_t *result,
- void *user_data),
- void (*fork_cb) (int pid, void *user_data));
+int stonith__execute_async(stonith_action_t *action, void *userdata,
+ void (*done) (int pid,
+ const pcmk__action_result_t *result,
+ void *user_data),
+ void (*fork_cb) (int pid, void *user_data));
+
+int stonith__metadata_async(const char *agent, int timeout_sec,
+ void (*callback)(int pid,
+ const pcmk__action_result_t *result,
+ void *user_data),
+ void *user_data);
xmlNode *create_level_registration_xml(const char *node, const char *pattern,
const char *attr, const char *value,
int level,
stonith_key_value_t *device_list);
xmlNode *create_device_registration_xml(const char *id,
enum stonith_namespace namespace,
const char *agent,
stonith_key_value_t *params,
const char *rsc_provides);
void stonith__register_messages(pcmk__output_t *out);
GList *stonith__parse_targets(const char *hosts);
const char *stonith__later_succeeded(stonith_history_t *event,
stonith_history_t *top_history);
stonith_history_t *stonith__sort_history(stonith_history_t *history);
void stonith__device_parameter_flags(uint32_t *device_flags,
const char *device_name,
xmlNode *metadata);
# define ST_LEVEL_MAX 10
# define F_STONITH_CLIENTID "st_clientid"
# define F_STONITH_CALLOPTS "st_callopt"
# define F_STONITH_CALLID "st_callid"
# define F_STONITH_CALLDATA "st_calldata"
# define F_STONITH_OPERATION "st_op"
# define F_STONITH_TARGET "st_target"
# define F_STONITH_REMOTE_OP_ID "st_remote_op"
# define F_STONITH_REMOTE_OP_ID_RELAY "st_remote_op_relay"
# define F_STONITH_RC "st_rc"
# define F_STONITH_OUTPUT "st_output"
/*! Timeout period per a device execution */
# define F_STONITH_TIMEOUT "st_timeout"
# define F_STONITH_TOLERANCE "st_tolerance"
# define F_STONITH_DELAY "st_delay"
/*! Action specific timeout period returned in query of fencing devices. */
# define F_STONITH_ACTION_TIMEOUT "st_action_timeout"
/*! Host in query result is not allowed to run this action */
# define F_STONITH_ACTION_DISALLOWED "st_action_disallowed"
/*! Maximum of random fencing delay for a device */
# define F_STONITH_DELAY_MAX "st_delay_max"
/*! Base delay used for a fencing delay */
# define F_STONITH_DELAY_BASE "st_delay_base"
/*! Has this device been verified using a monitor type
* operation (monitor, list, status) */
# define F_STONITH_DEVICE_VERIFIED "st_monitor_verified"
/*! device is required for this action */
# define F_STONITH_DEVICE_REQUIRED "st_required"
/*! number of available devices in query result */
# define F_STONITH_AVAILABLE_DEVICES "st-available-devices"
# define F_STONITH_CALLBACK_TOKEN "st_async_id"
# define F_STONITH_CLIENTNAME "st_clientname"
# define F_STONITH_CLIENTNODE "st_clientnode"
# define F_STONITH_NOTIFY_ACTIVATE "st_notify_activate"
# define F_STONITH_NOTIFY_DEACTIVATE "st_notify_deactivate"
# define F_STONITH_DELEGATE "st_delegate"
# define F_STONITH_DEVICE_SUPPORT_FLAGS "st_device_support_flags"
/*! The node initiating the stonith operation. If an operation
* is relayed, this is the last node the operation lands on. When
* in standalone mode, origin is the client's id that originated the
* operation. */
# define F_STONITH_ORIGIN "st_origin"
# define F_STONITH_HISTORY_LIST "st_history"
# define F_STONITH_DATE "st_date"
# define F_STONITH_DATE_NSEC "st_date_nsec"
# define F_STONITH_STATE "st_state"
# define F_STONITH_ACTIVE "st_active"
# define F_STONITH_DIFFERENTIAL "st_differential"
# define F_STONITH_DEVICE "st_device_id"
# define F_STONITH_ACTION "st_device_action"
# define F_STONITH_MERGED "st_op_merged"
# define T_STONITH_NG "stonith-ng"
# define T_STONITH_REPLY "st-reply"
/*! For async operations, an event from the server containing
* the total amount of time the server is allowing for the operation
* to take place is returned to the client. */
# define T_STONITH_TIMEOUT_VALUE "st-async-timeout-value"
# define T_STONITH_NOTIFY "st_notify"
# define STONITH_ATTR_ACTION_OP "action"
# define STONITH_OP_EXEC "st_execute"
# define STONITH_OP_TIMEOUT_UPDATE "st_timeout_update"
# define STONITH_OP_QUERY "st_query"
# define STONITH_OP_FENCE "st_fence"
# define STONITH_OP_RELAY "st_relay"
# define STONITH_OP_DEVICE_ADD "st_device_register"
# define STONITH_OP_DEVICE_DEL "st_device_remove"
# define STONITH_OP_FENCE_HISTORY "st_fence_history"
# define STONITH_OP_LEVEL_ADD "st_level_add"
# define STONITH_OP_LEVEL_DEL "st_level_remove"
# define STONITH_WATCHDOG_AGENT "fence_watchdog"
/* Don't change 2 below as it would break rolling upgrade */
# define STONITH_WATCHDOG_AGENT_INTERNAL "#watchdog"
# define STONITH_WATCHDOG_ID "watchdog"
stonith_history_t *stonith__first_matching_event(stonith_history_t *history,
bool (*matching_fn)(stonith_history_t *, void *),
void *user_data);
bool stonith__event_state_pending(stonith_history_t *history, void *user_data);
bool stonith__event_state_eq(stonith_history_t *history, void *user_data);
bool stonith__event_state_neq(stonith_history_t *history, void *user_data);
int stonith__legacy2status(int rc);
int stonith__exit_status(stonith_callback_data_t *data);
int stonith__execution_status(stonith_callback_data_t *data);
const char *stonith__exit_reason(stonith_callback_data_t *data);
int stonith__event_exit_status(stonith_event_t *event);
int stonith__event_execution_status(stonith_event_t *event);
const char *stonith__event_exit_reason(stonith_event_t *event);
char *stonith__event_description(stonith_event_t *event);
gchar *stonith__history_description(stonith_history_t *event, bool full_history,
const char *later_succeeded,
uint32_t show_opts);
/*!
* \internal
* \brief Is a fencing operation in pending state?
*
* \param[in] state State as enum op_state value
*
* \return A boolean
*/
static inline bool
stonith__op_state_pending(enum op_state state)
{
return state != st_failed && state != st_done;
}
gboolean stonith__watchdog_fencing_enabled_for_node(const char *node);
gboolean stonith__watchdog_fencing_enabled_for_node_api(stonith_t *st, const char *node);
#endif
diff --git a/include/crm/lrmd_internal.h b/include/crm/lrmd_internal.h
index 284c4d6a9a..5cb00d5e4d 100644
--- a/include/crm/lrmd_internal.h
+++ b/include/crm/lrmd_internal.h
@@ -1,83 +1,89 @@
/*
- * Copyright 2015-2021 the Pacemaker project contributors
+ * Copyright 2015-2022 the Pacemaker project contributors
*
* The version control history for this file may have further details.
*
* This source code is licensed under the GNU Lesser General Public License
* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
*/
#ifndef LRMD_INTERNAL__H
#define LRMD_INTERNAL__H
#include <stdint.h> // uint32_t
#include <glib.h> // GList, GHashTable, gpointer
#include <libxml/tree.h> // xmlNode
#include <crm/common/ipc.h> // crm_ipc_t
#include <crm/common/mainloop.h> // mainloop_io_t, ipc_client_callbacks
#include <crm/common/output_internal.h> // pcmk__output_t
#include <crm/common/remote_internal.h> // pcmk__remote_t
-#include <crm/lrmd.h> // lrmd_t, lrmd_event_data_t
+#include <crm/lrmd.h> // lrmd_t, lrmd_event_data_t, lrmd_rsc_info_t
int lrmd__new(lrmd_t **api, const char *nodename, const char *server, int port);
int lrmd_send_attribute_alert(lrmd_t *lrmd, GList *alert_list,
const char *node, uint32_t nodeid,
const char *attr_name, const char *attr_value);
int lrmd_send_node_alert(lrmd_t *lrmd, GList *alert_list,
const char *node, uint32_t nodeid, const char *state);
int lrmd_send_fencing_alert(lrmd_t *lrmd, GList *alert_list,
const char *target, const char *task,
const char *desc, int op_rc);
int lrmd_send_resource_alert(lrmd_t *lrmd, GList *alert_list,
const char *node, lrmd_event_data_t *op);
int lrmd__remote_send_xml(pcmk__remote_t *session, xmlNode *msg, uint32_t id,
const char *msg_type);
+int lrmd__metadata_async(lrmd_rsc_info_t *rsc,
+ void (*callback)(int pid,
+ const pcmk__action_result_t *result,
+ void *user_data),
+ void *user_data);
+
void lrmd__set_result(lrmd_event_data_t *event, enum ocf_exitcode rc,
int op_status, const char *exit_reason);
void lrmd__reset_result(lrmd_event_data_t *event);
/* Shared functions for IPC proxy back end */
typedef struct remote_proxy_s {
char *node_name;
char *session_id;
gboolean is_local;
crm_ipc_t *ipc;
mainloop_io_t *source;
uint32_t last_request_id;
lrmd_t *lrm;
} remote_proxy_t;
remote_proxy_t *remote_proxy_new(lrmd_t *lrmd,
struct ipc_client_callbacks *proxy_callbacks,
const char *node_name, const char *session_id,
const char *channel);
int lrmd__validate_remote_settings(lrmd_t *lrmd, GHashTable *hash);
void remote_proxy_cb(lrmd_t *lrmd, const char *node_name, xmlNode *msg);
void remote_proxy_ack_shutdown(lrmd_t *lrmd);
void remote_proxy_nack_shutdown(lrmd_t *lrmd);
int remote_proxy_dispatch(const char *buffer, ssize_t length,
gpointer userdata);
void remote_proxy_disconnected(gpointer data);
void remote_proxy_free(gpointer data);
void remote_proxy_relay_event(remote_proxy_t *proxy, xmlNode *msg);
void remote_proxy_relay_response(remote_proxy_t *proxy, xmlNode *msg,
int msg_id);
void lrmd__register_messages(pcmk__output_t *out);
#ifdef HAVE_GNUTLS_GNUTLS_H
int lrmd__init_remote_key(gnutls_datum_t *key);
#endif
#endif
diff --git a/include/crm/stonith-ng.h b/include/crm/stonith-ng.h
index aa2c54922b..ffc71653b4 100644
--- a/include/crm/stonith-ng.h
+++ b/include/crm/stonith-ng.h
@@ -1,588 +1,595 @@
/*
* Copyright 2004-2022 the Pacemaker project contributors
*
* The version control history for this file may have further details.
*
* This source code is licensed under the GNU Lesser General Public License
* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
*/
#ifndef PCMK__CRM_STONITH_NG__H
# define PCMK__CRM_STONITH_NG__H
#ifdef __cplusplus
extern "C" {
#endif
/**
* \file
* \brief Fencing aka. STONITH
* \ingroup fencing
*/
/* IMPORTANT: DLM source code includes this file directly, without having access
* to other Pacemaker headers on its include path, so this file should *not*
* include any other Pacemaker headers. (DLM might be updated to avoid the
* issue, but we should still follow this guideline for a long time after.)
*/
# include <dlfcn.h>
# include <errno.h>
# include <stdbool.h> // bool
# include <stdint.h> // uint32_t
# include <time.h> // time_t
# define T_STONITH_NOTIFY_DISCONNECT "st_notify_disconnect"
# define T_STONITH_NOTIFY_FENCE "st_notify_fence"
# define T_STONITH_NOTIFY_HISTORY "st_notify_history"
# define T_STONITH_NOTIFY_HISTORY_SYNCED "st_notify_history_synced"
/* *INDENT-OFF* */
enum stonith_state {
stonith_connected_command,
stonith_connected_query,
stonith_disconnected,
};
enum stonith_call_options {
st_opt_none = 0x00000000,
st_opt_verbose = 0x00000001,
st_opt_allow_suicide = 0x00000002,
st_opt_manual_ack = 0x00000008,
st_opt_discard_reply = 0x00000010,
/* st_opt_all_replies = 0x00000020, */
st_opt_topology = 0x00000040,
st_opt_scope_local = 0x00000100,
st_opt_cs_nodeid = 0x00000200,
st_opt_sync_call = 0x00001000,
/*! Allow the timeout period for a callback to be adjusted
* based on the time the server reports the operation will take. */
st_opt_timeout_updates = 0x00002000,
/*! Only report back if operation is a success in callback */
st_opt_report_only_success = 0x00004000,
/* used where ever apropriate - e.g. cleanup of history */
st_opt_cleanup = 0x000080000,
/* used where ever apropriate - e.g. send out a history query to all nodes */
st_opt_broadcast = 0x000100000,
};
/*! Order matters here, do not change values */
enum op_state
{
st_query,
st_exec,
st_done,
st_duplicate,
st_failed,
};
// Supported fence agent interface standards
enum stonith_namespace {
st_namespace_invalid,
st_namespace_any,
st_namespace_internal, // Implemented internally by Pacemaker
/* Neither of these projects are active any longer, but the fence agent
* interfaces they created are still in use and supported by Pacemaker.
*/
st_namespace_rhcs, // Red Hat Cluster Suite compatible
st_namespace_lha, // Linux-HA compatible
};
enum stonith_namespace stonith_text2namespace(const char *namespace_s);
const char *stonith_namespace2text(enum stonith_namespace st_namespace);
enum stonith_namespace stonith_get_namespace(const char *agent,
const char *namespace_s);
typedef struct stonith_key_value_s {
char *key;
char *value;
struct stonith_key_value_s *next;
} stonith_key_value_t;
typedef struct stonith_history_s {
char *target;
char *action;
char *origin;
char *delegate;
char *client;
int state;
time_t completed;
struct stonith_history_s *next;
long completed_nsec;
char *exit_reason;
} stonith_history_t;
typedef struct stonith_s stonith_t;
typedef struct stonith_event_s
{
char *id;
char *type; //!< \deprecated Will be removed in future release
char *message; //!< \deprecated Will be removed in future release
char *operation;
int result;
char *origin;
char *target;
char *action;
char *executioner;
char *device;
/*! The name of the client that initiated the action. */
char *client_origin;
//! \internal This field should be treated as internal to Pacemaker
void *opaque;
} stonith_event_t;
typedef struct stonith_callback_data_s {
int rc;
int call_id;
void *userdata;
//! \internal This field should be treated as internal to Pacemaker
void *opaque;
} stonith_callback_data_t;
typedef struct stonith_api_operations_s
{
/*!
* \brief Destroy the stonith api structure.
*/
int (*free) (stonith_t *st);
/*!
* \brief Connect to the local stonith daemon.
*
* \return Legacy Pacemaker return code
*/
int (*connect) (stonith_t *st, const char *name, int *stonith_fd);
/*!
* \brief Disconnect from the local stonith daemon.
*
* \return Legacy Pacemaker return code
*/
int (*disconnect)(stonith_t *st);
/*!
* \brief Unregister a fence device with the local fencer
*
* \return pcmk_ok (if synchronous) or positive call ID (if asynchronous)
* on success, otherwise a negative legacy Pacemaker return code
*/
int (*remove_device)(
stonith_t *st, int options, const char *name);
/*!
* \brief Register a fence device with the local fencer
*
* \return pcmk_ok (if synchronous) or positive call ID (if asynchronous)
* on success, otherwise a negative legacy Pacemaker return code
*/
int (*register_device)(
stonith_t *st, int options, const char *id,
const char *provider, const char *agent, stonith_key_value_t *params);
/*!
* \brief Unregister a fencing level for specified node with local fencer
*
* \return pcmk_ok (if synchronous) or positive call ID (if asynchronous)
* on success, otherwise a negative legacy Pacemaker return code
*/
int (*remove_level)(
stonith_t *st, int options, const char *node, int level);
/*!
* \brief Register a fencing level for specified node with local fencer
*
* \return pcmk_ok (if synchronous) or positive call ID (if asynchronous)
* on success, otherwise a negative legacy Pacemaker return code
*/
int (*register_level)(
stonith_t *st, int options, const char *node, int level, stonith_key_value_t *device_list);
/*!
- * \brief Get the metadata documentation for a resource.
+ * \brief Retrieve a fence agent's metadata
*
- * \note Value is returned in output. Output must be freed when set.
+ * \param[in,out] stonith Fencer connection
+ * \param[in] call_options Group of enum stonith_call_options
+ * (currently ignored)
+ * \param[in] agent Fence agent to query
+ * \param[in] namespace Namespace of fence agent to query (optional)
+ * \param[out] output Where to store metadata
+ * \param[in] timeout_sec Error if not complete within this time
*
* \return Legacy Pacemaker return code
+ * \note The caller is responsible for freeing *output using free().
*/
- int (*metadata)(stonith_t *st, int options,
- const char *device, const char *provider, char **output, int timeout);
+ int (*metadata)(stonith_t *stonith, int call_options, const char *agent,
+ const char *namespace, char **output, int timeout_sec);
/*!
* \brief Retrieve a list of installed stonith agents
*
* \note if provider is not provided, all known agents will be returned
* \note list must be freed using stonith_key_value_freeall()
* \note call_options parameter is not used, it is reserved for future use.
*
* \return Number of items in list on success, or negative errno otherwise
*/
int (*list_agents)(stonith_t *stonith, int call_options, const char *provider,
stonith_key_value_t **devices, int timeout);
/*!
* \brief Retrieve string listing hosts and port assignments from a local stonith device.
*
* \return pcmk_ok (if synchronous) or positive call ID (if asynchronous)
* on success, otherwise a negative legacy Pacemaker return code
*/
int (*list)(stonith_t *st, int options, const char *id, char **list_output, int timeout);
/*!
* \brief Check to see if a local stonith device is reachable
*
* \return pcmk_ok (if synchronous) or positive call ID (if asynchronous)
* on success, otherwise a negative legacy Pacemaker return code
*/
int (*monitor)(stonith_t *st, int options, const char *id, int timeout);
/*!
* \brief Check to see if a local stonith device's port is reachable
*
* \return pcmk_ok (if synchronous) or positive call ID (if asynchronous)
* on success, otherwise a negative legacy Pacemaker return code
*/
int (*status)(stonith_t *st, int options, const char *id, const char *port, int timeout);
/*!
* \brief Retrieve a list of registered stonith devices.
*
* \note If node is provided, only devices that can fence the node id
* will be returned.
*
* \return Number of items in list on success, or negative errno otherwise
*/
int (*query)(stonith_t *st, int options, const char *node,
stonith_key_value_t **devices, int timeout);
/*!
* \brief Issue a fencing action against a node.
*
* \note Possible actions are, 'on', 'off', and 'reboot'.
*
* \param st, stonith connection
* \param options, call options
* \param node, The target node to fence
* \param action, The fencing action to take
* \param timeout, The default per device timeout to use with each device
* capable of fencing the target.
*
* \return pcmk_ok (if synchronous) or positive call ID (if asynchronous)
* on success, otherwise a negative legacy Pacemaker return code
*/
int (*fence)(stonith_t *st, int options, const char *node, const char *action,
int timeout, int tolerance);
/*!
* \brief Manually confirm that a node is down.
*
* \return pcmk_ok (if synchronous) or positive call ID (if asynchronous)
* on success, otherwise a negative legacy Pacemaker return code
*/
int (*confirm)(stonith_t *st, int options, const char *node);
/*!
* \brief Retrieve a list of fencing operations that have occurred for a specific node.
*
* \return Legacy Pacemaker return code
*/
int (*history)(stonith_t *st, int options, const char *node, stonith_history_t **output, int timeout);
int (*register_notification)(
stonith_t *st, const char *event,
void (*notify)(stonith_t *st, stonith_event_t *e));
/*!
* \brief Remove a previously registered notification for \c event, or all
* notifications if NULL.
*
* \param[in] st Fencer connection to use
* \param[in] event The event to remove notifications for (may be NULL).
*
* \return Legacy Pacemaker return code
*/
int (*remove_notification)(stonith_t *st, const char *event);
/*!
* \brief Register a callback to receive the result of an asynchronous call
*
* \param[in] call_id The call ID to register callback for
* \param[in] timeout Default time to wait until callback expires
* \param[in] options Bitmask of \c stonith_call_options (respects
* \c st_opt_timeout_updates and
* \c st_opt_report_only_success)
* \param[in] userdata Pointer that will be given to callback
* \param[in] callback_name Unique name to identify callback
* \param[in] callback The callback function to register
*
* \return \c TRUE on success, \c FALSE if call_id is negative, -errno otherwise
*/
int (*register_callback)(stonith_t *st,
int call_id,
int timeout,
int options,
void *userdata,
const char *callback_name,
void (*callback)(stonith_t *st, stonith_callback_data_t *data));
/*!
* \brief Remove a registered callback for a given call id
*
* \return pcmk_ok
*/
int (*remove_callback)(stonith_t *st, int call_id, bool all_callbacks);
/*!
* \brief Unregister fencing level for specified node, pattern or attribute
*
* \param[in] st Fencer connection to use
* \param[in] options Bitmask of stonith_call_options to pass to the fencer
* \param[in] node If not NULL, target level by this node name
* \param[in] pattern If not NULL, target by node name using this regex
* \param[in] attr If not NULL, target by this node attribute
* \param[in] value If not NULL, target by this node attribute value
* \param[in] level Index number of level to remove
*
* \return pcmk_ok (if synchronous) or positive call ID (if asynchronous)
* on success, otherwise a negative legacy Pacemaker return code
*
* \note The caller should set only one of node, pattern or attr/value.
*/
int (*remove_level_full)(stonith_t *st, int options,
const char *node, const char *pattern,
const char *attr, const char *value, int level);
/*!
* \brief Register fencing level for specified node, pattern or attribute
*
* \param[in] st Fencer connection to use
* \param[in] options Bitmask of stonith_call_options to pass to fencer
* \param[in] node If not NULL, target level by this node name
* \param[in] pattern If not NULL, target by node name using this regex
* \param[in] attr If not NULL, target by this node attribute
* \param[in] value If not NULL, target by this node attribute value
* \param[in] level Index number of level to add
* \param[in] device_list Devices to use in level
*
* \return pcmk_ok (if synchronous) or positive call ID (if asynchronous)
* on success, otherwise a negative legacy Pacemaker return code
*
* \note The caller should set only one of node, pattern or attr/value.
*/
int (*register_level_full)(stonith_t *st, int options,
const char *node, const char *pattern,
const char *attr, const char *value,
int level, stonith_key_value_t *device_list);
/*!
* \brief Validate an arbitrary stonith device configuration
*
* \param[in] st Stonithd connection to use
* \param[in] call_options Bitmask of stonith_call_options to use with fencer
* \param[in] rsc_id ID used to replace CIB secrets in params
* \param[in] namespace_s Namespace of fence agent to validate (optional)
* \param[in] agent Fence agent to validate
* \param[in] params Configuration parameters to pass to fence agent
* \param[in] timeout Fail if no response within this many seconds
* \param[out] output If non-NULL, where to store any agent output
* \param[out] error_output If non-NULL, where to store agent error output
*
* \return pcmk_ok if validation succeeds, -errno otherwise
*
* \note If pcmk_ok is returned, the caller is responsible for freeing
* the output (if requested).
*/
int (*validate)(stonith_t *st, int call_options, const char *rsc_id,
const char *namespace_s, const char *agent,
stonith_key_value_t *params, int timeout, char **output,
char **error_output);
/*!
* \brief Issue a fencing action against a node with requested fencing delay.
*
* \note Possible actions are, 'on', 'off', and 'reboot'.
*
* \param st, stonith connection
* \param options, call options
* \param node, The target node to fence
* \param action, The fencing action to take
* \param timeout, The default per device timeout to use with each device
* capable of fencing the target.
* \param delay, Apply a fencing delay. Value -1 means disable also any
* static/random fencing delays from pcmk_delay_base/max
*
* \return pcmk_ok (if synchronous) or positive call ID (if asynchronous)
* on success, otherwise a negative legacy Pacemaker return code
*/
int (*fence_with_delay)(stonith_t *st, int options, const char *node, const char *action,
int timeout, int tolerance, int delay);
} stonith_api_operations_t;
struct stonith_s
{
enum stonith_state state;
int call_id;
int call_timeout; //!< \deprecated Unused
void *st_private;
stonith_api_operations_t *cmds;
};
/* *INDENT-ON* */
/* Core functions */
stonith_t *stonith_api_new(void);
void stonith_api_delete(stonith_t * st);
void stonith_dump_pending_callbacks(stonith_t * st);
bool stonith_dispatch(stonith_t * st);
stonith_key_value_t *stonith_key_value_add(stonith_key_value_t * kvp, const char *key,
const char *value);
void stonith_key_value_freeall(stonith_key_value_t * kvp, int keys, int values);
void stonith_history_free(stonith_history_t *history);
// Convenience functions
int stonith_api_connect_retry(stonith_t *st, const char *name,
int max_attempts);
const char *stonith_op_state_str(enum op_state state);
/* Basic helpers that allows nodes to be fenced and the history to be
* queried without mainloop or the caller understanding the full API
*
* At least one of nodeid and uname are required
*/
int stonith_api_kick(uint32_t nodeid, const char *uname, int timeout, bool off);
time_t stonith_api_time(uint32_t nodeid, const char *uname, bool in_progress);
/*
* Helpers for using the above functions without install-time dependencies
*
* Usage:
* #include <crm/stonith-ng.h>
*
* To turn a node off by corosync nodeid:
* stonith_api_kick_helper(nodeid, 120, 1);
*
* To check the last fence date/time (also by nodeid):
* last = stonith_api_time_helper(nodeid, 0);
*
* To check if fencing is in progress:
* if(stonith_api_time_helper(nodeid, 1) > 0) { ... }
*
* eg.
#include <stdio.h>
#include <time.h>
#include <crm/stonith-ng.h>
int
main(int argc, char ** argv)
{
int rc = 0;
int nodeid = 102;
rc = stonith_api_time_helper(nodeid, 0);
printf("%d last fenced at %s\n", nodeid, ctime(rc));
rc = stonith_api_kick_helper(nodeid, 120, 1);
printf("%d fence result: %d\n", nodeid, rc);
rc = stonith_api_time_helper(nodeid, 0);
printf("%d last fenced at %s\n", nodeid, ctime(rc));
return 0;
}
*/
# define STONITH_LIBRARY "libstonithd.so.26"
typedef int (*st_api_kick_fn) (int nodeid, const char *uname, int timeout, bool off);
typedef time_t (*st_api_time_fn) (int nodeid, const char *uname, bool in_progress);
static inline int
stonith_api_kick_helper(uint32_t nodeid, int timeout, bool off)
{
static void *st_library = NULL;
static st_api_kick_fn st_kick_fn;
if (st_library == NULL) {
st_library = dlopen(STONITH_LIBRARY, RTLD_LAZY);
}
if (st_library && st_kick_fn == NULL) {
st_kick_fn = (st_api_kick_fn) dlsym(st_library, "stonith_api_kick");
}
if (st_kick_fn == NULL) {
#ifdef ELIBACC
return -ELIBACC;
#else
return -ENOSYS;
#endif
}
return (*st_kick_fn) (nodeid, NULL, timeout, off);
}
static inline time_t
stonith_api_time_helper(uint32_t nodeid, bool in_progress)
{
static void *st_library = NULL;
static st_api_time_fn st_time_fn;
if (st_library == NULL) {
st_library = dlopen(STONITH_LIBRARY, RTLD_LAZY);
}
if (st_library && st_time_fn == NULL) {
st_time_fn = (st_api_time_fn) dlsym(st_library, "stonith_api_time");
}
if (st_time_fn == NULL) {
return 0;
}
return (*st_time_fn) (nodeid, NULL, in_progress);
}
/**
* Does the given agent describe a stonith resource that can exist?
*
* \param[in] agent What is the name of the agent?
* \param[in] timeout Timeout to use when querying. If 0 is given,
* use a default of 120.
*
* \return A boolean
*/
bool stonith_agent_exists(const char *agent, int timeout);
/*!
* \brief Turn stonith action into a more readable string.
*
* \param action Stonith action
*/
const char *stonith_action_str(const char *action);
#if !defined(PCMK_ALLOW_DEPRECATED) || (PCMK_ALLOW_DEPRECATED == 1)
/* Normally we'd put this section in a separate file (crm/fencing/compat.h), but
* we can't do that for the reason noted at the top of this file. That does mean
* we have to duplicate these declarations where they're implemented.
*/
//! \deprecated Use stonith_get_namespace() instead
const char *get_stonith_provider(const char *agent, const char *provider);
#endif
#ifdef __cplusplus
}
#endif
#endif
diff --git a/lib/common/results.c b/lib/common/results.c
index 972250ab16..814e489345 100644
--- a/lib/common/results.c
+++ b/lib/common/results.c
@@ -1,1028 +1,1030 @@
/*
* Copyright 2004-2022 the Pacemaker project contributors
*
* The version control history for this file may have further details.
*
* This source code is licensed under the GNU Lesser General Public License
* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
*/
#include <crm_internal.h>
#ifndef _GNU_SOURCE
# define _GNU_SOURCE
#endif
#include <bzlib.h>
#include <errno.h>
#include <stdlib.h>
#include <string.h>
#include <qb/qbdefs.h>
#include <crm/common/mainloop.h>
#include <crm/common/xml.h>
G_DEFINE_QUARK(pcmk-rc-error-quark, pcmk__rc_error)
G_DEFINE_QUARK(pcmk-exitc-error-quark, pcmk__exitc_error)
// General (all result code types)
/*!
* \brief Get the name and description of a given result code
*
* A result code can be interpreted as a member of any one of several families.
*
* \param[in] code The result code to look up
* \param[in] type How \p code should be interpreted
* \param[out] name Where to store the result code's name
* \param[out] desc Where to store the result code's description
*
* \return Standard Pacemaker return code
*/
int
pcmk_result_get_strings(int code, enum pcmk_result_type type, const char **name,
const char **desc)
{
const char *code_name = NULL;
const char *code_desc = NULL;
switch (type) {
case pcmk_result_legacy:
code_name = pcmk_errorname(code);
code_desc = pcmk_strerror(code);
break;
case pcmk_result_rc:
code_name = pcmk_rc_name(code);
code_desc = pcmk_rc_str(code);
break;
case pcmk_result_exitcode:
code_name = crm_exit_name(code);
code_desc = crm_exit_str((crm_exit_t) code);
break;
default:
return pcmk_rc_undetermined;
}
if (name != NULL) {
*name = code_name;
}
if (desc != NULL) {
*desc = code_desc;
}
return pcmk_rc_ok;
}
/*!
* \internal
* \brief Get the lower and upper bounds of a result code family
*
* \param[in] type Type of result code
* \param[out] lower Where to store the lower bound
* \param[out] upper Where to store the upper bound
*
* \return Standard Pacemaker return code
*
* \note There is no true upper bound on standard Pacemaker return codes or
* legacy return codes. All system \p errno values are valid members of
* these result code families, and there is no global upper limit nor a
* constant by which to refer to the highest \p errno value on a given
* system.
*/
int
pcmk__result_bounds(enum pcmk_result_type type, int *lower, int *upper)
{
CRM_ASSERT((lower != NULL) && (upper != NULL));
switch (type) {
case pcmk_result_legacy:
*lower = pcmk_ok;
*upper = 256; // should be enough for almost any system error code
break;
case pcmk_result_rc:
*lower = pcmk_rc_error - pcmk__n_rc + 1;
*upper = 256;
break;
case pcmk_result_exitcode:
*lower = CRM_EX_OK;
*upper = CRM_EX_MAX;
break;
default:
*lower = 0;
*upper = -1;
return pcmk_rc_undetermined;
}
return pcmk_rc_ok;
}
// @COMPAT Legacy function return codes
//! \deprecated Use standard return codes and pcmk_rc_name() instead
const char *
pcmk_errorname(int rc)
{
rc = abs(rc);
switch (rc) {
case pcmk_err_generic: return "pcmk_err_generic";
case pcmk_err_no_quorum: return "pcmk_err_no_quorum";
case pcmk_err_schema_validation: return "pcmk_err_schema_validation";
case pcmk_err_transform_failed: return "pcmk_err_transform_failed";
case pcmk_err_old_data: return "pcmk_err_old_data";
case pcmk_err_diff_failed: return "pcmk_err_diff_failed";
case pcmk_err_diff_resync: return "pcmk_err_diff_resync";
case pcmk_err_cib_modified: return "pcmk_err_cib_modified";
case pcmk_err_cib_backup: return "pcmk_err_cib_backup";
case pcmk_err_cib_save: return "pcmk_err_cib_save";
case pcmk_err_cib_corrupt: return "pcmk_err_cib_corrupt";
case pcmk_err_multiple: return "pcmk_err_multiple";
case pcmk_err_node_unknown: return "pcmk_err_node_unknown";
case pcmk_err_already: return "pcmk_err_already";
case pcmk_err_bad_nvpair: return "pcmk_err_bad_nvpair";
case pcmk_err_unknown_format: return "pcmk_err_unknown_format";
default: return pcmk_rc_name(rc); // system errno
}
}
//! \deprecated Use standard return codes and pcmk_rc_str() instead
const char *
pcmk_strerror(int rc)
{
return pcmk_rc_str(pcmk_legacy2rc(rc));
}
// Standard Pacemaker API return codes
/* This array is used only for nonzero values of pcmk_rc_e. Its values must be
* kept in the exact reverse order of the enum value numbering (i.e. add new
* values to the end of the array).
*/
static const struct pcmk__rc_info {
const char *name;
const char *desc;
int legacy_rc;
} pcmk__rcs[] = {
{ "pcmk_rc_error",
"Error",
-pcmk_err_generic,
},
{ "pcmk_rc_unknown_format",
"Unknown output format",
-pcmk_err_unknown_format,
},
{ "pcmk_rc_bad_nvpair",
"Bad name/value pair given",
-pcmk_err_bad_nvpair,
},
{ "pcmk_rc_already",
"Already in requested state",
-pcmk_err_already,
},
{ "pcmk_rc_node_unknown",
"Node not found",
-pcmk_err_node_unknown,
},
{ "pcmk_rc_multiple",
"Resource active on multiple nodes",
-pcmk_err_multiple,
},
{ "pcmk_rc_cib_corrupt",
"Could not parse on-disk configuration",
-pcmk_err_cib_corrupt,
},
{ "pcmk_rc_cib_save",
"Could not save new configuration to disk",
-pcmk_err_cib_save,
},
{ "pcmk_rc_cib_backup",
"Could not archive previous configuration",
-pcmk_err_cib_backup,
},
{ "pcmk_rc_cib_modified",
"On-disk configuration was manually modified",
-pcmk_err_cib_modified,
},
{ "pcmk_rc_diff_resync",
"Application of update diff failed, requesting full refresh",
-pcmk_err_diff_resync,
},
{ "pcmk_rc_diff_failed",
"Application of update diff failed",
-pcmk_err_diff_failed,
},
{ "pcmk_rc_old_data",
"Update was older than existing configuration",
-pcmk_err_old_data,
},
{ "pcmk_rc_transform_failed",
"Schema transform failed",
-pcmk_err_transform_failed,
},
{ "pcmk_rc_schema_unchanged",
"Schema is already the latest available",
-pcmk_err_schema_unchanged,
},
{ "pcmk_rc_schema_validation",
"Update does not conform to the configured schema",
-pcmk_err_schema_validation,
},
{ "pcmk_rc_no_quorum",
"Operation requires quorum",
-pcmk_err_no_quorum,
},
{ "pcmk_rc_ipc_unauthorized",
"IPC server is blocked by unauthorized process",
-pcmk_err_generic,
},
{ "pcmk_rc_ipc_unresponsive",
"IPC server is unresponsive",
-pcmk_err_generic,
},
{ "pcmk_rc_ipc_pid_only",
"IPC server process is active but not accepting connections",
-pcmk_err_generic,
},
{ "pcmk_rc_op_unsatisfied",
"Not applicable under current conditions",
-pcmk_err_generic,
},
{ "pcmk_rc_undetermined",
"Result undetermined",
-pcmk_err_generic,
},
{ "pcmk_rc_before_range",
"Result occurs before given range",
-pcmk_err_generic,
},
{ "pcmk_rc_within_range",
"Result occurs within given range",
-pcmk_err_generic,
},
{ "pcmk_rc_after_range",
"Result occurs after given range",
-pcmk_err_generic,
},
{ "pcmk_rc_no_output",
"Output message produced no output",
-pcmk_err_generic,
},
{ "pcmk_rc_no_input",
"Input file not available",
-pcmk_err_generic,
},
{ "pcmk_rc_underflow",
"Value too small to be stored in data type",
-pcmk_err_generic,
},
{ "pcmk_rc_dot_error",
"Error writing dot(1) file",
-pcmk_err_generic,
},
{ "pcmk_rc_graph_error",
"Error writing graph file",
-pcmk_err_generic,
},
{ "pcmk_rc_invalid_transition",
"Cluster simulation produced invalid transition",
-pcmk_err_generic,
},
{ "pcmk_rc_unpack_error",
"Unable to parse CIB XML",
-pcmk_err_generic,
},
{ "pcmk_rc_duplicate_id",
"Two or more XML elements have the same ID",
-pcmk_err_generic,
},
};
/*!
* \internal
* \brief The number of <tt>enum pcmk_rc_e</tt> values, excluding \c pcmk_rc_ok
*
* This constant stores the number of negative standard Pacemaker return codes.
* These represent Pacemaker-custom error codes. The count does not include
* positive system error numbers, nor does it include \c pcmk_rc_ok (success).
*/
const size_t pcmk__n_rc = PCMK__NELEM(pcmk__rcs);
/*!
* \brief Get a return code constant name as a string
*
* \param[in] rc Integer return code to convert
*
* \return String of constant name corresponding to rc
*/
const char *
pcmk_rc_name(int rc)
{
if ((rc <= pcmk_rc_error) && ((pcmk_rc_error - rc) < pcmk__n_rc)) {
return pcmk__rcs[pcmk_rc_error - rc].name;
}
switch (rc) {
case pcmk_rc_ok: return "pcmk_rc_ok";
case E2BIG: return "E2BIG";
case EACCES: return "EACCES";
case EADDRINUSE: return "EADDRINUSE";
case EADDRNOTAVAIL: return "EADDRNOTAVAIL";
case EAFNOSUPPORT: return "EAFNOSUPPORT";
case EAGAIN: return "EAGAIN";
case EALREADY: return "EALREADY";
case EBADF: return "EBADF";
case EBADMSG: return "EBADMSG";
case EBUSY: return "EBUSY";
case ECANCELED: return "ECANCELED";
case ECHILD: return "ECHILD";
case ECOMM: return "ECOMM";
case ECONNABORTED: return "ECONNABORTED";
case ECONNREFUSED: return "ECONNREFUSED";
case ECONNRESET: return "ECONNRESET";
/* case EDEADLK: return "EDEADLK"; */
case EDESTADDRREQ: return "EDESTADDRREQ";
case EDOM: return "EDOM";
case EDQUOT: return "EDQUOT";
case EEXIST: return "EEXIST";
case EFAULT: return "EFAULT";
case EFBIG: return "EFBIG";
case EHOSTDOWN: return "EHOSTDOWN";
case EHOSTUNREACH: return "EHOSTUNREACH";
case EIDRM: return "EIDRM";
case EILSEQ: return "EILSEQ";
case EINPROGRESS: return "EINPROGRESS";
case EINTR: return "EINTR";
case EINVAL: return "EINVAL";
case EIO: return "EIO";
case EISCONN: return "EISCONN";
case EISDIR: return "EISDIR";
case ELIBACC: return "ELIBACC";
case ELOOP: return "ELOOP";
case EMFILE: return "EMFILE";
case EMLINK: return "EMLINK";
case EMSGSIZE: return "EMSGSIZE";
#ifdef EMULTIHOP // Not available on OpenBSD
case EMULTIHOP: return "EMULTIHOP";
#endif
case ENAMETOOLONG: return "ENAMETOOLONG";
case ENETDOWN: return "ENETDOWN";
case ENETRESET: return "ENETRESET";
case ENETUNREACH: return "ENETUNREACH";
case ENFILE: return "ENFILE";
case ENOBUFS: return "ENOBUFS";
case ENODATA: return "ENODATA";
case ENODEV: return "ENODEV";
case ENOENT: return "ENOENT";
case ENOEXEC: return "ENOEXEC";
case ENOKEY: return "ENOKEY";
case ENOLCK: return "ENOLCK";
#ifdef ENOLINK // Not available on OpenBSD
case ENOLINK: return "ENOLINK";
#endif
case ENOMEM: return "ENOMEM";
case ENOMSG: return "ENOMSG";
case ENOPROTOOPT: return "ENOPROTOOPT";
case ENOSPC: return "ENOSPC";
#ifdef ENOSR
case ENOSR: return "ENOSR";
#endif
#ifdef ENOSTR
case ENOSTR: return "ENOSTR";
#endif
case ENOSYS: return "ENOSYS";
case ENOTBLK: return "ENOTBLK";
case ENOTCONN: return "ENOTCONN";
case ENOTDIR: return "ENOTDIR";
case ENOTEMPTY: return "ENOTEMPTY";
case ENOTSOCK: return "ENOTSOCK";
#if ENOTSUP != EOPNOTSUPP
case ENOTSUP: return "ENOTSUP";
#endif
case ENOTTY: return "ENOTTY";
case ENOTUNIQ: return "ENOTUNIQ";
case ENXIO: return "ENXIO";
case EOPNOTSUPP: return "EOPNOTSUPP";
case EOVERFLOW: return "EOVERFLOW";
case EPERM: return "EPERM";
case EPFNOSUPPORT: return "EPFNOSUPPORT";
case EPIPE: return "EPIPE";
case EPROTO: return "EPROTO";
case EPROTONOSUPPORT: return "EPROTONOSUPPORT";
case EPROTOTYPE: return "EPROTOTYPE";
case ERANGE: return "ERANGE";
case EREMOTE: return "EREMOTE";
case EREMOTEIO: return "EREMOTEIO";
case EROFS: return "EROFS";
case ESHUTDOWN: return "ESHUTDOWN";
case ESPIPE: return "ESPIPE";
case ESOCKTNOSUPPORT: return "ESOCKTNOSUPPORT";
case ESRCH: return "ESRCH";
case ESTALE: return "ESTALE";
case ETIME: return "ETIME";
case ETIMEDOUT: return "ETIMEDOUT";
case ETXTBSY: return "ETXTBSY";
#ifdef EUNATCH
case EUNATCH: return "EUNATCH";
#endif
case EUSERS: return "EUSERS";
/* case EWOULDBLOCK: return "EWOULDBLOCK"; */
case EXDEV: return "EXDEV";
#ifdef EBADE // Not available on OS X
case EBADE: return "EBADE";
case EBADFD: return "EBADFD";
case EBADSLT: return "EBADSLT";
case EDEADLOCK: return "EDEADLOCK";
case EBADR: return "EBADR";
case EBADRQC: return "EBADRQC";
case ECHRNG: return "ECHRNG";
#ifdef EISNAM // Not available on OS X, Illumos, Solaris
case EISNAM: return "EISNAM";
case EKEYEXPIRED: return "EKEYEXPIRED";
case EKEYREVOKED: return "EKEYREVOKED";
#endif
case EKEYREJECTED: return "EKEYREJECTED";
case EL2HLT: return "EL2HLT";
case EL2NSYNC: return "EL2NSYNC";
case EL3HLT: return "EL3HLT";
case EL3RST: return "EL3RST";
case ELIBBAD: return "ELIBBAD";
case ELIBMAX: return "ELIBMAX";
case ELIBSCN: return "ELIBSCN";
case ELIBEXEC: return "ELIBEXEC";
#ifdef ENOMEDIUM // Not available on OS X, Illumos, Solaris
case ENOMEDIUM: return "ENOMEDIUM";
case EMEDIUMTYPE: return "EMEDIUMTYPE";
#endif
case ENONET: return "ENONET";
case ENOPKG: return "ENOPKG";
case EREMCHG: return "EREMCHG";
case ERESTART: return "ERESTART";
case ESTRPIPE: return "ESTRPIPE";
#ifdef EUCLEAN // Not available on OS X, Illumos, Solaris
case EUCLEAN: return "EUCLEAN";
#endif
case EXFULL: return "EXFULL";
#endif // EBADE
default: return "Unknown";
}
}
/*!
* \brief Get a user-friendly description of a return code
*
* \param[in] rc Integer return code to convert
*
* \return String description of rc
*/
const char *
pcmk_rc_str(int rc)
{
if (rc == pcmk_rc_ok) {
return "OK";
}
if ((rc <= pcmk_rc_error) && ((pcmk_rc_error - rc) < pcmk__n_rc)) {
return pcmk__rcs[pcmk_rc_error - rc].desc;
}
if (rc < 0) {
return "Error";
}
// Handle values that could be defined by system or by portability.h
switch (rc) {
#ifdef PCMK__ENOTUNIQ
case ENOTUNIQ: return "Name not unique on network";
#endif
#ifdef PCMK__ECOMM
case ECOMM: return "Communication error on send";
#endif
#ifdef PCMK__ELIBACC
case ELIBACC: return "Can not access a needed shared library";
#endif
#ifdef PCMK__EREMOTEIO
case EREMOTEIO: return "Remote I/O error";
#endif
#ifdef PCMK__ENOKEY
case ENOKEY: return "Required key not available";
#endif
#ifdef PCMK__ENODATA
case ENODATA: return "No data available";
#endif
#ifdef PCMK__ETIME
case ETIME: return "Timer expired";
#endif
#ifdef PCMK__EKEYREJECTED
case EKEYREJECTED: return "Key was rejected by service";
#endif
default: return strerror(rc);
}
}
// This returns negative values for errors
//! \deprecated Use standard return codes instead
int
pcmk_rc2legacy(int rc)
{
if (rc >= 0) {
return -rc; // OK or system errno
}
if ((rc <= pcmk_rc_error) && ((pcmk_rc_error - rc) < pcmk__n_rc)) {
return pcmk__rcs[pcmk_rc_error - rc].legacy_rc;
}
return -pcmk_err_generic;
}
//! \deprecated Use standard return codes instead
int
pcmk_legacy2rc(int legacy_rc)
{
legacy_rc = abs(legacy_rc);
switch (legacy_rc) {
case pcmk_err_no_quorum: return pcmk_rc_no_quorum;
case pcmk_err_schema_validation: return pcmk_rc_schema_validation;
case pcmk_err_schema_unchanged: return pcmk_rc_schema_unchanged;
case pcmk_err_transform_failed: return pcmk_rc_transform_failed;
case pcmk_err_old_data: return pcmk_rc_old_data;
case pcmk_err_diff_failed: return pcmk_rc_diff_failed;
case pcmk_err_diff_resync: return pcmk_rc_diff_resync;
case pcmk_err_cib_modified: return pcmk_rc_cib_modified;
case pcmk_err_cib_backup: return pcmk_rc_cib_backup;
case pcmk_err_cib_save: return pcmk_rc_cib_save;
case pcmk_err_cib_corrupt: return pcmk_rc_cib_corrupt;
case pcmk_err_multiple: return pcmk_rc_multiple;
case pcmk_err_node_unknown: return pcmk_rc_node_unknown;
case pcmk_err_already: return pcmk_rc_already;
case pcmk_err_bad_nvpair: return pcmk_rc_bad_nvpair;
case pcmk_err_unknown_format: return pcmk_rc_unknown_format;
case pcmk_err_generic: return pcmk_rc_error;
case pcmk_ok: return pcmk_rc_ok;
default: return legacy_rc; // system errno
}
}
// Exit status codes
const char *
crm_exit_name(crm_exit_t exit_code)
{
switch (exit_code) {
case CRM_EX_OK: return "CRM_EX_OK";
case CRM_EX_ERROR: return "CRM_EX_ERROR";
case CRM_EX_INVALID_PARAM: return "CRM_EX_INVALID_PARAM";
case CRM_EX_UNIMPLEMENT_FEATURE: return "CRM_EX_UNIMPLEMENT_FEATURE";
case CRM_EX_INSUFFICIENT_PRIV: return "CRM_EX_INSUFFICIENT_PRIV";
case CRM_EX_NOT_INSTALLED: return "CRM_EX_NOT_INSTALLED";
case CRM_EX_NOT_CONFIGURED: return "CRM_EX_NOT_CONFIGURED";
case CRM_EX_NOT_RUNNING: return "CRM_EX_NOT_RUNNING";
case CRM_EX_PROMOTED: return "CRM_EX_PROMOTED";
case CRM_EX_FAILED_PROMOTED: return "CRM_EX_FAILED_PROMOTED";
case CRM_EX_USAGE: return "CRM_EX_USAGE";
case CRM_EX_DATAERR: return "CRM_EX_DATAERR";
case CRM_EX_NOINPUT: return "CRM_EX_NOINPUT";
case CRM_EX_NOUSER: return "CRM_EX_NOUSER";
case CRM_EX_NOHOST: return "CRM_EX_NOHOST";
case CRM_EX_UNAVAILABLE: return "CRM_EX_UNAVAILABLE";
case CRM_EX_SOFTWARE: return "CRM_EX_SOFTWARE";
case CRM_EX_OSERR: return "CRM_EX_OSERR";
case CRM_EX_OSFILE: return "CRM_EX_OSFILE";
case CRM_EX_CANTCREAT: return "CRM_EX_CANTCREAT";
case CRM_EX_IOERR: return "CRM_EX_IOERR";
case CRM_EX_TEMPFAIL: return "CRM_EX_TEMPFAIL";
case CRM_EX_PROTOCOL: return "CRM_EX_PROTOCOL";
case CRM_EX_NOPERM: return "CRM_EX_NOPERM";
case CRM_EX_CONFIG: return "CRM_EX_CONFIG";
case CRM_EX_FATAL: return "CRM_EX_FATAL";
case CRM_EX_PANIC: return "CRM_EX_PANIC";
case CRM_EX_DISCONNECT: return "CRM_EX_DISCONNECT";
case CRM_EX_DIGEST: return "CRM_EX_DIGEST";
case CRM_EX_NOSUCH: return "CRM_EX_NOSUCH";
case CRM_EX_QUORUM: return "CRM_EX_QUORUM";
case CRM_EX_UNSAFE: return "CRM_EX_UNSAFE";
case CRM_EX_EXISTS: return "CRM_EX_EXISTS";
case CRM_EX_MULTIPLE: return "CRM_EX_MULTIPLE";
case CRM_EX_EXPIRED: return "CRM_EX_EXPIRED";
case CRM_EX_NOT_YET_IN_EFFECT: return "CRM_EX_NOT_YET_IN_EFFECT";
case CRM_EX_INDETERMINATE: return "CRM_EX_INDETERMINATE";
case CRM_EX_UNSATISFIED: return "CRM_EX_UNSATISFIED";
case CRM_EX_OLD: return "CRM_EX_OLD";
case CRM_EX_TIMEOUT: return "CRM_EX_TIMEOUT";
case CRM_EX_DEGRADED: return "CRM_EX_DEGRADED";
case CRM_EX_DEGRADED_PROMOTED: return "CRM_EX_DEGRADED_PROMOTED";
case CRM_EX_NONE: return "CRM_EX_NONE";
case CRM_EX_MAX: return "CRM_EX_UNKNOWN";
}
return "CRM_EX_UNKNOWN";
}
const char *
crm_exit_str(crm_exit_t exit_code)
{
switch (exit_code) {
case CRM_EX_OK: return "OK";
case CRM_EX_ERROR: return "Error occurred";
case CRM_EX_INVALID_PARAM: return "Invalid parameter";
case CRM_EX_UNIMPLEMENT_FEATURE: return "Unimplemented";
case CRM_EX_INSUFFICIENT_PRIV: return "Insufficient privileges";
case CRM_EX_NOT_INSTALLED: return "Not installed";
case CRM_EX_NOT_CONFIGURED: return "Not configured";
case CRM_EX_NOT_RUNNING: return "Not running";
case CRM_EX_PROMOTED: return "Promoted";
case CRM_EX_FAILED_PROMOTED: return "Failed in promoted role";
case CRM_EX_USAGE: return "Incorrect usage";
case CRM_EX_DATAERR: return "Invalid data given";
case CRM_EX_NOINPUT: return "Input file not available";
case CRM_EX_NOUSER: return "User does not exist";
case CRM_EX_NOHOST: return "Host does not exist";
case CRM_EX_UNAVAILABLE: return "Necessary service unavailable";
case CRM_EX_SOFTWARE: return "Internal software bug";
case CRM_EX_OSERR: return "Operating system error occurred";
case CRM_EX_OSFILE: return "System file not available";
case CRM_EX_CANTCREAT: return "Cannot create output file";
case CRM_EX_IOERR: return "I/O error occurred";
case CRM_EX_TEMPFAIL: return "Temporary failure, try again";
case CRM_EX_PROTOCOL: return "Protocol violated";
case CRM_EX_NOPERM: return "Insufficient privileges";
case CRM_EX_CONFIG: return "Invalid configuration";
case CRM_EX_FATAL: return "Fatal error occurred, will not respawn";
case CRM_EX_PANIC: return "System panic required";
case CRM_EX_DISCONNECT: return "Not connected";
case CRM_EX_DIGEST: return "Digest mismatch";
case CRM_EX_NOSUCH: return "No such object";
case CRM_EX_QUORUM: return "Quorum required";
case CRM_EX_UNSAFE: return "Operation not safe";
case CRM_EX_EXISTS: return "Requested item already exists";
case CRM_EX_MULTIPLE: return "Multiple items match request";
case CRM_EX_EXPIRED: return "Requested item has expired";
case CRM_EX_NOT_YET_IN_EFFECT: return "Requested item is not yet in effect";
case CRM_EX_INDETERMINATE: return "Could not determine status";
case CRM_EX_UNSATISFIED: return "Not applicable under current conditions";
case CRM_EX_OLD: return "Update was older than existing configuration";
case CRM_EX_TIMEOUT: return "Timeout occurred";
case CRM_EX_DEGRADED: return "Service is active but might fail soon";
case CRM_EX_DEGRADED_PROMOTED: return "Service is promoted but might fail soon";
case CRM_EX_NONE: return "No exit status available";
case CRM_EX_MAX: return "Error occurred";
}
if ((exit_code > 128) && (exit_code < CRM_EX_MAX)) {
return "Interrupted by signal";
}
return "Unknown exit status";
}
/*!
* \brief Map a function return code to the most similar exit code
*
* \param[in] rc Function return code
*
* \return Most similar exit code
*/
crm_exit_t
pcmk_rc2exitc(int rc)
{
switch (rc) {
case pcmk_rc_ok:
return CRM_EX_OK;
case pcmk_rc_no_quorum:
return CRM_EX_QUORUM;
case pcmk_rc_old_data:
return CRM_EX_OLD;
case pcmk_rc_schema_validation:
case pcmk_rc_transform_failed:
case pcmk_rc_unpack_error:
return CRM_EX_CONFIG;
case pcmk_rc_bad_nvpair:
return CRM_EX_INVALID_PARAM;
case EACCES:
return CRM_EX_INSUFFICIENT_PRIV;
case EBADF:
case EINVAL:
case EFAULT:
case ENOSYS:
case EOVERFLOW:
case pcmk_rc_underflow:
return CRM_EX_SOFTWARE;
case EBADMSG:
case EMSGSIZE:
case ENOMSG:
case ENOPROTOOPT:
case EPROTO:
case EPROTONOSUPPORT:
case EPROTOTYPE:
return CRM_EX_PROTOCOL;
case ECOMM:
case ENOMEM:
return CRM_EX_OSERR;
case ECONNABORTED:
case ECONNREFUSED:
case ECONNRESET:
case ENOTCONN:
return CRM_EX_DISCONNECT;
case EEXIST:
case pcmk_rc_already:
return CRM_EX_EXISTS;
case EIO:
case pcmk_rc_no_output:
case pcmk_rc_dot_error:
case pcmk_rc_graph_error:
return CRM_EX_IOERR;
case ENOTSUP:
#if EOPNOTSUPP != ENOTSUP
case EOPNOTSUPP:
#endif
return CRM_EX_UNIMPLEMENT_FEATURE;
case ENOTUNIQ:
case pcmk_rc_multiple:
return CRM_EX_MULTIPLE;
+ case ENODEV:
+ case ENOENT:
case ENXIO:
case pcmk_rc_node_unknown:
case pcmk_rc_unknown_format:
return CRM_EX_NOSUCH;
case ETIME:
case ETIMEDOUT:
return CRM_EX_TIMEOUT;
case EAGAIN:
case EBUSY:
return CRM_EX_UNSATISFIED;
case pcmk_rc_before_range:
return CRM_EX_NOT_YET_IN_EFFECT;
case pcmk_rc_after_range:
return CRM_EX_EXPIRED;
case pcmk_rc_undetermined:
return CRM_EX_INDETERMINATE;
case pcmk_rc_op_unsatisfied:
return CRM_EX_UNSATISFIED;
case pcmk_rc_within_range:
return CRM_EX_OK;
case pcmk_rc_no_input:
return CRM_EX_NOINPUT;
case pcmk_rc_duplicate_id:
return CRM_EX_MULTIPLE;
default:
return CRM_EX_ERROR;
}
}
/*!
* \brief Map a function return code to the most similar OCF exit code
*
* \param[in] rc Function return code
*
* \return Most similar OCF exit code
*/
enum ocf_exitcode
pcmk_rc2ocf(int rc)
{
switch (rc) {
case pcmk_rc_ok:
return PCMK_OCF_OK;
case pcmk_rc_bad_nvpair:
return PCMK_OCF_INVALID_PARAM;
case EACCES:
return PCMK_OCF_INSUFFICIENT_PRIV;
case ENOTSUP:
#if EOPNOTSUPP != ENOTSUP
case EOPNOTSUPP:
#endif
return PCMK_OCF_UNIMPLEMENT_FEATURE;
default:
return PCMK_OCF_UNKNOWN_ERROR;
}
}
// Other functions
const char *
bz2_strerror(int rc)
{
// See ftp://sources.redhat.com/pub/bzip2/docs/manual_3.html#SEC17
switch (rc) {
case BZ_OK:
case BZ_RUN_OK:
case BZ_FLUSH_OK:
case BZ_FINISH_OK:
case BZ_STREAM_END:
return "Ok";
case BZ_CONFIG_ERROR:
return "libbz2 has been improperly compiled on your platform";
case BZ_SEQUENCE_ERROR:
return "library functions called in the wrong order";
case BZ_PARAM_ERROR:
return "parameter is out of range or otherwise incorrect";
case BZ_MEM_ERROR:
return "memory allocation failed";
case BZ_DATA_ERROR:
return "data integrity error is detected during decompression";
case BZ_DATA_ERROR_MAGIC:
return "the compressed stream does not start with the correct magic bytes";
case BZ_IO_ERROR:
return "error reading or writing in the compressed file";
case BZ_UNEXPECTED_EOF:
return "compressed file finishes before the logical end of stream is detected";
case BZ_OUTBUFF_FULL:
return "output data will not fit into the buffer provided";
}
return "Data compression error";
}
crm_exit_t
crm_exit(crm_exit_t rc)
{
/* A compiler could theoretically use any type for crm_exit_t, but an int
* should always hold it, so cast to int to keep static analysis happy.
*/
if ((((int) rc) < 0) || (((int) rc) > CRM_EX_MAX)) {
rc = CRM_EX_ERROR;
}
mainloop_cleanup();
crm_xml_cleanup();
pcmk__cli_option_cleanup();
if (crm_system_name) {
crm_info("Exiting %s " CRM_XS " with status %d", crm_system_name, rc);
free(crm_system_name);
} else {
crm_trace("Exiting with status %d", rc);
}
qb_log_fini(); // Don't log anything after this point
exit(rc);
}
/*
* External action results
*/
/*!
* \internal
* \brief Set the result of an action
*
* \param[out] result Where to set action result
* \param[in] exit_status OCF exit status to set
* \param[in] exec_status Execution status to set
* \param[in] exit_reason Human-friendly description of event to set
*/
void
pcmk__set_result(pcmk__action_result_t *result, int exit_status,
enum pcmk_exec_status exec_status, const char *exit_reason)
{
if (result == NULL) {
return;
}
result->exit_status = exit_status;
result->execution_status = exec_status;
if (!pcmk__str_eq(result->exit_reason, exit_reason, pcmk__str_none)) {
free(result->exit_reason);
result->exit_reason = (exit_reason == NULL)? NULL : strdup(exit_reason);
}
}
/*!
* \internal
* \brief Set the result of an action, with a formatted exit reason
*
* \param[out] result Where to set action result
* \param[in] exit_status OCF exit status to set
* \param[in] exec_status Execution status to set
* \param[in] format printf-style format for a human-friendly
* description of reason for result
* \param[in] ... arguments for \p format
*/
G_GNUC_PRINTF(4, 5)
void
pcmk__format_result(pcmk__action_result_t *result, int exit_status,
enum pcmk_exec_status exec_status,
const char *format, ...)
{
va_list ap;
int len = 0;
char *reason = NULL;
if (result == NULL) {
return;
}
result->exit_status = exit_status;
result->execution_status = exec_status;
if (format != NULL) {
va_start(ap, format);
len = vasprintf(&reason, format, ap);
CRM_ASSERT(len > 0);
va_end(ap);
}
free(result->exit_reason);
result->exit_reason = reason;
}
/*!
* \internal
* \brief Set the output of an action
*
* \param[out] result Action result to set output for
* \param[in] out Action output to set (must be dynamically
* allocated)
* \param[in] err Action error output to set (must be dynamically
* allocated)
*
* \note \p result will take ownership of \p out and \p err, so the caller
* should not free them.
*/
void
pcmk__set_result_output(pcmk__action_result_t *result, char *out, char *err)
{
if (result == NULL) {
return;
}
free(result->action_stdout);
result->action_stdout = out;
free(result->action_stderr);
result->action_stderr = err;
}
/*!
* \internal
* \brief Clear a result's exit reason, output, and error output
*
* \param[in] result Result to reset
*/
void
pcmk__reset_result(pcmk__action_result_t *result)
{
if (result == NULL) {
return;
}
free(result->exit_reason);
result->exit_reason = NULL;
free(result->action_stdout);
result->action_stdout = NULL;
free(result->action_stderr);
result->action_stderr = NULL;
}
/*!
* \internal
* \brief Copy the result of an action
*
* \param[in] src Result to copy
* \param[out] dst Where to copy \p src to
*/
void
pcmk__copy_result(pcmk__action_result_t *src, pcmk__action_result_t *dst)
{
CRM_CHECK((src != NULL) && (dst != NULL), return);
dst->exit_status = src->exit_status;
dst->execution_status = src->execution_status;
pcmk__str_update(&src->exit_reason, dst->exit_reason);
pcmk__str_update(&src->action_stdout, dst->action_stdout);
pcmk__str_update(&src->action_stderr, dst->action_stderr);
}
// Deprecated functions kept only for backward API compatibility
// LCOV_EXCL_START
#include <crm/common/results_compat.h>
crm_exit_t
crm_errno2exit(int rc)
{
return pcmk_rc2exitc(pcmk_legacy2rc(rc));
}
// LCOV_EXCL_STOP
// End deprecated API
diff --git a/lib/fencing/st_actions.c b/lib/fencing/st_actions.c
index 38618ab937..d7fc170a5b 100644
--- a/lib/fencing/st_actions.c
+++ b/lib/fencing/st_actions.c
@@ -1,695 +1,710 @@
/*
* Copyright 2004-2022 the Pacemaker project contributors
*
* The version control history for this file may have further details.
*
* This source code is licensed under the GNU Lesser General Public License
* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
*/
#include <crm_internal.h>
+#include <stdbool.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <libgen.h>
#include <inttypes.h>
#include <sys/types.h>
#include <glib.h>
#include <crm/crm.h>
#include <crm/stonith-ng.h>
#include <crm/fencing/internal.h>
#include <crm/msg_xml.h>
#include <crm/services_internal.h>
#include "fencing_private.h"
struct stonith_action_s {
/*! user defined data */
char *agent;
char *action;
GHashTable *args;
int timeout;
- int async;
+ bool async;
void *userdata;
void (*done_cb) (int pid, const pcmk__action_result_t *result,
void *user_data);
void (*fork_cb) (int pid, void *user_data);
svc_action_t *svc_action;
/*! internal timing information */
time_t initial_start_time;
int tries;
int remaining_timeout;
int max_retries;
int pid;
pcmk__action_result_t result;
};
static int internal_stonith_action_execute(stonith_action_t *action);
static void log_action(stonith_action_t *action, pid_t pid);
/*!
* \internal
* \brief Set an action's result based on services library result
*
* \param[in] action Fence action to set result for
* \param[in] svc_action Service action to get result from
*/
static void
set_result_from_svc_action(stonith_action_t *action, svc_action_t *svc_action)
{
pcmk__set_result(&(action->result), svc_action->rc, svc_action->status,
services__exit_reason(svc_action));
pcmk__set_result_output(&(action->result),
services__grab_stdout(svc_action),
services__grab_stderr(svc_action));
}
static void
log_action(stonith_action_t *action, pid_t pid)
{
/* The services library has already logged the output at info or debug
* level, so just raise to warning for stderr.
*/
if (action->result.action_stderr != NULL) {
/* Logging the whole string confuses syslog when the string is xml */
char *prefix = crm_strdup_printf("%s[%d] stderr:", action->agent, pid);
crm_log_output(LOG_WARNING, prefix, action->result.action_stderr);
free(prefix);
}
}
static void
append_config_arg(gpointer key, gpointer value, gpointer user_data)
{
/* The fencer will filter "action" out when it registers the device,
* but ignore it here in case any external API users don't.
*
* Also filter out parameters handled directly by Pacemaker.
*/
if (!pcmk__str_eq(key, STONITH_ATTR_ACTION_OP, pcmk__str_casei)
&& !pcmk_stonith_param(key)
&& (strstr(key, CRM_META) == NULL)
&& !pcmk__str_eq(key, "crm_feature_set", pcmk__str_casei)) {
crm_trace("Passing %s=%s with fence action",
(const char *) key, (const char *) (value? value : ""));
g_hash_table_insert((GHashTable *) user_data,
strdup(key), strdup(value? value : ""));
}
}
/*!
* \internal
* \brief Create a table of arguments for a fencing action
*
* \param[in] agent Fencing agent name
* \param[in] action Name of fencing action
* \param[in] target Name of target node for fencing action
* \param[in] target_nodeid Node ID of target node for fencing action
* \param[in] device_args Fence device parameters
* \param[in] port_map Target node-to-port mapping for fence device
* \param[in] host_arg Argument name for passing target
*
* \return Newly created hash table of arguments for fencing action
*/
static GHashTable *
make_args(const char *agent, const char *action, const char *target,
uint32_t target_nodeid, GHashTable *device_args,
GHashTable *port_map, const char *host_arg)
{
GHashTable *arg_list = NULL;
const char *value = NULL;
CRM_CHECK(action != NULL, return NULL);
arg_list = pcmk__strkey_table(free, free);
// Add action to arguments (using an alias if requested)
if (device_args) {
char buffer[512];
snprintf(buffer, sizeof(buffer), "pcmk_%s_action", action);
value = g_hash_table_lookup(device_args, buffer);
if (value) {
crm_debug("Substituting '%s' for fence action %s targeting %s",
value, action, pcmk__s(target, "no node"));
action = value;
}
}
g_hash_table_insert(arg_list, strdup(STONITH_ATTR_ACTION_OP),
strdup(action));
/* If this is a fencing operation against another node, add more standard
* arguments.
*/
if ((target != NULL) && (device_args != NULL)) {
const char *param = NULL;
/* Always pass the target's name, per
* https://github.com/ClusterLabs/fence-agents/blob/main/doc/FenceAgentAPI.md
*/
g_hash_table_insert(arg_list, strdup("nodename"), strdup(target));
// If the target's node ID was specified, pass it, too
if (target_nodeid != 0) {
char *nodeid = crm_strdup_printf("%" PRIu32, target_nodeid);
// cts-fencing looks for this log message
crm_info("Passing '%s' as nodeid with fence action '%s' targeting %s",
nodeid, action, pcmk__s(target, "no node"));
g_hash_table_insert(arg_list, strdup("nodeid"), nodeid);
}
// Check whether target must be specified in some other way
param = g_hash_table_lookup(device_args, PCMK_STONITH_HOST_ARGUMENT);
if (!pcmk__str_eq(agent, "fence_legacy", pcmk__str_none)
&& !pcmk__str_eq(param, PCMK__VALUE_NONE, pcmk__str_casei)) {
if (param == NULL) {
/* Use the caller's default for pcmk_host_argument, or "port" if
* none was given
*/
param = (host_arg == NULL)? "port" : host_arg;
}
value = g_hash_table_lookup(device_args, param);
if (pcmk__str_eq(value, "dynamic",
pcmk__str_casei|pcmk__str_null_matches)) {
/* If the host argument was "dynamic" or not explicitly specified,
* add it with the target
*/
const char *alias = NULL;
if (port_map) {
alias = g_hash_table_lookup(port_map, target);
}
if (alias == NULL) {
alias = target;
}
crm_debug("Passing %s='%s' with fence action %s targeting %s",
param, alias, action, pcmk__s(target, "no node"));
g_hash_table_insert(arg_list, strdup(param), strdup(alias));
}
}
}
if (device_args) {
g_hash_table_foreach(device_args, append_config_arg, arg_list);
}
return arg_list;
}
/*!
* \internal
* \brief Free all memory used by a stonith action
*
* \param[in,out] action Action to free
*/
void
stonith__destroy_action(stonith_action_t *action)
{
if (action) {
free(action->agent);
if (action->args) {
g_hash_table_destroy(action->args);
}
free(action->action);
if (action->svc_action) {
services_action_free(action->svc_action);
}
pcmk__reset_result(&(action->result));
free(action);
}
}
/*!
* \internal
* \brief Get the result of an executed stonith action
*
* \param[in] action Executed action
*
* \return Pointer to action's result (or NULL if \p action is NULL)
*/
pcmk__action_result_t *
stonith__action_result(stonith_action_t *action)
{
return (action == NULL)? NULL : &(action->result);
}
#define FAILURE_MAX_RETRIES 2
+
+/*!
+ * \internal
+ * \brief Create a new fencing action to be executed
+ *
+ * \param[in] agent Fence agent to use
+ * \param[in] action_name Fencing action to be executed
+ * \param[in] target Name of target of fencing action (if known)
+ * \param[in] target_nodeid Node ID of target of fencing action (if known)
+ * \param[in] timeout_sec Timeout to be used when executing action
+ * \param[in] device_args Parameters to pass to fence agent
+ * \param[in] port_map Mapping of target names to device ports
+ * \param[in] host_arg Agent parameter used to pass target name
+ *
+ * \return Newly created fencing action (asserts on error, never NULL)
+ */
stonith_action_t *
-stonith__action_create(const char *agent, const char *_action,
+stonith__action_create(const char *agent, const char *action_name,
const char *target, uint32_t target_nodeid,
- int timeout, GHashTable *device_args,
+ int timeout_sec, GHashTable *device_args,
GHashTable *port_map, const char *host_arg)
{
- stonith_action_t *action;
+ stonith_action_t *action = calloc(1, sizeof(stonith_action_t));
- action = calloc(1, sizeof(stonith_action_t));
CRM_ASSERT(action != NULL);
- action->args = make_args(agent, _action, target, target_nodeid,
+ action->args = make_args(agent, action_name, target, target_nodeid,
device_args, port_map, host_arg);
crm_debug("Preparing '%s' action targeting %s using agent %s",
- _action, pcmk__s(target, "no node"), agent);
+ action_name, pcmk__s(target, "no node"), agent);
action->agent = strdup(agent);
- action->action = strdup(_action);
- action->timeout = action->remaining_timeout = timeout;
+ action->action = strdup(action_name);
+ action->timeout = action->remaining_timeout = timeout_sec;
action->max_retries = FAILURE_MAX_RETRIES;
pcmk__set_result(&(action->result), PCMK_OCF_UNKNOWN, PCMK_EXEC_UNKNOWN,
"Initialization bug in fencing library");
if (device_args) {
char buffer[512];
const char *value = NULL;
- snprintf(buffer, sizeof(buffer), "pcmk_%s_retries", _action);
+ snprintf(buffer, sizeof(buffer), "pcmk_%s_retries", action_name);
value = g_hash_table_lookup(device_args, buffer);
if (value) {
action->max_retries = atoi(value);
}
}
return action;
}
static gboolean
update_remaining_timeout(stonith_action_t * action)
{
int diff = time(NULL) - action->initial_start_time;
if (action->tries >= action->max_retries) {
crm_info("Attempted to execute agent %s (%s) the maximum number of times (%d) allowed",
action->agent, action->action, action->max_retries);
action->remaining_timeout = 0;
} else if ((action->result.execution_status != PCMK_EXEC_TIMEOUT)
&& (diff < (action->timeout * 0.7))) {
/* only set remaining timeout period if there is 30%
* or greater of the original timeout period left */
action->remaining_timeout = action->timeout - diff;
} else {
action->remaining_timeout = 0;
}
return action->remaining_timeout ? TRUE : FALSE;
}
/*!
* \internal
* \brief Map a fencing action result to a standard return code
*
* \param[in] result Fencing action result to map
*
* \return Standard Pacemaker return code that best corresponds to \p result
*/
int
stonith__result2rc(const pcmk__action_result_t *result)
{
if (pcmk__result_ok(result)) {
return pcmk_rc_ok;
}
switch (result->execution_status) {
case PCMK_EXEC_PENDING: return EINPROGRESS;
case PCMK_EXEC_CANCELLED: return ECANCELED;
case PCMK_EXEC_TIMEOUT: return ETIME;
case PCMK_EXEC_NOT_INSTALLED: return ENOENT;
case PCMK_EXEC_NOT_SUPPORTED: return EOPNOTSUPP;
case PCMK_EXEC_NOT_CONNECTED: return ENOTCONN;
case PCMK_EXEC_NO_FENCE_DEVICE: return ENODEV;
case PCMK_EXEC_NO_SECRETS: return EACCES;
/* For the fencing API, PCMK_EXEC_INVALID is used with fencer API
* operations that don't involve executing an agent (for example,
* registering devices). This allows us to use the CRM_EX_* codes in the
* exit status for finer-grained responses.
*/
case PCMK_EXEC_INVALID:
switch (result->exit_status) {
case CRM_EX_INVALID_PARAM: return EINVAL;
case CRM_EX_INSUFFICIENT_PRIV: return EACCES;
case CRM_EX_PROTOCOL: return EPROTO;
/* CRM_EX_EXPIRED is used for orphaned fencing operations left
* over from a previous instance of the fencer. For API backward
* compatibility, this is mapped to the previously used code for
* this case, EHOSTUNREACH.
*/
case CRM_EX_EXPIRED: return EHOSTUNREACH;
default: break;
}
break;
default:
break;
}
// Try to provide useful error code based on result's error output
if (result->action_stderr == NULL) {
return ENODATA;
} else if (strcasestr(result->action_stderr, "timed out")
|| strcasestr(result->action_stderr, "timeout")) {
return ETIME;
} else if (strcasestr(result->action_stderr, "unrecognised action")
|| strcasestr(result->action_stderr, "unrecognized action")
|| strcasestr(result->action_stderr, "unsupported action")) {
return EOPNOTSUPP;
}
// Oh well, we tried
return pcmk_rc_error;
}
/*!
* \internal
* \brief Determine execution status equivalent of legacy fencer return code
*
* Fence action notifications, and fence action callbacks from older fencers
* (<=2.1.2) in a rolling upgrade, will have only a legacy return code. Map this
* to an execution status as best as possible (essentially, the inverse of
* stonith__result2rc()).
*
* \param[in] rc Legacy return code from fencer
*
* \return Execution status best corresponding to \p rc
*/
int
stonith__legacy2status(int rc)
{
if (rc >= 0) {
return PCMK_EXEC_DONE;
}
switch (-rc) {
case EACCES: return PCMK_EXEC_NO_SECRETS;
case ECANCELED: return PCMK_EXEC_CANCELLED;
case EHOSTUNREACH: return PCMK_EXEC_INVALID;
case EINPROGRESS: return PCMK_EXEC_PENDING;
case ENODEV: return PCMK_EXEC_NO_FENCE_DEVICE;
case ENOENT: return PCMK_EXEC_NOT_INSTALLED;
case ENOTCONN: return PCMK_EXEC_NOT_CONNECTED;
case EOPNOTSUPP: return PCMK_EXEC_NOT_SUPPORTED;
case EPROTO: return PCMK_EXEC_INVALID;
case EPROTONOSUPPORT: return PCMK_EXEC_NOT_SUPPORTED;
case ETIME: return PCMK_EXEC_TIMEOUT;
case ETIMEDOUT: return PCMK_EXEC_TIMEOUT;
default: return PCMK_EXEC_ERROR;
}
}
/*!
* \internal
* \brief Add a fencing result to an XML element as attributes
*
* \param[in] xml XML element to add result to
* \param[in] result Fencing result to add (assume success if NULL)
*/
void
stonith__xe_set_result(xmlNode *xml, const pcmk__action_result_t *result)
{
int exit_status = CRM_EX_OK;
enum pcmk_exec_status execution_status = PCMK_EXEC_DONE;
const char *exit_reason = NULL;
const char *action_stdout = NULL;
int rc = pcmk_ok;
CRM_CHECK(xml != NULL, return);
if (result != NULL) {
exit_status = result->exit_status;
execution_status = result->execution_status;
exit_reason = result->exit_reason;
action_stdout = result->action_stdout;
rc = pcmk_rc2legacy(stonith__result2rc(result));
}
crm_xml_add_int(xml, XML_LRM_ATTR_OPSTATUS, (int) execution_status);
crm_xml_add_int(xml, XML_LRM_ATTR_RC, exit_status);
crm_xml_add(xml, XML_LRM_ATTR_EXIT_REASON, exit_reason);
crm_xml_add(xml, F_STONITH_OUTPUT, action_stdout);
/* @COMPAT Peers in rolling upgrades, Pacemaker Remote nodes, and external
* code that use libstonithd <=2.1.2 don't check for the full result, and
* need a legacy return code instead.
*/
crm_xml_add_int(xml, F_STONITH_RC, rc);
}
/*!
* \internal
* \brief Find a fencing result beneath an XML element
*
* \param[in] xml XML element to search
*
* \return \p xml or descendent of it that contains a fencing result, else NULL
*/
xmlNode *
stonith__find_xe_with_result(xmlNode *xml)
{
xmlNode *match = get_xpath_object("//@" XML_LRM_ATTR_RC, xml, LOG_NEVER);
if (match == NULL) {
/* @COMPAT Peers <=2.1.2 in a rolling upgrade provide only a legacy
* return code, not a full result, so check for that.
*/
match = get_xpath_object("//@" F_STONITH_RC, xml, LOG_ERR);
}
return match;
}
/*!
* \internal
* \brief Get a fencing result from an XML element's attributes
*
* \param[in] xml XML element with fencing result
* \param[out] result Where to store fencing result
*/
void
stonith__xe_get_result(xmlNode *xml, pcmk__action_result_t *result)
{
int exit_status = CRM_EX_OK;
int execution_status = PCMK_EXEC_DONE;
const char *exit_reason = NULL;
char *action_stdout = NULL;
CRM_CHECK((xml != NULL) && (result != NULL), return);
exit_reason = crm_element_value(xml, XML_LRM_ATTR_EXIT_REASON);
action_stdout = crm_element_value_copy(xml, F_STONITH_OUTPUT);
// A result must include an exit status and execution status
if ((crm_element_value_int(xml, XML_LRM_ATTR_RC, &exit_status) < 0)
|| (crm_element_value_int(xml, XML_LRM_ATTR_OPSTATUS,
&execution_status) < 0)) {
int rc = pcmk_ok;
exit_status = CRM_EX_ERROR;
/* @COMPAT Peers <=2.1.2 in rolling upgrades provide only a legacy
* return code, not a full result, so check for that.
*/
if (crm_element_value_int(xml, F_STONITH_RC, &rc) == 0) {
if ((rc == pcmk_ok) || (rc == -EINPROGRESS)) {
exit_status = CRM_EX_OK;
}
execution_status = stonith__legacy2status(rc);
exit_reason = pcmk_strerror(rc);
} else {
execution_status = PCMK_EXEC_ERROR;
exit_reason = "Fencer reply contained neither a full result "
"nor a legacy return code (bug?)";
}
}
pcmk__set_result(result, exit_status, execution_status, exit_reason);
pcmk__set_result_output(result, action_stdout, NULL);
}
static void
stonith_action_async_done(svc_action_t *svc_action)
{
stonith_action_t *action = (stonith_action_t *) svc_action->cb_data;
set_result_from_svc_action(action, svc_action);
svc_action->params = NULL;
log_action(action, action->pid);
if (!pcmk__result_ok(&(action->result))
&& update_remaining_timeout(action)) {
int rc = internal_stonith_action_execute(action);
if (rc == pcmk_ok) {
return;
}
}
if (action->done_cb) {
action->done_cb(action->pid, &(action->result), action->userdata);
}
action->svc_action = NULL; // don't remove our caller
stonith__destroy_action(action);
}
static void
stonith_action_async_forked(svc_action_t *svc_action)
{
stonith_action_t *action = (stonith_action_t *) svc_action->cb_data;
action->pid = svc_action->pid;
action->svc_action = svc_action;
if (action->fork_cb) {
(action->fork_cb) (svc_action->pid, action->userdata);
}
pcmk__set_result(&(action->result), PCMK_OCF_UNKNOWN, PCMK_EXEC_PENDING,
NULL);
crm_trace("Child process %d performing action '%s' successfully forked",
action->pid, action->action);
}
static int
internal_stonith_action_execute(stonith_action_t * action)
{
int rc = -EPROTO;
int is_retry = 0;
svc_action_t *svc_action = NULL;
static int stonith_sequence = 0;
char *buffer = NULL;
CRM_CHECK(action != NULL, return -EINVAL);
if ((action->action == NULL) || (action->args == NULL)
|| (action->agent == NULL)) {
pcmk__set_result(&(action->result), PCMK_OCF_UNKNOWN_ERROR,
PCMK_EXEC_ERROR_FATAL, "Bug in fencing library");
return -EINVAL;
}
if (!action->tries) {
action->initial_start_time = time(NULL);
}
action->tries++;
if (action->tries > 1) {
crm_info("Attempt %d to execute %s (%s). remaining timeout is %d",
action->tries, action->agent, action->action, action->remaining_timeout);
is_retry = 1;
}
buffer = crm_strdup_printf(PCMK__FENCE_BINDIR "/%s",
basename(action->agent));
svc_action = services_action_create_generic(buffer, NULL);
free(buffer);
if (svc_action->rc != PCMK_OCF_UNKNOWN) {
set_result_from_svc_action(action, svc_action);
services_action_free(svc_action);
return -E2BIG;
}
svc_action->timeout = 1000 * action->remaining_timeout;
svc_action->standard = strdup(PCMK_RESOURCE_CLASS_STONITH);
svc_action->id = crm_strdup_printf("%s_%s_%dof%d", basename(action->agent),
action->action, action->tries,
action->max_retries);
svc_action->agent = strdup(action->agent);
svc_action->sequence = stonith_sequence++;
svc_action->params = action->args;
svc_action->cb_data = (void *) action;
svc_action->flags = pcmk__set_flags_as(__func__, __LINE__,
LOG_TRACE, "Action",
svc_action->id, svc_action->flags,
SVC_ACTION_NON_BLOCKED,
"SVC_ACTION_NON_BLOCKED");
/* keep retries from executing out of control and free previous results */
if (is_retry) {
pcmk__reset_result(&(action->result));
sleep(1);
}
if (action->async) {
// We never create a recurring action, so this should always return TRUE
CRM_LOG_ASSERT(services_action_async_fork_notify(svc_action,
&stonith_action_async_done,
&stonith_action_async_forked));
return pcmk_ok;
} else if (services_action_sync(svc_action)) { // sync success
rc = pcmk_ok;
} else { // sync failure
rc = -ECONNABORTED;
}
set_result_from_svc_action(action, svc_action);
svc_action->params = NULL;
services_action_free(svc_action);
return rc;
}
/*!
* \internal
* \brief Kick off execution of an async stonith action
*
* \param[in,out] action Action to be executed
* \param[in,out] userdata Datapointer to be passed to callbacks
* \param[in] done Callback to notify action has failed/succeeded
* \param[in] fork_callback Callback to notify successful fork of child
*
* \return pcmk_ok if ownership of action has been taken, -errno otherwise
*/
int
-stonith_action_execute_async(stonith_action_t * action,
- void *userdata,
- void (*done) (int pid,
- const pcmk__action_result_t *result,
- void *user_data),
- void (*fork_cb) (int pid, void *user_data))
+stonith__execute_async(stonith_action_t * action, void *userdata,
+ void (*done) (int pid,
+ const pcmk__action_result_t *result,
+ void *user_data),
+ void (*fork_cb) (int pid, void *user_data))
{
if (!action) {
return -EINVAL;
}
action->userdata = userdata;
action->done_cb = done;
action->fork_cb = fork_cb;
- action->async = 1;
+ action->async = true;
return internal_stonith_action_execute(action);
}
/*!
* \internal
* \brief Execute a stonith action
*
* \param[in,out] action Action to execute
*
* \return pcmk_ok on success, -errno otherwise
*/
int
stonith__execute(stonith_action_t *action)
{
int rc = pcmk_ok;
CRM_CHECK(action != NULL, return -EINVAL);
// Keep trying until success, max retries, or timeout
do {
rc = internal_stonith_action_execute(action);
} while ((rc != pcmk_ok) && update_remaining_timeout(action));
return rc;
}
diff --git a/lib/fencing/st_client.c b/lib/fencing/st_client.c
index 1f04124e18..8a5503d7fb 100644
--- a/lib/fencing/st_client.c
+++ b/lib/fencing/st_client.c
@@ -1,2609 +1,2699 @@
/*
* Copyright 2004-2022 the Pacemaker project contributors
*
* The version control history for this file may have further details.
*
* This source code is licensed under the GNU Lesser General Public License
* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
*/
#include <crm_internal.h>
#include <stdlib.h>
#include <stdio.h>
#include <stdbool.h>
#include <string.h>
#include <ctype.h>
#include <inttypes.h>
#include <sys/types.h>
#include <glib.h>
#include <crm/crm.h>
#include <crm/stonith-ng.h>
#include <crm/fencing/internal.h>
#include <crm/msg_xml.h>
#include <crm/common/mainloop.h>
#include "fencing_private.h"
CRM_TRACE_INIT_DATA(stonith);
// Used as stonith_t:st_private
typedef struct stonith_private_s {
char *token;
crm_ipc_t *ipc;
mainloop_io_t *source;
GHashTable *stonith_op_callback_table;
GList *notify_list;
int notify_refcnt;
bool notify_deletes;
void (*op_callback) (stonith_t * st, stonith_callback_data_t * data);
} stonith_private_t;
// Used as stonith_event_t:opaque
struct event_private {
pcmk__action_result_t result;
};
typedef struct stonith_notify_client_s {
const char *event;
const char *obj_id; /* implement one day */
const char *obj_type; /* implement one day */
void (*notify) (stonith_t * st, stonith_event_t * e);
bool delete;
} stonith_notify_client_t;
typedef struct stonith_callback_client_s {
void (*callback) (stonith_t * st, stonith_callback_data_t * data);
const char *id;
void *user_data;
gboolean only_success;
gboolean allow_timeout_updates;
struct timer_rec_s *timer;
} stonith_callback_client_t;
struct notify_blob_s {
stonith_t *stonith;
xmlNode *xml;
};
struct timer_rec_s {
int call_id;
int timeout;
guint ref;
stonith_t *stonith;
};
typedef int (*stonith_op_t) (const char *, int, const char *, xmlNode *,
xmlNode *, xmlNode *, xmlNode **, xmlNode **);
bool stonith_dispatch(stonith_t * st);
xmlNode *stonith_create_op(int call_id, const char *token, const char *op, xmlNode * data,
int call_options);
static int stonith_send_command(stonith_t *stonith, const char *op,
xmlNode *data, xmlNode **output_data,
int call_options, int timeout);
static void stonith_connection_destroy(gpointer user_data);
static void stonith_send_notification(gpointer data, gpointer user_data);
static int stonith_api_del_notification(stonith_t *stonith,
const char *event);
/*!
* \brief Get agent namespace by name
*
* \param[in] namespace_s Name of namespace as string
*
* \return Namespace as enum value
*/
enum stonith_namespace
stonith_text2namespace(const char *namespace_s)
{
if (pcmk__str_eq(namespace_s, "any", pcmk__str_null_matches)) {
return st_namespace_any;
} else if (!strcmp(namespace_s, "redhat")
|| !strcmp(namespace_s, "stonith-ng")) {
return st_namespace_rhcs;
} else if (!strcmp(namespace_s, "internal")) {
return st_namespace_internal;
} else if (!strcmp(namespace_s, "heartbeat")) {
return st_namespace_lha;
}
return st_namespace_invalid;
}
/*!
* \brief Get agent namespace name
*
* \param[in] namespace Namespace as enum value
*
* \return Namespace name as string
*/
const char *
stonith_namespace2text(enum stonith_namespace st_namespace)
{
switch (st_namespace) {
case st_namespace_any: return "any";
case st_namespace_rhcs: return "stonith-ng";
case st_namespace_internal: return "internal";
case st_namespace_lha: return "heartbeat";
default: break;
}
return "unsupported";
}
/*!
* \brief Determine namespace of a fence agent
*
* \param[in] agent Fence agent type
* \param[in] namespace_s Name of agent namespace as string, if known
*
* \return Namespace of specified agent, as enum value
*/
enum stonith_namespace
stonith_get_namespace(const char *agent, const char *namespace_s)
{
if (pcmk__str_eq(namespace_s, "internal", pcmk__str_none)) {
return st_namespace_internal;
}
if (stonith__agent_is_rhcs(agent)) {
return st_namespace_rhcs;
}
#if HAVE_STONITH_STONITH_H
if (stonith__agent_is_lha(agent)) {
return st_namespace_lha;
}
#endif
crm_err("Unknown fence agent: %s", agent);
return st_namespace_invalid;
}
gboolean
stonith__watchdog_fencing_enabled_for_node_api(stonith_t *st, const char *node)
{
gboolean rv = FALSE;
stonith_t *stonith_api = st?st:stonith_api_new();
char *list = NULL;
if(stonith_api) {
if (stonith_api->state == stonith_disconnected) {
int rc = stonith_api->cmds->connect(stonith_api, "stonith-api", NULL);
if (rc != pcmk_ok) {
crm_err("Failed connecting to Stonith-API for watchdog-fencing-query.");
}
}
if (stonith_api->state != stonith_disconnected) {
/* caveat!!!
* this might fail when when stonithd is just updating the device-list
* probably something we should fix as well for other api-calls */
int rc = stonith_api->cmds->list(stonith_api, st_opt_sync_call, STONITH_WATCHDOG_ID, &list, 0);
if ((rc != pcmk_ok) || (list == NULL)) {
/* due to the race described above it can happen that
* we drop in here - so as not to make remote nodes
* panic on that answer
*/
if (rc == -ENODEV) {
crm_notice("Cluster does not have watchdog fencing device");
} else {
crm_warn("Could not check for watchdog fencing device: %s",
pcmk_strerror(rc));
}
} else if (list[0] == '\0') {
rv = TRUE;
} else {
GList *targets = stonith__parse_targets(list);
rv = pcmk__str_in_list(node, targets, pcmk__str_casei);
g_list_free_full(targets, free);
}
free(list);
if (!st) {
/* if we're provided the api we still might have done the
* connection - but let's assume the caller won't bother
*/
stonith_api->cmds->disconnect(stonith_api);
}
}
if (!st) {
stonith_api_delete(stonith_api);
}
} else {
crm_err("Stonith-API for watchdog-fencing-query couldn't be created.");
}
crm_trace("Pacemaker assumes node %s %sto do watchdog-fencing.",
node, rv?"":"not ");
return rv;
}
gboolean
stonith__watchdog_fencing_enabled_for_node(const char *node)
{
return stonith__watchdog_fencing_enabled_for_node_api(NULL, node);
}
/* when cycling through the list we don't want to delete items
so just mark them and when we know nobody is using the list
loop over it to remove the marked items
*/
static void
foreach_notify_entry (stonith_private_t *private,
GFunc func,
gpointer user_data)
{
private->notify_refcnt++;
g_list_foreach(private->notify_list, func, user_data);
private->notify_refcnt--;
if ((private->notify_refcnt == 0) &&
private->notify_deletes) {
GList *list_item = private->notify_list;
private->notify_deletes = FALSE;
while (list_item != NULL)
{
stonith_notify_client_t *list_client = list_item->data;
GList *next = g_list_next(list_item);
if (list_client->delete) {
free(list_client);
private->notify_list =
g_list_delete_link(private->notify_list, list_item);
}
list_item = next;
}
}
}
static void
stonith_connection_destroy(gpointer user_data)
{
stonith_t *stonith = user_data;
stonith_private_t *native = NULL;
struct notify_blob_s blob;
crm_trace("Sending destroyed notification");
blob.stonith = stonith;
blob.xml = create_xml_node(NULL, "notify");
native = stonith->st_private;
native->ipc = NULL;
native->source = NULL;
free(native->token); native->token = NULL;
stonith->state = stonith_disconnected;
crm_xml_add(blob.xml, F_TYPE, T_STONITH_NOTIFY);
crm_xml_add(blob.xml, F_SUBTYPE, T_STONITH_NOTIFY_DISCONNECT);
foreach_notify_entry(native, stonith_send_notification, &blob);
free_xml(blob.xml);
}
xmlNode *
create_device_registration_xml(const char *id, enum stonith_namespace namespace,
const char *agent, stonith_key_value_t *params,
const char *rsc_provides)
{
xmlNode *data = create_xml_node(NULL, F_STONITH_DEVICE);
xmlNode *args = create_xml_node(data, XML_TAG_ATTRS);
#if HAVE_STONITH_STONITH_H
if (namespace == st_namespace_any) {
namespace = stonith_get_namespace(agent, NULL);
}
if (namespace == st_namespace_lha) {
hash2field((gpointer) "plugin", (gpointer) agent, args);
agent = "fence_legacy";
}
#endif
crm_xml_add(data, XML_ATTR_ID, id);
crm_xml_add(data, F_STONITH_ORIGIN, __func__);
crm_xml_add(data, "agent", agent);
if ((namespace != st_namespace_any) && (namespace != st_namespace_invalid)) {
crm_xml_add(data, "namespace", stonith_namespace2text(namespace));
}
if (rsc_provides) {
crm_xml_add(data, "rsc_provides", rsc_provides);
}
for (; params; params = params->next) {
hash2field((gpointer) params->key, (gpointer) params->value, args);
}
return data;
}
static int
stonith_api_register_device(stonith_t * st, int call_options,
const char *id, const char *namespace, const char *agent,
stonith_key_value_t * params)
{
int rc = 0;
xmlNode *data = NULL;
data = create_device_registration_xml(id, stonith_text2namespace(namespace),
agent, params, NULL);
rc = stonith_send_command(st, STONITH_OP_DEVICE_ADD, data, NULL, call_options, 0);
free_xml(data);
return rc;
}
static int
stonith_api_remove_device(stonith_t * st, int call_options, const char *name)
{
int rc = 0;
xmlNode *data = NULL;
data = create_xml_node(NULL, F_STONITH_DEVICE);
crm_xml_add(data, F_STONITH_ORIGIN, __func__);
crm_xml_add(data, XML_ATTR_ID, name);
rc = stonith_send_command(st, STONITH_OP_DEVICE_DEL, data, NULL, call_options, 0);
free_xml(data);
return rc;
}
static int
stonith_api_remove_level_full(stonith_t *st, int options,
const char *node, const char *pattern,
const char *attr, const char *value, int level)
{
int rc = 0;
xmlNode *data = NULL;
CRM_CHECK(node || pattern || (attr && value), return -EINVAL);
data = create_xml_node(NULL, XML_TAG_FENCING_LEVEL);
crm_xml_add(data, F_STONITH_ORIGIN, __func__);
if (node) {
crm_xml_add(data, XML_ATTR_STONITH_TARGET, node);
} else if (pattern) {
crm_xml_add(data, XML_ATTR_STONITH_TARGET_PATTERN, pattern);
} else {
crm_xml_add(data, XML_ATTR_STONITH_TARGET_ATTRIBUTE, attr);
crm_xml_add(data, XML_ATTR_STONITH_TARGET_VALUE, value);
}
crm_xml_add_int(data, XML_ATTR_STONITH_INDEX, level);
rc = stonith_send_command(st, STONITH_OP_LEVEL_DEL, data, NULL, options, 0);
free_xml(data);
return rc;
}
static int
stonith_api_remove_level(stonith_t * st, int options, const char *node, int level)
{
return stonith_api_remove_level_full(st, options, node,
NULL, NULL, NULL, level);
}
/*!
* \internal
* \brief Create XML for fence topology level registration request
*
* \param[in] node If not NULL, target level by this node name
* \param[in] pattern If not NULL, target by node name using this regex
* \param[in] attr If not NULL, target by this node attribute
* \param[in] value If not NULL, target by this node attribute value
* \param[in] level Index number of level to register
* \param[in] device_list List of devices in level
*
* \return Newly allocated XML tree on success, NULL otherwise
*
* \note The caller should set only one of node, pattern or attr/value.
*/
xmlNode *
create_level_registration_xml(const char *node, const char *pattern,
const char *attr, const char *value,
int level, stonith_key_value_t *device_list)
{
size_t len = 0;
char *list = NULL;
xmlNode *data;
CRM_CHECK(node || pattern || (attr && value), return NULL);
data = create_xml_node(NULL, XML_TAG_FENCING_LEVEL);
CRM_CHECK(data, return NULL);
crm_xml_add(data, F_STONITH_ORIGIN, __func__);
crm_xml_add_int(data, XML_ATTR_ID, level);
crm_xml_add_int(data, XML_ATTR_STONITH_INDEX, level);
if (node) {
crm_xml_add(data, XML_ATTR_STONITH_TARGET, node);
} else if (pattern) {
crm_xml_add(data, XML_ATTR_STONITH_TARGET_PATTERN, pattern);
} else {
crm_xml_add(data, XML_ATTR_STONITH_TARGET_ATTRIBUTE, attr);
crm_xml_add(data, XML_ATTR_STONITH_TARGET_VALUE, value);
}
// cppcheck seems not to understand the abort logic behind pcmk__realloc
// cppcheck-suppress memleak
for (; device_list; device_list = device_list->next) {
pcmk__add_separated_word(&list, &len, device_list->value, ",");
}
crm_xml_add(data, XML_ATTR_STONITH_DEVICES, list);
free(list);
return data;
}
static int
stonith_api_register_level_full(stonith_t * st, int options, const char *node,
const char *pattern,
const char *attr, const char *value,
int level, stonith_key_value_t *device_list)
{
int rc = 0;
xmlNode *data = create_level_registration_xml(node, pattern, attr, value,
level, device_list);
CRM_CHECK(data != NULL, return -EINVAL);
rc = stonith_send_command(st, STONITH_OP_LEVEL_ADD, data, NULL, options, 0);
free_xml(data);
return rc;
}
static int
stonith_api_register_level(stonith_t * st, int options, const char *node, int level,
stonith_key_value_t * device_list)
{
return stonith_api_register_level_full(st, options, node, NULL, NULL, NULL,
level, device_list);
}
static int
stonith_api_device_list(stonith_t * stonith, int call_options, const char *namespace,
stonith_key_value_t ** devices, int timeout)
{
int count = 0;
enum stonith_namespace ns = stonith_text2namespace(namespace);
if (devices == NULL) {
crm_err("Parameter error: stonith_api_device_list");
return -EFAULT;
}
#if HAVE_STONITH_STONITH_H
// Include Linux-HA agents if requested
if ((ns == st_namespace_any) || (ns == st_namespace_lha)) {
count += stonith__list_lha_agents(devices);
}
#endif
// Include Red Hat agents if requested
if ((ns == st_namespace_any) || (ns == st_namespace_rhcs)) {
count += stonith__list_rhcs_agents(devices);
}
return count;
}
+// See stonith_api_operations_t:metadata() documentation
static int
-stonith_api_device_metadata(stonith_t * stonith, int call_options, const char *agent,
- const char *namespace, char **output, int timeout)
+stonith_api_device_metadata(stonith_t *stonith, int call_options,
+ const char *agent, const char *namespace,
+ char **output, int timeout_sec)
{
/* By executing meta-data directly, we can get it from stonith_admin when
* the cluster is not running, which is important for higher-level tools.
*/
enum stonith_namespace ns = stonith_get_namespace(agent, namespace);
+ if (timeout_sec <= 0) {
+ timeout_sec = CRMD_METADATA_CALL_TIMEOUT;
+ }
+
crm_trace("Looking up metadata for %s agent %s",
stonith_namespace2text(ns), agent);
switch (ns) {
case st_namespace_rhcs:
- return stonith__rhcs_metadata(agent, timeout, output);
+ return stonith__rhcs_metadata(agent, timeout_sec, output);
#if HAVE_STONITH_STONITH_H
case st_namespace_lha:
- return stonith__lha_metadata(agent, timeout, output);
+ return stonith__lha_metadata(agent, timeout_sec, output);
#endif
default:
crm_err("Can't get fence agent '%s' meta-data: No such agent",
agent);
break;
}
return -ENODEV;
}
static int
stonith_api_query(stonith_t * stonith, int call_options, const char *target,
stonith_key_value_t ** devices, int timeout)
{
int rc = 0, lpc = 0, max = 0;
xmlNode *data = NULL;
xmlNode *output = NULL;
xmlXPathObjectPtr xpathObj = NULL;
CRM_CHECK(devices != NULL, return -EINVAL);
data = create_xml_node(NULL, F_STONITH_DEVICE);
crm_xml_add(data, F_STONITH_ORIGIN, __func__);
crm_xml_add(data, F_STONITH_TARGET, target);
crm_xml_add(data, F_STONITH_ACTION, "off");
rc = stonith_send_command(stonith, STONITH_OP_QUERY, data, &output, call_options, timeout);
if (rc < 0) {
return rc;
}
xpathObj = xpath_search(output, "//@agent");
if (xpathObj) {
max = numXpathResults(xpathObj);
for (lpc = 0; lpc < max; lpc++) {
xmlNode *match = getXpathResult(xpathObj, lpc);
CRM_LOG_ASSERT(match != NULL);
if(match != NULL) {
xmlChar *match_path = xmlGetNodePath(match);
crm_info("%s[%d] = %s", "//@agent", lpc, match_path);
free(match_path);
*devices = stonith_key_value_add(*devices, NULL, crm_element_value(match, XML_ATTR_ID));
}
}
freeXpathObject(xpathObj);
}
free_xml(output);
free_xml(data);
return max;
}
/*!
* \internal
* \brief Make a STONITH_OP_EXEC request
*
* \param[in] stonith Fencer connection
* \param[in] call_options Bitmask of \c stonith_call_options
* \param[in] id Fence device ID that request is for
* \param[in] action Agent action to request (list, status, or monitor)
* \param[in] target Name of target node for requested action
* \param[in] timeout_sec Error if not completed within this many seconds
* \param[out] output Where to set agent output
*/
static int
stonith_api_call(stonith_t *stonith, int call_options, const char *id,
const char *action, const char *target, int timeout_sec,
xmlNode **output)
{
int rc = 0;
xmlNode *data = NULL;
data = create_xml_node(NULL, F_STONITH_DEVICE);
crm_xml_add(data, F_STONITH_ORIGIN, __func__);
crm_xml_add(data, F_STONITH_DEVICE, id);
crm_xml_add(data, F_STONITH_ACTION, action);
crm_xml_add(data, F_STONITH_TARGET, target);
rc = stonith_send_command(stonith, STONITH_OP_EXEC, data, output,
call_options, timeout_sec);
free_xml(data);
return rc;
}
static int
stonith_api_list(stonith_t * stonith, int call_options, const char *id, char **list_info,
int timeout)
{
int rc;
xmlNode *output = NULL;
rc = stonith_api_call(stonith, call_options, id, "list", NULL, timeout, &output);
if (output && list_info) {
const char *list_str;
list_str = crm_element_value(output, F_STONITH_OUTPUT);
if (list_str) {
*list_info = strdup(list_str);
}
}
if (output) {
free_xml(output);
}
return rc;
}
static int
stonith_api_monitor(stonith_t * stonith, int call_options, const char *id, int timeout)
{
return stonith_api_call(stonith, call_options, id, "monitor", NULL, timeout, NULL);
}
static int
stonith_api_status(stonith_t * stonith, int call_options, const char *id, const char *port,
int timeout)
{
return stonith_api_call(stonith, call_options, id, "status", port, timeout, NULL);
}
static int
stonith_api_fence_with_delay(stonith_t * stonith, int call_options, const char *node,
const char *action, int timeout, int tolerance, int delay)
{
int rc = 0;
xmlNode *data = NULL;
data = create_xml_node(NULL, __func__);
crm_xml_add(data, F_STONITH_TARGET, node);
crm_xml_add(data, F_STONITH_ACTION, action);
crm_xml_add_int(data, F_STONITH_TIMEOUT, timeout);
crm_xml_add_int(data, F_STONITH_TOLERANCE, tolerance);
crm_xml_add_int(data, F_STONITH_DELAY, delay);
rc = stonith_send_command(stonith, STONITH_OP_FENCE, data, NULL, call_options, timeout);
free_xml(data);
return rc;
}
static int
stonith_api_fence(stonith_t * stonith, int call_options, const char *node, const char *action,
int timeout, int tolerance)
{
return stonith_api_fence_with_delay(stonith, call_options, node, action,
timeout, tolerance, 0);
}
static int
stonith_api_confirm(stonith_t * stonith, int call_options, const char *target)
{
stonith__set_call_options(call_options, target, st_opt_manual_ack);
return stonith_api_fence(stonith, call_options, target, "off", 0, 0);
}
static int
stonith_api_history(stonith_t * stonith, int call_options, const char *node,
stonith_history_t ** history, int timeout)
{
int rc = 0;
xmlNode *data = NULL;
xmlNode *output = NULL;
stonith_history_t *last = NULL;
*history = NULL;
if (node) {
data = create_xml_node(NULL, __func__);
crm_xml_add(data, F_STONITH_TARGET, node);
}
stonith__set_call_options(call_options, node, st_opt_sync_call);
rc = stonith_send_command(stonith, STONITH_OP_FENCE_HISTORY, data, &output,
call_options, timeout);
free_xml(data);
if (rc == 0) {
xmlNode *op = NULL;
xmlNode *reply = get_xpath_object("//" F_STONITH_HISTORY_LIST, output,
LOG_NEVER);
for (op = pcmk__xml_first_child(reply); op != NULL;
op = pcmk__xml_next(op)) {
stonith_history_t *kvp;
long long completed;
long long completed_nsec = 0L;
kvp = calloc(1, sizeof(stonith_history_t));
kvp->target = crm_element_value_copy(op, F_STONITH_TARGET);
kvp->action = crm_element_value_copy(op, F_STONITH_ACTION);
kvp->origin = crm_element_value_copy(op, F_STONITH_ORIGIN);
kvp->delegate = crm_element_value_copy(op, F_STONITH_DELEGATE);
kvp->client = crm_element_value_copy(op, F_STONITH_CLIENTNAME);
crm_element_value_ll(op, F_STONITH_DATE, &completed);
kvp->completed = (time_t) completed;
crm_element_value_ll(op, F_STONITH_DATE_NSEC, &completed_nsec);
kvp->completed_nsec = completed_nsec;
crm_element_value_int(op, F_STONITH_STATE, &kvp->state);
kvp->exit_reason = crm_element_value_copy(op,
XML_LRM_ATTR_EXIT_REASON);
if (last) {
last->next = kvp;
} else {
*history = kvp;
}
last = kvp;
}
}
free_xml(output);
return rc;
}
void stonith_history_free(stonith_history_t *history)
{
stonith_history_t *hp, *hp_old;
for (hp = history; hp; hp_old = hp, hp = hp->next, free(hp_old)) {
free(hp->target);
free(hp->action);
free(hp->origin);
free(hp->delegate);
free(hp->client);
free(hp->exit_reason);
}
}
static gint
stonithlib_GCompareFunc(gconstpointer a, gconstpointer b)
{
int rc = 0;
const stonith_notify_client_t *a_client = a;
const stonith_notify_client_t *b_client = b;
if (a_client->delete || b_client->delete) {
/* make entries marked for deletion not findable */
return -1;
}
CRM_CHECK(a_client->event != NULL && b_client->event != NULL, return 0);
rc = strcmp(a_client->event, b_client->event);
if (rc == 0) {
if (a_client->notify == NULL || b_client->notify == NULL) {
return 0;
} else if (a_client->notify == b_client->notify) {
return 0;
} else if (((long)a_client->notify) < ((long)b_client->notify)) {
crm_err("callbacks for %s are not equal: %p vs. %p",
a_client->event, a_client->notify, b_client->notify);
return -1;
}
crm_err("callbacks for %s are not equal: %p vs. %p",
a_client->event, a_client->notify, b_client->notify);
return 1;
}
return rc;
}
xmlNode *
stonith_create_op(int call_id, const char *token, const char *op, xmlNode * data, int call_options)
{
xmlNode *op_msg = create_xml_node(NULL, "stonith_command");
CRM_CHECK(op_msg != NULL, return NULL);
CRM_CHECK(token != NULL, return NULL);
crm_xml_add(op_msg, F_XML_TAGNAME, "stonith_command");
crm_xml_add(op_msg, F_TYPE, T_STONITH_NG);
crm_xml_add(op_msg, F_STONITH_CALLBACK_TOKEN, token);
crm_xml_add(op_msg, F_STONITH_OPERATION, op);
crm_xml_add_int(op_msg, F_STONITH_CALLID, call_id);
crm_trace("Sending call options: %.8lx, %d", (long)call_options, call_options);
crm_xml_add_int(op_msg, F_STONITH_CALLOPTS, call_options);
if (data != NULL) {
add_message_xml(op_msg, F_STONITH_CALLDATA, data);
}
return op_msg;
}
static void
stonith_destroy_op_callback(gpointer data)
{
stonith_callback_client_t *blob = data;
if (blob->timer && blob->timer->ref > 0) {
g_source_remove(blob->timer->ref);
}
free(blob->timer);
free(blob);
}
static int
stonith_api_signoff(stonith_t * stonith)
{
stonith_private_t *native = stonith->st_private;
crm_debug("Disconnecting from the fencer");
if (native->source != NULL) {
/* Attached to mainloop */
mainloop_del_ipc_client(native->source);
native->source = NULL;
native->ipc = NULL;
} else if (native->ipc) {
/* Not attached to mainloop */
crm_ipc_t *ipc = native->ipc;
native->ipc = NULL;
crm_ipc_close(ipc);
crm_ipc_destroy(ipc);
}
free(native->token); native->token = NULL;
stonith->state = stonith_disconnected;
return pcmk_ok;
}
static int
stonith_api_del_callback(stonith_t * stonith, int call_id, bool all_callbacks)
{
stonith_private_t *private = stonith->st_private;
if (all_callbacks) {
private->op_callback = NULL;
g_hash_table_destroy(private->stonith_op_callback_table);
private->stonith_op_callback_table = pcmk__intkey_table(stonith_destroy_op_callback);
} else if (call_id == 0) {
private->op_callback = NULL;
} else {
pcmk__intkey_table_remove(private->stonith_op_callback_table, call_id);
}
return pcmk_ok;
}
/*!
* \internal
* \brief Invoke a (single) specified fence action callback
*
* \param[in] st Fencer API connection
* \param[in] call_id If positive, call ID of completed fence action, otherwise
* legacy return code for early action failure
* \param[in] result Full result for action
* \param[in] userdata User data to pass to callback
* \param[in] callback Fence action callback to invoke
*/
static void
invoke_fence_action_callback(stonith_t *st, int call_id,
pcmk__action_result_t *result,
void *userdata,
void (*callback) (stonith_t *st,
stonith_callback_data_t *data))
{
stonith_callback_data_t data = { 0, };
data.call_id = call_id;
data.rc = pcmk_rc2legacy(stonith__result2rc(result));
data.userdata = userdata;
data.opaque = (void *) result;
callback(st, &data);
}
/*!
* \internal
* \brief Invoke any callbacks registered for a specified fence action result
*
* Given a fence action result from the fencer, invoke any callback registered
* for that action, as well as any global callback registered.
*
* \param[in] st Fencer API connection
* \param[in] msg If non-NULL, fencer reply
* \param[in] call_id If \p msg is NULL, call ID of action that timed out
*/
static void
invoke_registered_callbacks(stonith_t *stonith, xmlNode *msg, int call_id)
{
stonith_private_t *private = NULL;
stonith_callback_client_t *cb_info = NULL;
pcmk__action_result_t result = PCMK__UNKNOWN_RESULT;
CRM_CHECK(stonith != NULL, return);
CRM_CHECK(stonith->st_private != NULL, return);
private = stonith->st_private;
if (msg == NULL) {
// Fencer didn't reply in time
pcmk__set_result(&result, CRM_EX_ERROR, PCMK_EXEC_TIMEOUT,
"Fencer accepted request but did not reply in time");
CRM_LOG_ASSERT(call_id > 0);
} else {
// We have the fencer reply
if ((crm_element_value_int(msg, F_STONITH_CALLID, &call_id) != 0)
|| (call_id <= 0)) {
crm_log_xml_warn(msg, "Bad fencer reply");
}
stonith__xe_get_result(msg, &result);
}
if (call_id > 0) {
cb_info = pcmk__intkey_table_lookup(private->stonith_op_callback_table,
call_id);
}
if ((cb_info != NULL) && (cb_info->callback != NULL)
&& (pcmk__result_ok(&result) || !(cb_info->only_success))) {
crm_trace("Invoking callback %s for call %d",
pcmk__s(cb_info->id, "without ID"), call_id);
invoke_fence_action_callback(stonith, call_id, &result,
cb_info->user_data, cb_info->callback);
} else if ((private->op_callback == NULL) && !pcmk__result_ok(&result)) {
crm_warn("Fencing action without registered callback failed: %d (%s%s%s)",
result.exit_status,
pcmk_exec_status_str(result.execution_status),
((result.exit_reason == NULL)? "" : ": "),
((result.exit_reason == NULL)? "" : result.exit_reason));
crm_log_xml_debug(msg, "Failed fence update");
}
if (private->op_callback != NULL) {
crm_trace("Invoking global callback for call %d", call_id);
invoke_fence_action_callback(stonith, call_id, &result, NULL,
private->op_callback);
}
if (cb_info != NULL) {
stonith_api_del_callback(stonith, call_id, FALSE);
}
pcmk__reset_result(&result);
}
static gboolean
stonith_async_timeout_handler(gpointer data)
{
struct timer_rec_s *timer = data;
crm_err("Async call %d timed out after %dms", timer->call_id, timer->timeout);
invoke_registered_callbacks(timer->stonith, NULL, timer->call_id);
/* Always return TRUE, never remove the handler
* We do that in stonith_del_callback()
*/
return TRUE;
}
static void
set_callback_timeout(stonith_callback_client_t * callback, stonith_t * stonith, int call_id,
int timeout)
{
struct timer_rec_s *async_timer = callback->timer;
if (timeout <= 0) {
return;
}
if (!async_timer) {
async_timer = calloc(1, sizeof(struct timer_rec_s));
callback->timer = async_timer;
}
async_timer->stonith = stonith;
async_timer->call_id = call_id;
/* Allow a fair bit of grace to allow the server to tell us of a timeout
* This is only a fallback
*/
async_timer->timeout = (timeout + 60) * 1000;
if (async_timer->ref) {
g_source_remove(async_timer->ref);
}
async_timer->ref =
g_timeout_add(async_timer->timeout, stonith_async_timeout_handler, async_timer);
}
static void
update_callback_timeout(int call_id, int timeout, stonith_t * st)
{
stonith_callback_client_t *callback = NULL;
stonith_private_t *private = st->st_private;
callback = pcmk__intkey_table_lookup(private->stonith_op_callback_table,
call_id);
if (!callback || !callback->allow_timeout_updates) {
return;
}
set_callback_timeout(callback, st, call_id, timeout);
}
static int
stonith_dispatch_internal(const char *buffer, ssize_t length, gpointer userdata)
{
const char *type = NULL;
struct notify_blob_s blob;
stonith_t *st = userdata;
stonith_private_t *private = NULL;
CRM_ASSERT(st != NULL);
private = st->st_private;
blob.stonith = st;
blob.xml = string2xml(buffer);
if (blob.xml == NULL) {
crm_warn("Received malformed message from fencer: %s", buffer);
return 0;
}
/* do callbacks */
type = crm_element_value(blob.xml, F_TYPE);
crm_trace("Activating %s callbacks...", type);
if (pcmk__str_eq(type, T_STONITH_NG, pcmk__str_none)) {
invoke_registered_callbacks(st, blob.xml, 0);
} else if (pcmk__str_eq(type, T_STONITH_NOTIFY, pcmk__str_none)) {
foreach_notify_entry(private, stonith_send_notification, &blob);
} else if (pcmk__str_eq(type, T_STONITH_TIMEOUT_VALUE, pcmk__str_none)) {
int call_id = 0;
int timeout = 0;
crm_element_value_int(blob.xml, F_STONITH_TIMEOUT, &timeout);
crm_element_value_int(blob.xml, F_STONITH_CALLID, &call_id);
update_callback_timeout(call_id, timeout, st);
} else {
crm_err("Unknown message type: %s", type);
crm_log_xml_warn(blob.xml, "BadReply");
}
free_xml(blob.xml);
return 1;
}
static int
stonith_api_signon(stonith_t * stonith, const char *name, int *stonith_fd)
{
int rc = pcmk_ok;
stonith_private_t *native = NULL;
const char *display_name = name? name : "client";
struct ipc_client_callbacks st_callbacks = {
.dispatch = stonith_dispatch_internal,
.destroy = stonith_connection_destroy
};
CRM_CHECK(stonith != NULL, return -EINVAL);
native = stonith->st_private;
CRM_ASSERT(native != NULL);
crm_debug("Attempting fencer connection by %s with%s mainloop",
display_name, (stonith_fd? "out" : ""));
stonith->state = stonith_connected_command;
if (stonith_fd) {
/* No mainloop */
native->ipc = crm_ipc_new("stonith-ng", 0);
if (native->ipc && crm_ipc_connect(native->ipc)) {
*stonith_fd = crm_ipc_get_fd(native->ipc);
} else if (native->ipc) {
crm_ipc_close(native->ipc);
crm_ipc_destroy(native->ipc);
native->ipc = NULL;
}
} else {
/* With mainloop */
native->source =
mainloop_add_ipc_client("stonith-ng", G_PRIORITY_MEDIUM, 0, stonith, &st_callbacks);
native->ipc = mainloop_get_ipc_client(native->source);
}
if (native->ipc == NULL) {
rc = -ENOTCONN;
} else {
xmlNode *reply = NULL;
xmlNode *hello = create_xml_node(NULL, "stonith_command");
crm_xml_add(hello, F_TYPE, T_STONITH_NG);
crm_xml_add(hello, F_STONITH_OPERATION, CRM_OP_REGISTER);
crm_xml_add(hello, F_STONITH_CLIENTNAME, name);
rc = crm_ipc_send(native->ipc, hello, crm_ipc_client_response, -1, &reply);
if (rc < 0) {
crm_debug("Couldn't register with the fencer: %s "
CRM_XS " rc=%d", pcmk_strerror(rc), rc);
rc = -ECOMM;
} else if (reply == NULL) {
crm_debug("Couldn't register with the fencer: no reply");
rc = -EPROTO;
} else {
const char *msg_type = crm_element_value(reply, F_STONITH_OPERATION);
native->token = crm_element_value_copy(reply, F_STONITH_CLIENTID);
if (!pcmk__str_eq(msg_type, CRM_OP_REGISTER, pcmk__str_none)) {
crm_debug("Couldn't register with the fencer: invalid reply type '%s'",
(msg_type? msg_type : "(missing)"));
crm_log_xml_debug(reply, "Invalid fencer reply");
rc = -EPROTO;
} else if (native->token == NULL) {
crm_debug("Couldn't register with the fencer: no token in reply");
crm_log_xml_debug(reply, "Invalid fencer reply");
rc = -EPROTO;
} else {
crm_debug("Connection to fencer by %s succeeded (registration token: %s)",
display_name, native->token);
rc = pcmk_ok;
}
}
free_xml(reply);
free_xml(hello);
}
if (rc != pcmk_ok) {
crm_debug("Connection attempt to fencer by %s failed: %s "
CRM_XS " rc=%d", display_name, pcmk_strerror(rc), rc);
stonith->cmds->disconnect(stonith);
}
return rc;
}
static int
stonith_set_notification(stonith_t * stonith, const char *callback, int enabled)
{
int rc = pcmk_ok;
xmlNode *notify_msg = create_xml_node(NULL, __func__);
stonith_private_t *native = stonith->st_private;
if (stonith->state != stonith_disconnected) {
crm_xml_add(notify_msg, F_STONITH_OPERATION, T_STONITH_NOTIFY);
if (enabled) {
crm_xml_add(notify_msg, F_STONITH_NOTIFY_ACTIVATE, callback);
} else {
crm_xml_add(notify_msg, F_STONITH_NOTIFY_DEACTIVATE, callback);
}
rc = crm_ipc_send(native->ipc, notify_msg, crm_ipc_client_response, -1, NULL);
if (rc < 0) {
crm_perror(LOG_DEBUG, "Couldn't register for fencing notifications: %d", rc);
rc = -ECOMM;
} else {
rc = pcmk_ok;
}
}
free_xml(notify_msg);
return rc;
}
static int
stonith_api_add_notification(stonith_t * stonith, const char *event,
void (*callback) (stonith_t * stonith, stonith_event_t * e))
{
GList *list_item = NULL;
stonith_notify_client_t *new_client = NULL;
stonith_private_t *private = NULL;
private = stonith->st_private;
crm_trace("Adding callback for %s events (%d)", event, g_list_length(private->notify_list));
new_client = calloc(1, sizeof(stonith_notify_client_t));
new_client->event = event;
new_client->notify = callback;
list_item = g_list_find_custom(private->notify_list, new_client, stonithlib_GCompareFunc);
if (list_item != NULL) {
crm_warn("Callback already present");
free(new_client);
return -ENOTUNIQ;
} else {
private->notify_list = g_list_append(private->notify_list, new_client);
stonith_set_notification(stonith, event, 1);
crm_trace("Callback added (%d)", g_list_length(private->notify_list));
}
return pcmk_ok;
}
static void
del_notify_entry(gpointer data, gpointer user_data)
{
stonith_notify_client_t *entry = data;
stonith_t * stonith = user_data;
if (!entry->delete) {
crm_debug("Removing callback for %s events", entry->event);
stonith_api_del_notification(stonith, entry->event);
}
}
static int
stonith_api_del_notification(stonith_t * stonith, const char *event)
{
GList *list_item = NULL;
stonith_notify_client_t *new_client = NULL;
stonith_private_t *private = stonith->st_private;
if (event == NULL) {
foreach_notify_entry(private, del_notify_entry, stonith);
crm_trace("Removed callback");
return pcmk_ok;
}
crm_debug("Removing callback for %s events", event);
new_client = calloc(1, sizeof(stonith_notify_client_t));
new_client->event = event;
new_client->notify = NULL;
list_item = g_list_find_custom(private->notify_list, new_client, stonithlib_GCompareFunc);
stonith_set_notification(stonith, event, 0);
if (list_item != NULL) {
stonith_notify_client_t *list_client = list_item->data;
if (private->notify_refcnt) {
list_client->delete = TRUE;
private->notify_deletes = TRUE;
} else {
private->notify_list = g_list_remove(private->notify_list, list_client);
free(list_client);
}
crm_trace("Removed callback");
} else {
crm_trace("Callback not present");
}
free(new_client);
return pcmk_ok;
}
static int
stonith_api_add_callback(stonith_t * stonith, int call_id, int timeout, int options,
void *user_data, const char *callback_name,
void (*callback) (stonith_t * st, stonith_callback_data_t * data))
{
stonith_callback_client_t *blob = NULL;
stonith_private_t *private = NULL;
CRM_CHECK(stonith != NULL, return -EINVAL);
CRM_CHECK(stonith->st_private != NULL, return -EINVAL);
private = stonith->st_private;
if (call_id == 0) { // Add global callback
private->op_callback = callback;
} else if (call_id < 0) { // Call failed immediately, so call callback now
if (!(options & st_opt_report_only_success)) {
pcmk__action_result_t result = PCMK__UNKNOWN_RESULT;
crm_trace("Call failed, calling %s: %s", callback_name, pcmk_strerror(call_id));
pcmk__set_result(&result, CRM_EX_ERROR,
stonith__legacy2status(call_id), NULL);
invoke_fence_action_callback(stonith, call_id, &result,
user_data, callback);
} else {
crm_warn("Fencer call failed: %s", pcmk_strerror(call_id));
}
return FALSE;
}
blob = calloc(1, sizeof(stonith_callback_client_t));
blob->id = callback_name;
blob->only_success = (options & st_opt_report_only_success) ? TRUE : FALSE;
blob->user_data = user_data;
blob->callback = callback;
blob->allow_timeout_updates = (options & st_opt_timeout_updates) ? TRUE : FALSE;
if (timeout > 0) {
set_callback_timeout(blob, stonith, call_id, timeout);
}
pcmk__intkey_table_insert(private->stonith_op_callback_table, call_id,
blob);
crm_trace("Added callback to %s for call %d", callback_name, call_id);
return TRUE;
}
static void
stonith_dump_pending_op(gpointer key, gpointer value, gpointer user_data)
{
int call = GPOINTER_TO_INT(key);
stonith_callback_client_t *blob = value;
crm_debug("Call %d (%s): pending", call, pcmk__s(blob->id, "no ID"));
}
void
stonith_dump_pending_callbacks(stonith_t * stonith)
{
stonith_private_t *private = stonith->st_private;
if (private->stonith_op_callback_table == NULL) {
return;
}
return g_hash_table_foreach(private->stonith_op_callback_table, stonith_dump_pending_op, NULL);
}
/*!
* \internal
* \brief Get the data section of a fencer notification
*
* \param[in] msg Notification XML
* \param[in] ntype Notification type
*/
static xmlNode *
get_event_data_xml(xmlNode *msg, const char *ntype)
{
char *data_addr = crm_strdup_printf("//%s", ntype);
xmlNode *data = get_xpath_object(data_addr, msg, LOG_DEBUG);
free(data_addr);
return data;
}
/*
<notify t="st_notify" subt="st_device_register" st_op="st_device_register" st_rc="0" >
<st_calldata >
<stonith_command t="stonith-ng" st_async_id="088fb640-431a-48b9-b2fc-c4ff78d0a2d9" st_op="st_device_register" st_callid="2" st_callopt="4096" st_timeout="0" st_clientid="088fb640-431a-48b9-b2fc-c4ff78d0a2d9" st_clientname="cts-fence-helper" >
<st_calldata >
<st_device_id id="test-id" origin="create_device_registration_xml" agent="fence_virsh" namespace="stonith-ng" >
<attributes ipaddr="localhost" pcmk-portmal="some-host=pcmk-1 pcmk-3=3,4" login="root" identity_file="/root/.ssh/id_dsa" />
</st_device_id>
</st_calldata>
</stonith_command>
</st_calldata>
</notify>
<notify t="st_notify" subt="st_notify_fence" st_op="st_notify_fence" st_rc="0" >
<st_calldata >
<st_notify_fence st_rc="0" st_target="some-host" st_op="st_fence" st_delegate="test-id" st_origin="61dd7759-e229-4be7-b1f8-ef49dd14d9f0" />
</st_calldata>
</notify>
*/
static stonith_event_t *
xml_to_event(xmlNode *msg)
{
stonith_event_t *event = calloc(1, sizeof(stonith_event_t));
struct event_private *event_private = NULL;
CRM_ASSERT(event != NULL);
event->opaque = calloc(1, sizeof(struct event_private));
CRM_ASSERT(event->opaque != NULL);
event_private = (struct event_private *) event->opaque;
crm_log_xml_trace(msg, "stonith_notify");
// All notification types have the operation result and notification subtype
stonith__xe_get_result(msg, &event_private->result);
event->operation = crm_element_value_copy(msg, F_STONITH_OPERATION);
// @COMPAT The API originally provided the result as a legacy return code
event->result = pcmk_rc2legacy(stonith__result2rc(&event_private->result));
// Some notification subtypes have additional information
if (pcmk__str_eq(event->operation, T_STONITH_NOTIFY_FENCE,
pcmk__str_none)) {
xmlNode *data = get_event_data_xml(msg, event->operation);
if (data == NULL) {
crm_err("No data for %s event", event->operation);
crm_log_xml_notice(msg, "BadEvent");
} else {
event->origin = crm_element_value_copy(data, F_STONITH_ORIGIN);
event->action = crm_element_value_copy(data, F_STONITH_ACTION);
event->target = crm_element_value_copy(data, F_STONITH_TARGET);
event->executioner = crm_element_value_copy(data, F_STONITH_DELEGATE);
event->id = crm_element_value_copy(data, F_STONITH_REMOTE_OP_ID);
event->client_origin = crm_element_value_copy(data, F_STONITH_CLIENTNAME);
event->device = crm_element_value_copy(data, F_STONITH_DEVICE);
}
} else if (pcmk__str_any_of(event->operation,
STONITH_OP_DEVICE_ADD, STONITH_OP_DEVICE_DEL,
STONITH_OP_LEVEL_ADD, STONITH_OP_LEVEL_DEL,
NULL)) {
xmlNode *data = get_event_data_xml(msg, event->operation);
if (data == NULL) {
crm_err("No data for %s event", event->operation);
crm_log_xml_notice(msg, "BadEvent");
} else {
event->device = crm_element_value_copy(data, F_STONITH_DEVICE);
}
}
return event;
}
static void
event_free(stonith_event_t * event)
{
struct event_private *event_private = event->opaque;
free(event->id);
free(event->type);
free(event->message);
free(event->operation);
free(event->origin);
free(event->action);
free(event->target);
free(event->executioner);
free(event->device);
free(event->client_origin);
pcmk__reset_result(&event_private->result);
free(event->opaque);
free(event);
}
static void
stonith_send_notification(gpointer data, gpointer user_data)
{
struct notify_blob_s *blob = user_data;
stonith_notify_client_t *entry = data;
stonith_event_t *st_event = NULL;
const char *event = NULL;
if (blob->xml == NULL) {
crm_warn("Skipping callback - NULL message");
return;
}
event = crm_element_value(blob->xml, F_SUBTYPE);
if (entry == NULL) {
crm_warn("Skipping callback - NULL callback client");
return;
} else if (entry->delete) {
crm_trace("Skipping callback - marked for deletion");
return;
} else if (entry->notify == NULL) {
crm_warn("Skipping callback - NULL callback");
return;
} else if (!pcmk__str_eq(entry->event, event, pcmk__str_none)) {
crm_trace("Skipping callback - event mismatch %p/%s vs. %s", entry, entry->event, event);
return;
}
st_event = xml_to_event(blob->xml);
crm_trace("Invoking callback for %p/%s event...", entry, event);
entry->notify(blob->stonith, st_event);
crm_trace("Callback invoked...");
event_free(st_event);
}
/*!
* \internal
* \brief Create and send an API request
*
* \param[in] stonith Stonith connection
* \param[in] op API operation to request
* \param[in] data Data to attach to request
* \param[out] output_data If not NULL, will be set to reply if synchronous
* \param[in] call_options Bitmask of stonith_call_options to use
* \param[in] timeout Error if not completed within this many seconds
*
* \return pcmk_ok (for synchronous requests) or positive call ID
* (for asynchronous requests) on success, -errno otherwise
*/
static int
stonith_send_command(stonith_t * stonith, const char *op, xmlNode * data, xmlNode ** output_data,
int call_options, int timeout)
{
int rc = 0;
int reply_id = -1;
xmlNode *op_msg = NULL;
xmlNode *op_reply = NULL;
stonith_private_t *native = NULL;
CRM_ASSERT(stonith && stonith->st_private && op);
native = stonith->st_private;
if (output_data != NULL) {
*output_data = NULL;
}
if ((stonith->state == stonith_disconnected) || (native->token == NULL)) {
return -ENOTCONN;
}
/* Increment the call ID, which must be positive to avoid conflicting with
* error codes. This shouldn't be a problem unless the client mucked with
* it or the counter wrapped around.
*/
stonith->call_id++;
if (stonith->call_id < 1) {
stonith->call_id = 1;
}
op_msg = stonith_create_op(stonith->call_id, native->token, op, data, call_options);
if (op_msg == NULL) {
return -EINVAL;
}
crm_xml_add_int(op_msg, F_STONITH_TIMEOUT, timeout);
crm_trace("Sending %s message to fencer with timeout %ds", op, timeout);
if (data) {
const char *delay_s = crm_element_value(data, F_STONITH_DELAY);
if (delay_s) {
crm_xml_add(op_msg, F_STONITH_DELAY, delay_s);
}
}
{
enum crm_ipc_flags ipc_flags = crm_ipc_flags_none;
if (call_options & st_opt_sync_call) {
pcmk__set_ipc_flags(ipc_flags, "stonith command",
crm_ipc_client_response);
}
rc = crm_ipc_send(native->ipc, op_msg, ipc_flags,
1000 * (timeout + 60), &op_reply);
}
free_xml(op_msg);
if (rc < 0) {
crm_perror(LOG_ERR, "Couldn't perform %s operation (timeout=%ds): %d", op, timeout, rc);
rc = -ECOMM;
goto done;
}
crm_log_xml_trace(op_reply, "Reply");
if (!(call_options & st_opt_sync_call)) {
crm_trace("Async call %d, returning", stonith->call_id);
free_xml(op_reply);
return stonith->call_id;
}
rc = pcmk_ok;
crm_element_value_int(op_reply, F_STONITH_CALLID, &reply_id);
if (reply_id == stonith->call_id) {
pcmk__action_result_t result = PCMK__UNKNOWN_RESULT;
crm_trace("Synchronous reply %d received", reply_id);
stonith__xe_get_result(op_reply, &result);
rc = pcmk_rc2legacy(stonith__result2rc(&result));
pcmk__reset_result(&result);
if ((call_options & st_opt_discard_reply) || output_data == NULL) {
crm_trace("Discarding reply");
} else {
*output_data = op_reply;
op_reply = NULL; /* Prevent subsequent free */
}
} else if (reply_id <= 0) {
crm_err("Received bad reply: No id set");
crm_log_xml_err(op_reply, "Bad reply");
free_xml(op_reply);
rc = -ENOMSG;
} else {
crm_err("Received bad reply: %d (wanted %d)", reply_id, stonith->call_id);
crm_log_xml_err(op_reply, "Old reply");
free_xml(op_reply);
rc = -ENOMSG;
}
done:
if (crm_ipc_connected(native->ipc) == FALSE) {
crm_err("Fencer disconnected");
free(native->token); native->token = NULL;
stonith->state = stonith_disconnected;
}
free_xml(op_reply);
return rc;
}
/* Not used with mainloop */
bool
stonith_dispatch(stonith_t * st)
{
gboolean stay_connected = TRUE;
stonith_private_t *private = NULL;
CRM_ASSERT(st != NULL);
private = st->st_private;
while (crm_ipc_ready(private->ipc)) {
if (crm_ipc_read(private->ipc) > 0) {
const char *msg = crm_ipc_buffer(private->ipc);
stonith_dispatch_internal(msg, strlen(msg), st);
}
if (crm_ipc_connected(private->ipc) == FALSE) {
crm_err("Connection closed");
stay_connected = FALSE;
}
}
return stay_connected;
}
static int
stonith_api_free(stonith_t * stonith)
{
int rc = pcmk_ok;
crm_trace("Destroying %p", stonith);
if (stonith->state != stonith_disconnected) {
crm_trace("Disconnecting %p first", stonith);
rc = stonith->cmds->disconnect(stonith);
}
if (stonith->state == stonith_disconnected) {
stonith_private_t *private = stonith->st_private;
crm_trace("Removing %d callbacks", g_hash_table_size(private->stonith_op_callback_table));
g_hash_table_destroy(private->stonith_op_callback_table);
crm_trace("Destroying %d notification clients", g_list_length(private->notify_list));
g_list_free_full(private->notify_list, free);
free(stonith->st_private);
free(stonith->cmds);
free(stonith);
} else {
crm_err("Not free'ing active connection: %s (%d)", pcmk_strerror(rc), rc);
}
return rc;
}
void
stonith_api_delete(stonith_t * stonith)
{
crm_trace("Destroying %p", stonith);
if(stonith) {
stonith->cmds->free(stonith);
}
}
static int
stonith_api_validate(stonith_t *st, int call_options, const char *rsc_id,
const char *namespace_s, const char *agent,
- stonith_key_value_t *params, int timeout, char **output,
- char **error_output)
+ stonith_key_value_t *params, int timeout_sec,
+ char **output, char **error_output)
{
/* Validation should be done directly via the agent, so we can get it from
* stonith_admin when the cluster is not running, which is important for
* higher-level tools.
*/
int rc = pcmk_ok;
/* Use a dummy node name in case the agent requires a target. We assume the
* actual target doesn't matter for validation purposes (if in practice,
* that is incorrect, we will need to allow the caller to pass the target).
*/
const char *target = "node1";
const char *host_arg = NULL;
GHashTable *params_table = pcmk__strkey_table(free, free);
// Convert parameter list to a hash table
for (; params; params = params->next) {
if (pcmk__str_eq(params->key, PCMK_STONITH_HOST_ARGUMENT,
pcmk__str_none)) {
host_arg = params->value;
}
if (!pcmk_stonith_param(params->key)) {
g_hash_table_insert(params_table, strdup(params->key),
strdup(params->value));
}
}
#if SUPPORT_CIBSECRETS
rc = pcmk__substitute_secrets(rsc_id, params_table);
if (rc != pcmk_rc_ok) {
crm_warn("Could not replace secret parameters for validation of %s: %s",
agent, pcmk_rc_str(rc));
// rc is standard return value, don't return it in this function
}
#endif
if (output) {
*output = NULL;
}
if (error_output) {
*error_output = NULL;
}
+ if (timeout_sec <= 0) {
+ timeout_sec = CRMD_METADATA_CALL_TIMEOUT; // Questionable
+ }
+
switch (stonith_get_namespace(agent, namespace_s)) {
case st_namespace_rhcs:
rc = stonith__rhcs_validate(st, call_options, target, agent,
- params_table, host_arg, timeout,
+ params_table, host_arg, timeout_sec,
output, error_output);
break;
#if HAVE_STONITH_STONITH_H
case st_namespace_lha:
rc = stonith__lha_validate(st, call_options, target, agent,
- params_table, timeout, output,
+ params_table, timeout_sec, output,
error_output);
break;
#endif
case st_namespace_invalid:
errno = ENOENT;
rc = -errno;
if (error_output) {
*error_output = crm_strdup_printf("Agent %s not found", agent);
} else {
crm_err("Agent %s not found", agent);
}
break;
default:
errno = EOPNOTSUPP;
rc = -errno;
if (error_output) {
*error_output = crm_strdup_printf("Agent %s does not support validation",
agent);
} else {
crm_err("Agent %s does not support validation", agent);
}
break;
}
g_hash_table_destroy(params_table);
return rc;
}
stonith_t *
stonith_api_new(void)
{
stonith_t *new_stonith = NULL;
stonith_private_t *private = NULL;
new_stonith = calloc(1, sizeof(stonith_t));
if (new_stonith == NULL) {
return NULL;
}
private = calloc(1, sizeof(stonith_private_t));
if (private == NULL) {
free(new_stonith);
return NULL;
}
new_stonith->st_private = private;
private->stonith_op_callback_table = pcmk__intkey_table(stonith_destroy_op_callback);
private->notify_list = NULL;
private->notify_refcnt = 0;
private->notify_deletes = FALSE;
new_stonith->call_id = 1;
new_stonith->state = stonith_disconnected;
new_stonith->cmds = calloc(1, sizeof(stonith_api_operations_t));
if (new_stonith->cmds == NULL) {
free(new_stonith->st_private);
free(new_stonith);
return NULL;
}
/* *INDENT-OFF* */
new_stonith->cmds->free = stonith_api_free;
new_stonith->cmds->connect = stonith_api_signon;
new_stonith->cmds->disconnect = stonith_api_signoff;
new_stonith->cmds->list = stonith_api_list;
new_stonith->cmds->monitor = stonith_api_monitor;
new_stonith->cmds->status = stonith_api_status;
new_stonith->cmds->fence = stonith_api_fence;
new_stonith->cmds->fence_with_delay = stonith_api_fence_with_delay;
new_stonith->cmds->confirm = stonith_api_confirm;
new_stonith->cmds->history = stonith_api_history;
new_stonith->cmds->list_agents = stonith_api_device_list;
new_stonith->cmds->metadata = stonith_api_device_metadata;
new_stonith->cmds->query = stonith_api_query;
new_stonith->cmds->remove_device = stonith_api_remove_device;
new_stonith->cmds->register_device = stonith_api_register_device;
new_stonith->cmds->remove_level = stonith_api_remove_level;
new_stonith->cmds->remove_level_full = stonith_api_remove_level_full;
new_stonith->cmds->register_level = stonith_api_register_level;
new_stonith->cmds->register_level_full = stonith_api_register_level_full;
new_stonith->cmds->remove_callback = stonith_api_del_callback;
new_stonith->cmds->register_callback = stonith_api_add_callback;
new_stonith->cmds->remove_notification = stonith_api_del_notification;
new_stonith->cmds->register_notification = stonith_api_add_notification;
new_stonith->cmds->validate = stonith_api_validate;
/* *INDENT-ON* */
return new_stonith;
}
/*!
* \brief Make a blocking connection attempt to the fencer
*
* \param[in,out] st Fencer API object
* \param[in] name Client name to use with fencer
* \param[in] max_attempts Return error if this many attempts fail
*
* \return pcmk_ok on success, result of last attempt otherwise
*/
int
stonith_api_connect_retry(stonith_t *st, const char *name, int max_attempts)
{
int rc = -EINVAL; // if max_attempts is not positive
for (int attempt = 1; attempt <= max_attempts; attempt++) {
rc = st->cmds->connect(st, name, NULL);
if (rc == pcmk_ok) {
return pcmk_ok;
} else if (attempt < max_attempts) {
crm_notice("Fencer connection attempt %d of %d failed (retrying in 2s): %s "
CRM_XS " rc=%d",
attempt, max_attempts, pcmk_strerror(rc), rc);
sleep(2);
}
}
crm_notice("Could not connect to fencer: %s " CRM_XS " rc=%d",
pcmk_strerror(rc), rc);
return rc;
}
stonith_key_value_t *
stonith_key_value_add(stonith_key_value_t * head, const char *key, const char *value)
{
stonith_key_value_t *p, *end;
p = calloc(1, sizeof(stonith_key_value_t));
pcmk__str_update(&p->key, key);
pcmk__str_update(&p->value, value);
end = head;
while (end && end->next) {
end = end->next;
}
if (end) {
end->next = p;
} else {
head = p;
}
return head;
}
void
stonith_key_value_freeall(stonith_key_value_t * head, int keys, int values)
{
stonith_key_value_t *p;
while (head) {
p = head->next;
if (keys) {
free(head->key);
}
if (values) {
free(head->value);
}
free(head);
head = p;
}
}
#define api_log_open() openlog("stonith-api", LOG_CONS | LOG_NDELAY | LOG_PID, LOG_DAEMON)
#define api_log(level, fmt, args...) syslog(level, "%s: "fmt, __func__, args)
int
stonith_api_kick(uint32_t nodeid, const char *uname, int timeout, bool off)
{
int rc = pcmk_ok;
stonith_t *st = stonith_api_new();
const char *action = off? "off" : "reboot";
api_log_open();
if (st == NULL) {
api_log(LOG_ERR, "API initialization failed, could not kick (%s) node %u/%s",
action, nodeid, uname);
return -EPROTO;
}
rc = st->cmds->connect(st, "stonith-api", NULL);
if (rc != pcmk_ok) {
api_log(LOG_ERR, "Connection failed, could not kick (%s) node %u/%s : %s (%d)",
action, nodeid, uname, pcmk_strerror(rc), rc);
} else {
char *name = (uname == NULL)? pcmk__itoa(nodeid) : strdup(uname);
int opts = 0;
stonith__set_call_options(opts, name,
st_opt_sync_call|st_opt_allow_suicide);
if ((uname == NULL) && (nodeid > 0)) {
stonith__set_call_options(opts, name, st_opt_cs_nodeid);
}
rc = st->cmds->fence(st, opts, name, action, timeout, 0);
free(name);
if (rc != pcmk_ok) {
api_log(LOG_ERR, "Could not kick (%s) node %u/%s : %s (%d)",
action, nodeid, uname, pcmk_strerror(rc), rc);
} else {
api_log(LOG_NOTICE, "Node %u/%s kicked: %s", nodeid, uname, action);
}
}
stonith_api_delete(st);
return rc;
}
time_t
stonith_api_time(uint32_t nodeid, const char *uname, bool in_progress)
{
int rc = pcmk_ok;
time_t when = 0;
stonith_t *st = stonith_api_new();
stonith_history_t *history = NULL, *hp = NULL;
if (st == NULL) {
api_log(LOG_ERR, "Could not retrieve fence history for %u/%s: "
"API initialization failed", nodeid, uname);
return when;
}
rc = st->cmds->connect(st, "stonith-api", NULL);
if (rc != pcmk_ok) {
api_log(LOG_NOTICE, "Connection failed: %s (%d)", pcmk_strerror(rc), rc);
} else {
int entries = 0;
int progress = 0;
int completed = 0;
int opts = 0;
char *name = (uname == NULL)? pcmk__itoa(nodeid) : strdup(uname);
stonith__set_call_options(opts, name, st_opt_sync_call);
if ((uname == NULL) && (nodeid > 0)) {
stonith__set_call_options(opts, name, st_opt_cs_nodeid);
}
rc = st->cmds->history(st, opts, name, &history, 120);
free(name);
for (hp = history; hp; hp = hp->next) {
entries++;
if (in_progress) {
progress++;
if (hp->state != st_done && hp->state != st_failed) {
when = time(NULL);
}
} else if (hp->state == st_done) {
completed++;
if (hp->completed > when) {
when = hp->completed;
}
}
}
stonith_history_free(history);
if(rc == pcmk_ok) {
api_log(LOG_INFO, "Found %d entries for %u/%s: %d in progress, %d completed", entries, nodeid, uname, progress, completed);
} else {
api_log(LOG_ERR, "Could not retrieve fence history for %u/%s: %s (%d)", nodeid, uname, pcmk_strerror(rc), rc);
}
}
stonith_api_delete(st);
if(when) {
api_log(LOG_INFO, "Node %u/%s last kicked at: %ld", nodeid, uname, (long int)when);
}
return when;
}
bool
stonith_agent_exists(const char *agent, int timeout)
{
stonith_t *st = NULL;
stonith_key_value_t *devices = NULL;
stonith_key_value_t *dIter = NULL;
bool rc = FALSE;
if (agent == NULL) {
return rc;
}
st = stonith_api_new();
if (st == NULL) {
crm_err("Could not list fence agents: API memory allocation failed");
return FALSE;
}
st->cmds->list_agents(st, st_opt_sync_call, NULL, &devices, timeout == 0 ? 120 : timeout);
for (dIter = devices; dIter != NULL; dIter = dIter->next) {
if (pcmk__str_eq(dIter->value, agent, pcmk__str_none)) {
rc = TRUE;
break;
}
}
stonith_key_value_freeall(devices, 1, 1);
stonith_api_delete(st);
return rc;
}
const char *
stonith_action_str(const char *action)
{
if (action == NULL) {
return "fencing";
} else if (!strcmp(action, "on")) {
return "unfencing";
} else if (!strcmp(action, "off")) {
return "turning off";
} else {
return action;
}
}
/*!
* \internal
* \brief Parse a target name from one line of a target list string
*
* \param[in] line One line of a target list string
* \param[in] len String length of line
* \param[in,out] output List to add newly allocated target name to
*/
static void
parse_list_line(const char *line, int len, GList **output)
{
size_t i = 0;
size_t entry_start = 0;
/* Skip complaints about additional parameters device doesn't understand
*
* @TODO Document or eliminate the implied restriction of target names
*/
if (strstr(line, "invalid") || strstr(line, "variable")) {
crm_debug("Skipping list output line: %s", line);
return;
}
// Process line content, character by character
for (i = 0; i <= len; i++) {
if (isspace(line[i]) || (line[i] == ',') || (line[i] == ';')
|| (line[i] == '\0')) {
// We've found a separator (i.e. the end of an entry)
int rc = 0;
char *entry = NULL;
if (i == entry_start) {
// Skip leading and sequential separators
entry_start = i + 1;
continue;
}
entry = calloc(i - entry_start + 1, sizeof(char));
CRM_ASSERT(entry != NULL);
/* Read entry, stopping at first separator
*
* @TODO Document or eliminate these character restrictions
*/
rc = sscanf(line + entry_start, "%[a-zA-Z0-9_-.]", entry);
if (rc != 1) {
crm_warn("Could not parse list output entry: %s "
CRM_XS " entry_start=%d position=%d",
line + entry_start, entry_start, i);
free(entry);
} else if (pcmk__strcase_any_of(entry, "on", "off", NULL)) {
/* Some agents print the target status in the list output,
* though none are known now (the separate list-status command
* is used for this, but it can also print "UNKNOWN"). To handle
* this possibility, skip such entries.
*
* @TODO Document or eliminate the implied restriction of target
* names.
*/
free(entry);
} else {
// We have a valid entry
*output = g_list_append(*output, entry);
}
entry_start = i + 1;
}
}
}
/*!
* \internal
* \brief Parse a list of targets from a string
*
* \param[in] list_output Target list as a string
*
* \return List of target names
* \note The target list string format is flexible, to allow for user-specified
* lists such pcmk_host_list and the output of an agent's list action
* (whether direct or via the API, which escapes newlines). There may be
* multiple lines, separated by either a newline or an escaped newline
* (backslash n). Each line may have one or more target names, separated
* by any combination of whitespace, commas, and semi-colons. Lines
* containing "invalid" or "variable" will be ignored entirely. Target
* names "on" or "off" (case-insensitive) will be ignored. Target names
* may contain only alphanumeric characters, underbars (_), dashes (-),
* and dots (.) (if any other character occurs in the name, it and all
* subsequent characters in the name will be ignored).
* \note The caller is responsible for freeing the result with
* g_list_free_full(result, free).
*/
GList *
stonith__parse_targets(const char *target_spec)
{
GList *targets = NULL;
if (target_spec != NULL) {
size_t out_len = strlen(target_spec);
size_t line_start = 0; // Starting index of line being processed
for (size_t i = 0; i <= out_len; ++i) {
if ((target_spec[i] == '\n') || (target_spec[i] == '\0')
|| ((target_spec[i] == '\\') && (target_spec[i + 1] == 'n'))) {
// We've reached the end of one line of output
int len = i - line_start;
if (len > 0) {
char *line = strndup(target_spec + line_start, len);
line[len] = '\0'; // Because it might be a newline
parse_list_line(line, len, &targets);
free(line);
}
if (target_spec[i] == '\\') {
++i; // backslash-n takes up two positions
}
line_start = i + 1;
}
}
}
return targets;
}
/*!
* \internal
* \brief Check whether a fencing failure was followed by an equivalent success
*
* \param[in] event Fencing failure
* \param[in] top_history Complete fencing history (must be sorted by
* stonith__sort_history() beforehand)
*
* \return The name of the node that executed the fencing if a later successful
* event exists, or NULL if no such event exists
*/
const char *
stonith__later_succeeded(stonith_history_t *event, stonith_history_t *top_history)
{
const char *other = NULL;
for (stonith_history_t *prev_hp = top_history; prev_hp; prev_hp = prev_hp->next) {
if (prev_hp == event) {
break;
}
if ((prev_hp->state == st_done) &&
pcmk__str_eq(event->target, prev_hp->target, pcmk__str_casei) &&
pcmk__str_eq(event->action, prev_hp->action, pcmk__str_none) &&
((event->completed < prev_hp->completed) ||
((event->completed == prev_hp->completed) && (event->completed_nsec < prev_hp->completed_nsec)))) {
if ((event->delegate == NULL)
|| pcmk__str_eq(event->delegate, prev_hp->delegate,
pcmk__str_casei)) {
// Prefer equivalent fencing by same executioner
return prev_hp->delegate;
} else if (other == NULL) {
// Otherwise remember first successful executioner
other = (prev_hp->delegate == NULL)? "some node" : prev_hp->delegate;
}
}
}
return other;
}
/*!
* \internal
* \brief Sort the stonith-history
* sort by competed most current on the top
* pending actions lacking a completed-stamp are gathered at the top
*
* \param[in] history List of stonith actions
*
*/
stonith_history_t *
stonith__sort_history(stonith_history_t *history)
{
stonith_history_t *new = NULL, *pending = NULL, *hp, *np, *tmp;
for (hp = history; hp; ) {
tmp = hp->next;
if ((hp->state == st_done) || (hp->state == st_failed)) {
/* sort into new */
if ((!new) || (hp->completed > new->completed) ||
((hp->completed == new->completed) && (hp->completed_nsec > new->completed_nsec))) {
hp->next = new;
new = hp;
} else {
np = new;
do {
if ((!np->next) || (hp->completed > np->next->completed) ||
((hp->completed == np->next->completed) && (hp->completed_nsec > np->next->completed_nsec))) {
hp->next = np->next;
np->next = hp;
break;
}
np = np->next;
} while (1);
}
} else {
/* put into pending */
hp->next = pending;
pending = hp;
}
hp = tmp;
}
/* pending actions don't have a completed-stamp so make them go front */
if (pending) {
stonith_history_t *last_pending = pending;
while (last_pending->next) {
last_pending = last_pending->next;
}
last_pending->next = new;
new = pending;
}
return new;
}
/*!
* \brief Return string equivalent of an operation state value
*
* \param[in] state Fencing operation state value
*
* \return Human-friendly string equivalent of state
*/
const char *
stonith_op_state_str(enum op_state state)
{
switch (state) {
case st_query: return "querying";
case st_exec: return "executing";
case st_done: return "completed";
case st_duplicate: return "duplicate";
case st_failed: return "failed";
}
return "unknown";
}
stonith_history_t *
stonith__first_matching_event(stonith_history_t *history,
bool (*matching_fn)(stonith_history_t *, void *),
void *user_data)
{
for (stonith_history_t *hp = history; hp; hp = hp->next) {
if (matching_fn(hp, user_data)) {
return hp;
}
}
return NULL;
}
bool
stonith__event_state_pending(stonith_history_t *history, void *user_data)
{
return history->state != st_failed && history->state != st_done;
}
bool
stonith__event_state_eq(stonith_history_t *history, void *user_data)
{
return history->state == GPOINTER_TO_INT(user_data);
}
bool
stonith__event_state_neq(stonith_history_t *history, void *user_data)
{
return history->state != GPOINTER_TO_INT(user_data);
}
void
stonith__device_parameter_flags(uint32_t *device_flags, const char *device_name,
xmlNode *metadata)
{
xmlXPathObjectPtr xpath = NULL;
int max = 0;
int lpc = 0;
CRM_CHECK((device_flags != NULL) && (metadata != NULL), return);
xpath = xpath_search(metadata, "//parameter");
max = numXpathResults(xpath);
if (max <= 0) {
freeXpathObject(xpath);
return;
}
for (lpc = 0; lpc < max; lpc++) {
const char *parameter = NULL;
xmlNode *match = getXpathResult(xpath, lpc);
CRM_LOG_ASSERT(match != NULL);
if (match == NULL) {
continue;
}
parameter = crm_element_value(match, "name");
if (pcmk__str_eq(parameter, "plug", pcmk__str_casei)) {
stonith__set_device_flags(*device_flags, device_name,
st_device_supports_parameter_plug);
} else if (pcmk__str_eq(parameter, "port", pcmk__str_casei)) {
stonith__set_device_flags(*device_flags, device_name,
st_device_supports_parameter_port);
}
}
freeXpathObject(xpath);
}
+/*!
+ * \internal
+ * \brief Retrieve fence agent meta-data asynchronously
+ *
+ * \param[in] agent Agent to execute
+ * \param[in] timeout_sec Error if not complete within this time
+ * \param[in] callback Function to call with result (this will always be
+ * called, whether by this function directly or later
+ * via the main loop, and on success the metadata will
+ * be in its result argument's action_stdout)
+ * \param[in] user_data User data to pass to callback
+ *
+ * \return Standard Pacemaker return code
+ * \note The caller must use a main loop. This function is not a
+ * stonith_api_operations_t method because it does not need a stonith_t
+ * object and does not go through the fencer, but executes the agent
+ * directly.
+ */
+int
+stonith__metadata_async(const char *agent, int timeout_sec,
+ void (*callback)(int pid,
+ const pcmk__action_result_t *result,
+ void *user_data),
+ void *user_data)
+{
+ switch (stonith_get_namespace(agent, NULL)) {
+ case st_namespace_rhcs:
+ {
+ stonith_action_t *action = NULL;
+ int rc = pcmk_ok;
+
+ action = stonith__action_create(agent, "metadata", NULL, 0,
+ timeout_sec, NULL, NULL, NULL);
+
+ rc = stonith__execute_async(action, user_data, callback, NULL);
+ if (rc != pcmk_ok) {
+ callback(0, stonith__action_result(action), user_data);
+ stonith__destroy_action(action);
+ }
+ return pcmk_legacy2rc(rc);
+ }
+
+#if HAVE_STONITH_STONITH_H
+ case st_namespace_lha:
+ // LHA metadata is simply synthesized, so simulate async
+ {
+ pcmk__action_result_t result = {
+ .exit_status = CRM_EX_OK,
+ .execution_status = PCMK_EXEC_DONE,
+ .exit_reason = NULL,
+ .action_stdout = NULL,
+ .action_stderr = NULL,
+ };
+
+ stonith__lha_metadata(agent, timeout_sec,
+ &result.action_stdout);
+ callback(0, &result, user_data);
+ pcmk__reset_result(&result);
+ return pcmk_rc_ok;
+ }
+#endif
+
+ default:
+ {
+ pcmk__action_result_t result = {
+ .exit_status = CRM_EX_NOSUCH,
+ .execution_status = PCMK_EXEC_ERROR_HARD,
+ .exit_reason = crm_strdup_printf("No such agent '%s'",
+ agent),
+ .action_stdout = NULL,
+ .action_stderr = NULL,
+ };
+
+ callback(0, &result, user_data);
+ pcmk__reset_result(&result);
+ return ENOENT;
+ }
+ }
+}
+
/*!
* \internal
* \brief Return the exit status from an async action callback
*
* \param[in] data Callback data
*
* \return Exit status from callback data
*/
int
stonith__exit_status(stonith_callback_data_t *data)
{
if ((data == NULL) || (data->opaque == NULL)) {
return CRM_EX_ERROR;
}
return ((pcmk__action_result_t *) data->opaque)->exit_status;
}
/*!
* \internal
* \brief Return the execution status from an async action callback
*
* \param[in] data Callback data
*
* \return Execution status from callback data
*/
int
stonith__execution_status(stonith_callback_data_t *data)
{
if ((data == NULL) || (data->opaque == NULL)) {
return PCMK_EXEC_UNKNOWN;
}
return ((pcmk__action_result_t *) data->opaque)->execution_status;
}
/*!
* \internal
* \brief Return the exit reason from an async action callback
*
* \param[in] data Callback data
*
* \return Exit reason from callback data
*/
const char *
stonith__exit_reason(stonith_callback_data_t *data)
{
if ((data == NULL) || (data->opaque == NULL)) {
return NULL;
}
return ((pcmk__action_result_t *) data->opaque)->exit_reason;
}
/*!
* \internal
* \brief Return the exit status from an event notification
*
* \param[in] event Event
*
* \return Exit status from event
*/
int
stonith__event_exit_status(stonith_event_t *event)
{
if ((event == NULL) || (event->opaque == NULL)) {
return CRM_EX_ERROR;
} else {
struct event_private *event_private = event->opaque;
return event_private->result.exit_status;
}
}
/*!
* \internal
* \brief Return the execution status from an event notification
*
* \param[in] event Event
*
* \return Execution status from event
*/
int
stonith__event_execution_status(stonith_event_t *event)
{
if ((event == NULL) || (event->opaque == NULL)) {
return PCMK_EXEC_UNKNOWN;
} else {
struct event_private *event_private = event->opaque;
return event_private->result.execution_status;
}
}
/*!
* \internal
* \brief Return the exit reason from an event notification
*
* \param[in] event Event
*
* \return Exit reason from event
*/
const char *
stonith__event_exit_reason(stonith_event_t *event)
{
if ((event == NULL) || (event->opaque == NULL)) {
return NULL;
} else {
struct event_private *event_private = event->opaque;
return event_private->result.exit_reason;
}
}
/*!
* \internal
* \brief Return a human-friendly description of a fencing event
*
* \param[in] event Event to describe
*
* \return Newly allocated string with description of \p event
* \note The caller is responsible for freeing the return value.
* This function asserts on memory errors and never returns NULL.
*/
char *
stonith__event_description(stonith_event_t *event)
{
// Use somewhat readable defaults
const char *origin = pcmk__s(event->client_origin, "a client");
const char *origin_node = pcmk__s(event->origin, "a node");
const char *executioner = pcmk__s(event->executioner, "the cluster");
const char *device = pcmk__s(event->device, "unknown");
const char *action = pcmk__s(event->action, event->operation);
const char *target = pcmk__s(event->target, "no node");
const char *reason = stonith__event_exit_reason(event);
const char *status;
if (action == NULL) {
action = "(unknown)";
}
if (stonith__event_execution_status(event) != PCMK_EXEC_DONE) {
status = pcmk_exec_status_str(stonith__event_execution_status(event));
} else if (stonith__event_exit_status(event) != CRM_EX_OK) {
status = pcmk_exec_status_str(PCMK_EXEC_ERROR);
} else {
status = crm_exit_str(CRM_EX_OK);
}
if (pcmk__str_eq(event->operation, T_STONITH_NOTIFY_HISTORY,
pcmk__str_none)) {
return crm_strdup_printf("Fencing history may have changed");
} else if (pcmk__str_eq(event->operation, STONITH_OP_DEVICE_ADD,
pcmk__str_none)) {
return crm_strdup_printf("A fencing device (%s) was added", device);
} else if (pcmk__str_eq(event->operation, STONITH_OP_DEVICE_DEL,
pcmk__str_none)) {
return crm_strdup_printf("A fencing device (%s) was removed", device);
} else if (pcmk__str_eq(event->operation, STONITH_OP_LEVEL_ADD,
pcmk__str_none)) {
return crm_strdup_printf("A fencing topology level (%s) was added",
device);
} else if (pcmk__str_eq(event->operation, STONITH_OP_LEVEL_DEL,
pcmk__str_none)) {
return crm_strdup_printf("A fencing topology level (%s) was removed",
device);
}
// event->operation should be T_STONITH_NOTIFY_FENCE at this point
return crm_strdup_printf("Operation %s of %s by %s for %s@%s: %s%s%s%s (ref=%s)",
action, target, executioner, origin, origin_node,
status,
((reason == NULL)? "" : " ("), pcmk__s(reason, ""),
((reason == NULL)? "" : ")"),
pcmk__s(event->id, "(none)"));
}
// Deprecated functions kept only for backward API compatibility
// LCOV_EXCL_START
const char *get_stonith_provider(const char *agent, const char *provider);
const char *
get_stonith_provider(const char *agent, const char *provider)
{
return stonith_namespace2text(stonith_get_namespace(agent, provider));
}
// LCOV_EXCL_STOP
// End deprecated API
diff --git a/lib/fencing/st_rhcs.c b/lib/fencing/st_rhcs.c
index db00831fca..ec80793ca9 100644
--- a/lib/fencing/st_rhcs.c
+++ b/lib/fencing/st_rhcs.c
@@ -1,319 +1,316 @@
/*
* Copyright 2004-2022 the Pacemaker project contributors
*
* The version control history for this file may have further details.
*
* This source code is licensed under the GNU Lesser General Public License
* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
*/
#include <crm_internal.h>
#include <stdio.h>
#include <string.h>
#include <sys/stat.h>
#include <glib.h>
#include <dirent.h>
#include <crm/crm.h>
#include <crm/stonith-ng.h>
#include <crm/fencing/internal.h>
#include "fencing_private.h"
#define RH_STONITH_PREFIX "fence_"
/*!
* \internal
* \brief Add available RHCS-compatible agents to a list
*
* \param[in,out] List to add to
*
* \return Number of agents added
*/
int
stonith__list_rhcs_agents(stonith_key_value_t **devices)
{
// Essentially: ls -1 @sbin_dir@/fence_*
int count = 0, i;
struct dirent **namelist;
const int file_num = scandir(PCMK__FENCE_BINDIR, &namelist, 0, alphasort);
#if _POSIX_C_SOURCE < 200809L && !(defined(O_SEARCH) || defined(O_PATH))
char buffer[FILENAME_MAX + 1];
#elif defined(O_SEARCH)
const int dirfd = open(PCMK__FENCE_BINDIR, O_SEARCH);
#else
const int dirfd = open(PCMK__FENCE_BINDIR, O_PATH);
#endif
for (i = 0; i < file_num; i++) {
struct stat prop;
if (pcmk__starts_with(namelist[i]->d_name, RH_STONITH_PREFIX)) {
#if _POSIX_C_SOURCE < 200809L && !(defined(O_SEARCH) || defined(O_PATH))
snprintf(buffer, sizeof(buffer), "%s/%s", PCMK__FENCE_BINDIR,
namelist[i]->d_name);
if (stat(buffer, &prop) == 0 && S_ISREG(prop.st_mode)) {
#else
if (dirfd == -1) {
if (i == 0) {
crm_notice("Problem with listing %s directory"
CRM_XS "errno=%d", RH_STONITH_PREFIX, errno);
}
free(namelist[i]);
continue;
}
/* note: we can possibly prevent following symlinks here,
which may be a good idea, but fall on the nose when
these agents are moved elsewhere & linked back */
if (fstatat(dirfd, namelist[i]->d_name, &prop, 0) == 0
&& S_ISREG(prop.st_mode)) {
#endif
*devices = stonith_key_value_add(*devices, NULL,
namelist[i]->d_name);
count++;
}
}
free(namelist[i]);
}
if (file_num > 0) {
free(namelist);
}
#if _POSIX_C_SOURCE >= 200809L || defined(O_SEARCH) || defined(O_PATH)
if (dirfd >= 0) {
close(dirfd);
}
#endif
return count;
}
static void
stonith_rhcs_parameter_not_required(xmlNode *metadata, const char *parameter)
{
char *xpath = NULL;
xmlXPathObject *xpathObj = NULL;
CRM_CHECK(metadata != NULL, return);
CRM_CHECK(parameter != NULL, return);
xpath = crm_strdup_printf("//parameter[@name='%s']", parameter);
/* Fudge metadata so that the parameter isn't required in config
* Pacemaker handles and adds it */
xpathObj = xpath_search(metadata, xpath);
if (numXpathResults(xpathObj) > 0) {
xmlNode *tmp = getXpathResult(xpathObj, 0);
crm_xml_add(tmp, "required", "0");
}
freeXpathObject(xpathObj);
free(xpath);
}
/*!
- * \brief Execute RHCS-compatible agent's meta-data action
+ * \brief Execute RHCS-compatible agent's metadata action
*
- * \param[in] agent Agent to execute
- * \param[in] timeout Action timeout
- * \param[out] metadata Where to store output xmlNode (or NULL to ignore)
- *
- * \todo timeout is currently ignored; shouldn't we use it?
+ * \param[in] agent Agent to execute
+ * \param[in] timeout_sec Action timeout
+ * \param[out] metadata Where to store output xmlNode (or NULL to ignore)
*/
static int
-stonith__rhcs_get_metadata(const char *agent, int timeout, xmlNode **metadata)
+stonith__rhcs_get_metadata(const char *agent, int timeout_sec,
+ xmlNode **metadata)
{
xmlNode *xml = NULL;
xmlNode *actions = NULL;
xmlXPathObject *xpathObj = NULL;
- pcmk__action_result_t *result = NULL;
stonith_action_t *action = stonith__action_create(agent, "metadata", NULL,
- 0, 5, NULL, NULL, NULL);
+ 0, timeout_sec, NULL,
+ NULL, NULL);
int rc = stonith__execute(action);
- result = stonith__action_result(action);
+ pcmk__action_result_t *result = stonith__action_result(action);
if (result == NULL) {
if (rc < 0) {
crm_warn("Could not execute metadata action for %s: %s "
CRM_XS " rc=%d", agent, pcmk_strerror(rc), rc);
}
stonith__destroy_action(action);
return rc;
}
if (result->execution_status != PCMK_EXEC_DONE) {
crm_warn("Could not execute metadata action for %s: %s",
agent, pcmk_exec_status_str(result->execution_status));
rc = pcmk_rc2legacy(stonith__result2rc(result));
stonith__destroy_action(action);
return rc;
}
if (!pcmk__result_ok(result)) {
crm_warn("Metadata action for %s returned error code %d",
agent, result->exit_status);
rc = pcmk_rc2legacy(stonith__result2rc(result));
stonith__destroy_action(action);
return rc;
}
if (result->action_stdout == NULL) {
crm_warn("Metadata action for %s returned no data", agent);
stonith__destroy_action(action);
return -ENODATA;
}
xml = string2xml(result->action_stdout);
stonith__destroy_action(action);
if (xml == NULL) {
crm_warn("Metadata for %s is invalid", agent);
return -pcmk_err_schema_validation;
}
xpathObj = xpath_search(xml, "//actions");
if (numXpathResults(xpathObj) > 0) {
actions = getXpathResult(xpathObj, 0);
}
freeXpathObject(xpathObj);
// Add start and stop (implemented by pacemaker, not agent) to meta-data
xpathObj = xpath_search(xml, "//action[@name='stop']");
if (numXpathResults(xpathObj) <= 0) {
xmlNode *tmp = NULL;
tmp = create_xml_node(actions, "action");
crm_xml_add(tmp, "name", "stop");
crm_xml_add(tmp, "timeout", CRM_DEFAULT_OP_TIMEOUT_S);
tmp = create_xml_node(actions, "action");
crm_xml_add(tmp, "name", "start");
crm_xml_add(tmp, "timeout", CRM_DEFAULT_OP_TIMEOUT_S);
}
freeXpathObject(xpathObj);
// Fudge metadata so parameters are not required in config (pacemaker adds them)
stonith_rhcs_parameter_not_required(xml, "action");
stonith_rhcs_parameter_not_required(xml, "plug");
stonith_rhcs_parameter_not_required(xml, "port");
if (metadata) {
*metadata = xml;
} else {
free_xml(xml);
}
return pcmk_ok;
}
/*!
- * \brief Execute RHCS-compatible agent's meta-data action
- *
- * \param[in] agent Agent to execute
- * \param[in] timeout Action timeout
- * \param[out] output Where to store action output (or NULL to ignore)
+ * \brief Retrieve metadata for RHCS-compatible fence agent
*
- * \todo timeout is currently ignored; shouldn't we use it?
+ * \param[in] agent Agent to execute
+ * \param[in] timeout_sec Action timeout
+ * \param[out] output Where to store action output (or NULL to ignore)
*/
int
-stonith__rhcs_metadata(const char *agent, int timeout, char **output)
+stonith__rhcs_metadata(const char *agent, int timeout_sec, char **output)
{
char *buffer = NULL;
xmlNode *xml = NULL;
- int rc = stonith__rhcs_get_metadata(agent, timeout, &xml);
+ int rc = stonith__rhcs_get_metadata(agent, timeout_sec, &xml);
if (rc != pcmk_ok) {
free_xml(xml);
return rc;
}
buffer = dump_xml_formatted_with_text(xml);
free_xml(xml);
if (buffer == NULL) {
return -pcmk_err_schema_validation;
}
if (output) {
*output = buffer;
} else {
free(buffer);
}
return pcmk_ok;
}
bool
stonith__agent_is_rhcs(const char *agent)
{
struct stat prop;
char *buffer = crm_strdup_printf(PCMK__FENCE_BINDIR "/%s", agent);
int rc = stat(buffer, &prop);
free(buffer);
return (rc >= 0) && S_ISREG(prop.st_mode);
}
int
stonith__rhcs_validate(stonith_t *st, int call_options, const char *target,
const char *agent, GHashTable *params,
const char * host_arg, int timeout,
char **output, char **error_output)
{
int rc = pcmk_ok;
int remaining_timeout = timeout;
xmlNode *metadata = NULL;
stonith_action_t *action = NULL;
pcmk__action_result_t *result = NULL;
if (host_arg == NULL) {
time_t start_time = time(NULL);
rc = stonith__rhcs_get_metadata(agent, remaining_timeout, &metadata);
if (rc == pcmk_ok) {
uint32_t device_flags = 0;
stonith__device_parameter_flags(&device_flags, agent, metadata);
if (pcmk_is_set(device_flags, st_device_supports_parameter_port)) {
host_arg = "port";
} else if (pcmk_is_set(device_flags,
st_device_supports_parameter_plug)) {
host_arg = "plug";
}
}
free_xml(metadata);
remaining_timeout -= time(NULL) - start_time;
if (rc == -ETIME || remaining_timeout <= 0 ) {
return -ETIME;
}
} else if (pcmk__str_eq(host_arg, PCMK__VALUE_NONE, pcmk__str_casei)) {
host_arg = NULL;
}
action = stonith__action_create(agent, "validate-all", target, 0,
remaining_timeout, params, NULL, host_arg);
rc = stonith__execute(action);
result = stonith__action_result(action);
if (result != NULL) {
rc = pcmk_rc2legacy(stonith__result2rc(result));
// Take ownership of output so stonith__destroy_action() doesn't free it
if (output != NULL) {
*output = result->action_stdout;
result->action_stdout = NULL;
}
if (error_output != NULL) {
*error_output = result->action_stderr;
result->action_stderr = NULL;
}
}
stonith__destroy_action(action);
return rc;
}
diff --git a/lib/lrmd/lrmd_client.c b/lib/lrmd/lrmd_client.c
index f6b797d675..06ffd66229 100644
--- a/lib/lrmd/lrmd_client.c
+++ b/lib/lrmd/lrmd_client.c
@@ -1,2386 +1,2502 @@
/*
* Copyright 2012-2022 the Pacemaker project contributors
*
* The version control history for this file may have further details.
*
* This source code is licensed under the GNU Lesser General Public License
* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
*/
#include <crm_internal.h>
#include <unistd.h>
#include <stdlib.h>
#include <stdio.h>
#include <stdint.h> // uint32_t, uint64_t
#include <stdarg.h>
#include <string.h>
#include <ctype.h>
#include <errno.h>
#include <sys/types.h>
#include <sys/wait.h>
#include <glib.h>
#include <dirent.h>
#include <crm/crm.h>
#include <crm/lrmd.h>
#include <crm/lrmd_internal.h>
#include <crm/services.h>
#include <crm/services_internal.h>
#include <crm/common/mainloop.h>
#include <crm/common/ipc_internal.h>
#include <crm/common/remote_internal.h>
#include <crm/msg_xml.h>
#include <crm/stonith-ng.h>
#include <crm/fencing/internal.h>
#ifdef HAVE_GNUTLS_GNUTLS_H
# include <gnutls/gnutls.h>
#endif
#include <sys/socket.h>
#include <netinet/in.h>
#include <netinet/ip.h>
#include <arpa/inet.h>
#include <netdb.h>
#define MAX_TLS_RECV_WAIT 10000
CRM_TRACE_INIT_DATA(lrmd);
static int lrmd_api_disconnect(lrmd_t * lrmd);
static int lrmd_api_is_connected(lrmd_t * lrmd);
/* IPC proxy functions */
int lrmd_internal_proxy_send(lrmd_t * lrmd, xmlNode *msg);
static void lrmd_internal_proxy_dispatch(lrmd_t *lrmd, xmlNode *msg);
void lrmd_internal_set_proxy_callback(lrmd_t * lrmd, void *userdata, void (*callback)(lrmd_t *lrmd, void *userdata, xmlNode *msg));
#ifdef HAVE_GNUTLS_GNUTLS_H
# define LRMD_CLIENT_HANDSHAKE_TIMEOUT 5000 /* 5 seconds */
gnutls_psk_client_credentials_t psk_cred_s;
static void lrmd_tls_disconnect(lrmd_t * lrmd);
static int global_remote_msg_id = 0;
static void lrmd_tls_connection_destroy(gpointer userdata);
#endif
typedef struct lrmd_private_s {
uint64_t type;
char *token;
mainloop_io_t *source;
/* IPC parameters */
crm_ipc_t *ipc;
pcmk__remote_t *remote;
/* Extra TLS parameters */
char *remote_nodename;
#ifdef HAVE_GNUTLS_GNUTLS_H
char *server;
int port;
gnutls_psk_client_credentials_t psk_cred_c;
/* while the async connection is occurring, this is the id
* of the connection timeout timer. */
int async_timer;
int sock;
/* since tls requires a round trip across the network for a
* request/reply, there are times where we just want to be able
* to send a request from the client and not wait around (or even care
* about) what the reply is. */
int expected_late_replies;
GList *pending_notify;
crm_trigger_t *process_notify;
#endif
lrmd_event_callback callback;
/* Internal IPC proxy msg passing for remote guests */
void (*proxy_callback)(lrmd_t *lrmd, void *userdata, xmlNode *msg);
void *proxy_callback_userdata;
char *peer_version;
} lrmd_private_t;
static lrmd_list_t *
lrmd_list_add(lrmd_list_t * head, const char *value)
{
lrmd_list_t *p, *end;
p = calloc(1, sizeof(lrmd_list_t));
p->val = strdup(value);
end = head;
while (end && end->next) {
end = end->next;
}
if (end) {
end->next = p;
} else {
head = p;
}
return head;
}
void
lrmd_list_freeall(lrmd_list_t * head)
{
lrmd_list_t *p;
while (head) {
char *val = (char *)head->val;
p = head->next;
free(val);
free(head);
head = p;
}
}
lrmd_key_value_t *
lrmd_key_value_add(lrmd_key_value_t * head, const char *key, const char *value)
{
lrmd_key_value_t *p, *end;
p = calloc(1, sizeof(lrmd_key_value_t));
p->key = strdup(key);
p->value = strdup(value);
end = head;
while (end && end->next) {
end = end->next;
}
if (end) {
end->next = p;
} else {
head = p;
}
return head;
}
void
lrmd_key_value_freeall(lrmd_key_value_t * head)
{
lrmd_key_value_t *p;
while (head) {
p = head->next;
free(head->key);
free(head->value);
free(head);
head = p;
}
}
/*!
* Create a new lrmd_event_data_t object
*
* \param[in] rsc_id ID of resource involved in event
* \param[in] task Action name
* \param[in] interval_ms Action interval
*
* \return Newly allocated and initialized lrmd_event_data_t
* \note This functions asserts on memory errors, so the return value is
* guaranteed to be non-NULL. The caller is responsible for freeing the
* result with lrmd_free_event().
*/
lrmd_event_data_t *
lrmd_new_event(const char *rsc_id, const char *task, guint interval_ms)
{
lrmd_event_data_t *event = calloc(1, sizeof(lrmd_event_data_t));
CRM_ASSERT(event != NULL);
pcmk__str_update((char **) &event->rsc_id, rsc_id);
pcmk__str_update((char **) &event->op_type, task);
event->interval_ms = interval_ms;
return event;
}
lrmd_event_data_t *
lrmd_copy_event(lrmd_event_data_t * event)
{
lrmd_event_data_t *copy = NULL;
copy = calloc(1, sizeof(lrmd_event_data_t));
copy->type = event->type;
pcmk__str_update((char **) &copy->rsc_id, event->rsc_id);
pcmk__str_update((char **) &copy->op_type, event->op_type);
pcmk__str_update((char **) &copy->user_data, event->user_data);
copy->call_id = event->call_id;
copy->timeout = event->timeout;
copy->interval_ms = event->interval_ms;
copy->start_delay = event->start_delay;
copy->rsc_deleted = event->rsc_deleted;
copy->rc = event->rc;
copy->op_status = event->op_status;
pcmk__str_update((char **) &copy->output, event->output);
copy->t_run = event->t_run;
copy->t_rcchange = event->t_rcchange;
copy->exec_time = event->exec_time;
copy->queue_time = event->queue_time;
copy->connection_rc = event->connection_rc;
copy->params = pcmk__str_table_dup(event->params);
pcmk__str_update((char **) &copy->remote_nodename, event->remote_nodename);
pcmk__str_update((char **) &copy->exit_reason, event->exit_reason);
return copy;
}
/*!
* \brief Free an executor event
*
* \param[in] Executor event object to free
*/
void
lrmd_free_event(lrmd_event_data_t *event)
{
if (event == NULL) {
return;
}
// @TODO Why are these const char *?
free((void *) event->rsc_id);
free((void *) event->op_type);
free((void *) event->user_data);
free((void *) event->remote_nodename);
lrmd__reset_result(event);
if (event->params != NULL) {
g_hash_table_destroy(event->params);
}
free(event);
}
static void
lrmd_dispatch_internal(lrmd_t * lrmd, xmlNode * msg)
{
const char *type;
const char *proxy_session = crm_element_value(msg, F_LRMD_IPC_SESSION);
lrmd_private_t *native = lrmd->lrmd_private;
lrmd_event_data_t event = { 0, };
if (proxy_session != NULL) {
/* this is proxy business */
lrmd_internal_proxy_dispatch(lrmd, msg);
return;
} else if (!native->callback) {
/* no callback set */
crm_trace("notify event received but client has not set callback");
return;
}
event.remote_nodename = native->remote_nodename;
type = crm_element_value(msg, F_LRMD_OPERATION);
crm_element_value_int(msg, F_LRMD_CALLID, &event.call_id);
event.rsc_id = crm_element_value(msg, F_LRMD_RSC_ID);
if (pcmk__str_eq(type, LRMD_OP_RSC_REG, pcmk__str_none)) {
event.type = lrmd_event_register;
} else if (pcmk__str_eq(type, LRMD_OP_RSC_UNREG, pcmk__str_none)) {
event.type = lrmd_event_unregister;
} else if (pcmk__str_eq(type, LRMD_OP_RSC_EXEC, pcmk__str_none)) {
time_t epoch = 0;
crm_element_value_int(msg, F_LRMD_TIMEOUT, &event.timeout);
crm_element_value_ms(msg, F_LRMD_RSC_INTERVAL, &event.interval_ms);
crm_element_value_int(msg, F_LRMD_RSC_START_DELAY, &event.start_delay);
crm_element_value_int(msg, F_LRMD_EXEC_RC, (int *)&event.rc);
crm_element_value_int(msg, F_LRMD_OP_STATUS, &event.op_status);
crm_element_value_int(msg, F_LRMD_RSC_DELETED, &event.rsc_deleted);
crm_element_value_epoch(msg, F_LRMD_RSC_RUN_TIME, &epoch);
event.t_run = (unsigned int) epoch;
crm_element_value_epoch(msg, F_LRMD_RSC_RCCHANGE_TIME, &epoch);
event.t_rcchange = (unsigned int) epoch;
crm_element_value_int(msg, F_LRMD_RSC_EXEC_TIME, (int *)&event.exec_time);
crm_element_value_int(msg, F_LRMD_RSC_QUEUE_TIME, (int *)&event.queue_time);
event.op_type = crm_element_value(msg, F_LRMD_RSC_ACTION);
event.user_data = crm_element_value(msg, F_LRMD_RSC_USERDATA_STR);
event.type = lrmd_event_exec_complete;
/* output and exit_reason may be freed by a callback */
event.output = crm_element_value_copy(msg, F_LRMD_RSC_OUTPUT);
lrmd__set_result(&event, event.rc, event.op_status,
crm_element_value(msg, F_LRMD_RSC_EXIT_REASON));
event.params = xml2list(msg);
} else if (pcmk__str_eq(type, LRMD_OP_NEW_CLIENT, pcmk__str_none)) {
event.type = lrmd_event_new_client;
} else if (pcmk__str_eq(type, LRMD_OP_POKE, pcmk__str_none)) {
event.type = lrmd_event_poke;
} else {
return;
}
crm_trace("op %s notify event received", type);
native->callback(&event);
if (event.params) {
g_hash_table_destroy(event.params);
}
lrmd__reset_result(&event);
}
// \return Always 0, to indicate that IPC mainloop source should be kept
static int
lrmd_ipc_dispatch(const char *buffer, ssize_t length, gpointer userdata)
{
lrmd_t *lrmd = userdata;
lrmd_private_t *native = lrmd->lrmd_private;
if (native->callback != NULL) {
xmlNode *msg = string2xml(buffer);
lrmd_dispatch_internal(lrmd, msg);
free_xml(msg);
}
return 0;
}
#ifdef HAVE_GNUTLS_GNUTLS_H
static void
lrmd_free_xml(gpointer userdata)
{
free_xml((xmlNode *) userdata);
}
static bool
remote_executor_connected(lrmd_t * lrmd)
{
lrmd_private_t *native = lrmd->lrmd_private;
return (native->remote->tls_session != NULL);
}
/*!
* \internal
* \brief TLS dispatch function (for both trigger and file descriptor sources)
*
* \param[in] userdata API connection
*
* \return Always return a nonnegative value, which as a file descriptor
* dispatch function means keep the mainloop source, and as a
* trigger dispatch function, 0 means remove the trigger from the
* mainloop while 1 means keep it (and job completed)
*/
static int
lrmd_tls_dispatch(gpointer userdata)
{
lrmd_t *lrmd = userdata;
lrmd_private_t *native = lrmd->lrmd_private;
xmlNode *xml = NULL;
int rc = pcmk_rc_ok;
if (!remote_executor_connected(lrmd)) {
crm_trace("TLS dispatch triggered after disconnect");
return 0;
}
crm_trace("TLS dispatch triggered");
/* First check if there are any pending notifies to process that came
* while we were waiting for replies earlier. */
if (native->pending_notify) {
GList *iter = NULL;
crm_trace("Processing pending notifies");
for (iter = native->pending_notify; iter; iter = iter->next) {
lrmd_dispatch_internal(lrmd, iter->data);
}
g_list_free_full(native->pending_notify, lrmd_free_xml);
native->pending_notify = NULL;
}
/* Next read the current buffer and see if there are any messages to handle. */
switch (pcmk__remote_ready(native->remote, 0)) {
case pcmk_rc_ok:
rc = pcmk__read_remote_message(native->remote, -1);
xml = pcmk__remote_message_xml(native->remote);
break;
case ETIME:
// Nothing to read, check if a full message is already in buffer
xml = pcmk__remote_message_xml(native->remote);
break;
default:
rc = ENOTCONN;
break;
}
while (xml) {
const char *msg_type = crm_element_value(xml, F_LRMD_REMOTE_MSG_TYPE);
if (pcmk__str_eq(msg_type, "notify", pcmk__str_casei)) {
lrmd_dispatch_internal(lrmd, xml);
} else if (pcmk__str_eq(msg_type, "reply", pcmk__str_casei)) {
if (native->expected_late_replies > 0) {
native->expected_late_replies--;
} else {
int reply_id = 0;
crm_element_value_int(xml, F_LRMD_CALLID, &reply_id);
/* if this happens, we want to know about it */
crm_err("Got outdated Pacemaker Remote reply %d", reply_id);
}
}
free_xml(xml);
xml = pcmk__remote_message_xml(native->remote);
}
if (rc == ENOTCONN) {
crm_info("Lost %s executor connection while reading data",
(native->remote_nodename? native->remote_nodename : "local"));
lrmd_tls_disconnect(lrmd);
return 0;
}
return 1;
}
#endif
/* Not used with mainloop */
int
lrmd_poll(lrmd_t * lrmd, int timeout)
{
lrmd_private_t *native = lrmd->lrmd_private;
switch (native->type) {
case pcmk__client_ipc:
return crm_ipc_ready(native->ipc);
#ifdef HAVE_GNUTLS_GNUTLS_H
case pcmk__client_tls:
if (native->pending_notify) {
return 1;
} else {
int rc = pcmk__remote_ready(native->remote, 0);
switch (rc) {
case pcmk_rc_ok:
return 1;
case ETIME:
return 0;
default:
return pcmk_rc2legacy(rc);
}
}
#endif
default:
crm_err("Unsupported executor connection type (bug?): %d",
native->type);
return -EPROTONOSUPPORT;
}
}
/* Not used with mainloop */
bool
lrmd_dispatch(lrmd_t * lrmd)
{
lrmd_private_t *private = NULL;
CRM_ASSERT(lrmd != NULL);
private = lrmd->lrmd_private;
switch (private->type) {
case pcmk__client_ipc:
while (crm_ipc_ready(private->ipc)) {
if (crm_ipc_read(private->ipc) > 0) {
const char *msg = crm_ipc_buffer(private->ipc);
lrmd_ipc_dispatch(msg, strlen(msg), lrmd);
}
}
break;
#ifdef HAVE_GNUTLS_GNUTLS_H
case pcmk__client_tls:
lrmd_tls_dispatch(lrmd);
break;
#endif
default:
crm_err("Unsupported executor connection type (bug?): %d",
private->type);
}
if (lrmd_api_is_connected(lrmd) == FALSE) {
crm_err("Connection closed");
return FALSE;
}
return TRUE;
}
static xmlNode *
lrmd_create_op(const char *token, const char *op, xmlNode *data, int timeout,
enum lrmd_call_options options)
{
xmlNode *op_msg = create_xml_node(NULL, "lrmd_command");
CRM_CHECK(op_msg != NULL, return NULL);
CRM_CHECK(token != NULL, return NULL);
crm_xml_add(op_msg, F_XML_TAGNAME, "lrmd_command");
crm_xml_add(op_msg, F_TYPE, T_LRMD);
crm_xml_add(op_msg, F_LRMD_CALLBACK_TOKEN, token);
crm_xml_add(op_msg, F_LRMD_OPERATION, op);
crm_xml_add_int(op_msg, F_LRMD_TIMEOUT, timeout);
crm_xml_add_int(op_msg, F_LRMD_CALLOPTS, options);
if (data != NULL) {
add_message_xml(op_msg, F_LRMD_CALLDATA, data);
}
crm_trace("Created executor %s command with call options %.8lx (%d)",
op, (long)options, options);
return op_msg;
}
static void
lrmd_ipc_connection_destroy(gpointer userdata)
{
lrmd_t *lrmd = userdata;
lrmd_private_t *native = lrmd->lrmd_private;
crm_info("IPC connection destroyed");
/* Prevent these from being cleaned up in lrmd_api_disconnect() */
native->ipc = NULL;
native->source = NULL;
if (native->callback) {
lrmd_event_data_t event = { 0, };
event.type = lrmd_event_disconnect;
event.remote_nodename = native->remote_nodename;
native->callback(&event);
}
}
#ifdef HAVE_GNUTLS_GNUTLS_H
static void
lrmd_tls_connection_destroy(gpointer userdata)
{
lrmd_t *lrmd = userdata;
lrmd_private_t *native = lrmd->lrmd_private;
crm_info("TLS connection destroyed");
if (native->remote->tls_session) {
gnutls_bye(*native->remote->tls_session, GNUTLS_SHUT_RDWR);
gnutls_deinit(*native->remote->tls_session);
gnutls_free(native->remote->tls_session);
}
if (native->psk_cred_c) {
gnutls_psk_free_client_credentials(native->psk_cred_c);
}
if (native->sock) {
close(native->sock);
}
if (native->process_notify) {
mainloop_destroy_trigger(native->process_notify);
native->process_notify = NULL;
}
if (native->pending_notify) {
g_list_free_full(native->pending_notify, lrmd_free_xml);
native->pending_notify = NULL;
}
free(native->remote->buffer);
native->remote->buffer = NULL;
native->source = 0;
native->sock = 0;
native->psk_cred_c = NULL;
native->remote->tls_session = NULL;
native->sock = 0;
if (native->callback) {
lrmd_event_data_t event = { 0, };
event.remote_nodename = native->remote_nodename;
event.type = lrmd_event_disconnect;
native->callback(&event);
}
return;
}
// \return Standard Pacemaker return code
int
lrmd__remote_send_xml(pcmk__remote_t *session, xmlNode *msg, uint32_t id,
const char *msg_type)
{
crm_xml_add_int(msg, F_LRMD_REMOTE_MSG_ID, id);
crm_xml_add(msg, F_LRMD_REMOTE_MSG_TYPE, msg_type);
return pcmk__remote_send_xml(session, msg);
}
// \return Standard Pacemaker return code
static int
read_remote_reply(lrmd_t *lrmd, int total_timeout, int expected_reply_id,
xmlNode **reply)
{
lrmd_private_t *native = lrmd->lrmd_private;
time_t start = time(NULL);
const char *msg_type = NULL;
int reply_id = 0;
int remaining_timeout = 0;
int rc = pcmk_rc_ok;
/* A timeout of 0 here makes no sense. We have to wait a period of time
* for the response to come back. If -1 or 0, default to 10 seconds. */
if (total_timeout <= 0 || total_timeout > MAX_TLS_RECV_WAIT) {
total_timeout = MAX_TLS_RECV_WAIT;
}
for (*reply = NULL; *reply == NULL; ) {
*reply = pcmk__remote_message_xml(native->remote);
if (*reply == NULL) {
/* read some more off the tls buffer if we still have time left. */
if (remaining_timeout) {
remaining_timeout = total_timeout - ((time(NULL) - start) * 1000);
} else {
remaining_timeout = total_timeout;
}
if (remaining_timeout <= 0) {
return ETIME;
}
rc = pcmk__read_remote_message(native->remote, remaining_timeout);
if (rc != pcmk_rc_ok) {
return rc;
}
*reply = pcmk__remote_message_xml(native->remote);
if (*reply == NULL) {
return ENOMSG;
}
}
crm_element_value_int(*reply, F_LRMD_REMOTE_MSG_ID, &reply_id);
msg_type = crm_element_value(*reply, F_LRMD_REMOTE_MSG_TYPE);
if (!msg_type) {
crm_err("Empty msg type received while waiting for reply");
free_xml(*reply);
*reply = NULL;
} else if (pcmk__str_eq(msg_type, "notify", pcmk__str_casei)) {
/* got a notify while waiting for reply, trigger the notify to be processed later */
crm_info("queueing notify");
native->pending_notify = g_list_append(native->pending_notify, *reply);
if (native->process_notify) {
crm_info("notify trigger set.");
mainloop_set_trigger(native->process_notify);
}
*reply = NULL;
} else if (!pcmk__str_eq(msg_type, "reply", pcmk__str_casei)) {
/* msg isn't a reply, make some noise */
crm_err("Expected a reply, got %s", msg_type);
free_xml(*reply);
*reply = NULL;
} else if (reply_id != expected_reply_id) {
if (native->expected_late_replies > 0) {
native->expected_late_replies--;
} else {
crm_err("Got outdated reply, expected id %d got id %d", expected_reply_id, reply_id);
}
free_xml(*reply);
*reply = NULL;
}
}
if (native->remote->buffer && native->process_notify) {
mainloop_set_trigger(native->process_notify);
}
return rc;
}
// \return Standard Pacemaker return code
static int
send_remote_message(lrmd_t *lrmd, xmlNode *msg)
{
int rc = pcmk_rc_ok;
lrmd_private_t *native = lrmd->lrmd_private;
global_remote_msg_id++;
if (global_remote_msg_id <= 0) {
global_remote_msg_id = 1;
}
rc = lrmd__remote_send_xml(native->remote, msg, global_remote_msg_id,
"request");
if (rc != pcmk_rc_ok) {
crm_err("Disconnecting because TLS message could not be sent to "
"Pacemaker Remote: %s", pcmk_rc_str(rc));
lrmd_tls_disconnect(lrmd);
}
return rc;
}
static int
lrmd_tls_send_recv(lrmd_t * lrmd, xmlNode * msg, int timeout, xmlNode ** reply)
{
int rc = 0;
xmlNode *xml = NULL;
if (!remote_executor_connected(lrmd)) {
return -ENOTCONN;
}
rc = send_remote_message(lrmd, msg);
if (rc != pcmk_rc_ok) {
return pcmk_rc2legacy(rc);
}
rc = read_remote_reply(lrmd, timeout, global_remote_msg_id, &xml);
if (rc != pcmk_rc_ok) {
crm_err("Disconnecting remote after request %d reply not received: %s "
CRM_XS " rc=%d timeout=%dms",
global_remote_msg_id, pcmk_rc_str(rc), rc, timeout);
lrmd_tls_disconnect(lrmd);
}
if (reply) {
*reply = xml;
} else {
free_xml(xml);
}
return pcmk_rc2legacy(rc);
}
#endif
static int
lrmd_send_xml(lrmd_t * lrmd, xmlNode * msg, int timeout, xmlNode ** reply)
{
int rc = pcmk_ok;
lrmd_private_t *native = lrmd->lrmd_private;
switch (native->type) {
case pcmk__client_ipc:
rc = crm_ipc_send(native->ipc, msg, crm_ipc_client_response, timeout, reply);
break;
#ifdef HAVE_GNUTLS_GNUTLS_H
case pcmk__client_tls:
rc = lrmd_tls_send_recv(lrmd, msg, timeout, reply);
break;
#endif
default:
crm_err("Unsupported executor connection type (bug?): %d",
native->type);
rc = -EPROTONOSUPPORT;
}
return rc;
}
static int
lrmd_send_xml_no_reply(lrmd_t * lrmd, xmlNode * msg)
{
int rc = pcmk_ok;
lrmd_private_t *native = lrmd->lrmd_private;
switch (native->type) {
case pcmk__client_ipc:
rc = crm_ipc_send(native->ipc, msg, crm_ipc_flags_none, 0, NULL);
break;
#ifdef HAVE_GNUTLS_GNUTLS_H
case pcmk__client_tls:
rc = send_remote_message(lrmd, msg);
if (rc == pcmk_rc_ok) {
/* we don't want to wait around for the reply, but
* since the request/reply protocol needs to behave the same
* as libqb, a reply will eventually come later anyway. */
native->expected_late_replies++;
}
rc = pcmk_rc2legacy(rc);
break;
#endif
default:
crm_err("Unsupported executor connection type (bug?): %d",
native->type);
rc = -EPROTONOSUPPORT;
}
return rc;
}
static int
lrmd_api_is_connected(lrmd_t * lrmd)
{
lrmd_private_t *native = lrmd->lrmd_private;
switch (native->type) {
case pcmk__client_ipc:
return crm_ipc_connected(native->ipc);
#ifdef HAVE_GNUTLS_GNUTLS_H
case pcmk__client_tls:
return remote_executor_connected(lrmd);
#endif
default:
crm_err("Unsupported executor connection type (bug?): %d",
native->type);
return 0;
}
}
/*!
* \internal
* \brief Send a prepared API command to the executor
*
* \param[in] lrmd Existing connection to the executor
* \param[in] op Name of API command to send
* \param[in] data Command data XML to add to the sent command
* \param[out] output_data If expecting a reply, it will be stored here
* \param[in] timeout Timeout in milliseconds (if 0, defaults to
* a sensible value per the type of connection,
* standard vs. pacemaker remote);
* also propagated to the command XML
* \param[in] call_options Call options to pass to server when sending
* \param[in] expect_reply If TRUE, wait for a reply from the server;
* must be TRUE for IPC (as opposed to TLS) clients
*
* \return pcmk_ok on success, -errno on error
*/
static int
lrmd_send_command(lrmd_t *lrmd, const char *op, xmlNode *data,
xmlNode **output_data, int timeout,
enum lrmd_call_options options, gboolean expect_reply)
{
int rc = pcmk_ok;
lrmd_private_t *native = lrmd->lrmd_private;
xmlNode *op_msg = NULL;
xmlNode *op_reply = NULL;
if (!lrmd_api_is_connected(lrmd)) {
return -ENOTCONN;
}
if (op == NULL) {
crm_err("No operation specified");
return -EINVAL;
}
CRM_CHECK(native->token != NULL,;
);
crm_trace("Sending %s op to executor", op);
op_msg = lrmd_create_op(native->token, op, data, timeout, options);
if (op_msg == NULL) {
return -EINVAL;
}
if (expect_reply) {
rc = lrmd_send_xml(lrmd, op_msg, timeout, &op_reply);
} else {
rc = lrmd_send_xml_no_reply(lrmd, op_msg);
goto done;
}
if (rc < 0) {
crm_perror(LOG_ERR, "Couldn't perform %s operation (timeout=%d): %d", op, timeout, rc);
goto done;
} else if(op_reply == NULL) {
rc = -ENOMSG;
goto done;
}
rc = pcmk_ok;
crm_trace("%s op reply received", op);
if (crm_element_value_int(op_reply, F_LRMD_RC, &rc) != 0) {
rc = -ENOMSG;
goto done;
}
crm_log_xml_trace(op_reply, "Reply");
if (output_data) {
*output_data = op_reply;
op_reply = NULL; /* Prevent subsequent free */
}
done:
if (lrmd_api_is_connected(lrmd) == FALSE) {
crm_err("Executor disconnected");
}
free_xml(op_msg);
free_xml(op_reply);
return rc;
}
static int
lrmd_api_poke_connection(lrmd_t * lrmd)
{
int rc;
lrmd_private_t *native = lrmd->lrmd_private;
xmlNode *data = create_xml_node(NULL, F_LRMD_RSC);
crm_xml_add(data, F_LRMD_ORIGIN, __func__);
rc = lrmd_send_command(lrmd, LRMD_OP_POKE, data, NULL, 0, 0,
(native->type == pcmk__client_ipc));
free_xml(data);
return rc < 0 ? rc : pcmk_ok;
}
// \return Standard Pacemaker return code
int
lrmd__validate_remote_settings(lrmd_t *lrmd, GHashTable *hash)
{
int rc = pcmk_rc_ok;
const char *value;
lrmd_private_t *native = lrmd->lrmd_private;
xmlNode *data = create_xml_node(NULL, F_LRMD_OPERATION);
crm_xml_add(data, F_LRMD_ORIGIN, __func__);
value = g_hash_table_lookup(hash, "stonith-watchdog-timeout");
if ((value) &&
(stonith__watchdog_fencing_enabled_for_node(native->remote_nodename))) {
crm_xml_add(data, F_LRMD_WATCHDOG, value);
}
rc = lrmd_send_command(lrmd, LRMD_OP_CHECK, data, NULL, 0, 0,
(native->type == pcmk__client_ipc));
free_xml(data);
return (rc < 0)? pcmk_legacy2rc(rc) : pcmk_rc_ok;
}
static int
lrmd_handshake(lrmd_t * lrmd, const char *name)
{
int rc = pcmk_ok;
lrmd_private_t *native = lrmd->lrmd_private;
xmlNode *reply = NULL;
xmlNode *hello = create_xml_node(NULL, "lrmd_command");
crm_xml_add(hello, F_TYPE, T_LRMD);
crm_xml_add(hello, F_LRMD_OPERATION, CRM_OP_REGISTER);
crm_xml_add(hello, F_LRMD_CLIENTNAME, name);
crm_xml_add(hello, F_LRMD_PROTOCOL_VERSION, LRMD_PROTOCOL_VERSION);
/* advertise that we are a proxy provider */
if (native->proxy_callback) {
pcmk__xe_set_bool_attr(hello, F_LRMD_IS_IPC_PROVIDER, true);
}
rc = lrmd_send_xml(lrmd, hello, -1, &reply);
if (rc < 0) {
crm_perror(LOG_DEBUG, "Couldn't complete registration with the executor API: %d", rc);
rc = -ECOMM;
} else if (reply == NULL) {
crm_err("Did not receive registration reply");
rc = -EPROTO;
} else {
const char *version = crm_element_value(reply, F_LRMD_PROTOCOL_VERSION);
const char *msg_type = crm_element_value(reply, F_LRMD_OPERATION);
const char *tmp_ticket = crm_element_value(reply, F_LRMD_CLIENTID);
crm_element_value_int(reply, F_LRMD_RC, &rc);
if (rc == -EPROTO) {
crm_err("Executor protocol version mismatch between client (%s) and server (%s)",
LRMD_PROTOCOL_VERSION, version);
crm_log_xml_err(reply, "Protocol Error");
} else if (!pcmk__str_eq(msg_type, CRM_OP_REGISTER, pcmk__str_casei)) {
crm_err("Invalid registration message: %s", msg_type);
crm_log_xml_err(reply, "Bad reply");
rc = -EPROTO;
} else if (tmp_ticket == NULL) {
crm_err("No registration token provided");
crm_log_xml_err(reply, "Bad reply");
rc = -EPROTO;
} else {
crm_trace("Obtained registration token: %s", tmp_ticket);
native->token = strdup(tmp_ticket);
native->peer_version = strdup(version?version:"1.0"); /* Included since 1.1 */
rc = pcmk_ok;
}
}
free_xml(reply);
free_xml(hello);
if (rc != pcmk_ok) {
lrmd_api_disconnect(lrmd);
}
return rc;
}
static int
lrmd_ipc_connect(lrmd_t * lrmd, int *fd)
{
int rc = pcmk_ok;
lrmd_private_t *native = lrmd->lrmd_private;
struct ipc_client_callbacks lrmd_callbacks = {
.dispatch = lrmd_ipc_dispatch,
.destroy = lrmd_ipc_connection_destroy
};
crm_info("Connecting to executor");
if (fd) {
/* No mainloop */
native->ipc = crm_ipc_new(CRM_SYSTEM_LRMD, 0);
if (native->ipc && crm_ipc_connect(native->ipc)) {
*fd = crm_ipc_get_fd(native->ipc);
} else if (native->ipc) {
crm_perror(LOG_ERR, "Connection to executor failed");
rc = -ENOTCONN;
}
} else {
native->source = mainloop_add_ipc_client(CRM_SYSTEM_LRMD, G_PRIORITY_HIGH, 0, lrmd, &lrmd_callbacks);
native->ipc = mainloop_get_ipc_client(native->source);
}
if (native->ipc == NULL) {
crm_debug("Could not connect to the executor API");
rc = -ENOTCONN;
}
return rc;
}
#ifdef HAVE_GNUTLS_GNUTLS_H
static void
copy_gnutls_datum(gnutls_datum_t *dest, gnutls_datum_t *source)
{
CRM_ASSERT((dest != NULL) && (source != NULL) && (source->data != NULL));
dest->data = gnutls_malloc(source->size);
CRM_ASSERT(dest->data);
memcpy(dest->data, source->data, source->size);
dest->size = source->size;
}
static void
clear_gnutls_datum(gnutls_datum_t *datum)
{
gnutls_free(datum->data);
datum->data = NULL;
datum->size = 0;
}
#define KEY_READ_LEN 256 // Chunk size for reading key from file
// \return Standard Pacemaker return code
static int
read_gnutls_key(const char *location, gnutls_datum_t *key)
{
FILE *stream = NULL;
size_t buf_len = KEY_READ_LEN;
if ((location == NULL) || (key == NULL)) {
return EINVAL;
}
stream = fopen(location, "r");
if (stream == NULL) {
return errno;
}
key->data = gnutls_malloc(buf_len);
key->size = 0;
while (!feof(stream)) {
int next = fgetc(stream);
if (next == EOF) {
if (!feof(stream)) {
crm_warn("Pacemaker Remote key read was partially successful "
"(copy in memory may be corrupted)");
}
break;
}
if (key->size == buf_len) {
buf_len = key->size + KEY_READ_LEN;
key->data = gnutls_realloc(key->data, buf_len);
CRM_ASSERT(key->data);
}
key->data[key->size++] = (unsigned char) next;
}
fclose(stream);
if (key->size == 0) {
clear_gnutls_datum(key);
return ENOKEY;
}
return pcmk_rc_ok;
}
// Cache the most recently used Pacemaker Remote authentication key
struct key_cache_s {
time_t updated; // When cached key was read (valid for 1 minute)
const char *location; // Where cached key was read from
gnutls_datum_t key; // Cached key
};
static bool
key_is_cached(struct key_cache_s *key_cache)
{
return key_cache->updated != 0;
}
static bool
key_cache_expired(struct key_cache_s *key_cache)
{
return (time(NULL) - key_cache->updated) >= 60;
}
static void
clear_key_cache(struct key_cache_s *key_cache)
{
clear_gnutls_datum(&(key_cache->key));
if ((key_cache->updated != 0) || (key_cache->location != NULL)) {
key_cache->updated = 0;
key_cache->location = NULL;
crm_debug("Cleared Pacemaker Remote key cache");
}
}
static void
get_cached_key(struct key_cache_s *key_cache, gnutls_datum_t *key)
{
copy_gnutls_datum(key, &(key_cache->key));
crm_debug("Using cached Pacemaker Remote key from %s",
pcmk__s(key_cache->location, "unknown location"));
}
static void
cache_key(struct key_cache_s *key_cache, gnutls_datum_t *key,
const char *location)
{
key_cache->updated = time(NULL);
key_cache->location = location;
copy_gnutls_datum(&(key_cache->key), key);
crm_debug("Using (and cacheing) Pacemaker Remote key from %s",
pcmk__s(location, "unknown location"));
}
/*!
* \internal
* \brief Get Pacemaker Remote authentication key from file or cache
*
* \param[in] location Path to key file to try (this memory must
* persist across all calls of this function)
* \param[out] key Key from location or cache
*
* \return Standard Pacemaker return code
*/
static int
get_remote_key(const char *location, gnutls_datum_t *key)
{
static struct key_cache_s key_cache = { 0, };
int rc = pcmk_rc_ok;
if ((location == NULL) || (key == NULL)) {
return EINVAL;
}
if (key_is_cached(&key_cache)) {
if (key_cache_expired(&key_cache)) {
clear_key_cache(&key_cache);
} else {
get_cached_key(&key_cache, key);
return pcmk_rc_ok;
}
}
rc = read_gnutls_key(location, key);
if (rc != pcmk_rc_ok) {
return rc;
}
cache_key(&key_cache, key, location);
return pcmk_rc_ok;
}
/*!
* \internal
* \brief Initialize the Pacemaker Remote authentication key
*
* Try loading the Pacemaker Remote authentication key from cache if available,
* otherwise from these locations, in order of preference: the value of the
* PCMK_authkey_location environment variable, if set; the Pacemaker default key
* file location; or (for historical reasons) /etc/corosync/authkey.
*
* \param[out] key Where to store key
*
* \return Standard Pacemaker return code
*/
int
lrmd__init_remote_key(gnutls_datum_t *key)
{
static const char *env_location = NULL;
static bool need_env = true;
int env_rc = pcmk_rc_ok;
int default_rc = pcmk_rc_ok;
int alt_rc = pcmk_rc_ok;
bool env_is_default = false;
bool env_is_fallback = false;
if (need_env) {
env_location = getenv("PCMK_authkey_location");
need_env = false;
}
// Try location in environment variable, if set
if (env_location != NULL) {
env_rc = get_remote_key(env_location, key);
if (env_rc == pcmk_rc_ok) {
return pcmk_rc_ok;
}
env_is_default = !strcmp(env_location, DEFAULT_REMOTE_KEY_LOCATION);
env_is_fallback = !strcmp(env_location, ALT_REMOTE_KEY_LOCATION);
/* @TODO It would be more secure to fail, rather than fall back to the
* default, if an explicitly set key location is not readable, and it
* would be better to never use the Corosync location as a fallback.
* However, that would break any deployments currently working with the
* fallbacks.
*/
}
// Try default location, if environment wasn't explicitly set to it
if (env_is_default) {
default_rc = env_rc;
} else {
default_rc = get_remote_key(DEFAULT_REMOTE_KEY_LOCATION, key);
}
// Try fallback location, if environment wasn't set to it and default failed
if (env_is_fallback) {
alt_rc = env_rc;
} else if (default_rc != pcmk_rc_ok) {
alt_rc = get_remote_key(ALT_REMOTE_KEY_LOCATION, key);
}
// We have all results, so log and return
if ((env_rc != pcmk_rc_ok) && (default_rc != pcmk_rc_ok)
&& (alt_rc != pcmk_rc_ok)) { // Environment set, everything failed
crm_warn("Could not read Pacemaker Remote key from %s (%s%s%s%s%s): %s",
env_location,
env_is_default? "" : "or default location ",
env_is_default? "" : DEFAULT_REMOTE_KEY_LOCATION,
!env_is_default && !env_is_fallback? " " : "",
env_is_fallback? "" : "or fallback location ",
env_is_fallback? "" : ALT_REMOTE_KEY_LOCATION,
pcmk_rc_str(env_rc));
return ENOKEY;
}
if (env_rc != pcmk_rc_ok) { // Environment set but failed, using a default
crm_warn("Could not read Pacemaker Remote key from %s "
"(using %s location %s instead): %s",
env_location,
(default_rc == pcmk_rc_ok)? "default" : "fallback",
(default_rc == pcmk_rc_ok)? DEFAULT_REMOTE_KEY_LOCATION : ALT_REMOTE_KEY_LOCATION,
pcmk_rc_str(env_rc));
return pcmk_rc_ok;
}
if ((default_rc != pcmk_rc_ok) && (alt_rc != pcmk_rc_ok)) {
// Environment unset, defaults failed
crm_warn("Could not read Pacemaker Remote key from default location %s"
" (or fallback location %s): %s",
DEFAULT_REMOTE_KEY_LOCATION, ALT_REMOTE_KEY_LOCATION,
pcmk_rc_str(default_rc));
return ENOKEY;
}
return pcmk_rc_ok; // Environment variable unset, a default worked
}
static void
lrmd_gnutls_global_init(void)
{
static int gnutls_init = 0;
if (!gnutls_init) {
crm_gnutls_global_init();
}
gnutls_init = 1;
}
#endif
static void
report_async_connection_result(lrmd_t * lrmd, int rc)
{
lrmd_private_t *native = lrmd->lrmd_private;
if (native->callback) {
lrmd_event_data_t event = { 0, };
event.type = lrmd_event_connect;
event.remote_nodename = native->remote_nodename;
event.connection_rc = rc;
native->callback(&event);
}
}
#ifdef HAVE_GNUTLS_GNUTLS_H
static inline int
lrmd__tls_client_handshake(pcmk__remote_t *remote)
{
return pcmk__tls_client_handshake(remote, LRMD_CLIENT_HANDSHAKE_TIMEOUT);
}
/*!
* \internal
* \brief Add trigger and file descriptor mainloop sources for TLS
*
* \param[in] lrmd API connection with established TLS session
* \param[in] do_handshake Whether to perform executor handshake
*
* \return Standard Pacemaker return code
*/
static int
add_tls_to_mainloop(lrmd_t *lrmd, bool do_handshake)
{
lrmd_private_t *native = lrmd->lrmd_private;
int rc = pcmk_rc_ok;
char *name = crm_strdup_printf("pacemaker-remote-%s:%d",
native->server, native->port);
struct mainloop_fd_callbacks tls_fd_callbacks = {
.dispatch = lrmd_tls_dispatch,
.destroy = lrmd_tls_connection_destroy,
};
native->process_notify = mainloop_add_trigger(G_PRIORITY_HIGH,
lrmd_tls_dispatch, lrmd);
native->source = mainloop_add_fd(name, G_PRIORITY_HIGH, native->sock, lrmd,
&tls_fd_callbacks);
/* Async connections lose the client name provided by the API caller, so we
* have to use our generated name here to perform the executor handshake.
*
* @TODO Keep track of the caller-provided name. Perhaps we should be using
* that name in this function instead of generating one anyway.
*/
if (do_handshake) {
rc = lrmd_handshake(lrmd, name);
rc = pcmk_legacy2rc(rc);
}
free(name);
return rc;
}
static void
lrmd_tcp_connect_cb(void *userdata, int rc, int sock)
{
lrmd_t *lrmd = userdata;
lrmd_private_t *native = lrmd->lrmd_private;
gnutls_datum_t psk_key = { NULL, 0 };
native->async_timer = 0;
if (rc != pcmk_rc_ok) {
lrmd_tls_connection_destroy(lrmd);
crm_info("Could not connect to Pacemaker Remote at %s:%d: %s "
CRM_XS " rc=%d",
native->server, native->port, pcmk_rc_str(rc), rc);
report_async_connection_result(lrmd, pcmk_rc2legacy(rc));
return;
}
/* The TCP connection was successful, so establish the TLS connection.
* @TODO make this async to avoid blocking code in client
*/
native->sock = sock;
rc = lrmd__init_remote_key(&psk_key);
if (rc != pcmk_rc_ok) {
crm_info("Could not connect to Pacemaker Remote at %s:%d: %s "
CRM_XS " rc=%d",
native->server, native->port, pcmk_rc_str(rc), rc);
lrmd_tls_connection_destroy(lrmd);
report_async_connection_result(lrmd, pcmk_rc2legacy(rc));
return;
}
gnutls_psk_allocate_client_credentials(&native->psk_cred_c);
gnutls_psk_set_client_credentials(native->psk_cred_c, DEFAULT_REMOTE_USERNAME, &psk_key, GNUTLS_PSK_KEY_RAW);
gnutls_free(psk_key.data);
native->remote->tls_session = pcmk__new_tls_session(sock, GNUTLS_CLIENT,
GNUTLS_CRD_PSK,
native->psk_cred_c);
if (native->remote->tls_session == NULL) {
lrmd_tls_connection_destroy(lrmd);
report_async_connection_result(lrmd, -EPROTO);
return;
}
if (lrmd__tls_client_handshake(native->remote) != pcmk_rc_ok) {
crm_warn("Disconnecting after TLS handshake with Pacemaker Remote server %s:%d failed",
native->server, native->port);
gnutls_deinit(*native->remote->tls_session);
gnutls_free(native->remote->tls_session);
native->remote->tls_session = NULL;
lrmd_tls_connection_destroy(lrmd);
report_async_connection_result(lrmd, -EKEYREJECTED);
return;
}
crm_info("TLS connection to Pacemaker Remote server %s:%d succeeded",
native->server, native->port);
rc = add_tls_to_mainloop(lrmd, true);
report_async_connection_result(lrmd, pcmk_rc2legacy(rc));
}
static int
lrmd_tls_connect_async(lrmd_t * lrmd, int timeout /*ms */ )
{
int rc;
int timer_id = 0;
lrmd_private_t *native = lrmd->lrmd_private;
lrmd_gnutls_global_init();
native->sock = -1;
rc = pcmk__connect_remote(native->server, native->port, timeout, &timer_id,
&(native->sock), lrmd, lrmd_tcp_connect_cb);
if (rc != pcmk_rc_ok) {
crm_warn("Pacemaker Remote connection to %s:%d failed: %s "
CRM_XS " rc=%d",
native->server, native->port, pcmk_rc_str(rc), rc);
return pcmk_rc2legacy(rc);
}
native->async_timer = timer_id;
return pcmk_ok;
}
static int
lrmd_tls_connect(lrmd_t * lrmd, int *fd)
{
int rc;
lrmd_private_t *native = lrmd->lrmd_private;
gnutls_datum_t psk_key = { NULL, 0 };
lrmd_gnutls_global_init();
native->sock = -1;
rc = pcmk__connect_remote(native->server, native->port, 0, NULL,
&(native->sock), NULL, NULL);
if (rc != pcmk_rc_ok) {
crm_warn("Pacemaker Remote connection to %s:%d failed: %s "
CRM_XS " rc=%d",
native->server, native->port, pcmk_rc_str(rc), rc);
lrmd_tls_connection_destroy(lrmd);
return -ENOTCONN;
}
rc = lrmd__init_remote_key(&psk_key);
if (rc != pcmk_rc_ok) {
lrmd_tls_connection_destroy(lrmd);
return pcmk_rc2legacy(rc);
}
gnutls_psk_allocate_client_credentials(&native->psk_cred_c);
gnutls_psk_set_client_credentials(native->psk_cred_c, DEFAULT_REMOTE_USERNAME, &psk_key, GNUTLS_PSK_KEY_RAW);
gnutls_free(psk_key.data);
native->remote->tls_session = pcmk__new_tls_session(native->sock, GNUTLS_CLIENT,
GNUTLS_CRD_PSK,
native->psk_cred_c);
if (native->remote->tls_session == NULL) {
lrmd_tls_connection_destroy(lrmd);
return -EPROTO;
}
if (lrmd__tls_client_handshake(native->remote) != pcmk_rc_ok) {
crm_err("Session creation for %s:%d failed", native->server, native->port);
gnutls_deinit(*native->remote->tls_session);
gnutls_free(native->remote->tls_session);
native->remote->tls_session = NULL;
lrmd_tls_connection_destroy(lrmd);
return -EKEYREJECTED;
}
crm_info("Client TLS connection established with Pacemaker Remote server %s:%d", native->server,
native->port);
if (fd) {
*fd = native->sock;
} else {
add_tls_to_mainloop(lrmd, false);
}
return pcmk_ok;
}
#endif
static int
lrmd_api_connect(lrmd_t * lrmd, const char *name, int *fd)
{
int rc = -ENOTCONN;
lrmd_private_t *native = lrmd->lrmd_private;
switch (native->type) {
case pcmk__client_ipc:
rc = lrmd_ipc_connect(lrmd, fd);
break;
#ifdef HAVE_GNUTLS_GNUTLS_H
case pcmk__client_tls:
rc = lrmd_tls_connect(lrmd, fd);
break;
#endif
default:
crm_err("Unsupported executor connection type (bug?): %d",
native->type);
rc = -EPROTONOSUPPORT;
}
if (rc == pcmk_ok) {
rc = lrmd_handshake(lrmd, name);
}
return rc;
}
static int
lrmd_api_connect_async(lrmd_t * lrmd, const char *name, int timeout)
{
int rc = pcmk_ok;
lrmd_private_t *native = lrmd->lrmd_private;
CRM_CHECK(native && native->callback, return -EINVAL);
switch (native->type) {
case pcmk__client_ipc:
/* fake async connection with ipc. it should be fast
* enough that we gain very little from async */
rc = lrmd_api_connect(lrmd, name, NULL);
if (!rc) {
report_async_connection_result(lrmd, rc);
}
break;
#ifdef HAVE_GNUTLS_GNUTLS_H
case pcmk__client_tls:
rc = lrmd_tls_connect_async(lrmd, timeout);
if (rc) {
/* connection failed, report rc now */
report_async_connection_result(lrmd, rc);
}
break;
#endif
default:
crm_err("Unsupported executor connection type (bug?): %d",
native->type);
rc = -EPROTONOSUPPORT;
}
return rc;
}
static void
lrmd_ipc_disconnect(lrmd_t * lrmd)
{
lrmd_private_t *native = lrmd->lrmd_private;
if (native->source != NULL) {
/* Attached to mainloop */
mainloop_del_ipc_client(native->source);
native->source = NULL;
native->ipc = NULL;
} else if (native->ipc) {
/* Not attached to mainloop */
crm_ipc_t *ipc = native->ipc;
native->ipc = NULL;
crm_ipc_close(ipc);
crm_ipc_destroy(ipc);
}
}
#ifdef HAVE_GNUTLS_GNUTLS_H
static void
lrmd_tls_disconnect(lrmd_t * lrmd)
{
lrmd_private_t *native = lrmd->lrmd_private;
if (native->remote->tls_session) {
gnutls_bye(*native->remote->tls_session, GNUTLS_SHUT_RDWR);
gnutls_deinit(*native->remote->tls_session);
gnutls_free(native->remote->tls_session);
native->remote->tls_session = 0;
}
if (native->async_timer) {
g_source_remove(native->async_timer);
native->async_timer = 0;
}
if (native->source != NULL) {
/* Attached to mainloop */
mainloop_del_ipc_client(native->source);
native->source = NULL;
} else if (native->sock) {
close(native->sock);
native->sock = 0;
}
if (native->pending_notify) {
g_list_free_full(native->pending_notify, lrmd_free_xml);
native->pending_notify = NULL;
}
}
#endif
static int
lrmd_api_disconnect(lrmd_t * lrmd)
{
lrmd_private_t *native = lrmd->lrmd_private;
int rc = pcmk_ok;
crm_info("Disconnecting %s %s executor connection",
pcmk__client_type_str(native->type),
(native->remote_nodename? native->remote_nodename : "local"));
switch (native->type) {
case pcmk__client_ipc:
lrmd_ipc_disconnect(lrmd);
break;
#ifdef HAVE_GNUTLS_GNUTLS_H
case pcmk__client_tls:
lrmd_tls_disconnect(lrmd);
break;
#endif
default:
crm_err("Unsupported executor connection type (bug?): %d",
native->type);
rc = -EPROTONOSUPPORT;
}
free(native->token);
native->token = NULL;
free(native->peer_version);
native->peer_version = NULL;
return rc;
}
static int
lrmd_api_register_rsc(lrmd_t * lrmd,
const char *rsc_id,
const char *class,
const char *provider, const char *type, enum lrmd_call_options options)
{
int rc = pcmk_ok;
xmlNode *data = NULL;
if (!class || !type || !rsc_id) {
return -EINVAL;
}
if (pcmk_is_set(pcmk_get_ra_caps(class), pcmk_ra_cap_provider)
&& (provider == NULL)) {
return -EINVAL;
}
data = create_xml_node(NULL, F_LRMD_RSC);
crm_xml_add(data, F_LRMD_ORIGIN, __func__);
crm_xml_add(data, F_LRMD_RSC_ID, rsc_id);
crm_xml_add(data, F_LRMD_CLASS, class);
crm_xml_add(data, F_LRMD_PROVIDER, provider);
crm_xml_add(data, F_LRMD_TYPE, type);
rc = lrmd_send_command(lrmd, LRMD_OP_RSC_REG, data, NULL, 0, options, TRUE);
free_xml(data);
return rc;
}
static int
lrmd_api_unregister_rsc(lrmd_t * lrmd, const char *rsc_id, enum lrmd_call_options options)
{
int rc = pcmk_ok;
xmlNode *data = create_xml_node(NULL, F_LRMD_RSC);
crm_xml_add(data, F_LRMD_ORIGIN, __func__);
crm_xml_add(data, F_LRMD_RSC_ID, rsc_id);
rc = lrmd_send_command(lrmd, LRMD_OP_RSC_UNREG, data, NULL, 0, options, TRUE);
free_xml(data);
return rc;
}
lrmd_rsc_info_t *
lrmd_new_rsc_info(const char *rsc_id, const char *standard,
const char *provider, const char *type)
{
lrmd_rsc_info_t *rsc_info = calloc(1, sizeof(lrmd_rsc_info_t));
CRM_ASSERT(rsc_info);
pcmk__str_update(&rsc_info->id, rsc_id);
pcmk__str_update(&rsc_info->standard, standard);
pcmk__str_update(&rsc_info->provider, provider);
pcmk__str_update(&rsc_info->type, type);
return rsc_info;
}
lrmd_rsc_info_t *
lrmd_copy_rsc_info(lrmd_rsc_info_t * rsc_info)
{
return lrmd_new_rsc_info(rsc_info->id, rsc_info->standard,
rsc_info->provider, rsc_info->type);
}
void
lrmd_free_rsc_info(lrmd_rsc_info_t * rsc_info)
{
if (!rsc_info) {
return;
}
free(rsc_info->id);
free(rsc_info->type);
free(rsc_info->standard);
free(rsc_info->provider);
free(rsc_info);
}
static lrmd_rsc_info_t *
lrmd_api_get_rsc_info(lrmd_t * lrmd, const char *rsc_id, enum lrmd_call_options options)
{
lrmd_rsc_info_t *rsc_info = NULL;
xmlNode *data = create_xml_node(NULL, F_LRMD_RSC);
xmlNode *output = NULL;
const char *class = NULL;
const char *provider = NULL;
const char *type = NULL;
crm_xml_add(data, F_LRMD_ORIGIN, __func__);
crm_xml_add(data, F_LRMD_RSC_ID, rsc_id);
lrmd_send_command(lrmd, LRMD_OP_RSC_INFO, data, &output, 0, options, TRUE);
free_xml(data);
if (!output) {
return NULL;
}
class = crm_element_value(output, F_LRMD_CLASS);
provider = crm_element_value(output, F_LRMD_PROVIDER);
type = crm_element_value(output, F_LRMD_TYPE);
if (!class || !type) {
free_xml(output);
return NULL;
} else if (pcmk_is_set(pcmk_get_ra_caps(class), pcmk_ra_cap_provider)
&& !provider) {
free_xml(output);
return NULL;
}
rsc_info = lrmd_new_rsc_info(rsc_id, class, provider, type);
free_xml(output);
return rsc_info;
}
void
lrmd_free_op_info(lrmd_op_info_t *op_info)
{
if (op_info) {
free(op_info->rsc_id);
free(op_info->action);
free(op_info->interval_ms_s);
free(op_info->timeout_ms_s);
free(op_info);
}
}
static int
lrmd_api_get_recurring_ops(lrmd_t *lrmd, const char *rsc_id, int timeout_ms,
enum lrmd_call_options options, GList **output)
{
xmlNode *data = NULL;
xmlNode *output_xml = NULL;
int rc = pcmk_ok;
if (output == NULL) {
return -EINVAL;
}
*output = NULL;
// Send request
if (rsc_id) {
data = create_xml_node(NULL, F_LRMD_RSC);
crm_xml_add(data, F_LRMD_ORIGIN, __func__);
crm_xml_add(data, F_LRMD_RSC_ID, rsc_id);
}
rc = lrmd_send_command(lrmd, LRMD_OP_GET_RECURRING, data, &output_xml,
timeout_ms, options, TRUE);
if (data) {
free_xml(data);
}
// Process reply
if ((rc != pcmk_ok) || (output_xml == NULL)) {
return rc;
}
for (xmlNode *rsc_xml = first_named_child(output_xml, F_LRMD_RSC);
(rsc_xml != NULL) && (rc == pcmk_ok);
rsc_xml = crm_next_same_xml(rsc_xml)) {
rsc_id = crm_element_value(rsc_xml, F_LRMD_RSC_ID);
if (rsc_id == NULL) {
crm_err("Could not parse recurring operation information from executor");
continue;
}
for (xmlNode *op_xml = first_named_child(rsc_xml, T_LRMD_RSC_OP);
op_xml != NULL; op_xml = crm_next_same_xml(op_xml)) {
lrmd_op_info_t *op_info = calloc(1, sizeof(lrmd_op_info_t));
if (op_info == NULL) {
rc = -ENOMEM;
break;
}
op_info->rsc_id = strdup(rsc_id);
op_info->action = crm_element_value_copy(op_xml, F_LRMD_RSC_ACTION);
op_info->interval_ms_s = crm_element_value_copy(op_xml,
F_LRMD_RSC_INTERVAL);
op_info->timeout_ms_s = crm_element_value_copy(op_xml,
F_LRMD_TIMEOUT);
*output = g_list_prepend(*output, op_info);
}
}
free_xml(output_xml);
return rc;
}
static void
lrmd_api_set_callback(lrmd_t * lrmd, lrmd_event_callback callback)
{
lrmd_private_t *native = lrmd->lrmd_private;
native->callback = callback;
}
void
lrmd_internal_set_proxy_callback(lrmd_t * lrmd, void *userdata, void (*callback)(lrmd_t *lrmd, void *userdata, xmlNode *msg))
{
lrmd_private_t *native = lrmd->lrmd_private;
native->proxy_callback = callback;
native->proxy_callback_userdata = userdata;
}
void
lrmd_internal_proxy_dispatch(lrmd_t *lrmd, xmlNode *msg)
{
lrmd_private_t *native = lrmd->lrmd_private;
if (native->proxy_callback) {
crm_log_xml_trace(msg, "PROXY_INBOUND");
native->proxy_callback(lrmd, native->proxy_callback_userdata, msg);
}
}
int
lrmd_internal_proxy_send(lrmd_t * lrmd, xmlNode *msg)
{
if (lrmd == NULL) {
return -ENOTCONN;
}
crm_xml_add(msg, F_LRMD_OPERATION, CRM_OP_IPC_FWD);
crm_log_xml_trace(msg, "PROXY_OUTBOUND");
return lrmd_send_xml_no_reply(lrmd, msg);
}
static int
stonith_get_metadata(const char *provider, const char *type, char **output)
{
int rc = pcmk_ok;
stonith_t *stonith_api = stonith_api_new();
if (stonith_api == NULL) {
crm_err("Could not get fence agent meta-data: API memory allocation failed");
return -ENOMEM;
}
rc = stonith_api->cmds->metadata(stonith_api, st_opt_sync_call, type,
provider, output, 0);
if ((rc == pcmk_ok) && (*output == NULL)) {
rc = -EIO;
}
stonith_api->cmds->free(stonith_api);
return rc;
}
static int
lrmd_api_get_metadata(lrmd_t *lrmd, const char *standard, const char *provider,
const char *type, char **output,
enum lrmd_call_options options)
{
return lrmd->cmds->get_metadata_params(lrmd, standard, provider, type,
output, options, NULL);
}
static int
lrmd_api_get_metadata_params(lrmd_t *lrmd, const char *standard,
const char *provider, const char *type,
char **output, enum lrmd_call_options options,
lrmd_key_value_t *params)
{
svc_action_t *action = NULL;
GHashTable *params_table = NULL;
if (!standard || !type) {
lrmd_key_value_freeall(params);
return -EINVAL;
}
if (pcmk__str_eq(standard, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_casei)) {
lrmd_key_value_freeall(params);
return stonith_get_metadata(provider, type, output);
}
params_table = pcmk__strkey_table(free, free);
for (const lrmd_key_value_t *param = params; param; param = param->next) {
g_hash_table_insert(params_table, strdup(param->key), strdup(param->value));
}
action = services__create_resource_action(type, standard, provider, type,
CRMD_ACTION_METADATA, 0,
CRMD_METADATA_CALL_TIMEOUT,
params_table, 0);
lrmd_key_value_freeall(params);
if (action == NULL) {
return -ENOMEM;
}
if (action->rc != PCMK_OCF_UNKNOWN) {
services_action_free(action);
return -EINVAL;
}
if (!services_action_sync(action)) {
crm_err("Failed to retrieve meta-data for %s:%s:%s",
standard, provider, type);
services_action_free(action);
return -EIO;
}
if (!action->stdout_data) {
crm_err("Failed to receive meta-data for %s:%s:%s",
standard, provider, type);
services_action_free(action);
return -EIO;
}
*output = strdup(action->stdout_data);
services_action_free(action);
return pcmk_ok;
}
static int
lrmd_api_exec(lrmd_t *lrmd, const char *rsc_id, const char *action,
const char *userdata, guint interval_ms,
int timeout, /* ms */
int start_delay, /* ms */
enum lrmd_call_options options, lrmd_key_value_t * params)
{
int rc = pcmk_ok;
xmlNode *data = create_xml_node(NULL, F_LRMD_RSC);
xmlNode *args = create_xml_node(data, XML_TAG_ATTRS);
lrmd_key_value_t *tmp = NULL;
crm_xml_add(data, F_LRMD_ORIGIN, __func__);
crm_xml_add(data, F_LRMD_RSC_ID, rsc_id);
crm_xml_add(data, F_LRMD_RSC_ACTION, action);
crm_xml_add(data, F_LRMD_RSC_USERDATA_STR, userdata);
crm_xml_add_ms(data, F_LRMD_RSC_INTERVAL, interval_ms);
crm_xml_add_int(data, F_LRMD_TIMEOUT, timeout);
crm_xml_add_int(data, F_LRMD_RSC_START_DELAY, start_delay);
for (tmp = params; tmp; tmp = tmp->next) {
hash2smartfield((gpointer) tmp->key, (gpointer) tmp->value, args);
}
rc = lrmd_send_command(lrmd, LRMD_OP_RSC_EXEC, data, NULL, timeout, options, TRUE);
free_xml(data);
lrmd_key_value_freeall(params);
return rc;
}
/* timeout is in ms */
static int
lrmd_api_exec_alert(lrmd_t *lrmd, const char *alert_id, const char *alert_path,
int timeout, lrmd_key_value_t *params)
{
int rc = pcmk_ok;
xmlNode *data = create_xml_node(NULL, F_LRMD_ALERT);
xmlNode *args = create_xml_node(data, XML_TAG_ATTRS);
lrmd_key_value_t *tmp = NULL;
crm_xml_add(data, F_LRMD_ORIGIN, __func__);
crm_xml_add(data, F_LRMD_ALERT_ID, alert_id);
crm_xml_add(data, F_LRMD_ALERT_PATH, alert_path);
crm_xml_add_int(data, F_LRMD_TIMEOUT, timeout);
for (tmp = params; tmp; tmp = tmp->next) {
hash2smartfield((gpointer) tmp->key, (gpointer) tmp->value, args);
}
rc = lrmd_send_command(lrmd, LRMD_OP_ALERT_EXEC, data, NULL, timeout,
lrmd_opt_notify_orig_only, TRUE);
free_xml(data);
lrmd_key_value_freeall(params);
return rc;
}
static int
lrmd_api_cancel(lrmd_t *lrmd, const char *rsc_id, const char *action,
guint interval_ms)
{
int rc = pcmk_ok;
xmlNode *data = create_xml_node(NULL, F_LRMD_RSC);
crm_xml_add(data, F_LRMD_ORIGIN, __func__);
crm_xml_add(data, F_LRMD_RSC_ACTION, action);
crm_xml_add(data, F_LRMD_RSC_ID, rsc_id);
crm_xml_add_ms(data, F_LRMD_RSC_INTERVAL, interval_ms);
rc = lrmd_send_command(lrmd, LRMD_OP_RSC_CANCEL, data, NULL, 0, 0, TRUE);
free_xml(data);
return rc;
}
static int
list_stonith_agents(lrmd_list_t ** resources)
{
int rc = 0;
stonith_t *stonith_api = stonith_api_new();
stonith_key_value_t *stonith_resources = NULL;
stonith_key_value_t *dIter = NULL;
if (stonith_api == NULL) {
crm_err("Could not list fence agents: API memory allocation failed");
return -ENOMEM;
}
stonith_api->cmds->list_agents(stonith_api, st_opt_sync_call, NULL,
&stonith_resources, 0);
stonith_api->cmds->free(stonith_api);
for (dIter = stonith_resources; dIter; dIter = dIter->next) {
rc++;
if (resources) {
*resources = lrmd_list_add(*resources, dIter->value);
}
}
stonith_key_value_freeall(stonith_resources, 1, 0);
return rc;
}
static int
lrmd_api_list_agents(lrmd_t * lrmd, lrmd_list_t ** resources, const char *class,
const char *provider)
{
int rc = 0;
int stonith_count = 0; // Initially, whether to include stonith devices
if (pcmk__str_eq(class, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_casei)) {
stonith_count = 1;
} else {
GList *gIter = NULL;
GList *agents = resources_list_agents(class, provider);
for (gIter = agents; gIter != NULL; gIter = gIter->next) {
*resources = lrmd_list_add(*resources, (const char *)gIter->data);
rc++;
}
g_list_free_full(agents, free);
if (!class) {
stonith_count = 1;
}
}
if (stonith_count) {
// Now, if stonith devices are included, how many there are
stonith_count = list_stonith_agents(resources);
if (stonith_count > 0) {
rc += stonith_count;
}
}
if (rc == 0) {
crm_notice("No agents found for class %s", class);
rc = -EPROTONOSUPPORT;
}
return rc;
}
static bool
does_provider_have_agent(const char *agent, const char *provider, const char *class)
{
bool found = false;
GList *agents = NULL;
GList *gIter2 = NULL;
agents = resources_list_agents(class, provider);
for (gIter2 = agents; gIter2 != NULL; gIter2 = gIter2->next) {
if (pcmk__str_eq(agent, gIter2->data, pcmk__str_casei)) {
found = true;
}
}
g_list_free_full(agents, free);
return found;
}
static int
lrmd_api_list_ocf_providers(lrmd_t * lrmd, const char *agent, lrmd_list_t ** providers)
{
int rc = pcmk_ok;
char *provider = NULL;
GList *ocf_providers = NULL;
GList *gIter = NULL;
ocf_providers = resources_list_providers(PCMK_RESOURCE_CLASS_OCF);
for (gIter = ocf_providers; gIter != NULL; gIter = gIter->next) {
provider = gIter->data;
if (!agent || does_provider_have_agent(agent, provider,
PCMK_RESOURCE_CLASS_OCF)) {
*providers = lrmd_list_add(*providers, (const char *)gIter->data);
rc++;
}
}
g_list_free_full(ocf_providers, free);
return rc;
}
static int
lrmd_api_list_standards(lrmd_t * lrmd, lrmd_list_t ** supported)
{
int rc = 0;
GList *standards = NULL;
GList *gIter = NULL;
standards = resources_list_standards();
for (gIter = standards; gIter != NULL; gIter = gIter->next) {
*supported = lrmd_list_add(*supported, (const char *)gIter->data);
rc++;
}
if (list_stonith_agents(NULL) > 0) {
*supported = lrmd_list_add(*supported, PCMK_RESOURCE_CLASS_STONITH);
rc++;
}
g_list_free_full(standards, free);
return rc;
}
/*!
* \internal
* \brief Create an executor API object
*
* \param[out] api Will be set to newly created API object (it is the
* caller's responsibility to free this value with
* lrmd_api_delete() if this function succeeds)
* \param[in] nodename If the object will be used for a remote connection,
* the node name to use in cluster for remote executor
* \param[in] server If the object will be used for a remote connection,
* the resolvable host name to connect to
* \param[in] port If the object will be used for a remote connection,
* port number on \p server to connect to
*
* \return Standard Pacemaker return code
* \note If the caller leaves one of \p nodename or \p server NULL, the other's
* value will be used for both. If the caller leaves both NULL, an API
* object will be created for a local executor connection.
*/
int
lrmd__new(lrmd_t **api, const char *nodename, const char *server, int port)
{
lrmd_private_t *pvt = NULL;
if (api == NULL) {
return EINVAL;
}
*api = NULL;
// Allocate all memory needed
*api = calloc(1, sizeof(lrmd_t));
if (*api == NULL) {
return ENOMEM;
}
pvt = calloc(1, sizeof(lrmd_private_t));
if (pvt == NULL) {
lrmd_api_delete(*api);
*api = NULL;
return ENOMEM;
}
(*api)->lrmd_private = pvt;
// @TODO Do we need to do this for local connections?
pvt->remote = calloc(1, sizeof(pcmk__remote_t));
(*api)->cmds = calloc(1, sizeof(lrmd_api_operations_t));
if ((pvt->remote == NULL) || ((*api)->cmds == NULL)) {
lrmd_api_delete(*api);
*api = NULL;
return ENOMEM;
}
// Set methods
(*api)->cmds->connect = lrmd_api_connect;
(*api)->cmds->connect_async = lrmd_api_connect_async;
(*api)->cmds->is_connected = lrmd_api_is_connected;
(*api)->cmds->poke_connection = lrmd_api_poke_connection;
(*api)->cmds->disconnect = lrmd_api_disconnect;
(*api)->cmds->register_rsc = lrmd_api_register_rsc;
(*api)->cmds->unregister_rsc = lrmd_api_unregister_rsc;
(*api)->cmds->get_rsc_info = lrmd_api_get_rsc_info;
(*api)->cmds->get_recurring_ops = lrmd_api_get_recurring_ops;
(*api)->cmds->set_callback = lrmd_api_set_callback;
(*api)->cmds->get_metadata = lrmd_api_get_metadata;
(*api)->cmds->exec = lrmd_api_exec;
(*api)->cmds->cancel = lrmd_api_cancel;
(*api)->cmds->list_agents = lrmd_api_list_agents;
(*api)->cmds->list_ocf_providers = lrmd_api_list_ocf_providers;
(*api)->cmds->list_standards = lrmd_api_list_standards;
(*api)->cmds->exec_alert = lrmd_api_exec_alert;
(*api)->cmds->get_metadata_params = lrmd_api_get_metadata_params;
if ((nodename == NULL) && (server == NULL)) {
pvt->type = pcmk__client_ipc;
} else {
#ifdef HAVE_GNUTLS_GNUTLS_H
if (nodename == NULL) {
nodename = server;
} else if (server == NULL) {
server = nodename;
}
pvt->type = pcmk__client_tls;
pvt->remote_nodename = strdup(nodename);
pvt->server = strdup(server);
if ((pvt->remote_nodename == NULL) || (pvt->server == NULL)) {
lrmd_api_delete(*api);
*api = NULL;
return ENOMEM;
}
pvt->port = port;
if (pvt->port == 0) {
pvt->port = crm_default_remote_port();
}
#else
crm_err("Cannot communicate with Pacemaker Remote "
"because GnuTLS is not enabled for this build");
lrmd_api_delete(*api);
*api = NULL;
return EOPNOTSUPP;
#endif
}
return pcmk_rc_ok;
}
lrmd_t *
lrmd_api_new(void)
{
lrmd_t *api = NULL;
CRM_ASSERT(lrmd__new(&api, NULL, NULL, 0) == pcmk_rc_ok);
return api;
}
lrmd_t *
lrmd_remote_api_new(const char *nodename, const char *server, int port)
{
lrmd_t *api = NULL;
CRM_ASSERT(lrmd__new(&api, nodename, server, port) == pcmk_rc_ok);
return api;
}
void
lrmd_api_delete(lrmd_t * lrmd)
{
if (lrmd == NULL) {
return;
}
if (lrmd->cmds != NULL) { // Never NULL, but make static analysis happy
if (lrmd->cmds->disconnect != NULL) { // Also never really NULL
lrmd->cmds->disconnect(lrmd); // No-op if already disconnected
}
free(lrmd->cmds);
}
if (lrmd->lrmd_private != NULL) {
lrmd_private_t *native = lrmd->lrmd_private;
#ifdef HAVE_GNUTLS_GNUTLS_H
free(native->server);
#endif
free(native->remote_nodename);
free(native->remote);
free(native->token);
free(native->peer_version);
free(lrmd->lrmd_private);
}
free(lrmd);
}
+struct metadata_cb {
+ void (*callback)(int pid, const pcmk__action_result_t *result,
+ void *user_data);
+ void *user_data;
+};
+
+/*!
+ * \internal
+ * \brief Process asynchronous metadata completion
+ *
+ * \param[in] action Metadata action that completed
+ */
+static void
+metadata_complete(svc_action_t *action)
+{
+ struct metadata_cb *metadata_cb = (struct metadata_cb *) action->cb_data;
+ pcmk__action_result_t result = PCMK__UNKNOWN_RESULT;
+
+ pcmk__set_result(&result, action->rc, action->status,
+ services__exit_reason(action));
+ pcmk__set_result_output(&result, action->stdout_data, action->stderr_data);
+
+ metadata_cb->callback(0, &result, metadata_cb->user_data);
+ result.action_stdout = NULL; // Prevent free, because action owns it
+ result.action_stderr = NULL; // Prevent free, because action owns it
+ pcmk__reset_result(&result);
+ free(metadata_cb);
+}
+
+/*!
+ * \internal
+ * \brief Retrieve agent metadata asynchronously
+ *
+ * \param[in] rsc Resource agent specification
+ * \param[in] callback Function to call with result (this will always be
+ * called, whether by this function directly or later via
+ * the main loop, and on success the metadata will be in
+ * its result argument's action_stdout)
+ * \param[in] user_data User data to pass to callback
+ *
+ * \return Standard Pacemaker return code
+ * \note This function is not a lrmd_api_operations_t method because it does not
+ * need an lrmd_t object and does not go through the executor, but
+ * executes the agent directly.
+ */
+int
+lrmd__metadata_async(lrmd_rsc_info_t *rsc,
+ void (*callback)(int pid,
+ const pcmk__action_result_t *result,
+ void *user_data),
+ void *user_data)
+{
+ svc_action_t *action = NULL;
+ struct metadata_cb *metadata_cb = NULL;
+ pcmk__action_result_t result = PCMK__UNKNOWN_RESULT;
+
+ CRM_CHECK(callback != NULL, return EINVAL);
+
+ if ((rsc == NULL) || (rsc->standard == NULL) || (rsc->type == NULL)) {
+ pcmk__set_result(&result, PCMK_OCF_NOT_CONFIGURED,
+ PCMK_EXEC_ERROR_FATAL,
+ "Invalid resource specification");
+ callback(0, &result, user_data);
+ pcmk__reset_result(&result);
+ return EINVAL;
+ }
+
+ if (strcmp(rsc->standard, PCMK_RESOURCE_CLASS_STONITH) == 0) {
+ return stonith__metadata_async(rsc->type,
+ CRMD_METADATA_CALL_TIMEOUT / 1000,
+ callback, user_data);
+ }
+
+ action = services__create_resource_action(pcmk__s(rsc->id, rsc->type),
+ rsc->standard, rsc->provider,
+ rsc->type, CRMD_ACTION_METADATA,
+ 0, CRMD_METADATA_CALL_TIMEOUT,
+ NULL, 0);
+ if (action == NULL) {
+ pcmk__set_result(&result, PCMK_OCF_UNKNOWN_ERROR, PCMK_EXEC_ERROR,
+ "Out of memory");
+ callback(0, &result, user_data);
+ pcmk__reset_result(&result);
+ return ENOMEM;
+ }
+ if (action->rc != PCMK_OCF_UNKNOWN) {
+ pcmk__set_result(&result, action->rc, action->status,
+ services__exit_reason(action));
+ callback(0, &result, user_data);
+ pcmk__reset_result(&result);
+ services_action_free(action);
+ return EINVAL;
+ }
+
+ action->cb_data = calloc(1, sizeof(struct metadata_cb));
+ if (action->cb_data == NULL) {
+ services_action_free(action);
+ pcmk__set_result(&result, PCMK_OCF_UNKNOWN_ERROR, PCMK_EXEC_ERROR,
+ "Out of memory");
+ callback(0, &result, user_data);
+ pcmk__reset_result(&result);
+ return ENOMEM;
+ }
+
+ metadata_cb = (struct metadata_cb *) action->cb_data;
+ metadata_cb->callback = callback;
+ metadata_cb->user_data = user_data;
+ if (!services_action_async(action, metadata_complete)) {
+ services_action_free(action);
+ return pcmk_rc_error; // @TODO Derive from action->rc and ->status
+ }
+
+ // The services library has taken responsibility for action
+ return pcmk_rc_ok;
+}
+
/*!
* \internal
* \brief Set the result of an executor event
*
* \param[in,out] event Executor event to set
* \param[in] rc OCF exit status of event
* \param[in] op_status Executor status of event
* \param[in] exit_reason Human-friendly description of event
*/
void
lrmd__set_result(lrmd_event_data_t *event, enum ocf_exitcode rc, int op_status,
const char *exit_reason)
{
if (event == NULL) {
return;
}
event->rc = rc;
event->op_status = op_status;
pcmk__str_update((char **) &event->exit_reason, exit_reason);
}
/*!
* \internal
* \brief Clear an executor event's exit reason, output, and error output
*
* \param[in] event Executor event to reset
*/
void
lrmd__reset_result(lrmd_event_data_t *event)
{
if (event == NULL) {
return;
}
free((void *) event->exit_reason);
event->exit_reason = NULL;
free((void *) event->output);
event->output = NULL;
}
diff --git a/lib/pacemaker/pcmk_graph_producer.c b/lib/pacemaker/pcmk_graph_producer.c
index ffdf58efaa..a352783955 100644
--- a/lib/pacemaker/pcmk_graph_producer.c
+++ b/lib/pacemaker/pcmk_graph_producer.c
@@ -1,1122 +1,1121 @@
/*
* Copyright 2004-2022 the Pacemaker project contributors
*
* The version control history for this file may have further details.
*
* This source code is licensed under the GNU General Public License version 2
* or later (GPLv2+) WITHOUT ANY WARRANTY.
*/
#include <crm_internal.h>
#include <sys/param.h>
#include <crm/crm.h>
#include <crm/cib.h>
#include <crm/msg_xml.h>
#include <crm/common/xml.h>
#include <glib.h>
#include <pacemaker-internal.h>
#include "libpacemaker_private.h"
// Convenience macros for logging action properties
#define action_type_str(flags) \
(pcmk_is_set((flags), pe_action_pseudo)? "pseudo-action" : "action")
#define action_optional_str(flags) \
(pcmk_is_set((flags), pe_action_optional)? "optional" : "required")
#define action_runnable_str(flags) \
(pcmk_is_set((flags), pe_action_runnable)? "runnable" : "unrunnable")
#define action_node_str(a) \
(((a)->node == NULL)? "no node" : (a)->node->details->uname)
/*!
* \internal
* \brief Add an XML node tag for a specified ID
*
* \param[in] id Node UUID to add
* \param[in,out] xml Parent XML tag to add to
*/
static xmlNode*
add_node_to_xml_by_id(const char *id, xmlNode *xml)
{
xmlNode *node_xml;
node_xml = create_xml_node(xml, XML_CIB_TAG_NODE);
crm_xml_add(node_xml, XML_ATTR_UUID, id);
return node_xml;
}
/*!
* \internal
* \brief Add an XML node tag for a specified node
*
* \param[in] node Node to add
* \param[in,out] xml XML to add node to
*/
static void
add_node_to_xml(const pe_node_t *node, void *xml)
{
add_node_to_xml_by_id(node->details->id, (xmlNode *) xml);
}
/*!
* \internal
* \brief Add XML with nodes that need an update of their maintenance state
*
* \param[in,out] xml Parent XML tag to add to
* \param[in] data_set Working set for cluster
*/
static int
add_maintenance_nodes(xmlNode *xml, const pe_working_set_t *data_set)
{
GList *gIter = NULL;
xmlNode *maintenance =
xml?create_xml_node(xml, XML_GRAPH_TAG_MAINTENANCE):NULL;
int count = 0;
for (gIter = data_set->nodes; gIter != NULL;
gIter = gIter->next) {
pe_node_t *node = (pe_node_t *) gIter->data;
struct pe_node_shared_s *details = node->details;
if (!pe__is_guest_or_remote_node(node)) {
continue; /* just remote nodes need to know atm */
}
if (details->maintenance != details->remote_maintenance) {
if (maintenance) {
crm_xml_add(
add_node_to_xml_by_id(node->details->id, maintenance),
XML_NODE_IS_MAINTENANCE, details->maintenance?"1":"0");
}
count++;
}
}
crm_trace("%s %d nodes to adjust maintenance-mode "
"to transition", maintenance?"Added":"Counted", count);
return count;
}
/*!
* \internal
* \brief Add pseudo action with nodes needing maintenance state update
*
* \param[in,out] data_set Working set for cluster
*/
static void
add_maintenance_update(pe_working_set_t *data_set)
{
pe_action_t *action = NULL;
if (add_maintenance_nodes(NULL, data_set)) {
crm_trace("adding maintenance state update pseudo action");
action = get_pseudo_op(CRM_OP_MAINTENANCE_NODES, data_set);
pe__set_action_flags(action, pe_action_print_always);
}
}
/*!
* \internal
* \brief Add XML with nodes that an action is expected to bring down
*
* If a specified action is expected to bring any nodes down, add an XML block
* with their UUIDs. When a node is lost, this allows the controller to
* determine whether it was expected.
*
* \param[in,out] xml Parent XML tag to add to
* \param[in] action Action to check for downed nodes
* \param[in] data_set Working set for cluster
*/
static void
add_downed_nodes(xmlNode *xml, const pe_action_t *action,
const pe_working_set_t *data_set)
{
CRM_CHECK(xml && action && action->node && data_set, return);
if (pcmk__str_eq(action->task, CRM_OP_SHUTDOWN, pcmk__str_casei)) {
/* Shutdown makes the action's node down */
xmlNode *downed = create_xml_node(xml, XML_GRAPH_TAG_DOWNED);
add_node_to_xml_by_id(action->node->details->id, downed);
} else if (pcmk__str_eq(action->task, CRM_OP_FENCE, pcmk__str_casei)) {
/* Fencing makes the action's node and any hosted guest nodes down */
const char *fence = g_hash_table_lookup(action->meta, "stonith_action");
if (pcmk__is_fencing_action(fence)) {
xmlNode *downed = create_xml_node(xml, XML_GRAPH_TAG_DOWNED);
add_node_to_xml_by_id(action->node->details->id, downed);
pe_foreach_guest_node(data_set, action->node, add_node_to_xml, downed);
}
} else if (action->rsc && action->rsc->is_remote_node
&& pcmk__str_eq(action->task, CRMD_ACTION_STOP, pcmk__str_casei)) {
/* Stopping a remote connection resource makes connected node down,
* unless it's part of a migration
*/
GList *iter;
pe_action_t *input;
gboolean migrating = FALSE;
for (iter = action->actions_before; iter != NULL; iter = iter->next) {
input = ((pe_action_wrapper_t *) iter->data)->action;
if (input->rsc && pcmk__str_eq(action->rsc->id, input->rsc->id, pcmk__str_casei)
&& pcmk__str_eq(input->task, CRMD_ACTION_MIGRATED, pcmk__str_casei)) {
migrating = TRUE;
break;
}
}
if (!migrating) {
xmlNode *downed = create_xml_node(xml, XML_GRAPH_TAG_DOWNED);
add_node_to_xml_by_id(action->rsc->id, downed);
}
}
}
/*!
* \internal
* \brief Create a transition graph operation key for a clone action
*
* \param[in] action Clone action
* \param[in] interval_ms Action interval in milliseconds
*
* \return Newly allocated string with transition graph operation key
*/
static char *
clone_op_key(pe_action_t *action, guint interval_ms)
{
if (pcmk__str_eq(action->task, RSC_NOTIFY, pcmk__str_none)) {
const char *n_type = g_hash_table_lookup(action->meta, "notify_type");
const char *n_task = g_hash_table_lookup(action->meta,
"notify_operation");
CRM_LOG_ASSERT((n_type != NULL) && (n_task != NULL));
return pcmk__notify_key(action->rsc->clone_name, n_type, n_task);
} else if (action->cancel_task != NULL) {
return pcmk__op_key(action->rsc->clone_name, action->cancel_task,
interval_ms);
} else {
return pcmk__op_key(action->rsc->clone_name, action->task, interval_ms);
}
}
/*!
* \internal
* \brief Add node details to transition graph action XML
*
* \param[in] action Scheduled action
* \param[in] xml Transition graph action XML for \p action
*/
static void
add_node_details(pe_action_t *action, xmlNode *xml)
{
pe_node_t *router_node = pcmk__connection_host_for_action(action);
crm_xml_add(xml, XML_LRM_ATTR_TARGET, action->node->details->uname);
crm_xml_add(xml, XML_LRM_ATTR_TARGET_UUID, action->node->details->id);
if (router_node != NULL) {
crm_xml_add(xml, XML_LRM_ATTR_ROUTER_NODE, router_node->details->uname);
}
}
/*!
* \internal
* \brief Add resource details to transition graph action XML
*
* \param[in] action Scheduled action
* \param[in] action_xml Transition graph action XML for \p action
*/
static void
add_resource_details(pe_action_t *action, xmlNode *action_xml)
{
xmlNode *rsc_xml = NULL;
const char *attr_list[] = {
XML_AGENT_ATTR_CLASS,
XML_AGENT_ATTR_PROVIDER,
XML_ATTR_TYPE
};
/* If a resource is locked to a node via shutdown-lock, mark its actions
* so the controller can preserve the lock when the action completes.
*/
if (pcmk__action_locks_rsc_to_node(action)) {
crm_xml_add_ll(action_xml, XML_CONFIG_ATTR_SHUTDOWN_LOCK,
(long long) action->rsc->lock_time);
}
// List affected resource
rsc_xml = create_xml_node(action_xml, crm_element_name(action->rsc->xml));
if (pcmk_is_set(action->rsc->flags, pe_rsc_orphan)
&& (action->rsc->clone_name != NULL)) {
/* Use the numbered instance name here, because if there is more
* than one instance on a node, we need to make sure the command
* goes to the right one.
*
* This is important even for anonymous clones, because the clone's
* unique meta-attribute might have just been toggled from on to
* off.
*/
crm_debug("Using orphan clone name %s instead of %s",
action->rsc->id, action->rsc->clone_name);
crm_xml_add(rsc_xml, XML_ATTR_ID, action->rsc->clone_name);
crm_xml_add(rsc_xml, XML_ATTR_ID_LONG, action->rsc->id);
} else if (!pcmk_is_set(action->rsc->flags, pe_rsc_unique)) {
const char *xml_id = ID(action->rsc->xml);
crm_debug("Using anonymous clone name %s for %s (aka %s)",
xml_id, action->rsc->id, action->rsc->clone_name);
/* ID is what we'd like client to use
* ID_LONG is what they might know it as instead
*
* ID_LONG is only strictly needed /here/ during the
* transition period until all nodes in the cluster
* are running the new software /and/ have rebooted
* once (meaning that they've only ever spoken to a DC
* supporting this feature).
*
* If anyone toggles the unique flag to 'on', the
* 'instance free' name will correspond to an orphan
* and fall into the clause above instead
*/
crm_xml_add(rsc_xml, XML_ATTR_ID, xml_id);
if ((action->rsc->clone_name != NULL)
&& !pcmk__str_eq(xml_id, action->rsc->clone_name,
pcmk__str_none)) {
crm_xml_add(rsc_xml, XML_ATTR_ID_LONG, action->rsc->clone_name);
} else {
crm_xml_add(rsc_xml, XML_ATTR_ID_LONG, action->rsc->id);
}
} else {
CRM_ASSERT(action->rsc->clone_name == NULL);
crm_xml_add(rsc_xml, XML_ATTR_ID, action->rsc->id);
}
for (int lpc = 0; lpc < PCMK__NELEM(attr_list); lpc++) {
crm_xml_add(rsc_xml, attr_list[lpc],
g_hash_table_lookup(action->rsc->meta, attr_list[lpc]));
}
}
/*!
* \internal
* \brief Add action attributes to transition graph action XML
*
* \param[in] action Scheduled action
* \param[in] action_xml Transition graph action XML for \p action
*/
static void
add_action_attributes(pe_action_t *action, xmlNode *action_xml)
{
xmlNode *args_xml = NULL;
/* We create free-standing XML to start, so we can sort the attributes
* before adding it to action_xml, which keeps the scheduler regression
* test graphs comparable.
*/
args_xml = create_xml_node(NULL, XML_TAG_ATTRS);
crm_xml_add(args_xml, XML_ATTR_CRM_VERSION, CRM_FEATURE_SET);
g_hash_table_foreach(action->extra, hash2field, args_xml);
if ((action->rsc != NULL) && (action->node != NULL)) {
// Get the resource instance attributes, evaluated properly for node
GHashTable *params = pe_rsc_params(action->rsc, action->node,
action->rsc->cluster);
pcmk__substitute_remote_addr(action->rsc, params);
g_hash_table_foreach(params, hash2smartfield, args_xml);
#if ENABLE_VERSIONED_ATTRS
{
xmlNode *versioned_parameters = create_xml_node(NULL, XML_TAG_RSC_VER_ATTRS);
pe_get_versioned_attributes(versioned_parameters, action->rsc,
action->node, action->rsc->cluster);
if (xml_has_children(versioned_parameters)) {
add_node_copy(action_xml, versioned_parameters);
}
free_xml(versioned_parameters);
}
#endif
} else if ((action->rsc != NULL) && (action->rsc->variant <= pe_native)) {
GHashTable *params = pe_rsc_params(action->rsc, NULL,
action->rsc->cluster);
g_hash_table_foreach(params, hash2smartfield, args_xml);
#if ENABLE_VERSIONED_ATTRS
if (xml_has_children(action->rsc->versioned_parameters)) {
add_node_copy(action_xml, action->rsc->versioned_parameters);
}
#endif
}
#if ENABLE_VERSIONED_ATTRS
if (rsc_details != NULL) {
if (xml_has_children(rsc_details->versioned_parameters)) {
add_node_copy(action_xml, rsc_details->versioned_parameters);
}
if (xml_has_children(rsc_details->versioned_meta)) {
add_node_copy(action_xml, rsc_details->versioned_meta);
}
}
#endif
g_hash_table_foreach(action->meta, hash2metafield, args_xml);
if (action->rsc != NULL) {
const char *value = g_hash_table_lookup(action->rsc->meta,
"external-ip");
pe_resource_t *parent = action->rsc;
while (parent != NULL) {
parent->cmds->append_meta(parent, args_xml);
parent = parent->parent;
}
if (value != NULL) {
hash2smartfield((gpointer) "pcmk_external_ip", (gpointer) value,
(gpointer) args_xml);
}
pcmk__add_bundle_meta_to_xml(args_xml, action);
} else if (pcmk__str_eq(action->task, CRM_OP_FENCE, pcmk__str_none)
&& (action->node != NULL)) {
/* Pass the node's attributes as meta-attributes.
*
* @TODO: Determine whether it is still necessary to do this. It was
* added in 33d99707, probably for the libfence-based implementation in
* c9a90bd, which is no longer used.
*/
g_hash_table_foreach(action->node->details->attrs, hash2metafield, args_xml);
}
sorted_xml(args_xml, action_xml, FALSE);
free_xml(args_xml);
}
/*!
* \internal
* \brief Create the transition graph XML for a scheduled action
*
* \param[in] parent Parent XML element to add action to
* \param[in] action Scheduled action
* \param[in] skip_details If false, add action details as sub-elements
* \param[in] data_set Cluster working set
*/
static void
create_graph_action(xmlNode *parent, pe_action_t *action, bool skip_details,
pe_working_set_t *data_set)
{
bool needs_node_info = true;
bool needs_maintenance_info = false;
xmlNode *action_xml = NULL;
#if ENABLE_VERSIONED_ATTRS
pe_rsc_action_details_t *rsc_details = NULL;
#endif
if ((action == NULL) || (data_set == NULL)) {
return;
}
// Create the top-level element based on task
if (pcmk__str_eq(action->task, CRM_OP_FENCE, pcmk__str_casei)) {
/* All fences need node info; guest node fences are pseudo-events */
action_xml = create_xml_node(parent,
pcmk_is_set(action->flags, pe_action_pseudo)?
XML_GRAPH_TAG_PSEUDO_EVENT :
XML_GRAPH_TAG_CRM_EVENT);
} else if (pcmk__str_any_of(action->task,
CRM_OP_SHUTDOWN,
- CRM_OP_CLEAR_FAILCOUNT,
- CRM_OP_LRM_REFRESH, NULL)) {
+ CRM_OP_CLEAR_FAILCOUNT, NULL)) {
action_xml = create_xml_node(parent, XML_GRAPH_TAG_CRM_EVENT);
} else if (pcmk__str_eq(action->task, CRM_OP_LRM_DELETE, pcmk__str_none)) {
// CIB-only clean-up for shutdown locks
action_xml = create_xml_node(parent, XML_GRAPH_TAG_CRM_EVENT);
crm_xml_add(action_xml, PCMK__XA_MODE, XML_TAG_CIB);
} else if (pcmk_is_set(action->flags, pe_action_pseudo)) {
if (pcmk__str_eq(action->task, CRM_OP_MAINTENANCE_NODES,
pcmk__str_none)) {
needs_maintenance_info = true;
}
action_xml = create_xml_node(parent, XML_GRAPH_TAG_PSEUDO_EVENT);
needs_node_info = false;
} else {
action_xml = create_xml_node(parent, XML_GRAPH_TAG_RSC_OP);
#if ENABLE_VERSIONED_ATTRS
rsc_details = pe_rsc_action_details(action);
#endif
}
crm_xml_add_int(action_xml, XML_ATTR_ID, action->id);
crm_xml_add(action_xml, XML_LRM_ATTR_TASK, action->task);
if ((action->rsc != NULL) && (action->rsc->clone_name != NULL)) {
char *clone_key = NULL;
guint interval_ms;
if (pcmk__guint_from_hash(action->meta, XML_LRM_ATTR_INTERVAL_MS, 0,
&interval_ms) != pcmk_rc_ok) {
interval_ms = 0;
}
clone_key = clone_op_key(action, interval_ms);
crm_xml_add(action_xml, XML_LRM_ATTR_TASK_KEY, clone_key);
crm_xml_add(action_xml, "internal_" XML_LRM_ATTR_TASK_KEY, action->uuid);
free(clone_key);
} else {
crm_xml_add(action_xml, XML_LRM_ATTR_TASK_KEY, action->uuid);
}
if (needs_node_info && (action->node != NULL)) {
add_node_details(action, action_xml);
g_hash_table_insert(action->meta, strdup(XML_LRM_ATTR_TARGET),
strdup(action->node->details->uname));
g_hash_table_insert(action->meta, strdup(XML_LRM_ATTR_TARGET_UUID),
strdup(action->node->details->id));
}
if (skip_details) {
return;
}
if ((action->rsc != NULL)
&& !pcmk_is_set(action->flags, pe_action_pseudo)) {
// This is a real resource action, so add resource details
add_resource_details(action, action_xml);
}
/* List any attributes in effect */
add_action_attributes(action, action_xml);
/* List any nodes this action is expected to make down */
if (needs_node_info && (action->node != NULL)) {
add_downed_nodes(action_xml, action, data_set);
}
if (needs_maintenance_info) {
add_maintenance_nodes(action_xml, data_set);
}
}
/*!
* \internal
* \brief Check whether an action should be added to the transition graph
*
* \param[in] action Action to check
*
* \return true if action should be added to graph, otherwise false
*/
static bool
should_add_action_to_graph(pe_action_t *action)
{
if (!pcmk_is_set(action->flags, pe_action_runnable)) {
crm_trace("Ignoring action %s (%d): unrunnable",
action->uuid, action->id);
return false;
}
if (pcmk_is_set(action->flags, pe_action_optional)
&& !pcmk_is_set(action->flags, pe_action_print_always)) {
crm_trace("Ignoring action %s (%d): optional",
action->uuid, action->id);
return false;
}
/* Actions for unmanaged resources should be excluded from the graph,
* with the exception of monitors and cancellation of recurring monitors.
*/
if ((action->rsc != NULL)
&& !pcmk_is_set(action->rsc->flags, pe_rsc_managed)
&& !pcmk__str_eq(action->task, RSC_STATUS, pcmk__str_none)) {
const char *interval_ms_s;
/* A cancellation of a recurring monitor will get here because the task
* is cancel rather than monitor, but the interval can still be used to
* recognize it. The interval has been normalized to milliseconds by
* this point, so a string comparison is sufficient.
*/
interval_ms_s = g_hash_table_lookup(action->meta,
XML_LRM_ATTR_INTERVAL_MS);
if (pcmk__str_eq(interval_ms_s, "0", pcmk__str_null_matches)) {
crm_trace("Ignoring action %s (%d): for unmanaged resource (%s)",
action->uuid, action->id, action->rsc->id);
return false;
}
}
/* Always add pseudo-actions, fence actions, and shutdown actions (already
* determined to be required and runnable by this point)
*/
if (pcmk_is_set(action->flags, pe_action_pseudo)
|| pcmk__strcase_any_of(action->task, CRM_OP_FENCE, CRM_OP_SHUTDOWN,
NULL)) {
return true;
}
if (action->node == NULL) {
pe_err("Skipping action %s (%d) "
"because it was not allocated to a node (bug?)",
action->uuid, action->id);
pcmk__log_action("Unallocated", action, false);
return false;
}
if (pcmk_is_set(action->flags, pe_action_dc)) {
crm_trace("Action %s (%d) should be dumped: "
"can run on DC instead of %s",
action->uuid, action->id, pe__node_name(action->node));
} else if (pe__is_guest_node(action->node)
&& !action->node->details->remote_requires_reset) {
crm_trace("Action %s (%d) should be dumped: "
"assuming will be runnable on guest %s",
action->uuid, action->id, pe__node_name(action->node));
} else if (!action->node->details->online) {
pe_err("Skipping action %s (%d) "
"because it was scheduled for offline node (bug?)",
action->uuid, action->id);
pcmk__log_action("Offline node", action, false);
return false;
} else if (action->node->details->unclean) {
pe_err("Skipping action %s (%d) "
"because it was scheduled for unclean node (bug?)",
action->uuid, action->id);
pcmk__log_action("Unclean node", action, false);
return false;
}
return true;
}
/*!
* \internal
* \brief Check whether an ordering's flags can change an action
*
* \param[in] ordering Ordering to check
*
* \return true if ordering has flags that can change an action, false otherwise
*/
static bool
ordering_can_change_actions(pe_action_wrapper_t *ordering)
{
return pcmk_any_flags_set(ordering->type, ~(pe_order_implies_first_printed
|pe_order_implies_then_printed
|pe_order_optional));
}
/*!
* \internal
* \brief Check whether an action input should be in the transition graph
*
* \param[in] action Action to check
* \param[in,out] input Action input to check
*
* \return true if input should be in graph, false otherwise
* \note This function may not only check an input, but disable it under certian
* circumstances (load or anti-colocation orderings that are not needed).
*/
static bool
should_add_input_to_graph(pe_action_t *action, pe_action_wrapper_t *input)
{
if (input->state == pe_link_dumped) {
return true;
}
if (input->type == pe_order_none) {
crm_trace("Ignoring %s (%d) input %s (%d): "
"ordering disabled",
action->uuid, action->id,
input->action->uuid, input->action->id);
return false;
} else if (!pcmk_is_set(input->action->flags, pe_action_runnable)
&& !ordering_can_change_actions(input)) {
crm_trace("Ignoring %s (%d) input %s (%d): "
"optional and input unrunnable",
action->uuid, action->id,
input->action->uuid, input->action->id);
return false;
} else if (!pcmk_is_set(input->action->flags, pe_action_runnable)
&& pcmk_is_set(input->type, pe_order_one_or_more)) {
crm_trace("Ignoring %s (%d) input %s (%d): "
"one-or-more and input unrunnable",
action->uuid, action->id,
input->action->uuid, input->action->id);
return false;
} else if (pcmk_is_set(input->type, pe_order_implies_first_migratable)
&& !pcmk_is_set(input->action->flags, pe_action_runnable)) {
crm_trace("Ignoring %s (%d) input %s (%d): "
"implies input migratable but input unrunnable",
action->uuid, action->id,
input->action->uuid, input->action->id);
return false;
} else if (pcmk_is_set(input->type, pe_order_apply_first_non_migratable)
&& pcmk_is_set(input->action->flags, pe_action_migrate_runnable)) {
crm_trace("Ignoring %s (%d) input %s (%d): "
"only if input unmigratable but input unrunnable",
action->uuid, action->id,
input->action->uuid, input->action->id);
return false;
} else if ((input->type == pe_order_optional)
&& pcmk_is_set(input->action->flags, pe_action_migrate_runnable)
&& pcmk__ends_with(input->action->uuid, "_stop_0")) {
crm_trace("Ignoring %s (%d) input %s (%d): "
"optional but stop in migration",
action->uuid, action->id,
input->action->uuid, input->action->id);
return false;
} else if (input->type == pe_order_load) {
pe_node_t *input_node = input->action->node;
// load orderings are relevant only if actions are for same node
if (action->rsc && pcmk__str_eq(action->task, RSC_MIGRATE, pcmk__str_casei)) {
pe_node_t *allocated = action->rsc->allocated_to;
/* For load_stopped -> migrate_to orderings, we care about where it
* has been allocated to, not where it will be executed.
*/
if ((input_node == NULL) || (allocated == NULL)
|| (input_node->details != allocated->details)) {
crm_trace("Ignoring %s (%d) input %s (%d): "
"load ordering node mismatch %s vs %s",
action->uuid, action->id,
input->action->uuid, input->action->id,
(allocated? allocated->details->uname : "<none>"),
(input_node? input_node->details->uname : "<none>"));
input->type = pe_order_none;
return false;
}
} else if ((input_node == NULL) || (action->node == NULL)
|| (input_node->details != action->node->details)) {
crm_trace("Ignoring %s (%d) input %s (%d): "
"load ordering node mismatch %s vs %s",
action->uuid, action->id,
input->action->uuid, input->action->id,
(action->node? action->node->details->uname : "<none>"),
(input_node? input_node->details->uname : "<none>"));
input->type = pe_order_none;
return false;
} else if (pcmk_is_set(input->action->flags, pe_action_optional)) {
crm_trace("Ignoring %s (%d) input %s (%d): "
"load ordering input optional",
action->uuid, action->id,
input->action->uuid, input->action->id);
input->type = pe_order_none;
return false;
}
} else if (input->type == pe_order_anti_colocation) {
if (input->action->node && action->node
&& (input->action->node->details != action->node->details)) {
crm_trace("Ignoring %s (%d) input %s (%d): "
"anti-colocation node mismatch %s vs %s",
action->uuid, action->id,
input->action->uuid, input->action->id,
pe__node_name(action->node),
pe__node_name(input->action->node));
input->type = pe_order_none;
return false;
} else if (pcmk_is_set(input->action->flags, pe_action_optional)) {
crm_trace("Ignoring %s (%d) input %s (%d): "
"anti-colocation input optional",
action->uuid, action->id,
input->action->uuid, input->action->id);
input->type = pe_order_none;
return false;
}
} else if (input->action->rsc
&& input->action->rsc != action->rsc
&& pcmk_is_set(input->action->rsc->flags, pe_rsc_failed)
&& !pcmk_is_set(input->action->rsc->flags, pe_rsc_managed)
&& pcmk__ends_with(input->action->uuid, "_stop_0")
&& action->rsc && pe_rsc_is_clone(action->rsc)) {
crm_warn("Ignoring requirement that %s complete before %s:"
" unmanaged failed resources cannot prevent clone shutdown",
input->action->uuid, action->uuid);
return false;
} else if (pcmk_is_set(input->action->flags, pe_action_optional)
&& !pcmk_any_flags_set(input->action->flags,
pe_action_print_always|pe_action_dumped)
&& !should_add_action_to_graph(input->action)) {
crm_trace("Ignoring %s (%d) input %s (%d): "
"input optional",
action->uuid, action->id,
input->action->uuid, input->action->id);
return false;
}
crm_trace("%s (%d) input %s %s (%d) on %s should be dumped: %s %s %#.6x",
action->uuid, action->id, action_type_str(input->action->flags),
input->action->uuid, input->action->id,
action_node_str(input->action),
action_runnable_str(input->action->flags),
action_optional_str(input->action->flags), input->type);
return true;
}
/*!
* \internal
* \brief Check whether an ordering creates an ordering loop
*
* \param[in] init_action "First" action in ordering
* \param[in] action Callers should always set this the same as
* \p init_action (this function may use a different
* value for recursive calls)
* \param[in] input Action wrapper for "then" action in ordering
*
* \return true if the ordering creates a loop, otherwise false
*/
bool
pcmk__graph_has_loop(pe_action_t *init_action, pe_action_t *action,
pe_action_wrapper_t *input)
{
bool has_loop = false;
if (pcmk_is_set(input->action->flags, pe_action_tracking)) {
crm_trace("Breaking tracking loop: %s@%s -> %s@%s (%#.6x)",
input->action->uuid,
input->action->node? input->action->node->details->uname : "",
action->uuid,
action->node? action->node->details->uname : "",
input->type);
return false;
}
// Don't need to check inputs that won't be used
if (!should_add_input_to_graph(action, input)) {
return false;
}
if (input->action == init_action) {
crm_debug("Input loop found in %s@%s ->...-> %s@%s",
action->uuid,
action->node? action->node->details->uname : "",
init_action->uuid,
init_action->node? init_action->node->details->uname : "");
return true;
}
pe__set_action_flags(input->action, pe_action_tracking);
crm_trace("Checking inputs of action %s@%s input %s@%s (%#.6x)"
"for graph loop with %s@%s ",
action->uuid,
action->node? action->node->details->uname : "",
input->action->uuid,
input->action->node? input->action->node->details->uname : "",
input->type,
init_action->uuid,
init_action->node? init_action->node->details->uname : "");
// Recursively check input itself for loops
for (GList *iter = input->action->actions_before;
iter != NULL; iter = iter->next) {
if (pcmk__graph_has_loop(init_action, input->action,
(pe_action_wrapper_t *) iter->data)) {
// Recursive call already logged a debug message
has_loop = true;
break;
}
}
pe__clear_action_flags(input->action, pe_action_tracking);
if (!has_loop) {
crm_trace("No input loop found in %s@%s -> %s@%s (%#.6x)",
input->action->uuid,
input->action->node? input->action->node->details->uname : "",
action->uuid,
action->node? action->node->details->uname : "",
input->type);
}
return has_loop;
}
/*!
* \internal
* \brief Create a synapse XML element for a transition graph
*
* \param[in] action Action that synapse is for
* \param[in] data_set Cluster working set containing graph
*
* \return Newly added XML element for new graph synapse
*/
static xmlNode *
create_graph_synapse(pe_action_t *action, pe_working_set_t *data_set)
{
int synapse_priority = 0;
xmlNode *syn = create_xml_node(data_set->graph, "synapse");
crm_xml_add_int(syn, XML_ATTR_ID, data_set->num_synapse);
data_set->num_synapse++;
if (action->rsc != NULL) {
synapse_priority = action->rsc->priority;
}
if (action->priority > synapse_priority) {
synapse_priority = action->priority;
}
if (synapse_priority > 0) {
crm_xml_add_int(syn, XML_CIB_ATTR_PRIORITY, synapse_priority);
}
return syn;
}
/*!
* \internal
* \brief Add an action to the transition graph XML if appropriate
*
* \param[in] data Action to possibly add
* \param[in] user_data Cluster working set
*
* \note This will de-duplicate the action inputs, meaning that the
* pe_action_wrapper_t:type flags can no longer be relied on to retain
* their original settings. That means this MUST be called after
* pcmk__apply_orderings() is complete, and nothing after this should rely
* on those type flags. (For example, some code looks for type equal to
* some flag rather than whether the flag is set, and some code looks for
* particular combinations of flags -- such code must be done before
* pcmk__create_graph().)
*/
static void
add_action_to_graph(gpointer data, gpointer user_data)
{
pe_action_t *action = (pe_action_t *) data;
pe_working_set_t *data_set = (pe_working_set_t *) user_data;
xmlNode *syn = NULL;
xmlNode *set = NULL;
xmlNode *in = NULL;
/* If we haven't already, de-duplicate inputs (even if we won't be adding
* the action to the graph, so that crm_simulate's dot graphs don't have
* duplicates).
*/
if (!pcmk_is_set(action->flags, pe_action_dedup)) {
pcmk__deduplicate_action_inputs(action);
pe__set_action_flags(action, pe_action_dedup);
}
if (pcmk_is_set(action->flags, pe_action_dumped) // Already added, or
|| !should_add_action_to_graph(action)) { // shouldn't be added
return;
}
pe__set_action_flags(action, pe_action_dumped);
crm_trace("Adding action %d (%s%s%s) to graph",
action->id, action->uuid,
((action->node == NULL)? "" : " on "),
((action->node == NULL)? "" : action->node->details->uname));
syn = create_graph_synapse(action, data_set);
set = create_xml_node(syn, "action_set");
in = create_xml_node(syn, "inputs");
create_graph_action(set, action, false, data_set);
for (GList *lpc = action->actions_before; lpc != NULL; lpc = lpc->next) {
pe_action_wrapper_t *input = (pe_action_wrapper_t *) lpc->data;
if (should_add_input_to_graph(action, input)) {
xmlNode *input_xml = create_xml_node(in, "trigger");
input->state = pe_link_dumped;
create_graph_action(input_xml, input->action, true, data_set);
}
}
}
static int transition_id = -1;
/*!
* \internal
* \brief Log a message after calculating a transition
*
* \param[in] filename Where transition input is stored
*/
void
pcmk__log_transition_summary(const char *filename)
{
if (was_processing_error) {
crm_err("Calculated transition %d (with errors)%s%s",
transition_id,
(filename == NULL)? "" : ", saving inputs in ",
(filename == NULL)? "" : filename);
} else if (was_processing_warning) {
crm_warn("Calculated transition %d (with warnings)%s%s",
transition_id,
(filename == NULL)? "" : ", saving inputs in ",
(filename == NULL)? "" : filename);
} else {
crm_notice("Calculated transition %d%s%s",
transition_id,
(filename == NULL)? "" : ", saving inputs in ",
(filename == NULL)? "" : filename);
}
if (crm_config_error) {
crm_notice("Configuration errors found during scheduler processing,"
" please run \"crm_verify -L\" to identify issues");
}
}
/*!
* \internal
* \brief Add a resource's actions to the transition graph
*
* \param[in] rsc Resource whose actions should be added
*/
void
pcmk__add_rsc_actions_to_graph(pe_resource_t *rsc)
{
GList *iter = NULL;
CRM_ASSERT(rsc != NULL);
pe_rsc_trace(rsc, "Adding actions for %s to graph", rsc->id);
// First add the resource's own actions
g_list_foreach(rsc->actions, add_action_to_graph, rsc->cluster);
// Then recursively add its children's actions (appropriate to variant)
for (iter = rsc->children; iter != NULL; iter = iter->next) {
pe_resource_t *child_rsc = (pe_resource_t *) iter->data;
child_rsc->cmds->add_actions_to_graph(child_rsc);
}
}
/*!
* \internal
* \brief Create a transition graph with all cluster actions needed
*
* \param[in] data_set Cluster working set
*/
void
pcmk__create_graph(pe_working_set_t *data_set)
{
GList *iter = NULL;
const char *value = NULL;
long long limit = 0LL;
transition_id++;
crm_trace("Creating transition graph %d", transition_id);
data_set->graph = create_xml_node(NULL, XML_TAG_GRAPH);
value = pe_pref(data_set->config_hash, "cluster-delay");
crm_xml_add(data_set->graph, "cluster-delay", value);
value = pe_pref(data_set->config_hash, "stonith-timeout");
crm_xml_add(data_set->graph, "stonith-timeout", value);
crm_xml_add(data_set->graph, "failed-stop-offset", "INFINITY");
if (pcmk_is_set(data_set->flags, pe_flag_start_failure_fatal)) {
crm_xml_add(data_set->graph, "failed-start-offset", "INFINITY");
} else {
crm_xml_add(data_set->graph, "failed-start-offset", "1");
}
value = pe_pref(data_set->config_hash, "batch-limit");
crm_xml_add(data_set->graph, "batch-limit", value);
crm_xml_add_int(data_set->graph, "transition_id", transition_id);
value = pe_pref(data_set->config_hash, "migration-limit");
if ((pcmk__scan_ll(value, &limit, 0LL) == pcmk_rc_ok) && (limit > 0)) {
crm_xml_add(data_set->graph, "migration-limit", value);
}
if (data_set->recheck_by > 0) {
char *recheck_epoch = NULL;
recheck_epoch = crm_strdup_printf("%llu",
(long long) data_set->recheck_by);
crm_xml_add(data_set->graph, "recheck-by", recheck_epoch);
free(recheck_epoch);
}
/* The following code will de-duplicate action inputs, so nothing past this
* should rely on the action input type flags retaining their original
* values.
*/
// Add resource actions to graph
for (iter = data_set->resources; iter != NULL; iter = iter->next) {
pe_resource_t *rsc = (pe_resource_t *) iter->data;
pe_rsc_trace(rsc, "Processing actions for %s", rsc->id);
rsc->cmds->add_actions_to_graph(rsc);
}
// Add pseudo-action for list of nodes with maintenance state update
add_maintenance_update(data_set);
// Add non-resource (node) actions
for (iter = data_set->actions; iter != NULL; iter = iter->next) {
pe_action_t *action = (pe_action_t *) iter->data;
if ((action->rsc != NULL)
&& (action->node != NULL)
&& action->node->details->shutdown
&& !pcmk_is_set(action->rsc->flags, pe_rsc_maintenance)
&& !pcmk_any_flags_set(action->flags,
pe_action_optional|pe_action_runnable)
&& pcmk__str_eq(action->task, RSC_STOP, pcmk__str_none)) {
/* Eventually we should just ignore the 'fence' case, but for now
* it's the best way to detect (in CTS) when CIB resource updates
* are being lost.
*/
if (pcmk_is_set(data_set->flags, pe_flag_have_quorum)
|| (data_set->no_quorum_policy == no_quorum_ignore)) {
crm_crit("Cannot %s %s because of %s:%s%s (%s)",
action->node->details->unclean? "fence" : "shut down",
pe__node_name(action->node), action->rsc->id,
pcmk_is_set(action->rsc->flags, pe_rsc_managed)? " blocked" : " unmanaged",
pcmk_is_set(action->rsc->flags, pe_rsc_failed)? " failed" : "",
action->uuid);
}
}
add_action_to_graph((gpointer) action, (gpointer) data_set);
}
crm_log_xml_trace(data_set->graph, "graph");
}
diff --git a/lib/pengine/common.c b/lib/pengine/common.c
index 9be835da74..fc79e6e188 100644
--- a/lib/pengine/common.c
+++ b/lib/pengine/common.c
@@ -1,568 +1,566 @@
/*
* Copyright 2004-2022 the Pacemaker project contributors
*
* The version control history for this file may have further details.
*
* This source code is licensed under the GNU Lesser General Public License
* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
*/
#include <crm_internal.h>
#include <crm/crm.h>
#include <crm/msg_xml.h>
#include <crm/common/xml.h>
#include <crm/common/util.h>
#include <glib.h>
#include <crm/pengine/internal.h>
gboolean was_processing_error = FALSE;
gboolean was_processing_warning = FALSE;
static bool
check_placement_strategy(const char *value)
{
return pcmk__strcase_any_of(value, "default", "utilization", "minimal",
"balanced", NULL);
}
static pcmk__cluster_option_t pe_opts[] = {
/* name, old name, type, allowed values,
* default value, validator,
* short description,
* long description
*/
{
"no-quorum-policy", NULL, "select", "stop, freeze, ignore, demote, suicide",
"stop", pcmk__valid_quorum,
"What to do when the cluster does not have quorum",
NULL
},
{
"symmetric-cluster", NULL, "boolean", NULL,
"true", pcmk__valid_boolean,
"Whether resources can run on any node by default",
NULL
},
{
"maintenance-mode", NULL, "boolean", NULL,
"false", pcmk__valid_boolean,
"Whether the cluster should refrain from monitoring, starting, "
"and stopping resources",
NULL
},
{
"start-failure-is-fatal", NULL, "boolean", NULL,
"true", pcmk__valid_boolean,
"Whether a start failure should prevent a resource from being "
"recovered on the same node",
"When true, the cluster will immediately ban a resource from a node "
"if it fails to start there. When false, the cluster will instead "
"check the resource's fail count against its migration-threshold."
},
{
"enable-startup-probes", NULL, "boolean", NULL,
"true", pcmk__valid_boolean,
"Whether the cluster should check for active resources during start-up",
NULL
},
{
XML_CONFIG_ATTR_SHUTDOWN_LOCK, NULL, "boolean", NULL,
"false", pcmk__valid_boolean,
"Whether to lock resources to a cleanly shut down node",
"When true, resources active on a node when it is cleanly shut down "
"are kept \"locked\" to that node (not allowed to run elsewhere) "
"until they start again on that node after it rejoins (or for at "
"most shutdown-lock-limit, if set). Stonith resources and "
"Pacemaker Remote connections are never locked. Clone and bundle "
"instances and the promoted role of promotable clones are currently"
" never locked, though support could be added in a future release."
},
{
XML_CONFIG_ATTR_SHUTDOWN_LOCK_LIMIT, NULL, "time", NULL,
"0", pcmk__valid_interval_spec,
"Do not lock resources to a cleanly shut down node longer than this",
"If shutdown-lock is true and this is set to a nonzero time duration, "
"shutdown locks will expire after this much time has passed since "
"the shutdown was initiated, even if the node has not rejoined."
},
// Fencing-related options
{
"stonith-enabled", NULL, "boolean", NULL,
"true", pcmk__valid_boolean,
"*** Advanced Use Only *** "
"Whether nodes may be fenced as part of recovery",
"If false, unresponsive nodes are immediately assumed to be harmless, "
"and resources that were active on them may be recovered "
"elsewhere. This can result in a \"split-brain\" situation, "
"potentially leading to data loss and/or service unavailability."
},
{
"stonith-action", NULL, "select", "reboot, off, poweroff",
"reboot", pcmk__is_fencing_action,
"Action to send to fence device when a node needs to be fenced "
"(\"poweroff\" is a deprecated alias for \"off\")",
NULL
},
{
"stonith-timeout", NULL, "time", NULL,
"60s", pcmk__valid_interval_spec,
"*** Advanced Use Only *** Unused by Pacemaker",
"This value is not used by Pacemaker, but is kept for backward "
"compatibility, and certain legacy fence agents might use it."
},
{
XML_ATTR_HAVE_WATCHDOG, NULL, "boolean", NULL,
"false", pcmk__valid_boolean,
N_("Whether watchdog integration is enabled"),
"This is set automatically by the cluster according to whether SBD "
"is detected to be in use. User-configured values are ignored. "
"The value `true` is meaningful if diskless SBD is used and "
"`stonith-watchdog-timeout` is nonzero. In that case, if fencing "
"is required, watchdog-based self-fencing will be performed via "
"SBD without requiring a fencing resource explicitly configured."
},
{
"concurrent-fencing", NULL, "boolean", NULL,
PCMK__CONCURRENT_FENCING_DEFAULT, pcmk__valid_boolean,
"Allow performing fencing operations in parallel",
NULL
},
{
"startup-fencing", NULL, "boolean", NULL,
"true", pcmk__valid_boolean,
"*** Advanced Use Only *** Whether to fence unseen nodes at start-up",
"Setting this to false may lead to a \"split-brain\" situation,"
"potentially leading to data loss and/or service unavailability."
},
{
XML_CONFIG_ATTR_PRIORITY_FENCING_DELAY, NULL, "time", NULL,
"0", pcmk__valid_interval_spec,
"Apply fencing delay targeting the lost nodes with the highest total resource priority",
"Apply specified delay for the fencings that are targeting the lost "
"nodes with the highest total resource priority in case we don't "
"have the majority of the nodes in our cluster partition, so that "
"the more significant nodes potentially win any fencing match, "
"which is especially meaningful under split-brain of 2-node "
"cluster. A promoted resource instance takes the base priority + 1 "
"on calculation if the base priority is not 0. Any static/random "
"delays that are introduced by `pcmk_delay_base/max` configured "
"for the corresponding fencing resources will be added to this "
"delay. This delay should be significantly greater than, safely "
"twice, the maximum `pcmk_delay_base/max`. By default, priority "
"fencing delay is disabled."
},
{
"cluster-delay", NULL, "time", NULL,
"60s", pcmk__valid_interval_spec,
"Maximum time for node-to-node communication",
"The node elected Designated Controller (DC) will consider an action "
"failed if it does not get a response from the node executing the "
"action within this time (after considering the action's own "
"timeout). The \"correct\" value will depend on the speed and "
"load of your network and cluster nodes."
},
{
"batch-limit", NULL, "integer", NULL,
"0", pcmk__valid_number,
"Maximum number of jobs that the cluster may execute in parallel "
"across all nodes",
"The \"correct\" value will depend on the speed and load of your "
"network and cluster nodes. If set to 0, the cluster will "
"impose a dynamically calculated limit when any node has a "
"high load."
},
{
"migration-limit", NULL, "integer", NULL,
"-1", pcmk__valid_number,
"The number of live migration actions that the cluster is allowed "
"to execute in parallel on a node (-1 means no limit)"
},
/* Orphans and stopping */
{
"stop-all-resources", NULL, "boolean", NULL,
"false", pcmk__valid_boolean,
"Whether the cluster should stop all active resources",
NULL
},
{
"stop-orphan-resources", NULL, "boolean", NULL,
"true", pcmk__valid_boolean,
"Whether to stop resources that were removed from the configuration",
NULL
},
{
"stop-orphan-actions", NULL, "boolean", NULL,
"true", pcmk__valid_boolean,
"Whether to cancel recurring actions removed from the configuration",
NULL
},
{
"remove-after-stop", NULL, "boolean", NULL,
"false", pcmk__valid_boolean,
"*** Deprecated *** Whether to remove stopped resources from "
"the executor",
"Values other than default are poorly tested and potentially dangerous."
" This option will be removed in a future release."
},
/* Storing inputs */
{
"pe-error-series-max", NULL, "integer", NULL,
"-1", pcmk__valid_number,
"The number of scheduler inputs resulting in errors to save",
"Zero to disable, -1 to store unlimited."
},
{
"pe-warn-series-max", NULL, "integer", NULL,
"5000", pcmk__valid_number,
"The number of scheduler inputs resulting in warnings to save",
"Zero to disable, -1 to store unlimited."
},
{
"pe-input-series-max", NULL, "integer", NULL,
"4000", pcmk__valid_number,
"The number of scheduler inputs without errors or warnings to save",
"Zero to disable, -1 to store unlimited."
},
/* Node health */
{
PCMK__OPT_NODE_HEALTH_STRATEGY, NULL, "select",
PCMK__VALUE_NONE ", " PCMK__VALUE_MIGRATE_ON_RED ", "
PCMK__VALUE_ONLY_GREEN ", " PCMK__VALUE_PROGRESSIVE ", "
PCMK__VALUE_CUSTOM,
PCMK__VALUE_NONE, pcmk__validate_health_strategy,
"How cluster should react to node health attributes",
"Requires external entities to create node attributes (named with "
"the prefix \"#health\") with values \"" PCMK__VALUE_RED "\", "
"\"" PCMK__VALUE_YELLOW "\", or \"" PCMK__VALUE_GREEN "\"."
},
{
PCMK__OPT_NODE_HEALTH_BASE, NULL, "integer", NULL,
"0", pcmk__valid_number,
"Base health score assigned to a node",
"Only used when " PCMK__OPT_NODE_HEALTH_STRATEGY " is set to "
PCMK__VALUE_PROGRESSIVE "."
},
{
PCMK__OPT_NODE_HEALTH_GREEN, NULL, "integer", NULL,
"0", pcmk__valid_number,
"The score to use for a node health attribute whose value is \""
PCMK__VALUE_GREEN "\"",
"Only used when " PCMK__OPT_NODE_HEALTH_STRATEGY " is set to "
PCMK__VALUE_CUSTOM " or " PCMK__VALUE_PROGRESSIVE "."
},
{
PCMK__OPT_NODE_HEALTH_YELLOW, NULL, "integer", NULL,
"0", pcmk__valid_number,
"The score to use for a node health attribute whose value is \""
PCMK__VALUE_YELLOW "\"",
"Only used when " PCMK__OPT_NODE_HEALTH_STRATEGY " is set to "
PCMK__VALUE_CUSTOM " or " PCMK__VALUE_PROGRESSIVE "."
},
{
PCMK__OPT_NODE_HEALTH_RED, NULL, "integer", NULL,
"-INFINITY", pcmk__valid_number,
"The score to use for a node health attribute whose value is \""
PCMK__VALUE_RED "\"",
"Only used when " PCMK__OPT_NODE_HEALTH_STRATEGY " is set to "
PCMK__VALUE_CUSTOM " or " PCMK__VALUE_PROGRESSIVE "."
},
/*Placement Strategy*/
{
"placement-strategy", NULL, "select",
"default, utilization, minimal, balanced",
"default", check_placement_strategy,
"How the cluster should allocate resources to nodes",
NULL
},
};
void
pe_metadata(pcmk__output_t *out)
{
char *s = pcmk__format_option_metadata("pacemaker-schedulerd",
"Pacemaker scheduler options",
"Cluster options used by Pacemaker's scheduler",
pe_opts, PCMK__NELEM(pe_opts));
out->output_xml(out, "metadata", s);
free(s);
}
void
verify_pe_options(GHashTable * options)
{
pcmk__validate_cluster_options(options, pe_opts, PCMK__NELEM(pe_opts));
}
const char *
pe_pref(GHashTable * options, const char *name)
{
return pcmk__cluster_option(options, pe_opts, PCMK__NELEM(pe_opts), name);
}
const char *
fail2text(enum action_fail_response fail)
{
const char *result = "<unknown>";
switch (fail) {
case action_fail_ignore:
result = "ignore";
break;
case action_fail_demote:
result = "demote";
break;
case action_fail_block:
result = "block";
break;
case action_fail_recover:
result = "recover";
break;
case action_fail_migrate:
result = "migrate";
break;
case action_fail_stop:
result = "stop";
break;
case action_fail_fence:
result = "fence";
break;
case action_fail_standby:
result = "standby";
break;
case action_fail_restart_container:
result = "restart-container";
break;
case action_fail_reset_remote:
result = "reset-remote";
break;
}
return result;
}
enum action_tasks
text2task(const char *task)
{
if (pcmk__str_eq(task, CRMD_ACTION_STOP, pcmk__str_casei)) {
return stop_rsc;
} else if (pcmk__str_eq(task, CRMD_ACTION_STOPPED, pcmk__str_casei)) {
return stopped_rsc;
} else if (pcmk__str_eq(task, CRMD_ACTION_START, pcmk__str_casei)) {
return start_rsc;
} else if (pcmk__str_eq(task, CRMD_ACTION_STARTED, pcmk__str_casei)) {
return started_rsc;
} else if (pcmk__str_eq(task, CRM_OP_SHUTDOWN, pcmk__str_casei)) {
return shutdown_crm;
} else if (pcmk__str_eq(task, CRM_OP_FENCE, pcmk__str_casei)) {
return stonith_node;
} else if (pcmk__str_eq(task, CRMD_ACTION_STATUS, pcmk__str_casei)) {
return monitor_rsc;
} else if (pcmk__str_eq(task, CRMD_ACTION_NOTIFY, pcmk__str_casei)) {
return action_notify;
} else if (pcmk__str_eq(task, CRMD_ACTION_NOTIFIED, pcmk__str_casei)) {
return action_notified;
} else if (pcmk__str_eq(task, CRMD_ACTION_PROMOTE, pcmk__str_casei)) {
return action_promote;
} else if (pcmk__str_eq(task, CRMD_ACTION_DEMOTE, pcmk__str_casei)) {
return action_demote;
} else if (pcmk__str_eq(task, CRMD_ACTION_PROMOTED, pcmk__str_casei)) {
return action_promoted;
} else if (pcmk__str_eq(task, CRMD_ACTION_DEMOTED, pcmk__str_casei)) {
return action_demoted;
}
#if SUPPORT_TRACING
if (pcmk__str_eq(task, CRMD_ACTION_CANCEL, pcmk__str_casei)) {
return no_action;
} else if (pcmk__str_eq(task, CRMD_ACTION_DELETE, pcmk__str_casei)) {
return no_action;
} else if (pcmk__str_eq(task, CRMD_ACTION_STATUS, pcmk__str_casei)) {
return no_action;
- } else if (pcmk__str_eq(task, CRM_OP_LRM_REFRESH, pcmk__str_casei)) {
- return no_action;
} else if (pcmk__str_eq(task, CRMD_ACTION_MIGRATE, pcmk__str_casei)) {
return no_action;
} else if (pcmk__str_eq(task, CRMD_ACTION_MIGRATED, pcmk__str_casei)) {
return no_action;
}
crm_trace("Unsupported action: %s", task);
#endif
return no_action;
}
const char *
task2text(enum action_tasks task)
{
const char *result = "<unknown>";
switch (task) {
case no_action:
result = "no_action";
break;
case stop_rsc:
result = CRMD_ACTION_STOP;
break;
case stopped_rsc:
result = CRMD_ACTION_STOPPED;
break;
case start_rsc:
result = CRMD_ACTION_START;
break;
case started_rsc:
result = CRMD_ACTION_STARTED;
break;
case shutdown_crm:
result = CRM_OP_SHUTDOWN;
break;
case stonith_node:
result = CRM_OP_FENCE;
break;
case monitor_rsc:
result = CRMD_ACTION_STATUS;
break;
case action_notify:
result = CRMD_ACTION_NOTIFY;
break;
case action_notified:
result = CRMD_ACTION_NOTIFIED;
break;
case action_promote:
result = CRMD_ACTION_PROMOTE;
break;
case action_promoted:
result = CRMD_ACTION_PROMOTED;
break;
case action_demote:
result = CRMD_ACTION_DEMOTE;
break;
case action_demoted:
result = CRMD_ACTION_DEMOTED;
break;
}
return result;
}
const char *
role2text(enum rsc_role_e role)
{
switch (role) {
case RSC_ROLE_UNKNOWN:
return RSC_ROLE_UNKNOWN_S;
case RSC_ROLE_STOPPED:
return RSC_ROLE_STOPPED_S;
case RSC_ROLE_STARTED:
return RSC_ROLE_STARTED_S;
case RSC_ROLE_UNPROMOTED:
#ifdef PCMK__COMPAT_2_0
return RSC_ROLE_UNPROMOTED_LEGACY_S;
#else
return RSC_ROLE_UNPROMOTED_S;
#endif
case RSC_ROLE_PROMOTED:
#ifdef PCMK__COMPAT_2_0
return RSC_ROLE_PROMOTED_LEGACY_S;
#else
return RSC_ROLE_PROMOTED_S;
#endif
}
CRM_CHECK(role >= RSC_ROLE_UNKNOWN, return RSC_ROLE_UNKNOWN_S);
CRM_CHECK(role < RSC_ROLE_MAX, return RSC_ROLE_UNKNOWN_S);
// coverity[dead_error_line]
return RSC_ROLE_UNKNOWN_S;
}
enum rsc_role_e
text2role(const char *role)
{
CRM_ASSERT(role != NULL);
if (pcmk__str_eq(role, RSC_ROLE_STOPPED_S, pcmk__str_casei)) {
return RSC_ROLE_STOPPED;
} else if (pcmk__str_eq(role, RSC_ROLE_STARTED_S, pcmk__str_casei)) {
return RSC_ROLE_STARTED;
} else if (pcmk__strcase_any_of(role, RSC_ROLE_UNPROMOTED_S,
RSC_ROLE_UNPROMOTED_LEGACY_S, NULL)) {
return RSC_ROLE_UNPROMOTED;
} else if (pcmk__strcase_any_of(role, RSC_ROLE_PROMOTED_S,
RSC_ROLE_PROMOTED_LEGACY_S, NULL)) {
return RSC_ROLE_PROMOTED;
} else if (pcmk__str_eq(role, RSC_ROLE_UNKNOWN_S, pcmk__str_casei)) {
return RSC_ROLE_UNKNOWN;
}
crm_err("Unknown role: %s", role);
return RSC_ROLE_UNKNOWN;
}
void
add_hash_param(GHashTable * hash, const char *name, const char *value)
{
CRM_CHECK(hash != NULL, return);
crm_trace("Adding name='%s' value='%s' to hash table",
pcmk__s(name, "<null>"), pcmk__s(value, "<null>"));
if (name == NULL || value == NULL) {
return;
} else if (pcmk__str_eq(value, "#default", pcmk__str_casei)) {
return;
} else if (g_hash_table_lookup(hash, name) == NULL) {
g_hash_table_insert(hash, strdup(name), strdup(value));
}
}
const char *
pe_node_attribute_calculated(const pe_node_t *node, const char *name,
const pe_resource_t *rsc)
{
const char *source;
if(node == NULL) {
return NULL;
} else if(rsc == NULL) {
return g_hash_table_lookup(node->details->attrs, name);
}
source = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_TARGET);
if(source == NULL || !pcmk__str_eq("host", source, pcmk__str_casei)) {
return g_hash_table_lookup(node->details->attrs, name);
}
/* Use attributes set for the containers location
* instead of for the container itself
*
* Useful when the container is using the host's local
* storage
*/
CRM_ASSERT(node->details->remote_rsc);
CRM_ASSERT(node->details->remote_rsc->container);
if(node->details->remote_rsc->container->running_on) {
pe_node_t *host = node->details->remote_rsc->container->running_on->data;
pe_rsc_trace(rsc, "%s: Looking for %s on the container host %s",
rsc->id, name, pe__node_name(host));
return g_hash_table_lookup(host->details->attrs, name);
}
pe_rsc_trace(rsc, "%s: Not looking for %s on the container host: %s is inactive",
rsc->id, name, node->details->remote_rsc->container->id);
return NULL;
}
const char *
pe_node_attribute_raw(pe_node_t *node, const char *name)
{
if(node == NULL) {
return NULL;
}
return g_hash_table_lookup(node->details->attrs, name);
}

File Metadata

Mime Type
text/x-diff
Expires
Tue, Jul 8, 6:14 PM (21 h, 41 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
1985444
Default Alt Text
(835 KB)

Event Timeline