No OneTemporary
Actions

Size

243 KB

Referenced Files

None

Subscribers

None

View Options

	diff --git a/devel/Makefile.am b/devel/Makefile.am
	index abbb40becf..9c0532b180 100644
	--- a/devel/Makefile.am
	+++ b/devel/Makefile.am
	@@ -1,335 +1,335 @@
	#
	# Copyright 2020-2023 the Pacemaker project contributors
	#
	# The version control history for this file may have further details.
	#
	# This source code is licensed under the GNU General Public License version 2
	# or later (GPLv2+) WITHOUT ANY WARRANTY.
	#

	include $(top_srcdir)/mk/common.mk
	include $(top_srcdir)/mk/release.mk

	# Coccinelle is a tool that takes special patch-like files (called semantic patches) and
	# applies them throughout a source tree. This is useful when refactoring, changing APIs,
	# catching dangerous or incorrect code, and other similar tasks. It's not especially
	# easy to write a semantic patch but most users should only be concerned about running
	# the target and inspecting the results.
	#
	# Documentation (including examples, which are the most useful):
	# https://coccinelle.gitlabpages.inria.fr/website/docs/
	#
	# Run the "make cocci" target to just output what would be done, or "make cocci-inplace"
	# to apply the changes to the source tree.
	#
	# COCCI_FILES may be set on the command line, if you want to test just a single file
	# while it's under development. Otherwise, it is a list of all the files that are ready
	# to be run.
	#
	# ref-passed-variables-inited.cocci seems to be returning some false positives around
	# GHashTableIters, so it is disabled for the moment.
	COCCI_FILES ?= coccinelle/string-any-of.cocci \
	coccinelle/string-empty.cocci \
	coccinelle/string-null-matches.cocci \
	coccinelle/use-func.cocci


	dist_noinst_SCRIPTS = coccinelle/test/testrunner.sh
	EXTRA_DIST = README \
	gdbhelpers \
	$(COCCI_FILES) \
	coccinelle/ref-passed-variables-inited.cocci \
	coccinelle/rename-fn.cocci \
	coccinelle/test/ref-passed-variables-inited.input.c \
	coccinelle/test/ref-passed-variables-inited.output

	# Any file in this list is allowed to use any of the pcmk__ internal functions.
	# Coccinelle can use any transformation that depends on "internal" to rewrite
	# code to use the internal functions.
	MAY_USE_INTERNAL_FILES = $(shell find .. -path "../lib/.c" -o -path "../lib/private.h" -o -path "../tools/.c" -o -path "../daemons/.c" -o -path '../include/pcmki/h' -o -name 'internal.h')

	# And then any file in this list is public API, which may not use internal
	# functions. Thus, only those transformations that do not depend on "internal"
	# may be applied.
	OTHER_FILES = $(shell find ../include -name 'h' -a \! -name 'internal.h' -a \! -path '../include/pcmki/*')

	.PHONY: cocci
	cocci:
	-for cf in $(COCCI_FILES); do \
	for f in $(MAY_USE_INTERNAL_FILES); do \
	spatch $(_SPATCH_FLAGS) -D internal --very-quiet --local-includes --preprocess --sp-file $$cf $$f; \
	done ; \
	for f in $(OTHER_FILES); do \
	spatch $(_SPATCH_FLAGS) --very-quiet --local-includes --preprocess --sp-file $$cf $$f; \
	done ; \
	done

	.PHONY: cocci-inplace
	cocci-inplace:
	$(MAKE) $(AM_MAKEFLAGS) _SPATCH_FLAGS=--in-place cocci

	.PHONY: cocci-test
	cocci-test:
	for f in coccinelle/test/*.c; do \
	coccinelle/test/testrunner.sh $$f; \
	done

	#
	# Static analysis
	#

	## clang

	# See scan-build(1) for possible checkers (leave empty to use default set)
	CLANG_checkers ?=

	.PHONY: clang
	clang:
	OUT=$$(cd $(top_builddir) \
	&& scan-build $(CLANG_checkers:%=-enable-checker %) \
	$(MAKE) $(AM_MAKEFLAGS) CFLAGS="-std=c99 $(CFLAGS)" \
	clean all 2>&1); \
	REPORT=$$(echo "$$OUT" \
	\| sed -n -e "s/.'scan-view $.$'.*/\1/p"); \
	[ -z "$$REPORT" ] && echo "$$OUT" \|\| scan-view "$$REPORT"

	## coverity

	# Aggressiveness (low, medium, or high)
	COVLEVEL ?= low

	# Generated outputs
	COVERITY_DIR = $(abs_top_builddir)/coverity-$(TAG)
	COVTAR = $(abs_top_builddir)/$(PACKAGE)-coverity-$(TAG).tgz
	COVEMACS = $(abs_top_builddir)/$(TAG).coverity
	COVHTML = $(COVERITY_DIR)/output/errors

	# Coverity outputs are phony so they get rebuilt every invocation

	.PHONY: $(COVERITY_DIR)
	$(COVERITY_DIR): coverity-clean
	$(MAKE) $(AM_MAKEFLAGS) -C $(top_builddir) init core-clean
	$(AM_V_GEN)cd $(top_builddir) \
	&& cov-build --dir "$@" $(MAKE) $(AM_MAKEFLAGS) core

	# Public coverity instance

	.PHONY: $(COVTAR)
	$(COVTAR): $(COVERITY_DIR)
	$(AM_V_GEN)tar czf "$@" --transform="s@.*$(TAG)@cov-int@" "$<"

	.PHONY: coverity
	coverity: $(COVTAR)
	@echo "Now go to https://scan.coverity.com/users/sign_in and upload:"
	@echo " $(COVTAR)"
	@echo "then make clean at the top level"

	# Licensed coverity instance
	#
	# The prerequisites are a little hacky; rather than actually required, some
	# of them are designed so that things execute in the proper order (which is
	# not the same as GNU make's order-only prerequisites).

	.PHONY: coverity-analyze
	coverity-analyze: $(COVERITY_DIR)
	@echo ""
	@echo "Analyzing (waiting for coverity license if necessary) ..."
	cd $(top_builddir) && cov-analyze --dir "$<" --wait-for-license \
	--security --aggressiveness-level "$(COVLEVEL)"

	.PHONY: $(COVEMACS)
	$(COVEMACS): coverity-analyze
	$(AM_V_GEN)cd $(top_builddir) \
	&& cov-format-errors --dir "$(COVERITY_DIR)" --emacs-style > "$@"

	.PHONY: $(COVHTML)
	$(COVHTML): $(COVEMACS)
	$(AM_V_GEN)cd $(top_builddir) \
	&& cov-format-errors --dir "$(COVERITY_DIR)" --html-output "$@"

	.PHONY: coverity-corp
	coverity-corp: $(COVHTML)
	$(MAKE) $(AM_MAKEFLAGS) -C $(top_builddir) core-clean
	@echo "Done. See:"
	@echo " file://$(COVHTML)/index.html"
	@echo "When no longer needed, make coverity-clean"

	# Remove all outputs regardless of tag
	.PHONY: coverity-clean
	coverity-clean:
	-rm -rf "$(abs_builddir)"/coverity-* \
	"$(abs_builddir)"/$(PACKAGE)-coverity-*.tgz \
	"$(abs_builddir)"/*.coverity


	## cppcheck

	GLIB_CFLAGS ?= $(pkg-config --cflags glib-2.0)
	GLIB_INCL_DEF_CFLAGS = $(shell echo $(GLIB_CFLAGS) \
	\| tr ' ' '\n' \| grep '^-[IDU]' \| paste -d ' ')

	# Use CPPCHECK_ARGS to pass extra cppcheck options, e.g.:
	# --enable={warning,style,performance,portability,information,all}
	# --inconclusive --std=posix
	-# -DBUILD_PUBLIC_LIBPACEMAKER -DDEFAULT_CONCURRENT_FENCING_TRUE
	+# -DDEFAULT_CONCURRENT_FENCING_TRUE
	CPPCHECK_ARGS ?=

	CPPCHECK_DIRS = replace lib daemons tools
	CPPCHECK_OUT = $(abs_top_builddir)/cppcheck.out

	.PHONY: cppcheck
	cppcheck:
	cppcheck $(CPPCHECK_ARGS) -I $(top_srcdir)/include \
	--output-file=$(CPPCHECK_OUT) \
	--max-configs=30 --inline-suppr -q \
	--library=posix --library=gnu --library=gtk \
	$(GLIB_INCL_DEF_CFLAGS) -D__GNUC__ \
	$(foreach dir,$(CPPCHECK_DIRS),$(top_srcdir)/$(dir))
	@echo "Done: See $(CPPCHECK_OUT)"
	@echo "When no longer needed, make cppcheck-clean"

	.PHONY: cppcheck-clean
	cppcheck-clean:
	-rm -f "$(CPPCHECK_OUT)"

	#
	# Coverage/profiling
	#

	COVERAGE_DIR = $(top_builddir)/coverage

	# Check coverage of unit tests
	.PHONY: coverage
	coverage: coverage-partial-clean
	cd $(top_builddir) \
	&& $(MAKE) $(AM_MAKEFLAGS) \
	&& lcov --no-external --exclude='*_test.c' -c -i -d . \
	-o pacemaker_base.info \
	&& $(MAKE) $(AM_MAKEFLAGS) check \
	&& lcov --no-external --exclude='*_test.c' -c -d . \
	-o pacemaker_test.info \
	&& lcov -a pacemaker_base.info -a pacemaker_test.info \
	-o pacemaker_total.info \
	&& lcov --remove pacemaker_total.info -o pacemaker_filtered.info\
	"$(abs_top_builddir)/tools/*" \
	"$(abs_top_builddir)/daemons//" \
	"$(abs_top_builddir)/replace/*" \
	genhtml $(top_builddir)/pacemaker_filtered.info -o $(COVERAGE_DIR) -s -t "Pacemaker code coverage"

	# Check coverage of CLI regression tests
	.PHONY: coverage-cts
	coverage-cts: coverage-partial-clean
	cd $(top_builddir) \
	&& $(MAKE) $(AM_MAKEFLAGS) \
	&& lcov --no-external -c -i -d tools -o pacemaker_base.info \
	&& cts/cts-cli \
	&& lcov --no-external -c -d tools -o pacemaker_test.info \
	&& lcov -a pacemaker_base.info -a pacemaker_test.info \
	-o pacemaker_total.info
	genhtml $(top_builddir)/pacemaker_total.info -o $(COVERAGE_DIR) -s

	# Remove coverage-related files that aren't needed across runs
	.PHONY: coverage-partial-clean
	coverage-partial-clean:
	-rm -f $(top_builddir)/pacemaker_*.info
	-rm -rf $(COVERAGE_DIR)
	-find $(top_builddir) -name "*.gcda" -exec rm -f \{\} \;

	# This target removes all coverage-related files. It is only to be run when
	# done with coverage analysis and you are ready to go back to normal development,
	# starting with re-running ./configure. It is not to be run in between
	# "make coverage" runs.
	#
	# In particular, the *.gcno files are generated when the source is built.
	# Removing those files will break "make coverage" until the whole source tree
	# has been built and the *.gcno files generated again.
	.PHONY: coverage-clean
	coverage-clean: coverage-partial-clean
	-find $(top_builddir) -name "*.gcno" -exec rm -f \{\} \;

	#
	# indent cannot cope with all our exceptions and needs heavy manual editing
	#

	# indent target: Limit indent to these directories
	INDENT_DIRS ?= .

	# indent target: Extra options to pass to indent
	INDENT_OPTS ?=

	INDENT_IGNORE_PATHS = daemons/controld/controld_fsa.h
	INDENT_PACEMAKER_STYLE = --blank-lines-after-declarations \
	--blank-lines-after-procedures \
	--braces-after-func-def-line \
	--braces-on-if-line \
	--braces-on-struct-decl-line \
	--break-before-boolean-operator \
	--case-brace-indentation4 \
	--case-indentation4 \
	--comment-indentation0 \
	--continuation-indentation4 \
	--continue-at-parentheses \
	--cuddle-do-while \
	--cuddle-else \
	--declaration-comment-column0 \
	--declaration-indentation1 \
	--else-endif-column0 \
	--honour-newlines \
	--indent-label0 \
	--indent-level4 \
	--line-comments-indentation0 \
	--line-length80 \
	--no-blank-lines-after-commas \
	--no-comment-delimiters-on-blank-lines \
	--no-space-after-function-call-names \
	--no-space-after-parentheses \
	--no-tabs \
	--preprocessor-indentation2 \
	--procnames-start-lines \
	--space-after-cast \
	--start-left-side-of-comments \
	--swallow-optional-blank-lines \
	--tab-size8

	.PHONY: indent
	indent:
	VERSION_CONTROL=none \
	find $(INDENT_DIRS) -type f -name "*.[ch]" \
	$(INDENT_IGNORE_PATHS:%= ! -path '%') \
	-exec indent $(INDENT_PACEMAKER_STYLE) $(INDENT_OPTS) \{\} \;

	#
	# Check whether copyrights have been updated appropriately
	# (Set COMMIT to desired commit or commit range to check, defaulting to HEAD,
	# or set it empty to check uncommitted changes)
	#
	YEAR = $(shell date +%Y)
	MODIFIED_FILES = $(shell case "$(COMMIT)" in \
	[0-9a-f]*$(rparen) \
	git diff-tree --no-commit-id \
	--name-only "$(COMMIT)" -r ;; \
	*$(rparen) \
	cd "$(top_srcdir)"; \
	git ls-files --modified ;; \
	esac)

	.PHONY: copyright
	copyright:
	@cd "$(top_srcdir)" && for file in $(MODIFIED_FILES); do \
	if ! grep 'opyright .$(YEAR). Pacemaker' "$$file" \
	>/dev/null 2>&1; then \
	echo "$$file"; \
	fi; \
	done

	#
	# Scratch file for ad-hoc testing
	#

	EXTRA_PROGRAMS = scratch
	nodist_scratch_SOURCES = scratch.c
	scratch_LDADD = $(top_builddir)/lib/common/libcrmcommon.la

	.PHONY: clean-local
	clean-local: coverage-clean coverity-clean cppcheck-clean
	-rm -f $(EXTRA_PROGRAMS)
	diff --git a/include/crm/fencing/internal.h b/include/crm/fencing/internal.h
	index 5a5c013cb2..d46dab7ab9 100644
	--- a/include/crm/fencing/internal.h
	+++ b/include/crm/fencing/internal.h
	@@ -1,178 +1,199 @@
	/*
	* Copyright 2011-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#ifndef PCMK__CRM_FENCING_INTERNAL__H
	#define PCMK__CRM_FENCING_INTERNAL__H

	#include <glib.h>
	#include <crm/common/ipc.h>
	#include <crm/common/xml.h>
	#include <crm/common/output_internal.h>
	#include <crm/common/results_internal.h>
	#include <crm/stonith-ng.h>

	#ifdef __cplusplus
	extern "C" {
	#endif

	enum st_device_flags {
	st_device_supports_none = (0 << 0),
	st_device_supports_list = (1 << 0),
	st_device_supports_status = (1 << 1),
	st_device_supports_reboot = (1 << 2),
	st_device_supports_parameter_plug = (1 << 3),
	st_device_supports_parameter_port = (1 << 4),
	st_device_supports_on = (1 << 5),
	};

	#define stonith__set_device_flags(device_flags, device_id, flags_to_set) do { \
	device_flags = pcmk__set_flags_as(__func__, __LINE__, LOG_TRACE, \
	"Fence device", device_id, \
	(device_flags), (flags_to_set), \
	#flags_to_set); \
	} while (0)

	#define stonith__set_call_options(st_call_opts, call_for, flags_to_set) do { \
	st_call_opts = pcmk__set_flags_as(__func__, __LINE__, LOG_TRACE, \
	"Fencer call", (call_for), \
	(st_call_opts), (flags_to_set), \
	#flags_to_set); \
	} while (0)

	#define stonith__clear_call_options(st_call_opts, call_for, flags_to_clear) do { \
	st_call_opts = pcmk__clear_flags_as(__func__, __LINE__, LOG_TRACE, \
	"Fencer call", (call_for), \
	(st_call_opts), (flags_to_clear), \
	#flags_to_clear); \
	} while (0)

	struct stonith_action_s;
	typedef struct stonith_action_s stonith_action_t;

	stonith_action_t stonith__action_create(const char agent,
	const char *action_name,
	const char *target,
	uint32_t target_nodeid,
	int timeout_sec,
	GHashTable *device_args,
	GHashTable *port_map,
	const char *host_arg);
	void stonith__destroy_action(stonith_action_t *action);
	pcmk__action_result_t stonith__action_result(stonith_action_t action);
	int stonith__result2rc(const pcmk__action_result_t *result);
	void stonith__xe_set_result(xmlNode xml, const pcmk__action_result_t result);
	void stonith__xe_get_result(const xmlNode xml, pcmk__action_result_t result);
	xmlNode stonith__find_xe_with_result(xmlNode xml);

	int stonith__execute_async(stonith_action_t action, void userdata,
	void (*done) (int pid,
	const pcmk__action_result_t *result,
	void *user_data),
	void (fork_cb) (int pid, void user_data));

	int stonith__metadata_async(const char *agent, int timeout_sec,
	void (*callback)(int pid,
	const pcmk__action_result_t *result,
	void *user_data),
	void *user_data);

	xmlNode create_level_registration_xml(const char node, const char *pattern,
	const char attr, const char value,
	int level,
	const stonith_key_value_t *device_list);

	xmlNode create_device_registration_xml(const char id,
	enum stonith_namespace standard,
	const char *agent,
	const stonith_key_value_t *params,
	const char *rsc_provides);

	void stonith__register_messages(pcmk__output_t *out);

	GList stonith__parse_targets(const char hosts);

	const char stonith__later_succeeded(const stonith_history_t event,
	const stonith_history_t *top_history);
	stonith_history_t stonith__sort_history(stonith_history_t history);

	void stonith__device_parameter_flags(uint32_t *device_flags,
	const char *device_name,
	xmlNode *metadata);

	/* Only 1-9 is allowed for fencing topology levels,
	* however, 0 is used to unregister all levels in
	* unregister requests.
	*/
	# define ST__LEVEL_COUNT 10

	# define STONITH_ATTR_ACTION_OP "action"

	# define STONITH_OP_EXEC "st_execute"
	# define STONITH_OP_TIMEOUT_UPDATE "st_timeout_update"
	# define STONITH_OP_QUERY "st_query"
	# define STONITH_OP_FENCE "st_fence"
	# define STONITH_OP_RELAY "st_relay"
	# define STONITH_OP_DEVICE_ADD "st_device_register"
	# define STONITH_OP_DEVICE_DEL "st_device_remove"
	# define STONITH_OP_FENCE_HISTORY "st_fence_history"
	# define STONITH_OP_LEVEL_ADD "st_level_add"
	# define STONITH_OP_LEVEL_DEL "st_level_remove"
	# define STONITH_OP_NOTIFY "st_notify"
	# define STONITH_OP_POKE "poke"


	# define STONITH_WATCHDOG_AGENT "fence_watchdog"
	/* Don't change 2 below as it would break rolling upgrade */
	# define STONITH_WATCHDOG_AGENT_INTERNAL "#watchdog"
	# define STONITH_WATCHDOG_ID "watchdog"

	stonith_history_t stonith__first_matching_event(stonith_history_t history,
	bool (matching_fn)(stonith_history_t , void *),
	void *user_data);
	bool stonith__event_state_pending(stonith_history_t history, void user_data);
	bool stonith__event_state_eq(stonith_history_t history, void user_data);
	bool stonith__event_state_neq(stonith_history_t history, void user_data);

	int stonith__legacy2status(int rc);

	int stonith__exit_status(const stonith_callback_data_t *data);
	int stonith__execution_status(const stonith_callback_data_t *data);
	const char stonith__exit_reason(const stonith_callback_data_t data);

	int stonith__event_exit_status(const stonith_event_t *event);
	int stonith__event_execution_status(const stonith_event_t *event);
	const char stonith__event_exit_reason(const stonith_event_t event);
	char stonith__event_description(const stonith_event_t event);
	gchar stonith__history_description(const stonith_history_t event,
	bool full_history,
	const char *later_succeeded,
	uint32_t show_opts);

	/*!
	* \internal
	* \brief Is a fencing operation in pending state?
	*
	* \param[in] state State as enum op_state value
	*
	* \return A boolean
	*/
	static inline bool
	stonith__op_state_pending(enum op_state state)
	{
	return state != st_failed && state != st_done;
	}

	gboolean stonith__watchdog_fencing_enabled_for_node(const char *node);
	gboolean stonith__watchdog_fencing_enabled_for_node_api(stonith_t st, const char node);

	+/*!
	+ * \internal
	+ * \brief Validate a fencing configuration
	+ *
	+ * \param[in,out] st Fencer connection to use
	+ * \param[in] call_options Group of enum stonith_call_options
	+ * \param[in] rsc_id Resource to validate
	+ * \param[in] namespace_s Type of fence agent to search for
	+ * \param[in] agent Fence agent to validate
	+ * \param[in,out] params Fence device configuration parameters
	+ * \param[in] timeout_sec How long to wait for operation to complete
	+ * \param[in,out] output If non-NULL, where to store any agent output
	+ * \param[in,out] error_output If non-NULL, where to store agent error output
	+ *
	+ * \return Standard Pacemaker return code
	+ */
	+int stonith__validate(stonith_t st, int call_options, const char rsc_id,
	+ const char namespace_s, const char agent,
	+ GHashTable params, int timeout_sec, char *output,
	+ char **error_output);
	+
	#ifdef __cplusplus
	}
	#endif

	#endif // PCMK__CRM_FENCING_INTERNAL__H
	diff --git a/include/pacemaker.h b/include/pacemaker.h
	index ed52f8aa2d..90e251c52f 100644
	--- a/include/pacemaker.h
	+++ b/include/pacemaker.h
	@@ -1,715 +1,701 @@
	/*
	* Copyright 2019-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#ifndef PCMK__PACEMAKER__H
	# define PCMK__PACEMAKER__H

	# include <glib.h>
	# include <libxml/tree.h>
	# include <crm/common/scheduler.h>
	# include <crm/cib/cib_types.h>

	# include <crm/stonith-ng.h>

	#ifdef __cplusplus
	extern "C" {
	#endif

	/**
	* \file
	* \brief High Level API
	* \ingroup pacemaker
	*/


	/*!
	* \brief Modify operation of running a cluster simulation.
	*/
	enum pcmk_sim_flags {
	pcmk_sim_none = 0,
	pcmk_sim_all_actions = 1 << 0,
	pcmk_sim_show_pending = 1 << 1,
	pcmk_sim_process = 1 << 2,
	pcmk_sim_show_scores = 1 << 3,
	pcmk_sim_show_utilization = 1 << 4,
	pcmk_sim_simulate = 1 << 5,
	pcmk_sim_sanitized = 1 << 6,
	pcmk_sim_verbose = 1 << 7,
	};

	/*!
	* \brief Synthetic cluster events that can be injected into the cluster
	* for running simulations.
	*/
	typedef struct {
	/! A list of node names (gchar ) to simulate bringing online */
	GList *node_up;
	/! A list of node names (gchar ) to simulate bringing offline */
	GList *node_down;
	/! A list of node names (gchar ) to simulate failing */
	GList *node_fail;
	/! A list of operations (gchar ) to inject. The format of these strings
	* is described in the "Operation Specification" section of crm_simulate
	* help output.
	*/
	GList *op_inject;
	/! A list of operations (gchar ) that should return a given error code
	* if they fail. The format of these strings is described in the
	* "Operation Specification" section of crm_simulate help output.
	*/
	GList *op_fail;
	/! A list of tickets (gchar ) to simulate granting */
	GList *ticket_grant;
	/! A list of tickets (gchar ) to simulate revoking */
	GList *ticket_revoke;
	/! A list of tickets (gchar ) to simulate putting on standby */
	GList *ticket_standby;
	/! A list of tickets (gchar ) to simulate activating */
	GList *ticket_activate;
	/! Does the cluster have an active watchdog device? /
	char *watchdog;
	/! Does the cluster have quorum? /
	char *quorum;
	} pcmk_injections_t;

	/*!
	* \brief Get and output controller status
	*
	* \param[in,out] xml Destination for the result, as an XML tree
	* \param[in] node_name Name of node whose status is desired
	* (\p NULL for DC)
	* \param[in] message_timeout_ms How long to wait for a reply from the
	* controller API. If 0,
	* \p pcmk_ipc_dispatch_sync will be used.
	* Otherwise, \p pcmk_ipc_dispatch_poll will
	* be used.
	*
	* \return Standard Pacemaker return code
	*/
	int pcmk_controller_status(xmlNodePtr xml, const char node_name,
	unsigned int message_timeout_ms);

	/*!
	* \brief Get and output designated controller node name
	*
	* \param[in,out] xml Destination for the result, as an XML tree
	* \param[in] message_timeout_ms How long to wait for a reply from the
	* controller API. If 0,
	* \p pcmk_ipc_dispatch_sync will be used.
	* Otherwise, \p pcmk_ipc_dispatch_poll will
	* be used.
	*
	* \return Standard Pacemaker return code
	*/
	int pcmk_designated_controller(xmlNodePtr *xml,
	unsigned int message_timeout_ms);

	/*!
	* \brief Free a :pcmk_injections_t structure
	*
	* \param[in,out] injections The structure to be freed
	*/
	void pcmk_free_injections(pcmk_injections_t *injections);

	/*!
	* \brief Get and optionally output node info corresponding to a node ID from
	* the controller
	*
	* \param[in,out] xml Destination for the result, as an XML tree
	* \param[in,out] node_id ID of node whose name to get. If \p NULL
	* or 0, get the local node name. If not
	* \p NULL, store the true node ID here on
	* success.
	* \param[out] node_name If not \p NULL, where to store the node
	* name
	* \param[out] uuid If not \p NULL, where to store the node
	* UUID
	* \param[out] state If not \p NULL, where to store the
	* membership state
	* \param[out] is_remote If not \p NULL, where to store whether the
	* node is a Pacemaker Remote node
	* \param[out] have_quorum If not \p NULL, where to store whether the
	* node has quorum
	* \param[in] show_output Whether to output the node info
	* \param[in] message_timeout_ms How long to wait for a reply from the
	* controller API. If 0,
	* \p pcmk_ipc_dispatch_sync will be used.
	* Otherwise, \p pcmk_ipc_dispatch_poll will
	* be used.
	*
	* \return Standard Pacemaker return code
	*
	* \note The caller is responsible for freeing \p node_name, \p uuid, and
	* \p *state using \p free().
	*/
	int pcmk_query_node_info(xmlNodePtr xml, uint32_t node_id, char **node_name,
	char uuid, char state, bool *have_quorum,
	bool *is_remote, bool show_output,
	unsigned int message_timeout_ms);

	/*!
	* \brief Get the node name corresponding to a node ID from the controller
	*
	* \param[in,out] xml Destination for the result, as an XML tree
	* \param[in,out] node_id ID of node whose name to get (or 0 for the
	* local node)
	* \param[out] node_name If not \p NULL, where to store the node
	* name
	* \param[in] message_timeout_ms How long to wait for a reply from the
	* controller API. If 0,
	* \p pcmk_ipc_dispatch_sync will be used.
	* Otherwise, \p pcmk_ipc_dispatch_poll will
	* be used.
	*
	* \return Standard Pacemaker return code
	*
	* \note The caller is responsible for freeing \p *node_name using \p free().
	*/
	static inline int
	pcmk_query_node_name(xmlNodePtr xml, uint32_t node_id, char *node_name,
	unsigned int message_timeout_ms)
	{
	return pcmk_query_node_info(xml, &node_id, node_name, NULL, NULL, NULL,
	NULL, false, message_timeout_ms);
	}

	/*!
	* \brief Get and output \p pacemakerd status
	*
	* \param[in,out] xml Destination for the result, as an XML tree
	* \param[in] ipc_name IPC name for request
	* \param[in] message_timeout_ms How long to wait for a reply from the
	* \p pacemakerd API. If 0,
	* \p pcmk_ipc_dispatch_sync will be used.
	* Otherwise, \p pcmk_ipc_dispatch_poll will
	* be used.
	*
	* \return Standard Pacemaker return code
	*/
	int pcmk_pacemakerd_status(xmlNodePtr xml, const char ipc_name,
	unsigned int message_timeout_ms);

	/*!
	* \brief Remove a resource
	*
	* \param[in,out] xml Destination for the result, as an XML tree
	* \param[in] rsc_id Resource to remove
	* \param[in] rsc_type Type of the resource ("primitive", "group", etc.)
	*
	* \return Standard Pacemaker return code
	* \note This function will return \p pcmk_rc_ok if \p rsc_id doesn't exist
	* or if \p rsc_type is incorrect for \p rsc_id (deleting something
	* that doesn't exist always succeeds).
	*/
	int pcmk_resource_delete(xmlNodePtr xml, const char rsc_id, const char *rsc_type);

	/*!
	* \brief Calculate and output resource operation digests
	*
	* \param[out] xml Where to store XML with result
	* \param[in,out] rsc Resource to calculate digests for
	* \param[in] node Node whose operation history should be used
	* \param[in] overrides Hash table of configuration parameters to override
	*
	* \return Standard Pacemaker return code
	*/
	int pcmk_resource_digests(xmlNodePtr xml, pcmk_resource_t rsc,
	const pcmk_node_t node, GHashTable overrides);

	/*!
	* \brief Simulate a cluster's response to events
	*
	* This high-level function essentially implements crm_simulate(8). It operates
	* on an input CIB file and various lists of events that can be simulated. It
	* optionally writes out a variety of artifacts to show the results of the
	* simulation. Output can be modified with various flags.
	*
	* \param[in,out] xml The destination for the result, as an XML tree
	* \param[in,out] scheduler Scheduler data
	* \param[in] injections A structure containing cluster events
	* (node up/down, tickets, injected operations)
	* \param[in] flags A bitfield of :pcmk_sim_flags to modify
	* operation of the simulation
	* \param[in] section_opts Which portions of the cluster status output
	* should be displayed?
	* \param[in] use_date Date to set the cluster's time to (may be NULL)
	* \param[in] input_file The source CIB file, which may be overwritten by
	* this function (may be NULL)
	* \param[in] graph_file Where to write the XML-formatted transition graph
	* (may be NULL, in which case no file will be
	* written)
	* \param[in] dot_file Where to write the dot(1) formatted transition
	* graph (may be NULL, in which case no file will
	* be written)
	*
	* \return Standard Pacemaker return code
	*/
	int pcmk_simulate(xmlNodePtr xml, pcmk_scheduler_t scheduler,
	const pcmk_injections_t *injections, unsigned int flags,
	unsigned int section_opts, const char *use_date,
	const char input_file, const char graph_file,
	const char *dot_file);

	/*!
	* \brief Verify that a CIB is error-free or output errors and warnings
	*
	* This high-level function essentially implements crm_verify(8). It operates
	* on an input CIB file, which can be inputted through one of several ways. It
	* writes out XML-formatted output.
	*
	* \param[in,out] xml The destination for the result, as an XML tree
	* \param[in] cib_source Source of the CIB:
	* NULL -> use live cib, "-" -> stdin
	* "<..." -> xml str, otherwise -> xml file name
	*
	* \return Standard Pacemaker return code
	*/
	int pcmk_verify(xmlNodePtr xml, const char cib_source);

	/*!
	* \brief Get nodes list
	*
	* \param[in,out] xml The destination for the result, as an XML tree
	* \param[in] node_types Node type(s) to return (default: all)
	*
	* \return Standard Pacemaker return code
	*/
	int pcmk_list_nodes(xmlNodePtr xml, const char node_types);

	/*!
	* \brief Output cluster status formatted like `crm_mon --output-as=xml`
	*
	* \param[in,out] xml The destination for the result, as an XML tree
	*
	* \return Standard Pacemaker return code
	*/
	int pcmk_status(xmlNodePtr *xml);

	/*!
	* \brief Check whether each rule in a list is in effect
	*
	* \param[in,out] xml The destination for the result, as an XML tree
	* \param[in] input The CIB XML to check (if \c NULL, use current CIB)
	* \param[in] date Check whether the rule is in effect at this date and
	* time (if \c NULL, use current date and time)
	* \param[in] rule_ids The IDs of the rules to check, as a <tt>NULL</tt>-
	* terminated list.
	*
	* \return Standard Pacemaker return code
	*/
	int pcmk_check_rules(xmlNodePtr xml, xmlNodePtr input, const crm_time_t date,
	const char **rule_ids);

	/*!
	* \brief Check whether a given rule is in effect
	*
	* \param[in,out] xml The destination for the result, as an XML tree
	* \param[in] input The CIB XML to check (if \c NULL, use current CIB)
	* \param[in] date Check whether the rule is in effect at this date and
	* time (if \c NULL, use current date and time)
	* \param[in] rule_ids The ID of the rule to check
	*
	* \return Standard Pacemaker return code
	*/
	static inline int
	pcmk_check_rule(xmlNodePtr xml, xmlNodePtr input, const crm_time_t date,
	const char *rule_id)
	{
	const char *rule_ids[] = {rule_id, NULL};
	return pcmk_check_rules(xml, input, date, rule_ids);
	}

	/*!
	* \enum pcmk_rc_disp_flags
	* \brief Bit flags to control which fields of result code info are displayed
	*/
	enum pcmk_rc_disp_flags {
	pcmk_rc_disp_none = 0, //!< (Does nothing)
	pcmk_rc_disp_code = (1 << 0), //!< Display result code number
	pcmk_rc_disp_name = (1 << 1), //!< Display result code name
	pcmk_rc_disp_desc = (1 << 2), //!< Display result code description
	};

	/*!
	* \brief Display the name and/or description of a result code
	*
	* \param[in,out] xml The destination for the result, as an XML tree
	* \param[in] code The result code
	* \param[in] type Interpret \c code as this type of result code.
	* Supported values: \c pcmk_result_legacy,
	* \c pcmk_result_rc, \c pcmk_result_exitcode.
	* \param[in] flags Group of \c pcmk_rc_disp_flags
	*
	* \return Standard Pacemaker return code
	*/
	int pcmk_show_result_code(xmlNodePtr *xml, int code, enum pcmk_result_type type,
	uint32_t flags);

	/*!
	* \brief List all valid result codes in a particular family
	*
	* \param[in,out] xml The destination for the result, as an XML tree
	* \param[in] type The family of result codes to list. Supported
	* values: \c pcmk_result_legacy, \c pcmk_result_rc,
	* \c pcmk_result_exitcode.
	* \param[in] flags Group of \c pcmk_rc_disp_flags
	*
	* \return Standard Pacemaker return code
	*/
	int pcmk_list_result_codes(xmlNodePtr *xml, enum pcmk_result_type type,
	uint32_t flags);

	/*!
	* \brief List available providers for the given OCF agent
	*
	* \param[in,out] xml The destination for the result, as an XML tree
	* \param[in] agent_spec Resource agent name
	*
	* \return Standard Pacemaker return code
	*/
	int pcmk_list_alternatives(xmlNodePtr xml, const char agent_spec);

	/*!
	* \brief List all agents available for the named standard and/or provider
	*
	* \param[in,out] xml The destination for the result, as an XML tree
	* \param[in] agent_spec STD[:PROV]
	*
	* \return Standard Pacemaker return code
	*/
	int pcmk_list_agents(xmlNodePtr xml, char agent_spec);

	/*!
	* \brief List all available OCF providers for the given agent
	*
	* \param[in,out] xml The destination for the result, as an XML tree
	* \param[in] agent_spec Resource agent name
	*
	* \return Standard Pacemaker return code
	*/
	int pcmk_list_providers(xmlNodePtr xml, const char agent_spec);

	/*!
	* \brief List all available resource agent standards
	*
	* \param[in,out] xml The destination for the result, as an XML tree
	*
	* \return Standard Pacemaker return code
	*/
	int pcmk_list_standards(xmlNodePtr *xml);

	/*!
	* \brief List all available cluster options
	*
	* These are options that affect the entire cluster.
	*
	* \param[in,out] xml The destination for the result, as an XML tree
	* \param[in] all If \c true, include advanced and deprecated options
	* (currently always treated as true)
	*
	* \return Standard Pacemaker return code
	*/
	int pcmk_list_cluster_options(xmlNode **xml, bool all);

	/*!
	* \brief List common fencing resource parameters
	*
	* These are parameters that are available for all fencing resources, regardless
	* of type. They are processed by Pacemaker, rather than by the fence agent or
	* the fencing library.
	*
	* \param[in,out] xml The destination for the result, as an XML tree
	* \param[in] all If \c true, include advanced and deprecated options
	* (currently always treated as true)
	*
	* \return Standard Pacemaker return code
	*/
	int pcmk_list_fencing_params(xmlNode **xml, bool all);

	/*!
	* \internal
	* \brief List meta-attributes applicable to primitive resources as OCF-like XML
	*
	* \param[in,out] out Output object
	* \param[in] all If \c true, include advanced and deprecated options (this
	* is always treated as true for XML output objects)
	*
	* \return Standard Pacemaker return code
	*/
	int pcmk_list_primitive_meta(xmlNode **xml, bool all);

	/*!
	* \brief Return constraints that apply to the given ticket
	*
	* \param[in,out] xml The destination for the result, as an XML tree
	* \param[in] ticket_id Ticket to find constraint for, or \c NULL for
	* all ticket constraints
	*
	* \return Standard Pacemaker return code
	*/
	int pcmk_ticket_constraints(xmlNodePtr xml, const char ticket_id);


	/*!
	* \brief Delete a ticket's state from the local cluster site
	*
	* \param[in,out] xml The destination for the result, as an XML tree
	* \param[in] ticket_id Ticket to delete
	* \param[in] force If \c true, delete the ticket even if it has
	* been granted
	*
	* \return Standard Pacemaker return code
	*/
	int pcmk_ticket_delete(xmlNodePtr xml, const char ticket_id, bool force);

	/*!
	* \brief Return the value of a ticket's attribute
	*
	* \param[in,out] xml The destination for the result, as an XML tree
	* \param[in] ticket_id Ticket to find attribute value for
	* \param[in] attr_name Attribute's name to find value for
	* \param[in] attr_default If either the ticket or the attribute do not
	* exist, use this as the value in \p xml
	*
	* \return Standard Pacemaker return code
	*/
	int pcmk_ticket_get_attr(xmlNodePtr xml, const char ticket_id,
	const char attr_name, const char attr_default);

	/*!
	* \brief Return information about the given ticket
	*
	* \param[in,out] xml The destination for the result, as an XML tree
	* \param[in] ticket_id Ticket to find info value for, or \c NULL for
	* all tickets
	*
	* \return Standard Pacemaker return code
	*/
	int pcmk_ticket_info(xmlNodePtr xml, const char ticket_id);

	/*!
	* \brief Remove the given attribute(s) from a ticket
	*
	* \param[in,out] xml The destination for the result, as an XML tree
	* \param[in] ticket_id Ticket to remove attributes from
	* \param[in] attr_delete A list of attribute names
	* \param[in] force Attempting to remove the granted attribute of
	* \p ticket_id will cause this function to return
	* \c EACCES unless \p force is set to \c true
	*
	* \return Standard Pacemaker return code
	*/
	int pcmk_ticket_remove_attr(xmlNodePtr xml, const char ticket_id, GList *attr_delete,
	bool force);

	/*!
	* \brief Set the given attribute(s) on a ticket
	*
	* \param[in,out] xml The destination for the result, as an XML tree
	* \param[in] ticket_id Ticket to set attributes on
	* \param[in] attr_set A hash table of attributes, where keys are the
	* attribute names and the values are the attribute
	* values
	* \param[in] force Attempting to change the granted status of
	* \p ticket_id will cause this function to return
	* \c EACCES unless \p force is set to \c true
	*
	* \return Standard Pacemaker return code
	*
	* \note If no \p ticket_id attribute exists but \p attr_set is non-NULL, the
	* ticket will be created with the given attributes.
	*/
	int pcmk_ticket_set_attr(xmlNodePtr xml, const char ticket_id, GHashTable *attr_set,
	bool force);

	/*!
	* \brief Return a ticket's state XML
	*
	* \param[in,out] xml The destination for the result, as an XML tree
	* \param[in] ticket_id Ticket to find state for, or \c NULL for all
	* tickets
	*
	* \return Standard Pacemaker return code
	*
	* \note If \p ticket_id is not \c NULL and more than one ticket exists with
	* that ID, this function returns \c pcmk_rc_duplicate_id.
	*/
	int pcmk_ticket_state(xmlNodePtr xml, const char ticket_id);

	-#ifdef BUILD_PUBLIC_LIBPACEMAKER
	-
	/*!
	* \brief Ask the cluster to perform fencing
	*
	- * \param[in,out] st A connection to the fencer API
	+ * \param[in,out] xml The destination for the result, as an XML tree
	* \param[in] target The node that should be fenced
	* \param[in] action The fencing action (on, off, reboot) to perform
	* \param[in] name Who requested the fence action?
	* \param[in] timeout How long to wait for operation to complete (in ms)
	* \param[in] tolerance If a successful action for \p target happened within
	* this many ms, return 0 without performing the action
	* again
	* \param[in] delay Apply this delay (in milliseconds) before initiating
	* fencing action (-1 applies no delay and also
	* disables any fencing delay from pcmk_delay_base and
	* pcmk_delay_max)
	* \param[out] reason If not NULL, where to put descriptive failure reason
	*
	* \return Standard Pacemaker return code
	* \note If \p reason is not NULL, the caller is responsible for freeing its
	* returned value.
	*/
	-int pcmk_request_fencing(stonith_t st, const char target, const char *action,
	+int pcmk_request_fencing(xmlNodePtr xml, const char target, const char *action,
	const char *name, unsigned int timeout,
	unsigned int tolerance, int delay, char **reason);

	/*!
	* \brief List the fencing operations that have occurred for a specific node
	*
	* \note If \p xml is not NULL, it will be freed first and the previous
	* contents lost.
	*
	* \param[in,out] xml The destination for the result, as an XML tree
	- * \param[in,out] st A connection to the fencer API
	* \param[in] target The node to get history for
	* \param[in] timeout How long to wait for operation to complete (in ms)
	* \param[in] quiet Suppress most output
	* \param[in] verbose Include additional output
	* \param[in] broadcast Gather fencing history from all nodes
	* \param[in] cleanup Clean up fencing history after listing
	*
	* \return Standard Pacemaker return code
	*/
	-int pcmk_fence_history(xmlNodePtr xml, stonith_t st, const char *target,
	- unsigned int timeout, bool quiet, int verbose,
	- bool broadcast, bool cleanup);
	+int pcmk_fence_history(xmlNodePtr xml, const char target, unsigned int timeout,
	+ bool quiet, int verbose, bool broadcast, bool cleanup);

	/*!
	* \brief List all installed fence agents
	*
	* \param[in,out] xml The destination for the result, as an XML tree (if
	* not NULL, previous contents will be freed and lost)
	- * \param[in,out] st A connection to the fencer API
	* \param[in] timeout How long to wait for operation to complete (in ms)
	*
	* \return Standard Pacemaker return code
	*/
	-int pcmk_fence_installed(xmlNodePtr xml, stonith_t st, unsigned int timeout);
	+int pcmk_fence_installed(xmlNodePtr *xml, unsigned int timeout);

	/*!
	* \brief When was a device last fenced?
	*
	* \param[in,out] xml The destination for the result, as an XML tree (if
	* not NULL, previous contents will be freed and lost)
	* \param[in] target The node that was fenced
	* \param[in] as_nodeid If true, \p target has node ID rather than name
	*
	* \return Standard Pacemaker return code
	*/
	int pcmk_fence_last(xmlNodePtr xml, const char target, bool as_nodeid);

	/*!
	* \brief List nodes that can be fenced
	*
	* \param[in,out] xml The destination for the result, as an XML tree (if
	* not NULL, previous contents will be freed and lost)
	- * \param[in,out] st A connection to the fencer API
	* \param[in] device_id Resource ID of fence device to check
	* \param[in] timeout How long to wait for operation to complete (in ms)
	*
	* \return Standard Pacemaker return code
	*/
	-int pcmk_fence_list_targets(xmlNodePtr xml, stonith_t st,
	- const char *device_id, unsigned int timeout);
	+int pcmk_fence_list_targets(xmlNodePtr xml, const char device_id,
	+ unsigned int timeout);

	/*!
	* \brief Get metadata for a fence agent
	*
	* \note If \p xml is not NULL, it will be freed first and the previous
	* contents lost.
	*
	* \param[in,out] xml The destination for the result, as an XML tree (if
	* not NULL, previous contents will be freed and lost)
	- * \param[in,out] st A connection to the fencer API
	* \param[in] agent The fence agent to get metadata for
	* \param[in] timeout How long to wait for operation to complete (in ms)
	*
	* \return Standard Pacemaker return code
	*/
	-int pcmk_fence_metadata(xmlNodePtr xml, stonith_t st, const char *agent,
	- unsigned int timeout);
	+int pcmk_fence_metadata(xmlNodePtr xml, const char agent, unsigned int timeout);

	/*!
	* \brief List registered fence devices
	*
	* \param[in,out] xml The destination for the result, as an XML tree (if
	* not NULL, previous contents will be freed and lost)
	- * \param[in,out] st A connection to the fencer API
	* \param[in] target If not NULL, return only devices that can fence this
	* \param[in] timeout How long to wait for operation to complete (in ms)
	*
	* \return Standard Pacemaker return code
	*/
	-int pcmk_fence_registered(xmlNodePtr xml, stonith_t st, const char *target,
	- unsigned int timeout);
	+int pcmk_fence_registered(xmlNodePtr xml, const char target, unsigned int timeout);

	/*!
	* \brief Register a fencing topology level
	*
	- * \param[in,out] st A connection to the fencer API
	+ * \param[in,out] xml The destination for the result, as an XML tree (if
	+ * not NULL, previous contents will be freed and lost)
	* \param[in] target What fencing level targets (as "name=value" to
	* target by given node attribute, or "@pattern" to
	* target by node name pattern, or a node name)
	* \param[in] fence_level Index number of level to add
	- * \param[in] devices Devices to use in level
	+ * \param[in] devices Devices to use in level as a list of char *
	*
	* \return Standard Pacemaker return code
	*/
	-int pcmk_fence_register_level(stonith_t st, const char target,
	- int fence_level,
	- const stonith_key_value_t *devices);
	+int pcmk_fence_register_level(xmlNodePtr xml, const char target, int fence_level,
	+ GList *devices);

	/*!
	* \brief Unregister a fencing topology level
	*
	- * \param[in,out] st A connection to the fencer API
	+ * \param[in,out] xml The destination for the result, as an XML tree (if
	+ * not NULL, previous contents will be freed and lost)
	* \param[in] target What fencing level targets (as "name=value" to
	* target by given node attribute, or "@pattern" to
	* target by node name pattern, or a node name)
	* \param[in] fence_level Index number of level to remove
	*
	* \return Standard Pacemaker return code
	*/
	-int pcmk_fence_unregister_level(stonith_t st, const char target,
	- int fence_level);
	+int pcmk_fence_unregister_level(xmlNodePtr xml, const char target, int fence_level);

	/*!
	* \brief Validate a fence device configuration
	*
	* \param[in,out] xml The destination for the result, as an XML tree (if
	* not NULL, previous contents will be freed and lost)
	- * \param[in,out] st A connection to the fencer API
	* \param[in] agent The agent to validate (for example, "fence_xvm")
	* \param[in] id Fence device ID (may be NULL)
	* \param[in] params Fence device configuration parameters
	* \param[in] timeout How long to wait for operation to complete (in ms)
	*
	* \return Standard Pacemaker return code
	*/
	-int pcmk_fence_validate(xmlNodePtr xml, stonith_t st, const char *agent,
	- const char id, const stonith_key_value_t params,
	- unsigned int timeout);
	-
	-#endif
	+int pcmk_fence_validate(xmlNodePtr xml, const char agent, const char *id,
	+ GHashTable *params, unsigned int timeout);

	#ifdef __cplusplus
	}
	#endif

	#endif
	diff --git a/include/pcmki/pcmki_fence.h b/include/pcmki/pcmki_fence.h
	index 83b5afde11..cfcc4333c5 100644
	--- a/include/pcmki/pcmki_fence.h
	+++ b/include/pcmki/pcmki_fence.h
	@@ -1,259 +1,257 @@
	/*
	* Copyright 2019-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/
	#ifndef PCMK__PCMKI_PCMKI_FENCE__H
	#define PCMK__PCMKI_PCMKI_FENCE__H

	#include <crm/stonith-ng.h>
	#include <crm/common/output_internal.h>

	#ifdef __cplusplus
	extern "C" {
	#endif

	/*!
	* \brief Control how much of the fencing history is output.
	*/
	enum pcmk__fence_history {
	pcmk__fence_history_none,
	pcmk__fence_history_reduced,
	pcmk__fence_history_full
	};

	/*!
	* \brief Ask the cluster to perform fencing
	*
	* \note This is the internal version of pcmk_request_fencing(). External users
	* of the pacemaker API should use that function instead.
	*
	* \param[in,out] st A connection to the fencer API
	* \param[in] target The node that should be fenced
	* \param[in] action The fencing action (on, off, reboot) to perform
	* \param[in] name Who requested the fence action?
	* \param[in] timeout How long to wait for operation to complete (in ms)
	* \param[in] tolerance If a successful action for \p target happened within
	* this many milliseconds, return success without
	* performing the action again
	* \param[in] delay Apply this delay (in milliseconds) before initiating
	* fencing action (a value of -1 applies no delay and
	* disables any fencing delay from pcmk_delay_base and
	* pcmk_delay_max)
	* \param[out] reason If not NULL, where to put descriptive failure reason
	*
	* \return Standard Pacemaker return code
	* \note If \p reason is not NULL, the caller is responsible for freeing its
	* returned value.
	* \todo delay is eventually used with g_timeout_add() and should be guint
	*/
	int pcmk__request_fencing(stonith_t st, const char target, const char *action,
	const char *name, unsigned int timeout,
	unsigned int tolerance, int delay, char **reason);

	/*!
	* \brief List the fencing operations that have occurred for a specific node
	*
	* \note This is the internal version of pcmk_fence_history(). External users
	* of the pacemaker API should use that function instead.
	*
	* \note \p out should be initialized with pcmk__output_new() before calling this
	* function and destroyed with out->finish and pcmk__output_free() before
	* reusing it with any other functions in this library.
	*
	* \param[in,out] out The output functions structure
	* \param[in,out] st A connection to the fencer API
	* \param[in] target The node to get history for
	* \param[in] timeout How long to wait for operation to complete (in ms)
	* \param[in] verbose Include additional output
	* \param[in] broadcast Gather fencing history from all nodes
	* \param[in] cleanup Clean up fencing history after listing
	*
	* \return Standard Pacemaker return code
	*/
	int pcmk__fence_history(pcmk__output_t out, stonith_t st, const char *target,
	unsigned int timeout, int verbose, bool broadcast,
	bool cleanup);

	/*!
	* \brief List all installed fence agents
	*
	* \note This is the internal version of pcmk_fence_installed(). External users
	* of the pacemaker API should use that function instead.
	*
	* \note \p out should be initialized with pcmk__output_new() before calling this
	* function and destroyed with out->finish and pcmk__output_free() before
	* reusing it with any other functions in this library.
	*
	* \param[in,out] out The output functions structure
	* \param[in,out] st A connection to the fencer API
	* \param[in] timeout How long to wait for the operation to complete (in ms)
	*
	* \return Standard Pacemaker return code
	*/
	int pcmk__fence_installed(pcmk__output_t out, stonith_t st, unsigned int timeout);

	/*!
	* \brief When was a device last fenced?
	*
	* \note This is the internal version of pcmk_fence_last(). External users
	* of the pacemaker API should use that function instead.
	*
	* \note \p out should be initialized with pcmk__output_new() before calling this
	* function and destroyed with out->finish and pcmk__output_free() before
	* reusing it with any other functions in this library.
	*
	* \param[in,out] out The output functions structure.
	* \param[in] target The node that was fenced.
	* \param[in] as_nodeid
	*
	* \return Standard Pacemaker return code
	*/
	int pcmk__fence_last(pcmk__output_t out, const char target, bool as_nodeid);

	/*!
	* \brief List nodes that can be fenced
	*
	* \note This is the internal version of pcmk_fence_list_targets(). External users
	* of the pacemaker API should use that function instead.
	*
	* \note \p out should be initialized with pcmk__output_new() before calling this
	* function and destroyed with out->finish and pcmk__output_free() before
	* reusing it with any other functions in this library.
	*
	* \param[in,out] out The output functions structure
	* \param[in,out] st A connection to the fencer API
	* \param[in] device_id Resource ID of fence device to check
	* \param[in] timeout How long to wait for operation to complete (in ms)
	*
	* \return Standard Pacemaker return code
	*/
	int pcmk__fence_list_targets(pcmk__output_t out, stonith_t st,
	const char *device_id, unsigned int timeout);

	/*!
	* \brief Get metadata for a fence agent
	*
	* \note This is the internal version of pcmk_fence_metadata(). External users
	* of the pacemaker API should use that function instead.
	*
	* \note \p out should be initialized with pcmk__output_new() before calling this
	* function and destroyed with out->finish and pcmk__output_free() before
	* reusing it with any other functions in this library.
	*
	* \param[in,out] out The output functions structure
	* \param[in,out] st A connection to the fencer API
	* \param[in] agent The fence agent to get metadata for
	* \param[in] timeout How long to wait for the operation to complete (in ms)
	*
	* \return Standard Pacemaker return code
	*/
	int pcmk__fence_metadata(pcmk__output_t out, stonith_t st, const char *agent,
	unsigned int timeout);

	/*!
	* \brief List registered fence devices
	*
	* \note This is the internal version of pcmk_fence_metadata(). External users
	* of the pacemaker API should use that function instead.
	*
	* \note \p out should be initialized with pcmk__output_new() before calling this
	* function and destroyed with out->finish and pcmk__output_free() before
	* reusing it with any other functions in this library.
	*
	* \param[in,out] out The output functions structure
	* \param[in,out] st A connection to the fencer API
	* \param[in] target If not NULL, return only devices that can fence this
	* \param[in] timeout How long to wait for the operation to complete (in ms)
	*
	* \return Standard Pacemaker return code
	*/
	int pcmk__fence_registered(pcmk__output_t out, stonith_t st,
	const char *target, unsigned int timeout);

	/*!
	* \brief Register a fencing level for a specific node, node regex, or attribute
	*
	* \note This is the internal version of pcmk_fence_register_level(). External users
	* of the pacemaker API should use that function instead.
	*
	* \p target can take three different forms:
	* - name=value, in which case \p target is an attribute.
	* - @pattern, in which case \p target is a node regex.
	* - Otherwise, \p target is a node name.
	*
	* \param[in,out] st A connection to the fencer API
	* \param[in] target The object to register a fencing level for
	* \param[in] fence_level Index number of level to add
	- * \param[in] devices Devices to use in level
	+ * \param[in] devices Devices to use in level as a list of char *
	*
	* \return Standard Pacemaker return code
	*/
	int pcmk__fence_register_level(stonith_t st, const char target,
	- int fence_level,
	- const stonith_key_value_t *devices);
	+ int fence_level, GList *devices);

	/*!
	* \brief Unregister a fencing level for specific node, node regex, or attribute
	*
	* \note This is the internal version of pcmk_fence_unregister_level(). External users
	* of the pacemaker API should use that function instead.
	*
	* \p target can take three different forms:
	* - name=value, in which case \p target is an attribute.
	* - @pattern, in which case \p target is a node regex.
	* - Otherwise, \p target is a node name.
	*
	* \param[in,out] st A connection to the fencer API
	* \param[in] target The object to unregister a fencing level for
	* \param[in] fence_level Index number of level to remove
	*
	* \return Standard Pacemaker return code
	*/
	int pcmk__fence_unregister_level(stonith_t st, const char target,
	int fence_level);

	/*!
	* \brief Validate a fence device configuration
	*
	* \note This is the internal version of pcmk_stonith_validate(). External users
	* of the pacemaker API should use that function instead.
	*
	* \note \p out should be initialized with pcmk__output_new() before calling this
	* function and destroyed with out->finish and pcmk__output_free() before
	* reusing it with any other functions in this library.
	*
	* \param[in,out] out The output functions structure
	* \param[in,out] st A connection to the fencer API
	* \param[in] agent The agent to validate (for example, "fence_xvm")
	* \param[in] id Fence device ID (may be NULL)
	* \param[in] params Fence device configuration parameters
	* \param[in] timeout How long to wait for the operation to complete (in ms)
	*
	* \return Standard Pacemaker return code
	*/
	int pcmk__fence_validate(pcmk__output_t out, stonith_t st, const char *agent,
	- const char id, const stonith_key_value_t params,
	- unsigned int timeout);
	+ const char id, GHashTable params, unsigned int timeout);

	/*!
	* \brief Fetch fencing history, optionally reducing it
	*
	* \param[in,out] st A connection to the fencer API
	* \param[out] stonith_history Destination for storing the history
	* \param[in] fence_history How much of the fencing history to display
	*
	* \return Standard Pacemaker return code
	*/
	int
	pcmk__get_fencing_history(stonith_t st, stonith_history_t *stonith_history,
	enum pcmk__fence_history fence_history);

	#ifdef __cplusplus
	}
	#endif

	#endif // PCMK__PCMKI_PCMKI_FENCE__H
	diff --git a/lib/fencing/st_client.c b/lib/fencing/st_client.c
	index 8236e62ccb..6c24974e2f 100644
	--- a/lib/fencing/st_client.c
	+++ b/lib/fencing/st_client.c
	@@ -1,2703 +1,2728 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdlib.h>
	#include <stdio.h>
	#include <stdbool.h>
	#include <string.h>
	#include <ctype.h>
	#include <inttypes.h>
	#include <sys/types.h>
	#include <glib.h>

	#include <crm/crm.h>
	#include <crm/stonith-ng.h>
	#include <crm/fencing/internal.h>
	#include <crm/common/xml.h>

	#include <crm/common/mainloop.h>

	#include "fencing_private.h"

	CRM_TRACE_INIT_DATA(stonith);

	// Used as stonith_t:st_private
	typedef struct stonith_private_s {
	char *token;
	crm_ipc_t *ipc;
	mainloop_io_t *source;
	GHashTable *stonith_op_callback_table;
	GList *notify_list;
	int notify_refcnt;
	bool notify_deletes;

	void (op_callback) (stonith_t st, stonith_callback_data_t * data);

	} stonith_private_t;

	// Used as stonith_event_t:opaque
	struct event_private {
	pcmk__action_result_t result;
	};

	typedef struct stonith_notify_client_s {
	const char *event;
	const char obj_id; / implement one day */
	const char obj_type; / implement one day */
	void (notify) (stonith_t st, stonith_event_t * e);
	bool delete;

	} stonith_notify_client_t;

	typedef struct stonith_callback_client_s {
	void (callback) (stonith_t st, stonith_callback_data_t * data);
	const char *id;
	void *user_data;
	gboolean only_success;
	gboolean allow_timeout_updates;
	struct timer_rec_s *timer;

	} stonith_callback_client_t;

	struct notify_blob_s {
	stonith_t *stonith;
	xmlNode *xml;
	};

	struct timer_rec_s {
	int call_id;
	int timeout;
	guint ref;
	stonith_t *stonith;
	};

	typedef int (stonith_op_t) (const char , int, const char , xmlNode ,
	xmlNode , xmlNode , xmlNode , xmlNode );

	bool stonith_dispatch(stonith_t * st);
	xmlNode stonith_create_op(int call_id, const char token, const char op, xmlNode data,
	int call_options);
	static int stonith_send_command(stonith_t stonith, const char op,
	xmlNode data, xmlNode *output_data,
	int call_options, int timeout);

	static void stonith_connection_destroy(gpointer user_data);
	static void stonith_send_notification(gpointer data, gpointer user_data);
	static int stonith_api_del_notification(stonith_t *stonith,
	const char *event);
	/*!
	* \brief Get agent namespace by name
	*
	* \param[in] namespace_s Name of namespace as string
	*
	* \return Namespace as enum value
	*/
	enum stonith_namespace
	stonith_text2namespace(const char *namespace_s)
	{
	if (pcmk__str_eq(namespace_s, "any", pcmk__str_null_matches)) {
	return st_namespace_any;

	} else if (!strcmp(namespace_s, "redhat")
	\|\| !strcmp(namespace_s, "stonith-ng")) {
	return st_namespace_rhcs;

	} else if (!strcmp(namespace_s, "internal")) {
	return st_namespace_internal;

	} else if (!strcmp(namespace_s, "heartbeat")) {
	return st_namespace_lha;
	}
	return st_namespace_invalid;
	}

	/*!
	* \brief Get agent namespace name
	*
	* \param[in] namespace Namespace as enum value
	*
	* \return Namespace name as string
	*/
	const char *
	stonith_namespace2text(enum stonith_namespace st_namespace)
	{
	switch (st_namespace) {
	case st_namespace_any: return "any";
	case st_namespace_rhcs: return "stonith-ng";
	case st_namespace_internal: return "internal";
	case st_namespace_lha: return "heartbeat";
	default: break;
	}
	return "unsupported";
	}

	/*!
	* \brief Determine namespace of a fence agent
	*
	* \param[in] agent Fence agent type
	* \param[in] namespace_s Name of agent namespace as string, if known
	*
	* \return Namespace of specified agent, as enum value
	*/
	enum stonith_namespace
	stonith_get_namespace(const char agent, const char namespace_s)
	{
	if (pcmk__str_eq(namespace_s, "internal", pcmk__str_none)) {
	return st_namespace_internal;
	}

	if (stonith__agent_is_rhcs(agent)) {
	return st_namespace_rhcs;
	}

	#if HAVE_STONITH_STONITH_H
	if (stonith__agent_is_lha(agent)) {
	return st_namespace_lha;
	}
	#endif

	crm_err("Unknown fence agent: %s", agent);
	return st_namespace_invalid;
	}

	gboolean
	stonith__watchdog_fencing_enabled_for_node_api(stonith_t st, const char node)
	{
	gboolean rv = FALSE;
	stonith_t *stonith_api = st?st:stonith_api_new();
	char *list = NULL;

	if(stonith_api) {
	if (stonith_api->state == stonith_disconnected) {
	int rc = stonith_api->cmds->connect(stonith_api, "stonith-api", NULL);

	if (rc != pcmk_ok) {
	crm_err("Failed connecting to Stonith-API for watchdog-fencing-query.");
	}
	}

	if (stonith_api->state != stonith_disconnected) {
	/* caveat!!!
	* this might fail when when stonithd is just updating the device-list
	* probably something we should fix as well for other api-calls */
	int rc = stonith_api->cmds->list(stonith_api, st_opt_sync_call, STONITH_WATCHDOG_ID, &list, 0);
	if ((rc != pcmk_ok) \|\| (list == NULL)) {
	/* due to the race described above it can happen that
	* we drop in here - so as not to make remote nodes
	* panic on that answer
	*/
	if (rc == -ENODEV) {
	crm_notice("Cluster does not have watchdog fencing device");
	} else {
	crm_warn("Could not check for watchdog fencing device: %s",
	pcmk_strerror(rc));
	}
	} else if (list[0] == '\0') {
	rv = TRUE;
	} else {
	GList *targets = stonith__parse_targets(list);
	rv = pcmk__str_in_list(node, targets, pcmk__str_casei);
	g_list_free_full(targets, free);
	}
	free(list);
	if (!st) {
	/* if we're provided the api we still might have done the
	* connection - but let's assume the caller won't bother
	*/
	stonith_api->cmds->disconnect(stonith_api);
	}
	}

	if (!st) {
	stonith_api_delete(stonith_api);
	}
	} else {
	crm_err("Stonith-API for watchdog-fencing-query couldn't be created.");
	}
	crm_trace("Pacemaker assumes node %s %sto do watchdog-fencing.",
	node, rv?"":"not ");
	return rv;
	}

	gboolean
	stonith__watchdog_fencing_enabled_for_node(const char *node)
	{
	return stonith__watchdog_fencing_enabled_for_node_api(NULL, node);
	}

	/* when cycling through the list we don't want to delete items
	so just mark them and when we know nobody is using the list
	loop over it to remove the marked items
	*/
	static void
	foreach_notify_entry (stonith_private_t *private,
	GFunc func,
	gpointer user_data)
	{
	private->notify_refcnt++;
	g_list_foreach(private->notify_list, func, user_data);
	private->notify_refcnt--;
	if ((private->notify_refcnt == 0) &&
	private->notify_deletes) {
	GList *list_item = private->notify_list;

	private->notify_deletes = FALSE;
	while (list_item != NULL)
	{
	stonith_notify_client_t *list_client = list_item->data;
	GList *next = g_list_next(list_item);

	if (list_client->delete) {
	free(list_client);
	private->notify_list =
	g_list_delete_link(private->notify_list, list_item);
	}
	list_item = next;
	}
	}
	}

	static void
	stonith_connection_destroy(gpointer user_data)
	{
	stonith_t *stonith = user_data;
	stonith_private_t *native = NULL;
	struct notify_blob_s blob;

	crm_trace("Sending destroyed notification");
	blob.stonith = stonith;
	blob.xml = pcmk__xe_create(NULL, PCMK__XE_NOTIFY);

	native = stonith->st_private;
	native->ipc = NULL;
	native->source = NULL;

	free(native->token); native->token = NULL;
	stonith->state = stonith_disconnected;
	crm_xml_add(blob.xml, PCMK__XA_T, PCMK__VALUE_ST_NOTIFY);
	crm_xml_add(blob.xml, PCMK__XA_SUBT, PCMK__VALUE_ST_NOTIFY_DISCONNECT);

	foreach_notify_entry(native, stonith_send_notification, &blob);
	pcmk__xml_free(blob.xml);
	}

	xmlNode *
	create_device_registration_xml(const char *id, enum stonith_namespace standard,
	const char *agent,
	const stonith_key_value_t *params,
	const char *rsc_provides)
	{
	xmlNode *data = pcmk__xe_create(NULL, PCMK__XE_ST_DEVICE_ID);
	xmlNode *args = pcmk__xe_create(data, PCMK__XE_ATTRIBUTES);

	#if HAVE_STONITH_STONITH_H
	if (standard == st_namespace_any) {
	standard = stonith_get_namespace(agent, NULL);
	}
	if (standard == st_namespace_lha) {
	hash2field((gpointer) "plugin", (gpointer) agent, args);
	agent = "fence_legacy";
	}
	#endif

	crm_xml_add(data, PCMK_XA_ID, id);
	crm_xml_add(data, PCMK__XA_ST_ORIGIN, __func__);
	crm_xml_add(data, PCMK_XA_AGENT, agent);
	if ((standard != st_namespace_any) && (standard != st_namespace_invalid)) {
	crm_xml_add(data, PCMK__XA_NAMESPACE,
	stonith_namespace2text(standard));
	}
	if (rsc_provides) {
	crm_xml_add(data, PCMK__XA_RSC_PROVIDES, rsc_provides);
	}

	for (; params; params = params->next) {
	hash2field((gpointer) params->key, (gpointer) params->value, args);
	}

	return data;
	}

	static int
	stonith_api_register_device(stonith_t *st, int call_options,
	const char id, const char namespace_s,
	const char *agent,
	const stonith_key_value_t *params)
	{
	int rc = 0;
	xmlNode *data = NULL;

	data = create_device_registration_xml(id,
	stonith_text2namespace(namespace_s),
	agent, params, NULL);

	rc = stonith_send_command(st, STONITH_OP_DEVICE_ADD, data, NULL, call_options, 0);
	pcmk__xml_free(data);

	return rc;
	}

	static int
	stonith_api_remove_device(stonith_t * st, int call_options, const char *name)
	{
	int rc = 0;
	xmlNode *data = NULL;

	data = pcmk__xe_create(NULL, PCMK__XE_ST_DEVICE_ID);
	crm_xml_add(data, PCMK__XA_ST_ORIGIN, __func__);
	crm_xml_add(data, PCMK_XA_ID, name);
	rc = stonith_send_command(st, STONITH_OP_DEVICE_DEL, data, NULL, call_options, 0);
	pcmk__xml_free(data);

	return rc;
	}

	static int
	stonith_api_remove_level_full(stonith_t *st, int options,
	const char node, const char pattern,
	const char attr, const char value, int level)
	{
	int rc = 0;
	xmlNode *data = NULL;

	CRM_CHECK(node \|\| pattern \|\| (attr && value), return -EINVAL);

	data = pcmk__xe_create(NULL, PCMK_XE_FENCING_LEVEL);
	crm_xml_add(data, PCMK__XA_ST_ORIGIN, __func__);

	if (node) {
	crm_xml_add(data, PCMK_XA_TARGET, node);

	} else if (pattern) {
	crm_xml_add(data, PCMK_XA_TARGET_PATTERN, pattern);

	} else {
	crm_xml_add(data, PCMK_XA_TARGET_ATTRIBUTE, attr);
	crm_xml_add(data, PCMK_XA_TARGET_VALUE, value);
	}

	crm_xml_add_int(data, PCMK_XA_INDEX, level);
	rc = stonith_send_command(st, STONITH_OP_LEVEL_DEL, data, NULL, options, 0);
	pcmk__xml_free(data);

	return rc;
	}

	static int
	stonith_api_remove_level(stonith_t * st, int options, const char *node, int level)
	{
	return stonith_api_remove_level_full(st, options, node,
	NULL, NULL, NULL, level);
	}

	/*!
	* \internal
	* \brief Create XML for fence topology level registration request
	*
	* \param[in] node If not NULL, target level by this node name
	* \param[in] pattern If not NULL, target by node name using this regex
	* \param[in] attr If not NULL, target by this node attribute
	* \param[in] value If not NULL, target by this node attribute value
	* \param[in] level Index number of level to register
	* \param[in] device_list List of devices in level
	*
	* \return Newly allocated XML tree on success, NULL otherwise
	*
	* \note The caller should set only one of node, pattern or attr/value.
	*/
	xmlNode *
	create_level_registration_xml(const char node, const char pattern,
	const char attr, const char value,
	int level, const stonith_key_value_t *device_list)
	{
	GString *list = NULL;
	xmlNode *data;

	CRM_CHECK(node \|\| pattern \|\| (attr && value), return NULL);

	data = pcmk__xe_create(NULL, PCMK_XE_FENCING_LEVEL);

	crm_xml_add(data, PCMK__XA_ST_ORIGIN, __func__);
	crm_xml_add_int(data, PCMK_XA_ID, level);
	crm_xml_add_int(data, PCMK_XA_INDEX, level);

	if (node) {
	crm_xml_add(data, PCMK_XA_TARGET, node);

	} else if (pattern) {
	crm_xml_add(data, PCMK_XA_TARGET_PATTERN, pattern);

	} else {
	crm_xml_add(data, PCMK_XA_TARGET_ATTRIBUTE, attr);
	crm_xml_add(data, PCMK_XA_TARGET_VALUE, value);
	}

	for (; device_list; device_list = device_list->next) {
	pcmk__add_separated_word(&list, 1024, device_list->value, ",");
	}

	if (list != NULL) {
	crm_xml_add(data, PCMK_XA_DEVICES, (const char *) list->str);
	g_string_free(list, TRUE);
	}
	return data;
	}

	static int
	stonith_api_register_level_full(stonith_t st, int options, const char node,
	const char pattern, const char attr,
	const char *value, int level,
	const stonith_key_value_t *device_list)
	{
	int rc = 0;
	xmlNode *data = create_level_registration_xml(node, pattern, attr, value,
	level, device_list);
	CRM_CHECK(data != NULL, return -EINVAL);

	rc = stonith_send_command(st, STONITH_OP_LEVEL_ADD, data, NULL, options, 0);
	pcmk__xml_free(data);

	return rc;
	}

	static int
	stonith_api_register_level(stonith_t * st, int options, const char *node, int level,
	const stonith_key_value_t * device_list)
	{
	return stonith_api_register_level_full(st, options, node, NULL, NULL, NULL,
	level, device_list);
	}

	static int
	stonith_api_device_list(stonith_t *stonith, int call_options,
	const char namespace_s, stonith_key_value_t *devices,
	int timeout)
	{
	int count = 0;
	enum stonith_namespace ns = stonith_text2namespace(namespace_s);

	if (devices == NULL) {
	crm_err("Parameter error: stonith_api_device_list");
	return -EFAULT;
	}

	#if HAVE_STONITH_STONITH_H
	// Include Linux-HA agents if requested
	if ((ns == st_namespace_any) \|\| (ns == st_namespace_lha)) {
	count += stonith__list_lha_agents(devices);
	}
	#endif

	// Include Red Hat agents if requested
	if ((ns == st_namespace_any) \|\| (ns == st_namespace_rhcs)) {
	count += stonith__list_rhcs_agents(devices);
	}

	return count;
	}

	// See stonith_api_operations_t:metadata() documentation
	static int
	stonith_api_device_metadata(stonith_t *stonith, int call_options,
	const char agent, const char namespace_s,
	char **output, int timeout_sec)
	{
	/* By executing meta-data directly, we can get it from stonith_admin when
	* the cluster is not running, which is important for higher-level tools.
	*/

	enum stonith_namespace ns = stonith_get_namespace(agent, namespace_s);

	if (timeout_sec <= 0) {
	timeout_sec = PCMK_DEFAULT_ACTION_TIMEOUT_MS;
	}

	crm_trace("Looking up metadata for %s agent %s",
	stonith_namespace2text(ns), agent);

	switch (ns) {
	case st_namespace_rhcs:
	return stonith__rhcs_metadata(agent, timeout_sec, output);

	#if HAVE_STONITH_STONITH_H
	case st_namespace_lha:
	return stonith__lha_metadata(agent, timeout_sec, output);
	#endif

	default:
	crm_err("Can't get fence agent '%s' meta-data: No such agent",
	agent);
	break;
	}
	return -ENODEV;
	}

	static int
	stonith_api_query(stonith_t * stonith, int call_options, const char *target,
	stonith_key_value_t ** devices, int timeout)
	{
	int rc = 0, lpc = 0, max = 0;

	xmlNode *data = NULL;
	xmlNode *output = NULL;
	xmlXPathObjectPtr xpathObj = NULL;

	CRM_CHECK(devices != NULL, return -EINVAL);

	data = pcmk__xe_create(NULL, PCMK__XE_ST_DEVICE_ID);
	crm_xml_add(data, PCMK__XA_ST_ORIGIN, __func__);
	crm_xml_add(data, PCMK__XA_ST_TARGET, target);
	crm_xml_add(data, PCMK__XA_ST_DEVICE_ACTION, PCMK_ACTION_OFF);
	rc = stonith_send_command(stonith, STONITH_OP_QUERY, data, &output, call_options, timeout);

	if (rc < 0) {
	return rc;
	}

	xpathObj = xpath_search(output, "//@agent");
	if (xpathObj) {
	max = numXpathResults(xpathObj);

	for (lpc = 0; lpc < max; lpc++) {
	xmlNode *match = getXpathResult(xpathObj, lpc);

	CRM_LOG_ASSERT(match != NULL);
	if(match != NULL) {
	xmlChar *match_path = xmlGetNodePath(match);

	crm_info("%s[%d] = %s", "//@agent", lpc, match_path);
	free(match_path);
	devices = stonith_key_value_add(devices, NULL,
	crm_element_value(match,
	PCMK_XA_ID));
	}
	}

	freeXpathObject(xpathObj);
	}

	pcmk__xml_free(output);
	pcmk__xml_free(data);
	return max;
	}

	/*!
	* \internal
	* \brief Make a STONITH_OP_EXEC request
	*
	* \param[in,out] stonith Fencer connection
	* \param[in] call_options Bitmask of \c stonith_call_options
	* \param[in] id Fence device ID that request is for
	* \param[in] action Agent action to request (list, status, monitor)
	* \param[in] target Name of target node for requested action
	* \param[in] timeout_sec Error if not completed within this many seconds
	* \param[out] output Where to set agent output
	*/
	static int
	stonith_api_call(stonith_t stonith, int call_options, const char id,
	const char action, const char target, int timeout_sec,
	xmlNode **output)
	{
	int rc = 0;
	xmlNode *data = NULL;

	data = pcmk__xe_create(NULL, PCMK__XE_ST_DEVICE_ID);
	crm_xml_add(data, PCMK__XA_ST_ORIGIN, __func__);
	crm_xml_add(data, PCMK__XA_ST_DEVICE_ID, id);
	crm_xml_add(data, PCMK__XA_ST_DEVICE_ACTION, action);
	crm_xml_add(data, PCMK__XA_ST_TARGET, target);

	rc = stonith_send_command(stonith, STONITH_OP_EXEC, data, output,
	call_options, timeout_sec);
	pcmk__xml_free(data);

	return rc;
	}

	static int
	stonith_api_list(stonith_t * stonith, int call_options, const char id, char *list_info,
	int timeout)
	{
	int rc;
	xmlNode *output = NULL;

	rc = stonith_api_call(stonith, call_options, id, PCMK_ACTION_LIST, NULL,
	timeout, &output);

	if (output && list_info) {
	const char *list_str;

	list_str = crm_element_value(output, PCMK__XA_ST_OUTPUT);

	if (list_str) {
	*list_info = strdup(list_str);
	}
	}

	if (output) {
	pcmk__xml_free(output);
	}

	return rc;
	}

	static int
	stonith_api_monitor(stonith_t * stonith, int call_options, const char *id, int timeout)
	{
	return stonith_api_call(stonith, call_options, id, PCMK_ACTION_MONITOR,
	NULL, timeout, NULL);
	}

	static int
	stonith_api_status(stonith_t * stonith, int call_options, const char id, const char port,
	int timeout)
	{
	return stonith_api_call(stonith, call_options, id, PCMK_ACTION_STATUS, port,
	timeout, NULL);
	}

	static int
	stonith_api_fence_with_delay(stonith_t * stonith, int call_options, const char *node,
	const char *action, int timeout, int tolerance, int delay)
	{
	int rc = 0;
	xmlNode *data = NULL;

	data = pcmk__xe_create(NULL, __func__);
	crm_xml_add(data, PCMK__XA_ST_TARGET, node);
	crm_xml_add(data, PCMK__XA_ST_DEVICE_ACTION, action);
	crm_xml_add_int(data, PCMK__XA_ST_TIMEOUT, timeout);
	crm_xml_add_int(data, PCMK__XA_ST_TOLERANCE, tolerance);
	crm_xml_add_int(data, PCMK__XA_ST_DELAY, delay);

	rc = stonith_send_command(stonith, STONITH_OP_FENCE, data, NULL, call_options, timeout);
	pcmk__xml_free(data);

	return rc;
	}

	static int
	stonith_api_fence(stonith_t * stonith, int call_options, const char node, const char action,
	int timeout, int tolerance)
	{
	return stonith_api_fence_with_delay(stonith, call_options, node, action,
	timeout, tolerance, 0);
	}

	static int
	stonith_api_confirm(stonith_t * stonith, int call_options, const char *target)
	{
	stonith__set_call_options(call_options, target, st_opt_manual_ack);
	return stonith_api_fence(stonith, call_options, target, PCMK_ACTION_OFF, 0,
	0);
	}

	static int
	stonith_api_history(stonith_t * stonith, int call_options, const char *node,
	stonith_history_t ** history, int timeout)
	{
	int rc = 0;
	xmlNode *data = NULL;
	xmlNode *output = NULL;
	stonith_history_t *last = NULL;

	*history = NULL;

	if (node) {
	data = pcmk__xe_create(NULL, __func__);
	crm_xml_add(data, PCMK__XA_ST_TARGET, node);
	}

	stonith__set_call_options(call_options, node, st_opt_sync_call);
	rc = stonith_send_command(stonith, STONITH_OP_FENCE_HISTORY, data, &output,
	call_options, timeout);
	pcmk__xml_free(data);

	if (rc == 0) {
	xmlNode *op = NULL;
	xmlNode *reply = get_xpath_object("//" PCMK__XE_ST_HISTORY, output,
	LOG_NEVER);

	for (op = pcmk__xe_first_child(reply, NULL, NULL, NULL); op != NULL;
	op = pcmk__xe_next(op)) {
	stonith_history_t *kvp;
	long long completed;
	long long completed_nsec = 0L;

	kvp = pcmk__assert_alloc(1, sizeof(stonith_history_t));
	kvp->target = crm_element_value_copy(op, PCMK__XA_ST_TARGET);
	kvp->action = crm_element_value_copy(op, PCMK__XA_ST_DEVICE_ACTION);
	kvp->origin = crm_element_value_copy(op, PCMK__XA_ST_ORIGIN);
	kvp->delegate = crm_element_value_copy(op, PCMK__XA_ST_DELEGATE);
	kvp->client = crm_element_value_copy(op, PCMK__XA_ST_CLIENTNAME);
	crm_element_value_ll(op, PCMK__XA_ST_DATE, &completed);
	kvp->completed = (time_t) completed;
	crm_element_value_ll(op, PCMK__XA_ST_DATE_NSEC, &completed_nsec);
	kvp->completed_nsec = completed_nsec;
	crm_element_value_int(op, PCMK__XA_ST_STATE, &kvp->state);
	kvp->exit_reason = crm_element_value_copy(op, PCMK_XA_EXIT_REASON);

	if (last) {
	last->next = kvp;
	} else {
	*history = kvp;
	}
	last = kvp;
	}
	}

	pcmk__xml_free(output);

	return rc;
	}

	void stonith_history_free(stonith_history_t *history)
	{
	stonith_history_t hp, hp_old;

	for (hp = history; hp; hp_old = hp, hp = hp->next, free(hp_old)) {
	free(hp->target);
	free(hp->action);
	free(hp->origin);
	free(hp->delegate);
	free(hp->client);
	free(hp->exit_reason);
	}
	}

	static gint
	stonithlib_GCompareFunc(gconstpointer a, gconstpointer b)
	{
	int rc = 0;
	const stonith_notify_client_t *a_client = a;
	const stonith_notify_client_t *b_client = b;

	if (a_client->delete \|\| b_client->delete) {
	/* make entries marked for deletion not findable */
	return -1;
	}
	CRM_CHECK(a_client->event != NULL && b_client->event != NULL, return 0);
	rc = strcmp(a_client->event, b_client->event);
	if (rc == 0) {
	if (a_client->notify == NULL \|\| b_client->notify == NULL) {
	return 0;

	} else if (a_client->notify == b_client->notify) {
	return 0;

	} else if (((long)a_client->notify) < ((long)b_client->notify)) {
	crm_err("callbacks for %s are not equal: %p vs. %p",
	a_client->event, a_client->notify, b_client->notify);
	return -1;
	}
	crm_err("callbacks for %s are not equal: %p vs. %p",
	a_client->event, a_client->notify, b_client->notify);
	return 1;
	}
	return rc;
	}

	xmlNode *
	stonith_create_op(int call_id, const char token, const char op, xmlNode * data, int call_options)
	{
	xmlNode *op_msg = NULL;

	CRM_CHECK(token != NULL, return NULL);

	op_msg = pcmk__xe_create(NULL, PCMK__XE_STONITH_COMMAND);
	crm_xml_add(op_msg, PCMK__XA_T, PCMK__VALUE_STONITH_NG);
	crm_xml_add(op_msg, PCMK__XA_ST_OP, op);
	crm_xml_add_int(op_msg, PCMK__XA_ST_CALLID, call_id);
	crm_trace("Sending call options: %.8lx, %d", (long)call_options, call_options);
	crm_xml_add_int(op_msg, PCMK__XA_ST_CALLOPT, call_options);

	if (data != NULL) {
	xmlNode *wrapper = pcmk__xe_create(op_msg, PCMK__XE_ST_CALLDATA);

	pcmk__xml_copy(wrapper, data);
	}

	return op_msg;
	}

	static void
	stonith_destroy_op_callback(gpointer data)
	{
	stonith_callback_client_t *blob = data;

	if (blob->timer && blob->timer->ref > 0) {
	g_source_remove(blob->timer->ref);
	}
	free(blob->timer);
	free(blob);
	}

	static int
	stonith_api_signoff(stonith_t * stonith)
	{
	stonith_private_t *native = stonith->st_private;

	crm_debug("Disconnecting from the fencer");

	if (native->source != NULL) {
	/* Attached to mainloop */
	mainloop_del_ipc_client(native->source);
	native->source = NULL;
	native->ipc = NULL;

	} else if (native->ipc) {
	/* Not attached to mainloop */
	crm_ipc_t *ipc = native->ipc;

	native->ipc = NULL;
	crm_ipc_close(ipc);
	crm_ipc_destroy(ipc);
	}

	free(native->token); native->token = NULL;
	stonith->state = stonith_disconnected;
	return pcmk_ok;
	}

	static int
	stonith_api_del_callback(stonith_t * stonith, int call_id, bool all_callbacks)
	{
	stonith_private_t *private = stonith->st_private;

	if (all_callbacks) {
	private->op_callback = NULL;
	g_hash_table_destroy(private->stonith_op_callback_table);
	private->stonith_op_callback_table = pcmk__intkey_table(stonith_destroy_op_callback);

	} else if (call_id == 0) {
	private->op_callback = NULL;

	} else {
	pcmk__intkey_table_remove(private->stonith_op_callback_table, call_id);
	}
	return pcmk_ok;
	}

	/*!
	* \internal
	* \brief Invoke a (single) specified fence action callback
	*
	* \param[in,out] st Fencer API connection
	* \param[in] call_id If positive, call ID of completed fence action,
	* otherwise legacy return code for early failure
	* \param[in,out] result Full result for action
	* \param[in,out] userdata User data to pass to callback
	* \param[in] callback Fence action callback to invoke
	*/
	static void
	invoke_fence_action_callback(stonith_t *st, int call_id,
	pcmk__action_result_t *result,
	void *userdata,
	void (callback) (stonith_t st,
	stonith_callback_data_t *data))
	{
	stonith_callback_data_t data = { 0, };

	data.call_id = call_id;
	data.rc = pcmk_rc2legacy(stonith__result2rc(result));
	data.userdata = userdata;
	data.opaque = (void *) result;

	callback(st, &data);
	}

	/*!
	* \internal
	* \brief Invoke any callbacks registered for a specified fence action result
	*
	* Given a fence action result from the fencer, invoke any callback registered
	* for that action, as well as any global callback registered.
	*
	* \param[in,out] stonith Fencer API connection
	* \param[in] msg If non-NULL, fencer reply
	* \param[in] call_id If \p msg is NULL, call ID of action that timed out
	*/
	static void
	invoke_registered_callbacks(stonith_t stonith, const xmlNode msg, int call_id)
	{
	stonith_private_t *private = NULL;
	stonith_callback_client_t *cb_info = NULL;
	pcmk__action_result_t result = PCMK__UNKNOWN_RESULT;

	CRM_CHECK(stonith != NULL, return);
	CRM_CHECK(stonith->st_private != NULL, return);

	private = stonith->st_private;

	if (msg == NULL) {
	// Fencer didn't reply in time
	pcmk__set_result(&result, CRM_EX_ERROR, PCMK_EXEC_TIMEOUT,
	"Fencer accepted request but did not reply in time");
	CRM_LOG_ASSERT(call_id > 0);

	} else {
	// We have the fencer reply
	if ((crm_element_value_int(msg, PCMK__XA_ST_CALLID, &call_id) != 0)
	\|\| (call_id <= 0)) {
	crm_log_xml_warn(msg, "Bad fencer reply");
	}
	stonith__xe_get_result(msg, &result);
	}

	if (call_id > 0) {
	cb_info = pcmk__intkey_table_lookup(private->stonith_op_callback_table,
	call_id);
	}

	if ((cb_info != NULL) && (cb_info->callback != NULL)
	&& (pcmk__result_ok(&result) \|\| !(cb_info->only_success))) {
	crm_trace("Invoking callback %s for call %d",
	pcmk__s(cb_info->id, "without ID"), call_id);
	invoke_fence_action_callback(stonith, call_id, &result,
	cb_info->user_data, cb_info->callback);

	} else if ((private->op_callback == NULL) && !pcmk__result_ok(&result)) {
	crm_warn("Fencing action without registered callback failed: %d (%s%s%s)",
	result.exit_status,
	pcmk_exec_status_str(result.execution_status),
	((result.exit_reason == NULL)? "" : ": "),
	((result.exit_reason == NULL)? "" : result.exit_reason));
	crm_log_xml_debug(msg, "Failed fence update");
	}

	if (private->op_callback != NULL) {
	crm_trace("Invoking global callback for call %d", call_id);
	invoke_fence_action_callback(stonith, call_id, &result, NULL,
	private->op_callback);
	}

	if (cb_info != NULL) {
	stonith_api_del_callback(stonith, call_id, FALSE);
	}
	pcmk__reset_result(&result);
	}

	static gboolean
	stonith_async_timeout_handler(gpointer data)
	{
	struct timer_rec_s *timer = data;

	crm_err("Async call %d timed out after %dms", timer->call_id, timer->timeout);
	invoke_registered_callbacks(timer->stonith, NULL, timer->call_id);

	/* Always return TRUE, never remove the handler
	* We do that in stonith_del_callback()
	*/
	return TRUE;
	}

	static void
	set_callback_timeout(stonith_callback_client_t * callback, stonith_t * stonith, int call_id,
	int timeout)
	{
	struct timer_rec_s *async_timer = callback->timer;

	if (timeout <= 0) {
	return;
	}

	if (!async_timer) {
	async_timer = pcmk__assert_alloc(1, sizeof(struct timer_rec_s));
	callback->timer = async_timer;
	}

	async_timer->stonith = stonith;
	async_timer->call_id = call_id;
	/* Allow a fair bit of grace to allow the server to tell us of a timeout
	* This is only a fallback
	*/
	async_timer->timeout = (timeout + 60) * 1000;
	if (async_timer->ref) {
	g_source_remove(async_timer->ref);
	}
	async_timer->ref =
	g_timeout_add(async_timer->timeout, stonith_async_timeout_handler, async_timer);
	}

	static void
	update_callback_timeout(int call_id, int timeout, stonith_t * st)
	{
	stonith_callback_client_t *callback = NULL;
	stonith_private_t *private = st->st_private;

	callback = pcmk__intkey_table_lookup(private->stonith_op_callback_table,
	call_id);
	if (!callback \|\| !callback->allow_timeout_updates) {
	return;
	}

	set_callback_timeout(callback, st, call_id, timeout);
	}

	static int
	stonith_dispatch_internal(const char *buffer, ssize_t length, gpointer userdata)
	{
	const char *type = NULL;
	struct notify_blob_s blob;

	stonith_t *st = userdata;
	stonith_private_t *private = NULL;

	pcmk__assert(st != NULL);
	private = st->st_private;

	blob.stonith = st;
	blob.xml = pcmk__xml_parse(buffer);
	if (blob.xml == NULL) {
	crm_warn("Received malformed message from fencer: %s", buffer);
	return 0;
	}

	/* do callbacks */
	type = crm_element_value(blob.xml, PCMK__XA_T);
	crm_trace("Activating %s callbacks...", type);

	if (pcmk__str_eq(type, PCMK__VALUE_STONITH_NG, pcmk__str_none)) {
	invoke_registered_callbacks(st, blob.xml, 0);

	} else if (pcmk__str_eq(type, PCMK__VALUE_ST_NOTIFY, pcmk__str_none)) {
	foreach_notify_entry(private, stonith_send_notification, &blob);

	} else if (pcmk__str_eq(type, PCMK__VALUE_ST_ASYNC_TIMEOUT_VALUE,
	pcmk__str_none)) {
	int call_id = 0;
	int timeout = 0;

	crm_element_value_int(blob.xml, PCMK__XA_ST_TIMEOUT, &timeout);
	crm_element_value_int(blob.xml, PCMK__XA_ST_CALLID, &call_id);

	update_callback_timeout(call_id, timeout, st);
	} else {
	crm_err("Unknown message type: %s", type);
	crm_log_xml_warn(blob.xml, "BadReply");
	}

	pcmk__xml_free(blob.xml);
	return 1;
	}

	static int
	stonith_api_signon(stonith_t * stonith, const char name, int stonith_fd)
	{
	int rc = pcmk_ok;
	stonith_private_t *native = NULL;
	const char *display_name = name? name : "client";

	struct ipc_client_callbacks st_callbacks = {
	.dispatch = stonith_dispatch_internal,
	.destroy = stonith_connection_destroy
	};

	CRM_CHECK(stonith != NULL, return -EINVAL);

	native = stonith->st_private;
	pcmk__assert(native != NULL);

	crm_debug("Attempting fencer connection by %s with%s mainloop",
	display_name, (stonith_fd? "out" : ""));

	stonith->state = stonith_connected_command;
	if (stonith_fd) {
	/* No mainloop */
	native->ipc = crm_ipc_new("stonith-ng", 0);
	if (native->ipc != NULL) {
	rc = pcmk__connect_generic_ipc(native->ipc);
	if (rc == pcmk_rc_ok) {
	rc = pcmk__ipc_fd(native->ipc, stonith_fd);
	if (rc != pcmk_rc_ok) {
	crm_debug("Couldn't get file descriptor for IPC: %s",
	pcmk_rc_str(rc));
	}
	}
	if (rc != pcmk_rc_ok) {
	crm_ipc_close(native->ipc);
	crm_ipc_destroy(native->ipc);
	native->ipc = NULL;
	}
	}

	} else {
	/* With mainloop */
	native->source =
	mainloop_add_ipc_client("stonith-ng", G_PRIORITY_MEDIUM, 0, stonith, &st_callbacks);
	native->ipc = mainloop_get_ipc_client(native->source);
	}

	if (native->ipc == NULL) {
	rc = -ENOTCONN;
	} else {
	xmlNode *reply = NULL;
	xmlNode *hello = pcmk__xe_create(NULL, PCMK__XE_STONITH_COMMAND);

	crm_xml_add(hello, PCMK__XA_T, PCMK__VALUE_STONITH_NG);
	crm_xml_add(hello, PCMK__XA_ST_OP, CRM_OP_REGISTER);
	crm_xml_add(hello, PCMK__XA_ST_CLIENTNAME, name);
	rc = crm_ipc_send(native->ipc, hello, crm_ipc_client_response, -1, &reply);

	if (rc < 0) {
	crm_debug("Couldn't register with the fencer: %s "
	QB_XS " rc=%d", pcmk_strerror(rc), rc);
	rc = -ECOMM;

	} else if (reply == NULL) {
	crm_debug("Couldn't register with the fencer: no reply");
	rc = -EPROTO;

	} else {
	const char *msg_type = crm_element_value(reply, PCMK__XA_ST_OP);

	native->token = crm_element_value_copy(reply, PCMK__XA_ST_CLIENTID);
	if (!pcmk__str_eq(msg_type, CRM_OP_REGISTER, pcmk__str_none)) {
	crm_debug("Couldn't register with the fencer: invalid reply type '%s'",
	(msg_type? msg_type : "(missing)"));
	crm_log_xml_debug(reply, "Invalid fencer reply");
	rc = -EPROTO;

	} else if (native->token == NULL) {
	crm_debug("Couldn't register with the fencer: no token in reply");
	crm_log_xml_debug(reply, "Invalid fencer reply");
	rc = -EPROTO;

	} else {
	crm_debug("Connection to fencer by %s succeeded (registration token: %s)",
	display_name, native->token);
	rc = pcmk_ok;
	}
	}

	pcmk__xml_free(reply);
	pcmk__xml_free(hello);
	}

	if (rc != pcmk_ok) {
	crm_debug("Connection attempt to fencer by %s failed: %s "
	QB_XS " rc=%d", display_name, pcmk_strerror(rc), rc);
	stonith->cmds->disconnect(stonith);
	}
	return rc;
	}

	static int
	stonith_set_notification(stonith_t * stonith, const char *callback, int enabled)
	{
	int rc = pcmk_ok;
	xmlNode *notify_msg = pcmk__xe_create(NULL, __func__);
	stonith_private_t *native = stonith->st_private;

	if (stonith->state != stonith_disconnected) {

	crm_xml_add(notify_msg, PCMK__XA_ST_OP, STONITH_OP_NOTIFY);
	if (enabled) {
	crm_xml_add(notify_msg, PCMK__XA_ST_NOTIFY_ACTIVATE, callback);
	} else {
	crm_xml_add(notify_msg, PCMK__XA_ST_NOTIFY_DEACTIVATE, callback);
	}

	rc = crm_ipc_send(native->ipc, notify_msg, crm_ipc_client_response, -1, NULL);
	if (rc < 0) {
	crm_perror(LOG_DEBUG, "Couldn't register for fencing notifications: %d", rc);
	rc = -ECOMM;
	} else {
	rc = pcmk_ok;
	}
	}

	pcmk__xml_free(notify_msg);
	return rc;
	}

	static int
	stonith_api_add_notification(stonith_t * stonith, const char *event,
	void (callback) (stonith_t stonith, stonith_event_t * e))
	{
	GList *list_item = NULL;
	stonith_notify_client_t *new_client = NULL;
	stonith_private_t *private = NULL;

	private = stonith->st_private;
	crm_trace("Adding callback for %s events (%d)", event, g_list_length(private->notify_list));

	new_client = pcmk__assert_alloc(1, sizeof(stonith_notify_client_t));
	new_client->event = event;
	new_client->notify = callback;

	list_item = g_list_find_custom(private->notify_list, new_client, stonithlib_GCompareFunc);

	if (list_item != NULL) {
	crm_warn("Callback already present");
	free(new_client);
	return -ENOTUNIQ;

	} else {
	private->notify_list = g_list_append(private->notify_list, new_client);

	stonith_set_notification(stonith, event, 1);

	crm_trace("Callback added (%d)", g_list_length(private->notify_list));
	}
	return pcmk_ok;
	}

	static void
	del_notify_entry(gpointer data, gpointer user_data)
	{
	stonith_notify_client_t *entry = data;
	stonith_t * stonith = user_data;

	if (!entry->delete) {
	crm_debug("Removing callback for %s events", entry->event);
	stonith_api_del_notification(stonith, entry->event);
	}
	}

	static int
	stonith_api_del_notification(stonith_t * stonith, const char *event)
	{
	GList *list_item = NULL;
	stonith_notify_client_t *new_client = NULL;
	stonith_private_t *private = stonith->st_private;

	if (event == NULL) {
	foreach_notify_entry(private, del_notify_entry, stonith);
	crm_trace("Removed callback");

	return pcmk_ok;
	}

	crm_debug("Removing callback for %s events", event);

	new_client = pcmk__assert_alloc(1, sizeof(stonith_notify_client_t));
	new_client->event = event;
	new_client->notify = NULL;

	list_item = g_list_find_custom(private->notify_list, new_client, stonithlib_GCompareFunc);

	stonith_set_notification(stonith, event, 0);

	if (list_item != NULL) {
	stonith_notify_client_t *list_client = list_item->data;

	if (private->notify_refcnt) {
	list_client->delete = TRUE;
	private->notify_deletes = TRUE;
	} else {
	private->notify_list = g_list_remove(private->notify_list, list_client);
	free(list_client);
	}

	crm_trace("Removed callback");

	} else {
	crm_trace("Callback not present");
	}
	free(new_client);
	return pcmk_ok;
	}

	static int
	stonith_api_add_callback(stonith_t * stonith, int call_id, int timeout, int options,
	void user_data, const char callback_name,
	void (callback) (stonith_t st, stonith_callback_data_t * data))
	{
	stonith_callback_client_t *blob = NULL;
	stonith_private_t *private = NULL;

	CRM_CHECK(stonith != NULL, return -EINVAL);
	CRM_CHECK(stonith->st_private != NULL, return -EINVAL);
	private = stonith->st_private;

	if (call_id == 0) { // Add global callback
	private->op_callback = callback;

	} else if (call_id < 0) { // Call failed immediately, so call callback now
	if (!(options & st_opt_report_only_success)) {
	pcmk__action_result_t result = PCMK__UNKNOWN_RESULT;

	crm_trace("Call failed, calling %s: %s", callback_name, pcmk_strerror(call_id));
	pcmk__set_result(&result, CRM_EX_ERROR,
	stonith__legacy2status(call_id), NULL);
	invoke_fence_action_callback(stonith, call_id, &result,
	user_data, callback);
	} else {
	crm_warn("Fencer call failed: %s", pcmk_strerror(call_id));
	}
	return FALSE;
	}

	blob = pcmk__assert_alloc(1, sizeof(stonith_callback_client_t));
	blob->id = callback_name;
	blob->only_success = (options & st_opt_report_only_success) ? TRUE : FALSE;
	blob->user_data = user_data;
	blob->callback = callback;
	blob->allow_timeout_updates = (options & st_opt_timeout_updates) ? TRUE : FALSE;

	if (timeout > 0) {
	set_callback_timeout(blob, stonith, call_id, timeout);
	}

	pcmk__intkey_table_insert(private->stonith_op_callback_table, call_id,
	blob);
	crm_trace("Added callback to %s for call %d", callback_name, call_id);

	return TRUE;
	}

	static void
	stonith_dump_pending_op(gpointer key, gpointer value, gpointer user_data)
	{
	int call = GPOINTER_TO_INT(key);
	stonith_callback_client_t *blob = value;

	crm_debug("Call %d (%s): pending", call, pcmk__s(blob->id, "no ID"));
	}

	void
	stonith_dump_pending_callbacks(stonith_t * stonith)
	{
	stonith_private_t *private = stonith->st_private;

	if (private->stonith_op_callback_table == NULL) {
	return;
	}
	return g_hash_table_foreach(private->stonith_op_callback_table, stonith_dump_pending_op, NULL);
	}

	/*!
	* \internal
	* \brief Get the data section of a fencer notification
	*
	* \param[in] msg Notification XML
	* \param[in] ntype Notification type
	*/
	static xmlNode *
	get_event_data_xml(xmlNode msg, const char ntype)
	{
	char *data_addr = crm_strdup_printf("//%s", ntype);
	xmlNode *data = get_xpath_object(data_addr, msg, LOG_DEBUG);

	free(data_addr);
	return data;
	}

	/*
	<notify t="st_notify" subt="st_device_register" st_op="st_device_register" st_rc="0" >
	<st_calldata >
	<stonith_command t="stonith-ng" st_async_id="088fb640-431a-48b9-b2fc-c4ff78d0a2d9" st_op="st_device_register" st_callid="2" st_callopt="4096" st_timeout="0" st_clientid="088fb640-431a-48b9-b2fc-c4ff78d0a2d9" st_clientname="cts-fence-helper" >
	<st_calldata >
	<st_device_id id="test-id" origin="create_device_registration_xml" agent="fence_virsh" namespace="stonith-ng" >
	<attributes ipaddr="localhost" pcmk-portmal="some-host=pcmk-1 pcmk-3=3,4" login="root" identity_file="/root/.ssh/id_dsa" />
	</st_device_id>
	</st_calldata>
	</stonith_command>
	</st_calldata>
	</notify>

	<notify t="st_notify" subt="st_notify_fence" st_op="st_notify_fence" st_rc="0" >
	<st_calldata >
	<st_notify_fence st_rc="0" st_target="some-host" st_op="st_fence" st_delegate="test-id" st_origin="61dd7759-e229-4be7-b1f8-ef49dd14d9f0" />
	</st_calldata>
	</notify>
	*/
	static stonith_event_t *
	xml_to_event(xmlNode *msg)
	{
	stonith_event_t *event = pcmk__assert_alloc(1, sizeof(stonith_event_t));
	struct event_private *event_private = NULL;

	event->opaque = pcmk__assert_alloc(1, sizeof(struct event_private));
	event_private = (struct event_private *) event->opaque;

	crm_log_xml_trace(msg, "stonith_notify");

	// All notification types have the operation result and notification subtype
	stonith__xe_get_result(msg, &event_private->result);
	event->operation = crm_element_value_copy(msg, PCMK__XA_ST_OP);

	// @COMPAT The API originally provided the result as a legacy return code
	event->result = pcmk_rc2legacy(stonith__result2rc(&event_private->result));

	// Some notification subtypes have additional information

	if (pcmk__str_eq(event->operation, PCMK__VALUE_ST_NOTIFY_FENCE,
	pcmk__str_none)) {
	xmlNode *data = get_event_data_xml(msg, event->operation);

	if (data == NULL) {
	crm_err("No data for %s event", event->operation);
	crm_log_xml_notice(msg, "BadEvent");
	} else {
	event->origin = crm_element_value_copy(data, PCMK__XA_ST_ORIGIN);
	event->action = crm_element_value_copy(data,
	PCMK__XA_ST_DEVICE_ACTION);
	event->target = crm_element_value_copy(data, PCMK__XA_ST_TARGET);
	event->executioner = crm_element_value_copy(data,
	PCMK__XA_ST_DELEGATE);
	event->id = crm_element_value_copy(data, PCMK__XA_ST_REMOTE_OP);
	event->client_origin =
	crm_element_value_copy(data, PCMK__XA_ST_CLIENTNAME);
	event->device = crm_element_value_copy(data, PCMK__XA_ST_DEVICE_ID);
	}

	} else if (pcmk__str_any_of(event->operation,
	STONITH_OP_DEVICE_ADD, STONITH_OP_DEVICE_DEL,
	STONITH_OP_LEVEL_ADD, STONITH_OP_LEVEL_DEL,
	NULL)) {
	xmlNode *data = get_event_data_xml(msg, event->operation);

	if (data == NULL) {
	crm_err("No data for %s event", event->operation);
	crm_log_xml_notice(msg, "BadEvent");
	} else {
	event->device = crm_element_value_copy(data, PCMK__XA_ST_DEVICE_ID);
	}
	}

	return event;
	}

	static void
	event_free(stonith_event_t * event)
	{
	struct event_private *event_private = event->opaque;

	free(event->id);
	free(event->operation);
	free(event->origin);
	free(event->action);
	free(event->target);
	free(event->executioner);
	free(event->device);
	free(event->client_origin);
	pcmk__reset_result(&event_private->result);
	free(event->opaque);
	free(event);
	}

	static void
	stonith_send_notification(gpointer data, gpointer user_data)
	{
	struct notify_blob_s *blob = user_data;
	stonith_notify_client_t *entry = data;
	stonith_event_t *st_event = NULL;
	const char *event = NULL;

	if (blob->xml == NULL) {
	crm_warn("Skipping callback - NULL message");
	return;
	}

	event = crm_element_value(blob->xml, PCMK__XA_SUBT);

	if (entry == NULL) {
	crm_warn("Skipping callback - NULL callback client");
	return;

	} else if (entry->delete) {
	crm_trace("Skipping callback - marked for deletion");
	return;

	} else if (entry->notify == NULL) {
	crm_warn("Skipping callback - NULL callback");
	return;

	} else if (!pcmk__str_eq(entry->event, event, pcmk__str_none)) {
	crm_trace("Skipping callback - event mismatch %p/%s vs. %s", entry, entry->event, event);
	return;
	}

	st_event = xml_to_event(blob->xml);

	crm_trace("Invoking callback for %p/%s event...", entry, event);
	entry->notify(blob->stonith, st_event);
	crm_trace("Callback invoked...");

	event_free(st_event);
	}

	/*!
	* \internal
	* \brief Create and send an API request
	*
	* \param[in,out] stonith Stonith connection
	* \param[in] op API operation to request
	* \param[in] data Data to attach to request
	* \param[out] output_data If not NULL, will be set to reply if synchronous
	* \param[in] call_options Bitmask of stonith_call_options to use
	* \param[in] timeout Error if not completed within this many seconds
	*
	* \return pcmk_ok (for synchronous requests) or positive call ID
	* (for asynchronous requests) on success, -errno otherwise
	*/
	static int
	stonith_send_command(stonith_t * stonith, const char op, xmlNode data, xmlNode ** output_data,
	int call_options, int timeout)
	{
	int rc = 0;
	int reply_id = -1;

	xmlNode *op_msg = NULL;
	xmlNode *op_reply = NULL;
	stonith_private_t *native = NULL;

	pcmk__assert((stonith != NULL) && (stonith->st_private != NULL)
	&& (op != NULL));
	native = stonith->st_private;

	if (output_data != NULL) {
	*output_data = NULL;
	}

	if ((stonith->state == stonith_disconnected) \|\| (native->token == NULL)) {
	return -ENOTCONN;
	}

	/* Increment the call ID, which must be positive to avoid conflicting with
	* error codes. This shouldn't be a problem unless the client mucked with
	* it or the counter wrapped around.
	*/
	stonith->call_id++;
	if (stonith->call_id < 1) {
	stonith->call_id = 1;
	}

	op_msg = stonith_create_op(stonith->call_id, native->token, op, data, call_options);
	if (op_msg == NULL) {
	return -EINVAL;
	}

	crm_xml_add_int(op_msg, PCMK__XA_ST_TIMEOUT, timeout);
	crm_trace("Sending %s message to fencer with timeout %ds", op, timeout);

	if (data) {
	const char *delay_s = crm_element_value(data, PCMK__XA_ST_DELAY);

	if (delay_s) {
	crm_xml_add(op_msg, PCMK__XA_ST_DELAY, delay_s);
	}
	}

	{
	enum crm_ipc_flags ipc_flags = crm_ipc_flags_none;

	if (call_options & st_opt_sync_call) {
	pcmk__set_ipc_flags(ipc_flags, "stonith command",
	crm_ipc_client_response);
	}
	rc = crm_ipc_send(native->ipc, op_msg, ipc_flags,
	1000 * (timeout + 60), &op_reply);
	}
	pcmk__xml_free(op_msg);

	if (rc < 0) {
	crm_perror(LOG_ERR, "Couldn't perform %s operation (timeout=%ds): %d", op, timeout, rc);
	rc = -ECOMM;
	goto done;
	}

	crm_log_xml_trace(op_reply, "Reply");

	if (!(call_options & st_opt_sync_call)) {
	crm_trace("Async call %d, returning", stonith->call_id);
	pcmk__xml_free(op_reply);
	return stonith->call_id;
	}

	crm_element_value_int(op_reply, PCMK__XA_ST_CALLID, &reply_id);

	if (reply_id == stonith->call_id) {
	pcmk__action_result_t result = PCMK__UNKNOWN_RESULT;

	crm_trace("Synchronous reply %d received", reply_id);

	stonith__xe_get_result(op_reply, &result);
	rc = pcmk_rc2legacy(stonith__result2rc(&result));
	pcmk__reset_result(&result);

	if ((call_options & st_opt_discard_reply) \|\| output_data == NULL) {
	crm_trace("Discarding reply");

	} else {
	*output_data = op_reply;
	op_reply = NULL; /* Prevent subsequent free */
	}

	} else if (reply_id <= 0) {
	crm_err("Received bad reply: No id set");
	crm_log_xml_err(op_reply, "Bad reply");
	pcmk__xml_free(op_reply);
	op_reply = NULL;
	rc = -ENOMSG;

	} else {
	crm_err("Received bad reply: %d (wanted %d)", reply_id, stonith->call_id);
	crm_log_xml_err(op_reply, "Old reply");
	pcmk__xml_free(op_reply);
	op_reply = NULL;
	rc = -ENOMSG;
	}

	done:
	if (!crm_ipc_connected(native->ipc)) {
	crm_err("Fencer disconnected");
	free(native->token); native->token = NULL;
	stonith->state = stonith_disconnected;
	}

	pcmk__xml_free(op_reply);
	return rc;
	}

	/* Not used with mainloop */
	bool
	stonith_dispatch(stonith_t * st)
	{
	gboolean stay_connected = TRUE;
	stonith_private_t *private = NULL;

	pcmk__assert(st != NULL);
	private = st->st_private;

	while (crm_ipc_ready(private->ipc)) {

	if (crm_ipc_read(private->ipc) > 0) {
	const char *msg = crm_ipc_buffer(private->ipc);

	stonith_dispatch_internal(msg, strlen(msg), st);
	}

	if (!crm_ipc_connected(private->ipc)) {
	crm_err("Connection closed");
	stay_connected = FALSE;
	}
	}

	return stay_connected;
	}

	static int
	stonith_api_free(stonith_t * stonith)
	{
	int rc = pcmk_ok;

	crm_trace("Destroying %p", stonith);

	if (stonith->state != stonith_disconnected) {
	crm_trace("Unregistering notifications and disconnecting %p first",
	stonith);
	stonith->cmds->remove_notification(stonith, NULL);
	rc = stonith->cmds->disconnect(stonith);
	}

	if (stonith->state == stonith_disconnected) {
	stonith_private_t *private = stonith->st_private;

	crm_trace("Removing %d callbacks", g_hash_table_size(private->stonith_op_callback_table));
	g_hash_table_destroy(private->stonith_op_callback_table);

	crm_trace("Destroying %d notification clients", g_list_length(private->notify_list));
	g_list_free_full(private->notify_list, free);

	free(stonith->st_private);
	free(stonith->cmds);
	free(stonith);

	} else {
	crm_err("Not free'ing active connection: %s (%d)", pcmk_strerror(rc), rc);
	}

	return rc;
	}

	void
	stonith_api_delete(stonith_t * stonith)
	{
	crm_trace("Destroying %p", stonith);
	if(stonith) {
	stonith->cmds->free(stonith);
	}
	}

	-static int
	-stonith_api_validate(stonith_t st, int call_options, const char rsc_id,
	- const char namespace_s, const char agent,
	- const stonith_key_value_t *params, int timeout_sec,
	- char output, char error_output)
	+static gboolean
	+is_stonith_param(gpointer key, gpointer value, gpointer user_data)
	{
	- /* Validation should be done directly via the agent, so we can get it from
	- * stonith_admin when the cluster is not running, which is important for
	- * higher-level tools.
	- */
	+ return pcmk_stonith_param(key);
	+}

	- int rc = pcmk_ok;
	+int
	+stonith__validate(stonith_t st, int call_options, const char rsc_id,
	+ const char namespace_s, const char agent,
	+ GHashTable params, int timeout_sec, char *output,
	+ char **error_output)
	+{
	+ int rc = pcmk_rc_ok;

	/* Use a dummy node name in case the agent requires a target. We assume the
	* actual target doesn't matter for validation purposes (if in practice,
	* that is incorrect, we will need to allow the caller to pass the target).
	*/
	const char *target = "node1";
	const char *host_arg = NULL;

	- GHashTable *params_table = pcmk__strkey_table(free, free);
	+ if (params != NULL) {
	+ /* Remove special stonith params from the table before doing anything else */
	+ g_hash_table_foreach_remove(params, is_stonith_param, NULL);

	- // Convert parameter list to a hash table
	- for (; params; params = params->next) {
	- if (pcmk__str_eq(params->key, PCMK_STONITH_HOST_ARGUMENT,
	- pcmk__str_none)) {
	- host_arg = params->value;
	- }
	- if (!pcmk_stonith_param(params->key)) {
	- pcmk__insert_dup(params_table, params->key, params->value);
	- }
	+ host_arg = g_hash_table_lookup(params, PCMK_STONITH_HOST_ARGUMENT);
	}

	#if PCMK__ENABLE_CIBSECRETS
	- rc = pcmk__substitute_secrets(rsc_id, params_table);
	+ rc = pcmk__substitute_secrets(rsc_id, params);
	if (rc != pcmk_rc_ok) {
	crm_warn("Could not replace secret parameters for validation of %s: %s",
	agent, pcmk_rc_str(rc));
	// rc is standard return value, don't return it in this function
	}
	#endif

	if (output) {
	*output = NULL;
	}
	if (error_output) {
	*error_output = NULL;
	}

	if (timeout_sec <= 0) {
	timeout_sec = PCMK_DEFAULT_ACTION_TIMEOUT_MS;
	}

	switch (stonith_get_namespace(agent, namespace_s)) {
	case st_namespace_rhcs:
	rc = stonith__rhcs_validate(st, call_options, target, agent,
	- params_table, host_arg, timeout_sec,
	+ params, host_arg, timeout_sec,
	output, error_output);
	+ rc = pcmk_legacy2rc(rc);
	break;

	#if HAVE_STONITH_STONITH_H
	case st_namespace_lha:
	rc = stonith__lha_validate(st, call_options, target, agent,
	- params_table, timeout_sec, output,
	+ params, timeout_sec, output,
	error_output);
	+ rc = pcmk_legacy2rc(rc);
	break;
	#endif

	case st_namespace_invalid:
	errno = ENOENT;
	- rc = -errno;
	+ rc = errno;

	if (error_output) {
	*error_output = crm_strdup_printf("Agent %s not found", agent);
	} else {
	crm_err("Agent %s not found", agent);
	}

	break;

	default:
	errno = EOPNOTSUPP;
	- rc = -errno;
	+ rc = errno;

	if (error_output) {
	*error_output = crm_strdup_printf("Agent %s does not support validation",
	agent);
	} else {
	crm_err("Agent %s does not support validation", agent);
	}

	break;
	}

	+ return rc;
	+}
	+
	+static int
	+stonith_api_validate(stonith_t st, int call_options, const char rsc_id,
	+ const char namespace_s, const char agent,
	+ const stonith_key_value_t *params, int timeout_sec,
	+ char output, char error_output)
	+{
	+ /* Validation should be done directly via the agent, so we can get it from
	+ * stonith_admin when the cluster is not running, which is important for
	+ * higher-level tools.
	+ */
	+
	+ int rc = pcmk_ok;
	+
	+ GHashTable *params_table = pcmk__strkey_table(free, free);
	+
	+ // Convert parameter list to a hash table
	+ for (; params; params = params->next) {
	+ if (!pcmk_stonith_param(params->key)) {
	+ pcmk__insert_dup(params_table, params->key, params->value);
	+ }
	+ }
	+
	+ rc = stonith__validate(st, call_options, rsc_id, namespace_s, agent,
	+ params_table, timeout_sec, output, error_output);
	+
	g_hash_table_destroy(params_table);
	return rc;
	}

	stonith_t *
	stonith_api_new(void)
	{
	stonith_t *new_stonith = NULL;
	stonith_private_t *private = NULL;

	new_stonith = calloc(1, sizeof(stonith_t));
	if (new_stonith == NULL) {
	return NULL;
	}

	private = calloc(1, sizeof(stonith_private_t));
	if (private == NULL) {
	free(new_stonith);
	return NULL;
	}
	new_stonith->st_private = private;

	private->stonith_op_callback_table = pcmk__intkey_table(stonith_destroy_op_callback);
	private->notify_list = NULL;
	private->notify_refcnt = 0;
	private->notify_deletes = FALSE;

	new_stonith->call_id = 1;
	new_stonith->state = stonith_disconnected;

	new_stonith->cmds = calloc(1, sizeof(stonith_api_operations_t));
	if (new_stonith->cmds == NULL) {
	free(new_stonith->st_private);
	free(new_stonith);
	return NULL;
	}

	/* INDENT-OFF */
	new_stonith->cmds->free = stonith_api_free;
	new_stonith->cmds->connect = stonith_api_signon;
	new_stonith->cmds->disconnect = stonith_api_signoff;

	new_stonith->cmds->list = stonith_api_list;
	new_stonith->cmds->monitor = stonith_api_monitor;
	new_stonith->cmds->status = stonith_api_status;
	new_stonith->cmds->fence = stonith_api_fence;
	new_stonith->cmds->fence_with_delay = stonith_api_fence_with_delay;
	new_stonith->cmds->confirm = stonith_api_confirm;
	new_stonith->cmds->history = stonith_api_history;

	new_stonith->cmds->list_agents = stonith_api_device_list;
	new_stonith->cmds->metadata = stonith_api_device_metadata;

	new_stonith->cmds->query = stonith_api_query;
	new_stonith->cmds->remove_device = stonith_api_remove_device;
	new_stonith->cmds->register_device = stonith_api_register_device;

	new_stonith->cmds->remove_level = stonith_api_remove_level;
	new_stonith->cmds->remove_level_full = stonith_api_remove_level_full;
	new_stonith->cmds->register_level = stonith_api_register_level;
	new_stonith->cmds->register_level_full = stonith_api_register_level_full;

	new_stonith->cmds->remove_callback = stonith_api_del_callback;
	new_stonith->cmds->register_callback = stonith_api_add_callback;
	new_stonith->cmds->remove_notification = stonith_api_del_notification;
	new_stonith->cmds->register_notification = stonith_api_add_notification;

	new_stonith->cmds->validate = stonith_api_validate;
	/* INDENT-ON */

	return new_stonith;
	}

	/*!
	* \brief Make a blocking connection attempt to the fencer
	*
	* \param[in,out] st Fencer API object
	* \param[in] name Client name to use with fencer
	* \param[in] max_attempts Return error if this many attempts fail
	*
	* \return pcmk_ok on success, result of last attempt otherwise
	*/
	int
	stonith_api_connect_retry(stonith_t st, const char name, int max_attempts)
	{
	int rc = -EINVAL; // if max_attempts is not positive

	for (int attempt = 1; attempt <= max_attempts; attempt++) {
	rc = st->cmds->connect(st, name, NULL);
	if (rc == pcmk_ok) {
	return pcmk_ok;
	} else if (attempt < max_attempts) {
	crm_notice("Fencer connection attempt %d of %d failed (retrying in 2s): %s "
	QB_XS " rc=%d",
	attempt, max_attempts, pcmk_strerror(rc), rc);
	sleep(2);
	}
	}
	crm_notice("Could not connect to fencer: %s " QB_XS " rc=%d",
	pcmk_strerror(rc), rc);
	return rc;
	}

	stonith_key_value_t *
	stonith_key_value_add(stonith_key_value_t * head, const char key, const char value)
	{
	stonith_key_value_t p, end;

	p = pcmk__assert_alloc(1, sizeof(stonith_key_value_t));
	p->key = pcmk__str_copy(key);
	p->value = pcmk__str_copy(value);

	end = head;
	while (end && end->next) {
	end = end->next;
	}

	if (end) {
	end->next = p;
	} else {
	head = p;
	}

	return head;
	}

	void
	stonith_key_value_freeall(stonith_key_value_t * head, int keys, int values)
	{
	stonith_key_value_t *p;

	while (head) {
	p = head->next;
	if (keys) {
	free(head->key);
	}
	if (values) {
	free(head->value);
	}
	free(head);
	head = p;
	}
	}

	#define api_log_open() openlog("stonith-api", LOG_CONS \| LOG_NDELAY \| LOG_PID, LOG_DAEMON)
	#define api_log(level, fmt, args...) syslog(level, "%s: "fmt, __func__, args)

	int
	stonith_api_kick(uint32_t nodeid, const char *uname, int timeout, bool off)
	{
	int rc = pcmk_ok;
	stonith_t *st = stonith_api_new();
	const char *action = off? PCMK_ACTION_OFF : PCMK_ACTION_REBOOT;

	api_log_open();
	if (st == NULL) {
	api_log(LOG_ERR, "API initialization failed, could not kick (%s) node %u/%s",
	action, nodeid, uname);
	return -EPROTO;
	}

	rc = st->cmds->connect(st, "stonith-api", NULL);
	if (rc != pcmk_ok) {
	api_log(LOG_ERR, "Connection failed, could not kick (%s) node %u/%s : %s (%d)",
	action, nodeid, uname, pcmk_strerror(rc), rc);
	} else {
	char *name = (uname == NULL)? pcmk__itoa(nodeid) : strdup(uname);
	int opts = 0;

	stonith__set_call_options(opts, name,
	st_opt_sync_call\|st_opt_allow_self_fencing);
	if ((uname == NULL) && (nodeid > 0)) {
	stonith__set_call_options(opts, name, st_opt_cs_nodeid);
	}
	rc = st->cmds->fence(st, opts, name, action, timeout, 0);
	free(name);

	if (rc != pcmk_ok) {
	api_log(LOG_ERR, "Could not kick (%s) node %u/%s : %s (%d)",
	action, nodeid, uname, pcmk_strerror(rc), rc);
	} else {
	api_log(LOG_NOTICE, "Node %u/%s kicked: %s", nodeid, uname, action);
	}
	}

	stonith_api_delete(st);
	return rc;
	}

	time_t
	stonith_api_time(uint32_t nodeid, const char *uname, bool in_progress)
	{
	int rc = pcmk_ok;
	time_t when = 0;
	stonith_t *st = stonith_api_new();
	stonith_history_t history = NULL, hp = NULL;

	if (st == NULL) {
	api_log(LOG_ERR, "Could not retrieve fence history for %u/%s: "
	"API initialization failed", nodeid, uname);
	return when;
	}

	rc = st->cmds->connect(st, "stonith-api", NULL);
	if (rc != pcmk_ok) {
	api_log(LOG_NOTICE, "Connection failed: %s (%d)", pcmk_strerror(rc), rc);
	} else {
	int entries = 0;
	int progress = 0;
	int completed = 0;
	int opts = 0;
	char *name = (uname == NULL)? pcmk__itoa(nodeid) : strdup(uname);

	stonith__set_call_options(opts, name, st_opt_sync_call);
	if ((uname == NULL) && (nodeid > 0)) {
	stonith__set_call_options(opts, name, st_opt_cs_nodeid);
	}
	rc = st->cmds->history(st, opts, name, &history, 120);
	free(name);

	for (hp = history; hp; hp = hp->next) {
	entries++;
	if (in_progress) {
	progress++;
	if (hp->state != st_done && hp->state != st_failed) {
	when = time(NULL);
	}

	} else if (hp->state == st_done) {
	completed++;
	if (hp->completed > when) {
	when = hp->completed;
	}
	}
	}

	stonith_history_free(history);

	if(rc == pcmk_ok) {
	api_log(LOG_INFO, "Found %d entries for %u/%s: %d in progress, %d completed", entries, nodeid, uname, progress, completed);
	} else {
	api_log(LOG_ERR, "Could not retrieve fence history for %u/%s: %s (%d)", nodeid, uname, pcmk_strerror(rc), rc);
	}
	}

	stonith_api_delete(st);

	if(when) {
	api_log(LOG_INFO, "Node %u/%s last kicked at: %ld", nodeid, uname, (long int)when);
	}
	return when;
	}

	bool
	stonith_agent_exists(const char *agent, int timeout)
	{
	stonith_t *st = NULL;
	stonith_key_value_t *devices = NULL;
	stonith_key_value_t *dIter = NULL;
	bool rc = FALSE;

	if (agent == NULL) {
	return rc;
	}

	st = stonith_api_new();
	if (st == NULL) {
	crm_err("Could not list fence agents: API memory allocation failed");
	return FALSE;
	}
	st->cmds->list_agents(st, st_opt_sync_call, NULL, &devices, timeout == 0 ? 120 : timeout);

	for (dIter = devices; dIter != NULL; dIter = dIter->next) {
	if (pcmk__str_eq(dIter->value, agent, pcmk__str_none)) {
	rc = TRUE;
	break;
	}
	}

	stonith_key_value_freeall(devices, 1, 1);
	stonith_api_delete(st);
	return rc;
	}

	const char *
	stonith_action_str(const char *action)
	{
	if (action == NULL) {
	return "fencing";
	} else if (strcmp(action, PCMK_ACTION_ON) == 0) {
	return "unfencing";
	} else if (strcmp(action, PCMK_ACTION_OFF) == 0) {
	return "turning off";
	} else {
	return action;
	}
	}

	/*!
	* \internal
	* \brief Parse a target name from one line of a target list string
	*
	* \param[in] line One line of a target list string
	* \param[in] len String length of line
	* \param[in,out] output List to add newly allocated target name to
	*/
	static void
	parse_list_line(const char line, int len, GList *output)
	{
	size_t i = 0;
	size_t entry_start = 0;

	/* Skip complaints about additional parameters device doesn't understand
	*
	* @TODO Document or eliminate the implied restriction of target names
	*/
	if (strstr(line, "invalid") \|\| strstr(line, "variable")) {
	crm_debug("Skipping list output line: %s", line);
	return;
	}

	// Process line content, character by character
	for (i = 0; i <= len; i++) {

	if (isspace(line[i]) \|\| (line[i] == ',') \|\| (line[i] == ';')
	\|\| (line[i] == '\0')) {
	// We've found a separator (i.e. the end of an entry)

	int rc = 0;
	char *entry = NULL;

	if (i == entry_start) {
	// Skip leading and sequential separators
	entry_start = i + 1;
	continue;
	}

	entry = pcmk__assert_alloc(i - entry_start + 1, sizeof(char));

	/* Read entry, stopping at first separator
	*
	* @TODO Document or eliminate these character restrictions
	*/
	rc = sscanf(line + entry_start, "%[a-zA-Z0-9_-.]", entry);
	if (rc != 1) {
	crm_warn("Could not parse list output entry: %s "
	QB_XS " entry_start=%d position=%d",
	line + entry_start, entry_start, i);
	free(entry);

	} else if (pcmk__strcase_any_of(entry, PCMK_ACTION_ON,
	PCMK_ACTION_OFF, NULL)) {
	/* Some agents print the target status in the list output,
	* though none are known now (the separate list-status command
	* is used for this, but it can also print "UNKNOWN"). To handle
	* this possibility, skip such entries.
	*
	* @TODO Document or eliminate the implied restriction of target
	* names.
	*/
	free(entry);

	} else {
	// We have a valid entry
	output = g_list_append(output, entry);
	}
	entry_start = i + 1;
	}
	}
	}

	/*!
	* \internal
	* \brief Parse a list of targets from a string
	*
	* \param[in] list_output Target list as a string
	*
	* \return List of target names
	* \note The target list string format is flexible, to allow for user-specified
	* lists such pcmk_host_list and the output of an agent's list action
	* (whether direct or via the API, which escapes newlines). There may be
	* multiple lines, separated by either a newline or an escaped newline
	* (backslash n). Each line may have one or more target names, separated
	* by any combination of whitespace, commas, and semi-colons. Lines
	* containing "invalid" or "variable" will be ignored entirely. Target
	* names "on" or "off" (case-insensitive) will be ignored. Target names
	* may contain only alphanumeric characters, underbars (_), dashes (-),
	* and dots (.) (if any other character occurs in the name, it and all
	* subsequent characters in the name will be ignored).
	* \note The caller is responsible for freeing the result with
	* g_list_free_full(result, free).
	*/
	GList *
	stonith__parse_targets(const char *target_spec)
	{
	GList *targets = NULL;

	if (target_spec != NULL) {
	size_t out_len = strlen(target_spec);
	size_t line_start = 0; // Starting index of line being processed

	for (size_t i = 0; i <= out_len; ++i) {
	if ((target_spec[i] == '\n') \|\| (target_spec[i] == '\0')
	\|\| ((target_spec[i] == '\\') && (target_spec[i + 1] == 'n'))) {
	// We've reached the end of one line of output

	int len = i - line_start;

	if (len > 0) {
	char *line = strndup(target_spec + line_start, len);

	line[len] = '\0'; // Because it might be a newline
	parse_list_line(line, len, &targets);
	free(line);
	}
	if (target_spec[i] == '\\') {
	++i; // backslash-n takes up two positions
	}
	line_start = i + 1;
	}
	}
	}
	return targets;
	}

	/*!
	* \internal
	* \brief Check whether a fencing failure was followed by an equivalent success
	*
	* \param[in] event Fencing failure
	* \param[in] top_history Complete fencing history (must be sorted by
	* stonith__sort_history() beforehand)
	*
	* \return The name of the node that executed the fencing if a later successful
	* event exists, or NULL if no such event exists
	*/
	const char *
	stonith__later_succeeded(const stonith_history_t *event,
	const stonith_history_t *top_history)
	{
	const char *other = NULL;

	for (const stonith_history_t *prev_hp = top_history;
	prev_hp != NULL; prev_hp = prev_hp->next) {
	if (prev_hp == event) {
	break;
	}
	if ((prev_hp->state == st_done) &&
	pcmk__str_eq(event->target, prev_hp->target, pcmk__str_casei) &&
	pcmk__str_eq(event->action, prev_hp->action, pcmk__str_none) &&
	((event->completed < prev_hp->completed) \|\|
	((event->completed == prev_hp->completed) && (event->completed_nsec < prev_hp->completed_nsec)))) {

	if ((event->delegate == NULL)
	\|\| pcmk__str_eq(event->delegate, prev_hp->delegate,
	pcmk__str_casei)) {
	// Prefer equivalent fencing by same executioner
	return prev_hp->delegate;

	} else if (other == NULL) {
	// Otherwise remember first successful executioner
	other = (prev_hp->delegate == NULL)? "some node" : prev_hp->delegate;
	}
	}
	}
	return other;
	}

	/*!
	* \internal
	* \brief Sort fencing history, pending first then by most recently completed
	*
	* \param[in,out] history List of stonith actions
	*
	* \return New head of sorted \p history
	*/
	stonith_history_t *
	stonith__sort_history(stonith_history_t *history)
	{
	stonith_history_t new = NULL, pending = NULL, hp, np, *tmp;

	for (hp = history; hp; ) {
	tmp = hp->next;
	if ((hp->state == st_done) \|\| (hp->state == st_failed)) {
	/* sort into new */
	if ((!new) \|\| (hp->completed > new->completed) \|\|
	((hp->completed == new->completed) && (hp->completed_nsec > new->completed_nsec))) {
	hp->next = new;
	new = hp;
	} else {
	np = new;
	do {
	if ((!np->next) \|\| (hp->completed > np->next->completed) \|\|
	((hp->completed == np->next->completed) && (hp->completed_nsec > np->next->completed_nsec))) {
	hp->next = np->next;
	np->next = hp;
	break;
	}
	np = np->next;
	} while (1);
	}
	} else {
	/* put into pending */
	hp->next = pending;
	pending = hp;
	}
	hp = tmp;
	}

	/* pending actions don't have a completed-stamp so make them go front */
	if (pending) {
	stonith_history_t *last_pending = pending;

	while (last_pending->next) {
	last_pending = last_pending->next;
	}

	last_pending->next = new;
	new = pending;
	}
	return new;
	}

	/*!
	* \brief Return string equivalent of an operation state value
	*
	* \param[in] state Fencing operation state value
	*
	* \return Human-friendly string equivalent of state
	*/
	const char *
	stonith_op_state_str(enum op_state state)
	{
	switch (state) {
	case st_query: return "querying";
	case st_exec: return "executing";
	case st_done: return "completed";
	case st_duplicate: return "duplicate";
	case st_failed: return "failed";
	}
	return "unknown";
	}

	stonith_history_t *
	stonith__first_matching_event(stonith_history_t *history,
	bool (matching_fn)(stonith_history_t , void *),
	void *user_data)
	{
	for (stonith_history_t *hp = history; hp; hp = hp->next) {
	if (matching_fn(hp, user_data)) {
	return hp;
	}
	}

	return NULL;
	}

	bool
	stonith__event_state_pending(stonith_history_t history, void user_data)
	{
	return history->state != st_failed && history->state != st_done;
	}

	bool
	stonith__event_state_eq(stonith_history_t history, void user_data)
	{
	return history->state == GPOINTER_TO_INT(user_data);
	}

	bool
	stonith__event_state_neq(stonith_history_t history, void user_data)
	{
	return history->state != GPOINTER_TO_INT(user_data);
	}

	void
	stonith__device_parameter_flags(uint32_t device_flags, const char device_name,
	xmlNode *metadata)
	{
	xmlXPathObjectPtr xpath = NULL;
	int max = 0;
	int lpc = 0;

	CRM_CHECK((device_flags != NULL) && (metadata != NULL), return);

	xpath = xpath_search(metadata, "//" PCMK_XE_PARAMETER);
	max = numXpathResults(xpath);

	if (max <= 0) {
	freeXpathObject(xpath);
	return;
	}

	for (lpc = 0; lpc < max; lpc++) {
	const char *parameter = NULL;
	xmlNode *match = getXpathResult(xpath, lpc);

	CRM_LOG_ASSERT(match != NULL);
	if (match == NULL) {
	continue;
	}

	parameter = crm_element_value(match, PCMK_XA_NAME);

	if (pcmk__str_eq(parameter, "plug", pcmk__str_casei)) {
	stonith__set_device_flags(*device_flags, device_name,
	st_device_supports_parameter_plug);

	} else if (pcmk__str_eq(parameter, "port", pcmk__str_casei)) {
	stonith__set_device_flags(*device_flags, device_name,
	st_device_supports_parameter_port);
	}
	}

	freeXpathObject(xpath);
	}

	/*!
	* \internal
	* \brief Retrieve fence agent meta-data asynchronously
	*
	* \param[in] agent Agent to execute
	* \param[in] timeout_sec Error if not complete within this time
	* \param[in] callback Function to call with result (this will always be
	* called, whether by this function directly or
	* later via the main loop, and on success the
	* metadata will be in its result argument's
	* action_stdout)
	* \param[in,out] user_data User data to pass to callback
	*
	* \return Standard Pacemaker return code
	* \note The caller must use a main loop. This function is not a
	* stonith_api_operations_t method because it does not need a stonith_t
	* object and does not go through the fencer, but executes the agent
	* directly.
	*/
	int
	stonith__metadata_async(const char *agent, int timeout_sec,
	void (*callback)(int pid,
	const pcmk__action_result_t *result,
	void *user_data),
	void *user_data)
	{
	switch (stonith_get_namespace(agent, NULL)) {
	case st_namespace_rhcs:
	{
	stonith_action_t *action = NULL;
	int rc = pcmk_ok;

	action = stonith__action_create(agent, PCMK_ACTION_METADATA,
	NULL, 0, timeout_sec, NULL,
	NULL, NULL);

	rc = stonith__execute_async(action, user_data, callback, NULL);
	if (rc != pcmk_ok) {
	callback(0, stonith__action_result(action), user_data);
	stonith__destroy_action(action);
	}
	return pcmk_legacy2rc(rc);
	}

	#if HAVE_STONITH_STONITH_H
	case st_namespace_lha:
	// LHA metadata is simply synthesized, so simulate async
	{
	pcmk__action_result_t result = {
	.exit_status = CRM_EX_OK,
	.execution_status = PCMK_EXEC_DONE,
	.exit_reason = NULL,
	.action_stdout = NULL,
	.action_stderr = NULL,
	};

	stonith__lha_metadata(agent, timeout_sec,
	&result.action_stdout);
	callback(0, &result, user_data);
	pcmk__reset_result(&result);
	return pcmk_rc_ok;
	}
	#endif

	default:
	{
	pcmk__action_result_t result = {
	.exit_status = CRM_EX_NOSUCH,
	.execution_status = PCMK_EXEC_ERROR_HARD,
	.exit_reason = crm_strdup_printf("No such agent '%s'",
	agent),
	.action_stdout = NULL,
	.action_stderr = NULL,
	};

	callback(0, &result, user_data);
	pcmk__reset_result(&result);
	return ENOENT;
	}
	}
	}

	/*!
	* \internal
	* \brief Return the exit status from an async action callback
	*
	* \param[in] data Callback data
	*
	* \return Exit status from callback data
	*/
	int
	stonith__exit_status(const stonith_callback_data_t *data)
	{
	if ((data == NULL) \|\| (data->opaque == NULL)) {
	return CRM_EX_ERROR;
	}
	return ((pcmk__action_result_t *) data->opaque)->exit_status;
	}

	/*!
	* \internal
	* \brief Return the execution status from an async action callback
	*
	* \param[in] data Callback data
	*
	* \return Execution status from callback data
	*/
	int
	stonith__execution_status(const stonith_callback_data_t *data)
	{
	if ((data == NULL) \|\| (data->opaque == NULL)) {
	return PCMK_EXEC_UNKNOWN;
	}
	return ((pcmk__action_result_t *) data->opaque)->execution_status;
	}

	/*!
	* \internal
	* \brief Return the exit reason from an async action callback
	*
	* \param[in] data Callback data
	*
	* \return Exit reason from callback data
	*/
	const char *
	stonith__exit_reason(const stonith_callback_data_t *data)
	{
	if ((data == NULL) \|\| (data->opaque == NULL)) {
	return NULL;
	}
	return ((pcmk__action_result_t *) data->opaque)->exit_reason;
	}

	/*!
	* \internal
	* \brief Return the exit status from an event notification
	*
	* \param[in] event Event
	*
	* \return Exit status from event
	*/
	int
	stonith__event_exit_status(const stonith_event_t *event)
	{
	if ((event == NULL) \|\| (event->opaque == NULL)) {
	return CRM_EX_ERROR;
	} else {
	struct event_private *event_private = event->opaque;

	return event_private->result.exit_status;
	}
	}

	/*!
	* \internal
	* \brief Return the execution status from an event notification
	*
	* \param[in] event Event
	*
	* \return Execution status from event
	*/
	int
	stonith__event_execution_status(const stonith_event_t *event)
	{
	if ((event == NULL) \|\| (event->opaque == NULL)) {
	return PCMK_EXEC_UNKNOWN;
	} else {
	struct event_private *event_private = event->opaque;

	return event_private->result.execution_status;
	}
	}

	/*!
	* \internal
	* \brief Return the exit reason from an event notification
	*
	* \param[in] event Event
	*
	* \return Exit reason from event
	*/
	const char *
	stonith__event_exit_reason(const stonith_event_t *event)
	{
	if ((event == NULL) \|\| (event->opaque == NULL)) {
	return NULL;
	} else {
	struct event_private *event_private = event->opaque;

	return event_private->result.exit_reason;
	}
	}

	/*!
	* \internal
	* \brief Return a human-friendly description of a fencing event
	*
	* \param[in] event Event to describe
	*
	* \return Newly allocated string with description of \p event
	* \note The caller is responsible for freeing the return value.
	* This function asserts on memory errors and never returns NULL.
	*/
	char *
	stonith__event_description(const stonith_event_t *event)
	{
	// Use somewhat readable defaults
	const char *origin = pcmk__s(event->client_origin, "a client");
	const char *origin_node = pcmk__s(event->origin, "a node");
	const char *executioner = pcmk__s(event->executioner, "the cluster");
	const char *device = pcmk__s(event->device, "unknown");
	const char *action = pcmk__s(event->action, event->operation);
	const char *target = pcmk__s(event->target, "no node");
	const char *reason = stonith__event_exit_reason(event);
	const char *status;

	if (action == NULL) {
	action = "(unknown)";
	}

	if (stonith__event_execution_status(event) != PCMK_EXEC_DONE) {
	status = pcmk_exec_status_str(stonith__event_execution_status(event));
	} else if (stonith__event_exit_status(event) != CRM_EX_OK) {
	status = pcmk_exec_status_str(PCMK_EXEC_ERROR);
	} else {
	status = crm_exit_str(CRM_EX_OK);
	}

	if (pcmk__str_eq(event->operation, PCMK__VALUE_ST_NOTIFY_HISTORY,
	pcmk__str_none)) {
	return crm_strdup_printf("Fencing history may have changed");

	} else if (pcmk__str_eq(event->operation, STONITH_OP_DEVICE_ADD,
	pcmk__str_none)) {
	return crm_strdup_printf("A fencing device (%s) was added", device);

	} else if (pcmk__str_eq(event->operation, STONITH_OP_DEVICE_DEL,
	pcmk__str_none)) {
	return crm_strdup_printf("A fencing device (%s) was removed", device);

	} else if (pcmk__str_eq(event->operation, STONITH_OP_LEVEL_ADD,
	pcmk__str_none)) {
	return crm_strdup_printf("A fencing topology level (%s) was added",
	device);

	} else if (pcmk__str_eq(event->operation, STONITH_OP_LEVEL_DEL,
	pcmk__str_none)) {
	return crm_strdup_printf("A fencing topology level (%s) was removed",
	device);
	}

	// event->operation should be PCMK__VALUE_ST_NOTIFY_FENCE at this point

	return crm_strdup_printf("Operation %s of %s by %s for %s@%s: %s%s%s%s (ref=%s)",
	action, target, executioner, origin, origin_node,
	status,
	((reason == NULL)? "" : " ("), pcmk__s(reason, ""),
	((reason == NULL)? "" : ")"),
	pcmk__s(event->id, "(none)"));
	}
	diff --git a/lib/pacemaker/libpacemaker_private.h b/lib/pacemaker/libpacemaker_private.h
	index 2c8e9f07d7..de58ec60a6 100644
	--- a/lib/pacemaker/libpacemaker_private.h
	+++ b/lib/pacemaker/libpacemaker_private.h
	@@ -1,1138 +1,1141 @@
	/*
	* Copyright 2021-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#ifndef PCMK__PACEMAKER_LIBPACEMAKER_PRIVATE__H
	#define PCMK__PACEMAKER_LIBPACEMAKER_PRIVATE__H

	/* This header is for the sole use of libpacemaker, so that functions can be
	* declared with G_GNUC_INTERNAL for efficiency.
	*/

	#include <stdio.h> // NULL
	#include <stdint.h> // uint32_t
	#include <stdbool.h> // bool, false
	#include <glib.h> // guint, gpointer, GList, GHashTable
	#include <libxml/tree.h> // xmlNode

	#include <crm/common/scheduler.h> // pcmk_action_t, pcmk_node_t, etc.
	#include <crm/common/scheduler_internal.h> // pcmk__location_t, etc.
	#include <crm/cib.h> // cib_t
	#include <crm/lrmd_events.h> // lrmd_event_data_t
	#include <crm/pengine/internal.h> // pe__const_top_resource(), etc.
	#include <pacemaker.h> // pcmk_injections_t
	#include <pacemaker-internal.h> // pcmk__colocation_t

	#ifdef __cplusplus
	extern "C" {
	#endif

	// Colocation flags
	enum pcmk__coloc_flags {
	pcmk__coloc_none = 0U,

	// Primary is affected even if already active
	pcmk__coloc_influence = (1U << 0),

	// Colocation was explicitly configured in CIB
	pcmk__coloc_explicit = (1U << 1),
	};

	// Flags to modify the behavior of add_colocated_node_scores()
	enum pcmk__coloc_select {
	// With no other flags, apply all "with this" colocations
	pcmk__coloc_select_default = 0,

	// Apply "this with" colocations instead of "with this" colocations
	pcmk__coloc_select_this_with = (1 << 0),

	// Apply only colocations with non-negative scores
	pcmk__coloc_select_nonnegative = (1 << 1),

	// Apply only colocations with at least one matching node
	pcmk__coloc_select_active = (1 << 2),
	};

	// Flags the update_ordered_actions() method can return
	enum pcmk__updated {
	pcmk__updated_none = 0, // Nothing changed
	pcmk__updated_first = (1 << 0), // First action was updated
	pcmk__updated_then = (1 << 1), // Then action was updated
	};

	#define pcmk__set_updated_flags(au_flags, action, flags_to_set) do { \
	au_flags = pcmk__set_flags_as(__func__, __LINE__, \
	LOG_TRACE, "Action update", \
	(action)->uuid, au_flags, \
	(flags_to_set), #flags_to_set); \
	} while (0)

	#define pcmk__clear_updated_flags(au_flags, action, flags_to_clear) do { \
	au_flags = pcmk__clear_flags_as(__func__, __LINE__, \
	LOG_TRACE, "Action update", \
	(action)->uuid, au_flags, \
	(flags_to_clear), #flags_to_clear); \
	} while (0)

	// Resource assignment methods
	struct pcmk__assignment_methods {
	/*!
	* \internal
	* \brief Assign a resource to a node
	*
	* \param[in,out] rsc Resource to assign to a node
	* \param[in] prefer Node to prefer, if all else is equal
	* \param[in] stop_if_fail If \c true and \p rsc can't be assigned to a
	* node, set next role to stopped and update
	* existing actions (if \p rsc is not a
	* primitive, this applies to its primitive
	* descendants instead)
	*
	* \return Node that \p rsc is assigned to, if assigned entirely to one node
	*
	* \note If \p stop_if_fail is \c false, then \c pcmk__unassign_resource()
	* can completely undo the assignment. A successful assignment can be
	* either undone or left alone as final. A failed assignment has the
	* same effect as calling pcmk__unassign_resource(); there are no side
	* effects on roles or actions.
	*/
	pcmk_node_t (assign)(pcmk_resource_t rsc, const pcmk_node_t prefer,
	bool stop_if_fail);

	/*!
	* \internal
	* \brief Create all actions needed for a given resource
	*
	* \param[in,out] rsc Resource to create actions for
	*/
	void (create_actions)(pcmk_resource_t rsc);

	/*!
	* \internal
	* \brief Schedule any probes needed for a resource on a node
	*
	* \param[in,out] rsc Resource to create probe for
	* \param[in,out] node Node to create probe on
	*
	* \return true if any probe was created, otherwise false
	*/
	bool (create_probe)(pcmk_resource_t rsc, pcmk_node_t *node);

	/*!
	* \internal
	* \brief Create implicit constraints needed for a resource
	*
	* \param[in,out] rsc Resource to create implicit constraints for
	*/
	void (internal_constraints)(pcmk_resource_t rsc);

	/*!
	* \internal
	* \brief Apply a colocation's score to node scores or resource priority
	*
	* Given a colocation constraint, apply its score to the dependent's
	* allowed node scores (if we are still placing resources) or priority (if
	* we are choosing promotable clone instance roles).
	*
	* \param[in,out] dependent Dependent resource in colocation
	* \param[in] primary Primary resource in colocation
	* \param[in] colocation Colocation constraint to apply
	* \param[in] for_dependent true if called on behalf of dependent
	*
	* \return The score added to the dependent's priority
	*/
	int (apply_coloc_score)(pcmk_resource_t dependent,
	const pcmk_resource_t *primary,
	const pcmk__colocation_t *colocation,
	bool for_dependent);

	/*!
	* \internal
	* \brief Create list of all resources in colocations with a given resource
	*
	* Given a resource, create a list of all resources involved in mandatory
	* colocations with it, whether directly or via chained colocations.
	*
	* \param[in] rsc Resource to add to colocated list
	* \param[in] orig_rsc Resource originally requested
	* \param[in,out] colocated_rscs Existing list
	*
	* \return List of given resource and all resources involved in colocations
	*
	* \note This function is recursive; top-level callers should pass NULL as
	* \p colocated_rscs and \p orig_rsc, and the desired resource as
	* \p rsc. The recursive calls will use other values.
	*/
	GList (colocated_resources)(const pcmk_resource_t *rsc,
	const pcmk_resource_t *orig_rsc,
	GList *colocated_rscs);

	/*!
	* \internal
	* \brief Add colocations affecting a resource as primary to a list
	*
	* Given a resource being assigned (\p orig_rsc) and a resource somewhere in
	* its chain of ancestors (\p rsc, which may be \p orig_rsc), get
	* colocations that affect the ancestor as primary and should affect the
	* resource, and add them to a given list.
	*
	* \param[in] rsc Resource whose colocations should be added
	* \param[in] orig_rsc Affected resource (\p rsc or a descendant)
	* \param[in,out] list List of colocations to add to
	*
	* \note All arguments should be non-NULL.
	* \note The pcmk__with_this_colocations() wrapper should usually be used
	* instead of using this method directly.
	*/
	void (with_this_colocations)(const pcmk_resource_t rsc,
	const pcmk_resource_t *orig_rsc,
	GList **list);

	/*!
	* \internal
	* \brief Add colocations affecting a resource as dependent to a list
	*
	* Given a resource being assigned (\p orig_rsc) and a resource somewhere in
	* its chain of ancestors (\p rsc, which may be \p orig_rsc), get
	* colocations that affect the ancestor as dependent and should affect the
	* resource, and add them to a given list.
	*
	*
	* \param[in] rsc Resource whose colocations should be added
	* \param[in] orig_rsc Affected resource (\p rsc or a descendant)
	* \param[in,out] list List of colocations to add to
	*
	* \note All arguments should be non-NULL.
	* \note The pcmk__this_with_colocations() wrapper should usually be used
	* instead of using this method directly.
	*/
	void (this_with_colocations)(const pcmk_resource_t rsc,
	const pcmk_resource_t *orig_rsc,
	GList **list);

	/*!
	* \internal
	* \brief Update nodes with scores of colocated resources' nodes
	*
	* Given a table of nodes and a resource, update the nodes' scores with the
	* scores of the best nodes matching the attribute used for each of the
	* resource's relevant colocations.
	*
	* \param[in,out] source_rsc Resource whose node scores to add
	* \param[in] target_rsc Resource on whose behalf to update \p *nodes
	* \param[in] log_id Resource ID for logs (if \c NULL, use
	* \p source_rsc ID)
	* \param[in,out] nodes Nodes to update (set initial contents to
	* \c NULL to copy allowed nodes from
	* \p source_rsc)
	* \param[in] colocation Original colocation constraint (used to get
	* configured primary resource's stickiness, and
	* to get colocation node attribute; if \c NULL,
	* <tt>source_rsc</tt>'s own matching node scores
	* will not be added, and \p *nodes must be
	* \c NULL as well)
	* \param[in] factor Incorporate scores multiplied by this factor
	* \param[in] flags Bitmask of enum pcmk__coloc_select values
	*
	* \note \c NULL \p target_rsc, \c NULL \p *nodes, \c NULL \p colocation,
	* and the \c pcmk__coloc_select_this_with flag are used together (and
	* only by \c cmp_resources()).
	* \note The caller remains responsible for freeing \p *nodes.
	*/
	void (add_colocated_node_scores)(pcmk_resource_t source_rsc,
	const pcmk_resource_t *target_rsc,
	const char log_id, GHashTable *nodes,
	const pcmk__colocation_t *colocation,
	float factor, uint32_t flags);

	/*!
	* \internal
	* \brief Apply a location constraint to a resource's allowed node scores
	*
	* \param[in,out] rsc Resource to apply constraint to
	* \param[in,out] location Location constraint to apply
	*/
	void (apply_location)(pcmk_resource_t rsc, pcmk__location_t *location);

	/*!
	* \internal
	* \brief Return action flags for a given resource action
	*
	* \param[in,out] action Action to get flags for
	* \param[in] node If not NULL, limit effects to this node
	*
	* \return Flags appropriate to \p action on \p node
	* \note For primitives, this will be the same as action->flags regardless
	* of node. For collective resources, the flags can differ due to
	* multiple instances possibly being involved.
	*/
	uint32_t (action_flags)(pcmk_action_t action, const pcmk_node_t *node);

	/*!
	* \internal
	* \brief Update two actions according to an ordering between them
	*
	* Given information about an ordering of two actions, update the actions'
	* flags (and runnable_before members if appropriate) as appropriate for the
	* ordering. Effects may cascade to other orderings involving the actions as
	* well.
	*
	* \param[in,out] first 'First' action in an ordering
	* \param[in,out] then 'Then' action in an ordering
	* \param[in] node If not NULL, limit scope of ordering to this
	* node (only used when interleaving instances)
	* \param[in] flags Action flags for \p first for ordering purposes
	* \param[in] filter Action flags to limit scope of certain updates
	* (may include pcmk__action_optional to affect
	* only mandatory actions and
	* pcmk__action_runnable to affect only runnable
	* actions)
	* \param[in] type Group of enum pcmk__action_relation_flags
	* \param[in,out] scheduler Scheduler data
	*
	* \return Group of enum pcmk__updated flags indicating what was updated
	*/
	uint32_t (update_ordered_actions)(pcmk_action_t first,
	pcmk_action_t *then,
	const pcmk_node_t *node, uint32_t flags,
	uint32_t filter, uint32_t type,
	pcmk_scheduler_t *scheduler);

	/*!
	* \internal
	* \brief Output a summary of scheduled actions for a resource
	*
	* \param[in,out] rsc Resource to output actions for
	*/
	void (output_actions)(pcmk_resource_t rsc);

	/*!
	* \internal
	* \brief Add a resource's actions to the transition graph
	*
	* \param[in,out] rsc Resource whose actions should be added
	*/
	void (add_actions_to_graph)(pcmk_resource_t rsc);

	/*!
	* \internal
	* \brief Add meta-attributes relevant to transition graph actions to XML
	*
	* If a given resource supports variant-specific meta-attributes that are
	* needed for transition graph actions, add them to a given XML element.
	*
	* \param[in] rsc Resource whose meta-attributes should be added
	* \param[in,out] xml Transition graph action attributes XML to add to
	*/
	void (add_graph_meta)(const pcmk_resource_t rsc, xmlNode *xml);

	/*!
	* \internal
	* \brief Add a resource's utilization to a table of utilization values
	*
	* This function is used when summing the utilization of a resource and all
	* resources colocated with it, to determine whether a node has sufficient
	* capacity. Given a resource and a table of utilization values, it will add
	* the resource's utilization to the existing values, if the resource has
	* not yet been assigned to a node.
	*
	* \param[in] rsc Resource with utilization to add
	* \param[in] orig_rsc Resource being assigned (for logging only)
	* \param[in] all_rscs List of all resources that will be summed
	* \param[in,out] utilization Table of utilization values to add to
	*/
	void (add_utilization)(const pcmk_resource_t rsc,
	const pcmk_resource_t orig_rsc, GList all_rscs,
	GHashTable *utilization);

	/*!
	* \internal
	* \brief Apply a shutdown lock for a resource, if appropriate
	*
	* \param[in,out] rsc Resource to check for shutdown lock
	*/
	void (shutdown_lock)(pcmk_resource_t rsc);
	};

	// Actions (pcmk_sched_actions.c)

	G_GNUC_INTERNAL
	void pcmk__update_action_for_orderings(pcmk_action_t *action,
	pcmk_scheduler_t *scheduler);

	G_GNUC_INTERNAL
	uint32_t pcmk__update_ordered_actions(pcmk_action_t first, pcmk_action_t then,
	const pcmk_node_t *node, uint32_t flags,
	uint32_t filter, uint32_t type,
	pcmk_scheduler_t *scheduler);

	G_GNUC_INTERNAL
	void pcmk__log_action(const char pre_text, const pcmk_action_t action,
	bool details);

	G_GNUC_INTERNAL
	pcmk_action_t pcmk__new_cancel_action(pcmk_resource_t rsc, const char *name,
	guint interval_ms,
	const pcmk_node_t *node);

	G_GNUC_INTERNAL
	pcmk_action_t pcmk__new_shutdown_action(pcmk_node_t node);

	G_GNUC_INTERNAL
	bool pcmk__action_locks_rsc_to_node(const pcmk_action_t *action);

	G_GNUC_INTERNAL
	void pcmk__deduplicate_action_inputs(pcmk_action_t *action);

	G_GNUC_INTERNAL
	void pcmk__output_actions(pcmk_scheduler_t *scheduler);

	G_GNUC_INTERNAL
	bool pcmk__check_action_config(pcmk_resource_t rsc, pcmk_node_t node,
	const xmlNode *xml_op);

	G_GNUC_INTERNAL
	void pcmk__handle_rsc_config_changes(pcmk_scheduler_t *scheduler);


	// Recurring actions (pcmk_sched_recurring.c)

	G_GNUC_INTERNAL
	void pcmk__create_recurring_actions(pcmk_resource_t *rsc);

	G_GNUC_INTERNAL
	void pcmk__schedule_cancel(pcmk_resource_t rsc, const char call_id,
	const char *task, guint interval_ms,
	const pcmk_node_t node, const char reason);

	G_GNUC_INTERNAL
	void pcmk__reschedule_recurring(pcmk_resource_t rsc, const char task,
	guint interval_ms, pcmk_node_t *node);

	G_GNUC_INTERNAL
	bool pcmk__action_is_recurring(const pcmk_action_t *action);


	// Producing transition graphs (pcmk_graph_producer.c)

	G_GNUC_INTERNAL
	bool pcmk__graph_has_loop(const pcmk_action_t *init_action,
	const pcmk_action_t *action,
	pcmk__related_action_t *input);

	G_GNUC_INTERNAL
	void pcmk__add_rsc_actions_to_graph(pcmk_resource_t *rsc);

	G_GNUC_INTERNAL
	void pcmk__create_graph(pcmk_scheduler_t *scheduler);


	// Fencing (pcmk_sched_fencing.c)

	G_GNUC_INTERNAL
	void pcmk__order_vs_fence(pcmk_action_t *stonith_op,
	pcmk_scheduler_t *scheduler);

	G_GNUC_INTERNAL
	void pcmk__order_vs_unfence(const pcmk_resource_t rsc, pcmk_node_t node,
	pcmk_action_t *action,
	enum pcmk__action_relation_flags order);

	G_GNUC_INTERNAL
	void pcmk__fence_guest(pcmk_node_t *node);

	G_GNUC_INTERNAL
	bool pcmk__node_unfenced(const pcmk_node_t *node);

	G_GNUC_INTERNAL
	void pcmk__order_restart_vs_unfence(gpointer data, gpointer user_data);


	// Injected scheduler inputs (pcmk_sched_injections.c)

	G_GNUC_INTERNAL
	void pcmk__inject_scheduler_input(pcmk_scheduler_t scheduler, cib_t cib,
	const pcmk_injections_t *injections);


	// Constraints of any type (pcmk_sched_constraints.c)

	G_GNUC_INTERNAL
	pcmk_resource_t pcmk__find_constraint_resource(GList rsc_list,
	const char *id);

	G_GNUC_INTERNAL
	int pcmk__parse_constraint_role(const char id, const char role_spec,
	enum rsc_role_e *role);

	G_GNUC_INTERNAL
	xmlNode pcmk__expand_tags_in_sets(xmlNode xml_obj,
	const pcmk_scheduler_t *scheduler);

	G_GNUC_INTERNAL
	bool pcmk__valid_resource_or_tag(const pcmk_scheduler_t *scheduler,
	const char id, pcmk_resource_t *rsc,
	pcmk__idref_t **tag);

	G_GNUC_INTERNAL
	bool pcmk__tag_to_set(xmlNode xml_obj, xmlNode rsc_set, const char attr,
	bool convert_rsc, const pcmk_scheduler_t *scheduler);

	G_GNUC_INTERNAL
	void pcmk__create_internal_constraints(pcmk_scheduler_t *scheduler);


	// Location constraints

	G_GNUC_INTERNAL
	void pcmk__unpack_location(xmlNode xml_obj, pcmk_scheduler_t scheduler);

	G_GNUC_INTERNAL
	pcmk__location_t pcmk__new_location(const char id, pcmk_resource_t *rsc,
	int node_score, const char *discover_mode,
	pcmk_node_t *foo_node);

	G_GNUC_INTERNAL
	void pcmk__apply_locations(pcmk_scheduler_t *scheduler);

	G_GNUC_INTERNAL
	void pcmk__apply_location(pcmk_resource_t rsc, pcmk__location_t constraint);


	// Colocation constraints (pcmk_sched_colocation.c)

	enum pcmk__coloc_affects {
	pcmk__coloc_affects_nothing = 0,
	pcmk__coloc_affects_location,
	pcmk__coloc_affects_role,
	};

	G_GNUC_INTERNAL
	const char pcmk__colocation_node_attr(const pcmk_node_t node,
	const char *attr,
	const pcmk_resource_t *rsc);

	G_GNUC_INTERNAL
	enum pcmk__coloc_affects pcmk__colocation_affects(const pcmk_resource_t
	*dependent,
	const pcmk_resource_t
	*primary,
	const pcmk__colocation_t
	*colocation,
	bool preview);

	G_GNUC_INTERNAL
	void pcmk__apply_coloc_to_scores(pcmk_resource_t *dependent,
	const pcmk_resource_t *primary,
	const pcmk__colocation_t *colocation);

	G_GNUC_INTERNAL
	int pcmk__apply_coloc_to_priority(pcmk_resource_t *dependent,
	const pcmk_resource_t *primary,
	const pcmk__colocation_t *colocation);

	G_GNUC_INTERNAL
	void pcmk__add_colocated_node_scores(pcmk_resource_t *source_rsc,
	const pcmk_resource_t *target_rsc,
	const char log_id, GHashTable *nodes,
	const pcmk__colocation_t *colocation,
	float factor, uint32_t flags);

	G_GNUC_INTERNAL
	void pcmk__add_dependent_scores(gpointer data, gpointer user_data);

	G_GNUC_INTERNAL
	void pcmk__colocation_intersect_nodes(pcmk_resource_t *dependent,
	const pcmk_resource_t *primary,
	const pcmk__colocation_t *colocation,
	const GList *primary_nodes,
	bool merge_scores);

	G_GNUC_INTERNAL
	void pcmk__unpack_colocation(xmlNode xml_obj, pcmk_scheduler_t scheduler);

	G_GNUC_INTERNAL
	void pcmk__add_this_with(GList *list, const pcmk__colocation_t colocation,
	const pcmk_resource_t *rsc);

	G_GNUC_INTERNAL
	void pcmk__add_this_with_list(GList *list, GList addition,
	const pcmk_resource_t *rsc);

	G_GNUC_INTERNAL
	void pcmk__add_with_this(GList *list, const pcmk__colocation_t colocation,
	const pcmk_resource_t *rsc);

	G_GNUC_INTERNAL
	void pcmk__add_with_this_list(GList *list, GList addition,
	const pcmk_resource_t *rsc);

	G_GNUC_INTERNAL
	GList pcmk__with_this_colocations(const pcmk_resource_t rsc);

	G_GNUC_INTERNAL
	GList pcmk__this_with_colocations(const pcmk_resource_t rsc);

	G_GNUC_INTERNAL
	void pcmk__new_colocation(const char id, const char node_attr, int score,
	pcmk_resource_t dependent, pcmk_resource_t primary,
	const char *dependent_role_spec,
	const char *primary_role_spec, uint32_t flags);

	G_GNUC_INTERNAL
	void pcmk__block_colocation_dependents(pcmk_action_t *action);

	G_GNUC_INTERNAL
	bool pcmk__colocation_has_influence(const pcmk__colocation_t *colocation,
	const pcmk_resource_t *rsc);


	// Ordering constraints (pcmk_sched_ordering.c)

	G_GNUC_INTERNAL
	void pcmk__new_ordering(pcmk_resource_t first_rsc, char first_task,
	pcmk_action_t first_action, pcmk_resource_t then_rsc,
	char then_task, pcmk_action_t then_action,
	uint32_t flags, pcmk_scheduler_t *sched);

	G_GNUC_INTERNAL
	void pcmk__unpack_ordering(xmlNode xml_obj, pcmk_scheduler_t scheduler);

	G_GNUC_INTERNAL
	void pcmk__disable_invalid_orderings(pcmk_scheduler_t *scheduler);

	G_GNUC_INTERNAL
	void pcmk__order_stops_before_shutdown(pcmk_node_t *node,
	pcmk_action_t *shutdown_op);

	G_GNUC_INTERNAL
	void pcmk__apply_orderings(pcmk_scheduler_t *sched);

	G_GNUC_INTERNAL
	void pcmk__order_after_each(pcmk_action_t after, GList list);


	/*!
	* \internal
	* \brief Create a new ordering between two resource actions
	*
	* \param[in,out] first_rsc Resource for 'first' action
	* \param[in,out] first_task Action key for 'first' action
	* \param[in] then_rsc Resource for 'then' action
	* \param[in,out] then_task Action key for 'then' action
	* \param[in] flags Group of enum pcmk__action_relation_flags
	*/
	#define pcmk__order_resource_actions(first_rsc, first_task, \
	then_rsc, then_task, flags) \
	pcmk__new_ordering((first_rsc), \
	pcmk__op_key((first_rsc)->id, (first_task), 0), \
	NULL, \
	(then_rsc), \
	pcmk__op_key((then_rsc)->id, (then_task), 0), \
	NULL, (flags), (first_rsc)->priv->scheduler)

	#define pcmk__order_starts(rsc1, rsc2, flags) \
	pcmk__order_resource_actions((rsc1), PCMK_ACTION_START, \
	(rsc2), PCMK_ACTION_START, (flags))

	#define pcmk__order_stops(rsc1, rsc2, flags) \
	pcmk__order_resource_actions((rsc1), PCMK_ACTION_STOP, \
	(rsc2), PCMK_ACTION_STOP, (flags))


	// Ticket constraints (pcmk_sched_tickets.c)

	G_GNUC_INTERNAL
	void pcmk__unpack_rsc_ticket(xmlNode xml_obj, pcmk_scheduler_t scheduler);


	// Promotable clone resources (pcmk_sched_promotable.c)

	G_GNUC_INTERNAL
	void pcmk__add_promotion_scores(pcmk_resource_t *rsc);

	G_GNUC_INTERNAL
	void pcmk__require_promotion_tickets(pcmk_resource_t *rsc);

	G_GNUC_INTERNAL
	void pcmk__set_instance_roles(pcmk_resource_t *rsc);

	G_GNUC_INTERNAL
	void pcmk__create_promotable_actions(pcmk_resource_t *clone);

	G_GNUC_INTERNAL
	void pcmk__promotable_restart_ordering(pcmk_resource_t *rsc);

	G_GNUC_INTERNAL
	void pcmk__order_promotable_instances(pcmk_resource_t *clone);

	G_GNUC_INTERNAL
	void pcmk__update_dependent_with_promotable(const pcmk_resource_t *primary,
	pcmk_resource_t *dependent,
	const pcmk__colocation_t
	*colocation);

	G_GNUC_INTERNAL
	int pcmk__update_promotable_dependent_priority(const pcmk_resource_t *primary,
	pcmk_resource_t *dependent,
	const pcmk__colocation_t
	*colocation);


	// Pacemaker Remote nodes (pcmk_sched_remote.c)

	G_GNUC_INTERNAL
	bool pcmk__is_failed_remote_node(const pcmk_node_t *node);

	G_GNUC_INTERNAL
	void pcmk__order_remote_connection_actions(pcmk_scheduler_t *scheduler);

	G_GNUC_INTERNAL
	bool pcmk__rsc_corresponds_to_guest(const pcmk_resource_t *rsc,
	const pcmk_node_t *node);

	G_GNUC_INTERNAL
	pcmk_node_t pcmk__connection_host_for_action(const pcmk_action_t action);

	G_GNUC_INTERNAL
	void pcmk__substitute_remote_addr(pcmk_resource_t rsc, GHashTable params);

	G_GNUC_INTERNAL
	void pcmk__add_guest_meta_to_xml(xmlNode *args_xml,
	const pcmk_action_t *action);


	// Primitives (pcmk_sched_primitive.c)

	G_GNUC_INTERNAL
	pcmk_node_t pcmk__primitive_assign(pcmk_resource_t rsc,
	const pcmk_node_t *prefer,
	bool stop_if_fail);

	G_GNUC_INTERNAL
	void pcmk__primitive_create_actions(pcmk_resource_t *rsc);

	G_GNUC_INTERNAL
	void pcmk__primitive_internal_constraints(pcmk_resource_t *rsc);

	G_GNUC_INTERNAL
	uint32_t pcmk__primitive_action_flags(pcmk_action_t *action,
	const pcmk_node_t *node);

	G_GNUC_INTERNAL
	int pcmk__primitive_apply_coloc_score(pcmk_resource_t *dependent,
	const pcmk_resource_t *primary,
	const pcmk__colocation_t *colocation,
	bool for_dependent);

	G_GNUC_INTERNAL
	void pcmk__with_primitive_colocations(const pcmk_resource_t *rsc,
	const pcmk_resource_t *orig_rsc,
	GList **list);

	G_GNUC_INTERNAL
	void pcmk__primitive_with_colocations(const pcmk_resource_t *rsc,
	const pcmk_resource_t *orig_rsc,
	GList **list);

	G_GNUC_INTERNAL
	void pcmk__schedule_cleanup(pcmk_resource_t rsc, const pcmk_node_t node,
	bool optional);

	G_GNUC_INTERNAL
	void pcmk__primitive_add_graph_meta(const pcmk_resource_t rsc, xmlNode xml);

	G_GNUC_INTERNAL
	void pcmk__primitive_add_utilization(const pcmk_resource_t *rsc,
	const pcmk_resource_t *orig_rsc,
	GList all_rscs, GHashTable utilization);

	G_GNUC_INTERNAL
	void pcmk__primitive_shutdown_lock(pcmk_resource_t *rsc);


	// Groups (pcmk_sched_group.c)

	G_GNUC_INTERNAL
	pcmk_node_t pcmk__group_assign(pcmk_resource_t rsc, const pcmk_node_t *prefer,
	bool stop_if_fail);

	G_GNUC_INTERNAL
	void pcmk__group_create_actions(pcmk_resource_t *rsc);

	G_GNUC_INTERNAL
	void pcmk__group_internal_constraints(pcmk_resource_t *rsc);

	G_GNUC_INTERNAL
	int pcmk__group_apply_coloc_score(pcmk_resource_t *dependent,
	const pcmk_resource_t *primary,
	const pcmk__colocation_t *colocation,
	bool for_dependent);

	G_GNUC_INTERNAL
	void pcmk__with_group_colocations(const pcmk_resource_t *rsc,
	const pcmk_resource_t *orig_rsc,
	GList **list);

	G_GNUC_INTERNAL
	void pcmk__group_with_colocations(const pcmk_resource_t *rsc,
	const pcmk_resource_t *orig_rsc,
	GList **list);

	G_GNUC_INTERNAL
	void pcmk__group_add_colocated_node_scores(pcmk_resource_t *source_rsc,
	const pcmk_resource_t *target_rsc,
	const char *log_id,
	GHashTable **nodes,
	const pcmk__colocation_t *colocation,
	float factor, uint32_t flags);

	G_GNUC_INTERNAL
	void pcmk__group_apply_location(pcmk_resource_t *rsc,
	pcmk__location_t *location);

	G_GNUC_INTERNAL
	uint32_t pcmk__group_action_flags(pcmk_action_t *action,
	const pcmk_node_t *node);

	G_GNUC_INTERNAL
	uint32_t pcmk__group_update_ordered_actions(pcmk_action_t *first,
	pcmk_action_t *then,
	const pcmk_node_t *node,
	uint32_t flags, uint32_t filter,
	uint32_t type,
	pcmk_scheduler_t *scheduler);

	G_GNUC_INTERNAL
	GList pcmk__group_colocated_resources(const pcmk_resource_t rsc,
	const pcmk_resource_t *orig_rsc,
	GList *colocated_rscs);

	G_GNUC_INTERNAL
	void pcmk__group_add_utilization(const pcmk_resource_t *rsc,
	const pcmk_resource_t *orig_rsc,
	GList all_rscs, GHashTable utilization);

	G_GNUC_INTERNAL
	void pcmk__group_shutdown_lock(pcmk_resource_t *rsc);


	// Clones (pcmk_sched_clone.c)

	G_GNUC_INTERNAL
	pcmk_node_t pcmk__clone_assign(pcmk_resource_t rsc, const pcmk_node_t *prefer,
	bool stop_if_fail);

	G_GNUC_INTERNAL
	void pcmk__clone_create_actions(pcmk_resource_t *rsc);

	G_GNUC_INTERNAL
	bool pcmk__clone_create_probe(pcmk_resource_t rsc, pcmk_node_t node);

	G_GNUC_INTERNAL
	void pcmk__clone_internal_constraints(pcmk_resource_t *rsc);

	G_GNUC_INTERNAL
	int pcmk__clone_apply_coloc_score(pcmk_resource_t *dependent,
	const pcmk_resource_t *primary,
	const pcmk__colocation_t *colocation,
	bool for_dependent);

	G_GNUC_INTERNAL
	void pcmk__with_clone_colocations(const pcmk_resource_t *rsc,
	const pcmk_resource_t *orig_rsc,
	GList **list);

	G_GNUC_INTERNAL
	void pcmk__clone_with_colocations(const pcmk_resource_t *rsc,
	const pcmk_resource_t *orig_rsc,
	GList **list);

	G_GNUC_INTERNAL
	void pcmk__clone_apply_location(pcmk_resource_t *rsc,
	pcmk__location_t *constraint);

	G_GNUC_INTERNAL
	uint32_t pcmk__clone_action_flags(pcmk_action_t *action,
	const pcmk_node_t *node);

	G_GNUC_INTERNAL
	void pcmk__clone_add_actions_to_graph(pcmk_resource_t *rsc);

	G_GNUC_INTERNAL
	void pcmk__clone_add_graph_meta(const pcmk_resource_t rsc, xmlNode xml);

	G_GNUC_INTERNAL
	void pcmk__clone_add_utilization(const pcmk_resource_t *rsc,
	const pcmk_resource_t *orig_rsc,
	GList all_rscs, GHashTable utilization);

	G_GNUC_INTERNAL
	void pcmk__clone_shutdown_lock(pcmk_resource_t *rsc);

	// Bundles (pcmk_sched_bundle.c)

	G_GNUC_INTERNAL
	pcmk_node_t pcmk__bundle_assign(pcmk_resource_t rsc,
	const pcmk_node_t *prefer, bool stop_if_fail);

	G_GNUC_INTERNAL
	void pcmk__bundle_create_actions(pcmk_resource_t *rsc);

	G_GNUC_INTERNAL
	bool pcmk__bundle_create_probe(pcmk_resource_t rsc, pcmk_node_t node);

	G_GNUC_INTERNAL
	void pcmk__bundle_internal_constraints(pcmk_resource_t *rsc);

	G_GNUC_INTERNAL
	int pcmk__bundle_apply_coloc_score(pcmk_resource_t *dependent,
	const pcmk_resource_t *primary,
	const pcmk__colocation_t *colocation,
	bool for_dependent);

	G_GNUC_INTERNAL
	void pcmk__with_bundle_colocations(const pcmk_resource_t *rsc,
	const pcmk_resource_t *orig_rsc,
	GList **list);

	G_GNUC_INTERNAL
	void pcmk__bundle_with_colocations(const pcmk_resource_t *rsc,
	const pcmk_resource_t *orig_rsc,
	GList **list);

	G_GNUC_INTERNAL
	void pcmk__bundle_apply_location(pcmk_resource_t *rsc,
	pcmk__location_t *constraint);

	G_GNUC_INTERNAL
	uint32_t pcmk__bundle_action_flags(pcmk_action_t *action,
	const pcmk_node_t *node);

	G_GNUC_INTERNAL
	void pcmk__output_bundle_actions(pcmk_resource_t *rsc);

	G_GNUC_INTERNAL
	void pcmk__bundle_add_actions_to_graph(pcmk_resource_t *rsc);

	G_GNUC_INTERNAL
	void pcmk__bundle_add_utilization(const pcmk_resource_t *rsc,
	const pcmk_resource_t *orig_rsc,
	GList all_rscs, GHashTable utilization);

	G_GNUC_INTERNAL
	void pcmk__bundle_shutdown_lock(pcmk_resource_t *rsc);


	// Clone instances or bundle replica containers (pcmk_sched_instances.c)

	G_GNUC_INTERNAL
	void pcmk__assign_instances(pcmk_resource_t collective, GList instances,
	int max_total, int max_per_node);

	G_GNUC_INTERNAL
	void pcmk__create_instance_actions(pcmk_resource_t rsc, GList instances);

	G_GNUC_INTERNAL
	bool pcmk__instance_matches(const pcmk_resource_t *instance,
	const pcmk_node_t *node, enum rsc_role_e role,
	bool current);

	G_GNUC_INTERNAL
	pcmk_resource_t pcmk__find_compatible_instance(const pcmk_resource_t match_rsc,
	const pcmk_resource_t *rsc,
	enum rsc_role_e role,
	bool current);

	G_GNUC_INTERNAL
	uint32_t pcmk__instance_update_ordered_actions(pcmk_action_t *first,
	pcmk_action_t *then,
	const pcmk_node_t *node,
	uint32_t flags, uint32_t filter,
	uint32_t type,
	pcmk_scheduler_t *scheduler);

	G_GNUC_INTERNAL
	uint32_t pcmk__collective_action_flags(pcmk_action_t *action,
	const GList *instances,
	const pcmk_node_t *node);


	// Injections (pcmk_injections.c)

	G_GNUC_INTERNAL
	xmlNode pcmk__inject_node(cib_t cib_conn, const char node, const char uuid);

	G_GNUC_INTERNAL
	xmlNode pcmk__inject_node_state_change(cib_t cib_conn, const char *node,
	bool up);

	G_GNUC_INTERNAL
	xmlNode pcmk__inject_resource_history(pcmk__output_t out, xmlNode *cib_node,
	const char *resource,
	const char *lrm_name,
	const char *rclass,
	const char *rtype,
	const char *rprovider);

	G_GNUC_INTERNAL
	void pcmk__inject_failcount(pcmk__output_t out, cib_t cib_conn,
	xmlNode cib_node, const char resource,
	const char *task, guint interval_ms, int rc);

	G_GNUC_INTERNAL
	xmlNode pcmk__inject_action_result(xmlNode cib_resource,
	lrmd_event_data_t op, const char node,
	int target_rc);


	// Nodes (pcmk_sched_nodes.c)

	G_GNUC_INTERNAL
	bool pcmk__node_available(const pcmk_node_t *node, bool consider_score,
	bool consider_guest);

	G_GNUC_INTERNAL
	bool pcmk__any_node_available(GHashTable *nodes);

	G_GNUC_INTERNAL
	GHashTable pcmk__copy_node_table(GHashTable nodes);

	G_GNUC_INTERNAL
	void pcmk__copy_node_tables(const pcmk_resource_t rsc, GHashTable *copy);

	G_GNUC_INTERNAL
	void pcmk__restore_node_tables(pcmk_resource_t rsc, GHashTable backup);

	G_GNUC_INTERNAL
	GList pcmk__sort_nodes(GList nodes, pcmk_node_t *active_node);

	G_GNUC_INTERNAL
	void pcmk__apply_node_health(pcmk_scheduler_t *scheduler);

	G_GNUC_INTERNAL
	pcmk_node_t pcmk__top_allowed_node(const pcmk_resource_t rsc,
	const pcmk_node_t *node);


	// Functions applying to more than one variant (pcmk_sched_resource.c)

	G_GNUC_INTERNAL
	void pcmk__set_assignment_methods(pcmk_scheduler_t *scheduler);

	G_GNUC_INTERNAL
	bool pcmk__rsc_agent_changed(pcmk_resource_t rsc, pcmk_node_t node,
	const xmlNode *rsc_entry, bool active_on_node);

	G_GNUC_INTERNAL
	GList pcmk__rscs_matching_id(const char id,
	const pcmk_scheduler_t *scheduler);

	G_GNUC_INTERNAL
	GList pcmk__colocated_resources(const pcmk_resource_t rsc,
	const pcmk_resource_t *orig_rsc,
	GList *colocated_rscs);

	G_GNUC_INTERNAL
	void pcmk__noop_add_graph_meta(const pcmk_resource_t rsc, xmlNode xml);

	G_GNUC_INTERNAL
	void pcmk__output_resource_actions(pcmk_resource_t *rsc);

	G_GNUC_INTERNAL
	bool pcmk__assign_resource(pcmk_resource_t rsc, pcmk_node_t node, bool force,
	bool stop_if_fail);

	G_GNUC_INTERNAL
	void pcmk__unassign_resource(pcmk_resource_t *rsc);

	G_GNUC_INTERNAL
	bool pcmk__threshold_reached(pcmk_resource_t rsc, const pcmk_node_t node,
	pcmk_resource_t **failed);

	G_GNUC_INTERNAL
	void pcmk__sort_resources(pcmk_scheduler_t *scheduler);

	G_GNUC_INTERNAL
	gint pcmk__cmp_instance(gconstpointer a, gconstpointer b);

	G_GNUC_INTERNAL
	gint pcmk__cmp_instance_number(gconstpointer a, gconstpointer b);


	// Functions related to probes (pcmk_sched_probes.c)

	G_GNUC_INTERNAL
	bool pcmk__probe_rsc_on_node(pcmk_resource_t rsc, pcmk_node_t node);

	G_GNUC_INTERNAL
	void pcmk__order_probes(pcmk_scheduler_t *scheduler);

	G_GNUC_INTERNAL
	bool pcmk__probe_resource_list(GList rscs, pcmk_node_t node);

	G_GNUC_INTERNAL
	void pcmk__schedule_probes(pcmk_scheduler_t *scheduler);


	// Functions related to live migration (pcmk_sched_migration.c)

	void pcmk__create_migration_actions(pcmk_resource_t *rsc,
	const pcmk_node_t *current);

	void pcmk__abort_dangling_migration(void data, void user_data);

	bool pcmk__rsc_can_migrate(const pcmk_resource_t *rsc,
	const pcmk_node_t *current);

	void pcmk__order_migration_equivalents(pcmk__action_relation_t *order);


	// Functions related to node utilization (pcmk_sched_utilization.c)

	G_GNUC_INTERNAL
	int pcmk__compare_node_capacities(const pcmk_node_t *node1,
	const pcmk_node_t *node2);

	G_GNUC_INTERNAL
	void pcmk__consume_node_capacity(GHashTable *current_utilization,
	const pcmk_resource_t *rsc);

	G_GNUC_INTERNAL
	void pcmk__release_node_capacity(GHashTable *current_utilization,
	const pcmk_resource_t *rsc);

	G_GNUC_INTERNAL
	const pcmk_node_t pcmk__ban_insufficient_capacity(pcmk_resource_t rsc);

	G_GNUC_INTERNAL
	void pcmk__create_utilization_constraints(pcmk_resource_t *rsc,
	const GList *allowed_nodes);

	G_GNUC_INTERNAL
	void pcmk__show_node_capacities(const char desc, pcmk_scheduler_t scheduler);


	// Functions related to the scheduler (pcmk_scheduler.c)

	G_GNUC_INTERNAL
	int pcmk__init_scheduler(pcmk__output_t out, xmlNodePtr input, const crm_time_t date,
	pcmk_scheduler_t **scheduler);


	// General setup functions (pcmk_setup.c)

	G_GNUC_INTERNAL
	int pcmk__setup_output_cib_sched(pcmk__output_t out, cib_t cib,
	pcmk_scheduler_t scheduler, xmlNode xml);

	+G_GNUC_INTERNAL
	+int pcmk__setup_output_fencing(pcmk__output_t out, stonith_t st, xmlNode **xml);
	+
	#ifdef __cplusplus
	}
	#endif

	#endif // PCMK__PACEMAKER_LIBPACEMAKER_PRIVATE__H
	diff --git a/lib/pacemaker/pcmk_fence.c b/lib/pacemaker/pcmk_fence.c
	index f8aa67d4e8..e2d6d0e794 100644
	--- a/lib/pacemaker/pcmk_fence.c
	+++ b/lib/pacemaker/pcmk_fence.c
	@@ -1,635 +1,677 @@
	/*
	* Copyright 2009-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>
	#include <crm/common/mainloop.h>
	#include <crm/common/results.h>
	#include <crm/common/output.h>
	#include <crm/common/output_internal.h>
	#include <crm/stonith-ng.h>
	#include <crm/fencing/internal.h> // stonith__*

	#include <glib.h>
	#include <libxml/tree.h>
	#include <pacemaker.h>
	#include <pacemaker-internal.h>

	+#include "libpacemaker_private.h"
	+
	static const int st_opts = st_opt_sync_call\|st_opt_allow_self_fencing;

	static GMainLoop *mainloop = NULL;

	static struct {
	stonith_t *st;
	const char *target;
	const char *action;
	char *name;
	unsigned int timeout;
	unsigned int tolerance;
	int delay;
	pcmk__action_result_t result;
	} async_fence_data = { NULL, };

	static int
	-handle_level(stonith_t st, const char target, int fence_level,
	- const stonith_key_value_t *devices, bool added)
	+handle_level(stonith_t st, const char target, int fence_level, GList *devices,
	+ bool added)
	{
	const char *node = NULL;
	const char *pattern = NULL;
	const char *name = NULL;
	char *value = NULL;
	int rc = pcmk_rc_ok;

	if (target == NULL) {
	// Not really possible, but makes static analysis happy
	return EINVAL;
	}

	/* Determine if targeting by attribute, node name pattern or node name */
	value = strchr(target, '=');
	if (value != NULL) {
	name = target;
	*value++ = '\0';
	} else if (*target == '@') {
	pattern = target + 1;
	} else {
	node = target;
	}

	/* Register or unregister level as appropriate */
	if (added) {
	- rc = st->cmds->register_level_full(st, st_opts, node, pattern,
	- name, value, fence_level,
	- devices);
	+ stonith_key_value_t *kvs = NULL;
	+
	+ for (GList *iter = devices; iter != NULL; iter = iter->next) {
	+ kvs = stonith_key_value_add(kvs, NULL, iter->data);
	+ }
	+
	+ rc = st->cmds->register_level_full(st, st_opts, node, pattern, name,
	+ value, fence_level, kvs);
	+ stonith_key_value_freeall(kvs, 0, 1);
	} else {
	rc = st->cmds->remove_level_full(st, st_opts, node, pattern,
	name, value, fence_level);
	}

	return pcmk_legacy2rc(rc);
	}

	static stonith_history_t *
	reduce_fence_history(stonith_history_t *history)
	{
	stonith_history_t new, hp, *np;

	if (!history) {
	return history;
	}

	new = history;
	hp = new->next;
	new->next = NULL;

	while (hp) {
	stonith_history_t *hp_next = hp->next;

	hp->next = NULL;

	for (np = new; ; np = np->next) {
	if ((hp->state == st_done) \|\| (hp->state == st_failed)) {
	/* action not in progress */
	if (pcmk__str_eq(hp->target, np->target, pcmk__str_casei)
	&& pcmk__str_eq(hp->action, np->action, pcmk__str_none)
	&& (hp->state == np->state)
	&& ((hp->state == st_done)
	\|\| pcmk__str_eq(hp->delegate, np->delegate,
	pcmk__str_casei))) {
	/* purge older hp */
	stonith_history_free(hp);
	break;
	}
	}

	if (!np->next) {
	np->next = hp;
	break;
	}
	}
	hp = hp_next;
	}

	return new;
	}

	static void
	notify_callback(stonith_t * st, stonith_event_t * e)
	{
	if (pcmk__str_eq(async_fence_data.target, e->target, pcmk__str_casei)
	&& pcmk__str_eq(async_fence_data.action, e->action, pcmk__str_none)) {

	pcmk__set_result(&async_fence_data.result,
	stonith__event_exit_status(e),
	stonith__event_execution_status(e),
	stonith__event_exit_reason(e));
	g_main_loop_quit(mainloop);
	}
	}

	static void
	fence_callback(stonith_t * stonith, stonith_callback_data_t * data)
	{
	pcmk__set_result(&async_fence_data.result, stonith__exit_status(data),
	stonith__execution_status(data),
	stonith__exit_reason(data));
	g_main_loop_quit(mainloop);
	}

	static gboolean
	async_fence_helper(gpointer user_data)
	{
	stonith_t *st = async_fence_data.st;
	int call_id = 0;
	int rc = stonith_api_connect_retry(st, async_fence_data.name, 10);
	int timeout = 0;

	if (rc != pcmk_ok) {
	g_main_loop_quit(mainloop);
	pcmk__set_result(&async_fence_data.result, CRM_EX_ERROR,
	PCMK_EXEC_NOT_CONNECTED, pcmk_strerror(rc));
	return TRUE;
	}

	st->cmds->register_notification(st, PCMK__VALUE_ST_NOTIFY_FENCE,
	notify_callback);

	call_id = st->cmds->fence_with_delay(st,
	st_opt_allow_self_fencing,
	async_fence_data.target,
	async_fence_data.action,
	async_fence_data.timeout/1000,
	async_fence_data.tolerance/1000,
	async_fence_data.delay);

	if (call_id < 0) {
	g_main_loop_quit(mainloop);
	pcmk__set_result(&async_fence_data.result, CRM_EX_ERROR,
	PCMK_EXEC_ERROR, pcmk_strerror(call_id));
	return TRUE;
	}

	timeout = async_fence_data.timeout / 1000;
	if (async_fence_data.delay > 0) {
	timeout += async_fence_data.delay;
	}
	st->cmds->register_callback(st, call_id, timeout, st_opt_timeout_updates,
	NULL, "callback", fence_callback);
	return TRUE;
	}

	int
	pcmk__request_fencing(stonith_t st, const char target, const char *action,
	const char *name, unsigned int timeout,
	unsigned int tolerance, int delay, char **reason)
	{
	crm_trigger_t *trig;
	int rc = pcmk_rc_ok;

	async_fence_data.st = st;
	async_fence_data.name = strdup(name);
	async_fence_data.target = target;
	async_fence_data.action = action;
	async_fence_data.timeout = timeout;
	async_fence_data.tolerance = tolerance;
	async_fence_data.delay = delay;
	pcmk__set_result(&async_fence_data.result, CRM_EX_ERROR, PCMK_EXEC_UNKNOWN,
	NULL);

	trig = mainloop_add_trigger(G_PRIORITY_HIGH, async_fence_helper, NULL);
	mainloop_set_trigger(trig);

	mainloop = g_main_loop_new(NULL, FALSE);
	g_main_loop_run(mainloop);

	free(async_fence_data.name);

	if (reason != NULL) {
	// Give the caller ownership of the exit reason
	*reason = async_fence_data.result.exit_reason;
	async_fence_data.result.exit_reason = NULL;
	}
	rc = stonith__result2rc(&async_fence_data.result);
	pcmk__reset_result(&async_fence_data.result);
	return rc;
	}

	-#ifdef BUILD_PUBLIC_LIBPACEMAKER
	int
	-pcmk_request_fencing(stonith_t st, const char target, const char *action,
	+pcmk_request_fencing(xmlNodePtr xml, const char target, const char *action,
	const char *name, unsigned int timeout,
	unsigned int tolerance, int delay, char **reason)
	{
	- return pcmk__request_fencing(st, target, action, name, timeout, tolerance,
	- delay, reason);
	+ stonith_t *st = NULL;
	+ pcmk__output_t *out = NULL;
	+ int rc = pcmk_rc_ok;
	+
	+ rc = pcmk__setup_output_fencing(&out, &st, xml);
	+ if (rc != pcmk_rc_ok) {
	+ return rc;
	+ }
	+
	+ rc = pcmk__request_fencing(st, target, action, name, timeout, tolerance,
	+ delay, reason);
	+ pcmk__xml_output_finish(out, pcmk_rc2exitc(rc), xml);
	+
	+ st->cmds->disconnect(st);
	+ stonith_api_delete(st);
	+ return rc;
	}
	-#endif

	int
	pcmk__fence_history(pcmk__output_t out, stonith_t st, const char *target,
	unsigned int timeout, int verbose, bool broadcast,
	bool cleanup)
	{
	- stonith_history_t history = NULL, hp, *latest = NULL;
	+ stonith_history_t *history = NULL;
	+ stonith_history_t *latest = NULL;
	int rc = pcmk_rc_ok;
	int opts = 0;

	if (cleanup) {
	out->info(out, "cleaning up fencing-history%s%s",
	target ? " for node " : "", target ? target : "");
	}
	+
	if (broadcast) {
	out->info(out, "gather fencing-history from all nodes");
	}

	stonith__set_call_options(opts, target, st_opts);
	+
	if (cleanup) {
	stonith__set_call_options(opts, target, st_opt_cleanup);
	}
	+
	if (broadcast) {
	stonith__set_call_options(opts, target, st_opt_broadcast);
	}
	+
	if (pcmk__str_eq(target, "*", pcmk__str_none)) {
	target = NULL;
	}
	+
	rc = st->cmds->history(st, opts, target, &history, (timeout / 1000));

	if (cleanup) {
	// Cleanup doesn't return a history list
	stonith_history_free(history);
	return pcmk_legacy2rc(rc);
	}

	out->begin_list(out, "event", "events", "Fencing history");

	history = stonith__sort_history(history);
	- for (hp = history; hp; hp = hp->next) {
	+ for (stonith_history_t *hp = history; hp != NULL; hp = hp->next) {
	if (hp->state == st_done) {
	latest = hp;
	}

	if (out->is_quiet(out) \|\| !verbose) {
	continue;
	}

	out->message(out, "stonith-event", hp, true, false,
	stonith__later_succeeded(hp, history),
	(uint32_t) pcmk_show_failed_detail);
	out->increment_list(out);
	}

	if (latest) {
	if (out->is_quiet(out)) {
	out->message(out, "stonith-event", latest, false, true, NULL,
	(uint32_t) pcmk_show_failed_detail);
	} else if (!verbose) { // already printed if verbose
	out->message(out, "stonith-event", latest, false, false, NULL,
	(uint32_t) pcmk_show_failed_detail);
	out->increment_list(out);
	}
	}

	out->end_list(out);

	stonith_history_free(history);
	return pcmk_legacy2rc(rc);
	}

	-#ifdef BUILD_PUBLIC_LIBPACEMAKER
	int
	-pcmk_fence_history(xmlNodePtr xml, stonith_t st, const char *target,
	- unsigned int timeout, bool quiet, int verbose,
	- bool broadcast, bool cleanup)
	+pcmk_fence_history(xmlNodePtr xml, const char target, unsigned int timeout,
	+ bool quiet, int verbose, bool broadcast, bool cleanup)
	{
	+ stonith_t *st = NULL;
	pcmk__output_t *out = NULL;
	int rc = pcmk_rc_ok;

	- rc = pcmk__xml_output_new(&out, xml);
	+ rc = pcmk__setup_output_fencing(&out, &st, xml);
	if (rc != pcmk_rc_ok) {
	return rc;
	}

	- stonith__register_messages(out);
	-
	out->quiet = quiet;

	rc = pcmk__fence_history(out, st, target, timeout, verbose, broadcast,
	cleanup);
	pcmk__xml_output_finish(out, pcmk_rc2exitc(rc), xml);
	+
	+ st->cmds->disconnect(st);
	+ stonith_api_delete(st);
	return rc;
	}
	-#endif

	int
	pcmk__fence_installed(pcmk__output_t out, stonith_t st, unsigned int timeout)
	{
	stonith_key_value_t *devices = NULL;
	int rc = pcmk_rc_ok;

	rc = st->cmds->list_agents(st, st_opt_sync_call, NULL, &devices,
	(timeout / 1000));
	// rc is a negative error code or a positive number of agents
	if (rc < 0) {
	return pcmk_legacy2rc(rc);
	}

	out->begin_list(out, "fence device", "fence devices",
	"Installed fence devices");
	for (stonith_key_value_t *iter = devices; iter != NULL; iter = iter->next) {
	out->list_item(out, "device", "%s", iter->value);
	}
	out->end_list(out);

	stonith_key_value_freeall(devices, 1, 1);
	return pcmk_rc_ok;
	}

	-#ifdef BUILD_PUBLIC_LIBPACEMAKER
	int
	-pcmk_fence_installed(xmlNodePtr xml, stonith_t st, unsigned int timeout)
	+pcmk_fence_installed(xmlNodePtr *xml, unsigned int timeout)
	{
	+ stonith_t *st = NULL;
	pcmk__output_t *out = NULL;
	int rc = pcmk_rc_ok;

	- rc = pcmk__xml_output_new(&out, xml);
	+ rc = pcmk__setup_output_fencing(&out, &st, xml);
	if (rc != pcmk_rc_ok) {
	return rc;
	}

	- stonith__register_messages(out);
	-
	rc = pcmk__fence_installed(out, st, timeout);
	pcmk__xml_output_finish(out, pcmk_rc2exitc(rc), xml);
	+
	+ st->cmds->disconnect(st);
	+ stonith_api_delete(st);
	return rc;
	}
	-#endif

	int
	pcmk__fence_last(pcmk__output_t out, const char target, bool as_nodeid)
	{
	time_t when = 0;

	if (target == NULL) {
	return pcmk_rc_ok;
	}

	if (as_nodeid) {
	when = stonith_api_time(atol(target), NULL, FALSE);
	} else {
	when = stonith_api_time(0, target, FALSE);
	}

	return out->message(out, "last-fenced", target, when);
	}

	-#ifdef BUILD_PUBLIC_LIBPACEMAKER
	int
	pcmk_fence_last(xmlNodePtr xml, const char target, bool as_nodeid)
	{
	pcmk__output_t *out = NULL;
	int rc = pcmk_rc_ok;

	rc = pcmk__xml_output_new(&out, xml);
	if (rc != pcmk_rc_ok) {
	return rc;
	}

	stonith__register_messages(out);

	rc = pcmk__fence_last(out, target, as_nodeid);
	pcmk__xml_output_finish(out, pcmk_rc2exitc(rc), xml);
	return rc;
	}
	-#endif

	int
	pcmk__fence_list_targets(pcmk__output_t out, stonith_t st,
	const char *device_id, unsigned int timeout)
	{
	GList *targets = NULL;
	char *lists = NULL;
	int rc = pcmk_rc_ok;

	rc = st->cmds->list(st, st_opts, device_id, &lists, timeout/1000);
	if (rc != pcmk_rc_ok) {
	return pcmk_legacy2rc(rc);
	}

	targets = stonith__parse_targets(lists);

	out->begin_list(out, "fence target", "fence targets", "Fence Targets");
	while (targets != NULL) {
	out->list_item(out, NULL, "%s", (const char *) targets->data);
	targets = targets->next;
	}
	out->end_list(out);

	free(lists);
	return rc;
	}

	-#ifdef BUILD_PUBLIC_LIBPACEMAKER
	int
	-pcmk_fence_list_targets(xmlNodePtr xml, stonith_t st, const char *device_id,
	- unsigned int timeout)
	+pcmk_fence_list_targets(xmlNodePtr xml, const char device_id, unsigned int timeout)
	{
	+ stonith_t *st = NULL;
	pcmk__output_t *out = NULL;
	int rc = pcmk_rc_ok;

	- rc = pcmk__xml_output_new(&out, xml);
	+ rc = pcmk__setup_output_fencing(&out, &st, xml);
	if (rc != pcmk_rc_ok) {
	return rc;
	}

	- stonith__register_messages(out);
	-
	rc = pcmk__fence_list_targets(out, st, device_id, timeout);
	pcmk__xml_output_finish(out, pcmk_rc2exitc(rc), xml);
	+
	+ st->cmds->disconnect(st);
	+ stonith_api_delete(st);
	return rc;
	}
	-#endif

	int
	pcmk__fence_metadata(pcmk__output_t out, stonith_t st, const char *agent,
	unsigned int timeout)
	{
	char *buffer = NULL;
	int rc = st->cmds->metadata(st, st_opt_sync_call, agent, NULL, &buffer,
	timeout/1000);

	if (rc != pcmk_rc_ok) {
	return pcmk_legacy2rc(rc);
	}

	out->output_xml(out, PCMK_XE_METADATA, buffer);
	free(buffer);
	return rc;
	}

	-#ifdef BUILD_PUBLIC_LIBPACEMAKER
	int
	-pcmk_fence_metadata(xmlNodePtr xml, stonith_t st, const char *agent,
	- unsigned int timeout)
	+pcmk_fence_metadata(xmlNodePtr xml, const char agent, unsigned int timeout)
	{
	+ stonith_t *st = NULL;
	pcmk__output_t *out = NULL;
	int rc = pcmk_rc_ok;

	- rc = pcmk__xml_output_new(&out, xml);
	+ rc = pcmk__setup_output_fencing(&out, &st, xml);
	if (rc != pcmk_rc_ok) {
	return rc;
	}

	- stonith__register_messages(out);
	-
	rc = pcmk__fence_metadata(out, st, agent, timeout);
	pcmk__xml_output_finish(out, pcmk_rc2exitc(rc), xml);
	+
	+ st->cmds->disconnect(st);
	+ stonith_api_delete(st);
	return rc;
	}
	-#endif

	int
	pcmk__fence_registered(pcmk__output_t out, stonith_t st, const char *target,
	unsigned int timeout)
	{
	stonith_key_value_t *devices = NULL;
	int rc = pcmk_rc_ok;

	rc = st->cmds->query(st, st_opts, target, &devices, timeout/1000);
	/* query returns a negative error code or a positive number of results. */
	if (rc < 0) {
	return pcmk_legacy2rc(rc);
	}

	out->begin_list(out, "fence device", "fence devices",
	"Registered fence devices");
	for (stonith_key_value_t *iter = devices; iter != NULL; iter = iter->next) {
	out->list_item(out, "device", "%s", iter->value);
	}
	out->end_list(out);

	stonith_key_value_freeall(devices, 1, 1);

	/* Return pcmk_rc_ok here, not the number of results. Callers probably
	* don't care.
	*/
	return pcmk_rc_ok;
	}

	-#ifdef BUILD_PUBLIC_LIBPACEMAKER
	int
	-pcmk_fence_registered(xmlNodePtr xml, stonith_t st, const char *target,
	- unsigned int timeout)
	+pcmk_fence_registered(xmlNodePtr xml, const char target, unsigned int timeout)
	{
	+ stonith_t *st = NULL;
	pcmk__output_t *out = NULL;
	int rc = pcmk_rc_ok;

	- rc = pcmk__xml_output_new(&out, xml);
	+ rc = pcmk__setup_output_fencing(&out, &st, xml);
	if (rc != pcmk_rc_ok) {
	return rc;
	}

	- stonith__register_messages(out);
	-
	rc = pcmk__fence_registered(out, st, target, timeout);
	pcmk__xml_output_finish(out, pcmk_rc2exitc(rc), xml);
	+
	+ st->cmds->disconnect(st);
	+ stonith_api_delete(st);
	return rc;
	}
	-#endif

	int
	pcmk__fence_register_level(stonith_t st, const char target, int fence_level,
	- const stonith_key_value_t *devices)
	+ GList *devices)
	{
	return handle_level(st, target, fence_level, devices, true);
	}

	-#ifdef BUILD_PUBLIC_LIBPACEMAKER
	int
	-pcmk_fence_register_level(stonith_t st, const char target, int fence_level,
	- const stonith_key_value_t *devices)
	+pcmk_fence_register_level(xmlNodePtr xml, const char target, int fence_level,
	+ GList *devices)
	{
	- return pcmk__fence_register_level(st, target, fence_level, devices);
	+ stonith_t* st = NULL;
	+ pcmk__output_t *out = NULL;
	+ int rc = pcmk_rc_ok;
	+
	+ rc = pcmk__setup_output_fencing(&out, &st, xml);
	+ if (rc != pcmk_rc_ok) {
	+ return rc;
	+ }
	+
	+ rc = pcmk__fence_register_level(st, target, fence_level, devices);
	+ pcmk__xml_output_finish(out, pcmk_rc2exitc(rc), xml);
	+
	+ st->cmds->disconnect(st);
	+ stonith_api_delete(st);
	+ return rc;
	}
	-#endif

	int
	pcmk__fence_unregister_level(stonith_t st, const char target, int fence_level)
	{
	return handle_level(st, target, fence_level, NULL, false);
	}

	-#ifdef BUILD_PUBLIC_LIBPACEMAKER
	int
	-pcmk_fence_unregister_level(stonith_t st, const char target, int fence_level)
	+pcmk_fence_unregister_level(xmlNodePtr xml, const char target, int fence_level)
	{
	- return pcmk__fence_unregister_level(st, target, fence_level);
	+ stonith_t* st = NULL;
	+ pcmk__output_t *out = NULL;
	+ int rc = pcmk_rc_ok;
	+
	+ rc = pcmk__setup_output_fencing(&out, &st, xml);
	+ if (rc != pcmk_rc_ok) {
	+ return rc;
	+ }
	+
	+ rc = pcmk__fence_unregister_level(st, target, fence_level);
	+ pcmk__xml_output_finish(out, pcmk_rc2exitc(rc), xml);
	+
	+ st->cmds->disconnect(st);
	+ stonith_api_delete(st);
	+ return rc;
	}
	-#endif

	int
	pcmk__fence_validate(pcmk__output_t out, stonith_t st, const char *agent,
	- const char id, const stonith_key_value_t params,
	- unsigned int timeout)
	+ const char id, GHashTable params, unsigned int timeout)
	{
	char *output = NULL;
	char *error_output = NULL;
	int rc;

	- rc = st->cmds->validate(st, st_opt_sync_call, id, NULL, agent, params,
	- timeout/1000, &output, &error_output);
	+ rc = stonith__validate(st, st_opt_sync_call, id, NULL, agent, params,
	+ timeout/1000, &output, &error_output);
	out->message(out, "validate", agent, id, output, error_output, rc);
	return pcmk_legacy2rc(rc);
	}

	-#ifdef BUILD_PUBLIC_LIBPACEMAKER
	int
	-pcmk_fence_validate(xmlNodePtr xml, stonith_t st, const char *agent,
	- const char id, const stonith_key_value_t params,
	- unsigned int timeout)
	+pcmk_fence_validate(xmlNodePtr xml, const char agent, const char *id,
	+ GHashTable *params, unsigned int timeout)
	{
	+ stonith_t *st = NULL;
	pcmk__output_t *out = NULL;
	int rc = pcmk_rc_ok;

	- rc = pcmk__xml_output_new(&out, xml);
	+ rc = pcmk__setup_output_fencing(&out, &st, xml);
	if (rc != pcmk_rc_ok) {
	return rc;
	}

	- stonith__register_messages(out);
	-
	rc = pcmk__fence_validate(out, st, agent, id, params, timeout);
	pcmk__xml_output_finish(out, pcmk_rc2exitc(rc), xml);
	+
	+ st->cmds->disconnect(st);
	+ stonith_api_delete(st);
	return rc;
	}
	-#endif

	int
	pcmk__get_fencing_history(stonith_t st, stonith_history_t *stonith_history,
	enum pcmk__fence_history fence_history)
	{
	int rc = pcmk_rc_ok;

	if ((st == NULL) \|\| (st->state == stonith_disconnected)) {
	rc = ENOTCONN;
	} else if (fence_history != pcmk__fence_history_none) {
	rc = st->cmds->history(st, st_opt_sync_call, NULL, stonith_history,
	120);

	rc = pcmk_legacy2rc(rc);
	if (rc != pcmk_rc_ok) {
	return rc;
	}

	stonith_history = stonith__sort_history(stonith_history);
	if (fence_history == pcmk__fence_history_reduced) {
	stonith_history = reduce_fence_history(stonith_history);
	}
	}

	return rc;
	}
	diff --git a/lib/pacemaker/pcmk_setup.c b/lib/pacemaker/pcmk_setup.c
	index a4679117ad..42dd3f7b0c 100644
	--- a/lib/pacemaker/pcmk_setup.c
	+++ b/lib/pacemaker/pcmk_setup.c
	@@ -1,78 +1,121 @@
	/*
	* Copyright 2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>
	#include <crm/cib/internal.h>
	#include <crm/common/output.h>
	#include <crm/common/results.h>
	#include <crm/common/scheduler.h>
	+#include <crm/fencing/internal.h>
	#include <pacemaker-internal.h>
	#include <pacemaker.h>

	#include "libpacemaker_private.h"

	/*!
	* \internal
	* \brief Set up a pcmk__output_t, (optionally) cib_t, and
	* (optionally) pcmk_scheduler_t for use in implementing
	* public/private API function pairs
	*
	* \param[in,out] out Where to store a \c pcmk__output_t object
	* \param[in,out] cib Where to store a \c cib_t object
	* (may be \c NULL if a CIB is not needed)
	* \param[in,out] scheduler Where to store a \c pcmk_scheduler_t object
	* (may be \c NULL if a scheduler is not needed)
	* \param[in,out] xml Where to write any result XML
	*
	* \note The \p cib and \p scheduler arguments will only be valid if there
	* are no errors in this function. However, \p out will always be
	* valid unless there are errors setting it up so that other errors
	* may still be reported.
	*
	* \return Standard Pacemaker return code
	*/
	int
	pcmk__setup_output_cib_sched(pcmk__output_t out, cib_t cib,
	pcmk_scheduler_t scheduler, xmlNode xml)
	{
	int rc = pcmk_rc_ok;

	rc = pcmk__xml_output_new(out, xml);
	if (rc != pcmk_rc_ok) {
	return rc;
	}

	if (cib != NULL) {
	*cib = cib_new();
	if (*cib == NULL) {
	return pcmk_rc_cib_corrupt;
	}

	rc = (cib)->cmds->signon(cib, crm_system_name, cib_command);
	rc = pcmk_legacy2rc(rc);

	if (rc != pcmk_rc_ok) {
	cib__clean_up_connection(cib);
	return rc;
	}
	}

	if (scheduler != NULL) {
	rc = pcmk__init_scheduler(*out, NULL, NULL, scheduler);
	if (rc != pcmk_rc_ok && cib != NULL) {
	cib__clean_up_connection(cib);
	return rc;
	}

	pcmk__unpack_constraints(*scheduler);
	}

	pcmk__register_lib_messages(*out);
	return rc;
	}
	+
	+/*!
	+ * \internal
	+ * \brief Set up a pcmk__output_t and stonith_t for use in implementing
	+ * public/private API function pairs
	+ *
	+ * \param[in,out] out Where to store a \c pcmk__output_t object
	+ * \param[in,out] st Where to store a \c stonith_t object
	+ * \param[in,out] xml Where to write any result XML
	+ *
	+ * \note The \p st argument will only be valid if there are no errors in this
	+ * function. However, \p out will always be valid unless there are
	+ * errors setting it up so that other errors may still be reported.
	+ *
	+ * \return Standard Pacemaker return code
	+ */
	+int
	+pcmk__setup_output_fencing(pcmk__output_t out, stonith_t st, xmlNode **xml)
	+{
	+ int rc = pcmk_rc_ok;
	+
	+ rc = pcmk__xml_output_new(out, xml);
	+ if (rc != pcmk_rc_ok) {
	+ return rc;
	+ }
	+
	+ *st = stonith_api_new();
	+ if (*st == NULL) {
	+ return ENOMEM;
	+ }
	+
	+ rc = (st)->cmds->connect(st, crm_system_name, NULL);
	+ if (rc < 0) {
	+ rc = pcmk_legacy2rc(rc);
	+ stonith_api_delete(*st);
	+ return rc;
	+ }
	+
	+ pcmk__register_lib_messages(*out);
	+ stonith__register_messages(*out);
	+ return rc;
	+}
	diff --git a/tools/stonith_admin.c b/tools/stonith_admin.c
	index 990a5ba971..39c4f18ef9 100644
	--- a/tools/stonith_admin.c
	+++ b/tools/stonith_admin.c
	@@ -1,687 +1,719 @@
	/*
	* Copyright 2009-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <sys/param.h>
	#include <stdio.h>
	#include <sys/types.h>
	#include <sys/stat.h>
	#include <unistd.h>
	#include <sys/utsname.h>

	#include <errno.h>
	#include <fcntl.h>
	#include <stdbool.h>
	#include <stdlib.h>
	#include <string.h>

	#include <crm/crm.h>
	#include <crm/common/ipc.h>
	#include <crm/cluster/internal.h>
	#include <crm/common/cmdline_internal.h>
	#include <crm/common/output_internal.h>

	#include <crm/stonith-ng.h>
	#include <crm/fencing/internal.h> // stonith__register_messages()
	#include <crm/cib.h>
	#include <crm/pengine/status.h>

	#include <crm/common/xml.h>
	#include <pacemaker-internal.h>

	#define SUMMARY "stonith_admin - Access the Pacemaker fencing API"

	char action = 0;

	struct {
	gboolean as_nodeid;
	gboolean broadcast;
	gboolean cleanup;
	gboolean installed;
	gboolean metadata;
	gboolean registered;
	gboolean validate_cfg;
	- stonith_key_value_t *devices;
	- stonith_key_value_t *params;
	+ GList *devices;
	+ GHashTable *params;
	int fence_level;
	int timeout ;
	long long tolerance_ms;
	int delay;
	char *agent;
	char *confirm_host;
	char *fence_host;
	char *history;
	char *last_fenced;
	char *query;
	char *reboot_host;
	char *register_dev;
	char *register_level;
	char *targets;
	char *terminate;
	char *unfence_host;
	char *unregister_dev;
	char *unregister_level;
	} options = {
	.timeout = 120,
	.delay = 0
	};

	gboolean add_env_params(const gchar option_name, const gchar optarg, gpointer data, GError **error);
	gboolean add_stonith_device(const gchar option_name, const gchar optarg, gpointer data, GError **error);
	gboolean add_stonith_params(const gchar option_name, const gchar optarg, gpointer data, GError **error);
	gboolean add_tolerance(const gchar option_name, const gchar optarg, gpointer data, GError **error);
	gboolean set_tag(const gchar option_name, const gchar optarg, gpointer data, GError **error);

	#define INDENT " "

	/* INDENT-OFF */
	static GOptionEntry defn_entries[] = {
	{ "register", 'R', 0, G_OPTION_ARG_STRING, &options.register_dev,
	"Register the named stonith device. Requires: --agent.\n"
	INDENT "Optional: --option, --env-option.",
	"DEVICE" },
	{ "deregister", 'D', 0, G_OPTION_ARG_STRING, &options.unregister_dev,
	"De-register the named stonith device.",
	"DEVICE" },
	{ "register-level", 'r', 0, G_OPTION_ARG_STRING, &options.register_level,
	"Register a stonith level for the named target,\n"
	INDENT "specified as one of NAME, @PATTERN, or ATTR=VALUE.\n"
	INDENT "Requires: --index and one or more --device entries.",
	"TARGET" },
	{ "deregister-level", 'd', 0, G_OPTION_ARG_STRING, &options.unregister_level,
	"Unregister a stonith level for the named target,\n"
	INDENT "specified as for --register-level. Requires: --index",
	"TARGET" },

	{ NULL }
	};

	static GOptionEntry query_entries[] = {
	{ "list", 'l', 0, G_OPTION_ARG_STRING, &options.terminate,
	"List devices that can terminate the specified host.\n"
	INDENT "Optional: --timeout",
	"HOST" },
	{ "list-registered", 'L', 0, G_OPTION_ARG_NONE, &options.registered,
	"List all registered devices. Optional: --timeout.",
	NULL },
	{ "list-installed", 'I', 0, G_OPTION_ARG_NONE, &options.installed,
	"List all installed devices. Optional: --timeout.",
	NULL },
	{ "list-targets", 's', 0, G_OPTION_ARG_STRING, &options.targets,
	"List the targets that can be fenced by the\n"
	INDENT "named device. Optional: --timeout.",
	"DEVICE" },
	{ "metadata", 'M', 0, G_OPTION_ARG_NONE, &options.metadata,
	"Show agent metadata. Requires: --agent.\n"
	INDENT "Optional: --timeout.",
	NULL },
	{ "query", 'Q', 0, G_OPTION_ARG_STRING, &options.query,
	"Check the named device's status. Optional: --timeout.",
	"DEVICE" },
	{ "history", 'H', 0, G_OPTION_ARG_STRING, &options.history,
	"Show last successful fencing operation for named node\n"
	INDENT "(or '*' for all nodes). Optional: --timeout, --cleanup,\n"
	INDENT "--quiet (show only the operation's epoch timestamp),\n"
	INDENT "--verbose (show all recorded and pending operations),\n"
	INDENT "--broadcast (update history from all nodes available).",
	"NODE" },
	{ "last", 'h', 0, G_OPTION_ARG_STRING, &options.last_fenced,
	"Indicate when the named node was last fenced.\n"
	INDENT "Optional: --as-node-id.",
	"NODE" },
	{ "validate", 'K', 0, G_OPTION_ARG_NONE, &options.validate_cfg,
	"Validate a fence device configuration.\n"
	INDENT "Requires: --agent. Optional: --option, --env-option,\n"
	INDENT "--quiet (print no output, only return status).",
	NULL },

	{ NULL }
	};

	static GOptionEntry fence_entries[] = {
	{ "fence", 'F', 0, G_OPTION_ARG_STRING, &options.fence_host,
	"Fence named host. Optional: --timeout, --tolerance, --delay.",
	"HOST" },
	{ "unfence", 'U', 0, G_OPTION_ARG_STRING, &options.unfence_host,
	"Unfence named host. Optional: --timeout, --tolerance, --delay.",
	"HOST" },
	{ "reboot", 'B', 0, G_OPTION_ARG_STRING, &options.reboot_host,
	"Reboot named host. Optional: --timeout, --tolerance, --delay.",
	"HOST" },
	{ "confirm", 'C', 0, G_OPTION_ARG_STRING, &options.confirm_host,
	"Tell cluster that named host is now safely down.",
	"HOST", },

	{ NULL }
	};

	static GOptionEntry addl_entries[] = {
	{ "cleanup", 'c', 0, G_OPTION_ARG_NONE, &options.cleanup,
	"Cleanup wherever appropriate. Requires --history.",
	NULL },
	{ "broadcast", 'b', 0, G_OPTION_ARG_NONE, &options.broadcast,
	"Broadcast wherever appropriate.",
	NULL },
	{ "agent", 'a', 0, G_OPTION_ARG_STRING, &options.agent,
	"The agent to use (for example, fence_xvm;\n"
	INDENT "with --register, --metadata, --validate).",
	"AGENT" },
	{ "option", 'o', 0, G_OPTION_ARG_CALLBACK, add_stonith_params,
	"Specify a device configuration parameter as NAME=VALUE\n"
	INDENT "(may be specified multiple times; with --register,\n"
	INDENT "--validate).",
	"PARAM" },
	{ "env-option", 'e', 0, G_OPTION_ARG_CALLBACK, add_env_params,
	"Specify a device configuration parameter with the\n"
	INDENT "specified name, using the value of the\n"
	INDENT "environment variable of the same name prefixed with\n"
	INDENT "OCF_RESKEY_ (may be specified multiple times;\n"
	INDENT "with --register, --validate).",
	"PARAM" },
	{ "tag", 'T', 0, G_OPTION_ARG_CALLBACK, set_tag,
	"Identify fencing operations in logs with the specified\n"
	INDENT "tag; useful when multiple entities might invoke\n"
	INDENT "stonith_admin (used with most commands).",
	"TAG" },
	{ "device", 'v', 0, G_OPTION_ARG_CALLBACK, add_stonith_device,
	"Device ID (with --register-level, device to associate with\n"
	INDENT "a given host and level; may be specified multiple times)"
	#if PCMK__ENABLE_CIBSECRETS
	"\n" INDENT "(with --validate, name to use to load CIB secrets)"
	#endif
	".",
	"DEVICE" },
	{ "index", 'i', 0, G_OPTION_ARG_INT, &options.fence_level,
	"The stonith level (1-9) (with --register-level,\n"
	INDENT "--deregister-level).",
	"LEVEL" },
	{ "timeout", 't', 0, G_OPTION_ARG_INT, &options.timeout,
	"Operation timeout in seconds (default 120;\n"
	INDENT "used with most commands).",
	"SECONDS" },
	{ "delay", 'y', 0, G_OPTION_ARG_INT, &options.delay,
	"Apply a fencing delay in seconds. Any static/random delays from\n"
	INDENT "pcmk_delay_base/max will be added, otherwise all\n"
	INDENT "disabled with the value -1\n"
	INDENT "(default 0; with --fence, --reboot, --unfence).",
	"SECONDS" },
	{ "as-node-id", 'n', 0, G_OPTION_ARG_NONE, &options.as_nodeid,
	"(Advanced) The supplied node is the corosync node ID\n"
	INDENT "(with --last).",
	NULL },
	{ "tolerance", 0, 0, G_OPTION_ARG_CALLBACK, add_tolerance,
	"(Advanced) Do nothing if an equivalent --fence request\n"
	INDENT "succeeded less than this many seconds earlier\n"
	INDENT "(with --fence, --unfence, --reboot).",
	"SECONDS" },

	{ NULL }
	};
	/* INDENT-ON */

	static pcmk__supported_format_t formats[] = {
	PCMK__SUPPORTED_FORMAT_HTML,
	PCMK__SUPPORTED_FORMAT_NONE,
	PCMK__SUPPORTED_FORMAT_TEXT,
	PCMK__SUPPORTED_FORMAT_XML,
	{ NULL, NULL, NULL }
	};

	static const int st_opts = st_opt_sync_call\|st_opt_allow_self_fencing;

	static char *name = NULL;

	gboolean
	add_env_params(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	char *key = crm_strdup_printf("OCF_RESKEY_%s", optarg);
	const char *env = getenv(key);
	gboolean retval = TRUE;

	if (env == NULL) {
	g_set_error(error, PCMK__EXITC_ERROR, CRM_EX_INVALID_PARAM, "Invalid option: -e %s", optarg);
	retval = FALSE;
	} else {
	crm_info("Got: '%s'='%s'", optarg, env);
	- options.params = stonith_key_value_add(options.params, optarg, env);
	+
	+ if (options.params != NULL) {
	+ options.params = pcmk__strkey_table(free, free);
	+ }
	+
	+ pcmk__insert_dup(options.params, optarg, env);
	}

	free(key);
	return retval;
	}

	gboolean
	add_stonith_device(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	- options.devices = stonith_key_value_add(options.devices, NULL, optarg);
	+ options.devices = g_list_append(options.devices, pcmk__str_copy(optarg));
	return TRUE;
	}

	gboolean
	add_tolerance(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	// pcmk__request_fencing() expects an unsigned int
	options.tolerance_ms = crm_get_msec(optarg);

	if (options.tolerance_ms < 0) {
	crm_warn("Ignoring invalid tolerance '%s'", optarg);
	options.tolerance_ms = 0;
	} else {
	options.tolerance_ms = QB_MIN(options.tolerance_ms, UINT_MAX);
	}
	return TRUE;
	}

	gboolean
	add_stonith_params(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	char *name = NULL;
	char *value = NULL;
	int rc = 0;
	gboolean retval = TRUE;

	crm_info("Scanning: -o %s", optarg);

	rc = pcmk__scan_nvpair(optarg, &name, &value);

	if (rc != 2) {
	rc = pcmk_legacy2rc(rc);
	g_set_error(error, PCMK__RC_ERROR, rc, "Invalid option: -o %s: %s", optarg, pcmk_rc_str(rc));
	retval = FALSE;
	} else {
	crm_info("Got: '%s'='%s'", name, value);
	- options.params = stonith_key_value_add(options.params, name, value);
	+
	+ if (options.params == NULL) {
	+ options.params = pcmk__strkey_table(free, free);
	+ }
	+
	+ pcmk__insert_dup(options.params, name, value);
	}

	free(name);
	free(value);
	return retval;
	}

	gboolean
	set_tag(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	free(name);
	name = crm_strdup_printf("%s.%s", crm_system_name, optarg);
	return TRUE;
	}

	static GOptionContext *
	build_arg_context(pcmk__common_args_t args, GOptionGroup *group) {
	GOptionContext *context = NULL;

	GOptionEntry extra_prog_entries[] = {
	{ "quiet", 'q', 0, G_OPTION_ARG_NONE, &(args->quiet),
	"Be less descriptive in output.",
	NULL },

	{ NULL }
	};

	context = pcmk__build_arg_context(args, "text (default), html, xml", group, NULL);

	/* Add the -q option, which cannot be part of the globally supported options
	* because some tools use that flag for something else.
	*/
	pcmk__add_main_args(context, extra_prog_entries);

	pcmk__add_arg_group(context, "definition", "Device Definition Commands:",
	"Show device definition help", defn_entries);
	pcmk__add_arg_group(context, "queries", "Queries:",
	"Show query help", query_entries);
	pcmk__add_arg_group(context, "fence", "Fencing Commands:",
	"Show fence help", fence_entries);
	pcmk__add_arg_group(context, "additional", "Additional Options:",
	"Show additional options", addl_entries);
	return context;
	}

	// \return Standard Pacemaker return code
	static int
	request_fencing(stonith_t st, const char target, const char *command,
	GError **error)
	{
	char *reason = NULL;
	int rc = pcmk__request_fencing(st, target, command, name,
	options.timeout * 1000,
	options.tolerance_ms, options.delay,
	&reason);

	if (rc != pcmk_rc_ok) {
	const char *rc_str = pcmk_rc_str(rc);
	const char *what = "fence";

	if (strcmp(command, PCMK_ACTION_ON) == 0) {
	what = "unfence";
	}

	// If reason is identical to return code string, don't display it twice
	if (pcmk__str_eq(rc_str, reason, pcmk__str_none)) {
	free(reason);
	reason = NULL;
	}

	g_set_error(error, PCMK__RC_ERROR, rc,
	"Couldn't %s %s: %s%s%s%s",
	what, target, rc_str,
	((reason == NULL)? "" : " ("),
	((reason == NULL)? "" : reason),
	((reason == NULL)? "" : ")"));
	}
	free(reason);
	return rc;
	}

	int
	main(int argc, char **argv)
	{
	int rc = 0;
	crm_exit_t exit_code = CRM_EX_OK;
	bool no_connect = false;
	bool required_agent = false;

	char *target = NULL;
	const char *device = NULL;
	stonith_t *st = NULL;

	GError *error = NULL;

	pcmk__output_t *out = NULL;

	GOptionGroup *output_group = NULL;
	pcmk__common_args_t *args = pcmk__new_common_args(SUMMARY);
	gchar **processed_args = pcmk__cmdline_preproc(argv, "adehilorstvyBCDFHQRTU");
	GOptionContext *context = build_arg_context(args, &output_group);

	pcmk__register_formats(output_group, formats);
	if (!g_option_context_parse_strv(context, &processed_args, &error)) {
	exit_code = CRM_EX_USAGE;
	goto done;
	}

	pcmk__cli_init_logging("stonith_admin", args->verbosity);

	if (name == NULL) {
	name = strdup(crm_system_name);
	}

	rc = pcmk__output_new(&out, args->output_ty, args->output_dest, argv);
	if (rc != pcmk_rc_ok) {
	exit_code = CRM_EX_ERROR;
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code, "Error creating output format %s: %s",
	args->output_ty, pcmk_rc_str(rc));
	goto done;
	}

	pcmk__output_enable_list_element(out);

	stonith__register_messages(out);

	if (args->version) {
	out->version(out, false);
	goto done;
	}

	if (options.validate_cfg) {
	required_agent = true;
	no_connect = true;
	action = 'K';
	}

	if (options.installed) {
	no_connect = true;
	action = 'I';
	}

	if (options.registered) {
	action = 'L';
	}

	if (options.register_dev != NULL) {
	required_agent = true;
	action = 'R';
	device = options.register_dev;
	}

	if (options.query != NULL) {
	action = 'Q';
	device = options.query;
	}

	if (options.unregister_dev != NULL) {
	action = 'D';
	device = options.unregister_dev;
	}

	if (options.targets != NULL) {
	action = 's';
	device = options.targets;
	}

	if (options.terminate != NULL) {
	action = 'L';
	target = options.terminate;
	}

	if (options.metadata) {
	no_connect = true;
	required_agent = true;
	action = 'M';
	}

	if (options.reboot_host != NULL) {
	no_connect = true;
	action = 'B';
	target = options.reboot_host;
	crm_log_args(argc, argv);
	}

	if (options.fence_host != NULL) {
	no_connect = true;
	action = 'F';
	target = options.fence_host;
	crm_log_args(argc, argv);
	}

	if (options.unfence_host != NULL) {
	no_connect = true;
	action = 'U';
	target = options.unfence_host;
	crm_log_args(argc, argv);
	}

	if (options.confirm_host != NULL) {
	action = 'C';
	target = options.confirm_host;
	crm_log_args(argc, argv);
	}

	if (options.last_fenced != NULL) {
	action = 'h';
	target = options.last_fenced;
	}

	if (options.history != NULL) {
	action = 'H';
	target = options.history;
	}

	if (options.register_level != NULL) {
	action = 'r';
	target = options.register_level;
	}

	if (options.unregister_level != NULL) {
	action = 'd';
	target = options.unregister_level;
	}

	if (action == 0) {
	char *help = g_option_context_get_help(context, TRUE, NULL);

	out->err(out, "%s", help);
	g_free(help);
	exit_code = CRM_EX_USAGE;
	goto done;
	}

	if (required_agent && options.agent == NULL) {
	char *help = g_option_context_get_help(context, TRUE, NULL);

	out->err(out, "Please specify an agent to query using -a,--agent [value]");
	out->err(out, "%s", help);
	g_free(help);
	exit_code = CRM_EX_USAGE;
	goto done;
	}

	out->quiet = args->quiet;

	st = stonith_api_new();
	if (st == NULL) {
	rc = -ENOMEM;
	} else if (!no_connect) {
	rc = st->cmds->connect(st, name, NULL);
	}
	if (rc < 0) {
	out->err(out, "Could not connect to fencer: %s", pcmk_strerror(rc));
	exit_code = CRM_EX_DISCONNECT;
	goto done;
	}

	switch (action) {
	case 'I':
	rc = pcmk__fence_installed(out, st, options.timeout*1000);
	if (rc != pcmk_rc_ok) {
	out->err(out, "Failed to list installed devices: %s", pcmk_rc_str(rc));
	}

	break;

	case 'L':
	rc = pcmk__fence_registered(out, st, target, options.timeout*1000);
	if (rc != pcmk_rc_ok) {
	out->err(out, "Failed to list registered devices: %s", pcmk_rc_str(rc));
	}

	break;

	case 'Q':
	rc = st->cmds->monitor(st, st_opts, device, options.timeout);
	if (rc != pcmk_rc_ok) {
	rc = st->cmds->list(st, st_opts, device, NULL, options.timeout);
	}
	rc = pcmk_legacy2rc(rc);
	break;

	case 's':
	rc = pcmk__fence_list_targets(out, st, device, options.timeout*1000);
	if (rc != pcmk_rc_ok) {
	out->err(out, "Couldn't list targets: %s", pcmk_rc_str(rc));
	}

	break;

	- case 'R':
	+ case 'R': {
	+ /* register_device wants a stonith_key_value_t instead of a GHashTable */
	+ stonith_key_value_t *params = NULL;
	+ GHashTableIter iter;
	+ gpointer key, val;
	+
	+ if (options.params != NULL) {
	+ g_hash_table_iter_init(&iter, options.params);
	+ while (g_hash_table_iter_next(&iter, &key, &val)) {
	+ params = stonith_key_value_add(params, key, val);
	+ }
	+ }
	rc = st->cmds->register_device(st, st_opts, device, NULL, options.agent,
	- options.params);
	+ params);
	+ stonith_key_value_freeall(params, 1, 1);
	+
	rc = pcmk_legacy2rc(rc);
	if (rc != pcmk_rc_ok) {
	out->err(out, "Can't register device %s using agent %s: %s",
	device, options.agent, pcmk_rc_str(rc));
	}
	break;
	+ }

	case 'D':
	rc = st->cmds->remove_device(st, st_opts, device);
	rc = pcmk_legacy2rc(rc);
	if (rc != pcmk_rc_ok) {
	out->err(out, "Can't unregister device %s: %s",
	device, pcmk_rc_str(rc));
	}
	break;

	case 'd':
	rc = pcmk__fence_unregister_level(st, target, options.fence_level);
	if (rc != pcmk_rc_ok) {
	out->err(out, "Can't unregister topology level %d for %s: %s",
	options.fence_level, target, pcmk_rc_str(rc));
	}
	break;

	case 'r':
	rc = pcmk__fence_register_level(st, target, options.fence_level, options.devices);
	if (rc != pcmk_rc_ok) {
	out->err(out, "Can't register topology level %d for %s: %s",
	options.fence_level, target, pcmk_rc_str(rc));
	}
	break;

	case 'M':
	rc = pcmk__fence_metadata(out, st, options.agent, options.timeout*1000);
	if (rc != pcmk_rc_ok) {
	out->err(out, "Can't get fence agent meta-data: %s",
	pcmk_rc_str(rc));
	}

	break;

	case 'C':
	rc = st->cmds->confirm(st, st_opts, target);
	rc = pcmk_legacy2rc(rc);
	break;

	case 'B':
	rc = request_fencing(st, target, PCMK_ACTION_REBOOT, &error);
	break;

	case 'F':
	rc = request_fencing(st, target, PCMK_ACTION_OFF, &error);
	break;

	case 'U':
	rc = request_fencing(st, target, PCMK_ACTION_ON, &error);
	break;

	case 'h':
	rc = pcmk__fence_last(out, target, options.as_nodeid);
	break;

	case 'H':
	rc = pcmk__fence_history(out, st, target, options.timeout*1000, args->verbosity,
	options.broadcast, options.cleanup);
	break;

	case 'K':
	- device = options.devices ? options.devices->key : NULL;
	+ device = NULL;
	+ if (options.devices != NULL) {
	+ device = g_list_nth_data(options.devices, 0);
	+ }
	+
	rc = pcmk__fence_validate(out, st, options.agent, device, options.params,
	options.timeout*1000);
	break;
	}

	crm_info("Command returned: %s (%d)", pcmk_rc_str(rc), rc);
	exit_code = pcmk_rc2exitc(rc);

	done:
	g_strfreev(processed_args);
	pcmk__free_arg_context(context);

	pcmk__output_and_clear_error(&error, out);

	if (out != NULL) {
	out->finish(out, exit_code, true, NULL);
	pcmk__output_free(out);
	}
	pcmk__unregister_formats();
	free(name);
	- stonith_key_value_freeall(options.params, 1, 1);
	+ g_list_free_full(options.devices, free);
	+
	+ if (options.params != NULL) {
	+ g_hash_table_destroy(options.params);
	+ }

	if (st != NULL) {
	st->cmds->disconnect(st);
	stonith_api_delete(st);
	}

	return exit_code;
	}

File Metadata

Mime Type: text/x-diff
Expires: Tue, Oct 29, 7:04 PM (1 d, 14 h)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 939781
Default Alt Text: (243 KB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions