No OneTemporary
Actions

Size

109 KB

Referenced Files

None

Subscribers

None

View Options

	diff --git a/daemons/controld/controld_messages.c b/daemons/controld/controld_messages.c
	index 2ffd97fed7..9e94e35a88 100644
	--- a/daemons/controld/controld_messages.c
	+++ b/daemons/controld/controld_messages.c
	@@ -1,1336 +1,1337 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <sys/param.h>
	#include <string.h>
	#include <time.h>

	#include <crm/crm.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>
	#include <crm/cluster/internal.h>
	#include <crm/cib.h>
	#include <crm/common/ipc_internal.h>

	#include <pacemaker-controld.h>

	extern void crm_shutdown(int nsig);

	static enum crmd_fsa_input handle_message(xmlNode *msg,
	enum crmd_fsa_cause cause);
	static void handle_response(xmlNode *stored_msg);
	static enum crmd_fsa_input handle_request(xmlNode *stored_msg,
	enum crmd_fsa_cause cause);
	static enum crmd_fsa_input handle_shutdown_request(xmlNode *stored_msg);
	static void send_msg_via_ipc(xmlNode * msg, const char *sys);

	/* debug only, can wrap all it likes */
	static int last_data_id = 0;

	void
	register_fsa_error_adv(enum crmd_fsa_cause cause, enum crmd_fsa_input input,
	fsa_data_t * cur_data, void new_data, const char raised_from)
	{
	/* save the current actions if any */
	if (controld_globals.fsa_actions != A_NOTHING) {
	register_fsa_input_adv(cur_data ? cur_data->fsa_cause : C_FSA_INTERNAL,
	I_NULL, cur_data ? cur_data->data : NULL,
	controld_globals.fsa_actions, TRUE, __func__);
	}

	/* reset the action list */
	crm_info("Resetting the current action list");
	fsa_dump_actions(controld_globals.fsa_actions, "Drop");
	controld_globals.fsa_actions = A_NOTHING;

	/* register the error */
	register_fsa_input_adv(cause, input, new_data, A_NOTHING, TRUE, raised_from);
	}

	void
	register_fsa_input_adv(enum crmd_fsa_cause cause, enum crmd_fsa_input input,
	void *data, uint64_t with_actions,
	gboolean prepend, const char *raised_from)
	{
	unsigned old_len = g_list_length(controld_globals.fsa_message_queue);
	fsa_data_t *fsa_data = NULL;

	if (raised_from == NULL) {
	raised_from = "<unknown>";
	}

	if (input == I_NULL && with_actions == A_NOTHING /* && data == NULL */ ) {
	/* no point doing anything */
	crm_err("Cannot add entry to queue: no input and no action");
	return;
	}

	if (input == I_WAIT_FOR_EVENT) {
	controld_set_global_flags(controld_fsa_is_stalled);
	crm_debug("Stalling the FSA pending further input: source=%s cause=%s data=%p queue=%d",
	raised_from, fsa_cause2string(cause), data, old_len);

	if (old_len > 0) {
	fsa_dump_queue(LOG_TRACE);
	prepend = FALSE;
	}

	if (data == NULL) {
	controld_set_fsa_action_flags(with_actions);
	fsa_dump_actions(with_actions, "Restored");
	return;
	}

	/* Store everything in the new event and reset
	* controld_globals.fsa_actions
	*/
	with_actions \|= controld_globals.fsa_actions;
	controld_globals.fsa_actions = A_NOTHING;
	}

	last_data_id++;
	crm_trace("%s %s FSA input %d (%s) due to %s, %s data",
	raised_from, (prepend? "prepended" : "appended"), last_data_id,
	fsa_input2string(input), fsa_cause2string(cause),
	(data? "with" : "without"));

	fsa_data = calloc(1, sizeof(fsa_data_t));
	fsa_data->id = last_data_id;
	fsa_data->fsa_input = input;
	fsa_data->fsa_cause = cause;
	fsa_data->origin = raised_from;
	fsa_data->data = NULL;
	fsa_data->data_type = fsa_dt_none;
	fsa_data->actions = with_actions;

	if (with_actions != A_NOTHING) {
	crm_trace("Adding actions %.16llx to input",
	(unsigned long long) with_actions);
	}

	if (data != NULL) {
	switch (cause) {
	case C_FSA_INTERNAL:
	case C_CRMD_STATUS_CALLBACK:
	case C_IPC_MESSAGE:
	case C_HA_MESSAGE:
	CRM_CHECK(((ha_msg_input_t *) data)->msg != NULL,
	crm_err("Bogus data from %s", raised_from));
	crm_trace("Copying %s data from %s as cluster message data",
	fsa_cause2string(cause), raised_from);
	fsa_data->data = copy_ha_msg_input(data);
	fsa_data->data_type = fsa_dt_ha_msg;
	break;

	case C_LRM_OP_CALLBACK:
	crm_trace("Copying %s data from %s as lrmd_event_data_t",
	fsa_cause2string(cause), raised_from);
	fsa_data->data = lrmd_copy_event((lrmd_event_data_t *) data);
	fsa_data->data_type = fsa_dt_lrm;
	break;

	case C_TIMER_POPPED:
	case C_SHUTDOWN:
	case C_UNKNOWN:
	case C_STARTUP:
	crm_crit("Copying %s data (from %s) is not yet implemented",
	fsa_cause2string(cause), raised_from);
	crmd_exit(CRM_EX_SOFTWARE);
	break;
	}
	}

	/* make sure to free it properly later */
	if (prepend) {
	controld_globals.fsa_message_queue
	= g_list_prepend(controld_globals.fsa_message_queue, fsa_data);
	} else {
	controld_globals.fsa_message_queue
	= g_list_append(controld_globals.fsa_message_queue, fsa_data);
	}

	crm_trace("FSA message queue length is %d",
	g_list_length(controld_globals.fsa_message_queue));

	/* fsa_dump_queue(LOG_TRACE); */

	if (old_len == g_list_length(controld_globals.fsa_message_queue)) {
	crm_err("Couldn't add message to the queue");
	}

	if (input != I_WAIT_FOR_EVENT) {
	controld_trigger_fsa();
	}
	}

	void
	fsa_dump_queue(int log_level)
	{
	int offset = 0;

	for (GList *iter = controld_globals.fsa_message_queue; iter != NULL;
	iter = iter->next) {
	fsa_data_t data = (fsa_data_t ) iter->data;

	do_crm_log_unlikely(log_level,
	"queue[%d.%d]: input %s raised by %s(%p.%d)\t(cause=%s)",
	offset++, data->id, fsa_input2string(data->fsa_input),
	data->origin, data->data, data->data_type,
	fsa_cause2string(data->fsa_cause));
	}
	}

	ha_msg_input_t *
	copy_ha_msg_input(ha_msg_input_t * orig)
	{
	ha_msg_input_t *copy = calloc(1, sizeof(ha_msg_input_t));

	CRM_ASSERT(copy != NULL);
	copy->msg = (orig && orig->msg)? copy_xml(orig->msg) : NULL;
	copy->xml = get_message_xml(copy->msg, F_CRM_DATA);
	return copy;
	}

	void
	delete_fsa_input(fsa_data_t * fsa_data)
	{
	lrmd_event_data_t *op = NULL;
	xmlNode *foo = NULL;

	if (fsa_data == NULL) {
	return;
	}
	crm_trace("About to free %s data", fsa_cause2string(fsa_data->fsa_cause));

	if (fsa_data->data != NULL) {
	switch (fsa_data->data_type) {
	case fsa_dt_ha_msg:
	delete_ha_msg_input(fsa_data->data);
	break;

	case fsa_dt_xml:
	foo = fsa_data->data;
	free_xml(foo);
	break;

	case fsa_dt_lrm:
	op = (lrmd_event_data_t *) fsa_data->data;
	lrmd_free_event(op);
	break;

	case fsa_dt_none:
	if (fsa_data->data != NULL) {
	crm_err("Don't know how to free %s data from %s",
	fsa_cause2string(fsa_data->fsa_cause), fsa_data->origin);
	crmd_exit(CRM_EX_SOFTWARE);
	}
	break;
	}
	crm_trace("%s data freed", fsa_cause2string(fsa_data->fsa_cause));
	}

	free(fsa_data);
	}

	/* returns the next message */
	fsa_data_t *
	get_message(void)
	{
	fsa_data_t *message
	= (fsa_data_t *) controld_globals.fsa_message_queue->data;

	controld_globals.fsa_message_queue
	= g_list_remove(controld_globals.fsa_message_queue, message);
	crm_trace("Processing input %d", message->id);
	return message;
	}

	void *
	fsa_typed_data_adv(fsa_data_t * fsa_data, enum fsa_data_type a_type, const char *caller)
	{
	void *ret_val = NULL;

	if (fsa_data == NULL) {
	crm_err("%s: No FSA data available", caller);

	} else if (fsa_data->data == NULL) {
	crm_err("%s: No message data available. Origin: %s", caller, fsa_data->origin);

	} else if (fsa_data->data_type != a_type) {
	crm_crit("%s: Message data was the wrong type! %d vs. requested=%d. Origin: %s",
	caller, fsa_data->data_type, a_type, fsa_data->origin);
	CRM_ASSERT(fsa_data->data_type == a_type);
	} else {
	ret_val = fsa_data->data;
	}

	return ret_val;
	}

	/* A_MSG_ROUTE */
	void
	do_msg_route(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	ha_msg_input_t *input = fsa_typed_data(fsa_dt_ha_msg);

	route_message(msg_data->fsa_cause, input->msg);
	}

	void
	route_message(enum crmd_fsa_cause cause, xmlNode * input)
	{
	ha_msg_input_t fsa_input;
	enum crmd_fsa_input result = I_NULL;

	fsa_input.msg = input;
	CRM_CHECK(cause == C_IPC_MESSAGE \|\| cause == C_HA_MESSAGE, return);

	/* try passing the buck first */
	if (relay_message(input, cause == C_IPC_MESSAGE)) {
	return;
	}

	/* handle locally */
	result = handle_message(input, cause);

	/* done or process later? */
	switch (result) {
	case I_NULL:
	case I_CIB_OP:
	case I_ROUTER:
	case I_NODE_JOIN:
	case I_JOIN_REQUEST:
	case I_JOIN_RESULT:
	break;
	default:
	/* Defering local processing of message */
	register_fsa_input_later(cause, result, &fsa_input);
	return;
	}

	if (result != I_NULL) {
	/* add to the front of the queue */
	register_fsa_input(cause, result, &fsa_input);
	}
	}

	gboolean
	relay_message(xmlNode * msg, gboolean originated_locally)
	{
	enum crm_ais_msg_types dest = crm_msg_ais;
	bool is_for_dc = false;
	bool is_for_dcib = false;
	bool is_for_te = false;
	bool is_for_crm = false;
	bool is_for_cib = false;
	bool is_local = false;
	bool broadcast = false;
	const char *host_to = NULL;
	const char *sys_to = NULL;
	const char *sys_from = NULL;
	const char *type = NULL;
	const char *task = NULL;
	const char *ref = NULL;
	crm_node_t *node_to = NULL;

	CRM_CHECK(msg != NULL, return TRUE);

	host_to = crm_element_value(msg, F_CRM_HOST_TO);
	sys_to = crm_element_value(msg, F_CRM_SYS_TO);
	sys_from = crm_element_value(msg, F_CRM_SYS_FROM);
	type = crm_element_value(msg, F_TYPE);
	task = crm_element_value(msg, F_CRM_TASK);
	ref = crm_element_value(msg, XML_ATTR_REFERENCE);

	broadcast = pcmk__str_empty(host_to);

	if (ref == NULL) {
	ref = "without reference ID";
	}

	if (pcmk__str_eq(task, CRM_OP_HELLO, pcmk__str_casei)) {
	crm_trace("Received hello %s from %s (no processing needed)",
	ref, pcmk__s(sys_from, "unidentified source"));
	crm_log_xml_trace(msg, "hello");
	return TRUE;
	}

	// Require message type (set by create_request())
	if (!pcmk__str_eq(type, T_CRM, pcmk__str_casei)) {
	crm_warn("Ignoring invalid message %s with type '%s' (not '" T_CRM "')",
	ref, pcmk__s(type, ""));
	crm_log_xml_trace(msg, "ignored");
	return TRUE;
	}

	// Require a destination subsystem (also set by create_request())
	if (sys_to == NULL) {
	crm_warn("Ignoring invalid message %s with no " F_CRM_SYS_TO, ref);
	crm_log_xml_trace(msg, "ignored");
	return TRUE;
	}

	// Get the message type appropriate to the destination subsystem
	if (is_corosync_cluster()) {
	dest = text2msg_type(sys_to);
	if ((dest < crm_msg_ais) \|\| (dest > crm_msg_stonith_ng)) {
	/* Unrecognized value, use a sane default
	*
	* @TODO Maybe we should bail instead
	*/
	dest = crm_msg_crmd;
	}
	}

	is_for_dc = (strcasecmp(CRM_SYSTEM_DC, sys_to) == 0);
	is_for_dcib = (strcasecmp(CRM_SYSTEM_DCIB, sys_to) == 0);
	is_for_te = (strcasecmp(CRM_SYSTEM_TENGINE, sys_to) == 0);
	is_for_cib = (strcasecmp(CRM_SYSTEM_CIB, sys_to) == 0);
	is_for_crm = (strcasecmp(CRM_SYSTEM_CRMD, sys_to) == 0);

	// Check whether message should be processed locally
	is_local = false;
	if (broadcast) {
	if (is_for_dc \|\| is_for_te) {
	is_local = false;

	} else if (is_for_crm) {
	if (pcmk__strcase_any_of(task, CRM_OP_NODE_INFO,
	PCMK__CONTROLD_CMD_NODES, NULL)) {
	/* Node info requests do not specify a host, which is normally
	* treated as "all hosts", because the whole point is that the
	* client may not know the local node name. Always handle these
	* requests locally.
	*/
	is_local = true;
	} else {
	is_local = !originated_locally;
	}

	} else {
	is_local = true;
	}

	} else if (pcmk__str_eq(controld_globals.our_nodename, host_to,
	pcmk__str_casei)) {
	is_local = true;

	} else if (is_for_crm && pcmk__str_eq(task, CRM_OP_LRM_DELETE, pcmk__str_casei)) {
	xmlNode *msg_data = get_message_xml(msg, F_CRM_DATA);
	const char *mode = crm_element_value(msg_data, PCMK__XA_MODE);

	if (pcmk__str_eq(mode, XML_TAG_CIB, pcmk__str_casei)) {
	// Local delete of an offline node's resource history
	is_local = true;
	}
	}

	// Check whether message should be relayed

	if (is_for_dc \|\| is_for_dcib \|\| is_for_te) {
	if (AM_I_DC) {
	if (is_for_te) {
	crm_trace("Route message %s locally as transition request",
	ref);
	crm_log_xml_trace(msg, sys_to);
	send_msg_via_ipc(msg, sys_to);
	return TRUE; // No further processing of message is needed
	}
	crm_trace("Route message %s locally as DC request", ref);
	return FALSE; // More to be done by caller
	}

	if (originated_locally
	&& !pcmk__strcase_any_of(sys_from, CRM_SYSTEM_PENGINE,
	CRM_SYSTEM_TENGINE, NULL)) {
	crm_trace("Relay message %s to DC (via %s)",
	ref, pcmk__s(host_to, "broadcast"));
	crm_log_xml_trace(msg, "relayed");
	if (!broadcast) {
	node_to = crm_get_peer(0, host_to);
	}
	send_cluster_message(node_to, dest, msg, TRUE);
	return TRUE;
	}

	/* Transition engine and scheduler messages are sent only to the DC on
	* the same node. If we are no longer the DC, discard this message.
	*/
	crm_trace("Ignoring message %s because we are no longer DC", ref);
	crm_log_xml_trace(msg, "ignored");
	return TRUE; // No further processing of message is needed
	}

	if (is_local) {
	if (is_for_crm \|\| is_for_cib) {
	crm_trace("Route message %s locally as controller request", ref);
	return FALSE; // More to be done by caller
	}
	crm_trace("Relay message %s locally to %s", ref, sys_to);
	crm_log_xml_trace(msg, "IPC-relay");
	send_msg_via_ipc(msg, sys_to);
	return TRUE;
	}

	if (!broadcast) {
	node_to = pcmk__search_cluster_node_cache(0, host_to, NULL);
	if (node_to == NULL) {
	crm_warn("Ignoring message %s because node %s is unknown",
	ref, host_to);
	crm_log_xml_trace(msg, "ignored");
	return TRUE;
	}
	}

	crm_trace("Relay message %s to %s",
	ref, pcmk__s(host_to, "all peers"));
	crm_log_xml_trace(msg, "relayed");
	send_cluster_message(node_to, dest, msg, TRUE);
	return TRUE;
	}

	// Return true if field contains a positive integer
	static bool
	authorize_version(xmlNode message_data, const char field,
	const char client_name, const char ref, const char *uuid)
	{
	const char *version = crm_element_value(message_data, field);
	long long version_num;

	if ((pcmk__scan_ll(version, &version_num, -1LL) != pcmk_rc_ok)
	\|\| (version_num < 0LL)) {

	crm_warn("Rejected IPC hello from %s: '%s' is not a valid protocol %s "
	CRM_XS " ref=%s uuid=%s",
	client_name, ((version == NULL)? "" : version),
	field, (ref? ref : "none"), uuid);
	return false;
	}
	return true;
	}

	/*!
	* \internal
	* \brief Check whether a client IPC message is acceptable
	*
	* If a given client IPC message is a hello, "authorize" it by ensuring it has
	* valid information such as a protocol version, and return false indicating
	* that nothing further needs to be done with the message. If the message is not
	* a hello, just return true to indicate it needs further processing.
	*
	* \param[in] client_msg XML of IPC message
	* \param[in,out] curr_client If IPC is not proxied, client that sent message
	* \param[in] proxy_session If IPC is proxied, the session ID
	*
	* \return true if message needs further processing, false if it doesn't
	*/
	bool
	controld_authorize_ipc_message(const xmlNode client_msg, pcmk__client_t curr_client,
	const char *proxy_session)
	{
	xmlNode *message_data = NULL;
	const char *client_name = NULL;
	const char *op = crm_element_value(client_msg, F_CRM_TASK);
	const char *ref = crm_element_value(client_msg, XML_ATTR_REFERENCE);
	const char *uuid = (curr_client? curr_client->id : proxy_session);

	if (uuid == NULL) {
	crm_warn("IPC message from client rejected: No client identifier "
	CRM_XS " ref=%s", (ref? ref : "none"));
	goto rejected;
	}

	if (!pcmk__str_eq(CRM_OP_HELLO, op, pcmk__str_casei)) {
	// Only hello messages need to be authorized
	return true;
	}

	message_data = get_message_xml(client_msg, F_CRM_DATA);

	client_name = crm_element_value(message_data, "client_name");
	if (pcmk__str_empty(client_name)) {
	crm_warn("IPC hello from client rejected: No client name",
	CRM_XS " ref=%s uuid=%s", (ref? ref : "none"), uuid);
	goto rejected;
	}
	if (!authorize_version(message_data, "major_version", client_name, ref,
	uuid)) {
	goto rejected;
	}
	if (!authorize_version(message_data, "minor_version", client_name, ref,
	uuid)) {
	goto rejected;
	}

	crm_trace("Validated IPC hello from client %s", client_name);
	crm_log_xml_trace(client_msg, "hello");
	if (curr_client) {
	curr_client->userdata = strdup(client_name);
	}
	controld_trigger_fsa();
	return false;

	rejected:
	crm_log_xml_trace(client_msg, "rejected");
	if (curr_client) {
	qb_ipcs_disconnect(curr_client->ipcs);
	}
	return false;
	}

	static enum crmd_fsa_input
	handle_message(xmlNode *msg, enum crmd_fsa_cause cause)
	{
	const char *type = NULL;

	CRM_CHECK(msg != NULL, return I_NULL);

	type = crm_element_value(msg, F_CRM_MSG_TYPE);
	if (pcmk__str_eq(type, PCMK__VALUE_REQUEST, pcmk__str_none)) {
	return handle_request(msg, cause);
	+ }

	- } else if (pcmk__str_eq(type, XML_ATTR_RESPONSE, pcmk__str_none)) {
	+ if (pcmk__str_eq(type, PCMK__VALUE_RESPONSE, pcmk__str_none)) {
	handle_response(msg);
	return I_NULL;
	}

	crm_warn("Ignoring message with unknown " F_CRM_MSG_TYPE " '%s'",
	pcmk__s(type, ""));
	crm_log_xml_trace(msg, "bad");
	return I_NULL;
	}

	static enum crmd_fsa_input
	handle_failcount_op(xmlNode * stored_msg)
	{
	const char *rsc = NULL;
	const char *uname = NULL;
	const char *op = NULL;
	char *interval_spec = NULL;
	guint interval_ms = 0;
	gboolean is_remote_node = FALSE;
	xmlNode *xml_op = get_message_xml(stored_msg, F_CRM_DATA);

	if (xml_op) {
	xmlNode *xml_rsc = first_named_child(xml_op, XML_CIB_TAG_RESOURCE);
	xmlNode *xml_attrs = first_named_child(xml_op, XML_TAG_ATTRS);

	if (xml_rsc) {
	rsc = ID(xml_rsc);
	}
	if (xml_attrs) {
	op = crm_element_value(xml_attrs,
	CRM_META "_" PCMK__META_CLEAR_FAILURE_OP);
	crm_element_value_ms(xml_attrs,
	CRM_META "_" PCMK__META_CLEAR_FAILURE_INTERVAL,
	&interval_ms);
	}
	}
	uname = crm_element_value(xml_op, XML_LRM_ATTR_TARGET);

	if ((rsc == NULL) \|\| (uname == NULL)) {
	crm_log_xml_warn(stored_msg, "invalid failcount op");
	return I_NULL;
	}

	if (crm_element_value(xml_op, XML_LRM_ATTR_ROUTER_NODE)) {
	is_remote_node = TRUE;
	}

	crm_debug("Clearing failures for %s-interval %s on %s "
	"from attribute manager, CIB, and executor state",
	pcmk__readable_interval(interval_ms), rsc, uname);

	if (interval_ms) {
	interval_spec = crm_strdup_printf("%ums", interval_ms);
	}
	update_attrd_clear_failures(uname, rsc, op, interval_spec, is_remote_node);
	free(interval_spec);

	controld_cib_delete_last_failure(rsc, uname, op, interval_ms);

	lrm_clear_last_failure(rsc, uname, op, interval_ms);

	return I_NULL;
	}

	static enum crmd_fsa_input
	handle_lrm_delete(xmlNode *stored_msg)
	{
	const char *mode = NULL;
	xmlNode *msg_data = get_message_xml(stored_msg, F_CRM_DATA);

	CRM_CHECK(msg_data != NULL, return I_NULL);

	/* CRM_OP_LRM_DELETE has two distinct modes. The default behavior is to
	* relay the operation to the affected node, which will unregister the
	* resource from the local executor, clear the resource's history from the
	* CIB, and do some bookkeeping in the controller.
	*
	* However, if the affected node is offline, the client will specify
	* mode="cib" which means the controller receiving the operation should
	* clear the resource's history from the CIB and nothing else. This is used
	* to clear shutdown locks.
	*/
	mode = crm_element_value(msg_data, PCMK__XA_MODE);
	if ((mode == NULL) \|\| strcmp(mode, XML_TAG_CIB)) {
	// Relay to affected node
	crm_xml_add(stored_msg, F_CRM_SYS_TO, CRM_SYSTEM_LRMD);
	return I_ROUTER;

	} else {
	// Delete CIB history locally (compare with do_lrm_delete())
	const char *from_sys = NULL;
	const char *user_name = NULL;
	const char *rsc_id = NULL;
	const char *node = NULL;
	xmlNode *rsc_xml = NULL;
	int rc = pcmk_rc_ok;

	rsc_xml = first_named_child(msg_data, XML_CIB_TAG_RESOURCE);
	CRM_CHECK(rsc_xml != NULL, return I_NULL);

	rsc_id = ID(rsc_xml);
	from_sys = crm_element_value(stored_msg, F_CRM_SYS_FROM);
	node = crm_element_value(msg_data, XML_LRM_ATTR_TARGET);
	user_name = pcmk__update_acl_user(stored_msg, F_CRM_USER, NULL);
	crm_debug("Handling " CRM_OP_LRM_DELETE " for %s on %s locally%s%s "
	"(clearing CIB resource history only)", rsc_id, node,
	(user_name? " for user " : ""), (user_name? user_name : ""));
	rc = controld_delete_resource_history(rsc_id, node, user_name,
	cib_dryrun\|cib_sync_call);
	if (rc == pcmk_rc_ok) {
	rc = controld_delete_resource_history(rsc_id, node, user_name,
	crmd_cib_smart_opt());
	}

	//Notify client and tengine.(Only notify tengine if mode = "cib" and CRM_OP_LRM_DELETE.)
	if (from_sys) {
	lrmd_event_data_t *op = NULL;
	const char *from_host = crm_element_value(stored_msg,
	F_CRM_HOST_FROM);
	const char *transition;

	if (strcmp(from_sys, CRM_SYSTEM_TENGINE)) {
	transition = crm_element_value(msg_data,
	XML_ATTR_TRANSITION_KEY);
	} else {
	transition = crm_element_value(stored_msg,
	XML_ATTR_TRANSITION_KEY);
	}

	crm_info("Notifying %s on %s that %s was%s deleted",
	from_sys, (from_host? from_host : "local node"), rsc_id,
	((rc == pcmk_rc_ok)? "" : " not"));
	op = lrmd_new_event(rsc_id, PCMK_ACTION_DELETE, 0);
	op->type = lrmd_event_exec_complete;
	op->user_data = strdup(transition? transition : FAKE_TE_ID);
	op->params = pcmk__strkey_table(free, free);
	g_hash_table_insert(op->params, strdup(PCMK_XA_CRM_FEATURE_SET),
	strdup(CRM_FEATURE_SET));
	controld_rc2event(op, rc);
	controld_ack_event_directly(from_host, from_sys, NULL, op, rsc_id);
	lrmd_free_event(op);
	controld_trigger_delete_refresh(from_sys, rsc_id);
	}
	return I_NULL;
	}
	}

	/*!
	* \brief Handle a CRM_OP_REMOTE_STATE message by updating remote peer cache
	*
	* \param[in] msg Message XML
	*
	* \return Next FSA input
	*/
	static enum crmd_fsa_input
	handle_remote_state(const xmlNode *msg)
	{
	const char *conn_host = NULL;
	const char *remote_uname = ID(msg);
	crm_node_t *remote_peer;
	bool remote_is_up = false;
	int rc = pcmk_rc_ok;

	rc = pcmk__xe_get_bool_attr(msg, PCMK__XA_IN_CCM, &remote_is_up);

	CRM_CHECK(remote_uname && rc == pcmk_rc_ok, return I_NULL);

	remote_peer = crm_remote_peer_get(remote_uname);
	CRM_CHECK(remote_peer, return I_NULL);

	pcmk__update_peer_state(__func__, remote_peer,
	remote_is_up ? CRM_NODE_MEMBER : CRM_NODE_LOST,
	0);

	conn_host = crm_element_value(msg, PCMK__XA_CONN_HOST);
	if (conn_host) {
	pcmk__str_update(&remote_peer->conn_host, conn_host);
	} else if (remote_peer->conn_host) {
	free(remote_peer->conn_host);
	remote_peer->conn_host = NULL;
	}

	return I_NULL;
	}

	/*!
	* \brief Handle a CRM_OP_PING message
	*
	* \param[in] msg Message XML
	*
	* \return Next FSA input
	*/
	static enum crmd_fsa_input
	handle_ping(const xmlNode *msg)
	{
	const char *value = NULL;
	xmlNode *ping = NULL;
	xmlNode *reply = NULL;

	// Build reply

	ping = create_xml_node(NULL, XML_CRM_TAG_PING);
	value = crm_element_value(msg, F_CRM_SYS_TO);
	crm_xml_add(ping, XML_PING_ATTR_SYSFROM, value);

	// Add controller state
	value = fsa_state2string(controld_globals.fsa_state);
	crm_xml_add(ping, XML_PING_ATTR_CRMDSTATE, value);
	crm_notice("Current ping state: %s", value); // CTS needs this

	// Add controller health
	// @TODO maybe do some checks to determine meaningful status
	crm_xml_add(ping, XML_PING_ATTR_STATUS, "ok");

	// Send reply
	reply = create_reply(msg, ping);
	free_xml(ping);
	if (reply != NULL) {
	(void) relay_message(reply, TRUE);
	free_xml(reply);
	}

	// Nothing further to do
	return I_NULL;
	}

	/*!
	* \brief Handle a PCMK__CONTROLD_CMD_NODES message
	*
	* \param[in] request Message XML
	*
	* \return Next FSA input
	*/
	static enum crmd_fsa_input
	handle_node_list(const xmlNode *request)
	{
	GHashTableIter iter;
	crm_node_t *node = NULL;
	xmlNode *reply = NULL;
	xmlNode *reply_data = NULL;

	// Create message data for reply
	reply_data = create_xml_node(NULL, XML_CIB_TAG_NODES);
	g_hash_table_iter_init(&iter, crm_peer_cache);
	while (g_hash_table_iter_next(&iter, NULL, (gpointer *) & node)) {
	xmlNode *xml = create_xml_node(reply_data, XML_CIB_TAG_NODE);

	crm_xml_add_ll(xml, PCMK_XA_ID, (long long) node->id); // uint32_t
	crm_xml_add(xml, XML_ATTR_UNAME, node->uname);
	crm_xml_add(xml, PCMK__XA_IN_CCM, node->state);
	}

	// Create and send reply
	reply = create_reply(request, reply_data);
	free_xml(reply_data);
	if (reply) {
	(void) relay_message(reply, TRUE);
	free_xml(reply);
	}

	// Nothing further to do
	return I_NULL;
	}

	/*!
	* \brief Handle a CRM_OP_NODE_INFO request
	*
	* \param[in] msg Message XML
	*
	* \return Next FSA input
	*/
	static enum crmd_fsa_input
	handle_node_info_request(const xmlNode *msg)
	{
	const char *value = NULL;
	crm_node_t *node = NULL;
	int node_id = 0;
	xmlNode *reply = NULL;
	xmlNode *reply_data = NULL;

	// Build reply

	reply_data = create_xml_node(NULL, XML_CIB_TAG_NODE);
	crm_xml_add(reply_data, XML_PING_ATTR_SYSFROM, CRM_SYSTEM_CRMD);

	// Add whether current partition has quorum
	pcmk__xe_set_bool_attr(reply_data, PCMK_XA_HAVE_QUORUM,
	pcmk_is_set(controld_globals.flags,
	controld_has_quorum));

	// Check whether client requested node info by ID and/or name
	crm_element_value_int(msg, PCMK_XA_ID, &node_id);
	if (node_id < 0) {
	node_id = 0;
	}
	value = crm_element_value(msg, XML_ATTR_UNAME);

	// Default to local node if none given
	if ((node_id == 0) && (value == NULL)) {
	value = controld_globals.our_nodename;
	}

	node = pcmk__search_node_caches(node_id, value, CRM_GET_PEER_ANY);
	if (node) {
	crm_xml_add(reply_data, PCMK_XA_ID, node->uuid);
	crm_xml_add(reply_data, XML_ATTR_UNAME, node->uname);
	crm_xml_add(reply_data, PCMK__XA_CRMD, node->state);
	pcmk__xe_set_bool_attr(reply_data, XML_NODE_IS_REMOTE,
	pcmk_is_set(node->flags, crm_remote_node));
	}

	// Send reply
	reply = create_reply(msg, reply_data);
	free_xml(reply_data);
	if (reply != NULL) {
	(void) relay_message(reply, TRUE);
	free_xml(reply);
	}

	// Nothing further to do
	return I_NULL;
	}

	static void
	verify_feature_set(xmlNode *msg)
	{
	const char *dc_version = crm_element_value(msg, PCMK_XA_CRM_FEATURE_SET);

	if (dc_version == NULL) {
	/* All we really know is that the DC feature set is older than 3.1.0,
	* but that's also all that really matters.
	*/
	dc_version = "3.0.14";
	}

	if (feature_set_compatible(dc_version, CRM_FEATURE_SET)) {
	crm_trace("Local feature set (%s) is compatible with DC's (%s)",
	CRM_FEATURE_SET, dc_version);
	} else {
	crm_err("Local feature set (%s) is incompatible with DC's (%s)",
	CRM_FEATURE_SET, dc_version);

	// Nothing is likely to improve without administrator involvement
	controld_set_fsa_input_flags(R_STAYDOWN);
	crmd_exit(CRM_EX_FATAL);
	}
	}

	// DC gets own shutdown all-clear
	static enum crmd_fsa_input
	handle_shutdown_self_ack(xmlNode *stored_msg)
	{
	const char *host_from = crm_element_value(stored_msg, F_CRM_HOST_FROM);

	if (pcmk_is_set(controld_globals.fsa_input_register, R_SHUTDOWN)) {
	// The expected case -- we initiated own shutdown sequence
	crm_info("Shutting down controller");
	return I_STOP;
	}

	if (pcmk__str_eq(host_from, controld_globals.dc_name, pcmk__str_casei)) {
	// Must be logic error -- DC confirming its own unrequested shutdown
	crm_err("Shutting down controller immediately due to "
	"unexpected shutdown confirmation");
	return I_TERMINATE;
	}

	if (controld_globals.fsa_state != S_STOPPING) {
	// Shouldn't happen -- non-DC confirming unrequested shutdown
	crm_err("Starting new DC election because %s is "
	"confirming shutdown we did not request",
	(host_from? host_from : "another node"));
	return I_ELECTION;
	}

	// Shouldn't happen, but we are already stopping anyway
	crm_debug("Ignoring unexpected shutdown confirmation from %s",
	(host_from? host_from : "another node"));
	return I_NULL;
	}

	// Non-DC gets shutdown all-clear from DC
	static enum crmd_fsa_input
	handle_shutdown_ack(xmlNode *stored_msg)
	{
	const char *host_from = crm_element_value(stored_msg, F_CRM_HOST_FROM);

	if (host_from == NULL) {
	crm_warn("Ignoring shutdown request without origin specified");
	return I_NULL;
	}

	if (pcmk__str_eq(host_from, controld_globals.dc_name,
	pcmk__str_null_matches\|pcmk__str_casei)) {

	if (pcmk_is_set(controld_globals.fsa_input_register, R_SHUTDOWN)) {
	crm_info("Shutting down controller after confirmation from %s",
	host_from);
	} else {
	crm_err("Shutting down controller after unexpected "
	"shutdown request from %s", host_from);
	controld_set_fsa_input_flags(R_STAYDOWN);
	}
	return I_STOP;
	}

	crm_warn("Ignoring shutdown request from %s because DC is %s",
	host_from, controld_globals.dc_name);
	return I_NULL;
	}

	static enum crmd_fsa_input
	handle_request(xmlNode *stored_msg, enum crmd_fsa_cause cause)
	{
	xmlNode *msg = NULL;
	const char *op = crm_element_value(stored_msg, F_CRM_TASK);

	/* Optimize this for the DC - it has the most to do */

	crm_log_xml_trace(stored_msg, "request");
	if (op == NULL) {
	crm_warn("Ignoring request without " F_CRM_TASK);
	return I_NULL;
	}

	if (strcmp(op, CRM_OP_SHUTDOWN_REQ) == 0) {
	const char *from = crm_element_value(stored_msg, F_CRM_HOST_FROM);
	crm_node_t *node = pcmk__search_cluster_node_cache(0, from, NULL);

	pcmk__update_peer_expected(__func__, node, CRMD_JOINSTATE_DOWN);
	if(AM_I_DC == FALSE) {
	return I_NULL; /* Done */
	}
	}

	/========== DC-Only Actions ==========/
	if (AM_I_DC) {
	if (strcmp(op, CRM_OP_JOIN_ANNOUNCE) == 0) {
	return I_NODE_JOIN;

	} else if (strcmp(op, CRM_OP_JOIN_REQUEST) == 0) {
	return I_JOIN_REQUEST;

	} else if (strcmp(op, CRM_OP_JOIN_CONFIRM) == 0) {
	return I_JOIN_RESULT;

	} else if (strcmp(op, CRM_OP_SHUTDOWN) == 0) {
	return handle_shutdown_self_ack(stored_msg);

	} else if (strcmp(op, CRM_OP_SHUTDOWN_REQ) == 0) {
	// Another controller wants to shut down its node
	return handle_shutdown_request(stored_msg);
	}
	}

	/========== common actions ==========/
	if (strcmp(op, CRM_OP_NOVOTE) == 0) {
	ha_msg_input_t fsa_input;

	fsa_input.msg = stored_msg;
	register_fsa_input_adv(C_HA_MESSAGE, I_NULL, &fsa_input,
	A_ELECTION_COUNT \| A_ELECTION_CHECK, FALSE,
	__func__);

	} else if (strcmp(op, CRM_OP_REMOTE_STATE) == 0) {
	/* a remote connection host is letting us know the node state */
	return handle_remote_state(stored_msg);

	} else if (strcmp(op, CRM_OP_THROTTLE) == 0) {
	throttle_update(stored_msg);
	if (AM_I_DC && (controld_globals.transition_graph != NULL)
	&& !controld_globals.transition_graph->complete) {

	crm_debug("The throttle changed. Trigger a graph.");
	trigger_graph();
	}
	return I_NULL;

	} else if (strcmp(op, CRM_OP_CLEAR_FAILCOUNT) == 0) {
	return handle_failcount_op(stored_msg);

	} else if (strcmp(op, CRM_OP_VOTE) == 0) {
	/* count the vote and decide what to do after that */
	ha_msg_input_t fsa_input;

	fsa_input.msg = stored_msg;
	register_fsa_input_adv(C_HA_MESSAGE, I_NULL, &fsa_input,
	A_ELECTION_COUNT \| A_ELECTION_CHECK, FALSE,
	__func__);

	/* Sometimes we _must_ go into S_ELECTION */
	if (controld_globals.fsa_state == S_HALT) {
	crm_debug("Forcing an election from S_HALT");
	return I_ELECTION;
	}

	} else if (strcmp(op, CRM_OP_JOIN_OFFER) == 0) {
	verify_feature_set(stored_msg);
	crm_debug("Raising I_JOIN_OFFER: join-%s", crm_element_value(stored_msg, F_CRM_JOIN_ID));
	return I_JOIN_OFFER;

	} else if (strcmp(op, CRM_OP_JOIN_ACKNAK) == 0) {
	crm_debug("Raising I_JOIN_RESULT: join-%s", crm_element_value(stored_msg, F_CRM_JOIN_ID));
	return I_JOIN_RESULT;

	} else if (strcmp(op, CRM_OP_LRM_DELETE) == 0) {
	return handle_lrm_delete(stored_msg);

	} else if ((strcmp(op, CRM_OP_LRM_FAIL) == 0)
	\|\| (strcmp(op, CRM_OP_LRM_REFRESH) == 0) // @COMPAT
	\|\| (strcmp(op, CRM_OP_REPROBE) == 0)) {

	crm_xml_add(stored_msg, F_CRM_SYS_TO, CRM_SYSTEM_LRMD);
	return I_ROUTER;

	} else if (strcmp(op, CRM_OP_NOOP) == 0) {
	return I_NULL;

	} else if (strcmp(op, CRM_OP_LOCAL_SHUTDOWN) == 0) {

	crm_shutdown(SIGTERM);
	/return I_SHUTDOWN; /
	return I_NULL;

	} else if (strcmp(op, CRM_OP_PING) == 0) {
	return handle_ping(stored_msg);

	} else if (strcmp(op, CRM_OP_NODE_INFO) == 0) {
	return handle_node_info_request(stored_msg);

	} else if (strcmp(op, CRM_OP_RM_NODE_CACHE) == 0) {
	int id = 0;
	const char *name = NULL;

	crm_element_value_int(stored_msg, PCMK_XA_ID, &id);
	name = crm_element_value(stored_msg, XML_ATTR_UNAME);

	if(cause == C_IPC_MESSAGE) {
	msg = create_request(CRM_OP_RM_NODE_CACHE, NULL, NULL, CRM_SYSTEM_CRMD, CRM_SYSTEM_CRMD, NULL);
	if (send_cluster_message(NULL, crm_msg_crmd, msg, TRUE) == FALSE) {
	crm_err("Could not instruct peers to remove references to node %s/%u", name, id);
	} else {
	crm_notice("Instructing peers to remove references to node %s/%u", name, id);
	}
	free_xml(msg);

	} else {
	reap_crm_member(id, name);

	/* If we're forgetting this node, also forget any failures to fence
	* it, so we don't carry that over to any node added later with the
	* same name.
	*/
	st_fail_count_reset(name);
	}

	} else if (strcmp(op, CRM_OP_MAINTENANCE_NODES) == 0) {
	xmlNode *xml = get_message_xml(stored_msg, F_CRM_DATA);

	remote_ra_process_maintenance_nodes(xml);

	} else if (strcmp(op, PCMK__CONTROLD_CMD_NODES) == 0) {
	return handle_node_list(stored_msg);

	/========== (NOT_DC)-Only Actions ==========/
	} else if (!AM_I_DC) {

	if (strcmp(op, CRM_OP_SHUTDOWN) == 0) {
	return handle_shutdown_ack(stored_msg);
	}

	} else {
	crm_err("Unexpected request (%s) sent to %s", op, AM_I_DC ? "the DC" : "non-DC node");
	crm_log_xml_err(stored_msg, "Unexpected");
	}

	return I_NULL;
	}

	static void
	handle_response(xmlNode *stored_msg)
	{
	const char *op = crm_element_value(stored_msg, F_CRM_TASK);

	crm_log_xml_trace(stored_msg, "reply");
	if (op == NULL) {
	crm_warn("Ignoring reply without " F_CRM_TASK);

	} else if (AM_I_DC && strcmp(op, CRM_OP_PECALC) == 0) {
	// Check whether scheduler answer been superseded by subsequent request
	const char *msg_ref = crm_element_value(stored_msg, XML_ATTR_REFERENCE);

	if (msg_ref == NULL) {
	crm_err("%s - Ignoring calculation with no reference", op);

	} else if (pcmk__str_eq(msg_ref, controld_globals.fsa_pe_ref,
	pcmk__str_none)) {
	ha_msg_input_t fsa_input;

	controld_stop_sched_timer();
	fsa_input.msg = stored_msg;
	register_fsa_input_later(C_IPC_MESSAGE, I_PE_SUCCESS, &fsa_input);

	} else {
	crm_info("%s calculation %s is obsolete", op, msg_ref);
	}

	} else if (strcmp(op, CRM_OP_VOTE) == 0
	\|\| strcmp(op, CRM_OP_SHUTDOWN_REQ) == 0 \|\| strcmp(op, CRM_OP_SHUTDOWN) == 0) {

	} else {
	const char *host_from = crm_element_value(stored_msg, F_CRM_HOST_FROM);

	crm_err("Unexpected response (op=%s, src=%s) sent to the %s",
	op, host_from, AM_I_DC ? "DC" : "controller");
	}
	}

	static enum crmd_fsa_input
	handle_shutdown_request(xmlNode * stored_msg)
	{
	/* handle here to avoid potential version issues
	* where the shutdown message/procedure may have
	* been changed in later versions.
	*
	* This way the DC is always in control of the shutdown
	*/

	char *now_s = NULL;
	const char *host_from = crm_element_value(stored_msg, F_CRM_HOST_FROM);

	if (host_from == NULL) {
	/* we're shutting down and the DC */
	host_from = controld_globals.our_nodename;
	}

	crm_info("Creating shutdown request for %s (state=%s)", host_from,
	fsa_state2string(controld_globals.fsa_state));
	crm_log_xml_trace(stored_msg, "message");

	now_s = pcmk__ttoa(time(NULL));
	update_attrd(host_from, XML_CIB_ATTR_SHUTDOWN, now_s, NULL, FALSE);
	free(now_s);

	/* will be picked up by the TE as long as its running */
	return I_NULL;
	}

	static void
	send_msg_via_ipc(xmlNode * msg, const char *sys)
	{
	pcmk__client_t *client_channel = NULL;

	CRM_CHECK(sys != NULL, return);

	client_channel = pcmk__find_client_by_id(sys);

	if (crm_element_value(msg, F_CRM_HOST_FROM) == NULL) {
	crm_xml_add(msg, F_CRM_HOST_FROM, controld_globals.our_nodename);
	}

	if (client_channel != NULL) {
	/* Transient clients such as crmadmin */
	pcmk__ipc_send_xml(client_channel, 0, msg, crm_ipc_server_event);

	} else if (pcmk__str_eq(sys, CRM_SYSTEM_TENGINE, pcmk__str_none)) {
	xmlNode *data = get_message_xml(msg, F_CRM_DATA);

	process_te_message(msg, data);

	} else if (pcmk__str_eq(sys, CRM_SYSTEM_LRMD, pcmk__str_none)) {
	fsa_data_t fsa_data;
	ha_msg_input_t fsa_input;

	fsa_input.msg = msg;
	fsa_input.xml = get_message_xml(msg, F_CRM_DATA);

	fsa_data.id = 0;
	fsa_data.actions = 0;
	fsa_data.data = &fsa_input;
	fsa_data.fsa_input = I_MESSAGE;
	fsa_data.fsa_cause = C_IPC_MESSAGE;
	fsa_data.origin = __func__;
	fsa_data.data_type = fsa_dt_ha_msg;

	do_lrm_invoke(A_LRM_INVOKE, C_IPC_MESSAGE, controld_globals.fsa_state,
	I_MESSAGE, &fsa_data);

	} else if (crmd_is_proxy_session(sys)) {
	crmd_proxy_send(sys, msg);

	} else {
	crm_info("Received invalid request: unknown subsystem '%s'", sys);
	}
	}

	void
	delete_ha_msg_input(ha_msg_input_t * orig)
	{
	if (orig == NULL) {
	return;
	}
	free_xml(orig->msg);
	free(orig);
	}

	/*!
	* \internal
	* \brief Notify the cluster of a remote node state change
	*
	* \param[in] node_name Node's name
	* \param[in] node_up true if node is up, false if down
	*/
	void
	broadcast_remote_state_message(const char *node_name, bool node_up)
	{
	xmlNode *msg = create_request(CRM_OP_REMOTE_STATE, NULL, NULL,
	CRM_SYSTEM_CRMD, CRM_SYSTEM_CRMD, NULL);

	crm_info("Notifying cluster of Pacemaker Remote node %s %s",
	node_name, node_up? "coming up" : "going down");

	crm_xml_add(msg, PCMK_XA_ID, node_name);
	pcmk__xe_set_bool_attr(msg, PCMK__XA_IN_CCM, node_up);

	if (node_up) {
	crm_xml_add(msg, PCMK__XA_CONN_HOST, controld_globals.our_nodename);
	}

	send_cluster_message(NULL, crm_msg_crmd, msg, TRUE);
	free_xml(msg);
	}

	diff --git a/daemons/schedulerd/schedulerd_messages.c b/daemons/schedulerd/schedulerd_messages.c
	index 93ab0bbeb0..c8439131ad 100644
	--- a/daemons/schedulerd/schedulerd_messages.c
	+++ b/daemons/schedulerd/schedulerd_messages.c
	@@ -1,335 +1,335 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <crm/crm.h>
	#include <crm/msg_xml.h>
	#include <pacemaker-internal.h>

	#include <stdbool.h>
	#include <sys/stat.h>
	#include <sys/types.h>
	#include <unistd.h>

	#include "pacemaker-schedulerd.h"

	static GHashTable *schedulerd_handlers = NULL;

	static pcmk_scheduler_t *
	init_working_set(void)
	{
	pcmk_scheduler_t *scheduler = pe_new_working_set();

	CRM_ASSERT(scheduler != NULL);

	crm_config_error = FALSE;
	crm_config_warning = FALSE;

	was_processing_error = FALSE;
	was_processing_warning = FALSE;

	scheduler->priv = logger_out;
	return scheduler;
	}

	static xmlNode *
	handle_pecalc_request(pcmk__request_t *request)
	{
	static struct series_s {
	const char *name;
	const char *param;

	/* Maximum number of inputs of this kind to save to disk.
	* If -1, save all; if 0, save none.
	*/
	int wrap;
	} series[] = {
	{ "pe-error", PCMK_OPT_PE_ERROR_SERIES_MAX, -1 },
	{ "pe-warn", PCMK_OPT_PE_WARN_SERIES_MAX, 5000 },
	{ "pe-input", PCMK_OPT_PE_INPUT_SERIES_MAX, 4000 },
	};

	xmlNode *msg = request->xml;
	xmlNode *xml_data = get_message_xml(msg, F_CRM_DATA);

	static char *last_digest = NULL;
	static char *filename = NULL;

	unsigned int seq;
	int series_id = 0;
	int series_wrap = 0;
	char *digest = NULL;
	const char *value = NULL;
	time_t execution_date = time(NULL);
	xmlNode *converted = NULL;
	xmlNode *reply = NULL;
	bool is_repoke = false;
	bool process = true;
	pcmk_scheduler_t *scheduler = init_working_set();

	pcmk__ipc_send_ack(request->ipc_client, request->ipc_id, request->ipc_flags,
	"ack", NULL, CRM_EX_INDETERMINATE);

	digest = calculate_xml_versioned_digest(xml_data, FALSE, FALSE,
	CRM_FEATURE_SET);
	converted = copy_xml(xml_data);
	if (!cli_config_update(&converted, NULL, TRUE)) {
	scheduler->graph = create_xml_node(NULL, XML_TAG_GRAPH);
	crm_xml_add_int(scheduler->graph, "transition_id", 0);
	crm_xml_add_int(scheduler->graph, PCMK_OPT_CLUSTER_DELAY, 0);
	process = false;
	free(digest);

	} else if (pcmk__str_eq(digest, last_digest, pcmk__str_casei)) {
	is_repoke = true;
	free(digest);

	} else {
	free(last_digest);
	last_digest = digest;
	}

	if (process) {
	pcmk__schedule_actions(converted,
	pcmk_sched_no_counts
	\|pcmk_sched_no_compat
	\|pcmk_sched_show_utilization, scheduler);
	}

	// Get appropriate index into series[] array
	if (was_processing_error \|\| crm_config_error) {
	series_id = 0;
	} else if (was_processing_warning \|\| crm_config_warning) {
	series_id = 1;
	} else {
	series_id = 2;
	}

	value = pe_pref(scheduler->config_hash, series[series_id].param);
	if ((value == NULL)
	\|\| (pcmk__scan_min_int(value, &series_wrap, -1) != pcmk_rc_ok)) {
	series_wrap = series[series_id].wrap;
	}

	if (pcmk__read_series_sequence(PE_STATE_DIR, series[series_id].name,
	&seq) != pcmk_rc_ok) {
	// @TODO maybe handle errors better ...
	seq = 0;
	}
	crm_trace("Series %s: wrap=%d, seq=%u, pref=%s",
	series[series_id].name, series_wrap, seq, value);

	scheduler->input = NULL;
	reply = create_reply(msg, scheduler->graph);

	if (reply == NULL) {
	pcmk__format_result(&request->result, CRM_EX_ERROR, PCMK_EXEC_ERROR,
	"Failed building ping reply for client %s",
	pcmk__client_name(request->ipc_client));
	goto done;
	}

	if (series_wrap == 0) { // Don't save any inputs of this kind
	free(filename);
	filename = NULL;

	} else if (!is_repoke) { // Input changed, save to disk
	free(filename);
	filename = pcmk__series_filename(PE_STATE_DIR,
	series[series_id].name, seq, true);
	}

	crm_xml_add(reply, F_CRM_TGRAPH_INPUT, filename);
	crm_xml_add_int(reply, PCMK__XA_GRAPH_ERRORS, was_processing_error);
	crm_xml_add_int(reply, PCMK__XA_GRAPH_WARNINGS, was_processing_warning);
	crm_xml_add_int(reply, PCMK__XA_CONFIG_ERRORS, crm_config_error);
	crm_xml_add_int(reply, PCMK__XA_CONFIG_WARNINGS, crm_config_warning);

	pcmk__log_transition_summary(filename);

	if (series_wrap == 0) {
	crm_debug("Not saving input to disk (disabled by configuration)");

	} else if (is_repoke) {
	crm_info("Input has not changed since last time, not saving to disk");

	} else {
	unlink(filename);
	crm_xml_add_ll(xml_data, "execution-date", (long long) execution_date);
	write_xml_file(xml_data, filename, TRUE);
	pcmk__write_series_sequence(PE_STATE_DIR, series[series_id].name,
	++seq, series_wrap);
	}

	pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);

	done:
	free_xml(converted);
	pe_free_working_set(scheduler);

	return reply;
	}

	static xmlNode *
	handle_unknown_request(pcmk__request_t *request)
	{
	pcmk__ipc_send_ack(request->ipc_client, request->ipc_id, request->ipc_flags,
	"ack", NULL, CRM_EX_INVALID_PARAM);

	pcmk__format_result(&request->result, CRM_EX_PROTOCOL, PCMK_EXEC_INVALID,
	"Unknown IPC request type '%s' (bug?)",
	pcmk__client_name(request->ipc_client));
	return NULL;
	}

	static xmlNode *
	handle_hello_request(pcmk__request_t *request)
	{
	pcmk__ipc_send_ack(request->ipc_client, request->ipc_id, request->ipc_flags,
	"ack", NULL, CRM_EX_INDETERMINATE);

	crm_trace("Received IPC hello from %s", pcmk__client_name(request->ipc_client));

	pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
	return NULL;
	}

	static void
	schedulerd_register_handlers(void)
	{
	pcmk__server_command_t handlers[] = {
	{ CRM_OP_HELLO, handle_hello_request },
	{ CRM_OP_PECALC, handle_pecalc_request },
	{ NULL, handle_unknown_request },
	};

	schedulerd_handlers = pcmk__register_handlers(handlers);
	}

	static int32_t
	pe_ipc_accept(qb_ipcs_connection_t * c, uid_t uid, gid_t gid)
	{
	crm_trace("Connection %p", c);
	if (pcmk__new_client(c, uid, gid) == NULL) {
	return -EIO;
	}
	return 0;
	}

	static int32_t
	pe_ipc_dispatch(qb_ipcs_connection_t * qbc, void *data, size_t size)
	{
	uint32_t id = 0;
	uint32_t flags = 0;
	xmlNode *msg = NULL;
	pcmk__client_t *c = pcmk__find_client(qbc);
	const char *sys_to = NULL;

	CRM_CHECK(c != NULL, return 0);

	if (schedulerd_handlers == NULL) {
	schedulerd_register_handlers();
	}

	msg = pcmk__client_data2xml(c, data, &id, &flags);
	if (msg == NULL) {
	pcmk__ipc_send_ack(c, id, flags, "ack", NULL, CRM_EX_PROTOCOL);
	return 0;
	}

	sys_to = crm_element_value(msg, F_CRM_SYS_TO);

	if (pcmk__str_eq(crm_element_value(msg, F_CRM_MSG_TYPE),
	- XML_ATTR_RESPONSE, pcmk__str_none)) {
	+ PCMK__VALUE_RESPONSE, pcmk__str_none)) {
	pcmk__ipc_send_ack(c, id, flags, "ack", NULL, CRM_EX_INDETERMINATE);
	crm_info("Ignoring IPC reply from %s", pcmk__client_name(c));

	} else if (!pcmk__str_eq(sys_to, CRM_SYSTEM_PENGINE, pcmk__str_none)) {
	pcmk__ipc_send_ack(c, id, flags, "ack", NULL, CRM_EX_INDETERMINATE);
	crm_info("Ignoring invalid IPC message: to '%s' not "
	CRM_SYSTEM_PENGINE, pcmk__s(sys_to, ""));

	} else {
	char *log_msg = NULL;
	const char *reason = NULL;
	xmlNode *reply = NULL;

	pcmk__request_t request = {
	.ipc_client = c,
	.ipc_id = id,
	.ipc_flags = flags,
	.peer = NULL,
	.xml = msg,
	.call_options = 0,
	.result = PCMK__UNKNOWN_RESULT,
	};

	request.op = crm_element_value_copy(request.xml, F_CRM_TASK);
	CRM_CHECK(request.op != NULL, return 0);

	reply = pcmk__process_request(&request, schedulerd_handlers);

	if (reply != NULL) {
	pcmk__ipc_send_xml(c, id, reply, crm_ipc_server_event);
	free_xml(reply);
	}

	reason = request.result.exit_reason;

	log_msg = crm_strdup_printf("Processed %s request from %s %s: %s%s%s%s",
	request.op, pcmk__request_origin_type(&request),
	pcmk__request_origin(&request),
	pcmk_exec_status_str(request.result.execution_status),
	(reason == NULL)? "" : " (",
	(reason == NULL)? "" : reason,
	(reason == NULL)? "" : ")");

	if (!pcmk__result_ok(&request.result)) {
	crm_warn("%s", log_msg);
	} else {
	crm_debug("%s", log_msg);
	}

	free(log_msg);
	pcmk__reset_request(&request);
	}

	free_xml(msg);
	return 0;
	}

	/* Error code means? */
	static int32_t
	pe_ipc_closed(qb_ipcs_connection_t * c)
	{
	pcmk__client_t *client = pcmk__find_client(c);

	if (client == NULL) {
	return 0;
	}
	crm_trace("Connection %p", c);
	pcmk__free_client(client);
	return 0;
	}

	static void
	pe_ipc_destroy(qb_ipcs_connection_t * c)
	{
	crm_trace("Connection %p", c);
	pe_ipc_closed(c);
	}

	struct qb_ipcs_service_handlers ipc_callbacks = {
	.connection_accept = pe_ipc_accept,
	.connection_created = NULL,
	.msg_process = pe_ipc_dispatch,
	.connection_closed = pe_ipc_closed,
	.connection_destroyed = pe_ipc_destroy
	};
	diff --git a/include/crm/common/options_internal.h b/include/crm/common/options_internal.h
	index 8efb045c81..92a5e55501 100644
	--- a/include/crm/common/options_internal.h
	+++ b/include/crm/common/options_internal.h
	@@ -1,172 +1,173 @@
	/*
	* Copyright 2006-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#ifndef PCMK__OPTIONS_INTERNAL__H
	# define PCMK__OPTIONS_INTERNAL__H

	# ifndef PCMK__CONFIG_H
	# define PCMK__CONFIG_H
	# include <config.h> // _Noreturn
	# endif

	# include <glib.h> // GHashTable
	# include <stdbool.h> // bool

	#include <crm/common/util.h> // pcmk_parse_interval_spec()

	_Noreturn void pcmk__cli_help(char cmd);


	/*
	* Environment variable option handling
	*/

	const char pcmk__env_option(const char option);
	void pcmk__set_env_option(const char option, const char value, bool compat);
	bool pcmk__env_option_enabled(const char daemon, const char option);


	/*
	* Cluster option handling
	*/

	typedef struct pcmk__cluster_option_s {
	const char *name;
	const char *alt_name;
	const char *type;
	const char *values;
	const char *default_value;

	bool (is_valid)(const char );

	const char *description_short;
	const char *description_long;

	} pcmk__cluster_option_t;

	const char pcmk__cluster_option(GHashTable options,
	const pcmk__cluster_option_t *option_list,
	int len, const char *name);

	gchar pcmk__format_option_metadata(const char name, const char *desc_short,
	const char *desc_long,
	pcmk__cluster_option_t *option_list,
	int len);

	void pcmk__validate_cluster_options(GHashTable *options,
	pcmk__cluster_option_t *option_list,
	int len);

	bool pcmk__valid_interval_spec(const char *value);
	bool pcmk__valid_boolean(const char *value);
	bool pcmk__valid_int(const char *value);
	bool pcmk__valid_positive_int(const char *value);
	bool pcmk__valid_no_quorum_policy(const char *value);
	bool pcmk__valid_percentage(const char *value);
	bool pcmk__valid_script(const char *value);

	// from watchdog.c
	long pcmk__get_sbd_timeout(void);
	bool pcmk__get_sbd_sync_resource_startup(void);
	long pcmk__auto_watchdog_timeout(void);
	bool pcmk__valid_sbd_timeout(const char *value);

	// Constants for environment variable names
	#define PCMK__ENV_AUTHKEY_LOCATION "authkey_location"
	#define PCMK__ENV_BLACKBOX "blackbox"
	#define PCMK__ENV_CALLGRIND_ENABLED "callgrind_enabled"
	#define PCMK__ENV_CLUSTER_TYPE "cluster_type"
	#define PCMK__ENV_DEBUG "debug"
	#define PCMK__ENV_DH_MAX_BITS "dh_max_bits"
	#define PCMK__ENV_DH_MIN_BITS "dh_min_bits"
	#define PCMK__ENV_FAIL_FAST "fail_fast"
	#define PCMK__ENV_IPC_BUFFER "ipc_buffer"
	#define PCMK__ENV_IPC_TYPE "ipc_type"
	#define PCMK__ENV_LOGFACILITY "logfacility"
	#define PCMK__ENV_LOGFILE "logfile"
	#define PCMK__ENV_LOGFILE_MODE "logfile_mode"
	#define PCMK__ENV_LOGPRIORITY "logpriority"
	#define PCMK__ENV_NODE_ACTION_LIMIT "node_action_limit"
	#define PCMK__ENV_NODE_START_STATE "node_start_state"
	#define PCMK__ENV_PANIC_ACTION "panic_action"
	#define PCMK__ENV_REMOTE_ADDRESS "remote_address"
	#define PCMK__ENV_REMOTE_SCHEMA_DIR "remote_schema_directory"
	#define PCMK__ENV_REMOTE_PID1 "remote_pid1"
	#define PCMK__ENV_REMOTE_PORT "remote_port"
	#define PCMK__ENV_RESPAWNED "respawned"
	#define PCMK__ENV_SCHEMA_DIRECTORY "schema_directory"
	#define PCMK__ENV_SERVICE "service"
	#define PCMK__ENV_STDERR "stderr"
	#define PCMK__ENV_TLS_PRIORITIES "tls_priorities"
	#define PCMK__ENV_TRACE_BLACKBOX "trace_blackbox"
	#define PCMK__ENV_TRACE_FILES "trace_files"
	#define PCMK__ENV_TRACE_FORMATS "trace_formats"
	#define PCMK__ENV_TRACE_FUNCTIONS "trace_functions"
	#define PCMK__ENV_TRACE_TAGS "trace_tags"
	#define PCMK__ENV_VALGRIND_ENABLED "valgrind_enabled"

	// @COMPAT Drop at 3.0.0; default is plenty
	#define PCMK__ENV_CIB_TIMEOUT "cib_timeout"

	// @COMPAT Drop at 3.0.0; likely last used in 1.1.24
	#define PCMK__ENV_MCP "mcp"

	// @COMPAT Drop at 3.0.0; added unused in 1.1.9
	#define PCMK__ENV_QUORUM_TYPE "quorum_type"

	/* @COMPAT Drop at 3.0.0; added to debug shutdown issues when Pacemaker is
	* managed by systemd, but no longer useful.
	*/
	#define PCMK__ENV_SHUTDOWN_DELAY "shutdown_delay"

	// @COMPAT Deprecated since 2.1.0
	#define PCMK__OPT_REMOVE_AFTER_STOP "remove-after-stop"

	// Constants for meta-attribute names
	#define PCMK__META_CLONE_INSTANCE_NUM "clone"
	#define PCMK__META_CONTAINER "container"
	#define PCMK__META_INTERNAL_RSC "internal_rsc"
	#define PCMK__META_PHYSICAL_HOST "physical-host"

	/* @TODO Plug these in. Currently, they're never set. These are op attrs for use
	* with https://projects.clusterlabs.org/T382.
	*/
	#define PCMK__META_CLEAR_FAILURE_OP "clear_failure_op"
	#define PCMK__META_CLEAR_FAILURE_INTERVAL "clear_failure_interval"

	// @COMPAT Deprecated alias for PCMK__META_PROMOTED_MAX since 2.0.0
	#define PCMK__META_PROMOTED_MAX_LEGACY "master-max"

	// @COMPAT Deprecated alias for PCMK__META_PROMOTED_NODE_MAX since 2.0.0
	#define PCMK__META_PROMOTED_NODE_MAX_LEGACY "master-node-max"

	// @COMPAT Deprecated meta-attribute since 2.0.0
	#define PCMK__META_RESTART_TYPE "restart-type"

	// @COMPAT Deprecated meta-attribute since 2.0.0
	#define PCMK__META_ROLE_AFTER_FAILURE "role_after_failure"

	// Constants for enumerated values for various options
	#define PCMK__VALUE_CLUSTER "cluster"
	#define PCMK__VALUE_CUSTOM "custom"
	#define PCMK__VALUE_FENCING "fencing"
	#define PCMK__VALUE_GREEN "green"
	#define PCMK__VALUE_LOCAL "local"
	#define PCMK__VALUE_MIGRATE_ON_RED "migrate-on-red"
	#define PCMK__VALUE_NONE "none"
	#define PCMK__VALUE_NOTHING "nothing"
	#define PCMK__VALUE_ONLY_GREEN "only-green"
	#define PCMK__VALUE_PROGRESSIVE "progressive"
	#define PCMK__VALUE_QUORUM "quorum"
	#define PCMK__VALUE_RED "red"
	#define PCMK__VALUE_REQUEST "request"
	+#define PCMK__VALUE_RESPONSE "response"
	#define PCMK__VALUE_UNFENCING "unfencing"
	#define PCMK__VALUE_YELLOW "yellow"

	#endif // PCMK__OPTIONS_INTERNAL__H
	diff --git a/lib/common/ipc_controld.c b/lib/common/ipc_controld.c
	index d5266910a7..7f5a491b5a 100644
	--- a/lib/common/ipc_controld.c
	+++ b/lib/common/ipc_controld.c
	@@ -1,657 +1,657 @@
	/*
	* Copyright 2020-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdio.h>
	#include <stdbool.h>
	#include <errno.h>
	#include <libxml/tree.h>

	#include <crm/crm.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>
	#include <crm/common/ipc.h>
	#include <crm/common/ipc_internal.h>
	#include <crm/common/ipc_controld.h>
	#include "crmcommon_private.h"

	struct controld_api_private_s {
	char *client_uuid;
	unsigned int replies_expected;
	};

	/*!
	* \internal
	* \brief Get a string representation of a controller API reply type
	*
	* \param[in] reply Controller API reply type
	*
	* \return String representation of a controller API reply type
	*/
	const char *
	pcmk__controld_api_reply2str(enum pcmk_controld_api_reply reply)
	{
	switch (reply) {
	case pcmk_controld_reply_reprobe:
	return "reprobe";
	case pcmk_controld_reply_info:
	return "info";
	case pcmk_controld_reply_resource:
	return "resource";
	case pcmk_controld_reply_ping:
	return "ping";
	case pcmk_controld_reply_nodes:
	return "nodes";
	default:
	return "unknown";
	}
	}

	// \return Standard Pacemaker return code
	static int
	new_data(pcmk_ipc_api_t *api)
	{
	struct controld_api_private_s *private = NULL;

	api->api_data = calloc(1, sizeof(struct controld_api_private_s));

	if (api->api_data == NULL) {
	return errno;
	}

	private = api->api_data;

	/* This is set to the PID because that's how it was always done, but PIDs
	* are not unique because clients can be remote. The value appears to be
	* unused other than as part of F_CRM_SYS_FROM in IPC requests, which is
	* only compared against the internal system names (CRM_SYSTEM_TENGINE,
	* etc.), so it shouldn't be a problem.
	*/
	private->client_uuid = pcmk__getpid_s();

	/* @TODO Implement a call ID model similar to the CIB, executor, and fencer
	* IPC APIs, so that requests and replies can be matched, and
	* duplicate replies can be discarded.
	*/
	return pcmk_rc_ok;
	}

	static void
	free_data(void *data)
	{
	free(((struct controld_api_private_s *) data)->client_uuid);
	free(data);
	}

	// \return Standard Pacemaker return code
	static int
	post_connect(pcmk_ipc_api_t *api)
	{
	/* The controller currently requires clients to register via a hello
	* request, but does not reply back.
	*/
	struct controld_api_private_s *private = api->api_data;
	const char *client_name = crm_system_name? crm_system_name : "client";
	xmlNode *hello;
	int rc;

	hello = create_hello_message(private->client_uuid, client_name,
	PCMK__CONTROLD_API_MAJOR,
	PCMK__CONTROLD_API_MINOR);
	rc = pcmk__send_ipc_request(api, hello);
	free_xml(hello);
	if (rc != pcmk_rc_ok) {
	crm_info("Could not send IPC hello to %s: %s " CRM_XS " rc=%s",
	pcmk_ipc_name(api, true), pcmk_rc_str(rc), rc);
	} else {
	crm_debug("Sent IPC hello to %s", pcmk_ipc_name(api, true));
	}
	return rc;
	}

	static void
	set_node_info_data(pcmk_controld_api_reply_t data, xmlNode msg_data)
	{
	data->reply_type = pcmk_controld_reply_info;
	if (msg_data == NULL) {
	return;
	}
	data->data.node_info.have_quorum =
	pcmk__xe_attr_is_true(msg_data, PCMK_XA_HAVE_QUORUM);
	data->data.node_info.is_remote = pcmk__xe_attr_is_true(msg_data, XML_NODE_IS_REMOTE);

	/* Integer node_info.id is currently valid only for Corosync nodes.
	*
	* @TODO: Improve handling after crm_node_t is refactored to handle layer-
	* specific data better.
	*/
	crm_element_value_int(msg_data, PCMK_XA_ID, &(data->data.node_info.id));

	data->data.node_info.uuid = crm_element_value(msg_data, PCMK_XA_ID);
	data->data.node_info.uname = crm_element_value(msg_data, XML_ATTR_UNAME);
	data->data.node_info.state = crm_element_value(msg_data, PCMK__XA_CRMD);
	}

	static void
	set_ping_data(pcmk_controld_api_reply_t data, xmlNode msg_data)
	{
	data->reply_type = pcmk_controld_reply_ping;
	if (msg_data == NULL) {
	return;
	}
	data->data.ping.sys_from = crm_element_value(msg_data,
	XML_PING_ATTR_SYSFROM);
	data->data.ping.fsa_state = crm_element_value(msg_data,
	XML_PING_ATTR_CRMDSTATE);
	data->data.ping.result = crm_element_value(msg_data, XML_PING_ATTR_STATUS);
	}

	static void
	set_nodes_data(pcmk_controld_api_reply_t data, xmlNode msg_data)
	{
	pcmk_controld_api_node_t *node_info;

	data->reply_type = pcmk_controld_reply_nodes;
	for (xmlNode *node = first_named_child(msg_data, XML_CIB_TAG_NODE);
	node != NULL; node = crm_next_same_xml(node)) {

	long long id_ll = 0;

	node_info = calloc(1, sizeof(pcmk_controld_api_node_t));
	crm_element_value_ll(node, PCMK_XA_ID, &id_ll);
	if (id_ll > 0) {
	node_info->id = id_ll;
	}
	node_info->uname = crm_element_value(node, XML_ATTR_UNAME);
	node_info->state = crm_element_value(node, PCMK__XA_IN_CCM);
	data->data.nodes = g_list_prepend(data->data.nodes, node_info);
	}
	}

	static bool
	reply_expected(pcmk_ipc_api_t api, const xmlNode request)
	{
	// We only need to handle commands that API functions can send
	return pcmk__str_any_of(crm_element_value(request, F_CRM_TASK),
	PCMK__CONTROLD_CMD_NODES,
	CRM_OP_LRM_DELETE,
	CRM_OP_LRM_FAIL,
	CRM_OP_NODE_INFO,
	CRM_OP_PING,
	CRM_OP_REPROBE,
	CRM_OP_RM_NODE_CACHE,
	NULL);
	}

	static bool
	dispatch(pcmk_ipc_api_t api, xmlNode reply)
	{
	struct controld_api_private_s *private = api->api_data;
	crm_exit_t status = CRM_EX_OK;
	xmlNode *msg_data = NULL;
	const char *value = NULL;
	pcmk_controld_api_reply_t reply_data = {
	pcmk_controld_reply_unknown, NULL, NULL,
	};

	if (pcmk__xe_is(reply, "ack")) {
	/* ACKs are trivial responses that do not count toward expected replies,
	* and do not have all the fields that validation requires, so skip that
	* processing.
	*/
	return private->replies_expected > 0;
	}

	if (private->replies_expected > 0) {
	private->replies_expected--;
	}

	// Do some basic validation of the reply

	/* @TODO We should be able to verify that value is always a response, but
	* currently the controller doesn't always properly set the type. Even
	* if we fix the controller, we'll still need to handle replies from
	* old versions (feature set could be used to differentiate).
	*/
	value = crm_element_value(reply, F_CRM_MSG_TYPE);
	- if (!pcmk__str_any_of(value, PCMK__VALUE_REQUEST, XML_ATTR_RESPONSE,
	+ if (!pcmk__str_any_of(value, PCMK__VALUE_REQUEST, PCMK__VALUE_RESPONSE,
	NULL)) {
	crm_info("Unrecognizable message from controller: "
	"invalid message type '%s'", pcmk__s(value, ""));
	status = CRM_EX_PROTOCOL;
	goto done;
	}

	if (pcmk__str_empty(crm_element_value(reply, XML_ATTR_REFERENCE))) {
	crm_info("Unrecognizable message from controller: no reference");
	status = CRM_EX_PROTOCOL;
	goto done;
	}

	value = crm_element_value(reply, F_CRM_TASK);
	if (pcmk__str_empty(value)) {
	crm_info("Unrecognizable message from controller: no command name");
	status = CRM_EX_PROTOCOL;
	goto done;
	}

	// Parse useful info from reply

	reply_data.feature_set = crm_element_value(reply, PCMK_XA_VERSION);
	reply_data.host_from = crm_element_value(reply, F_CRM_HOST_FROM);
	msg_data = get_message_xml(reply, F_CRM_DATA);

	if (!strcmp(value, CRM_OP_REPROBE)) {
	reply_data.reply_type = pcmk_controld_reply_reprobe;

	} else if (!strcmp(value, CRM_OP_NODE_INFO)) {
	set_node_info_data(&reply_data, msg_data);

	} else if (!strcmp(value, CRM_OP_INVOKE_LRM)) {
	reply_data.reply_type = pcmk_controld_reply_resource;
	reply_data.data.resource.node_state = msg_data;

	} else if (!strcmp(value, CRM_OP_PING)) {
	set_ping_data(&reply_data, msg_data);

	} else if (!strcmp(value, PCMK__CONTROLD_CMD_NODES)) {
	set_nodes_data(&reply_data, msg_data);

	} else {
	crm_info("Unrecognizable message from controller: unknown command '%s'",
	value);
	status = CRM_EX_PROTOCOL;
	}

	done:
	pcmk__call_ipc_callback(api, pcmk_ipc_event_reply, status, &reply_data);

	// Free any reply data that was allocated
	if (pcmk__str_eq(value, PCMK__CONTROLD_CMD_NODES, pcmk__str_casei)) {
	g_list_free_full(reply_data.data.nodes, free);
	}

	return false; // No further replies needed
	}

	pcmk__ipc_methods_t *
	pcmk__controld_api_methods(void)
	{
	pcmk__ipc_methods_t *cmds = calloc(1, sizeof(pcmk__ipc_methods_t));

	if (cmds != NULL) {
	cmds->new_data = new_data;
	cmds->free_data = free_data;
	cmds->post_connect = post_connect;
	cmds->reply_expected = reply_expected;
	cmds->dispatch = dispatch;
	}
	return cmds;
	}

	/*!
	* \internal
	* \brief Create XML for a controller IPC request
	*
	* \param[in] api Controller connection
	* \param[in] op Controller IPC command name
	* \param[in] node Node name to set as destination host
	* \param[in] msg_data XML to attach to request as message data
	*
	* \return Newly allocated XML for request
	*/
	static xmlNode *
	create_controller_request(const pcmk_ipc_api_t api, const char op,
	const char node, xmlNode msg_data)
	{
	struct controld_api_private_s *private = NULL;
	const char *sys_to = NULL;

	if (api == NULL) {
	return NULL;
	}
	private = api->api_data;
	if ((node == NULL) && !strcmp(op, CRM_OP_PING)) {
	sys_to = CRM_SYSTEM_DC;
	} else {
	sys_to = CRM_SYSTEM_CRMD;
	}
	return create_request(op, msg_data, node, sys_to,
	(crm_system_name? crm_system_name : "client"),
	private->client_uuid);
	}

	// \return Standard Pacemaker return code
	static int
	send_controller_request(pcmk_ipc_api_t api, const xmlNode request,
	bool reply_is_expected)
	{
	if (crm_element_value(request, XML_ATTR_REFERENCE) == NULL) {
	return EINVAL;
	}
	if (reply_is_expected) {
	struct controld_api_private_s *private = api->api_data;

	private->replies_expected++;
	}
	return pcmk__send_ipc_request(api, request);
	}

	static xmlNode *
	create_reprobe_message_data(const char target_node, const char router_node)
	{
	xmlNode *msg_data;

	msg_data = create_xml_node(NULL, "data_for_" CRM_OP_REPROBE);
	crm_xml_add(msg_data, XML_LRM_ATTR_TARGET, target_node);
	if ((router_node != NULL) && !pcmk__str_eq(router_node, target_node, pcmk__str_casei)) {
	crm_xml_add(msg_data, XML_LRM_ATTR_ROUTER_NODE, router_node);
	}
	return msg_data;
	}

	/*!
	* \brief Send a reprobe controller operation
	*
	* \param[in,out] api Controller connection
	* \param[in] target_node Name of node to reprobe
	* \param[in] router_node Router node for host
	*
	* \return Standard Pacemaker return code
	* \note Event callback will get a reply of type pcmk_controld_reply_reprobe.
	*/
	int
	pcmk_controld_api_reprobe(pcmk_ipc_api_t api, const char target_node,
	const char *router_node)
	{
	xmlNode *request;
	xmlNode *msg_data;
	int rc = pcmk_rc_ok;

	if (api == NULL) {
	return EINVAL;
	}
	if (router_node == NULL) {
	router_node = target_node;
	}
	crm_debug("Sending %s IPC request to reprobe %s via %s",
	pcmk_ipc_name(api, true), pcmk__s(target_node, "local node"),
	pcmk__s(router_node, "local node"));
	msg_data = create_reprobe_message_data(target_node, router_node);
	request = create_controller_request(api, CRM_OP_REPROBE, router_node,
	msg_data);
	rc = send_controller_request(api, request, true);
	free_xml(msg_data);
	free_xml(request);
	return rc;
	}

	/*!
	* \brief Send a "node info" controller operation
	*
	* \param[in,out] api Controller connection
	* \param[in] nodeid ID of node to get info for (or 0 for local node)
	*
	* \return Standard Pacemaker return code
	* \note Event callback will get a reply of type pcmk_controld_reply_info.
	*/
	int
	pcmk_controld_api_node_info(pcmk_ipc_api_t *api, uint32_t nodeid)
	{
	xmlNode *request;
	int rc = pcmk_rc_ok;

	request = create_controller_request(api, CRM_OP_NODE_INFO, NULL, NULL);
	if (request == NULL) {
	return EINVAL;
	}
	if (nodeid > 0) {
	crm_xml_set_id(request, "%lu", (unsigned long) nodeid);
	}

	rc = send_controller_request(api, request, true);
	free_xml(request);
	return rc;
	}

	/*!
	* \brief Ask the controller for status
	*
	* \param[in,out] api Controller connection
	* \param[in] node_name Name of node whose status is desired (NULL for DC)
	*
	* \return Standard Pacemaker return code
	* \note Event callback will get a reply of type pcmk_controld_reply_ping.
	*/
	int
	pcmk_controld_api_ping(pcmk_ipc_api_t api, const char node_name)
	{
	xmlNode *request;
	int rc = pcmk_rc_ok;

	request = create_controller_request(api, CRM_OP_PING, node_name, NULL);
	if (request == NULL) {
	return EINVAL;
	}
	rc = send_controller_request(api, request, true);
	free_xml(request);
	return rc;
	}

	/*!
	* \brief Ask the controller for cluster information
	*
	* \param[in,out] api Controller connection
	*
	* \return Standard Pacemaker return code
	* \note Event callback will get a reply of type pcmk_controld_reply_nodes.
	*/
	int
	pcmk_controld_api_list_nodes(pcmk_ipc_api_t *api)
	{
	xmlNode *request;
	int rc = EINVAL;

	request = create_controller_request(api, PCMK__CONTROLD_CMD_NODES, NULL,
	NULL);
	if (request != NULL) {
	rc = send_controller_request(api, request, true);
	free_xml(request);
	}
	return rc;
	}

	// \return Standard Pacemaker return code
	static int
	controller_resource_op(pcmk_ipc_api_t api, const char op,
	const char target_node, const char router_node,
	bool cib_only, const char *rsc_id,
	const char rsc_long_id, const char standard,
	const char provider, const char type)
	{
	int rc = pcmk_rc_ok;
	char *key;
	xmlNode request, msg_data, xml_rsc, params;

	if (api == NULL) {
	return EINVAL;
	}
	if (router_node == NULL) {
	router_node = target_node;
	}

	msg_data = create_xml_node(NULL, XML_GRAPH_TAG_RSC_OP);

	/* The controller logs the transition key from resource op requests, so we
	* need to have something for it.
	* @TODO don't use "crm-resource"
	*/
	key = pcmk__transition_key(0, getpid(), 0,
	"xxxxxxxx-xrsc-opxx-xcrm-resourcexxxx");
	crm_xml_add(msg_data, XML_ATTR_TRANSITION_KEY, key);
	free(key);

	crm_xml_add(msg_data, XML_LRM_ATTR_TARGET, target_node);
	if (!pcmk__str_eq(router_node, target_node, pcmk__str_casei)) {
	crm_xml_add(msg_data, XML_LRM_ATTR_ROUTER_NODE, router_node);
	}

	if (cib_only) {
	// Indicate that only the CIB needs to be cleaned
	crm_xml_add(msg_data, PCMK__XA_MODE, XML_TAG_CIB);
	}

	xml_rsc = create_xml_node(msg_data, XML_CIB_TAG_RESOURCE);
	crm_xml_add(xml_rsc, PCMK_XA_ID, rsc_id);
	crm_xml_add(xml_rsc, PCMK__XA_LONG_ID, rsc_long_id);
	crm_xml_add(xml_rsc, PCMK_XA_CLASS, standard);
	crm_xml_add(xml_rsc, PCMK_XA_PROVIDER, provider);
	crm_xml_add(xml_rsc, PCMK_XA_TYPE, type);

	params = create_xml_node(msg_data, XML_TAG_ATTRS);
	crm_xml_add(params, PCMK_XA_CRM_FEATURE_SET, CRM_FEATURE_SET);

	// The controller parses the timeout from the request
	key = crm_meta_name(XML_ATTR_TIMEOUT);
	crm_xml_add(params, key, "60000"); /* 1 minute */ //@TODO pass as arg
	free(key);

	request = create_controller_request(api, op, router_node, msg_data);
	rc = send_controller_request(api, request, true);
	free_xml(msg_data);
	free_xml(request);
	return rc;
	}

	/*!
	* \brief Ask the controller to fail a resource
	*
	* \param[in,out] api Controller connection
	* \param[in] target_node Name of node resource is on
	* \param[in] router_node Router node for target
	* \param[in] rsc_id ID of resource to fail
	* \param[in] rsc_long_id Long ID of resource (if any)
	* \param[in] standard Standard of resource
	* \param[in] provider Provider of resource (if any)
	* \param[in] type Type of resource to fail
	*
	* \return Standard Pacemaker return code
	* \note Event callback will get a reply of type pcmk_controld_reply_resource.
	*/
	int
	pcmk_controld_api_fail(pcmk_ipc_api_t *api,
	const char target_node, const char router_node,
	const char rsc_id, const char rsc_long_id,
	const char standard, const char provider,
	const char *type)
	{
	crm_debug("Sending %s IPC request to fail %s (a.k.a. %s) on %s via %s",
	pcmk_ipc_name(api, true), pcmk__s(rsc_id, "unknown resource"),
	pcmk__s(rsc_long_id, "no other names"),
	pcmk__s(target_node, "unspecified node"),
	pcmk__s(router_node, "unspecified node"));
	return controller_resource_op(api, CRM_OP_LRM_FAIL, target_node,
	router_node, false, rsc_id, rsc_long_id,
	standard, provider, type);
	}

	/*!
	* \brief Ask the controller to refresh a resource
	*
	* \param[in,out] api Controller connection
	* \param[in] target_node Name of node resource is on
	* \param[in] router_node Router node for target
	* \param[in] rsc_id ID of resource to refresh
	* \param[in] rsc_long_id Long ID of resource (if any)
	* \param[in] standard Standard of resource
	* \param[in] provider Provider of resource (if any)
	* \param[in] type Type of resource
	* \param[in] cib_only If true, clean resource from CIB only
	*
	* \return Standard Pacemaker return code
	* \note Event callback will get a reply of type pcmk_controld_reply_resource.
	*/
	int
	pcmk_controld_api_refresh(pcmk_ipc_api_t api, const char target_node,
	const char *router_node,
	const char rsc_id, const char rsc_long_id,
	const char standard, const char provider,
	const char *type, bool cib_only)
	{
	crm_debug("Sending %s IPC request to refresh %s (a.k.a. %s) on %s via %s",
	pcmk_ipc_name(api, true), pcmk__s(rsc_id, "unknown resource"),
	pcmk__s(rsc_long_id, "no other names"),
	pcmk__s(target_node, "unspecified node"),
	pcmk__s(router_node, "unspecified node"));
	return controller_resource_op(api, CRM_OP_LRM_DELETE, target_node,
	router_node, cib_only, rsc_id, rsc_long_id,
	standard, provider, type);
	}

	/*!
	* \brief Get the number of IPC replies currently expected from the controller
	*
	* \param[in] api Controller IPC API connection
	*
	* \return Number of replies expected
	*/
	unsigned int
	pcmk_controld_api_replies_expected(const pcmk_ipc_api_t *api)
	{
	struct controld_api_private_s *private = api->api_data;

	return private->replies_expected;
	}

	/*!
	* \brief Create XML for a controller IPC "hello" message
	*
	* \deprecated This function is deprecated as part of the public C API.
	*/
	// \todo make this static to this file when breaking API backward compatibility
	xmlNode *
	create_hello_message(const char uuid, const char client_name,
	const char major_version, const char minor_version)
	{
	xmlNode *hello_node = NULL;
	xmlNode *hello = NULL;

	if (pcmk__str_empty(uuid) \|\| pcmk__str_empty(client_name)
	\|\| pcmk__str_empty(major_version) \|\| pcmk__str_empty(minor_version)) {
	crm_err("Could not create IPC hello message from %s (UUID %s): "
	"missing information",
	client_name? client_name : "unknown client",
	uuid? uuid : "unknown");
	return NULL;
	}

	hello_node = create_xml_node(NULL, XML_TAG_OPTIONS);
	if (hello_node == NULL) {
	crm_err("Could not create IPC hello message from %s (UUID %s): "
	"Message data creation failed", client_name, uuid);
	return NULL;
	}

	crm_xml_add(hello_node, "major_version", major_version);
	crm_xml_add(hello_node, "minor_version", minor_version);
	crm_xml_add(hello_node, "client_name", client_name);
	crm_xml_add(hello_node, "client_uuid", uuid);

	hello = create_request(CRM_OP_HELLO, hello_node, NULL, NULL, client_name, uuid);
	if (hello == NULL) {
	crm_err("Could not create IPC hello message from %s (UUID %s): "
	"Request creation failed", client_name, uuid);
	return NULL;
	}
	free_xml(hello_node);

	crm_trace("Created hello message from %s (UUID %s)", client_name, uuid);
	return hello;
	}
	diff --git a/lib/common/ipc_pacemakerd.c b/lib/common/ipc_pacemakerd.c
	index e13fa0a6e3..8994a6a152 100644
	--- a/lib/common/ipc_pacemakerd.c
	+++ b/lib/common/ipc_pacemakerd.c
	@@ -1,317 +1,316 @@
	/*
	* Copyright 2020-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdlib.h>
	#include <time.h>

	#include <crm/crm.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>
	#include <crm/common/ipc.h>
	#include <crm/common/ipc_internal.h>
	#include <crm/common/ipc_pacemakerd.h>
	#include "crmcommon_private.h"

	typedef struct pacemakerd_api_private_s {
	enum pcmk_pacemakerd_state state;
	char *client_uuid;
	} pacemakerd_api_private_t;

	static const char *pacemakerd_state_str[] = {
	XML_PING_ATTR_PACEMAKERDSTATE_INIT,
	XML_PING_ATTR_PACEMAKERDSTATE_STARTINGDAEMONS,
	XML_PING_ATTR_PACEMAKERDSTATE_WAITPING,
	XML_PING_ATTR_PACEMAKERDSTATE_RUNNING,
	XML_PING_ATTR_PACEMAKERDSTATE_SHUTTINGDOWN,
	XML_PING_ATTR_PACEMAKERDSTATE_SHUTDOWNCOMPLETE,
	XML_PING_ATTR_PACEMAKERDSTATE_REMOTE,
	};

	enum pcmk_pacemakerd_state
	pcmk_pacemakerd_api_daemon_state_text2enum(const char *state)
	{
	int i;

	if (state == NULL) {
	return pcmk_pacemakerd_state_invalid;
	}
	for (i=pcmk_pacemakerd_state_init; i <= pcmk_pacemakerd_state_max;
	i++) {
	if (pcmk__str_eq(state, pacemakerd_state_str[i], pcmk__str_none)) {
	return i;
	}
	}
	return pcmk_pacemakerd_state_invalid;
	}

	const char *
	pcmk_pacemakerd_api_daemon_state_enum2text(
	enum pcmk_pacemakerd_state state)
	{
	if ((state >= pcmk_pacemakerd_state_init) &&
	(state <= pcmk_pacemakerd_state_max)) {
	return pacemakerd_state_str[state];
	}
	return "invalid";
	}

	/*!
	* \internal
	* \brief Return a friendly string representation of a \p pacemakerd state
	*
	* \param[in] state \p pacemakerd state
	*
	* \return A user-friendly string representation of \p state, or
	* <tt>"Invalid pacemakerd state"</tt>
	*/
	const char *
	pcmk__pcmkd_state_enum2friendly(enum pcmk_pacemakerd_state state)
	{
	switch (state) {
	case pcmk_pacemakerd_state_init:
	return "Initializing pacemaker";
	case pcmk_pacemakerd_state_starting_daemons:
	return "Pacemaker daemons are starting";
	case pcmk_pacemakerd_state_wait_for_ping:
	return "Waiting for startup trigger from SBD";
	case pcmk_pacemakerd_state_running:
	return "Pacemaker is running";
	case pcmk_pacemakerd_state_shutting_down:
	return "Pacemaker daemons are shutting down";
	case pcmk_pacemakerd_state_shutdown_complete:
	/* Assuming pacemakerd won't process messages while in
	* shutdown_complete state unless reporting to SBD
	*/
	return "Pacemaker daemons are shut down (reporting to SBD)";
	case pcmk_pacemakerd_state_remote:
	return "pacemaker-remoted is running (on a Pacemaker Remote node)";
	default:
	return "Invalid pacemakerd state";
	}
	}

	/*!
	* \internal
	* \brief Get a string representation of a \p pacemakerd API reply type
	*
	* \param[in] reply \p pacemakerd API reply type
	*
	* \return String representation of a \p pacemakerd API reply type
	*/
	const char *
	pcmk__pcmkd_api_reply2str(enum pcmk_pacemakerd_api_reply reply)
	{
	switch (reply) {
	case pcmk_pacemakerd_reply_ping:
	return "ping";
	case pcmk_pacemakerd_reply_shutdown:
	return "shutdown";
	default:
	return "unknown";
	}
	}

	// \return Standard Pacemaker return code
	static int
	new_data(pcmk_ipc_api_t *api)
	{
	struct pacemakerd_api_private_s *private = NULL;

	api->api_data = calloc(1, sizeof(struct pacemakerd_api_private_s));

	if (api->api_data == NULL) {
	return errno;
	}

	private = api->api_data;
	private->state = pcmk_pacemakerd_state_invalid;
	/* other as with cib, controld, ... we are addressing pacemakerd just
	from the local node -> pid is unique and thus sufficient as an ID
	*/
	private->client_uuid = pcmk__getpid_s();

	return pcmk_rc_ok;
	}

	static void
	free_data(void *data)
	{
	free(((struct pacemakerd_api_private_s *) data)->client_uuid);
	free(data);
	}

	// \return Standard Pacemaker return code
	static int
	post_connect(pcmk_ipc_api_t *api)
	{
	struct pacemakerd_api_private_s *private = NULL;

	if (api->api_data == NULL) {
	return EINVAL;
	}
	private = api->api_data;
	private->state = pcmk_pacemakerd_state_invalid;

	return pcmk_rc_ok;
	}

	static void
	post_disconnect(pcmk_ipc_api_t *api)
	{
	struct pacemakerd_api_private_s *private = NULL;

	if (api->api_data == NULL) {
	return;
	}
	private = api->api_data;
	private->state = pcmk_pacemakerd_state_invalid;

	return;
	}

	static bool
	reply_expected(pcmk_ipc_api_t api, const xmlNode request)
	{
	const char *command = crm_element_value(request, F_CRM_TASK);

	if (command == NULL) {
	return false;
	}

	// We only need to handle commands that functions in this file can send
	return pcmk__str_any_of(command, CRM_OP_PING, CRM_OP_QUIT, NULL);
	}

	static bool
	dispatch(pcmk_ipc_api_t api, xmlNode reply)
	{
	crm_exit_t status = CRM_EX_OK;
	xmlNode *msg_data = NULL;
	pcmk_pacemakerd_api_reply_t reply_data = {
	pcmk_pacemakerd_reply_unknown
	};
	const char *value = NULL;
	long long value_ll = 0;

	if (pcmk__str_eq((const char *) reply->name, "ack", pcmk__str_none)) {
	long long int ack_status = 0;
	pcmk__scan_ll(crm_element_value(reply, "status"), &ack_status, CRM_EX_OK);
	return ack_status == CRM_EX_INDETERMINATE;
	}

	value = crm_element_value(reply, F_CRM_MSG_TYPE);
	- if (pcmk__str_empty(value)
	- \|\| !pcmk__str_eq(value, XML_ATTR_RESPONSE, pcmk__str_none)) {
	+ if (!pcmk__str_eq(value, PCMK__VALUE_RESPONSE, pcmk__str_none)) {
	crm_info("Unrecognizable message from %s: "
	- "message type '%s' not '" XML_ATTR_RESPONSE "'",
	+ "message type '%s' not '" PCMK__VALUE_RESPONSE "'",
	pcmk_ipc_name(api, true), pcmk__s(value, ""));
	status = CRM_EX_PROTOCOL;
	goto done;
	}

	if (pcmk__str_empty(crm_element_value(reply, XML_ATTR_REFERENCE))) {
	crm_info("Unrecognizable message from %s: no reference",
	pcmk_ipc_name(api, true));
	status = CRM_EX_PROTOCOL;
	goto done;
	}

	value = crm_element_value(reply, F_CRM_TASK);

	// Parse useful info from reply
	msg_data = get_message_xml(reply, F_CRM_DATA);
	crm_element_value_ll(msg_data, PCMK_XA_CRM_TIMESTAMP, &value_ll);

	if (pcmk__str_eq(value, CRM_OP_PING, pcmk__str_none)) {
	reply_data.reply_type = pcmk_pacemakerd_reply_ping;
	reply_data.data.ping.state =
	pcmk_pacemakerd_api_daemon_state_text2enum(
	crm_element_value(msg_data, XML_PING_ATTR_PACEMAKERDSTATE));
	reply_data.data.ping.status =
	pcmk__str_eq(crm_element_value(msg_data, XML_PING_ATTR_STATUS), "ok",
	pcmk__str_casei)?pcmk_rc_ok:pcmk_rc_error;
	reply_data.data.ping.last_good = (value_ll < 0)? 0 : (time_t) value_ll;
	reply_data.data.ping.sys_from = crm_element_value(msg_data,
	XML_PING_ATTR_SYSFROM);
	} else if (pcmk__str_eq(value, CRM_OP_QUIT, pcmk__str_none)) {
	reply_data.reply_type = pcmk_pacemakerd_reply_shutdown;
	reply_data.data.shutdown.status = atoi(crm_element_value(msg_data, XML_LRM_ATTR_OPSTATUS));
	} else {
	crm_info("Unrecognizable message from %s: unknown command '%s'",
	pcmk_ipc_name(api, true), pcmk__s(value, ""));
	status = CRM_EX_PROTOCOL;
	goto done;
	}

	done:
	pcmk__call_ipc_callback(api, pcmk_ipc_event_reply, status, &reply_data);
	return false;
	}

	pcmk__ipc_methods_t *
	pcmk__pacemakerd_api_methods(void)
	{
	pcmk__ipc_methods_t *cmds = calloc(1, sizeof(pcmk__ipc_methods_t));

	if (cmds != NULL) {
	cmds->new_data = new_data;
	cmds->free_data = free_data;
	cmds->post_connect = post_connect;
	cmds->reply_expected = reply_expected;
	cmds->dispatch = dispatch;
	cmds->post_disconnect = post_disconnect;
	}
	return cmds;
	}

	static int
	do_pacemakerd_api_call(pcmk_ipc_api_t api, const char ipc_name, const char *task)
	{
	pacemakerd_api_private_t *private;
	xmlNode *cmd;
	int rc;

	if (api == NULL) {
	return EINVAL;
	}

	private = api->api_data;
	CRM_ASSERT(private != NULL);

	cmd = create_request(task, NULL, NULL, CRM_SYSTEM_MCP,
	pcmk__ipc_sys_name(ipc_name, "client"),
	private->client_uuid);

	if (cmd) {
	rc = pcmk__send_ipc_request(api, cmd);
	if (rc != pcmk_rc_ok) {
	crm_debug("Couldn't send request to %s: %s rc=%d",
	pcmk_ipc_name(api, true), pcmk_rc_str(rc), rc);
	}
	free_xml(cmd);
	} else {
	rc = ENOMSG;
	}

	return rc;
	}

	int
	pcmk_pacemakerd_api_ping(pcmk_ipc_api_t api, const char ipc_name)
	{
	return do_pacemakerd_api_call(api, ipc_name, CRM_OP_PING);
	}

	int
	pcmk_pacemakerd_api_shutdown(pcmk_ipc_api_t api, const char ipc_name)
	{
	return do_pacemakerd_api_call(api, ipc_name, CRM_OP_QUIT);
	}
	diff --git a/lib/common/ipc_schedulerd.c b/lib/common/ipc_schedulerd.c
	index cf788e5c60..65033991c5 100644
	--- a/lib/common/ipc_schedulerd.c
	+++ b/lib/common/ipc_schedulerd.c
	@@ -1,180 +1,180 @@
	/*
	* Copyright 2021-2023 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdlib.h>
	#include <time.h>

	#include <crm/crm.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>
	#include <crm/common/ipc.h>
	#include <crm/common/ipc_internal.h>
	#include <crm/common/ipc_schedulerd.h>
	#include "crmcommon_private.h"

	typedef struct schedulerd_api_private_s {
	char *client_uuid;
	} schedulerd_api_private_t;

	// \return Standard Pacemaker return code
	static int
	new_data(pcmk_ipc_api_t *api)
	{
	struct schedulerd_api_private_s *private = NULL;

	api->api_data = calloc(1, sizeof(struct schedulerd_api_private_s));

	if (api->api_data == NULL) {
	return errno;
	}

	private = api->api_data;
	/* See comments in ipc_pacemakerd.c. */
	private->client_uuid = pcmk__getpid_s();

	return pcmk_rc_ok;
	}

	static void
	free_data(void *data)
	{
	free(((struct schedulerd_api_private_s *) data)->client_uuid);
	free(data);
	}

	// \return Standard Pacemaker return code
	static int
	post_connect(pcmk_ipc_api_t *api)
	{
	if (api->api_data == NULL) {
	return EINVAL;
	}

	return pcmk_rc_ok;
	}

	static bool
	reply_expected(pcmk_ipc_api_t api, const xmlNode request)
	{
	const char *command = crm_element_value(request, F_CRM_TASK);

	if (command == NULL) {
	return false;
	}

	// We only need to handle commands that functions in this file can send
	return pcmk__str_any_of(command, CRM_OP_PECALC, NULL);
	}

	static bool
	dispatch(pcmk_ipc_api_t api, xmlNode reply)
	{
	crm_exit_t status = CRM_EX_OK;
	xmlNode *msg_data = NULL;
	pcmk_schedulerd_api_reply_t reply_data = {
	pcmk_schedulerd_reply_unknown
	};
	const char *value = NULL;

	if (pcmk__str_eq((const char *) reply->name, "ack", pcmk__str_casei)) {
	return false;
	}

	value = crm_element_value(reply, F_CRM_MSG_TYPE);
	- if (!pcmk__str_eq(value, XML_ATTR_RESPONSE, pcmk__str_none)) {
	+ if (!pcmk__str_eq(value, PCMK__VALUE_RESPONSE, pcmk__str_none)) {
	crm_info("Unrecognizable message from schedulerd: "
	- "message type '%s' not '" XML_ATTR_RESPONSE "'",
	+ "message type '%s' not '" PCMK__VALUE_RESPONSE "'",
	pcmk__s(value, ""));
	status = CRM_EX_PROTOCOL;
	goto done;
	}

	if (pcmk__str_empty(crm_element_value(reply, XML_ATTR_REFERENCE))) {
	crm_info("Unrecognizable message from schedulerd: no reference");
	status = CRM_EX_PROTOCOL;
	goto done;
	}

	// Parse useful info from reply
	msg_data = get_message_xml(reply, F_CRM_DATA);
	value = crm_element_value(reply, F_CRM_TASK);

	if (pcmk__str_eq(value, CRM_OP_PECALC, pcmk__str_none)) {
	reply_data.reply_type = pcmk_schedulerd_reply_graph;
	reply_data.data.graph.reference = crm_element_value(reply, XML_ATTR_REFERENCE);
	reply_data.data.graph.input = crm_element_value(reply, F_CRM_TGRAPH_INPUT);
	reply_data.data.graph.tgraph = msg_data;
	} else {
	crm_info("Unrecognizable message from schedulerd: "
	"unknown command '%s'", pcmk__s(value, ""));
	status = CRM_EX_PROTOCOL;
	goto done;
	}

	done:
	pcmk__call_ipc_callback(api, pcmk_ipc_event_reply, status, &reply_data);
	return false;
	}

	pcmk__ipc_methods_t *
	pcmk__schedulerd_api_methods(void)
	{
	pcmk__ipc_methods_t *cmds = calloc(1, sizeof(pcmk__ipc_methods_t));

	if (cmds != NULL) {
	cmds->new_data = new_data;
	cmds->free_data = free_data;
	cmds->post_connect = post_connect;
	cmds->reply_expected = reply_expected;
	cmds->dispatch = dispatch;
	}
	return cmds;
	}

	static int
	do_schedulerd_api_call(pcmk_ipc_api_t api, const char task, xmlNode cib, char *ref)
	{
	schedulerd_api_private_t *private;
	xmlNode *cmd = NULL;
	int rc;

	if (!pcmk_ipc_is_connected(api)) {
	return ENOTCONN;
	}

	private = api->api_data;
	CRM_ASSERT(private != NULL);

	cmd = create_request(task, cib, NULL, CRM_SYSTEM_PENGINE,
	crm_system_name? crm_system_name : "client",
	private->client_uuid);

	if (cmd) {
	rc = pcmk__send_ipc_request(api, cmd);
	if (rc != pcmk_rc_ok) {
	crm_debug("Couldn't send request to schedulerd: %s rc=%d",
	pcmk_rc_str(rc), rc);
	}

	*ref = strdup(crm_element_value(cmd, F_CRM_REFERENCE));
	free_xml(cmd);
	} else {
	rc = ENOMSG;
	}

	return rc;
	}

	int
	pcmk_schedulerd_api_graph(pcmk_ipc_api_t api, xmlNode cib, char **ref)
	{
	return do_schedulerd_api_call(api, CRM_OP_PECALC, cib, ref);
	}
	diff --git a/lib/common/messages.c b/lib/common/messages.c
	index 36e59f86b0..f227753ccf 100644
	--- a/lib/common/messages.c
	+++ b/lib/common/messages.c
	@@ -1,293 +1,293 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdio.h>
	#include <sys/types.h>

	#include <glib.h>
	#include <libxml/tree.h>

	#include <crm/msg_xml.h>
	#include <crm/common/xml_internal.h>

	/*!
	* \brief Create a Pacemaker request (for IPC or cluster layer)
	*
	* \param[in] task What to set as the request's task
	* \param[in] msg_data What to add as the request's data contents
	* \param[in] host_to What to set as the request's destination host
	* \param[in] sys_to What to set as the request's destination system
	* \param[in] sys_from If not NULL, set as request's origin system
	* \param[in] uuid_from If not NULL, use in request's origin system
	* \param[in] origin Name of function that called this one
	*
	* \return XML of new request
	*
	* \note One of sys_from or uuid_from must be non-NULL
	* \note This function should not be called directly, but via the
	* create_request() wrapper.
	* \note The caller is responsible for freeing the result using free_xml().
	*/
	xmlNode *
	create_request_adv(const char task, xmlNode msg_data,
	const char host_to, const char sys_to,
	const char sys_from, const char uuid_from,
	const char *origin)
	{
	static uint ref_counter = 0;

	char *true_from = NULL;
	xmlNode *request = NULL;
	char *reference = crm_strdup_printf("%s-%s-%lld-%u",
	(task? task : "_empty_"),
	(sys_from? sys_from : "_empty_"),
	(long long) time(NULL), ref_counter++);

	if (uuid_from != NULL) {
	true_from = crm_strdup_printf("%s_%s", uuid_from,
	(sys_from? sys_from : "none"));
	} else if (sys_from != NULL) {
	true_from = strdup(sys_from);
	} else {
	crm_err("Cannot create IPC request: No originating system specified");
	}

	// host_from will get set for us if necessary by the controller when routed
	request = create_xml_node(NULL, __func__);
	crm_xml_add(request, F_CRM_ORIGIN, origin);
	crm_xml_add(request, F_TYPE, T_CRM);
	crm_xml_add(request, F_CRM_VERSION, CRM_FEATURE_SET);
	crm_xml_add(request, F_CRM_MSG_TYPE, PCMK__VALUE_REQUEST);
	crm_xml_add(request, F_CRM_REFERENCE, reference);
	crm_xml_add(request, F_CRM_TASK, task);
	crm_xml_add(request, F_CRM_SYS_TO, sys_to);
	crm_xml_add(request, F_CRM_SYS_FROM, true_from);

	/* HOSTTO will be ignored if it is to the DC anyway. */
	if (host_to != NULL && strlen(host_to) > 0) {
	crm_xml_add(request, F_CRM_HOST_TO, host_to);
	}

	if (msg_data != NULL) {
	add_message_xml(request, F_CRM_DATA, msg_data);
	}
	free(reference);
	free(true_from);

	return request;
	}

	/*!
	* \brief Create a Pacemaker reply (for IPC or cluster layer)
	*
	* \param[in] original_request XML of request this is a reply to
	* \param[in] xml_response_data XML to copy as data section of reply
	* \param[in] origin Name of function that called this one
	*
	* \return XML of new reply
	*
	* \note This function should not be called directly, but via the
	* create_reply() wrapper.
	* \note The caller is responsible for freeing the result using free_xml().
	*/
	xmlNode *
	create_reply_adv(const xmlNode original_request, xmlNode xml_response_data,
	const char *origin)
	{
	xmlNode *reply = NULL;

	const char *host_from = crm_element_value(original_request, F_CRM_HOST_FROM);
	const char *sys_from = crm_element_value(original_request, F_CRM_SYS_FROM);
	const char *sys_to = crm_element_value(original_request, F_CRM_SYS_TO);
	const char *type = crm_element_value(original_request, F_CRM_MSG_TYPE);
	const char *operation = crm_element_value(original_request, F_CRM_TASK);
	const char *crm_msg_reference = crm_element_value(original_request, F_CRM_REFERENCE);

	if (type == NULL) {
	crm_err("Cannot create new_message, no message type in original message");
	CRM_ASSERT(type != NULL);
	return NULL;
	}

	if (strcmp(type, PCMK__VALUE_REQUEST) != 0) {
	/* Replies should only be generated for request messages, but it's possible
	* we expect replies to other messages right now so this can't be enforced.
	*/
	crm_trace("Creating a reply for a non-request original message");
	}
	reply = create_xml_node(NULL, __func__);
	if (reply == NULL) {
	crm_err("Cannot create new_message, malloc failed");
	return NULL;
	}

	crm_xml_add(reply, F_CRM_ORIGIN, origin);
	crm_xml_add(reply, F_TYPE, T_CRM);
	crm_xml_add(reply, F_CRM_VERSION, CRM_FEATURE_SET);
	- crm_xml_add(reply, F_CRM_MSG_TYPE, XML_ATTR_RESPONSE);
	+ crm_xml_add(reply, F_CRM_MSG_TYPE, PCMK__VALUE_RESPONSE);
	crm_xml_add(reply, F_CRM_REFERENCE, crm_msg_reference);
	crm_xml_add(reply, F_CRM_TASK, operation);

	/* since this is a reply, we reverse the from and to */
	crm_xml_add(reply, F_CRM_SYS_TO, sys_from);
	crm_xml_add(reply, F_CRM_SYS_FROM, sys_to);

	/* HOSTTO will be ignored if it is to the DC anyway. */
	if (host_from != NULL && strlen(host_from) > 0) {
	crm_xml_add(reply, F_CRM_HOST_TO, host_from);
	}

	if (xml_response_data != NULL) {
	add_message_xml(reply, F_CRM_DATA, xml_response_data);
	}

	return reply;
	}

	xmlNode *
	get_message_xml(const xmlNode msg, const char field)
	{
	return pcmk__xml_first_child(first_named_child(msg, field));
	}

	gboolean
	add_message_xml(xmlNode msg, const char field, xmlNode *xml)
	{
	xmlNode *holder = create_xml_node(msg, field);

	add_node_copy(holder, xml);
	return TRUE;
	}

	/*!
	* \brief Get name to be used as identifier for cluster messages
	*
	* \param[in] name Actual system name to check
	*
	* \return Non-NULL cluster message identifier corresponding to name
	*
	* \note The Pacemaker daemons were renamed in version 2.0.0, but the old names
	* must continue to be used as the identifier for cluster messages, so
	* that mixed-version clusters are possible during a rolling upgrade.
	*/
	const char *
	pcmk__message_name(const char *name)
	{
	if (name == NULL) {
	return "unknown";

	} else if (!strcmp(name, "pacemaker-attrd")) {
	return "attrd";

	} else if (!strcmp(name, "pacemaker-based")) {
	return CRM_SYSTEM_CIB;

	} else if (!strcmp(name, "pacemaker-controld")) {
	return CRM_SYSTEM_CRMD;

	} else if (!strcmp(name, "pacemaker-execd")) {
	return CRM_SYSTEM_LRMD;

	} else if (!strcmp(name, "pacemaker-fenced")) {
	return "stonith-ng";

	} else if (!strcmp(name, "pacemaker-schedulerd")) {
	return CRM_SYSTEM_PENGINE;

	} else {
	return name;
	}
	}

	/*!
	* \internal
	* \brief Register handlers for server commands
	*
	* \param[in] handlers Array of handler functions for supported server commands
	* (the final entry must have a NULL command name, and if
	* it has a handler it will be used as the default handler
	* for unrecognized commands)
	*
	* \return Newly created hash table with commands and handlers
	* \note The caller is responsible for freeing the return value with
	* g_hash_table_destroy().
	*/
	GHashTable *
	pcmk__register_handlers(const pcmk__server_command_t handlers[])
	{
	GHashTable *commands = g_hash_table_new(g_str_hash, g_str_equal);

	if (handlers != NULL) {
	int i;

	for (i = 0; handlers[i].command != NULL; ++i) {
	g_hash_table_insert(commands, (gpointer) handlers[i].command,
	handlers[i].handler);
	}
	if (handlers[i].handler != NULL) {
	// g_str_hash() can't handle NULL, so use empty string for default
	g_hash_table_insert(commands, (gpointer) "", handlers[i].handler);
	}
	}
	return commands;
	}

	/*!
	* \internal
	* \brief Process an incoming request
	*
	* \param[in,out] request Request to process
	* \param[in] handlers Command table created by pcmk__register_handlers()
	*
	* \return XML to send as reply (or NULL if no reply is needed)
	*/
	xmlNode *
	pcmk__process_request(pcmk__request_t request, GHashTable handlers)
	{
	xmlNode (handler)(pcmk__request_t *request) = NULL;

	CRM_CHECK((request != NULL) && (request->op != NULL) && (handlers != NULL),
	return NULL);

	if (pcmk_is_set(request->flags, pcmk__request_sync)
	&& (request->ipc_client != NULL)) {
	CRM_CHECK(request->ipc_client->request_id == request->ipc_id,
	return NULL);
	}

	handler = g_hash_table_lookup(handlers, request->op);
	if (handler == NULL) {
	handler = g_hash_table_lookup(handlers, ""); // Default handler
	if (handler == NULL) {
	crm_info("Ignoring %s request from %s %s with no handler",
	request->op, pcmk__request_origin_type(request),
	pcmk__request_origin(request));
	return NULL;
	}
	}

	return (*handler)(request);
	}

	/*!
	* \internal
	* \brief Free memory used within a request (but not the request itself)
	*
	* \param[in,out] request Request to reset
	*/
	void
	pcmk__reset_request(pcmk__request_t *request)
	{
	free(request->op);
	request->op = NULL;

	pcmk__reset_result(&(request->result));
	}

File Metadata

Mime Type: text/x-diff
Expires: Mon, Apr 21, 6:01 PM (1 d, 7 h)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 1665017
Default Alt Text: (109 KB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions