No OneTemporary
Actions

Size

95 KB

Referenced Files

None

Subscribers

None

View Options

	diff --git a/daemons/attrd/attrd_alerts.c b/daemons/attrd/attrd_alerts.c
	index 641aa76cfe..32d808a22f 100644
	--- a/daemons/attrd/attrd_alerts.c
	+++ b/daemons/attrd/attrd_alerts.c
	@@ -1,171 +1,155 @@
	/*
	* Copyright 2015-2023 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>
	#include <crm/crm.h>
	#include <crm/cib/internal.h>
	#include <crm/msg_xml.h>
	#include <crm/cluster/internal.h>
	#include <crm/cluster/election_internal.h>
	#include <crm/common/alerts_internal.h>
	#include <crm/pengine/rules_internal.h>
	#include <crm/lrmd_internal.h>
	#include "pacemaker-attrd.h"

	static GList *attrd_alert_list = NULL;

	static void
	attrd_lrmd_callback(lrmd_event_data_t * op)
	{
	CRM_CHECK(op != NULL, return);
	switch (op->type) {
	case lrmd_event_disconnect:
	crm_info("Lost connection to executor");
	attrd_lrmd_disconnect();
	break;
	default:
	break;
	}
	}

	static lrmd_t *
	attrd_lrmd_connect(void)
	{
	if (the_lrmd == NULL) {
	the_lrmd = lrmd_api_new();
	the_lrmd->cmds->set_callback(the_lrmd, attrd_lrmd_callback);
	}

	if (!the_lrmd->cmds->is_connected(the_lrmd)) {
	const unsigned int max_attempts = 10;
	int ret = -ENOTCONN;

	for (int fails = 0; fails < max_attempts; ++fails) {
	ret = the_lrmd->cmds->connect(the_lrmd, T_ATTRD, NULL);
	if (ret == pcmk_ok) {
	break;
	}

	crm_debug("Could not connect to executor, %d tries remaining",
	(max_attempts - fails));
	/* @TODO We don't want to block here with sleep, but we should wait
	* some time between connection attempts. We could possibly add a
	* timer with a callback, but then we'd likely need an alert queue.
	*/
	}

	if (ret != pcmk_ok) {
	attrd_lrmd_disconnect();
	}
	}

	return the_lrmd;
	}

	void
	attrd_lrmd_disconnect(void) {
	if (the_lrmd) {
	lrmd_t *conn = the_lrmd;

	the_lrmd = NULL; /* in case we're called recursively */
	lrmd_api_delete(conn); /* will disconnect if necessary */
	}
	}

	static void
	config_query_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
	{
	xmlNode *crmalerts = NULL;

	if (rc == -ENXIO) {
	crm_debug("Local CIB has no alerts section");
	return;
	} else if (rc != pcmk_ok) {
	crm_notice("Could not query local CIB: %s", pcmk_strerror(rc));
	return;
	}

	crmalerts = output;
	if ((crmalerts != NULL) && !pcmk__xe_is(crmalerts, XML_CIB_TAG_ALERTS)) {
	crmalerts = first_named_child(crmalerts, XML_CIB_TAG_ALERTS);
	}
	if (!crmalerts) {
	crm_notice("CIB query result has no " XML_CIB_TAG_ALERTS " section");
	return;
	}

	pe_free_alert_list(attrd_alert_list);
	attrd_alert_list = pe_unpack_alerts(crmalerts);
	}

	#define XPATH_ALERTS \
	"/" XML_TAG_CIB "/" XML_CIB_TAG_CONFIGURATION "/" XML_CIB_TAG_ALERTS

	gboolean
	attrd_read_options(gpointer user_data)
	{
	int call_id;

	CRM_CHECK(the_cib != NULL, return TRUE);

	call_id = the_cib->cmds->query(the_cib, XPATH_ALERTS, NULL,
	cib_xpath \| cib_scope_local);

	the_cib->cmds->register_callback_full(the_cib, call_id, 120, FALSE, NULL,
	"config_query_callback",
	config_query_callback, free);

	crm_trace("Querying the CIB... call %d", call_id);
	return TRUE;
	}

	void
	attrd_cib_updated_cb(const char event, xmlNode msg)
	{
	- int rc = pcmk_err_generic;
	const xmlNode *patchset = NULL;

	if (attrd_shutting_down(true)) {
	return;
	}

	- if (msg == NULL) {
	- crm_warn("CIB %s%snotification received with no message XML",
	- pcmk__s(event, ""), ((event != NULL)? " " : ""));
	- return;
	- }
	-
	- if ((crm_element_value_int(msg, F_CIB_RC, &rc) != 0) \|\| (rc != pcmk_ok)) {
	- crm_warn("Ignore failed CIB update: %s (%d)", pcmk_strerror(rc), rc);
	- crm_log_xml_debug(msg, "failed");
	- return;
	- }
	-
	- patchset = get_message_xml(msg, F_CIB_UPDATE_RESULT);
	- if (patchset == NULL) {
	- crm_warn("CIB %s notification received with no patchset XML",
	- pcmk__s(event, ""), ((event != NULL)? " " : ""));
	+ if (cib__get_notify_patchset(msg, &patchset) != pcmk_rc_ok) {
	return;
	}

	if (cib__element_in_patchset(patchset, XML_CIB_TAG_ALERTS)) {
	mainloop_set_trigger(attrd_config_read);
	}
	}

	int
	attrd_send_attribute_alert(const char *node, int nodeid,
	const char attr, const char value)
	{
	if (attrd_alert_list == NULL) {
	return pcmk_ok;
	}
	return lrmd_send_attribute_alert(attrd_lrmd_connect(), attrd_alert_list,
	node, nodeid, attr, value);
	}
	diff --git a/daemons/controld/controld_cib.c b/daemons/controld/controld_cib.c
	index 0ac309b5e7..969cbed860 100644
	--- a/daemons/controld/controld_cib.c
	+++ b/daemons/controld/controld_cib.c
	@@ -1,1188 +1,1172 @@
	/*
	* Copyright 2004-2023 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <unistd.h> /* sleep */

	#include <crm/common/alerts_internal.h>
	#include <crm/common/xml.h>
	#include <crm/crm.h>
	#include <crm/msg_xml.h>
	#include <crm/lrmd_internal.h>

	#include <pacemaker-controld.h>

	// Call ID of the most recent in-progress CIB resource update (or 0 if none)
	static int pending_rsc_update = 0;

	// Call IDs of requested CIB replacements that won't trigger a new election
	// (used as a set of gint values)
	static GHashTable *cib_replacements = NULL;

	/*!
	* \internal
	* \brief Store the call ID of a CIB replacement that the controller requested
	*
	* The \p do_cib_replaced() callback function will avoid triggering a new
	* election when we're notified of one of these expected replacements.
	*
	* \param[in] call_id CIB call ID (or 0 for a synchronous call)
	*
	* \note This function should be called after making any asynchronous CIB
	* request (or before making any synchronous CIB request) that may replace
	* part of the nodes or status section. This may include CIB sync calls.
	*/
	void
	controld_record_cib_replace_call(int call_id)
	{
	CRM_CHECK(call_id >= 0, return);

	if (cib_replacements == NULL) {
	cib_replacements = g_hash_table_new(NULL, NULL);
	}

	/* If the call ID is already present in the table, then it's old. We may not
	* be removing them properly, and we could improperly ignore replacement
	* notifications if cib_t:call_id wraps around.
	*/
	CRM_LOG_ASSERT(g_hash_table_add(cib_replacements,
	GINT_TO_POINTER((gint) call_id)));
	}

	/*!
	* \internal
	* \brief Remove the call ID of a CIB replacement from the replacements table
	*
	* \param[in] call_id CIB call ID (or 0 for a synchronous call)
	*
	* \return \p true if \p call_id was found in the table, or \p false otherwise
	*
	* \note CIB notifications run before CIB callbacks. If this function is called
	* from within a callback, \p do_cib_replaced() will have removed
	* \p call_id from the table first if relevant changes triggered a
	* notification.
	*/
	bool
	controld_forget_cib_replace_call(int call_id)
	{
	CRM_CHECK(call_id >= 0, return false);

	if (cib_replacements == NULL) {
	return false;
	}
	return g_hash_table_remove(cib_replacements,
	GINT_TO_POINTER((gint) call_id));
	}

	/*!
	* \internal
	* \brief Empty the hash table containing call IDs of CIB replacement requests
	*/
	void
	controld_forget_all_cib_replace_calls(void)
	{
	if (cib_replacements != NULL) {
	g_hash_table_remove_all(cib_replacements);
	}
	}

	/*!
	* \internal
	* \brief Free the hash table containing call IDs of CIB replacement requests
	*/
	void
	controld_destroy_cib_replacements_table(void)
	{
	if (cib_replacements != NULL) {
	g_hash_table_destroy(cib_replacements);
	cib_replacements = NULL;
	}
	}

	/*!
	* \internal
	* \brief Respond to a dropped CIB connection
	*
	* \param[in] user_data CIB connection that dropped
	*/
	static void
	handle_cib_disconnect(gpointer user_data)
	{
	CRM_LOG_ASSERT(user_data == controld_globals.cib_conn);

	controld_trigger_fsa();
	controld_globals.cib_conn->state = cib_disconnected;

	if (pcmk_is_set(controld_globals.fsa_input_register, R_CIB_CONNECTED)) {
	// @TODO This should trigger a reconnect, not a shutdown
	crm_crit("Lost connection to the CIB manager, shutting down");
	register_fsa_input(C_FSA_INTERNAL, I_ERROR, NULL);
	controld_clear_fsa_input_flags(R_CIB_CONNECTED);

	} else { // Expected
	crm_info("Disconnected from the CIB manager");
	}
	}

	static void
	do_cib_updated(const char event, xmlNode msg)
	{
	- int rc = pcmk_err_generic;
	const xmlNode *patchset = NULL;

	- if (msg == NULL) {
	- crm_warn("CIB %s%snotification received with no message XML",
	- pcmk__s(event, ""), ((event != NULL)? " " : ""));
	- return;
	- }
	-
	- if ((crm_element_value_int(msg, F_CIB_RC, &rc) != 0) \|\| (rc != pcmk_ok)) {
	- crm_warn("Ignore failed CIB update: %s (%d)", pcmk_strerror(rc), rc);
	- crm_log_xml_debug(msg, "failed");
	- return;
	- }
	-
	- patchset = get_message_xml(msg, F_CIB_UPDATE_RESULT);
	- if (patchset == NULL) {
	- crm_warn("CIB %s notification received with no patchset XML",
	- pcmk__s(event, ""), ((event != NULL)? " " : ""));
	+ if (cib__get_notify_patchset(msg, &patchset) != pcmk_rc_ok) {
	return;
	}

	if (cib__element_in_patchset(patchset, XML_CIB_TAG_ALERTS)
	\|\| cib__element_in_patchset(patchset, XML_CIB_TAG_CRMCONFIG)) {

	controld_trigger_config();
	}
	}

	static void
	do_cib_replaced(const char event, xmlNode msg)
	{
	int call_id = 0;
	const char *client_id = crm_element_value(msg, F_CIB_CLIENTID);
	uint32_t change_section = cib_change_section_nodes
	\|cib_change_section_status;
	long long value = 0;

	crm_debug("Updating the CIB after a replace: DC=%s", pcmk__btoa(AM_I_DC));
	if (!AM_I_DC) {
	return;
	}

	if ((crm_element_value_int(msg, F_CIB_CALLID, &call_id) == 0)
	&& pcmk__str_eq(client_id, controld_globals.cib_client_id,
	pcmk__str_none)
	&& controld_forget_cib_replace_call(call_id)) {
	// We requested this replace op. No need to restart the join.
	return;
	}

	if ((crm_element_value_ll(msg, F_CIB_CHANGE_SECTION, &value) < 0)
	\|\| (value < 0) \|\| (value > UINT32_MAX)) {

	crm_trace("Couldn't parse '%s' from message", F_CIB_CHANGE_SECTION);
	} else {
	change_section = (uint32_t) value;
	}

	if (pcmk_any_flags_set(change_section, cib_change_section_nodes
	\|cib_change_section_status)) {

	/* start the join process again so we get everyone's LRM status */
	populate_cib_nodes(node_update_quick\|node_update_all, __func__);

	register_fsa_input(C_FSA_INTERNAL, I_ELECTION, NULL);
	}
	}

	void
	controld_disconnect_cib_manager(void)
	{
	cib_t *cib_conn = controld_globals.cib_conn;

	CRM_ASSERT(cib_conn != NULL);

	crm_debug("Disconnecting from the CIB manager");

	controld_clear_fsa_input_flags(R_CIB_CONNECTED);

	cib_conn->cmds->del_notify_callback(cib_conn, T_CIB_REPLACE_NOTIFY,
	do_cib_replaced);
	cib_conn->cmds->del_notify_callback(cib_conn, T_CIB_DIFF_NOTIFY,
	do_cib_updated);
	cib_free_callbacks(cib_conn);

	if (cib_conn->state != cib_disconnected) {
	cib_conn->cmds->set_secondary(cib_conn,
	cib_scope_local\|cib_discard_reply);
	cib_conn->cmds->signoff(cib_conn);
	}
	}

	/* A_CIB_STOP, A_CIB_START, O_CIB_RESTART */
	void
	do_cib_control(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	static int cib_retries = 0;

	cib_t *cib_conn = controld_globals.cib_conn;

	void (*dnotify_fn) (gpointer user_data) = handle_cib_disconnect;
	void (replace_cb) (const char event, xmlNodePtr msg) = do_cib_replaced;
	void (update_cb) (const char event, xmlNodePtr msg) = do_cib_updated;

	int rc = pcmk_ok;

	CRM_ASSERT(cib_conn != NULL);

	if (pcmk_is_set(action, A_CIB_STOP)) {
	if ((cib_conn->state != cib_disconnected)
	&& (pending_rsc_update != 0)) {

	crm_info("Waiting for resource update %d to complete",
	pending_rsc_update);
	crmd_fsa_stall(FALSE);
	return;
	}
	controld_disconnect_cib_manager();
	}

	if (!pcmk_is_set(action, A_CIB_START)) {
	return;
	}

	if (cur_state == S_STOPPING) {
	crm_err("Ignoring request to connect to the CIB manager after "
	"shutdown");
	return;
	}

	rc = cib_conn->cmds->signon(cib_conn, CRM_SYSTEM_CRMD,
	cib_command_nonblocking);

	if (rc != pcmk_ok) {
	// A short wait that usually avoids stalling the FSA
	sleep(1);
	rc = cib_conn->cmds->signon(cib_conn, CRM_SYSTEM_CRMD,
	cib_command_nonblocking);
	}

	if (rc != pcmk_ok) {
	crm_info("Could not connect to the CIB manager: %s", pcmk_strerror(rc));

	} else if (cib_conn->cmds->set_connection_dnotify(cib_conn,
	dnotify_fn) != pcmk_ok) {
	crm_err("Could not set dnotify callback");

	} else if (cib_conn->cmds->add_notify_callback(cib_conn,
	T_CIB_REPLACE_NOTIFY,
	replace_cb) != pcmk_ok) {
	crm_err("Could not set CIB notification callback (replace)");

	} else if (cib_conn->cmds->add_notify_callback(cib_conn,
	T_CIB_DIFF_NOTIFY,
	update_cb) != pcmk_ok) {
	crm_err("Could not set CIB notification callback (update)");

	} else {
	controld_set_fsa_input_flags(R_CIB_CONNECTED);
	cib_retries = 0;
	cib_conn->cmds->client_id(cib_conn, &controld_globals.cib_client_id,
	NULL);
	}

	if (!pcmk_is_set(controld_globals.fsa_input_register, R_CIB_CONNECTED)) {
	cib_retries++;

	if (cib_retries < 30) {
	crm_warn("Couldn't complete CIB registration %d times... "
	"pause and retry", cib_retries);
	controld_start_wait_timer();
	crmd_fsa_stall(FALSE);

	} else {
	crm_err("Could not complete CIB registration %d times... "
	"hard error", cib_retries);
	register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
	}
	}
	}

	#define MIN_CIB_OP_TIMEOUT (30)

	/*!
	* \internal
	* \brief Get the timeout (in seconds) that should be used with CIB operations
	*
	* \return The maximum of 30 seconds, the value of the PCMK_cib_timeout
	* environment variable, or 10 seconds times one more than the number of
	* nodes in the cluster.
	*/
	unsigned int
	cib_op_timeout(void)
	{
	static int env_timeout = -1;
	unsigned int calculated_timeout = 0;

	if (env_timeout == -1) {
	const char *env = getenv("PCMK_cib_timeout");

	pcmk__scan_min_int(env, &env_timeout, MIN_CIB_OP_TIMEOUT);
	crm_trace("Minimum CIB op timeout: %ds (environment: %s)",
	env_timeout, (env? env : "none"));
	}

	calculated_timeout = 1 + crm_active_peers();
	if (crm_remote_peer_cache) {
	calculated_timeout += g_hash_table_size(crm_remote_peer_cache);
	}
	calculated_timeout *= 10;

	calculated_timeout = QB_MAX(calculated_timeout, env_timeout);
	crm_trace("Calculated timeout: %us", calculated_timeout);

	if (controld_globals.cib_conn) {
	controld_globals.cib_conn->call_timeout = calculated_timeout;
	}
	return calculated_timeout;
	}

	/*!
	* \internal
	* \brief Get CIB call options to use local scope if primary is unavailable
	*
	* \return CIB call options
	*/
	int
	crmd_cib_smart_opt(void)
	{
	int call_opt = cib_none;

	if ((controld_globals.fsa_state == S_ELECTION)
	\|\| (controld_globals.fsa_state == S_PENDING)) {
	crm_info("Sending update to local CIB in state: %s",
	fsa_state2string(controld_globals.fsa_state));
	cib__set_call_options(call_opt, "update", cib_scope_local);
	}
	return call_opt;
	}

	static void
	cib_delete_callback(xmlNode msg, int call_id, int rc, xmlNode output,
	void *user_data)
	{
	char *desc = user_data;

	if (rc == 0) {
	crm_debug("Deletion of %s (via CIB call %d) succeeded", desc, call_id);
	} else {
	crm_warn("Deletion of %s (via CIB call %d) failed: %s " CRM_XS " rc=%d",
	desc, call_id, pcmk_strerror(rc), rc);
	}
	}

	// Searches for various portions of node_state to delete

	// Match a particular node's node_state (takes node name 1x)
	#define XPATH_NODE_STATE "//" XML_CIB_TAG_STATE "[@" XML_ATTR_UNAME "='%s']"

	// Node's lrm section (name 1x)
	#define XPATH_NODE_LRM XPATH_NODE_STATE "/" XML_CIB_TAG_LRM

	/* Node's lrm_rsc_op entries and lrm_resource entries without unexpired lock
	* (name 2x, (seconds_since_epoch - XML_CONFIG_ATTR_SHUTDOWN_LOCK_LIMIT) 1x)
	*/
	#define XPATH_NODE_LRM_UNLOCKED XPATH_NODE_STATE "//" XML_LRM_TAG_RSC_OP \
	"\|" XPATH_NODE_STATE \
	"//" XML_LRM_TAG_RESOURCE \
	"[not(@" XML_CONFIG_ATTR_SHUTDOWN_LOCK ") " \
	"or " XML_CONFIG_ATTR_SHUTDOWN_LOCK "<%lld]"

	// Node's transient_attributes section (name 1x)
	#define XPATH_NODE_ATTRS XPATH_NODE_STATE "/" XML_TAG_TRANSIENT_NODEATTRS

	// Everything under node_state (name 1x)
	#define XPATH_NODE_ALL XPATH_NODE_STATE "/*"

	/* Unlocked history + transient attributes
	* (name 2x, (seconds_since_epoch - XML_CONFIG_ATTR_SHUTDOWN_LOCK_LIMIT) 1x,
	* name 1x)
	*/
	#define XPATH_NODE_ALL_UNLOCKED XPATH_NODE_LRM_UNLOCKED "\|" XPATH_NODE_ATTRS

	/*!
	* \internal
	* \brief Get the XPath and description of a node state section to be deleted
	*
	* \param[in] uname Desired node
	* \param[in] section Subsection of node_state to be deleted
	* \param[out] xpath Where to store XPath of \p section
	* \param[out] desc If not \c NULL, where to store description of \p section
	*/
	void
	controld_node_state_deletion_strings(const char *uname,
	enum controld_section_e section,
	char xpath, char desc)
	{
	const char *desc_pre = NULL;

	// Shutdown locks that started before this time are expired
	long long expire = (long long) time(NULL)
	- controld_globals.shutdown_lock_limit;

	switch (section) {
	case controld_section_lrm:
	*xpath = crm_strdup_printf(XPATH_NODE_LRM, uname);
	desc_pre = "resource history";
	break;
	case controld_section_lrm_unlocked:
	*xpath = crm_strdup_printf(XPATH_NODE_LRM_UNLOCKED,
	uname, uname, expire);
	desc_pre = "resource history (other than shutdown locks)";
	break;
	case controld_section_attrs:
	*xpath = crm_strdup_printf(XPATH_NODE_ATTRS, uname);
	desc_pre = "transient attributes";
	break;
	case controld_section_all:
	*xpath = crm_strdup_printf(XPATH_NODE_ALL, uname);
	desc_pre = "all state";
	break;
	case controld_section_all_unlocked:
	*xpath = crm_strdup_printf(XPATH_NODE_ALL_UNLOCKED,
	uname, uname, expire, uname);
	desc_pre = "all state (other than shutdown locks)";
	break;
	default:
	// We called this function incorrectly
	CRM_ASSERT(false);
	break;
	}

	if (desc != NULL) {
	*desc = crm_strdup_printf("%s for node %s", desc_pre, uname);
	}
	}

	/*!
	* \internal
	* \brief Delete subsection of a node's CIB node_state
	*
	* \param[in] uname Desired node
	* \param[in] section Subsection of node_state to delete
	* \param[in] options CIB call options to use
	*/
	void
	controld_delete_node_state(const char *uname, enum controld_section_e section,
	int options)
	{
	cib_t *cib = controld_globals.cib_conn;
	char *xpath = NULL;
	char *desc = NULL;
	int cib_rc = pcmk_ok;

	CRM_ASSERT((uname != NULL) && (cib != NULL));

	controld_node_state_deletion_strings(uname, section, &xpath, &desc);

	cib__set_call_options(options, "node state deletion",
	cib_xpath\|cib_multiple);
	cib_rc = cib->cmds->remove(cib, xpath, NULL, options);
	fsa_register_cib_callback(cib_rc, desc, cib_delete_callback);
	crm_info("Deleting %s (via CIB call %d) " CRM_XS " xpath=%s",
	desc, cib_rc, xpath);

	// CIB library handles freeing desc
	free(xpath);
	}

	// Takes node name and resource ID
	#define XPATH_RESOURCE_HISTORY "//" XML_CIB_TAG_STATE \
	"[@" XML_ATTR_UNAME "='%s']/" \
	XML_CIB_TAG_LRM "/" XML_LRM_TAG_RESOURCES \
	"/" XML_LRM_TAG_RESOURCE \
	"[@" XML_ATTR_ID "='%s']"
	// @TODO could add "and @XML_CONFIG_ATTR_SHUTDOWN_LOCK" to limit to locks

	/*!
	* \internal
	* \brief Clear resource history from CIB for a given resource and node
	*
	* \param[in] rsc_id ID of resource to be cleared
	* \param[in] node Node whose resource history should be cleared
	* \param[in] user_name ACL user name to use
	* \param[in] call_options CIB call options
	*
	* \return Standard Pacemaker return code
	*/
	int
	controld_delete_resource_history(const char rsc_id, const char node,
	const char *user_name, int call_options)
	{
	char *desc = NULL;
	char *xpath = NULL;
	int rc = pcmk_rc_ok;

	CRM_CHECK((rsc_id != NULL) && (node != NULL), return EINVAL);

	desc = crm_strdup_printf("resource history for %s on %s", rsc_id, node);
	if (controld_globals.cib_conn == NULL) {
	crm_err("Unable to clear %s: no CIB connection", desc);
	free(desc);
	return ENOTCONN;
	}

	// Ask CIB to delete the entry
	xpath = crm_strdup_printf(XPATH_RESOURCE_HISTORY, node, rsc_id);
	rc = cib_internal_op(controld_globals.cib_conn, PCMK__CIB_REQUEST_DELETE,
	NULL, xpath, NULL, NULL, call_options\|cib_xpath,
	user_name);

	if (rc < 0) {
	rc = pcmk_legacy2rc(rc);
	crm_err("Could not delete resource status of %s on %s%s%s: %s "
	CRM_XS " rc=%d", rsc_id, node,
	(user_name? " for user " : ""), (user_name? user_name : ""),
	pcmk_rc_str(rc), rc);
	free(desc);
	free(xpath);
	return rc;
	}

	if (pcmk_is_set(call_options, cib_sync_call)) {
	if (pcmk_is_set(call_options, cib_dryrun)) {
	crm_debug("Deletion of %s would succeed", desc);
	} else {
	crm_debug("Deletion of %s succeeded", desc);
	}
	free(desc);

	} else {
	crm_info("Clearing %s (via CIB call %d) " CRM_XS " xpath=%s",
	desc, rc, xpath);
	fsa_register_cib_callback(rc, desc, cib_delete_callback);
	// CIB library handles freeing desc
	}

	free(xpath);
	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief Build XML and string of parameters meeting some criteria, for digest
	*
	* \param[in] op Executor event with parameter table to use
	* \param[in] metadata Parsed meta-data for executed resource agent
	* \param[in] param_type Flag used for selection criteria
	* \param[out] result Will be set to newly created XML with selected
	* parameters as attributes
	*
	* \return Newly allocated space-separated string of parameter names
	* \note Selection criteria varies by param_type: for the restart digest, we
	* want parameters that are not marked reloadable (OCF 1.1) or that
	* are marked unique (pre-1.1), for both string and XML results; for the
	* secure digest, we want parameters that are marked private for the
	* string, but parameters that are not marked private for the XML.
	* \note It is the caller's responsibility to free the string return value with
	* \p g_string_free() and the XML result with \p free_xml().
	*/
	static GString *
	build_parameter_list(const lrmd_event_data_t *op,
	const struct ra_metadata_s *metadata,
	enum ra_param_flags_e param_type, xmlNode **result)
	{
	GString *list = NULL;

	*result = create_xml_node(NULL, XML_TAG_PARAMS);

	/* Consider all parameters only except private ones to be consistent with
	* what scheduler does with calculate_secure_digest().
	*/
	if (param_type == ra_param_private
	&& compare_version(controld_globals.dc_version, "3.16.0") >= 0) {
	g_hash_table_foreach(op->params, hash2field, *result);
	pcmk__filter_op_for_digest(*result);
	}

	for (GList *iter = metadata->ra_params; iter != NULL; iter = iter->next) {
	struct ra_param_s param = (struct ra_param_s ) iter->data;

	bool accept_for_list = false;
	bool accept_for_xml = false;

	switch (param_type) {
	case ra_param_reloadable:
	accept_for_list = !pcmk_is_set(param->rap_flags, param_type);
	accept_for_xml = accept_for_list;
	break;

	case ra_param_unique:
	accept_for_list = pcmk_is_set(param->rap_flags, param_type);
	accept_for_xml = accept_for_list;
	break;

	case ra_param_private:
	accept_for_list = pcmk_is_set(param->rap_flags, param_type);
	accept_for_xml = !accept_for_list;
	break;
	}

	if (accept_for_list) {
	crm_trace("Attr %s is %s", param->rap_name, ra_param_flag2text(param_type));

	if (list == NULL) {
	// We will later search for " WORD ", so start list with a space
	pcmk__add_word(&list, 256, " ");
	}
	pcmk__add_word(&list, 0, param->rap_name);

	} else {
	crm_trace("Rejecting %s for %s", param->rap_name, ra_param_flag2text(param_type));
	}

	if (accept_for_xml) {
	const char *v = g_hash_table_lookup(op->params, param->rap_name);

	if (v != NULL) {
	crm_trace("Adding attr %s=%s to the xml result", param->rap_name, v);
	crm_xml_add(*result, param->rap_name, v);
	}

	} else {
	crm_trace("Removing attr %s from the xml result", param->rap_name);
	xml_remove_prop(*result, param->rap_name);
	}
	}

	if (list != NULL) {
	// We will later search for " WORD ", so end list with a space
	pcmk__add_word(&list, 0, " ");
	}
	return list;
	}

	static void
	append_restart_list(lrmd_event_data_t op, struct ra_metadata_s metadata,
	xmlNode update, const char version)
	{
	GString *list = NULL;
	char *digest = NULL;
	xmlNode *restart = NULL;

	CRM_LOG_ASSERT(op->params != NULL);

	if (op->interval_ms > 0) {
	/* monitors are not reloadable */
	return;
	}

	if (pcmk_is_set(metadata->ra_flags, ra_supports_reload_agent)) {
	// Add parameters not marked reloadable to the "op-force-restart" list
	list = build_parameter_list(op, metadata, ra_param_reloadable,
	&restart);

	} else if (pcmk_is_set(metadata->ra_flags, ra_supports_legacy_reload)) {
	/* @COMPAT pre-OCF-1.1 resource agents
	*
	* Before OCF 1.1, Pacemaker abused "unique=0" to indicate
	* reloadability. Add any parameters with unique="1" to the
	* "op-force-restart" list.
	*/
	list = build_parameter_list(op, metadata, ra_param_unique, &restart);

	} else {
	// Resource does not support agent reloads
	return;
	}

	digest = calculate_operation_digest(restart, version);
	/* Add "op-force-restart" and "op-restart-digest" to indicate the resource supports reload,
	* no matter if it actually supports any parameters with unique="1"). */
	crm_xml_add(update, XML_LRM_ATTR_OP_RESTART,
	(list == NULL)? "" : (const char *) list->str);
	crm_xml_add(update, XML_LRM_ATTR_RESTART_DIGEST, digest);

	if ((list != NULL) && (list->len > 0)) {
	crm_trace("%s: %s, %s", op->rsc_id, digest, (const char *) list->str);
	} else {
	crm_trace("%s: %s", op->rsc_id, digest);
	}

	if (list != NULL) {
	g_string_free(list, TRUE);
	}
	free_xml(restart);
	free(digest);
	}

	static void
	append_secure_list(lrmd_event_data_t op, struct ra_metadata_s metadata,
	xmlNode update, const char version)
	{
	GString *list = NULL;
	char *digest = NULL;
	xmlNode *secure = NULL;

	CRM_LOG_ASSERT(op->params != NULL);

	/*
	* To keep XML_LRM_ATTR_OP_SECURE short, we want it to contain the
	* secure parameters but XML_LRM_ATTR_SECURE_DIGEST to be based on
	* the insecure ones
	*/
	list = build_parameter_list(op, metadata, ra_param_private, &secure);

	if (list != NULL) {
	digest = calculate_operation_digest(secure, version);
	crm_xml_add(update, XML_LRM_ATTR_OP_SECURE, (const char *) list->str);
	crm_xml_add(update, XML_LRM_ATTR_SECURE_DIGEST, digest);

	crm_trace("%s: %s, %s", op->rsc_id, digest, (const char *) list->str);
	g_string_free(list, TRUE);
	} else {
	crm_trace("%s: no secure parameters", op->rsc_id);
	}

	free_xml(secure);
	free(digest);
	}

	/*!
	* \internal
	* \brief Create XML for a resource history entry
	*
	* \param[in] func Function name of caller
	* \param[in,out] parent XML to add entry to
	* \param[in] rsc Affected resource
	* \param[in,out] op Action to add an entry for (or NULL to do nothing)
	* \param[in] node_name Node where action occurred
	*/
	void
	controld_add_resource_history_xml_as(const char func, xmlNode parent,
	const lrmd_rsc_info_t *rsc,
	lrmd_event_data_t *op,
	const char *node_name)
	{
	int target_rc = 0;
	xmlNode *xml_op = NULL;
	struct ra_metadata_s *metadata = NULL;
	const char *caller_version = NULL;
	lrm_state_t *lrm_state = NULL;

	if (op == NULL) {
	return;
	}

	target_rc = rsc_op_expected_rc(op);

	caller_version = g_hash_table_lookup(op->params, XML_ATTR_CRM_VERSION);
	CRM_CHECK(caller_version != NULL, caller_version = CRM_FEATURE_SET);

	xml_op = pcmk__create_history_xml(parent, op, caller_version, target_rc,
	controld_globals.our_nodename, func);
	if (xml_op == NULL) {
	return;
	}

	if ((rsc == NULL) \|\| (op->params == NULL)
	\|\| !crm_op_needs_metadata(rsc->standard, op->op_type)) {

	crm_trace("No digests needed for %s action on %s (params=%p rsc=%p)",
	op->op_type, op->rsc_id, op->params, rsc);
	return;
	}

	lrm_state = lrm_state_find(node_name);
	if (lrm_state == NULL) {
	crm_warn("Cannot calculate digests for operation " PCMK__OP_FMT
	" because we have no connection to executor for %s",
	op->rsc_id, op->op_type, op->interval_ms, node_name);
	return;
	}

	/* Ideally the metadata is cached, and the agent is just a fallback.
	*
	* @TODO Go through all callers and ensure they get metadata asynchronously
	* first.
	*/
	metadata = controld_get_rsc_metadata(lrm_state, rsc,
	controld_metadata_from_agent
	\|controld_metadata_from_cache);
	if (metadata == NULL) {
	return;
	}

	crm_trace("Including additional digests for %s:%s:%s",
	rsc->standard, rsc->provider, rsc->type);
	append_restart_list(op, metadata, xml_op, caller_version);
	append_secure_list(op, metadata, xml_op, caller_version);

	return;
	}

	/*!
	* \internal
	* \brief Record an action as pending in the CIB, if appropriate
	*
	* \param[in] node_name Node where the action is pending
	* \param[in] rsc Resource that action is for
	* \param[in,out] op Pending action
	*
	* \return true if action was recorded in CIB, otherwise false
	*/
	bool
	controld_record_pending_op(const char node_name, const lrmd_rsc_info_t rsc,
	lrmd_event_data_t *op)
	{
	const char *record_pending = NULL;

	CRM_CHECK((node_name != NULL) && (rsc != NULL) && (op != NULL),
	return false);

	// Never record certain operation types as pending
	if ((op->op_type == NULL) \|\| (op->params == NULL)
	\|\| !controld_action_is_recordable(op->op_type)) {
	return false;
	}

	// Check action's record-pending meta-attribute (defaults to true)
	record_pending = crm_meta_value(op->params, XML_OP_ATTR_PENDING);
	if ((record_pending != NULL) && !crm_is_true(record_pending)) {
	return false;
	}

	op->call_id = -1;
	op->t_run = time(NULL);
	op->t_rcchange = op->t_run;

	lrmd__set_result(op, PCMK_OCF_UNKNOWN, PCMK_EXEC_PENDING, NULL);

	crm_debug("Recording pending %s-interval %s for %s on %s in the CIB",
	pcmk__readable_interval(op->interval_ms), op->op_type, op->rsc_id,
	node_name);
	controld_update_resource_history(node_name, rsc, op, 0);
	return true;
	}

	static void
	cib_rsc_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
	{
	switch (rc) {
	case pcmk_ok:
	case -pcmk_err_diff_failed:
	case -pcmk_err_diff_resync:
	crm_trace("Resource history update completed (call=%d rc=%d)",
	call_id, rc);
	break;
	default:
	if (call_id > 0) {
	crm_warn("Resource history update %d failed: %s "
	CRM_XS " rc=%d", call_id, pcmk_strerror(rc), rc);
	} else {
	crm_warn("Resource history update failed: %s " CRM_XS " rc=%d",
	pcmk_strerror(rc), rc);
	}
	}

	if (call_id == pending_rsc_update) {
	pending_rsc_update = 0;
	controld_trigger_fsa();
	}
	}

	/* Only successful stops, and probes that found the resource inactive, get locks
	* recorded in the history. This ensures the resource stays locked to the node
	* until it is active there again after the node comes back up.
	*/
	static bool
	should_preserve_lock(lrmd_event_data_t *op)
	{
	if (!pcmk_is_set(controld_globals.flags, controld_shutdown_lock_enabled)) {
	return false;
	}
	if (!strcmp(op->op_type, PCMK_ACTION_STOP) && (op->rc == PCMK_OCF_OK)) {
	return true;
	}
	if (!strcmp(op->op_type, PCMK_ACTION_MONITOR)
	&& (op->rc == PCMK_OCF_NOT_RUNNING)) {
	return true;
	}
	return false;
	}

	/*!
	* \internal
	* \brief Request a CIB update
	*
	* \param[in] section Section of CIB to update
	* \param[in] data New XML of CIB section to update
	* \param[in] options CIB call options
	* \param[in] callback If not \c NULL, set this as the operation callback
	*
	* \return Standard Pacemaker return code
	*
	* \note If \p callback is \p cib_rsc_callback(), the CIB update's call ID is
	* stored in \p pending_rsc_update on success.
	*/
	int
	controld_update_cib(const char section, xmlNode data, int options,
	void (callback)(xmlNode , int, int, xmlNode , void ))
	{
	int cib_rc = -ENOTCONN;

	CRM_ASSERT(data != NULL);

	if (controld_globals.cib_conn != NULL) {
	cib_rc = cib_internal_op(controld_globals.cib_conn,
	PCMK__CIB_REQUEST_MODIFY, NULL, section,
	data, NULL, options, NULL);
	if (cib_rc >= 0) {
	crm_debug("Submitted CIB update %d for %s section",
	cib_rc, section);
	}
	}

	if (callback == NULL) {
	if (cib_rc < 0) {
	crm_err("Failed to update CIB %s section: %s",
	section, pcmk_rc_str(pcmk_legacy2rc(cib_rc)));
	}

	} else {
	if ((cib_rc >= 0) && (callback == cib_rsc_callback)) {
	/* Checking for a particular callback is a little hacky, but it
	* didn't seem worth adding an output argument for cib_rc for just
	* one use case.
	*/
	pending_rsc_update = cib_rc;
	}
	fsa_register_cib_callback(cib_rc, NULL, callback);
	}

	return (cib_rc >= 0)? pcmk_rc_ok : pcmk_legacy2rc(cib_rc);
	}

	/*!
	* \internal
	* \brief Update resource history entry in CIB
	*
	* \param[in] node_name Node where action occurred
	* \param[in] rsc Resource that action is for
	* \param[in,out] op Action to record
	* \param[in] lock_time If nonzero, when resource was locked to node
	*
	* \note On success, the CIB update's call ID will be stored in
	* pending_rsc_update.
	*/
	void
	controld_update_resource_history(const char *node_name,
	const lrmd_rsc_info_t *rsc,
	lrmd_event_data_t *op, time_t lock_time)
	{
	xmlNode *update = NULL;
	xmlNode *xml = NULL;
	int call_opt = crmd_cib_smart_opt();
	const char *node_id = NULL;
	const char *container = NULL;

	CRM_CHECK((node_name != NULL) && (op != NULL), return);

	if (rsc == NULL) {
	crm_warn("Resource %s no longer exists in the executor", op->rsc_id);
	controld_ack_event_directly(NULL, NULL, rsc, op, op->rsc_id);
	return;
	}

	// <status>
	update = create_xml_node(NULL, XML_CIB_TAG_STATUS);

	// <node_state ...>
	xml = create_xml_node(update, XML_CIB_TAG_STATE);
	if (pcmk__str_eq(node_name, controld_globals.our_nodename,
	pcmk__str_casei)) {
	node_id = controld_globals.our_uuid;
	} else {
	node_id = node_name;
	pcmk__xe_set_bool_attr(xml, XML_NODE_IS_REMOTE, true);
	}
	crm_xml_add(xml, XML_ATTR_ID, node_id);
	crm_xml_add(xml, XML_ATTR_UNAME, node_name);
	crm_xml_add(xml, XML_ATTR_ORIGIN, __func__);

	// <lrm ...>
	xml = create_xml_node(xml, XML_CIB_TAG_LRM);
	crm_xml_add(xml, XML_ATTR_ID, node_id);

	// <lrm_resources>
	xml = create_xml_node(xml, XML_LRM_TAG_RESOURCES);

	// <lrm_resource ...>
	xml = create_xml_node(xml, XML_LRM_TAG_RESOURCE);
	crm_xml_add(xml, XML_ATTR_ID, op->rsc_id);
	crm_xml_add(xml, XML_AGENT_ATTR_CLASS, rsc->standard);
	crm_xml_add(xml, XML_AGENT_ATTR_PROVIDER, rsc->provider);
	crm_xml_add(xml, XML_ATTR_TYPE, rsc->type);
	if (lock_time != 0) {
	/* Actions on a locked resource should either preserve the lock by
	* recording it with the action result, or clear it.
	*/
	if (!should_preserve_lock(op)) {
	lock_time = 0;
	}
	crm_xml_add_ll(xml, XML_CONFIG_ATTR_SHUTDOWN_LOCK,
	(long long) lock_time);
	}
	if (op->params != NULL) {
	container = g_hash_table_lookup(op->params,
	CRM_META "_" XML_RSC_ATTR_CONTAINER);
	if (container != NULL) {
	crm_trace("Resource %s is a part of container resource %s",
	op->rsc_id, container);
	crm_xml_add(xml, XML_RSC_ATTR_CONTAINER, container);
	}
	}

	// <lrm_resource_op ...> (possibly more than one)
	controld_add_resource_history_xml(xml, rsc, op, node_name);

	/* Update CIB asynchronously. Even if it fails, the resource state should be
	* discovered during the next election. Worst case, the node is wrongly
	* fenced for running a resource it isn't.
	*/
	crm_log_xml_trace(update, __func__);
	controld_update_cib(XML_CIB_TAG_STATUS, update, call_opt, cib_rsc_callback);
	free_xml(update);
	}

	/*!
	* \internal
	* \brief Erase an LRM history entry from the CIB, given the operation data
	*
	* \param[in] op Operation whose history should be deleted
	*/
	void
	controld_delete_action_history(const lrmd_event_data_t *op)
	{
	xmlNode *xml_top = NULL;

	CRM_CHECK(op != NULL, return);

	xml_top = create_xml_node(NULL, XML_LRM_TAG_RSC_OP);
	crm_xml_add_int(xml_top, XML_LRM_ATTR_CALLID, op->call_id);
	crm_xml_add(xml_top, XML_ATTR_TRANSITION_KEY, op->user_data);

	if (op->interval_ms > 0) {
	char *op_id = pcmk__op_key(op->rsc_id, op->op_type, op->interval_ms);

	/* Avoid deleting last_failure too (if it was a result of this recurring op failing) */
	crm_xml_add(xml_top, XML_ATTR_ID, op_id);
	free(op_id);
	}

	crm_debug("Erasing resource operation history for " PCMK__OP_FMT " (call=%d)",
	op->rsc_id, op->op_type, op->interval_ms, op->call_id);

	controld_globals.cib_conn->cmds->remove(controld_globals.cib_conn,
	XML_CIB_TAG_STATUS, xml_top,
	cib_none);

	crm_log_xml_trace(xml_top, "op:cancel");
	free_xml(xml_top);
	}

	/* Define xpath to find LRM resource history entry by node and resource */
	#define XPATH_HISTORY \
	"/" XML_TAG_CIB "/" XML_CIB_TAG_STATUS \
	"/" XML_CIB_TAG_STATE "[@" XML_ATTR_UNAME "='%s']" \
	"/" XML_CIB_TAG_LRM "/" XML_LRM_TAG_RESOURCES \
	"/" XML_LRM_TAG_RESOURCE "[@" XML_ATTR_ID "='%s']" \
	"/" XML_LRM_TAG_RSC_OP

	/* ... and also by operation key */
	#define XPATH_HISTORY_ID XPATH_HISTORY \
	"[@" XML_ATTR_ID "='%s']"

	/* ... and also by operation key and operation call ID */
	#define XPATH_HISTORY_CALL XPATH_HISTORY \
	"[@" XML_ATTR_ID "='%s' and @" XML_LRM_ATTR_CALLID "='%d']"

	/* ... and also by operation key and original operation key */
	#define XPATH_HISTORY_ORIG XPATH_HISTORY \
	"[@" XML_ATTR_ID "='%s' and @" XML_LRM_ATTR_TASK_KEY "='%s']"

	/*!
	* \internal
	* \brief Delete a last_failure resource history entry from the CIB
	*
	* \param[in] rsc_id Name of resource to clear history for
	* \param[in] node Name of node to clear history for
	* \param[in] action If specified, delete only if this was failed action
	* \param[in] interval_ms If \p action is specified, it has this interval
	*/
	void
	controld_cib_delete_last_failure(const char rsc_id, const char node,
	const char *action, guint interval_ms)
	{
	char *xpath = NULL;
	char *last_failure_key = NULL;

	CRM_CHECK((rsc_id != NULL) && (node != NULL), return);

	// Generate XPath to match desired entry
	last_failure_key = pcmk__op_key(rsc_id, "last_failure", 0);
	if (action == NULL) {
	xpath = crm_strdup_printf(XPATH_HISTORY_ID, node, rsc_id,
	last_failure_key);
	} else {
	char *action_key = pcmk__op_key(rsc_id, action, interval_ms);

	xpath = crm_strdup_printf(XPATH_HISTORY_ORIG, node, rsc_id,
	last_failure_key, action_key);
	free(action_key);
	}
	free(last_failure_key);

	controld_globals.cib_conn->cmds->remove(controld_globals.cib_conn, xpath,
	NULL, cib_xpath);
	free(xpath);
	}

	/*!
	* \internal
	* \brief Delete resource history entry from the CIB, given operation key
	*
	* \param[in] rsc_id Name of resource to clear history for
	* \param[in] node Name of node to clear history for
	* \param[in] key Operation key of operation to clear history for
	* \param[in] call_id If specified, delete entry only if it has this call ID
	*/
	void
	controld_delete_action_history_by_key(const char rsc_id, const char node,
	const char *key, int call_id)
	{
	char *xpath = NULL;

	CRM_CHECK((rsc_id != NULL) && (node != NULL) && (key != NULL), return);

	if (call_id > 0) {
	xpath = crm_strdup_printf(XPATH_HISTORY_CALL, node, rsc_id, key,
	call_id);
	} else {
	xpath = crm_strdup_printf(XPATH_HISTORY_ID, node, rsc_id, key);
	}
	controld_globals.cib_conn->cmds->remove(controld_globals.cib_conn, xpath,
	NULL, cib_xpath);
	free(xpath);
	}
	diff --git a/include/crm/cib/internal.h b/include/crm/cib/internal.h
	index 50fb6db5b7..fccabfd6ad 100644
	--- a/include/crm/cib/internal.h
	+++ b/include/crm/cib/internal.h
	@@ -1,334 +1,336 @@
	/*
	* Copyright 2004-2023 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#ifndef CIB_INTERNAL__H
	# define CIB_INTERNAL__H
	# include <crm/cib.h>
	# include <crm/common/ipc_internal.h>
	# include <crm/common/output_internal.h>

	// Request types for CIB manager IPC/CPG
	#define PCMK__CIB_REQUEST_SECONDARY "cib_slave"
	#define PCMK__CIB_REQUEST_PRIMARY "cib_master"
	#define PCMK__CIB_REQUEST_SYNC_TO_ALL "cib_sync"
	#define PCMK__CIB_REQUEST_SYNC_TO_ONE "cib_sync_one"
	#define PCMK__CIB_REQUEST_IS_PRIMARY "cib_ismaster"
	#define PCMK__CIB_REQUEST_BUMP "cib_bump"
	#define PCMK__CIB_REQUEST_QUERY "cib_query"
	#define PCMK__CIB_REQUEST_CREATE "cib_create"
	#define PCMK__CIB_REQUEST_MODIFY "cib_modify"
	#define PCMK__CIB_REQUEST_DELETE "cib_delete"
	#define PCMK__CIB_REQUEST_ERASE "cib_erase"
	#define PCMK__CIB_REQUEST_REPLACE "cib_replace"
	#define PCMK__CIB_REQUEST_APPLY_PATCH "cib_apply_diff"
	#define PCMK__CIB_REQUEST_UPGRADE "cib_upgrade"
	#define PCMK__CIB_REQUEST_ABS_DELETE "cib_delete_alt"
	#define PCMK__CIB_REQUEST_NOOP "noop"
	#define PCMK__CIB_REQUEST_SHUTDOWN "cib_shutdown_req"
	#define PCMK__CIB_REQUEST_COMMIT_TRANSACT "cib_commit_transact"

	# define F_CIB_CLIENTID "cib_clientid"
	# define F_CIB_CALLOPTS "cib_callopt"
	# define F_CIB_CALLID "cib_callid"
	# define F_CIB_CALLDATA "cib_calldata"
	# define F_CIB_OPERATION "cib_op"
	# define F_CIB_ISREPLY "cib_isreplyto"
	# define F_CIB_SECTION "cib_section"
	# define F_CIB_HOST "cib_host"
	# define F_CIB_RC "cib_rc"
	# define F_CIB_UPGRADE_RC "cib_upgrade_rc"
	# define F_CIB_DELEGATED "cib_delegated_from"
	# define F_CIB_OBJID "cib_object"
	# define F_CIB_OBJTYPE "cib_object_type"
	# define F_CIB_EXISTING "cib_existing_object"
	# define F_CIB_SEENCOUNT "cib_seen"
	# define F_CIB_TIMEOUT "cib_timeout"
	# define F_CIB_UPDATE "cib_update"
	# define F_CIB_GLOBAL_UPDATE "cib_update"
	# define F_CIB_UPDATE_RESULT "cib_update_result"
	# define F_CIB_CLIENTNAME "cib_clientname"
	# define F_CIB_NOTIFY_TYPE "cib_notify_type"
	# define F_CIB_NOTIFY_ACTIVATE "cib_notify_activate"
	# define F_CIB_UPDATE_DIFF "cib_update_diff"
	# define F_CIB_USER "cib_user"
	# define F_CIB_LOCAL_NOTIFY_ID "cib_local_notify_id"
	# define F_CIB_PING_ID "cib_ping_id"
	# define F_CIB_SCHEMA_MAX "cib_schema_max"
	# define F_CIB_CHANGE_SECTION "cib_change_section"

	# define T_CIB "cib"
	# define T_CIB_COMMAND "cib_command"
	# define T_CIB_NOTIFY "cib_notify"
	/* notify sub-types */
	# define T_CIB_PRE_NOTIFY "cib_pre_notify"
	# define T_CIB_POST_NOTIFY "cib_post_notify"
	# define T_CIB_TRANSACTION "cib_transaction"
	# define T_CIB_UPDATE_CONFIRM "cib_update_confirmation"
	# define T_CIB_REPLACE_NOTIFY "cib_refresh_notify"

	/*!
	* \internal
	* \enum cib_change_section_info
	* \brief Flags to indicate which sections of the CIB have changed
	*/
	enum cib_change_section_info {
	cib_change_section_none = 0, //!< No sections have changed
	cib_change_section_nodes = (1 << 0), //!< The nodes section has changed
	cib_change_section_alerts = (1 << 1), //!< The alerts section has changed
	cib_change_section_status = (1 << 2), //!< The status section has changed
	};

	/*!
	* \internal
	* \enum cib__op_attr
	* \brief Flags for CIB operation attributes
	*/
	enum cib__op_attr {
	cib__op_attr_none = 0, //!< No special attributes
	cib__op_attr_modifies = (1 << 1), //!< Modifies CIB
	cib__op_attr_privileged = (1 << 2), //!< Requires privileges
	cib__op_attr_local = (1 << 3), //!< Must only be processed locally
	cib__op_attr_replaces = (1 << 4), //!< Replaces CIB
	cib__op_attr_writes_through = (1 << 5), //!< Writes to disk on success
	cib__op_attr_transaction = (1 << 6), //!< Supported in a transaction
	};

	/*!
	* \internal
	* \enum cib__op_type
	* \brief Types of CIB operations
	*/
	enum cib__op_type {
	cib__op_abs_delete,
	cib__op_apply_patch,
	cib__op_bump,
	cib__op_commit_transact,
	cib__op_create,
	cib__op_delete,
	cib__op_erase,
	cib__op_is_primary,
	cib__op_modify,
	cib__op_noop,
	cib__op_ping,
	cib__op_primary,
	cib__op_query,
	cib__op_replace,
	cib__op_secondary,
	cib__op_shutdown,
	cib__op_sync_all,
	cib__op_sync_one,
	cib__op_upgrade,
	};

	/*!
	* \internal
	* \brief Set given <tt>enum cib_change_section_info</tt> flags
	*
	* \param[in,out] flags_orig Group of flags to update
	* \param[in] flags_to_set Flags to clear from \p flags_orig
	*/
	#define pcmk__set_change_section(flags_orig, flags_to_set) do { \
	flags_orig = pcmk__set_flags_as(__func__, __LINE__, LOG_TRACE, \
	"CIB change section", \
	"change_section", flags_orig, \
	flags_to_set, #flags_to_set); \
	} while (0)

	gboolean cib_diff_version_details(xmlNode * diff, int admin_epoch, int epoch, int *updates,
	int _admin_epoch, int _epoch, int *_updates);

	gboolean cib_read_config(GHashTable * options, xmlNode * current_cib);

	typedef int (cib__op_fn_t)(const char , int, const char , xmlNode ,
	xmlNode , xmlNode , xmlNode , xmlNode );

	typedef struct cib__operation_s {
	const char *name;
	enum cib__op_type type;
	uint32_t flags; //!< Group of <tt>enum cib__op_attr</tt> flags
	} cib__operation_t;

	typedef struct cib_notify_client_s {
	const char *event;
	const char obj_id; / implement one day */
	const char obj_type; / implement one day */
	void (callback) (const char event, xmlNode * msg);

	} cib_notify_client_t;

	typedef struct cib_callback_client_s {
	void (callback) (xmlNode , int, int, xmlNode , void );
	const char *id;
	void *user_data;
	gboolean only_success;
	struct timer_rec_s *timer;
	void (free_func)(void );
	} cib_callback_client_t;

	struct timer_rec_s {
	int call_id;
	int timeout;
	guint ref;
	cib_t *cib;
	};

	#define cib__set_call_options(cib_call_opts, call_for, flags_to_set) do { \
	cib_call_opts = pcmk__set_flags_as(__func__, __LINE__, \
	LOG_TRACE, "CIB call", (call_for), (cib_call_opts), \
	(flags_to_set), #flags_to_set); \
	} while (0)

	#define cib__clear_call_options(cib_call_opts, call_for, flags_to_clear) do { \
	cib_call_opts = pcmk__clear_flags_as(__func__, __LINE__, \
	LOG_TRACE, "CIB call", (call_for), (cib_call_opts), \
	(flags_to_clear), #flags_to_clear); \
	} while (0)

	cib_t *cib_new_variant(void);

	+int cib__get_notify_patchset(const xmlNode msg, const xmlNode *patchset);
	+
	bool cib__element_in_patchset(const xmlNode patchset, const char element);

	int cib_perform_op(const char *op, int call_options, cib__op_fn_t fn,
	bool is_query, const char section, xmlNode req,
	xmlNode input, bool manage_counters, bool config_changed,
	xmlNode current_cib, xmlNode result_cib, xmlNode **diff,
	xmlNode **output);

	int cib__create_op(cib_t cib, const char op, const char *host,
	const char section, xmlNode data, int call_options,
	const char user_name, const char client_name,
	xmlNode **op_msg);

	int cib__extend_transaction(cib_t cib, xmlNode request);

	void cib_native_callback(cib_t * cib, xmlNode * msg, int call_id, int rc);
	void cib_native_notify(gpointer data, gpointer user_data);

	int cib__get_operation(const char op, const cib__operation_t *operation);

	int cib_process_query(const char op, int options, const char section, xmlNode * req,
	xmlNode * input, xmlNode * existing_cib, xmlNode ** result_cib,
	xmlNode ** answer);

	int cib_process_erase(const char op, int options, const char section, xmlNode * req,
	xmlNode * input, xmlNode * existing_cib, xmlNode ** result_cib,
	xmlNode ** answer);

	int cib_process_bump(const char op, int options, const char section, xmlNode * req,
	xmlNode * input, xmlNode * existing_cib, xmlNode ** result_cib,
	xmlNode ** answer);

	int cib_process_replace(const char op, int options, const char section, xmlNode * req,
	xmlNode * input, xmlNode * existing_cib, xmlNode ** result_cib,
	xmlNode ** answer);

	int cib_process_create(const char op, int options, const char section, xmlNode * req,
	xmlNode * input, xmlNode * existing_cib, xmlNode ** result_cib,
	xmlNode ** answer);

	int cib_process_modify(const char op, int options, const char section, xmlNode * req,
	xmlNode * input, xmlNode * existing_cib, xmlNode ** result_cib,
	xmlNode ** answer);

	int cib_process_delete(const char op, int options, const char section, xmlNode * req,
	xmlNode * input, xmlNode * existing_cib, xmlNode ** result_cib,
	xmlNode ** answer);

	int cib_process_diff(const char op, int options, const char section, xmlNode * req,
	xmlNode * input, xmlNode * existing_cib, xmlNode ** result_cib,
	xmlNode ** answer);

	int cib_process_upgrade(const char op, int options, const char section, xmlNode * req,
	xmlNode * input, xmlNode * existing_cib, xmlNode ** result_cib,
	xmlNode ** answer);

	/*!
	* \internal
	* \brief Query or modify a CIB
	*
	* \param[in] op PCMK__CIB_REQUEST_* operation to be performed
	* \param[in] options Flag set of \c cib_call_options
	* \param[in] section XPath to query or modify
	* \param[in] req unused
	* \param[in] input Portion of CIB to modify (used with
	* PCMK__CIB_REQUEST_CREATE,
	* PCMK__CIB_REQUEST_MODIFY, and
	* PCMK__CIB_REQUEST_REPLACE)
	* \param[in,out] existing_cib Input CIB (used with PCMK__CIB_REQUEST_QUERY)
	* \param[in,out] result_cib CIB copy to make changes in (used with
	* PCMK__CIB_REQUEST_CREATE,
	* PCMK__CIB_REQUEST_MODIFY,
	* PCMK__CIB_REQUEST_DELETE, and
	* PCMK__CIB_REQUEST_REPLACE)
	* \param[out] answer Query result (used with PCMK__CIB_REQUEST_QUERY)
	*
	* \return Legacy Pacemaker return code
	*/
	int cib_process_xpath(const char op, int options, const char section,
	const xmlNode req, xmlNode input, xmlNode *existing_cib,
	xmlNode result_cib, xmlNode answer);

	bool cib__config_changed_v1(xmlNode last, xmlNode next, xmlNode **diff);

	int cib_internal_op(cib_t * cib, const char op, const char host,
	const char section, xmlNode data,
	xmlNode ** output_data, int call_options, const char *user_name);


	int cib_file_read_and_verify(const char filename, const char sigfile,
	xmlNode **root);
	int cib_file_write_with_digest(xmlNode cib_root, const char cib_dirname,
	const char *cib_filename);

	void cib__set_output(cib_t cib, pcmk__output_t out);

	cib_callback_client_t* cib__lookup_id (int call_id);

	/*!
	* \internal
	* \brief Connect to, query, and optionally disconnect from the CIB
	*
	* Open a read-write connection to the CIB manager if an already connected
	* client is not passed in. Then query the CIB and store the resulting XML.
	* Finally, disconnect if the CIB connection isn't being returned to the caller.
	*
	* \param[in,out] out Output object (may be \p NULL)
	* \param[in,out] cib If not \p NULL, where to store CIB connection
	* \param[out] cib_object Where to store query result
	*
	* \return Standard Pacemaker return code
	*
	* \note If \p cib is not \p NULL, the caller is responsible for freeing \p *cib
	* using \p cib_delete().
	* \note If \p *cib points to an existing \p cib_t object, this function will
	* reuse it instead of creating a new one. If the existing client is
	* already connected, the connection will be reused, even if it's
	* read-only.
	*/
	int cib__signon_query(pcmk__output_t out, cib_t cib, xmlNode *cib_object);

	int cib__clean_up_connection(cib_t **cib);

	int cib__update_node_attr(pcmk__output_t out, cib_t cib, int call_options,
	const char section, const char node_uuid, const char *set_type,
	const char set_name, const char attr_id, const char *attr_name,
	const char attr_value, const char user_name,
	const char *node_type);

	int cib__get_node_attrs(pcmk__output_t out, cib_t cib, const char *section,
	const char node_uuid, const char set_type, const char *set_name,
	const char attr_id, const char attr_name, const char *user_name,
	xmlNode **result);

	int cib__delete_node_attr(pcmk__output_t out, cib_t cib, int options,
	const char section, const char node_uuid, const char *set_type,
	const char set_name, const char attr_id, const char *attr_name,
	const char attr_value, const char user_name);

	#endif
	diff --git a/lib/cib/cib_utils.c b/lib/cib/cib_utils.c
	index 166e9754eb..6ea06e7127 100644
	--- a/lib/cib/cib_utils.c
	+++ b/lib/cib/cib_utils.c
	@@ -1,1114 +1,1152 @@
	/*
	* Original copyright 2004 International Business Machines
	* Later changes copyright 2008-2023 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/
	#include <crm_internal.h>
	#include <unistd.h>
	#include <stdlib.h>
	#include <stdio.h>
	#include <stdarg.h>
	#include <string.h>
	#include <sys/utsname.h>

	#include <glib.h>

	#include <crm/crm.h>
	#include <crm/cib/internal.h>
	#include <crm/msg_xml.h>
	#include <crm/common/cib_internal.h>
	#include <crm/common/xml.h>
	#include <crm/common/xml_internal.h>
	#include <crm/pengine/rules.h>

	xmlNode *
	cib_get_generation(cib_t * cib)
	{
	xmlNode *the_cib = NULL;
	xmlNode *generation = create_xml_node(NULL, XML_CIB_TAG_GENERATION_TUPPLE);

	cib->cmds->query(cib, NULL, &the_cib, cib_scope_local \| cib_sync_call);
	if (the_cib != NULL) {
	copy_in_properties(generation, the_cib);
	free_xml(the_cib);
	}

	return generation;
	}

	gboolean
	cib_version_details(xmlNode * cib, int admin_epoch, int epoch, int *updates)
	{
	*epoch = -1;
	*updates = -1;
	*admin_epoch = -1;

	if (cib == NULL) {
	return FALSE;

	} else {
	crm_element_value_int(cib, XML_ATTR_GENERATION, epoch);
	crm_element_value_int(cib, XML_ATTR_NUMUPDATES, updates);
	crm_element_value_int(cib, XML_ATTR_GENERATION_ADMIN, admin_epoch);
	}
	return TRUE;
	}

	gboolean
	cib_diff_version_details(xmlNode * diff, int admin_epoch, int epoch, int *updates,
	int _admin_epoch, int _epoch, int *_updates)
	{
	int add[] = { 0, 0, 0 };
	int del[] = { 0, 0, 0 };

	xml_patch_versions(diff, add, del);

	*admin_epoch = add[0];
	*epoch = add[1];
	*updates = add[2];

	*_admin_epoch = del[0];
	*_epoch = del[1];
	*_updates = del[2];

	return TRUE;
	}

	+/*!
	+ * \internal
	+ * \brief Get the XML patchset from a CIB diff notification
	+ *
	+ * \param[in] msg CIB diff notification
	+ * \param[out] patchset Where to store XML patchset
	+ *
	+ * \return Standard Pacemaker return code
	+ */
	+int
	+cib__get_notify_patchset(const xmlNode msg, const xmlNode *patchset)
	+{
	+ int rc = pcmk_err_generic;
	+
	+ CRM_ASSERT(patchset != NULL);
	+ *patchset = NULL;
	+
	+ if (msg == NULL) {
	+ crm_err("CIB diff notification received with no XML");
	+ return ENOMSG;
	+ }
	+
	+ if ((crm_element_value_int(msg, F_CIB_RC, &rc) != 0) \|\| (rc != pcmk_ok)) {
	+ crm_warn("Ignore failed CIB update: %s " CRM_XS " rc=%d",
	+ pcmk_strerror(rc), rc);
	+ crm_log_xml_debug(msg, "failed");
	+ return pcmk_legacy2rc(rc);
	+ }
	+
	+ *patchset = get_message_xml(msg, F_CIB_UPDATE_RESULT);
	+
	+ if (*patchset == NULL) {
	+ crm_err("CIB diff notification received with no patchset");
	+ return ENOMSG;
	+ }
	+ return pcmk_rc_ok;
	+}
	+
	#define XPATH_DIFF_V1 "//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_ADDED

	/*!
	* \internal
	* \brief Check whether a given CIB element was modified in a CIB patchset (v1)
	*
	* \param[in] patchset CIB XML patchset
	* \param[in] element XML tag of CIB element to check (\c NULL is equivalent
	* to \c XML_TAG_CIB)
	*
	* \return \c true if \p element was modified, or \c false otherwise
	*/
	static bool
	element_in_patchset_v1(const xmlNode patchset, const char element)
	{
	char *xpath = crm_strdup_printf(XPATH_DIFF_V1 "//%s",
	pcmk__s(element, XML_TAG_CIB));
	xmlXPathObject *xpath_obj = xpath_search(patchset, xpath);

	free(xpath);

	if (xpath_obj == NULL) {
	return false;
	}
	freeXpathObject(xpath_obj);
	return true;
	}

	/*!
	* \internal
	* \brief Check whether a given CIB element was modified in a CIB patchset (v2)
	*
	* \param[in] patchset CIB XML patchset
	* \param[in] element XML tag of CIB element to check (\c NULL is equivalent
	* to \c XML_TAG_CIB). Supported values include any CIB
	* element supported by \c pcmk__cib_abs_xpath_for().
	*
	* \return \c true if \p element was modified, or \c false otherwise
	*/
	static bool
	element_in_patchset_v2(const xmlNode patchset, const char element)
	{
	const char *element_xpath = pcmk__cib_abs_xpath_for(element);
	const char *parent_xpath = pcmk_cib_parent_name_for(element);
	char *element_regex = NULL;
	bool rc = false;

	CRM_CHECK(element_xpath != NULL, return false); // Unsupported element

	// Matches if and only if element_xpath is part of a changed path
	element_regex = crm_strdup_printf("^%s(/\|$)", element_xpath);

	for (const xmlNode *change = first_named_child(patchset, XML_DIFF_CHANGE);
	change != NULL; change = crm_next_same_xml(change)) {

	const char *op = crm_element_value(change, F_CIB_OPERATION);
	const char *diff_xpath = crm_element_value(change, XML_DIFF_PATH);

	if (pcmk__str_eq(diff_xpath, element_regex, pcmk__str_regex)) {
	// Change to an existing element
	rc = true;
	break;
	}

	if (pcmk__str_eq(op, "create", pcmk__str_none)
	&& pcmk__str_eq(diff_xpath, parent_xpath, pcmk__str_none)
	&& pcmk__xe_is(pcmk__xml_first_child(change), element)) {

	// Newly added element
	rc = true;
	break;
	}
	}

	free(element_regex);
	return rc;
	}

	/*!
	* \internal
	* \brief Check whether a given CIB element was modified in a CIB patchset
	*
	* \param[in] patchset CIB XML patchset
	* \param[in] element XML tag of CIB element to check (\c NULL is equivalent
	* to \c XML_TAG_CIB). Supported values include any CIB
	* element supported by \c pcmk__cib_abs_xpath_for().
	*
	* \return \c true if \p element was modified, or \c false otherwise
	*/
	bool
	cib__element_in_patchset(const xmlNode patchset, const char element)
	{
	int format = 1;

	CRM_ASSERT(patchset != NULL);

	crm_element_value_int(patchset, "format", &format);
	switch (format) {
	case 1:
	return element_in_patchset_v1(patchset, element);

	case 2:
	return element_in_patchset_v2(patchset, element);

	default:
	crm_warn("Unknown patch format: %d", format);
	return false;
	}
	}

	/*!
	* \brief Create XML for a new (empty) CIB
	*
	* \param[in] cib_epoch What to use as "epoch" CIB property
	*
	* \return Newly created XML for empty CIB
	* \note It is the caller's responsibility to free the result with free_xml().
	*/
	xmlNode *
	createEmptyCib(int cib_epoch)
	{
	xmlNode cib_root = NULL, config = NULL;

	cib_root = create_xml_node(NULL, XML_TAG_CIB);
	crm_xml_add(cib_root, XML_ATTR_CRM_VERSION, CRM_FEATURE_SET);
	crm_xml_add(cib_root, XML_ATTR_VALIDATION, xml_latest_schema());

	crm_xml_add_int(cib_root, XML_ATTR_GENERATION, cib_epoch);
	crm_xml_add_int(cib_root, XML_ATTR_NUMUPDATES, 0);
	crm_xml_add_int(cib_root, XML_ATTR_GENERATION_ADMIN, 0);

	config = create_xml_node(cib_root, XML_CIB_TAG_CONFIGURATION);
	create_xml_node(cib_root, XML_CIB_TAG_STATUS);

	create_xml_node(config, XML_CIB_TAG_CRMCONFIG);
	create_xml_node(config, XML_CIB_TAG_NODES);
	create_xml_node(config, XML_CIB_TAG_RESOURCES);
	create_xml_node(config, XML_CIB_TAG_CONSTRAINTS);

	#if PCMK__RESOURCE_STICKINESS_DEFAULT != 0
	{
	xmlNode *rsc_defaults = create_xml_node(config, XML_CIB_TAG_RSCCONFIG);
	xmlNode *meta = create_xml_node(rsc_defaults, XML_TAG_META_SETS);
	xmlNode *nvpair = create_xml_node(meta, XML_CIB_TAG_NVPAIR);

	crm_xml_add(meta, XML_ATTR_ID, "build-resource-defaults");
	crm_xml_add(nvpair, XML_ATTR_ID, "build-" XML_RSC_ATTR_STICKINESS);
	crm_xml_add(nvpair, XML_NVPAIR_ATTR_NAME, XML_RSC_ATTR_STICKINESS);
	crm_xml_add_int(nvpair, XML_NVPAIR_ATTR_VALUE,
	PCMK__RESOURCE_STICKINESS_DEFAULT);
	}
	#endif
	return cib_root;
	}

	static bool
	cib_acl_enabled(xmlNode xml, const char user)
	{
	bool rc = FALSE;

	if(pcmk_acl_required(user)) {
	const char *value = NULL;
	GHashTable *options = pcmk__strkey_table(free, free);

	cib_read_config(options, xml);
	value = cib_pref(options, "enable-acl");
	rc = crm_is_true(value);
	g_hash_table_destroy(options);
	}

	crm_trace("CIB ACL is %s", rc ? "enabled" : "disabled");
	return rc;
	}

	/*!
	* \internal
	* \brief Determine whether to perform operations on a scratch copy of the CIB
	*
	* \param[in] op CIB operation
	* \param[in] section CIB section
	* \param[in] call_options CIB call options
	*
	* \return \p true if we should make a copy of the CIB, or \p false otherwise
	*/
	static bool
	should_copy_cib(const char op, const char section, int call_options)
	{
	if (pcmk_is_set(call_options, cib_dryrun)) {
	// cib_dryrun implies a scratch copy by definition; no side effects
	return true;
	}

	if (pcmk__str_eq(op, PCMK__CIB_REQUEST_COMMIT_TRANSACT, pcmk__str_none)) {
	/* Commit-transaction must make a copy for atomicity. We must revert to
	* the original CIB if the entire transaction cannot be applied
	* successfully.
	*/
	return true;
	}

	if (pcmk_is_set(call_options, cib_transaction)) {
	/* If cib_transaction is set, then we're in the process of committing a
	* transaction. The commit-transaction request already made a scratch
	* copy, and we're accumulating changes in that copy.
	*/
	return false;
	}

	if (pcmk__str_eq(section, XML_CIB_TAG_STATUS, pcmk__str_none)) {
	/* Copying large CIBs accounts for a huge percentage of our CIB usage,
	* and this avoids some of it.
	*
	* @TODO: Is this safe? See discussion at
	* https://github.com/ClusterLabs/pacemaker/pull/3094#discussion_r1211400690.
	*/
	return false;
	}

	// Default behavior is to operate on a scratch copy
	return true;
	}

	int
	cib_perform_op(const char *op, int call_options, cib__op_fn_t fn, bool is_query,
	const char section, xmlNode req, xmlNode *input,
	bool manage_counters, bool *config_changed,
	xmlNode current_cib, xmlNode result_cib, xmlNode **diff,
	xmlNode **output)
	{
	int rc = pcmk_ok;
	bool check_schema = true;
	bool make_copy = true;
	xmlNode *top = NULL;
	xmlNode *scratch = NULL;
	xmlNode *patchset_cib = NULL;
	xmlNode *local_diff = NULL;

	const char *new_version = NULL;
	const char *user = crm_element_value(req, F_CIB_USER);
	bool with_digest = false;

	pcmk__output_t *out = NULL;
	int out_rc = pcmk_rc_no_output;

	crm_trace("Begin %s%s%s op",
	(pcmk_is_set(call_options, cib_dryrun)? "dry run of " : ""),
	(is_query? "read-only " : ""), op);

	CRM_CHECK(output != NULL, return -ENOMSG);
	CRM_CHECK(current_cib != NULL, return -ENOMSG);
	CRM_CHECK(result_cib != NULL, return -ENOMSG);
	CRM_CHECK(config_changed != NULL, return -ENOMSG);

	if(output) {
	*output = NULL;
	}

	*result_cib = NULL;
	*config_changed = false;

	if (fn == NULL) {
	return -EINVAL;
	}

	if (is_query) {
	xmlNode cib_ro = current_cib;
	xmlNode *cib_filtered = NULL;

	if (cib_acl_enabled(cib_ro, user)
	&& xml_acl_filtered_copy(user, current_cib, current_cib,
	&cib_filtered)) {

	if (cib_filtered == NULL) {
	crm_debug("Pre-filtered the entire cib");
	return -EACCES;
	}
	cib_ro = cib_filtered;
	crm_log_xml_trace(cib_ro, "filtered");
	}

	rc = (*fn) (op, call_options, section, req, input, cib_ro, result_cib, output);

	if(output == NULL \|\| *output == NULL) {
	/* nothing */

	} else if(cib_filtered == *output) {
	cib_filtered = NULL; /* Let them have this copy */

	} else if (output == current_cib) {
	/* They already know not to free it */

	} else if(cib_filtered && (*output)->doc == cib_filtered->doc) {
	/* We're about to free the document of which output is a part /
	output = copy_xml(output);

	} else if ((output)->doc == (current_cib)->doc) {
	/* Give them a copy they can free */
	output = copy_xml(output);
	}

	free_xml(cib_filtered);
	return rc;
	}

	make_copy = should_copy_cib(op, section, call_options);

	if (!make_copy) {
	/* Conditional on v2 patch style */

	scratch = *current_cib;

	// Make a copy of the top-level element to store version details
	top = create_xml_node(NULL, (const char *) scratch->name);
	copy_in_properties(top, scratch);
	patchset_cib = top;

	xml_track_changes(scratch, user, NULL, cib_acl_enabled(scratch, user));
	rc = (*fn) (op, call_options, section, req, input, scratch, &scratch, output);

	/* If scratch points to a new object now (for example, after an erase
	* operation), then *current_cib should point to the same object.
	*/
	*current_cib = scratch;

	} else {
	scratch = copy_xml(*current_cib);
	patchset_cib = *current_cib;

	xml_track_changes(scratch, user, NULL, cib_acl_enabled(scratch, user));
	rc = (fn) (op, call_options, section, req, input, current_cib,
	&scratch, output);

	if ((scratch != NULL) && !xml_tracking_changes(scratch)) {
	crm_trace("Inferring changes after %s op", op);
	xml_track_changes(scratch, user, *current_cib,
	cib_acl_enabled(*current_cib, user));
	xml_calculate_changes(*current_cib, scratch);
	}
	CRM_CHECK(*current_cib != scratch, return -EINVAL);
	}

	xml_acl_disable(scratch); /* Allow the system to make any additional changes */

	if (rc == pcmk_ok && scratch == NULL) {
	rc = -EINVAL;
	goto done;

	} else if(rc == pcmk_ok && xml_acl_denied(scratch)) {
	crm_trace("ACL rejected part or all of the proposed changes");
	rc = -EACCES;
	goto done;

	} else if (rc != pcmk_ok) {
	goto done;
	}

	if (scratch) {
	new_version = crm_element_value(scratch, XML_ATTR_CRM_VERSION);

	if (new_version && compare_version(new_version, CRM_FEATURE_SET) > 0) {
	crm_err("Discarding update with feature set '%s' greater than our own '%s'",
	new_version, CRM_FEATURE_SET);
	rc = -EPROTONOSUPPORT;
	goto done;
	}
	}

	if (patchset_cib != NULL) {
	int old = 0;
	int new = 0;

	crm_element_value_int(scratch, XML_ATTR_GENERATION_ADMIN, &new);
	crm_element_value_int(patchset_cib, XML_ATTR_GENERATION_ADMIN, &old);

	if (old > new) {
	crm_err("%s went backwards: %d -> %d (Opts: %#x)",
	XML_ATTR_GENERATION_ADMIN, old, new, call_options);
	crm_log_xml_warn(req, "Bad Op");
	crm_log_xml_warn(input, "Bad Data");
	rc = -pcmk_err_old_data;

	} else if (old == new) {
	crm_element_value_int(scratch, XML_ATTR_GENERATION, &new);
	crm_element_value_int(patchset_cib, XML_ATTR_GENERATION, &old);
	if (old > new) {
	crm_err("%s went backwards: %d -> %d (Opts: %#x)",
	XML_ATTR_GENERATION, old, new, call_options);
	crm_log_xml_warn(req, "Bad Op");
	crm_log_xml_warn(input, "Bad Data");
	rc = -pcmk_err_old_data;
	}
	}
	}

	crm_trace("Massaging CIB contents");
	pcmk__strip_xml_text(scratch);
	fix_plus_plus_recursive(scratch);

	if (!make_copy) {
	/* At this point, patchset_cib is just the "cib" tag and its properties.
	*
	* The v1 format would barf on this, but we know the v2 patch
	* format only needs it for the top-level version fields
	*/
	local_diff = xml_create_patchset(2, patchset_cib, scratch,
	config_changed, manage_counters);

	} else {
	static time_t expires = 0;
	time_t tm_now = time(NULL);

	if (expires < tm_now) {
	expires = tm_now + 60; /* Validate clients are correctly applying v2-style diffs at most once a minute */
	with_digest = true;
	}

	local_diff = xml_create_patchset(0, patchset_cib, scratch,
	config_changed, manage_counters);
	}

	// Create a log output object only if we're going to use it
	pcmk__if_tracing(
	{
	rc = pcmk_rc2legacy(pcmk__log_output_new(&out));
	CRM_CHECK(rc == pcmk_ok, goto done);

	pcmk__output_set_log_level(out, LOG_TRACE);
	out_rc = pcmk__xml_show_changes(out, scratch);
	},
	{}
	);
	xml_accept_changes(scratch);

	if(local_diff) {
	int temp_rc = pcmk_rc_no_output;

	patchset_process_digest(local_diff, patchset_cib, scratch, with_digest);

	if (out == NULL) {
	rc = pcmk_rc2legacy(pcmk__log_output_new(&out));
	CRM_CHECK(rc == pcmk_ok, goto done);
	}
	pcmk__output_set_log_level(out, LOG_INFO);
	temp_rc = out->message(out, "xml-patchset", local_diff);
	out_rc = pcmk__output_select_rc(rc, temp_rc);

	crm_log_xml_trace(local_diff, "raw patch");
	}

	if (out != NULL) {
	out->finish(out, pcmk_rc2exitc(out_rc), true, NULL);
	pcmk__output_free(out);
	out = NULL;
	}

	if (make_copy && (local_diff != NULL)) {
	// Original to compare against doesn't exist
	pcmk__if_tracing(
	{
	// Validate the calculated patch set
	int test_rc = pcmk_ok;
	int format = 1;
	xmlNode *cib_copy = copy_xml(patchset_cib);

	crm_element_value_int(local_diff, "format", &format);
	test_rc = xml_apply_patchset(cib_copy, local_diff,
	manage_counters);

	if (test_rc != pcmk_ok) {
	save_xml_to_file(cib_copy, "PatchApply:calculated", NULL);
	save_xml_to_file(patchset_cib, "PatchApply:input", NULL);
	save_xml_to_file(scratch, "PatchApply:actual", NULL);
	save_xml_to_file(local_diff, "PatchApply:diff", NULL);
	crm_err("v%d patchset error, patch failed to apply: %s "
	"(%d)",
	format, pcmk_rc_str(pcmk_legacy2rc(test_rc)),
	test_rc);
	}
	free_xml(cib_copy);
	},
	{}
	);
	}

	if (pcmk__str_eq(section, XML_CIB_TAG_STATUS, pcmk__str_casei)) {
	/* Throttle the amount of costly validation we perform due to status updates
	* a) we don't really care whats in the status section
	* b) we don't validate any of its contents at the moment anyway
	*/
	check_schema = false;
	}

	/* === scratch must not be modified after this point ===
	* Exceptions, anything in:

	static filter_t filter[] = {
	{ 0, XML_ATTR_ORIGIN },
	{ 0, XML_CIB_ATTR_WRITTEN },
	{ 0, XML_ATTR_UPDATE_ORIG },
	{ 0, XML_ATTR_UPDATE_CLIENT },
	{ 0, XML_ATTR_UPDATE_USER },
	};
	*/

	if (*config_changed && !pcmk_is_set(call_options, cib_no_mtime)) {
	const char *schema = crm_element_value(scratch, XML_ATTR_VALIDATION);

	pcmk__xe_add_last_written(scratch);
	if (schema) {
	static int minimum_schema = 0;
	int current_schema = get_schema_version(schema);

	if (minimum_schema == 0) {
	minimum_schema = get_schema_version("pacemaker-1.2");
	}

	/* Does the CIB support the "update-" attributes... /
	if (current_schema >= minimum_schema) {
	/* Ensure values of origin, client, and user in scratch match
	* the values in req
	*/
	const char *origin = crm_element_value(req, F_ORIG);
	const char *client = crm_element_value(req, F_CIB_CLIENTNAME);

	if (origin != NULL) {
	crm_xml_add(scratch, XML_ATTR_UPDATE_ORIG, origin);
	} else {
	xml_remove_prop(scratch, XML_ATTR_UPDATE_ORIG);
	}

	if (client != NULL) {
	crm_xml_add(scratch, XML_ATTR_UPDATE_CLIENT, user);
	} else {
	xml_remove_prop(scratch, XML_ATTR_UPDATE_CLIENT);
	}

	if (user != NULL) {
	crm_xml_add(scratch, XML_ATTR_UPDATE_USER, user);
	} else {
	xml_remove_prop(scratch, XML_ATTR_UPDATE_USER);
	}
	}
	}
	}

	crm_trace("Perform validation: %s", pcmk__btoa(check_schema));
	if ((rc == pcmk_ok) && check_schema && !validate_xml(scratch, NULL, true)) {
	const char *current_schema = crm_element_value(scratch,
	XML_ATTR_VALIDATION);

	crm_warn("Updated CIB does not validate against %s schema",
	pcmk__s(current_schema, "unspecified"));
	rc = -pcmk_err_schema_validation;
	}

	done:

	*result_cib = scratch;

	/* @TODO: This may not work correctly with !make_copy, since we don't
	* keep the original CIB.
	*/
	if ((rc != pcmk_ok) && cib_acl_enabled(patchset_cib, user)
	&& xml_acl_filtered_copy(user, patchset_cib, scratch, result_cib)) {

	if (*result_cib == NULL) {
	crm_debug("Pre-filtered the entire cib result");
	}
	free_xml(scratch);
	}

	if(diff) {
	*diff = local_diff;
	} else {
	free_xml(local_diff);
	}

	free_xml(top);
	crm_trace("Done");
	return rc;
	}

	int
	cib__create_op(cib_t cib, const char op, const char *host,
	const char section, xmlNode data, int call_options,
	const char user_name, const char client_name,
	xmlNode **op_msg)
	{
	CRM_CHECK((cib != NULL) && (op_msg != NULL), return -EPROTO);

	*op_msg = create_xml_node(NULL, T_CIB_COMMAND);
	if (*op_msg == NULL) {
	return -EPROTO;
	}

	cib->call_id++;
	if (cib->call_id < 1) {
	cib->call_id = 1;
	}

	crm_xml_add(*op_msg, F_XML_TAGNAME, T_CIB_COMMAND);
	crm_xml_add(*op_msg, F_TYPE, T_CIB);
	crm_xml_add(*op_msg, F_CIB_OPERATION, op);
	crm_xml_add(*op_msg, F_CIB_HOST, host);
	crm_xml_add(*op_msg, F_CIB_SECTION, section);
	crm_xml_add(*op_msg, F_CIB_USER, user_name);
	crm_xml_add(*op_msg, F_CIB_CLIENTNAME, client_name);
	crm_xml_add_int(*op_msg, F_CIB_CALLID, cib->call_id);

	crm_trace("Sending call options: %.8lx, %d", (long)call_options, call_options);
	crm_xml_add_int(*op_msg, F_CIB_CALLOPTS, call_options);

	if (data != NULL) {
	add_message_xml(*op_msg, F_CIB_CALLDATA, data);
	}

	if (pcmk_is_set(call_options, cib_inhibit_bcast)) {
	CRM_CHECK(pcmk_is_set(call_options, cib_scope_local),
	free_xml(*op_msg); return -EPROTO);
	}
	return pcmk_ok;
	}

	/*!
	* \internal
	* \brief Check whether a CIB request is supported in a transaction
	*
	* \param[in] request CIB request
	*
	* \return Standard Pacemaker return code
	*/
	static int
	validate_transaction_request(const xmlNode *request)
	{
	const char *op = crm_element_value(request, F_CIB_OPERATION);
	const char *host = crm_element_value(request, F_CIB_HOST);
	const cib__operation_t *operation = NULL;
	int rc = cib__get_operation(op, &operation);

	if (rc != pcmk_rc_ok) {
	// cib__get_operation() logs error
	return rc;
	}

	if (!pcmk_is_set(operation->flags, cib__op_attr_transaction)) {
	crm_err("Operation %s is not supported in CIB transactions", op);
	return EOPNOTSUPP;
	}

	if (host != NULL) {
	crm_err("Operation targeting a specific node (%s) is not supported in "
	"a CIB transaction",
	host);
	return EOPNOTSUPP;
	}
	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief Append a CIB request to a CIB transaction
	*
	* \param[in,out] cib CIB client whose transaction to extend
	* \param[in,out] request Request to add to transaction
	*
	* \return Legacy Pacemaker return code
	*/
	int
	cib__extend_transaction(cib_t cib, xmlNode request)
	{
	int rc = pcmk_rc_ok;

	CRM_ASSERT((cib != NULL) && (request != NULL));

	rc = validate_transaction_request(request);

	if ((rc == pcmk_rc_ok) && (cib->transaction == NULL)) {
	rc = pcmk_rc_no_transaction;
	}

	if (rc == pcmk_rc_ok) {
	add_node_copy(cib->transaction, request);

	} else {
	const char *op = crm_element_value(request, F_CIB_OPERATION);
	const char *client_id = NULL;

	cib->cmds->client_id(cib, NULL, &client_id);
	crm_err("Failed to add '%s' operation to transaction for client %s: %s",
	op, pcmk__s(client_id, "(unidentified)"), pcmk_rc_str(rc));
	crm_log_xml_info(request, "failed");
	}
	return pcmk_rc2legacy(rc);
	}

	void
	cib_native_callback(cib_t * cib, xmlNode * msg, int call_id, int rc)
	{
	xmlNode *output = NULL;
	cib_callback_client_t *blob = NULL;

	if (msg != NULL) {
	crm_element_value_int(msg, F_CIB_RC, &rc);
	crm_element_value_int(msg, F_CIB_CALLID, &call_id);
	output = get_message_xml(msg, F_CIB_CALLDATA);
	}

	blob = cib__lookup_id(call_id);

	if (blob == NULL) {
	crm_trace("No callback found for call %d", call_id);
	}

	if (cib == NULL) {
	crm_debug("No cib object supplied");
	}

	if (rc == -pcmk_err_diff_resync) {
	/* This is an internal value that clients do not and should not care about */
	rc = pcmk_ok;
	}

	if (blob && blob->callback && (rc == pcmk_ok \|\| blob->only_success == FALSE)) {
	crm_trace("Invoking callback %s for call %d",
	pcmk__s(blob->id, "without ID"), call_id);
	blob->callback(msg, call_id, rc, output, blob->user_data);

	} else if (cib && cib->op_callback == NULL && rc != pcmk_ok) {
	crm_warn("CIB command failed: %s", pcmk_strerror(rc));
	crm_log_xml_debug(msg, "Failed CIB Update");
	}

	/* This may free user_data, so do it after the callback */
	if (blob) {
	remove_cib_op_callback(call_id, FALSE);
	}

	if (cib && cib->op_callback != NULL) {
	crm_trace("Invoking global callback for call %d", call_id);
	cib->op_callback(msg, call_id, rc, output);
	}
	crm_trace("OP callback activated for %d", call_id);
	}

	void
	cib_native_notify(gpointer data, gpointer user_data)
	{
	xmlNode *msg = user_data;
	cib_notify_client_t *entry = data;
	const char *event = NULL;

	if (msg == NULL) {
	crm_warn("Skipping callback - NULL message");
	return;
	}

	event = crm_element_value(msg, F_SUBTYPE);

	if (entry == NULL) {
	crm_warn("Skipping callback - NULL callback client");
	return;

	} else if (entry->callback == NULL) {
	crm_warn("Skipping callback - NULL callback");
	return;

	} else if (!pcmk__str_eq(entry->event, event, pcmk__str_casei)) {
	crm_trace("Skipping callback - event mismatch %p/%s vs. %s", entry, entry->event, event);
	return;
	}

	crm_trace("Invoking callback for %p/%s event...", entry, event);
	entry->callback(event, msg);
	crm_trace("Callback invoked...");
	}

	static pcmk__cluster_option_t cib_opts[] = {
	/* name, legacy name, type, allowed values,
	* default value, validator,
	* short description,
	* long description
	*/
	{
	"enable-acl", NULL, "boolean", NULL,
	"false", pcmk__valid_boolean,
	N_("Enable Access Control Lists (ACLs) for the CIB"),
	NULL
	},
	{
	"cluster-ipc-limit", NULL, "integer", NULL,
	"500", pcmk__valid_positive_number,
	N_("Maximum IPC message backlog before disconnecting a cluster daemon"),
	N_("Raise this if log has \"Evicting client\" messages for cluster daemon"
	" PIDs (a good value is the number of resources in the cluster"
	" multiplied by the number of nodes).")
	},
	};

	void
	cib_metadata(void)
	{
	const char *desc_short = "Cluster Information Base manager options";
	const char *desc_long = "Cluster options used by Pacemaker's Cluster "
	"Information Base manager";

	gchar *s = pcmk__format_option_metadata("pacemaker-based", desc_short,
	desc_long, cib_opts,
	PCMK__NELEM(cib_opts));
	printf("%s", s);
	g_free(s);
	}

	static void
	verify_cib_options(GHashTable *options)
	{
	pcmk__validate_cluster_options(options, cib_opts, PCMK__NELEM(cib_opts));
	}

	const char *
	cib_pref(GHashTable * options, const char *name)
	{
	return pcmk__cluster_option(options, cib_opts, PCMK__NELEM(cib_opts),
	name);
	}

	gboolean
	cib_read_config(GHashTable * options, xmlNode * current_cib)
	{
	xmlNode *config = NULL;
	crm_time_t *now = NULL;

	if (options == NULL \|\| current_cib == NULL) {
	return FALSE;
	}

	now = crm_time_new(NULL);

	g_hash_table_remove_all(options);

	config = pcmk_find_cib_element(current_cib, XML_CIB_TAG_CRMCONFIG);
	if (config) {
	pe_unpack_nvpairs(current_cib, config, XML_CIB_TAG_PROPSET, NULL,
	options, CIB_OPTIONS_FIRST, TRUE, now, NULL);
	}

	verify_cib_options(options);

	crm_time_free(now);

	return TRUE;
	}

	int
	cib_internal_op(cib_t * cib, const char op, const char host,
	const char section, xmlNode data,
	xmlNode ** output_data, int call_options, const char *user_name)
	{
	int (delegate) (cib_t cib, const char op, const char host,
	const char section, xmlNode data,
	xmlNode ** output_data, int call_options, const char *user_name) =
	cib->delegate_fn;

	if(user_name == NULL) {
	user_name = getenv("CIB_user");
	}

	return delegate(cib, op, host, section, data, output_data, call_options, user_name);
	}

	/*!
	* \brief Apply a CIB update patch to a given CIB
	*
	* \param[in] event CIB update patch
	* \param[in] input CIB to patch
	* \param[out] output Resulting CIB after patch
	* \param[in] level Log the patch at this log level (unless LOG_CRIT)
	*
	* \return Legacy Pacemaker return code
	* \note sbd calls this function
	*/
	int
	cib_apply_patch_event(xmlNode event, xmlNode input, xmlNode **output,
	int level)
	{
	int rc = pcmk_err_generic;

	xmlNode *diff = NULL;

	CRM_ASSERT(event);
	CRM_ASSERT(input);
	CRM_ASSERT(output);

	crm_element_value_int(event, F_CIB_RC, &rc);
	diff = get_message_xml(event, F_CIB_UPDATE_RESULT);

	if (rc < pcmk_ok \|\| diff == NULL) {
	return rc;
	}

	if (level > LOG_CRIT) {
	pcmk__output_t *out = NULL;

	rc = pcmk_rc2legacy(pcmk__log_output_new(&out));
	CRM_CHECK(rc == pcmk_ok, return rc);

	pcmk__output_set_log_level(out, level);
	rc = out->message(out, "xml-patchset", diff);
	out->finish(out, pcmk_rc2exitc(rc), true, NULL);
	pcmk__output_free(out);
	rc = pcmk_ok;
	}

	if (input != NULL) {
	rc = cib_process_diff(NULL, cib_none, NULL, event, diff, input, output,
	NULL);

	if (rc != pcmk_ok) {
	crm_debug("Update didn't apply: %s (%d) %p",
	pcmk_strerror(rc), rc, *output);

	if (rc == -pcmk_err_old_data) {
	crm_trace("Masking error, we already have the supplied update");
	return pcmk_ok;
	}
	free_xml(*output);
	*output = NULL;
	return rc;
	}
	}
	return rc;
	}

	#define log_signon_query_err(out, fmt, args...) do { \
	if (out != NULL) { \
	out->err(out, fmt, ##args); \
	} else { \
	crm_err(fmt, ##args); \
	} \
	} while (0)

	int
	cib__signon_query(pcmk__output_t out, cib_t cib, xmlNode *cib_object)
	{
	int rc = pcmk_rc_ok;
	cib_t *cib_conn = NULL;

	CRM_ASSERT(cib_object != NULL);

	if (cib == NULL) {
	cib_conn = cib_new();
	} else {
	if (*cib == NULL) {
	*cib = cib_new();
	}
	cib_conn = *cib;
	}

	if (cib_conn == NULL) {
	return ENOMEM;
	}

	if (cib_conn->state == cib_disconnected) {
	rc = cib_conn->cmds->signon(cib_conn, crm_system_name, cib_command);
	rc = pcmk_legacy2rc(rc);
	}

	if (rc != pcmk_rc_ok) {
	log_signon_query_err(out, "Could not connect to the CIB: %s",
	pcmk_rc_str(rc));
	goto done;
	}

	if (out != NULL) {
	out->transient(out, "Querying CIB...");
	}
	rc = cib_conn->cmds->query(cib_conn, NULL, cib_object,
	cib_scope_local\|cib_sync_call);
	rc = pcmk_legacy2rc(rc);

	if (rc != pcmk_rc_ok) {
	log_signon_query_err(out, "CIB query failed: %s", pcmk_rc_str(rc));
	}

	done:
	if (cib == NULL) {
	cib__clean_up_connection(&cib_conn);
	}

	if ((rc == pcmk_rc_ok) && (*cib_object == NULL)) {
	return pcmk_rc_no_input;
	}
	return rc;
	}

	int
	cib__clean_up_connection(cib_t **cib)
	{
	int rc;

	if (*cib == NULL) {
	return pcmk_rc_ok;
	}

	rc = (cib)->cmds->signoff(cib);
	cib_delete(*cib);
	*cib = NULL;
	return pcmk_legacy2rc(rc);
	}

	// Deprecated functions kept only for backward API compatibility
	// LCOV_EXCL_START

	#include <crm/cib/util_compat.h>

	const char *
	get_object_path(const char *object_type)
	{
	return pcmk_cib_xpath_for(object_type);
	}

	const char *
	get_object_parent(const char *object_type)
	{
	return pcmk_cib_parent_name_for(object_type);
	}

	xmlNode *
	get_object_root(const char object_type, xmlNode the_root)
	{
	return pcmk_find_cib_element(the_root, object_type);
	}

	// LCOV_EXCL_STOP
	// End deprecated API

File Metadata

Mime Type: text/x-diff
Expires: Mon, Apr 21, 7:00 PM (23 h, 16 m)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 1665283
Default Alt Text: (95 KB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions