No OneTemporary
Actions

Size

68 KB

Referenced Files

None

Subscribers

None

View Options

	diff --git a/daemons/controld/controld_cib.c b/daemons/controld/controld_cib.c
	index b07b66c1ff..9887314dd0 100644
	--- a/daemons/controld/controld_cib.c
	+++ b/daemons/controld/controld_cib.c
	@@ -1,1068 +1,1072 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <unistd.h> /* sleep */

	#include <crm/common/alerts_internal.h>
	#include <crm/common/xml.h>
	#include <crm/crm.h>
	#include <crm/msg_xml.h>
	#include <crm/lrmd_internal.h>

	#include <pacemaker-controld.h>

	// Call ID of the most recent in-progress CIB resource update (or 0 if none)
	static int pending_rsc_update = 0;

	/*!
	* \internal
	* \brief Respond to a dropped CIB connection
	*
	* \param[in] user_data CIB connection that dropped
	*/
	static void
	handle_cib_disconnect(gpointer user_data)
	{
	CRM_LOG_ASSERT(user_data == controld_globals.cib_conn);

	controld_trigger_fsa();
	controld_globals.cib_conn->state = cib_disconnected;

	if (pcmk_is_set(controld_globals.fsa_input_register, R_CIB_CONNECTED)) {
	// @TODO This should trigger a reconnect, not a shutdown
	crm_crit("Lost connection to the CIB manager, shutting down");
	register_fsa_input(C_FSA_INTERNAL, I_ERROR, NULL);
	controld_clear_fsa_input_flags(R_CIB_CONNECTED);

	} else { // Expected
	crm_info("Disconnected from the CIB manager");
	}
	}

	static void
	do_cib_updated(const char event, xmlNode msg)
	{
	const xmlNode *patchset = NULL;
	const char *client_name = NULL;

	crm_debug("Received CIB diff notification: DC=%s", pcmk__btoa(AM_I_DC));

	if (cib__get_notify_patchset(msg, &patchset) != pcmk_rc_ok) {
	return;
	}

	if (cib__element_in_patchset(patchset, XML_CIB_TAG_ALERTS)
	\|\| cib__element_in_patchset(patchset, XML_CIB_TAG_CRMCONFIG)) {

	controld_trigger_config();
	}

	if (!AM_I_DC) {
	// We're not in control of the join sequence
	return;
	}

	client_name = crm_element_value(msg, F_CIB_CLIENTNAME);
	if (!cib__client_triggers_refresh(client_name)) {
	// The CIB is still accurate
	return;
	}

	if (cib__element_in_patchset(patchset, XML_CIB_TAG_NODES)
	\|\| cib__element_in_patchset(patchset, XML_CIB_TAG_STATUS)) {

	/* An unsafe client modified the nodes or status section. Ensure the
	* node list is up-to-date, and start the join process again so we get
	* everyone's current resource history.
	*/
	if (client_name == NULL) {
	client_name = crm_element_value(msg, F_CIB_CLIENTID);
	}
	crm_notice("Populating nodes and starting an election after %s event "
	"triggered by %s",
	event, pcmk__s(client_name, "(unidentified client)"));

	populate_cib_nodes(node_update_quick\|node_update_all, __func__);
	register_fsa_input(C_FSA_INTERNAL, I_ELECTION, NULL);
	}
	}

	void
	controld_disconnect_cib_manager(void)
	{
	cib_t *cib_conn = controld_globals.cib_conn;

	CRM_ASSERT(cib_conn != NULL);

	crm_debug("Disconnecting from the CIB manager");

	controld_clear_fsa_input_flags(R_CIB_CONNECTED);

	cib_conn->cmds->del_notify_callback(cib_conn, T_CIB_DIFF_NOTIFY,
	do_cib_updated);
	cib_free_callbacks(cib_conn);

	if (cib_conn->state != cib_disconnected) {
	cib_conn->cmds->set_secondary(cib_conn,
	cib_scope_local\|cib_discard_reply);
	cib_conn->cmds->signoff(cib_conn);
	}
	}

	/* A_CIB_STOP, A_CIB_START, O_CIB_RESTART */
	void
	do_cib_control(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	static int cib_retries = 0;

	cib_t *cib_conn = controld_globals.cib_conn;

	void (*dnotify_fn) (gpointer user_data) = handle_cib_disconnect;
	void (update_cb) (const char event, xmlNodePtr msg) = do_cib_updated;

	int rc = pcmk_ok;

	CRM_ASSERT(cib_conn != NULL);

	if (pcmk_is_set(action, A_CIB_STOP)) {
	if ((cib_conn->state != cib_disconnected)
	&& (pending_rsc_update != 0)) {

	crm_info("Waiting for resource update %d to complete",
	pending_rsc_update);
	crmd_fsa_stall(FALSE);
	return;
	}
	controld_disconnect_cib_manager();
	}

	if (!pcmk_is_set(action, A_CIB_START)) {
	return;
	}

	if (cur_state == S_STOPPING) {
	crm_err("Ignoring request to connect to the CIB manager after "
	"shutdown");
	return;
	}

	rc = cib_conn->cmds->signon(cib_conn, CRM_SYSTEM_CRMD,
	cib_command_nonblocking);

	if (rc != pcmk_ok) {
	// A short wait that usually avoids stalling the FSA
	sleep(1);
	rc = cib_conn->cmds->signon(cib_conn, CRM_SYSTEM_CRMD,
	cib_command_nonblocking);
	}

	if (rc != pcmk_ok) {
	crm_info("Could not connect to the CIB manager: %s", pcmk_strerror(rc));

	} else if (cib_conn->cmds->set_connection_dnotify(cib_conn,
	dnotify_fn) != pcmk_ok) {
	crm_err("Could not set dnotify callback");

	} else if (cib_conn->cmds->add_notify_callback(cib_conn,
	T_CIB_DIFF_NOTIFY,
	update_cb) != pcmk_ok) {
	crm_err("Could not set CIB notification callback (update)");

	} else {
	controld_set_fsa_input_flags(R_CIB_CONNECTED);
	cib_retries = 0;
	}

	if (!pcmk_is_set(controld_globals.fsa_input_register, R_CIB_CONNECTED)) {
	cib_retries++;

	if (cib_retries < 30) {
	crm_warn("Couldn't complete CIB registration %d times... "
	"pause and retry", cib_retries);
	controld_start_wait_timer();
	crmd_fsa_stall(FALSE);

	} else {
	crm_err("Could not complete CIB registration %d times... "
	"hard error", cib_retries);
	register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
	}
	}
	}

	#define MIN_CIB_OP_TIMEOUT (30)

	/*!
	* \internal
	* \brief Get the timeout (in seconds) that should be used with CIB operations
	*
	* \return The maximum of 30 seconds, the value of the PCMK_cib_timeout
	* environment variable, or 10 seconds times one more than the number of
	* nodes in the cluster.
	*/
	unsigned int
	cib_op_timeout(void)
	{
	// @COMPAT: Drop env_timeout at 3.0.0
	static int env_timeout = -1;
	unsigned int calculated_timeout = 0;

	if (env_timeout == -1) {
	const char *env = pcmk__env_option(PCMK__ENV_CIB_TIMEOUT);

	pcmk__scan_min_int(env, &env_timeout, MIN_CIB_OP_TIMEOUT);
	crm_trace("Minimum CIB op timeout: %ds (environment: %s)",
	env_timeout, (env? env : "none"));
	}

	calculated_timeout = 1 + crm_active_peers();
	if (crm_remote_peer_cache) {
	calculated_timeout += g_hash_table_size(crm_remote_peer_cache);
	}
	calculated_timeout *= 10;

	calculated_timeout = QB_MAX(calculated_timeout, env_timeout);
	crm_trace("Calculated timeout: %us", calculated_timeout);

	if (controld_globals.cib_conn) {
	controld_globals.cib_conn->call_timeout = calculated_timeout;
	}
	return calculated_timeout;
	}

	/*!
	* \internal
	* \brief Get CIB call options to use local scope if primary is unavailable
	*
	* \return CIB call options
	*/
	int
	crmd_cib_smart_opt(void)
	{
	int call_opt = cib_none;

	if ((controld_globals.fsa_state == S_ELECTION)
	\|\| (controld_globals.fsa_state == S_PENDING)) {
	crm_info("Sending update to local CIB in state: %s",
	fsa_state2string(controld_globals.fsa_state));
	cib__set_call_options(call_opt, "update", cib_scope_local);
	}
	return call_opt;
	}

	static void
	cib_delete_callback(xmlNode msg, int call_id, int rc, xmlNode output,
	void *user_data)
	{
	char *desc = user_data;

	if (rc == 0) {
	crm_debug("Deletion of %s (via CIB call %d) succeeded", desc, call_id);
	} else {
	crm_warn("Deletion of %s (via CIB call %d) failed: %s " CRM_XS " rc=%d",
	desc, call_id, pcmk_strerror(rc), rc);
	}
	}

	// Searches for various portions of node_state to delete

	// Match a particular node's node_state (takes node name 1x)
	#define XPATH_NODE_STATE "//" XML_CIB_TAG_STATE \
	"[@" PCMK_XA_UNAME "='%s']"

	// Node's lrm section (name 1x)
	#define XPATH_NODE_LRM XPATH_NODE_STATE "/" XML_CIB_TAG_LRM

	/* Node's lrm_rsc_op entries and lrm_resource entries without unexpired lock
	* (name 2x, (seconds_since_epoch - PCMK_OPT_SHUTDOWN_LOCK_LIMIT) 1x)
	*/
	#define XPATH_NODE_LRM_UNLOCKED XPATH_NODE_STATE "//" XML_LRM_TAG_RSC_OP \
	"\|" XPATH_NODE_STATE \
	"//" XML_LRM_TAG_RESOURCE \
	"[not(@" PCMK_OPT_SHUTDOWN_LOCK ") " \
	"or " PCMK_OPT_SHUTDOWN_LOCK "<%lld]"

	// Node's transient_attributes section (name 1x)
	#define XPATH_NODE_ATTRS XPATH_NODE_STATE "/" XML_TAG_TRANSIENT_NODEATTRS

	// Everything under node_state (name 1x)
	#define XPATH_NODE_ALL XPATH_NODE_STATE "/*"

	/* Unlocked history + transient attributes
	* (name 2x, (seconds_since_epoch - PCMK_OPT_SHUTDOWN_LOCK_LIMIT) 1x, name 1x)
	*/
	#define XPATH_NODE_ALL_UNLOCKED XPATH_NODE_LRM_UNLOCKED "\|" XPATH_NODE_ATTRS

	/*!
	* \internal
	* \brief Get the XPath and description of a node state section to be deleted
	*
	* \param[in] uname Desired node
	* \param[in] section Subsection of node_state to be deleted
	* \param[out] xpath Where to store XPath of \p section
	* \param[out] desc If not \c NULL, where to store description of \p section
	*/
	void
	controld_node_state_deletion_strings(const char *uname,
	enum controld_section_e section,
	char xpath, char desc)
	{
	const char *desc_pre = NULL;

	// Shutdown locks that started before this time are expired
	long long expire = (long long) time(NULL)
	- controld_globals.shutdown_lock_limit;

	switch (section) {
	case controld_section_lrm:
	*xpath = crm_strdup_printf(XPATH_NODE_LRM, uname);
	desc_pre = "resource history";
	break;
	case controld_section_lrm_unlocked:
	*xpath = crm_strdup_printf(XPATH_NODE_LRM_UNLOCKED,
	uname, uname, expire);
	desc_pre = "resource history (other than shutdown locks)";
	break;
	case controld_section_attrs:
	*xpath = crm_strdup_printf(XPATH_NODE_ATTRS, uname);
	desc_pre = "transient attributes";
	break;
	case controld_section_all:
	*xpath = crm_strdup_printf(XPATH_NODE_ALL, uname);
	desc_pre = "all state";
	break;
	case controld_section_all_unlocked:
	*xpath = crm_strdup_printf(XPATH_NODE_ALL_UNLOCKED,
	uname, uname, expire, uname);
	desc_pre = "all state (other than shutdown locks)";
	break;
	default:
	// We called this function incorrectly
	CRM_ASSERT(false);
	break;
	}

	if (desc != NULL) {
	*desc = crm_strdup_printf("%s for node %s", desc_pre, uname);
	}
	}

	/*!
	* \internal
	* \brief Delete subsection of a node's CIB node_state
	*
	* \param[in] uname Desired node
	* \param[in] section Subsection of node_state to delete
	* \param[in] options CIB call options to use
	*/
	void
	controld_delete_node_state(const char *uname, enum controld_section_e section,
	int options)
	{
	cib_t *cib = controld_globals.cib_conn;
	char *xpath = NULL;
	char *desc = NULL;
	int cib_rc = pcmk_ok;

	CRM_ASSERT((uname != NULL) && (cib != NULL));

	controld_node_state_deletion_strings(uname, section, &xpath, &desc);

	cib__set_call_options(options, "node state deletion",
	cib_xpath\|cib_multiple);
	cib_rc = cib->cmds->remove(cib, xpath, NULL, options);
	fsa_register_cib_callback(cib_rc, desc, cib_delete_callback);
	crm_info("Deleting %s (via CIB call %d) " CRM_XS " xpath=%s",
	desc, cib_rc, xpath);

	// CIB library handles freeing desc
	free(xpath);
	}

	// Takes node name and resource ID
	#define XPATH_RESOURCE_HISTORY "//" XML_CIB_TAG_STATE \
	"[@" PCMK_XA_UNAME "='%s']/" \
	XML_CIB_TAG_LRM "/" XML_LRM_TAG_RESOURCES \
	"/" XML_LRM_TAG_RESOURCE \
	"[@" PCMK_XA_ID "='%s']"
	// @TODO could add "and @PCMK_OPT_SHUTDOWN_LOCK" to limit to locks

	/*!
	* \internal
	* \brief Clear resource history from CIB for a given resource and node
	*
	* \param[in] rsc_id ID of resource to be cleared
	* \param[in] node Node whose resource history should be cleared
	* \param[in] user_name ACL user name to use
	* \param[in] call_options CIB call options
	*
	* \return Standard Pacemaker return code
	*/
	int
	controld_delete_resource_history(const char rsc_id, const char node,
	const char *user_name, int call_options)
	{
	char *desc = NULL;
	char *xpath = NULL;
	int rc = pcmk_rc_ok;
	cib_t *cib = controld_globals.cib_conn;

	CRM_CHECK((rsc_id != NULL) && (node != NULL), return EINVAL);

	desc = crm_strdup_printf("resource history for %s on %s", rsc_id, node);
	if (cib == NULL) {
	crm_err("Unable to clear %s: no CIB connection", desc);
	free(desc);
	return ENOTCONN;
	}

	// Ask CIB to delete the entry
	xpath = crm_strdup_printf(XPATH_RESOURCE_HISTORY, node, rsc_id);

	cib->cmds->set_user(cib, user_name);
	rc = cib->cmds->remove(cib, xpath, NULL, call_options\|cib_xpath);
	cib->cmds->set_user(cib, NULL);

	if (rc < 0) {
	rc = pcmk_legacy2rc(rc);
	crm_err("Could not delete resource status of %s on %s%s%s: %s "
	CRM_XS " rc=%d", rsc_id, node,
	(user_name? " for user " : ""), (user_name? user_name : ""),
	pcmk_rc_str(rc), rc);
	free(desc);
	free(xpath);
	return rc;
	}

	if (pcmk_is_set(call_options, cib_sync_call)) {
	if (pcmk_is_set(call_options, cib_dryrun)) {
	crm_debug("Deletion of %s would succeed", desc);
	} else {
	crm_debug("Deletion of %s succeeded", desc);
	}
	free(desc);

	} else {
	crm_info("Clearing %s (via CIB call %d) " CRM_XS " xpath=%s",
	desc, rc, xpath);
	fsa_register_cib_callback(rc, desc, cib_delete_callback);
	// CIB library handles freeing desc
	}

	free(xpath);
	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief Build XML and string of parameters meeting some criteria, for digest
	*
	* \param[in] op Executor event with parameter table to use
	* \param[in] metadata Parsed meta-data for executed resource agent
	* \param[in] param_type Flag used for selection criteria
	* \param[out] result Will be set to newly created XML with selected
	* parameters as attributes
	*
	* \return Newly allocated space-separated string of parameter names
	* \note Selection criteria varies by param_type: for the restart digest, we
	* want parameters that are not marked reloadable (OCF 1.1) or that
	* are marked unique (pre-1.1), for both string and XML results; for the
	* secure digest, we want parameters that are marked private for the
	* string, but parameters that are not marked private for the XML.
	* \note It is the caller's responsibility to free the string return value with
	* \p g_string_free() and the XML result with \p free_xml().
	*/
	static GString *
	build_parameter_list(const lrmd_event_data_t *op,
	const struct ra_metadata_s *metadata,
	enum ra_param_flags_e param_type, xmlNode **result)
	{
	GString *list = NULL;

	*result = create_xml_node(NULL, XML_TAG_PARAMS);

	/* Consider all parameters only except private ones to be consistent with
	* what scheduler does with calculate_secure_digest().
	*/
	if (param_type == ra_param_private
	&& compare_version(controld_globals.dc_version, "3.16.0") >= 0) {
	g_hash_table_foreach(op->params, hash2field, *result);
	pcmk__filter_op_for_digest(*result);
	}

	for (GList *iter = metadata->ra_params; iter != NULL; iter = iter->next) {
	struct ra_param_s param = (struct ra_param_s ) iter->data;

	bool accept_for_list = false;
	bool accept_for_xml = false;

	switch (param_type) {
	case ra_param_reloadable:
	accept_for_list = !pcmk_is_set(param->rap_flags, param_type);
	accept_for_xml = accept_for_list;
	break;

	case ra_param_unique:
	accept_for_list = pcmk_is_set(param->rap_flags, param_type);
	accept_for_xml = accept_for_list;
	break;

	case ra_param_private:
	accept_for_list = pcmk_is_set(param->rap_flags, param_type);
	accept_for_xml = !accept_for_list;
	break;
	}

	if (accept_for_list) {
	crm_trace("Attr %s is %s", param->rap_name, ra_param_flag2text(param_type));

	if (list == NULL) {
	// We will later search for " WORD ", so start list with a space
	pcmk__add_word(&list, 256, " ");
	}
	pcmk__add_word(&list, 0, param->rap_name);

	} else {
	crm_trace("Rejecting %s for %s", param->rap_name, ra_param_flag2text(param_type));
	}

	if (accept_for_xml) {
	const char *v = g_hash_table_lookup(op->params, param->rap_name);

	if (v != NULL) {
	crm_trace("Adding attr %s=%s to the xml result", param->rap_name, v);
	crm_xml_add(*result, param->rap_name, v);
	}

	} else {
	crm_trace("Removing attr %s from the xml result", param->rap_name);
	xml_remove_prop(*result, param->rap_name);
	}
	}

	if (list != NULL) {
	// We will later search for " WORD ", so end list with a space
	pcmk__add_word(&list, 0, " ");
	}
	return list;
	}

	static void
	append_restart_list(lrmd_event_data_t op, struct ra_metadata_s metadata,
	xmlNode update, const char version)
	{
	GString *list = NULL;
	char *digest = NULL;
	xmlNode *restart = NULL;

	CRM_LOG_ASSERT(op->params != NULL);

	if (op->interval_ms > 0) {
	/* monitors are not reloadable */
	return;
	}

	if (pcmk_is_set(metadata->ra_flags, ra_supports_reload_agent)) {
	- // Add parameters not marked reloadable to the "op-force-restart" list
	+ /* Add parameters not marked reloadable to the PCMK__XA_OP_FORCE_RESTART
	+ * list
	+ */
	list = build_parameter_list(op, metadata, ra_param_reloadable,
	&restart);

	} else if (pcmk_is_set(metadata->ra_flags, ra_supports_legacy_reload)) {
	/* @COMPAT pre-OCF-1.1 resource agents
	*
	* Before OCF 1.1, Pacemaker abused "unique=0" to indicate
	* reloadability. Add any parameters with unique="1" to the
	- * "op-force-restart" list.
	+ * PCMK__XA_OP_FORCE_RESTART list.
	*/
	list = build_parameter_list(op, metadata, ra_param_unique, &restart);

	} else {
	// Resource does not support agent reloads
	return;
	}

	digest = calculate_operation_digest(restart, version);
	- /* Add "op-force-restart" and "op-restart-digest" to indicate the resource supports reload,
	- * no matter if it actually supports any parameters with unique="1"). */
	- crm_xml_add(update, XML_LRM_ATTR_OP_RESTART,
	+ /* Add PCMK__XA_OP_FORCE_RESTART and "op-restart-digest" to indicate the
	+ * resource supports reload, no matter if it actually supports any
	+ * reloadable parameters
	+ */
	+ crm_xml_add(update, PCMK__XA_OP_FORCE_RESTART,
	(list == NULL)? "" : (const char *) list->str);
	crm_xml_add(update, XML_LRM_ATTR_RESTART_DIGEST, digest);

	if ((list != NULL) && (list->len > 0)) {
	crm_trace("%s: %s, %s", op->rsc_id, digest, (const char *) list->str);
	} else {
	crm_trace("%s: %s", op->rsc_id, digest);
	}

	if (list != NULL) {
	g_string_free(list, TRUE);
	}
	free_xml(restart);
	free(digest);
	}

	static void
	append_secure_list(lrmd_event_data_t op, struct ra_metadata_s metadata,
	xmlNode update, const char version)
	{
	GString *list = NULL;
	char *digest = NULL;
	xmlNode *secure = NULL;

	CRM_LOG_ASSERT(op->params != NULL);

	/*
	* To keep XML_LRM_ATTR_OP_SECURE short, we want it to contain the
	* secure parameters but XML_LRM_ATTR_SECURE_DIGEST to be based on
	* the insecure ones
	*/
	list = build_parameter_list(op, metadata, ra_param_private, &secure);

	if (list != NULL) {
	digest = calculate_operation_digest(secure, version);
	crm_xml_add(update, XML_LRM_ATTR_OP_SECURE, (const char *) list->str);
	crm_xml_add(update, XML_LRM_ATTR_SECURE_DIGEST, digest);

	crm_trace("%s: %s, %s", op->rsc_id, digest, (const char *) list->str);
	g_string_free(list, TRUE);
	} else {
	crm_trace("%s: no secure parameters", op->rsc_id);
	}

	free_xml(secure);
	free(digest);
	}

	/*!
	* \internal
	* \brief Create XML for a resource history entry
	*
	* \param[in] func Function name of caller
	* \param[in,out] parent XML to add entry to
	* \param[in] rsc Affected resource
	* \param[in,out] op Action to add an entry for (or NULL to do nothing)
	* \param[in] node_name Node where action occurred
	*/
	void
	controld_add_resource_history_xml_as(const char func, xmlNode parent,
	const lrmd_rsc_info_t *rsc,
	lrmd_event_data_t *op,
	const char *node_name)
	{
	int target_rc = 0;
	xmlNode *xml_op = NULL;
	struct ra_metadata_s *metadata = NULL;
	const char *caller_version = NULL;
	lrm_state_t *lrm_state = NULL;

	if (op == NULL) {
	return;
	}

	target_rc = rsc_op_expected_rc(op);

	caller_version = g_hash_table_lookup(op->params, PCMK_XA_CRM_FEATURE_SET);
	CRM_CHECK(caller_version != NULL, caller_version = CRM_FEATURE_SET);

	xml_op = pcmk__create_history_xml(parent, op, caller_version, target_rc,
	controld_globals.our_nodename, func);
	if (xml_op == NULL) {
	return;
	}

	if ((rsc == NULL) \|\| (op->params == NULL)
	\|\| !crm_op_needs_metadata(rsc->standard, op->op_type)) {

	crm_trace("No digests needed for %s action on %s (params=%p rsc=%p)",
	op->op_type, op->rsc_id, op->params, rsc);
	return;
	}

	lrm_state = lrm_state_find(node_name);
	if (lrm_state == NULL) {
	crm_warn("Cannot calculate digests for operation " PCMK__OP_FMT
	" because we have no connection to executor for %s",
	op->rsc_id, op->op_type, op->interval_ms, node_name);
	return;
	}

	/* Ideally the metadata is cached, and the agent is just a fallback.
	*
	* @TODO Go through all callers and ensure they get metadata asynchronously
	* first.
	*/
	metadata = controld_get_rsc_metadata(lrm_state, rsc,
	controld_metadata_from_agent
	\|controld_metadata_from_cache);
	if (metadata == NULL) {
	return;
	}

	crm_trace("Including additional digests for %s:%s:%s",
	rsc->standard, rsc->provider, rsc->type);
	append_restart_list(op, metadata, xml_op, caller_version);
	append_secure_list(op, metadata, xml_op, caller_version);

	return;
	}

	/*!
	* \internal
	* \brief Record an action as pending in the CIB, if appropriate
	*
	* \param[in] node_name Node where the action is pending
	* \param[in] rsc Resource that action is for
	* \param[in,out] op Pending action
	*
	* \return true if action was recorded in CIB, otherwise false
	*/
	bool
	controld_record_pending_op(const char node_name, const lrmd_rsc_info_t rsc,
	lrmd_event_data_t *op)
	{
	const char *record_pending = NULL;

	CRM_CHECK((node_name != NULL) && (rsc != NULL) && (op != NULL),
	return false);

	// Never record certain operation types as pending
	if ((op->op_type == NULL) \|\| (op->params == NULL)
	\|\| !controld_action_is_recordable(op->op_type)) {
	return false;
	}

	// Check action's PCMK_META_RECORD_PENDING meta-attribute (defaults to true)
	record_pending = crm_meta_value(op->params, PCMK_META_RECORD_PENDING);
	if ((record_pending != NULL) && !crm_is_true(record_pending)) {
	return false;
	}

	op->call_id = -1;
	op->t_run = time(NULL);
	op->t_rcchange = op->t_run;

	lrmd__set_result(op, PCMK_OCF_UNKNOWN, PCMK_EXEC_PENDING, NULL);

	crm_debug("Recording pending %s-interval %s for %s on %s in the CIB",
	pcmk__readable_interval(op->interval_ms), op->op_type, op->rsc_id,
	node_name);
	controld_update_resource_history(node_name, rsc, op, 0);
	return true;
	}

	static void
	cib_rsc_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
	{
	switch (rc) {
	case pcmk_ok:
	case -pcmk_err_diff_failed:
	case -pcmk_err_diff_resync:
	crm_trace("Resource history update completed (call=%d rc=%d)",
	call_id, rc);
	break;
	default:
	if (call_id > 0) {
	crm_warn("Resource history update %d failed: %s "
	CRM_XS " rc=%d", call_id, pcmk_strerror(rc), rc);
	} else {
	crm_warn("Resource history update failed: %s " CRM_XS " rc=%d",
	pcmk_strerror(rc), rc);
	}
	}

	if (call_id == pending_rsc_update) {
	pending_rsc_update = 0;
	controld_trigger_fsa();
	}
	}

	/* Only successful stops, and probes that found the resource inactive, get locks
	* recorded in the history. This ensures the resource stays locked to the node
	* until it is active there again after the node comes back up.
	*/
	static bool
	should_preserve_lock(lrmd_event_data_t *op)
	{
	if (!pcmk_is_set(controld_globals.flags, controld_shutdown_lock_enabled)) {
	return false;
	}
	if (!strcmp(op->op_type, PCMK_ACTION_STOP) && (op->rc == PCMK_OCF_OK)) {
	return true;
	}
	if (!strcmp(op->op_type, PCMK_ACTION_MONITOR)
	&& (op->rc == PCMK_OCF_NOT_RUNNING)) {
	return true;
	}
	return false;
	}

	/*!
	* \internal
	* \brief Request a CIB update
	*
	* \param[in] section Section of CIB to update
	* \param[in] data New XML of CIB section to update
	* \param[in] options CIB call options
	* \param[in] callback If not \c NULL, set this as the operation callback
	*
	* \return Standard Pacemaker return code
	*
	* \note If \p callback is \p cib_rsc_callback(), the CIB update's call ID is
	* stored in \p pending_rsc_update on success.
	*/
	int
	controld_update_cib(const char section, xmlNode data, int options,
	void (callback)(xmlNode , int, int, xmlNode , void ))
	{
	cib_t *cib = controld_globals.cib_conn;
	int cib_rc = -ENOTCONN;

	CRM_ASSERT(data != NULL);

	if (cib != NULL) {
	cib_rc = cib->cmds->modify(cib, section, data, options);
	if (cib_rc >= 0) {
	crm_debug("Submitted CIB update %d for %s section",
	cib_rc, section);
	}
	}

	if (callback == NULL) {
	if (cib_rc < 0) {
	crm_err("Failed to update CIB %s section: %s",
	section, pcmk_rc_str(pcmk_legacy2rc(cib_rc)));
	}

	} else {
	if ((cib_rc >= 0) && (callback == cib_rsc_callback)) {
	/* Checking for a particular callback is a little hacky, but it
	* didn't seem worth adding an output argument for cib_rc for just
	* one use case.
	*/
	pending_rsc_update = cib_rc;
	}
	fsa_register_cib_callback(cib_rc, NULL, callback);
	}

	return (cib_rc >= 0)? pcmk_rc_ok : pcmk_legacy2rc(cib_rc);
	}

	/*!
	* \internal
	* \brief Update resource history entry in CIB
	*
	* \param[in] node_name Node where action occurred
	* \param[in] rsc Resource that action is for
	* \param[in,out] op Action to record
	* \param[in] lock_time If nonzero, when resource was locked to node
	*
	* \note On success, the CIB update's call ID will be stored in
	* pending_rsc_update.
	*/
	void
	controld_update_resource_history(const char *node_name,
	const lrmd_rsc_info_t *rsc,
	lrmd_event_data_t *op, time_t lock_time)
	{
	xmlNode *update = NULL;
	xmlNode *xml = NULL;
	int call_opt = crmd_cib_smart_opt();
	const char *node_id = NULL;
	const char *container = NULL;

	CRM_CHECK((node_name != NULL) && (op != NULL), return);

	if (rsc == NULL) {
	crm_warn("Resource %s no longer exists in the executor", op->rsc_id);
	controld_ack_event_directly(NULL, NULL, rsc, op, op->rsc_id);
	return;
	}

	// <status>
	update = create_xml_node(NULL, XML_CIB_TAG_STATUS);

	// <node_state ...>
	xml = create_xml_node(update, XML_CIB_TAG_STATE);
	if (pcmk__str_eq(node_name, controld_globals.our_nodename,
	pcmk__str_casei)) {
	node_id = controld_globals.our_uuid;
	} else {
	node_id = node_name;
	pcmk__xe_set_bool_attr(xml, XML_NODE_IS_REMOTE, true);
	}
	crm_xml_add(xml, PCMK_XA_ID, node_id);
	crm_xml_add(xml, PCMK_XA_UNAME, node_name);
	crm_xml_add(xml, PCMK_XA_CRM_DEBUG_ORIGIN, __func__);

	// <lrm ...>
	xml = create_xml_node(xml, XML_CIB_TAG_LRM);
	crm_xml_add(xml, PCMK_XA_ID, node_id);

	// <lrm_resources>
	xml = create_xml_node(xml, XML_LRM_TAG_RESOURCES);

	// <lrm_resource ...>
	xml = create_xml_node(xml, XML_LRM_TAG_RESOURCE);
	crm_xml_add(xml, PCMK_XA_ID, op->rsc_id);
	crm_xml_add(xml, PCMK_XA_CLASS, rsc->standard);
	crm_xml_add(xml, PCMK_XA_PROVIDER, rsc->provider);
	crm_xml_add(xml, PCMK_XA_TYPE, rsc->type);
	if (lock_time != 0) {
	/* Actions on a locked resource should either preserve the lock by
	* recording it with the action result, or clear it.
	*/
	if (!should_preserve_lock(op)) {
	lock_time = 0;
	}
	crm_xml_add_ll(xml, PCMK_OPT_SHUTDOWN_LOCK, (long long) lock_time);
	}
	if (op->params != NULL) {
	container = g_hash_table_lookup(op->params,
	CRM_META "_" PCMK__META_CONTAINER);
	if (container != NULL) {
	crm_trace("Resource %s is a part of container resource %s",
	op->rsc_id, container);
	crm_xml_add(xml, PCMK__META_CONTAINER, container);
	}
	}

	// <lrm_resource_op ...> (possibly more than one)
	controld_add_resource_history_xml(xml, rsc, op, node_name);

	/* Update CIB asynchronously. Even if it fails, the resource state should be
	* discovered during the next election. Worst case, the node is wrongly
	* fenced for running a resource it isn't.
	*/
	crm_log_xml_trace(update, __func__);
	controld_update_cib(XML_CIB_TAG_STATUS, update, call_opt, cib_rsc_callback);
	free_xml(update);
	}

	/*!
	* \internal
	* \brief Erase an LRM history entry from the CIB, given the operation data
	*
	* \param[in] op Operation whose history should be deleted
	*/
	void
	controld_delete_action_history(const lrmd_event_data_t *op)
	{
	xmlNode *xml_top = NULL;

	CRM_CHECK(op != NULL, return);

	xml_top = create_xml_node(NULL, XML_LRM_TAG_RSC_OP);
	crm_xml_add_int(xml_top, PCMK__XA_CALL_ID, op->call_id);
	crm_xml_add(xml_top, PCMK__XA_TRANSITION_KEY, op->user_data);

	if (op->interval_ms > 0) {
	char *op_id = pcmk__op_key(op->rsc_id, op->op_type, op->interval_ms);

	/* Avoid deleting last_failure too (if it was a result of this recurring op failing) */
	crm_xml_add(xml_top, PCMK_XA_ID, op_id);
	free(op_id);
	}

	crm_debug("Erasing resource operation history for " PCMK__OP_FMT " (call=%d)",
	op->rsc_id, op->op_type, op->interval_ms, op->call_id);

	controld_globals.cib_conn->cmds->remove(controld_globals.cib_conn,
	XML_CIB_TAG_STATUS, xml_top,
	cib_none);
	crm_log_xml_trace(xml_top, "op:cancel");
	free_xml(xml_top);
	}

	/* Define xpath to find LRM resource history entry by node and resource */
	#define XPATH_HISTORY \
	"/" XML_TAG_CIB "/" XML_CIB_TAG_STATUS \
	"/" XML_CIB_TAG_STATE "[@" PCMK_XA_UNAME "='%s']" \
	"/" XML_CIB_TAG_LRM "/" XML_LRM_TAG_RESOURCES \
	"/" XML_LRM_TAG_RESOURCE "[@" PCMK_XA_ID "='%s']" \
	"/" XML_LRM_TAG_RSC_OP

	/* ... and also by operation key */
	#define XPATH_HISTORY_ID XPATH_HISTORY "[@" PCMK_XA_ID "='%s']"

	/* ... and also by operation key and operation call ID */
	#define XPATH_HISTORY_CALL XPATH_HISTORY \
	"[@" PCMK_XA_ID "='%s' and @" PCMK__XA_CALL_ID "='%d']"

	/* ... and also by operation key and original operation key */
	#define XPATH_HISTORY_ORIG XPATH_HISTORY \
	"[@" PCMK_XA_ID "='%s' and @" PCMK__XA_OPERATION_KEY "='%s']"

	/*!
	* \internal
	* \brief Delete a last_failure resource history entry from the CIB
	*
	* \param[in] rsc_id Name of resource to clear history for
	* \param[in] node Name of node to clear history for
	* \param[in] action If specified, delete only if this was failed action
	* \param[in] interval_ms If \p action is specified, it has this interval
	*/
	void
	controld_cib_delete_last_failure(const char rsc_id, const char node,
	const char *action, guint interval_ms)
	{
	char *xpath = NULL;
	char *last_failure_key = NULL;
	CRM_CHECK((rsc_id != NULL) && (node != NULL), return);

	// Generate XPath to match desired entry
	last_failure_key = pcmk__op_key(rsc_id, "last_failure", 0);
	if (action == NULL) {
	xpath = crm_strdup_printf(XPATH_HISTORY_ID, node, rsc_id,
	last_failure_key);
	} else {
	char *action_key = pcmk__op_key(rsc_id, action, interval_ms);

	xpath = crm_strdup_printf(XPATH_HISTORY_ORIG, node, rsc_id,
	last_failure_key, action_key);
	free(action_key);
	}
	free(last_failure_key);

	controld_globals.cib_conn->cmds->remove(controld_globals.cib_conn, xpath,
	NULL, cib_xpath);
	free(xpath);
	}

	/*!
	* \internal
	* \brief Delete resource history entry from the CIB, given operation key
	*
	* \param[in] rsc_id Name of resource to clear history for
	* \param[in] node Name of node to clear history for
	* \param[in] key Operation key of operation to clear history for
	* \param[in] call_id If specified, delete entry only if it has this call ID
	*/
	void
	controld_delete_action_history_by_key(const char rsc_id, const char node,
	const char *key, int call_id)
	{
	char *xpath = NULL;

	CRM_CHECK((rsc_id != NULL) && (node != NULL) && (key != NULL), return);

	if (call_id > 0) {
	xpath = crm_strdup_printf(XPATH_HISTORY_CALL, node, rsc_id, key,
	call_id);
	} else {
	xpath = crm_strdup_printf(XPATH_HISTORY_ID, node, rsc_id, key);
	}
	controld_globals.cib_conn->cmds->remove(controld_globals.cib_conn, xpath,
	NULL, cib_xpath);
	free(xpath);
	}
	diff --git a/include/crm_internal.h b/include/crm_internal.h
	index 3960652613..c14fe2195b 100644
	--- a/include/crm_internal.h
	+++ b/include/crm_internal.h
	@@ -1,197 +1,198 @@
	/*
	* Copyright 2006-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#ifndef CRM_INTERNAL__H
	# define CRM_INTERNAL__H

	# ifndef PCMK__CONFIG_H
	# define PCMK__CONFIG_H
	# include <config.h>
	# endif

	# include <portability.h>

	/* Our minimum glib dependency is 2.42. Define that as both the minimum and
	* maximum glib APIs that are allowed (i.e. APIs that were already deprecated
	* in 2.42, and APIs introduced after 2.42, cannot be used by Pacemaker code).
	*/
	#define GLIB_VERSION_MIN_REQUIRED GLIB_VERSION_2_42
	#define GLIB_VERSION_MAX_ALLOWED GLIB_VERSION_2_42

	# include <glib.h>
	# include <stdbool.h>
	# include <libxml/tree.h>

	/* Public API headers can guard including deprecated API headers with this
	* symbol, thus preventing internal code (which includes this header) from using
	* deprecated APIs, while still allowing external code to use them by default.
	*/
	#define PCMK_ALLOW_DEPRECATED 0

	# include <crm/lrmd.h>
	# include <crm/common/logging.h>
	# include <crm/common/logging_internal.h>
	# include <crm/common/ipc_internal.h>
	# include <crm/common/options_internal.h>
	# include <crm/common/output_internal.h>
	# include <crm/common/xml_internal.h>
	# include <crm/common/internal.h>
	# include <locale.h>
	# include <gettext.h>

	#define N_(String) (String)

	#ifdef ENABLE_NLS
	# define _(String) gettext(String)
	#else
	# define _(String) (String)
	#endif


	/*
	* XML element names used only by internal code
	*/

	// @COMPAT Deprecated since 2.1.7
	#define PCMK__XE_DIFF_ADDED "diff-added"

	// @COMPAT Deprecated since 2.1.7
	#define PCMK__XE_DIFF_REMOVED "diff-removed"

	/* @COMPAT Deprecated since 2.0.0; alias for <clone> with PCMK_META_PROMOTABLE
	* set to "true"
	*/
	#define PCMK__XE_PROMOTABLE_LEGACY "master"


	/*
	* XML attribute names used only by internal code
	*/

	#define PCMK__XA_ATTR_DAMPENING "attr_dampening"
	#define PCMK__XA_ATTR_FORCE "attrd_is_force_write"
	#define PCMK__XA_ATTR_INTERVAL "attr_clear_interval"
	#define PCMK__XA_ATTR_IS_PRIVATE "attr_is_private"
	#define PCMK__XA_ATTR_IS_REMOTE "attr_is_remote"
	#define PCMK__XA_ATTR_NAME "attr_name"
	#define PCMK__XA_ATTR_NODE_ID "attr_host_id"
	#define PCMK__XA_ATTR_NODE_NAME "attr_host"
	#define PCMK__XA_ATTR_OPERATION "attr_clear_operation"
	#define PCMK__XA_ATTR_PATTERN "attr_regex"
	#define PCMK__XA_ATTR_RESOURCE "attr_resource"
	#define PCMK__XA_ATTR_SECTION "attr_section"
	#define PCMK__XA_ATTR_SET "attr_set"
	#define PCMK__XA_ATTR_SET_TYPE "attr_set_type"
	#define PCMK__XA_ATTR_SYNC_POINT "attr_sync_point"
	#define PCMK__XA_ATTR_USER "attr_user"
	#define PCMK__XA_ATTR_UUID "attr_key"
	#define PCMK__XA_ATTR_VALUE "attr_value"
	#define PCMK__XA_ATTR_VERSION "attr_version"
	#define PCMK__XA_ATTR_WRITER "attr_writer"
	#define PCMK__XA_CALL_ID "call-id"
	#define PCMK__XA_CONFIG_ERRORS "config-errors"
	#define PCMK__XA_CONFIG_WARNINGS "config-warnings"
	#define PCMK__XA_CONFIRM "confirm"
	#define PCMK__XA_CONN_HOST "connection_host"
	#define PCMK__XA_CRMD "crmd"
	#define PCMK__XA_CRMD_STATE "crmd_state"
	#define PCMK__XA_CRM_HOST_TO "crm_host_to"
	#define PCMK__XA_CRM_LIMIT_MAX "crm-limit-max"
	#define PCMK__XA_CRM_LIMIT_MODE "crm-limit-mode"
	#define PCMK__XA_CRM_SUBSYSTEM "crm_subsystem"
	#define PCMK__XA_CRM_SYS_FROM "crm_sys_from"
	#define PCMK__XA_CRM_SYS_TO "crm_sys_to"
	#define PCMK__XA_CRM_TASK "crm_task"
	#define PCMK__XA_CRM_TGRAPH_IN "crm-tgraph-in"
	#define PCMK__XA_CRM_USER "crm_user"
	#define PCMK__XA_DC_LEAVING "dc-leaving"
	#define PCMK__XA_DIGEST "digest"
	#define PCMK__XA_ELECTION_AGE_SEC "election-age-sec"
	#define PCMK__XA_ELECTION_AGE_NANO_SEC "election-age-nano-sec"
	#define PCMK__XA_ELECTION_ID "election-id"
	#define PCMK__XA_ELECTION_OWNER "election-owner"
	#define PCMK__XA_EXPECTED "expected"
	#define PCMK__XA_FILE "file"
	#define PCMK__XA_GRAPH_ERRORS "graph-errors"
	#define PCMK__XA_GRAPH_WARNINGS "graph-warnings"
	#define PCMK__XA_IN_CCM "in_ccm"
	#define PCMK__XA_JOIN "join"
	#define PCMK__XA_JOIN_ID "join_id"
	#define PCMK__XA_LONG_ID "long-id"
	#define PCMK__XA_MODE "mode"
	#define PCMK__XA_NODE_START_STATE "node_start_state"
	#define PCMK__XA_OBJECT_TYPE "object_type"
	#define PCMK__XA_OPERATION_KEY "operation_key"
	#define PCMK__XA_OP_DIGEST "op-digest"
	+#define PCMK__XA_OP_FORCE_RESTART "op-force-restart"
	#define PCMK__XA_OP_STATUS "op-status"
	#define PCMK__XA_PACEMAKERD_STATE "pacemakerd_state"
	#define PCMK__XA_PRIORITY "priority"
	#define PCMK__XA_RC_CODE "rc-code"
	#define PCMK__XA_REAP "reap"

	/* Actions to be executed on Pacemaker Remote nodes are routed through the
	* controller on the cluster node hosting the remote connection. That cluster
	* node is considered the router node for the action.
	*/
	#define PCMK__XA_ROUTER_NODE "router_node"

	#define PCMK__XA_RSC_ID "rsc-id"
	#define PCMK__XA_SCHEMA "schema"
	#define PCMK__XA_SCHEMAS "schemas"
	#define PCMK__XA_SRC "src"
	#define PCMK__XA_SUBT "subt" // subtype
	#define PCMK__XA_T "t" // type
	#define PCMK__XA_TASK "task"
	#define PCMK__XA_TRANSITION_KEY "transition-key"
	#define PCMK__XA_TRANSITION_MAGIC "transition-magic"
	#define PCMK__XA_UPTIME "uptime"

	// @COMPAT Deprecated since 2.1.5
	#define PCMK__XA_FIRST_INSTANCE "first-instance"

	// @COMPAT Deprecated since 2.1.6
	#define PCMK__XA_REPLACE "replace"

	// @COMPAT Deprecated since 2.1.5
	#define PCMK__XA_RSC_INSTANCE "rsc-instance"

	// @COMPAT Deprecated since 2.1.5
	#define PCMK__XA_THEN_INSTANCE "then-instance"

	// @COMPAT Deprecated since 2.1.5
	#define PCMK__XA_WITH_RSC_INSTANCE "with-rsc-instance"


	/*
	* IPC service names that are only used internally
	*/

	# define PCMK__SERVER_BASED_RO "cib_ro"
	# define PCMK__SERVER_BASED_RW "cib_rw"
	# define PCMK__SERVER_BASED_SHM "cib_shm"

	/*
	* IPC commands that can be sent to Pacemaker daemons
	*/

	#define PCMK__ATTRD_CMD_PEER_REMOVE "peer-remove"
	#define PCMK__ATTRD_CMD_UPDATE "update"
	#define PCMK__ATTRD_CMD_UPDATE_BOTH "update-both"
	#define PCMK__ATTRD_CMD_UPDATE_DELAY "update-delay"
	#define PCMK__ATTRD_CMD_QUERY "query"
	#define PCMK__ATTRD_CMD_REFRESH "refresh"
	#define PCMK__ATTRD_CMD_FLUSH "flush"
	#define PCMK__ATTRD_CMD_SYNC "sync"
	#define PCMK__ATTRD_CMD_SYNC_RESPONSE "sync-response"
	#define PCMK__ATTRD_CMD_CLEAR_FAILURE "clear-failure"
	#define PCMK__ATTRD_CMD_CONFIRM "confirm"

	#define PCMK__CONTROLD_CMD_NODES "list-nodes"

	#endif /* CRM_INTERNAL__H */
	diff --git a/lib/pengine/pe_digest.c b/lib/pengine/pe_digest.c
	index e53136b829..a702cc7434 100644
	--- a/lib/pengine/pe_digest.c
	+++ b/lib/pengine/pe_digest.c
	@@ -1,609 +1,609 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <glib.h>
	#include <stdbool.h>

	#include <crm/crm.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>
	#include <crm/common/xml_internal.h>
	#include <crm/pengine/internal.h>
	#include "pe_status_private.h"

	extern bool pcmk__is_daemon;

	/*!
	* \internal
	* \brief Free an operation digest cache entry
	*
	* \param[in,out] ptr Pointer to cache entry to free
	*
	* \note The argument is a gpointer so this can be used as a hash table
	* free function.
	*/
	void
	pe__free_digests(gpointer ptr)
	{
	pcmk__op_digest_t *data = ptr;

	if (data != NULL) {
	free_xml(data->params_all);
	free_xml(data->params_secure);
	free_xml(data->params_restart);

	free(data->digest_all_calc);
	free(data->digest_restart_calc);
	free(data->digest_secure_calc);

	free(data);
	}
	}

	// Return true if XML attribute name is not substring of a given string
	static bool
	attr_not_in_string(xmlAttrPtr a, void *user_data)
	{
	bool filter = false;
	char name = crm_strdup_printf(" %s ", (const char ) a->name);

	if (strstr((const char *) user_data, name) == NULL) {
	crm_trace("Filtering %s (not found in '%s')",
	(const char ) a->name, (const char ) user_data);
	filter = true;
	}
	free(name);
	return filter;
	}

	// Return true if XML attribute name is substring of a given string
	static bool
	attr_in_string(xmlAttrPtr a, void *user_data)
	{
	bool filter = false;
	char name = crm_strdup_printf(" %s ", (const char ) a->name);

	if (strstr((const char *) user_data, name) != NULL) {
	crm_trace("Filtering %s (found in '%s')",
	(const char ) a->name, (const char ) user_data);
	filter = true;
	}
	free(name);
	return filter;
	}

	/*!
	* \internal
	* \brief Add digest of all parameters to a digest cache entry
	*
	* \param[out] data Digest cache entry to modify
	* \param[in,out] rsc Resource that action was for
	* \param[in] node Node action was performed on
	* \param[in] params Resource parameters evaluated for node
	* \param[in] task Name of action performed
	* \param[in,out] interval_ms Action's interval (will be reset if in overrides)
	* \param[in] xml_op Unused
	* \param[in] op_version CRM feature set to use for digest calculation
	* \param[in] overrides Key/value table to override resource parameters
	* \param[in,out] scheduler Scheduler data
	*/
	static void
	calculate_main_digest(pcmk__op_digest_t data, pcmk_resource_t rsc,
	const pcmk_node_t node, GHashTable params,
	const char task, guint interval_ms,
	const xmlNode xml_op, const char op_version,
	GHashTable overrides, pcmk_scheduler_t scheduler)
	{
	xmlNode *action_config = NULL;

	data->params_all = create_xml_node(NULL, XML_TAG_PARAMS);

	/* REMOTE_CONTAINER_HACK: Allow Pacemaker Remote nodes to run containers
	* that themselves are Pacemaker Remote nodes
	*/
	(void) pe__add_bundle_remote_name(rsc, scheduler, data->params_all,
	PCMK_REMOTE_RA_ADDR);

	if (overrides != NULL) {
	// If interval was overridden, reset it
	const char *meta_name = CRM_META "_" PCMK_META_INTERVAL;
	const char *interval_s = g_hash_table_lookup(overrides, meta_name);

	if (interval_s != NULL) {
	long long value_ll;

	if ((pcmk__scan_ll(interval_s, &value_ll, 0LL) == pcmk_rc_ok)
	&& (value_ll >= 0) && (value_ll <= G_MAXUINT)) {
	*interval_ms = (guint) value_ll;
	}
	}

	// Add overrides to list of all parameters
	g_hash_table_foreach(overrides, hash2field, data->params_all);
	}

	// Add provided instance parameters
	g_hash_table_foreach(params, hash2field, data->params_all);

	// Find action configuration XML in CIB
	action_config = pcmk__find_action_config(rsc, task, *interval_ms, true);

	/* Add action-specific resource instance attributes to the digest list.
	*
	* If this is a one-time action with action-specific instance attributes,
	* enforce a restart instead of reload-agent in case the main digest doesn't
	* match, even if the restart digest does. This ensures any changes of the
	* action-specific parameters get applied for this specific action, and
	* digests calculated for the resulting history will be correct. Default the
	* result to RSC_DIGEST_RESTART for the case where the main digest doesn't
	* match.
	*/
	params = pcmk__unpack_action_rsc_params(action_config, node->details->attrs,
	scheduler);
	if ((*interval_ms == 0) && (g_hash_table_size(params) > 0)) {
	data->rc = pcmk__digest_restart;
	}
	g_hash_table_foreach(params, hash2field, data->params_all);
	g_hash_table_destroy(params);

	// Add action meta-attributes
	params = pcmk__unpack_action_meta(rsc, node, task, *interval_ms,
	action_config);
	g_hash_table_foreach(params, hash2metafield, data->params_all);
	g_hash_table_destroy(params);

	pcmk__filter_op_for_digest(data->params_all);

	data->digest_all_calc = calculate_operation_digest(data->params_all,
	op_version);
	}

	// Return true if XML attribute name is a Pacemaker-defined fencing parameter
	static bool
	is_fence_param(xmlAttrPtr attr, void *user_data)
	{
	return pcmk_stonith_param((const char *) attr->name);
	}

	/*!
	* \internal
	* \brief Add secure digest to a digest cache entry
	*
	* \param[out] data Digest cache entry to modify
	* \param[in] rsc Resource that action was for
	* \param[in] params Resource parameters evaluated for node
	* \param[in] xml_op XML of operation in CIB status (if available)
	* \param[in] op_version CRM feature set to use for digest calculation
	* \param[in] overrides Key/value hash table to override resource parameters
	*/
	static void
	calculate_secure_digest(pcmk__op_digest_t data, const pcmk_resource_t rsc,
	GHashTable params, const xmlNode xml_op,
	const char op_version, GHashTable overrides)
	{
	const char *class = crm_element_value(rsc->xml, PCMK_XA_CLASS);
	const char *secure_list = NULL;
	bool old_version = (compare_version(op_version, "3.16.0") < 0);

	if (xml_op == NULL) {
	secure_list = " passwd password user ";
	} else {
	secure_list = crm_element_value(xml_op, XML_LRM_ATTR_OP_SECURE);
	}

	if (old_version) {
	data->params_secure = create_xml_node(NULL, XML_TAG_PARAMS);
	if (overrides != NULL) {
	g_hash_table_foreach(overrides, hash2field, data->params_secure);
	}

	g_hash_table_foreach(params, hash2field, data->params_secure);

	} else {
	// Start with a copy of all parameters
	data->params_secure = copy_xml(data->params_all);
	}

	if (secure_list != NULL) {
	pcmk__xe_remove_matching_attrs(data->params_secure, attr_in_string,
	(void *) secure_list);
	}
	if (old_version
	&& pcmk_is_set(pcmk_get_ra_caps(class),
	pcmk_ra_cap_fence_params)) {
	/* For stonith resources, Pacemaker adds special parameters,
	* but these are not listed in fence agent meta-data, so with older
	* versions of DC, the controller will not hash them. That means we have
	* to filter them out before calculating our hash for comparison.
	*/
	pcmk__xe_remove_matching_attrs(data->params_secure, is_fence_param,
	NULL);
	}
	pcmk__filter_op_for_digest(data->params_secure);

	/* CRM_meta_timeout should be part of a digest for recurring operations.
	* However, with older versions of DC, the controller does not add timeout
	* to secure digests, because it only includes parameters declared by the
	* resource agent.
	* Remove any timeout that made it this far, to match.
	*/
	if (old_version) {
	xml_remove_prop(data->params_secure, CRM_META "_" PCMK_META_TIMEOUT);
	}

	data->digest_secure_calc = calculate_operation_digest(data->params_secure,
	op_version);
	}

	/*!
	* \internal
	* \brief Add restart digest to a digest cache entry
	*
	* \param[out] data Digest cache entry to modify
	* \param[in] xml_op XML of operation in CIB status (if available)
	* \param[in] op_version CRM feature set to use for digest calculation
	*
	* \note This function doesn't need to handle overrides because it starts with
	* data->params_all, which already has overrides applied.
	*/
	static void
	calculate_restart_digest(pcmk__op_digest_t data, const xmlNode xml_op,
	const char *op_version)
	{
	const char *value = NULL;

	// We must have XML of resource operation history
	if (xml_op == NULL) {
	return;
	}

	// And the history must have a restart digest to compare against
	if (crm_element_value(xml_op, XML_LRM_ATTR_RESTART_DIGEST) == NULL) {
	return;
	}

	// Start with a copy of all parameters
	data->params_restart = copy_xml(data->params_all);

	// Then filter out reloadable parameters, if any
	- value = crm_element_value(xml_op, XML_LRM_ATTR_OP_RESTART);
	+ value = crm_element_value(xml_op, PCMK__XA_OP_FORCE_RESTART);
	if (value != NULL) {
	pcmk__xe_remove_matching_attrs(data->params_restart, attr_not_in_string,
	(void *) value);
	}

	value = crm_element_value(xml_op, PCMK_XA_CRM_FEATURE_SET);
	data->digest_restart_calc = calculate_operation_digest(data->params_restart,
	value);
	}

	/*!
	* \internal
	* \brief Create a new digest cache entry with calculated digests
	*
	* \param[in,out] rsc Resource that action was for
	* \param[in] task Name of action performed
	* \param[in,out] interval_ms Action's interval (will be reset if in overrides)
	* \param[in] node Node action was performed on
	* \param[in] xml_op XML of operation in CIB status (if available)
	* \param[in] overrides Key/value table to override resource parameters
	* \param[in] calc_secure Whether to calculate secure digest
	* \param[in,out] scheduler Scheduler data
	*
	* \return Pointer to new digest cache entry (or NULL on memory error)
	* \note It is the caller's responsibility to free the result using
	* pe__free_digests().
	*/
	pcmk__op_digest_t *
	pe__calculate_digests(pcmk_resource_t rsc, const char task,
	guint interval_ms, const pcmk_node_t node,
	const xmlNode xml_op, GHashTable overrides,
	bool calc_secure, pcmk_scheduler_t *scheduler)
	{
	pcmk__op_digest_t *data = calloc(1, sizeof(pcmk__op_digest_t));
	const char *op_version = NULL;
	GHashTable *params = NULL;

	if (data == NULL) {
	pcmk__sched_err("Could not allocate memory for operation digest");
	return NULL;
	}

	data->rc = pcmk__digest_match;

	if (xml_op != NULL) {
	op_version = crm_element_value(xml_op, PCMK_XA_CRM_FEATURE_SET);
	}

	if (op_version == NULL && scheduler != NULL && scheduler->input != NULL) {
	op_version = crm_element_value(scheduler->input,
	PCMK_XA_CRM_FEATURE_SET);
	}

	if (op_version == NULL) {
	op_version = CRM_FEATURE_SET;
	}

	params = pe_rsc_params(rsc, node, scheduler);
	calculate_main_digest(data, rsc, node, params, task, interval_ms, xml_op,
	op_version, overrides, scheduler);
	if (calc_secure) {
	calculate_secure_digest(data, rsc, params, xml_op, op_version,
	overrides);
	}
	calculate_restart_digest(data, xml_op, op_version);
	return data;
	}

	/*!
	* \internal
	* \brief Calculate action digests and store in node's digest cache
	*
	* \param[in,out] rsc Resource that action was for
	* \param[in] task Name of action performed
	* \param[in] interval_ms Action's interval
	* \param[in,out] node Node action was performed on
	* \param[in] xml_op XML of operation in CIB status (if available)
	* \param[in] calc_secure Whether to calculate secure digest
	* \param[in,out] scheduler Scheduler data
	*
	* \return Pointer to node's digest cache entry
	*/
	static pcmk__op_digest_t *
	rsc_action_digest(pcmk_resource_t rsc, const char task, guint interval_ms,
	pcmk_node_t node, const xmlNode xml_op,
	bool calc_secure, pcmk_scheduler_t *scheduler)
	{
	pcmk__op_digest_t *data = NULL;
	char *key = pcmk__op_key(rsc->id, task, interval_ms);

	data = g_hash_table_lookup(node->details->digest_cache, key);
	if (data == NULL) {
	data = pe__calculate_digests(rsc, task, &interval_ms, node, xml_op,
	NULL, calc_secure, scheduler);
	CRM_ASSERT(data != NULL);
	g_hash_table_insert(node->details->digest_cache, strdup(key), data);
	}
	free(key);
	return data;
	}

	/*!
	* \internal
	* \brief Calculate operation digests and compare against an XML history entry
	*
	* \param[in,out] rsc Resource to check
	* \param[in] xml_op Resource history XML
	* \param[in,out] node Node to use for digest calculation
	* \param[in,out] scheduler Scheduler data
	*
	* \return Pointer to node's digest cache entry, with comparison result set
	*/
	pcmk__op_digest_t *
	rsc_action_digest_cmp(pcmk_resource_t rsc, const xmlNode xml_op,
	pcmk_node_t node, pcmk_scheduler_t scheduler)
	{
	pcmk__op_digest_t *data = NULL;
	guint interval_ms = 0;

	const char *op_version;
	const char *task = crm_element_value(xml_op, PCMK_XA_OPERATION);
	const char *digest_all;
	const char *digest_restart;

	CRM_ASSERT(node != NULL);

	op_version = crm_element_value(xml_op, PCMK_XA_CRM_FEATURE_SET);
	digest_all = crm_element_value(xml_op, PCMK__XA_OP_DIGEST);
	digest_restart = crm_element_value(xml_op, XML_LRM_ATTR_RESTART_DIGEST);

	crm_element_value_ms(xml_op, PCMK_META_INTERVAL, &interval_ms);
	data = rsc_action_digest(rsc, task, interval_ms, node, xml_op,
	pcmk_is_set(scheduler->flags,
	pcmk_sched_sanitized),
	scheduler);

	if (digest_restart && data->digest_restart_calc && strcmp(data->digest_restart_calc, digest_restart) != 0) {
	pcmk__rsc_info(rsc,
	"Parameters to %ums-interval %s action for %s on %s "
	"changed: hash was %s vs. now %s (restart:%s) %s",
	interval_ms, task, rsc->id, pe__node_name(node),
	pcmk__s(digest_restart, "missing"),
	data->digest_restart_calc, op_version,
	crm_element_value(xml_op, PCMK__XA_TRANSITION_MAGIC));
	data->rc = pcmk__digest_restart;

	} else if (digest_all == NULL) {
	/* it is unknown what the previous op digest was */
	data->rc = pcmk__digest_unknown;

	} else if (strcmp(digest_all, data->digest_all_calc) != 0) {
	/* Given a non-recurring operation with extra parameters configured,
	* in case that the main digest doesn't match, even if the restart
	* digest matches, enforce a restart rather than a reload-agent anyway.
	* So that it ensures any changes of the extra parameters get applied
	* for this specific operation, and the digests calculated for the
	* resulting lrm_rsc_op will be correct.
	* Preserve the implied rc pcmk__digest_restart for the case that the
	* main digest doesn't match.
	*/
	if ((interval_ms == 0) && (data->rc == pcmk__digest_restart)) {
	pcmk__rsc_info(rsc,
	"Parameters containing extra ones to %ums-interval"
	" %s action for %s on %s "
	"changed: hash was %s vs. now %s (restart:%s) %s",
	interval_ms, task, rsc->id, pe__node_name(node),
	pcmk__s(digest_all, "missing"),
	data->digest_all_calc, op_version,
	crm_element_value(xml_op,
	PCMK__XA_TRANSITION_MAGIC));

	} else {
	pcmk__rsc_info(rsc,
	"Parameters to %ums-interval %s action for %s on %s "
	"changed: hash was %s vs. now %s (%s:%s) %s",
	interval_ms, task, rsc->id, pe__node_name(node),
	pcmk__s(digest_all, "missing"),
	data->digest_all_calc,
	(interval_ms > 0)? "reschedule" : "reload",
	op_version,
	crm_element_value(xml_op,
	PCMK__XA_TRANSITION_MAGIC));
	data->rc = pcmk__digest_mismatch;
	}

	} else {
	data->rc = pcmk__digest_match;
	}
	return data;
	}

	/*!
	* \internal
	* \brief Create an unfencing summary for use in special node attribute
	*
	* Create a string combining a fence device's resource ID, agent type, and
	* parameter digest (whether for all parameters or just non-private parameters).
	* This can be stored in a special node attribute, allowing us to detect changes
	* in either the agent type or parameters, to know whether unfencing must be
	* redone or can be safely skipped when the device's history is cleaned.
	*
	* \param[in] rsc_id Fence device resource ID
	* \param[in] agent_type Fence device agent
	* \param[in] param_digest Fence device parameter digest
	*
	* \return Newly allocated string with unfencing digest
	* \note The caller is responsible for freeing the result.
	*/
	static inline char *
	create_unfencing_summary(const char rsc_id, const char agent_type,
	const char *param_digest)
	{
	return crm_strdup_printf("%s:%s:%s", rsc_id, agent_type, param_digest);
	}

	/*!
	* \internal
	* \brief Check whether a node can skip unfencing
	*
	* Check whether a fence device's current definition matches a node's
	* stored summary of when it was last unfenced by the device.
	*
	* \param[in] rsc_id Fence device's resource ID
	* \param[in] agent Fence device's agent type
	* \param[in] digest_calc Fence device's current parameter digest
	* \param[in] node_summary Value of node's special unfencing node attribute
	* (a comma-separated list of unfencing summaries for
	* all devices that have unfenced this node)
	*
	* \return TRUE if digest matches, FALSE otherwise
	*/
	static bool
	unfencing_digest_matches(const char rsc_id, const char agent,
	const char digest_calc, const char node_summary)
	{
	bool matches = FALSE;

	if (rsc_id && agent && digest_calc && node_summary) {
	char *search_secure = create_unfencing_summary(rsc_id, agent,
	digest_calc);

	/* The digest was calculated including the device ID and agent,
	* so there is no risk of collision using strstr().
	*/
	matches = (strstr(node_summary, search_secure) != NULL);
	crm_trace("Calculated unfencing digest '%s' %sfound in '%s'",
	search_secure, matches? "" : "not ", node_summary);
	free(search_secure);
	}
	return matches;
	}

	/* Magic string to use as action name for digest cache entries used for
	* unfencing checks. This is not a real action name (i.e. "on"), so
	* pcmk__check_action_config() won't confuse these entries with real actions.
	*/
	#define STONITH_DIGEST_TASK "stonith-on"

	/*!
	* \internal
	* \brief Calculate fence device digests and digest comparison result
	*
	* \param[in,out] rsc Fence device resource
	* \param[in] agent Fence device's agent type
	* \param[in,out] node Node with digest cache to use
	* \param[in,out] scheduler Scheduler data
	*
	* \return Node's digest cache entry
	*/
	pcmk__op_digest_t *
	pe__compare_fencing_digest(pcmk_resource_t rsc, const char agent,
	pcmk_node_t node, pcmk_scheduler_t scheduler)
	{
	const char *node_summary = NULL;

	// Calculate device's current parameter digests
	pcmk__op_digest_t *data = rsc_action_digest(rsc, STONITH_DIGEST_TASK, 0U,
	node, NULL, TRUE, scheduler);

	// Check whether node has special unfencing summary node attribute
	node_summary = pe_node_attribute_raw(node, CRM_ATTR_DIGESTS_ALL);
	if (node_summary == NULL) {
	data->rc = pcmk__digest_unknown;
	return data;
	}

	// Check whether full parameter digest matches
	if (unfencing_digest_matches(rsc->id, agent, data->digest_all_calc,
	node_summary)) {
	data->rc = pcmk__digest_match;
	return data;
	}

	// Check whether secure parameter digest matches
	node_summary = pe_node_attribute_raw(node, CRM_ATTR_DIGESTS_SECURE);
	if (unfencing_digest_matches(rsc->id, agent, data->digest_secure_calc,
	node_summary)) {
	data->rc = pcmk__digest_match;
	if (!pcmk__is_daemon && scheduler->priv != NULL) {
	pcmk__output_t *out = scheduler->priv;
	out->info(out, "Only 'private' parameters to %s "
	"for unfencing %s changed", rsc->id,
	pe__node_name(node));
	}
	return data;
	}

	// Parameters don't match
	data->rc = pcmk__digest_mismatch;
	if (pcmk_is_set(scheduler->flags, pcmk_sched_sanitized)
	&& (data->digest_secure_calc != NULL)) {

	if (scheduler->priv != NULL) {
	pcmk__output_t *out = scheduler->priv;
	char *digest = create_unfencing_summary(rsc->id, agent,
	data->digest_secure_calc);

	out->info(out, "Parameters to %s for unfencing "
	"%s changed, try '%s'", rsc->id,
	pe__node_name(node), digest);
	free(digest);
	} else if (!pcmk__is_daemon) {
	char *digest = create_unfencing_summary(rsc->id, agent,
	data->digest_secure_calc);

	printf("Parameters to %s for unfencing %s changed, try '%s'\n",
	rsc->id, pe__node_name(node), digest);
	free(digest);
	}
	}
	return data;
	}

File Metadata

Mime Type: text/x-diff
Expires: Thu, Jun 26, 7:47 PM (19 h, 12 m)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 1959600
Default Alt Text: (68 KB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions