No OneTemporary
Actions

Size

122 KB

Referenced Files

None

Subscribers

None

View Options

	diff --git a/daemons/based/based_callbacks.c b/daemons/based/based_callbacks.c
	index df684bb967..437c100266 100644
	--- a/daemons/based/based_callbacks.c
	+++ b/daemons/based/based_callbacks.c
	@@ -1,1633 +1,1652 @@
	/*
	* Copyright 2004-2022 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <sys/param.h>
	#include <stdio.h>
	#include <sys/types.h>
	#include <unistd.h>

	#include <stdlib.h>
	#include <stdint.h> // uint32_t, uint64_t, UINT64_C()
	#include <errno.h>
	#include <fcntl.h>
	#include <inttypes.h> // PRIu64

	#include <crm/crm.h>
	#include <crm/cib.h>
	#include <crm/msg_xml.h>
	#include <crm/cluster/internal.h>

	#include <crm/common/xml.h>
	#include <crm/common/remote_internal.h>

	#include <pacemaker-based.h>

	#define EXIT_ESCALATION_MS 10000

	static unsigned long cib_local_bcast_num = 0;

	typedef struct cib_local_notify_s {
	xmlNode *notify_src;
	char *client_id;
	gboolean from_peer;
	gboolean sync_reply;
	} cib_local_notify_t;

	int next_client_id = 0;

	gboolean legacy_mode = FALSE;

	qb_ipcs_service_t *ipcs_ro = NULL;
	qb_ipcs_service_t *ipcs_rw = NULL;
	qb_ipcs_service_t *ipcs_shm = NULL;

	void send_cib_replace(const xmlNode * sync_request, const char *host);
	static void cib_process_request(xmlNode *request, gboolean privileged,
	const pcmk__client_t *cib_client);


	static int cib_process_command(xmlNode request, xmlNode *reply,
	xmlNode **cib_diff, gboolean privileged);

	gboolean cib_common_callback(qb_ipcs_connection_t * c, void *data, size_t size,
	gboolean privileged);

	gboolean cib_legacy_mode(void)
	{
	return legacy_mode;
	}


	static int32_t
	cib_ipc_accept(qb_ipcs_connection_t * c, uid_t uid, gid_t gid)
	{
	if (cib_shutdown_flag) {
	crm_info("Ignoring new IPC client [%d] during shutdown",
	pcmk__client_pid(c));
	return -EPERM;
	}

	if (pcmk__new_client(c, uid, gid) == NULL) {
	return -EIO;
	}
	return 0;
	}

	static int32_t
	cib_ipc_dispatch_rw(qb_ipcs_connection_t * c, void *data, size_t size)
	{
	pcmk__client_t *client = pcmk__find_client(c);

	crm_trace("%p message from %s", c, client->id);
	return cib_common_callback(c, data, size, TRUE);
	}

	static int32_t
	cib_ipc_dispatch_ro(qb_ipcs_connection_t * c, void *data, size_t size)
	{
	pcmk__client_t *client = pcmk__find_client(c);

	crm_trace("%p message from %s", c, client->id);
	return cib_common_callback(c, data, size, FALSE);
	}

	/* Error code means? */
	static int32_t
	cib_ipc_closed(qb_ipcs_connection_t * c)
	{
	pcmk__client_t *client = pcmk__find_client(c);

	if (client == NULL) {
	return 0;
	}
	crm_trace("Connection %p", c);
	pcmk__free_client(client);
	return 0;
	}

	static void
	cib_ipc_destroy(qb_ipcs_connection_t * c)
	{
	crm_trace("Connection %p", c);
	cib_ipc_closed(c);
	if (cib_shutdown_flag) {
	cib_shutdown(0);
	}
	}

	struct qb_ipcs_service_handlers ipc_ro_callbacks = {
	.connection_accept = cib_ipc_accept,
	.connection_created = NULL,
	.msg_process = cib_ipc_dispatch_ro,
	.connection_closed = cib_ipc_closed,
	.connection_destroyed = cib_ipc_destroy
	};

	struct qb_ipcs_service_handlers ipc_rw_callbacks = {
	.connection_accept = cib_ipc_accept,
	.connection_created = NULL,
	.msg_process = cib_ipc_dispatch_rw,
	.connection_closed = cib_ipc_closed,
	.connection_destroyed = cib_ipc_destroy
	};

	void
	cib_common_callback_worker(uint32_t id, uint32_t flags, xmlNode * op_request,
	pcmk__client_t *cib_client, gboolean privileged)
	{
	const char *op = crm_element_value(op_request, F_CIB_OPERATION);

	if (pcmk__str_eq(op, CRM_OP_REGISTER, pcmk__str_none)) {
	if (flags & crm_ipc_client_response) {
	xmlNode *ack = create_xml_node(NULL, __func__);

	crm_xml_add(ack, F_CIB_OPERATION, CRM_OP_REGISTER);
	crm_xml_add(ack, F_CIB_CLIENTID, cib_client->id);
	pcmk__ipc_send_xml(cib_client, id, ack, flags);
	cib_client->request_id = 0;
	free_xml(ack);
	}
	return;

	} else if (pcmk__str_eq(op, T_CIB_NOTIFY, pcmk__str_none)) {
	/* Update the notify filters for this client */
	int on_off = 0;
	crm_exit_t status = CRM_EX_OK;
	uint64_t bit = UINT64_C(0);
	const char *type = crm_element_value(op_request, F_CIB_NOTIFY_TYPE);

	crm_element_value_int(op_request, F_CIB_NOTIFY_ACTIVATE, &on_off);

	crm_debug("Setting %s callbacks %s for client %s",
	type, (on_off? "on" : "off"), pcmk__client_name(cib_client));

	if (pcmk__str_eq(type, T_CIB_POST_NOTIFY, pcmk__str_casei)) {
	bit = cib_notify_post;

	} else if (pcmk__str_eq(type, T_CIB_PRE_NOTIFY, pcmk__str_casei)) {
	bit = cib_notify_pre;

	} else if (pcmk__str_eq(type, T_CIB_UPDATE_CONFIRM, pcmk__str_casei)) {
	bit = cib_notify_confirm;

	} else if (pcmk__str_eq(type, T_CIB_DIFF_NOTIFY, pcmk__str_casei)) {
	bit = cib_notify_diff;

	} else if (pcmk__str_eq(type, T_CIB_REPLACE_NOTIFY, pcmk__str_casei)) {
	bit = cib_notify_replace;

	} else {
	status = CRM_EX_INVALID_PARAM;
	}

	if (bit != 0) {
	if (on_off) {
	pcmk__set_client_flags(cib_client, bit);
	} else {
	pcmk__clear_client_flags(cib_client, bit);
	}
	}

	pcmk__ipc_send_ack(cib_client, id, flags, "ack", NULL, status);
	return;
	}

	cib_process_request(op_request, privileged, cib_client);
	}

	int32_t
	cib_common_callback(qb_ipcs_connection_t * c, void *data, size_t size, gboolean privileged)
	{
	uint32_t id = 0;
	uint32_t flags = 0;
	int call_options = 0;
	pcmk__client_t *cib_client = pcmk__find_client(c);
	xmlNode *op_request = pcmk__client_data2xml(cib_client, data, &id, &flags);

	if (op_request) {
	crm_element_value_int(op_request, F_CIB_CALLOPTS, &call_options);
	}

	if (op_request == NULL) {
	crm_trace("Invalid message from %p", c);
	pcmk__ipc_send_ack(cib_client, id, flags, "nack", NULL, CRM_EX_PROTOCOL);
	return 0;

	} else if(cib_client == NULL) {
	crm_trace("Invalid client %p", c);
	return 0;
	}

	if (pcmk_is_set(call_options, cib_sync_call)) {
	CRM_LOG_ASSERT(flags & crm_ipc_client_response);
	CRM_LOG_ASSERT(cib_client->request_id == 0); /* This means the client has two synchronous events in-flight */
	cib_client->request_id = id; /* Reply only to the last one */
	}

	if (cib_client->name == NULL) {
	const char *value = crm_element_value(op_request, F_CIB_CLIENTNAME);

	if (value == NULL) {
	cib_client->name = pcmk__itoa(cib_client->pid);
	} else {
	cib_client->name = strdup(value);
	if (crm_is_daemon_name(value)) {
	pcmk__set_client_flags(cib_client, cib_is_daemon);
	}
	}
	}

	/* Allow cluster daemons more leeway before being evicted */
	if (pcmk_is_set(cib_client->flags, cib_is_daemon)) {
	const char *qmax = cib_config_lookup("cluster-ipc-limit");

	if (pcmk__set_client_queue_max(cib_client, qmax)) {
	crm_trace("IPC threshold for client %s[%u] is now %u",
	pcmk__client_name(cib_client), cib_client->pid,
	cib_client->queue_max);
	}
	}

	crm_xml_add(op_request, F_CIB_CLIENTID, cib_client->id);
	crm_xml_add(op_request, F_CIB_CLIENTNAME, cib_client->name);

	CRM_LOG_ASSERT(cib_client->user != NULL);
	pcmk__update_acl_user(op_request, F_CIB_USER, cib_client->user);

	cib_common_callback_worker(id, flags, op_request, cib_client, privileged);
	free_xml(op_request);

	return 0;
	}

	static uint64_t ping_seq = 0;
	static char *ping_digest = NULL;
	static bool ping_modified_since = FALSE;
	int sync_our_cib(xmlNode * request, gboolean all);

	static gboolean
	cib_digester_cb(gpointer data)
	{
	if (based_is_primary) {
	char buffer[32];
	xmlNode *ping = create_xml_node(NULL, "ping");

	ping_seq++;
	free(ping_digest);
	ping_digest = NULL;
	ping_modified_since = FALSE;
	snprintf(buffer, 32, "%" PRIu64, ping_seq);
	crm_trace("Requesting peer digests (%s)", buffer);

	crm_xml_add(ping, F_TYPE, "cib");
	crm_xml_add(ping, F_CIB_OPERATION, CRM_OP_PING);
	crm_xml_add(ping, F_CIB_PING_ID, buffer);

	crm_xml_add(ping, XML_ATTR_CRM_VERSION, CRM_FEATURE_SET);
	send_cluster_message(NULL, crm_msg_cib, ping, TRUE);

	free_xml(ping);
	}
	return FALSE;
	}

	static void
	process_ping_reply(xmlNode *reply)
	{
	uint64_t seq = 0;
	const char *host = crm_element_value(reply, F_ORIG);

	xmlNode *pong = get_message_xml(reply, F_CIB_CALLDATA);
	const char *seq_s = crm_element_value(pong, F_CIB_PING_ID);
	const char *digest = crm_element_value(pong, XML_ATTR_DIGEST);

	if (seq_s == NULL) {
	crm_debug("Ignoring ping reply with no " F_CIB_PING_ID);
	return;

	} else {
	long long seq_ll;

	if (pcmk__scan_ll(seq_s, &seq_ll, 0LL) != pcmk_rc_ok) {
	return;
	}
	seq = (uint64_t) seq_ll;
	}

	if(digest == NULL) {
	crm_trace("Ignoring ping reply %s from %s with no digest", seq_s, host);

	} else if(seq != ping_seq) {
	crm_trace("Ignoring out of sequence ping reply %s from %s", seq_s, host);

	} else if(ping_modified_since) {
	crm_trace("Ignoring ping reply %s from %s: cib updated since", seq_s, host);

	} else {
	const char *version = crm_element_value(pong, XML_ATTR_CRM_VERSION);

	if(ping_digest == NULL) {
	crm_trace("Calculating new digest");
	ping_digest = calculate_xml_versioned_digest(the_cib, FALSE, TRUE, version);
	}

	crm_trace("Processing ping reply %s from %s (%s)", seq_s, host, digest);
	if (!pcmk__str_eq(ping_digest, digest, pcmk__str_casei)) {
	xmlNode *remote_cib = get_message_xml(pong, F_CIB_CALLDATA);

	crm_notice("Local CIB %s.%s.%s.%s differs from %s: %s.%s.%s.%s %p",
	crm_element_value(the_cib, XML_ATTR_GENERATION_ADMIN),
	crm_element_value(the_cib, XML_ATTR_GENERATION),
	crm_element_value(the_cib, XML_ATTR_NUMUPDATES),
	ping_digest, host,
	remote_cib?crm_element_value(remote_cib, XML_ATTR_GENERATION_ADMIN):"_",
	remote_cib?crm_element_value(remote_cib, XML_ATTR_GENERATION):"_",
	remote_cib?crm_element_value(remote_cib, XML_ATTR_NUMUPDATES):"_",
	digest, remote_cib);

	if(remote_cib && remote_cib->children) {
	/* Additional debug */
	xml_calculate_changes(the_cib, remote_cib);
	xml_log_changes(LOG_INFO, __func__, remote_cib);
	crm_trace("End of differences");
	}

	free_xml(remote_cib);
	sync_our_cib(reply, FALSE);
	}
	}
	}

	static void
	do_local_notify(xmlNode * notify_src, const char *client_id,
	gboolean sync_reply, gboolean from_peer)
	{
	int rid = 0;
	int call_id = 0;
	pcmk__client_t *client_obj = NULL;

	CRM_ASSERT(notify_src && client_id);

	crm_element_value_int(notify_src, F_CIB_CALLID, &call_id);

	client_obj = pcmk__find_client_by_id(client_id);
	if (client_obj == NULL) {
	crm_debug("Could not send response %d: client %s not found",
	call_id, client_id);
	return;
	}

	if (sync_reply) {
	if (client_obj->ipcs) {
	CRM_LOG_ASSERT(client_obj->request_id);

	rid = client_obj->request_id;
	client_obj->request_id = 0;

	crm_trace("Sending response %d to client %s%s",
	rid, pcmk__client_name(client_obj),
	(from_peer? " (originator of delegated request)" : ""));
	} else {
	crm_trace("Sending response (call %d) to client %s%s",
	call_id, pcmk__client_name(client_obj),
	(from_peer? " (originator of delegated request)" : ""));
	}

	} else {
	crm_trace("Sending event %d to client %s%s",
	call_id, pcmk__client_name(client_obj),
	(from_peer? " (originator of delegated request)" : ""));
	}

	switch (PCMK__CLIENT_TYPE(client_obj)) {
	case pcmk__client_ipc:
	{
	int rc = pcmk__ipc_send_xml(client_obj, rid, notify_src,
	(sync_reply? crm_ipc_flags_none
	: crm_ipc_server_event));

	if (rc != pcmk_rc_ok) {
	crm_warn("%s reply to client %s failed: %s " CRM_XS " rc=%d",
	(sync_reply? "Synchronous" : "Asynchronous"),
	pcmk__client_name(client_obj), pcmk_rc_str(rc),
	rc);
	}
	}
	break;
	#ifdef HAVE_GNUTLS_GNUTLS_H
	case pcmk__client_tls:
	#endif
	case pcmk__client_tcp:
	pcmk__remote_send_xml(client_obj->remote, notify_src);
	break;
	default:
	crm_err("Unknown transport for client %s "
	CRM_XS " flags=%#016" PRIx64,
	pcmk__client_name(client_obj), client_obj->flags);
	}
	}

	static void
	local_notify_destroy_callback(gpointer data)
	{
	cib_local_notify_t *notify = data;

	free_xml(notify->notify_src);
	free(notify->client_id);
	free(notify);
	}

	static void
	check_local_notify(int bcast_id)
	{
	cib_local_notify_t *notify = NULL;

	if (!local_notify_queue) {
	return;
	}

	notify = pcmk__intkey_table_lookup(local_notify_queue, bcast_id);

	if (notify) {
	do_local_notify(notify->notify_src, notify->client_id, notify->sync_reply,
	notify->from_peer);
	pcmk__intkey_table_remove(local_notify_queue, bcast_id);
	}
	}

	static void
	queue_local_notify(xmlNode * notify_src, const char *client_id, gboolean sync_reply,
	gboolean from_peer)
	{
	cib_local_notify_t *notify = calloc(1, sizeof(cib_local_notify_t));

	notify->notify_src = notify_src;
	notify->client_id = strdup(client_id);
	notify->sync_reply = sync_reply;
	notify->from_peer = from_peer;

	if (!local_notify_queue) {
	local_notify_queue = pcmk__intkey_table(local_notify_destroy_callback);
	}
	pcmk__intkey_table_insert(local_notify_queue, cib_local_bcast_num, notify);
	// cppcheck doesn't know notify will get freed when hash table is destroyed
	// cppcheck-suppress memleak
	}

	static void
	parse_local_options_v1(const pcmk__client_t *cib_client, int call_type,
	int call_options, const char host, const char op,
	gboolean local_notify, gboolean needs_reply,
	gboolean process, gboolean needs_forward)
	{
	if (cib_op_modifies(call_type)
	&& !(call_options & cib_inhibit_bcast)) {
	/* we need to send an update anyway */
	*needs_reply = TRUE;
	} else {
	*needs_reply = FALSE;
	}

	if (host == NULL && (call_options & cib_scope_local)) {
	crm_trace("Processing locally scoped %s op from client %s",
	op, pcmk__client_name(cib_client));
	*local_notify = TRUE;

	} else if ((host == NULL) && based_is_primary) {
	crm_trace("Processing %s op locally from client %s as primary",
	op, pcmk__client_name(cib_client));
	*local_notify = TRUE;

	} else if (pcmk__str_eq(host, cib_our_uname, pcmk__str_casei)) {
	crm_trace("Processing locally addressed %s op from client %s",
	op, pcmk__client_name(cib_client));
	*local_notify = TRUE;

	} else if (stand_alone) {
	*needs_forward = FALSE;
	*local_notify = TRUE;
	*process = TRUE;

	} else {
	crm_trace("%s op from %s needs to be forwarded to client %s",
	op, pcmk__client_name(cib_client),
	pcmk__s(host, "the primary instance"));
	*needs_forward = TRUE;
	*process = FALSE;
	}
	}

	static void
	parse_local_options_v2(const pcmk__client_t *cib_client, int call_type,
	int call_options, const char host, const char op,
	gboolean local_notify, gboolean needs_reply,
	gboolean process, gboolean needs_forward)
	{
	if (cib_op_modifies(call_type)) {
	if (pcmk__str_any_of(op, PCMK__CIB_REQUEST_PRIMARY,
	PCMK__CIB_REQUEST_SECONDARY, NULL)) {
	/* Always handle these locally */
	*process = TRUE;
	*needs_reply = FALSE;
	*local_notify = TRUE;
	*needs_forward = FALSE;
	return;

	} else {
	/* Redirect all other updates via CPG */
	*needs_reply = TRUE;
	*needs_forward = TRUE;
	*process = FALSE;
	crm_trace("%s op from %s needs to be forwarded to client %s",
	op, pcmk__client_name(cib_client),
	pcmk__s(host, "the primary instance"));
	return;
	}
	}


	*process = TRUE;
	*needs_reply = FALSE;
	*local_notify = TRUE;
	*needs_forward = FALSE;

	if (stand_alone) {
	crm_trace("Processing %s op from client %s (stand-alone)",
	op, pcmk__client_name(cib_client));

	} else if (host == NULL) {
	crm_trace("Processing unaddressed %s op from client %s",
	op, pcmk__client_name(cib_client));

	} else if (pcmk__str_eq(host, cib_our_uname, pcmk__str_casei)) {
	crm_trace("Processing locally addressed %s op from client %s",
	op, pcmk__client_name(cib_client));

	} else {
	crm_trace("%s op from %s needs to be forwarded to client %s",
	op, pcmk__client_name(cib_client), host);
	*needs_forward = TRUE;
	*process = FALSE;
	}
	}

	static void
	parse_local_options(const pcmk__client_t *cib_client, int call_type,
	int call_options, const char host, const char op,
	gboolean local_notify, gboolean needs_reply,
	gboolean process, gboolean needs_forward)
	{
	if(cib_legacy_mode()) {
	parse_local_options_v1(cib_client, call_type, call_options, host,
	op, local_notify, needs_reply, process, needs_forward);
	} else {
	parse_local_options_v2(cib_client, call_type, call_options, host,
	op, local_notify, needs_reply, process, needs_forward);
	}
	}

	static gboolean
	parse_peer_options_v1(int call_type, xmlNode * request,
	gboolean * local_notify, gboolean * needs_reply, gboolean * process,
	gboolean * needs_forward)
	{
	const char *op = NULL;
	const char *host = NULL;
	const char *delegated = NULL;
	const char *originator = crm_element_value(request, F_ORIG);
	const char *reply_to = crm_element_value(request, F_CIB_ISREPLY);

	gboolean is_reply = pcmk__str_eq(reply_to, cib_our_uname, pcmk__str_casei);

	if (pcmk__xe_attr_is_true(request, F_CIB_GLOBAL_UPDATE)) {
	*needs_reply = FALSE;
	if (is_reply) {
	*local_notify = TRUE;
	crm_trace("Processing global/peer update from %s"
	" that originated from us", originator);
	} else {
	crm_trace("Processing global/peer update from %s", originator);
	}
	return TRUE;
	}

	op = crm_element_value(request, F_CIB_OPERATION);
	crm_trace("Processing %s request sent by %s", op, originator);
	if (pcmk__str_eq(op, PCMK__CIB_REQUEST_SHUTDOWN, pcmk__str_none)) {
	/* Always process these */
	*local_notify = FALSE;
	if (reply_to == NULL \|\| is_reply) {
	*process = TRUE;
	}
	if (is_reply) {
	*needs_reply = FALSE;
	}
	return *process;
	}

	if (is_reply && pcmk__str_eq(op, CRM_OP_PING, pcmk__str_casei)) {
	process_ping_reply(request);
	return FALSE;
	}

	if (is_reply) {
	crm_trace("Forward reply sent from %s to local clients", originator);
	*process = FALSE;
	*needs_reply = FALSE;
	*local_notify = TRUE;
	return TRUE;
	}

	host = crm_element_value(request, F_CIB_HOST);
	if (host != NULL && pcmk__str_eq(host, cib_our_uname, pcmk__str_casei)) {
	crm_trace("Processing %s request sent to us from %s", op, originator);
	return TRUE;

	} else if(is_reply == FALSE && pcmk__str_eq(op, CRM_OP_PING, pcmk__str_casei)) {
	crm_trace("Processing %s request sent to %s by %s", op, host?host:"everyone", originator);
	*needs_reply = TRUE;
	return TRUE;

	} else if ((host == NULL) && based_is_primary) {
	crm_trace("Processing %s request sent to primary instance from %s",
	op, originator);
	return TRUE;
	}

	delegated = crm_element_value(request, F_CIB_DELEGATED);
	if (delegated != NULL) {
	crm_trace("Ignoring message for primary instance");

	} else if (host != NULL) {
	/* this is for a specific instance and we're not it */
	crm_trace("Ignoring msg for instance on %s", host);

	} else if ((reply_to == NULL) && !based_is_primary) {
	// This is for the primary instance, and we're not it
	crm_trace("Ignoring reply for primary instance");

	} else if (pcmk__str_eq(op, PCMK__CIB_REQUEST_SHUTDOWN, pcmk__str_none)) {
	if (reply_to != NULL) {
	crm_debug("Processing %s from %s", op, originator);
	*needs_reply = FALSE;

	} else {
	crm_debug("Processing %s reply from %s", op, originator);
	}
	return TRUE;

	} else {
	crm_err("Nothing for us to do?");
	crm_log_xml_err(request, "Peer[inbound]");
	}

	return FALSE;
	}

	static gboolean
	parse_peer_options_v2(int call_type, xmlNode * request,
	gboolean * local_notify, gboolean * needs_reply, gboolean * process,
	gboolean * needs_forward)
	{
	const char *host = NULL;
	const char *delegated = crm_element_value(request, F_CIB_DELEGATED);
	const char *op = crm_element_value(request, F_CIB_OPERATION);
	const char *originator = crm_element_value(request, F_ORIG);
	const char *reply_to = crm_element_value(request, F_CIB_ISREPLY);

	gboolean is_reply = pcmk__str_eq(reply_to, cib_our_uname, pcmk__str_casei);

	if (pcmk__str_eq(op, PCMK__CIB_REQUEST_REPLACE, pcmk__str_none)) {
	/* sync_our_cib() sets F_CIB_ISREPLY */
	if (reply_to) {
	delegated = reply_to;
	}
	goto skip_is_reply;

	} else if (pcmk__str_eq(op, PCMK__CIB_REQUEST_SYNC_TO_ALL,
	pcmk__str_none)) {
	// Nothing to do

	} else if (is_reply && pcmk__str_eq(op, CRM_OP_PING, pcmk__str_casei)) {
	process_ping_reply(request);
	return FALSE;

	} else if (pcmk__str_eq(op, PCMK__CIB_REQUEST_UPGRADE, pcmk__str_none)) {
	/* Only the DC (node with the oldest software) should process
	* this operation if F_CIB_SCHEMA_MAX is unset
	*
	* If the DC is happy it will then send out another
	* PCMK__CIB_REQUEST_UPGRADE which will tell all nodes to do the actual
	* upgrade.
	*
	* Except this time F_CIB_SCHEMA_MAX will be set which puts a
	* limit on how far newer nodes will go
	*/
	const char *max = crm_element_value(request, F_CIB_SCHEMA_MAX);
	const char *upgrade_rc = crm_element_value(request, F_CIB_UPGRADE_RC);

	crm_trace("Parsing %s operation%s for %s with max=%s and upgrade_rc=%s",
	op, (is_reply? " reply" : ""),
	(based_is_primary? "primary" : "secondary"),
	(max? max : "none"), (upgrade_rc? upgrade_rc : "none"));

	if (upgrade_rc != NULL) {
	// Our upgrade request was rejected by DC, notify clients of result
	crm_xml_add(request, F_CIB_RC, upgrade_rc);

	} else if ((max == NULL) && based_is_primary) {
	/* We are the DC, check if this upgrade is allowed */
	goto skip_is_reply;

	} else if(max) {
	/* Ok, go ahead and upgrade to 'max' */
	goto skip_is_reply;

	} else {
	// Ignore broadcast client requests when we're not DC
	return FALSE;
	}

	} else if (pcmk__xe_attr_is_true(request, F_CIB_GLOBAL_UPDATE)) {
	crm_info("Detected legacy %s global update from %s", op, originator);
	send_sync_request(NULL);
	legacy_mode = TRUE;
	return FALSE;

	} else if (is_reply && cib_op_modifies(call_type)) {
	crm_trace("Ignoring legacy %s reply sent from %s to local clients", op, originator);
	return FALSE;

	} else if (pcmk__str_eq(op, PCMK__CIB_REQUEST_SHUTDOWN, pcmk__str_none)) {
	/* Legacy handling */
	crm_debug("Legacy handling of %s message from %s", op, originator);
	*local_notify = FALSE;
	if (reply_to == NULL) {
	*process = TRUE;
	}
	return *process;
	}

	if(is_reply) {
	crm_trace("Handling %s reply sent from %s to local clients", op, originator);
	*process = FALSE;
	*needs_reply = FALSE;
	*local_notify = TRUE;
	return TRUE;
	}

	skip_is_reply:
	*process = TRUE;
	*needs_reply = FALSE;

	if(pcmk__str_eq(delegated, cib_our_uname, pcmk__str_casei)) {
	*local_notify = TRUE;
	} else {
	*local_notify = FALSE;
	}

	host = crm_element_value(request, F_CIB_HOST);
	if (host != NULL && pcmk__str_eq(host, cib_our_uname, pcmk__str_casei)) {
	crm_trace("Processing %s request sent to us from %s", op, originator);
	*needs_reply = TRUE;
	return TRUE;

	} else if (host != NULL) {
	/* this is for a specific instance and we're not it */
	crm_trace("Ignoring %s operation for instance on %s", op, host);
	return FALSE;

	} else if(is_reply == FALSE && pcmk__str_eq(op, CRM_OP_PING, pcmk__str_casei)) {
	*needs_reply = TRUE;
	}

	crm_trace("Processing %s request sent to everyone by %s/%s on %s %s", op,
	crm_element_value(request, F_CIB_CLIENTNAME),
	crm_element_value(request, F_CIB_CALLID),
	originator, (*local_notify)?"(notify)":"");
	return TRUE;
	}

	static gboolean
	parse_peer_options(int call_type, xmlNode * request,
	gboolean * local_notify, gboolean * needs_reply, gboolean * process,
	gboolean * needs_forward)
	{
	/* TODO: What happens when an update comes in after node A
	* requests the CIB from node B, but before it gets the reply (and
	* sends out the replace operation)
	*/
	if(cib_legacy_mode()) {
	return parse_peer_options_v1(
	call_type, request, local_notify, needs_reply, process, needs_forward);
	} else {
	return parse_peer_options_v2(
	call_type, request, local_notify, needs_reply, process, needs_forward);
	}
	}

	static void
	forward_request(xmlNode *request, int call_options)
	{
	const char *op = crm_element_value(request, F_CIB_OPERATION);
	const char *host = crm_element_value(request, F_CIB_HOST);

	crm_xml_add(request, F_CIB_DELEGATED, cib_our_uname);

	if (host != NULL) {
	crm_trace("Forwarding %s op to %s", op, host);
	send_cluster_message(crm_get_peer(0, host), crm_msg_cib, request, FALSE);

	} else {
	crm_trace("Forwarding %s op to primary instance", op);
	send_cluster_message(NULL, crm_msg_cib, request, FALSE);
	}

	/* Return the request to its original state */
	xml_remove_prop(request, F_CIB_DELEGATED);

	if (call_options & cib_discard_reply) {
	crm_trace("Client not interested in reply");
	}
	}

	static gboolean
	send_peer_reply(xmlNode * msg, xmlNode * result_diff, const char *originator, gboolean broadcast)
	{
	CRM_ASSERT(msg != NULL);

	if (broadcast) {
	/* this (successful) call modified the CIB _and_ the
	* change needs to be broadcast...
	* send via HA to other nodes
	*/
	int diff_add_updates = 0;
	int diff_add_epoch = 0;
	int diff_add_admin_epoch = 0;

	int diff_del_updates = 0;
	int diff_del_epoch = 0;
	int diff_del_admin_epoch = 0;

	const char *digest = NULL;
	int format = 1;

	CRM_LOG_ASSERT(result_diff != NULL);
	digest = crm_element_value(result_diff, XML_ATTR_DIGEST);
	crm_element_value_int(result_diff, "format", &format);

	cib_diff_version_details(result_diff,
	&diff_add_admin_epoch, &diff_add_epoch, &diff_add_updates,
	&diff_del_admin_epoch, &diff_del_epoch, &diff_del_updates);

	crm_trace("Sending update diff %d.%d.%d -> %d.%d.%d %s",
	diff_del_admin_epoch, diff_del_epoch, diff_del_updates,
	diff_add_admin_epoch, diff_add_epoch, diff_add_updates, digest);

	crm_xml_add(msg, F_CIB_ISREPLY, originator);
	pcmk__xe_set_bool_attr(msg, F_CIB_GLOBAL_UPDATE, true);
	crm_xml_add(msg, F_CIB_OPERATION, PCMK__CIB_REQUEST_APPLY_PATCH);
	crm_xml_add(msg, F_CIB_USER, CRM_DAEMON_USER);

	if (format == 1) {
	CRM_ASSERT(digest != NULL);
	}

	add_message_xml(msg, F_CIB_UPDATE_DIFF, result_diff);
	crm_log_xml_explicit(msg, "copy");
	return send_cluster_message(NULL, crm_msg_cib, msg, TRUE);

	} else if (originator != NULL) {
	/* send reply via HA to originating node */
	crm_trace("Sending request result to %s only", originator);
	crm_xml_add(msg, F_CIB_ISREPLY, originator);
	return send_cluster_message(crm_get_peer(0, originator), crm_msg_cib, msg, FALSE);
	}

	return FALSE;
	}

	/*!
	* \internal
	* \brief Handle an IPC or CPG message containing a request
	*
	* \param[in,out] request Request XML
	* \param[in] privileged Whether privileged commands may be run
	* (see cib_server_ops[] definition)
	* \param[in] cib_client IPC client that sent request (or NULL if CPG)
	*/
	static void
	cib_process_request(xmlNode *request, gboolean privileged,
	const pcmk__client_t *cib_client)
	{
	int call_type = 0;
	int call_options = 0;

	gboolean process = TRUE; // Whether to process request locally now
	gboolean is_update = TRUE; // Whether request would modify CIB
	gboolean needs_reply = TRUE; // Whether to build a reply
	gboolean local_notify = FALSE; // Whether to notify (local) requester
	gboolean needs_forward = FALSE; // Whether to forward request somewhere else

	xmlNode *op_reply = NULL;
	xmlNode *result_diff = NULL;

	int rc = pcmk_ok;
	const char *op = crm_element_value(request, F_CIB_OPERATION);
	const char *originator = crm_element_value(request, F_ORIG);
	const char *host = crm_element_value(request, F_CIB_HOST);
	const char *target = NULL;
	const char *call_id = crm_element_value(request, F_CIB_CALLID);
	const char *client_id = crm_element_value(request, F_CIB_CLIENTID);
	const char *client_name = crm_element_value(request, F_CIB_CLIENTNAME);
	const char *reply_to = crm_element_value(request, F_CIB_ISREPLY);

	crm_element_value_int(request, F_CIB_CALLOPTS, &call_options);

	if ((host != NULL) && (*host == '\0')) {
	host = NULL;
	}

	if (host) {
	target = host;

	} else if (call_options & cib_scope_local) {
	target = "local host";

	} else {
	target = "primary";
	}

	if (cib_client == NULL) {
	crm_trace("Processing peer %s operation from %s/%s on %s intended for %s (reply=%s)",
	op, client_name, call_id, originator, target, reply_to);
	} else {
	crm_xml_add(request, F_ORIG, cib_our_uname);
	crm_trace("Processing local %s operation from %s/%s intended for %s", op, client_name, call_id, target);
	}

	rc = cib_get_operation_id(op, &call_type);
	if (rc != pcmk_ok) {
	/* TODO: construct error reply? */
	crm_err("Pre-processing of command failed: %s", pcmk_strerror(rc));
	return;
	}

	if (cib_client != NULL) {
	parse_local_options(cib_client, call_type, call_options, host, op,
	&local_notify, &needs_reply, &process, &needs_forward);

	} else if (parse_peer_options(call_type, request, &local_notify,
	&needs_reply, &process, &needs_forward) == FALSE) {
	return;
	}

	is_update = cib_op_modifies(call_type);

	if (call_options & cib_discard_reply) {
	/* If the request will modify the CIB, and we are in legacy mode, we
	* need to build a reply so we can broadcast a diff, even if the
	* requester doesn't want one.
	*/
	needs_reply = is_update && cib_legacy_mode();
	local_notify = FALSE;
	}

	if (needs_forward) {
	const char *section = crm_element_value(request, F_CIB_SECTION);
	int log_level = LOG_INFO;

	if (pcmk__str_eq(op, PCMK__CIB_REQUEST_NOOP, pcmk__str_none)) {
	log_level = LOG_DEBUG;
	}

	do_crm_log(log_level,
	"Forwarding %s operation for section %s to %s (origin=%s/%s/%s)",
	op,
	section ? section : "'all'",
	pcmk__s(host, (cib_legacy_mode() ? "primary" : "all")),
	originator ? originator : "local",
	client_name, call_id);

	forward_request(request, call_options);
	return;
	}

	if (cib_status != pcmk_ok) {
	const char *call = crm_element_value(request, F_CIB_CALLID);

	rc = cib_status;
	crm_err("Operation ignored, cluster configuration is invalid."
	" Please repair and restart: %s", pcmk_strerror(cib_status));

	op_reply = create_xml_node(NULL, "cib-reply");
	crm_xml_add(op_reply, F_TYPE, T_CIB);
	crm_xml_add(op_reply, F_CIB_OPERATION, op);
	crm_xml_add(op_reply, F_CIB_CALLID, call);
	crm_xml_add(op_reply, F_CIB_CLIENTID, client_id);
	crm_xml_add_int(op_reply, F_CIB_CALLOPTS, call_options);
	crm_xml_add_int(op_reply, F_CIB_RC, rc);

	crm_trace("Attaching reply output");
	add_message_xml(op_reply, F_CIB_CALLDATA, the_cib);

	crm_log_xml_explicit(op_reply, "cib:reply");

	} else if (process) {
	time_t finished = 0;
	time_t now = time(NULL);
	int level = LOG_INFO;
	const char *section = crm_element_value(request, F_CIB_SECTION);

	rc = cib_process_command(request, &op_reply, &result_diff, privileged);

	if (!is_update) {
	level = LOG_TRACE;

	} else if (pcmk__xe_attr_is_true(request, F_CIB_GLOBAL_UPDATE)) {
	switch (rc) {
	case pcmk_ok:
	level = LOG_INFO;
	break;
	case -pcmk_err_old_data:
	case -pcmk_err_diff_resync:
	case -pcmk_err_diff_failed:
	level = LOG_TRACE;
	break;
	default:
	level = LOG_ERR;
	}

	} else if (rc != pcmk_ok) {
	level = LOG_WARNING;
	}

	do_crm_log(level,
	"Completed %s operation for section %s: %s (rc=%d, origin=%s/%s/%s, version=%s.%s.%s)",
	op, section ? section : "'all'", pcmk_strerror(rc), rc,
	originator ? originator : "local", client_name, call_id,
	the_cib ? crm_element_value(the_cib, XML_ATTR_GENERATION_ADMIN) : "0",
	the_cib ? crm_element_value(the_cib, XML_ATTR_GENERATION) : "0",
	the_cib ? crm_element_value(the_cib, XML_ATTR_NUMUPDATES) : "0");

	finished = time(NULL);
	if ((finished - now) > 3) {
	crm_trace("%s operation took %lds to complete", op, (long)(finished - now));
	crm_write_blackbox(0, NULL);
	}

	if (op_reply == NULL && (needs_reply \|\| local_notify)) {
	crm_err("Unexpected NULL reply to message");
	crm_log_xml_err(request, "null reply");
	needs_reply = FALSE;
	local_notify = FALSE;
	}
	}

	if (is_update && !cib_legacy_mode()) {
	crm_trace("Completed pre-sync update from %s/%s/%s%s",
	originator ? originator : "local", client_name, call_id,
	local_notify?" with local notification":"");

	} else if (!needs_reply \|\| stand_alone) {
	// This was a non-originating secondary update
	crm_trace("Completed update as secondary");

	} else if (cib_legacy_mode() &&
	rc == pcmk_ok && result_diff != NULL && !(call_options & cib_inhibit_bcast)) {
	gboolean broadcast = FALSE;

	cib_local_bcast_num++;
	crm_xml_add_int(request, F_CIB_LOCAL_NOTIFY_ID, cib_local_bcast_num);
	broadcast = send_peer_reply(request, result_diff, originator, TRUE);

	if (broadcast && client_id && local_notify && op_reply) {

	/* If we have been asked to sync the reply,
	* and a bcast msg has gone out, we queue the local notify
	* until we know the bcast message has been received */
	local_notify = FALSE;
	crm_trace("Queuing local %ssync notification for %s",
	(call_options & cib_sync_call) ? "" : "a-", client_id);

	queue_local_notify(op_reply, client_id,
	pcmk_is_set(call_options, cib_sync_call),
	(cib_client == NULL));
	op_reply = NULL; /* the reply is queued, so don't free here */
	}

	} else if (call_options & cib_discard_reply) {
	crm_trace("Caller isn't interested in reply");

	} else if (cib_client == NULL) {
	if (is_update == FALSE \|\| result_diff == NULL) {
	crm_trace("Request not broadcast: R/O call");

	} else if (call_options & cib_inhibit_bcast) {
	crm_trace("Request not broadcast: inhibited");

	} else if (rc != pcmk_ok) {
	crm_trace("Request not broadcast: call failed: %s", pcmk_strerror(rc));

	} else {
	crm_trace("Directing reply to %s", originator);
	}

	send_peer_reply(op_reply, result_diff, originator, FALSE);
	}

	if (local_notify && client_id) {
	crm_trace("Performing local %ssync notification for %s",
	(pcmk_is_set(call_options, cib_sync_call)? "" : "a"),
	client_id);
	if (process == FALSE) {
	do_local_notify(request, client_id,
	pcmk_is_set(call_options, cib_sync_call),
	(cib_client == NULL));
	} else {
	do_local_notify(op_reply, client_id,
	pcmk_is_set(call_options, cib_sync_call),
	(cib_client == NULL));
	}
	}

	free_xml(op_reply);
	free_xml(result_diff);

	return;
	}

	static char *
	calculate_section_digest(const char xpath, xmlNode xml_obj)
	{
	xmlNode *xml_section = NULL;

	if (xml_obj == NULL) {
	return NULL;
	}

	xml_section = get_xpath_object(xpath, xml_obj, LOG_TRACE);
	if (xml_section == NULL) {
	return NULL;
	}
	return calculate_xml_versioned_digest(xml_section, FALSE, TRUE, CRM_FEATURE_SET);

	}

	static int
	cib_process_command(xmlNode * request, xmlNode reply, xmlNode cib_diff, gboolean privileged)
	{
	xmlNode *input = NULL;
	xmlNode *output = NULL;
	xmlNode *result_cib = NULL;
	xmlNode *current_cib = NULL;

	int call_type = 0;
	int call_options = 0;

	const char *op = NULL;
	const char *section = NULL;
	const char *call_id = crm_element_value(request, F_CIB_CALLID);

	int rc = pcmk_ok;
	int rc2 = pcmk_ok;

	gboolean send_r_notify = FALSE;
	gboolean global_update = FALSE;
	gboolean config_changed = FALSE;
	gboolean manage_counters = TRUE;

	static mainloop_timer_t *digest_timer = NULL;

	char *current_nodes_digest = NULL;
	char *current_alerts_digest = NULL;
	char *current_status_digest = NULL;
	- int change_section = cib_change_section_nodes \| cib_change_section_alerts \| cib_change_section_status;
	+ uint32_t change_section = cib_change_section_nodes
	+ \|cib_change_section_alerts
	+ \|cib_change_section_status;

	CRM_ASSERT(cib_status == pcmk_ok);

	if(digest_timer == NULL) {
	digest_timer = mainloop_timer_add("digester", 5000, FALSE, cib_digester_cb, NULL);
	}

	*reply = NULL;
	*cib_diff = NULL;
	current_cib = the_cib;

	/* Start processing the request... */
	op = crm_element_value(request, F_CIB_OPERATION);
	crm_element_value_int(request, F_CIB_CALLOPTS, &call_options);
	rc = cib_get_operation_id(op, &call_type);

	if (rc == pcmk_ok && privileged == FALSE) {
	rc = cib_op_can_run(call_type, call_options, privileged, global_update);
	}

	rc2 = cib_op_prepare(call_type, request, &input, &section);
	if (rc == pcmk_ok) {
	rc = rc2;
	}

	if (rc != pcmk_ok) {
	crm_trace("Call setup failed: %s", pcmk_strerror(rc));
	goto done;

	} else if (cib_op_modifies(call_type) == FALSE) {
	rc = cib_perform_op(op, call_options, cib_op_func(call_type), TRUE,
	section, request, input, FALSE, &config_changed,
	current_cib, &result_cib, NULL, &output);

	CRM_CHECK(result_cib == NULL, free_xml(result_cib));
	goto done;
	}

	/* Handle a valid write action */
	if (pcmk__xe_attr_is_true(request, F_CIB_GLOBAL_UPDATE)) {
	/* legacy code */
	manage_counters = FALSE;
	cib__set_call_options(call_options, "call", cib_force_diff);
	crm_trace("Global update detected");

	CRM_CHECK(call_type == 3 \|\| call_type == 4, crm_err("Call type: %d", call_type);
	crm_log_xml_err(request, "bad op"));
	}

	if (rc == pcmk_ok) {
	ping_modified_since = TRUE;
	if (call_options & cib_inhibit_bcast) {
	/* skip */
	crm_trace("Skipping update: inhibit broadcast");
	manage_counters = FALSE;
	}

	if (!pcmk_is_set(call_options, cib_dryrun)
	&& pcmk__str_eq(section, XML_CIB_TAG_STATUS, pcmk__str_casei)) {
	/* Copying large CIBs accounts for a huge percentage of our CIB usage */
	cib__set_call_options(call_options, "call", cib_zero_copy);
	} else {
	cib__clear_call_options(call_options, "call", cib_zero_copy);
	}

	/* Calculate the hash value of the section before the change. */
	if (pcmk__str_eq(PCMK__CIB_REQUEST_REPLACE, op, pcmk__str_none)) {
	current_nodes_digest = calculate_section_digest("//" XML_TAG_CIB "/" XML_CIB_TAG_CONFIGURATION "/" XML_CIB_TAG_NODES, current_cib);
	current_alerts_digest = calculate_section_digest("//" XML_TAG_CIB "/" XML_CIB_TAG_CONFIGURATION "/" XML_CIB_TAG_ALERTS, current_cib);
	current_status_digest = calculate_section_digest("//" XML_TAG_CIB "/" XML_CIB_TAG_STATUS, current_cib);
	crm_trace("current-digest %s:%s:%s", current_nodes_digest, current_alerts_digest, current_status_digest);
	}

	/* result_cib must not be modified after cib_perform_op() returns */
	rc = cib_perform_op(op, call_options, cib_op_func(call_type), FALSE,
	section, request, input, manage_counters, &config_changed,
	current_cib, &result_cib, cib_diff, &output);

	if (manage_counters == FALSE) {
	int format = 1;
	/* Legacy code
	* If the diff is NULL at this point, it's because nothing changed
	*/
	if (*cib_diff) {
	crm_element_value_int(*cib_diff, "format", &format);
	}

	if (format == 1) {
	config_changed = cib_config_changed(NULL, NULL, cib_diff);
	}
	}

	/* Always write to disk for replace ops,
	* this also negates the need to detect ordering changes
	*/
	if (pcmk__str_eq(PCMK__CIB_REQUEST_REPLACE, op, pcmk__str_none)) {
	config_changed = TRUE;
	}
	}

	if (rc == pcmk_ok && !pcmk_is_set(call_options, cib_dryrun)) {
	crm_trace("Activating %s->%s%s%s",
	crm_element_value(current_cib, XML_ATTR_NUMUPDATES),
	crm_element_value(result_cib, XML_ATTR_NUMUPDATES),
	(pcmk_is_set(call_options, cib_zero_copy)? " zero-copy" : ""),
	(config_changed? " changed" : ""));
	if (!pcmk_is_set(call_options, cib_zero_copy)) {
	rc = activateCibXml(result_cib, config_changed, op);
	crm_trace("Activated %s (%d)",
	crm_element_value(current_cib, XML_ATTR_NUMUPDATES), rc);
	}

	if (rc == pcmk_ok && cib_internal_config_changed(*cib_diff)) {
	cib_read_config(config_hash, result_cib);
	}

	if (pcmk__str_eq(PCMK__CIB_REQUEST_REPLACE, op, pcmk__str_none)) {
	char *result_nodes_digest = NULL;
	char *result_alerts_digest = NULL;
	char *result_status_digest = NULL;

	/* Calculate the hash value of the changed section. */
	result_nodes_digest = calculate_section_digest("//" XML_TAG_CIB "/" XML_CIB_TAG_CONFIGURATION "/" XML_CIB_TAG_NODES, result_cib);
	result_alerts_digest = calculate_section_digest("//" XML_TAG_CIB "/" XML_CIB_TAG_CONFIGURATION "/" XML_CIB_TAG_ALERTS, result_cib);
	result_status_digest = calculate_section_digest("//" XML_TAG_CIB "/" XML_CIB_TAG_STATUS, result_cib);
	crm_trace("result-digest %s:%s:%s", result_nodes_digest, result_alerts_digest, result_status_digest);

	- if (pcmk__str_eq(current_nodes_digest, result_nodes_digest, pcmk__str_none)) {
	- change_section = change_section ^ cib_change_section_nodes;
	+ if (pcmk__str_eq(current_nodes_digest, result_nodes_digest,
	+ pcmk__str_none)) {
	+ change_section =
	+ pcmk__clear_flags_as(__func__, __LINE__, LOG_TRACE,
	+ "CIB change section",
	+ "change_section", change_section,
	+ cib_change_section_nodes, "nodes");
	}
	- if (pcmk__str_eq(current_alerts_digest, result_alerts_digest, pcmk__str_none)) {
	- change_section = change_section ^ cib_change_section_alerts;
	+
	+ if (pcmk__str_eq(current_alerts_digest, result_alerts_digest,
	+ pcmk__str_none)) {
	+ change_section =
	+ pcmk__clear_flags_as(__func__, __LINE__, LOG_TRACE,
	+ "CIB change section",
	+ "change_section", change_section,
	+ cib_change_section_alerts, "alerts");
	}
	- if (pcmk__str_eq(current_status_digest, result_status_digest, pcmk__str_none)) {
	- change_section = change_section ^ cib_change_section_status;
	+
	+ if (pcmk__str_eq(current_status_digest, result_status_digest,
	+ pcmk__str_none)) {
	+ change_section =
	+ pcmk__clear_flags_as(__func__, __LINE__, LOG_TRACE,
	+ "CIB change section",
	+ "change_section", change_section,
	+ cib_change_section_status, "status");
	}

	- if (change_section) {
	+ if (change_section != cib_change_section_none) {
	send_r_notify = TRUE;
	}

	free(result_nodes_digest);
	free(result_alerts_digest);
	free(result_status_digest);

	} else if (pcmk__str_eq(PCMK__CIB_REQUEST_ERASE, op, pcmk__str_none)) {
	send_r_notify = TRUE;
	}

	mainloop_timer_stop(digest_timer);
	mainloop_timer_start(digest_timer);

	} else if (rc == -pcmk_err_schema_validation) {
	CRM_ASSERT(!pcmk_is_set(call_options, cib_zero_copy));

	if (output != NULL) {
	crm_log_xml_info(output, "cib:output");
	free_xml(output);
	}

	output = result_cib;

	} else {
	crm_trace("Not activating %d %d %s", rc,
	pcmk_is_set(call_options, cib_dryrun),
	crm_element_value(result_cib, XML_ATTR_NUMUPDATES));
	if (!pcmk_is_set(call_options, cib_zero_copy)) {
	free_xml(result_cib);
	}
	}

	if ((call_options & (cib_inhibit_notify\|cib_dryrun)) == 0) {
	const char *client = crm_element_value(request, F_CIB_CLIENTNAME);

	crm_trace("Sending notifications %d",
	pcmk_is_set(call_options, cib_dryrun));
	cib_diff_notify(call_options, client, call_id, op, input, rc, *cib_diff);
	}

	if (send_r_notify) {
	const char *origin = crm_element_value(request, F_ORIG);

	cib_replace_notify(origin, the_cib, rc, *cib_diff, change_section);
	}

	xml_log_patchset(LOG_TRACE, "cib:diff", *cib_diff);
	done:
	if (!pcmk_is_set(call_options, cib_discard_reply) \|\| cib_legacy_mode()) {
	const char *caller = crm_element_value(request, F_CIB_CLIENTID);

	*reply = create_xml_node(NULL, "cib-reply");
	crm_xml_add(*reply, F_TYPE, T_CIB);
	crm_xml_add(*reply, F_CIB_OPERATION, op);
	crm_xml_add(*reply, F_CIB_CALLID, call_id);
	crm_xml_add(*reply, F_CIB_CLIENTID, caller);
	crm_xml_add_int(*reply, F_CIB_CALLOPTS, call_options);
	crm_xml_add_int(*reply, F_CIB_RC, rc);

	if (output != NULL) {
	crm_trace("Attaching reply output");
	add_message_xml(*reply, F_CIB_CALLDATA, output);
	}

	crm_log_xml_explicit(*reply, "cib:reply");
	}

	crm_trace("cleanup");

	if (cib_op_modifies(call_type) == FALSE && output != current_cib) {
	free_xml(output);
	output = NULL;
	}

	if (call_type >= 0) {
	cib_op_cleanup(call_type, call_options, &input, &output);
	}

	free(current_nodes_digest);
	free(current_alerts_digest);
	free(current_status_digest);

	crm_trace("done");
	return rc;
	}

	void
	cib_peer_callback(xmlNode * msg, void *private_data)
	{
	const char *reason = NULL;
	const char *originator = crm_element_value(msg, F_ORIG);

	if (cib_legacy_mode() && pcmk__str_eq(originator, cib_our_uname, pcmk__str_null_matches)) {
	/* message is from ourselves */
	int bcast_id = 0;

	if (!(crm_element_value_int(msg, F_CIB_LOCAL_NOTIFY_ID, &bcast_id))) {
	check_local_notify(bcast_id);
	}
	return;

	} else if (crm_peer_cache == NULL) {
	reason = "membership not established";
	goto bail;
	}

	if (crm_element_value(msg, F_CIB_CLIENTNAME) == NULL) {
	crm_xml_add(msg, F_CIB_CLIENTNAME, originator);
	}

	/* crm_log_xml_trace(msg, "Peer[inbound]"); */
	cib_process_request(msg, TRUE, NULL);
	return;

	bail:
	if (reason) {
	const char *seq = crm_element_value(msg, F_SEQ);
	const char *op = crm_element_value(msg, F_CIB_OPERATION);

	crm_warn("Discarding %s message (%s) from %s: %s", op, seq, originator, reason);
	}
	}

	static gboolean
	cib_force_exit(gpointer data)
	{
	crm_notice("Forcing exit!");
	terminate_cib(__func__, CRM_EX_ERROR);
	return FALSE;
	}

	static void
	disconnect_remote_client(gpointer key, gpointer value, gpointer user_data)
	{
	pcmk__client_t *a_client = value;

	crm_err("Can't disconnect client %s: Not implemented",
	pcmk__client_name(a_client));
	}

	void
	cib_shutdown(int nsig)
	{
	struct qb_ipcs_stats srv_stats;

	if (cib_shutdown_flag == FALSE) {
	int disconnects = 0;
	qb_ipcs_connection_t *c = NULL;

	cib_shutdown_flag = TRUE;

	c = qb_ipcs_connection_first_get(ipcs_rw);
	while (c != NULL) {
	qb_ipcs_connection_t *last = c;

	c = qb_ipcs_connection_next_get(ipcs_rw, last);

	crm_debug("Disconnecting r/w client %p...", last);
	qb_ipcs_disconnect(last);
	qb_ipcs_connection_unref(last);
	disconnects++;
	}

	c = qb_ipcs_connection_first_get(ipcs_ro);
	while (c != NULL) {
	qb_ipcs_connection_t *last = c;

	c = qb_ipcs_connection_next_get(ipcs_ro, last);

	crm_debug("Disconnecting r/o client %p...", last);
	qb_ipcs_disconnect(last);
	qb_ipcs_connection_unref(last);
	disconnects++;
	}

	c = qb_ipcs_connection_first_get(ipcs_shm);
	while (c != NULL) {
	qb_ipcs_connection_t *last = c;

	c = qb_ipcs_connection_next_get(ipcs_shm, last);

	crm_debug("Disconnecting non-blocking r/w client %p...", last);
	qb_ipcs_disconnect(last);
	qb_ipcs_connection_unref(last);
	disconnects++;
	}

	disconnects += pcmk__ipc_client_count();

	crm_debug("Disconnecting %d remote clients", pcmk__ipc_client_count());
	pcmk__foreach_ipc_client(disconnect_remote_client, NULL);
	crm_info("Disconnected %d clients", disconnects);
	}

	qb_ipcs_stats_get(ipcs_rw, &srv_stats, QB_FALSE);

	if (pcmk__ipc_client_count() == 0) {
	crm_info("All clients disconnected (%d)", srv_stats.active_connections);
	initiate_exit();

	} else {
	crm_info("Waiting on %d clients to disconnect (%d)",
	pcmk__ipc_client_count(), srv_stats.active_connections);
	}
	}

	void
	initiate_exit(void)
	{
	int active = 0;
	xmlNode *leaving = NULL;

	active = crm_active_peers();
	if (active < 2) {
	terminate_cib(__func__, 0);
	return;
	}

	crm_info("Sending disconnect notification to %d peers...", active);

	leaving = create_xml_node(NULL, "exit-notification");
	crm_xml_add(leaving, F_TYPE, "cib");
	crm_xml_add(leaving, F_CIB_OPERATION, PCMK__CIB_REQUEST_SHUTDOWN);

	send_cluster_message(NULL, crm_msg_cib, leaving, TRUE);
	free_xml(leaving);

	g_timeout_add(EXIT_ESCALATION_MS, cib_force_exit, NULL);
	}

	extern int remote_fd;
	extern int remote_tls_fd;

	/*!
	* \internal
	* \brief Close remote sockets, free the global CIB and quit
	*
	* \param[in] caller Name of calling function (for log message)
	* \param[in] fast If -1, skip disconnect; if positive, exit that
	*/
	void
	terminate_cib(const char *caller, int fast)
	{
	crm_info("%s: Exiting%s...", caller,
	(fast > 0)? " fast" : mainloop ? " from mainloop" : "");

	if (remote_fd > 0) {
	close(remote_fd);
	remote_fd = 0;
	}
	if (remote_tls_fd > 0) {
	close(remote_tls_fd);
	remote_tls_fd = 0;
	}

	uninitializeCib();

	if (fast > 0) {
	/* Quit fast on error */
	pcmk__stop_based_ipc(ipcs_ro, ipcs_rw, ipcs_shm);
	crm_exit(fast);

	} else if ((mainloop != NULL) && g_main_loop_is_running(mainloop)) {
	/* Quit via returning from the main loop. If fast == -1, we skip the
	* disconnect here, and it will be done when the main loop returns
	* (this allows the peer status callback to avoid messing with the
	* peer caches).
	*/
	if (fast == 0) {
	crm_cluster_disconnect(&crm_cluster);
	}
	g_main_loop_quit(mainloop);

	} else {
	/* Quit via clean exit. Even the peer status callback can disconnect
	* here, because we're not returning control to the caller. */
	crm_cluster_disconnect(&crm_cluster);
	pcmk__stop_based_ipc(ipcs_ro, ipcs_rw, ipcs_shm);
	crm_exit(CRM_EX_OK);
	}
	}
	diff --git a/daemons/based/based_notify.c b/daemons/based/based_notify.c
	index c2d8a0aa61..38e2658b6f 100644
	--- a/daemons/based/based_notify.c
	+++ b/daemons/based/based_notify.c
	@@ -1,283 +1,285 @@
	/*
	* Copyright 2004-2022 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <sys/param.h>
	#include <stdio.h>
	#include <sys/types.h>
	#include <unistd.h>
	#include <inttypes.h> // PRIx64

	#include <stdlib.h>
	#include <errno.h>
	#include <fcntl.h>

	#include <time.h>

	#include <crm/crm.h>
	#include <crm/cib/internal.h>
	#include <crm/msg_xml.h>

	#include <crm/common/xml.h>
	#include <crm/common/remote_internal.h>
	#include <pacemaker-based.h>

	int pending_updates = 0;

	struct cib_notification_s {
	xmlNode *msg;
	struct iovec *iov;
	int32_t iov_size;
	};

	void attach_cib_generation(xmlNode * msg, const char field, xmlNode a_cib);

	static void do_cib_notify(int options, const char op, xmlNode update,
	int result, xmlNode * result_data,
	const char *msg_type);

	static void
	cib_notify_send_one(gpointer key, gpointer value, gpointer user_data)
	{
	const char *type = NULL;
	gboolean do_send = FALSE;
	int rc = pcmk_rc_ok;

	pcmk__client_t *client = value;
	struct cib_notification_s *update = user_data;

	if (client->ipcs == NULL && client->remote == NULL) {
	crm_warn("Skipping client with NULL channel");
	return;
	}

	type = crm_element_value(update->msg, F_SUBTYPE);
	CRM_LOG_ASSERT(type != NULL);

	if (pcmk_is_set(client->flags, cib_notify_diff)
	&& pcmk__str_eq(type, T_CIB_DIFF_NOTIFY, pcmk__str_casei)) {

	do_send = TRUE;

	} else if (pcmk_is_set(client->flags, cib_notify_replace)
	&& pcmk__str_eq(type, T_CIB_REPLACE_NOTIFY, pcmk__str_casei)) {
	do_send = TRUE;

	} else if (pcmk_is_set(client->flags, cib_notify_confirm)
	&& pcmk__str_eq(type, T_CIB_UPDATE_CONFIRM, pcmk__str_casei)) {
	do_send = TRUE;

	} else if (pcmk_is_set(client->flags, cib_notify_pre)
	&& pcmk__str_eq(type, T_CIB_PRE_NOTIFY, pcmk__str_casei)) {
	do_send = TRUE;

	} else if (pcmk_is_set(client->flags, cib_notify_post)
	&& pcmk__str_eq(type, T_CIB_POST_NOTIFY, pcmk__str_casei)) {

	do_send = TRUE;
	}

	if (do_send) {
	switch (PCMK__CLIENT_TYPE(client)) {
	case pcmk__client_ipc:
	rc = pcmk__ipc_send_iov(client, update->iov,
	crm_ipc_server_event);
	if (rc != pcmk_rc_ok) {
	crm_warn("Could not notify client %s: %s " CRM_XS " id=%s",
	pcmk__client_name(client), pcmk_rc_str(rc),
	client->id);
	}
	break;
	#ifdef HAVE_GNUTLS_GNUTLS_H
	case pcmk__client_tls:
	#endif
	case pcmk__client_tcp:
	crm_debug("Sent %s notification to client %s (id %s)",
	type, pcmk__client_name(client), client->id);
	pcmk__remote_send_xml(client->remote, update->msg);
	break;
	default:
	crm_err("Unknown transport for client %s "
	CRM_XS " flags=%#016" PRIx64,
	pcmk__client_name(client), client->flags);
	}
	}
	}

	static void
	cib_notify_send(xmlNode * xml)
	{
	struct iovec *iov;
	struct cib_notification_s update;

	ssize_t bytes = 0;
	int rc = pcmk__ipc_prepare_iov(0, xml, 0, &iov, &bytes);

	if (rc == pcmk_rc_ok) {
	update.msg = xml;
	update.iov = iov;
	update.iov_size = bytes;
	pcmk__foreach_ipc_client(cib_notify_send_one, &update);

	} else {
	crm_notice("Could not notify clients: %s " CRM_XS " rc=%d",
	pcmk_rc_str(rc), rc);
	}
	pcmk_free_ipc_event(iov);
	}

	void
	cib_diff_notify(int options, const char client, const char call_id, const char *op,
	xmlNode * update, int result, xmlNode * diff)
	{
	int add_updates = 0;
	int add_epoch = 0;
	int add_admin_epoch = 0;

	int del_updates = 0;
	int del_epoch = 0;
	int del_admin_epoch = 0;

	int log_level = LOG_TRACE;

	if (diff == NULL) {
	return;
	}

	if (result != pcmk_ok) {
	log_level = LOG_WARNING;
	}

	cib_diff_version_details(diff, &add_admin_epoch, &add_epoch, &add_updates,
	&del_admin_epoch, &del_epoch, &del_updates);

	if (add_updates != del_updates) {
	do_crm_log(log_level,
	"Update (client: %s%s%s): %d.%d.%d -> %d.%d.%d (%s)",
	client, call_id ? ", call:" : "", call_id ? call_id : "",
	del_admin_epoch, del_epoch, del_updates,
	add_admin_epoch, add_epoch, add_updates, pcmk_strerror(result));

	} else if (diff != NULL) {
	do_crm_log(log_level,
	"Local-only Change (client:%s%s%s): %d.%d.%d (%s)",
	client, call_id ? ", call: " : "", call_id ? call_id : "",
	add_admin_epoch, add_epoch, add_updates, pcmk_strerror(result));
	}

	do_cib_notify(options, op, update, result, diff, T_CIB_DIFF_NOTIFY);
	}

	static void
	do_cib_notify(int options, const char op, xmlNode update,
	int result, xmlNode * result_data, const char *msg_type)
	{
	xmlNode *update_msg = NULL;
	const char *id = NULL;

	update_msg = create_xml_node(NULL, "notify");

	if (result_data != NULL) {
	id = crm_element_value(result_data, XML_ATTR_ID);
	}

	crm_xml_add(update_msg, F_TYPE, T_CIB_NOTIFY);
	crm_xml_add(update_msg, F_SUBTYPE, msg_type);
	crm_xml_add(update_msg, F_CIB_OPERATION, op);
	crm_xml_add_int(update_msg, F_CIB_RC, result);

	if (id != NULL) {
	crm_xml_add(update_msg, F_CIB_OBJID, id);
	}

	if (update != NULL) {
	crm_trace("Setting type to update->name: %s", crm_element_name(update));
	crm_xml_add(update_msg, F_CIB_OBJTYPE, crm_element_name(update));

	} else if (result_data != NULL) {
	crm_trace("Setting type to new_obj->name: %s", crm_element_name(result_data));
	crm_xml_add(update_msg, F_CIB_OBJTYPE, crm_element_name(result_data));

	} else {
	crm_trace("Not Setting type");
	}

	attach_cib_generation(update_msg, "cib_generation", the_cib);
	if (update != NULL) {
	add_message_xml(update_msg, F_CIB_UPDATE, update);
	}
	if (result_data != NULL) {
	add_message_xml(update_msg, F_CIB_UPDATE_RESULT, result_data);
	}

	cib_notify_send(update_msg);
	free_xml(update_msg);
	}

	void
	attach_cib_generation(xmlNode * msg, const char field, xmlNode a_cib)
	{
	xmlNode *generation = create_xml_node(NULL, XML_CIB_TAG_GENERATION_TUPPLE);

	if (a_cib != NULL) {
	copy_in_properties(generation, a_cib);
	}
	add_message_xml(msg, field, generation);
	free_xml(generation);
	}

	void
	-cib_replace_notify(const char origin, xmlNode update, int result, xmlNode * diff, int change_section)
	+cib_replace_notify(const char origin, xmlNode update, int result,
	+ xmlNode *diff, uint32_t change_section)
	{
	xmlNode *replace_msg = NULL;

	int add_updates = 0;
	int add_epoch = 0;
	int add_admin_epoch = 0;

	int del_updates = 0;
	int del_epoch = 0;
	int del_admin_epoch = 0;

	if (diff == NULL) {
	return;
	}

	cib_diff_version_details(diff, &add_admin_epoch, &add_epoch, &add_updates,
	&del_admin_epoch, &del_epoch, &del_updates);

	if (del_updates < 0) {
	crm_log_xml_debug(diff, "Bad replace diff");
	}

	if (add_updates != del_updates) {
	crm_info("Replaced: %d.%d.%d -> %d.%d.%d from %s",
	del_admin_epoch, del_epoch, del_updates,
	add_admin_epoch, add_epoch, add_updates,
	pcmk__s(origin, "unspecified peer"));
	} else if (diff != NULL) {
	crm_info("Local-only Replace: %d.%d.%d from %s",
	add_admin_epoch, add_epoch, add_updates,
	pcmk__s(origin, "unspecified peer"));
	}

	replace_msg = create_xml_node(NULL, "notify-replace");
	crm_xml_add(replace_msg, F_TYPE, T_CIB_NOTIFY);
	crm_xml_add(replace_msg, F_SUBTYPE, T_CIB_REPLACE_NOTIFY);
	crm_xml_add(replace_msg, F_CIB_OPERATION, PCMK__CIB_REQUEST_REPLACE);
	crm_xml_add_int(replace_msg, F_CIB_RC, result);
	- crm_xml_add_int(replace_msg, F_CIB_CHANGE_SECTION, change_section);
	+ crm_xml_add_ll(replace_msg, F_CIB_CHANGE_SECTION,
	+ (long long) change_section);
	attach_cib_generation(replace_msg, "cib-replace-generation", update);

	crm_log_xml_trace(replace_msg, "CIB Replaced");

	cib_notify_send(replace_msg);
	free_xml(replace_msg);
	}
	diff --git a/daemons/based/pacemaker-based.h b/daemons/based/pacemaker-based.h
	index 92f9e369a0..b2683938a7 100644
	--- a/daemons/based/pacemaker-based.h
	+++ b/daemons/based/pacemaker-based.h
	@@ -1,157 +1,157 @@
	/*
	* Copyright 2004-2022 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#ifndef PACEMAKER_BASED__H
	# define PACEMAKER_BASED__H

	#include <stdio.h>
	#include <sys/types.h>
	#include <unistd.h>
	#include <stdlib.h>
	#include <glib.h>
	#include <errno.h>
	#include <fcntl.h>

	#include <crm/crm.h>
	#include <crm/cib.h>
	#include <crm/common/xml.h>
	#include <crm/cluster.h>
	#include <crm/common/ipc_internal.h>
	#include <crm/common/mainloop.h>
	#include <crm/cib/internal.h>

	#ifdef HAVE_GNUTLS_GNUTLS_H
	# include <gnutls/gnutls.h>
	#endif

	// CIB-specific client flags
	enum cib_client_flags {
	// Notifications
	cib_notify_pre = (UINT64_C(1) << 0),
	cib_notify_post = (UINT64_C(1) << 1),
	cib_notify_replace = (UINT64_C(1) << 2),
	cib_notify_confirm = (UINT64_C(1) << 3),
	cib_notify_diff = (UINT64_C(1) << 4),

	// Whether client is another cluster daemon
	cib_is_daemon = (UINT64_C(1) << 12),
	};

	typedef struct cib_operation_s {
	const char *operation;
	gboolean modifies_cib;
	gboolean needs_privileges;
	gboolean needs_quorum;
	int (prepare) (xmlNode , xmlNode , const char );
	int (cleanup) (int, xmlNode , xmlNode *);
	int (fn) (const char , int, const char , xmlNode ,
	xmlNode , xmlNode , xmlNode , xmlNode );
	} cib_operation_t;

	extern bool based_is_primary;
	extern GHashTable *peer_hash;
	extern GHashTable *config_hash;
	extern xmlNode *the_cib;
	extern crm_trigger_t *cib_writer;
	extern gboolean cib_writes_enabled;

	extern GMainLoop *mainloop;
	extern crm_cluster_t crm_cluster;
	extern GHashTable *local_notify_queue;
	extern gboolean legacy_mode;
	extern gboolean stand_alone;
	extern gboolean cib_shutdown_flag;
	extern const char *cib_root;
	extern char *cib_our_uname;
	extern int cib_status;
	extern FILE *msg_cib_strm;

	extern struct qb_ipcs_service_handlers ipc_ro_callbacks;
	extern struct qb_ipcs_service_handlers ipc_rw_callbacks;
	extern qb_ipcs_service_t *ipcs_ro;
	extern qb_ipcs_service_t *ipcs_rw;
	extern qb_ipcs_service_t *ipcs_shm;

	void cib_peer_callback(xmlNode msg, void private_data);
	void cib_common_callback_worker(uint32_t id, uint32_t flags,
	xmlNode op_request, pcmk__client_t cib_client,
	gboolean privileged);
	void cib_shutdown(int nsig);
	void initiate_exit(void);
	void terminate_cib(const char *caller, int fast);
	gboolean cib_legacy_mode(void);

	gboolean uninitializeCib(void);
	xmlNode readCibXml(char buffer);
	xmlNode readCibXmlFile(const char dir, const char *file,
	gboolean discard_status);
	int activateCibXml(xmlNode doc, gboolean to_disk, const char op);

	xmlNode createCibRequest(gboolean isLocal, const char operation,
	const char section, const char verbose,
	xmlNode *data);
	int cib_process_shutdown_req(const char op, int options, const char section,
	xmlNode req, xmlNode input,
	xmlNode existing_cib, xmlNode *result_cib,
	xmlNode **answer);
	int cib_process_default(const char op, int options, const char section,
	xmlNode req, xmlNode input, xmlNode *existing_cib,
	xmlNode result_cib, xmlNode answer);
	int cib_process_ping(const char op, int options, const char section,
	xmlNode req, xmlNode input, xmlNode *existing_cib,
	xmlNode result_cib, xmlNode answer);
	int cib_process_readwrite(const char op, int options, const char section,
	xmlNode req, xmlNode input, xmlNode *existing_cib,
	xmlNode result_cib, xmlNode answer);
	int cib_process_replace_svr(const char op, int options, const char section,
	xmlNode req, xmlNode input, xmlNode *existing_cib,
	xmlNode result_cib, xmlNode answer);
	int cib_server_process_diff(const char op, int options, const char section,
	xmlNode req, xmlNode input, xmlNode *existing_cib,
	xmlNode result_cib, xmlNode answer);
	int cib_process_sync(const char op, int options, const char section,
	xmlNode req, xmlNode input, xmlNode *existing_cib,
	xmlNode result_cib, xmlNode answer);
	int cib_process_sync_one(const char op, int options, const char section,
	xmlNode req, xmlNode input, xmlNode *existing_cib,
	xmlNode result_cib, xmlNode answer);
	int cib_process_delete_absolute(const char *op, int options,
	const char section, xmlNode req,
	xmlNode input, xmlNode existing_cib,
	xmlNode result_cib, xmlNode answer);
	int cib_process_upgrade_server(const char op, int options, const char section,
	xmlNode req, xmlNode input,
	xmlNode existing_cib, xmlNode *result_cib,
	xmlNode **answer);
	void send_sync_request(const char *host);

	xmlNode cib_msg_copy(xmlNode msg, gboolean with_data);
	xmlNode cib_construct_reply(xmlNode request, xmlNode *output, int rc);
	int cib_get_operation_id(const char op, int operation);
	cib_op_t *cib_op_func(int call_type);
	gboolean cib_op_modifies(int call_type);
	int cib_op_prepare(int call_type, xmlNode request, xmlNode *input,
	const char **section);
	int cib_op_cleanup(int call_type, int options, xmlNode **input,
	xmlNode **output);
	int cib_op_can_run(int call_type, int call_options, gboolean privileged,
	gboolean global_update);
	void cib_diff_notify(int options, const char client, const char call_id,
	const char op, xmlNode update, int result,
	xmlNode *old_cib);
	void cib_replace_notify(const char origin, xmlNode update, int result,
	- xmlNode *diff, int change_section);
	+ xmlNode *diff, uint32_t change_section);

	static inline const char *
	cib_config_lookup(const char *opt)
	{
	return g_hash_table_lookup(config_hash, opt);
	}

	#endif // PACEMAKER_BASED__H
	diff --git a/daemons/controld/controld_based.c b/daemons/controld/controld_based.c
	index 23f0baa0e9..6334ef2bfd 100644
	--- a/daemons/controld/controld_based.c
	+++ b/daemons/controld/controld_based.c
	@@ -1,353 +1,364 @@
	/*
	* Copyright 2004-2022 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <unistd.h> /* sleep */

	#include <crm/common/alerts_internal.h>
	#include <crm/common/xml.h>
	#include <crm/crm.h>
	#include <crm/msg_xml.h>

	#include <pacemaker-controld.h>

	int cib_retries = 0;

	void
	do_cib_updated(const char event, xmlNode msg)
	{
	if (pcmk__alert_in_patchset(msg, TRUE)) {
	mainloop_set_trigger(config_read);
	}
	}

	void
	do_cib_replaced(const char event, xmlNode msg)
	{
	- int change_section = cib_change_section_nodes \| cib_change_section_status;
	+ uint32_t change_section = cib_change_section_nodes
	+ \|cib_change_section_status;
	+ long long value = 0;

	crm_debug("Updating the CIB after a replace: DC=%s", pcmk__btoa(AM_I_DC));
	if (AM_I_DC == FALSE) {
	return;

	} else if ((fsa_state == S_FINALIZE_JOIN)
	&& pcmk_is_set(fsa_input_register, R_CIB_ASKED)) {
	/* no need to restart the join - we asked for this replace op */
	return;
	}

	- crm_element_value_int(msg, F_CIB_CHANGE_SECTION, &change_section);
	- if (change_section & (cib_change_section_nodes \| cib_change_section_status)) {
	+ if ((crm_element_value_ll(msg, F_CIB_CHANGE_SECTION, &value) < 0)
	+ \|\| (value < 0) \|\| (value > UINT32_MAX)) {
	+
	+ crm_trace("Couldn't parse '%s' from message", F_CIB_CHANGE_SECTION);
	+ } else {
	+ change_section = (uint32_t) value;
	+ }
	+
	+ if (pcmk_any_flags_set(change_section, cib_change_section_nodes
	+ \|cib_change_section_status)) {
	+
	/* start the join process again so we get everyone's LRM status */
	populate_cib_nodes(node_update_quick\|node_update_all, __func__);

	register_fsa_input(C_FSA_INTERNAL, I_ELECTION, NULL);
	}
	}

	void
	controld_disconnect_cib_manager(void)
	{
	CRM_ASSERT(fsa_cib_conn != NULL);

	crm_info("Disconnecting from the CIB manager");

	controld_clear_fsa_input_flags(R_CIB_CONNECTED);

	fsa_cib_conn->cmds->del_notify_callback(fsa_cib_conn, T_CIB_REPLACE_NOTIFY, do_cib_replaced);
	fsa_cib_conn->cmds->del_notify_callback(fsa_cib_conn, T_CIB_DIFF_NOTIFY, do_cib_updated);
	cib_free_callbacks(fsa_cib_conn);
	if (fsa_cib_conn->state != cib_disconnected) {
	fsa_cib_conn->cmds->set_secondary(fsa_cib_conn,
	cib_scope_local\|cib_discard_reply);
	fsa_cib_conn->cmds->signoff(fsa_cib_conn);
	}

	crm_notice("Disconnected from the CIB manager");
	}

	/* A_CIB_STOP, A_CIB_START, O_CIB_RESTART */
	void
	do_cib_control(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	CRM_ASSERT(fsa_cib_conn != NULL);

	if (action & A_CIB_STOP) {

	if (fsa_cib_conn->state != cib_disconnected && last_resource_update != 0) {
	crm_info("Waiting for resource update %d to complete", last_resource_update);
	crmd_fsa_stall(FALSE);
	return;
	}

	controld_disconnect_cib_manager();

	}

	if (action & A_CIB_START) {
	int rc = pcmk_ok;

	if (cur_state == S_STOPPING) {
	crm_err("Ignoring request to connect to the CIB manager after shutdown");
	return;
	}

	rc = fsa_cib_conn->cmds->signon(fsa_cib_conn, CRM_SYSTEM_CRMD, cib_command_nonblocking);

	if (rc != pcmk_ok) {
	/* a short wait that usually avoids stalling the FSA */
	sleep(1);
	rc = fsa_cib_conn->cmds->signon(fsa_cib_conn, CRM_SYSTEM_CRMD, cib_command_nonblocking);
	}

	if (rc != pcmk_ok) {
	crm_info("Could not connect to the CIB manager: %s", pcmk_strerror(rc));

	} else if (pcmk_ok !=
	fsa_cib_conn->cmds->set_connection_dnotify(fsa_cib_conn,
	crmd_cib_connection_destroy)) {
	crm_err("Could not set dnotify callback");

	} else if (pcmk_ok !=
	fsa_cib_conn->cmds->add_notify_callback(fsa_cib_conn, T_CIB_REPLACE_NOTIFY,
	do_cib_replaced)) {
	crm_err("Could not set CIB notification callback (replace)");

	} else if (pcmk_ok !=
	fsa_cib_conn->cmds->add_notify_callback(fsa_cib_conn, T_CIB_DIFF_NOTIFY,
	do_cib_updated)) {
	crm_err("Could not set CIB notification callback (update)");

	} else {
	controld_set_fsa_input_flags(R_CIB_CONNECTED);
	cib_retries = 0;
	}

	if (!pcmk_is_set(fsa_input_register, R_CIB_CONNECTED)) {

	cib_retries++;
	crm_warn("Couldn't complete CIB registration %d"
	" times... pause and retry", cib_retries);

	if (cib_retries < 30) {
	controld_start_timer(wait_timer);
	crmd_fsa_stall(FALSE);

	} else {
	crm_err("Could not complete CIB"
	" registration %d times..." " hard error", cib_retries);
	register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
	}
	}
	}
	}

	/*!
	* \internal
	* \brief Get CIB call options to use local scope if primary is unavailable
	*
	* \return CIB call options
	*/
	int
	crmd_cib_smart_opt(void)
	{
	int call_opt = cib_quorum_override;

	if (fsa_state == S_ELECTION \|\| fsa_state == S_PENDING) {
	crm_info("Sending update to local CIB in state: %s", fsa_state2string(fsa_state));
	cib__set_call_options(call_opt, "update", cib_scope_local);
	}
	return call_opt;
	}

	/*!
	* \internal
	* \brief Check whether an action type should be recorded in the CIB
	*
	* \param[in] action Action type
	*
	* \return TRUE if action should be recorded, FALSE otherwise
	*/
	bool
	controld_action_is_recordable(const char *action)
	{
	return !pcmk__strcase_any_of(action, CRMD_ACTION_CANCEL, CRMD_ACTION_DELETE,
	CRMD_ACTION_NOTIFY, CRMD_ACTION_METADATA, NULL);
	}

	static void
	cib_delete_callback(xmlNode msg, int call_id, int rc, xmlNode output,
	void *user_data)
	{
	char *desc = user_data;

	if (rc == 0) {
	crm_debug("Deletion of %s (via CIB call %d) succeeded", desc, call_id);
	} else {
	crm_warn("Deletion of %s (via CIB call %d) failed: %s " CRM_XS " rc=%d",
	desc, call_id, pcmk_strerror(rc), rc);
	}
	}

	// Searches for various portions of node_state to delete

	// Match a particular node's node_state (takes node name 1x)
	#define XPATH_NODE_STATE "//" XML_CIB_TAG_STATE "[@" XML_ATTR_UNAME "='%s']"

	// Node's lrm section (name 1x)
	#define XPATH_NODE_LRM XPATH_NODE_STATE "/" XML_CIB_TAG_LRM

	// Node's lrm_rsc_op entries and lrm_resource entries without lock (name 2x)
	#define XPATH_NODE_LRM_UNLOCKED XPATH_NODE_STATE "//" XML_LRM_TAG_RSC_OP \
	"\|" XPATH_NODE_STATE \
	"//" XML_LRM_TAG_RESOURCE \
	"[not(@" XML_CONFIG_ATTR_SHUTDOWN_LOCK ")]"

	// Node's transient_attributes section (name 1x)
	#define XPATH_NODE_ATTRS XPATH_NODE_STATE "/" XML_TAG_TRANSIENT_NODEATTRS

	// Everything under node_state (name 1x)
	#define XPATH_NODE_ALL XPATH_NODE_STATE "/*"

	// Unlocked history + transient attributes (name 3x)
	#define XPATH_NODE_ALL_UNLOCKED XPATH_NODE_LRM_UNLOCKED "\|" XPATH_NODE_ATTRS

	/*!
	* \internal
	* \brief Delete subsection of a node's CIB node_state
	*
	* \param[in] uname Desired node
	* \param[in] section Subsection of node_state to delete
	* \param[in] options CIB call options to use
	*/
	void
	controld_delete_node_state(const char *uname, enum controld_section_e section,
	int options)
	{
	char *xpath = NULL;
	char *desc = NULL;

	CRM_CHECK(uname != NULL, return);
	switch (section) {
	case controld_section_lrm:
	xpath = crm_strdup_printf(XPATH_NODE_LRM, uname);
	desc = crm_strdup_printf("resource history for node %s", uname);
	break;
	case controld_section_lrm_unlocked:
	xpath = crm_strdup_printf(XPATH_NODE_LRM_UNLOCKED, uname, uname);
	desc = crm_strdup_printf("resource history (other than shutdown "
	"locks) for node %s", uname);
	break;
	case controld_section_attrs:
	xpath = crm_strdup_printf(XPATH_NODE_ATTRS, uname);
	desc = crm_strdup_printf("transient attributes for node %s", uname);
	break;
	case controld_section_all:
	xpath = crm_strdup_printf(XPATH_NODE_ALL, uname);
	desc = crm_strdup_printf("all state for node %s", uname);
	break;
	case controld_section_all_unlocked:
	xpath = crm_strdup_printf(XPATH_NODE_ALL_UNLOCKED,
	uname, uname, uname);
	desc = crm_strdup_printf("all state (other than shutdown locks) "
	"for node %s", uname);
	break;
	}

	if (fsa_cib_conn == NULL) {
	crm_warn("Unable to delete %s: no CIB connection", desc);
	free(desc);
	} else {
	int call_id;

	cib__set_call_options(options, "node state deletion",
	cib_quorum_override\|cib_xpath\|cib_multiple);
	call_id = fsa_cib_conn->cmds->remove(fsa_cib_conn, xpath, NULL, options);
	crm_info("Deleting %s (via CIB call %d) " CRM_XS " xpath=%s",
	desc, call_id, xpath);
	fsa_register_cib_callback(call_id, FALSE, desc, cib_delete_callback);
	// CIB library handles freeing desc
	}
	free(xpath);
	}

	// Takes node name and resource ID
	#define XPATH_RESOURCE_HISTORY "//" XML_CIB_TAG_STATE \
	"[@" XML_ATTR_UNAME "='%s']/" \
	XML_CIB_TAG_LRM "/" XML_LRM_TAG_RESOURCES \
	"/" XML_LRM_TAG_RESOURCE \
	"[@" XML_ATTR_ID "='%s']"
	// @TODO could add "and @XML_CONFIG_ATTR_SHUTDOWN_LOCK" to limit to locks

	/*!
	* \internal
	* \brief Clear resource history from CIB for a given resource and node
	*
	* \param[in] rsc_id ID of resource to be cleared
	* \param[in] node Node whose resource history should be cleared
	* \param[in] user_name ACL user name to use
	* \param[in] call_options CIB call options
	*
	* \return Standard Pacemaker return code
	*/
	int
	controld_delete_resource_history(const char rsc_id, const char node,
	const char *user_name, int call_options)
	{
	char *desc = NULL;
	char *xpath = NULL;
	int rc = pcmk_rc_ok;

	CRM_CHECK((rsc_id != NULL) && (node != NULL), return EINVAL);

	desc = crm_strdup_printf("resource history for %s on %s", rsc_id, node);
	if (fsa_cib_conn == NULL) {
	crm_err("Unable to clear %s: no CIB connection", desc);
	free(desc);
	return ENOTCONN;
	}

	// Ask CIB to delete the entry
	xpath = crm_strdup_printf(XPATH_RESOURCE_HISTORY, node, rsc_id);
	rc = cib_internal_op(fsa_cib_conn, PCMK__CIB_REQUEST_DELETE, NULL, xpath,
	NULL, NULL, call_options\|cib_xpath, user_name);

	if (rc < 0) {
	rc = pcmk_legacy2rc(rc);
	crm_err("Could not delete resource status of %s on %s%s%s: %s "
	CRM_XS " rc=%d", rsc_id, node,
	(user_name? " for user " : ""), (user_name? user_name : ""),
	pcmk_rc_str(rc), rc);
	free(desc);
	free(xpath);
	return rc;
	}

	if (pcmk_is_set(call_options, cib_sync_call)) {
	if (pcmk_is_set(call_options, cib_dryrun)) {
	crm_debug("Deletion of %s would succeed", desc);
	} else {
	crm_debug("Deletion of %s succeeded", desc);
	}
	free(desc);

	} else {
	crm_info("Clearing %s (via CIB call %d) " CRM_XS " xpath=%s",
	desc, rc, xpath);
	fsa_register_cib_callback(rc, FALSE, desc, cib_delete_callback);
	// CIB library handles freeing desc
	}

	free(xpath);
	return pcmk_rc_ok;
	}
	diff --git a/daemons/controld/controld_te_callbacks.c b/daemons/controld/controld_te_callbacks.c
	index 123edc0ff3..87ad861a2a 100644
	--- a/daemons/controld/controld_te_callbacks.c
	+++ b/daemons/controld/controld_te_callbacks.c
	@@ -1,678 +1,685 @@
	/*
	* Copyright 2004-2022 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <sys/stat.h>

	#include <crm/crm.h>
	#include <crm/common/xml.h>
	#include <crm/common/xml_internal.h>
	#include <crm/msg_xml.h>
	#include <crm/cluster.h> /* For ONLINESTATUS etc */

	#include <pacemaker-controld.h>

	void te_update_confirm(const char event, xmlNode msg);

	extern char *te_uuid;
	gboolean shuttingdown = FALSE;
	pcmk__graph_t *transition_graph;
	crm_trigger_t *transition_trigger = NULL;

	/* #define RSC_OP_TEMPLATE "//"XML_TAG_DIFF_ADDED"//"XML_TAG_CIB"//"XML_CIB_TAG_STATE"[@uname='%s']"//"XML_LRM_TAG_RSC_OP"[@id='%s]" */
	#define RSC_OP_TEMPLATE "//"XML_TAG_DIFF_ADDED"//"XML_TAG_CIB"//"XML_LRM_TAG_RSC_OP"[@id='%s']"

	// An explicit shutdown-lock of 0 means the lock has been cleared
	static bool
	shutdown_lock_cleared(xmlNode *lrm_resource)
	{
	time_t shutdown_lock = 0;

	return (crm_element_value_epoch(lrm_resource, XML_CONFIG_ATTR_SHUTDOWN_LOCK,
	&shutdown_lock) == pcmk_ok)
	&& (shutdown_lock == 0);
	}

	static void
	te_update_diff_v1(const char event, xmlNode diff)
	{
	int lpc, max;
	xmlXPathObject *xpathObj = NULL;

	CRM_CHECK(diff != NULL, return);

	xml_log_patchset(LOG_TRACE, __func__, diff);
	if (cib_config_changed(NULL, NULL, &diff)) {
	abort_transition(INFINITY, pcmk__graph_restart, "Non-status change",
	diff);
	goto bail; /* configuration changed */
	}

	/* Tickets Attributes - Added/Updated */
	xpathObj =
	xpath_search(diff,
	"//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_ADDED "//" XML_CIB_TAG_TICKETS);
	if (numXpathResults(xpathObj) > 0) {
	xmlNode *aborted = getXpathResult(xpathObj, 0);

	abort_transition(INFINITY, pcmk__graph_restart,
	"Ticket attribute: update", aborted);
	goto bail;

	}
	freeXpathObject(xpathObj);

	/* Tickets Attributes - Removed */
	xpathObj =
	xpath_search(diff,
	"//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_REMOVED "//" XML_CIB_TAG_TICKETS);
	if (numXpathResults(xpathObj) > 0) {
	xmlNode *aborted = getXpathResult(xpathObj, 0);

	abort_transition(INFINITY, pcmk__graph_restart,
	"Ticket attribute: removal", aborted);
	goto bail;
	}
	freeXpathObject(xpathObj);

	/* Transient Attributes - Removed */
	xpathObj =
	xpath_search(diff,
	"//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_REMOVED "//"
	XML_TAG_TRANSIENT_NODEATTRS);
	if (numXpathResults(xpathObj) > 0) {
	xmlNode *aborted = getXpathResult(xpathObj, 0);

	abort_transition(INFINITY, pcmk__graph_restart,
	"Transient attribute: removal", aborted);
	goto bail;

	}
	freeXpathObject(xpathObj);

	// Check for lrm_resource entries
	xpathObj = xpath_search(diff,
	"//" F_CIB_UPDATE_RESULT
	"//" XML_TAG_DIFF_ADDED
	"//" XML_LRM_TAG_RESOURCE);
	max = numXpathResults(xpathObj);

	/*
	* Updates by, or in response to, graph actions will never affect more than
	* one resource at a time, so such updates indicate an LRM refresh. In that
	* case, start a new transition rather than check each result individually,
	* which can result in _huge_ speedups in large clusters.
	*
	* Unfortunately, we can only do so when there are no pending actions.
	* Otherwise, we could mistakenly throw away those results here, and
	* the cluster will stall waiting for them and time out the operation.
	*/
	if ((transition_graph->pending == 0) && (max > 1)) {
	crm_debug("Ignoring resource operation updates due to history refresh of %d resources",
	max);
	crm_log_xml_trace(diff, "lrm-refresh");
	abort_transition(INFINITY, pcmk__graph_restart, "History refresh",
	NULL);
	goto bail;
	}

	if (max == 1) {
	xmlNode *lrm_resource = getXpathResult(xpathObj, 0);

	if (shutdown_lock_cleared(lrm_resource)) {
	// @TODO would be more efficient to abort once after transition done
	abort_transition(INFINITY, pcmk__graph_restart,
	"Shutdown lock cleared", lrm_resource);
	// Still process results, so we stop timers and update failcounts
	}
	}
	freeXpathObject(xpathObj);

	/* Process operation updates */
	xpathObj =
	xpath_search(diff,
	"//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_ADDED "//" XML_LRM_TAG_RSC_OP);
	max = numXpathResults(xpathObj);
	if (max > 0) {
	int lpc = 0;

	for (lpc = 0; lpc < max; lpc++) {
	xmlNode *rsc_op = getXpathResult(xpathObj, lpc);
	const char *node = get_node_id(rsc_op);

	process_graph_event(rsc_op, node);
	}
	}
	freeXpathObject(xpathObj);

	/* Detect deleted (as opposed to replaced or added) actions - eg. crm_resource -C */
	xpathObj = xpath_search(diff, "//" XML_TAG_DIFF_REMOVED "//" XML_LRM_TAG_RSC_OP);
	max = numXpathResults(xpathObj);
	for (lpc = 0; lpc < max; lpc++) {
	int path_max = 0;
	const char *op_id = NULL;
	char *rsc_op_xpath = NULL;
	xmlXPathObject *op_match = NULL;
	xmlNode *match = getXpathResult(xpathObj, lpc);

	CRM_LOG_ASSERT(match != NULL);
	if(match == NULL) { continue; };

	op_id = ID(match);

	path_max = strlen(RSC_OP_TEMPLATE) + strlen(op_id) + 1;
	rsc_op_xpath = calloc(1, path_max);
	snprintf(rsc_op_xpath, path_max, RSC_OP_TEMPLATE, op_id);

	op_match = xpath_search(diff, rsc_op_xpath);
	if (numXpathResults(op_match) == 0) {
	/* Prevent false positives by matching cancelations too */
	const char *node = get_node_id(match);
	pcmk__graph_action_t *cancelled = get_cancel_action(op_id, node);

	if (cancelled == NULL) {
	crm_debug("No match for deleted action %s (%s on %s)", rsc_op_xpath, op_id,
	node);
	abort_transition(INFINITY, pcmk__graph_restart,
	"Resource op removal", match);
	freeXpathObject(op_match);
	free(rsc_op_xpath);
	goto bail;

	} else {
	crm_debug("Deleted lrm_rsc_op %s on %s was for graph event %d",
	op_id, node, cancelled->id);
	}
	}

	freeXpathObject(op_match);
	free(rsc_op_xpath);
	}

	bail:
	freeXpathObject(xpathObj);
	}

	static void
	process_lrm_resource_diff(xmlNode lrm_resource, const char node)
	{
	for (xmlNode *rsc_op = pcmk__xml_first_child(lrm_resource); rsc_op != NULL;
	rsc_op = pcmk__xml_next(rsc_op)) {
	process_graph_event(rsc_op, node);
	}
	if (shutdown_lock_cleared(lrm_resource)) {
	// @TODO would be more efficient to abort once after transition done
	abort_transition(INFINITY, pcmk__graph_restart, "Shutdown lock cleared",
	lrm_resource);
	}
	}

	static void
	process_resource_updates(const char node, xmlNode xml, xmlNode *change,
	const char op, const char xpath)
	{
	xmlNode *rsc = NULL;

	if (xml == NULL) {
	return;
	}

	if (strcmp(TYPE(xml), XML_CIB_TAG_LRM) == 0) {
	xml = first_named_child(xml, XML_LRM_TAG_RESOURCES);
	CRM_CHECK(xml != NULL, return);
	}

	CRM_CHECK(strcmp(TYPE(xml), XML_LRM_TAG_RESOURCES) == 0, return);

	/*
	* Updates by, or in response to, TE actions will never contain updates
	* for more than one resource at a time, so such updates indicate an
	* LRM refresh.
	*
	* In that case, start a new transition rather than check each result
	* individually, which can result in _huge_ speedups in large clusters.
	*
	* Unfortunately, we can only do so when there are no pending actions.
	* Otherwise, we could mistakenly throw away those results here, and
	* the cluster will stall waiting for them and time out the operation.
	*/
	if ((transition_graph->pending == 0)
	&& xml->children && xml->children->next) {

	crm_log_xml_trace(change, "lrm-refresh");
	abort_transition(INFINITY, pcmk__graph_restart, "History refresh",
	NULL);
	return;
	}

	for (rsc = pcmk__xml_first_child(xml); rsc != NULL;
	rsc = pcmk__xml_next(rsc)) {
	crm_trace("Processing %s", ID(rsc));
	process_lrm_resource_diff(rsc, node);
	}
	}

	static char extract_node_uuid(const char xpath)
	{
	char *mutable_path = strdup(xpath);
	char *node_uuid = NULL;
	char *search = NULL;
	char *match = NULL;

	match = strstr(mutable_path, "node_state[@id=\'");
	if (match == NULL) {
	free(mutable_path);
	return NULL;
	}
	match += strlen("node_state[@id=\'");

	search = strchr(match, '\'');
	if (search == NULL) {
	free(mutable_path);
	return NULL;
	}
	search[0] = 0;

	node_uuid = strdup(match);
	free(mutable_path);
	return node_uuid;
	}

	static void
	abort_unless_down(const char xpath, const char op, xmlNode *change,
	const char *reason)
	{
	char *node_uuid = NULL;
	pcmk__graph_action_t *down = NULL;

	if(!pcmk__str_eq(op, "delete", pcmk__str_casei)) {
	abort_transition(INFINITY, pcmk__graph_restart, reason, change);
	return;
	}

	node_uuid = extract_node_uuid(xpath);
	if(node_uuid == NULL) {
	crm_err("Could not extract node ID from %s", xpath);
	abort_transition(INFINITY, pcmk__graph_restart, reason, change);
	return;
	}

	down = match_down_event(node_uuid);
	if (down == NULL) {
	crm_trace("Not expecting %s to be down (%s)", node_uuid, xpath);
	abort_transition(INFINITY, pcmk__graph_restart, reason, change);
	} else {
	crm_trace("Expecting changes to %s (%s)", node_uuid, xpath);
	}
	free(node_uuid);
	}

	static void
	process_op_deletion(const char xpath, xmlNode change)
	{
	char *mutable_key = strdup(xpath);
	char *key;
	char *node_uuid;

	// Extract the part of xpath between last pair of single quotes
	key = strrchr(mutable_key, '\'');
	if (key != NULL) {
	*key = '\0';
	key = strrchr(mutable_key, '\'');
	}
	if (key == NULL) {
	crm_warn("Ignoring malformed CIB update (resource deletion of %s)",
	xpath);
	free(mutable_key);
	return;
	}
	++key;

	node_uuid = extract_node_uuid(xpath);
	if (confirm_cancel_action(key, node_uuid) == FALSE) {
	abort_transition(INFINITY, pcmk__graph_restart,
	"Resource operation removal", change);
	}
	free(mutable_key);
	free(node_uuid);
	}

	static void
	process_delete_diff(const char xpath, const char op, xmlNode *change)
	{
	if (strstr(xpath, "/" XML_LRM_TAG_RSC_OP "[")) {
	process_op_deletion(xpath, change);

	} else if (strstr(xpath, "/" XML_CIB_TAG_LRM "[")) {
	abort_unless_down(xpath, op, change, "Resource state removal");

	} else if (strstr(xpath, "/" XML_CIB_TAG_STATE "[")) {
	abort_unless_down(xpath, op, change, "Node state removal");

	} else {
	crm_trace("Ignoring delete of %s", xpath);
	}
	}

	static void
	process_node_state_diff(xmlNode state, xmlNode change, const char *op,
	const char *xpath)
	{
	xmlNode *lrm = first_named_child(state, XML_CIB_TAG_LRM);

	process_resource_updates(ID(state), lrm, change, op, xpath);
	}

	static void
	process_status_diff(xmlNode status, xmlNode change, const char *op,
	const char *xpath)
	{
	for (xmlNode *state = pcmk__xml_first_child(status); state != NULL;
	state = pcmk__xml_next(state)) {
	process_node_state_diff(state, change, op, xpath);
	}
	}

	static void
	process_cib_diff(xmlNode cib, xmlNode change, const char *op,
	const char *xpath)
	{
	xmlNode *status = first_named_child(cib, XML_CIB_TAG_STATUS);
	xmlNode *config = first_named_child(cib, XML_CIB_TAG_CONFIGURATION);

	if (status) {
	process_status_diff(status, change, op, xpath);
	}
	if (config) {
	abort_transition(INFINITY, pcmk__graph_restart,
	"Non-status-only change", change);
	}
	}

	static void
	te_update_diff_v2(xmlNode *diff)
	{
	crm_log_xml_trace(diff, "Patch:Raw");

	for (xmlNode *change = pcmk__xml_first_child(diff); change != NULL;
	change = pcmk__xml_next(change)) {

	xmlNode *match = NULL;
	const char *name = NULL;
	const char *xpath = crm_element_value(change, XML_DIFF_PATH);

	// Possible ops: create, modify, delete, move
	const char *op = crm_element_value(change, XML_DIFF_OP);

	// Ignore uninteresting updates
	if (op == NULL) {
	continue;

	} else if (xpath == NULL) {
	crm_trace("Ignoring %s change for version field", op);
	continue;

	- } else if (strcmp(op, "move") == 0) {
	+ } else if ((strcmp(op, "move") == 0)
	+ && (strstr(xpath,
	+ "/" XML_TAG_CIB "/" XML_CIB_TAG_CONFIGURATION
	+ "/" XML_CIB_TAG_RESOURCES) == NULL)) {
	+ /* We still need to consider moves within the resources section,
	+ * since they affect placement order.
	+ */
	crm_trace("Ignoring move change at %s", xpath);
	continue;
	}

	// Find the result of create/modify ops
	if (strcmp(op, "create") == 0) {
	match = change->children;

	} else if (strcmp(op, "modify") == 0) {
	match = first_named_child(change, XML_DIFF_RESULT);
	if(match) {
	match = match->children;
	}

	- } else if (strcmp(op, "delete") != 0) {
	+ } else if (!pcmk__str_any_of(op, "delete", "move", NULL)) {
	crm_warn("Ignoring malformed CIB update (%s operation on %s is unrecognized)",
	op, xpath);
	continue;
	}

	if (match) {
	if (match->type == XML_COMMENT_NODE) {
	crm_trace("Ignoring %s operation for comment at %s", op, xpath);
	continue;
	}
	name = (const char *)match->name;
	}

	crm_trace("Handling %s operation for %s%s%s",
	op, (xpath? xpath : "CIB"),
	(name? " matched by " : ""), (name? name : ""));

	if (strstr(xpath, "/" XML_TAG_CIB "/" XML_CIB_TAG_CONFIGURATION)) {
	abort_transition(INFINITY, pcmk__graph_restart,
	"Configuration change", change);
	break; // Won't be packaged with operation results we may be waiting for

	} else if (strstr(xpath, "/" XML_CIB_TAG_TICKETS)
	- \|\| pcmk__str_eq(name, XML_CIB_TAG_TICKETS, pcmk__str_casei)) {
	+ \|\| pcmk__str_eq(name, XML_CIB_TAG_TICKETS, pcmk__str_none)) {
	abort_transition(INFINITY, pcmk__graph_restart,
	"Ticket attribute change", change);
	break; // Won't be packaged with operation results we may be waiting for

	} else if (strstr(xpath, "/" XML_TAG_TRANSIENT_NODEATTRS "[")
	- \|\| pcmk__str_eq(name, XML_TAG_TRANSIENT_NODEATTRS, pcmk__str_casei)) {
	+ \|\| pcmk__str_eq(name, XML_TAG_TRANSIENT_NODEATTRS,
	+ pcmk__str_none)) {
	abort_unless_down(xpath, op, change, "Transient attribute change");
	break; // Won't be packaged with operation results we may be waiting for

	} else if (strcmp(op, "delete") == 0) {
	process_delete_diff(xpath, op, change);

	} else if (name == NULL) {
	crm_warn("Ignoring malformed CIB update (%s at %s has no result)",
	op, xpath);

	} else if (strcmp(name, XML_TAG_CIB) == 0) {
	process_cib_diff(match, change, op, xpath);

	} else if (strcmp(name, XML_CIB_TAG_STATUS) == 0) {
	process_status_diff(match, change, op, xpath);

	} else if (strcmp(name, XML_CIB_TAG_STATE) == 0) {
	process_node_state_diff(match, change, op, xpath);

	} else if (strcmp(name, XML_CIB_TAG_LRM) == 0) {
	process_resource_updates(ID(match), match, change, op, xpath);

	} else if (strcmp(name, XML_LRM_TAG_RESOURCES) == 0) {
	char *local_node = pcmk__xpath_node_id(xpath, "lrm");

	process_resource_updates(local_node, match, change, op, xpath);
	free(local_node);

	} else if (strcmp(name, XML_LRM_TAG_RESOURCE) == 0) {
	char *local_node = pcmk__xpath_node_id(xpath, "lrm");

	process_lrm_resource_diff(match, local_node);
	free(local_node);

	} else if (strcmp(name, XML_LRM_TAG_RSC_OP) == 0) {
	char *local_node = pcmk__xpath_node_id(xpath, "lrm");

	process_graph_event(match, local_node);
	free(local_node);

	} else {
	crm_warn("Ignoring malformed CIB update (%s at %s has unrecognized result %s)",
	op, xpath, name);
	}
	}
	}

	void
	te_update_diff(const char event, xmlNode msg)
	{
	xmlNode *diff = NULL;
	const char *op = NULL;
	int rc = -EINVAL;
	int format = 1;
	int p_add[] = { 0, 0, 0 };
	int p_del[] = { 0, 0, 0 };

	CRM_CHECK(msg != NULL, return);
	crm_element_value_int(msg, F_CIB_RC, &rc);

	if (transition_graph == NULL) {
	crm_trace("No graph");
	return;

	} else if (rc < pcmk_ok) {
	crm_trace("Filter rc=%d (%s)", rc, pcmk_strerror(rc));
	return;

	} else if (transition_graph->complete
	&& fsa_state != S_IDLE
	&& fsa_state != S_TRANSITION_ENGINE
	&& fsa_state != S_POLICY_ENGINE) {
	crm_trace("Filter state=%s (complete)", fsa_state2string(fsa_state));
	return;
	}

	op = crm_element_value(msg, F_CIB_OPERATION);
	diff = get_message_xml(msg, F_CIB_UPDATE_RESULT);

	xml_patch_versions(diff, p_add, p_del);
	crm_debug("Processing (%s) diff: %d.%d.%d -> %d.%d.%d (%s)", op,
	p_del[0], p_del[1], p_del[2], p_add[0], p_add[1], p_add[2],
	fsa_state2string(fsa_state));

	crm_element_value_int(diff, "format", &format);
	switch (format) {
	case 1:
	te_update_diff_v1(event, diff);
	break;
	case 2:
	te_update_diff_v2(diff);
	break;
	default:
	crm_warn("Ignoring malformed CIB update (unknown patch format %d)",
	format);
	}
	}

	void
	process_te_message(xmlNode * msg, xmlNode * xml_data)
	{
	const char *value = NULL;
	xmlXPathObject *xpathObj = NULL;
	int nmatches = 0;

	CRM_CHECK(msg != NULL, return);

	// Transition requests must specify transition engine as subsystem
	value = crm_element_value(msg, F_CRM_SYS_TO);
	if (pcmk__str_empty(value)
	\|\| !pcmk__str_eq(value, CRM_SYSTEM_TENGINE, pcmk__str_none)) {
	crm_info("Received invalid transition request: subsystem '%s' not '"
	CRM_SYSTEM_TENGINE "'", pcmk__s(value, ""));
	return;
	}

	// Only the lrm_invoke command is supported as a transition request
	value = crm_element_value(msg, F_CRM_TASK);
	if (!pcmk__str_eq(value, CRM_OP_INVOKE_LRM, pcmk__str_none)) {
	crm_info("Received invalid transition request: command '%s' not '"
	CRM_OP_INVOKE_LRM "'", pcmk__s(value, ""));
	return;
	}

	// Transition requests must be marked as coming from the executor
	value = crm_element_value(msg, F_CRM_SYS_FROM);
	if (!pcmk__str_eq(value, CRM_SYSTEM_LRMD, pcmk__str_none)) {
	crm_info("Received invalid transition request: from '%s' not '"
	CRM_SYSTEM_LRMD "'", pcmk__s(value, ""));
	return;
	}

	crm_debug("Processing transition request with ref='%s' origin='%s'",
	pcmk__s(crm_element_value(msg, F_CRM_REFERENCE), ""),
	pcmk__s(crm_element_value(msg, F_ORIG), ""));

	xpathObj = xpath_search(xml_data, "//" XML_LRM_TAG_RSC_OP);
	nmatches = numXpathResults(xpathObj);
	if (nmatches == 0) {
	crm_err("Received transition request with no results (bug?)");
	} else {
	for (int lpc = 0; lpc < nmatches; lpc++) {
	xmlNode *rsc_op = getXpathResult(xpathObj, lpc);
	const char *node = get_node_id(rsc_op);

	process_graph_event(rsc_op, node);
	}
	}
	freeXpathObject(xpathObj);
	}

	void
	cib_action_updated(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
	{
	if (rc < pcmk_ok) {
	crm_err("Update %d FAILED: %s", call_id, pcmk_strerror(rc));
	}
	}

	/*!
	* \brief Handle a timeout in node-to-node communication
	*
	* \param[in] data Pointer to graph action
	*
	* \return FALSE (indicating that source should be not be re-added)
	*/
	gboolean
	action_timer_callback(gpointer data)
	{
	pcmk__graph_action_t action = (pcmk__graph_action_t ) data;
	const char *task = NULL;
	const char *on_node = NULL;
	const char *via_node = NULL;

	CRM_CHECK(data != NULL, return FALSE);

	stop_te_timer(action);

	task = crm_element_value(action->xml, XML_LRM_ATTR_TASK);
	on_node = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
	via_node = crm_element_value(action->xml, XML_LRM_ATTR_ROUTER_NODE);

	if (transition_graph->complete) {
	crm_notice("Node %s did not send %s result (via %s) within %dms "
	"(ignoring because transition not in progress)",
	(on_node? on_node : ""), (task? task : "unknown action"),
	(via_node? via_node : "controller"), action->timeout);
	} else {
	/* fail the action */

	crm_err("Node %s did not send %s result (via %s) within %dms "
	"(action timeout plus cluster-delay)",
	(on_node? on_node : ""), (task? task : "unknown action"),
	(via_node? via_node : "controller"),
	action->timeout + transition_graph->network_delay);
	pcmk__log_graph_action(LOG_ERR, action);

	pcmk__set_graph_action_flags(action, pcmk__graph_action_failed);

	te_action_confirmed(action, transition_graph);
	abort_transition(INFINITY, pcmk__graph_restart, "Action lost", NULL);

	// Record timeout in the CIB if appropriate
	if ((action->type == pcmk__rsc_graph_action)
	&& controld_action_is_recordable(task)) {
	controld_record_action_timeout(action);
	}
	}

	return FALSE;
	}
	diff --git a/include/crm/cib/internal.h b/include/crm/cib/internal.h
	index 31b823a15f..70cbebd123 100644
	--- a/include/crm/cib/internal.h
	+++ b/include/crm/cib/internal.h
	@@ -1,266 +1,271 @@
	/*
	* Copyright 2004-2022 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#ifndef CIB_INTERNAL__H
	# define CIB_INTERNAL__H
	# include <crm/cib.h>
	# include <crm/common/ipc_internal.h>
	# include <crm/common/output_internal.h>

	// Request types for CIB manager IPC/CPG
	#define PCMK__CIB_REQUEST_SECONDARY "cib_slave"
	#define PCMK__CIB_REQUEST_ALL_SECONDARY "cib_slave_all"
	#define PCMK__CIB_REQUEST_PRIMARY "cib_master"
	#define PCMK__CIB_REQUEST_SYNC_TO_ALL "cib_sync"
	#define PCMK__CIB_REQUEST_SYNC_TO_ONE "cib_sync_one"
	#define PCMK__CIB_REQUEST_IS_PRIMARY "cib_ismaster"
	#define PCMK__CIB_REQUEST_BUMP "cib_bump"
	#define PCMK__CIB_REQUEST_QUERY "cib_query"
	#define PCMK__CIB_REQUEST_CREATE "cib_create"
	#define PCMK__CIB_REQUEST_MODIFY "cib_modify"
	#define PCMK__CIB_REQUEST_DELETE "cib_delete"
	#define PCMK__CIB_REQUEST_ERASE "cib_erase"
	#define PCMK__CIB_REQUEST_REPLACE "cib_replace"
	#define PCMK__CIB_REQUEST_APPLY_PATCH "cib_apply_diff"
	#define PCMK__CIB_REQUEST_UPGRADE "cib_upgrade"
	#define PCMK__CIB_REQUEST_ABS_DELETE "cib_delete_alt"
	#define PCMK__CIB_REQUEST_NOOP "noop"
	#define PCMK__CIB_REQUEST_SHUTDOWN "cib_shutdown_req"

	# define F_CIB_CLIENTID "cib_clientid"
	# define F_CIB_CALLOPTS "cib_callopt"
	# define F_CIB_CALLID "cib_callid"
	# define F_CIB_CALLDATA "cib_calldata"
	# define F_CIB_OPERATION "cib_op"
	# define F_CIB_ISREPLY "cib_isreplyto"
	# define F_CIB_SECTION "cib_section"
	# define F_CIB_HOST "cib_host"
	# define F_CIB_RC "cib_rc"
	# define F_CIB_UPGRADE_RC "cib_upgrade_rc"
	# define F_CIB_DELEGATED "cib_delegated_from"
	# define F_CIB_OBJID "cib_object"
	# define F_CIB_OBJTYPE "cib_object_type"
	# define F_CIB_EXISTING "cib_existing_object"
	# define F_CIB_SEENCOUNT "cib_seen"
	# define F_CIB_TIMEOUT "cib_timeout"
	# define F_CIB_UPDATE "cib_update"
	# define F_CIB_CALLBACK_TOKEN "cib_async_id"
	# define F_CIB_GLOBAL_UPDATE "cib_update"
	# define F_CIB_UPDATE_RESULT "cib_update_result"
	# define F_CIB_CLIENTNAME "cib_clientname"
	# define F_CIB_NOTIFY_TYPE "cib_notify_type"
	# define F_CIB_NOTIFY_ACTIVATE "cib_notify_activate"
	# define F_CIB_UPDATE_DIFF "cib_update_diff"
	# define F_CIB_USER "cib_user"
	# define F_CIB_LOCAL_NOTIFY_ID "cib_local_notify_id"
	# define F_CIB_PING_ID "cib_ping_id"
	# define F_CIB_SCHEMA_MAX "cib_schema_max"
	# define F_CIB_CHANGE_SECTION "cib_change_section"

	# define T_CIB "cib"
	# define T_CIB_NOTIFY "cib_notify"
	/* notify sub-types */
	# define T_CIB_PRE_NOTIFY "cib_pre_notify"
	# define T_CIB_POST_NOTIFY "cib_post_notify"
	# define T_CIB_UPDATE_CONFIRM "cib_update_confirmation"
	# define T_CIB_REPLACE_NOTIFY "cib_refresh_notify"

	+/*!
	+ * \internal
	+ * \enum cib_change_section_info
	+ * \brief Flags to indicate which sections of the CIB have changed
	+ */
	enum cib_change_section_info {
	- cib_change_section_none = 0x00000000,
	- cib_change_section_nodes = 0x00000001,
	- cib_change_section_alerts = 0x00000002,
	- cib_change_section_status = 0x00000004
	+ cib_change_section_none = 0, //!< No sections have changed
	+ cib_change_section_nodes = (1 << 0), //!< The nodes section has changed
	+ cib_change_section_alerts = (1 << 1), //!< The alerts section has changed
	+ cib_change_section_status = (1 << 2), //!< The status section has changed
	};


	gboolean cib_diff_version_details(xmlNode * diff, int admin_epoch, int epoch, int *updates,
	int _admin_epoch, int _epoch, int *_updates);

	gboolean cib_read_config(GHashTable * options, xmlNode * current_cib);
	void verify_cib_options(GHashTable * options);
	gboolean cib_internal_config_changed(xmlNode * diff);

	typedef struct cib_notify_client_s {
	const char *event;
	const char obj_id; / implement one day */
	const char obj_type; / implement one day */
	void (callback) (const char event, xmlNode * msg);

	} cib_notify_client_t;

	typedef struct cib_callback_client_s {
	void (callback) (xmlNode , int, int, xmlNode , void );
	const char *id;
	void *user_data;
	gboolean only_success;
	struct timer_rec_s *timer;
	void (free_func)(void );
	} cib_callback_client_t;

	struct timer_rec_s {
	int call_id;
	int timeout;
	guint ref;
	cib_t *cib;
	};

	#define cib__set_call_options(cib_call_opts, call_for, flags_to_set) do { \
	cib_call_opts = pcmk__set_flags_as(__func__, __LINE__, \
	LOG_TRACE, "CIB call", (call_for), (cib_call_opts), \
	(flags_to_set), #flags_to_set); \
	} while (0)

	#define cib__clear_call_options(cib_call_opts, call_for, flags_to_clear) do { \
	cib_call_opts = pcmk__clear_flags_as(__func__, __LINE__, \
	LOG_TRACE, "CIB call", (call_for), (cib_call_opts), \
	(flags_to_clear), #flags_to_clear); \
	} while (0)

	typedef int (cib_op_t) (const char , int, const char , xmlNode ,
	xmlNode , xmlNode , xmlNode , xmlNode );

	cib_t *cib_new_variant(void);

	int cib_perform_op(const char op, int call_options, cib_op_t fn, gboolean is_query,
	const char section, xmlNode req, xmlNode * input,
	gboolean manage_counters, gboolean * config_changed,
	xmlNode * current_cib, xmlNode result_cib, xmlNode diff,
	xmlNode ** output);

	xmlNode cib_create_op(int call_id, const char token, const char op, const char host,
	const char section, xmlNode data, int call_options,
	const char *user_name);

	void cib_native_callback(cib_t * cib, xmlNode * msg, int call_id, int rc);
	void cib_native_notify(gpointer data, gpointer user_data);
	int cib_native_register_notification(cib_t * cib, const char *callback, int enabled);
	gboolean cib_client_register_callback(cib_t * cib, int call_id, int timeout, gboolean only_success,
	void user_data, const char callback_name,
	void (callback) (xmlNode , int, int, xmlNode , void ));
	gboolean cib_client_register_callback_full(cib_t *cib, int call_id,
	int timeout, gboolean only_success,
	void *user_data,
	const char *callback_name,
	void (callback)(xmlNode , int, int,
	xmlNode , void ),
	void (free_func)(void ));

	int cib_process_query(const char op, int options, const char section, xmlNode * req,
	xmlNode * input, xmlNode * existing_cib, xmlNode ** result_cib,
	xmlNode ** answer);

	int cib_process_erase(const char op, int options, const char section, xmlNode * req,
	xmlNode * input, xmlNode * existing_cib, xmlNode ** result_cib,
	xmlNode ** answer);

	int cib_process_bump(const char op, int options, const char section, xmlNode * req,
	xmlNode * input, xmlNode * existing_cib, xmlNode ** result_cib,
	xmlNode ** answer);

	int cib_process_replace(const char op, int options, const char section, xmlNode * req,
	xmlNode * input, xmlNode * existing_cib, xmlNode ** result_cib,
	xmlNode ** answer);

	int cib_process_create(const char op, int options, const char section, xmlNode * req,
	xmlNode * input, xmlNode * existing_cib, xmlNode ** result_cib,
	xmlNode ** answer);

	int cib_process_modify(const char op, int options, const char section, xmlNode * req,
	xmlNode * input, xmlNode * existing_cib, xmlNode ** result_cib,
	xmlNode ** answer);

	int cib_process_delete(const char op, int options, const char section, xmlNode * req,
	xmlNode * input, xmlNode * existing_cib, xmlNode ** result_cib,
	xmlNode ** answer);

	int cib_process_diff(const char op, int options, const char section, xmlNode * req,
	xmlNode * input, xmlNode * existing_cib, xmlNode ** result_cib,
	xmlNode ** answer);

	int cib_process_upgrade(const char op, int options, const char section, xmlNode * req,
	xmlNode * input, xmlNode * existing_cib, xmlNode ** result_cib,
	xmlNode ** answer);

	/*!
	* \internal
	* \brief Query or modify a CIB
	*
	* \param[in] op PCMK__CIB_REQUEST_* operation to be performed
	* \param[in] options Flag set of \c cib_call_options
	* \param[in] section XPath to query or modify
	* \param[in] req unused
	* \param[in] input Portion of CIB to modify (used with
	* PCMK__CIB_REQUEST_CREATE,
	* PCMK__CIB_REQUEST_MODIFY, and
	* PCMK__CIB_REQUEST_REPLACE)
	* \param[in] existing_cib Input CIB (used with PCMK__CIB_REQUEST_QUERY)
	* \param[in,out] result_cib CIB copy to make changes in (used with
	* PCMK__CIB_REQUEST_CREATE,
	* PCMK__CIB_REQUEST_MODIFY,
	* PCMK__CIB_REQUEST_DELETE, and
	* PCMK__CIB_REQUEST_REPLACE)
	* \param[out] answer Query result (used with PCMK__CIB_REQUEST_QUERY)
	*
	* \return Legacy Pacemaker return code
	*/
	int cib_process_xpath(const char op, int options, const char section, xmlNode * req,
	xmlNode * input, xmlNode * existing_cib, xmlNode ** result_cib,
	xmlNode ** answer);

	gboolean cib_config_changed(xmlNode * last, xmlNode * next, xmlNode ** diff);
	gboolean update_results(xmlNode * failed, xmlNode * target, const char *operation, int return_code);
	int cib_update_counter(xmlNode * xml_obj, const char *field, gboolean reset);

	int cib_internal_op(cib_t * cib, const char op, const char host,
	const char section, xmlNode data,
	xmlNode ** output_data, int call_options, const char *user_name);


	int cib_file_read_and_verify(const char filename, const char sigfile,
	xmlNode **root);
	int cib_file_write_with_digest(xmlNode cib_root, const char cib_dirname,
	const char *cib_filename);

	void cib__set_output(cib_t cib, pcmk__output_t out);

	cib_callback_client_t* cib__lookup_id (int call_id);

	/*!
	* \internal
	* \brief Connect to, query, and optionally disconnect from the CIB, returning
	* the resulting XML object.
	*
	* \param[out] cib If non-NULL, a pointer to where to store the CIB
	* connection. In this case, it is up to the caller to
	* disconnect from the CIB when finished.
	* \param[out] cib_object A pointer to where to store the XML query result.
	*
	* \return A standard Pacemaker return code
	*/
	int cib__signon_query(cib_t cib, xmlNode cib_object);

	int cib__clean_up_connection(cib_t **cib);

	int cib__update_node_attr(pcmk__output_t out, cib_t cib, int call_options,
	const char section, const char node_uuid, const char *set_type,
	const char set_name, const char attr_id, const char *attr_name,
	const char attr_value, const char user_name,
	const char *node_type);

	int cib__get_node_attrs(pcmk__output_t out, cib_t cib, const char *section,
	const char node_uuid, const char set_type, const char *set_name,
	const char attr_id, const char attr_name, const char *user_name,
	xmlNode **result);

	int cib__delete_node_attr(pcmk__output_t out, cib_t cib, int options,
	const char section, const char node_uuid, const char *set_type,
	const char set_name, const char attr_id, const char *attr_name,
	const char attr_value, const char user_name);

	#endif

File Metadata

Mime Type: text/x-diff
Expires: Sat, Jan 25, 11:27 AM (1 d, 15 h)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 1322365
Default Alt Text: (122 KB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions