No OneTemporary
Actions

Size

93 KB

Referenced Files

None

Subscribers

None

View Options

	diff --git a/daemons/based/based_callbacks.c b/daemons/based/based_callbacks.c
	index ccde937d30..de847e2cac 100644
	--- a/daemons/based/based_callbacks.c
	+++ b/daemons/based/based_callbacks.c
	@@ -1,1651 +1,1667 @@
	/*
	* Copyright 2004-2022 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <sys/param.h>
	#include <stdio.h>
	#include <sys/types.h>
	#include <unistd.h>

	#include <stdlib.h>
	#include <stdint.h> // uint32_t, uint64_t, UINT64_C()
	#include <errno.h>
	#include <fcntl.h>
	#include <inttypes.h> // PRIu64

	#include <crm/crm.h>
	#include <crm/cib.h>
	#include <crm/msg_xml.h>
	#include <crm/cluster/internal.h>

	#include <crm/common/xml.h>
	#include <crm/common/remote_internal.h>

	#include <pacemaker-based.h>

	#define EXIT_ESCALATION_MS 10000
	#define OUR_NODENAME (stand_alone? "localhost" : crm_cluster->uname)

	static unsigned long cib_local_bcast_num = 0;

	typedef struct cib_local_notify_s {
	xmlNode *notify_src;
	char *client_id;
	gboolean from_peer;
	gboolean sync_reply;
	} cib_local_notify_t;

	int next_client_id = 0;

	gboolean legacy_mode = FALSE;

	qb_ipcs_service_t *ipcs_ro = NULL;
	qb_ipcs_service_t *ipcs_rw = NULL;
	qb_ipcs_service_t *ipcs_shm = NULL;

	void send_cib_replace(const xmlNode * sync_request, const char *host);
	static void cib_process_request(xmlNode *request, gboolean privileged,
	const pcmk__client_t *cib_client);


	static int cib_process_command(xmlNode request, xmlNode *reply,
	xmlNode **cib_diff, gboolean privileged);

	gboolean cib_common_callback(qb_ipcs_connection_t * c, void *data, size_t size,
	gboolean privileged);

	gboolean cib_legacy_mode(void)
	{
	return legacy_mode;
	}


	static int32_t
	cib_ipc_accept(qb_ipcs_connection_t * c, uid_t uid, gid_t gid)
	{
	if (cib_shutdown_flag) {
	crm_info("Ignoring new IPC client [%d] during shutdown",
	pcmk__client_pid(c));
	return -EPERM;
	}

	if (pcmk__new_client(c, uid, gid) == NULL) {
	return -EIO;
	}
	return 0;
	}

	static int32_t
	cib_ipc_dispatch_rw(qb_ipcs_connection_t * c, void *data, size_t size)
	{
	pcmk__client_t *client = pcmk__find_client(c);

	crm_trace("%p message from %s", c, client->id);
	return cib_common_callback(c, data, size, TRUE);
	}

	static int32_t
	cib_ipc_dispatch_ro(qb_ipcs_connection_t * c, void *data, size_t size)
	{
	pcmk__client_t *client = pcmk__find_client(c);

	crm_trace("%p message from %s", c, client->id);
	return cib_common_callback(c, data, size, FALSE);
	}

	/* Error code means? */
	static int32_t
	cib_ipc_closed(qb_ipcs_connection_t * c)
	{
	pcmk__client_t *client = pcmk__find_client(c);

	if (client == NULL) {
	return 0;
	}
	crm_trace("Connection %p", c);
	pcmk__free_client(client);
	return 0;
	}

	static void
	cib_ipc_destroy(qb_ipcs_connection_t * c)
	{
	crm_trace("Connection %p", c);
	cib_ipc_closed(c);
	if (cib_shutdown_flag) {
	cib_shutdown(0);
	}
	}

	struct qb_ipcs_service_handlers ipc_ro_callbacks = {
	.connection_accept = cib_ipc_accept,
	.connection_created = NULL,
	.msg_process = cib_ipc_dispatch_ro,
	.connection_closed = cib_ipc_closed,
	.connection_destroyed = cib_ipc_destroy
	};

	struct qb_ipcs_service_handlers ipc_rw_callbacks = {
	.connection_accept = cib_ipc_accept,
	.connection_created = NULL,
	.msg_process = cib_ipc_dispatch_rw,
	.connection_closed = cib_ipc_closed,
	.connection_destroyed = cib_ipc_destroy
	};

	void
	cib_common_callback_worker(uint32_t id, uint32_t flags, xmlNode * op_request,
	pcmk__client_t *cib_client, gboolean privileged)
	{
	const char *op = crm_element_value(op_request, F_CIB_OPERATION);

	if (pcmk__str_eq(op, CRM_OP_REGISTER, pcmk__str_none)) {
	if (flags & crm_ipc_client_response) {
	xmlNode *ack = create_xml_node(NULL, __func__);

	crm_xml_add(ack, F_CIB_OPERATION, CRM_OP_REGISTER);
	crm_xml_add(ack, F_CIB_CLIENTID, cib_client->id);
	pcmk__ipc_send_xml(cib_client, id, ack, flags);
	cib_client->request_id = 0;
	free_xml(ack);
	}
	return;

	} else if (pcmk__str_eq(op, T_CIB_NOTIFY, pcmk__str_none)) {
	/* Update the notify filters for this client */
	int on_off = 0;
	crm_exit_t status = CRM_EX_OK;
	uint64_t bit = UINT64_C(0);
	const char *type = crm_element_value(op_request, F_CIB_NOTIFY_TYPE);

	crm_element_value_int(op_request, F_CIB_NOTIFY_ACTIVATE, &on_off);

	crm_debug("Setting %s callbacks %s for client %s",
	type, (on_off? "on" : "off"), pcmk__client_name(cib_client));

	if (pcmk__str_eq(type, T_CIB_POST_NOTIFY, pcmk__str_casei)) {
	bit = cib_notify_post;

	} else if (pcmk__str_eq(type, T_CIB_PRE_NOTIFY, pcmk__str_casei)) {
	bit = cib_notify_pre;

	} else if (pcmk__str_eq(type, T_CIB_UPDATE_CONFIRM, pcmk__str_casei)) {
	bit = cib_notify_confirm;

	} else if (pcmk__str_eq(type, T_CIB_DIFF_NOTIFY, pcmk__str_casei)) {
	bit = cib_notify_diff;

	} else if (pcmk__str_eq(type, T_CIB_REPLACE_NOTIFY, pcmk__str_casei)) {
	bit = cib_notify_replace;

	} else {
	status = CRM_EX_INVALID_PARAM;
	}

	if (bit != 0) {
	if (on_off) {
	pcmk__set_client_flags(cib_client, bit);
	} else {
	pcmk__clear_client_flags(cib_client, bit);
	}
	}

	pcmk__ipc_send_ack(cib_client, id, flags, "ack", NULL, status);
	return;
	}

	cib_process_request(op_request, privileged, cib_client);
	}

	int32_t
	cib_common_callback(qb_ipcs_connection_t * c, void *data, size_t size, gboolean privileged)
	{
	uint32_t id = 0;
	uint32_t flags = 0;
	int call_options = 0;
	pcmk__client_t *cib_client = pcmk__find_client(c);
	xmlNode *op_request = pcmk__client_data2xml(cib_client, data, &id, &flags);

	if (op_request) {
	crm_element_value_int(op_request, F_CIB_CALLOPTS, &call_options);
	}

	if (op_request == NULL) {
	crm_trace("Invalid message from %p", c);
	pcmk__ipc_send_ack(cib_client, id, flags, "nack", NULL, CRM_EX_PROTOCOL);
	return 0;

	} else if(cib_client == NULL) {
	crm_trace("Invalid client %p", c);
	return 0;
	}

	if (pcmk_is_set(call_options, cib_sync_call)) {
	CRM_LOG_ASSERT(flags & crm_ipc_client_response);
	CRM_LOG_ASSERT(cib_client->request_id == 0); /* This means the client has two synchronous events in-flight */
	cib_client->request_id = id; /* Reply only to the last one */
	}

	if (cib_client->name == NULL) {
	const char *value = crm_element_value(op_request, F_CIB_CLIENTNAME);

	if (value == NULL) {
	cib_client->name = pcmk__itoa(cib_client->pid);
	} else {
	cib_client->name = strdup(value);
	if (crm_is_daemon_name(value)) {
	pcmk__set_client_flags(cib_client, cib_is_daemon);
	}
	}
	}

	/* Allow cluster daemons more leeway before being evicted */
	if (pcmk_is_set(cib_client->flags, cib_is_daemon)) {
	const char *qmax = cib_config_lookup("cluster-ipc-limit");

	if (pcmk__set_client_queue_max(cib_client, qmax)) {
	crm_trace("IPC threshold for client %s[%u] is now %u",
	pcmk__client_name(cib_client), cib_client->pid,
	cib_client->queue_max);
	}
	}

	crm_xml_add(op_request, F_CIB_CLIENTID, cib_client->id);
	crm_xml_add(op_request, F_CIB_CLIENTNAME, cib_client->name);

	CRM_LOG_ASSERT(cib_client->user != NULL);
	pcmk__update_acl_user(op_request, F_CIB_USER, cib_client->user);

	cib_common_callback_worker(id, flags, op_request, cib_client, privileged);
	free_xml(op_request);

	return 0;
	}

	static uint64_t ping_seq = 0;
	static char *ping_digest = NULL;
	static bool ping_modified_since = FALSE;
	int sync_our_cib(xmlNode * request, gboolean all);

	static gboolean
	cib_digester_cb(gpointer data)
	{
	if (based_is_primary) {
	char buffer[32];
	xmlNode *ping = create_xml_node(NULL, "ping");

	ping_seq++;
	free(ping_digest);
	ping_digest = NULL;
	ping_modified_since = FALSE;
	snprintf(buffer, 32, "%" PRIu64, ping_seq);
	crm_trace("Requesting peer digests (%s)", buffer);

	crm_xml_add(ping, F_TYPE, "cib");
	crm_xml_add(ping, F_CIB_OPERATION, CRM_OP_PING);
	crm_xml_add(ping, F_CIB_PING_ID, buffer);

	crm_xml_add(ping, XML_ATTR_CRM_VERSION, CRM_FEATURE_SET);
	send_cluster_message(NULL, crm_msg_cib, ping, TRUE);

	free_xml(ping);
	}
	return FALSE;
	}

	static void
	process_ping_reply(xmlNode *reply)
	{
	uint64_t seq = 0;
	const char *host = crm_element_value(reply, F_ORIG);

	xmlNode *pong = get_message_xml(reply, F_CIB_CALLDATA);
	const char *seq_s = crm_element_value(pong, F_CIB_PING_ID);
	const char *digest = crm_element_value(pong, XML_ATTR_DIGEST);

	if (seq_s == NULL) {
	crm_debug("Ignoring ping reply with no " F_CIB_PING_ID);
	return;

	} else {
	long long seq_ll;

	if (pcmk__scan_ll(seq_s, &seq_ll, 0LL) != pcmk_rc_ok) {
	return;
	}
	seq = (uint64_t) seq_ll;
	}

	if(digest == NULL) {
	crm_trace("Ignoring ping reply %s from %s with no digest", seq_s, host);

	} else if(seq != ping_seq) {
	crm_trace("Ignoring out of sequence ping reply %s from %s", seq_s, host);

	} else if(ping_modified_since) {
	crm_trace("Ignoring ping reply %s from %s: cib updated since", seq_s, host);

	} else {
	const char *version = crm_element_value(pong, XML_ATTR_CRM_VERSION);

	if(ping_digest == NULL) {
	crm_trace("Calculating new digest");
	ping_digest = calculate_xml_versioned_digest(the_cib, FALSE, TRUE, version);
	}

	crm_trace("Processing ping reply %s from %s (%s)", seq_s, host, digest);
	if (!pcmk__str_eq(ping_digest, digest, pcmk__str_casei)) {
	xmlNode *remote_cib = get_message_xml(pong, F_CIB_CALLDATA);

	crm_notice("Local CIB %s.%s.%s.%s differs from %s: %s.%s.%s.%s %p",
	crm_element_value(the_cib, XML_ATTR_GENERATION_ADMIN),
	crm_element_value(the_cib, XML_ATTR_GENERATION),
	crm_element_value(the_cib, XML_ATTR_NUMUPDATES),
	ping_digest, host,
	remote_cib?crm_element_value(remote_cib, XML_ATTR_GENERATION_ADMIN):"_",
	remote_cib?crm_element_value(remote_cib, XML_ATTR_GENERATION):"_",
	remote_cib?crm_element_value(remote_cib, XML_ATTR_NUMUPDATES):"_",
	digest, remote_cib);

	if(remote_cib && remote_cib->children) {
	/* Additional debug */
	xml_calculate_changes(the_cib, remote_cib);
	xml_log_changes(LOG_INFO, __func__, remote_cib);
	crm_trace("End of differences");
	}

	free_xml(remote_cib);
	sync_our_cib(reply, FALSE);
	}
	}
	}

	static void
	do_local_notify(xmlNode * notify_src, const char *client_id,
	gboolean sync_reply, gboolean from_peer)
	{
	int rid = 0;
	int call_id = 0;
	pcmk__client_t *client_obj = NULL;

	CRM_ASSERT(notify_src && client_id);

	crm_element_value_int(notify_src, F_CIB_CALLID, &call_id);

	client_obj = pcmk__find_client_by_id(client_id);
	if (client_obj == NULL) {
	crm_debug("Could not send response %d: client %s not found",
	call_id, client_id);
	return;
	}

	if (sync_reply) {
	if (client_obj->ipcs) {
	CRM_LOG_ASSERT(client_obj->request_id);

	rid = client_obj->request_id;
	client_obj->request_id = 0;

	crm_trace("Sending response %d to client %s%s",
	rid, pcmk__client_name(client_obj),
	(from_peer? " (originator of delegated request)" : ""));
	} else {
	crm_trace("Sending response (call %d) to client %s%s",
	call_id, pcmk__client_name(client_obj),
	(from_peer? " (originator of delegated request)" : ""));
	}

	} else {
	crm_trace("Sending event %d to client %s%s",
	call_id, pcmk__client_name(client_obj),
	(from_peer? " (originator of delegated request)" : ""));
	}

	switch (PCMK__CLIENT_TYPE(client_obj)) {
	case pcmk__client_ipc:
	{
	int rc = pcmk__ipc_send_xml(client_obj, rid, notify_src,
	(sync_reply? crm_ipc_flags_none
	: crm_ipc_server_event));

	if (rc != pcmk_rc_ok) {
	crm_warn("%s reply to client %s failed: %s " CRM_XS " rc=%d",
	(sync_reply? "Synchronous" : "Asynchronous"),
	pcmk__client_name(client_obj), pcmk_rc_str(rc),
	rc);
	}
	}
	break;
	#ifdef HAVE_GNUTLS_GNUTLS_H
	case pcmk__client_tls:
	#endif
	case pcmk__client_tcp:
	pcmk__remote_send_xml(client_obj->remote, notify_src);
	break;
	default:
	crm_err("Unknown transport for client %s "
	CRM_XS " flags=%#016" PRIx64,
	pcmk__client_name(client_obj), client_obj->flags);
	}
	}

	static void
	local_notify_destroy_callback(gpointer data)
	{
	cib_local_notify_t *notify = data;

	free_xml(notify->notify_src);
	free(notify->client_id);
	free(notify);
	}

	static void
	check_local_notify(int bcast_id)
	{
	cib_local_notify_t *notify = NULL;

	if (!local_notify_queue) {
	return;
	}

	notify = pcmk__intkey_table_lookup(local_notify_queue, bcast_id);

	if (notify) {
	do_local_notify(notify->notify_src, notify->client_id, notify->sync_reply,
	notify->from_peer);
	pcmk__intkey_table_remove(local_notify_queue, bcast_id);
	}
	}

	static void
	queue_local_notify(xmlNode * notify_src, const char *client_id, gboolean sync_reply,
	gboolean from_peer)
	{
	cib_local_notify_t *notify = calloc(1, sizeof(cib_local_notify_t));

	notify->notify_src = notify_src;
	notify->client_id = strdup(client_id);
	notify->sync_reply = sync_reply;
	notify->from_peer = from_peer;

	if (!local_notify_queue) {
	local_notify_queue = pcmk__intkey_table(local_notify_destroy_callback);
	}
	pcmk__intkey_table_insert(local_notify_queue, cib_local_bcast_num, notify);
	// cppcheck doesn't know notify will get freed when hash table is destroyed
	// cppcheck-suppress memleak
	}

	static void
	parse_local_options_v1(const pcmk__client_t *cib_client, int call_type,
	int call_options, const char host, const char op,
	gboolean local_notify, gboolean needs_reply,
	gboolean process, gboolean needs_forward)
	{
	if (cib_op_modifies(call_type)
	&& !(call_options & cib_inhibit_bcast)) {
	/* we need to send an update anyway */
	*needs_reply = TRUE;
	} else {
	*needs_reply = FALSE;
	}

	if (host == NULL && (call_options & cib_scope_local)) {
	crm_trace("Processing locally scoped %s op from client %s",
	op, pcmk__client_name(cib_client));
	*local_notify = TRUE;

	} else if ((host == NULL) && based_is_primary) {
	crm_trace("Processing %s op locally from client %s as primary",
	op, pcmk__client_name(cib_client));
	*local_notify = TRUE;

	} else if (pcmk__str_eq(host, OUR_NODENAME, pcmk__str_casei)) {
	crm_trace("Processing locally addressed %s op from client %s",
	op, pcmk__client_name(cib_client));
	*local_notify = TRUE;

	} else if (stand_alone) {
	*needs_forward = FALSE;
	*local_notify = TRUE;
	*process = TRUE;

	} else {
	crm_trace("%s op from %s needs to be forwarded to client %s",
	op, pcmk__client_name(cib_client),
	pcmk__s(host, "the primary instance"));
	*needs_forward = TRUE;
	*process = FALSE;
	}
	}

	static void
	parse_local_options_v2(const pcmk__client_t *cib_client, int call_type,
	int call_options, const char host, const char op,
	gboolean local_notify, gboolean needs_reply,
	gboolean process, gboolean needs_forward)
	{
	if (cib_op_modifies(call_type)) {
	if (pcmk__str_any_of(op, PCMK__CIB_REQUEST_PRIMARY,
	PCMK__CIB_REQUEST_SECONDARY, NULL)) {
	/* Always handle these locally */
	*process = TRUE;
	*needs_reply = FALSE;
	*local_notify = TRUE;
	*needs_forward = FALSE;
	return;

	} else {
	/* Redirect all other updates via CPG */
	*needs_reply = TRUE;
	*needs_forward = TRUE;
	*process = FALSE;
	crm_trace("%s op from %s needs to be forwarded to client %s",
	op, pcmk__client_name(cib_client),
	pcmk__s(host, "the primary instance"));
	return;
	}
	}


	*process = TRUE;
	*needs_reply = FALSE;
	*local_notify = TRUE;
	*needs_forward = FALSE;

	if (stand_alone) {
	crm_trace("Processing %s op from client %s (stand-alone)",
	op, pcmk__client_name(cib_client));

	} else if (host == NULL) {
	crm_trace("Processing unaddressed %s op from client %s",
	op, pcmk__client_name(cib_client));

	} else if (pcmk__str_eq(host, OUR_NODENAME, pcmk__str_casei)) {
	crm_trace("Processing locally addressed %s op from client %s",
	op, pcmk__client_name(cib_client));

	} else {
	crm_trace("%s op from %s needs to be forwarded to client %s",
	op, pcmk__client_name(cib_client), host);
	*needs_forward = TRUE;
	*process = FALSE;
	}
	}

	static void
	parse_local_options(const pcmk__client_t *cib_client, int call_type,
	int call_options, const char host, const char op,
	gboolean local_notify, gboolean needs_reply,
	gboolean process, gboolean needs_forward)
	{
	if(cib_legacy_mode()) {
	parse_local_options_v1(cib_client, call_type, call_options, host,
	op, local_notify, needs_reply, process, needs_forward);
	} else {
	parse_local_options_v2(cib_client, call_type, call_options, host,
	op, local_notify, needs_reply, process, needs_forward);
	}
	}

	static gboolean
	parse_peer_options_v1(int call_type, xmlNode * request,
	gboolean * local_notify, gboolean * needs_reply, gboolean * process,
	gboolean * needs_forward)
	{
	const char *op = NULL;
	const char *host = NULL;
	const char *delegated = NULL;
	const char *originator = crm_element_value(request, F_ORIG);
	const char *reply_to = crm_element_value(request, F_CIB_ISREPLY);

	gboolean is_reply = pcmk__str_eq(reply_to, OUR_NODENAME, pcmk__str_casei);

	if (pcmk__xe_attr_is_true(request, F_CIB_GLOBAL_UPDATE)) {
	*needs_reply = FALSE;
	if (is_reply) {
	*local_notify = TRUE;
	crm_trace("Processing global/peer update from %s"
	" that originated from us", originator);
	} else {
	crm_trace("Processing global/peer update from %s", originator);
	}
	return TRUE;
	}

	op = crm_element_value(request, F_CIB_OPERATION);
	crm_trace("Processing %s request sent by %s", op, originator);
	if (pcmk__str_eq(op, PCMK__CIB_REQUEST_SHUTDOWN, pcmk__str_none)) {
	/* Always process these */
	*local_notify = FALSE;
	if (reply_to == NULL \|\| is_reply) {
	*process = TRUE;
	}
	if (is_reply) {
	*needs_reply = FALSE;
	}
	return *process;
	}

	if (is_reply && pcmk__str_eq(op, CRM_OP_PING, pcmk__str_casei)) {
	process_ping_reply(request);
	return FALSE;
	}

	if (is_reply) {
	crm_trace("Forward reply sent from %s to local clients", originator);
	*process = FALSE;
	*needs_reply = FALSE;
	*local_notify = TRUE;
	return TRUE;
	}

	host = crm_element_value(request, F_CIB_HOST);
	if (pcmk__str_eq(host, OUR_NODENAME, pcmk__str_casei)) {
	crm_trace("Processing %s request sent to us from %s", op, originator);
	return TRUE;

	} else if(is_reply == FALSE && pcmk__str_eq(op, CRM_OP_PING, pcmk__str_casei)) {
	crm_trace("Processing %s request sent to %s by %s", op, host?host:"everyone", originator);
	*needs_reply = TRUE;
	return TRUE;

	} else if ((host == NULL) && based_is_primary) {
	crm_trace("Processing %s request sent to primary instance from %s",
	op, originator);
	return TRUE;
	}

	delegated = crm_element_value(request, F_CIB_DELEGATED);
	if (delegated != NULL) {
	crm_trace("Ignoring message for primary instance");

	} else if (host != NULL) {
	/* this is for a specific instance and we're not it */
	crm_trace("Ignoring msg for instance on %s", host);

	} else if ((reply_to == NULL) && !based_is_primary) {
	// This is for the primary instance, and we're not it
	crm_trace("Ignoring reply for primary instance");

	} else if (pcmk__str_eq(op, PCMK__CIB_REQUEST_SHUTDOWN, pcmk__str_none)) {
	if (reply_to != NULL) {
	crm_debug("Processing %s from %s", op, originator);
	*needs_reply = FALSE;

	} else {
	crm_debug("Processing %s reply from %s", op, originator);
	}
	return TRUE;

	} else {
	crm_err("Nothing for us to do?");
	crm_log_xml_err(request, "Peer[inbound]");
	}

	return FALSE;
	}

	static gboolean
	parse_peer_options_v2(int call_type, xmlNode * request,
	gboolean * local_notify, gboolean * needs_reply, gboolean * process,
	gboolean * needs_forward)
	{
	const char *host = NULL;
	const char *delegated = crm_element_value(request, F_CIB_DELEGATED);
	const char *op = crm_element_value(request, F_CIB_OPERATION);
	const char *originator = crm_element_value(request, F_ORIG);
	const char *reply_to = crm_element_value(request, F_CIB_ISREPLY);

	gboolean is_reply = pcmk__str_eq(reply_to, OUR_NODENAME, pcmk__str_casei);

	if (pcmk__str_eq(op, PCMK__CIB_REQUEST_REPLACE, pcmk__str_none)) {
	/* sync_our_cib() sets F_CIB_ISREPLY */
	if (reply_to) {
	delegated = reply_to;
	}
	goto skip_is_reply;

	} else if (pcmk__str_eq(op, PCMK__CIB_REQUEST_SYNC_TO_ALL,
	pcmk__str_none)) {
	// Nothing to do

	} else if (is_reply && pcmk__str_eq(op, CRM_OP_PING, pcmk__str_casei)) {
	process_ping_reply(request);
	return FALSE;

	} else if (pcmk__str_eq(op, PCMK__CIB_REQUEST_UPGRADE, pcmk__str_none)) {
	/* Only the DC (node with the oldest software) should process
	* this operation if F_CIB_SCHEMA_MAX is unset
	*
	* If the DC is happy it will then send out another
	* PCMK__CIB_REQUEST_UPGRADE which will tell all nodes to do the actual
	* upgrade.
	*
	* Except this time F_CIB_SCHEMA_MAX will be set which puts a
	* limit on how far newer nodes will go
	*/
	const char *max = crm_element_value(request, F_CIB_SCHEMA_MAX);
	const char *upgrade_rc = crm_element_value(request, F_CIB_UPGRADE_RC);

	crm_trace("Parsing %s operation%s for %s with max=%s and upgrade_rc=%s",
	op, (is_reply? " reply" : ""),
	(based_is_primary? "primary" : "secondary"),
	(max? max : "none"), (upgrade_rc? upgrade_rc : "none"));

	if (upgrade_rc != NULL) {
	// Our upgrade request was rejected by DC, notify clients of result
	crm_xml_add(request, F_CIB_RC, upgrade_rc);

	} else if ((max == NULL) && based_is_primary) {
	/* We are the DC, check if this upgrade is allowed */
	goto skip_is_reply;

	} else if(max) {
	/* Ok, go ahead and upgrade to 'max' */
	goto skip_is_reply;

	} else {
	// Ignore broadcast client requests when we're not DC
	return FALSE;
	}

	} else if (pcmk__xe_attr_is_true(request, F_CIB_GLOBAL_UPDATE)) {
	crm_info("Detected legacy %s global update from %s", op, originator);
	send_sync_request(NULL);
	legacy_mode = TRUE;
	return FALSE;

	} else if (is_reply && cib_op_modifies(call_type)) {
	crm_trace("Ignoring legacy %s reply sent from %s to local clients", op, originator);
	return FALSE;

	} else if (pcmk__str_eq(op, PCMK__CIB_REQUEST_SHUTDOWN, pcmk__str_none)) {
	/* Legacy handling */
	crm_debug("Legacy handling of %s message from %s", op, originator);
	*local_notify = FALSE;
	if (reply_to == NULL) {
	*process = TRUE;
	}
	return *process;
	}

	if(is_reply) {
	crm_trace("Handling %s reply sent from %s to local clients", op, originator);
	*process = FALSE;
	*needs_reply = FALSE;
	*local_notify = TRUE;
	return TRUE;
	}

	skip_is_reply:
	*process = TRUE;
	*needs_reply = FALSE;

	*local_notify = pcmk__str_eq(delegated, OUR_NODENAME, pcmk__str_casei);

	host = crm_element_value(request, F_CIB_HOST);
	if (pcmk__str_eq(host, OUR_NODENAME, pcmk__str_casei)) {
	crm_trace("Processing %s request sent to us from %s", op, originator);
	*needs_reply = TRUE;
	return TRUE;

	} else if (host != NULL) {
	/* this is for a specific instance and we're not it */
	crm_trace("Ignoring %s operation for instance on %s", op, host);
	return FALSE;

	} else if(is_reply == FALSE && pcmk__str_eq(op, CRM_OP_PING, pcmk__str_casei)) {
	*needs_reply = TRUE;
	}

	crm_trace("Processing %s request sent to everyone by %s/%s on %s %s", op,
	crm_element_value(request, F_CIB_CLIENTNAME),
	crm_element_value(request, F_CIB_CALLID),
	originator, (*local_notify)?"(notify)":"");
	return TRUE;
	}

	static gboolean
	parse_peer_options(int call_type, xmlNode * request,
	gboolean * local_notify, gboolean * needs_reply, gboolean * process,
	gboolean * needs_forward)
	{
	/* TODO: What happens when an update comes in after node A
	* requests the CIB from node B, but before it gets the reply (and
	* sends out the replace operation)
	*/
	if(cib_legacy_mode()) {
	return parse_peer_options_v1(
	call_type, request, local_notify, needs_reply, process, needs_forward);
	} else {
	return parse_peer_options_v2(
	call_type, request, local_notify, needs_reply, process, needs_forward);
	}
	}

	static void
	forward_request(xmlNode *request, int call_options)
	{
	const char *op = crm_element_value(request, F_CIB_OPERATION);
	const char *host = crm_element_value(request, F_CIB_HOST);

	crm_xml_add(request, F_CIB_DELEGATED, OUR_NODENAME);

	if (host != NULL) {
	crm_trace("Forwarding %s op to %s", op, host);
	send_cluster_message(crm_get_peer(0, host), crm_msg_cib, request, FALSE);

	} else {
	crm_trace("Forwarding %s op to primary instance", op);
	send_cluster_message(NULL, crm_msg_cib, request, FALSE);
	}

	/* Return the request to its original state */
	xml_remove_prop(request, F_CIB_DELEGATED);

	if (call_options & cib_discard_reply) {
	crm_trace("Client not interested in reply");
	}
	}

	static gboolean
	send_peer_reply(xmlNode * msg, xmlNode * result_diff, const char *originator, gboolean broadcast)
	{
	CRM_ASSERT(msg != NULL);

	if (broadcast) {
	/* this (successful) call modified the CIB _and_ the
	* change needs to be broadcast...
	* send via HA to other nodes
	*/
	int diff_add_updates = 0;
	int diff_add_epoch = 0;
	int diff_add_admin_epoch = 0;

	int diff_del_updates = 0;
	int diff_del_epoch = 0;
	int diff_del_admin_epoch = 0;

	const char *digest = NULL;
	int format = 1;

	CRM_LOG_ASSERT(result_diff != NULL);
	digest = crm_element_value(result_diff, XML_ATTR_DIGEST);
	crm_element_value_int(result_diff, "format", &format);

	cib_diff_version_details(result_diff,
	&diff_add_admin_epoch, &diff_add_epoch, &diff_add_updates,
	&diff_del_admin_epoch, &diff_del_epoch, &diff_del_updates);

	crm_trace("Sending update diff %d.%d.%d -> %d.%d.%d %s",
	diff_del_admin_epoch, diff_del_epoch, diff_del_updates,
	diff_add_admin_epoch, diff_add_epoch, diff_add_updates, digest);

	crm_xml_add(msg, F_CIB_ISREPLY, originator);
	pcmk__xe_set_bool_attr(msg, F_CIB_GLOBAL_UPDATE, true);
	crm_xml_add(msg, F_CIB_OPERATION, PCMK__CIB_REQUEST_APPLY_PATCH);
	crm_xml_add(msg, F_CIB_USER, CRM_DAEMON_USER);

	if (format == 1) {
	CRM_ASSERT(digest != NULL);
	}

	add_message_xml(msg, F_CIB_UPDATE_DIFF, result_diff);
	crm_log_xml_explicit(msg, "copy");
	return send_cluster_message(NULL, crm_msg_cib, msg, TRUE);

	} else if (originator != NULL) {
	/* send reply via HA to originating node */
	crm_trace("Sending request result to %s only", originator);
	crm_xml_add(msg, F_CIB_ISREPLY, originator);
	return send_cluster_message(crm_get_peer(0, originator), crm_msg_cib, msg, FALSE);
	}

	return FALSE;
	}

	/*!
	* \internal
	* \brief Handle an IPC or CPG message containing a request
	*
	* \param[in,out] request Request XML
	* \param[in] privileged Whether privileged commands may be run
	* (see cib_server_ops[] definition)
	* \param[in] cib_client IPC client that sent request (or NULL if CPG)
	*/
	static void
	cib_process_request(xmlNode *request, gboolean privileged,
	const pcmk__client_t *cib_client)
	{
	int call_type = 0;
	int call_options = 0;

	gboolean process = TRUE; // Whether to process request locally now
	gboolean is_update = TRUE; // Whether request would modify CIB
	gboolean needs_reply = TRUE; // Whether to build a reply
	gboolean local_notify = FALSE; // Whether to notify (local) requester
	gboolean needs_forward = FALSE; // Whether to forward request somewhere else

	xmlNode *op_reply = NULL;
	xmlNode *result_diff = NULL;

	int rc = pcmk_ok;
	const char *op = crm_element_value(request, F_CIB_OPERATION);
	const char *originator = crm_element_value(request, F_ORIG);
	const char *host = crm_element_value(request, F_CIB_HOST);
	const char *target = NULL;
	const char *call_id = crm_element_value(request, F_CIB_CALLID);
	const char *client_id = crm_element_value(request, F_CIB_CLIENTID);
	const char *client_name = crm_element_value(request, F_CIB_CLIENTNAME);
	const char *reply_to = crm_element_value(request, F_CIB_ISREPLY);

	crm_element_value_int(request, F_CIB_CALLOPTS, &call_options);

	if ((host != NULL) && (*host == '\0')) {
	host = NULL;
	}

	if (host) {
	target = host;

	} else if (call_options & cib_scope_local) {
	target = "local host";

	} else {
	target = "primary";
	}

	if (cib_client == NULL) {
	crm_trace("Processing peer %s operation from %s/%s on %s intended for %s (reply=%s)",
	op, client_name, call_id, originator, target, reply_to);
	} else {
	crm_xml_add(request, F_ORIG, OUR_NODENAME);
	crm_trace("Processing local %s operation from %s/%s intended for %s", op, client_name, call_id, target);
	}

	rc = cib_get_operation_id(op, &call_type);
	if (rc != pcmk_ok) {
	/* TODO: construct error reply? */
	crm_err("Pre-processing of command failed: %s", pcmk_strerror(rc));
	return;
	}

	if (cib_client != NULL) {
	parse_local_options(cib_client, call_type, call_options, host, op,
	&local_notify, &needs_reply, &process, &needs_forward);

	} else if (parse_peer_options(call_type, request, &local_notify,
	&needs_reply, &process, &needs_forward) == FALSE) {
	return;
	}

	is_update = cib_op_modifies(call_type);

	if (call_options & cib_discard_reply) {
	/* If the request will modify the CIB, and we are in legacy mode, we
	* need to build a reply so we can broadcast a diff, even if the
	* requester doesn't want one.
	*/
	needs_reply = is_update && cib_legacy_mode();
	local_notify = FALSE;
	}

	if (needs_forward) {
	const char *section = crm_element_value(request, F_CIB_SECTION);
	int log_level = LOG_INFO;

	if (pcmk__str_eq(op, PCMK__CIB_REQUEST_NOOP, pcmk__str_none)) {
	log_level = LOG_DEBUG;
	}

	do_crm_log(log_level,
	"Forwarding %s operation for section %s to %s (origin=%s/%s/%s)",
	op,
	section ? section : "'all'",
	pcmk__s(host, (cib_legacy_mode() ? "primary" : "all")),
	originator ? originator : "local",
	client_name, call_id);

	forward_request(request, call_options);
	return;
	}

	if (cib_status != pcmk_ok) {
	const char *call = crm_element_value(request, F_CIB_CALLID);

	rc = cib_status;
	crm_err("Operation ignored, cluster configuration is invalid."
	" Please repair and restart: %s", pcmk_strerror(cib_status));

	op_reply = create_xml_node(NULL, "cib-reply");
	crm_xml_add(op_reply, F_TYPE, T_CIB);
	crm_xml_add(op_reply, F_CIB_OPERATION, op);
	crm_xml_add(op_reply, F_CIB_CALLID, call);
	crm_xml_add(op_reply, F_CIB_CLIENTID, client_id);
	crm_xml_add_int(op_reply, F_CIB_CALLOPTS, call_options);
	crm_xml_add_int(op_reply, F_CIB_RC, rc);

	crm_trace("Attaching reply output");
	add_message_xml(op_reply, F_CIB_CALLDATA, the_cib);

	crm_log_xml_explicit(op_reply, "cib:reply");

	} else if (process) {
	time_t finished = 0;
	time_t now = time(NULL);
	int level = LOG_INFO;
	const char *section = crm_element_value(request, F_CIB_SECTION);

	rc = cib_process_command(request, &op_reply, &result_diff, privileged);

	if (!is_update) {
	level = LOG_TRACE;

	} else if (pcmk__xe_attr_is_true(request, F_CIB_GLOBAL_UPDATE)) {
	switch (rc) {
	case pcmk_ok:
	level = LOG_INFO;
	break;
	case -pcmk_err_old_data:
	case -pcmk_err_diff_resync:
	case -pcmk_err_diff_failed:
	level = LOG_TRACE;
	break;
	default:
	level = LOG_ERR;
	}

	} else if (rc != pcmk_ok) {
	level = LOG_WARNING;
	}

	do_crm_log(level,
	"Completed %s operation for section %s: %s (rc=%d, origin=%s/%s/%s, version=%s.%s.%s)",
	op, section ? section : "'all'", pcmk_strerror(rc), rc,
	originator ? originator : "local", client_name, call_id,
	the_cib ? crm_element_value(the_cib, XML_ATTR_GENERATION_ADMIN) : "0",
	the_cib ? crm_element_value(the_cib, XML_ATTR_GENERATION) : "0",
	the_cib ? crm_element_value(the_cib, XML_ATTR_NUMUPDATES) : "0");

	finished = time(NULL);
	if ((finished - now) > 3) {
	crm_trace("%s operation took %lds to complete", op, (long)(finished - now));
	crm_write_blackbox(0, NULL);
	}

	if (op_reply == NULL && (needs_reply \|\| local_notify)) {
	crm_err("Unexpected NULL reply to message");
	crm_log_xml_err(request, "null reply");
	needs_reply = FALSE;
	local_notify = FALSE;
	}
	}

	if (is_update && !cib_legacy_mode()) {
	crm_trace("Completed pre-sync update from %s/%s/%s%s",
	originator ? originator : "local", client_name, call_id,
	local_notify?" with local notification":"");

	} else if (!needs_reply \|\| stand_alone) {
	// This was a non-originating secondary update
	crm_trace("Completed update as secondary");

	} else if (cib_legacy_mode() &&
	rc == pcmk_ok && result_diff != NULL && !(call_options & cib_inhibit_bcast)) {
	gboolean broadcast = FALSE;

	cib_local_bcast_num++;
	crm_xml_add_int(request, F_CIB_LOCAL_NOTIFY_ID, cib_local_bcast_num);
	broadcast = send_peer_reply(request, result_diff, originator, TRUE);

	if (broadcast && client_id && local_notify && op_reply) {

	/* If we have been asked to sync the reply,
	* and a bcast msg has gone out, we queue the local notify
	* until we know the bcast message has been received */
	local_notify = FALSE;
	crm_trace("Queuing local %ssync notification for %s",
	(call_options & cib_sync_call) ? "" : "a-", client_id);

	queue_local_notify(op_reply, client_id,
	pcmk_is_set(call_options, cib_sync_call),
	(cib_client == NULL));
	op_reply = NULL; /* the reply is queued, so don't free here */
	}

	} else if (call_options & cib_discard_reply) {
	crm_trace("Caller isn't interested in reply");

	} else if (cib_client == NULL) {
	if (is_update == FALSE \|\| result_diff == NULL) {
	crm_trace("Request not broadcast: R/O call");

	} else if (call_options & cib_inhibit_bcast) {
	crm_trace("Request not broadcast: inhibited");

	} else if (rc != pcmk_ok) {
	crm_trace("Request not broadcast: call failed: %s", pcmk_strerror(rc));

	} else {
	crm_trace("Directing reply to %s", originator);
	}

	send_peer_reply(op_reply, result_diff, originator, FALSE);
	}

	if (local_notify && client_id) {
	crm_trace("Performing local %ssync notification for %s",
	(pcmk_is_set(call_options, cib_sync_call)? "" : "a"),
	client_id);
	if (process == FALSE) {
	do_local_notify(request, client_id,
	pcmk_is_set(call_options, cib_sync_call),
	(cib_client == NULL));
	} else {
	do_local_notify(op_reply, client_id,
	pcmk_is_set(call_options, cib_sync_call),
	(cib_client == NULL));
	}
	}

	free_xml(op_reply);
	free_xml(result_diff);

	return;
	}

	static char *
	calculate_section_digest(const char xpath, xmlNode xml_obj)
	{
	xmlNode *xml_section = NULL;

	if (xml_obj == NULL) {
	return NULL;
	}

	xml_section = get_xpath_object(xpath, xml_obj, LOG_TRACE);
	if (xml_section == NULL) {
	return NULL;
	}
	return calculate_xml_versioned_digest(xml_section, FALSE, TRUE, CRM_FEATURE_SET);

	}

	static int
	cib_process_command(xmlNode * request, xmlNode reply, xmlNode cib_diff, gboolean privileged)
	{
	xmlNode *input = NULL;
	xmlNode *output = NULL;
	xmlNode *result_cib = NULL;
	xmlNode *current_cib = NULL;

	int call_type = 0;
	int call_options = 0;

	const char *op = NULL;
	const char *section = NULL;
	const char *call_id = crm_element_value(request, F_CIB_CALLID);

	int rc = pcmk_ok;
	int rc2 = pcmk_ok;

	gboolean send_r_notify = FALSE;
	gboolean global_update = FALSE;
	gboolean config_changed = FALSE;
	gboolean manage_counters = TRUE;

	static mainloop_timer_t *digest_timer = NULL;

	char *current_nodes_digest = NULL;
	char *current_alerts_digest = NULL;
	char *current_status_digest = NULL;
	uint32_t change_section = cib_change_section_nodes
	\|cib_change_section_alerts
	\|cib_change_section_status;

	CRM_ASSERT(cib_status == pcmk_ok);

	if(digest_timer == NULL) {
	digest_timer = mainloop_timer_add("digester", 5000, FALSE, cib_digester_cb, NULL);
	}

	*reply = NULL;
	*cib_diff = NULL;
	current_cib = the_cib;

	/* Start processing the request... */
	op = crm_element_value(request, F_CIB_OPERATION);
	crm_element_value_int(request, F_CIB_CALLOPTS, &call_options);
	rc = cib_get_operation_id(op, &call_type);

	if (rc == pcmk_ok && privileged == FALSE) {
	rc = cib_op_can_run(call_type, call_options, privileged, global_update);
	}

	rc2 = cib_op_prepare(call_type, request, &input, &section);
	if (rc == pcmk_ok) {
	rc = rc2;
	}

	if (rc != pcmk_ok) {
	crm_trace("Call setup failed: %s", pcmk_strerror(rc));
	goto done;

	} else if (cib_op_modifies(call_type) == FALSE) {
	rc = cib_perform_op(op, call_options, cib_op_func(call_type), TRUE,
	section, request, input, FALSE, &config_changed,
	current_cib, &result_cib, NULL, &output);

	CRM_CHECK(result_cib == NULL, free_xml(result_cib));
	goto done;
	}

	/* Handle a valid write action */
	if (pcmk__xe_attr_is_true(request, F_CIB_GLOBAL_UPDATE)) {
	/* legacy code */
	manage_counters = FALSE;
	cib__set_call_options(call_options, "call", cib_force_diff);
	crm_trace("Global update detected");

	CRM_CHECK(call_type == 3 \|\| call_type == 4, crm_err("Call type: %d", call_type);
	crm_log_xml_err(request, "bad op"));
	}

	- if (rc == pcmk_ok) {
	- ping_modified_since = TRUE;
	- if (call_options & cib_inhibit_bcast) {
	- /* skip */
	- crm_trace("Skipping update: inhibit broadcast");
	- manage_counters = FALSE;
	- }
	+ ping_modified_since = TRUE;
	+ if (pcmk_is_set(call_options, cib_inhibit_bcast)) {
	+ crm_trace("Skipping update: inhibit broadcast");
	+ manage_counters = FALSE;
	+ }

	- if (!pcmk_is_set(call_options, cib_dryrun)
	- && pcmk__str_eq(section, XML_CIB_TAG_STATUS, pcmk__str_casei)) {
	- /* Copying large CIBs accounts for a huge percentage of our CIB usage */
	- cib__set_call_options(call_options, "call", cib_zero_copy);
	- } else {
	- cib__clear_call_options(call_options, "call", cib_zero_copy);
	- }
	+ if (!pcmk_is_set(call_options, cib_dryrun)
	+ && pcmk__str_eq(section, XML_CIB_TAG_STATUS, pcmk__str_casei)) {
	+ // Copying large CIBs accounts for a huge percentage of our CIB usage
	+ cib__set_call_options(call_options, "call", cib_zero_copy);
	+ } else {
	+ cib__clear_call_options(call_options, "call", cib_zero_copy);
	+ }

	- /* Calculate the hash value of the section before the change. */
	- if (pcmk__str_eq(PCMK__CIB_REQUEST_REPLACE, op, pcmk__str_none)) {
	- current_nodes_digest = calculate_section_digest("//" XML_TAG_CIB "/" XML_CIB_TAG_CONFIGURATION "/" XML_CIB_TAG_NODES, current_cib);
	- current_alerts_digest = calculate_section_digest("//" XML_TAG_CIB "/" XML_CIB_TAG_CONFIGURATION "/" XML_CIB_TAG_ALERTS, current_cib);
	- current_status_digest = calculate_section_digest("//" XML_TAG_CIB "/" XML_CIB_TAG_STATUS, current_cib);
	- crm_trace("current-digest %s:%s:%s", current_nodes_digest, current_alerts_digest, current_status_digest);
	- }
	+#define XPATH_CONFIG "//" XML_TAG_CIB "/" XML_CIB_TAG_CONFIGURATION
	+#define XPATH_NODES XPATH_CONFIG "/" XML_CIB_TAG_NODES
	+#define XPATH_ALERTS XPATH_CONFIG "/" XML_CIB_TAG_ALERTS
	+#define XPATH_STATUS "//" XML_TAG_CIB "/" XML_CIB_TAG_STATUS

	- /* result_cib must not be modified after cib_perform_op() returns */
	- rc = cib_perform_op(op, call_options, cib_op_func(call_type), FALSE,
	- section, request, input, manage_counters, &config_changed,
	- current_cib, &result_cib, cib_diff, &output);
	-
	- if (manage_counters == FALSE) {
	- int format = 1;
	- /* Legacy code
	- * If the diff is NULL at this point, it's because nothing changed
	- */
	- if (*cib_diff) {
	- crm_element_value_int(*cib_diff, "format", &format);
	- }
	+ // Calculate the hash value of the section before the change
	+ if (pcmk__str_eq(PCMK__CIB_REQUEST_REPLACE, op, pcmk__str_none)) {
	+ current_nodes_digest = calculate_section_digest(XPATH_NODES,
	+ current_cib);
	+ current_alerts_digest = calculate_section_digest(XPATH_ALERTS,
	+ current_cib);
	+ current_status_digest = calculate_section_digest(XPATH_STATUS,
	+ current_cib);
	+ crm_trace("current-digest %s:%s:%s", current_nodes_digest,
	+ current_alerts_digest, current_status_digest);
	+ }

	- if (format == 1) {
	- config_changed = cib_config_changed(NULL, NULL, cib_diff);
	- }
	- }
	+ // result_cib must not be modified after cib_perform_op() returns
	+ rc = cib_perform_op(op, call_options, cib_op_func(call_type), FALSE,
	+ section, request, input, manage_counters,
	+ &config_changed, current_cib, &result_cib, cib_diff,
	+ &output);
	+
	+ if (!manage_counters) {
	+ int format = 1;

	- /* Always write to disk for replace ops,
	- * this also negates the need to detect ordering changes
	+ /* Legacy code
	+ * If the diff is NULL at this point, it's because nothing changed
	*/
	- if (pcmk__str_eq(PCMK__CIB_REQUEST_REPLACE, op, pcmk__str_none)) {
	- config_changed = TRUE;
	+ if (*cib_diff != NULL) {
	+ crm_element_value_int(*cib_diff, "format", &format);
	}
	+
	+ if (format == 1) {
	+ config_changed = cib_config_changed(NULL, NULL, cib_diff);
	+ }
	+ }
	+
	+ /* Always write to disk for successful replace and upgrade ops. This also
	+ * negates the need to detect ordering changes.
	+ */
	+ if ((rc == pcmk_ok)
	+ && pcmk__str_any_of(op,
	+ PCMK__CIB_REQUEST_REPLACE,
	+ PCMK__CIB_REQUEST_UPGRADE,
	+ NULL)) {
	+ config_changed = TRUE;
	}

	if (rc == pcmk_ok && !pcmk_is_set(call_options, cib_dryrun)) {
	crm_trace("Activating %s->%s%s%s",
	crm_element_value(current_cib, XML_ATTR_NUMUPDATES),
	crm_element_value(result_cib, XML_ATTR_NUMUPDATES),
	(pcmk_is_set(call_options, cib_zero_copy)? " zero-copy" : ""),
	(config_changed? " changed" : ""));
	if (!pcmk_is_set(call_options, cib_zero_copy)) {
	rc = activateCibXml(result_cib, config_changed, op);
	crm_trace("Activated %s (%d)",
	crm_element_value(current_cib, XML_ATTR_NUMUPDATES), rc);
	}

	if (rc == pcmk_ok && cib_internal_config_changed(*cib_diff)) {
	cib_read_config(config_hash, result_cib);
	}

	if (pcmk__str_eq(PCMK__CIB_REQUEST_REPLACE, op, pcmk__str_none)) {
	char *result_nodes_digest = NULL;
	char *result_alerts_digest = NULL;
	char *result_status_digest = NULL;

	/* Calculate the hash value of the changed section. */
	- result_nodes_digest = calculate_section_digest("//" XML_TAG_CIB "/" XML_CIB_TAG_CONFIGURATION "/" XML_CIB_TAG_NODES, result_cib);
	- result_alerts_digest = calculate_section_digest("//" XML_TAG_CIB "/" XML_CIB_TAG_CONFIGURATION "/" XML_CIB_TAG_ALERTS, result_cib);
	- result_status_digest = calculate_section_digest("//" XML_TAG_CIB "/" XML_CIB_TAG_STATUS, result_cib);
	- crm_trace("result-digest %s:%s:%s", result_nodes_digest, result_alerts_digest, result_status_digest);
	+ result_nodes_digest = calculate_section_digest(XPATH_NODES,
	+ result_cib);
	+ result_alerts_digest = calculate_section_digest(XPATH_ALERTS,
	+ result_cib);
	+ result_status_digest = calculate_section_digest(XPATH_STATUS,
	+ result_cib);
	+ crm_trace("result-digest %s:%s:%s", result_nodes_digest,
	+ result_alerts_digest, result_status_digest);

	if (pcmk__str_eq(current_nodes_digest, result_nodes_digest,
	pcmk__str_none)) {
	change_section =
	pcmk__clear_flags_as(__func__, __LINE__, LOG_TRACE,
	"CIB change section",
	"change_section", change_section,
	cib_change_section_nodes, "nodes");
	}

	if (pcmk__str_eq(current_alerts_digest, result_alerts_digest,
	pcmk__str_none)) {
	change_section =
	pcmk__clear_flags_as(__func__, __LINE__, LOG_TRACE,
	"CIB change section",
	"change_section", change_section,
	cib_change_section_alerts, "alerts");
	}

	if (pcmk__str_eq(current_status_digest, result_status_digest,
	pcmk__str_none)) {
	change_section =
	pcmk__clear_flags_as(__func__, __LINE__, LOG_TRACE,
	"CIB change section",
	"change_section", change_section,
	cib_change_section_status, "status");
	}

	if (change_section != cib_change_section_none) {
	send_r_notify = TRUE;
	}

	free(result_nodes_digest);
	free(result_alerts_digest);
	free(result_status_digest);

	} else if (pcmk__str_eq(PCMK__CIB_REQUEST_ERASE, op, pcmk__str_none)) {
	send_r_notify = TRUE;
	}

	mainloop_timer_stop(digest_timer);
	mainloop_timer_start(digest_timer);

	} else if (rc == -pcmk_err_schema_validation) {
	CRM_ASSERT(!pcmk_is_set(call_options, cib_zero_copy));

	if (output != NULL) {
	crm_log_xml_info(output, "cib:output");
	free_xml(output);
	}

	output = result_cib;

	} else {
	crm_trace("Not activating %d %d %s", rc,
	pcmk_is_set(call_options, cib_dryrun),
	crm_element_value(result_cib, XML_ATTR_NUMUPDATES));
	if (!pcmk_is_set(call_options, cib_zero_copy)) {
	free_xml(result_cib);
	}
	}

	if ((call_options & (cib_inhibit_notify\|cib_dryrun)) == 0) {
	const char *client = crm_element_value(request, F_CIB_CLIENTNAME);

	crm_trace("Sending notifications %d",
	pcmk_is_set(call_options, cib_dryrun));
	cib_diff_notify(call_options, client, call_id, op, input, rc, *cib_diff);
	}

	if (send_r_notify) {
	const char *origin = crm_element_value(request, F_ORIG);

	cib_replace_notify(origin, the_cib, rc, *cib_diff, change_section);
	}

	xml_log_patchset(LOG_TRACE, "cib:diff", *cib_diff);
	done:
	if (!pcmk_is_set(call_options, cib_discard_reply) \|\| cib_legacy_mode()) {
	const char *caller = crm_element_value(request, F_CIB_CLIENTID);

	*reply = create_xml_node(NULL, "cib-reply");
	crm_xml_add(*reply, F_TYPE, T_CIB);
	crm_xml_add(*reply, F_CIB_OPERATION, op);
	crm_xml_add(*reply, F_CIB_CALLID, call_id);
	crm_xml_add(*reply, F_CIB_CLIENTID, caller);
	crm_xml_add_int(*reply, F_CIB_CALLOPTS, call_options);
	crm_xml_add_int(*reply, F_CIB_RC, rc);

	if (output != NULL) {
	crm_trace("Attaching reply output");
	add_message_xml(*reply, F_CIB_CALLDATA, output);
	}

	crm_log_xml_explicit(*reply, "cib:reply");
	}

	crm_trace("cleanup");

	if (cib_op_modifies(call_type) == FALSE && output != current_cib) {
	free_xml(output);
	output = NULL;
	}

	if (call_type >= 0) {
	cib_op_cleanup(call_type, call_options, &input, &output);
	}

	free(current_nodes_digest);
	free(current_alerts_digest);
	free(current_status_digest);

	crm_trace("done");
	return rc;
	}

	void
	cib_peer_callback(xmlNode * msg, void *private_data)
	{
	const char *reason = NULL;
	const char *originator = crm_element_value(msg, F_ORIG);

	if (cib_legacy_mode()
	&& pcmk__str_eq(originator, OUR_NODENAME,
	pcmk__str_casei\|pcmk__str_null_matches)) {
	/* message is from ourselves */
	int bcast_id = 0;

	if (!(crm_element_value_int(msg, F_CIB_LOCAL_NOTIFY_ID, &bcast_id))) {
	check_local_notify(bcast_id);
	}
	return;

	} else if (crm_peer_cache == NULL) {
	reason = "membership not established";
	goto bail;
	}

	if (crm_element_value(msg, F_CIB_CLIENTNAME) == NULL) {
	crm_xml_add(msg, F_CIB_CLIENTNAME, originator);
	}

	/* crm_log_xml_trace(msg, "Peer[inbound]"); */
	cib_process_request(msg, TRUE, NULL);
	return;

	bail:
	if (reason) {
	const char *seq = crm_element_value(msg, F_SEQ);
	const char *op = crm_element_value(msg, F_CIB_OPERATION);

	crm_warn("Discarding %s message (%s) from %s: %s", op, seq, originator, reason);
	}
	}

	static gboolean
	cib_force_exit(gpointer data)
	{
	crm_notice("Forcing exit!");
	terminate_cib(__func__, CRM_EX_ERROR);
	return FALSE;
	}

	static void
	disconnect_remote_client(gpointer key, gpointer value, gpointer user_data)
	{
	pcmk__client_t *a_client = value;

	crm_err("Can't disconnect client %s: Not implemented",
	pcmk__client_name(a_client));
	}

	void
	cib_shutdown(int nsig)
	{
	struct qb_ipcs_stats srv_stats;

	if (cib_shutdown_flag == FALSE) {
	int disconnects = 0;
	qb_ipcs_connection_t *c = NULL;

	cib_shutdown_flag = TRUE;

	c = qb_ipcs_connection_first_get(ipcs_rw);
	while (c != NULL) {
	qb_ipcs_connection_t *last = c;

	c = qb_ipcs_connection_next_get(ipcs_rw, last);

	crm_debug("Disconnecting r/w client %p...", last);
	qb_ipcs_disconnect(last);
	qb_ipcs_connection_unref(last);
	disconnects++;
	}

	c = qb_ipcs_connection_first_get(ipcs_ro);
	while (c != NULL) {
	qb_ipcs_connection_t *last = c;

	c = qb_ipcs_connection_next_get(ipcs_ro, last);

	crm_debug("Disconnecting r/o client %p...", last);
	qb_ipcs_disconnect(last);
	qb_ipcs_connection_unref(last);
	disconnects++;
	}

	c = qb_ipcs_connection_first_get(ipcs_shm);
	while (c != NULL) {
	qb_ipcs_connection_t *last = c;

	c = qb_ipcs_connection_next_get(ipcs_shm, last);

	crm_debug("Disconnecting non-blocking r/w client %p...", last);
	qb_ipcs_disconnect(last);
	qb_ipcs_connection_unref(last);
	disconnects++;
	}

	disconnects += pcmk__ipc_client_count();

	crm_debug("Disconnecting %d remote clients", pcmk__ipc_client_count());
	pcmk__foreach_ipc_client(disconnect_remote_client, NULL);
	crm_info("Disconnected %d clients", disconnects);
	}

	qb_ipcs_stats_get(ipcs_rw, &srv_stats, QB_FALSE);

	if (pcmk__ipc_client_count() == 0) {
	crm_info("All clients disconnected (%d)", srv_stats.active_connections);
	initiate_exit();

	} else {
	crm_info("Waiting on %d clients to disconnect (%d)",
	pcmk__ipc_client_count(), srv_stats.active_connections);
	}
	}

	void
	initiate_exit(void)
	{
	int active = 0;
	xmlNode *leaving = NULL;

	active = crm_active_peers();
	if (active < 2) {
	terminate_cib(__func__, 0);
	return;
	}

	crm_info("Sending disconnect notification to %d peers...", active);

	leaving = create_xml_node(NULL, "exit-notification");
	crm_xml_add(leaving, F_TYPE, "cib");
	crm_xml_add(leaving, F_CIB_OPERATION, PCMK__CIB_REQUEST_SHUTDOWN);

	send_cluster_message(NULL, crm_msg_cib, leaving, TRUE);
	free_xml(leaving);

	g_timeout_add(EXIT_ESCALATION_MS, cib_force_exit, NULL);
	}

	extern int remote_fd;
	extern int remote_tls_fd;

	/*!
	* \internal
	* \brief Close remote sockets, free the global CIB and quit
	*
	* \param[in] caller Name of calling function (for log message)
	* \param[in] fast If -1, skip disconnect; if positive, exit that
	*/
	void
	terminate_cib(const char *caller, int fast)
	{
	crm_info("%s: Exiting%s...", caller,
	(fast > 0)? " fast" : mainloop ? " from mainloop" : "");

	if (remote_fd > 0) {
	close(remote_fd);
	remote_fd = 0;
	}
	if (remote_tls_fd > 0) {
	close(remote_tls_fd);
	remote_tls_fd = 0;
	}

	uninitializeCib();

	if (fast > 0) {
	/* Quit fast on error */
	pcmk__stop_based_ipc(ipcs_ro, ipcs_rw, ipcs_shm);
	crm_exit(fast);

	} else if ((mainloop != NULL) && g_main_loop_is_running(mainloop)) {
	/* Quit via returning from the main loop. If fast == -1, we skip the
	* disconnect here, and it will be done when the main loop returns
	* (this allows the peer status callback to avoid messing with the
	* peer caches).
	*/
	if (fast == 0) {
	crm_cluster_disconnect(crm_cluster);
	}
	g_main_loop_quit(mainloop);

	} else {
	/* Quit via clean exit. Even the peer status callback can disconnect
	* here, because we're not returning control to the caller. */
	crm_cluster_disconnect(crm_cluster);
	pcmk__stop_based_ipc(ipcs_ro, ipcs_rw, ipcs_shm);
	crm_exit(CRM_EX_OK);
	}
	}
	diff --git a/tools/cibadmin.c b/tools/cibadmin.c
	index 035533552f..7a3378fa09 100644
	--- a/tools/cibadmin.c
	+++ b/tools/cibadmin.c
	@@ -1,949 +1,950 @@
	/*
	* Copyright 2004-2022 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>
	#include <stdio.h>
	#include <crm/crm.h>
	#include <crm/msg_xml.h>
	#include <crm/common/cmdline_internal.h>
	#include <crm/common/ipc.h>
	#include <crm/common/xml.h>
	#include <crm/cib/internal.h>

	#include <pacemaker-internal.h>

	#define SUMMARY "query and edit the Pacemaker configuration"

	#define INDENT " "

	enum cibadmin_section_type {
	cibadmin_section_all = 0,
	cibadmin_section_scope,
	cibadmin_section_xpath,
	};

	static int request_id = 0;

	static cib_t *the_cib = NULL;
	static GMainLoop *mainloop = NULL;
	static crm_exit_t exit_code = CRM_EX_OK;

	static struct {
	const char *cib_action;
	int cmd_options;
	enum cibadmin_section_type section_type;
	char *cib_section;
	char *validate_with;
	gint message_timeout_sec;
	enum pcmk__acl_render_how acl_render_mode;
	gchar *cib_user;
	gchar *dest_node;
	gchar *input_file;
	gchar *input_xml;
	gboolean input_stdin;
	bool delete_all;
	gboolean allow_create;
	gboolean force;
	gboolean get_node_path;
	gboolean local;
	gboolean no_children;
	gboolean sync_call;

	/* @COMPAT: For "-!" version option. Not advertised nor marked as
	* deprecated, but accepted.
	*/
	gboolean extended_version;

	//! \deprecated
	gboolean no_bcast;
	} options;

	int do_init(void);
	static int do_work(xmlNode input, xmlNode *output);
	void cibadmin_op_callback(xmlNode msg, int call_id, int rc, xmlNode output,
	void *user_data);

	static void
	print_xml_output(xmlNode * xml)
	{
	char *buffer;

	if (!xml) {
	return;
	} else if (xml->type != XML_ELEMENT_NODE) {
	return;
	}

	if (pcmk_is_set(options.cmd_options, cib_xpath_address)) {
	const char *id = crm_element_value(xml, XML_ATTR_ID);

	if (pcmk__str_eq((const char *)xml->name, "xpath-query", pcmk__str_casei)) {
	xmlNode *child = NULL;

	for (child = xml->children; child; child = child->next) {
	print_xml_output(child);
	}

	} else if (id) {
	printf("%s\n", id);
	}

	} else {
	buffer = dump_xml_formatted(xml);
	fprintf(stdout, "%s", pcmk__s(buffer, "<null>\n"));
	free(buffer);
	}
	}

	// Upgrade requested but already at latest schema
	static void
	report_schema_unchanged(void)
	{
	const char *err = pcmk_rc_str(pcmk_rc_schema_unchanged);

	crm_info("Upgrade unnecessary: %s\n", err);
	printf("Upgrade unnecessary: %s\n", err);
	exit_code = CRM_EX_OK;
	}

	/*!
	* \internal
	* \brief Check whether the current CIB action is dangerous
	* \return true if \p options.cib_action is dangerous, or false otherwise
	*/
	static inline bool
	cib_action_is_dangerous(void)
	{
	return options.no_bcast \|\| options.delete_all
	\|\| pcmk__str_any_of(options.cib_action,
	PCMK__CIB_REQUEST_UPGRADE,
	PCMK__CIB_REQUEST_ERASE,
	NULL);
	}

	/*!
	* \internal
	* \brief Determine whether the given CIB scope is valid for \p cibadmin
	*
	* \param[in] scope Scope to validate
	*
	* \return true if \p scope is valid, or false otherwise
	* \note An invalid scope applies the operation to the entire CIB.
	*/
	static inline bool
	scope_is_valid(const char *scope)
	{
	return pcmk__str_any_of(scope,
	XML_CIB_TAG_CONFIGURATION,
	XML_CIB_TAG_NODES,
	XML_CIB_TAG_RESOURCES,
	XML_CIB_TAG_CONSTRAINTS,
	XML_CIB_TAG_CRMCONFIG,
	XML_CIB_TAG_RSCCONFIG,
	XML_CIB_TAG_OPCONFIG,
	XML_CIB_TAG_ACLS,
	XML_TAG_FENCING_TOPOLOGY,
	XML_CIB_TAG_TAGS,
	XML_CIB_TAG_ALERTS,
	XML_CIB_TAG_STATUS,
	NULL);
	}

	static gboolean
	command_cb(const gchar option_name, const gchar optarg, gpointer data,
	GError **error)
	{
	options.delete_all = false;

	if (pcmk__str_any_of(option_name, "-u", "--upgrade", NULL)) {
	options.cib_action = PCMK__CIB_REQUEST_UPGRADE;

	} else if (pcmk__str_any_of(option_name, "-Q", "--query", NULL)) {
	options.cib_action = PCMK__CIB_REQUEST_QUERY;

	} else if (pcmk__str_any_of(option_name, "-E", "--erase", NULL)) {
	options.cib_action = PCMK__CIB_REQUEST_ERASE;

	} else if (pcmk__str_any_of(option_name, "-B", "--bump", NULL)) {
	options.cib_action = PCMK__CIB_REQUEST_BUMP;

	} else if (pcmk__str_any_of(option_name, "-C", "--create", NULL)) {
	options.cib_action = PCMK__CIB_REQUEST_CREATE;

	} else if (pcmk__str_any_of(option_name, "-M", "--modify", NULL)) {
	options.cib_action = PCMK__CIB_REQUEST_MODIFY;

	} else if (pcmk__str_any_of(option_name, "-P", "--patch", NULL)) {
	options.cib_action = PCMK__CIB_REQUEST_APPLY_PATCH;

	} else if (pcmk__str_any_of(option_name, "-R", "--replace", NULL)) {
	options.cib_action = PCMK__CIB_REQUEST_REPLACE;

	} else if (pcmk__str_any_of(option_name, "-D", "--delete", NULL)) {
	options.cib_action = PCMK__CIB_REQUEST_DELETE;

	} else if (pcmk__str_any_of(option_name, "-d", "--delete-all", NULL)) {
	options.cib_action = PCMK__CIB_REQUEST_DELETE;
	options.delete_all = true;

	} else if (pcmk__str_any_of(option_name, "-a", "--empty", NULL)) {
	options.cib_action = "empty";
	pcmk__str_update(&options.validate_with, optarg);

	} else if (pcmk__str_any_of(option_name, "-5", "--md5-sum", NULL)) {
	options.cib_action = "md5-sum";

	} else if (pcmk__str_any_of(option_name, "-6", "--md5-sum-versioned",
	NULL)) {
	options.cib_action = "md5-sum-versioned";

	} else {
	// Should be impossible
	return FALSE;
	}

	return TRUE;
	}

	static gboolean
	show_access_cb(const gchar option_name, const gchar optarg, gpointer data,
	GError **error)
	{
	if (pcmk__str_eq(optarg, "auto", pcmk__str_null_matches)) {
	options.acl_render_mode = pcmk__acl_render_default;

	} else if (g_strcmp0(optarg, "namespace") == 0) {
	options.acl_render_mode = pcmk__acl_render_namespace;

	} else if (g_strcmp0(optarg, "text") == 0) {
	options.acl_render_mode = pcmk__acl_render_text;

	} else if (g_strcmp0(optarg, "color") == 0) {
	options.acl_render_mode = pcmk__acl_render_color;

	} else {
	g_set_error(error, PCMK__EXITC_ERROR, CRM_EX_USAGE,
	"Invalid value '%s' for option '%s'",
	optarg, option_name);
	return FALSE;
	}
	return TRUE;
	}

	static gboolean
	section_cb(const gchar option_name, const gchar optarg, gpointer data,
	GError **error)
	{
	if (pcmk__str_any_of(option_name, "-o", "--scope", NULL)) {
	options.section_type = cibadmin_section_scope;

	} else if (pcmk__str_any_of(option_name, "-A", "--xpath", NULL)) {
	options.section_type = cibadmin_section_xpath;

	} else {
	// Should be impossible
	return FALSE;
	}

	pcmk__str_update(&options.cib_section, optarg);
	return TRUE;
	}

	static GOptionEntry command_entries[] = {
	{ "upgrade", 'u', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, command_cb,
	"Upgrade the configuration to the latest syntax", NULL },

	{ "query", 'Q', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, command_cb,
	"Query the contents of the CIB", NULL },

	{ "erase", 'E', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, command_cb,
	"Erase the contents of the whole CIB", NULL },

	{ "bump", 'B', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, command_cb,
	"Increase the CIB's epoch value by 1", NULL },

	{ "create", 'C', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, command_cb,
	"Create an object in the CIB (will fail if object already exists)",
	NULL },

	{ "modify", 'M', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, command_cb,
	"Find object somewhere in CIB's XML tree and update it (fails if object "
	"does not exist unless -c is also specified)",
	NULL },

	{ "patch", 'P', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, command_cb,
	"Supply an update in the form of an XML diff (see crm_diff(8))", NULL },

	{ "replace", 'R', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, command_cb,
	"Recursively replace an object in the CIB", NULL },

	{ "delete", 'D', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, command_cb,
	"Delete first object matching supplied criteria (for example, "
	"<op id=\"rsc1_op1\" name=\"monitor\"/>).\n"
	INDENT "The XML element name and all attributes must match in order for "
	"the element to be deleted.",
	NULL },

	{ "delete-all", 'd', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK,
	command_cb,
	"When used with --xpath, remove all matching objects in the "
	"configuration instead of just the first one",
	NULL },

	{ "empty", 'a', G_OPTION_FLAG_OPTIONAL_ARG, G_OPTION_ARG_CALLBACK,
	command_cb,
	"Output an empty CIB. Accepts an optional schema name argument to use as "
	"the " XML_ATTR_VALIDATION " value.\n"
	INDENT "If no schema is given, the latest will be used.",
	"[schema]" },

	{ "md5-sum", '5', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, command_cb,
	"Calculate the on-disk CIB digest", NULL },

	{ "md5-sum-versioned", '6', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK,
	command_cb, "Calculate an on-the-wire versioned CIB digest", NULL },

	{ NULL }
	};

	static GOptionEntry data_entries[] = {
	/* @COMPAT: These arguments should be last-wins. We can have an enum option
	* that stores the input type, along with a single string option that stores
	* the XML string for --xml-text, filename for --xml-file, or NULL for
	* --xml-pipe.
	*/
	{ "xml-text", 'X', G_OPTION_FLAG_NONE, G_OPTION_ARG_STRING,
	&options.input_xml, "Retrieve XML from the supplied string", "value" },

	{ "xml-file", 'x', G_OPTION_FLAG_NONE, G_OPTION_ARG_FILENAME,
	&options.input_file, "Retrieve XML from the named file", "value" },

	{ "xml-pipe", 'p', G_OPTION_FLAG_NONE, G_OPTION_ARG_NONE,
	&options.input_stdin, "Retrieve XML from stdin", NULL },

	{ NULL }
	};

	static GOptionEntry addl_entries[] = {
	{ "force", 'f', G_OPTION_FLAG_NONE, G_OPTION_ARG_NONE, &options.force,
	"Force the action to be performed", NULL },

	{ "timeout", 't', G_OPTION_FLAG_NONE, G_OPTION_ARG_INT,
	&options.message_timeout_sec,
	"Time (in seconds) to wait before declaring the operation failed",
	"value" },

	{ "user", 'U', G_OPTION_FLAG_NONE, G_OPTION_ARG_STRING, &options.cib_user,
	"Run the command with permissions of the named user (valid only for the "
	"root and " CRM_DAEMON_USER " accounts)", "value" },

	{ "sync-call", 's', G_OPTION_FLAG_NONE, G_OPTION_ARG_NONE,
	&options.sync_call, "Wait for call to complete before returning", NULL },

	{ "local", 'l', G_OPTION_FLAG_NONE, G_OPTION_ARG_NONE, &options.local,
	"Command takes effect locally (should be used only for queries)", NULL },

	{ "scope", 'o', G_OPTION_FLAG_NONE, G_OPTION_ARG_CALLBACK, section_cb,
	"Limit scope of operation to specific section of CIB\n"
	INDENT "Valid values: " XML_CIB_TAG_CONFIGURATION ", " XML_CIB_TAG_NODES
	", " XML_CIB_TAG_RESOURCES ", " XML_CIB_TAG_CONSTRAINTS
	", " XML_CIB_TAG_CRMCONFIG ", " XML_CIB_TAG_RSCCONFIG ",\n"
	INDENT " " XML_CIB_TAG_OPCONFIG ", " XML_CIB_TAG_ACLS
	", " XML_TAG_FENCING_TOPOLOGY ", " XML_CIB_TAG_TAGS
	", " XML_CIB_TAG_ALERTS ", " XML_CIB_TAG_STATUS "\n"
	INDENT "If both --scope/-o and --xpath/-a are specified, the last one to "
	"appear takes effect",
	"value" },

	{ "xpath", 'A', G_OPTION_FLAG_NONE, G_OPTION_ARG_CALLBACK, section_cb,
	"A valid XPath to use instead of --scope/-o\n"
	INDENT "If both --scope/-o and --xpath/-a are specified, the last one to "
	"appear takes effect",
	"value" },

	{ "node-path", 'e', G_OPTION_FLAG_NONE, G_OPTION_ARG_NONE,
	&options.get_node_path,
	"When performing XPath queries, return paths of any matches found\n"
	INDENT "(for example, \"/cib/configuration/resources"
	"/clone[@id='dummy-clone']/primitive[@id='dummy']\")",
	NULL },

	{ "show-access", 'S', G_OPTION_FLAG_OPTIONAL_ARG, G_OPTION_ARG_CALLBACK,
	show_access_cb,
	"Whether to use syntax highlighting for ACLs (with -Q/--query and "
	"-U/--user)\n"
	INDENT "Allowed values: 'color' (default for terminal), 'text' (plain text, "
	"default for non-terminal),\n"
	INDENT " 'namespace', or 'auto' (use default value)\n"
	INDENT "Default value: 'auto'",
	"[value]" },

	{ "allow-create", 'c', G_OPTION_FLAG_NONE, G_OPTION_ARG_NONE,
	&options.allow_create,
	"(Advanced) Allow target of --modify/-M to be created if it does not "
	"exist",
	NULL },

	{ "no-children", 'n', G_OPTION_FLAG_NONE, G_OPTION_ARG_NONE,
	&options.no_children,
	"(Advanced) When querying an object, do not include its children in the "
	"result",
	NULL },

	{ "node", 'N', G_OPTION_FLAG_NONE, G_OPTION_ARG_STRING, &options.dest_node,
	"(Advanced) Send command to the specified host", "value" },

	// @COMPAT: Deprecated
	{ "no-bcast", 'b', G_OPTION_FLAG_HIDDEN, G_OPTION_ARG_NONE,
	&options.no_bcast, "deprecated", NULL },

	// @COMPAT: Deprecated
	{ "host", 'h', G_OPTION_FLAG_HIDDEN, G_OPTION_ARG_STRING,
	&options.dest_node, "deprecated", NULL },

	{ NULL }
	};

	static GOptionContext *
	build_arg_context(pcmk__common_args_t *args)
	{
	const char *desc = NULL;
	GOptionContext *context = NULL;

	GOptionEntry extra_prog_entries[] = {
	// @COMPAT: Deprecated
	{ "extended-version", '!', G_OPTION_FLAG_HIDDEN, G_OPTION_ARG_NONE,
	&options.extended_version, "deprecated", NULL },

	{ NULL }
	};

	desc = "Examples:\n\n"
	"Query the configuration from the local node:\n\n"
	"\t# cibadmin --query --local\n\n"
	"Query just the cluster options configuration:\n\n"
	"\t# cibadmin --query --scope crm_config\n\n"
	"Query all 'target-role' settings:\n\n"
	"\t# cibadmin --query --xpath \"//nvpair[@name='target-role']\"\n\n"
	"Remove all 'is-managed' settings:\n\n"
	"\t# cibadmin --delete-all --xpath "
	"\"//nvpair[@name='is-managed']\"\n\n"
	"Remove the resource named 'old':\n\n"
	"\t# cibadmin --delete --xml-text '<primitive id=\"old\"/>'\n\n"
	"Remove all resources from the configuration:\n\n"
	"\t# cibadmin --replace --scope resources --xml-text "
	"'<resources/>'\n\n"
	"Replace complete configuration with contents of "
	"$HOME/pacemaker.xml:\n\n"
	"\t# cibadmin --replace --xml-file $HOME/pacemaker.xml\n\n"
	"Replace constraints section of configuration with contents of "
	"$HOME/constraints.xml:\n\n"
	"\t# cibadmin --replace --scope constraints --xml-file "
	"$HOME/constraints.xml\n\n"
	"Increase configuration version to prevent old configurations from "
	"being loaded accidentally:\n\n"
	"\t# cibadmin --modify --xml-text "
	"'<cib admin_epoch=\"admin_epoch++\"/>'\n\n"
	"Edit the configuration with your favorite $EDITOR:\n\n"
	"\t# cibadmin --query > $HOME/local.xml\n\n"
	"\t# $EDITOR $HOME/local.xml\n\n"
	"\t# cibadmin --replace --xml-file $HOME/local.xml\n\n"
	"Assuming terminal, render configuration in color (green for "
	"writable, blue for readable, red for\n"
	"denied) to visualize permissions for user tony:\n\n"
	"\t# cibadmin --show-access=color --query --user tony \| less -r\n\n"
	"SEE ALSO:\n"
	" crm(8), pcs(8), crm_shadow(8), crm_diff(8)\n";

	context = pcmk__build_arg_context(args, NULL, NULL, "<command>");
	g_option_context_set_description(context, desc);

	pcmk__add_main_args(context, extra_prog_entries);

	pcmk__add_arg_group(context, "commands", "Commands:", "Show command help",
	command_entries);
	pcmk__add_arg_group(context, "data", "Data:", "Show data help",
	data_entries);
	pcmk__add_arg_group(context, "additional", "Additional Options:",
	"Show additional options", addl_entries);
	return context;
	}

	int
	main(int argc, char **argv)
	{
	int rc = pcmk_rc_ok;
	const char *source = NULL;
	xmlNode *output = NULL;
	xmlNode *input = NULL;
	gchar *acl_cred = NULL;

	GError *error = NULL;

	pcmk__common_args_t *args = pcmk__new_common_args(SUMMARY);
	gchar **processed_args = pcmk__cmdline_preproc(argv, "ANSUXhotx");
	GOptionContext *context = build_arg_context(args);

	if (!g_option_context_parse_strv(context, &processed_args, &error)) {
	exit_code = CRM_EX_USAGE;
	goto done;
	}

	if (g_strv_length(processed_args) > 1) {
	gchar *help = g_option_context_get_help(context, TRUE, NULL);
	GString *extra = g_string_sized_new(128);

	for (int lpc = 1; processed_args[lpc] != NULL; lpc++) {
	if (extra->len > 0) {
	g_string_append_c(extra, ' ');
	}
	g_string_append(extra, processed_args[lpc]);
	}

	exit_code = CRM_EX_USAGE;
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	"non-option ARGV-elements: %s\n\n%s", extra->str, help);
	g_free(help);
	g_string_free(extra, TRUE);
	goto done;
	}

	if (args->version \|\| options.extended_version) {
	g_strfreev(processed_args);
	pcmk__free_arg_context(context);

	/* FIXME: When cibadmin is converted to use formatted output, this can
	* be replaced by out->version with the appropriate boolean flag.
	*
	* options.extended_version is deprecated and will be removed in a
	* future release.
	*/
	pcmk__cli_help(options.extended_version? '!' : 'v', CRM_EX_OK);
	}

	/* At LOG_ERR, stderr for CIB calls is rather verbose. Several lines like
	*
	* (func@file:line) error: CIB <op> failures <XML>
	*
	* In cibadmin we explicitly output the XML portion without the prefixes. So
	* we default to LOG_CRIT.
	*/
	pcmk__cli_init_logging("cibadmin", 0);
	set_crm_log_level(LOG_CRIT);

	if (args->verbosity > 0) {
	cib__set_call_options(options.cmd_options, crm_system_name,
	cib_verbose);

	for (int i = 0; i < args->verbosity; i++) {
	crm_bump_log_level(argc, argv);
	}
	}

	if (options.cib_action == NULL) {
	// @COMPAT: Create a default command if other tools have one
	gchar *help = g_option_context_get_help(context, TRUE, NULL);

	exit_code = CRM_EX_USAGE;
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	"Must specify a command option\n\n%s", help);
	g_free(help);
	goto done;
	}

	if (strcmp(options.cib_action, "empty") == 0) {
	// Output an empty CIB
	char *buf = NULL;

	output = createEmptyCib(1);
	crm_xml_add(output, XML_ATTR_VALIDATION, options.validate_with);
	buf = dump_xml_formatted(output);
	fprintf(stdout, "%s", pcmk__s(buf, "<null>\n"));
	free(buf);
	goto done;
	}

	if (cib_action_is_dangerous() && !options.force) {
	exit_code = CRM_EX_UNSAFE;
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	"The supplied command is considered dangerous. To prevent "
	"accidental destruction of the cluster, the --force flag "
	"is required in order to proceed.");
	goto done;
	}

	if (options.message_timeout_sec < 1) {
	// Set default timeout
	options.message_timeout_sec = 30;
	}

	if (options.section_type == cibadmin_section_xpath) {
	// Enable getting section by XPath
	cib__set_call_options(options.cmd_options, crm_system_name,
	cib_xpath);

	} else if (options.section_type == cibadmin_section_scope) {
	if (!scope_is_valid(options.cib_section)) {
	// @COMPAT: Consider requiring --force to proceed
	fprintf(stderr,
	"Invalid value '%s' for '--scope'. Operation will apply "
	"to the entire CIB.\n", options.cib_section);
	}
	}

	if (options.allow_create) {
	// Allow target of --modify/-M to be created if it does not exist
	cib__set_call_options(options.cmd_options, crm_system_name,
	cib_can_create);
	}

	if (options.delete_all) {
	// With cibadmin_section_xpath, remove all matching objects
	cib__set_call_options(options.cmd_options, crm_system_name,
	cib_multiple);
	}

	if (options.force) {
	// Perform the action even without quorum
	cib__set_call_options(options.cmd_options, crm_system_name,
	cib_quorum_override);
	}

	if (options.get_node_path) {
	/* Enable getting node path of XPath query matches.
	* Meaningful only if options.section_type == cibadmin_section_xpath.
	*/
	cib__set_call_options(options.cmd_options, crm_system_name,
	cib_xpath_address);
	}

	if (options.local) {
	// Configure command to take effect only locally
	cib__set_call_options(options.cmd_options, crm_system_name,
	cib_scope_local);
	}

	// @COMPAT: Deprecated option
	if (options.no_bcast) {
	// Configure command to take effect only locally and not to broadcast
	cib__set_call_options(options.cmd_options, crm_system_name,
	cib_inhibit_bcast\|cib_scope_local);
	}

	if (options.no_children) {
	// When querying an object, don't include its children in the result
	cib__set_call_options(options.cmd_options, crm_system_name,
	cib_no_children);
	}

	if (options.sync_call
	\|\| (options.acl_render_mode != pcmk__acl_render_none)) {
	/* Wait for call to complete before returning.
	*
	* The ACL render modes work only with sync calls due to differences in
	* output handling between sync/async. It shouldn't matter to the user
	* whether the call is synchronous; for a CIB query, we have to wait for
	* the result in order to display it in any case.
	*/
	cib__set_call_options(options.cmd_options, crm_system_name,
	cib_sync_call);
	}

	if (options.input_file != NULL) {
	input = filename2xml(options.input_file);
	source = options.input_file;

	} else if (options.input_xml != NULL) {
	input = string2xml(options.input_xml);
	source = "input string";

	} else if (options.input_stdin) {
	source = "STDIN";
	input = stdin2xml();

	} else if (options.acl_render_mode != pcmk__acl_render_none) {
	char *username = pcmk__uid2username(geteuid());
	bool required = pcmk_acl_required(username);

	free(username);

	if (required) {
	if (options.force) {
	fprintf(stderr, "The supplied command can provide skewed"
	" result since it is run under user that also"
	" gets guarded per ACLs on their own right."
	" Continuing since --force flag was"
	" provided.\n");

	} else {
	exit_code = CRM_EX_UNSAFE;
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	"The supplied command can provide skewed result "
	"since it is run under user that also gets guarded "
	"per ACLs in their own right. To accept the risk "
	"of such a possible distortion (without even "
	"knowing it at this time), use the --force flag.");
	goto done;
	}
	}

	if (options.cib_user == NULL) {
	exit_code = CRM_EX_USAGE;
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	"The supplied command requires -U user specified.");
	goto done;
	}

	/* We already stopped/warned ACL-controlled users about consequences.
	*
	* Note: acl_cred takes ownership of options.cib_user here.
	* options.cib_user is set to NULL so that the CIB is obtained as the
	* user running the cibadmin command. The CIB must be obtained as a user
	* with full permissions in order to show the CIB correctly annotated
	* for the options.cib_user's permissions.
	*/
	acl_cred = options.cib_user;
	options.cib_user = NULL;
	}

	if (input != NULL) {
	crm_log_xml_debug(input, "[admin input]");

	} else if (source != NULL) {
	exit_code = CRM_EX_CONFIG;
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	"Couldn't parse input from %s.", source);
	goto done;
	}

	if (strcmp(options.cib_action, "md5-sum") == 0) {
	char *digest = NULL;

	if (input == NULL) {
	exit_code = CRM_EX_USAGE;
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	"Please supply XML to process with -X, -x, or -p");
	goto done;
	}

	digest = calculate_on_disk_digest(input);
	fprintf(stderr, "Digest: ");
	fprintf(stdout, "%s\n", pcmk__s(digest, "<null>"));
	free(digest);
	goto done;

	} else if (strcmp(options.cib_action, "md5-sum-versioned") == 0) {
	char *digest = NULL;
	const char *version = NULL;

	if (input == NULL) {
	exit_code = CRM_EX_USAGE;
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	"Please supply XML to process with -X, -x, or -p");
	goto done;
	}

	version = crm_element_value(input, XML_ATTR_CRM_VERSION);
	digest = calculate_xml_versioned_digest(input, FALSE, TRUE, version);
	fprintf(stderr, "Versioned (%s) digest: ", version);
	fprintf(stdout, "%s\n", pcmk__s(digest, "<null>"));
	free(digest);
	goto done;
	}

	rc = do_init();
	if (rc != pcmk_ok) {
	rc = pcmk_legacy2rc(rc);
	exit_code = pcmk_rc2exitc(rc);

	crm_err("Init failed, could not perform requested operations: %s",
	pcmk_rc_str(rc));
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	"Init failed, could not perform requested operations: %s",
	pcmk_rc_str(rc));
	goto done;
	}

	rc = do_work(input, &output);
	if (rc > 0) {
	/* wait for the reply by creating a mainloop and running it until
	* the callbacks are invoked...
	*/
	request_id = rc;

	the_cib->cmds->register_callback(the_cib, request_id,
	options.message_timeout_sec, FALSE,
	NULL, "cibadmin_op_callback",
	cibadmin_op_callback);

	mainloop = g_main_loop_new(NULL, FALSE);

	crm_trace("%s waiting for reply from the local CIB", crm_system_name);

	crm_info("Starting mainloop");
	g_main_loop_run(mainloop);

	} else if ((rc == -pcmk_err_schema_unchanged)
	&& (strcmp(options.cib_action,
	PCMK__CIB_REQUEST_UPGRADE) == 0)) {
	report_schema_unchanged();

	} else if (rc < 0) {
	rc = pcmk_legacy2rc(rc);
	crm_err("Call failed: %s", pcmk_rc_str(rc));
	fprintf(stderr, "Call failed: %s\n", pcmk_rc_str(rc));

	if (rc == pcmk_rc_schema_validation) {
	if (strcmp(options.cib_action, PCMK__CIB_REQUEST_UPGRADE) == 0) {
	xmlNode *obj = NULL;
	int version = 0;

	if (the_cib->cmds->query(the_cib, NULL, &obj,
	options.cmd_options) == pcmk_ok) {
	update_validation(&obj, &version, 0, TRUE, FALSE);
	}
	+ free_xml(obj);

	} else if (output) {
	validate_xml_verbose(output);
	}
	}
	exit_code = pcmk_rc2exitc(rc);
	}

	if ((output != NULL)
	&& (options.acl_render_mode != pcmk__acl_render_none)) {

	xmlDoc *acl_evaled_doc;
	rc = pcmk__acl_annotate_permissions(acl_cred, output->doc, &acl_evaled_doc);
	if (rc == pcmk_rc_ok) {
	xmlChar *rendered = NULL;

	rc = pcmk__acl_evaled_render(acl_evaled_doc,
	options.acl_render_mode, &rendered);
	if (rc != pcmk_rc_ok) {
	exit_code = CRM_EX_CONFIG;
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	"Could not render evaluated access: %s",
	pcmk_rc_str(rc));
	goto done;
	}
	printf("%s\n", (char *) rendered);
	free(rendered);

	} else {
	exit_code = CRM_EX_CONFIG;
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	"Could not evaluate access per request (%s, error: %s)",
	acl_cred, pcmk_rc_str(rc));
	goto done;
	}

	} else if (output != NULL) {
	print_xml_output(output);
	}

	crm_trace("%s exiting normally", crm_system_name);

	done:
	g_strfreev(processed_args);
	pcmk__free_arg_context(context);

	g_free(options.cib_user);
	g_free(options.dest_node);
	g_free(options.input_file);
	g_free(options.input_xml);
	free(options.cib_section);
	free(options.validate_with);

	g_free(acl_cred);
	free_xml(input);
	free_xml(output);

	rc = cib__clean_up_connection(&the_cib);
	if (exit_code == CRM_EX_OK) {
	exit_code = pcmk_rc2exitc(rc);
	}

	pcmk__output_and_clear_error(error, NULL);
	crm_exit(exit_code);
	}

	static int
	do_work(xmlNode input, xmlNode *output)
	{
	/* construct the request */
	the_cib->call_timeout = options.message_timeout_sec;
	if ((strcmp(options.cib_action, PCMK__CIB_REQUEST_REPLACE) == 0)
	&& pcmk__str_eq(crm_element_name(input), XML_TAG_CIB, pcmk__str_casei)) {
	xmlNode *status = pcmk_find_cib_element(input, XML_CIB_TAG_STATUS);

	if (status == NULL) {
	create_xml_node(input, XML_CIB_TAG_STATUS);
	}
	}

	crm_trace("Passing \"%s\" to variant_op...", options.cib_action);
	return cib_internal_op(the_cib, options.cib_action, options.dest_node,
	options.cib_section, input, output,
	options.cmd_options, options.cib_user);
	}

	int
	do_init(void)
	{
	int rc = pcmk_ok;

	the_cib = cib_new();
	rc = the_cib->cmds->signon(the_cib, crm_system_name, cib_command);
	if (rc != pcmk_ok) {
	crm_err("Could not connect to the CIB: %s", pcmk_strerror(rc));
	fprintf(stderr, "Could not connect to the CIB: %s\n",
	pcmk_strerror(rc));
	}

	return rc;
	}

	void
	cibadmin_op_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
	{
	rc = pcmk_legacy2rc(rc);
	exit_code = pcmk_rc2exitc(rc);

	if (rc == pcmk_rc_schema_unchanged) {
	report_schema_unchanged();

	} else if (rc != pcmk_rc_ok) {
	crm_warn("Call %s failed: %s " CRM_XS " rc=%d",
	options.cib_action, pcmk_rc_str(rc), rc);
	fprintf(stderr, "Call %s failed: %s\n",
	options.cib_action, pcmk_rc_str(rc));
	print_xml_output(output);

	} else if ((strcmp(options.cib_action, PCMK__CIB_REQUEST_QUERY) == 0)
	&& (output == NULL)) {
	crm_err("Query returned no output");
	crm_log_xml_err(msg, "no output");

	} else if (output == NULL) {
	crm_info("Call passed");

	} else {
	crm_info("Call passed");
	print_xml_output(output);
	}

	if (call_id == request_id) {
	g_main_loop_quit(mainloop);

	} else {
	crm_info("Message was not the response we were looking for (%d vs. %d)",
	call_id, request_id);
	}
	}

File Metadata

Mime Type: text/x-diff
Expires: Sat, Nov 23, 5:06 PM (11 h, 41 m)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 1019048
Default Alt Text: (93 KB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions