diff --git a/daemons/based/based_callbacks.c b/daemons/based/based_callbacks.c index cba7c5277e..7c10a19254 100644 --- a/daemons/based/based_callbacks.c +++ b/daemons/based/based_callbacks.c @@ -1,1739 +1,1739 @@ /* * Copyright 2004-2024 the Pacemaker project contributors * * The version control history for this file may have further details. * * This source code is licensed under the GNU General Public License version 2 * or later (GPLv2+) WITHOUT ANY WARRANTY. */ #include #include #include #include #include #include #include // uint32_t, uint64_t, UINT64_C() #include #include #include // PRIu64 #include #include #include #include #include #include #include #include #define EXIT_ESCALATION_MS 10000 static unsigned long cib_local_bcast_num = 0; typedef struct cib_local_notify_s { xmlNode *notify_src; char *client_id; gboolean from_peer; gboolean sync_reply; } cib_local_notify_t; int next_client_id = 0; gboolean legacy_mode = FALSE; qb_ipcs_service_t *ipcs_ro = NULL; qb_ipcs_service_t *ipcs_rw = NULL; qb_ipcs_service_t *ipcs_shm = NULL; static int cib_process_command(xmlNode *request, const cib__operation_t *operation, cib__op_fn_t op_function, xmlNode **reply, xmlNode **cib_diff, bool privileged); static gboolean cib_common_callback(qb_ipcs_connection_t *c, void *data, size_t size, gboolean privileged); gboolean cib_legacy_mode(void) { return legacy_mode; } static int32_t cib_ipc_accept(qb_ipcs_connection_t * c, uid_t uid, gid_t gid) { if (cib_shutdown_flag) { crm_info("Ignoring new IPC client [%d] during shutdown", pcmk__client_pid(c)); return -ECONNREFUSED; } if (pcmk__new_client(c, uid, gid) == NULL) { return -ENOMEM; } return 0; } static int32_t cib_ipc_dispatch_rw(qb_ipcs_connection_t * c, void *data, size_t size) { pcmk__client_t *client = pcmk__find_client(c); crm_trace("%p message from %s", c, client->id); return cib_common_callback(c, data, size, TRUE); } static int32_t cib_ipc_dispatch_ro(qb_ipcs_connection_t * c, void *data, size_t size) { pcmk__client_t *client = pcmk__find_client(c); crm_trace("%p message from %s", c, client->id); return cib_common_callback(c, data, size, FALSE); } /* Error code means? */ static int32_t cib_ipc_closed(qb_ipcs_connection_t * c) { pcmk__client_t *client = pcmk__find_client(c); if (client == NULL) { return 0; } crm_trace("Connection %p", c); pcmk__free_client(client); return 0; } static void cib_ipc_destroy(qb_ipcs_connection_t * c) { crm_trace("Connection %p", c); cib_ipc_closed(c); if (cib_shutdown_flag) { cib_shutdown(0); } } struct qb_ipcs_service_handlers ipc_ro_callbacks = { .connection_accept = cib_ipc_accept, .connection_created = NULL, .msg_process = cib_ipc_dispatch_ro, .connection_closed = cib_ipc_closed, .connection_destroyed = cib_ipc_destroy }; struct qb_ipcs_service_handlers ipc_rw_callbacks = { .connection_accept = cib_ipc_accept, .connection_created = NULL, .msg_process = cib_ipc_dispatch_rw, .connection_closed = cib_ipc_closed, .connection_destroyed = cib_ipc_destroy }; /*! * \internal * \brief Create reply XML for a CIB request * * \param[in] op CIB operation type * \param[in] call_id CIB call ID * \param[in] client_id CIB client ID * \param[in] call_options Group of enum cib_call_options flags * \param[in] rc Request return code * \param[in] call_data Request output data * * \return Reply XML * * \note The caller is responsible for freeing the return value using * \p free_xml(). */ static xmlNode * create_cib_reply(const char *op, const char *call_id, const char *client_id, int call_options, int rc, xmlNode *call_data) { xmlNode *reply = create_xml_node(NULL, "cib-reply"); CRM_ASSERT(reply != NULL); crm_xml_add(reply, PCMK__XA_T, PCMK__VALUE_CIB); crm_xml_add(reply, PCMK__XA_CIB_OP, op); crm_xml_add(reply, PCMK__XA_CIB_CALLID, call_id); crm_xml_add(reply, PCMK__XA_CIB_CLIENTID, client_id); crm_xml_add_int(reply, PCMK__XA_CIB_CALLOPT, call_options); crm_xml_add_int(reply, PCMK__XA_CIB_RC, rc); if (call_data != NULL) { crm_trace("Attaching reply output"); add_message_xml(reply, PCMK__XA_CIB_CALLDATA, call_data); } crm_log_xml_explicit(reply, "cib:reply"); return reply; } static void do_local_notify(const xmlNode *notify_src, const char *client_id, bool sync_reply, bool from_peer) { int rid = 0; int call_id = 0; pcmk__client_t *client_obj = NULL; CRM_ASSERT(notify_src && client_id); crm_element_value_int(notify_src, PCMK__XA_CIB_CALLID, &call_id); client_obj = pcmk__find_client_by_id(client_id); if (client_obj == NULL) { crm_debug("Could not send response %d: client %s not found", call_id, client_id); return; } if (sync_reply) { if (client_obj->ipcs) { CRM_LOG_ASSERT(client_obj->request_id); rid = client_obj->request_id; client_obj->request_id = 0; crm_trace("Sending response %d to client %s%s", rid, pcmk__client_name(client_obj), (from_peer? " (originator of delegated request)" : "")); } else { crm_trace("Sending response (call %d) to client %s%s", call_id, pcmk__client_name(client_obj), (from_peer? " (originator of delegated request)" : "")); } } else { crm_trace("Sending event %d to client %s%s", call_id, pcmk__client_name(client_obj), (from_peer? " (originator of delegated request)" : "")); } switch (PCMK__CLIENT_TYPE(client_obj)) { case pcmk__client_ipc: { int rc = pcmk__ipc_send_xml(client_obj, rid, notify_src, (sync_reply? crm_ipc_flags_none : crm_ipc_server_event)); if (rc != pcmk_rc_ok) { crm_warn("%s reply to client %s failed: %s " CRM_XS " rc=%d", (sync_reply? "Synchronous" : "Asynchronous"), pcmk__client_name(client_obj), pcmk_rc_str(rc), rc); } } break; #ifdef HAVE_GNUTLS_GNUTLS_H case pcmk__client_tls: #endif case pcmk__client_tcp: pcmk__remote_send_xml(client_obj->remote, notify_src); break; default: crm_err("Unknown transport for client %s " CRM_XS " flags=%#016" PRIx64, pcmk__client_name(client_obj), client_obj->flags); } } void cib_common_callback_worker(uint32_t id, uint32_t flags, xmlNode * op_request, pcmk__client_t *cib_client, gboolean privileged) { const char *op = crm_element_value(op_request, PCMK__XA_CIB_OP); int call_options = cib_none; crm_element_value_int(op_request, PCMK__XA_CIB_CALLOPT, &call_options); /* Requests with cib_transaction set should not be sent to based directly * (outside of a commit-transaction request) */ if (pcmk_is_set(call_options, cib_transaction)) { return; } if (pcmk__str_eq(op, CRM_OP_REGISTER, pcmk__str_none)) { if (flags & crm_ipc_client_response) { xmlNode *ack = create_xml_node(NULL, __func__); crm_xml_add(ack, PCMK__XA_CIB_OP, CRM_OP_REGISTER); crm_xml_add(ack, PCMK__XA_CIB_CLIENTID, cib_client->id); pcmk__ipc_send_xml(cib_client, id, ack, flags); cib_client->request_id = 0; free_xml(ack); } return; - } else if (pcmk__str_eq(op, T_CIB_NOTIFY, pcmk__str_none)) { + } else if (pcmk__str_eq(op, PCMK__VALUE_CIB_NOTIFY, pcmk__str_none)) { /* Update the notify filters for this client */ int on_off = 0; crm_exit_t status = CRM_EX_OK; uint64_t bit = UINT64_C(0); const char *type = crm_element_value(op_request, PCMK__XA_CIB_NOTIFY_TYPE); crm_element_value_int(op_request, PCMK__XA_CIB_NOTIFY_ACTIVATE, &on_off); crm_debug("Setting %s callbacks %s for client %s", type, (on_off? "on" : "off"), pcmk__client_name(cib_client)); if (pcmk__str_eq(type, T_CIB_POST_NOTIFY, pcmk__str_casei)) { bit = cib_notify_post; } else if (pcmk__str_eq(type, T_CIB_PRE_NOTIFY, pcmk__str_casei)) { bit = cib_notify_pre; } else if (pcmk__str_eq(type, T_CIB_UPDATE_CONFIRM, pcmk__str_casei)) { bit = cib_notify_confirm; } else if (pcmk__str_eq(type, T_CIB_DIFF_NOTIFY, pcmk__str_casei)) { bit = cib_notify_diff; } else { status = CRM_EX_INVALID_PARAM; } if (bit != 0) { if (on_off) { pcmk__set_client_flags(cib_client, bit); } else { pcmk__clear_client_flags(cib_client, bit); } } pcmk__ipc_send_ack(cib_client, id, flags, PCMK__XE_ACK, NULL, status); return; } cib_process_request(op_request, privileged, cib_client); } int32_t cib_common_callback(qb_ipcs_connection_t * c, void *data, size_t size, gboolean privileged) { uint32_t id = 0; uint32_t flags = 0; int call_options = 0; pcmk__client_t *cib_client = pcmk__find_client(c); xmlNode *op_request = pcmk__client_data2xml(cib_client, data, &id, &flags); if (op_request) { crm_element_value_int(op_request, PCMK__XA_CIB_CALLOPT, &call_options); } if (op_request == NULL) { crm_trace("Invalid message from %p", c); pcmk__ipc_send_ack(cib_client, id, flags, PCMK__XE_NACK, NULL, CRM_EX_PROTOCOL); return 0; } else if(cib_client == NULL) { crm_trace("Invalid client %p", c); return 0; } if (pcmk_is_set(call_options, cib_sync_call)) { CRM_LOG_ASSERT(flags & crm_ipc_client_response); CRM_LOG_ASSERT(cib_client->request_id == 0); /* This means the client has two synchronous events in-flight */ cib_client->request_id = id; /* Reply only to the last one */ } if (cib_client->name == NULL) { const char *value = crm_element_value(op_request, PCMK__XA_CIB_CLIENTNAME); if (value == NULL) { cib_client->name = pcmk__itoa(cib_client->pid); } else { cib_client->name = strdup(value); if (crm_is_daemon_name(value)) { pcmk__set_client_flags(cib_client, cib_is_daemon); } } } /* Allow cluster daemons more leeway before being evicted */ if (pcmk_is_set(cib_client->flags, cib_is_daemon)) { const char *qmax = cib_config_lookup(PCMK_OPT_CLUSTER_IPC_LIMIT); if (pcmk__set_client_queue_max(cib_client, qmax)) { crm_trace("IPC threshold for client %s[%u] is now %u", pcmk__client_name(cib_client), cib_client->pid, cib_client->queue_max); } } crm_xml_add(op_request, PCMK__XA_CIB_CLIENTID, cib_client->id); crm_xml_add(op_request, PCMK__XA_CIB_CLIENTNAME, cib_client->name); CRM_LOG_ASSERT(cib_client->user != NULL); pcmk__update_acl_user(op_request, PCMK__XA_CIB_USER, cib_client->user); cib_common_callback_worker(id, flags, op_request, cib_client, privileged); free_xml(op_request); return 0; } static uint64_t ping_seq = 0; static char *ping_digest = NULL; static bool ping_modified_since = FALSE; static gboolean cib_digester_cb(gpointer data) { if (based_is_primary) { char buffer[32]; xmlNode *ping = create_xml_node(NULL, PCMK__XE_PING); ping_seq++; free(ping_digest); ping_digest = NULL; ping_modified_since = FALSE; snprintf(buffer, 32, "%" PRIu64, ping_seq); crm_trace("Requesting peer digests (%s)", buffer); crm_xml_add(ping, PCMK__XA_T, PCMK__VALUE_CIB); crm_xml_add(ping, PCMK__XA_CIB_OP, CRM_OP_PING); crm_xml_add(ping, PCMK__XA_CIB_PING_ID, buffer); crm_xml_add(ping, PCMK_XA_CRM_FEATURE_SET, CRM_FEATURE_SET); send_cluster_message(NULL, crm_msg_cib, ping, TRUE); free_xml(ping); } return FALSE; } static void process_ping_reply(xmlNode *reply) { uint64_t seq = 0; const char *host = crm_element_value(reply, PCMK__XA_SRC); xmlNode *pong = get_message_xml(reply, PCMK__XA_CIB_CALLDATA); const char *seq_s = crm_element_value(pong, PCMK__XA_CIB_PING_ID); const char *digest = crm_element_value(pong, PCMK__XA_DIGEST); if (seq_s == NULL) { crm_debug("Ignoring ping reply with no " PCMK__XA_CIB_PING_ID); return; } else { long long seq_ll; if (pcmk__scan_ll(seq_s, &seq_ll, 0LL) != pcmk_rc_ok) { return; } seq = (uint64_t) seq_ll; } if(digest == NULL) { crm_trace("Ignoring ping reply %s from %s with no digest", seq_s, host); } else if(seq != ping_seq) { crm_trace("Ignoring out of sequence ping reply %s from %s", seq_s, host); } else if(ping_modified_since) { crm_trace("Ignoring ping reply %s from %s: cib updated since", seq_s, host); } else { const char *version = crm_element_value(pong, PCMK_XA_CRM_FEATURE_SET); if(ping_digest == NULL) { crm_trace("Calculating new digest"); ping_digest = calculate_xml_versioned_digest(the_cib, FALSE, TRUE, version); } crm_trace("Processing ping reply %s from %s (%s)", seq_s, host, digest); if (!pcmk__str_eq(ping_digest, digest, pcmk__str_casei)) { xmlNode *remote_cib = get_message_xml(pong, PCMK__XA_CIB_CALLDATA); const char *admin_epoch_s = NULL; const char *epoch_s = NULL; const char *num_updates_s = NULL; if (remote_cib != NULL) { admin_epoch_s = crm_element_value(remote_cib, PCMK_XA_ADMIN_EPOCH); epoch_s = crm_element_value(remote_cib, PCMK_XA_EPOCH); num_updates_s = crm_element_value(remote_cib, PCMK_XA_NUM_UPDATES); } crm_notice("Local CIB %s.%s.%s.%s differs from %s: %s.%s.%s.%s %p", crm_element_value(the_cib, PCMK_XA_ADMIN_EPOCH), crm_element_value(the_cib, PCMK_XA_EPOCH), crm_element_value(the_cib, PCMK_XA_NUM_UPDATES), ping_digest, host, pcmk__s(admin_epoch_s, "_"), pcmk__s(epoch_s, "_"), pcmk__s(num_updates_s, "_"), digest, remote_cib); if(remote_cib && remote_cib->children) { // Additional debug xml_calculate_changes(the_cib, remote_cib); pcmk__log_xml_changes(LOG_INFO, remote_cib); crm_trace("End of differences"); } free_xml(remote_cib); sync_our_cib(reply, FALSE); } } } static void local_notify_destroy_callback(gpointer data) { cib_local_notify_t *notify = data; free_xml(notify->notify_src); free(notify->client_id); free(notify); } static void check_local_notify(int bcast_id) { const cib_local_notify_t *notify = NULL; if (!local_notify_queue) { return; } notify = pcmk__intkey_table_lookup(local_notify_queue, bcast_id); if (notify) { do_local_notify(notify->notify_src, notify->client_id, notify->sync_reply, notify->from_peer); pcmk__intkey_table_remove(local_notify_queue, bcast_id); } } static void queue_local_notify(xmlNode * notify_src, const char *client_id, gboolean sync_reply, gboolean from_peer) { cib_local_notify_t *notify = calloc(1, sizeof(cib_local_notify_t)); notify->notify_src = notify_src; notify->client_id = strdup(client_id); notify->sync_reply = sync_reply; notify->from_peer = from_peer; if (!local_notify_queue) { local_notify_queue = pcmk__intkey_table(local_notify_destroy_callback); } pcmk__intkey_table_insert(local_notify_queue, cib_local_bcast_num, notify); // cppcheck doesn't know notify will get freed when hash table is destroyed // cppcheck-suppress memleak } static void parse_local_options_v1(const pcmk__client_t *cib_client, const cib__operation_t *operation, int call_options, const char *host, const char *op, gboolean *local_notify, gboolean *needs_reply, gboolean *process, gboolean *needs_forward) { if (pcmk_is_set(operation->flags, cib__op_attr_modifies) && !pcmk_is_set(call_options, cib_inhibit_bcast)) { /* we need to send an update anyway */ *needs_reply = TRUE; } else { *needs_reply = FALSE; } if (host == NULL && (call_options & cib_scope_local)) { crm_trace("Processing locally scoped %s op from client %s", op, pcmk__client_name(cib_client)); *local_notify = TRUE; } else if ((host == NULL) && based_is_primary) { crm_trace("Processing %s op locally from client %s as primary", op, pcmk__client_name(cib_client)); *local_notify = TRUE; } else if (pcmk__str_eq(host, OUR_NODENAME, pcmk__str_casei)) { crm_trace("Processing locally addressed %s op from client %s", op, pcmk__client_name(cib_client)); *local_notify = TRUE; } else if (stand_alone) { *needs_forward = FALSE; *local_notify = TRUE; *process = TRUE; } else { crm_trace("%s op from %s needs to be forwarded to client %s", op, pcmk__client_name(cib_client), pcmk__s(host, "the primary instance")); *needs_forward = TRUE; *process = FALSE; } } static void parse_local_options_v2(const pcmk__client_t *cib_client, const cib__operation_t *operation, int call_options, const char *host, const char *op, gboolean *local_notify, gboolean *needs_reply, gboolean *process, gboolean *needs_forward) { // Process locally and notify local client *process = TRUE; *needs_reply = FALSE; *local_notify = TRUE; *needs_forward = FALSE; if (pcmk_is_set(operation->flags, cib__op_attr_local)) { /* Always process locally if cib__op_attr_local is set. * * @COMPAT: Currently host is ignored. At a compatibility break, throw * an error (from cib_process_request() or earlier) if host is not NULL or * OUR_NODENAME. */ crm_trace("Processing always-local %s op from client %s", op, pcmk__client_name(cib_client)); if (!pcmk__str_eq(host, OUR_NODENAME, pcmk__str_casei|pcmk__str_null_matches)) { crm_warn("Operation '%s' is always local but its target host is " "set to '%s'", op, host); } return; } if (pcmk_is_set(operation->flags, cib__op_attr_modifies) || !pcmk__str_eq(host, OUR_NODENAME, pcmk__str_casei|pcmk__str_null_matches)) { // Forward modifying and non-local requests via cluster *process = FALSE; *needs_reply = FALSE; *local_notify = FALSE; *needs_forward = TRUE; crm_trace("%s op from %s needs to be forwarded to %s", op, pcmk__client_name(cib_client), pcmk__s(host, "all nodes")); return; } if (stand_alone) { crm_trace("Processing %s op from client %s (stand-alone)", op, pcmk__client_name(cib_client)); } else { crm_trace("Processing %saddressed %s op from client %s", ((host != NULL)? "locally " : "un"), op, pcmk__client_name(cib_client)); } } static void parse_local_options(const pcmk__client_t *cib_client, const cib__operation_t *operation, int call_options, const char *host, const char *op, gboolean *local_notify, gboolean *needs_reply, gboolean *process, gboolean *needs_forward) { if(cib_legacy_mode()) { parse_local_options_v1(cib_client, operation, call_options, host, op, local_notify, needs_reply, process, needs_forward); } else { parse_local_options_v2(cib_client, operation, call_options, host, op, local_notify, needs_reply, process, needs_forward); } } static gboolean parse_peer_options_v1(const cib__operation_t *operation, xmlNode *request, gboolean *local_notify, gboolean *needs_reply, gboolean *process) { const char *op = NULL; const char *host = NULL; const char *delegated = NULL; const char *originator = crm_element_value(request, PCMK__XA_SRC); const char *reply_to = crm_element_value(request, PCMK__XA_CIB_ISREPLYTO); gboolean is_reply = pcmk__str_eq(reply_to, OUR_NODENAME, pcmk__str_casei); if (pcmk__xe_attr_is_true(request, PCMK__XA_CIB_UPDATE)) { *needs_reply = FALSE; if (is_reply) { *local_notify = TRUE; crm_trace("Processing global/peer update from %s" " that originated from us", originator); } else { crm_trace("Processing global/peer update from %s", originator); } return TRUE; } op = crm_element_value(request, PCMK__XA_CIB_OP); crm_trace("Processing legacy %s request sent by %s", op, originator); if (pcmk__str_eq(op, PCMK__CIB_REQUEST_SHUTDOWN, pcmk__str_none)) { /* Always process these */ *local_notify = FALSE; if (reply_to == NULL || is_reply) { *process = TRUE; } if (is_reply) { *needs_reply = FALSE; } return *process; } if (is_reply && pcmk__str_eq(op, CRM_OP_PING, pcmk__str_casei)) { process_ping_reply(request); return FALSE; } if (is_reply) { crm_trace("Forward reply sent from %s to local clients", originator); *process = FALSE; *needs_reply = FALSE; *local_notify = TRUE; return TRUE; } host = crm_element_value(request, PCMK__XA_CIB_HOST); if (pcmk__str_eq(host, OUR_NODENAME, pcmk__str_casei)) { crm_trace("Processing %s request sent to us from %s", op, originator); return TRUE; } else if(is_reply == FALSE && pcmk__str_eq(op, CRM_OP_PING, pcmk__str_casei)) { crm_trace("Processing %s request sent to %s by %s", op, host?host:"everyone", originator); *needs_reply = TRUE; return TRUE; } else if ((host == NULL) && based_is_primary) { crm_trace("Processing %s request sent to primary instance from %s", op, originator); return TRUE; } delegated = crm_element_value(request, PCMK__XA_CIB_DELEGATED_FROM); if (delegated != NULL) { crm_trace("Ignoring message for primary instance"); } else if (host != NULL) { /* this is for a specific instance and we're not it */ crm_trace("Ignoring msg for instance on %s", host); } else if ((reply_to == NULL) && !based_is_primary) { // This is for the primary instance, and we're not it crm_trace("Ignoring reply for primary instance"); } else if (pcmk__str_eq(op, PCMK__CIB_REQUEST_SHUTDOWN, pcmk__str_none)) { if (reply_to != NULL) { crm_debug("Processing %s from %s", op, originator); *needs_reply = FALSE; } else { crm_debug("Processing %s reply from %s", op, originator); } return TRUE; } else { crm_err("Nothing for us to do?"); crm_log_xml_err(request, "Peer[inbound]"); } return FALSE; } static gboolean parse_peer_options_v2(const cib__operation_t *operation, xmlNode *request, gboolean *local_notify, gboolean *needs_reply, gboolean *process) { const char *host = NULL; const char *delegated = crm_element_value(request, PCMK__XA_CIB_DELEGATED_FROM); const char *op = crm_element_value(request, PCMK__XA_CIB_OP); const char *originator = crm_element_value(request, PCMK__XA_SRC); const char *reply_to = crm_element_value(request, PCMK__XA_CIB_ISREPLYTO); gboolean is_reply = pcmk__str_eq(reply_to, OUR_NODENAME, pcmk__str_casei); if (originator == NULL) { // Shouldn't be possible originator = "peer"; } if (pcmk__str_eq(op, PCMK__CIB_REQUEST_REPLACE, pcmk__str_none)) { // sync_our_cib() sets PCMK__XA_CIB_ISREPLYTO if (reply_to) { delegated = reply_to; } goto skip_is_reply; } else if (pcmk__str_eq(op, PCMK__CIB_REQUEST_SYNC_TO_ALL, pcmk__str_none)) { // Nothing to do } else if (is_reply && pcmk__str_eq(op, CRM_OP_PING, pcmk__str_casei)) { process_ping_reply(request); return FALSE; } else if (pcmk__str_eq(op, PCMK__CIB_REQUEST_UPGRADE, pcmk__str_none)) { /* Only the DC (node with the oldest software) should process * this operation if PCMK__XA_CIB_SCHEMA_MAX is unset. * * If the DC is happy it will then send out another * PCMK__CIB_REQUEST_UPGRADE which will tell all nodes to do the actual * upgrade. * * Except this time PCMK__XA_CIB_SCHEMA_MAX will be set which puts a * limit on how far newer nodes will go */ const char *max = crm_element_value(request, PCMK__XA_CIB_SCHEMA_MAX); const char *upgrade_rc = crm_element_value(request, PCMK__XA_CIB_UPGRADE_RC); crm_trace("Parsing upgrade %s for %s with max=%s and upgrade_rc=%s", (is_reply? "reply" : "request"), (based_is_primary? "primary" : "secondary"), pcmk__s(max, "none"), pcmk__s(upgrade_rc, "none")); if (upgrade_rc != NULL) { // Our upgrade request was rejected by DC, notify clients of result crm_xml_add(request, PCMK__XA_CIB_RC, upgrade_rc); } else if ((max == NULL) && based_is_primary) { /* We are the DC, check if this upgrade is allowed */ goto skip_is_reply; } else if(max) { /* Ok, go ahead and upgrade to 'max' */ goto skip_is_reply; } else { // Ignore broadcast client requests when we're not primary return FALSE; } } else if (pcmk__xe_attr_is_true(request, PCMK__XA_CIB_UPDATE)) { crm_info("Detected legacy %s global update from %s", op, originator); send_sync_request(NULL); legacy_mode = TRUE; return FALSE; } else if (is_reply && pcmk_is_set(operation->flags, cib__op_attr_modifies)) { crm_trace("Ignoring legacy %s reply sent from %s to local clients", op, originator); return FALSE; } else if (pcmk__str_eq(op, PCMK__CIB_REQUEST_SHUTDOWN, pcmk__str_none)) { *local_notify = FALSE; if (reply_to == NULL) { *process = TRUE; } else { // Not possible? crm_debug("Ignoring shutdown request from %s because reply_to=%s", originator, reply_to); } return *process; } if (is_reply) { crm_trace("Will notify local clients for %s reply from %s", op, originator); *process = FALSE; *needs_reply = FALSE; *local_notify = TRUE; return TRUE; } skip_is_reply: *process = TRUE; *needs_reply = FALSE; *local_notify = pcmk__str_eq(delegated, OUR_NODENAME, pcmk__str_casei); host = crm_element_value(request, PCMK__XA_CIB_HOST); if (pcmk__str_eq(host, OUR_NODENAME, pcmk__str_casei)) { crm_trace("Processing %s request sent to us from %s", op, originator); *needs_reply = TRUE; return TRUE; } else if (host != NULL) { crm_trace("Ignoring %s request intended for CIB manager on %s", op, host); return FALSE; } else if(is_reply == FALSE && pcmk__str_eq(op, CRM_OP_PING, pcmk__str_casei)) { *needs_reply = TRUE; } crm_trace("Processing %s request broadcast by %s call %s on %s " "(local clients will%s be notified)", op, pcmk__s(crm_element_value(request, PCMK__XA_CIB_CLIENTNAME), "client"), pcmk__s(crm_element_value(request, PCMK__XA_CIB_CALLID), "without ID"), originator, (*local_notify? "" : "not")); return TRUE; } static gboolean parse_peer_options(const cib__operation_t *operation, xmlNode *request, gboolean *local_notify, gboolean *needs_reply, gboolean *process) { /* TODO: What happens when an update comes in after node A * requests the CIB from node B, but before it gets the reply (and * sends out the replace operation) */ if(cib_legacy_mode()) { return parse_peer_options_v1(operation, request, local_notify, needs_reply, process); } else { return parse_peer_options_v2(operation, request, local_notify, needs_reply, process); } } /*! * \internal * \brief Forward a CIB request to the appropriate target host(s) * * \param[in] request CIB request to forward */ static void forward_request(xmlNode *request) { const char *op = crm_element_value(request, PCMK__XA_CIB_OP); const char *section = crm_element_value(request, PCMK__XA_CIB_SECTION); const char *host = crm_element_value(request, PCMK__XA_CIB_HOST); const char *originator = crm_element_value(request, PCMK__XA_SRC); const char *client_name = crm_element_value(request, PCMK__XA_CIB_CLIENTNAME); const char *call_id = crm_element_value(request, PCMK__XA_CIB_CALLID); crm_node_t *peer = NULL; int log_level = LOG_INFO; if (pcmk__str_eq(op, PCMK__CIB_REQUEST_NOOP, pcmk__str_none)) { log_level = LOG_DEBUG; } do_crm_log(log_level, "Forwarding %s operation for section %s to %s (origin=%s/%s/%s)", pcmk__s(op, "invalid"), pcmk__s(section, "all"), pcmk__s(host, (cib_legacy_mode()? "primary" : "all")), pcmk__s(originator, "local"), pcmk__s(client_name, "unspecified"), pcmk__s(call_id, "unspecified")); crm_xml_add(request, PCMK__XA_CIB_DELEGATED_FROM, OUR_NODENAME); if (host != NULL) { peer = pcmk__get_node(0, host, NULL, pcmk__node_search_cluster); } send_cluster_message(peer, crm_msg_cib, request, FALSE); // Return the request to its original state xml_remove_prop(request, PCMK__XA_CIB_DELEGATED_FROM); } static gboolean send_peer_reply(xmlNode * msg, xmlNode * result_diff, const char *originator, gboolean broadcast) { CRM_ASSERT(msg != NULL); if (broadcast) { /* @COMPAT: Legacy code * * This successful call modified the CIB, and the change needs to be * broadcast (sent via cluster to all nodes). */ int diff_add_updates = 0; int diff_add_epoch = 0; int diff_add_admin_epoch = 0; int diff_del_updates = 0; int diff_del_epoch = 0; int diff_del_admin_epoch = 0; const char *digest = NULL; int format = 1; CRM_LOG_ASSERT(result_diff != NULL); digest = crm_element_value(result_diff, PCMK__XA_DIGEST); crm_element_value_int(result_diff, PCMK_XA_FORMAT, &format); cib_diff_version_details(result_diff, &diff_add_admin_epoch, &diff_add_epoch, &diff_add_updates, &diff_del_admin_epoch, &diff_del_epoch, &diff_del_updates); crm_trace("Sending update diff %d.%d.%d -> %d.%d.%d %s", diff_del_admin_epoch, diff_del_epoch, diff_del_updates, diff_add_admin_epoch, diff_add_epoch, diff_add_updates, digest); crm_xml_add(msg, PCMK__XA_CIB_ISREPLYTO, originator); pcmk__xe_set_bool_attr(msg, PCMK__XA_CIB_UPDATE, true); crm_xml_add(msg, PCMK__XA_CIB_OP, PCMK__CIB_REQUEST_APPLY_PATCH); crm_xml_add(msg, PCMK__XA_CIB_USER, CRM_DAEMON_USER); if (format == 1) { CRM_ASSERT(digest != NULL); } add_message_xml(msg, PCMK__XA_CIB_UPDATE_DIFF, result_diff); crm_log_xml_explicit(msg, "copy"); return send_cluster_message(NULL, crm_msg_cib, msg, TRUE); } else if (originator != NULL) { /* send reply via HA to originating node */ crm_trace("Sending request result to %s only", originator); crm_xml_add(msg, PCMK__XA_CIB_ISREPLYTO, originator); return send_cluster_message(pcmk__get_node(0, originator, NULL, pcmk__node_search_cluster), crm_msg_cib, msg, FALSE); } return FALSE; } /*! * \internal * \brief Handle an IPC or CPG message containing a request * * \param[in,out] request Request XML * \param[in] privileged Whether privileged commands may be run * (see cib_server_ops[] definition) * \param[in] cib_client IPC client that sent request (or NULL if CPG) * * \return Legacy Pacemaker return code */ int cib_process_request(xmlNode *request, gboolean privileged, const pcmk__client_t *cib_client) { // @TODO: Break into multiple smaller functions int call_options = 0; gboolean process = TRUE; // Whether to process request locally now gboolean is_update = TRUE; // Whether request would modify CIB gboolean needs_reply = TRUE; // Whether to build a reply gboolean local_notify = FALSE; // Whether to notify (local) requester gboolean needs_forward = FALSE; // Whether to forward request somewhere else xmlNode *op_reply = NULL; xmlNode *result_diff = NULL; int rc = pcmk_ok; const char *op = crm_element_value(request, PCMK__XA_CIB_OP); const char *originator = crm_element_value(request, PCMK__XA_SRC); const char *host = crm_element_value(request, PCMK__XA_CIB_HOST); const char *target = NULL; const char *call_id = crm_element_value(request, PCMK__XA_CIB_CALLID); const char *client_id = crm_element_value(request, PCMK__XA_CIB_CLIENTID); const char *client_name = crm_element_value(request, PCMK__XA_CIB_CLIENTNAME); const char *reply_to = crm_element_value(request, PCMK__XA_CIB_ISREPLYTO); const cib__operation_t *operation = NULL; cib__op_fn_t op_function = NULL; crm_element_value_int(request, PCMK__XA_CIB_CALLOPT, &call_options); if ((host != NULL) && (*host == '\0')) { host = NULL; } // @TODO: Improve trace messages. Target is accurate only for legacy mode. if (host) { target = host; } else if (call_options & cib_scope_local) { target = "local host"; } else { target = "primary"; } if (cib_client == NULL) { crm_trace("Processing peer %s operation from %s/%s on %s intended for %s (reply=%s)", op, client_name, call_id, originator, target, reply_to); } else { crm_xml_add(request, PCMK__XA_SRC, OUR_NODENAME); crm_trace("Processing local %s operation from %s/%s intended for %s", op, client_name, call_id, target); } rc = cib__get_operation(op, &operation); rc = pcmk_rc2legacy(rc); if (rc != pcmk_ok) { /* TODO: construct error reply? */ crm_err("Pre-processing of command failed: %s", pcmk_strerror(rc)); return rc; } op_function = based_get_op_function(operation); if (op_function == NULL) { crm_err("Operation %s not supported by CIB manager", op); return -EOPNOTSUPP; } if (cib_client != NULL) { parse_local_options(cib_client, operation, call_options, host, op, &local_notify, &needs_reply, &process, &needs_forward); } else if (!parse_peer_options(operation, request, &local_notify, &needs_reply, &process)) { return rc; } if (pcmk_is_set(call_options, cib_transaction)) { /* All requests in a transaction are processed locally against a working * CIB copy, and we don't notify for individual requests because the * entire transaction is atomic. * * We still call the option parser functions above, for the sake of log * messages and checking whether we're the target for peer requests. */ process = TRUE; needs_reply = FALSE; local_notify = FALSE; needs_forward = FALSE; } is_update = pcmk_is_set(operation->flags, cib__op_attr_modifies); if (pcmk_is_set(call_options, cib_discard_reply)) { /* If the request will modify the CIB, and we are in legacy mode, we * need to build a reply so we can broadcast a diff, even if the * requester doesn't want one. */ needs_reply = is_update && cib_legacy_mode(); local_notify = FALSE; crm_trace("Client is not interested in the reply"); } if (needs_forward) { forward_request(request); return rc; } if (cib_status != pcmk_ok) { rc = cib_status; crm_err("Operation ignored, cluster configuration is invalid." " Please repair and restart: %s", pcmk_strerror(cib_status)); op_reply = create_cib_reply(op, call_id, client_id, call_options, rc, the_cib); } else if (process) { time_t finished = 0; time_t now = time(NULL); int level = LOG_INFO; const char *section = crm_element_value(request, PCMK__XA_CIB_SECTION); const char *admin_epoch_s = NULL; const char *epoch_s = NULL; const char *num_updates_s = NULL; rc = cib_process_command(request, operation, op_function, &op_reply, &result_diff, privileged); if (!is_update) { level = LOG_TRACE; } else if (pcmk__xe_attr_is_true(request, PCMK__XA_CIB_UPDATE)) { switch (rc) { case pcmk_ok: level = LOG_INFO; break; case -pcmk_err_old_data: case -pcmk_err_diff_resync: case -pcmk_err_diff_failed: level = LOG_TRACE; break; default: level = LOG_ERR; } } else if (rc != pcmk_ok) { level = LOG_WARNING; } if (the_cib != NULL) { admin_epoch_s = crm_element_value(the_cib, PCMK_XA_ADMIN_EPOCH); epoch_s = crm_element_value(the_cib, PCMK_XA_EPOCH); num_updates_s = crm_element_value(the_cib, PCMK_XA_NUM_UPDATES); } do_crm_log(level, "Completed %s operation for section %s: %s (rc=%d, origin=%s/%s/%s, version=%s.%s.%s)", op, section ? section : "'all'", pcmk_strerror(rc), rc, originator ? originator : "local", client_name, call_id, pcmk__s(admin_epoch_s, "0"), pcmk__s(epoch_s, "0"), pcmk__s(num_updates_s, "0")); finished = time(NULL); if ((finished - now) > 3) { crm_trace("%s operation took %lds to complete", op, (long)(finished - now)); crm_write_blackbox(0, NULL); } if (op_reply == NULL && (needs_reply || local_notify)) { crm_err("Unexpected NULL reply to message"); crm_log_xml_err(request, "null reply"); needs_reply = FALSE; local_notify = FALSE; } } if (is_update && !cib_legacy_mode()) { crm_trace("Completed pre-sync update from %s/%s/%s%s", originator ? originator : "local", client_name, call_id, local_notify?" with local notification":""); } else if (!needs_reply || stand_alone) { // This was a non-originating secondary update crm_trace("Completed update as secondary"); } else if (cib_legacy_mode() && rc == pcmk_ok && result_diff != NULL && !(call_options & cib_inhibit_bcast)) { gboolean broadcast = FALSE; cib_local_bcast_num++; crm_xml_add_int(request, PCMK__XA_CIB_LOCAL_NOTIFY_ID, cib_local_bcast_num); broadcast = send_peer_reply(request, result_diff, originator, TRUE); if (broadcast && client_id && local_notify && op_reply) { /* If we have been asked to sync the reply, * and a bcast msg has gone out, we queue the local notify * until we know the bcast message has been received */ local_notify = FALSE; crm_trace("Queuing local %ssync notification for %s", (call_options & cib_sync_call) ? "" : "a-", client_id); queue_local_notify(op_reply, client_id, pcmk_is_set(call_options, cib_sync_call), (cib_client == NULL)); op_reply = NULL; /* the reply is queued, so don't free here */ } } else if ((cib_client == NULL) && !pcmk_is_set(call_options, cib_discard_reply)) { if (is_update == FALSE || result_diff == NULL) { crm_trace("Request not broadcast: R/O call"); } else if (call_options & cib_inhibit_bcast) { crm_trace("Request not broadcast: inhibited"); } else if (rc != pcmk_ok) { crm_trace("Request not broadcast: call failed: %s", pcmk_strerror(rc)); } else { crm_trace("Directing reply to %s", originator); } send_peer_reply(op_reply, result_diff, originator, FALSE); } if (local_notify && client_id) { crm_trace("Performing local %ssync notification for %s", (pcmk_is_set(call_options, cib_sync_call)? "" : "a"), client_id); if (process == FALSE) { do_local_notify(request, client_id, pcmk_is_set(call_options, cib_sync_call), (cib_client == NULL)); } else { do_local_notify(op_reply, client_id, pcmk_is_set(call_options, cib_sync_call), (cib_client == NULL)); } } free_xml(op_reply); free_xml(result_diff); return rc; } /*! * \internal * \brief Get a CIB operation's input from the request XML * * \param[in] request CIB request XML * \param[in] type CIB operation type * \param[out] section Where to store CIB section name * * \return Input XML for CIB operation * * \note If not \c NULL, the return value is a non-const pointer to part of * \p request. The caller should not free it directly. */ static xmlNode * prepare_input(const xmlNode *request, enum cib__op_type type, const char **section) { xmlNode *input = NULL; *section = NULL; switch (type) { case cib__op_apply_patch: if (pcmk__xe_attr_is_true(request, PCMK__XA_CIB_UPDATE)) { input = get_message_xml(request, PCMK__XA_CIB_UPDATE_DIFF); } else { input = get_message_xml(request, PCMK__XA_CIB_CALLDATA); } break; default: input = get_message_xml(request, PCMK__XA_CIB_CALLDATA); *section = crm_element_value(request, PCMK__XA_CIB_SECTION); break; } // Grab the specified section if ((*section != NULL) && pcmk__xe_is(input, PCMK_XE_CIB)) { input = pcmk_find_cib_element(input, *section); } return input; } // v1 and v2 patch formats #define XPATH_CONFIG_CHANGE \ "//" PCMK_XE_CRM_CONFIG " | " \ "//" PCMK_XE_CHANGE \ "[contains(@" PCMK_XA_PATH ",'/" PCMK_XE_CRM_CONFIG "/')]" static bool contains_config_change(xmlNode *diff) { bool changed = false; if (diff) { xmlXPathObject *xpathObj = xpath_search(diff, XPATH_CONFIG_CHANGE); if (numXpathResults(xpathObj) > 0) { changed = true; } freeXpathObject(xpathObj); } return changed; } static int cib_process_command(xmlNode *request, const cib__operation_t *operation, cib__op_fn_t op_function, xmlNode **reply, xmlNode **cib_diff, bool privileged) { xmlNode *input = NULL; xmlNode *output = NULL; xmlNode *result_cib = NULL; int call_options = 0; const char *op = NULL; const char *section = NULL; const char *call_id = crm_element_value(request, PCMK__XA_CIB_CALLID); const char *client_id = crm_element_value(request, PCMK__XA_CIB_CLIENTID); const char *client_name = crm_element_value(request, PCMK__XA_CIB_CLIENTNAME); const char *originator = crm_element_value(request, PCMK__XA_SRC); int rc = pcmk_ok; bool config_changed = false; bool manage_counters = true; static mainloop_timer_t *digest_timer = NULL; CRM_ASSERT(cib_status == pcmk_ok); if(digest_timer == NULL) { digest_timer = mainloop_timer_add("digester", 5000, FALSE, cib_digester_cb, NULL); } *reply = NULL; *cib_diff = NULL; /* Start processing the request... */ op = crm_element_value(request, PCMK__XA_CIB_OP); crm_element_value_int(request, PCMK__XA_CIB_CALLOPT, &call_options); if (!privileged && pcmk_is_set(operation->flags, cib__op_attr_privileged)) { rc = -EACCES; crm_trace("Failed due to lack of privileges: %s", pcmk_strerror(rc)); goto done; } input = prepare_input(request, operation->type, §ion); if (!pcmk_is_set(operation->flags, cib__op_attr_modifies)) { rc = cib_perform_op(NULL, op, call_options, op_function, true, section, request, input, false, &config_changed, &the_cib, &result_cib, NULL, &output); CRM_CHECK(result_cib == NULL, free_xml(result_cib)); goto done; } /* @COMPAT: Handle a valid write action (legacy) * * @TODO: Re-evaluate whether this is all truly legacy. The cib_force_diff * portion is. However, PCMK__XA_CIB_UPDATE may be set by a sync operation * even in non-legacy mode, and manage_counters tells xml_create_patchset() * whether to update version/epoch info. */ if (pcmk__xe_attr_is_true(request, PCMK__XA_CIB_UPDATE)) { manage_counters = false; cib__set_call_options(call_options, "call", cib_force_diff); crm_trace("Global update detected"); CRM_LOG_ASSERT(pcmk__str_any_of(op, PCMK__CIB_REQUEST_APPLY_PATCH, PCMK__CIB_REQUEST_REPLACE, NULL)); } ping_modified_since = TRUE; if (pcmk_is_set(call_options, cib_inhibit_bcast)) { crm_trace("Skipping update: inhibit broadcast"); manage_counters = false; } // result_cib must not be modified after cib_perform_op() returns rc = cib_perform_op(NULL, op, call_options, op_function, false, section, request, input, manage_counters, &config_changed, &the_cib, &result_cib, cib_diff, &output); // @COMPAT: Legacy code if (!manage_counters) { int format = 1; // If the diff is NULL at this point, it's because nothing changed if (*cib_diff != NULL) { crm_element_value_int(*cib_diff, PCMK_XA_FORMAT, &format); } if (format == 1) { config_changed = cib__config_changed_v1(NULL, NULL, cib_diff); } } /* Always write to disk for successful ops with the flag set. This also * negates the need to detect ordering changes. */ if ((rc == pcmk_ok) && pcmk_is_set(operation->flags, cib__op_attr_writes_through)) { config_changed = true; } if ((rc == pcmk_ok) && !pcmk_any_flags_set(call_options, cib_dryrun|cib_transaction)) { if (result_cib != the_cib) { if (pcmk_is_set(operation->flags, cib__op_attr_writes_through)) { config_changed = true; } crm_trace("Activating %s->%s%s", crm_element_value(the_cib, PCMK_XA_NUM_UPDATES), crm_element_value(result_cib, PCMK_XA_NUM_UPDATES), (config_changed? " changed" : "")); rc = activateCibXml(result_cib, config_changed, op); if (rc != pcmk_ok) { crm_err("Failed to activate new CIB: %s", pcmk_strerror(rc)); } } if ((rc == pcmk_ok) && contains_config_change(*cib_diff)) { cib_read_config(config_hash, result_cib); } /* @COMPAT Nodes older than feature set 3.19.0 don't support * transactions. In a mixed-version cluster with nodes <3.19.0, we must * sync the updated CIB, so that the older nodes receive the changes. * Any node that has already applied the transaction will ignore the * synced CIB. * * To ensure the updated CIB is synced from only one node, we sync it * from the originator. */ if ((operation->type == cib__op_commit_transact) && pcmk__str_eq(originator, OUR_NODENAME, pcmk__str_casei) && compare_version(crm_element_value(the_cib, PCMK_XA_CRM_FEATURE_SET), "3.19.0") < 0) { sync_our_cib(request, TRUE); } mainloop_timer_stop(digest_timer); mainloop_timer_start(digest_timer); } else if (rc == -pcmk_err_schema_validation) { CRM_ASSERT(result_cib != the_cib); if (output != NULL) { crm_log_xml_info(output, "cib:output"); free_xml(output); } output = result_cib; } else { crm_trace("Not activating %d %d %s", rc, pcmk_is_set(call_options, cib_dryrun), crm_element_value(result_cib, PCMK_XA_NUM_UPDATES)); if (result_cib != the_cib) { free_xml(result_cib); } } if (!pcmk_any_flags_set(call_options, cib_dryrun|cib_inhibit_notify|cib_transaction)) { crm_trace("Sending notifications %d", pcmk_is_set(call_options, cib_dryrun)); cib_diff_notify(op, rc, call_id, client_id, client_name, originator, input, *cib_diff); } pcmk__log_xml_patchset(LOG_TRACE, *cib_diff); done: if (!pcmk_is_set(call_options, cib_discard_reply) || cib_legacy_mode()) { *reply = create_cib_reply(op, call_id, client_id, call_options, rc, output); } if (output != the_cib) { free_xml(output); } crm_trace("done"); return rc; } void cib_peer_callback(xmlNode * msg, void *private_data) { const char *reason = NULL; const char *originator = crm_element_value(msg, PCMK__XA_SRC); if (cib_legacy_mode() && pcmk__str_eq(originator, OUR_NODENAME, pcmk__str_casei|pcmk__str_null_matches)) { /* message is from ourselves */ int bcast_id = 0; if (crm_element_value_int(msg, PCMK__XA_CIB_LOCAL_NOTIFY_ID, &bcast_id) == 0) { check_local_notify(bcast_id); } return; } else if (crm_peer_cache == NULL) { reason = "membership not established"; goto bail; } if (crm_element_value(msg, PCMK__XA_CIB_CLIENTNAME) == NULL) { crm_xml_add(msg, PCMK__XA_CIB_CLIENTNAME, originator); } /* crm_log_xml_trace(msg, "Peer[inbound]"); */ cib_process_request(msg, TRUE, NULL); return; bail: if (reason) { const char *op = crm_element_value(msg, PCMK__XA_CIB_OP); crm_warn("Discarding %s message from %s: %s", op, originator, reason); } } static gboolean cib_force_exit(gpointer data) { crm_notice("Forcing exit!"); terminate_cib(__func__, CRM_EX_ERROR); return FALSE; } static void disconnect_remote_client(gpointer key, gpointer value, gpointer user_data) { pcmk__client_t *a_client = value; crm_err("Can't disconnect client %s: Not implemented", pcmk__client_name(a_client)); } static void initiate_exit(void) { int active = 0; xmlNode *leaving = NULL; active = crm_active_peers(); if (active < 2) { // This is the last active node terminate_cib(__func__, 0); return; } crm_info("Sending shutdown request to %d peers", active); leaving = create_xml_node(NULL, "exit-notification"); crm_xml_add(leaving, PCMK__XA_T, PCMK__VALUE_CIB); crm_xml_add(leaving, PCMK__XA_CIB_OP, PCMK__CIB_REQUEST_SHUTDOWN); send_cluster_message(NULL, crm_msg_cib, leaving, TRUE); free_xml(leaving); g_timeout_add(EXIT_ESCALATION_MS, cib_force_exit, NULL); } void cib_shutdown(int nsig) { struct qb_ipcs_stats srv_stats; if (cib_shutdown_flag == FALSE) { int disconnects = 0; qb_ipcs_connection_t *c = NULL; cib_shutdown_flag = TRUE; c = qb_ipcs_connection_first_get(ipcs_rw); while (c != NULL) { qb_ipcs_connection_t *last = c; c = qb_ipcs_connection_next_get(ipcs_rw, last); crm_debug("Disconnecting r/w client %p...", last); qb_ipcs_disconnect(last); qb_ipcs_connection_unref(last); disconnects++; } c = qb_ipcs_connection_first_get(ipcs_ro); while (c != NULL) { qb_ipcs_connection_t *last = c; c = qb_ipcs_connection_next_get(ipcs_ro, last); crm_debug("Disconnecting r/o client %p...", last); qb_ipcs_disconnect(last); qb_ipcs_connection_unref(last); disconnects++; } c = qb_ipcs_connection_first_get(ipcs_shm); while (c != NULL) { qb_ipcs_connection_t *last = c; c = qb_ipcs_connection_next_get(ipcs_shm, last); crm_debug("Disconnecting non-blocking r/w client %p...", last); qb_ipcs_disconnect(last); qb_ipcs_connection_unref(last); disconnects++; } disconnects += pcmk__ipc_client_count(); crm_debug("Disconnecting %d remote clients", pcmk__ipc_client_count()); pcmk__foreach_ipc_client(disconnect_remote_client, NULL); crm_info("Disconnected %d clients", disconnects); } qb_ipcs_stats_get(ipcs_rw, &srv_stats, QB_FALSE); if (pcmk__ipc_client_count() == 0) { crm_info("All clients disconnected (%d)", srv_stats.active_connections); initiate_exit(); } else { crm_info("Waiting on %d clients to disconnect (%d)", pcmk__ipc_client_count(), srv_stats.active_connections); } } extern int remote_fd; extern int remote_tls_fd; /*! * \internal * \brief Close remote sockets, free the global CIB and quit * * \param[in] caller Name of calling function (for log message) * \param[in] fast If -1, skip disconnect; if positive, exit that */ void terminate_cib(const char *caller, int fast) { crm_info("%s: Exiting%s...", caller, (fast > 0)? " fast" : mainloop ? " from mainloop" : ""); if (remote_fd > 0) { close(remote_fd); remote_fd = 0; } if (remote_tls_fd > 0) { close(remote_tls_fd); remote_tls_fd = 0; } uninitializeCib(); if (fast > 0) { /* Quit fast on error */ pcmk__stop_based_ipc(ipcs_ro, ipcs_rw, ipcs_shm); crm_exit(fast); } else if ((mainloop != NULL) && g_main_loop_is_running(mainloop)) { /* Quit via returning from the main loop. If fast == -1, we skip the * disconnect here, and it will be done when the main loop returns * (this allows the peer status callback to avoid messing with the * peer caches). */ if (fast == 0) { crm_cluster_disconnect(crm_cluster); } g_main_loop_quit(mainloop); } else { /* Quit via clean exit. Even the peer status callback can disconnect * here, because we're not returning control to the caller. */ crm_cluster_disconnect(crm_cluster); pcmk__stop_based_ipc(ipcs_ro, ipcs_rw, ipcs_shm); crm_exit(CRM_EX_OK); } } diff --git a/daemons/based/based_notify.c b/daemons/based/based_notify.c index 839a350447..a744fa3a46 100644 --- a/daemons/based/based_notify.c +++ b/daemons/based/based_notify.c @@ -1,232 +1,232 @@ /* * Copyright 2004-2024 the Pacemaker project contributors * * The version control history for this file may have further details. * * This source code is licensed under the GNU General Public License version 2 * or later (GPLv2+) WITHOUT ANY WARRANTY. */ #include #include #include #include #include #include // PRIx64 #include #include #include #include #include #include #include #include #include #include #include struct cib_notification_s { const xmlNode *msg; struct iovec *iov; int32_t iov_size; }; static void cib_notify_send_one(gpointer key, gpointer value, gpointer user_data) { const char *type = NULL; gboolean do_send = FALSE; int rc = pcmk_rc_ok; pcmk__client_t *client = value; struct cib_notification_s *update = user_data; if (client->ipcs == NULL && client->remote == NULL) { crm_warn("Skipping client with NULL channel"); return; } type = crm_element_value(update->msg, PCMK__XA_SUBT); CRM_LOG_ASSERT(type != NULL); if (pcmk_is_set(client->flags, cib_notify_diff) && pcmk__str_eq(type, T_CIB_DIFF_NOTIFY, pcmk__str_casei)) { do_send = TRUE; } else if (pcmk_is_set(client->flags, cib_notify_confirm) && pcmk__str_eq(type, T_CIB_UPDATE_CONFIRM, pcmk__str_casei)) { do_send = TRUE; } else if (pcmk_is_set(client->flags, cib_notify_pre) && pcmk__str_eq(type, T_CIB_PRE_NOTIFY, pcmk__str_casei)) { do_send = TRUE; } else if (pcmk_is_set(client->flags, cib_notify_post) && pcmk__str_eq(type, T_CIB_POST_NOTIFY, pcmk__str_casei)) { do_send = TRUE; } if (do_send) { switch (PCMK__CLIENT_TYPE(client)) { case pcmk__client_ipc: rc = pcmk__ipc_send_iov(client, update->iov, crm_ipc_server_event); if (rc != pcmk_rc_ok) { crm_warn("Could not notify client %s: %s " CRM_XS " id=%s", pcmk__client_name(client), pcmk_rc_str(rc), client->id); } break; #ifdef HAVE_GNUTLS_GNUTLS_H case pcmk__client_tls: #endif case pcmk__client_tcp: crm_debug("Sent %s notification to client %s (id %s)", type, pcmk__client_name(client), client->id); pcmk__remote_send_xml(client->remote, update->msg); break; default: crm_err("Unknown transport for client %s " CRM_XS " flags=%#016" PRIx64, pcmk__client_name(client), client->flags); } } } static void cib_notify_send(const xmlNode *xml) { struct iovec *iov; struct cib_notification_s update; ssize_t bytes = 0; int rc = pcmk__ipc_prepare_iov(0, xml, 0, &iov, &bytes); if (rc == pcmk_rc_ok) { update.msg = xml; update.iov = iov; update.iov_size = bytes; pcmk__foreach_ipc_client(cib_notify_send_one, &update); } else { crm_notice("Could not notify clients: %s " CRM_XS " rc=%d", pcmk_rc_str(rc), rc); } pcmk_free_ipc_event(iov); } static void attach_cib_generation(xmlNode *msg) { xmlNode *generation = create_xml_node(NULL, PCMK__XE_GENERATION_TUPLE); if (generation == NULL) { return; } if (the_cib != NULL) { copy_in_properties(generation, the_cib); } add_message_xml(msg, PCMK__XE_CIB_GENERATION, generation); free_xml(generation); } void cib_diff_notify(const char *op, int result, const char *call_id, const char *client_id, const char *client_name, const char *origin, xmlNode *update, xmlNode *diff) { int add_updates = 0; int add_epoch = 0; int add_admin_epoch = 0; int del_updates = 0; int del_epoch = 0; int del_admin_epoch = 0; uint8_t log_level = LOG_TRACE; xmlNode *update_msg = NULL; const char *type = NULL; if (diff == NULL) { return; } if (result != pcmk_ok) { log_level = LOG_WARNING; } cib_diff_version_details(diff, &add_admin_epoch, &add_epoch, &add_updates, &del_admin_epoch, &del_epoch, &del_updates); if ((add_admin_epoch != del_admin_epoch) || (add_epoch != del_epoch) || (add_updates != del_updates)) { do_crm_log(log_level, "Updated CIB generation %d.%d.%d to %d.%d.%d from client " "%s%s%s (%s) (%s)", del_admin_epoch, del_epoch, del_updates, add_admin_epoch, add_epoch, add_updates, client_name, ((call_id != NULL)? " call " : ""), pcmk__s(call_id, ""), pcmk__s(origin, "unspecified peer"), pcmk_strerror(result)); } else if ((add_admin_epoch != 0) || (add_epoch != 0) || (add_updates != 0)) { do_crm_log(log_level, "Local-only change to CIB generation %d.%d.%d from client " "%s%s%s (%s) (%s)", add_admin_epoch, add_epoch, add_updates, client_name, ((call_id != NULL)? " call " : ""), pcmk__s(call_id, ""), pcmk__s(origin, "unspecified peer"), pcmk_strerror(result)); } update_msg = create_xml_node(NULL, "notify"); - crm_xml_add(update_msg, PCMK__XA_T, T_CIB_NOTIFY); + crm_xml_add(update_msg, PCMK__XA_T, PCMK__VALUE_CIB_NOTIFY); crm_xml_add(update_msg, PCMK__XA_SUBT, T_CIB_DIFF_NOTIFY); crm_xml_add(update_msg, PCMK__XA_CIB_OP, op); crm_xml_add(update_msg, PCMK__XA_CIB_CLIENTID, client_id); crm_xml_add(update_msg, PCMK__XA_CIB_CLIENTNAME, client_name); crm_xml_add(update_msg, PCMK__XA_CIB_CALLID, call_id); crm_xml_add(update_msg, PCMK__XA_SRC, origin); crm_xml_add_int(update_msg, PCMK__XA_CIB_RC, result); // @COMPAT Unused internally, drop at 3.0.0 if (update != NULL) { type = (const char *) update->name; crm_trace("Setting type to update->name: %s", type); } else { type = (const char *) diff->name; crm_trace("Setting type to new_obj->name: %s", type); } // @COMPAT Unused internally, drop at 3.0.0 crm_xml_add(update_msg, PCMK__XA_CIB_OBJECT, pcmk__xe_id(diff)); crm_xml_add(update_msg, PCMK__XA_CIB_OBJECT_TYPE, type); attach_cib_generation(update_msg); // @COMPAT Unused internally, drop at 3.0.0 if (update != NULL) { add_message_xml(update_msg, PCMK__XE_CIB_UPDATE, update); } add_message_xml(update_msg, PCMK__XA_CIB_UPDATE_RESULT, diff); crm_log_xml_trace(update_msg, "diff-notify"); cib_notify_send(update_msg); free_xml(update_msg); } diff --git a/include/crm/cib/internal.h b/include/crm/cib/internal.h index 22d95377d5..9d9ca11019 100644 --- a/include/crm/cib/internal.h +++ b/include/crm/cib/internal.h @@ -1,316 +1,315 @@ /* * Copyright 2004-2024 the Pacemaker project contributors * * The version control history for this file may have further details. * * This source code is licensed under the GNU Lesser General Public License * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. */ #ifndef CIB_INTERNAL__H # define CIB_INTERNAL__H # include # include # include // Request types for CIB manager IPC/CPG #define PCMK__CIB_REQUEST_SECONDARY "cib_slave" #define PCMK__CIB_REQUEST_PRIMARY "cib_master" #define PCMK__CIB_REQUEST_SYNC_TO_ALL "cib_sync" #define PCMK__CIB_REQUEST_SYNC_TO_ONE "cib_sync_one" #define PCMK__CIB_REQUEST_IS_PRIMARY "cib_ismaster" #define PCMK__CIB_REQUEST_BUMP "cib_bump" #define PCMK__CIB_REQUEST_QUERY "cib_query" #define PCMK__CIB_REQUEST_CREATE "cib_create" #define PCMK__CIB_REQUEST_MODIFY "cib_modify" #define PCMK__CIB_REQUEST_DELETE "cib_delete" #define PCMK__CIB_REQUEST_ERASE "cib_erase" #define PCMK__CIB_REQUEST_REPLACE "cib_replace" #define PCMK__CIB_REQUEST_APPLY_PATCH "cib_apply_diff" #define PCMK__CIB_REQUEST_UPGRADE "cib_upgrade" #define PCMK__CIB_REQUEST_ABS_DELETE "cib_delete_alt" #define PCMK__CIB_REQUEST_NOOP "noop" #define PCMK__CIB_REQUEST_SHUTDOWN "cib_shutdown_req" #define PCMK__CIB_REQUEST_COMMIT_TRANSACT "cib_commit_transact" #define PCMK__CIB_REQUEST_SCHEMAS "cib_schemas" -# define T_CIB_NOTIFY "cib_notify" /* notify sub-types */ # define T_CIB_PRE_NOTIFY "cib_pre_notify" # define T_CIB_POST_NOTIFY "cib_post_notify" # define T_CIB_TRANSACTION "cib_transaction" # define T_CIB_UPDATE_CONFIRM "cib_update_confirmation" /*! * \internal * \enum cib__op_attr * \brief Flags for CIB operation attributes */ enum cib__op_attr { cib__op_attr_none = 0, //!< No special attributes cib__op_attr_modifies = (1 << 1), //!< Modifies CIB cib__op_attr_privileged = (1 << 2), //!< Requires privileges cib__op_attr_local = (1 << 3), //!< Must only be processed locally cib__op_attr_replaces = (1 << 4), //!< Replaces CIB cib__op_attr_writes_through = (1 << 5), //!< Writes to disk on success cib__op_attr_transaction = (1 << 6), //!< Supported in a transaction }; /*! * \internal * \enum cib__op_type * \brief Types of CIB operations */ enum cib__op_type { cib__op_abs_delete, cib__op_apply_patch, cib__op_bump, cib__op_commit_transact, cib__op_create, cib__op_delete, cib__op_erase, cib__op_is_primary, cib__op_modify, cib__op_noop, cib__op_ping, cib__op_primary, cib__op_query, cib__op_replace, cib__op_secondary, cib__op_shutdown, cib__op_sync_all, cib__op_sync_one, cib__op_upgrade, cib__op_schemas, }; gboolean cib_diff_version_details(xmlNode * diff, int *admin_epoch, int *epoch, int *updates, int *_admin_epoch, int *_epoch, int *_updates); gboolean cib_read_config(GHashTable * options, xmlNode * current_cib); typedef int (*cib__op_fn_t)(const char *, int, const char *, xmlNode *, xmlNode *, xmlNode *, xmlNode **, xmlNode **); typedef struct cib__operation_s { const char *name; enum cib__op_type type; uint32_t flags; //!< Group of enum cib__op_attr flags } cib__operation_t; typedef struct cib_notify_client_s { const char *event; const char *obj_id; /* implement one day */ const char *obj_type; /* implement one day */ void (*callback) (const char *event, xmlNode * msg); } cib_notify_client_t; typedef struct cib_callback_client_s { void (*callback) (xmlNode *, int, int, xmlNode *, void *); const char *id; void *user_data; gboolean only_success; struct timer_rec_s *timer; void (*free_func)(void *); } cib_callback_client_t; struct timer_rec_s { int call_id; int timeout; guint ref; cib_t *cib; }; #define cib__set_call_options(cib_call_opts, call_for, flags_to_set) do { \ cib_call_opts = pcmk__set_flags_as(__func__, __LINE__, \ LOG_TRACE, "CIB call", (call_for), (cib_call_opts), \ (flags_to_set), #flags_to_set); \ } while (0) #define cib__clear_call_options(cib_call_opts, call_for, flags_to_clear) do { \ cib_call_opts = pcmk__clear_flags_as(__func__, __LINE__, \ LOG_TRACE, "CIB call", (call_for), (cib_call_opts), \ (flags_to_clear), #flags_to_clear); \ } while (0) cib_t *cib_new_variant(void); /*! * \internal * \brief Check whether a given CIB client's update should trigger a refresh * * Here, "refresh" means that Pacemaker daemons write out their current state. * * If a Pacemaker daemon or one of certain Pacemaker CLI tools modifies the CIB, * we can assume that the CIB hasn't diverged from the true cluster state. A * "safe" CLI tool requests that all relevant daemons update their state before * the tool requests any CIB modifications directly. * * In contrast, other "unsafe" tools (for example, \c cibadmin and external * tools) may request arbitrary CIB changes. * * A Pacemaker daemon can write out its current state to the CIB when it's * notified of an update from an unsafe client, to ensure the CIB still contains * the daemon's correct state. * * \param[in] name CIB client name * * \return \c true if the CIB client should trigger a refresh, or \c false * otherwise */ static inline bool cib__client_triggers_refresh(const char *name) { return !crm_is_daemon_name(name) && !pcmk__str_any_of(name, "attrd_updater", "crm_attribute", "crm_node", "crm_resource", "crm_ticket", NULL); } int cib__get_notify_patchset(const xmlNode *msg, const xmlNode **patchset); bool cib__element_in_patchset(const xmlNode *patchset, const char *element); int cib_perform_op(cib_t *cib, const char *op, int call_options, cib__op_fn_t fn, bool is_query, const char *section, xmlNode *req, xmlNode *input, bool manage_counters, bool *config_changed, xmlNode **current_cib, xmlNode **result_cib, xmlNode **diff, xmlNode **output); int cib__create_op(cib_t *cib, const char *op, const char *host, const char *section, xmlNode *data, int call_options, const char *user_name, const char *client_name, xmlNode **op_msg); int cib__extend_transaction(cib_t *cib, xmlNode *request); void cib_native_callback(cib_t * cib, xmlNode * msg, int call_id, int rc); void cib_native_notify(gpointer data, gpointer user_data); int cib__get_operation(const char *op, const cib__operation_t **operation); int cib_process_query(const char *op, int options, const char *section, xmlNode * req, xmlNode * input, xmlNode * existing_cib, xmlNode ** result_cib, xmlNode ** answer); int cib_process_erase(const char *op, int options, const char *section, xmlNode * req, xmlNode * input, xmlNode * existing_cib, xmlNode ** result_cib, xmlNode ** answer); int cib_process_bump(const char *op, int options, const char *section, xmlNode * req, xmlNode * input, xmlNode * existing_cib, xmlNode ** result_cib, xmlNode ** answer); int cib_process_replace(const char *op, int options, const char *section, xmlNode * req, xmlNode * input, xmlNode * existing_cib, xmlNode ** result_cib, xmlNode ** answer); int cib_process_create(const char *op, int options, const char *section, xmlNode * req, xmlNode * input, xmlNode * existing_cib, xmlNode ** result_cib, xmlNode ** answer); int cib_process_modify(const char *op, int options, const char *section, xmlNode * req, xmlNode * input, xmlNode * existing_cib, xmlNode ** result_cib, xmlNode ** answer); int cib_process_delete(const char *op, int options, const char *section, xmlNode * req, xmlNode * input, xmlNode * existing_cib, xmlNode ** result_cib, xmlNode ** answer); int cib_process_diff(const char *op, int options, const char *section, xmlNode * req, xmlNode * input, xmlNode * existing_cib, xmlNode ** result_cib, xmlNode ** answer); int cib_process_upgrade(const char *op, int options, const char *section, xmlNode * req, xmlNode * input, xmlNode * existing_cib, xmlNode ** result_cib, xmlNode ** answer); /*! * \internal * \brief Query or modify a CIB * * \param[in] op PCMK__CIB_REQUEST_* operation to be performed * \param[in] options Flag set of \c cib_call_options * \param[in] section XPath to query or modify * \param[in] req unused * \param[in] input Portion of CIB to modify (used with * PCMK__CIB_REQUEST_CREATE, * PCMK__CIB_REQUEST_MODIFY, and * PCMK__CIB_REQUEST_REPLACE) * \param[in,out] existing_cib Input CIB (used with PCMK__CIB_REQUEST_QUERY) * \param[in,out] result_cib CIB copy to make changes in (used with * PCMK__CIB_REQUEST_CREATE, * PCMK__CIB_REQUEST_MODIFY, * PCMK__CIB_REQUEST_DELETE, and * PCMK__CIB_REQUEST_REPLACE) * \param[out] answer Query result (used with PCMK__CIB_REQUEST_QUERY) * * \return Legacy Pacemaker return code */ int cib_process_xpath(const char *op, int options, const char *section, const xmlNode *req, xmlNode *input, xmlNode *existing_cib, xmlNode **result_cib, xmlNode ** answer); bool cib__config_changed_v1(xmlNode *last, xmlNode *next, xmlNode **diff); int cib_internal_op(cib_t * cib, const char *op, const char *host, const char *section, xmlNode * data, xmlNode ** output_data, int call_options, const char *user_name); int cib_file_read_and_verify(const char *filename, const char *sigfile, xmlNode **root); int cib_file_write_with_digest(xmlNode *cib_root, const char *cib_dirname, const char *cib_filename); void cib__set_output(cib_t *cib, pcmk__output_t *out); cib_callback_client_t* cib__lookup_id (int call_id); /*! * \internal * \brief Connect to, query, and optionally disconnect from the CIB * * Open a read-write connection to the CIB manager if an already connected * client is not passed in. Then query the CIB and store the resulting XML. * Finally, disconnect if the CIB connection isn't being returned to the caller. * * \param[in,out] out Output object (may be \p NULL) * \param[in,out] cib If not \p NULL, where to store CIB connection * \param[out] cib_object Where to store query result * * \return Standard Pacemaker return code * * \note If \p cib is not \p NULL, the caller is responsible for freeing \p *cib * using \p cib_delete(). * \note If \p *cib points to an existing \p cib_t object, this function will * reuse it instead of creating a new one. If the existing client is * already connected, the connection will be reused, even if it's * read-only. */ int cib__signon_query(pcmk__output_t *out, cib_t **cib, xmlNode **cib_object); int cib__clean_up_connection(cib_t **cib); int cib__update_node_attr(pcmk__output_t *out, cib_t *cib, int call_options, const char *section, const char *node_uuid, const char *set_type, const char *set_name, const char *attr_id, const char *attr_name, const char *attr_value, const char *user_name, const char *node_type); int cib__get_node_attrs(pcmk__output_t *out, cib_t *cib, const char *section, const char *node_uuid, const char *set_type, const char *set_name, const char *attr_id, const char *attr_name, const char *user_name, xmlNode **result); int cib__delete_node_attr(pcmk__output_t *out, cib_t *cib, int options, const char *section, const char *node_uuid, const char *set_type, const char *set_name, const char *attr_id, const char *attr_name, const char *attr_value, const char *user_name); #endif diff --git a/include/crm/common/options_internal.h b/include/crm/common/options_internal.h index 62e33212b1..8aef8163d7 100644 --- a/include/crm/common/options_internal.h +++ b/include/crm/common/options_internal.h @@ -1,225 +1,226 @@ /* * Copyright 2006-2024 the Pacemaker project contributors * * The version control history for this file may have further details. * * This source code is licensed under the GNU Lesser General Public License * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. */ #ifndef PCMK__OPTIONS_INTERNAL__H # define PCMK__OPTIONS_INTERNAL__H # ifndef PCMK__CONFIG_H # define PCMK__CONFIG_H # include // _Noreturn # endif # include // GHashTable # include // bool #include // pcmk_parse_interval_spec() _Noreturn void pcmk__cli_help(char cmd); /* * Environment variable option handling */ const char *pcmk__env_option(const char *option); void pcmk__set_env_option(const char *option, const char *value, bool compat); bool pcmk__env_option_enabled(const char *daemon, const char *option); /* * Cluster option handling */ /*! * \internal * \enum pcmk__opt_context * \brief Context flags for options */ enum pcmk__opt_context { // @COMPAT Used only for daemon metadata pcmk__opt_context_none = 0, //!< No additional context pcmk__opt_context_based = (1 << 1), //!< CIB manager metadata pcmk__opt_context_controld = (1 << 2), //!< Controller metadata pcmk__opt_context_schedulerd = (1 << 3), //!< Scheduler metadata }; typedef struct pcmk__cluster_option_s { const char *name; const char *alt_name; const char *type; const char *values; const char *default_value; bool (*is_valid)(const char *); // @COMPAT context is used only for daemon meta-data enum pcmk__opt_context context; const char *description_short; const char *description_long; } pcmk__cluster_option_t; const char *pcmk__cluster_option(GHashTable *options, const char *name); char *pcmk__format_option_metadata(const char *name, const char *desc_short, const char *desc_long, enum pcmk__opt_context filter, pcmk__cluster_option_t *option_list, int len); char *pcmk__cluster_option_metadata(const char *name, const char *desc_short, const char *desc_long, enum pcmk__opt_context filter); void pcmk__validate_cluster_options(GHashTable *options); bool pcmk__valid_interval_spec(const char *value); bool pcmk__valid_boolean(const char *value); bool pcmk__valid_int(const char *value); bool pcmk__valid_positive_int(const char *value); bool pcmk__valid_no_quorum_policy(const char *value); bool pcmk__valid_percentage(const char *value); bool pcmk__valid_script(const char *value); bool pcmk__valid_placement_strategy(const char *value); // from watchdog.c long pcmk__get_sbd_watchdog_timeout(void); bool pcmk__get_sbd_sync_resource_startup(void); long pcmk__auto_stonith_watchdog_timeout(void); bool pcmk__valid_stonith_watchdog_timeout(const char *value); // Constants for environment variable names #define PCMK__ENV_AUTHKEY_LOCATION "authkey_location" #define PCMK__ENV_BLACKBOX "blackbox" #define PCMK__ENV_CALLGRIND_ENABLED "callgrind_enabled" #define PCMK__ENV_CLUSTER_TYPE "cluster_type" #define PCMK__ENV_DEBUG "debug" #define PCMK__ENV_DH_MAX_BITS "dh_max_bits" #define PCMK__ENV_DH_MIN_BITS "dh_min_bits" #define PCMK__ENV_FAIL_FAST "fail_fast" #define PCMK__ENV_IPC_BUFFER "ipc_buffer" #define PCMK__ENV_IPC_TYPE "ipc_type" #define PCMK__ENV_LOGFACILITY "logfacility" #define PCMK__ENV_LOGFILE "logfile" #define PCMK__ENV_LOGFILE_MODE "logfile_mode" #define PCMK__ENV_LOGPRIORITY "logpriority" #define PCMK__ENV_NODE_ACTION_LIMIT "node_action_limit" #define PCMK__ENV_NODE_START_STATE "node_start_state" #define PCMK__ENV_PANIC_ACTION "panic_action" #define PCMK__ENV_REMOTE_ADDRESS "remote_address" #define PCMK__ENV_REMOTE_SCHEMA_DIRECTORY "remote_schema_directory" #define PCMK__ENV_REMOTE_PID1 "remote_pid1" #define PCMK__ENV_REMOTE_PORT "remote_port" #define PCMK__ENV_RESPAWNED "respawned" #define PCMK__ENV_SCHEMA_DIRECTORY "schema_directory" #define PCMK__ENV_SERVICE "service" #define PCMK__ENV_STDERR "stderr" #define PCMK__ENV_TLS_PRIORITIES "tls_priorities" #define PCMK__ENV_TRACE_BLACKBOX "trace_blackbox" #define PCMK__ENV_TRACE_FILES "trace_files" #define PCMK__ENV_TRACE_FORMATS "trace_formats" #define PCMK__ENV_TRACE_FUNCTIONS "trace_functions" #define PCMK__ENV_TRACE_TAGS "trace_tags" #define PCMK__ENV_VALGRIND_ENABLED "valgrind_enabled" // @COMPAT Drop at 3.0.0; default is plenty #define PCMK__ENV_CIB_TIMEOUT "cib_timeout" // @COMPAT Drop at 3.0.0; likely last used in 1.1.24 #define PCMK__ENV_MCP "mcp" // @COMPAT Drop at 3.0.0; added unused in 1.1.9 #define PCMK__ENV_QUORUM_TYPE "quorum_type" /* @COMPAT Drop at 3.0.0; added to debug shutdown issues when Pacemaker is * managed by systemd, but no longer useful. */ #define PCMK__ENV_SHUTDOWN_DELAY "shutdown_delay" // @COMPAT Deprecated since 2.1.0 #define PCMK__OPT_REMOVE_AFTER_STOP "remove-after-stop" // Constants for meta-attribute names #define PCMK__META_CLONE "clone" #define PCMK__META_CONTAINER "container" #define PCMK__META_DIGESTS_ALL "digests-all" #define PCMK__META_DIGESTS_SECURE "digests-secure" #define PCMK__META_INTERNAL_RSC "internal_rsc" #define PCMK__META_MIGRATE_SOURCE "migrate_source" #define PCMK__META_MIGRATE_TARGET "migrate_target" #define PCMK__META_ON_NODE "on_node" #define PCMK__META_ON_NODE_UUID "on_node_uuid" #define PCMK__META_OP_NO_WAIT "op_no_wait" #define PCMK__META_OP_TARGET_RC "op_target_rc" #define PCMK__META_PHYSICAL_HOST "physical-host" #define PCMK__META_STONITH_ACTION "stonith_action" /* @TODO Plug these in. Currently, they're never set. These are op attrs for use * with https://projects.clusterlabs.org/T382. */ #define PCMK__META_CLEAR_FAILURE_OP "clear_failure_op" #define PCMK__META_CLEAR_FAILURE_INTERVAL "clear_failure_interval" // @COMPAT Deprecated meta-attribute since 2.1.0 #define PCMK__META_CAN_FAIL "can_fail" // @COMPAT Deprecated alias for PCMK__META_PROMOTED_MAX since 2.0.0 #define PCMK__META_PROMOTED_MAX_LEGACY "master-max" // @COMPAT Deprecated alias for PCMK__META_PROMOTED_NODE_MAX since 2.0.0 #define PCMK__META_PROMOTED_NODE_MAX_LEGACY "master-node-max" // @COMPAT Deprecated meta-attribute since 2.0.0 #define PCMK__META_RESTART_TYPE "restart-type" // @COMPAT Deprecated meta-attribute since 2.0.0 #define PCMK__META_ROLE_AFTER_FAILURE "role_after_failure" // Constants for enumerated values #define PCMK__VALUE_ATTRD "attrd" #define PCMK__VALUE_BOLD "bold" #define PCMK__VALUE_BROADCAST "broadcast" #define PCMK__VALUE_CIB "cib" +#define PCMK__VALUE_CIB_NOTIFY "cib_notify" #define PCMK__VALUE_CLUSTER "cluster" #define PCMK__VALUE_CRMD "crmd" #define PCMK__VALUE_EN "en" #define PCMK__VALUE_EPOCH "epoch" #define PCMK__VALUE_HEALTH_RED "health_red" #define PCMK__VALUE_HEALTH_YELLOW "health_yellow" #define PCMK__VALUE_INIT "init" #define PCMK__VALUE_LOCAL "local" #define PCMK__VALUE_MAINT "maint" #define PCMK__VALUE_OUTPUT "output" #define PCMK__VALUE_PASSWORD "password" #define PCMK__VALUE_PING "ping" #define PCMK__VALUE_REFRESH "refresh" #define PCMK__VALUE_REQUEST "request" #define PCMK__VALUE_RESPONSE "response" #define PCMK__VALUE_RSC_FAILED "rsc-failed" #define PCMK__VALUE_RSC_FAILURE_IGNORED "rsc-failure-ignored" #define PCMK__VALUE_RSC_MANAGED "rsc-managed" #define PCMK__VALUE_RSC_MULTIPLE "rsc-multiple" #define PCMK__VALUE_RSC_OK "rsc-ok" #define PCMK__VALUE_RUNNING "running" #define PCMK__VALUE_SHUTDOWN_COMPLETE "shutdown_complete" #define PCMK__VALUE_SHUTTING_DOWN "shutting_down" #define PCMK__VALUE_ST_ASYNC_TIMEOUT_VALUE "st-async-timeout-value" #define PCMK__VALUE_ST_NOTIFY "st_notify" #define PCMK__VALUE_STARTING_DAEMONS "starting_daemons" #define PCMK__VALUE_STONITH_NG "stonith-ng" #define PCMK__VALUE_WAIT_FOR_PING "wait_for_ping" #define PCMK__VALUE_WARNING "warning" /* @COMPAT Deprecated since 2.1.7 (used with PCMK__XA_ORDERING attribute of * resource sets) */ #define PCMK__VALUE_GROUP "group" #endif // PCMK__OPTIONS_INTERNAL__H diff --git a/lib/cib/cib_native.c b/lib/cib/cib_native.c index 344e35751e..9766205bb6 100644 --- a/lib/cib/cib_native.c +++ b/lib/cib/cib_native.c @@ -1,509 +1,509 @@ /* * Copyright 2004 International Business Machines * Later changes copyright 2004-2024 the Pacemaker project contributors * * The version control history for this file may have further details. * * This source code is licensed under the GNU Lesser General Public License * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. */ #include #ifndef _GNU_SOURCE # define _GNU_SOURCE #endif #include #include #include #include #include #include #include #include #include #include #include #include typedef struct cib_native_opaque_s { char *token; crm_ipc_t *ipc; void (*dnotify_fn) (gpointer user_data); mainloop_io_t *source; } cib_native_opaque_t; static int cib_native_perform_op_delegate(cib_t *cib, const char *op, const char *host, const char *section, xmlNode *data, xmlNode **output_data, int call_options, const char *user_name) { int rc = pcmk_ok; int reply_id = 0; enum crm_ipc_flags ipc_flags = crm_ipc_flags_none; xmlNode *op_msg = NULL; xmlNode *op_reply = NULL; cib_native_opaque_t *native = cib->variant_opaque; if (cib->state == cib_disconnected) { return -ENOTCONN; } if (output_data != NULL) { *output_data = NULL; } if (op == NULL) { crm_err("No operation specified"); return -EINVAL; } if (call_options & cib_sync_call) { pcmk__set_ipc_flags(ipc_flags, "client", crm_ipc_client_response); } rc = cib__create_op(cib, op, host, section, data, call_options, user_name, NULL, &op_msg); if (rc != pcmk_ok) { return rc; } if (pcmk_is_set(call_options, cib_transaction)) { rc = cib__extend_transaction(cib, op_msg); goto done; } crm_trace("Sending %s message to the CIB manager (timeout=%ds)", op, cib->call_timeout); rc = crm_ipc_send(native->ipc, op_msg, ipc_flags, cib->call_timeout * 1000, &op_reply); if (rc < 0) { crm_err("Couldn't perform %s operation (timeout=%ds): %s (%d)", op, cib->call_timeout, pcmk_strerror(rc), rc); rc = -ECOMM; goto done; } crm_log_xml_trace(op_reply, "Reply"); if (!(call_options & cib_sync_call)) { crm_trace("Async call, returning %d", cib->call_id); CRM_CHECK(cib->call_id != 0, return -ENOMSG); free_xml(op_reply); return cib->call_id; } rc = pcmk_ok; crm_element_value_int(op_reply, PCMK__XA_CIB_CALLID, &reply_id); if (reply_id == cib->call_id) { xmlNode *tmp = get_message_xml(op_reply, PCMK__XA_CIB_CALLDATA); crm_trace("Synchronous reply %d received", reply_id); if (crm_element_value_int(op_reply, PCMK__XA_CIB_RC, &rc) != 0) { rc = -EPROTO; } if (output_data == NULL || (call_options & cib_discard_reply)) { crm_trace("Discarding reply"); } else if (tmp != NULL) { *output_data = copy_xml(tmp); } } else if (reply_id <= 0) { crm_err("Received bad reply: No id set"); crm_log_xml_err(op_reply, "Bad reply"); rc = -ENOMSG; goto done; } else { crm_err("Received bad reply: %d (wanted %d)", reply_id, cib->call_id); crm_log_xml_err(op_reply, "Old reply"); rc = -ENOMSG; goto done; } if (op_reply == NULL && cib->state == cib_disconnected) { rc = -ENOTCONN; } else if (rc == pcmk_ok && op_reply == NULL) { rc = -ETIME; } switch (rc) { case pcmk_ok: case -EPERM: break; /* This is an internal value that clients do not and should not care about */ case -pcmk_err_diff_resync: rc = pcmk_ok; break; /* These indicate internal problems */ case -EPROTO: case -ENOMSG: crm_err("Call failed: %s", pcmk_strerror(rc)); if (op_reply) { crm_log_xml_err(op_reply, "Invalid reply"); } break; default: if (!pcmk__str_eq(op, PCMK__CIB_REQUEST_QUERY, pcmk__str_none)) { crm_warn("Call failed: %s", pcmk_strerror(rc)); } } done: if (!crm_ipc_connected(native->ipc)) { crm_err("The CIB manager disconnected"); cib->state = cib_disconnected; } free_xml(op_msg); free_xml(op_reply); return rc; } static int cib_native_dispatch_internal(const char *buffer, ssize_t length, gpointer userdata) { const char *type = NULL; xmlNode *msg = NULL; cib_t *cib = userdata; crm_trace("dispatching %p", userdata); if (cib == NULL) { crm_err("No CIB!"); return 0; } msg = string2xml(buffer); if (msg == NULL) { crm_warn("Received a NULL message from the CIB manager"); return 0; } /* do callbacks */ type = crm_element_value(msg, PCMK__XA_T); crm_trace("Activating %s callbacks...", type); crm_log_xml_explicit(msg, "cib-reply"); if (pcmk__str_eq(type, PCMK__VALUE_CIB, pcmk__str_none)) { cib_native_callback(cib, msg, 0, 0); - } else if (pcmk__str_eq(type, T_CIB_NOTIFY, pcmk__str_casei)) { + } else if (pcmk__str_eq(type, PCMK__VALUE_CIB_NOTIFY, pcmk__str_none)) { g_list_foreach(cib->notify_list, cib_native_notify, msg); } else { crm_err("Unknown message type: %s", type); } free_xml(msg); return 0; } static void cib_native_destroy(void *userdata) { cib_t *cib = userdata; cib_native_opaque_t *native = cib->variant_opaque; crm_trace("destroying %p", userdata); cib->state = cib_disconnected; native->source = NULL; native->ipc = NULL; if (native->dnotify_fn) { native->dnotify_fn(userdata); } } static int cib_native_signoff(cib_t *cib) { cib_native_opaque_t *native = cib->variant_opaque; crm_debug("Disconnecting from the CIB manager"); cib_free_notify(cib); remove_cib_op_callback(0, TRUE); if (native->source != NULL) { /* Attached to mainloop */ mainloop_del_ipc_client(native->source); native->source = NULL; native->ipc = NULL; } else if (native->ipc) { /* Not attached to mainloop */ crm_ipc_t *ipc = native->ipc; native->ipc = NULL; crm_ipc_close(ipc); crm_ipc_destroy(ipc); } cib->cmds->end_transaction(cib, false, cib_none); cib->state = cib_disconnected; cib->type = cib_no_connection; return pcmk_ok; } static int cib_native_signon_raw(cib_t *cib, const char *name, enum cib_conn_type type, int *async_fd) { int rc = pcmk_ok; const char *channel = NULL; cib_native_opaque_t *native = cib->variant_opaque; xmlNode *hello = NULL; struct ipc_client_callbacks cib_callbacks = { .dispatch = cib_native_dispatch_internal, .destroy = cib_native_destroy }; cib->call_timeout = PCMK__IPC_TIMEOUT; if (type == cib_command) { cib->state = cib_connected_command; channel = PCMK__SERVER_BASED_RW; } else if (type == cib_command_nonblocking) { cib->state = cib_connected_command; channel = PCMK__SERVER_BASED_SHM; } else if (type == cib_query) { cib->state = cib_connected_query; channel = PCMK__SERVER_BASED_RO; } else { return -ENOTCONN; } crm_trace("Connecting %s channel", channel); if (async_fd != NULL) { native->ipc = crm_ipc_new(channel, 0); if (native->ipc != NULL) { rc = pcmk__connect_generic_ipc(native->ipc); if (rc == pcmk_rc_ok) { rc = pcmk__ipc_fd(native->ipc, async_fd); if (rc != pcmk_rc_ok) { crm_info("Couldn't get file descriptor for %s IPC", channel); } } rc = pcmk_rc2legacy(rc); } } else { native->source = mainloop_add_ipc_client(channel, G_PRIORITY_HIGH, 512 * 1024 /* 512k */ , cib, &cib_callbacks); native->ipc = mainloop_get_ipc_client(native->source); } if (rc != pcmk_ok || native->ipc == NULL || !crm_ipc_connected(native->ipc)) { crm_info("Could not connect to CIB manager for %s", name); rc = -ENOTCONN; } if (rc == pcmk_ok) { rc = cib__create_op(cib, CRM_OP_REGISTER, NULL, NULL, NULL, cib_sync_call, NULL, name, &hello); } if (rc == pcmk_ok) { xmlNode *reply = NULL; if (crm_ipc_send(native->ipc, hello, crm_ipc_client_response, -1, &reply) > 0) { const char *msg_type = crm_element_value(reply, PCMK__XA_CIB_OP); crm_log_xml_trace(reply, "reg-reply"); if (!pcmk__str_eq(msg_type, CRM_OP_REGISTER, pcmk__str_casei)) { crm_info("Reply to CIB registration message has unknown type " "'%s'", msg_type); rc = -EPROTO; } else { native->token = crm_element_value_copy(reply, PCMK__XA_CIB_CLIENTID); if (native->token == NULL) { rc = -EPROTO; } } free_xml(reply); } else { rc = -ECOMM; } free_xml(hello); } if (rc == pcmk_ok) { crm_info("Successfully connected to CIB manager for %s", name); return pcmk_ok; } crm_info("Connection to CIB manager for %s failed: %s", name, pcmk_strerror(rc)); cib_native_signoff(cib); return rc; } static int cib_native_signon(cib_t *cib, const char *name, enum cib_conn_type type) { return cib_native_signon_raw(cib, name, type, NULL); } static int cib_native_free(cib_t *cib) { int rc = pcmk_ok; if (cib->state != cib_disconnected) { rc = cib_native_signoff(cib); } if (cib->state == cib_disconnected) { cib_native_opaque_t *native = cib->variant_opaque; free(native->token); free(cib->variant_opaque); free(cib->cmds); free(cib->user); free(cib); } return rc; } static int cib_native_register_notification(cib_t *cib, const char *callback, int enabled) { int rc = pcmk_ok; xmlNode *notify_msg = create_xml_node(NULL, "cib-callback"); cib_native_opaque_t *native = cib->variant_opaque; if (cib->state != cib_disconnected) { - crm_xml_add(notify_msg, PCMK__XA_CIB_OP, T_CIB_NOTIFY); + crm_xml_add(notify_msg, PCMK__XA_CIB_OP, PCMK__VALUE_CIB_NOTIFY); crm_xml_add(notify_msg, PCMK__XA_CIB_NOTIFY_TYPE, callback); crm_xml_add_int(notify_msg, PCMK__XA_CIB_NOTIFY_ACTIVATE, enabled); rc = crm_ipc_send(native->ipc, notify_msg, crm_ipc_client_response, 1000 * cib->call_timeout, NULL); if (rc <= 0) { crm_trace("Notification not registered: %d", rc); rc = -ECOMM; } } free_xml(notify_msg); return rc; } static int cib_native_set_connection_dnotify(cib_t *cib, void (*dnotify) (gpointer user_data)) { cib_native_opaque_t *native = NULL; if (cib == NULL) { crm_err("No CIB!"); return FALSE; } native = cib->variant_opaque; native->dnotify_fn = dnotify; return pcmk_ok; } /*! * \internal * \brief Get the given CIB connection's unique client identifier * * These can be used to check whether this client requested the action that * triggered a CIB notification. * * \param[in] cib CIB connection * \param[out] async_id If not \p NULL, where to store asynchronous client ID * \param[out] sync_id If not \p NULL, where to store synchronous client ID * * \return Legacy Pacemaker return code (specifically, \p pcmk_ok) * * \note This is the \p cib_native variant implementation of * \p cib_api_operations_t:client_id(). * \note For \p cib_native objects, \p async_id and \p sync_id are the same. * \note The client ID is assigned during CIB sign-on. */ static int cib_native_client_id(const cib_t *cib, const char **async_id, const char **sync_id) { cib_native_opaque_t *native = cib->variant_opaque; if (async_id != NULL) { *async_id = native->token; } if (sync_id != NULL) { *sync_id = native->token; } return pcmk_ok; } cib_t * cib_native_new(void) { cib_native_opaque_t *native = NULL; cib_t *cib = cib_new_variant(); if (cib == NULL) { return NULL; } native = calloc(1, sizeof(cib_native_opaque_t)); if (native == NULL) { free(cib); return NULL; } cib->variant = cib_native; cib->variant_opaque = native; native->ipc = NULL; native->source = NULL; native->dnotify_fn = NULL; /* assign variant specific ops */ cib->delegate_fn = cib_native_perform_op_delegate; cib->cmds->signon = cib_native_signon; cib->cmds->signon_raw = cib_native_signon_raw; cib->cmds->signoff = cib_native_signoff; cib->cmds->free = cib_native_free; cib->cmds->register_notification = cib_native_register_notification; cib->cmds->set_connection_dnotify = cib_native_set_connection_dnotify; cib->cmds->client_id = cib_native_client_id; return cib; } diff --git a/lib/cib/cib_remote.c b/lib/cib/cib_remote.c index bb1f0d4bbb..1fc9290be6 100644 --- a/lib/cib/cib_remote.c +++ b/lib/cib/cib_remote.c @@ -1,647 +1,647 @@ /* * Copyright 2008-2024 the Pacemaker project contributors * * The version control history for this file may have further details. * * This source code is licensed under the GNU Lesser General Public License * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef HAVE_GNUTLS_GNUTLS_H # include # define TLS_HANDSHAKE_TIMEOUT_MS 5000 static gnutls_anon_client_credentials_t anon_cred_c; static gboolean remote_gnutls_credentials_init = FALSE; #endif // HAVE_GNUTLS_GNUTLS_H #include typedef struct cib_remote_opaque_s { int port; char *server; char *user; char *passwd; gboolean encrypted; pcmk__remote_t command; pcmk__remote_t callback; pcmk__output_t *out; } cib_remote_opaque_t; static int cib_remote_perform_op(cib_t *cib, const char *op, const char *host, const char *section, xmlNode *data, xmlNode **output_data, int call_options, const char *user_name) { int rc; int remaining_time = 0; time_t start_time; xmlNode *op_msg = NULL; xmlNode *op_reply = NULL; cib_remote_opaque_t *private = cib->variant_opaque; if (cib->state == cib_disconnected) { return -ENOTCONN; } if (output_data != NULL) { *output_data = NULL; } if (op == NULL) { crm_err("No operation specified"); return -EINVAL; } rc = cib__create_op(cib, op, host, section, data, call_options, user_name, NULL, &op_msg); if (rc != pcmk_ok) { return rc; } if (pcmk_is_set(call_options, cib_transaction)) { rc = cib__extend_transaction(cib, op_msg); free_xml(op_msg); return rc; } crm_trace("Sending %s message to the CIB manager", op); if (!(call_options & cib_sync_call)) { pcmk__remote_send_xml(&private->callback, op_msg); } else { pcmk__remote_send_xml(&private->command, op_msg); } free_xml(op_msg); if ((call_options & cib_discard_reply)) { crm_trace("Discarding reply"); return pcmk_ok; } else if (!(call_options & cib_sync_call)) { return cib->call_id; } crm_trace("Waiting for a synchronous reply"); start_time = time(NULL); remaining_time = cib->call_timeout ? cib->call_timeout : 60; rc = pcmk_rc_ok; while (remaining_time > 0 && (rc != ENOTCONN)) { int reply_id = -1; int msg_id = cib->call_id; rc = pcmk__read_remote_message(&private->command, remaining_time * 1000); op_reply = pcmk__remote_message_xml(&private->command); if (!op_reply) { break; } crm_element_value_int(op_reply, PCMK__XA_CIB_CALLID, &reply_id); if (reply_id == msg_id) { break; } else if (reply_id < msg_id) { crm_debug("Received old reply: %d (wanted %d)", reply_id, msg_id); crm_log_xml_trace(op_reply, "Old reply"); } else if ((reply_id - 10000) > msg_id) { /* wrap-around case */ crm_debug("Received old reply: %d (wanted %d)", reply_id, msg_id); crm_log_xml_trace(op_reply, "Old reply"); } else { crm_err("Received a __future__ reply:" " %d (wanted %d)", reply_id, msg_id); } free_xml(op_reply); op_reply = NULL; /* wasn't the right reply, try and read some more */ remaining_time = time(NULL) - start_time; } /* if(IPC_ISRCONN(native->command_channel) == FALSE) { */ /* crm_err("The CIB manager disconnected: %d", */ /* native->command_channel->ch_status); */ /* cib->state = cib_disconnected; */ /* } */ if (rc == ENOTCONN) { crm_err("Disconnected while waiting for reply."); return -ENOTCONN; } else if (op_reply == NULL) { crm_err("No reply message - empty"); return -ENOMSG; } crm_trace("Synchronous reply received"); /* Start processing the reply... */ if (crm_element_value_int(op_reply, PCMK__XA_CIB_RC, &rc) != 0) { rc = -EPROTO; } if (rc == -pcmk_err_diff_resync) { /* This is an internal value that clients do not and should not care about */ rc = pcmk_ok; } if (rc == pcmk_ok || rc == -EPERM) { crm_log_xml_debug(op_reply, "passed"); } else { /* } else if(rc == -ETIME) { */ crm_err("Call failed: %s", pcmk_strerror(rc)); crm_log_xml_warn(op_reply, "failed"); } if (output_data == NULL) { /* do nothing more */ } else if (!(call_options & cib_discard_reply)) { xmlNode *tmp = get_message_xml(op_reply, PCMK__XA_CIB_CALLDATA); if (tmp == NULL) { crm_trace("No output in reply to \"%s\" command %d", op, cib->call_id - 1); } else { *output_data = copy_xml(tmp); } } free_xml(op_reply); return rc; } static int cib_remote_callback_dispatch(gpointer user_data) { int rc; cib_t *cib = user_data; cib_remote_opaque_t *private = cib->variant_opaque; xmlNode *msg = NULL; crm_info("Message on callback channel"); rc = pcmk__read_remote_message(&private->callback, -1); msg = pcmk__remote_message_xml(&private->callback); while (msg) { const char *type = crm_element_value(msg, PCMK__XA_T); crm_trace("Activating %s callbacks...", type); if (pcmk__str_eq(type, PCMK__VALUE_CIB, pcmk__str_none)) { cib_native_callback(cib, msg, 0, 0); - } else if (pcmk__str_eq(type, T_CIB_NOTIFY, pcmk__str_casei)) { + } else if (pcmk__str_eq(type, PCMK__VALUE_CIB_NOTIFY, pcmk__str_none)) { g_list_foreach(cib->notify_list, cib_native_notify, msg); } else { crm_err("Unknown message type: %s", type); } free_xml(msg); msg = pcmk__remote_message_xml(&private->callback); } if (rc == ENOTCONN) { return -1; } return 0; } static int cib_remote_command_dispatch(gpointer user_data) { int rc; cib_t *cib = user_data; cib_remote_opaque_t *private = cib->variant_opaque; rc = pcmk__read_remote_message(&private->command, -1); free(private->command.buffer); private->command.buffer = NULL; crm_err("received late reply for remote cib connection, discarding"); if (rc == ENOTCONN) { return -1; } return 0; } static int cib_tls_close(cib_t *cib) { cib_remote_opaque_t *private = cib->variant_opaque; #ifdef HAVE_GNUTLS_GNUTLS_H if (private->encrypted) { if (private->command.tls_session) { gnutls_bye(*(private->command.tls_session), GNUTLS_SHUT_RDWR); gnutls_deinit(*(private->command.tls_session)); gnutls_free(private->command.tls_session); } if (private->callback.tls_session) { gnutls_bye(*(private->callback.tls_session), GNUTLS_SHUT_RDWR); gnutls_deinit(*(private->callback.tls_session)); gnutls_free(private->callback.tls_session); } private->command.tls_session = NULL; private->callback.tls_session = NULL; if (remote_gnutls_credentials_init) { gnutls_anon_free_client_credentials(anon_cred_c); gnutls_global_deinit(); remote_gnutls_credentials_init = FALSE; } } #endif if (private->command.tcp_socket) { shutdown(private->command.tcp_socket, SHUT_RDWR); /* no more receptions */ close(private->command.tcp_socket); } if (private->callback.tcp_socket) { shutdown(private->callback.tcp_socket, SHUT_RDWR); /* no more receptions */ close(private->callback.tcp_socket); } private->command.tcp_socket = 0; private->callback.tcp_socket = 0; free(private->command.buffer); free(private->callback.buffer); private->command.buffer = NULL; private->callback.buffer = NULL; return 0; } static void cib_remote_connection_destroy(gpointer user_data) { crm_err("Connection destroyed"); #ifdef HAVE_GNUTLS_GNUTLS_H cib_tls_close(user_data); #endif } static int cib_tls_signon(cib_t *cib, pcmk__remote_t *connection, gboolean event_channel) { cib_remote_opaque_t *private = cib->variant_opaque; int rc; xmlNode *answer = NULL; xmlNode *login = NULL; static struct mainloop_fd_callbacks cib_fd_callbacks = { 0, }; cib_fd_callbacks.dispatch = event_channel ? cib_remote_callback_dispatch : cib_remote_command_dispatch; cib_fd_callbacks.destroy = cib_remote_connection_destroy; connection->tcp_socket = -1; #ifdef HAVE_GNUTLS_GNUTLS_H connection->tls_session = NULL; #endif rc = pcmk__connect_remote(private->server, private->port, 0, NULL, &(connection->tcp_socket), NULL, NULL); if (rc != pcmk_rc_ok) { crm_info("Remote connection to %s:%d failed: %s " CRM_XS " rc=%d", private->server, private->port, pcmk_rc_str(rc), rc); return -ENOTCONN; } if (private->encrypted) { /* initialize GnuTls lib */ #ifdef HAVE_GNUTLS_GNUTLS_H if (remote_gnutls_credentials_init == FALSE) { crm_gnutls_global_init(); gnutls_anon_allocate_client_credentials(&anon_cred_c); remote_gnutls_credentials_init = TRUE; } /* bind the socket to GnuTls lib */ connection->tls_session = pcmk__new_tls_session(connection->tcp_socket, GNUTLS_CLIENT, GNUTLS_CRD_ANON, anon_cred_c); if (connection->tls_session == NULL) { cib_tls_close(cib); return -1; } if (pcmk__tls_client_handshake(connection, TLS_HANDSHAKE_TIMEOUT_MS) != pcmk_rc_ok) { crm_err("Session creation for %s:%d failed", private->server, private->port); gnutls_deinit(*connection->tls_session); gnutls_free(connection->tls_session); connection->tls_session = NULL; cib_tls_close(cib); return -1; } #else return -EPROTONOSUPPORT; #endif } /* login to server */ login = create_xml_node(NULL, PCMK__XE_CIB_COMMAND); crm_xml_add(login, PCMK_XA_OP, "authenticate"); crm_xml_add(login, PCMK_XA_USER, private->user); crm_xml_add(login, PCMK__XA_PASSWORD, private->passwd); crm_xml_add(login, PCMK__XA_HIDDEN, PCMK__VALUE_PASSWORD); pcmk__remote_send_xml(connection, login); free_xml(login); rc = pcmk_ok; if (pcmk__read_remote_message(connection, -1) == ENOTCONN) { rc = -ENOTCONN; } answer = pcmk__remote_message_xml(connection); crm_log_xml_trace(answer, "Reply"); if (answer == NULL) { rc = -EPROTO; } else { /* grab the token */ const char *msg_type = crm_element_value(answer, PCMK__XA_CIB_OP); const char *tmp_ticket = crm_element_value(answer, PCMK__XA_CIB_CLIENTID); if (!pcmk__str_eq(msg_type, CRM_OP_REGISTER, pcmk__str_casei)) { crm_err("Invalid registration message: %s", msg_type); rc = -EPROTO; } else if (tmp_ticket == NULL) { rc = -EPROTO; } else { connection->token = strdup(tmp_ticket); } } free_xml(answer); answer = NULL; if (rc != 0) { cib_tls_close(cib); return rc; } crm_trace("remote client connection established"); connection->source = mainloop_add_fd("cib-remote", G_PRIORITY_HIGH, connection->tcp_socket, cib, &cib_fd_callbacks); return rc; } static int cib_remote_signon(cib_t *cib, const char *name, enum cib_conn_type type) { int rc = pcmk_ok; cib_remote_opaque_t *private = cib->variant_opaque; xmlNode *hello = NULL; if (private->passwd == NULL) { if (private->out == NULL) { /* If no pcmk__output_t is set, just assume that a text prompt * is good enough. */ pcmk__text_prompt("Password", false, &(private->passwd)); } else { private->out->prompt("Password", false, &(private->passwd)); } } if (private->server == NULL || private->user == NULL) { rc = -EINVAL; } if (rc == pcmk_ok) { rc = cib_tls_signon(cib, &(private->command), FALSE); } if (rc == pcmk_ok) { rc = cib_tls_signon(cib, &(private->callback), TRUE); } if (rc == pcmk_ok) { rc = cib__create_op(cib, CRM_OP_REGISTER, NULL, NULL, NULL, cib_none, NULL, name, &hello); } if (rc == pcmk_ok) { rc = pcmk__remote_send_xml(&private->command, hello); rc = pcmk_rc2legacy(rc); free_xml(hello); } if (rc == pcmk_ok) { crm_info("Opened connection to %s:%d for %s", private->server, private->port, name); cib->state = cib_connected_command; cib->type = cib_command; } else { crm_info("Connection to %s:%d for %s failed: %s\n", private->server, private->port, name, pcmk_strerror(rc)); } return rc; } static int cib_remote_signoff(cib_t *cib) { int rc = pcmk_ok; crm_debug("Disconnecting from the CIB manager"); #ifdef HAVE_GNUTLS_GNUTLS_H cib_tls_close(cib); #endif cib->cmds->end_transaction(cib, false, cib_none); cib->state = cib_disconnected; cib->type = cib_no_connection; return rc; } static int cib_remote_free(cib_t *cib) { int rc = pcmk_ok; crm_warn("Freeing CIB"); if (cib->state != cib_disconnected) { rc = cib_remote_signoff(cib); if (rc == pcmk_ok) { cib_remote_opaque_t *private = cib->variant_opaque; free(private->server); free(private->user); free(private->passwd); free(cib->cmds); free(cib->user); free(private); free(cib); } } return rc; } static int cib_remote_inputfd(cib_t * cib) { cib_remote_opaque_t *private = cib->variant_opaque; return private->callback.tcp_socket; } static int cib_remote_register_notification(cib_t * cib, const char *callback, int enabled) { xmlNode *notify_msg = create_xml_node(NULL, PCMK__XE_CIB_COMMAND); cib_remote_opaque_t *private = cib->variant_opaque; - crm_xml_add(notify_msg, PCMK__XA_CIB_OP, T_CIB_NOTIFY); + crm_xml_add(notify_msg, PCMK__XA_CIB_OP, PCMK__VALUE_CIB_NOTIFY); crm_xml_add(notify_msg, PCMK__XA_CIB_NOTIFY_TYPE, callback); crm_xml_add_int(notify_msg, PCMK__XA_CIB_NOTIFY_ACTIVATE, enabled); pcmk__remote_send_xml(&private->callback, notify_msg); free_xml(notify_msg); return pcmk_ok; } static int cib_remote_set_connection_dnotify(cib_t * cib, void (*dnotify) (gpointer user_data)) { return -EPROTONOSUPPORT; } /*! * \internal * \brief Get the given CIB connection's unique client identifiers * * These can be used to check whether this client requested the action that * triggered a CIB notification. * * \param[in] cib CIB connection * \param[out] async_id If not \p NULL, where to store asynchronous client ID * \param[out] sync_id If not \p NULL, where to store synchronous client ID * * \return Legacy Pacemaker return code (specifically, \p pcmk_ok) * * \note This is the \p cib_remote variant implementation of * \p cib_api_operations_t:client_id(). * \note The client IDs are assigned during CIB sign-on. */ static int cib_remote_client_id(const cib_t *cib, const char **async_id, const char **sync_id) { cib_remote_opaque_t *private = cib->variant_opaque; if (async_id != NULL) { // private->callback is the channel for async requests *async_id = private->callback.token; } if (sync_id != NULL) { // private->command is the channel for sync requests *sync_id = private->command.token; } return pcmk_ok; } cib_t * cib_remote_new(const char *server, const char *user, const char *passwd, int port, gboolean encrypted) { cib_remote_opaque_t *private = NULL; cib_t *cib = cib_new_variant(); if (cib == NULL) { return NULL; } private = calloc(1, sizeof(cib_remote_opaque_t)); if (private == NULL) { free(cib); return NULL; } cib->variant = cib_remote; cib->variant_opaque = private; pcmk__str_update(&private->server, server); pcmk__str_update(&private->user, user); pcmk__str_update(&private->passwd, passwd); private->port = port; private->encrypted = encrypted; /* assign variant specific ops */ cib->delegate_fn = cib_remote_perform_op; cib->cmds->signon = cib_remote_signon; cib->cmds->signoff = cib_remote_signoff; cib->cmds->free = cib_remote_free; cib->cmds->inputfd = cib_remote_inputfd; // Deprecated method cib->cmds->register_notification = cib_remote_register_notification; cib->cmds->set_connection_dnotify = cib_remote_set_connection_dnotify; cib->cmds->client_id = cib_remote_client_id; return cib; } void cib__set_output(cib_t *cib, pcmk__output_t *out) { cib_remote_opaque_t *private; if (cib->variant != cib_remote) { return; } private = cib->variant_opaque; private->out = out; }