diff --git a/daemons/attrd/attrd_messages.c b/daemons/attrd/attrd_messages.c index 4e8a3de0ee..7d146a9389 100644 --- a/daemons/attrd/attrd_messages.c +++ b/daemons/attrd/attrd_messages.c @@ -1,352 +1,352 @@ /* * Copyright 2022-2025 the Pacemaker project contributors * * The version control history for this file may have further details. * * This source code is licensed under the GNU General Public License version 2 * or later (GPLv2+) WITHOUT ANY WARRANTY. */ #include #include // PRIu32 #include #include #include // pcmk__get_node() #include #include "pacemaker-attrd.h" int minimum_protocol_version = -1; static GHashTable *attrd_handlers = NULL; static bool is_sync_point_attr(xmlAttrPtr attr, void *data) { return pcmk__str_eq((const char *) attr->name, PCMK__XA_ATTR_SYNC_POINT, pcmk__str_none); } static int remove_sync_point_attribute(xmlNode *xml, void *data) { pcmk__xe_remove_matching_attrs(xml, false, is_sync_point_attr, NULL); pcmk__xe_foreach_child(xml, PCMK_XE_OP, remove_sync_point_attribute, NULL); return pcmk_rc_ok; } /* Sync points on a multi-update IPC message to an attrd too old to support * multi-update messages won't work. Strip the sync point attribute off here * so we don't pretend to support this situation and instead ACK the client * immediately. */ static void remove_unsupported_sync_points(pcmk__request_t *request) { if (request->xml->children != NULL && !ATTRD_SUPPORTS_MULTI_MESSAGE(minimum_protocol_version) && attrd_request_has_sync_point(request->xml)) { pcmk__warn("Ignoring sync point in request from %s because not all " "nodes support it", pcmk__request_origin(request)); remove_sync_point_attribute(request->xml, NULL); } } static xmlNode * handle_unknown_request(pcmk__request_t *request) { pcmk__err("Unknown IPC request %s from %s %s", request->op, pcmk__request_origin_type(request), pcmk__request_origin(request)); pcmk__format_result(&request->result, CRM_EX_PROTOCOL, PCMK_EXEC_INVALID, "Unknown request type '%s' (bug?)", request->op); return NULL; } static xmlNode * handle_clear_failure_request(pcmk__request_t *request) { if (request->peer != NULL) { /* It is not currently possible to receive this as a peer command, * but will be, if we one day enable propagating this operation. */ attrd_peer_clear_failure(request); pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL); return NULL; } else { remove_unsupported_sync_points(request); if (attrd_request_has_sync_point(request->xml)) { /* If this client supplied a sync point it wants to wait for, add it to * the wait list. Clients on this list will not receive an ACK until * their sync point is hit which will result in the client stalled there * until it receives a response. * * All other clients will receive the expected response as normal. */ attrd_add_client_to_waitlist(request); } else { /* If the client doesn't want to wait for a sync point, go ahead and send * the ACK immediately. Otherwise, we'll send the ACK when the appropriate * sync point is reached. */ attrd_send_ack(request->ipc_client, request->ipc_id, request->ipc_flags); } return attrd_client_clear_failure(request); } } static xmlNode * handle_confirm_request(pcmk__request_t *request) { if (request->peer != NULL) { int callid; pcmk__debug("Received confirmation from %s", request->peer); if (pcmk__xe_get_int(request->xml, PCMK__XA_CALL_ID, &callid) != pcmk_rc_ok) { pcmk__set_result(&request->result, CRM_EX_PROTOCOL, PCMK_EXEC_INVALID, "Could not get callid from XML"); } else { attrd_handle_confirmation(callid, request->peer); } pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL); return NULL; } else { return handle_unknown_request(request); } } static xmlNode * handle_query_request(pcmk__request_t *request) { if (request->peer != NULL) { return handle_unknown_request(request); } else { return attrd_client_query(request); } } static xmlNode * handle_remove_request(pcmk__request_t *request) { if (request->peer != NULL) { const char *host = pcmk__xe_get(request->xml, PCMK__XA_ATTR_HOST); bool reap = false; if (pcmk__xe_get_bool_attr(request->xml, PCMK__XA_REAP, &reap) != pcmk_rc_ok) { reap = true; // Default to true for backward compatibility } attrd_peer_remove(host, reap, request->peer); pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL); return NULL; } else { return attrd_client_peer_remove(request); } } static xmlNode * handle_refresh_request(pcmk__request_t *request) { if (request->peer != NULL) { return handle_unknown_request(request); } else { return attrd_client_refresh(request); } } static xmlNode * handle_sync_response_request(pcmk__request_t *request) { if (request->ipc_client != NULL) { return handle_unknown_request(request); } else { if (request->peer != NULL) { pcmk__node_status_t *peer = pcmk__get_node(0, request->peer, NULL, pcmk__node_search_cluster_member); bool peer_won = attrd_check_for_new_writer(peer, request->xml); if (!pcmk__str_eq(peer->name, attrd_cluster->priv->node_name, pcmk__str_casei)) { attrd_peer_sync_response(peer, peer_won, request->xml); } } pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL); return NULL; } } static xmlNode * handle_update_request(pcmk__request_t *request) { if (request->peer != NULL) { const char *host = pcmk__xe_get(request->xml, PCMK__XA_ATTR_HOST); pcmk__node_status_t *peer = pcmk__get_node(0, request->peer, NULL, pcmk__node_search_cluster_member); attrd_peer_update(peer, request->xml, host, false); pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL); return NULL; } else { remove_unsupported_sync_points(request); if (attrd_request_has_sync_point(request->xml)) { /* If this client supplied a sync point it wants to wait for, add it to * the wait list. Clients on this list will not receive an ACK until * their sync point is hit which will result in the client stalled there * until it receives a response. * * All other clients will receive the expected response as normal. */ attrd_add_client_to_waitlist(request); } else { /* If the client doesn't want to wait for a sync point, go ahead and send * the ACK immediately. Otherwise, we'll send the ACK when the appropriate * sync point is reached. * * In the normal case, attrd_client_update can be called recursively which * makes where to send the ACK tricky. Doing it here ensures the client * only ever receives one. */ attrd_send_ack(request->ipc_client, request->ipc_id, request->flags|crm_ipc_client_response); } return attrd_client_update(request); } } static void attrd_register_handlers(void) { pcmk__server_command_t handlers[] = { { PCMK__ATTRD_CMD_CLEAR_FAILURE, handle_clear_failure_request }, { PCMK__ATTRD_CMD_CONFIRM, handle_confirm_request }, { PCMK__ATTRD_CMD_PEER_REMOVE, handle_remove_request }, { PCMK__ATTRD_CMD_QUERY, handle_query_request }, { PCMK__ATTRD_CMD_REFRESH, handle_refresh_request }, { PCMK__ATTRD_CMD_SYNC_RESPONSE, handle_sync_response_request }, { PCMK__ATTRD_CMD_UPDATE, handle_update_request }, { PCMK__ATTRD_CMD_UPDATE_DELAY, handle_update_request }, { PCMK__ATTRD_CMD_UPDATE_BOTH, handle_update_request }, { NULL, handle_unknown_request }, }; attrd_handlers = pcmk__register_handlers(handlers); } void attrd_unregister_handlers(void) { if (attrd_handlers != NULL) { g_hash_table_destroy(attrd_handlers); attrd_handlers = NULL; } } void attrd_handle_request(pcmk__request_t *request) { xmlNode *reply = NULL; char *log_msg = NULL; const char *exec_status_s = NULL; const char *reason = NULL; if (attrd_handlers == NULL) { attrd_register_handlers(); } reply = pcmk__process_request(request, attrd_handlers); if (reply != NULL) { - crm_log_xml_trace(reply, "Reply"); + pcmk__log_xml_trace(reply, "Reply"); if (request->ipc_client != NULL) { pcmk__ipc_send_xml(request->ipc_client, request->ipc_id, reply, request->ipc_flags); } else { pcmk__err("Not sending CPG reply to client"); } pcmk__xml_free(reply); } exec_status_s = pcmk_exec_status_str(request->result.execution_status); reason = request->result.exit_reason; log_msg = pcmk__assert_asprintf("Processed %s request from %s %s: %s%s%s%s", request->op, pcmk__request_origin_type(request), pcmk__request_origin(request), exec_status_s, (reason == NULL)? "" : " (", pcmk__s(reason, ""), (reason == NULL)? "" : ")"); if (!pcmk__result_ok(&request->result)) { pcmk__warn("%s", log_msg); } else { pcmk__debug("%s", log_msg); } free(log_msg); pcmk__reset_request(request); } /*! \internal \brief Broadcast private attribute for local node with protocol version */ void attrd_broadcast_protocol(void) { xmlNode *attrd_op = pcmk__xe_create(NULL, __func__); pcmk__xe_set(attrd_op, PCMK__XA_T, PCMK__VALUE_ATTRD); pcmk__xe_set(attrd_op, PCMK__XA_SRC, crm_system_name); pcmk__xe_set(attrd_op, PCMK_XA_TASK, PCMK__ATTRD_CMD_UPDATE); pcmk__xe_set(attrd_op, PCMK__XA_ATTR_NAME, CRM_ATTR_PROTOCOL); pcmk__xe_set(attrd_op, PCMK__XA_ATTR_VALUE, ATTRD_PROTOCOL_VERSION); pcmk__xe_set_int(attrd_op, PCMK__XA_ATTR_IS_PRIVATE, 1); pcmk__xe_set(attrd_op, PCMK__XA_ATTR_HOST, attrd_cluster->priv->node_name); pcmk__xe_set(attrd_op, PCMK__XA_ATTR_HOST_ID, attrd_cluster->priv->node_xml_id); pcmk__debug("Broadcasting attrd protocol version %s for node %s", ATTRD_PROTOCOL_VERSION, attrd_cluster->priv->node_name); attrd_send_message(NULL, attrd_op, false); /* ends up at attrd_peer_message() */ pcmk__xml_free(attrd_op); } gboolean attrd_send_message(pcmk__node_status_t *node, xmlNode *data, bool confirm) { const char *op = pcmk__xe_get(data, PCMK_XA_TASK); pcmk__xe_set(data, PCMK__XA_T, PCMK__VALUE_ATTRD); pcmk__xe_set(data, PCMK__XA_ATTR_VERSION, ATTRD_PROTOCOL_VERSION); /* Request a confirmation from the destination peer node (which could * be all if node is NULL) that the message has been received and * acted upon. */ if (!pcmk__str_eq(op, PCMK__ATTRD_CMD_CONFIRM, pcmk__str_none)) { pcmk__xe_set_bool_attr(data, PCMK__XA_CONFIRM, confirm); } attrd_xml_add_writer(data); return pcmk__cluster_send_message(node, pcmk_ipc_attrd, data); } diff --git a/daemons/based/based_callbacks.c b/daemons/based/based_callbacks.c index fea6b91816..4d2df2a371 100644 --- a/daemons/based/based_callbacks.c +++ b/daemons/based/based_callbacks.c @@ -1,1407 +1,1407 @@ /* * Copyright 2004-2025 the Pacemaker project contributors * * The version control history for this file may have further details. * * This source code is licensed under the GNU General Public License version 2 * or later (GPLv2+) WITHOUT ANY WARRANTY. */ #include #include #include #include #include #include #include // uint32_t, uint64_t, UINT64_C() #include #include #include // PRIu64 #include #include #include // xmlXPathObject, etc. #include #include #include #include #include #include #define EXIT_ESCALATION_MS 10000 qb_ipcs_service_t *ipcs_ro = NULL; qb_ipcs_service_t *ipcs_rw = NULL; qb_ipcs_service_t *ipcs_shm = NULL; static int cib_process_command(xmlNode *request, const cib__operation_t *operation, cib__op_fn_t op_function, xmlNode **reply, xmlNode **cib_diff, bool privileged); static gboolean cib_common_callback(qb_ipcs_connection_t *c, void *data, size_t size, gboolean privileged); static int32_t cib_ipc_accept(qb_ipcs_connection_t * c, uid_t uid, gid_t gid) { if (cib_shutdown_flag) { pcmk__info("Ignoring new IPC client [%d] during shutdown", pcmk__client_pid(c)); return -ECONNREFUSED; } if (pcmk__new_client(c, uid, gid) == NULL) { return -ENOMEM; } return 0; } static int32_t cib_ipc_dispatch_rw(qb_ipcs_connection_t * c, void *data, size_t size) { pcmk__client_t *client = pcmk__find_client(c); pcmk__trace("%p message from %s", c, client->id); return cib_common_callback(c, data, size, TRUE); } static int32_t cib_ipc_dispatch_ro(qb_ipcs_connection_t * c, void *data, size_t size) { pcmk__client_t *client = pcmk__find_client(c); pcmk__trace("%p message from %s", c, client->id); return cib_common_callback(c, data, size, FALSE); } /* Error code means? */ static int32_t cib_ipc_closed(qb_ipcs_connection_t * c) { pcmk__client_t *client = pcmk__find_client(c); if (client == NULL) { return 0; } pcmk__trace("Connection %p", c); pcmk__free_client(client); return 0; } static void cib_ipc_destroy(qb_ipcs_connection_t * c) { pcmk__trace("Connection %p", c); cib_ipc_closed(c); if (cib_shutdown_flag) { cib_shutdown(0); } } struct qb_ipcs_service_handlers ipc_ro_callbacks = { .connection_accept = cib_ipc_accept, .connection_created = NULL, .msg_process = cib_ipc_dispatch_ro, .connection_closed = cib_ipc_closed, .connection_destroyed = cib_ipc_destroy }; struct qb_ipcs_service_handlers ipc_rw_callbacks = { .connection_accept = cib_ipc_accept, .connection_created = NULL, .msg_process = cib_ipc_dispatch_rw, .connection_closed = cib_ipc_closed, .connection_destroyed = cib_ipc_destroy }; /*! * \internal * \brief Create reply XML for a CIB request * * \param[in] op CIB operation type * \param[in] call_id CIB call ID * \param[in] client_id CIB client ID * \param[in] call_options Group of enum cib_call_options flags * \param[in] rc Request return code * \param[in] call_data Request output data * * \return Reply XML (guaranteed not to be \c NULL) * * \note The caller is responsible for freeing the return value using * \p pcmk__xml_free(). */ static xmlNode * create_cib_reply(const char *op, const char *call_id, const char *client_id, uint32_t call_options, int rc, xmlNode *call_data) { xmlNode *reply = pcmk__xe_create(NULL, PCMK__XE_CIB_REPLY); pcmk__xe_set(reply, PCMK__XA_T, PCMK__VALUE_CIB); pcmk__xe_set(reply, PCMK__XA_CIB_OP, op); pcmk__xe_set(reply, PCMK__XA_CIB_CALLID, call_id); pcmk__xe_set(reply, PCMK__XA_CIB_CLIENTID, client_id); pcmk__xe_set_int(reply, PCMK__XA_CIB_CALLOPT, call_options); pcmk__xe_set_int(reply, PCMK__XA_CIB_RC, rc); if (call_data != NULL) { xmlNode *wrapper = pcmk__xe_create(reply, PCMK__XE_CIB_CALLDATA); pcmk__trace("Attaching reply output"); pcmk__xml_copy(wrapper, call_data); } crm_log_xml_explicit(reply, "cib:reply"); return reply; } static void do_local_notify(const xmlNode *notify_src, const char *client_id, bool sync_reply, bool from_peer) { int msg_id = 0; int rc = pcmk_rc_ok; pcmk__client_t *client_obj = NULL; uint32_t flags = crm_ipc_server_event; CRM_CHECK((notify_src != NULL) && (client_id != NULL), return); pcmk__xe_get_int(notify_src, PCMK__XA_CIB_CALLID, &msg_id); client_obj = pcmk__find_client_by_id(client_id); if (client_obj == NULL) { pcmk__debug("Could not notify client %s%s %s of call %d result: " "client no longer exists", client_id, (from_peer? " (originator of delegated request)" : ""), (sync_reply? "synchronously" : "asynchronously"), msg_id); return; } if (sync_reply) { flags = crm_ipc_flags_none; if (client_obj->ipcs != NULL) { msg_id = client_obj->request_id; client_obj->request_id = 0; } } switch (PCMK__CLIENT_TYPE(client_obj)) { case pcmk__client_ipc: rc = pcmk__ipc_send_xml(client_obj, msg_id, notify_src, flags); break; case pcmk__client_tls: case pcmk__client_tcp: rc = pcmk__remote_send_xml(client_obj->remote, notify_src); break; default: rc = EPROTONOSUPPORT; break; } if (rc == pcmk_rc_ok) { pcmk__trace("Notified %s client %s%s %s of call %d result", pcmk__client_type_str(PCMK__CLIENT_TYPE(client_obj)), pcmk__client_name(client_obj), (from_peer? " (originator of delegated request)" : ""), (sync_reply? "synchronously" : "asynchronously"), msg_id); } else { pcmk__warn("Could not notify %s client %s%s %s of call %d result: %s", pcmk__client_type_str(PCMK__CLIENT_TYPE(client_obj)), pcmk__client_name(client_obj), (from_peer? " (originator of delegated request)" : ""), (sync_reply? "synchronously" : "asynchronously"), msg_id, pcmk_rc_str(rc)); } } void cib_common_callback_worker(uint32_t id, uint32_t flags, xmlNode * op_request, pcmk__client_t *cib_client, gboolean privileged) { const char *op = pcmk__xe_get(op_request, PCMK__XA_CIB_OP); uint32_t call_options = cib_none; int rc = pcmk_rc_ok; rc = pcmk__xe_get_flags(op_request, PCMK__XA_CIB_CALLOPT, &call_options, cib_none); if (rc != pcmk_rc_ok) { pcmk__warn("Couldn't parse options from request: %s", pcmk_rc_str(rc)); } /* Requests with cib_transaction set should not be sent to based directly * (outside of a commit-transaction request) */ if (pcmk__is_set(call_options, cib_transaction)) { return; } if (pcmk__str_eq(op, CRM_OP_REGISTER, pcmk__str_none)) { if (flags & crm_ipc_client_response) { xmlNode *ack = pcmk__xe_create(NULL, __func__); pcmk__xe_set(ack, PCMK__XA_CIB_OP, CRM_OP_REGISTER); pcmk__xe_set(ack, PCMK__XA_CIB_CLIENTID, cib_client->id); pcmk__ipc_send_xml(cib_client, id, ack, flags); cib_client->request_id = 0; pcmk__xml_free(ack); } return; } else if (pcmk__str_eq(op, PCMK__VALUE_CIB_NOTIFY, pcmk__str_none)) { /* Update the notify filters for this client */ int on_off = 0; crm_exit_t status = CRM_EX_OK; uint64_t bit = UINT64_C(0); const char *type = pcmk__xe_get(op_request, PCMK__XA_CIB_NOTIFY_TYPE); pcmk__xe_get_int(op_request, PCMK__XA_CIB_NOTIFY_ACTIVATE, &on_off); pcmk__debug("Setting %s callbacks %s for client %s", type, (on_off? "on" : "off"), pcmk__client_name(cib_client)); if (pcmk__str_eq(type, PCMK__VALUE_CIB_POST_NOTIFY, pcmk__str_none)) { bit = cib_notify_post; } else if (pcmk__str_eq(type, PCMK__VALUE_CIB_PRE_NOTIFY, pcmk__str_none)) { bit = cib_notify_pre; } else if (pcmk__str_eq(type, PCMK__VALUE_CIB_UPDATE_CONFIRMATION, pcmk__str_none)) { bit = cib_notify_confirm; } else if (pcmk__str_eq(type, PCMK__VALUE_CIB_DIFF_NOTIFY, pcmk__str_none)) { bit = cib_notify_diff; } else { status = CRM_EX_INVALID_PARAM; } if (bit != 0) { if (on_off) { pcmk__set_client_flags(cib_client, bit); } else { pcmk__clear_client_flags(cib_client, bit); } } pcmk__ipc_send_ack(cib_client, id, flags, PCMK__XE_ACK, NULL, status); return; } cib_process_request(op_request, privileged, cib_client); } int32_t cib_common_callback(qb_ipcs_connection_t * c, void *data, size_t size, gboolean privileged) { uint32_t id = 0; uint32_t flags = 0; uint32_t call_options = cib_none; pcmk__client_t *cib_client = pcmk__find_client(c); xmlNode *op_request = pcmk__client_data2xml(cib_client, data, &id, &flags); if (op_request) { int rc = pcmk_rc_ok; rc = pcmk__xe_get_flags(op_request, PCMK__XA_CIB_CALLOPT, &call_options, cib_none); if (rc != pcmk_rc_ok) { pcmk__warn("Couldn't parse options from request: %s", pcmk_rc_str(rc)); } } if (op_request == NULL) { pcmk__trace("Invalid message from %p", c); pcmk__ipc_send_ack(cib_client, id, flags, PCMK__XE_NACK, NULL, CRM_EX_PROTOCOL); return 0; } else if(cib_client == NULL) { pcmk__trace("Invalid client %p", c); return 0; } if (pcmk__is_set(call_options, cib_sync_call)) { CRM_LOG_ASSERT(flags & crm_ipc_client_response); CRM_LOG_ASSERT(cib_client->request_id == 0); /* This means the client has two synchronous events in-flight */ cib_client->request_id = id; /* Reply only to the last one */ } if (cib_client->name == NULL) { const char *value = pcmk__xe_get(op_request, PCMK__XA_CIB_CLIENTNAME); if (value == NULL) { cib_client->name = pcmk__itoa(cib_client->pid); } else { cib_client->name = pcmk__str_copy(value); if (pcmk__parse_server(value) != pcmk_ipc_unknown) { pcmk__set_client_flags(cib_client, cib_is_daemon); } } } /* Allow cluster daemons more leeway before being evicted */ if (pcmk__is_set(cib_client->flags, cib_is_daemon)) { const char *qmax = cib_config_lookup(PCMK_OPT_CLUSTER_IPC_LIMIT); pcmk__set_client_queue_max(cib_client, qmax); } pcmk__xe_set(op_request, PCMK__XA_CIB_CLIENTID, cib_client->id); pcmk__xe_set(op_request, PCMK__XA_CIB_CLIENTNAME, cib_client->name); CRM_LOG_ASSERT(cib_client->user != NULL); pcmk__update_acl_user(op_request, PCMK__XA_CIB_USER, cib_client->user); cib_common_callback_worker(id, flags, op_request, cib_client, privileged); pcmk__xml_free(op_request); return 0; } static uint64_t ping_seq = 0; static char *ping_digest = NULL; static bool ping_modified_since = FALSE; static gboolean cib_digester_cb(gpointer data) { if (based_is_primary) { char buffer[32]; xmlNode *ping = pcmk__xe_create(NULL, PCMK__XE_PING); ping_seq++; free(ping_digest); ping_digest = NULL; ping_modified_since = FALSE; snprintf(buffer, 32, "%" PRIu64, ping_seq); pcmk__trace("Requesting peer digests (%s)", buffer); pcmk__xe_set(ping, PCMK__XA_T, PCMK__VALUE_CIB); pcmk__xe_set(ping, PCMK__XA_CIB_OP, CRM_OP_PING); pcmk__xe_set(ping, PCMK__XA_CIB_PING_ID, buffer); pcmk__xe_set(ping, PCMK_XA_CRM_FEATURE_SET, CRM_FEATURE_SET); pcmk__cluster_send_message(NULL, pcmk_ipc_based, ping); pcmk__xml_free(ping); } return FALSE; } static void process_ping_reply(xmlNode *reply) { uint64_t seq = 0; const char *host = pcmk__xe_get(reply, PCMK__XA_SRC); xmlNode *wrapper = pcmk__xe_first_child(reply, PCMK__XE_CIB_CALLDATA, NULL, NULL); xmlNode *pong = pcmk__xe_first_child(wrapper, NULL, NULL, NULL); const char *seq_s = pcmk__xe_get(pong, PCMK__XA_CIB_PING_ID); const char *digest = pcmk__xe_get(pong, PCMK__XA_DIGEST); if (seq_s == NULL) { pcmk__debug("Ignoring ping reply with no " PCMK__XA_CIB_PING_ID); return; } else { long long seq_ll; int rc = pcmk__scan_ll(seq_s, &seq_ll, 0LL); if (rc != pcmk_rc_ok) { pcmk__debug("Ignoring ping reply with invalid " PCMK__XA_CIB_PING_ID " '%s': %s", seq_s, pcmk_rc_str(rc)); return; } seq = (uint64_t) seq_ll; } if(digest == NULL) { pcmk__trace("Ignoring ping reply %s from %s with no digest", seq_s, host); } else if(seq != ping_seq) { pcmk__trace("Ignoring out of sequence ping reply %s from %s", seq_s, host); } else if(ping_modified_since) { pcmk__trace("Ignoring ping reply %s from %s: cib updated since", seq_s, host); } else { if(ping_digest == NULL) { pcmk__trace("Calculating new digest"); ping_digest = pcmk__digest_xml(the_cib, true); } pcmk__trace("Processing ping reply %s from %s (%s)", seq_s, host, digest); if (!pcmk__str_eq(ping_digest, digest, pcmk__str_casei)) { xmlNode *wrapper = pcmk__xe_first_child(pong, PCMK__XE_CIB_CALLDATA, NULL, NULL); xmlNode *remote_cib = pcmk__xe_first_child(wrapper, NULL, NULL, NULL); const char *admin_epoch_s = NULL; const char *epoch_s = NULL; const char *num_updates_s = NULL; if (remote_cib != NULL) { admin_epoch_s = pcmk__xe_get(remote_cib, PCMK_XA_ADMIN_EPOCH); epoch_s = pcmk__xe_get(remote_cib, PCMK_XA_EPOCH); num_updates_s = pcmk__xe_get(remote_cib, PCMK_XA_NUM_UPDATES); } pcmk__notice("Local CIB %s.%s.%s.%s differs from %s: %s.%s.%s.%s " "%p", pcmk__xe_get(the_cib, PCMK_XA_ADMIN_EPOCH), pcmk__xe_get(the_cib, PCMK_XA_EPOCH), pcmk__xe_get(the_cib, PCMK_XA_NUM_UPDATES), ping_digest, host, pcmk__s(admin_epoch_s, "_"), pcmk__s(epoch_s, "_"), pcmk__s(num_updates_s, "_"), digest, remote_cib); if(remote_cib && remote_cib->children) { // Additional debug pcmk__xml_mark_changes(the_cib, remote_cib); pcmk__log_xml_changes(LOG_INFO, remote_cib); pcmk__trace("End of differences"); } pcmk__xml_free(remote_cib); sync_our_cib(reply, FALSE); } } } static void parse_local_options(const pcmk__client_t *cib_client, const cib__operation_t *operation, const char *host, const char *op, gboolean *local_notify, gboolean *needs_reply, gboolean *process, gboolean *needs_forward) { // Process locally and notify local client *process = TRUE; *needs_reply = FALSE; *local_notify = TRUE; *needs_forward = FALSE; if (pcmk__is_set(operation->flags, cib__op_attr_local)) { /* Always process locally if cib__op_attr_local is set. * * @COMPAT: Currently host is ignored. At a compatibility break, throw * an error (from cib_process_request() or earlier) if host is not NULL or * OUR_NODENAME. */ pcmk__trace("Processing always-local %s op from client %s", op, pcmk__client_name(cib_client)); if (!pcmk__str_eq(host, OUR_NODENAME, pcmk__str_casei|pcmk__str_null_matches)) { pcmk__warn("Operation '%s' is always local but its target host is " "set to '%s'", op, host); } return; } if (pcmk__is_set(operation->flags, cib__op_attr_modifies) || !pcmk__str_eq(host, OUR_NODENAME, pcmk__str_casei|pcmk__str_null_matches)) { // Forward modifying and non-local requests via cluster *process = FALSE; *needs_reply = FALSE; *local_notify = FALSE; *needs_forward = TRUE; pcmk__trace("%s op from %s needs to be forwarded to %s", op, pcmk__client_name(cib_client), pcmk__s(host, "all nodes")); return; } if (stand_alone) { pcmk__trace("Processing %s op from client %s (stand-alone)", op, pcmk__client_name(cib_client)); } else { pcmk__trace("Processing %saddressed %s op from client %s", ((host != NULL)? "locally " : "un"), op, pcmk__client_name(cib_client)); } } static gboolean parse_peer_options(const cib__operation_t *operation, xmlNode *request, gboolean *local_notify, gboolean *needs_reply, gboolean *process) { /* TODO: What happens when an update comes in after node A * requests the CIB from node B, but before it gets the reply (and * sends out the replace operation)? * * (This may no longer be relevant since legacy mode was dropped; need to * trace code more closely to check.) */ const char *host = NULL; const char *delegated = pcmk__xe_get(request, PCMK__XA_CIB_DELEGATED_FROM); const char *op = pcmk__xe_get(request, PCMK__XA_CIB_OP); const char *originator = pcmk__xe_get(request, PCMK__XA_SRC); const char *reply_to = pcmk__xe_get(request, PCMK__XA_CIB_ISREPLYTO); gboolean is_reply = pcmk__str_eq(reply_to, OUR_NODENAME, pcmk__str_casei); if (originator == NULL) { // Shouldn't be possible originator = "peer"; } if (pcmk__str_eq(op, PCMK__CIB_REQUEST_REPLACE, pcmk__str_none)) { // sync_our_cib() sets PCMK__XA_CIB_ISREPLYTO if (reply_to) { delegated = reply_to; } goto skip_is_reply; } else if (pcmk__str_eq(op, PCMK__CIB_REQUEST_SYNC_TO_ALL, pcmk__str_none)) { // Nothing to do } else if (is_reply && pcmk__str_eq(op, CRM_OP_PING, pcmk__str_casei)) { process_ping_reply(request); return FALSE; } else if (pcmk__str_eq(op, PCMK__CIB_REQUEST_UPGRADE, pcmk__str_none)) { /* Only the DC (node with the oldest software) should process * this operation if PCMK__XA_CIB_SCHEMA_MAX is unset. * * If the DC is happy it will then send out another * PCMK__CIB_REQUEST_UPGRADE which will tell all nodes to do the actual * upgrade. * * Except this time PCMK__XA_CIB_SCHEMA_MAX will be set which puts a * limit on how far newer nodes will go */ const char *max = pcmk__xe_get(request, PCMK__XA_CIB_SCHEMA_MAX); const char *upgrade_rc = pcmk__xe_get(request, PCMK__XA_CIB_UPGRADE_RC); pcmk__trace("Parsing upgrade %s for %s with max=%s and upgrade_rc=%s", (is_reply? "reply" : "request"), (based_is_primary? "primary" : "secondary"), pcmk__s(max, "none"), pcmk__s(upgrade_rc, "none")); if (upgrade_rc != NULL) { // Our upgrade request was rejected by DC, notify clients of result pcmk__xe_set(request, PCMK__XA_CIB_RC, upgrade_rc); } else if ((max == NULL) && based_is_primary) { /* We are the DC, check if this upgrade is allowed */ goto skip_is_reply; } else if(max) { /* Ok, go ahead and upgrade to 'max' */ goto skip_is_reply; } else { // Ignore broadcast client requests when we're not primary return FALSE; } } else if (pcmk__xe_attr_is_true(request, PCMK__XA_CIB_UPDATE)) { pcmk__info("Detected legacy %s global update from %s", op, originator); send_sync_request(NULL); return FALSE; } else if (is_reply && pcmk__is_set(operation->flags, cib__op_attr_modifies)) { pcmk__trace("Ignoring legacy %s reply sent from %s to local clients", op, originator); return FALSE; } else if (pcmk__str_eq(op, PCMK__CIB_REQUEST_SHUTDOWN, pcmk__str_none)) { *local_notify = FALSE; if (reply_to == NULL) { *process = TRUE; } else { // Not possible? pcmk__debug("Ignoring shutdown request from %s because reply_to=%s", originator, reply_to); } return *process; } if (is_reply) { pcmk__trace("Will notify local clients for %s reply from %s", op, originator); *process = FALSE; *needs_reply = FALSE; *local_notify = TRUE; return TRUE; } skip_is_reply: *process = TRUE; *needs_reply = FALSE; *local_notify = pcmk__str_eq(delegated, OUR_NODENAME, pcmk__str_casei); host = pcmk__xe_get(request, PCMK__XA_CIB_HOST); if (pcmk__str_eq(host, OUR_NODENAME, pcmk__str_casei)) { pcmk__trace("Processing %s request sent to us from %s", op, originator); *needs_reply = TRUE; return TRUE; } else if (host != NULL) { pcmk__trace("Ignoring %s request intended for CIB manager on %s", op, host); return FALSE; } else if(is_reply == FALSE && pcmk__str_eq(op, CRM_OP_PING, pcmk__str_casei)) { *needs_reply = TRUE; } pcmk__trace("Processing %s request broadcast by %s call %s on %s (local " "clients will%s be notified)", op, pcmk__s(pcmk__xe_get(request, PCMK__XA_CIB_CLIENTNAME), "client"), pcmk__s(pcmk__xe_get(request, PCMK__XA_CIB_CALLID), "without ID"), originator, (*local_notify? "" : "not")); return TRUE; } /*! * \internal * \brief Forward a CIB request to the appropriate target host(s) * * \param[in] request CIB request to forward */ static void forward_request(xmlNode *request) { const char *op = pcmk__xe_get(request, PCMK__XA_CIB_OP); const char *section = pcmk__xe_get(request, PCMK__XA_CIB_SECTION); const char *host = pcmk__xe_get(request, PCMK__XA_CIB_HOST); const char *originator = pcmk__xe_get(request, PCMK__XA_SRC); const char *client_name = pcmk__xe_get(request, PCMK__XA_CIB_CLIENTNAME); const char *call_id = pcmk__xe_get(request, PCMK__XA_CIB_CALLID); pcmk__node_status_t *peer = NULL; int log_level = LOG_INFO; if (pcmk__str_eq(op, PCMK__CIB_REQUEST_NOOP, pcmk__str_none)) { log_level = LOG_DEBUG; } do_crm_log(log_level, "Forwarding %s operation for section %s to %s (origin=%s/%s/%s)", pcmk__s(op, "invalid"), pcmk__s(section, "all"), pcmk__s(host, "all"), pcmk__s(originator, "local"), pcmk__s(client_name, "unspecified"), pcmk__s(call_id, "unspecified")); pcmk__xe_set(request, PCMK__XA_CIB_DELEGATED_FROM, OUR_NODENAME); if (host != NULL) { peer = pcmk__get_node(0, host, NULL, pcmk__node_search_cluster_member); } pcmk__cluster_send_message(peer, pcmk_ipc_based, request); // Return the request to its original state pcmk__xe_remove_attr(request, PCMK__XA_CIB_DELEGATED_FROM); } static void send_peer_reply(xmlNode *msg, const char *originator) { const pcmk__node_status_t *node = NULL; if ((msg == NULL) || (originator == NULL)) { return; } // Send reply via cluster to originating node node = pcmk__get_node(0, originator, NULL, pcmk__node_search_cluster_member); pcmk__trace("Sending request result to %s only", originator); pcmk__xe_set(msg, PCMK__XA_CIB_ISREPLYTO, originator); pcmk__cluster_send_message(node, pcmk_ipc_based, msg); } /*! * \internal * \brief Handle an IPC or CPG message containing a request * * \param[in,out] request Request XML * \param[in] privileged Whether privileged commands may be run * (see cib_server_ops[] definition) * \param[in] cib_client IPC client that sent request (or NULL if CPG) * * \return Legacy Pacemaker return code */ int cib_process_request(xmlNode *request, gboolean privileged, const pcmk__client_t *cib_client) { // @TODO: Break into multiple smaller functions uint32_t call_options = cib_none; gboolean process = TRUE; // Whether to process request locally now gboolean is_update = TRUE; // Whether request would modify CIB gboolean needs_reply = TRUE; // Whether to build a reply gboolean local_notify = FALSE; // Whether to notify (local) requester gboolean needs_forward = FALSE; // Whether to forward request somewhere else xmlNode *op_reply = NULL; xmlNode *result_diff = NULL; int rc = pcmk_ok; const char *op = pcmk__xe_get(request, PCMK__XA_CIB_OP); const char *originator = pcmk__xe_get(request, PCMK__XA_SRC); const char *host = pcmk__xe_get(request, PCMK__XA_CIB_HOST); const char *call_id = pcmk__xe_get(request, PCMK__XA_CIB_CALLID); const char *client_id = pcmk__xe_get(request, PCMK__XA_CIB_CLIENTID); const char *client_name = pcmk__xe_get(request, PCMK__XA_CIB_CLIENTNAME); const char *reply_to = pcmk__xe_get(request, PCMK__XA_CIB_ISREPLYTO); const cib__operation_t *operation = NULL; cib__op_fn_t op_function = NULL; rc = pcmk__xe_get_flags(request, PCMK__XA_CIB_CALLOPT, &call_options, cib_none); if (rc != pcmk_rc_ok) { pcmk__warn("Couldn't parse options from request: %s", pcmk_rc_str(rc)); } if ((host != NULL) && (*host == '\0')) { host = NULL; } if (cib_client == NULL) { pcmk__trace("Processing peer %s operation from %s/%s on %s intended " "for %s (reply=%s)", op, pcmk__s(client_name, "client"), call_id, originator, pcmk__s(host, "all"), reply_to); } else { pcmk__xe_set(request, PCMK__XA_SRC, OUR_NODENAME); pcmk__trace("Processing local %s operation from %s/%s intended for %s", op, pcmk__s(client_name, "client"), call_id, pcmk__s(host, "all")); } rc = cib__get_operation(op, &operation); rc = pcmk_rc2legacy(rc); if (rc != pcmk_ok) { /* TODO: construct error reply? */ pcmk__err("Pre-processing of command failed: %s", pcmk_strerror(rc)); return rc; } op_function = based_get_op_function(operation); if (op_function == NULL) { pcmk__err("Operation %s not supported by CIB manager", op); return -EOPNOTSUPP; } if (cib_client != NULL) { parse_local_options(cib_client, operation, host, op, &local_notify, &needs_reply, &process, &needs_forward); } else if (!parse_peer_options(operation, request, &local_notify, &needs_reply, &process)) { return rc; } if (pcmk__is_set(call_options, cib_transaction)) { /* All requests in a transaction are processed locally against a working * CIB copy, and we don't notify for individual requests because the * entire transaction is atomic. * * We still call the option parser functions above, for the sake of log * messages and checking whether we're the target for peer requests. */ process = TRUE; needs_reply = FALSE; local_notify = FALSE; needs_forward = FALSE; } is_update = pcmk__is_set(operation->flags, cib__op_attr_modifies); if (pcmk__is_set(call_options, cib_discard_reply)) { /* If the request will modify the CIB, and we are in legacy mode, we * need to build a reply so we can broadcast a diff, even if the * requester doesn't want one. */ needs_reply = FALSE; local_notify = FALSE; pcmk__trace("Client is not interested in the reply"); } if (needs_forward) { forward_request(request); return rc; } if (cib_status != pcmk_ok) { rc = cib_status; pcmk__err("Ignoring request because cluster configuration is invalid " "(please repair and restart): %s", pcmk_strerror(rc)); op_reply = create_cib_reply(op, call_id, client_id, call_options, rc, the_cib); } else if (process) { time_t finished = 0; time_t now = time(NULL); int level = LOG_INFO; const char *section = pcmk__xe_get(request, PCMK__XA_CIB_SECTION); const char *admin_epoch_s = NULL; const char *epoch_s = NULL; const char *num_updates_s = NULL; rc = cib_process_command(request, operation, op_function, &op_reply, &result_diff, privileged); if (!is_update) { level = PCMK__LOG_TRACE; } else if (pcmk__xe_attr_is_true(request, PCMK__XA_CIB_UPDATE)) { switch (rc) { case pcmk_ok: level = LOG_INFO; break; case -pcmk_err_old_data: case -pcmk_err_diff_resync: case -pcmk_err_diff_failed: level = PCMK__LOG_TRACE; break; default: level = LOG_ERR; } } else if (rc != pcmk_ok) { level = LOG_WARNING; } if (the_cib != NULL) { admin_epoch_s = pcmk__xe_get(the_cib, PCMK_XA_ADMIN_EPOCH); epoch_s = pcmk__xe_get(the_cib, PCMK_XA_EPOCH); num_updates_s = pcmk__xe_get(the_cib, PCMK_XA_NUM_UPDATES); } do_crm_log(level, "Completed %s operation for section %s: %s (rc=%d, origin=%s/%s/%s, version=%s.%s.%s)", op, section ? section : "'all'", pcmk_strerror(rc), rc, originator ? originator : "local", pcmk__s(client_name, "client"), call_id, pcmk__s(admin_epoch_s, "0"), pcmk__s(epoch_s, "0"), pcmk__s(num_updates_s, "0")); finished = time(NULL); if ((finished - now) > 3) { pcmk__trace("%s operation took %llds to complete", op, (long long) (finished - now)); crm_write_blackbox(0, NULL); } if (op_reply == NULL && (needs_reply || local_notify)) { pcmk__err("Unexpected NULL reply to message"); pcmk__log_xml_err(request, "null reply"); needs_reply = FALSE; local_notify = FALSE; } } if (is_update) { pcmk__trace("Completed pre-sync update from %s/%s/%s%s", pcmk__s(originator, "local"), pcmk__s(client_name, "client"), call_id, (local_notify? " with local notification" : "")); } else if (!needs_reply || stand_alone) { // This was a non-originating secondary update pcmk__trace("Completed update as secondary"); } else if ((cib_client == NULL) && !pcmk__is_set(call_options, cib_discard_reply)) { if (is_update == FALSE || result_diff == NULL) { pcmk__trace("Request not broadcast: R/O call"); } else if (rc != pcmk_ok) { pcmk__trace("Request not broadcast: call failed: %s", pcmk_strerror(rc)); } else { pcmk__trace("Directing reply to %s", originator); } send_peer_reply(op_reply, originator); } if (local_notify && client_id) { pcmk__trace("Performing local %ssync notification for %s", (pcmk__is_set(call_options, cib_sync_call)? "" : "a"), client_id); if (process == FALSE) { do_local_notify(request, client_id, pcmk__is_set(call_options, cib_sync_call), (cib_client == NULL)); } else { do_local_notify(op_reply, client_id, pcmk__is_set(call_options, cib_sync_call), (cib_client == NULL)); } } pcmk__xml_free(op_reply); pcmk__xml_free(result_diff); return rc; } /*! * \internal * \brief Get a CIB operation's input from the request XML * * \param[in] request CIB request XML * \param[in] type CIB operation type * \param[out] section Where to store CIB section name * * \return Input XML for CIB operation * * \note If not \c NULL, the return value is a non-const pointer to part of * \p request. The caller should not free it directly. */ static xmlNode * prepare_input(const xmlNode *request, enum cib__op_type type, const char **section) { xmlNode *wrapper = pcmk__xe_first_child(request, PCMK__XE_CIB_CALLDATA, NULL, NULL); xmlNode *input = pcmk__xe_first_child(wrapper, NULL, NULL, NULL); if (type == cib__op_apply_patch) { *section = NULL; } else { *section = pcmk__xe_get(request, PCMK__XA_CIB_SECTION); } // Grab the specified section if ((*section != NULL) && pcmk__xe_is(input, PCMK_XE_CIB)) { input = pcmk_find_cib_element(input, *section); } return input; } #define XPATH_CONFIG_CHANGE \ "//" PCMK_XE_CHANGE \ "[contains(@" PCMK_XA_PATH ",'/" PCMK_XE_CRM_CONFIG "/')]" static bool contains_config_change(xmlNode *diff) { bool changed = false; if (diff) { xmlXPathObject *xpathObj = pcmk__xpath_search(diff->doc, XPATH_CONFIG_CHANGE); if (pcmk__xpath_num_results(xpathObj) > 0) { changed = true; } xmlXPathFreeObject(xpathObj); } return changed; } static int cib_process_command(xmlNode *request, const cib__operation_t *operation, cib__op_fn_t op_function, xmlNode **reply, xmlNode **cib_diff, bool privileged) { xmlNode *input = NULL; xmlNode *output = NULL; xmlNode *result_cib = NULL; uint32_t call_options = cib_none; const char *op = NULL; const char *section = NULL; const char *call_id = pcmk__xe_get(request, PCMK__XA_CIB_CALLID); const char *client_id = pcmk__xe_get(request, PCMK__XA_CIB_CLIENTID); const char *client_name = pcmk__xe_get(request, PCMK__XA_CIB_CLIENTNAME); const char *originator = pcmk__xe_get(request, PCMK__XA_SRC); int rc = pcmk_ok; bool config_changed = false; bool manage_counters = true; static mainloop_timer_t *digest_timer = NULL; pcmk__assert(cib_status == pcmk_ok); if(digest_timer == NULL) { digest_timer = mainloop_timer_add("digester", 5000, FALSE, cib_digester_cb, NULL); } *reply = NULL; *cib_diff = NULL; /* Start processing the request... */ op = pcmk__xe_get(request, PCMK__XA_CIB_OP); rc = pcmk__xe_get_flags(request, PCMK__XA_CIB_CALLOPT, &call_options, cib_none); if (rc != pcmk_rc_ok) { pcmk__warn("Couldn't parse options from request: %s", pcmk_rc_str(rc)); } if (!privileged && pcmk__is_set(operation->flags, cib__op_attr_privileged)) { rc = -EACCES; pcmk__trace("Failed due to lack of privileges: %s", pcmk_strerror(rc)); goto done; } input = prepare_input(request, operation->type, §ion); if (!pcmk__is_set(operation->flags, cib__op_attr_modifies)) { rc = cib_perform_op(NULL, op, call_options, op_function, true, section, request, input, false, &config_changed, &the_cib, &result_cib, NULL, &output); CRM_CHECK(result_cib == NULL, pcmk__xml_free(result_cib)); goto done; } /* @COMPAT: Handle a valid write action (legacy) * * @TODO: Re-evaluate whether this is all truly legacy. The cib_force_diff * portion is. However, PCMK__XA_CIB_UPDATE may be set by a sync operation * even in non-legacy mode, and manage_counters tells xml_create_patchset() * whether to update version/epoch info. */ if (pcmk__xe_attr_is_true(request, PCMK__XA_CIB_UPDATE)) { manage_counters = false; cib__set_call_options(call_options, "call", cib_force_diff); pcmk__trace("Global update detected"); CRM_LOG_ASSERT(pcmk__str_any_of(op, PCMK__CIB_REQUEST_APPLY_PATCH, PCMK__CIB_REQUEST_REPLACE, NULL)); } ping_modified_since = TRUE; // result_cib must not be modified after cib_perform_op() returns rc = cib_perform_op(NULL, op, call_options, op_function, false, section, request, input, manage_counters, &config_changed, &the_cib, &result_cib, cib_diff, &output); /* Always write to disk for successful ops with the flag set. This also * negates the need to detect ordering changes. */ if ((rc == pcmk_ok) && pcmk__is_set(operation->flags, cib__op_attr_writes_through)) { config_changed = true; } if ((rc == pcmk_ok) && !pcmk__any_flags_set(call_options, cib_dryrun|cib_transaction)) { if (result_cib != the_cib) { if (pcmk__is_set(operation->flags, cib__op_attr_writes_through)) { config_changed = true; } pcmk__trace("Activating %s->%s%s", pcmk__xe_get(the_cib, PCMK_XA_NUM_UPDATES), pcmk__xe_get(result_cib, PCMK_XA_NUM_UPDATES), (config_changed? " changed" : "")); rc = activateCibXml(result_cib, config_changed, op); if (rc != pcmk_ok) { pcmk__err("Failed to activate new CIB: %s", pcmk_strerror(rc)); } } if ((rc == pcmk_ok) && contains_config_change(*cib_diff)) { cib_read_config(config_hash, result_cib); } /* @COMPAT Nodes older than feature set 3.19.0 don't support * transactions. In a mixed-version cluster with nodes <3.19.0, we must * sync the updated CIB, so that the older nodes receive the changes. * Any node that has already applied the transaction will ignore the * synced CIB. * * To ensure the updated CIB is synced from only one node, we sync it * from the originator. */ if ((operation->type == cib__op_commit_transact) && pcmk__str_eq(originator, OUR_NODENAME, pcmk__str_casei) && (pcmk__compare_versions(pcmk__xe_get(the_cib, PCMK_XA_CRM_FEATURE_SET), "3.19.0") < 0)) { sync_our_cib(request, TRUE); } mainloop_timer_stop(digest_timer); mainloop_timer_start(digest_timer); } else if (rc == -pcmk_err_schema_validation) { pcmk__assert(result_cib != the_cib); if (output != NULL) { pcmk__log_xml_info(output, "cib:output"); pcmk__xml_free(output); } output = result_cib; } else { pcmk__trace("Not activating %d %d %s", rc, pcmk__is_set(call_options, cib_dryrun), pcmk__xe_get(result_cib, PCMK_XA_NUM_UPDATES)); if (result_cib != the_cib) { pcmk__xml_free(result_cib); } } if (!pcmk__any_flags_set(call_options, cib_dryrun|cib_inhibit_notify|cib_transaction)) { pcmk__trace("Sending notifications %d", pcmk__is_set(call_options, cib_dryrun)); cib_diff_notify(op, rc, call_id, client_id, client_name, originator, input, *cib_diff); } pcmk__log_xml_patchset(PCMK__LOG_TRACE, *cib_diff); done: if (!pcmk__is_set(call_options, cib_discard_reply)) { *reply = create_cib_reply(op, call_id, client_id, call_options, rc, output); } if (output != the_cib) { pcmk__xml_free(output); } pcmk__trace("done"); return rc; } void cib_peer_callback(xmlNode * msg, void *private_data) { const char *reason = NULL; const char *originator = pcmk__xe_get(msg, PCMK__XA_SRC); if (pcmk__peer_cache == NULL) { reason = "membership not established"; goto bail; } if (pcmk__xe_get(msg, PCMK__XA_CIB_CLIENTNAME) == NULL) { pcmk__xe_set(msg, PCMK__XA_CIB_CLIENTNAME, originator); } - /* crm_log_xml_trace(msg, "Peer[inbound]"); */ + // pcmk__log_xml_trace(msg, "Peer[inbound]"); cib_process_request(msg, TRUE, NULL); return; bail: if (reason) { const char *op = pcmk__xe_get(msg, PCMK__XA_CIB_OP); pcmk__warn("Discarding %s message from %s: %s", op, originator, reason); } } static gboolean cib_force_exit(gpointer data) { pcmk__notice("Exiting immediately after %s without shutdown acknowledgment", pcmk__readable_interval(EXIT_ESCALATION_MS)); terminate_cib(CRM_EX_ERROR); return FALSE; } static void disconnect_remote_client(gpointer key, gpointer value, gpointer user_data) { pcmk__client_t *a_client = value; pcmk__err("Can't disconnect client %s: Not implemented", pcmk__client_name(a_client)); } static void initiate_exit(void) { int active = 0; xmlNode *leaving = NULL; active = pcmk__cluster_num_active_nodes(); if (active < 2) { // This is the last active node pcmk__info("Exiting without sending shutdown request (no active " "peers)"); terminate_cib(CRM_EX_OK); return; } pcmk__info("Sending shutdown request to %d peers", active); leaving = pcmk__xe_create(NULL, PCMK__XE_EXIT_NOTIFICATION); pcmk__xe_set(leaving, PCMK__XA_T, PCMK__VALUE_CIB); pcmk__xe_set(leaving, PCMK__XA_CIB_OP, PCMK__CIB_REQUEST_SHUTDOWN); pcmk__cluster_send_message(NULL, pcmk_ipc_based, leaving); pcmk__xml_free(leaving); pcmk__create_timer(EXIT_ESCALATION_MS, cib_force_exit, NULL); } void cib_shutdown(int nsig) { struct qb_ipcs_stats srv_stats; if (cib_shutdown_flag == FALSE) { int disconnects = 0; qb_ipcs_connection_t *c = NULL; cib_shutdown_flag = TRUE; c = qb_ipcs_connection_first_get(ipcs_rw); while (c != NULL) { qb_ipcs_connection_t *last = c; c = qb_ipcs_connection_next_get(ipcs_rw, last); pcmk__debug("Disconnecting r/w client %p...", last); qb_ipcs_disconnect(last); qb_ipcs_connection_unref(last); disconnects++; } c = qb_ipcs_connection_first_get(ipcs_ro); while (c != NULL) { qb_ipcs_connection_t *last = c; c = qb_ipcs_connection_next_get(ipcs_ro, last); pcmk__debug("Disconnecting r/o client %p...", last); qb_ipcs_disconnect(last); qb_ipcs_connection_unref(last); disconnects++; } c = qb_ipcs_connection_first_get(ipcs_shm); while (c != NULL) { qb_ipcs_connection_t *last = c; c = qb_ipcs_connection_next_get(ipcs_shm, last); pcmk__debug("Disconnecting non-blocking r/w client %p...", last); qb_ipcs_disconnect(last); qb_ipcs_connection_unref(last); disconnects++; } disconnects += pcmk__ipc_client_count(); pcmk__debug("Disconnecting %d remote clients", pcmk__ipc_client_count()); pcmk__foreach_ipc_client(disconnect_remote_client, NULL); pcmk__info("Disconnected %d clients", disconnects); } qb_ipcs_stats_get(ipcs_rw, &srv_stats, QB_FALSE); if (pcmk__ipc_client_count() == 0) { pcmk__info("All clients disconnected (%d)", srv_stats.active_connections); initiate_exit(); } else { pcmk__info("Waiting on %d clients to disconnect (%d)", pcmk__ipc_client_count(), srv_stats.active_connections); } } extern int remote_fd; extern int remote_tls_fd; /*! * \internal * \brief Close remote sockets, free the global CIB and quit * * \param[in] exit_status What exit status to use (if -1, use CRM_EX_OK, but * skip disconnecting from the cluster layer) */ void terminate_cib(int exit_status) { if (remote_fd > 0) { close(remote_fd); remote_fd = 0; } if (remote_tls_fd > 0) { close(remote_tls_fd); remote_tls_fd = 0; } uninitializeCib(); // Exit immediately on error if (exit_status > CRM_EX_OK) { pcmk__stop_based_ipc(ipcs_ro, ipcs_rw, ipcs_shm); crm_exit(exit_status); return; } if ((mainloop != NULL) && g_main_loop_is_running(mainloop)) { /* Quit via returning from the main loop. If exit_status has the special * value -1, we skip the disconnect here, and it will be done when the * main loop returns (this allows the peer status callback to avoid * messing with the peer caches). */ if (exit_status == CRM_EX_OK) { pcmk_cluster_disconnect(crm_cluster); } g_main_loop_quit(mainloop); return; } /* Exit cleanly. Even the peer status callback can disconnect here, because * we're not returning control to the caller. */ pcmk_cluster_disconnect(crm_cluster); pcmk__stop_based_ipc(ipcs_ro, ipcs_rw, ipcs_shm); crm_exit(CRM_EX_OK); } diff --git a/daemons/based/based_io.c b/daemons/based/based_io.c index bca21a4d9c..79c2bb13c2 100644 --- a/daemons/based/based_io.c +++ b/daemons/based/based_io.c @@ -1,428 +1,428 @@ /* * Copyright 2004-2025 the Pacemaker project contributors * * The version control history for this file may have further details. * * This source code is licensed under the GNU General Public License version 2 * or later (GPLv2+) WITHOUT ANY WARRANTY. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include crm_trigger_t *cib_writer = NULL; int write_cib_contents(gpointer p); static void cib_rename(const char *old) { int new_fd; char *new = pcmk__assert_asprintf("%s/cib.auto.XXXXXX", cib_root); umask(S_IWGRP | S_IWOTH | S_IROTH); new_fd = mkstemp(new); if ((new_fd < 0) || (rename(old, new) < 0)) { pcmk__err("Couldn't archive unusable file %s (disabling disk writes " "and continuing)", old); cib_writes_enabled = FALSE; } else { pcmk__err("Archived unusable file %s as %s", old, new); } if (new_fd > 0) { close(new_fd); } free(new); } /* * It is the callers responsibility to free the output of this function */ static xmlNode * retrieveCib(const char *filename, const char *sigfile) { xmlNode *root = NULL; int rc = cib_file_read_and_verify(filename, sigfile, &root); if (rc == pcmk_ok) { pcmk__info("Loaded CIB from %s (with digest %s)", filename, sigfile); } else { pcmk__warn("Continuing but NOT using CIB from %s (with digest %s): %s", filename, sigfile, pcmk_strerror(rc)); if (rc == -pcmk_err_cib_modified) { // Archive the original files so the contents are not lost cib_rename(filename); cib_rename(sigfile); } } return root; } static int cib_archive_filter(const struct dirent * a) { int rc = 0; // Looking for regular files starting with "cib-" and not ending in .sig struct stat s; char *a_path = pcmk__assert_asprintf("%s/%s", cib_root, a->d_name); if(stat(a_path, &s) != 0) { rc = errno; pcmk__trace("%s - stat failed: %s (%d)", a->d_name, pcmk_rc_str(rc), rc); rc = 0; } else if (!S_ISREG(s.st_mode)) { pcmk__trace("%s - wrong type (%#o)", a->d_name, (unsigned int) (s.st_mode & S_IFMT)); } else if(strstr(a->d_name, "cib-") != a->d_name) { pcmk__trace("%s - wrong prefix", a->d_name); } else if (pcmk__ends_with_ext(a->d_name, ".sig")) { pcmk__trace("%s - wrong suffix", a->d_name); } else { pcmk__debug("%s - candidate", a->d_name); rc = 1; } free(a_path); return rc; } static int cib_archive_sort(const struct dirent ** a, const struct dirent **b) { /* Order by creation date - most recently created file first */ int rc = 0; struct stat buf; time_t a_age = 0; time_t b_age = 0; char *a_path = pcmk__assert_asprintf("%s/%s", cib_root, a[0]->d_name); char *b_path = pcmk__assert_asprintf("%s/%s", cib_root, b[0]->d_name); if(stat(a_path, &buf) == 0) { a_age = buf.st_ctime; } if(stat(b_path, &buf) == 0) { b_age = buf.st_ctime; } free(a_path); free(b_path); if(a_age > b_age) { rc = 1; } else if(a_age < b_age) { rc = -1; } pcmk__trace("%s (%lu) vs. %s (%lu) : %d", a[0]->d_name, (unsigned long)a_age, b[0]->d_name, (unsigned long)b_age, rc); return rc; } xmlNode * readCibXmlFile(const char *dir, const char *file, gboolean discard_status) { struct dirent **namelist = NULL; int lpc = 0; char *sigfile = NULL; char *sigfilepath = NULL; char *filename = NULL; const char *name = NULL; const char *value = NULL; const char *use_valgrind = pcmk__env_option(PCMK__ENV_VALGRIND_ENABLED); xmlNode *root = NULL; xmlNode *status = NULL; sigfile = pcmk__assert_asprintf("%s.sig", file); if (pcmk__daemon_can_write(dir, file) == FALSE || pcmk__daemon_can_write(dir, sigfile) == FALSE) { cib_status = -EACCES; return NULL; } filename = pcmk__assert_asprintf("%s/%s", dir, file); sigfilepath = pcmk__assert_asprintf("%s/%s", dir, sigfile); free(sigfile); cib_status = pcmk_ok; root = retrieveCib(filename, sigfilepath); free(filename); free(sigfilepath); if (root == NULL) { lpc = scandir(cib_root, &namelist, cib_archive_filter, cib_archive_sort); if (lpc < 0) { pcmk__err("Could not check for CIB backups in %s: %s", cib_root, pcmk_rc_str(errno)); } } while (root == NULL && lpc > 1) { int rc = pcmk_ok; lpc--; filename = pcmk__assert_asprintf("%s/%s", cib_root, namelist[lpc]->d_name); sigfile = pcmk__assert_asprintf("%s.sig", filename); rc = cib_file_read_and_verify(filename, sigfile, &root); if (rc == pcmk_ok) { pcmk__notice("Loaded CIB from last valid backup %s (with digest " "%s)", filename, sigfile); } else { pcmk__warn("Not using next most recent CIB backup from %s (with " "digest %s): %s", filename, sigfile, pcmk_strerror(rc)); } free(namelist[lpc]); free(filename); free(sigfile); } free(namelist); if (root == NULL) { root = createEmptyCib(0); pcmk__warn("Continuing with an empty configuration"); } if (cib_writes_enabled && (use_valgrind != NULL) && (pcmk__is_true(use_valgrind) || (strstr(use_valgrind, PCMK__SERVER_BASED) != NULL))) { cib_writes_enabled = FALSE; pcmk__err("*** Disabling disk writes to avoid confusing Valgrind ***"); } status = pcmk__xe_first_child(root, PCMK_XE_STATUS, NULL, NULL); if (discard_status && status != NULL) { // Strip out the PCMK_XE_STATUS section if there is one pcmk__xml_free(status); status = NULL; } if (status == NULL) { pcmk__xe_create(root, PCMK_XE_STATUS); } /* Do this before schema validation happens */ /* fill in some defaults */ value = pcmk__xe_get(root, PCMK_XA_ADMIN_EPOCH); if (value == NULL) { // Not possible with schema validation enabled pcmk__warn("Defaulting missing " PCMK_XA_ADMIN_EPOCH " to 0, but " "cluster may get confused about which node's configuration " "is most recent"); pcmk__xe_set_int(root, PCMK_XA_ADMIN_EPOCH, 0); } name = PCMK_XA_EPOCH; value = pcmk__xe_get(root, name); if (value == NULL) { pcmk__xe_set_int(root, name, 0); } name = PCMK_XA_NUM_UPDATES; value = pcmk__xe_get(root, name); if (value == NULL) { pcmk__xe_set_int(root, name, 0); } // Unset (DC should set appropriate value) pcmk__xe_remove_attr(root, PCMK_XA_DC_UUID); if (discard_status) { - crm_log_xml_trace(root, "[on-disk]"); + pcmk__log_xml_trace(root, "[on-disk]"); } if (!pcmk__configured_schema_validates(root)) { cib_status = -pcmk_err_schema_validation; } return root; } gboolean uninitializeCib(void) { xmlNode *tmp_cib = the_cib; if (tmp_cib == NULL) { return FALSE; } the_cib = NULL; pcmk__xml_free(tmp_cib); return TRUE; } /* * This method will free the old CIB pointer on success and the new one * on failure. */ int activateCibXml(xmlNode * new_cib, gboolean to_disk, const char *op) { if (new_cib) { xmlNode *saved_cib = the_cib; pcmk__assert(new_cib != saved_cib); the_cib = new_cib; pcmk__xml_free(saved_cib); if (cib_writes_enabled && cib_status == pcmk_ok && to_disk) { pcmk__debug("Triggering CIB write for %s op", op); mainloop_set_trigger(cib_writer); } return pcmk_ok; } pcmk__err("Ignoring invalid CIB"); if (the_cib) { pcmk__warn("Reverting to last known CIB"); } else { pcmk__crit("Could not write out new CIB and no saved version to revert " "to"); } return -ENODATA; } static void cib_diskwrite_complete(mainloop_child_t * p, pid_t pid, int core, int signo, int exitcode) { const char *errmsg = "Could not write CIB to disk"; if ((exitcode != 0) && cib_writes_enabled) { cib_writes_enabled = FALSE; errmsg = "Disabling CIB disk writes after failure"; } if ((signo == 0) && (exitcode == 0)) { pcmk__trace("Disk write [%d] succeeded", (int) pid); } else if (signo == 0) { pcmk__err("%s: process %d exited %d", errmsg, (int) pid, exitcode); } else { pcmk__err("%s: process %d terminated with signal %d (%s)%s", errmsg, (int) pid, signo, strsignal(signo), ((core != 0)? " and dumped core" : "")); } mainloop_trigger_complete(cib_writer); } int write_cib_contents(gpointer p) { int exit_rc = pcmk_ok; xmlNode *cib_local = NULL; /* Make a copy of the CIB to write (possibly in a forked child) */ if (p) { /* Synchronous write out */ cib_local = pcmk__xml_copy(NULL, p); } else { int pid = 0; int bb_state = qb_log_ctl(QB_LOG_BLACKBOX, QB_LOG_CONF_STATE_GET, 0); /* Turn it off before the fork() to avoid: * - 2 processes writing to the same shared mem * - the child needing to disable it * (which would close it from underneath the parent) * This way, the shared mem files are already closed */ qb_log_ctl(QB_LOG_BLACKBOX, QB_LOG_CONF_ENABLED, QB_FALSE); pid = fork(); if (pid < 0) { pcmk__err("Disabling disk writes after fork failure: %s", pcmk_rc_str(errno)); cib_writes_enabled = FALSE; return FALSE; } if (pid) { /* Parent */ mainloop_child_add(pid, 0, "disk-writer", NULL, cib_diskwrite_complete); if (bb_state == QB_LOG_STATE_ENABLED) { /* Re-enable now that it it safe */ qb_log_ctl(QB_LOG_BLACKBOX, QB_LOG_CONF_ENABLED, QB_TRUE); } return -1; /* -1 means 'still work to do' */ } /* Asynchronous write-out after a fork() */ /* In theory, we can scribble on the_cib here and not affect the parent, * but let's be safe anyway. */ cib_local = pcmk__xml_copy(NULL, the_cib); } /* Write the CIB */ exit_rc = cib_file_write_with_digest(cib_local, cib_root, "cib.xml"); /* A nonzero exit code will cause further writes to be disabled */ pcmk__xml_free(cib_local); if (p == NULL) { crm_exit_t exit_code = CRM_EX_OK; switch (exit_rc) { case pcmk_ok: exit_code = CRM_EX_OK; break; case pcmk_err_cib_modified: exit_code = CRM_EX_DIGEST; // Existing CIB doesn't match digest break; case pcmk_err_cib_backup: // Existing CIB couldn't be backed up case pcmk_err_cib_save: // New CIB couldn't be saved exit_code = CRM_EX_CANTCREAT; break; default: exit_code = CRM_EX_ERROR; break; } /* Use _exit() because exit() could affect the parent adversely */ pcmk_common_cleanup(); _exit(exit_code); } return exit_rc; } diff --git a/daemons/based/based_notify.c b/daemons/based/based_notify.c index 66ae13b250..0ca3ef4dc1 100644 --- a/daemons/based/based_notify.c +++ b/daemons/based/based_notify.c @@ -1,198 +1,198 @@ /* * Copyright 2004-2025 the Pacemaker project contributors * * The version control history for this file may have further details. * * This source code is licensed under the GNU General Public License version 2 * or later (GPLv2+) WITHOUT ANY WARRANTY. */ #include #include #include #include #include #include // PRIx64 #include #include #include #include #include #include #include #include #include #include #include struct cib_notification_s { const xmlNode *msg; struct iovec *iov; int32_t iov_size; }; static void cib_notify_send_one(gpointer key, gpointer value, gpointer user_data) { const char *type = NULL; gboolean do_send = FALSE; int rc = pcmk_rc_ok; pcmk__client_t *client = value; struct cib_notification_s *update = user_data; if (client->ipcs == NULL && client->remote == NULL) { pcmk__warn("Skipping client with NULL channel"); return; } type = pcmk__xe_get(update->msg, PCMK__XA_SUBT); CRM_LOG_ASSERT(type != NULL); if (pcmk__is_set(client->flags, cib_notify_diff) && pcmk__str_eq(type, PCMK__VALUE_CIB_DIFF_NOTIFY, pcmk__str_none)) { do_send = TRUE; } else if (pcmk__is_set(client->flags, cib_notify_confirm) && pcmk__str_eq(type, PCMK__VALUE_CIB_UPDATE_CONFIRMATION, pcmk__str_none)) { do_send = TRUE; } else if (pcmk__is_set(client->flags, cib_notify_pre) && pcmk__str_eq(type, PCMK__VALUE_CIB_PRE_NOTIFY, pcmk__str_none)) { do_send = TRUE; } else if (pcmk__is_set(client->flags, cib_notify_post) && pcmk__str_eq(type, PCMK__VALUE_CIB_POST_NOTIFY, pcmk__str_none)) { do_send = TRUE; } if (do_send) { switch (PCMK__CLIENT_TYPE(client)) { case pcmk__client_ipc: rc = pcmk__ipc_send_iov(client, update->iov, crm_ipc_server_event); if (rc != pcmk_rc_ok) { pcmk__warn("Could not notify client %s: %s " QB_XS " id=%s", pcmk__client_name(client), pcmk_rc_str(rc), client->id); } break; case pcmk__client_tls: case pcmk__client_tcp: pcmk__debug("Sent %s notification to client %s (id %s)", type, pcmk__client_name(client), client->id); pcmk__remote_send_xml(client->remote, update->msg); break; default: pcmk__err("Unknown transport for client %s " QB_XS " flags=%#016" PRIx64, pcmk__client_name(client), client->flags); } } } static void cib_notify_send(const xmlNode *xml) { struct iovec *iov; struct cib_notification_s update; ssize_t bytes = 0; int rc = pcmk__ipc_prepare_iov(0, xml, 0, &iov, &bytes); if (rc == pcmk_rc_ok) { update.msg = xml; update.iov = iov; update.iov_size = bytes; pcmk__foreach_ipc_client(cib_notify_send_one, &update); } else { pcmk__notice("Could not notify clients: %s " QB_XS " rc=%d", pcmk_rc_str(rc), rc); } pcmk_free_ipc_event(iov); } void cib_diff_notify(const char *op, int result, const char *call_id, const char *client_id, const char *client_name, const char *origin, xmlNode *update, xmlNode *diff) { int add_updates = 0; int add_epoch = 0; int add_admin_epoch = 0; int del_updates = 0; int del_epoch = 0; int del_admin_epoch = 0; uint8_t log_level = PCMK__LOG_TRACE; xmlNode *update_msg = NULL; xmlNode *wrapper = NULL; if (diff == NULL) { return; } if (result != pcmk_ok) { log_level = LOG_WARNING; } cib_diff_version_details(diff, &add_admin_epoch, &add_epoch, &add_updates, &del_admin_epoch, &del_epoch, &del_updates); if ((add_admin_epoch != del_admin_epoch) || (add_epoch != del_epoch) || (add_updates != del_updates)) { do_crm_log(log_level, "Updated CIB generation %d.%d.%d to %d.%d.%d from client " "%s%s%s (%s) (%s)", del_admin_epoch, del_epoch, del_updates, add_admin_epoch, add_epoch, add_updates, client_name, ((call_id != NULL)? " call " : ""), pcmk__s(call_id, ""), pcmk__s(origin, "unspecified peer"), pcmk_strerror(result)); } else if ((add_admin_epoch != 0) || (add_epoch != 0) || (add_updates != 0)) { do_crm_log(log_level, "Local-only change to CIB generation %d.%d.%d from client " "%s%s%s (%s) (%s)", add_admin_epoch, add_epoch, add_updates, client_name, ((call_id != NULL)? " call " : ""), pcmk__s(call_id, ""), pcmk__s(origin, "unspecified peer"), pcmk_strerror(result)); } update_msg = pcmk__xe_create(NULL, PCMK__XE_NOTIFY); pcmk__xe_set(update_msg, PCMK__XA_T, PCMK__VALUE_CIB_NOTIFY); pcmk__xe_set(update_msg, PCMK__XA_SUBT, PCMK__VALUE_CIB_DIFF_NOTIFY); pcmk__xe_set(update_msg, PCMK__XA_CIB_OP, op); pcmk__xe_set(update_msg, PCMK__XA_CIB_CLIENTID, client_id); pcmk__xe_set(update_msg, PCMK__XA_CIB_CLIENTNAME, client_name); pcmk__xe_set(update_msg, PCMK__XA_CIB_CALLID, call_id); pcmk__xe_set(update_msg, PCMK__XA_SRC, origin); pcmk__xe_set_int(update_msg, PCMK__XA_CIB_RC, result); wrapper = pcmk__xe_create(update_msg, PCMK__XE_CIB_UPDATE_RESULT); pcmk__xml_copy(wrapper, diff); - crm_log_xml_trace(update_msg, "diff-notify"); + pcmk__log_xml_trace(update_msg, "diff-notify"); cib_notify_send(update_msg); pcmk__xml_free(update_msg); } diff --git a/daemons/based/based_remote.c b/daemons/based/based_remote.c index 799f3b2121..4581699bc1 100644 --- a/daemons/based/based_remote.c +++ b/daemons/based/based_remote.c @@ -1,671 +1,671 @@ /* * Copyright 2004-2025 the Pacemaker project contributors * * The version control history for this file may have further details. * * This source code is licensed under the GNU General Public License version 2 * or later (GPLv2+) WITHOUT ANY WARRANTY. */ #include #include #include #include #include #include #include #include // PRIx64 #include #include #include #include #include #include #include #include #include #include #include #include #include #include "pacemaker-based.h" #include #include #include #if HAVE_SECURITY_PAM_APPL_H # include # define HAVE_PAM 1 #elif HAVE_PAM_PAM_APPL_H # include # define HAVE_PAM 1 #endif static pcmk__tls_t *tls = NULL; extern int remote_tls_fd; extern gboolean cib_shutdown_flag; int init_remote_listener(int port, gboolean encrypted); void cib_remote_connection_destroy(gpointer user_data); // @TODO This is rather short for someone to type their password #define REMOTE_AUTH_TIMEOUT 10000 int num_clients; static bool authenticate_user(const char *user, const char *passwd); static int cib_remote_listen(gpointer data); static int cib_remote_msg(gpointer data); static void remote_connection_destroy(gpointer user_data) { pcmk__info("No longer listening for remote connections"); return; } int init_remote_listener(int port, gboolean encrypted) { int rc; int *ssock = NULL; struct sockaddr_in saddr; int optval; static struct mainloop_fd_callbacks remote_listen_fd_callbacks = { .dispatch = cib_remote_listen, .destroy = remote_connection_destroy, }; if (port <= 0) { /* don't start it */ return 0; } if (encrypted) { bool use_cert = pcmk__x509_enabled(); pcmk__notice("Starting TLS listener on port %d", port); rc = pcmk__init_tls(&tls, true, use_cert ? GNUTLS_CRD_CERTIFICATE : GNUTLS_CRD_ANON); if (rc != pcmk_rc_ok) { return -1; } } else { pcmk__warn("Starting plain-text listener on port %d", port); } #ifndef HAVE_PAM pcmk__warn("This build does not support remote administrators because PAM " "support is not available"); #endif /* create server socket */ ssock = pcmk__assert_alloc(1, sizeof(int)); *ssock = socket(AF_INET, SOCK_STREAM, 0); if (*ssock == -1) { pcmk__err("Listener socket creation failed: %s", pcmk_rc_str(errno)); free(ssock); return -1; } /* reuse address */ optval = 1; rc = setsockopt(*ssock, SOL_SOCKET, SO_REUSEADDR, &optval, sizeof(optval)); if (rc < 0) { pcmk__err("Local address reuse not allowed on listener socket: %s", pcmk_rc_str(errno)); } /* bind server socket */ memset(&saddr, '\0', sizeof(saddr)); saddr.sin_family = AF_INET; saddr.sin_addr.s_addr = INADDR_ANY; saddr.sin_port = htons(port); if (bind(*ssock, (struct sockaddr *)&saddr, sizeof(saddr)) == -1) { pcmk__err("Cannot bind to listener socket: %s", pcmk_rc_str(errno)); close(*ssock); free(ssock); return -2; } if (listen(*ssock, 10) == -1) { pcmk__err("Cannot listen on socket: %s", pcmk_rc_str(errno)); close(*ssock); free(ssock); return -3; } mainloop_add_fd("cib-remote", G_PRIORITY_DEFAULT, *ssock, ssock, &remote_listen_fd_callbacks); pcmk__debug("Started listener on port %d", port); return *ssock; } static int check_group_membership(const char *usr, const char *grp) { int index = 0; gid_t gid = 0; struct group *group = NULL; int rc = pcmk_rc_ok; rc = pcmk__lookup_user(usr, NULL, &gid); if (rc != pcmk_rc_ok) { pcmk__notice("Rejecting remote client: could not find user '%s': %s", usr, pcmk_rc_str(rc)); return FALSE; } group = getgrgid(gid); if (group != NULL && pcmk__str_eq(grp, group->gr_name, pcmk__str_none)) { return TRUE; } group = getgrnam(grp); if (group == NULL) { pcmk__err("Rejecting remote client: '%s' is not a valid group", grp); return FALSE; } while (TRUE) { char *member = group->gr_mem[index++]; if (member == NULL) { break; } else if (pcmk__str_eq(usr, member, pcmk__str_none)) { return TRUE; } } pcmk__notice("Rejecting remote client: User '%s' is not a member of group " "'%s'", usr, grp); return FALSE; } static gboolean cib_remote_auth(xmlNode * login) { const char *user = NULL; const char *pass = NULL; const char *tmp = NULL; if (login == NULL) { return FALSE; } if (!pcmk__xe_is(login, PCMK__XE_CIB_COMMAND)) { pcmk__warn("Rejecting remote client: Unrecognizable message (element " "'%s' not '" PCMK__XE_CIB_COMMAND "')", login->name); pcmk__log_xml_debug(login, "bad"); return FALSE; } tmp = pcmk__xe_get(login, PCMK_XA_OP); if (!pcmk__str_eq(tmp, "authenticate", pcmk__str_casei)) { pcmk__warn("Rejecting remote client: Unrecognizable message (operation " "'%s' not 'authenticate')", tmp); pcmk__log_xml_debug(login, "bad"); return FALSE; } user = pcmk__xe_get(login, PCMK_XA_USER); pass = pcmk__xe_get(login, PCMK__XA_PASSWORD); if (!user || !pass) { pcmk__warn("Rejecting remote client: No %s given", ((user == NULL)? "username" : "password")); pcmk__log_xml_debug(login, "bad"); return FALSE; } pcmk__log_xml_debug(login, "auth"); return check_group_membership(user, CRM_DAEMON_GROUP) && authenticate_user(user, pass); } static gboolean remote_auth_timeout_cb(gpointer data) { pcmk__client_t *client = data; client->remote->auth_timeout = 0; if (pcmk__is_set(client->flags, pcmk__client_authenticated)) { return FALSE; } mainloop_del_fd(client->remote->source); pcmk__err("Remote client authentication timed out"); return FALSE; } static int cib_remote_listen(gpointer data) { int csock = -1; unsigned laddr; struct sockaddr_storage addr; char ipstr[INET6_ADDRSTRLEN]; int ssock = *(int *)data; int rc; pcmk__client_t *new_client = NULL; static struct mainloop_fd_callbacks remote_client_fd_callbacks = { .dispatch = cib_remote_msg, .destroy = cib_remote_connection_destroy, }; /* accept the connection */ laddr = sizeof(addr); memset(&addr, 0, sizeof(addr)); csock = accept(ssock, (struct sockaddr *)&addr, &laddr); if (csock == -1) { pcmk__warn("Could not accept remote connection: %s", pcmk_rc_str(errno)); return TRUE; } pcmk__sockaddr2str(&addr, ipstr); rc = pcmk__set_nonblocking(csock); if (rc != pcmk_rc_ok) { pcmk__warn("Dropping remote connection from %s because it could not be " "set to non-blocking: %s", ipstr, pcmk_rc_str(rc)); close(csock); return TRUE; } num_clients++; new_client = pcmk__new_unauth_client(NULL); new_client->remote = pcmk__assert_alloc(1, sizeof(pcmk__remote_t)); if (ssock == remote_tls_fd) { pcmk__set_client_flags(new_client, pcmk__client_tls); /* create gnutls session for the server socket */ new_client->remote->tls_session = pcmk__new_tls_session(tls, csock); if (new_client->remote->tls_session == NULL) { close(csock); return TRUE; } } else { pcmk__set_client_flags(new_client, pcmk__client_tcp); new_client->remote->tcp_socket = csock; } // Require the client to authenticate within this time new_client->remote->auth_timeout = pcmk__create_timer(REMOTE_AUTH_TIMEOUT, remote_auth_timeout_cb, new_client); pcmk__info("%s connection from %s pending authentication for client %s", ((ssock == remote_tls_fd)? "Encrypted" : "Clear-text"), ipstr, new_client->id); new_client->remote->source = mainloop_add_fd("cib-remote-client", G_PRIORITY_DEFAULT, csock, new_client, &remote_client_fd_callbacks); return TRUE; } void cib_remote_connection_destroy(gpointer user_data) { pcmk__client_t *client = user_data; int csock = -1; if (client == NULL) { return; } pcmk__trace("Cleaning up after client %s disconnect", pcmk__client_name(client)); num_clients--; pcmk__trace("Num unfree'd clients: %d", num_clients); switch (PCMK__CLIENT_TYPE(client)) { case pcmk__client_tcp: csock = client->remote->tcp_socket; break; case pcmk__client_tls: if (client->remote->tls_session) { csock = pcmk__tls_get_client_sock(client->remote); if (pcmk__is_set(client->flags, pcmk__client_tls_handshake_complete)) { gnutls_bye(client->remote->tls_session, GNUTLS_SHUT_WR); } gnutls_deinit(client->remote->tls_session); client->remote->tls_session = NULL; } break; default: pcmk__warn("Unknown transport for client %s " QB_XS " flags=%#016" PRIx64, pcmk__client_name(client), client->flags); } if (csock >= 0) { close(csock); } pcmk__free_client(client); pcmk__trace("Freed the cib client"); if (cib_shutdown_flag) { cib_shutdown(0); } return; } static void cib_handle_remote_msg(pcmk__client_t *client, xmlNode *command) { if (!pcmk__xe_is(command, PCMK__XE_CIB_COMMAND)) { - crm_log_xml_trace(command, "bad"); + pcmk__log_xml_trace(command, "bad"); return; } if (client->name == NULL) { client->name = pcmk__str_copy(client->id); } /* unset dangerous options */ pcmk__xe_remove_attr(command, PCMK__XA_SRC); pcmk__xe_remove_attr(command, PCMK__XA_CIB_HOST); pcmk__xe_remove_attr(command, PCMK__XA_CIB_UPDATE); pcmk__xe_set(command, PCMK__XA_T, PCMK__VALUE_CIB); pcmk__xe_set(command, PCMK__XA_CIB_CLIENTID, client->id); pcmk__xe_set(command, PCMK__XA_CIB_CLIENTNAME, client->name); pcmk__xe_set(command, PCMK__XA_CIB_USER, client->user); if (pcmk__xe_get(command, PCMK__XA_CIB_CALLID) == NULL) { char *call_uuid = pcmk__generate_uuid(); /* fix the command */ pcmk__xe_set(command, PCMK__XA_CIB_CALLID, call_uuid); free(call_uuid); } if (pcmk__xe_get(command, PCMK__XA_CIB_CALLOPT) == NULL) { pcmk__xe_set_int(command, PCMK__XA_CIB_CALLOPT, 0); } - crm_log_xml_trace(command, "Remote command: "); + pcmk__log_xml_trace(command, "Remote command: "); cib_common_callback_worker(0, 0, command, client, TRUE); } static int cib_remote_msg(gpointer data) { xmlNode *command = NULL; pcmk__client_t *client = data; int rc; const char *client_name = pcmk__client_name(client); pcmk__trace("Remote %s message received for client %s", pcmk__client_type_str(PCMK__CLIENT_TYPE(client)), client_name); if ((PCMK__CLIENT_TYPE(client) == pcmk__client_tls) && !pcmk__is_set(client->flags, pcmk__client_tls_handshake_complete)) { int rc = pcmk__read_handshake_data(client); if (rc == EAGAIN) { /* No more data is available at the moment. Just return for now; * we'll get invoked again once the client sends more. */ return 0; } else if (rc != pcmk_rc_ok) { return -1; } pcmk__debug("Completed TLS handshake with remote client %s", client_name); pcmk__set_client_flags(client, pcmk__client_tls_handshake_complete); if (client->remote->auth_timeout) { g_source_remove(client->remote->auth_timeout); } /* Now that the handshake is done, see if any client TLS certificate is * close to its expiration date and log if so. If a TLS certificate is not * in use, this function will just return so we don't need to check for the * session type here. */ pcmk__tls_check_cert_expiration(client->remote->tls_session); // Require the client to authenticate within this time client->remote->auth_timeout = pcmk__create_timer(REMOTE_AUTH_TIMEOUT, remote_auth_timeout_cb, client); return 0; } rc = pcmk__read_available_remote_data(client->remote); switch (rc) { case pcmk_rc_ok: break; case EAGAIN: /* We haven't read the whole message yet */ return 0; default: /* Error */ pcmk__trace("Error reading from remote client: %s", pcmk_rc_str(rc)); return -1; } /* must pass auth before we will process anything else */ if (!pcmk__is_set(client->flags, pcmk__client_authenticated)) { xmlNode *reg; const char *user = NULL; command = pcmk__remote_message_xml(client->remote); if (cib_remote_auth(command) == FALSE) { pcmk__xml_free(command); return -1; } pcmk__set_client_flags(client, pcmk__client_authenticated); g_source_remove(client->remote->auth_timeout); client->remote->auth_timeout = 0; client->name = pcmk__xe_get_copy(command, PCMK_XA_NAME); user = pcmk__xe_get(command, PCMK_XA_USER); if (user) { client->user = pcmk__str_copy(user); } pcmk__notice("Remote connection accepted for authenticated user %s " QB_XS " client %s", pcmk__s(user, ""), client_name); /* send ACK */ reg = pcmk__xe_create(NULL, PCMK__XE_CIB_RESULT); pcmk__xe_set(reg, PCMK__XA_CIB_OP, CRM_OP_REGISTER); pcmk__xe_set(reg, PCMK__XA_CIB_CLIENTID, client->id); pcmk__remote_send_xml(client->remote, reg); pcmk__xml_free(reg); pcmk__xml_free(command); } command = pcmk__remote_message_xml(client->remote); if (command != NULL) { pcmk__trace("Remote message received from client %s", client_name); cib_handle_remote_msg(client, command); pcmk__xml_free(command); } return 0; } #ifdef HAVE_PAM /*! * \internal * \brief Pass remote user's password to PAM * * \param[in] num_msg Number of entries in \p msg * \param[in] msg Array of PAM messages * \param[out] response Where to set response to PAM * \param[in] data User data (the password string) * * \return PAM return code (PAM_BUF_ERR for memory errors, PAM_CONV_ERR for all * other errors, or PAM_SUCCESS on success) * \note See pam_conv(3) for more explanation */ static int construct_pam_passwd(int num_msg, const struct pam_message **msg, struct pam_response **response, void *data) { /* In theory, multiple messages are allowed, but due to OS compatibility * issues, PAM implementations are recommended to only send one message at a * time. We can require that here for simplicity. */ CRM_CHECK((num_msg == 1) && (msg != NULL) && (response != NULL) && (data != NULL), return PAM_CONV_ERR); switch (msg[0]->msg_style) { case PAM_PROMPT_ECHO_OFF: case PAM_PROMPT_ECHO_ON: // Password requested break; case PAM_TEXT_INFO: pcmk__info("PAM: %s", msg[0]->msg); data = NULL; break; case PAM_ERROR_MSG: /* In theory we should show msg[0]->msg, but that might * contain the password, which we don't want in the logs */ pcmk__err("PAM reported an error"); data = NULL; break; default: pcmk__warn("Ignoring PAM message of unrecognized type %d", msg[0]->msg_style); return PAM_CONV_ERR; } *response = calloc(1, sizeof(struct pam_response)); if (*response == NULL) { return PAM_BUF_ERR; } (*response)->resp_retcode = 0; (*response)->resp = pcmk__str_copy((const char *) data); // Caller will free return PAM_SUCCESS; } #endif /*! * \internal * \brief Verify the username and password passed for a remote CIB connection * * \param[in] user Username passed for remote CIB connection * \param[in] passwd Password passed for remote CIB connection * * \return \c true if the username and password are accepted, otherwise \c false * \note This function rejects all credentials when built without PAM support. */ static bool authenticate_user(const char *user, const char *passwd) { #ifdef HAVE_PAM int rc = 0; bool pass = false; const void *p_user = NULL; struct pam_conv p_conv; struct pam_handle *pam_h = NULL; static const char *pam_name = NULL; if (pam_name == NULL) { pam_name = getenv("CIB_pam_service"); if (pam_name == NULL) { pam_name = "login"; } } p_conv.conv = construct_pam_passwd; p_conv.appdata_ptr = (void *) passwd; rc = pam_start(pam_name, user, &p_conv, &pam_h); if (rc != PAM_SUCCESS) { pcmk__warn("Rejecting remote client for user %s because PAM " "initialization failed: %s", user, pam_strerror(pam_h, rc)); goto bail; } // Check user credentials rc = pam_authenticate(pam_h, PAM_SILENT); if (rc != PAM_SUCCESS) { pcmk__notice("Access for remote user %s denied: %s", user, pam_strerror(pam_h, rc)); goto bail; } /* Get the authenticated user name (PAM modules can map the original name to * something else). Since the CIB manager runs as the daemon user (not * root), that is the only user that can be successfully authenticated. */ rc = pam_get_item(pam_h, PAM_USER, &p_user); if (rc != PAM_SUCCESS) { pcmk__warn("Rejecting remote client for user %s because PAM failed to " "return final user name: %s", user, pam_strerror(pam_h, rc)); goto bail; } if (p_user == NULL) { pcmk__warn("Rejecting remote client for user %s because PAM returned " "no final user name", user); goto bail; } // @TODO Why do we require these to match? if (!pcmk__str_eq(p_user, user, pcmk__str_none)) { pcmk__warn("Rejecting remote client for user %s because PAM returned " "different final user name %s", user, p_user); goto bail; } // Check user account restrictions (expiration, etc.) rc = pam_acct_mgmt(pam_h, PAM_SILENT); if (rc != PAM_SUCCESS) { pcmk__notice("Access for remote user %s denied: %s", user, pam_strerror(pam_h, rc)); goto bail; } pass = true; bail: pam_end(pam_h, rc); return pass; #else // @TODO Implement for non-PAM environments pcmk__warn("Rejecting remote user %s because this build does not have PAM " "support", user); return false; #endif } diff --git a/daemons/based/based_transaction.c b/daemons/based/based_transaction.c index 4d1562d09f..ca3c660dfe 100644 --- a/daemons/based/based_transaction.c +++ b/daemons/based/based_transaction.c @@ -1,164 +1,164 @@ /* * Copyright 2023-2025 the Pacemaker project contributors * * The version control history for this file may have further details. * * This source code is licensed under the GNU General Public License version 2 * or later (GPLv2+) WITHOUT ANY WARRANTY. */ #include #include #include #include "pacemaker-based.h" /*! * \internal * \brief Create a string describing the source of a commit-transaction request * * \param[in] client CIB client * \param[in] origin Host where the commit request originated * * \return String describing the request source * * \note The caller is responsible for freeing the return value using \c free(). */ char * based_transaction_source_str(const pcmk__client_t *client, const char *origin) { if (client != NULL) { return pcmk__assert_asprintf("client %s (%s)%s%s", pcmk__client_name(client), pcmk__s(client->id, "unidentified"), ((origin != NULL)? " on " : ""), pcmk__s(origin, "")); } else { return pcmk__str_copy(pcmk__s(origin, "unknown source")); } } /*! * \internal * \brief Process requests in a transaction * * Stop when a request fails or when all requests have been processed. * * \param[in,out] transaction Transaction to process * \param[in] client CIB client * \param[in] source String describing the commit request source * * \return Standard Pacemaker return code */ static int process_transaction_requests(xmlNodePtr transaction, const pcmk__client_t *client, const char *source) { for (xmlNode *request = pcmk__xe_first_child(transaction, PCMK__XE_CIB_COMMAND, NULL, NULL); request != NULL; request = pcmk__xe_next(request, PCMK__XE_CIB_COMMAND)) { const char *op = pcmk__xe_get(request, PCMK__XA_CIB_OP); const char *host = pcmk__xe_get(request, PCMK__XA_CIB_HOST); const cib__operation_t *operation = NULL; int rc = cib__get_operation(op, &operation); if (rc == pcmk_rc_ok) { if (!pcmk__is_set(operation->flags, cib__op_attr_transaction) || (host != NULL)) { rc = EOPNOTSUPP; } else { /* Commit-transaction is a privileged operation. If we reached * this point, the request came from a privileged connection. */ rc = cib_process_request(request, TRUE, client); rc = pcmk_legacy2rc(rc); } } if (rc != pcmk_rc_ok) { pcmk__err("Aborting CIB transaction for %s due to failed %s " "request: %s", source, op, pcmk_rc_str(rc)); pcmk__log_xml_info(request, "Failed request"); return rc; } pcmk__trace("Applied %s request to transaction working CIB for %s", op, source); - crm_log_xml_trace(request, "Successful request"); + pcmk__log_xml_trace(request, "Successful request"); } return pcmk_rc_ok; } /*! * \internal * \brief Commit a given CIB client's transaction to a working CIB copy * * \param[in] transaction Transaction to commit * \param[in] client CIB client * \param[in] origin Host where the commit request originated * \param[in,out] result_cib Where to store result CIB * * \return Standard Pacemaker return code * * \note This function is expected to be called only by * \p cib_process_commit_transaction(). * \note \p result_cib is expected to be a copy of the current CIB as created by * \p cib_perform_op(). * \note The caller is responsible for activating and syncing \p result_cib on * success, and for freeing it on failure. */ int based_commit_transaction(xmlNodePtr transaction, const pcmk__client_t *client, const char *origin, xmlNodePtr *result_cib) { xmlNodePtr saved_cib = the_cib; int rc = pcmk_rc_ok; char *source = NULL; pcmk__assert(result_cib != NULL); CRM_CHECK(pcmk__xe_is(transaction, PCMK__XE_CIB_TRANSACTION), return pcmk_rc_no_transaction); /* *result_cib should be a copy of the_cib (created by cib_perform_op()). If * not, make a copy now. Change tracking isn't strictly required here * because: * * Each request in the transaction will have changes tracked and ACLs * checked if appropriate. * * cib_perform_op() will infer changes for the commit request at the end. */ CRM_CHECK((*result_cib != NULL) && (*result_cib != the_cib), *result_cib = pcmk__xml_copy(NULL, the_cib)); source = based_transaction_source_str(client, origin); pcmk__trace("Committing transaction for %s to working CIB", source); // Apply all changes to a working copy of the CIB the_cib = *result_cib; rc = process_transaction_requests(transaction, client, origin); pcmk__trace("Transaction commit %s for %s", ((rc == pcmk_rc_ok)? "succeeded" : "failed"), source); /* Some request types (for example, erase) may have freed the_cib (the * working copy) and pointed it at a new XML object. In that case, it * follows that *result_cib (the working copy) was freed. * * Point *result_cib at the updated working copy stored in the_cib. */ *result_cib = the_cib; // Point the_cib back to the unchanged original copy the_cib = saved_cib; free(source); return rc; } diff --git a/daemons/controld/controld_callbacks.c b/daemons/controld/controld_callbacks.c index 58ede25cfc..a226cc8e4b 100644 --- a/daemons/controld/controld_callbacks.c +++ b/daemons/controld/controld_callbacks.c @@ -1,403 +1,403 @@ /* * Copyright 2004-2025 the Pacemaker project contributors * * The version control history for this file may have further details. * * This source code is licensed under the GNU General Public License version 2 * or later (GPLv2+) WITHOUT ANY WARRANTY. */ #include #include // PRIu32 #include // bool #include // NULL #include #include #include #include // PCMK_SCORE_INFINITY #include #include #include #include /* From join_dc... */ extern gboolean check_join_state(enum crmd_fsa_state cur_state, const char *source); void crmd_ha_msg_filter(xmlNode * msg) { if (AM_I_DC) { const char *sys_from = pcmk__xe_get(msg, PCMK__XA_CRM_SYS_FROM); if (pcmk__str_eq(sys_from, CRM_SYSTEM_DC, pcmk__str_casei)) { const char *from = pcmk__xe_get(msg, PCMK__XA_SRC); if (!controld_is_local_node(from)) { int level = LOG_INFO; const char *op = pcmk__xe_get(msg, PCMK__XA_CRM_TASK); /* make sure the election happens NOW */ if (controld_globals.fsa_state != S_ELECTION) { ha_msg_input_t new_input; level = LOG_WARNING; new_input.msg = msg; register_fsa_error_adv(C_FSA_INTERNAL, I_ELECTION, NULL, &new_input, __func__); } do_crm_log(level, "Another DC detected: %s (op=%s)", from, op); goto done; } } } else { const char *sys_to = pcmk__xe_get(msg, PCMK__XA_CRM_SYS_TO); if (pcmk__str_eq(sys_to, CRM_SYSTEM_DC, pcmk__str_casei)) { return; } } - /* crm_log_xml_trace(msg, "HA[inbound]"); */ + // pcmk__log_xml_trace(msg, "HA[inbound]"); route_message(C_HA_MESSAGE, msg); done: controld_trigger_fsa(); } /*! * \internal * \brief Check whether a node is online * * \param[in] node Node to check * * \retval -1 if completely dead * \retval 0 if partially alive * \retval 1 if completely alive */ static int node_alive(const pcmk__node_status_t *node) { if (pcmk__is_set(node->flags, pcmk__node_status_remote)) { // Pacemaker Remote nodes can't be partially alive if (pcmk__str_eq(node->state, PCMK_VALUE_MEMBER, pcmk__str_none)) { return 1; } return -1; } else if (pcmk__cluster_is_node_active(node)) { // Completely up cluster node: both cluster member and peer return 1; } else if (!pcmk__is_set(node->processes, crm_get_cluster_proc()) && !pcmk__str_eq(node->state, PCMK_VALUE_MEMBER, pcmk__str_none)) { // Completely down cluster node: neither cluster member nor peer return -1; } // Partially up cluster node: only cluster member or only peer return 0; } #define state_text(state) ((state)? (const char *)(state) : "in unknown state") // @TODO This is insanely long, and some parts should be functionized void peer_update_callback(enum pcmk__node_update type, pcmk__node_status_t *node, const void *data) { uint32_t old = 0; bool appeared = FALSE; bool is_remote = pcmk__is_set(node->flags, pcmk__node_status_remote); controld_node_pending_timer(node); /* The controller waits to receive some information from the membership * layer before declaring itself operational. If this is being called for a * cluster node, indicate that we have it. */ if (!is_remote) { controld_set_fsa_input_flags(R_PEER_DATA); } if ((type == pcmk__node_update_processes) && pcmk__is_set(node->processes, crm_get_cluster_proc()) && !AM_I_DC && !is_remote) { /* relay_message() on the recipient ignores these messages, but * libcrmcluster will have cached the node name by then */ xmlNode *query = pcmk__new_request(pcmk_ipc_controld, CRM_SYSTEM_CRMD, NULL, CRM_SYSTEM_CRMD, CRM_OP_HELLO, NULL); pcmk__debug("Sending hello to node %" PRIu32 " so that it learns our " "node name", node->cluster_layer_id); pcmk__cluster_send_message(node, pcmk_ipc_controld, query); pcmk__xml_free(query); } if (node->name == NULL) { return; } switch (type) { case pcmk__node_update_name: /* If we've never seen the node, then it also won't be in the status section */ pcmk__info("%s node %s is now %s", (is_remote? "Remote" : "Cluster"), node->name, state_text(node->state)); return; case pcmk__node_update_state: /* This callback should not be called unless the state actually * changed, but here's a failsafe just in case. */ CRM_CHECK(!pcmk__str_eq(data, node->state, pcmk__str_casei), return); pcmk__info("%s node %s is now %s (was %s)", (is_remote? "Remote" : "Cluster"), node->name, state_text(node->state), state_text(data)); if (pcmk__str_eq(PCMK_VALUE_MEMBER, node->state, pcmk__str_none)) { appeared = TRUE; if (!is_remote) { remove_stonith_cleanup(node->name); } } else { controld_remove_failed_sync_node(node->name); controld_remove_voter(node->name); } crmd_alert_node_event(node); break; case pcmk__node_update_processes: CRM_CHECK(data != NULL, return); old = *(const uint32_t *)data; appeared = pcmk__is_set(node->processes, crm_get_cluster_proc()); { const char *dc_s = controld_globals.dc_name; if ((dc_s == NULL) && AM_I_DC) { dc_s = PCMK_VALUE_TRUE; } pcmk__info("Node %s is %s a peer " QB_XS " DC=%s old=%#07x new=%#07x", node->name, (appeared? "now" : "no longer"), pcmk__s(dc_s, ""), old, node->processes); } if (!pcmk__is_set((node->processes ^ old), crm_get_cluster_proc())) { /* Peer status did not change. This should not be possible, * since we don't track process flags other than peer status. */ pcmk__trace("Process flag %#7x did not change from %#7x to " "%#7x", crm_get_cluster_proc(), old, node->processes); return; } if (!appeared) { node->peer_lost = time(NULL); controld_remove_failed_sync_node(node->name); controld_remove_voter(node->name); } if (!pcmk__is_set(controld_globals.fsa_input_register, R_CIB_CONNECTED)) { pcmk__trace("Ignoring peer status change because not connected " "to CIB"); return; } else if (controld_globals.fsa_state == S_STOPPING) { pcmk__trace("Ignoring peer status change because stopping"); return; } if (!appeared && controld_is_local_node(node->name)) { /* Did we get evicted? */ pcmk__notice("Our peer connection failed"); register_fsa_input(C_CRMD_STATUS_CALLBACK, I_ERROR, NULL); } else if (pcmk__str_eq(node->name, controld_globals.dc_name, pcmk__str_casei) && !pcmk__cluster_is_node_active(node)) { /* The DC has left, so delete its transient attributes and * trigger a new election. * * A DC sends its shutdown request to all peers, who update the * DC's expected state to down. This avoids fencing upon * deletion of its transient attributes. */ pcmk__notice("Our peer on the DC (%s) is dead", controld_globals.dc_name); register_fsa_input(C_CRMD_STATUS_CALLBACK, I_ELECTION, NULL); controld_delete_node_state(node->name, controld_section_attrs, cib_none); } else if (AM_I_DC || pcmk__is_set(controld_globals.flags, controld_dc_left) || (controld_globals.dc_name == NULL)) { /* This only needs to be done once, so normally the DC should do * it. However if there is no DC, every node must do it, since * there is no other way to ensure some one node does it. */ if (appeared) { te_trigger_stonith_history_sync(FALSE); } else { controld_delete_node_state(node->name, controld_section_attrs, cib_none); } } break; } if (AM_I_DC) { xmlNode *update = NULL; int flags = node_update_peer; int alive = node_alive(node); pcmk__graph_action_t *down = match_down_event(node->xml_id); pcmk__trace("Alive=%d, appeared=%d, down=%d", alive, appeared, ((down != NULL)? down->id : -1)); if (appeared && (alive > 0) && !is_remote) { register_fsa_input_before(C_FSA_INTERNAL, I_NODE_JOIN, NULL); } if (down) { const char *task = pcmk__xe_get(down->xml, PCMK_XA_OPERATION); if (pcmk__str_eq(task, PCMK_ACTION_STONITH, pcmk__str_casei)) { const bool confirmed = pcmk__is_set(down->flags, pcmk__graph_action_confirmed); /* tengine_stonith_callback() confirms fence actions */ pcmk__trace("Updating CIB %s fencer reported fencing of %s " "complete", (confirmed? "after" : "before"), node->name); } else if (!appeared && pcmk__str_eq(task, PCMK_ACTION_DO_SHUTDOWN, pcmk__str_casei)) { // Shutdown actions are immediately confirmed (i.e. no_wait) if (!is_remote) { flags |= node_update_join | node_update_expected; crmd_peer_down(node, FALSE); check_join_state(controld_globals.fsa_state, __func__); } if (alive >= 0) { pcmk__info("%s of peer %s is in progress " QB_XS " action=%d", task, node->name, down->id); } else { pcmk__notice("%s of peer %s is complete " QB_XS " action=%d", task, node->name, down->id); pcmk__update_graph(controld_globals.transition_graph, down); trigger_graph(); } } else { const char *liveness = "alive"; if (alive == 0) { liveness = "partially alive"; } else if (alive < 0) { liveness = "dead"; } pcmk__trace("Node %s is %s, was expected to %s (op %d)", node->name, liveness, task, down->id); } } else if (appeared == FALSE) { if ((controld_globals.transition_graph == NULL) || (controld_globals.transition_graph->id <= 0)) { pcmk__info("Stonith/shutdown of node %s is unknown to the " "current DC", node->name); } else { pcmk__warn("Stonith/shutdown of node %s was not expected", node->name); } if (!is_remote) { crm_update_peer_join(__func__, node, controld_join_none); check_join_state(controld_globals.fsa_state, __func__); } abort_transition(PCMK_SCORE_INFINITY, pcmk__graph_restart, "Node failure", NULL); fail_incompletable_actions(controld_globals.transition_graph, node->xml_id); } else { pcmk__trace("Node %s came up, was not expected to be down", node->name); } if (is_remote) { /* A pacemaker_remote node won't have its cluster status updated * in the CIB by membership-layer callbacks, so do it here. */ flags |= node_update_cluster; /* Trigger resource placement on newly integrated nodes */ if (appeared) { abort_transition(PCMK_SCORE_INFINITY, pcmk__graph_restart, "Pacemaker Remote node integrated", NULL); } } if (!appeared && (type == pcmk__node_update_processes) && (node->when_member > 1)) { /* The node left CPG but is still a cluster member. Set its * membership time to 1 to record it in the cluster state as a * boolean, so we don't fence it due to * PCMK_OPT_NODE_PENDING_TIMEOUT. */ node->when_member = 1; flags |= node_update_cluster; controld_node_pending_timer(node); } /* Update the CIB node state */ update = create_node_state_update(node, flags, NULL, __func__); if (update == NULL) { pcmk__debug("Node state update not yet possible for %s", node->name); } else { fsa_cib_anon_update(PCMK_XE_STATUS, update); } pcmk__xml_free(update); } controld_trigger_fsa(); } gboolean crm_fsa_trigger(gpointer user_data) { pcmk__trace("Invoked (queue len: %u)", g_list_length(controld_globals.fsa_message_queue)); s_crmd_fsa(C_FSA_INTERNAL); pcmk__trace("Exited (queue len: %u)", g_list_length(controld_globals.fsa_message_queue)); return TRUE; } diff --git a/daemons/controld/controld_cib.c b/daemons/controld/controld_cib.c index 91e95835e2..559a2d1107 100644 --- a/daemons/controld/controld_cib.c +++ b/daemons/controld/controld_cib.c @@ -1,1073 +1,1073 @@ /* * Copyright 2004-2025 the Pacemaker project contributors * * The version control history for this file may have further details. * * This source code is licensed under the GNU General Public License version 2 * or later (GPLv2+) WITHOUT ANY WARRANTY. */ #include #include /* sleep */ #include #include // crm_meta_value() #include #include #include #include // Call ID of the most recent in-progress CIB resource update (or 0 if none) static int pending_rsc_update = 0; /*! * \internal * \brief Respond to a dropped CIB connection * * \param[in] user_data CIB connection that dropped */ static void handle_cib_disconnect(gpointer user_data) { CRM_LOG_ASSERT(user_data == controld_globals.cib_conn); controld_trigger_fsa(); controld_globals.cib_conn->state = cib_disconnected; if (pcmk__is_set(controld_globals.fsa_input_register, R_CIB_CONNECTED)) { // @TODO This should trigger a reconnect, not a shutdown pcmk__crit("Lost connection to the CIB manager, shutting down"); register_fsa_input(C_FSA_INTERNAL, I_ERROR, NULL); controld_clear_fsa_input_flags(R_CIB_CONNECTED); } else { // Expected pcmk__info("Disconnected from the CIB manager"); } } static void do_cib_updated(const char *event, xmlNode * msg) { const xmlNode *patchset = NULL; const char *client_name = NULL; pcmk__debug("Received CIB diff notification: DC=%s", pcmk__btoa(AM_I_DC)); if (cib__get_notify_patchset(msg, &patchset) != pcmk_rc_ok) { return; } if (pcmk__cib_element_in_patchset(patchset, PCMK_XE_ALERTS) || pcmk__cib_element_in_patchset(patchset, PCMK_XE_CRM_CONFIG)) { controld_trigger_config(); } if (!AM_I_DC) { // We're not in control of the join sequence return; } client_name = pcmk__xe_get(msg, PCMK__XA_CIB_CLIENTNAME); if (!cib__client_triggers_refresh(client_name)) { // The CIB is still accurate return; } if (pcmk__cib_element_in_patchset(patchset, PCMK_XE_NODES) || pcmk__cib_element_in_patchset(patchset, PCMK_XE_STATUS)) { /* An unsafe client modified the PCMK_XE_NODES or PCMK_XE_STATUS * section. Ensure the node list is up-to-date, and start the join * process again so we get everyone's current resource history. */ if (client_name == NULL) { client_name = pcmk__xe_get(msg, PCMK__XA_CIB_CLIENTID); } pcmk__notice("Populating nodes and starting an election after %s event " "triggered by %s", event, pcmk__s(client_name, "(unidentified client)")); populate_cib_nodes(node_update_quick|node_update_all, __func__); register_fsa_input(C_FSA_INTERNAL, I_ELECTION, NULL); } } void controld_disconnect_cib_manager(void) { cib_t *cib_conn = controld_globals.cib_conn; pcmk__assert(cib_conn != NULL); pcmk__debug("Disconnecting from the CIB manager"); controld_clear_fsa_input_flags(R_CIB_CONNECTED); cib_conn->cmds->del_notify_callback(cib_conn, PCMK__VALUE_CIB_DIFF_NOTIFY, do_cib_updated); cib_free_callbacks(cib_conn); if (cib_conn->state != cib_disconnected) { cib_conn->cmds->set_secondary(cib_conn, cib_discard_reply); cib_conn->cmds->signoff(cib_conn); } } /* A_CIB_STOP, A_CIB_START, O_CIB_RESTART */ void do_cib_control(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data) { static int cib_retries = 0; cib_t *cib_conn = controld_globals.cib_conn; void (*dnotify_fn) (gpointer user_data) = handle_cib_disconnect; void (*update_cb) (const char *event, xmlNodePtr msg) = do_cib_updated; int rc = pcmk_ok; pcmk__assert(cib_conn != NULL); if (pcmk__is_set(action, A_CIB_STOP)) { if ((cib_conn->state != cib_disconnected) && (pending_rsc_update != 0)) { pcmk__info("Waiting for resource update %d to complete", pending_rsc_update); crmd_fsa_stall(FALSE); return; } controld_disconnect_cib_manager(); } if (!pcmk__is_set(action, A_CIB_START)) { return; } if (cur_state == S_STOPPING) { pcmk__err("Ignoring request to connect to the CIB manager after " "shutdown"); return; } rc = cib_conn->cmds->signon(cib_conn, crm_system_name, cib_command_nonblocking); if (rc != pcmk_ok) { // A short wait that usually avoids stalling the FSA sleep(1); rc = cib_conn->cmds->signon(cib_conn, crm_system_name, cib_command_nonblocking); } if (rc != pcmk_ok) { pcmk__info("Could not connect to the CIB manager: %s", pcmk_strerror(rc)); } else if (cib_conn->cmds->set_connection_dnotify(cib_conn, dnotify_fn) != pcmk_ok) { pcmk__err("Could not set dnotify callback"); } else if (cib_conn->cmds->add_notify_callback(cib_conn, PCMK__VALUE_CIB_DIFF_NOTIFY, update_cb) != pcmk_ok) { pcmk__err("Could not set CIB notification callback (update)"); } else { controld_set_fsa_input_flags(R_CIB_CONNECTED); cib_retries = 0; } if (!pcmk__is_set(controld_globals.fsa_input_register, R_CIB_CONNECTED)) { cib_retries++; if (cib_retries < 30) { pcmk__warn("Couldn't complete CIB registration %d times... pause " "and retry", cib_retries); controld_start_wait_timer(); crmd_fsa_stall(FALSE); } else { pcmk__err("Could not complete CIB registration %d times... hard " "error", cib_retries); register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL); } } } #define MIN_CIB_OP_TIMEOUT (30) /*! * \internal * \brief Get the timeout (in seconds) that should be used with CIB operations * * \return The maximum of 30 seconds, the value of the PCMK_cib_timeout * environment variable, or 10 seconds times one more than the number of * nodes in the cluster. */ unsigned int cib_op_timeout(void) { unsigned int calculated_timeout = 10U * (pcmk__cluster_num_active_nodes() + pcmk__cluster_num_remote_nodes() + 1U); calculated_timeout = QB_MAX(calculated_timeout, MIN_CIB_OP_TIMEOUT); pcmk__trace("Calculated timeout: %s", pcmk__readable_interval(calculated_timeout * 1000)); if (controld_globals.cib_conn) { controld_globals.cib_conn->call_timeout = calculated_timeout; } return calculated_timeout; } /*! * \internal * \brief Get CIB call options to use local scope if primary is unavailable * * \return CIB call options */ int crmd_cib_smart_opt(void) { int call_opt = cib_none; if ((controld_globals.fsa_state == S_ELECTION) || (controld_globals.fsa_state == S_PENDING)) { pcmk__info("Sending update to local CIB in state: %s", fsa_state2string(controld_globals.fsa_state)); cib__set_call_options(call_opt, "update", cib_none); } return call_opt; } static void cib_delete_callback(xmlNode *msg, int call_id, int rc, xmlNode *output, void *user_data) { char *desc = user_data; if (rc == 0) { pcmk__debug("Deletion of %s (via CIB call %d) succeeded", desc, call_id); } else { pcmk__warn("Deletion of %s (via CIB call %d) failed: %s " QB_XS " rc=%d", desc, call_id, pcmk_strerror(rc), rc); } } // Searches for various portions of PCMK__XE_NODE_STATE to delete // Match a particular node's PCMK__XE_NODE_STATE (takes node name 1x) #define XPATH_NODE_STATE "//" PCMK__XE_NODE_STATE "[@" PCMK_XA_UNAME "='%s']" // Node's lrm section (name 1x) #define XPATH_NODE_LRM XPATH_NODE_STATE "/" PCMK__XE_LRM /* Node's PCMK__XE_LRM_RSC_OP entries and PCMK__XE_LRM_RESOURCE entries without * unexpired lock * (name 2x, (seconds_since_epoch - PCMK_OPT_SHUTDOWN_LOCK_LIMIT) 1x) */ #define XPATH_NODE_LRM_UNLOCKED XPATH_NODE_STATE "//" PCMK__XE_LRM_RSC_OP \ "|" XPATH_NODE_STATE \ "//" PCMK__XE_LRM_RESOURCE \ "[not(@" PCMK_OPT_SHUTDOWN_LOCK ") " \ "or " PCMK_OPT_SHUTDOWN_LOCK "<%lld]" // Node's PCMK__XE_TRANSIENT_ATTRIBUTES section (name 1x) #define XPATH_NODE_ATTRS XPATH_NODE_STATE "/" PCMK__XE_TRANSIENT_ATTRIBUTES // Everything under PCMK__XE_NODE_STATE (name 1x) #define XPATH_NODE_ALL XPATH_NODE_STATE "/*" /* Unlocked history + transient attributes * (name 2x, (seconds_since_epoch - PCMK_OPT_SHUTDOWN_LOCK_LIMIT) 1x, name 1x) */ #define XPATH_NODE_ALL_UNLOCKED XPATH_NODE_LRM_UNLOCKED "|" XPATH_NODE_ATTRS /*! * \internal * \brief Get the XPath and description of a node state section to be deleted * * \param[in] uname Desired node * \param[in] section Subsection of \c PCMK__XE_NODE_STATE to be deleted * \param[out] xpath Where to store XPath of \p section * \param[out] desc If not \c NULL, where to store description of \p section */ void controld_node_state_deletion_strings(const char *uname, enum controld_section_e section, char **xpath, char **desc) { const char *desc_pre = NULL; // Shutdown locks that started before this time are expired long long expire = (long long) time(NULL) - controld_globals.shutdown_lock_limit; switch (section) { case controld_section_lrm: *xpath = pcmk__assert_asprintf(XPATH_NODE_LRM, uname); desc_pre = "resource history"; break; case controld_section_lrm_unlocked: *xpath = pcmk__assert_asprintf(XPATH_NODE_LRM_UNLOCKED, uname, uname, expire); desc_pre = "resource history (other than shutdown locks)"; break; case controld_section_attrs: *xpath = pcmk__assert_asprintf(XPATH_NODE_ATTRS, uname); desc_pre = "transient attributes"; break; case controld_section_all: *xpath = pcmk__assert_asprintf(XPATH_NODE_ALL, uname); desc_pre = "all state"; break; case controld_section_all_unlocked: *xpath = pcmk__assert_asprintf(XPATH_NODE_ALL_UNLOCKED, uname, uname, expire, uname); desc_pre = "all state (other than shutdown locks)"; break; default: // We called this function incorrectly pcmk__assert(false); break; } if (desc != NULL) { *desc = pcmk__assert_asprintf("%s for node %s", desc_pre, uname); } } /*! * \internal * \brief Delete subsection of a node's CIB \c PCMK__XE_NODE_STATE * * \param[in] uname Desired node * \param[in] section Subsection of \c PCMK__XE_NODE_STATE to delete * \param[in] options CIB call options to use */ void controld_delete_node_state(const char *uname, enum controld_section_e section, int options) { cib_t *cib = controld_globals.cib_conn; char *xpath = NULL; char *desc = NULL; int cib_rc = pcmk_ok; pcmk__assert((uname != NULL) && (cib != NULL)); controld_node_state_deletion_strings(uname, section, &xpath, &desc); cib__set_call_options(options, "node state deletion", cib_xpath|cib_multiple); cib_rc = cib->cmds->remove(cib, xpath, NULL, options); fsa_register_cib_callback(cib_rc, desc, cib_delete_callback); pcmk__info("Deleting %s (via CIB call %d) " QB_XS " xpath=%s", desc, cib_rc, xpath); // CIB library handles freeing desc free(xpath); } // Takes node name and resource ID #define XPATH_RESOURCE_HISTORY "//" PCMK__XE_NODE_STATE \ "[@" PCMK_XA_UNAME "='%s']/" \ PCMK__XE_LRM "/" PCMK__XE_LRM_RESOURCES \ "/" PCMK__XE_LRM_RESOURCE \ "[@" PCMK_XA_ID "='%s']" // @TODO could add "and @PCMK_OPT_SHUTDOWN_LOCK" to limit to locks /*! * \internal * \brief Clear resource history from CIB for a given resource and node * * \param[in] rsc_id ID of resource to be cleared * \param[in] node Node whose resource history should be cleared * \param[in] user_name ACL user name to use * \param[in] call_options CIB call options * * \return Standard Pacemaker return code */ int controld_delete_resource_history(const char *rsc_id, const char *node, const char *user_name, int call_options) { char *desc = NULL; char *xpath = NULL; int rc = pcmk_rc_ok; cib_t *cib = controld_globals.cib_conn; CRM_CHECK((rsc_id != NULL) && (node != NULL), return EINVAL); desc = pcmk__assert_asprintf("resource history for %s on %s", rsc_id, node); if (cib == NULL) { pcmk__err("Unable to clear %s: no CIB connection", desc); free(desc); return ENOTCONN; } // Ask CIB to delete the entry xpath = pcmk__assert_asprintf(XPATH_RESOURCE_HISTORY, node, rsc_id); cib->cmds->set_user(cib, user_name); rc = cib->cmds->remove(cib, xpath, NULL, call_options|cib_xpath); cib->cmds->set_user(cib, NULL); if (rc < 0) { rc = pcmk_legacy2rc(rc); pcmk__err("Could not delete resource status of %s on %s%s%s: %s " QB_XS " rc=%d", rsc_id, node, ((user_name != NULL)? " for user " : ""), pcmk__s(user_name, ""), pcmk_rc_str(rc), rc); free(desc); free(xpath); return rc; } if (pcmk__is_set(call_options, cib_sync_call)) { if (pcmk__is_set(call_options, cib_dryrun)) { pcmk__debug("Deletion of %s would succeed", desc); } else { pcmk__debug("Deletion of %s succeeded", desc); } free(desc); } else { pcmk__info("Clearing %s (via CIB call %d) " QB_XS " xpath=%s", desc, rc, xpath); fsa_register_cib_callback(rc, desc, cib_delete_callback); // CIB library handles freeing desc } free(xpath); return pcmk_rc_ok; } /*! * \internal * \brief Build XML and string of parameters meeting some criteria, for digest * * \param[in] op Executor event with parameter table to use * \param[in] metadata Parsed meta-data for executed resource agent * \param[in] param_type Flag used for selection criteria * \param[out] result Will be set to newly created XML with selected * parameters as attributes * * \return Newly allocated space-separated string of parameter names * \note Selection criteria varies by param_type: for the restart digest, we * want parameters that are *not* marked reloadable (OCF 1.1) or that * *are* marked unique (pre-1.1), for both string and XML results; for the * secure digest, we want parameters that *are* marked private for the * string, but parameters that are *not* marked private for the XML. * \note It is the caller's responsibility to free the string return value with * \p g_string_free() and the XML result with \p pcmk__xml_free(). */ static GString * build_parameter_list(const lrmd_event_data_t *op, const struct ra_metadata_s *metadata, enum ra_param_flags_e param_type, xmlNode **result) { GString *list = NULL; *result = pcmk__xe_create(NULL, PCMK_XE_PARAMETERS); /* Consider all parameters only except private ones to be consistent with * what scheduler does with calculate_secure_digest(). */ if ((param_type == ra_param_private) && (pcmk__compare_versions(controld_globals.dc_version, "3.16.0") >= 0)) { g_hash_table_foreach(op->params, hash2field, *result); pcmk__filter_op_for_digest(*result); } for (GList *iter = metadata->ra_params; iter != NULL; iter = iter->next) { struct ra_param_s *param = (struct ra_param_s *) iter->data; bool accept_for_list = false; bool accept_for_xml = false; switch (param_type) { case ra_param_reloadable: accept_for_list = !pcmk__is_set(param->rap_flags, param_type); accept_for_xml = accept_for_list; break; case ra_param_unique: accept_for_list = pcmk__is_set(param->rap_flags, param_type); accept_for_xml = accept_for_list; break; case ra_param_private: accept_for_list = pcmk__is_set(param->rap_flags, param_type); accept_for_xml = !accept_for_list; break; } if (accept_for_list) { pcmk__trace("Attr %s is %s", param->rap_name, ra_param_flag2text(param_type)); if (list == NULL) { // We will later search for " WORD ", so start list with a space pcmk__add_word(&list, 256, " "); } pcmk__add_word(&list, 0, param->rap_name); } else { pcmk__trace("Rejecting %s for %s", param->rap_name, ra_param_flag2text(param_type)); } if (accept_for_xml) { const char *v = g_hash_table_lookup(op->params, param->rap_name); if (v != NULL) { pcmk__trace("Adding attr %s=%s to the xml result", param->rap_name, v); pcmk__xe_set(*result, param->rap_name, v); } } else { pcmk__trace("Removing attr %s from the xml result", param->rap_name); pcmk__xe_remove_attr(*result, param->rap_name); } } if (list != NULL) { // We will later search for " WORD ", so end list with a space pcmk__add_word(&list, 0, " "); } return list; } static void append_restart_list(lrmd_event_data_t *op, struct ra_metadata_s *metadata, xmlNode *update, const char *version) { GString *list = NULL; char *digest = NULL; xmlNode *restart = NULL; CRM_LOG_ASSERT(op->params != NULL); if (op->interval_ms > 0) { /* monitors are not reloadable */ return; } if (pcmk__is_set(metadata->ra_flags, ra_supports_reload_agent)) { /* Add parameters not marked reloadable to the PCMK__XA_OP_FORCE_RESTART * list */ list = build_parameter_list(op, metadata, ra_param_reloadable, &restart); } else if (pcmk__is_set(metadata->ra_flags, ra_supports_legacy_reload)) { /* @COMPAT pre-OCF-1.1 resource agents * * Before OCF 1.1, Pacemaker abused "unique=0" to indicate * reloadability. Add any parameters with unique="1" to the * PCMK__XA_OP_FORCE_RESTART list. */ list = build_parameter_list(op, metadata, ra_param_unique, &restart); } else { // Resource does not support agent reloads return; } digest = pcmk__digest_operation(restart); /* Add PCMK__XA_OP_FORCE_RESTART and PCMK__XA_OP_RESTART_DIGEST to indicate * the resource supports reload, no matter if it actually supports any * reloadable parameters */ pcmk__xe_set(update, PCMK__XA_OP_FORCE_RESTART, (list == NULL)? "" : (const char *) list->str); pcmk__xe_set(update, PCMK__XA_OP_RESTART_DIGEST, digest); if ((list != NULL) && (list->len > 0)) { pcmk__trace("%s: %s, %s", op->rsc_id, digest, list->str); } else { pcmk__trace("%s: %s", op->rsc_id, digest); } if (list != NULL) { g_string_free(list, TRUE); } pcmk__xml_free(restart); free(digest); } static void append_secure_list(lrmd_event_data_t *op, struct ra_metadata_s *metadata, xmlNode *update, const char *version) { GString *list = NULL; char *digest = NULL; xmlNode *secure = NULL; CRM_LOG_ASSERT(op->params != NULL); /* To keep PCMK__XA_OP_SECURE_PARAMS short, we want it to contain the secure * parameters but PCMK__XA_OP_SECURE_DIGEST to be based on the insecure ones */ list = build_parameter_list(op, metadata, ra_param_private, &secure); if (list != NULL) { digest = pcmk__digest_operation(secure); pcmk__xe_set(update, PCMK__XA_OP_SECURE_PARAMS, (const char *) list->str); pcmk__xe_set(update, PCMK__XA_OP_SECURE_DIGEST, digest); pcmk__trace("%s: %s, %s", op->rsc_id, digest, list->str); g_string_free(list, TRUE); } else { pcmk__trace("%s: no secure parameters", op->rsc_id); } pcmk__xml_free(secure); free(digest); } /*! * \internal * \brief Create XML for a resource history entry * * \param[in] func Function name of caller * \param[in,out] parent XML to add entry to * \param[in] rsc Affected resource * \param[in,out] op Action to add an entry for (or NULL to do nothing) * \param[in] node_name Node where action occurred */ void controld_add_resource_history_xml_as(const char *func, xmlNode *parent, const lrmd_rsc_info_t *rsc, lrmd_event_data_t *op, const char *node_name) { int target_rc = 0; xmlNode *xml_op = NULL; struct ra_metadata_s *metadata = NULL; const char *caller_version = NULL; lrm_state_t *lrm_state = NULL; if (op == NULL) { return; } target_rc = rsc_op_expected_rc(op); caller_version = g_hash_table_lookup(op->params, PCMK_XA_CRM_FEATURE_SET); CRM_CHECK(caller_version != NULL, caller_version = CRM_FEATURE_SET); xml_op = pcmk__create_history_xml(parent, op, caller_version, target_rc, controld_globals.cluster->priv->node_name, func); if (xml_op == NULL) { return; } if ((rsc == NULL) || (op->params == NULL) || !crm_op_needs_metadata(rsc->standard, op->op_type)) { pcmk__trace("No digests needed for %s action on %s (params=%p rsc=%p)", op->op_type, op->rsc_id, op->params, rsc); return; } lrm_state = controld_get_executor_state(node_name, false); if (lrm_state == NULL) { pcmk__warn("Cannot calculate digests for operation " PCMK__OP_FMT " because we have no connection to executor for %s", op->rsc_id, op->op_type, op->interval_ms, node_name); return; } /* Ideally the metadata is cached, and the agent is just a fallback. * * @TODO Go through all callers and ensure they get metadata asynchronously * first. */ metadata = controld_get_rsc_metadata(lrm_state, rsc, controld_metadata_from_agent |controld_metadata_from_cache); if (metadata == NULL) { return; } pcmk__trace("Including additional digests for %s:%s:%s", rsc->standard, rsc->provider, rsc->type); append_restart_list(op, metadata, xml_op, caller_version); append_secure_list(op, metadata, xml_op, caller_version); return; } /*! * \internal * \brief Record an action as pending in the CIB, if appropriate * * \param[in] node_name Node where the action is pending * \param[in] rsc Resource that action is for * \param[in,out] op Pending action * * \return true if action was recorded in CIB, otherwise false */ bool controld_record_pending_op(const char *node_name, const lrmd_rsc_info_t *rsc, lrmd_event_data_t *op) { const char *record_pending = NULL; CRM_CHECK((node_name != NULL) && (rsc != NULL) && (op != NULL), return false); // Never record certain operation types as pending if ((op->op_type == NULL) || (op->params == NULL) || !controld_action_is_recordable(op->op_type)) { return false; } // Check action's PCMK_META_RECORD_PENDING meta-attribute (defaults to true) record_pending = crm_meta_value(op->params, PCMK_META_RECORD_PENDING); if ((record_pending != NULL) && !pcmk__is_true(record_pending)) { pcmk__warn_once(pcmk__wo_record_pending, "The " PCMK_META_RECORD_PENDING " option (for example, " "for the %s resource's %s operation) is deprecated and " "will be removed in a future release", rsc->id, op->op_type); return false; } op->call_id = -1; op->t_run = time(NULL); op->t_rcchange = op->t_run; lrmd__set_result(op, PCMK_OCF_UNKNOWN, PCMK_EXEC_PENDING, NULL); pcmk__debug("Recording pending %s-interval %s for %s on %s in the CIB", pcmk__readable_interval(op->interval_ms), op->op_type, op->rsc_id, node_name); controld_update_resource_history(node_name, rsc, op, 0); return true; } static void cib_rsc_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data) { switch (rc) { case pcmk_ok: case -pcmk_err_diff_failed: case -pcmk_err_diff_resync: pcmk__trace("Resource history update completed (call=%d rc=%d)", call_id, rc); break; default: if (call_id > 0) { pcmk__warn("Resource history update %d failed: %s " QB_XS " rc=%d", call_id, pcmk_strerror(rc), rc); } else { pcmk__warn("Resource history update failed: %s " QB_XS " rc=%d", pcmk_strerror(rc), rc); } } if (call_id == pending_rsc_update) { pending_rsc_update = 0; controld_trigger_fsa(); } } /* Only successful stops, and probes that found the resource inactive, get locks * recorded in the history. This ensures the resource stays locked to the node * until it is active there again after the node comes back up. */ static bool should_preserve_lock(lrmd_event_data_t *op) { if (!pcmk__is_set(controld_globals.flags, controld_shutdown_lock_enabled)) { return false; } if (!strcmp(op->op_type, PCMK_ACTION_STOP) && (op->rc == PCMK_OCF_OK)) { return true; } if (!strcmp(op->op_type, PCMK_ACTION_MONITOR) && (op->rc == PCMK_OCF_NOT_RUNNING)) { return true; } return false; } /*! * \internal * \brief Request a CIB update * * \param[in] section Section of CIB to update * \param[in] data New XML of CIB section to update * \param[in] options CIB call options * \param[in] callback If not \c NULL, set this as the operation callback * * \return Standard Pacemaker return code * * \note If \p callback is \p cib_rsc_callback(), the CIB update's call ID is * stored in \p pending_rsc_update on success. */ int controld_update_cib(const char *section, xmlNode *data, int options, void (*callback)(xmlNode *, int, int, xmlNode *, void *)) { cib_t *cib = controld_globals.cib_conn; int cib_rc = -ENOTCONN; pcmk__assert(data != NULL); if (cib != NULL) { cib_rc = cib->cmds->modify(cib, section, data, options); if (cib_rc >= 0) { pcmk__debug("Submitted CIB update %d for %s section", cib_rc, section); } } if (callback == NULL) { if (cib_rc < 0) { pcmk__err("Failed to update CIB %s section: %s", section, pcmk_rc_str(pcmk_legacy2rc(cib_rc))); } } else { if ((cib_rc >= 0) && (callback == cib_rsc_callback)) { /* Checking for a particular callback is a little hacky, but it * didn't seem worth adding an output argument for cib_rc for just * one use case. */ pending_rsc_update = cib_rc; } fsa_register_cib_callback(cib_rc, NULL, callback); } return (cib_rc >= 0)? pcmk_rc_ok : pcmk_legacy2rc(cib_rc); } /*! * \internal * \brief Update resource history entry in CIB * * \param[in] node_name Node where action occurred * \param[in] rsc Resource that action is for * \param[in,out] op Action to record * \param[in] lock_time If nonzero, when resource was locked to node * * \note On success, the CIB update's call ID will be stored in * pending_rsc_update. */ void controld_update_resource_history(const char *node_name, const lrmd_rsc_info_t *rsc, lrmd_event_data_t *op, time_t lock_time) { xmlNode *update = NULL; xmlNode *xml = NULL; int call_opt = crmd_cib_smart_opt(); const char *node_id = NULL; const char *container = NULL; CRM_CHECK((node_name != NULL) && (op != NULL), return); if (rsc == NULL) { pcmk__warn("Resource %s no longer exists in the executor", op->rsc_id); controld_ack_event_directly(NULL, NULL, rsc, op, op->rsc_id); return; } // update = pcmk__xe_create(NULL, PCMK_XE_STATUS); // xml = pcmk__xe_create(update, PCMK__XE_NODE_STATE); if (controld_is_local_node(node_name)) { node_id = controld_globals.our_uuid; } else { node_id = node_name; pcmk__xe_set_bool_attr(xml, PCMK_XA_REMOTE_NODE, true); } pcmk__xe_set(xml, PCMK_XA_ID, node_id); pcmk__xe_set(xml, PCMK_XA_UNAME, node_name); pcmk__xe_set(xml, PCMK_XA_CRM_DEBUG_ORIGIN, __func__); // xml = pcmk__xe_create(xml, PCMK__XE_LRM); pcmk__xe_set(xml, PCMK_XA_ID, node_id); // xml = pcmk__xe_create(xml, PCMK__XE_LRM_RESOURCES); // xml = pcmk__xe_create(xml, PCMK__XE_LRM_RESOURCE); pcmk__xe_set(xml, PCMK_XA_ID, op->rsc_id); pcmk__xe_set(xml, PCMK_XA_CLASS, rsc->standard); pcmk__xe_set(xml, PCMK_XA_PROVIDER, rsc->provider); pcmk__xe_set(xml, PCMK_XA_TYPE, rsc->type); if (lock_time != 0) { /* Actions on a locked resource should either preserve the lock by * recording it with the action result, or clear it. */ if (!should_preserve_lock(op)) { lock_time = 0; } pcmk__xe_set_time(xml, PCMK_OPT_SHUTDOWN_LOCK, lock_time); } if (op->params != NULL) { container = g_hash_table_lookup(op->params, CRM_META "_" PCMK__META_CONTAINER); if (container != NULL) { pcmk__trace("Resource %s is a part of container resource %s", op->rsc_id, container); pcmk__xe_set(xml, PCMK__META_CONTAINER, container); } } // (possibly more than one) controld_add_resource_history_xml(xml, rsc, op, node_name); /* Update CIB asynchronously. Even if it fails, the resource state should be * discovered during the next election. Worst case, the node is wrongly * fenced for running a resource it isn't. */ - crm_log_xml_trace(update, __func__); + pcmk__log_xml_trace(update, __func__); controld_update_cib(PCMK_XE_STATUS, update, call_opt, cib_rsc_callback); pcmk__xml_free(update); } /*! * \internal * \brief Erase an LRM history entry from the CIB, given the operation data * * \param[in] op Operation whose history should be deleted */ void controld_delete_action_history(const lrmd_event_data_t *op) { xmlNode *xml_top = NULL; CRM_CHECK(op != NULL, return); xml_top = pcmk__xe_create(NULL, PCMK__XE_LRM_RSC_OP); pcmk__xe_set_int(xml_top, PCMK__XA_CALL_ID, op->call_id); pcmk__xe_set(xml_top, PCMK__XA_TRANSITION_KEY, op->user_data); if (op->interval_ms > 0) { char *op_id = pcmk__op_key(op->rsc_id, op->op_type, op->interval_ms); /* Avoid deleting last_failure too (if it was a result of this recurring op failing) */ pcmk__xe_set(xml_top, PCMK_XA_ID, op_id); free(op_id); } pcmk__debug("Erasing resource operation history for " PCMK__OP_FMT " (call=%d)", op->rsc_id, op->op_type, op->interval_ms, op->call_id); controld_globals.cib_conn->cmds->remove(controld_globals.cib_conn, PCMK_XE_STATUS, xml_top, cib_none); - crm_log_xml_trace(xml_top, "op:cancel"); + pcmk__log_xml_trace(xml_top, "op:cancel"); pcmk__xml_free(xml_top); } /* Define xpath to find LRM resource history entry by node and resource */ #define XPATH_HISTORY \ "/" PCMK_XE_CIB "/" PCMK_XE_STATUS \ "/" PCMK__XE_NODE_STATE "[@" PCMK_XA_UNAME "='%s']" \ "/" PCMK__XE_LRM "/" PCMK__XE_LRM_RESOURCES \ "/" PCMK__XE_LRM_RESOURCE "[@" PCMK_XA_ID "='%s']" \ "/" PCMK__XE_LRM_RSC_OP /* ... and also by operation key */ #define XPATH_HISTORY_ID XPATH_HISTORY "[@" PCMK_XA_ID "='%s']" /* ... and also by operation key and operation call ID */ #define XPATH_HISTORY_CALL XPATH_HISTORY \ "[@" PCMK_XA_ID "='%s' and @" PCMK__XA_CALL_ID "='%d']" /* ... and also by operation key and original operation key */ #define XPATH_HISTORY_ORIG XPATH_HISTORY \ "[@" PCMK_XA_ID "='%s' and @" PCMK__XA_OPERATION_KEY "='%s']" /*! * \internal * \brief Delete a last_failure resource history entry from the CIB * * \param[in] rsc_id Name of resource to clear history for * \param[in] node Name of node to clear history for * \param[in] action If specified, delete only if this was failed action * \param[in] interval_ms If \p action is specified, it has this interval */ void controld_cib_delete_last_failure(const char *rsc_id, const char *node, const char *action, guint interval_ms) { char *xpath = NULL; char *last_failure_key = NULL; CRM_CHECK((rsc_id != NULL) && (node != NULL), return); // Generate XPath to match desired entry last_failure_key = pcmk__op_key(rsc_id, "last_failure", 0); if (action == NULL) { xpath = pcmk__assert_asprintf(XPATH_HISTORY_ID, node, rsc_id, last_failure_key); } else { char *action_key = pcmk__op_key(rsc_id, action, interval_ms); xpath = pcmk__assert_asprintf(XPATH_HISTORY_ORIG, node, rsc_id, last_failure_key, action_key); free(action_key); } free(last_failure_key); controld_globals.cib_conn->cmds->remove(controld_globals.cib_conn, xpath, NULL, cib_xpath); free(xpath); } /*! * \internal * \brief Delete resource history entry from the CIB, given operation key * * \param[in] rsc_id Name of resource to clear history for * \param[in] node Name of node to clear history for * \param[in] key Operation key of operation to clear history for * \param[in] call_id If specified, delete entry only if it has this call ID */ void controld_delete_action_history_by_key(const char *rsc_id, const char *node, const char *key, int call_id) { char *xpath = NULL; CRM_CHECK((rsc_id != NULL) && (node != NULL) && (key != NULL), return); if (call_id > 0) { xpath = pcmk__assert_asprintf(XPATH_HISTORY_CALL, node, rsc_id, key, call_id); } else { xpath = pcmk__assert_asprintf(XPATH_HISTORY_ID, node, rsc_id, key); } controld_globals.cib_conn->cmds->remove(controld_globals.cib_conn, xpath, NULL, cib_xpath); free(xpath); } diff --git a/daemons/controld/controld_execd.c b/daemons/controld/controld_execd.c index 2061d66f2b..fe425a3b10 100644 --- a/daemons/controld/controld_execd.c +++ b/daemons/controld/controld_execd.c @@ -1,2437 +1,2437 @@ /* * Copyright 2004-2025 the Pacemaker project contributors * * The version control history for this file may have further details. * * This source code is licensed under the GNU General Public License version 2 * or later (GPLv2+) WITHOUT ANY WARRANTY. */ #include #include #include #include #include #include #include // lrmd_event_data_t, lrmd_rsc_info_t, etc. #include #include // crm_meta_name() #include #include #include #include #define START_DELAY_THRESHOLD 5 * 60 * 1000 #define MAX_LRM_REG_FAILS 30 struct delete_event_s { int rc; const char *rsc; lrm_state_t *lrm_state; }; static gboolean is_rsc_active(lrm_state_t * lrm_state, const char *rsc_id); static gboolean build_active_RAs(lrm_state_t * lrm_state, xmlNode * rsc_list); static gboolean stop_recurring_actions(gpointer key, gpointer value, gpointer user_data); static lrmd_event_data_t *construct_op(const lrm_state_t *lrm_state, const xmlNode *rsc_op, const char *rsc_id, const char *operation); static void do_lrm_rsc_op(lrm_state_t *lrm_state, lrmd_rsc_info_t *rsc, xmlNode *msg, struct ra_metadata_s *md); static gboolean lrm_state_verify_stopped(lrm_state_t * lrm_state, enum crmd_fsa_state cur_state, int log_level); static void lrm_connection_destroy(void) { if (pcmk__is_set(controld_globals.fsa_input_register, R_LRM_CONNECTED)) { pcmk__crit("Lost connection to local executor"); register_fsa_input(C_FSA_INTERNAL, I_ERROR, NULL); controld_clear_fsa_input_flags(R_LRM_CONNECTED); } } static char * make_stop_id(const char *rsc, int call_id) { return pcmk__assert_asprintf("%s:%d", rsc, call_id); } static void copy_instance_keys(gpointer key, gpointer value, gpointer user_data) { if (strstr(key, CRM_META "_") == NULL) { pcmk__insert_dup(user_data, (const char *) key, (const char *) value); } } static void copy_meta_keys(gpointer key, gpointer value, gpointer user_data) { if (strstr(key, CRM_META "_") != NULL) { pcmk__insert_dup(user_data, (const char *) key, (const char *) value); } } /*! * \internal * \brief Remove a recurring operation from a resource's history * * \param[in,out] history Resource history to modify * \param[in] op Operation to remove * * \return TRUE if the operation was found and removed, FALSE otherwise */ static gboolean history_remove_recurring_op(rsc_history_t *history, const lrmd_event_data_t *op) { GList *iter; for (iter = history->recurring_op_list; iter != NULL; iter = iter->next) { lrmd_event_data_t *existing = iter->data; if ((op->interval_ms == existing->interval_ms) && pcmk__str_eq(op->rsc_id, existing->rsc_id, pcmk__str_none) && pcmk__str_eq(op->op_type, existing->op_type, pcmk__str_casei)) { history->recurring_op_list = g_list_delete_link(history->recurring_op_list, iter); lrmd_free_event(existing); return TRUE; } } return FALSE; } /*! * \internal * \brief Free all recurring operations in resource history * * \param[in,out] history Resource history to modify */ static void history_free_recurring_ops(rsc_history_t *history) { GList *iter; for (iter = history->recurring_op_list; iter != NULL; iter = iter->next) { lrmd_free_event(iter->data); } g_list_free(history->recurring_op_list); history->recurring_op_list = NULL; } /*! * \internal * \brief Free resource history * * \param[in,out] history Resource history to free */ void history_free(gpointer data) { rsc_history_t *history = (rsc_history_t*)data; if (history->stop_params) { g_hash_table_destroy(history->stop_params); } /* Don't need to free history->rsc.id because it's set to history->id */ free(history->rsc.type); free(history->rsc.standard); free(history->rsc.provider); lrmd_free_event(history->failed); lrmd_free_event(history->last); free(history->id); history_free_recurring_ops(history); free(history); } static void update_history_cache(lrm_state_t * lrm_state, lrmd_rsc_info_t * rsc, lrmd_event_data_t * op) { int target_rc = 0; rsc_history_t *entry = NULL; if (op->rsc_deleted) { pcmk__debug("Purged history for '%s' after %s", op->rsc_id, op->op_type); controld_delete_resource_history(op->rsc_id, lrm_state->node_name, NULL, crmd_cib_smart_opt()); return; } if (pcmk__str_eq(op->op_type, PCMK_ACTION_NOTIFY, pcmk__str_casei)) { return; } pcmk__debug("Updating history for '%s' with %s op", op->rsc_id, op->op_type); entry = g_hash_table_lookup(lrm_state->resource_history, op->rsc_id); if (entry == NULL && rsc) { entry = pcmk__assert_alloc(1, sizeof(rsc_history_t)); entry->id = pcmk__str_copy(op->rsc_id); g_hash_table_insert(lrm_state->resource_history, entry->id, entry); entry->rsc.id = entry->id; entry->rsc.type = pcmk__str_copy(rsc->type); entry->rsc.standard = pcmk__str_copy(rsc->standard); entry->rsc.provider = pcmk__str_copy(rsc->provider); } else if (entry == NULL) { pcmk__info("Resource %s no longer exists, not updating cache", op->rsc_id); return; } entry->last_callid = op->call_id; target_rc = rsc_op_expected_rc(op); if (op->op_status == PCMK_EXEC_CANCELLED) { if (op->interval_ms > 0) { pcmk__trace("Removing cancelled recurring op: " PCMK__OP_FMT, op->rsc_id, op->op_type, op->interval_ms); history_remove_recurring_op(entry, op); return; } else { pcmk__trace("Skipping " PCMK__OP_FMT " rc=%d, status=%d", op->rsc_id, op->op_type, op->interval_ms, op->rc, op->op_status); } } else if (did_rsc_op_fail(op, target_rc)) { /* Store failed monitors here, otherwise the block below will cause them * to be forgotten when a stop happens. */ if (entry->failed) { lrmd_free_event(entry->failed); } entry->failed = lrmd_copy_event(op); } else if (op->interval_ms == 0) { if (entry->last) { lrmd_free_event(entry->last); } entry->last = lrmd_copy_event(op); if (op->params && pcmk__strcase_any_of(op->op_type, PCMK_ACTION_START, PCMK_ACTION_RELOAD, PCMK_ACTION_RELOAD_AGENT, PCMK_ACTION_MONITOR, NULL)) { if (entry->stop_params) { g_hash_table_destroy(entry->stop_params); } entry->stop_params = pcmk__strkey_table(free, free); g_hash_table_foreach(op->params, copy_instance_keys, entry->stop_params); } } if (op->interval_ms > 0) { /* Ensure there are no duplicates */ history_remove_recurring_op(entry, op); pcmk__trace("Adding recurring op: " PCMK__OP_FMT, op->rsc_id, op->op_type, op->interval_ms); entry->recurring_op_list = g_list_prepend(entry->recurring_op_list, lrmd_copy_event(op)); } else if ((entry->recurring_op_list != NULL) && !pcmk__str_eq(op->op_type, PCMK_ACTION_MONITOR, pcmk__str_casei)) { pcmk__trace("Dropping %u recurring ops because of: " PCMK__OP_FMT, g_list_length(entry->recurring_op_list), op->rsc_id, op->op_type, op->interval_ms); history_free_recurring_ops(entry); } } /*! * \internal * \brief Send a direct OK ack for a resource task * * \param[in] lrm_state LRM connection * \param[in] input Input message being ack'ed * \param[in] rsc_id ID of affected resource * \param[in] rsc Affected resource (if available) * \param[in] task Operation task being ack'ed * \param[in] ack_host Name of host to send ack to * \param[in] ack_sys IPC system name to ack */ static void send_task_ok_ack(const lrm_state_t *lrm_state, const ha_msg_input_t *input, const char *rsc_id, const lrmd_rsc_info_t *rsc, const char *task, const char *ack_host, const char *ack_sys) { lrmd_event_data_t *op = construct_op(lrm_state, input->xml, rsc_id, task); lrmd__set_result(op, PCMK_OCF_OK, PCMK_EXEC_DONE, NULL); controld_ack_event_directly(ack_host, ack_sys, rsc, op, rsc_id); lrmd_free_event(op); } static inline const char * op_node_name(lrmd_event_data_t *op) { return pcmk__s(op->remote_nodename, controld_globals.cluster->priv->node_name); } void lrm_op_callback(lrmd_event_data_t * op) { CRM_CHECK(op != NULL, return); switch (op->type) { case lrmd_event_disconnect: if (op->remote_nodename == NULL) { /* If this is the local executor IPC connection, set the right * bits in the controller when the connection goes down. */ lrm_connection_destroy(); } break; case lrmd_event_exec_complete: { lrm_state_t *lrm_state = controld_get_executor_state(op_node_name(op), false); pcmk__assert(lrm_state != NULL); process_lrm_event(lrm_state, op, NULL, NULL); } break; default: break; } } static void try_local_executor_connect(long long action, fsa_data_t *msg_data, lrm_state_t *lrm_state) { int rc = pcmk_rc_ok; pcmk__debug("Connecting to the local executor"); // If we can connect, great rc = controld_connect_local_executor(lrm_state); if (rc == pcmk_rc_ok) { controld_set_fsa_input_flags(R_LRM_CONNECTED); pcmk__info("Connection to the local executor established"); return; } // Otherwise, if we can try again, set a timer to do so if (lrm_state->num_lrm_register_fails < MAX_LRM_REG_FAILS) { pcmk__warn("Failed to connect to the local executor %d time%s " "(%d max): %s", lrm_state->num_lrm_register_fails, pcmk__plural_s(lrm_state->num_lrm_register_fails), MAX_LRM_REG_FAILS, pcmk_rc_str(rc)); controld_start_wait_timer(); crmd_fsa_stall(FALSE); return; } // Otherwise give up pcmk__err("Failed to connect to the executor the max allowed %d time%s: %s", lrm_state->num_lrm_register_fails, pcmk__plural_s(lrm_state->num_lrm_register_fails), pcmk_rc_str(rc)); register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL); } /* A_LRM_CONNECT */ void do_lrm_control(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data) { /* This only pertains to local executor connections. Remote connections are * handled as resources within the scheduler. Connecting and disconnecting * from remote executor instances is handled differently. */ lrm_state_t *lrm_state = NULL; if (controld_globals.cluster->priv->node_name == NULL) { return; // Shouldn't be possible } lrm_state = controld_get_executor_state(NULL, true); if (lrm_state == NULL) { register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL); return; } if (action & A_LRM_DISCONNECT) { if (lrm_state_verify_stopped(lrm_state, cur_state, LOG_INFO) == FALSE) { if (action == A_LRM_DISCONNECT) { crmd_fsa_stall(FALSE); return; } } controld_clear_fsa_input_flags(R_LRM_CONNECTED); lrm_state_disconnect(lrm_state); lrm_state_reset_tables(lrm_state, FALSE); } if (action & A_LRM_CONNECT) { try_local_executor_connect(action, msg_data, lrm_state); } if (action & ~(A_LRM_CONNECT | A_LRM_DISCONNECT)) { pcmk__err("Unexpected action %s in %s", fsa_action2string(action), __func__); } } static gboolean lrm_state_verify_stopped(lrm_state_t * lrm_state, enum crmd_fsa_state cur_state, int log_level) { int counter = 0; gboolean rc = TRUE; const char *when = "lrm disconnect"; GHashTableIter gIter; const char *key = NULL; rsc_history_t *entry = NULL; active_op_t *pending = NULL; pcmk__debug("Checking for active resources before exit"); if (cur_state == S_TERMINATE) { log_level = LOG_ERR; when = "shutdown"; } else if (pcmk__is_set(controld_globals.fsa_input_register, R_SHUTDOWN)) { when = "shutdown... waiting"; } if ((lrm_state->active_ops != NULL) && lrm_state_is_connected(lrm_state)) { guint removed = g_hash_table_foreach_remove(lrm_state->active_ops, stop_recurring_actions, lrm_state); guint nremaining = g_hash_table_size(lrm_state->active_ops); if (removed || nremaining) { pcmk__notice("Stopped %u recurring operation%s at %s (%u " "remaining)", removed, pcmk__plural_s(removed), when, nremaining); } } if (lrm_state->active_ops != NULL) { g_hash_table_iter_init(&gIter, lrm_state->active_ops); while (g_hash_table_iter_next(&gIter, NULL, (void **)&pending)) { /* Ignore recurring actions in the shutdown calculations */ if (pending->interval_ms == 0) { counter++; } } } if (counter > 0) { do_crm_log(log_level, "%d pending executor operation%s at %s", counter, pcmk__plural_s(counter), when); if ((cur_state == S_TERMINATE) || !pcmk__is_set(controld_globals.fsa_input_register, R_SENT_RSC_STOP)) { g_hash_table_iter_init(&gIter, lrm_state->active_ops); while (g_hash_table_iter_next(&gIter, (gpointer*)&key, (gpointer*)&pending)) { do_crm_log(log_level, "Pending action: %s (%s)", key, pending->op_key); } } else { rc = FALSE; } return rc; } if (lrm_state->resource_history == NULL) { return rc; } if (pcmk__is_set(controld_globals.fsa_input_register, R_SHUTDOWN)) { /* At this point we're not waiting, we're just shutting down */ when = "shutdown"; } counter = 0; g_hash_table_iter_init(&gIter, lrm_state->resource_history); while (g_hash_table_iter_next(&gIter, NULL, (gpointer*)&entry)) { if (is_rsc_active(lrm_state, entry->id) == FALSE) { continue; } counter++; if (log_level == LOG_ERR) { pcmk__info("Found %s active at %s", entry->id, when); } else { pcmk__trace("Found %s active at %s", entry->id, when); } if (lrm_state->active_ops != NULL) { GHashTableIter hIter; g_hash_table_iter_init(&hIter, lrm_state->active_ops); while (g_hash_table_iter_next(&hIter, (gpointer*)&key, (gpointer*)&pending)) { if (pcmk__str_eq(entry->id, pending->rsc_id, pcmk__str_none)) { const bool recurring = (pending->interval_ms != 0); pcmk__notice("%s %s (%s) incomplete at %s", (recurring? "Recurring action" : "Action"), key, pending->op_key, when); } } } } if (counter) { pcmk__err("%d resource%s active at %s", counter, ((counter == 1)? " was" : "s were"), when); } return rc; } static gboolean is_rsc_active(lrm_state_t * lrm_state, const char *rsc_id) { rsc_history_t *entry = NULL; entry = g_hash_table_lookup(lrm_state->resource_history, rsc_id); if (entry == NULL || entry->last == NULL) { return FALSE; } pcmk__trace("Processing %s: %s.%d=%d", rsc_id, entry->last->op_type, entry->last->interval_ms, entry->last->rc); if ((entry->last->rc == PCMK_OCF_OK) && pcmk__str_eq(entry->last->op_type, PCMK_ACTION_STOP, pcmk__str_casei)) { return FALSE; } else if (entry->last->rc == PCMK_OCF_OK && pcmk__str_eq(entry->last->op_type, PCMK_ACTION_MIGRATE_TO, pcmk__str_casei)) { // A stricter check is too complex ... leave that to the scheduler return FALSE; } else if (entry->last->rc == PCMK_OCF_NOT_RUNNING) { return FALSE; } else if ((entry->last->interval_ms == 0) && (entry->last->rc == PCMK_OCF_NOT_CONFIGURED)) { /* Badly configured resources can't be reliably stopped */ return FALSE; } return TRUE; } static gboolean build_active_RAs(lrm_state_t * lrm_state, xmlNode * rsc_list) { GHashTableIter iter; rsc_history_t *entry = NULL; g_hash_table_iter_init(&iter, lrm_state->resource_history); while (g_hash_table_iter_next(&iter, NULL, (void **)&entry)) { GList *gIter = NULL; xmlNode *xml_rsc = pcmk__xe_create(rsc_list, PCMK__XE_LRM_RESOURCE); pcmk__xe_set(xml_rsc, PCMK_XA_ID, entry->id); pcmk__xe_set(xml_rsc, PCMK_XA_TYPE, entry->rsc.type); pcmk__xe_set(xml_rsc, PCMK_XA_CLASS, entry->rsc.standard); pcmk__xe_set(xml_rsc, PCMK_XA_PROVIDER, entry->rsc.provider); if (entry->last && entry->last->params) { static const char *name = CRM_META "_" PCMK__META_CONTAINER; const char *container = g_hash_table_lookup(entry->last->params, name); if (container) { pcmk__trace("Resource %s is a part of container resource %s", entry->id, container); pcmk__xe_set(xml_rsc, PCMK__META_CONTAINER, container); } } controld_add_resource_history_xml(xml_rsc, &(entry->rsc), entry->failed, lrm_state->node_name); controld_add_resource_history_xml(xml_rsc, &(entry->rsc), entry->last, lrm_state->node_name); for (gIter = entry->recurring_op_list; gIter != NULL; gIter = gIter->next) { controld_add_resource_history_xml(xml_rsc, &(entry->rsc), gIter->data, lrm_state->node_name); } } return FALSE; } xmlNode * controld_query_executor_state(void) { // @TODO Ensure all callers handle NULL returns xmlNode *xml_state = NULL; xmlNode *xml_data = NULL; xmlNode *rsc_list = NULL; pcmk__node_status_t *peer = NULL; lrm_state_t *lrm_state = controld_get_executor_state(NULL, false); if (!lrm_state) { pcmk__err("Could not get executor state for local node"); return NULL; } peer = pcmk__get_node(0, lrm_state->node_name, NULL, pcmk__node_search_any); CRM_CHECK(peer != NULL, return NULL); xml_state = create_node_state_update(peer, node_update_cluster|node_update_peer, NULL, __func__); if (xml_state == NULL) { return NULL; } xml_data = pcmk__xe_create(xml_state, PCMK__XE_LRM); pcmk__xe_set(xml_data, PCMK_XA_ID, peer->xml_id); rsc_list = pcmk__xe_create(xml_data, PCMK__XE_LRM_RESOURCES); // Build a list of active (not necessarily running) resources build_active_RAs(lrm_state, rsc_list); - crm_log_xml_trace(xml_state, "Current executor state"); + pcmk__log_xml_trace(xml_state, "Current executor state"); return xml_state; } /*! * \internal * \brief Map standard Pacemaker return code to operation status and OCF code * * \param[out] event Executor event whose status and return code should be set * \param[in] rc Standard Pacemaker return code */ void controld_rc2event(lrmd_event_data_t *event, int rc) { /* This is called for cleanup requests from controller peers/clients, not * for resource actions, so no exit reason is needed. */ switch (rc) { case pcmk_rc_ok: lrmd__set_result(event, PCMK_OCF_OK, PCMK_EXEC_DONE, NULL); break; case EACCES: lrmd__set_result(event, PCMK_OCF_INSUFFICIENT_PRIV, PCMK_EXEC_ERROR, NULL); break; default: lrmd__set_result(event, PCMK_OCF_UNKNOWN_ERROR, PCMK_EXEC_ERROR, NULL); break; } } /*! * \internal * \brief Trigger a new transition after CIB status was deleted * * If a CIB status delete was not expected (as part of the transition graph), * trigger a new transition by updating the (arbitrary) "last-lrm-refresh" * cluster property. * * \param[in] from_sys IPC name that requested the delete * \param[in] rsc_id Resource whose status was deleted (for logging only) */ void controld_trigger_delete_refresh(const char *from_sys, const char *rsc_id) { if (!pcmk__str_eq(from_sys, CRM_SYSTEM_TENGINE, pcmk__str_casei)) { char *now_s = pcmk__assert_asprintf("%lld", (long long) time(NULL)); pcmk__debug("Triggering a refresh after %s cleaned %s", from_sys, rsc_id); cib__update_node_attr(controld_globals.logger_out, controld_globals.cib_conn, cib_none, PCMK_XE_CRM_CONFIG, NULL, NULL, NULL, NULL, "last-lrm-refresh", now_s, NULL, NULL); free(now_s); } } static void notify_deleted(lrm_state_t * lrm_state, ha_msg_input_t * input, const char *rsc_id, int rc) { lrmd_event_data_t *op = NULL; const char *from_sys = pcmk__xe_get(input->msg, PCMK__XA_CRM_SYS_FROM); const char *from_host = pcmk__xe_get(input->msg, PCMK__XA_SRC); pcmk__info("Notifying %s on %s that %s was%s deleted", from_sys, pcmk__s(from_host, "localhost"), rsc_id, ((rc == pcmk_ok)? "" : " not")); op = construct_op(lrm_state, input->xml, rsc_id, PCMK_ACTION_DELETE); controld_rc2event(op, pcmk_legacy2rc(rc)); controld_ack_event_directly(from_host, from_sys, NULL, op, rsc_id); lrmd_free_event(op); controld_trigger_delete_refresh(from_sys, rsc_id); } static gboolean lrm_remove_deleted_rsc(gpointer key, gpointer value, gpointer user_data) { struct delete_event_s *event = user_data; struct pending_deletion_op_s *op = value; if (pcmk__str_eq(event->rsc, op->rsc, pcmk__str_none)) { notify_deleted(event->lrm_state, op->input, event->rsc, event->rc); return TRUE; } return FALSE; } static gboolean lrm_remove_deleted_op(gpointer key, gpointer value, gpointer user_data) { const char *rsc = user_data; active_op_t *pending = value; if (pcmk__str_eq(rsc, pending->rsc_id, pcmk__str_none)) { pcmk__info("Removing op %s:%d for deleted resource %s", pending->op_key, pending->call_id, rsc); return TRUE; } return FALSE; } static void delete_rsc_entry(lrm_state_t *lrm_state, ha_msg_input_t *input, const char *rsc_id, GHashTableIter *rsc_iter, int rc, const char *user_name, bool from_cib) { struct delete_event_s event; CRM_CHECK(rsc_id != NULL, return); if (rc == pcmk_ok) { char *rsc_id_copy = pcmk__str_copy(rsc_id); if (rsc_iter) { g_hash_table_iter_remove(rsc_iter); } else { g_hash_table_remove(lrm_state->resource_history, rsc_id_copy); } if (from_cib) { controld_delete_resource_history(rsc_id_copy, lrm_state->node_name, user_name, crmd_cib_smart_opt()); } g_hash_table_foreach_remove(lrm_state->active_ops, lrm_remove_deleted_op, rsc_id_copy); free(rsc_id_copy); } if (input) { notify_deleted(lrm_state, input, rsc_id, rc); } event.rc = rc; event.rsc = rsc_id; event.lrm_state = lrm_state; g_hash_table_foreach_remove(lrm_state->deletion_ops, lrm_remove_deleted_rsc, &event); } static inline gboolean last_failed_matches_op(rsc_history_t *entry, const char *op, guint interval_ms) { if (entry == NULL) { return FALSE; } if (op == NULL) { return TRUE; } return (pcmk__str_eq(op, entry->failed->op_type, pcmk__str_casei) && (interval_ms == entry->failed->interval_ms)); } /*! * \internal * \brief Clear a resource's last failure * * Erase a resource's last failure on a particular node from both the * LRM resource history in the CIB, and the resource history remembered * for the LRM state. * * \param[in] rsc_id Resource name * \param[in] node_name Node name * \param[in] operation If specified, only clear if matching this operation * \param[in] interval_ms If operation is specified, it has this interval */ void lrm_clear_last_failure(const char *rsc_id, const char *node_name, const char *operation, guint interval_ms) { lrm_state_t *lrm_state = controld_get_executor_state(node_name, false); if (lrm_state == NULL) { return; } if (lrm_state->resource_history != NULL) { rsc_history_t *entry = g_hash_table_lookup(lrm_state->resource_history, rsc_id); if (last_failed_matches_op(entry, operation, interval_ms)) { lrmd_free_event(entry->failed); entry->failed = NULL; } } } /* Returns: gboolean - cancellation is in progress */ static gboolean cancel_op(lrm_state_t * lrm_state, const char *rsc_id, const char *key, int op, gboolean remove) { int rc = pcmk_ok; char *local_key = NULL; active_op_t *pending = NULL; CRM_CHECK(op != 0, return FALSE); CRM_CHECK(rsc_id != NULL, return FALSE); if (key == NULL) { local_key = make_stop_id(rsc_id, op); key = local_key; } pending = g_hash_table_lookup(lrm_state->active_ops, key); if (pending) { if (remove && !pcmk__is_set(pending->flags, active_op_remove)) { controld_set_active_op_flags(pending, active_op_remove); pcmk__debug("Scheduling %s for removal", key); } if (pcmk__is_set(pending->flags, active_op_cancelled)) { pcmk__debug("Operation %s already cancelled", key); free(local_key); return FALSE; } controld_set_active_op_flags(pending, active_op_cancelled); } else { pcmk__info("No pending op found for %s", key); free(local_key); return FALSE; } pcmk__debug("Cancelling op %d for %s (%s)", op, rsc_id, key); rc = lrm_state_cancel(lrm_state, pending->rsc_id, pending->op_type, pending->interval_ms); if (rc == pcmk_ok) { pcmk__debug("Op %d for %s (%s): cancelled", op, rsc_id, key); free(local_key); return TRUE; } pcmk__debug("Op %d for %s (%s): Nothing to cancel", op, rsc_id, key); /* The caller needs to make sure the entry is * removed from the active operations list * * Usually by returning TRUE inside the worker function * supplied to g_hash_table_foreach_remove() * * Not removing the entry from active operations will block * the node from shutting down */ free(local_key); return FALSE; } struct cancel_data { gboolean done; gboolean remove; const char *key; lrmd_rsc_info_t *rsc; lrm_state_t *lrm_state; }; static gboolean cancel_action_by_key(gpointer key, gpointer value, gpointer user_data) { gboolean remove = FALSE; struct cancel_data *data = user_data; active_op_t *op = value; if (pcmk__str_eq(op->op_key, data->key, pcmk__str_none)) { data->done = TRUE; remove = !cancel_op(data->lrm_state, data->rsc->id, key, op->call_id, data->remove); } return remove; } static gboolean cancel_op_key(lrm_state_t * lrm_state, lrmd_rsc_info_t * rsc, const char *key, gboolean remove) { guint removed = 0; struct cancel_data data; CRM_CHECK(rsc != NULL, return FALSE); CRM_CHECK(key != NULL, return FALSE); data.key = key; data.rsc = rsc; data.done = FALSE; data.remove = remove; data.lrm_state = lrm_state; removed = g_hash_table_foreach_remove(lrm_state->active_ops, cancel_action_by_key, &data); pcmk__trace("Removed %u op cache entries, new size: %u", removed, g_hash_table_size(lrm_state->active_ops)); return data.done; } /*! * \internal * \brief Retrieve resource information from LRM * * \param[in,out] lrm_state Executor connection state to use * \param[in] rsc_xml XML containing resource configuration * \param[in] do_create If true, register resource if not already * \param[out] rsc_info Where to store information obtained from executor * * \retval pcmk_ok Success (and rsc_info holds newly allocated result) * \retval -EINVAL Required information is missing from arguments * \retval -ENOTCONN No active connection to LRM * \retval -ENODEV Resource not found * \retval -errno Error communicating with executor when registering resource * * \note Caller is responsible for freeing result on success. */ static int get_lrm_resource(lrm_state_t *lrm_state, const xmlNode *rsc_xml, gboolean do_create, lrmd_rsc_info_t **rsc_info) { const char *id = pcmk__xe_id(rsc_xml); CRM_CHECK(lrm_state && rsc_xml && rsc_info, return -EINVAL); CRM_CHECK(id, return -EINVAL); if (lrm_state_is_connected(lrm_state) == FALSE) { return -ENOTCONN; } pcmk__trace("Retrieving resource information for %s from the executor", id); *rsc_info = lrm_state_get_rsc_info(lrm_state, id, 0); // If resource isn't known by ID, try clone name, if provided if (!*rsc_info) { const char *long_id = pcmk__xe_get(rsc_xml, PCMK__XA_LONG_ID); if (long_id) { *rsc_info = lrm_state_get_rsc_info(lrm_state, long_id, 0); } } if ((*rsc_info == NULL) && do_create) { const char *class = pcmk__xe_get(rsc_xml, PCMK_XA_CLASS); const char *provider = pcmk__xe_get(rsc_xml, PCMK_XA_PROVIDER); const char *type = pcmk__xe_get(rsc_xml, PCMK_XA_TYPE); int rc; pcmk__trace("Registering resource %s with the executor", id); rc = lrm_state_register_rsc(lrm_state, id, class, provider, type, lrmd_opt_drop_recurring); if (rc != pcmk_ok) { fsa_data_t *msg_data = NULL; pcmk__err("Could not register resource %s with the executor on %s: " "%s " QB_XS " rc=%d", id, lrm_state->node_name, pcmk_strerror(rc), rc); /* Register this as an internal error if this involves the local * executor. Otherwise, we're likely dealing with an unresponsive * remote node, which is not an FSA failure. */ if (lrm_state_is_local(lrm_state) == TRUE) { register_fsa_error(C_FSA_INTERNAL, I_FAIL, NULL); } return rc; } *rsc_info = lrm_state_get_rsc_info(lrm_state, id, 0); } return *rsc_info? pcmk_ok : -ENODEV; } static void delete_resource(lrm_state_t *lrm_state, const char *id, lrmd_rsc_info_t *rsc, GHashTableIter *iter, const char *sys, const char *user, ha_msg_input_t *request, bool unregister, bool from_cib) { int rc = pcmk_ok; pcmk__info("Removing resource %s from executor for %s%s%s", id, sys, ((user != NULL)? " as " : ""), pcmk__s(user, "")); if (rsc && unregister) { rc = lrm_state_unregister_rsc(lrm_state, id, 0); } if (rc == pcmk_ok) { pcmk__trace("Resource %s deleted from executor", id); } else if (rc == -EINPROGRESS) { pcmk__info("Deletion of resource '%s' from executor is pending", id); if (request) { struct pending_deletion_op_s *op = NULL; char *ref = pcmk__xe_get_copy(request->msg, PCMK_XA_REFERENCE); op = pcmk__assert_alloc(1, sizeof(struct pending_deletion_op_s)); op->rsc = pcmk__str_copy(rsc->id); op->input = copy_ha_msg_input(request); g_hash_table_insert(lrm_state->deletion_ops, ref, op); } return; } else { pcmk__warn("Could not delete '%s' from executor for %s%s%s: %s " QB_XS " rc=%d", id, sys, ((user != NULL)? " as " : ""), pcmk__s(user, ""), pcmk_strerror(rc), rc); } delete_rsc_entry(lrm_state, request, id, iter, rc, user, from_cib); } static int get_fake_call_id(lrm_state_t *lrm_state, const char *rsc_id) { int call_id = 999999999; rsc_history_t *entry = NULL; if(lrm_state) { entry = g_hash_table_lookup(lrm_state->resource_history, rsc_id); } /* Make sure the call id is greater than the last successful operation, * otherwise the failure will not result in a possible recovery of the resource * as it could appear the failure occurred before the successful start */ if (entry) { call_id = entry->last_callid + 1; } if (call_id < 0) { call_id = 1; } return call_id; } static void fake_op_status(lrm_state_t *lrm_state, lrmd_event_data_t *op, int op_status, enum ocf_exitcode op_exitcode, const char *exit_reason) { op->call_id = get_fake_call_id(lrm_state, op->rsc_id); op->t_run = time(NULL); op->t_rcchange = op->t_run; lrmd__set_result(op, op_exitcode, op_status, exit_reason); } static void force_reprobe(lrm_state_t *lrm_state, const char *from_sys, const char *from_host, const char *user_name, gboolean is_remote_node, bool reprobe_all_nodes) { GHashTableIter gIter; rsc_history_t *entry = NULL; pcmk__info("Clearing resource history on node %s", lrm_state->node_name); g_hash_table_iter_init(&gIter, lrm_state->resource_history); while (g_hash_table_iter_next(&gIter, NULL, (void **)&entry)) { /* only unregister the resource during a reprobe if it is not a remote connection * resource. otherwise unregistering the connection will terminate remote-node * membership */ bool unregister = true; if (is_remote_lrmd_ra(NULL, NULL, entry->id)) { unregister = false; if (reprobe_all_nodes) { lrm_state_t *remote_lrm_state = controld_get_executor_state(entry->id, false); if (remote_lrm_state != NULL) { /* If reprobing all nodes, be sure to reprobe the remote * node before clearing its connection resource */ force_reprobe(remote_lrm_state, from_sys, from_host, user_name, TRUE, reprobe_all_nodes); } } } /* Don't delete from the CIB, since we'll delete the whole node's LRM * state from the CIB soon */ delete_resource(lrm_state, entry->id, &entry->rsc, &gIter, from_sys, user_name, NULL, unregister, false); } /* Now delete the copy in the CIB */ controld_delete_node_state(lrm_state->node_name, controld_section_lrm, cib_none); } /*! * \internal * \brief Fail a requested action without actually executing it * * For an action that can't be executed, process it similarly to an actual * execution result, with specified error status (except for notify actions, * which will always be treated as successful). * * \param[in,out] lrm_state Executor connection that action is for * \param[in] action Action XML from request * \param[in] rc Desired return code to use * \param[in] op_status Desired operation status to use * \param[in] exit_reason Human-friendly detail, if error */ static void synthesize_lrmd_failure(lrm_state_t *lrm_state, const xmlNode *action, int op_status, enum ocf_exitcode rc, const char *exit_reason) { lrmd_event_data_t *op = NULL; const char *operation = pcmk__xe_get(action, PCMK_XA_OPERATION); const char *target_node = pcmk__xe_get(action, PCMK__META_ON_NODE); xmlNode *xml_rsc = pcmk__xe_first_child(action, PCMK_XE_PRIMITIVE, NULL, NULL); if ((xml_rsc == NULL) || (pcmk__xe_id(xml_rsc) == NULL)) { /* @TODO Should we do something else, like direct ack? */ pcmk__info("Can't fake %s failure (%d) on %s without resource " "configuration", pcmk__xe_get(action, PCMK__XA_OPERATION_KEY), rc, target_node); return; } else if(operation == NULL) { /* This probably came from crm_resource -C, nothing to do */ pcmk__info("Can't fake %s failure (%d) on %s without operation", pcmk__xe_id(xml_rsc), rc, target_node); return; } op = construct_op(lrm_state, action, pcmk__xe_id(xml_rsc), operation); if (pcmk__str_eq(operation, PCMK_ACTION_NOTIFY, pcmk__str_casei)) { // Notifications can't fail fake_op_status(lrm_state, op, PCMK_EXEC_DONE, PCMK_OCF_OK, NULL); } else { fake_op_status(lrm_state, op, op_status, rc, exit_reason); } pcmk__info("Faking " PCMK__OP_FMT " result (%d) on %s", op->rsc_id, op->op_type, op->interval_ms, op->rc, target_node); // Process the result as if it came from the LRM process_lrm_event(lrm_state, op, NULL, action); lrmd_free_event(op); } /*! * \internal * \brief Get target of an LRM operation (replacing \p NULL with local node * name) * * \param[in] xml LRM operation data XML * * \return LRM operation target node name (local node or Pacemaker Remote node) */ static const char * lrm_op_target(const xmlNode *xml) { const char *target = NULL; if (xml) { target = pcmk__xe_get(xml, PCMK__META_ON_NODE); } if (target == NULL) { target = controld_globals.cluster->priv->node_name; } return target; } static void fail_lrm_resource(xmlNode *xml, lrm_state_t *lrm_state, const char *user_name, const char *from_host, const char *from_sys) { lrmd_event_data_t *op = NULL; lrmd_rsc_info_t *rsc = NULL; xmlNode *xml_rsc = pcmk__xe_first_child(xml, PCMK_XE_PRIMITIVE, NULL, NULL); CRM_CHECK(xml_rsc != NULL, return); /* The executor simply executes operations and reports the results, without * any concept of success or failure, so to fail a resource, we must fake * what a failure looks like. * * To do this, we create a fake executor operation event for the resource, * and pass that event to the executor client callback so it will be * processed as if it came from the executor. */ op = construct_op(lrm_state, xml, pcmk__xe_id(xml_rsc), "asyncmon"); free((char*) op->user_data); op->user_data = NULL; op->interval_ms = 0; if (user_name && !pcmk__is_privileged(user_name)) { pcmk__err("%s does not have permission to fail %s", user_name, pcmk__xe_id(xml_rsc)); fake_op_status(lrm_state, op, PCMK_EXEC_ERROR, PCMK_OCF_INSUFFICIENT_PRIV, "Unprivileged user cannot fail resources"); controld_ack_event_directly(from_host, from_sys, NULL, op, pcmk__xe_id(xml_rsc)); lrmd_free_event(op); return; } if (get_lrm_resource(lrm_state, xml_rsc, TRUE, &rsc) == pcmk_ok) { pcmk__info("Failing resource %s...", rsc->id); fake_op_status(lrm_state, op, PCMK_EXEC_DONE, PCMK_OCF_UNKNOWN_ERROR, "Simulated failure"); process_lrm_event(lrm_state, op, NULL, xml); op->rc = PCMK_OCF_OK; // The request to fail the resource succeeded lrmd_free_rsc_info(rsc); } else { pcmk__info("Cannot find/create resource in order to fail it..."); pcmk__log_xml_warn(xml, "bad input"); fake_op_status(lrm_state, op, PCMK_EXEC_ERROR, PCMK_OCF_UNKNOWN_ERROR, "Cannot fail unknown resource"); } controld_ack_event_directly(from_host, from_sys, NULL, op, pcmk__xe_id(xml_rsc)); lrmd_free_event(op); } static void handle_reprobe_op(lrm_state_t *lrm_state, xmlNode *msg, const char *from_sys, const char *from_host, const char *user_name, gboolean is_remote_node, bool reprobe_all_nodes) { pcmk__notice("Forcing the status of all resources to be redetected"); force_reprobe(lrm_state, from_sys, from_host, user_name, is_remote_node, reprobe_all_nodes); if (!pcmk__strcase_any_of(from_sys, CRM_SYSTEM_PENGINE, CRM_SYSTEM_TENGINE, NULL)) { xmlNode *reply = pcmk__new_reply(msg, NULL); pcmk__debug("ACK'ing re-probe from %s (%s)", from_sys, from_host); if (relay_message(reply, TRUE) == FALSE) { pcmk__log_xml_err(reply, "Unable to route reply"); } pcmk__xml_free(reply); } } static bool do_lrm_cancel(ha_msg_input_t *input, lrm_state_t *lrm_state, lrmd_rsc_info_t *rsc, const char *from_host, const char *from_sys) { char *op_key = NULL; char *meta_key = NULL; int call = 0; const char *call_id = NULL; const char *op_task = NULL; guint interval_ms = 0; gboolean in_progress = FALSE; xmlNode *params = pcmk__xe_first_child(input->xml, PCMK__XE_ATTRIBUTES, NULL, NULL); CRM_CHECK(params != NULL, return FALSE); meta_key = crm_meta_name(PCMK_XA_OPERATION); op_task = pcmk__xe_get(params, meta_key); free(meta_key); CRM_CHECK(op_task != NULL, return FALSE); meta_key = crm_meta_name(PCMK_META_INTERVAL); if (pcmk__xe_get_guint(params, meta_key, &interval_ms) != pcmk_rc_ok) { free(meta_key); return FALSE; } free(meta_key); op_key = pcmk__op_key(rsc->id, op_task, interval_ms); meta_key = crm_meta_name(PCMK__XA_CALL_ID); call_id = pcmk__xe_get(params, meta_key); free(meta_key); pcmk__debug("Scheduler requested op %s (call=%s) be cancelled", op_key, pcmk__s(call_id, "NA")); pcmk__scan_min_int(call_id, &call, 0); if (call == 0) { // Normal case when the scheduler cancels a recurring op in_progress = cancel_op_key(lrm_state, rsc, op_key, TRUE); } else { // Normal case when the scheduler cancels an orphan op in_progress = cancel_op(lrm_state, rsc->id, NULL, call, TRUE); } // Acknowledge cancellation operation if for a remote connection resource if (!in_progress || is_remote_lrmd_ra(NULL, NULL, rsc->id)) { char *op_id = make_stop_id(rsc->id, call); if (is_remote_lrmd_ra(NULL, NULL, rsc->id) == FALSE) { pcmk__info("Nothing known about operation %d for %s", call, op_key); } controld_delete_action_history_by_key(rsc->id, lrm_state->node_name, op_key, call); send_task_ok_ack(lrm_state, input, rsc->id, rsc, op_task, from_host, from_sys); /* needed at least for cancellation of a remote operation */ if (lrm_state->active_ops != NULL) { g_hash_table_remove(lrm_state->active_ops, op_id); } free(op_id); } free(op_key); return TRUE; } static void do_lrm_delete(ha_msg_input_t *input, lrm_state_t *lrm_state, lrmd_rsc_info_t *rsc, const char *from_sys, const char *from_host, bool crm_rsc_delete, const char *user_name) { bool unregister = true; int cib_rc = controld_delete_resource_history(rsc->id, lrm_state->node_name, user_name, cib_dryrun|cib_sync_call); if (cib_rc != pcmk_rc_ok) { lrmd_event_data_t *op = NULL; op = construct_op(lrm_state, input->xml, rsc->id, PCMK_ACTION_DELETE); /* These are resource clean-ups, not actions, so no exit reason is * needed. */ lrmd__set_result(op, pcmk_rc2ocf(cib_rc), PCMK_EXEC_ERROR, NULL); controld_ack_event_directly(from_host, from_sys, NULL, op, rsc->id); lrmd_free_event(op); return; } if (crm_rsc_delete && is_remote_lrmd_ra(NULL, NULL, rsc->id)) { unregister = false; } delete_resource(lrm_state, rsc->id, rsc, NULL, from_sys, user_name, input, unregister, true); } // User data for asynchronous metadata execution struct metadata_cb_data { lrmd_rsc_info_t *rsc; // Copy of resource information xmlNode *input_xml; // Copy of FSA input XML }; static struct metadata_cb_data * new_metadata_cb_data(lrmd_rsc_info_t *rsc, xmlNode *input_xml) { struct metadata_cb_data *data = NULL; data = pcmk__assert_alloc(1, sizeof(struct metadata_cb_data)); data->input_xml = pcmk__xml_copy(NULL, input_xml); data->rsc = lrmd_copy_rsc_info(rsc); return data; } static void free_metadata_cb_data(struct metadata_cb_data *data) { lrmd_free_rsc_info(data->rsc); pcmk__xml_free(data->input_xml); free(data); } /*! * \internal * \brief Execute an action after metadata has been retrieved * * \param[in] pid Ignored * \param[in] result Result of metadata action * \param[in] user_data Metadata callback data */ static void metadata_complete(int pid, const pcmk__action_result_t *result, void *user_data) { struct metadata_cb_data *data = (struct metadata_cb_data *) user_data; struct ra_metadata_s *md = NULL; lrm_state_t *lrm_state = controld_get_executor_state(lrm_op_target(data->input_xml), false); if ((lrm_state != NULL) && pcmk__result_ok(result)) { md = controld_cache_metadata(lrm_state->metadata_cache, data->rsc, result->action_stdout); } if (!pcmk__is_set(controld_globals.fsa_input_register, R_HA_DISCONNECTED)) { do_lrm_rsc_op(lrm_state, data->rsc, data->input_xml, md); } free_metadata_cb_data(data); } /* A_LRM_INVOKE */ void do_lrm_invoke(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data) { lrm_state_t *lrm_state = NULL; const char *crm_op = NULL; const char *from_sys = NULL; const char *from_host = NULL; const char *operation = NULL; ha_msg_input_t *input = fsa_typed_data(fsa_dt_ha_msg); const char *user_name = NULL; const char *target_node = lrm_op_target(input->xml); gboolean is_remote_node = FALSE; bool crm_rsc_delete = FALSE; // Message routed to the local node is targeting a specific, non-local node is_remote_node = !controld_is_local_node(target_node); lrm_state = controld_get_executor_state(target_node, false); if ((lrm_state == NULL) && is_remote_node) { pcmk__err("Failing action because local node has never had connection " "to remote node %s", target_node); synthesize_lrmd_failure(NULL, input->xml, PCMK_EXEC_NOT_CONNECTED, PCMK_OCF_UNKNOWN_ERROR, "Local node has no connection to remote"); return; } pcmk__assert(lrm_state != NULL); user_name = pcmk__update_acl_user(input->msg, PCMK__XA_CRM_USER, NULL); crm_op = pcmk__xe_get(input->msg, PCMK__XA_CRM_TASK); from_sys = pcmk__xe_get(input->msg, PCMK__XA_CRM_SYS_FROM); if (!pcmk__str_eq(from_sys, CRM_SYSTEM_TENGINE, pcmk__str_none)) { from_host = pcmk__xe_get(input->msg, PCMK__XA_SRC); } if (pcmk__str_eq(crm_op, PCMK_ACTION_LRM_DELETE, pcmk__str_none)) { if (!pcmk__str_eq(from_sys, CRM_SYSTEM_TENGINE, pcmk__str_none)) { crm_rsc_delete = TRUE; // from crm_resource } operation = PCMK_ACTION_DELETE; } else if (input->xml != NULL) { operation = pcmk__xe_get(input->xml, PCMK_XA_OPERATION); } CRM_CHECK(!pcmk__str_empty(crm_op) || !pcmk__str_empty(operation), return); pcmk__trace("'%s' execution request from %s as %s user", pcmk__s(crm_op, operation), pcmk__s(from_sys, "unknown subsystem"), pcmk__s(user_name, "current")); if (pcmk__str_eq(crm_op, CRM_OP_LRM_FAIL, pcmk__str_none)) { fail_lrm_resource(input->xml, lrm_state, user_name, from_host, from_sys); } else if (pcmk__str_eq(crm_op, CRM_OP_REPROBE, pcmk__str_none) || pcmk__str_eq(operation, CRM_OP_REPROBE, pcmk__str_none)) { const char *raw_target = NULL; if (input->xml != NULL) { // For CRM_OP_REPROBE, a NULL target means we're targeting all nodes raw_target = pcmk__xe_get(input->xml, PCMK__META_ON_NODE); } handle_reprobe_op(lrm_state, input->msg, from_sys, from_host, user_name, is_remote_node, (raw_target == NULL)); } else if (operation != NULL) { lrmd_rsc_info_t *rsc = NULL; xmlNode *xml_rsc = pcmk__xe_first_child(input->xml, PCMK_XE_PRIMITIVE, NULL, NULL); gboolean create_rsc = !pcmk__str_eq(operation, PCMK_ACTION_DELETE, pcmk__str_none); int rc; // We can't return anything meaningful without a resource ID CRM_CHECK((xml_rsc != NULL) && (pcmk__xe_id(xml_rsc) != NULL), return); rc = get_lrm_resource(lrm_state, xml_rsc, create_rsc, &rsc); if (rc == -ENOTCONN) { synthesize_lrmd_failure(lrm_state, input->xml, PCMK_EXEC_NOT_CONNECTED, PCMK_OCF_UNKNOWN_ERROR, "Not connected to remote executor"); return; } else if ((rc < 0) && !create_rsc) { /* Delete of malformed or nonexistent resource * (deleting something that does not exist is a success) */ pcmk__debug("Not registering resource '%s' for a %s event " QB_XS " get-rc=%d (%s) transition-key=%s", pcmk__xe_id(xml_rsc), operation, rc, pcmk_strerror(rc), pcmk__xe_id(input->xml)); delete_rsc_entry(lrm_state, input, pcmk__xe_id(xml_rsc), NULL, pcmk_ok, user_name, true); return; } else if (rc == -EINVAL) { // Resource operation on malformed resource pcmk__err("Invalid resource definition for %s", pcmk__xe_id(xml_rsc)); pcmk__log_xml_warn(input->msg, "invalid resource"); synthesize_lrmd_failure(lrm_state, input->xml, PCMK_EXEC_ERROR, PCMK_OCF_NOT_CONFIGURED, // fatal error "Invalid resource definition"); return; } else if (rc < 0) { // Error communicating with the executor pcmk__err("Could not register resource '%s' with executor: %s " QB_XS " rc=%d", pcmk__xe_id(xml_rsc), pcmk_strerror(rc), rc); pcmk__log_xml_warn(input->msg, "failed registration"); synthesize_lrmd_failure(lrm_state, input->xml, PCMK_EXEC_ERROR, PCMK_OCF_INVALID_PARAM, // hard error "Could not register resource with executor"); return; } if (pcmk__str_eq(operation, PCMK_ACTION_CANCEL, pcmk__str_none)) { if (!do_lrm_cancel(input, lrm_state, rsc, from_host, from_sys)) { pcmk__log_xml_warn(input->xml, "Bad command"); } } else if (pcmk__str_eq(operation, PCMK_ACTION_DELETE, pcmk__str_none)) { do_lrm_delete(input, lrm_state, rsc, from_sys, from_host, crm_rsc_delete, user_name); } else { struct ra_metadata_s *md = NULL; /* Getting metadata from cache is OK except for start actions -- * always refresh from the agent for those, in case the resource * agent was updated. * * @TODO Only refresh metadata for starts if the agent actually * changed (using something like inotify, or a hash or modification * time of the agent executable). */ if (strcmp(operation, PCMK_ACTION_START) != 0) { md = controld_get_rsc_metadata(lrm_state, rsc, controld_metadata_from_cache); } if ((md == NULL) && crm_op_needs_metadata(rsc->standard, operation)) { /* Most likely, we'll need the agent metadata to record the * pending operation and the operation result. Get it now rather * than wait until then, so the metadata action doesn't eat into * the real action's timeout. * * @TODO Metadata is retrieved via direct execution of the * agent, which has a couple of related issues: the executor * should execute agents, not the controller; and metadata for * Pacemaker Remote nodes should be collected on those nodes, * not locally. */ struct metadata_cb_data *data = NULL; data = new_metadata_cb_data(rsc, input->xml); pcmk__info("Retrieving metadata for %s (%s%s%s:%s) " "asynchronously", rsc->id, rsc->standard, ((rsc->provider != NULL)? ":" : ""), pcmk__s(rsc->provider, ""), rsc->type); (void) lrmd__metadata_async(rsc, metadata_complete, (void *) data); } else { do_lrm_rsc_op(lrm_state, rsc, input->xml, md); } } lrmd_free_rsc_info(rsc); } else { pcmk__err("Invalid execution request: unknown command '%s' (bug?)", crm_op); register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL); } } static lrmd_event_data_t * construct_op(const lrm_state_t *lrm_state, const xmlNode *rsc_op, const char *rsc_id, const char *operation) { lrmd_event_data_t *op = NULL; const char *op_delay = NULL; const char *op_timeout = NULL; GHashTable *params = NULL; xmlNode *primitive = NULL; const char *class = NULL; const char *transition = NULL; pcmk__assert((rsc_id != NULL) && (operation != NULL)); op = lrmd_new_event(rsc_id, operation, 0); op->type = lrmd_event_exec_complete; op->timeout = 0; op->start_delay = 0; lrmd__set_result(op, PCMK_OCF_UNKNOWN, PCMK_EXEC_PENDING, NULL); if (rsc_op == NULL) { CRM_LOG_ASSERT(pcmk__str_eq(operation, PCMK_ACTION_STOP, pcmk__str_casei)); op->user_data = NULL; /* the stop_all_resources() case * by definition there is no DC (or they'd be shutting * us down). * So we should put our version here. */ op->params = pcmk__strkey_table(free, free); pcmk__insert_dup(op->params, PCMK_XA_CRM_FEATURE_SET, CRM_FEATURE_SET); pcmk__trace("Constructed %s op for %s", operation, rsc_id); return op; } params = xml2list(rsc_op); g_hash_table_remove(params, CRM_META "_" PCMK__META_OP_TARGET_RC); op_delay = crm_meta_value(params, PCMK_META_START_DELAY); pcmk__scan_min_int(op_delay, &op->start_delay, 0); op_timeout = crm_meta_value(params, PCMK_META_TIMEOUT); pcmk__scan_min_int(op_timeout, &op->timeout, 0); if (pcmk__guint_from_hash(params, CRM_META "_" PCMK_META_INTERVAL, 0, &(op->interval_ms)) != pcmk_rc_ok) { op->interval_ms = 0; } /* Use pcmk_monitor_timeout instead of meta timeout for stonith recurring monitor, if set */ primitive = pcmk__xe_first_child(rsc_op, PCMK_XE_PRIMITIVE, NULL, NULL); class = pcmk__xe_get(primitive, PCMK_XA_CLASS); if (pcmk__is_set(pcmk_get_ra_caps(class), pcmk_ra_cap_fence_params) && pcmk__str_eq(operation, PCMK_ACTION_MONITOR, pcmk__str_casei) && (op->interval_ms > 0)) { op_timeout = g_hash_table_lookup(params, "pcmk_monitor_timeout"); if (op_timeout != NULL) { long long timeout_ms = 0; if ((pcmk__parse_ms(op_timeout, &timeout_ms) == pcmk_rc_ok) && (timeout_ms >= 0)) { op->timeout = (int) QB_MIN(timeout_ms, INT_MAX); } } } if (!pcmk__str_eq(operation, PCMK_ACTION_STOP, pcmk__str_casei)) { op->params = params; } else { rsc_history_t *entry = NULL; if (lrm_state) { entry = g_hash_table_lookup(lrm_state->resource_history, rsc_id); } /* If we do not have stop parameters cached, use * whatever we are given */ if (!entry || !entry->stop_params) { op->params = params; } else { /* Copy the cached parameter list so that we stop the resource * with the old attributes, not the new ones */ op->params = pcmk__strkey_table(free, free); g_hash_table_foreach(params, copy_meta_keys, op->params); g_hash_table_foreach(entry->stop_params, copy_instance_keys, op->params); g_hash_table_destroy(params); params = NULL; } } /* sanity */ if (op->timeout <= 0) { op->timeout = op->interval_ms; } if (op->start_delay < 0) { op->start_delay = 0; } transition = pcmk__xe_get(rsc_op, PCMK__XA_TRANSITION_KEY); CRM_CHECK(transition != NULL, return op); op->user_data = pcmk__str_copy(transition); if (op->interval_ms != 0) { if (pcmk__strcase_any_of(operation, PCMK_ACTION_START, PCMK_ACTION_STOP, NULL)) { pcmk__err("Start and stop actions cannot have an interval: %u", op->interval_ms); op->interval_ms = 0; } } pcmk__trace("Constructed %s op for %s: interval=%u", operation, rsc_id, op->interval_ms); return op; } /*! * \internal * \brief Send a (synthesized) event result * * Reply with a synthesized event result directly, as opposed to going through * the executor. * * \param[in] to_host Host to send result to * \param[in] to_sys IPC name to send result (NULL for transition engine) * \param[in] rsc Type information about resource the result is for * \param[in,out] op Event with result to send * \param[in] rsc_id ID of resource the result is for */ void controld_ack_event_directly(const char *to_host, const char *to_sys, const lrmd_rsc_info_t *rsc, lrmd_event_data_t *op, const char *rsc_id) { xmlNode *reply = NULL; xmlNode *update, *iter; pcmk__node_status_t *peer = NULL; CRM_CHECK(op != NULL, return); if (op->rsc_id == NULL) { // op->rsc_id is a (const char *) but lrmd_free_event() frees it pcmk__assert(rsc_id != NULL); op->rsc_id = pcmk__str_copy(rsc_id); } if (to_sys == NULL) { to_sys = CRM_SYSTEM_TENGINE; } peer = controld_get_local_node_status(); update = create_node_state_update(peer, node_update_none, NULL, __func__); iter = pcmk__xe_create(update, PCMK__XE_LRM); pcmk__xe_set(iter, PCMK_XA_ID, controld_globals.our_uuid); iter = pcmk__xe_create(iter, PCMK__XE_LRM_RESOURCES); iter = pcmk__xe_create(iter, PCMK__XE_LRM_RESOURCE); pcmk__xe_set(iter, PCMK_XA_ID, op->rsc_id); controld_add_resource_history_xml(iter, rsc, op, controld_globals.cluster->priv->node_name); /* We don't have the original message ID, so use "direct-ack" (we just need * something non-NULL for this to create a reply) * * @TODO It would be better to use the server, message ID, and task from the * original request when callers have it available */ reply = pcmk__new_message(pcmk_ipc_controld, "direct-ack", CRM_SYSTEM_LRMD, to_host, to_sys, CRM_OP_INVOKE_LRM, update); - crm_log_xml_trace(update, "[direct ACK]"); + pcmk__log_xml_trace(update, "[direct ACK]"); pcmk__debug("ACK'ing resource op " PCMK__OP_FMT " from %s: %s", op->rsc_id, op->op_type, op->interval_ms, op->user_data, pcmk__xe_get(reply, PCMK_XA_REFERENCE)); if (relay_message(reply, TRUE) == FALSE) { pcmk__log_xml_err(reply, "Unable to route reply"); } pcmk__xml_free(update); pcmk__xml_free(reply); } gboolean verify_stopped(enum crmd_fsa_state cur_state, int log_level) { gboolean res = TRUE; GList *lrm_state_list = lrm_state_get_list(); GList *state_entry; for (state_entry = lrm_state_list; state_entry != NULL; state_entry = state_entry->next) { lrm_state_t *lrm_state = state_entry->data; if (!lrm_state_verify_stopped(lrm_state, cur_state, log_level)) { /* keep iterating through all even when false is returned */ res = FALSE; } } controld_set_fsa_input_flags(R_SENT_RSC_STOP); g_list_free(lrm_state_list); lrm_state_list = NULL; return res; } struct stop_recurring_action_s { lrmd_rsc_info_t *rsc; lrm_state_t *lrm_state; }; static gboolean stop_recurring_action_by_rsc(gpointer key, gpointer value, gpointer user_data) { gboolean remove = FALSE; struct stop_recurring_action_s *event = user_data; active_op_t *op = value; if ((op->interval_ms != 0) && pcmk__str_eq(op->rsc_id, event->rsc->id, pcmk__str_none)) { pcmk__debug("Cancelling op %d for %s (%s)", op->call_id, op->rsc_id, (const char *) key); remove = !cancel_op(event->lrm_state, event->rsc->id, key, op->call_id, FALSE); } return remove; } static gboolean stop_recurring_actions(gpointer key, gpointer value, gpointer user_data) { gboolean remove = FALSE; lrm_state_t *lrm_state = user_data; active_op_t *op = value; if (op->interval_ms != 0) { pcmk__info("Cancelling op %d for %s (%s)", op->call_id, op->rsc_id, (const char *) key); remove = !cancel_op(lrm_state, op->rsc_id, key, op->call_id, FALSE); } return remove; } /*! * \internal * \brief Check whether recurring actions should be cancelled before an action * * \param[in] rsc_id Resource that action is for * \param[in] action Action being performed * \param[in] interval_ms Operation interval of \p action (in milliseconds) * * \return true if recurring actions should be cancelled, otherwise false */ static bool should_cancel_recurring(const char *rsc_id, const char *action, guint interval_ms) { if (is_remote_lrmd_ra(NULL, NULL, rsc_id) && (interval_ms == 0) && (strcmp(action, PCMK_ACTION_MIGRATE_TO) == 0)) { /* Don't stop monitoring a migrating Pacemaker Remote connection * resource until the entire migration has completed. We must detect if * the connection is unexpectedly severed, even during a migration. */ return false; } // Cancel recurring actions before changing resource state return (interval_ms == 0) && !pcmk__str_any_of(action, PCMK_ACTION_MONITOR, PCMK_ACTION_NOTIFY, NULL); } /*! * \internal * \brief Check whether an action should not be performed at this time * * \param[in] operation Action to be performed * * \return Readable description of why action should not be performed, * or NULL if it should be performed */ static const char * should_nack_action(const char *action) { if (pcmk__is_set(controld_globals.fsa_input_register, R_SHUTDOWN) && pcmk__str_eq(action, PCMK_ACTION_START, pcmk__str_none)) { register_fsa_input(C_SHUTDOWN, I_SHUTDOWN, NULL); return "Not attempting start due to shutdown in progress"; } switch (controld_globals.fsa_state) { case S_NOT_DC: case S_POLICY_ENGINE: // Recalculating case S_TRANSITION_ENGINE: break; default: if (!pcmk__str_eq(action, PCMK_ACTION_STOP, pcmk__str_none)) { return "Controller cannot attempt actions at this time"; } break; } return NULL; } static void do_lrm_rsc_op(lrm_state_t *lrm_state, lrmd_rsc_info_t *rsc, xmlNode *msg, struct ra_metadata_s *md) { int rc; int call_id = 0; char *op_id = NULL; lrmd_event_data_t *op = NULL; fsa_data_t *msg_data = NULL; const char *transition = NULL; const char *operation = NULL; const char *nack_reason = NULL; CRM_CHECK((rsc != NULL) && (msg != NULL), return); operation = pcmk__xe_get(msg, PCMK_XA_OPERATION); CRM_CHECK(!pcmk__str_empty(operation), return); transition = pcmk__xe_get(msg, PCMK__XA_TRANSITION_KEY); if (pcmk__str_empty(transition)) { pcmk__log_xml_err(msg, "Missing transition number"); } if (lrm_state == NULL) { // This shouldn't be possible, but provide a failsafe just in case pcmk__err("Cannot execute %s of %s: No executor connection " QB_XS " transition_key=%s", operation, rsc->id, pcmk__s(transition, "")); synthesize_lrmd_failure(NULL, msg, PCMK_EXEC_INVALID, PCMK_OCF_UNKNOWN_ERROR, "No executor connection"); return; } if (pcmk__str_any_of(operation, PCMK_ACTION_RELOAD, PCMK_ACTION_RELOAD_AGENT, NULL)) { /* Pre-2.1.0 DCs will schedule reload actions only, and 2.1.0+ DCs * will schedule reload-agent actions only. In either case, we need * to map that to whatever the resource agent actually supports. * Default to the OCF 1.1 name. */ if ((md != NULL) && pcmk__is_set(md->ra_flags, ra_supports_legacy_reload)) { operation = PCMK_ACTION_RELOAD; } else { operation = PCMK_ACTION_RELOAD_AGENT; } } op = construct_op(lrm_state, msg, rsc->id, operation); CRM_CHECK(op != NULL, return); if (should_cancel_recurring(rsc->id, operation, op->interval_ms)) { guint removed = 0; struct stop_recurring_action_s data; data.rsc = rsc; data.lrm_state = lrm_state; removed = g_hash_table_foreach_remove(lrm_state->active_ops, stop_recurring_action_by_rsc, &data); if (removed) { pcmk__debug("Stopped %u recurring operation%s in preparation for " PCMK__OP_FMT, removed, pcmk__plural_s(removed), rsc->id, operation, op->interval_ms); } } nack_reason = should_nack_action(operation); if (nack_reason != NULL) { pcmk__notice("Not requesting local execution of %s operation for %s on " "%s in state %s: %s", pcmk__readable_action(op->op_type, op->interval_ms), rsc->id, lrm_state->node_name, fsa_state2string(controld_globals.fsa_state), nack_reason); lrmd__set_result(op, PCMK_OCF_UNKNOWN_ERROR, PCMK_EXEC_INVALID, nack_reason); controld_ack_event_directly(NULL, NULL, rsc, op, rsc->id); lrmd_free_event(op); free(op_id); return; } pcmk__notice("Requesting local execution of %s operation for %s on %s " QB_XS " transition %s", pcmk__readable_action(op->op_type, op->interval_ms), rsc->id, lrm_state->node_name, pcmk__s(transition, "")); controld_record_pending_op(lrm_state->node_name, rsc, op); op_id = pcmk__op_key(rsc->id, op->op_type, op->interval_ms); if (op->interval_ms > 0) { /* cancel it so we can then restart it without conflict */ cancel_op_key(lrm_state, rsc, op_id, FALSE); } rc = controld_execute_resource_agent(lrm_state, rsc->id, op->op_type, op->user_data, op->interval_ms, op->timeout, op->start_delay, op->params, &call_id); if (rc == pcmk_rc_ok) { /* record all operations so we can wait * for them to complete during shutdown */ char *call_id_s = make_stop_id(rsc->id, call_id); active_op_t *pending = NULL; pending = pcmk__assert_alloc(1, sizeof(active_op_t)); pcmk__trace("Recording pending op: %d - %s %s", call_id, op_id, call_id_s); pending->call_id = call_id; pending->interval_ms = op->interval_ms; pending->op_type = pcmk__str_copy(operation); pending->op_key = pcmk__str_copy(op_id); pending->rsc_id = pcmk__str_copy(rsc->id); pending->start_time = time(NULL); pending->user_data = pcmk__str_copy(op->user_data); pcmk__xe_get_time(msg, PCMK_OPT_SHUTDOWN_LOCK, &(pending->lock_time)); g_hash_table_replace(lrm_state->active_ops, call_id_s, pending); if ((op->interval_ms > 0) && (op->start_delay > START_DELAY_THRESHOLD)) { int target_rc = PCMK_OCF_OK; pcmk__info("Faking confirmation of %s: execution postponed for " "over 5 minutes", op_id); decode_transition_key(op->user_data, NULL, NULL, NULL, &target_rc); lrmd__set_result(op, target_rc, PCMK_EXEC_DONE, NULL); controld_ack_event_directly(NULL, NULL, rsc, op, rsc->id); } pending->params = op->params; op->params = NULL; } else if (lrm_state_is_local(lrm_state)) { pcmk__err("Could not initiate %s action for resource %s locally: %s " QB_XS " rc=%d", operation, rsc->id, pcmk_rc_str(rc), rc); fake_op_status(lrm_state, op, PCMK_EXEC_NOT_CONNECTED, PCMK_OCF_UNKNOWN_ERROR, pcmk_rc_str(rc)); process_lrm_event(lrm_state, op, NULL, NULL); register_fsa_error(C_FSA_INTERNAL, I_FAIL, NULL); } else { pcmk__err("Could not initiate %s action for resource %s remotely on " "%s: %s " QB_XS " rc=%d", operation, rsc->id, lrm_state->node_name, pcmk_rc_str(rc), rc); fake_op_status(lrm_state, op, PCMK_EXEC_NOT_CONNECTED, PCMK_OCF_UNKNOWN_ERROR, pcmk_rc_str(rc)); process_lrm_event(lrm_state, op, NULL, NULL); } free(op_id); lrmd_free_event(op); } static char * unescape_newlines(const char *string) { char *pch = NULL; char *ret = NULL; static const char *escaped_newline = "\\n"; if (!string) { return NULL; } ret = pcmk__str_copy(string); pch = strstr(ret, escaped_newline); while (pch != NULL) { /* Replace newline escape pattern with actual newline (and a space so we * don't have to shuffle the rest of the buffer) */ pch[0] = '\n'; pch[1] = ' '; pch = strstr(pch, escaped_newline); } return ret; } static bool did_lrm_rsc_op_fail(lrm_state_t *lrm_state, const char * rsc_id, const char * op_type, guint interval_ms) { rsc_history_t *entry = NULL; CRM_CHECK(lrm_state != NULL, return FALSE); CRM_CHECK(rsc_id != NULL, return FALSE); CRM_CHECK(op_type != NULL, return FALSE); entry = g_hash_table_lookup(lrm_state->resource_history, rsc_id); if (entry == NULL || entry->failed == NULL) { return FALSE; } if (pcmk__str_eq(entry->failed->rsc_id, rsc_id, pcmk__str_none) && pcmk__str_eq(entry->failed->op_type, op_type, pcmk__str_casei) && entry->failed->interval_ms == interval_ms) { return TRUE; } return FALSE; } /*! * \internal * \brief Log the result of an executor action (actual or synthesized) * * \param[in] op Executor action to log result for * \param[in] op_key Operation key for action * \param[in] node_name Name of node action was performed on, if known * \param[in] confirmed Whether to log that graph action was confirmed */ static void log_executor_event(const lrmd_event_data_t *op, const char *op_key, const char *node_name, gboolean confirmed) { int log_level = LOG_ERR; GString *str = g_string_sized_new(100); // reasonable starting size pcmk__g_strcat(str, "Result of ", pcmk__readable_action(op->op_type, op->interval_ms), " operation for ", op->rsc_id, NULL); if (node_name != NULL) { pcmk__g_strcat(str, " on ", node_name, NULL); } switch (op->op_status) { case PCMK_EXEC_DONE: log_level = LOG_NOTICE; pcmk__g_strcat(str, ": ", crm_exit_str((crm_exit_t) op->rc), NULL); break; case PCMK_EXEC_TIMEOUT: pcmk__g_strcat(str, ": ", pcmk_exec_status_str(op->op_status), " after ", pcmk__readable_interval(op->timeout), NULL); break; case PCMK_EXEC_CANCELLED: log_level = LOG_INFO; pcmk__g_strcat(str, ": ", pcmk_exec_status_str(op->op_status), NULL); break; default: pcmk__g_strcat(str, ": ", pcmk_exec_status_str(op->op_status), NULL); break; } if ((op->exit_reason != NULL) && ((op->op_status != PCMK_EXEC_DONE) || (op->rc != PCMK_OCF_OK))) { pcmk__g_strcat(str, " (", op->exit_reason, ")", NULL); } g_string_append(str, " " QB_XS); g_string_append_printf(str, " graph action %sconfirmed; call=%d key=%s", (confirmed? "" : "un"), op->call_id, op_key); if (op->op_status == PCMK_EXEC_DONE) { g_string_append_printf(str, " rc=%d", op->rc); } do_crm_log(log_level, "%s", str->str); g_string_free(str, TRUE); /* The services library has already logged the output at info or debug * level, so just raise to notice if it looks like a failure. */ if ((op->output != NULL) && (op->rc != PCMK_OCF_OK)) { char *prefix = pcmk__assert_asprintf(PCMK__OP_FMT "@%s output", op->rsc_id, op->op_type, op->interval_ms, node_name); crm_log_output(LOG_NOTICE, prefix, op->output); free(prefix); } } void process_lrm_event(lrm_state_t *lrm_state, lrmd_event_data_t *op, active_op_t *pending, const xmlNode *action_xml) { char *op_id = NULL; char *op_key = NULL; gboolean remove = FALSE; gboolean removed = FALSE; bool need_direct_ack = FALSE; lrmd_rsc_info_t *rsc = NULL; const char *node_name = NULL; CRM_CHECK(op != NULL, return); CRM_CHECK(op->rsc_id != NULL, return); // Remap new status codes for older DCs if (pcmk__compare_versions(controld_globals.dc_version, "3.2.0") < 0) { switch (op->op_status) { case PCMK_EXEC_NOT_CONNECTED: lrmd__set_result(op, PCMK_OCF_CONNECTION_DIED, PCMK_EXEC_ERROR, op->exit_reason); break; case PCMK_EXEC_INVALID: lrmd__set_result(op, CRM_DIRECT_NACK_RC, PCMK_EXEC_ERROR, op->exit_reason); break; default: break; } } op_id = make_stop_id(op->rsc_id, op->call_id); op_key = pcmk__op_key(op->rsc_id, op->op_type, op->interval_ms); // Get resource info if available (from executor state or action XML) if (lrm_state) { rsc = lrm_state_get_rsc_info(lrm_state, op->rsc_id, 0); } if ((rsc == NULL) && action_xml) { xmlNode *xml = pcmk__xe_first_child(action_xml, PCMK_XE_PRIMITIVE, NULL, NULL); const char *standard = pcmk__xe_get(xml, PCMK_XA_CLASS); const char *provider = pcmk__xe_get(xml, PCMK_XA_PROVIDER); const char *type = pcmk__xe_get(xml, PCMK_XA_TYPE); if (standard && type) { pcmk__info("%s agent information not cached, using %s%s%s:%s from " "action XML", op->rsc_id, standard, (provider? ":" : ""), pcmk__s(provider, ""), type); rsc = lrmd_new_rsc_info(op->rsc_id, standard, provider, type); } else { pcmk__err("Can't process %s result because %s agent information " "not cached or in XML", op_key, op->rsc_id); } } // Get node name if available (from executor state or action XML) if (lrm_state) { node_name = lrm_state->node_name; } else if (action_xml) { node_name = pcmk__xe_get(action_xml, PCMK__META_ON_NODE); } if(pending == NULL) { remove = TRUE; if (lrm_state) { pending = g_hash_table_lookup(lrm_state->active_ops, op_id); } } if (op->op_status == PCMK_EXEC_ERROR) { switch(op->rc) { case PCMK_OCF_NOT_RUNNING: case PCMK_OCF_RUNNING_PROMOTED: case PCMK_OCF_DEGRADED: case PCMK_OCF_DEGRADED_PROMOTED: // Leave it to the TE/scheduler to decide if this is an error op->op_status = PCMK_EXEC_DONE; break; default: /* Nothing to do */ break; } } if (op->op_status != PCMK_EXEC_CANCELLED) { /* We might not record the result, so directly acknowledge it to the * originator instead, so it doesn't time out waiting for the result * (especially important if part of a transition). */ need_direct_ack = TRUE; if (controld_action_is_recordable(op->op_type)) { if (node_name && rsc) { // We should record the result, and happily, we can time_t lock_time = (pending == NULL)? 0 : pending->lock_time; controld_update_resource_history(node_name, rsc, op, lock_time); need_direct_ack = FALSE; } else if (op->rsc_deleted) { /* We shouldn't record the result (likely the resource was * refreshed, cleaned, or removed while this operation was * in flight). */ pcmk__notice("Not recording %s result in CIB because resource " "information was removed since it was initiated", op_key); } else { /* This shouldn't be possible; the executor didn't consider the * resource deleted, but we couldn't find resource or node * information. */ const char *missing = "node name"; if (node_name != NULL) { missing = "resource information"; } pcmk__err("Unable to record %s result in CIB: No %s", op_key, missing); } } } else if (op->interval_ms == 0) { /* A non-recurring operation was cancelled. Most likely, the * never-initiated action was removed from the executor's pending * operations list upon resource removal. */ need_direct_ack = TRUE; } else if (pending == NULL) { /* This recurring operation was cancelled, but was not pending. No * transition actions are waiting on it, nothing needs to be done. */ } else if (op->user_data == NULL) { /* This recurring operation was cancelled and pending, but we don't * have a transition key. This should never happen. */ pcmk__err("Recurring operation %s was cancelled without transition " "information", op_key); } else if (pcmk__is_set(pending->flags, active_op_remove)) { /* This recurring operation was cancelled (by us) and pending, and we * have been waiting for it to finish. */ if (lrm_state) { controld_delete_action_history(op); } /* Directly acknowledge failed recurring actions here. The above call to * controld_delete_action_history() will not erase any corresponding * last_failure entry, which means that the DC won't confirm the * cancellation via process_op_deletion(), and the transition would * otherwise wait for the action timer to pop. */ if (did_lrm_rsc_op_fail(lrm_state, pending->rsc_id, pending->op_type, pending->interval_ms)) { need_direct_ack = TRUE; } } else if (op->rsc_deleted) { /* This recurring operation was cancelled (but not by us, and the * executor does not have resource information, likely due to resource * cleanup, refresh, or removal) and pending. */ pcmk__debug("Recurring op %s was cancelled due to resource deletion", op_key); need_direct_ack = TRUE; } else { /* This recurring operation was cancelled (but not by us, likely by the * executor before stopping the resource) and pending. We don't need to * do anything special. */ } if (need_direct_ack) { controld_ack_event_directly(NULL, NULL, NULL, op, op->rsc_id); } if(remove == FALSE) { /* The caller will do this afterwards, but keep the logging consistent */ removed = TRUE; } else if (lrm_state && ((op->interval_ms == 0) || (op->op_status == PCMK_EXEC_CANCELLED))) { gboolean found = g_hash_table_remove(lrm_state->active_ops, op_id); if (op->interval_ms != 0) { removed = TRUE; } else if (found) { removed = TRUE; pcmk__trace("Op %s (call=%d, stop-id=%s, remaining=%u): Confirmed", op_key, op->call_id, op_id, g_hash_table_size(lrm_state->active_ops)); } } log_executor_event(op, op_key, node_name, removed); if (lrm_state) { if (!pcmk__str_eq(op->op_type, PCMK_ACTION_META_DATA, pcmk__str_casei)) { crmd_alert_resource_op(lrm_state->node_name, op); } else if (rsc && (op->rc == PCMK_OCF_OK)) { char *metadata = unescape_newlines(op->output); controld_cache_metadata(lrm_state->metadata_cache, rsc, metadata); free(metadata); } } if (op->rsc_deleted) { pcmk__info("Deletion of resource '%s' complete after %s", op->rsc_id, op_key); if (lrm_state) { delete_rsc_entry(lrm_state, NULL, op->rsc_id, NULL, pcmk_ok, NULL, true); } } /* If a shutdown was escalated while operations were pending, * then the FSA will be stalled right now... allow it to continue */ controld_trigger_fsa(); if (lrm_state && rsc) { update_history_cache(lrm_state, rsc, op); } lrmd_free_rsc_info(rsc); free(op_key); free(op_id); } diff --git a/daemons/controld/controld_execd_state.c b/daemons/controld/controld_execd_state.c index e28e633ca4..7af912d71c 100644 --- a/daemons/controld/controld_execd_state.c +++ b/daemons/controld/controld_execd_state.c @@ -1,836 +1,836 @@ /* * Copyright 2012-2025 the Pacemaker project contributors * * The version control history for this file may have further details. * * This source code is licensed under the GNU General Public License version 2 * or later (GPLv2+) WITHOUT ANY WARRANTY. */ #include #include #include #include #include // pcmk_unpack_nvpair_blocks() #include #include #include #include static GHashTable *lrm_state_table = NULL; extern GHashTable *proxy_table; int lrmd_internal_proxy_send(lrmd_t * lrmd, xmlNode *msg); void lrmd_internal_set_proxy_callback(lrmd_t * lrmd, void *userdata, void (*callback)(lrmd_t *lrmd, void *userdata, xmlNode *msg)); static void free_rsc_info(gpointer value) { lrmd_rsc_info_t *rsc_info = value; lrmd_free_rsc_info(rsc_info); } static void free_deletion_op(gpointer value) { struct pending_deletion_op_s *op = value; free(op->rsc); delete_ha_msg_input(op->input); free(op); } static void free_recurring_op(gpointer value) { active_op_t *op = value; free(op->user_data); free(op->rsc_id); free(op->op_type); free(op->op_key); if (op->params) { g_hash_table_destroy(op->params); } free(op); } static gboolean fail_pending_op(gpointer key, gpointer value, gpointer user_data) { lrmd_event_data_t event = { 0, }; lrm_state_t *lrm_state = user_data; active_op_t *op = value; pcmk__trace("Pre-emptively failing " PCMK__OP_FMT " on %s (call=%s, %s)", op->rsc_id, op->op_type, op->interval_ms, lrm_state->node_name, (const char *) key, op->user_data); event.type = lrmd_event_exec_complete; event.rsc_id = op->rsc_id; event.op_type = op->op_type; event.user_data = op->user_data; event.timeout = 0; event.interval_ms = op->interval_ms; lrmd__set_result(&event, PCMK_OCF_UNKNOWN_ERROR, PCMK_EXEC_NOT_CONNECTED, "Action was pending when executor connection was dropped"); event.t_run = op->start_time; event.t_rcchange = op->start_time; event.call_id = op->call_id; event.remote_nodename = lrm_state->node_name; event.params = op->params; process_lrm_event(lrm_state, &event, op, NULL); lrmd__reset_result(&event); return TRUE; } gboolean lrm_state_is_local(lrm_state_t *lrm_state) { return (lrm_state != NULL) && controld_is_local_node(lrm_state->node_name); } /*! * \internal * \brief Create executor state entry for a node and add it to the state table * * \param[in] node_name Node to create entry for * * \return Newly allocated executor state object initialized for \p node_name */ static lrm_state_t * lrm_state_create(const char *node_name) { lrm_state_t *state = NULL; if (!node_name) { pcmk__err("No node name given for lrm state object"); return NULL; } state = pcmk__assert_alloc(1, sizeof(lrm_state_t)); state->node_name = pcmk__str_copy(node_name); state->rsc_info_cache = pcmk__strkey_table(NULL, free_rsc_info); state->deletion_ops = pcmk__strkey_table(free, free_deletion_op); state->active_ops = pcmk__strkey_table(free, free_recurring_op); state->resource_history = pcmk__strkey_table(NULL, history_free); state->metadata_cache = metadata_cache_new(); g_hash_table_insert(lrm_state_table, (char *)state->node_name, state); return state; } static gboolean remote_proxy_remove_by_node(gpointer key, gpointer value, gpointer user_data) { remote_proxy_t *proxy = value; const char *node_name = user_data; if (pcmk__str_eq(node_name, proxy->node_name, pcmk__str_casei)) { return TRUE; } return FALSE; } static remote_proxy_t * find_connected_proxy_by_node(const char * node_name) { GHashTableIter gIter; remote_proxy_t *proxy = NULL; CRM_CHECK(proxy_table != NULL, return NULL); g_hash_table_iter_init(&gIter, proxy_table); while (g_hash_table_iter_next(&gIter, NULL, (gpointer *) &proxy)) { if (proxy->source && pcmk__str_eq(node_name, proxy->node_name, pcmk__str_casei)) { return proxy; } } return NULL; } static void remote_proxy_disconnect_by_node(const char * node_name) { remote_proxy_t *proxy = NULL; CRM_CHECK(proxy_table != NULL, return); while ((proxy = find_connected_proxy_by_node(node_name)) != NULL) { /* mainloop_del_ipc_client() eventually calls remote_proxy_disconnected() * , which removes the entry from proxy_table. * Do not do this in a g_hash_table_iter_next() loop. */ if (proxy->source) { mainloop_del_ipc_client(proxy->source); } } return; } static void internal_lrm_state_destroy(gpointer data) { lrm_state_t *lrm_state = data; if (!lrm_state) { return; } /* Rather than directly remove the recorded proxy entries from proxy_table, * make sure any connected proxies get disconnected. So that * remote_proxy_disconnected() will be called and as well remove the * entries from proxy_table. */ remote_proxy_disconnect_by_node(lrm_state->node_name); pcmk__trace("Destroying proxy table %s with %u members", lrm_state->node_name, g_hash_table_size(proxy_table)); // Just in case there's still any leftovers in proxy_table g_hash_table_foreach_remove(proxy_table, remote_proxy_remove_by_node, (char *) lrm_state->node_name); remote_ra_cleanup(lrm_state); lrmd_api_delete(lrm_state->conn); if (lrm_state->rsc_info_cache) { pcmk__trace("Destroying rsc info cache with %u members", g_hash_table_size(lrm_state->rsc_info_cache)); g_hash_table_destroy(lrm_state->rsc_info_cache); } if (lrm_state->resource_history) { pcmk__trace("Destroying history op cache with %u members", g_hash_table_size(lrm_state->resource_history)); g_hash_table_destroy(lrm_state->resource_history); } if (lrm_state->deletion_ops) { pcmk__trace("Destroying deletion op cache with %u members", g_hash_table_size(lrm_state->deletion_ops)); g_hash_table_destroy(lrm_state->deletion_ops); } if (lrm_state->active_ops != NULL) { pcmk__trace("Destroying pending op cache with %u members", g_hash_table_size(lrm_state->active_ops)); g_hash_table_destroy(lrm_state->active_ops); } metadata_cache_free(lrm_state->metadata_cache); free((char *)lrm_state->node_name); free(lrm_state); } void lrm_state_reset_tables(lrm_state_t * lrm_state, gboolean reset_metadata) { if (lrm_state->resource_history) { pcmk__trace("Resetting resource history cache with %u members", g_hash_table_size(lrm_state->resource_history)); g_hash_table_remove_all(lrm_state->resource_history); } if (lrm_state->deletion_ops) { pcmk__trace("Resetting deletion operations cache with %u members", g_hash_table_size(lrm_state->deletion_ops)); g_hash_table_remove_all(lrm_state->deletion_ops); } if (lrm_state->active_ops != NULL) { pcmk__trace("Resetting active operations cache with %u members", g_hash_table_size(lrm_state->active_ops)); g_hash_table_remove_all(lrm_state->active_ops); } if (lrm_state->rsc_info_cache) { pcmk__trace("Resetting resource information cache with %u members", g_hash_table_size(lrm_state->rsc_info_cache)); g_hash_table_remove_all(lrm_state->rsc_info_cache); } if (reset_metadata) { metadata_cache_reset(lrm_state->metadata_cache); } } gboolean lrm_state_init_local(void) { if (lrm_state_table) { return TRUE; } lrm_state_table = pcmk__strikey_table(NULL, internal_lrm_state_destroy); if (!lrm_state_table) { return FALSE; } proxy_table = pcmk__strikey_table(NULL, remote_proxy_free); if (!proxy_table) { g_hash_table_destroy(lrm_state_table); lrm_state_table = NULL; return FALSE; } return TRUE; } void lrm_state_destroy_all(void) { if (lrm_state_table) { pcmk__trace("Destroying state table with %u members", g_hash_table_size(lrm_state_table)); g_hash_table_destroy(lrm_state_table); lrm_state_table = NULL; } if(proxy_table) { pcmk__trace("Destroying proxy table with %u members", g_hash_table_size(proxy_table)); g_hash_table_destroy(proxy_table); proxy_table = NULL; } } /*! * \internal * \brief Get executor state object * * \param[in] node_name Get executor state for this node (local node if NULL) * \param[in] create If true, create executor state if it doesn't exist * * \return Executor state object for \p node_name */ lrm_state_t * controld_get_executor_state(const char *node_name, bool create) { lrm_state_t *state = NULL; if ((node_name == NULL) && (controld_globals.cluster != NULL)) { node_name = controld_globals.cluster->priv->node_name; } if ((node_name == NULL) || (lrm_state_table == NULL)) { return NULL; } state = g_hash_table_lookup(lrm_state_table, node_name); if ((state == NULL) && create) { state = lrm_state_create(node_name); } return state; } /* @TODO the lone caller just needs to iterate over the values, so replace this * with a g_hash_table_foreach() wrapper instead */ GList * lrm_state_get_list(void) { if (lrm_state_table == NULL) { return NULL; } return g_hash_table_get_values(lrm_state_table); } void lrm_state_disconnect_only(lrm_state_t * lrm_state) { guint removed = 0; if (!lrm_state->conn) { return; } pcmk__trace("Disconnecting %s", lrm_state->node_name); remote_proxy_disconnect_by_node(lrm_state->node_name); ((lrmd_t *) lrm_state->conn)->cmds->disconnect(lrm_state->conn); if (!pcmk__is_set(controld_globals.fsa_input_register, R_SHUTDOWN)) { removed = g_hash_table_foreach_remove(lrm_state->active_ops, fail_pending_op, lrm_state); pcmk__trace("Synthesized %u operation failures for %s", removed, lrm_state->node_name); } } void lrm_state_disconnect(lrm_state_t * lrm_state) { if (!lrm_state->conn) { return; } lrm_state_disconnect_only(lrm_state); lrmd_api_delete(lrm_state->conn); lrm_state->conn = NULL; } int lrm_state_is_connected(lrm_state_t * lrm_state) { if (!lrm_state->conn) { return FALSE; } return ((lrmd_t *) lrm_state->conn)->cmds->is_connected(lrm_state->conn); } int lrm_state_poke_connection(lrm_state_t * lrm_state) { if (!lrm_state->conn) { return -ENOTCONN; } return ((lrmd_t *) lrm_state->conn)->cmds->poke_connection(lrm_state->conn); } // \return Standard Pacemaker return code int controld_connect_local_executor(lrm_state_t *lrm_state) { int rc = pcmk_rc_ok; if (lrm_state->conn == NULL) { lrmd_t *api = NULL; rc = lrmd__new(&api, NULL, NULL, 0); if (rc != pcmk_rc_ok) { return rc; } api->cmds->set_callback(api, lrm_op_callback); lrm_state->conn = api; } rc = ((lrmd_t *) lrm_state->conn)->cmds->connect(lrm_state->conn, CRM_SYSTEM_CRMD, NULL); rc = pcmk_legacy2rc(rc); if (rc == pcmk_rc_ok) { lrm_state->num_lrm_register_fails = 0; } else { lrm_state->num_lrm_register_fails++; } return rc; } static remote_proxy_t * crmd_remote_proxy_new(lrmd_t *lrmd, const char *node_name, const char *session_id, const char *channel) { struct ipc_client_callbacks proxy_callbacks = { .dispatch = remote_proxy_dispatch, .destroy = remote_proxy_disconnected }; remote_proxy_t *proxy = remote_proxy_new(lrmd, &proxy_callbacks, node_name, session_id, channel); return proxy; } gboolean crmd_is_proxy_session(const char *session) { return g_hash_table_lookup(proxy_table, session) ? TRUE : FALSE; } void crmd_proxy_send(const char *session, xmlNode *msg) { remote_proxy_t *proxy = g_hash_table_lookup(proxy_table, session); lrm_state_t *lrm_state = NULL; if (!proxy) { return; } - crm_log_xml_trace(msg, "to-proxy"); + pcmk__log_xml_trace(msg, "to-proxy"); lrm_state = controld_get_executor_state(proxy->node_name, false); if (lrm_state) { pcmk__trace("Sending event to %.8s on %s", proxy->session_id, proxy->node_name); remote_proxy_relay_event(proxy, msg); } } static void crmd_proxy_dispatch(const char *session, xmlNode *msg) { pcmk__trace("Processing proxied IPC message from session %s", session); - crm_log_xml_trace(msg, "controller[inbound]"); + pcmk__log_xml_trace(msg, "controller[inbound]"); pcmk__xe_set(msg, PCMK__XA_CRM_SYS_FROM, session); if (controld_authorize_ipc_message(msg, NULL, session)) { route_message(C_IPC_MESSAGE, msg); } controld_trigger_fsa(); } static void remote_config_check(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data) { if (rc != pcmk_ok) { pcmk__err("Query resulted in an error: %s", pcmk_strerror(rc)); if (rc == -EACCES || rc == -pcmk_err_schema_validation) { pcmk__err("The cluster is mis-configured - shutting down and " "staying down"); } } else { lrmd_t * lrmd = (lrmd_t *)user_data; crm_time_t *now = crm_time_new(NULL); GHashTable *config_hash = pcmk__strkey_table(free, free); pcmk_rule_input_t rule_input = { .now = now, }; pcmk__debug("Call %d : Parsing CIB options", call_id); pcmk_unpack_nvpair_blocks(output, PCMK_XE_CLUSTER_PROPERTY_SET, PCMK_VALUE_CIB_BOOTSTRAP_OPTIONS, &rule_input, config_hash, NULL); /* Now send it to the remote peer */ lrmd__validate_remote_settings(lrmd, config_hash); g_hash_table_destroy(config_hash); crm_time_free(now); } } static void crmd_remote_proxy_cb(lrmd_t *lrmd, void *userdata, xmlNode *msg) { lrm_state_t *lrm_state = userdata; const char *session = pcmk__xe_get(msg, PCMK__XA_LRMD_IPC_SESSION); remote_proxy_t *proxy = g_hash_table_lookup(proxy_table, session); const char *op = pcmk__xe_get(msg, PCMK__XA_LRMD_IPC_OP); if (pcmk__str_eq(op, LRMD_IPC_OP_NEW, pcmk__str_casei)) { const char *channel = pcmk__xe_get(msg, PCMK__XA_LRMD_IPC_SERVER); proxy = crmd_remote_proxy_new(lrmd, lrm_state->node_name, session, channel); if (!remote_ra_controlling_guest(lrm_state)) { if (proxy != NULL) { cib_t *cib_conn = controld_globals.cib_conn; /* Look up PCMK_OPT_STONITH_WATCHDOG_TIMEOUT and send to the * remote peer for validation */ int rc = cib_conn->cmds->query(cib_conn, PCMK_XE_CRM_CONFIG, NULL, cib_none); cib_conn->cmds->register_callback_full(cib_conn, rc, 10, FALSE, lrmd, "remote_config_check", remote_config_check, NULL); } } else { pcmk__debug("Skipping remote_config_check for guest-nodes"); } } else if (pcmk__str_eq(op, LRMD_IPC_OP_SHUTDOWN_REQ, pcmk__str_casei)) { char *now_s = NULL; pcmk__notice("%s requested shutdown of its remote connection", lrm_state->node_name); if (!remote_ra_is_in_maintenance(lrm_state)) { now_s = pcmk__ttoa(time(NULL)); update_attrd(lrm_state->node_name, PCMK__NODE_ATTR_SHUTDOWN, now_s, NULL, TRUE); free(now_s); remote_proxy_ack_shutdown(lrmd); pcmk__warn("Reconnection attempts to %s may result in failures " "that must be cleared", lrm_state->node_name); } else { remote_proxy_nack_shutdown(lrmd); pcmk__notice("Remote resource for %s is not managed so no ordered " "shutdown happening", lrm_state->node_name); } return; } else if (pcmk__str_eq(op, LRMD_IPC_OP_REQUEST, pcmk__str_casei) && proxy && proxy->is_local) { /* This is for the controller, which we are, so don't try * to send to ourselves over IPC -- do it directly. */ uint32_t flags = 0U; int rc = pcmk_rc_ok; xmlNode *wrapper = pcmk__xe_first_child(msg, PCMK__XE_LRMD_IPC_MSG, NULL, NULL); xmlNode *request = pcmk__xe_first_child(wrapper, NULL, NULL, NULL); CRM_CHECK(request != NULL, return); CRM_CHECK(lrm_state->node_name, return); pcmk__xe_set(request, PCMK_XE_ACL_ROLE, "pacemaker-remote"); pcmk__update_acl_user(request, PCMK__XA_LRMD_IPC_USER, lrm_state->node_name); /* Pacemaker Remote nodes don't know their own names (as known to the * cluster). When getting a node info request with no name or ID, add * the name, so we don't return info for ourselves instead of the * Pacemaker Remote node. */ if (pcmk__str_eq(pcmk__xe_get(request, PCMK__XA_CRM_TASK), CRM_OP_NODE_INFO, pcmk__str_none)) { int node_id = 0; pcmk__xe_get_int(request, PCMK_XA_ID, &node_id); if ((node_id <= 0) && (pcmk__xe_get(request, PCMK_XA_UNAME) == NULL)) { pcmk__xe_set(request, PCMK_XA_UNAME, lrm_state->node_name); } } crmd_proxy_dispatch(session, request); rc = pcmk__xe_get_flags(msg, PCMK__XA_LRMD_IPC_MSG_FLAGS, &flags, 0U); if (rc != pcmk_rc_ok) { pcmk__warn("Couldn't parse controller flags from remote request: " "%s", pcmk_rc_str(rc)); } if (pcmk__is_set(flags, crm_ipc_client_response)) { int msg_id = 0; xmlNode *op_reply = pcmk__xe_create(NULL, PCMK__XE_ACK); pcmk__xe_set(op_reply, PCMK_XA_FUNCTION, __func__); pcmk__xe_set_int(op_reply, PCMK__XA_LINE, __LINE__); pcmk__xe_get_int(msg, PCMK__XA_LRMD_IPC_MSG_ID, &msg_id); remote_proxy_relay_response(proxy, op_reply, msg_id); pcmk__xml_free(op_reply); } } else { remote_proxy_cb(lrmd, lrm_state->node_name, msg); } } // \return Standard Pacemaker return code int controld_connect_remote_executor(lrm_state_t *lrm_state, const char *server, int port, int timeout_ms) { int rc = pcmk_rc_ok; if (lrm_state->conn == NULL) { lrmd_t *api = NULL; rc = lrmd__new(&api, lrm_state->node_name, server, port); if (rc != pcmk_rc_ok) { pcmk__warn("Pacemaker Remote connection to %s:%s failed: %s " QB_XS " rc=%d", server, port, pcmk_rc_str(rc), rc); return rc; } lrm_state->conn = api; api->cmds->set_callback(api, remote_lrm_op_callback); lrmd_internal_set_proxy_callback(api, lrm_state, crmd_remote_proxy_cb); } pcmk__trace("Initiating remote connection to %s:%d with timeout %dms", server, port, timeout_ms); rc = ((lrmd_t *) lrm_state->conn)->cmds->connect_async(lrm_state->conn, lrm_state->node_name, timeout_ms); if (rc == pcmk_ok) { lrm_state->num_lrm_register_fails = 0; } else { lrm_state->num_lrm_register_fails++; // Ignored for remote connections } return pcmk_legacy2rc(rc); } int lrm_state_get_metadata(lrm_state_t * lrm_state, const char *class, const char *provider, const char *agent, char **output, enum lrmd_call_options options) { lrmd_key_value_t *params = NULL; if (!lrm_state->conn) { return -ENOTCONN; } /* Add the node name to the environment, as is done with normal resource * action calls. Meta-data calls shouldn't need it, but some agents are * written with an ocf_local_nodename call at the beginning regardless of * action. Without the environment variable, the agent would try to contact * the controller to get the node name -- but the controller would be * blocking on the synchronous meta-data call. * * At this point, we have to assume that agents are unlikely to make other * calls that require the controller, such as crm_node --quorum or * --cluster-id. * * @TODO Make meta-data calls asynchronous. (This will be part of a larger * project to make meta-data calls via the executor rather than directly.) */ params = lrmd_key_value_add(params, CRM_META "_" PCMK__META_ON_NODE, lrm_state->node_name); return ((lrmd_t *) lrm_state->conn)->cmds->get_metadata_params(lrm_state->conn, class, provider, agent, output, options, params); } int lrm_state_cancel(lrm_state_t *lrm_state, const char *rsc_id, const char *action, guint interval_ms) { if (!lrm_state->conn) { return -ENOTCONN; } /* Figure out a way to make this async? * NOTICE: Currently it's synced and directly acknowledged in do_lrm_invoke(). */ if (is_remote_lrmd_ra(NULL, NULL, rsc_id)) { return remote_ra_cancel(lrm_state, rsc_id, action, interval_ms); } return ((lrmd_t *) lrm_state->conn)->cmds->cancel(lrm_state->conn, rsc_id, action, interval_ms); } lrmd_rsc_info_t * lrm_state_get_rsc_info(lrm_state_t * lrm_state, const char *rsc_id, enum lrmd_call_options options) { lrmd_rsc_info_t *rsc = NULL; if (!lrm_state->conn) { return NULL; } if (is_remote_lrmd_ra(NULL, NULL, rsc_id)) { return remote_ra_get_rsc_info(lrm_state, rsc_id); } rsc = g_hash_table_lookup(lrm_state->rsc_info_cache, rsc_id); if (rsc == NULL) { /* only contact the lrmd if we don't already have a cached rsc info */ rsc = ((lrmd_t *) lrm_state->conn)->cmds->get_rsc_info(lrm_state->conn, rsc_id, options); if (rsc == NULL) { return NULL; } /* cache the result */ g_hash_table_insert(lrm_state->rsc_info_cache, rsc->id, rsc); } return lrmd_copy_rsc_info(rsc); } /*! * \internal * \brief Initiate a resource agent action * * \param[in,out] lrm_state Executor state object * \param[in] rsc_id ID of resource for action * \param[in] action Action to execute * \param[in] userdata String to copy and pass to execution callback * \param[in] interval_ms Action interval (in milliseconds) * \param[in] timeout_ms Action timeout (in milliseconds) * \param[in] start_delay_ms Delay (in ms) before initiating action * \param[in] parameters Hash table of resource parameters * \param[out] call_id Where to store call ID on success * * \return Standard Pacemaker return code */ int controld_execute_resource_agent(lrm_state_t *lrm_state, const char *rsc_id, const char *action, const char *userdata, guint interval_ms, int timeout_ms, int start_delay_ms, GHashTable *parameters, int *call_id) { int rc = pcmk_rc_ok; lrmd_key_value_t *params = NULL; if (lrm_state->conn == NULL) { return ENOTCONN; } // Convert parameters from hash table to list if (parameters != NULL) { const char *key = NULL; const char *value = NULL; GHashTableIter iter; g_hash_table_iter_init(&iter, parameters); while (g_hash_table_iter_next(&iter, (gpointer *) &key, (gpointer *) &value)) { params = lrmd_key_value_add(params, key, value); } } if (is_remote_lrmd_ra(NULL, NULL, rsc_id)) { rc = controld_execute_remote_agent(lrm_state, rsc_id, action, userdata, interval_ms, timeout_ms, start_delay_ms, params, call_id); } else { rc = ((lrmd_t *) lrm_state->conn)->cmds->exec(lrm_state->conn, rsc_id, action, userdata, interval_ms, timeout_ms, start_delay_ms, lrmd_opt_notify_changes_only, params); if (rc < 0) { rc = pcmk_legacy2rc(rc); } else { *call_id = rc; rc = pcmk_rc_ok; } } return rc; } int lrm_state_register_rsc(lrm_state_t * lrm_state, const char *rsc_id, const char *class, const char *provider, const char *agent, enum lrmd_call_options options) { lrmd_t *conn = (lrmd_t *) lrm_state->conn; if (conn == NULL) { return -ENOTCONN; } if (is_remote_lrmd_ra(agent, provider, NULL)) { return controld_get_executor_state(rsc_id, true)? pcmk_ok : -EINVAL; } /* @TODO Implement an asynchronous version of this (currently a blocking * call to the lrmd). */ return conn->cmds->register_rsc(lrm_state->conn, rsc_id, class, provider, agent, options); } int lrm_state_unregister_rsc(lrm_state_t * lrm_state, const char *rsc_id, enum lrmd_call_options options) { if (!lrm_state->conn) { return -ENOTCONN; } if (is_remote_lrmd_ra(NULL, NULL, rsc_id)) { g_hash_table_remove(lrm_state_table, rsc_id); return pcmk_ok; } g_hash_table_remove(lrm_state->rsc_info_cache, rsc_id); /* @TODO Optimize this ... this function is a blocking round trip from * client to daemon. The controld_execd_state.c code path that uses this * function should always treat it as an async operation. The executor API * should make an async version available. */ return ((lrmd_t *) lrm_state->conn)->cmds->unregister_rsc(lrm_state->conn, rsc_id, options); } diff --git a/daemons/controld/controld_fsa.c b/daemons/controld/controld_fsa.c index e95b96961e..fd0282cd43 100644 --- a/daemons/controld/controld_fsa.c +++ b/daemons/controld/controld_fsa.c @@ -1,686 +1,686 @@ /* * Copyright 2004-2025 the Pacemaker project contributors * * The version control history for this file may have further details. * * This source code is licensed under the GNU General Public License version 2 * or later (GPLv2+) WITHOUT ANY WARRANTY. */ #include #include // PRIx64 #include #include #include // uint64_t #include #include #include #include #include #include #include #include #include //! Triggers an FSA invocation static crm_trigger_t *fsa_trigger = NULL; static void do_state_transition(enum crmd_fsa_state cur_state, enum crmd_fsa_state next_state, fsa_data_t *msg_data); void s_crmd_fsa_actions(fsa_data_t * fsa_data); void log_fsa_input(fsa_data_t * stored_msg); static void do_fsa_action(fsa_data_t * fsa_data, long long an_action, void (*function) (long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input cur_input, fsa_data_t * msg_data)) { controld_clear_fsa_action_flags(an_action); function(an_action, fsa_data->fsa_cause, controld_globals.fsa_state, fsa_data->fsa_input, fsa_data); } static const uint64_t startup_actions = A_STARTUP | A_CIB_START | A_LRM_CONNECT | A_HA_CONNECT | A_READCONFIG | A_STARTED | A_CL_JOIN_QUERY; // A_LOG, A_WARN, A_ERROR void do_log(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t *msg_data) { unsigned log_type = PCMK__LOG_TRACE; if (action & A_LOG) { log_type = LOG_INFO; } else if (action & A_WARN) { log_type = LOG_WARNING; } else if (action & A_ERROR) { log_type = LOG_ERR; } do_crm_log(log_type, "Input %s received in state %s from %s", fsa_input2string(msg_data->fsa_input), fsa_state2string(cur_state), msg_data->origin); if (msg_data->data_type == fsa_dt_ha_msg) { ha_msg_input_t *input = fsa_typed_data(msg_data->data_type); pcmk__log_xml_debug(input->msg, __func__); } else if (msg_data->data_type == fsa_dt_xml) { xmlNode *input = fsa_typed_data(msg_data->data_type); pcmk__log_xml_debug(input, __func__); } else if (msg_data->data_type == fsa_dt_lrm) { lrmd_event_data_t *input = fsa_typed_data(msg_data->data_type); do_crm_log(log_type, "Resource %s: Call ID %d returned %d (%d)." " New status if rc=0: %s", input->rsc_id, input->call_id, input->rc, input->op_status, (char *)input->user_data); } } /*! * \internal * \brief Initialize the FSA trigger */ void controld_init_fsa_trigger(void) { fsa_trigger = mainloop_add_trigger(G_PRIORITY_HIGH, crm_fsa_trigger, NULL); } /*! * \internal * \brief Destroy the FSA trigger */ void controld_destroy_fsa_trigger(void) { // This basically will not work, since mainloop has a reference to it mainloop_destroy_trigger(fsa_trigger); fsa_trigger = NULL; } /*! * \internal * \brief Trigger an FSA invocation * * \param[in] fn Calling function name * \param[in] line Line number where call occurred */ void controld_trigger_fsa_as(const char *fn, int line) { if (fsa_trigger != NULL) { pcmk__trace("%s:%d - Triggered FSA invocation", fn, line); mainloop_set_trigger(fsa_trigger); } } enum crmd_fsa_state s_crmd_fsa(enum crmd_fsa_cause cause) { controld_globals_t *globals = &controld_globals; fsa_data_t *fsa_data = NULL; uint64_t register_copy = controld_globals.fsa_input_register; uint64_t new_actions = A_NOTHING; enum crmd_fsa_state last_state; pcmk__trace("FSA invoked with Cause: %s\tState: %s", fsa_cause2string(cause), fsa_state2string(globals->fsa_state)); fsa_dump_actions(controld_globals.fsa_actions, "Initial"); controld_clear_global_flags(controld_fsa_is_stalled); if ((controld_globals.fsa_message_queue == NULL) && (controld_globals.fsa_actions != A_NOTHING)) { /* fake the first message so we can get into the loop */ fsa_data = pcmk__assert_alloc(1, sizeof(fsa_data_t)); fsa_data->fsa_input = I_NULL; fsa_data->fsa_cause = C_FSA_INTERNAL; fsa_data->origin = __func__; fsa_data->data_type = fsa_dt_none; controld_globals.fsa_message_queue = g_list_append(controld_globals.fsa_message_queue, fsa_data); } while ((controld_globals.fsa_message_queue != NULL) && !pcmk__is_set(controld_globals.flags, controld_fsa_is_stalled)) { pcmk__trace("Checking messages (%u remaining)", g_list_length(controld_globals.fsa_message_queue)); fsa_data = get_message(); if(fsa_data == NULL) { continue; } log_fsa_input(fsa_data); /* add any actions back to the queue */ controld_set_fsa_action_flags(fsa_data->actions); fsa_dump_actions(fsa_data->actions, "Restored actions"); /* get the next batch of actions */ new_actions = controld_fsa_get_action(fsa_data->fsa_input); controld_set_fsa_action_flags(new_actions); fsa_dump_actions(new_actions, "New actions"); if (fsa_data->fsa_input != I_NULL && fsa_data->fsa_input != I_ROUTER) { pcmk__debug("Processing %s: [ state=%s cause=%s origin=%s ]", fsa_input2string(fsa_data->fsa_input), fsa_state2string(globals->fsa_state), fsa_cause2string(fsa_data->fsa_cause), fsa_data->origin); } /* logging : *before* the state is changed */ if (pcmk__is_set(controld_globals.fsa_actions, A_ERROR)) { do_fsa_action(fsa_data, A_ERROR, do_log); } if (pcmk__is_set(controld_globals.fsa_actions, A_WARN)) { do_fsa_action(fsa_data, A_WARN, do_log); } if (pcmk__is_set(controld_globals.fsa_actions, A_LOG)) { do_fsa_action(fsa_data, A_LOG, do_log); } /* update state variables */ last_state = globals->fsa_state; globals->fsa_state = controld_fsa_get_next_state(fsa_data->fsa_input); /* * Remove certain actions during shutdown */ if ((globals->fsa_state == S_STOPPING) || pcmk__is_set(controld_globals.fsa_input_register, R_SHUTDOWN)) { controld_clear_fsa_action_flags(startup_actions); } /* * Hook for change of state. * Allows actions to be added or removed when entering a state */ if (last_state != globals->fsa_state) { do_state_transition(last_state, globals->fsa_state, fsa_data); } /* start doing things... */ s_crmd_fsa_actions(fsa_data); delete_fsa_input(fsa_data); } if ((controld_globals.fsa_message_queue != NULL) || (controld_globals.fsa_actions != A_NOTHING) || pcmk__is_set(controld_globals.flags, controld_fsa_is_stalled)) { pcmk__debug("Exiting the FSA: queue=%d, fsa_actions=%" PRIx64 ", stalled=%s", g_list_length(controld_globals.fsa_message_queue), controld_globals.fsa_actions, pcmk__flag_text(controld_globals.flags, controld_fsa_is_stalled)); } else { pcmk__trace("Exiting the FSA"); } /* cleanup inputs? */ if (register_copy != controld_globals.fsa_input_register) { uint64_t same = register_copy & controld_globals.fsa_input_register; fsa_dump_inputs(LOG_DEBUG, "Added", controld_globals.fsa_input_register ^ same); fsa_dump_inputs(LOG_DEBUG, "Removed", register_copy ^ same); } fsa_dump_actions(controld_globals.fsa_actions, "Remaining"); fsa_dump_queue(LOG_DEBUG); return globals->fsa_state; } void s_crmd_fsa_actions(fsa_data_t * fsa_data) { /* * Process actions in order of priority but do only one * action at a time to avoid complicating the ordering. */ CRM_CHECK(fsa_data != NULL, return); while ((controld_globals.fsa_actions != A_NOTHING) && !pcmk__is_set(controld_globals.flags, controld_fsa_is_stalled)) { /* regular action processing in order of action priority * * Make sure all actions that connect to required systems * are performed first */ if (pcmk__is_set(controld_globals.fsa_actions, A_ERROR)) { do_fsa_action(fsa_data, A_ERROR, do_log); } else if (pcmk__is_set(controld_globals.fsa_actions, A_WARN)) { do_fsa_action(fsa_data, A_WARN, do_log); } else if (pcmk__is_set(controld_globals.fsa_actions, A_LOG)) { do_fsa_action(fsa_data, A_LOG, do_log); /* get out of here NOW! before anything worse happens */ } else if (pcmk__is_set(controld_globals.fsa_actions, A_EXIT_1)) { do_fsa_action(fsa_data, A_EXIT_1, do_exit); /* sub-system restart */ } else if (pcmk__all_flags_set(controld_globals.fsa_actions, O_LRM_RECONNECT)) { do_fsa_action(fsa_data, O_LRM_RECONNECT, do_lrm_control); } else if (pcmk__all_flags_set(controld_globals.fsa_actions, O_CIB_RESTART)) { do_fsa_action(fsa_data, O_CIB_RESTART, do_cib_control); } else if (pcmk__all_flags_set(controld_globals.fsa_actions, O_PE_RESTART)) { do_fsa_action(fsa_data, O_PE_RESTART, do_pe_control); } else if (pcmk__all_flags_set(controld_globals.fsa_actions, O_TE_RESTART)) { do_fsa_action(fsa_data, O_TE_RESTART, do_te_control); /* essential start tasks */ } else if (pcmk__is_set(controld_globals.fsa_actions, A_STARTUP)) { do_fsa_action(fsa_data, A_STARTUP, do_startup); } else if (pcmk__is_set(controld_globals.fsa_actions, A_CIB_START)) { do_fsa_action(fsa_data, A_CIB_START, do_cib_control); } else if (pcmk__is_set(controld_globals.fsa_actions, A_HA_CONNECT)) { do_fsa_action(fsa_data, A_HA_CONNECT, do_ha_control); } else if (pcmk__is_set(controld_globals.fsa_actions, A_READCONFIG)) { do_fsa_action(fsa_data, A_READCONFIG, do_read_config); /* sub-system start/connect */ } else if (pcmk__is_set(controld_globals.fsa_actions, A_LRM_CONNECT)) { do_fsa_action(fsa_data, A_LRM_CONNECT, do_lrm_control); } else if (pcmk__is_set(controld_globals.fsa_actions, A_TE_START)) { do_fsa_action(fsa_data, A_TE_START, do_te_control); } else if (pcmk__is_set(controld_globals.fsa_actions, A_PE_START)) { do_fsa_action(fsa_data, A_PE_START, do_pe_control); /* Timers */ } else if (pcmk__is_set(controld_globals.fsa_actions, A_DC_TIMER_STOP)) { do_fsa_action(fsa_data, A_DC_TIMER_STOP, do_timer_control); } else if (pcmk__is_set(controld_globals.fsa_actions, A_INTEGRATE_TIMER_STOP)) { do_fsa_action(fsa_data, A_INTEGRATE_TIMER_STOP, do_timer_control); } else if (pcmk__is_set(controld_globals.fsa_actions, A_INTEGRATE_TIMER_START)) { do_fsa_action(fsa_data, A_INTEGRATE_TIMER_START, do_timer_control); } else if (pcmk__is_set(controld_globals.fsa_actions, A_FINALIZE_TIMER_STOP)) { do_fsa_action(fsa_data, A_FINALIZE_TIMER_STOP, do_timer_control); } else if (pcmk__is_set(controld_globals.fsa_actions, A_FINALIZE_TIMER_START)) { do_fsa_action(fsa_data, A_FINALIZE_TIMER_START, do_timer_control); /* * Highest priority actions */ } else if (pcmk__is_set(controld_globals.fsa_actions, A_MSG_ROUTE)) { do_fsa_action(fsa_data, A_MSG_ROUTE, do_msg_route); } else if (pcmk__is_set(controld_globals.fsa_actions, A_RECOVER)) { do_fsa_action(fsa_data, A_RECOVER, do_recover); } else if (pcmk__is_set(controld_globals.fsa_actions, A_CL_JOIN_RESULT)) { do_fsa_action(fsa_data, A_CL_JOIN_RESULT, do_cl_join_finalize_respond); } else if (pcmk__is_set(controld_globals.fsa_actions, A_CL_JOIN_REQUEST)) { do_fsa_action(fsa_data, A_CL_JOIN_REQUEST, do_cl_join_offer_respond); } else if (pcmk__is_set(controld_globals.fsa_actions, A_SHUTDOWN_REQ)) { do_fsa_action(fsa_data, A_SHUTDOWN_REQ, do_shutdown_req); } else if (pcmk__is_set(controld_globals.fsa_actions, A_ELECTION_VOTE)) { do_fsa_action(fsa_data, A_ELECTION_VOTE, do_election_vote); } else if (pcmk__is_set(controld_globals.fsa_actions, A_ELECTION_COUNT)) { do_fsa_action(fsa_data, A_ELECTION_COUNT, do_election_count_vote); /* * High priority actions */ } else if (pcmk__is_set(controld_globals.fsa_actions, A_STARTED)) { do_fsa_action(fsa_data, A_STARTED, do_started); } else if (pcmk__is_set(controld_globals.fsa_actions, A_CL_JOIN_QUERY)) { do_fsa_action(fsa_data, A_CL_JOIN_QUERY, do_cl_join_query); } else if (pcmk__is_set(controld_globals.fsa_actions, A_DC_TIMER_START)) { do_fsa_action(fsa_data, A_DC_TIMER_START, do_timer_control); /* * Medium priority actions * - Membership */ } else if (pcmk__is_set(controld_globals.fsa_actions, A_DC_TAKEOVER)) { do_fsa_action(fsa_data, A_DC_TAKEOVER, do_dc_takeover); } else if (pcmk__is_set(controld_globals.fsa_actions, A_DC_RELEASE)) { do_fsa_action(fsa_data, A_DC_RELEASE, do_dc_release); } else if (pcmk__is_set(controld_globals.fsa_actions, A_DC_JOIN_FINAL)) { do_fsa_action(fsa_data, A_DC_JOIN_FINAL, do_dc_join_final); } else if (pcmk__is_set(controld_globals.fsa_actions, A_ELECTION_CHECK)) { do_fsa_action(fsa_data, A_ELECTION_CHECK, do_election_check); } else if (pcmk__is_set(controld_globals.fsa_actions, A_ELECTION_START)) { do_fsa_action(fsa_data, A_ELECTION_START, do_election_vote); } else if (pcmk__is_set(controld_globals.fsa_actions, A_DC_JOIN_OFFER_ALL)) { do_fsa_action(fsa_data, A_DC_JOIN_OFFER_ALL, do_dc_join_offer_all); } else if (pcmk__is_set(controld_globals.fsa_actions, A_DC_JOIN_OFFER_ONE)) { do_fsa_action(fsa_data, A_DC_JOIN_OFFER_ONE, do_dc_join_offer_one); } else if (pcmk__is_set(controld_globals.fsa_actions, A_DC_JOIN_PROCESS_REQ)) { do_fsa_action(fsa_data, A_DC_JOIN_PROCESS_REQ, do_dc_join_filter_offer); } else if (pcmk__is_set(controld_globals.fsa_actions, A_DC_JOIN_PROCESS_ACK)) { do_fsa_action(fsa_data, A_DC_JOIN_PROCESS_ACK, do_dc_join_ack); } else if (pcmk__is_set(controld_globals.fsa_actions, A_DC_JOIN_FINALIZE)) { do_fsa_action(fsa_data, A_DC_JOIN_FINALIZE, do_dc_join_finalize); } else if (pcmk__is_set(controld_globals.fsa_actions, A_CL_JOIN_ANNOUNCE)) { do_fsa_action(fsa_data, A_CL_JOIN_ANNOUNCE, do_cl_join_announce); /* * Low(er) priority actions * Make sure the CIB is always updated before invoking the * scheduler, and the scheduler before the transition engine. */ } else if (pcmk__is_set(controld_globals.fsa_actions, A_TE_HALT)) { do_fsa_action(fsa_data, A_TE_HALT, do_te_invoke); } else if (pcmk__is_set(controld_globals.fsa_actions, A_TE_CANCEL)) { do_fsa_action(fsa_data, A_TE_CANCEL, do_te_invoke); } else if (pcmk__is_set(controld_globals.fsa_actions, A_LRM_INVOKE)) { do_fsa_action(fsa_data, A_LRM_INVOKE, do_lrm_invoke); } else if (pcmk__is_set(controld_globals.fsa_actions, A_PE_INVOKE)) { do_fsa_action(fsa_data, A_PE_INVOKE, do_pe_invoke); } else if (pcmk__is_set(controld_globals.fsa_actions, A_TE_INVOKE)) { do_fsa_action(fsa_data, A_TE_INVOKE, do_te_invoke); /* Shutdown actions */ } else if (pcmk__is_set(controld_globals.fsa_actions, A_DC_RELEASED)) { do_fsa_action(fsa_data, A_DC_RELEASED, do_dc_release); } else if (pcmk__is_set(controld_globals.fsa_actions, A_PE_STOP)) { do_fsa_action(fsa_data, A_PE_STOP, do_pe_control); } else if (pcmk__is_set(controld_globals.fsa_actions, A_TE_STOP)) { do_fsa_action(fsa_data, A_TE_STOP, do_te_control); } else if (pcmk__is_set(controld_globals.fsa_actions, A_SHUTDOWN)) { do_fsa_action(fsa_data, A_SHUTDOWN, do_shutdown); } else if (pcmk__is_set(controld_globals.fsa_actions, A_LRM_DISCONNECT)) { do_fsa_action(fsa_data, A_LRM_DISCONNECT, do_lrm_control); } else if (pcmk__is_set(controld_globals.fsa_actions, A_HA_DISCONNECT)) { do_fsa_action(fsa_data, A_HA_DISCONNECT, do_ha_control); } else if (pcmk__is_set(controld_globals.fsa_actions, A_CIB_STOP)) { do_fsa_action(fsa_data, A_CIB_STOP, do_cib_control); } else if (pcmk__is_set(controld_globals.fsa_actions, A_STOP)) { do_fsa_action(fsa_data, A_STOP, do_stop); /* exit gracefully */ } else if (pcmk__is_set(controld_globals.fsa_actions, A_EXIT_0)) { do_fsa_action(fsa_data, A_EXIT_0, do_exit); /* Error checking and reporting */ } else { pcmk__err("Action %s not supported " QB_XS " %" PRIx64, fsa_action2string(controld_globals.fsa_actions), controld_globals.fsa_actions); register_fsa_error_adv(C_FSA_INTERNAL, I_ERROR, fsa_data, NULL, __func__); } } } void log_fsa_input(fsa_data_t * stored_msg) { pcmk__assert(stored_msg != NULL); pcmk__trace("Processing queued input %d", stored_msg->id); if (stored_msg->fsa_cause == C_LRM_OP_CALLBACK) { pcmk__trace("FSA processing LRM callback from %s", stored_msg->origin); } else if (stored_msg->data == NULL) { pcmk__trace("FSA processing input from %s", stored_msg->origin); } else { ha_msg_input_t *ha_input = fsa_typed_data_adv(stored_msg, fsa_dt_ha_msg, __func__); pcmk__trace("FSA processing XML message from %s", stored_msg->origin); - crm_log_xml_trace(ha_input->xml, "FSA message data"); + pcmk__log_xml_trace(ha_input->xml, "FSA message data"); } } static void check_join_counts(fsa_data_t *msg_data) { int count; guint npeers; count = crmd_join_phase_count(controld_join_finalized); if (count > 0) { pcmk__err("%d cluster node%s failed to confirm join", count, pcmk__plural_s(count)); crmd_join_phase_log(LOG_NOTICE); return; } npeers = pcmk__cluster_num_active_nodes(); count = crmd_join_phase_count(controld_join_confirmed); if (count == npeers) { if (npeers == 1) { pcmk__debug("Sole active cluster node is fully joined"); } else { pcmk__debug("All %d active cluster nodes are fully joined", count); } } else if (count > npeers) { pcmk__err("New election needed because more nodes confirmed join " "than are in membership (%d > %u)", count, npeers); register_fsa_input(C_FSA_INTERNAL, I_ELECTION, NULL); } else if (controld_globals.membership_id != controld_globals.peer_seq) { pcmk__info("New join needed because membership changed (%llu -> %llu)", controld_globals.membership_id, controld_globals.peer_seq); register_fsa_input_before(C_FSA_INTERNAL, I_NODE_JOIN, NULL); } else { pcmk__warn("Only %d of %u active cluster nodes fully joined (%d did " "not respond to offer)", count, npeers, crmd_join_phase_count(controld_join_welcomed)); } } static void do_state_transition(enum crmd_fsa_state cur_state, enum crmd_fsa_state next_state, fsa_data_t *msg_data) { int level = LOG_INFO; int count = 0; gboolean clear_recovery_bit = TRUE; #if 0 uint64_t original_fsa_actions = controld_globals.fsa_actions; #endif enum crmd_fsa_cause cause = msg_data->fsa_cause; enum crmd_fsa_input current_input = msg_data->fsa_input; const char *state_from = fsa_state2string(cur_state); const char *state_to = fsa_state2string(next_state); const char *input = fsa_input2string(current_input); CRM_LOG_ASSERT(cur_state != next_state); if (cur_state == S_IDLE || next_state == S_IDLE) { level = LOG_NOTICE; } else if (cur_state == S_NOT_DC || next_state == S_NOT_DC) { level = LOG_NOTICE; } else if (cur_state == S_ELECTION) { level = LOG_NOTICE; } else if (cur_state == S_STARTING) { level = LOG_NOTICE; } else if (next_state == S_RECOVERY) { level = LOG_WARNING; } do_crm_log(level, "State transition %s -> %s " QB_XS " input=%s cause=%s origin=%s", state_from, state_to, input, fsa_cause2string(cause), msg_data->origin); if (next_state != S_ELECTION && cur_state != S_RELEASE_DC) { controld_stop_current_election_timeout(); } if (next_state == S_INTEGRATION) { controld_set_fsa_action_flags(A_INTEGRATE_TIMER_START); } else { controld_set_fsa_action_flags(A_INTEGRATE_TIMER_STOP); } if (next_state == S_FINALIZE_JOIN) { controld_set_fsa_action_flags(A_FINALIZE_TIMER_START); } else { controld_set_fsa_action_flags(A_FINALIZE_TIMER_STOP); } if (next_state != S_PENDING) { controld_set_fsa_action_flags(A_DC_TIMER_STOP); } if (next_state != S_IDLE) { controld_stop_recheck_timer(); } if (cur_state == S_FINALIZE_JOIN && next_state == S_POLICY_ENGINE) { populate_cib_nodes(node_update_quick|node_update_all, __func__); } switch (next_state) { case S_PENDING: { cib_t *cib_conn = controld_globals.cib_conn; cib_conn->cmds->set_secondary(cib_conn, cib_none); } update_dc(NULL); break; case S_ELECTION: update_dc(NULL); break; case S_NOT_DC: controld_reset_counter_election_timer(); purge_stonith_cleanup(); if (pcmk__is_set(controld_globals.fsa_input_register, R_SHUTDOWN)) { pcmk__info("(Re)Issuing shutdown request now" " that we have a " "new DC"); controld_set_fsa_action_flags(A_SHUTDOWN_REQ); } CRM_LOG_ASSERT(controld_globals.dc_name != NULL); if (controld_globals.dc_name == NULL) { pcmk__err("Reached S_NOT_DC without a DC" " being recorded"); } break; case S_RECOVERY: clear_recovery_bit = FALSE; break; case S_FINALIZE_JOIN: CRM_LOG_ASSERT(AM_I_DC); if (cause == C_TIMER_POPPED) { pcmk__warn("Progressed to state %s after %s", fsa_state2string(next_state), fsa_cause2string(cause)); } count = crmd_join_phase_count(controld_join_welcomed); if (count > 0) { pcmk__warn("%d cluster node%s failed to respond to join offer", count, pcmk__plural_s(count)); crmd_join_phase_log(LOG_NOTICE); } else { pcmk__debug("All cluster nodes (%d) responded to join offer", crmd_join_phase_count(controld_join_integrated)); } break; case S_POLICY_ENGINE: controld_reset_counter_election_timer(); CRM_LOG_ASSERT(AM_I_DC); if (cause == C_TIMER_POPPED) { pcmk__info("Progressed to state %s after %s", fsa_state2string(next_state), fsa_cause2string(cause)); } check_join_counts(msg_data); break; case S_STOPPING: case S_TERMINATE: /* possibly redundant */ controld_set_fsa_input_flags(R_SHUTDOWN); break; case S_IDLE: CRM_LOG_ASSERT(AM_I_DC); if (pcmk__is_set(controld_globals.fsa_input_register, R_SHUTDOWN)) { pcmk__info("(Re)Issuing shutdown request now" " that we are " "the DC"); controld_set_fsa_action_flags(A_SHUTDOWN_REQ); } controld_start_recheck_timer(); break; default: break; } if (clear_recovery_bit && next_state != S_PENDING) { controld_clear_fsa_action_flags(A_RECOVER); } else if (clear_recovery_bit == FALSE) { controld_set_fsa_action_flags(A_RECOVER); } #if 0 if (original_fsa_actions != controld_globals.fsa_actions) { fsa_dump_actions(original_fsa_actions ^ controld_globals.fsa_actions, "New actions"); } #endif } diff --git a/daemons/controld/controld_messages.c b/daemons/controld/controld_messages.c index 7a076a4c8e..613df46ca3 100644 --- a/daemons/controld/controld_messages.c +++ b/daemons/controld/controld_messages.c @@ -1,1396 +1,1396 @@ /* * Copyright 2004-2025 the Pacemaker project contributors * * The version control history for this file may have further details. * * This source code is licensed under the GNU General Public License version 2 * or later (GPLv2+) WITHOUT ANY WARRANTY. */ #include #include // PRIx64 #include // uint64_t #include #include #include #include #include #include #include #include #include static enum crmd_fsa_input handle_message(xmlNode *msg, enum crmd_fsa_cause cause); static xmlNode* create_ping_reply(const xmlNode *msg); static void handle_response(xmlNode *stored_msg); static enum crmd_fsa_input handle_request(xmlNode *stored_msg, enum crmd_fsa_cause cause); static enum crmd_fsa_input handle_shutdown_request(xmlNode *stored_msg); static void send_msg_via_ipc(xmlNode * msg, const char *sys, const char *src); /* debug only, can wrap all it likes */ static int last_data_id = 0; void register_fsa_error_adv(enum crmd_fsa_cause cause, enum crmd_fsa_input input, fsa_data_t * cur_data, void *new_data, const char *raised_from) { /* save the current actions if any */ if (controld_globals.fsa_actions != A_NOTHING) { register_fsa_input_adv(cur_data ? cur_data->fsa_cause : C_FSA_INTERNAL, I_NULL, cur_data ? cur_data->data : NULL, controld_globals.fsa_actions, TRUE, __func__); } /* reset the action list */ pcmk__info("Resetting the current action list"); fsa_dump_actions(controld_globals.fsa_actions, "Drop"); controld_globals.fsa_actions = A_NOTHING; /* register the error */ register_fsa_input_adv(cause, input, new_data, A_NOTHING, TRUE, raised_from); } void register_fsa_input_adv(enum crmd_fsa_cause cause, enum crmd_fsa_input input, void *data, uint64_t with_actions, gboolean prepend, const char *raised_from) { unsigned old_len = g_list_length(controld_globals.fsa_message_queue); fsa_data_t *fsa_data = NULL; if (raised_from == NULL) { raised_from = ""; } if (input == I_NULL && with_actions == A_NOTHING /* && data == NULL */ ) { /* no point doing anything */ pcmk__err("Cannot add entry to queue: no input and no action"); return; } if (input == I_WAIT_FOR_EVENT) { controld_set_global_flags(controld_fsa_is_stalled); pcmk__debug("Stalling the FSA pending further input: source=%s " "cause=%s data=%p queue=%d", raised_from, fsa_cause2string(cause), data, old_len); if (old_len > 0) { fsa_dump_queue(PCMK__LOG_TRACE); prepend = FALSE; } if (data == NULL) { controld_set_fsa_action_flags(with_actions); fsa_dump_actions(with_actions, "Restored"); return; } /* Store everything in the new event and reset * controld_globals.fsa_actions */ with_actions |= controld_globals.fsa_actions; controld_globals.fsa_actions = A_NOTHING; } last_data_id++; pcmk__trace("%s %s FSA input %d (%s) due to %s, %s data", raised_from, (prepend? "prepended" : "appended"), last_data_id, fsa_input2string(input), fsa_cause2string(cause), ((data != NULL)? "with" : "without")); fsa_data = pcmk__assert_alloc(1, sizeof(fsa_data_t)); fsa_data->id = last_data_id; fsa_data->fsa_input = input; fsa_data->fsa_cause = cause; fsa_data->origin = raised_from; fsa_data->data = NULL; fsa_data->data_type = fsa_dt_none; fsa_data->actions = with_actions; if (with_actions != A_NOTHING) { pcmk__trace("Adding actions %.16" PRIx64 " to input", with_actions); } if (data != NULL) { switch (cause) { case C_FSA_INTERNAL: case C_CRMD_STATUS_CALLBACK: case C_IPC_MESSAGE: case C_HA_MESSAGE: CRM_CHECK(((ha_msg_input_t *) data)->msg != NULL, pcmk__err("Bogus data from %s", raised_from)); pcmk__trace("Copying %s data from %s as cluster message data", fsa_cause2string(cause), raised_from); fsa_data->data = copy_ha_msg_input(data); fsa_data->data_type = fsa_dt_ha_msg; break; case C_LRM_OP_CALLBACK: pcmk__trace("Copying %s data from %s as lrmd_event_data_t", fsa_cause2string(cause), raised_from); fsa_data->data = lrmd_copy_event((lrmd_event_data_t *) data); fsa_data->data_type = fsa_dt_lrm; break; case C_TIMER_POPPED: case C_SHUTDOWN: case C_UNKNOWN: case C_STARTUP: pcmk__crit("Copying %s data (from %s) is not yet implemented", fsa_cause2string(cause), raised_from); crmd_exit(CRM_EX_SOFTWARE); break; } } /* make sure to free it properly later */ if (prepend) { controld_globals.fsa_message_queue = g_list_prepend(controld_globals.fsa_message_queue, fsa_data); } else { controld_globals.fsa_message_queue = g_list_append(controld_globals.fsa_message_queue, fsa_data); } pcmk__trace("FSA message queue length is %u", g_list_length(controld_globals.fsa_message_queue)); /* fsa_dump_queue(PCMK__LOG_TRACE); */ if (old_len == g_list_length(controld_globals.fsa_message_queue)) { pcmk__err("Couldn't add message to the queue"); } if (input != I_WAIT_FOR_EVENT) { controld_trigger_fsa(); } } void fsa_dump_queue(int log_level) { int offset = 0; for (GList *iter = controld_globals.fsa_message_queue; iter != NULL; iter = iter->next) { fsa_data_t *data = (fsa_data_t *) iter->data; do_crm_log_unlikely(log_level, "queue[%d.%d]: input %s raised by %s(%p.%d)\t(cause=%s)", offset++, data->id, fsa_input2string(data->fsa_input), data->origin, data->data, data->data_type, fsa_cause2string(data->fsa_cause)); } } ha_msg_input_t * copy_ha_msg_input(ha_msg_input_t * orig) { xmlNode *wrapper = NULL; ha_msg_input_t *copy = pcmk__assert_alloc(1, sizeof(ha_msg_input_t)); copy->msg = (orig != NULL)? pcmk__xml_copy(NULL, orig->msg) : NULL; wrapper = pcmk__xe_first_child(copy->msg, PCMK__XE_CRM_XML, NULL, NULL); copy->xml = pcmk__xe_first_child(wrapper, NULL, NULL, NULL); return copy; } void delete_fsa_input(fsa_data_t * fsa_data) { lrmd_event_data_t *op = NULL; xmlNode *foo = NULL; if (fsa_data == NULL) { return; } pcmk__trace("About to free %s data", fsa_cause2string(fsa_data->fsa_cause)); if (fsa_data->data != NULL) { switch (fsa_data->data_type) { case fsa_dt_ha_msg: delete_ha_msg_input(fsa_data->data); break; case fsa_dt_xml: foo = fsa_data->data; pcmk__xml_free(foo); break; case fsa_dt_lrm: op = (lrmd_event_data_t *) fsa_data->data; lrmd_free_event(op); break; case fsa_dt_none: if (fsa_data->data != NULL) { pcmk__err("Don't know how to free %s data from %s", fsa_cause2string(fsa_data->fsa_cause), fsa_data->origin); crmd_exit(CRM_EX_SOFTWARE); } break; } pcmk__trace("%s data freed", fsa_cause2string(fsa_data->fsa_cause)); } free(fsa_data); } /* returns the next message */ fsa_data_t * get_message(void) { fsa_data_t *message = (fsa_data_t *) controld_globals.fsa_message_queue->data; controld_globals.fsa_message_queue = g_list_remove(controld_globals.fsa_message_queue, message); pcmk__trace("Processing input %d", message->id); return message; } void * fsa_typed_data_adv(fsa_data_t * fsa_data, enum fsa_data_type a_type, const char *caller) { void *ret_val = NULL; if (fsa_data == NULL) { pcmk__err("%s: No FSA data available", caller); } else if (fsa_data->data == NULL) { pcmk__err("%s: No message data available. Origin: %s", caller, fsa_data->origin); } else if (fsa_data->data_type != a_type) { pcmk__crit("%s: Message data was the wrong type! %d vs. requested=%d. " "Origin: %s", caller, fsa_data->data_type, a_type, fsa_data->origin); pcmk__assert(fsa_data->data_type == a_type); } else { ret_val = fsa_data->data; } return ret_val; } /* A_MSG_ROUTE */ void do_msg_route(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data) { ha_msg_input_t *input = fsa_typed_data(fsa_dt_ha_msg); route_message(msg_data->fsa_cause, input->msg); } void route_message(enum crmd_fsa_cause cause, xmlNode * input) { ha_msg_input_t fsa_input; enum crmd_fsa_input result = I_NULL; fsa_input.msg = input; CRM_CHECK(cause == C_IPC_MESSAGE || cause == C_HA_MESSAGE, return); /* try passing the buck first */ if (relay_message(input, cause == C_IPC_MESSAGE)) { return; } /* handle locally */ result = handle_message(input, cause); /* done or process later? */ switch (result) { case I_NULL: case I_ROUTER: case I_NODE_JOIN: case I_JOIN_REQUEST: case I_JOIN_RESULT: break; default: /* Defering local processing of message */ register_fsa_input_later(cause, result, &fsa_input); return; } if (result != I_NULL) { /* add to the front of the queue */ register_fsa_input(cause, result, &fsa_input); } } gboolean relay_message(xmlNode * msg, gboolean originated_locally) { enum pcmk_ipc_server dest = pcmk_ipc_unknown; bool is_for_dc = false; bool is_for_dcib = false; bool is_for_te = false; bool is_for_crm = false; bool is_for_cib = false; bool is_local = false; bool broadcast = false; const char *host_to = NULL; const char *sys_to = NULL; const char *sys_from = NULL; const char *type = NULL; const char *task = NULL; const char *ref = NULL; pcmk__node_status_t *node_to = NULL; CRM_CHECK(msg != NULL, return TRUE); host_to = pcmk__xe_get(msg, PCMK__XA_CRM_HOST_TO); sys_to = pcmk__xe_get(msg, PCMK__XA_CRM_SYS_TO); sys_from = pcmk__xe_get(msg, PCMK__XA_CRM_SYS_FROM); type = pcmk__xe_get(msg, PCMK__XA_T); task = pcmk__xe_get(msg, PCMK__XA_CRM_TASK); ref = pcmk__xe_get(msg, PCMK_XA_REFERENCE); broadcast = pcmk__str_empty(host_to); if (ref == NULL) { ref = "without reference ID"; } if (pcmk__str_eq(task, CRM_OP_HELLO, pcmk__str_casei)) { pcmk__trace("Received hello %s from %s (no processing needed)", ref, pcmk__s(sys_from, "unidentified source")); - crm_log_xml_trace(msg, "hello"); + pcmk__log_xml_trace(msg, "hello"); return TRUE; } // Require message type (set by pcmk__new_request()) if (!pcmk__str_eq(type, PCMK__VALUE_CRMD, pcmk__str_none)) { pcmk__warn("Ignoring invalid message %s with type '%s' " "(not '" PCMK__VALUE_CRMD "')", ref, pcmk__s(type, "")); - crm_log_xml_trace(msg, "ignored"); + pcmk__log_xml_trace(msg, "ignored"); return TRUE; } // Require a destination subsystem (also set by pcmk__new_request()) if (sys_to == NULL) { pcmk__warn("Ignoring invalid message %s with no " PCMK__XA_CRM_SYS_TO, ref); - crm_log_xml_trace(msg, "ignored"); + pcmk__log_xml_trace(msg, "ignored"); return TRUE; } // Get the message type appropriate to the destination subsystem if (pcmk_get_cluster_layer() == pcmk_cluster_layer_corosync) { dest = pcmk__parse_server(sys_to); if (dest == pcmk_ipc_unknown) { /* Unrecognized value, use a sane default * * @TODO Maybe we should bail instead */ dest = pcmk_ipc_controld; } } is_for_dc = (strcasecmp(CRM_SYSTEM_DC, sys_to) == 0); is_for_dcib = (strcasecmp(CRM_SYSTEM_DCIB, sys_to) == 0); is_for_te = (strcasecmp(CRM_SYSTEM_TENGINE, sys_to) == 0); is_for_cib = (strcasecmp(CRM_SYSTEM_CIB, sys_to) == 0); is_for_crm = (strcasecmp(CRM_SYSTEM_CRMD, sys_to) == 0); // Check whether message should be processed locally is_local = false; if (broadcast) { if (is_for_dc || is_for_te) { is_local = false; } else if (is_for_crm) { if (pcmk__strcase_any_of(task, CRM_OP_NODE_INFO, PCMK__CONTROLD_CMD_NODES, NULL)) { /* Node info requests do not specify a host, which is normally * treated as "all hosts", because the whole point is that the * client may not know the local node name. Always handle these * requests locally. */ is_local = true; } else { is_local = !originated_locally; } } else { is_local = true; } } else if (controld_is_local_node(host_to)) { is_local = true; } else if (is_for_crm && pcmk__str_eq(task, CRM_OP_LRM_DELETE, pcmk__str_casei)) { xmlNode *wrapper = pcmk__xe_first_child(msg, PCMK__XE_CRM_XML, NULL, NULL); xmlNode *msg_data = pcmk__xe_first_child(wrapper, NULL, NULL, NULL); const char *mode = pcmk__xe_get(msg_data, PCMK__XA_MODE); if (pcmk__str_eq(mode, PCMK__VALUE_CIB, pcmk__str_none)) { // Local delete of an offline node's resource history is_local = true; } } // If is for DC and DC is not yet selected if (is_for_dc && pcmk__str_eq(task, CRM_OP_PING, pcmk__str_casei) && (controld_globals.dc_name == NULL)) { xmlNode *reply = create_ping_reply(msg); sys_to = pcmk__xe_get(reply, PCMK__XA_CRM_SYS_TO); // Explicitly leave src empty. It indicates that dc is "not yet selected" send_msg_via_ipc(reply, sys_to, NULL); pcmk__xml_free(reply); return TRUE; } // Check whether message should be relayed if (is_for_dc || is_for_dcib || is_for_te) { if (AM_I_DC) { if (is_for_te) { pcmk__trace("Route message %s locally as transition request", ref); - crm_log_xml_trace(msg, sys_to); + pcmk__log_xml_trace(msg, sys_to); send_msg_via_ipc(msg, sys_to, controld_globals.cluster->priv->node_name); return TRUE; // No further processing of message is needed } pcmk__trace("Route message %s locally as DC request", ref); return FALSE; // More to be done by caller } if (originated_locally && !pcmk__strcase_any_of(sys_from, CRM_SYSTEM_PENGINE, CRM_SYSTEM_TENGINE, NULL)) { pcmk__trace("Relay message %s to DC (via %s)", ref, pcmk__s(host_to, "broadcast")); - crm_log_xml_trace(msg, "relayed"); + pcmk__log_xml_trace(msg, "relayed"); if (!broadcast) { node_to = pcmk__get_node(0, host_to, NULL, pcmk__node_search_cluster_member); } pcmk__cluster_send_message(node_to, dest, msg); return TRUE; } /* Transition engine and scheduler messages are sent only to the DC on * the same node. If we are no longer the DC, discard this message. */ pcmk__trace("Ignoring message %s because we are no longer DC", ref); - crm_log_xml_trace(msg, "ignored"); + pcmk__log_xml_trace(msg, "ignored"); return TRUE; // No further processing of message is needed } if (is_local) { if (is_for_crm || is_for_cib) { pcmk__trace("Route message %s locally as controller request", ref); return FALSE; // More to be done by caller } pcmk__trace("Relay message %s locally to %s", ref, sys_to); - crm_log_xml_trace(msg, "IPC-relay"); + pcmk__log_xml_trace(msg, "IPC-relay"); send_msg_via_ipc(msg, sys_to, controld_globals.cluster->priv->node_name); return TRUE; } if (!broadcast) { node_to = pcmk__search_node_caches(0, host_to, NULL, pcmk__node_search_cluster_member); if (node_to == NULL) { pcmk__warn("Ignoring message %s because node %s is unknown", ref, host_to); - crm_log_xml_trace(msg, "ignored"); + pcmk__log_xml_trace(msg, "ignored"); return TRUE; } } pcmk__trace("Relay message %s to %s", ref, pcmk__s(host_to, "all peers")); - crm_log_xml_trace(msg, "relayed"); + pcmk__log_xml_trace(msg, "relayed"); pcmk__cluster_send_message(node_to, dest, msg); return TRUE; } // Return true if field contains a positive integer static bool authorize_version(xmlNode *message_data, const char *field, const char *client_name, const char *ref, const char *uuid) { const char *version = pcmk__xe_get(message_data, field); long long version_num; if ((pcmk__scan_ll(version, &version_num, -1LL) != pcmk_rc_ok) || (version_num < 0LL)) { pcmk__warn("Rejected IPC hello from %s: '%s' is not a valid protocol %s " QB_XS " ref=%s uuid=%s", client_name, pcmk__s(version, ""), field, pcmk__s(ref, "none"), uuid); return false; } return true; } /*! * \internal * \brief Check whether a client IPC message is acceptable * * If a given client IPC message is a hello, "authorize" it by ensuring it has * valid information such as a protocol version, and return false indicating * that nothing further needs to be done with the message. If the message is not * a hello, just return true to indicate it needs further processing. * * \param[in] client_msg XML of IPC message * \param[in,out] curr_client If IPC is not proxied, client that sent message * \param[in] proxy_session If IPC is proxied, the session ID * * \return true if message needs further processing, false if it doesn't */ bool controld_authorize_ipc_message(const xmlNode *client_msg, pcmk__client_t *curr_client, const char *proxy_session) { xmlNode *wrapper = NULL; xmlNode *message_data = NULL; const char *client_name = NULL; const char *op = pcmk__xe_get(client_msg, PCMK__XA_CRM_TASK); const char *ref = pcmk__xe_get(client_msg, PCMK_XA_REFERENCE); const char *uuid = (curr_client? curr_client->id : proxy_session); if (uuid == NULL) { pcmk__warn("IPC message from client rejected: No client identifier " QB_XS " ref=%s", pcmk__s(ref, "none")); goto rejected; } if (!pcmk__str_eq(CRM_OP_HELLO, op, pcmk__str_casei)) { // Only hello messages need to be authorized return true; } wrapper = pcmk__xe_first_child(client_msg, PCMK__XE_CRM_XML, NULL, NULL); message_data = pcmk__xe_first_child(wrapper, NULL, NULL, NULL); client_name = pcmk__xe_get(message_data, PCMK__XA_CLIENT_NAME); if (pcmk__str_empty(client_name)) { pcmk__warn("IPC hello from client rejected: No client name", QB_XS " ref=%s uuid=%s", pcmk__s(ref, "none"), uuid); goto rejected; } if (!authorize_version(message_data, PCMK__XA_MAJOR_VERSION, client_name, ref, uuid)) { goto rejected; } if (!authorize_version(message_data, PCMK__XA_MINOR_VERSION, client_name, ref, uuid)) { goto rejected; } pcmk__trace("Validated IPC hello from client %s", client_name); - crm_log_xml_trace(client_msg, "hello"); + pcmk__log_xml_trace(client_msg, "hello"); if (curr_client) { curr_client->userdata = pcmk__str_copy(client_name); } controld_trigger_fsa(); return false; rejected: - crm_log_xml_trace(client_msg, "rejected"); + pcmk__log_xml_trace(client_msg, "rejected"); if (curr_client) { qb_ipcs_disconnect(curr_client->ipcs); } return false; } static enum crmd_fsa_input handle_message(xmlNode *msg, enum crmd_fsa_cause cause) { const char *type = NULL; CRM_CHECK(msg != NULL, return I_NULL); type = pcmk__xe_get(msg, PCMK__XA_SUBT); if (pcmk__str_eq(type, PCMK__VALUE_REQUEST, pcmk__str_none)) { return handle_request(msg, cause); } if (pcmk__str_eq(type, PCMK__VALUE_RESPONSE, pcmk__str_none)) { handle_response(msg); return I_NULL; } pcmk__warn("Ignoring message with unknown " PCMK__XA_SUBT" '%s'", pcmk__s(type, "")); - crm_log_xml_trace(msg, "bad"); + pcmk__log_xml_trace(msg, "bad"); return I_NULL; } static enum crmd_fsa_input handle_failcount_op(xmlNode * stored_msg) { const char *rsc = NULL; const char *uname = NULL; const char *op = NULL; char *interval_spec = NULL; guint interval_ms = 0; gboolean is_remote_node = FALSE; xmlNode *wrapper = pcmk__xe_first_child(stored_msg, PCMK__XE_CRM_XML, NULL, NULL); xmlNode *xml_op = pcmk__xe_first_child(wrapper, NULL, NULL, NULL); if (xml_op) { xmlNode *xml_rsc = pcmk__xe_first_child(xml_op, PCMK_XE_PRIMITIVE, NULL, NULL); xmlNode *xml_attrs = pcmk__xe_first_child(xml_op, PCMK__XE_ATTRIBUTES, NULL, NULL); if (xml_rsc) { rsc = pcmk__xe_id(xml_rsc); } if (xml_attrs) { op = pcmk__xe_get(xml_attrs, CRM_META "_" PCMK__META_CLEAR_FAILURE_OP); pcmk__xe_get_guint(xml_attrs, CRM_META "_" PCMK__META_CLEAR_FAILURE_INTERVAL, &interval_ms); } } uname = pcmk__xe_get(xml_op, PCMK__META_ON_NODE); if ((rsc == NULL) || (uname == NULL)) { pcmk__log_xml_warn(stored_msg, "invalid failcount op"); return I_NULL; } if (pcmk__xe_get(xml_op, PCMK__XA_ROUTER_NODE)) { is_remote_node = TRUE; } pcmk__debug("Clearing failures for %s-interval %s on %s from attribute " "manager, CIB, and executor state", pcmk__readable_interval(interval_ms), rsc, uname); if (interval_ms) { interval_spec = pcmk__assert_asprintf("%ums", interval_ms); } update_attrd_clear_failures(uname, rsc, op, interval_spec, is_remote_node); free(interval_spec); controld_cib_delete_last_failure(rsc, uname, op, interval_ms); lrm_clear_last_failure(rsc, uname, op, interval_ms); return I_NULL; } static enum crmd_fsa_input handle_lrm_delete(xmlNode *stored_msg) { const char *mode = NULL; xmlNode *wrapper = pcmk__xe_first_child(stored_msg, PCMK__XE_CRM_XML, NULL, NULL); xmlNode *msg_data = pcmk__xe_first_child(wrapper, NULL, NULL, NULL); CRM_CHECK(msg_data != NULL, return I_NULL); /* CRM_OP_LRM_DELETE has two distinct modes. The default behavior is to * relay the operation to the affected node, which will unregister the * resource from the local executor, clear the resource's history from the * CIB, and do some bookkeeping in the controller. * * However, if the affected node is offline, the client will specify * mode=PCMK__VALUE_CIB which means the controller receiving the operation * should clear the resource's history from the CIB and nothing else. This * is used to clear shutdown locks. */ mode = pcmk__xe_get(msg_data, PCMK__XA_MODE); if (!pcmk__str_eq(mode, PCMK__VALUE_CIB, pcmk__str_none)) { // Relay to affected node pcmk__xe_set(stored_msg, PCMK__XA_CRM_SYS_TO, CRM_SYSTEM_LRMD); return I_ROUTER; } else { // Delete CIB history locally (compare with do_lrm_delete()) const char *from_sys = NULL; const char *user_name = NULL; const char *rsc_id = NULL; const char *node = NULL; xmlNode *rsc_xml = NULL; int rc = pcmk_rc_ok; rsc_xml = pcmk__xe_first_child(msg_data, PCMK_XE_PRIMITIVE, NULL, NULL); CRM_CHECK(rsc_xml != NULL, return I_NULL); rsc_id = pcmk__xe_id(rsc_xml); from_sys = pcmk__xe_get(stored_msg, PCMK__XA_CRM_SYS_FROM); node = pcmk__xe_get(msg_data, PCMK__META_ON_NODE); user_name = pcmk__update_acl_user(stored_msg, PCMK__XA_CRM_USER, NULL); pcmk__debug("Handling " CRM_OP_LRM_DELETE " for %s on %s locally%s%s " "(clearing CIB resource history only)", rsc_id, node, ((user_name != NULL)? " for user " : ""), pcmk__s(user_name, "")); rc = controld_delete_resource_history(rsc_id, node, user_name, cib_dryrun|cib_sync_call); if (rc == pcmk_rc_ok) { rc = controld_delete_resource_history(rsc_id, node, user_name, crmd_cib_smart_opt()); } /* Notify client. Also notify tengine if mode=PCMK__VALUE_CIB and * op=CRM_OP_LRM_DELETE. */ if (from_sys) { lrmd_event_data_t *op = NULL; const char *from_host = pcmk__xe_get(stored_msg, PCMK__XA_SRC); const char *transition; if (strcmp(from_sys, CRM_SYSTEM_TENGINE)) { transition = pcmk__xe_get(msg_data, PCMK__XA_TRANSITION_KEY); } else { transition = pcmk__xe_get(stored_msg, PCMK__XA_TRANSITION_KEY); } pcmk__info("Notifying %s on %s that %s was%s deleted", from_sys, pcmk__s(from_host, "local node"), rsc_id, ((rc == pcmk_rc_ok)? "" : " not")); op = lrmd_new_event(rsc_id, PCMK_ACTION_DELETE, 0); op->type = lrmd_event_exec_complete; op->user_data = pcmk__str_copy(pcmk__s(transition, FAKE_TE_ID)); op->params = pcmk__strkey_table(free, free); pcmk__insert_dup(op->params, PCMK_XA_CRM_FEATURE_SET, CRM_FEATURE_SET); controld_rc2event(op, rc); controld_ack_event_directly(from_host, from_sys, NULL, op, rsc_id); lrmd_free_event(op); controld_trigger_delete_refresh(from_sys, rsc_id); } return I_NULL; } } /*! * \brief Handle a CRM_OP_REMOTE_STATE message by updating remote peer cache * * \param[in] msg Message XML * * \return Next FSA input */ static enum crmd_fsa_input handle_remote_state(const xmlNode *msg) { const char *conn_host = NULL; const char *remote_uname = pcmk__xe_id(msg); pcmk__node_status_t *remote_peer; bool remote_is_up = false; int rc = pcmk_rc_ok; rc = pcmk__xe_get_bool_attr(msg, PCMK__XA_IN_CCM, &remote_is_up); CRM_CHECK(remote_uname && rc == pcmk_rc_ok, return I_NULL); remote_peer = pcmk__cluster_lookup_remote_node(remote_uname); CRM_CHECK(remote_peer, return I_NULL); pcmk__update_peer_state(__func__, remote_peer, remote_is_up ? PCMK_VALUE_MEMBER : PCMK__VALUE_LOST, 0); conn_host = pcmk__xe_get(msg, PCMK__XA_CONNECTION_HOST); if (conn_host) { pcmk__str_update(&remote_peer->conn_host, conn_host); } else if (remote_peer->conn_host) { free(remote_peer->conn_host); remote_peer->conn_host = NULL; } return I_NULL; } /*! * \brief Handle a CRM_OP_PING message * * \param[in] msg Message XML * * \return Next FSA input */ static xmlNode* create_ping_reply(const xmlNode *msg) { const char *value = NULL; xmlNode *ping = NULL; xmlNode *reply = NULL; // Build reply ping = pcmk__xe_create(NULL, PCMK__XE_PING_RESPONSE); value = pcmk__xe_get(msg, PCMK__XA_CRM_SYS_TO); pcmk__xe_set(ping, PCMK__XA_CRM_SUBSYSTEM, value); // Add controller state value = fsa_state2string(controld_globals.fsa_state); pcmk__xe_set(ping, PCMK__XA_CRMD_STATE, value); pcmk__notice("Current ping state: %s", value); // CTS needs this // Add controller health // @TODO maybe do some checks to determine meaningful status pcmk__xe_set(ping, PCMK_XA_RESULT, "ok"); reply = pcmk__new_reply(msg, ping); pcmk__xml_free(ping); return reply; } static enum crmd_fsa_input handle_ping(const xmlNode *msg) { xmlNode *reply = create_ping_reply(msg); if (reply != NULL) { (void) relay_message(reply, TRUE); pcmk__xml_free(reply); } // Nothing further to do return I_NULL; } /*! * \brief Handle a PCMK__CONTROLD_CMD_NODES message * * \param[in] request Message XML * * \return Next FSA input */ static enum crmd_fsa_input handle_node_list(const xmlNode *request) { GHashTableIter iter; pcmk__node_status_t *node = NULL; xmlNode *reply = NULL; xmlNode *reply_data = NULL; // Create message data for reply reply_data = pcmk__xe_create(NULL, PCMK_XE_NODES); g_hash_table_iter_init(&iter, pcmk__peer_cache); while (g_hash_table_iter_next(&iter, NULL, (gpointer *) & node)) { xmlNode *xml = pcmk__xe_create(reply_data, PCMK_XE_NODE); pcmk__xe_set_ll(xml, PCMK_XA_ID, (long long) node->cluster_layer_id); // uint32_t pcmk__xe_set(xml, PCMK_XA_UNAME, node->name); pcmk__xe_set(xml, PCMK__XA_IN_CCM, node->state); } // Create and send reply reply = pcmk__new_reply(request, reply_data); pcmk__xml_free(reply_data); if (reply) { (void) relay_message(reply, TRUE); pcmk__xml_free(reply); } // Nothing further to do return I_NULL; } /*! * \brief Handle a CRM_OP_NODE_INFO request * * \param[in] msg Message XML * * \return Next FSA input */ static enum crmd_fsa_input handle_node_info_request(const xmlNode *msg) { const char *value = NULL; pcmk__node_status_t *node = NULL; int node_id = 0; xmlNode *reply = NULL; xmlNode *reply_data = NULL; // Build reply reply_data = pcmk__xe_create(NULL, PCMK_XE_NODE); pcmk__xe_set(reply_data, PCMK__XA_CRM_SUBSYSTEM, CRM_SYSTEM_CRMD); // Add whether current partition has quorum pcmk__xe_set_bool_attr(reply_data, PCMK_XA_HAVE_QUORUM, pcmk__is_set(controld_globals.flags, controld_has_quorum)); /* Check whether client requested node info by ID and/or name * * @TODO A Corosync-layer node ID is of type uint32_t. We should be able to * handle legitimate node IDs greater than INT_MAX, but currently we do not. */ pcmk__xe_get_int(msg, PCMK_XA_ID, &node_id); if (node_id < 0) { node_id = 0; } value = pcmk__xe_get(msg, PCMK_XA_UNAME); // Default to local node if none given if ((node_id == 0) && (value == NULL)) { value = controld_globals.cluster->priv->node_name; } node = pcmk__search_node_caches(node_id, value, NULL, pcmk__node_search_any); if (node) { pcmk__xe_set(reply_data, PCMK_XA_ID, node->xml_id); pcmk__xe_set(reply_data, PCMK_XA_UNAME, node->name); pcmk__xe_set(reply_data, PCMK_XA_CRMD, node->state); pcmk__xe_set_bool_attr(reply_data, PCMK_XA_REMOTE_NODE, pcmk__is_set(node->flags, pcmk__node_status_remote)); } // Send reply reply = pcmk__new_reply(msg, reply_data); pcmk__xml_free(reply_data); if (reply != NULL) { (void) relay_message(reply, TRUE); pcmk__xml_free(reply); } // Nothing further to do return I_NULL; } static void verify_feature_set(xmlNode *msg) { const char *dc_version = pcmk__xe_get(msg, PCMK_XA_CRM_FEATURE_SET); if (dc_version == NULL) { /* All we really know is that the DC feature set is older than 3.1.0, * but that's also all that really matters. */ dc_version = "3.0.14"; } if (feature_set_compatible(dc_version, CRM_FEATURE_SET)) { pcmk__trace("Local feature set (%s) is compatible with DC's (%s)", CRM_FEATURE_SET, dc_version); } else { pcmk__err("Local feature set (%s) is incompatible with DC's (%s)", CRM_FEATURE_SET, dc_version); // Nothing is likely to improve without administrator involvement controld_set_fsa_input_flags(R_STAYDOWN); crmd_exit(CRM_EX_FATAL); } } // DC gets own shutdown all-clear static enum crmd_fsa_input handle_shutdown_self_ack(xmlNode *stored_msg) { const char *host_from = pcmk__xe_get(stored_msg, PCMK__XA_SRC); if (pcmk__is_set(controld_globals.fsa_input_register, R_SHUTDOWN)) { // The expected case -- we initiated own shutdown sequence pcmk__info("Shutting down controller"); return I_STOP; } if (pcmk__str_eq(host_from, controld_globals.dc_name, pcmk__str_casei)) { // Must be logic error -- DC confirming its own unrequested shutdown pcmk__err("Shutting down controller immediately due to unexpected " "shutdown confirmation"); return I_TERMINATE; } if (controld_globals.fsa_state != S_STOPPING) { // Shouldn't happen -- non-DC confirming unrequested shutdown pcmk__err("Starting new DC election because %s is confirming shutdown " "we did not request", pcmk__s(host_from, "another node")); return I_ELECTION; } // Shouldn't happen, but we are already stopping anyway pcmk__debug("Ignoring unexpected shutdown confirmation from %s", pcmk__s(host_from, "another node")); return I_NULL; } // Non-DC gets shutdown all-clear from DC static enum crmd_fsa_input handle_shutdown_ack(xmlNode *stored_msg) { const char *host_from = pcmk__xe_get(stored_msg, PCMK__XA_SRC); if (host_from == NULL) { pcmk__warn("Ignoring shutdown request without origin specified"); return I_NULL; } if (pcmk__str_eq(host_from, controld_globals.dc_name, pcmk__str_null_matches|pcmk__str_casei)) { if (pcmk__is_set(controld_globals.fsa_input_register, R_SHUTDOWN)) { pcmk__info("Shutting down controller after confirmation from %s", host_from); } else { pcmk__err("Shutting down controller after unexpected " "shutdown request from %s", host_from); controld_set_fsa_input_flags(R_STAYDOWN); } return I_STOP; } pcmk__warn("Ignoring shutdown request from %s because DC is %s", host_from, controld_globals.dc_name); return I_NULL; } static enum crmd_fsa_input handle_request(xmlNode *stored_msg, enum crmd_fsa_cause cause) { xmlNode *msg = NULL; const char *op = pcmk__xe_get(stored_msg, PCMK__XA_CRM_TASK); /* Optimize this for the DC - it has the most to do */ - crm_log_xml_trace(stored_msg, "request"); + pcmk__log_xml_trace(stored_msg, "request"); if (op == NULL) { pcmk__warn("Ignoring request without " PCMK__XA_CRM_TASK); return I_NULL; } if (strcmp(op, CRM_OP_SHUTDOWN_REQ) == 0) { const char *from = pcmk__xe_get(stored_msg, PCMK__XA_SRC); pcmk__node_status_t *node = pcmk__search_node_caches(0, from, NULL, pcmk__node_search_cluster_member); pcmk__update_peer_expected(__func__, node, CRMD_JOINSTATE_DOWN); if(AM_I_DC == FALSE) { return I_NULL; /* Done */ } } /*========== DC-Only Actions ==========*/ if (AM_I_DC) { if (strcmp(op, CRM_OP_JOIN_ANNOUNCE) == 0) { return I_NODE_JOIN; } else if (strcmp(op, CRM_OP_JOIN_REQUEST) == 0) { return I_JOIN_REQUEST; } else if (strcmp(op, CRM_OP_JOIN_CONFIRM) == 0) { return I_JOIN_RESULT; } else if (strcmp(op, CRM_OP_SHUTDOWN) == 0) { return handle_shutdown_self_ack(stored_msg); } else if (strcmp(op, CRM_OP_SHUTDOWN_REQ) == 0) { // Another controller wants to shut down its node return handle_shutdown_request(stored_msg); } } /*========== common actions ==========*/ if (strcmp(op, CRM_OP_NOVOTE) == 0) { ha_msg_input_t fsa_input; fsa_input.msg = stored_msg; register_fsa_input_adv(C_HA_MESSAGE, I_NULL, &fsa_input, A_ELECTION_COUNT | A_ELECTION_CHECK, FALSE, __func__); } else if (strcmp(op, CRM_OP_REMOTE_STATE) == 0) { /* a remote connection host is letting us know the node state */ return handle_remote_state(stored_msg); } else if (strcmp(op, CRM_OP_THROTTLE) == 0) { throttle_update(stored_msg); if (AM_I_DC && (controld_globals.transition_graph != NULL) && !controld_globals.transition_graph->complete) { pcmk__debug("The throttle changed. Trigger a graph"); trigger_graph(); } return I_NULL; } else if (strcmp(op, CRM_OP_CLEAR_FAILCOUNT) == 0) { return handle_failcount_op(stored_msg); } else if (strcmp(op, CRM_OP_VOTE) == 0) { /* count the vote and decide what to do after that */ ha_msg_input_t fsa_input; fsa_input.msg = stored_msg; register_fsa_input_adv(C_HA_MESSAGE, I_NULL, &fsa_input, A_ELECTION_COUNT | A_ELECTION_CHECK, FALSE, __func__); /* Sometimes we _must_ go into S_ELECTION */ if (controld_globals.fsa_state == S_HALT) { pcmk__debug("Forcing an election from S_HALT"); return I_ELECTION; } } else if (strcmp(op, CRM_OP_JOIN_OFFER) == 0) { verify_feature_set(stored_msg); pcmk__debug("Raising I_JOIN_OFFER: join-%s", pcmk__xe_get(stored_msg, PCMK__XA_JOIN_ID)); return I_JOIN_OFFER; } else if (strcmp(op, CRM_OP_JOIN_ACKNAK) == 0) { pcmk__debug("Raising I_JOIN_RESULT: join-%s", pcmk__xe_get(stored_msg, PCMK__XA_JOIN_ID)); return I_JOIN_RESULT; } else if (strcmp(op, CRM_OP_LRM_DELETE) == 0) { return handle_lrm_delete(stored_msg); } else if ((strcmp(op, CRM_OP_LRM_FAIL) == 0) || (strcmp(op, CRM_OP_REPROBE) == 0)) { pcmk__xe_set(stored_msg, PCMK__XA_CRM_SYS_TO, CRM_SYSTEM_LRMD); return I_ROUTER; } else if (strcmp(op, CRM_OP_NOOP) == 0) { return I_NULL; } else if (strcmp(op, CRM_OP_PING) == 0) { return handle_ping(stored_msg); } else if (strcmp(op, CRM_OP_NODE_INFO) == 0) { return handle_node_info_request(stored_msg); } else if (strcmp(op, CRM_OP_RM_NODE_CACHE) == 0) { int id = 0; const char *name = NULL; pcmk__xe_get_int(stored_msg, PCMK_XA_ID, &id); name = pcmk__xe_get(stored_msg, PCMK_XA_UNAME); if(cause == C_IPC_MESSAGE) { msg = pcmk__new_request(pcmk_ipc_controld, CRM_SYSTEM_CRMD, NULL, CRM_SYSTEM_CRMD, CRM_OP_RM_NODE_CACHE, NULL); if (!pcmk__cluster_send_message(NULL, pcmk_ipc_controld, msg)) { pcmk__err("Could not instruct peers to remove references to " "node %s/%u", name, id); } else { pcmk__notice("Instructing peers to remove references to node " "%s/%u", name, id); } pcmk__xml_free(msg); } else { pcmk__cluster_forget_cluster_node(id, name); /* If we're forgetting this node, also forget any failures to fence * it, so we don't carry that over to any node added later with the * same name. */ st_fail_count_reset(name); } } else if (strcmp(op, CRM_OP_MAINTENANCE_NODES) == 0) { xmlNode *wrapper = pcmk__xe_first_child(stored_msg, PCMK__XE_CRM_XML, NULL, NULL); xmlNode *xml = pcmk__xe_first_child(wrapper, NULL, NULL, NULL); remote_ra_process_maintenance_nodes(xml); } else if (strcmp(op, PCMK__CONTROLD_CMD_NODES) == 0) { return handle_node_list(stored_msg); /*========== (NOT_DC)-Only Actions ==========*/ } else if (!AM_I_DC) { if (strcmp(op, CRM_OP_SHUTDOWN) == 0) { return handle_shutdown_ack(stored_msg); } } else { pcmk__err("Unexpected request (%s) sent to %s", op, (AM_I_DC ? "the DC" : "non-DC node")); pcmk__log_xml_err(stored_msg, "Unexpected"); } return I_NULL; } static void handle_response(xmlNode *stored_msg) { const char *op = pcmk__xe_get(stored_msg, PCMK__XA_CRM_TASK); - crm_log_xml_trace(stored_msg, "reply"); + pcmk__log_xml_trace(stored_msg, "reply"); if (op == NULL) { pcmk__warn("Ignoring reply without " PCMK__XA_CRM_TASK); } else if (AM_I_DC && strcmp(op, CRM_OP_PECALC) == 0) { // Check whether scheduler answer been superseded by subsequent request const char *msg_ref = pcmk__xe_get(stored_msg, PCMK_XA_REFERENCE); if (msg_ref == NULL) { pcmk__err("%s - Ignoring calculation with no reference", op); } else if (pcmk__str_eq(msg_ref, controld_globals.fsa_pe_ref, pcmk__str_none)) { ha_msg_input_t fsa_input; controld_stop_sched_timer(); fsa_input.msg = stored_msg; register_fsa_input_later(C_IPC_MESSAGE, I_PE_SUCCESS, &fsa_input); } else { pcmk__info("%s calculation %s is obsolete", op, msg_ref); } } else if (strcmp(op, CRM_OP_VOTE) == 0 || strcmp(op, CRM_OP_SHUTDOWN_REQ) == 0 || strcmp(op, CRM_OP_SHUTDOWN) == 0) { } else { const char *host_from = pcmk__xe_get(stored_msg, PCMK__XA_SRC); pcmk__err("Unexpected response (op=%s, src=%s) sent to the %s", op, host_from, (AM_I_DC ? "DC" : "controller")); } } static enum crmd_fsa_input handle_shutdown_request(xmlNode * stored_msg) { /* handle here to avoid potential version issues * where the shutdown message/procedure may have * been changed in later versions. * * This way the DC is always in control of the shutdown */ char *now_s = NULL; const char *host_from = pcmk__xe_get(stored_msg, PCMK__XA_SRC); if (host_from == NULL) { /* we're shutting down and the DC */ host_from = controld_globals.cluster->priv->node_name; } pcmk__info("Creating shutdown request for %s (state=%s)", host_from, fsa_state2string(controld_globals.fsa_state)); - crm_log_xml_trace(stored_msg, "message"); + pcmk__log_xml_trace(stored_msg, "message"); now_s = pcmk__ttoa(time(NULL)); update_attrd(host_from, PCMK__NODE_ATTR_SHUTDOWN, now_s, NULL, FALSE); free(now_s); /* will be picked up by the TE as long as its running */ return I_NULL; } static void send_msg_via_ipc(xmlNode * msg, const char *sys, const char *src) { pcmk__client_t *client_channel = NULL; CRM_CHECK(sys != NULL, return); client_channel = pcmk__find_client_by_id(sys); if (pcmk__xe_get(msg, PCMK__XA_SRC) == NULL) { pcmk__xe_set(msg, PCMK__XA_SRC, src); } if (client_channel != NULL) { /* Transient clients such as crmadmin */ pcmk__ipc_send_xml(client_channel, 0, msg, crm_ipc_server_event); } else if (pcmk__str_eq(sys, CRM_SYSTEM_TENGINE, pcmk__str_none)) { xmlNode *wrapper = pcmk__xe_first_child(msg, PCMK__XE_CRM_XML, NULL, NULL); xmlNode *data = pcmk__xe_first_child(wrapper, NULL, NULL, NULL); process_te_message(msg, data); } else if (pcmk__str_eq(sys, CRM_SYSTEM_LRMD, pcmk__str_none)) { fsa_data_t fsa_data; ha_msg_input_t fsa_input; xmlNode *wrapper = NULL; fsa_input.msg = msg; wrapper = pcmk__xe_first_child(msg, PCMK__XE_CRM_XML, NULL, NULL); fsa_input.xml = pcmk__xe_first_child(wrapper, NULL, NULL, NULL); fsa_data.id = 0; fsa_data.actions = 0; fsa_data.data = &fsa_input; fsa_data.fsa_input = I_MESSAGE; fsa_data.fsa_cause = C_IPC_MESSAGE; fsa_data.origin = __func__; fsa_data.data_type = fsa_dt_ha_msg; do_lrm_invoke(A_LRM_INVOKE, C_IPC_MESSAGE, controld_globals.fsa_state, I_MESSAGE, &fsa_data); } else if (crmd_is_proxy_session(sys)) { crmd_proxy_send(sys, msg); } else { pcmk__info("Received invalid request: unknown subsystem '%s'", sys); } } void delete_ha_msg_input(ha_msg_input_t * orig) { if (orig == NULL) { return; } pcmk__xml_free(orig->msg); free(orig); } /*! * \internal * \brief Notify the cluster of a remote node state change * * \param[in] node_name Node's name * \param[in] node_up true if node is up, false if down */ void broadcast_remote_state_message(const char *node_name, bool node_up) { xmlNode *msg = pcmk__new_request(pcmk_ipc_controld, CRM_SYSTEM_CRMD, NULL, CRM_SYSTEM_CRMD, CRM_OP_REMOTE_STATE, NULL); pcmk__info("Notifying cluster of Pacemaker Remote node %s %s", node_name, (node_up? "coming up" : "going down")); pcmk__xe_set(msg, PCMK_XA_ID, node_name); pcmk__xe_set_bool_attr(msg, PCMK__XA_IN_CCM, node_up); if (node_up) { pcmk__xe_set(msg, PCMK__XA_CONNECTION_HOST, controld_globals.cluster->priv->node_name); } pcmk__cluster_send_message(NULL, pcmk_ipc_controld, msg); pcmk__xml_free(msg); } diff --git a/daemons/controld/controld_te_callbacks.c b/daemons/controld/controld_te_callbacks.c index 59f9b5b6bc..ae1c966cd9 100644 --- a/daemons/controld/controld_te_callbacks.c +++ b/daemons/controld/controld_te_callbacks.c @@ -1,532 +1,532 @@ /* * Copyright 2004-2025 the Pacemaker project contributors * * The version control history for this file may have further details. * * This source code is licensed under the GNU General Public License version 2 * or later (GPLv2+) WITHOUT ANY WARRANTY. */ #include #include #include #include // PCMK_SCORE_INFINITY #include #include #include // xmlXPathObject, etc. #include // An explicit PCMK_OPT_SHUTDOWN_LOCK of 0 means the lock has been cleared static bool shutdown_lock_cleared(xmlNode *lrm_resource) { time_t shutdown_lock = 0; return (pcmk__xe_get_time(lrm_resource, PCMK_OPT_SHUTDOWN_LOCK, &shutdown_lock) == pcmk_rc_ok) && (shutdown_lock == 0); } static void process_lrm_resource_diff(xmlNode *lrm_resource, const char *node) { for (xmlNode *rsc_op = pcmk__xe_first_child(lrm_resource, NULL, NULL, NULL); rsc_op != NULL; rsc_op = pcmk__xe_next(rsc_op, NULL)) { process_graph_event(rsc_op, node); } if (shutdown_lock_cleared(lrm_resource)) { // @TODO would be more efficient to abort once after transition done abort_transition(PCMK_SCORE_INFINITY, pcmk__graph_restart, "Shutdown lock cleared", lrm_resource); } } static void process_resource_updates(const char *node, xmlNode *xml, xmlNode *change, const char *op, const char *xpath) { xmlNode *rsc = NULL; if (xml == NULL) { return; } if (pcmk__xe_is(xml, PCMK__XE_LRM)) { xml = pcmk__xe_first_child(xml, PCMK__XE_LRM_RESOURCES, NULL, NULL); CRM_CHECK(xml != NULL, return); } CRM_CHECK(pcmk__xe_is(xml, PCMK__XE_LRM_RESOURCES), return); /* * Updates by, or in response to, TE actions will never contain updates * for more than one resource at a time, so such updates indicate an * LRM refresh. * * In that case, start a new transition rather than check each result * individually, which can result in _huge_ speedups in large clusters. * * Unfortunately, we can only do so when there are no pending actions. * Otherwise, we could mistakenly throw away those results here, and * the cluster will stall waiting for them and time out the operation. */ if ((controld_globals.transition_graph->pending == 0) && (xml->children != NULL) && (xml->children->next != NULL)) { - crm_log_xml_trace(change, "lrm-refresh"); + pcmk__log_xml_trace(change, "lrm-refresh"); abort_transition(PCMK_SCORE_INFINITY, pcmk__graph_restart, "History refresh", NULL); return; } for (rsc = pcmk__xe_first_child(xml, NULL, NULL, NULL); rsc != NULL; rsc = pcmk__xe_next(rsc, NULL)) { pcmk__trace("Processing %s", pcmk__xe_id(rsc)); process_lrm_resource_diff(rsc, node); } } static char *extract_node_uuid(const char *xpath) { char *mutable_path = pcmk__str_copy(xpath); char *node_uuid = NULL; char *search = NULL; char *match = NULL; match = strstr(mutable_path, PCMK__XE_NODE_STATE "[@" PCMK_XA_ID "=\'"); if (match == NULL) { free(mutable_path); return NULL; } match += strlen(PCMK__XE_NODE_STATE "[@" PCMK_XA_ID "=\'"); search = strchr(match, '\''); if (search == NULL) { free(mutable_path); return NULL; } search[0] = 0; node_uuid = pcmk__str_copy(match); free(mutable_path); return node_uuid; } static void abort_unless_down(const char *xpath, const char *op, xmlNode *change, const char *reason) { char *node_uuid = NULL; pcmk__graph_action_t *down = NULL; if (!pcmk__str_eq(op, PCMK_VALUE_DELETE, pcmk__str_none)) { abort_transition(PCMK_SCORE_INFINITY, pcmk__graph_restart, reason, change); return; } node_uuid = extract_node_uuid(xpath); if(node_uuid == NULL) { pcmk__err("Could not extract node ID from %s", xpath); abort_transition(PCMK_SCORE_INFINITY, pcmk__graph_restart, reason, change); return; } down = match_down_event(node_uuid); if (down == NULL) { pcmk__trace("Not expecting %s to be down (%s)", node_uuid, xpath); abort_transition(PCMK_SCORE_INFINITY, pcmk__graph_restart, reason, change); } else { pcmk__trace("Expecting changes to %s (%s)", node_uuid, xpath); } free(node_uuid); } static void process_op_deletion(const char *xpath, xmlNode *change) { char *mutable_key = pcmk__str_copy(xpath); char *key; char *node_uuid; // Extract the part of xpath between last pair of single quotes key = strrchr(mutable_key, '\''); if (key != NULL) { *key = '\0'; key = strrchr(mutable_key, '\''); } if (key == NULL) { pcmk__warn("Ignoring malformed CIB update (resource deletion of %s)", xpath); free(mutable_key); return; } ++key; node_uuid = extract_node_uuid(xpath); if (confirm_cancel_action(key, node_uuid) == FALSE) { abort_transition(PCMK_SCORE_INFINITY, pcmk__graph_restart, "Resource operation removal", change); } free(mutable_key); free(node_uuid); } static void process_delete_diff(const char *xpath, const char *op, xmlNode *change) { if (strstr(xpath, "/" PCMK__XE_LRM_RSC_OP "[")) { process_op_deletion(xpath, change); } else if (strstr(xpath, "/" PCMK__XE_LRM "[")) { abort_unless_down(xpath, op, change, "Resource state removal"); } else if (strstr(xpath, "/" PCMK__XE_NODE_STATE "[")) { abort_unless_down(xpath, op, change, "Node state removal"); } else { pcmk__trace("Ignoring delete of %s", xpath); } } static void process_node_state_diff(xmlNode *state, xmlNode *change, const char *op, const char *xpath) { xmlNode *lrm = pcmk__xe_first_child(state, PCMK__XE_LRM, NULL, NULL); process_resource_updates(pcmk__xe_id(state), lrm, change, op, xpath); } static void process_status_diff(xmlNode *status, xmlNode *change, const char *op, const char *xpath) { for (xmlNode *state = pcmk__xe_first_child(status, NULL, NULL, NULL); state != NULL; state = pcmk__xe_next(state, NULL)) { process_node_state_diff(state, change, op, xpath); } } static void process_cib_diff(xmlNode *cib, xmlNode *change, const char *op, const char *xpath) { xmlNode *status = pcmk__xe_first_child(cib, PCMK_XE_STATUS, NULL, NULL); xmlNode *config = pcmk__xe_first_child(cib, PCMK_XE_CONFIGURATION, NULL, NULL); if (status) { process_status_diff(status, change, op, xpath); } if (config) { abort_transition(PCMK_SCORE_INFINITY, pcmk__graph_restart, "Non-status-only change", change); } } static int te_update_diff_element(xmlNode *change, void *userdata) { xmlNode *match = NULL; const char *name = NULL; const char *xpath = pcmk__xe_get(change, PCMK_XA_PATH); // Possible ops: create, modify, delete, move const char *op = pcmk__xe_get(change, PCMK_XA_OPERATION); // Ignore uninteresting updates if (op == NULL) { return pcmk_rc_ok; } else if (xpath == NULL) { pcmk__trace("Ignoring %s change for version field", op); return pcmk_rc_ok; } else if ((strcmp(op, PCMK_VALUE_MOVE) == 0) && (strstr(xpath, "/" PCMK_XE_CIB "/" PCMK_XE_CONFIGURATION "/" PCMK_XE_RESOURCES) == NULL)) { /* We still need to consider moves within the resources section, * since they affect placement order. */ pcmk__trace("Ignoring move change at %s", xpath); return pcmk_rc_ok; } // Find the result of create/modify ops if (strcmp(op, PCMK_VALUE_CREATE) == 0) { match = change->children; } else if (strcmp(op, PCMK_VALUE_MODIFY) == 0) { match = pcmk__xe_first_child(change, PCMK_XE_CHANGE_RESULT, NULL, NULL); if(match) { match = match->children; } } else if (!pcmk__str_any_of(op, PCMK_VALUE_DELETE, PCMK_VALUE_MOVE, NULL)) { pcmk__warn("Ignoring malformed CIB update (%s operation on %s is " "unrecognized)", op, xpath); return pcmk_rc_ok; } if (match) { if (match->type == XML_COMMENT_NODE) { pcmk__trace("Ignoring %s operation for comment at %s", op, xpath); return pcmk_rc_ok; } name = (const char *)match->name; } pcmk__trace("Handling %s operation for %s%s%s", op, pcmk__s(xpath, "CIB"), ((name != NULL)? " matched by " : ""), pcmk__s(name, "")); if (strstr(xpath, "/" PCMK_XE_CIB "/" PCMK_XE_CONFIGURATION)) { abort_transition(PCMK_SCORE_INFINITY, pcmk__graph_restart, "Configuration change", change); return pcmk_rc_cib_modified; // Won't be packaged with operation results we may be waiting for } else if (strstr(xpath, "/" PCMK_XE_TICKETS) || pcmk__str_eq(name, PCMK_XE_TICKETS, pcmk__str_none)) { abort_transition(PCMK_SCORE_INFINITY, pcmk__graph_restart, "Ticket attribute change", change); return pcmk_rc_cib_modified; // Won't be packaged with operation results we may be waiting for } else if (strstr(xpath, "/" PCMK__XE_TRANSIENT_ATTRIBUTES "[") || pcmk__str_eq(name, PCMK__XE_TRANSIENT_ATTRIBUTES, pcmk__str_none)) { abort_unless_down(xpath, op, change, "Transient attribute change"); return pcmk_rc_cib_modified; // Won't be packaged with operation results we may be waiting for } else if (strcmp(op, PCMK_VALUE_DELETE) == 0) { process_delete_diff(xpath, op, change); } else if (name == NULL) { pcmk__warn("Ignoring malformed CIB update (%s at %s has no result)", op, xpath); } else if (strcmp(name, PCMK_XE_CIB) == 0) { process_cib_diff(match, change, op, xpath); } else if (strcmp(name, PCMK_XE_STATUS) == 0) { process_status_diff(match, change, op, xpath); } else if (strcmp(name, PCMK__XE_NODE_STATE) == 0) { process_node_state_diff(match, change, op, xpath); } else if (strcmp(name, PCMK__XE_LRM) == 0) { process_resource_updates(pcmk__xe_id(match), match, change, op, xpath); } else if (strcmp(name, PCMK__XE_LRM_RESOURCES) == 0) { char *local_node = pcmk__xpath_node_id(xpath, PCMK__XE_LRM); process_resource_updates(local_node, match, change, op, xpath); free(local_node); } else if (strcmp(name, PCMK__XE_LRM_RESOURCE) == 0) { char *local_node = pcmk__xpath_node_id(xpath, PCMK__XE_LRM); process_lrm_resource_diff(match, local_node); free(local_node); } else if (strcmp(name, PCMK__XE_LRM_RSC_OP) == 0) { char *local_node = pcmk__xpath_node_id(xpath, PCMK__XE_LRM); process_graph_event(match, local_node); free(local_node); } else { pcmk__warn("Ignoring malformed CIB update (%s at %s has unrecognized " "result %s)", op, xpath, name); } return pcmk_rc_ok; } void te_update_diff(const char *event, xmlNode * msg) { xmlNode *wrapper = NULL; xmlNode *diff = NULL; const char *op = NULL; int rc = -EINVAL; int format = 1; int p_add[] = { 0, 0, 0 }; int p_del[] = { 0, 0, 0 }; CRM_CHECK(msg != NULL, return); pcmk__xe_get_int(msg, PCMK__XA_CIB_RC, &rc); if (controld_globals.transition_graph == NULL) { pcmk__trace("No graph"); return; } else if (rc < pcmk_ok) { pcmk__trace("Filter rc=%d (%s)", rc, pcmk_strerror(rc)); return; } else if (controld_globals.transition_graph->complete && (controld_globals.fsa_state != S_IDLE) && (controld_globals.fsa_state != S_TRANSITION_ENGINE) && (controld_globals.fsa_state != S_POLICY_ENGINE)) { pcmk__trace("Filter state=%s (complete)", fsa_state2string(controld_globals.fsa_state)); return; } op = pcmk__xe_get(msg, PCMK__XA_CIB_OP); wrapper = pcmk__xe_first_child(msg, PCMK__XE_CIB_UPDATE_RESULT, NULL, NULL); diff = pcmk__xe_first_child(wrapper, NULL, NULL, NULL); pcmk__xml_patchset_versions(diff, p_del, p_add); pcmk__debug("Processing (%s) diff: %d.%d.%d -> %d.%d.%d (%s)", op, p_del[0], p_del[1], p_del[2], p_add[0], p_add[1], p_add[2], fsa_state2string(controld_globals.fsa_state)); pcmk__xe_get_int(diff, PCMK_XA_FORMAT, &format); if (format == 2) { - crm_log_xml_trace(diff, "patch"); + pcmk__log_xml_trace(diff, "patch"); pcmk__xe_foreach_child(diff, NULL, te_update_diff_element, NULL); } else { pcmk__warn("Ignoring malformed CIB update (unknown patch format %d)", format); } controld_remove_all_outside_events(); } void process_te_message(xmlNode * msg, xmlNode * xml_data) { const char *value = NULL; xmlXPathObject *xpathObj = NULL; int nmatches = 0; CRM_CHECK(msg != NULL, return); // Transition requests must specify transition engine as subsystem value = pcmk__xe_get(msg, PCMK__XA_CRM_SYS_TO); if (pcmk__str_empty(value) || !pcmk__str_eq(value, CRM_SYSTEM_TENGINE, pcmk__str_none)) { pcmk__info("Received invalid transition request: subsystem '%s' not '" CRM_SYSTEM_TENGINE "'", pcmk__s(value, "")); return; } // Only the lrm_invoke command is supported as a transition request value = pcmk__xe_get(msg, PCMK__XA_CRM_TASK); if (!pcmk__str_eq(value, CRM_OP_INVOKE_LRM, pcmk__str_none)) { pcmk__info("Received invalid transition request: command '%s' not '" CRM_OP_INVOKE_LRM "'", pcmk__s(value, "")); return; } // Transition requests must be marked as coming from the executor value = pcmk__xe_get(msg, PCMK__XA_CRM_SYS_FROM); if (!pcmk__str_eq(value, CRM_SYSTEM_LRMD, pcmk__str_none)) { pcmk__info("Received invalid transition request: from '%s' not '" CRM_SYSTEM_LRMD "'", pcmk__s(value, "")); return; } pcmk__debug("Processing transition request with ref='%s' origin='%s'", pcmk__s(pcmk__xe_get(msg, PCMK_XA_REFERENCE), ""), pcmk__s(pcmk__xe_get(msg, PCMK__XA_SRC), "")); xpathObj = pcmk__xpath_search(xml_data->doc, "//" PCMK__XE_LRM_RSC_OP); nmatches = pcmk__xpath_num_results(xpathObj); if (nmatches == 0) { pcmk__err("Received transition request with no results (bug?)"); } else { for (int lpc = 0; lpc < nmatches; lpc++) { xmlNode *rsc_op = pcmk__xpath_result(xpathObj, lpc); if (rsc_op != NULL) { const char *node = get_node_id(rsc_op); process_graph_event(rsc_op, node); } } } xmlXPathFreeObject(xpathObj); } void cib_action_updated(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data) { if (rc < pcmk_ok) { pcmk__err("Update %d FAILED: %s", call_id, pcmk_strerror(rc)); } } /*! * \brief Handle a timeout in node-to-node communication * * \param[in,out] data Pointer to graph action * * \return FALSE (indicating that source should be not be re-added) */ gboolean action_timer_callback(gpointer data) { pcmk__graph_action_t *action = (pcmk__graph_action_t *) data; const char *task = NULL; const char *on_node = NULL; const char *via_node = NULL; CRM_CHECK(data != NULL, return FALSE); stop_te_timer(action); task = pcmk__xe_get(action->xml, PCMK_XA_OPERATION); on_node = pcmk__xe_get(action->xml, PCMK__META_ON_NODE); via_node = pcmk__xe_get(action->xml, PCMK__XA_ROUTER_NODE); if (controld_globals.transition_graph->complete) { pcmk__notice("Node %s did not send %s result (via %s) within %dms " "(ignoring because transition not in progress)", pcmk__s(on_node, ""), pcmk__s(task, "unknown action"), pcmk__s(via_node, "controller"), action->timeout); } else { /* fail the action */ pcmk__err("Node %s did not send %s result (via %s) within %dms " "(action timeout plus " PCMK_OPT_CLUSTER_DELAY ")", pcmk__s(on_node, ""), pcmk__s(task, "unknown action"), pcmk__s(via_node, "controller"), (action->timeout + controld_globals.transition_graph->network_delay)); pcmk__log_graph_action(LOG_ERR, action); pcmk__set_graph_action_flags(action, pcmk__graph_action_failed); te_action_confirmed(action, controld_globals.transition_graph); abort_transition(PCMK_SCORE_INFINITY, pcmk__graph_restart, "Action lost", NULL); // Record timeout in the CIB if appropriate if ((action->type == pcmk__rsc_graph_action) && controld_action_is_recordable(task)) { controld_record_action_timeout(action); } } return FALSE; } diff --git a/daemons/fenced/fenced_commands.c b/daemons/fenced/fenced_commands.c index 1ae2b48fdc..be5eade10a 100644 --- a/daemons/fenced/fenced_commands.c +++ b/daemons/fenced/fenced_commands.c @@ -1,3702 +1,3702 @@ /* * Copyright 2009-2025 the Pacemaker project contributors * * The version control history for this file may have further details. * * This source code is licensed under the GNU General Public License version 2 * or later (GPLv2+) WITHOUT ANY WARRANTY. */ #include #include #include #include #include #include #include #include #include #include #include #include #include // xmlNode #include // xmlXPathObject, etc. #include #include #include #include #include #include // hash2field(), xml2list() #include #include #include #include GHashTable *device_list = NULL; GHashTable *topology = NULL; static GList *cmd_list = NULL; static GHashTable *fenced_handlers = NULL; struct device_search_s { /* target of fence action */ char *host; /* requested fence action */ char *action; /* timeout to use if a device is queried dynamically for possible targets */ // @TODO This name is misleading now, it's the value of stonith-timeout int per_device_timeout; /* number of registered fencing devices at time of request */ int replies_needed; /* number of device replies received so far */ int replies_received; /* whether the target is eligible to perform requested action (or off) */ bool allow_self; /* private data to pass to search callback function */ void *user_data; /* function to call when all replies have been received */ void (*callback) (GList * devices, void *user_data); /* devices capable of performing requested action (or off if remapping) */ GList *capable; /* Whether to perform searches that support the action */ uint32_t support_action_only; }; static gboolean stonith_device_dispatch(gpointer user_data); static void st_child_done(int pid, const pcmk__action_result_t *result, void *user_data); static void search_devices_record_result(struct device_search_s *search, const char *device, gboolean can_fence); static int get_agent_metadata(const char *agent, xmlNode **metadata); static void read_action_metadata(stonith_device_t *device); static enum fenced_target_by unpack_level_kind(const xmlNode *level); typedef struct async_command_s { int id; int pid; int fd_stdout; uint32_t options; int default_timeout; /* seconds */ int timeout; /* seconds */ int start_delay; // seconds (-1 means disable static/random fencing delays) int delay_id; char *op; char *origin; char *client; char *client_name; char *remote_op_id; char *target; uint32_t target_nodeid; char *action; char *device; GList *device_list; GList *next_device_iter; // device_list entry for next device to execute void *internal_user_data; void (*done_cb) (int pid, const pcmk__action_result_t *result, void *user_data); guint timer_sigterm; guint timer_sigkill; /*! If the operation timed out, this is the last signal * we sent to the process to get it to terminate */ int last_timeout_signo; stonith_device_t *active_on; stonith_device_t *activating_on; } async_command_t; static xmlNode *construct_async_reply(const async_command_t *cmd, const pcmk__action_result_t *result); static gboolean is_action_required(const char *action, const stonith_device_t *device) { return (device != NULL) && device->automatic_unfencing && pcmk__str_eq(action, PCMK_ACTION_ON, pcmk__str_none); } static int get_action_delay_max(const stonith_device_t *device, const char *action) { const char *value = NULL; guint delay_max = 0U; if (!pcmk__is_fencing_action(action)) { return 0; } value = g_hash_table_lookup(device->params, PCMK_STONITH_DELAY_MAX); if (value) { pcmk_parse_interval_spec(value, &delay_max); delay_max /= 1000; } return (int) delay_max; } static int get_action_delay_base(const stonith_device_t *device, const char *action, const char *target) { char *hash_value = NULL; guint delay_base = 0U; if (!pcmk__is_fencing_action(action)) { return 0; } hash_value = g_hash_table_lookup(device->params, PCMK_STONITH_DELAY_BASE); if (hash_value) { char *value = pcmk__str_copy(hash_value); char *valptr = value; if (target != NULL) { for (char *val = strtok(value, "; \t"); val != NULL; val = strtok(NULL, "; \t")) { char *mapval = strchr(val, ':'); if (mapval == NULL || mapval[1] == 0) { pcmk__err("pcmk_delay_base: empty value in mapping", val); continue; } if (mapval != val && strncasecmp(target, val, (size_t)(mapval - val)) == 0) { value = mapval + 1; pcmk__debug("pcmk_delay_base mapped to %s for %s", value, target); break; } } } if (strchr(value, ':') == 0) { pcmk_parse_interval_spec(value, &delay_base); delay_base /= 1000; } free(valptr); } return (int) delay_base; } /*! * \internal * \brief Override STONITH timeout with pcmk_*_timeout if available * * \param[in] device STONITH device to use * \param[in] action STONITH action name * \param[in] default_timeout Timeout to use if device does not have * a pcmk_*_timeout parameter for action * * \return Value of pcmk_(action)_timeout if available, otherwise default_timeout * \note For consistency, it would be nice if reboot/off/on timeouts could be * set the same way as start/stop/monitor timeouts, i.e. with an * entry in the fencing resource configuration. However that * is insufficient because fencing devices may be registered directly via * the fencer's register_device() API instead of going through the CIB * (e.g. stonith_admin uses it for its -R option, and the executor uses it * to ensure a device is registered when a command is issued). As device * properties, pcmk_*_timeout parameters can be grabbed by the fencer when * the device is registered, whether by CIB change or API call. */ static int get_action_timeout(const stonith_device_t *device, const char *action, int default_timeout) { if (action && device && device->params) { char buffer[64] = { 0, }; const char *value = NULL; /* If "reboot" was requested but the device does not support it, * we will remap to "off", so check timeout for "off" instead */ if (pcmk__str_eq(action, PCMK_ACTION_REBOOT, pcmk__str_none) && !pcmk__is_set(device->flags, st_device_supports_reboot)) { pcmk__trace("%s doesn't support reboot, using timeout for off " "instead", device->id); action = PCMK_ACTION_OFF; } /* If the device config specified an action-specific timeout, use it */ snprintf(buffer, sizeof(buffer), "pcmk_%s_timeout", action); value = g_hash_table_lookup(device->params, buffer); if (value) { long long timeout_ms = 0; if ((pcmk__parse_ms(value, &timeout_ms) == pcmk_rc_ok) && (timeout_ms >= 0)) { int timeout_sec = 0; timeout_ms = QB_MIN(timeout_ms, UINT_MAX); timeout_sec = pcmk__timeout_ms2s((guint) timeout_ms); return QB_MIN(timeout_sec, INT_MAX); } } } return default_timeout; } /*! * \internal * \brief Get the currently executing device for a fencing operation * * \param[in] cmd Fencing operation to check * * \return Currently executing device for \p cmd if any, otherwise NULL */ static stonith_device_t * cmd_device(const async_command_t *cmd) { if ((cmd == NULL) || (cmd->device == NULL) || (device_list == NULL)) { return NULL; } return g_hash_table_lookup(device_list, cmd->device); } /*! * \internal * \brief Return the configured reboot action for a given device * * \param[in] device_id Device ID * * \return Configured reboot action for \p device_id */ const char * fenced_device_reboot_action(const char *device_id) { const char *action = NULL; if ((device_list != NULL) && (device_id != NULL)) { stonith_device_t *device = g_hash_table_lookup(device_list, device_id); if ((device != NULL) && (device->params != NULL)) { action = g_hash_table_lookup(device->params, "pcmk_reboot_action"); } } return pcmk__s(action, PCMK_ACTION_REBOOT); } /*! * \internal * \brief Check whether a given device supports the "on" action * * \param[in] device_id Device ID * * \return true if \p device_id supports "on", otherwise false */ bool fenced_device_supports_on(const char *device_id) { if ((device_list != NULL) && (device_id != NULL)) { stonith_device_t *device = g_hash_table_lookup(device_list, device_id); if (device != NULL) { return pcmk__is_set(device->flags, st_device_supports_on); } } return false; } static void free_async_command(async_command_t * cmd) { if (!cmd) { return; } if (cmd->delay_id) { g_source_remove(cmd->delay_id); } cmd_list = g_list_remove(cmd_list, cmd); g_list_free_full(cmd->device_list, free); free(cmd->device); free(cmd->action); free(cmd->target); free(cmd->remote_op_id); free(cmd->client); free(cmd->client_name); free(cmd->origin); free(cmd->op); free(cmd); } /*! * \internal * \brief Create a new asynchronous fencing operation from request XML * * \param[in] msg Fencing request XML (from IPC or CPG) * * \return Newly allocated fencing operation on success, otherwise NULL * * \note This asserts on memory errors, so a NULL return indicates an * unparseable message. */ static async_command_t * create_async_command(xmlNode *msg) { xmlNode *op = NULL; async_command_t *cmd = NULL; int rc = pcmk_rc_ok; if (msg == NULL) { return NULL; } op = pcmk__xpath_find_one(msg->doc, "//*[@" PCMK__XA_ST_DEVICE_ACTION "]", LOG_ERR); if (op == NULL) { return NULL; } cmd = pcmk__assert_alloc(1, sizeof(async_command_t)); // All messages must include these cmd->action = pcmk__xe_get_copy(op, PCMK__XA_ST_DEVICE_ACTION); cmd->op = pcmk__xe_get_copy(msg, PCMK__XA_ST_OP); cmd->client = pcmk__xe_get_copy(msg, PCMK__XA_ST_CLIENTID); if ((cmd->action == NULL) || (cmd->op == NULL) || (cmd->client == NULL)) { free_async_command(cmd); return NULL; } pcmk__xe_get_int(msg, PCMK__XA_ST_CALLID, &(cmd->id)); pcmk__xe_get_int(msg, PCMK__XA_ST_DELAY, &(cmd->start_delay)); pcmk__xe_get_int(msg, PCMK__XA_ST_TIMEOUT, &(cmd->default_timeout)); cmd->timeout = cmd->default_timeout; rc = pcmk__xe_get_flags(msg, PCMK__XA_ST_CALLOPT, &(cmd->options), st_opt_none); if (rc != pcmk_rc_ok) { pcmk__warn("Couldn't parse options from request: %s", pcmk_rc_str(rc)); } cmd->origin = pcmk__xe_get_copy(msg, PCMK__XA_SRC); cmd->remote_op_id = pcmk__xe_get_copy(msg, PCMK__XA_ST_REMOTE_OP); cmd->client_name = pcmk__xe_get_copy(msg, PCMK__XA_ST_CLIENTNAME); cmd->target = pcmk__xe_get_copy(op, PCMK__XA_ST_TARGET); cmd->device = pcmk__xe_get_copy(op, PCMK__XA_ST_DEVICE_ID); cmd->done_cb = st_child_done; // Track in global command list cmd_list = g_list_append(cmd_list, cmd); return cmd; } static int get_action_limit(stonith_device_t * device) { const char *value = NULL; int action_limit = 1; value = g_hash_table_lookup(device->params, PCMK_STONITH_ACTION_LIMIT); if ((value == NULL) || (pcmk__scan_min_int(value, &action_limit, INT_MIN) != pcmk_rc_ok) || (action_limit == 0)) { action_limit = 1; } return action_limit; } static int get_active_cmds(stonith_device_t * device) { int counter = 0; GList *gIter = NULL; GList *gIterNext = NULL; CRM_CHECK(device != NULL, return 0); for (gIter = cmd_list; gIter != NULL; gIter = gIterNext) { async_command_t *cmd = gIter->data; gIterNext = gIter->next; if (cmd->active_on == device) { counter++; } } return counter; } static void fork_cb(int pid, void *user_data) { async_command_t *cmd = (async_command_t *) user_data; stonith_device_t * device = /* in case of a retry we've done the move from activating_on to active_on already */ cmd->activating_on?cmd->activating_on:cmd->active_on; pcmk__assert(device != NULL); pcmk__debug("Operation '%s' [%d]%s%s using %s now running with %ds timeout", cmd->action, pid, ((cmd->target != NULL)? " targeting " : ""), pcmk__s(cmd->target, ""), device->id, cmd->timeout); cmd->active_on = device; cmd->activating_on = NULL; } static int get_agent_metadata_cb(gpointer data) { stonith_device_t *device = data; guint period_ms; switch (get_agent_metadata(device->agent, &device->agent_metadata)) { case pcmk_rc_ok: if (device->agent_metadata) { read_action_metadata(device); stonith__device_parameter_flags(&(device->flags), device->id, device->agent_metadata); } return G_SOURCE_REMOVE; case EAGAIN: period_ms = pcmk__mainloop_timer_get_period(device->timer); if (period_ms < 160 * 1000) { mainloop_timer_set_period(device->timer, 2 * period_ms); } return G_SOURCE_CONTINUE; default: return G_SOURCE_REMOVE; } } /*! * \internal * \brief Call a command's action callback for an internal (not library) result * * \param[in,out] cmd Command to report result for * \param[in] execution_status Execution status to use for result * \param[in] exit_status Exit status to use for result * \param[in] exit_reason Exit reason to use for result */ static void report_internal_result(async_command_t *cmd, int exit_status, int execution_status, const char *exit_reason) { pcmk__action_result_t result = PCMK__UNKNOWN_RESULT; pcmk__set_result(&result, exit_status, execution_status, exit_reason); cmd->done_cb(0, &result, cmd); pcmk__reset_result(&result); } static gboolean stonith_device_execute(stonith_device_t * device) { int exec_rc = 0; const char *action_str = NULL; const char *host_arg = NULL; async_command_t *cmd = NULL; stonith_action_t *action = NULL; int active_cmds = 0; int action_limit = 0; GList *gIter = NULL; GList *gIterNext = NULL; CRM_CHECK(device != NULL, return FALSE); active_cmds = get_active_cmds(device); action_limit = get_action_limit(device); if (action_limit > -1 && active_cmds >= action_limit) { pcmk__trace("%s is over its action limit of %d (%u active action%s)", device->id, action_limit, active_cmds, pcmk__plural_s(active_cmds)); return TRUE; } for (gIter = device->pending_ops; gIter != NULL; gIter = gIterNext) { async_command_t *pending_op = gIter->data; gIterNext = gIter->next; if (pending_op && pending_op->delay_id) { pcmk__trace("Operation '%s'%s%s using %s was asked to run too " "early, waiting for start delay of %ds", pending_op->action, ((pending_op->target != NULL)? " targeting ": ""), pcmk__s(pending_op->target, ""), device->id, pending_op->start_delay); continue; } device->pending_ops = g_list_remove_link(device->pending_ops, gIter); g_list_free_1(gIter); cmd = pending_op; break; } if (cmd == NULL) { pcmk__trace("No actions using %s are needed", device->id); return TRUE; } if (pcmk__str_any_of(device->agent, STONITH_WATCHDOG_AGENT, STONITH_WATCHDOG_AGENT_INTERNAL, NULL)) { if (pcmk__is_fencing_action(cmd->action)) { if (node_does_watchdog_fencing(fenced_get_local_node())) { pcmk__panic("Watchdog self-fencing required"); goto done; } } else { pcmk__info("Faking success for %s watchdog operation", cmd->action); report_internal_result(cmd, CRM_EX_OK, PCMK_EXEC_DONE, NULL); goto done; } } #if PCMK__ENABLE_CIBSECRETS exec_rc = pcmk__substitute_secrets(device->id, device->params); if (exec_rc != pcmk_rc_ok) { if (pcmk__str_eq(cmd->action, PCMK_ACTION_STOP, pcmk__str_none)) { pcmk__info("Proceeding with stop operation for %s despite being " "unable to load CIB secrets (%s)", device->id, pcmk_rc_str(exec_rc)); } else { pcmk__err("Considering %s unconfigured because unable to load CIB " "secrets: %s", device->id, pcmk_rc_str(exec_rc)); report_internal_result(cmd, CRM_EX_ERROR, PCMK_EXEC_NO_SECRETS, "Failed to get CIB secrets"); goto done; } } #endif action_str = cmd->action; if (pcmk__str_eq(cmd->action, PCMK_ACTION_REBOOT, pcmk__str_none) && !pcmk__is_set(device->flags, st_device_supports_reboot)) { pcmk__notice("Remapping 'reboot' action%s%s using %s to 'off' because " "agent '%s' does not support reboot", ((cmd->target == NULL)? "" : " targeting "), pcmk__s(cmd->target, ""), device->id, device->agent); action_str = PCMK_ACTION_OFF; } if (pcmk__is_set(device->flags, st_device_supports_parameter_port)) { host_arg = "port"; } else if (pcmk__is_set(device->flags, st_device_supports_parameter_plug)) { host_arg = "plug"; } action = stonith__action_create(device->agent, action_str, cmd->target, cmd->target_nodeid, cmd->timeout, device->params, device->aliases, host_arg); /* for async exec, exec_rc is negative for early error exit otherwise handling of success/errors is done via callbacks */ cmd->activating_on = device; exec_rc = stonith__execute_async(action, (void *)cmd, cmd->done_cb, fork_cb); if (exec_rc < 0) { cmd->activating_on = NULL; cmd->done_cb(0, stonith__action_result(action), cmd); stonith__destroy_action(action); } done: /* Device might get triggered to work by multiple fencing commands * simultaneously. Trigger the device again to make sure any * remaining concurrent commands get executed. */ if (device->pending_ops) { mainloop_set_trigger(device->work); } return TRUE; } static gboolean stonith_device_dispatch(gpointer user_data) { return stonith_device_execute(user_data); } static gboolean start_delay_helper(gpointer data) { async_command_t *cmd = data; stonith_device_t *device = cmd_device(cmd); cmd->delay_id = 0; if (device) { mainloop_set_trigger(device->work); } return FALSE; } static void schedule_stonith_command(async_command_t * cmd, stonith_device_t * device) { int delay_max = 0; int delay_base = 0; int requested_delay = cmd->start_delay; CRM_CHECK(cmd != NULL, return); CRM_CHECK(device != NULL, return); if (cmd->device) { free(cmd->device); } if (device->include_nodeid && (cmd->target != NULL)) { pcmk__node_status_t *node = pcmk__get_node(0, cmd->target, NULL, pcmk__node_search_cluster_member); cmd->target_nodeid = node->cluster_layer_id; } cmd->device = pcmk__str_copy(device->id); cmd->timeout = get_action_timeout(device, cmd->action, cmd->default_timeout); if (cmd->remote_op_id) { pcmk__debug("Scheduling '%s' action%s%s using %s for remote peer %s " "with op id %.8s and timeout %ds", cmd->action, ((cmd->target != NULL)? " targeting " : ""), pcmk__s(cmd->target, ""), device->id, cmd->origin, cmd->remote_op_id, cmd->timeout); } else { pcmk__debug("Scheduling '%s' action%s%s using %s for %s with timeout " "%ds", cmd->action, ((cmd->target != NULL)? " targeting " : ""), pcmk__s(cmd->target, ""), device->id, cmd->client, cmd->timeout); } device->pending_ops = g_list_append(device->pending_ops, cmd); mainloop_set_trigger(device->work); // Value -1 means disable any static/random fencing delays if (requested_delay < 0) { return; } delay_max = get_action_delay_max(device, cmd->action); delay_base = get_action_delay_base(device, cmd->action, cmd->target); if (delay_max == 0) { delay_max = delay_base; } if (delay_max < delay_base) { pcmk__warn(PCMK_STONITH_DELAY_BASE " (%ds) is larger than " PCMK_STONITH_DELAY_MAX " (%ds) for %s using %s (limiting to " "maximum delay)", delay_base, delay_max, cmd->action, device->id); delay_base = delay_max; } if (delay_max > 0) { // coverity[dontcall] It doesn't matter here if rand() is predictable cmd->start_delay += ((delay_max != delay_base)?(rand() % (delay_max - delay_base)):0) + delay_base; } if (cmd->start_delay > 0) { pcmk__notice("Delaying '%s' action%s%s using %s for %ds " QB_XS " timeout=%ds requested_delay=%ds base=%ds max=%ds", cmd->action, (cmd->target == NULL)? "" : " targeting ", pcmk__s(cmd->target, ""), device->id, cmd->start_delay, cmd->timeout, requested_delay, delay_base, delay_max); cmd->delay_id = pcmk__create_timer(cmd->start_delay * 1000, start_delay_helper, cmd); } } static void free_device(gpointer data) { GList *gIter = NULL; stonith_device_t *device = data; g_hash_table_destroy(device->params); g_hash_table_destroy(device->aliases); for (gIter = device->pending_ops; gIter != NULL; gIter = gIter->next) { async_command_t *cmd = gIter->data; pcmk__warn("Removal of device '%s' purged operation '%s'", device->id, cmd->action); report_internal_result(cmd, CRM_EX_ERROR, PCMK_EXEC_NO_FENCE_DEVICE, "Device was removed before action could be executed"); } g_list_free(device->pending_ops); g_list_free_full(device->targets, free); if (device->timer) { mainloop_timer_stop(device->timer); mainloop_timer_del(device->timer); } mainloop_destroy_trigger(device->work); pcmk__xml_free(device->agent_metadata); free(device->namespace); if (device->on_target_actions != NULL) { g_string_free(device->on_target_actions, TRUE); } free(device->agent); free(device->id); free(device); } void free_device_list(void) { if (device_list != NULL) { g_hash_table_destroy(device_list); device_list = NULL; } } void init_device_list(void) { if (device_list == NULL) { device_list = pcmk__strkey_table(NULL, free_device); } } static GHashTable * build_port_aliases(const char *hostmap, GList ** targets) { char *name = NULL; int last = 0, lpc = 0, max = 0, added = 0; GHashTable *aliases = pcmk__strikey_table(free, free); if (hostmap == NULL) { return aliases; } max = strlen(hostmap); for (; lpc <= max; lpc++) { switch (hostmap[lpc]) { /* Skip escaped chars */ case '\\': lpc++; break; /* Assignment chars */ case '=': case ':': if (lpc > last) { free(name); name = pcmk__assert_alloc(1, 1 + lpc - last); memcpy(name, hostmap + last, lpc - last); } last = lpc + 1; break; /* Delimeter chars */ /* case ',': Potentially used to specify multiple ports */ case 0: case ';': case ' ': case '\t': if (name) { char *value = NULL; int k = 0; value = pcmk__assert_alloc(1, 1 + lpc - last); memcpy(value, hostmap + last, lpc - last); for (int i = 0; value[i] != '\0'; i++) { if (value[i] != '\\') { value[k++] = value[i]; } } value[k] = '\0'; pcmk__debug("Adding alias '%s'='%s'", name, value); g_hash_table_replace(aliases, name, value); if (targets) { *targets = g_list_append(*targets, pcmk__str_copy(value)); } value = NULL; name = NULL; added++; } else if (lpc > last) { pcmk__debug("Parse error at offset %d near '%s'", (lpc - last), (hostmap + last)); } last = lpc + 1; break; } if (hostmap[lpc] == 0) { break; } } if (added == 0) { pcmk__info("No host mappings detected in '%s'", hostmap); } free(name); return aliases; } GHashTable *metadata_cache = NULL; void free_metadata_cache(void) { if (metadata_cache != NULL) { g_hash_table_destroy(metadata_cache); metadata_cache = NULL; } } static void init_metadata_cache(void) { if (metadata_cache == NULL) { metadata_cache = pcmk__strkey_table(free, free); } } int get_agent_metadata(const char *agent, xmlNode ** metadata) { char *buffer = NULL; if (metadata == NULL) { return EINVAL; } *metadata = NULL; if (pcmk__str_eq(agent, STONITH_WATCHDOG_AGENT_INTERNAL, pcmk__str_none)) { return pcmk_rc_ok; } init_metadata_cache(); buffer = g_hash_table_lookup(metadata_cache, agent); if (buffer == NULL) { stonith_t *st = stonith_api_new(); int rc; if (st == NULL) { pcmk__warn("Could not get agent meta-data: API memory allocation " "failed"); return EAGAIN; } rc = st->cmds->metadata(st, st_opt_sync_call, agent, NULL, &buffer, 10); stonith_api_delete(st); if (rc || !buffer) { pcmk__err("Could not retrieve metadata for fencing agent %s", agent); return EAGAIN; } g_hash_table_replace(metadata_cache, pcmk__str_copy(agent), buffer); } *metadata = pcmk__xml_parse(buffer); return pcmk_rc_ok; } static gboolean is_nodeid_required(xmlNode * xml) { xmlXPathObject *xpath = NULL; if (!xml) { return FALSE; } xpath = pcmk__xpath_search(xml->doc, "//" PCMK_XE_PARAMETER "[@" PCMK_XA_NAME "='nodeid']"); if (pcmk__xpath_num_results(xpath) == 0) { xmlXPathFreeObject(xpath); return FALSE; } xmlXPathFreeObject(xpath); return TRUE; } static void read_action_metadata(stonith_device_t *device) { xmlXPathObject *xpath = NULL; int max = 0; int lpc = 0; if (device->agent_metadata == NULL) { return; } xpath = pcmk__xpath_search(device->agent_metadata->doc, "//" PCMK_XE_ACTION); max = pcmk__xpath_num_results(xpath); if (max == 0) { xmlXPathFreeObject(xpath); return; } for (lpc = 0; lpc < max; lpc++) { const char *action = NULL; xmlNode *match = pcmk__xpath_result(xpath, lpc); CRM_LOG_ASSERT(match != NULL); if(match == NULL) { continue; }; action = pcmk__xe_get(match, PCMK_XA_NAME); if (pcmk__str_eq(action, PCMK_ACTION_LIST, pcmk__str_none)) { stonith__set_device_flags(device->flags, device->id, st_device_supports_list); } else if (pcmk__str_eq(action, PCMK_ACTION_STATUS, pcmk__str_none)) { stonith__set_device_flags(device->flags, device->id, st_device_supports_status); } else if (pcmk__str_eq(action, PCMK_ACTION_REBOOT, pcmk__str_none)) { stonith__set_device_flags(device->flags, device->id, st_device_supports_reboot); } else if (pcmk__str_eq(action, PCMK_ACTION_ON, pcmk__str_none)) { /* PCMK_XA_AUTOMATIC means the cluster will unfence a node when it * joins. * * @COMPAT PCMK__XA_REQUIRED is a deprecated synonym for * PCMK_XA_AUTOMATIC. */ if (pcmk__xe_attr_is_true(match, PCMK_XA_AUTOMATIC) || pcmk__xe_attr_is_true(match, PCMK__XA_REQUIRED)) { device->automatic_unfencing = TRUE; } stonith__set_device_flags(device->flags, device->id, st_device_supports_on); } if ((action != NULL) && pcmk__xe_attr_is_true(match, PCMK_XA_ON_TARGET)) { pcmk__add_word(&(device->on_target_actions), 64, action); } } xmlXPathFreeObject(xpath); } static const char * target_list_type(stonith_device_t * dev) { const char *check_type = NULL; check_type = g_hash_table_lookup(dev->params, PCMK_STONITH_HOST_CHECK); if (check_type == NULL) { if (g_hash_table_lookup(dev->params, PCMK_STONITH_HOST_LIST)) { check_type = PCMK_VALUE_STATIC_LIST; } else if (g_hash_table_lookup(dev->params, PCMK_STONITH_HOST_MAP)) { check_type = PCMK_VALUE_STATIC_LIST; } else if (pcmk__is_set(dev->flags, st_device_supports_list)) { check_type = PCMK_VALUE_DYNAMIC_LIST; } else if (pcmk__is_set(dev->flags, st_device_supports_status)) { check_type = PCMK_VALUE_STATUS; } else { check_type = PCMK_VALUE_NONE; } } return check_type; } static stonith_device_t * build_device_from_xml(xmlNode *dev) { const char *value; stonith_device_t *device = NULL; char *agent = pcmk__xe_get_copy(dev, PCMK_XA_AGENT); CRM_CHECK(agent != NULL, return device); device = pcmk__assert_alloc(1, sizeof(stonith_device_t)); device->id = pcmk__xe_get_copy(dev, PCMK_XA_ID); device->agent = agent; device->namespace = pcmk__xe_get_copy(dev, PCMK__XA_NAMESPACE); device->params = xml2list(dev); value = g_hash_table_lookup(device->params, PCMK_STONITH_HOST_LIST); if (value) { device->targets = stonith__parse_targets(value); } value = g_hash_table_lookup(device->params, PCMK_STONITH_HOST_MAP); device->aliases = build_port_aliases(value, &(device->targets)); value = target_list_type(device); if (!pcmk__str_eq(value, PCMK_VALUE_STATIC_LIST, pcmk__str_casei) && (device->targets != NULL)) { // device->targets is necessary only with PCMK_VALUE_STATIC_LIST g_list_free_full(device->targets, free); device->targets = NULL; } switch (get_agent_metadata(device->agent, &device->agent_metadata)) { case pcmk_rc_ok: if (device->agent_metadata) { read_action_metadata(device); stonith__device_parameter_flags(&(device->flags), device->id, device->agent_metadata); } break; case EAGAIN: if (device->timer == NULL) { device->timer = mainloop_timer_add("get_agent_metadata", 10 * 1000, TRUE, get_agent_metadata_cb, device); } if (!mainloop_timer_running(device->timer)) { mainloop_timer_start(device->timer); } break; default: break; } value = g_hash_table_lookup(device->params, "nodeid"); if (!value) { device->include_nodeid = is_nodeid_required(device->agent_metadata); } value = pcmk__xe_get(dev, PCMK__XA_RSC_PROVIDES); if (pcmk__str_eq(value, PCMK_VALUE_UNFENCING, pcmk__str_casei)) { device->automatic_unfencing = TRUE; } if (is_action_required(PCMK_ACTION_ON, device)) { pcmk__info("Fencing device '%s' requires unfencing", device->id); } if (device->on_target_actions != NULL) { pcmk__info("Fencing device '%s' requires actions (%s) to be executed " "on target", device->id, (const char *) device->on_target_actions->str); } device->work = mainloop_add_trigger(G_PRIORITY_HIGH, stonith_device_dispatch, device); /* TODO: Hook up priority */ return device; } static void schedule_internal_command(const char *origin, stonith_device_t * device, const char *action, const char *target, int timeout, void *internal_user_data, void (*done_cb) (int pid, const pcmk__action_result_t *result, void *user_data)) { async_command_t *cmd = NULL; cmd = pcmk__assert_alloc(1, sizeof(async_command_t)); cmd->id = -1; cmd->default_timeout = timeout ? timeout : 60; cmd->timeout = cmd->default_timeout; cmd->action = pcmk__str_copy(action); cmd->target = pcmk__str_copy(target); cmd->device = pcmk__str_copy(device->id); cmd->origin = pcmk__str_copy(origin); cmd->client = pcmk__str_copy(crm_system_name); cmd->client_name = pcmk__str_copy(crm_system_name); cmd->internal_user_data = internal_user_data; cmd->done_cb = done_cb; /* cmd, not internal_user_data, is passed to 'done_cb' as the userdata */ schedule_stonith_command(cmd, device); } // Fence agent status commands use custom exit status codes enum fence_status_code { fence_status_invalid = -1, fence_status_active = 0, fence_status_unknown = 1, fence_status_inactive = 2, }; static void status_search_cb(int pid, const pcmk__action_result_t *result, void *user_data) { async_command_t *cmd = user_data; struct device_search_s *search = cmd->internal_user_data; stonith_device_t *dev = cmd_device(cmd); gboolean can = FALSE; free_async_command(cmd); if (!dev) { search_devices_record_result(search, NULL, FALSE); return; } mainloop_set_trigger(dev->work); if (result->execution_status != PCMK_EXEC_DONE) { const char *reason = result->exit_reason; pcmk__warn("Assuming %s cannot fence %s because status could not be " "executed: %s%s%s%s", dev->id, search->host, pcmk_exec_status_str(result->execution_status), ((reason != NULL)? " (" : ""), pcmk__s(reason, ""), ((reason != NULL)? ")" : "")); search_devices_record_result(search, dev->id, FALSE); return; } switch (result->exit_status) { case fence_status_unknown: pcmk__trace("%s reported it cannot fence %s", dev->id, search->host); break; case fence_status_active: case fence_status_inactive: pcmk__trace("%s reported it can fence %s", dev->id, search->host); can = TRUE; break; default: pcmk__warn("Assuming %s cannot fence %s (status returned unknown " "code %d)", dev->id, search->host, result->exit_status); break; } search_devices_record_result(search, dev->id, can); } static void dynamic_list_search_cb(int pid, const pcmk__action_result_t *result, void *user_data) { async_command_t *cmd = user_data; struct device_search_s *search = cmd->internal_user_data; stonith_device_t *dev = cmd_device(cmd); gboolean can_fence = FALSE; free_async_command(cmd); /* Host/alias must be in the list output to be eligible to be fenced * * Will cause problems if down'd nodes aren't listed or (for virtual nodes) * if the guest is still listed despite being moved to another machine */ if (!dev) { search_devices_record_result(search, NULL, FALSE); return; } mainloop_set_trigger(dev->work); if (pcmk__result_ok(result)) { pcmk__info("Refreshing target list for %s", dev->id); g_list_free_full(dev->targets, free); dev->targets = stonith__parse_targets(result->action_stdout); dev->targets_age = time(NULL); } else if (dev->targets != NULL) { if (result->execution_status == PCMK_EXEC_DONE) { pcmk__info("Reusing most recent target list for %s because list " "returned error code %d", dev->id, result->exit_status); } else { const char *reason = result->exit_reason; pcmk__info("Reusing most recent target list for %s because list " "could not be executed: %s%s%s%s", dev->id, pcmk_exec_status_str(result->execution_status), ((reason != NULL)? " (" : ""), pcmk__s(reason, ""), ((reason != NULL)? ")" : "")); } } else { // We have never successfully executed list if (result->execution_status == PCMK_EXEC_DONE) { pcmk__warn("Assuming %s cannot fence %s because list returned " "error code %d", dev->id, search->host, result->exit_status); } else { const char *reason = result->exit_reason; pcmk__warn("Assuming %s cannot fence %s because list could not be " "executed: %s%s%s%s", dev->id, search->host, pcmk_exec_status_str(result->execution_status), ((reason != NULL)? " (" : ""), pcmk__s(reason, ""), ((reason != NULL)? ")" : "")); } /* Fall back to pcmk_host_check=PCMK_VALUE_STATUS if the user didn't * explicitly specify PCMK_VALUE_DYNAMIC_LIST */ if (g_hash_table_lookup(dev->params, PCMK_STONITH_HOST_CHECK) == NULL) { pcmk__notice("Switching to pcmk_host_check='status' for %s", dev->id); pcmk__insert_dup(dev->params, PCMK_STONITH_HOST_CHECK, PCMK_VALUE_STATUS); } } if (dev->targets) { const char *alias = g_hash_table_lookup(dev->aliases, search->host); if (!alias) { alias = search->host; } if (pcmk__str_in_list(alias, dev->targets, pcmk__str_casei)) { can_fence = TRUE; } } search_devices_record_result(search, dev->id, can_fence); } /*! * \internal * \brief Returns true if any key in first is not in second or second has a different value for key */ static int device_params_diff(GHashTable *first, GHashTable *second) { char *key = NULL; char *value = NULL; GHashTableIter gIter; g_hash_table_iter_init(&gIter, first); while (g_hash_table_iter_next(&gIter, (void **)&key, (void **)&value)) { if(strstr(key, "CRM_meta") == key) { continue; } else if (strcmp(key, PCMK_XA_CRM_FEATURE_SET) == 0) { continue; } else { char *other_value = g_hash_table_lookup(second, key); if (!other_value || !pcmk__str_eq(other_value, value, pcmk__str_casei)) { pcmk__trace("Different value for %s: %s != %s", key, other_value, value); return 1; } } } return 0; } /*! * \internal * \brief Checks to see if an identical device already exists in the device_list */ static stonith_device_t * device_has_duplicate(const stonith_device_t *device) { stonith_device_t *dup = g_hash_table_lookup(device_list, device->id); if (!dup) { pcmk__trace("No match for %s", device->id); return NULL; } else if (!pcmk__str_eq(dup->agent, device->agent, pcmk__str_casei)) { pcmk__trace("Different agent: %s != %s", dup->agent, device->agent); return NULL; } // Use pcmk__digest_operation() here? if (device_params_diff(device->params, dup->params) || device_params_diff(dup->params, device->params)) { return NULL; } pcmk__trace("Match"); return dup; } int stonith_device_register(xmlNode *dev, gboolean from_cib) { stonith_device_t *dup = NULL; stonith_device_t *device = build_device_from_xml(dev); guint ndevices = 0; int rv = pcmk_ok; CRM_CHECK(device != NULL, return -ENOMEM); /* do we have a watchdog-device? */ if (pcmk__str_eq(device->id, STONITH_WATCHDOG_ID, pcmk__str_none) || pcmk__str_any_of(device->agent, STONITH_WATCHDOG_AGENT, STONITH_WATCHDOG_AGENT_INTERNAL, NULL)) do { if (stonith_watchdog_timeout_ms <= 0) { pcmk__err("Ignoring watchdog fence device without " PCMK_OPT_STONITH_WATCHDOG_TIMEOUT " set"); rv = -ENODEV; /* fall through to cleanup & return */ } else if (!pcmk__str_any_of(device->agent, STONITH_WATCHDOG_AGENT, STONITH_WATCHDOG_AGENT_INTERNAL, NULL)) { pcmk__err("Ignoring watchdog fence device with unknown agent '%s' " "unequal to '" STONITH_WATCHDOG_AGENT "'", pcmk__s(device->agent, "")); rv = -ENODEV; /* fall through to cleanup & return */ } else if (!pcmk__str_eq(device->id, STONITH_WATCHDOG_ID, pcmk__str_none)) { pcmk__err("Ignoring watchdog fence device named " "'%s' != '" STONITH_WATCHDOG_ID "'", pcmk__s(device->id, "")); rv = -ENODEV; /* fall through to cleanup & return */ } else { const char *local_node_name = fenced_get_local_node(); if (pcmk__str_eq(device->agent, STONITH_WATCHDOG_AGENT, pcmk__str_none)) { /* this either has an empty list or the targets configured for watchdog-fencing */ g_list_free_full(stonith_watchdog_targets, free); stonith_watchdog_targets = device->targets; device->targets = NULL; } if (node_does_watchdog_fencing(local_node_name)) { g_list_free_full(device->targets, free); device->targets = stonith__parse_targets(local_node_name); pcmk__insert_dup(device->params, PCMK_STONITH_HOST_LIST, local_node_name); /* proceed as with any other stonith-device */ break; } pcmk__debug("Skip registration of watchdog fence device on node " "not in host-list."); /* cleanup and fall through to more cleanup and return */ device->targets = NULL; stonith_device_remove(device->id, from_cib); } free_device(device); return rv; } while (0); dup = device_has_duplicate(device); if (dup) { ndevices = g_hash_table_size(device_list); pcmk__debug("Device '%s' already in device list (%d active device%s)", device->id, ndevices, pcmk__plural_s(ndevices)); free_device(device); device = dup; dup = g_hash_table_lookup(device_list, device->id); dup->dirty = FALSE; } else { stonith_device_t *old = g_hash_table_lookup(device_list, device->id); if (from_cib && old && old->api_registered) { /* If the cib is writing over an entry that is shared with a stonith client, * copy any pending ops that currently exist on the old entry to the new one. * Otherwise the pending ops will be reported as failures */ pcmk__info("Overwriting existing entry for %s from CIB", device->id); device->pending_ops = old->pending_ops; device->api_registered = TRUE; old->pending_ops = NULL; if (device->pending_ops) { mainloop_set_trigger(device->work); } } g_hash_table_replace(device_list, device->id, device); ndevices = g_hash_table_size(device_list); pcmk__notice("Added '%s' to device list (%d active device%s)", device->id, ndevices, pcmk__plural_s(ndevices)); } if (from_cib) { device->cib_registered = TRUE; } else { device->api_registered = TRUE; } return pcmk_ok; } void stonith_device_remove(const char *id, bool from_cib) { stonith_device_t *device = g_hash_table_lookup(device_list, id); guint ndevices = 0; if (!device) { ndevices = g_hash_table_size(device_list); pcmk__info("Device '%s' not found (%d active device%s)", id, ndevices, pcmk__plural_s(ndevices)); return; } if (from_cib) { device->cib_registered = FALSE; } else { device->verified = FALSE; device->api_registered = FALSE; } if (!device->cib_registered && !device->api_registered) { g_hash_table_remove(device_list, id); ndevices = g_hash_table_size(device_list); pcmk__info("Removed '%s' from device list (%u active device%s)", id, ndevices, pcmk__plural_s(ndevices)); } else { pcmk__trace("Not removing '%s' from device list (%u active) because " "still registered via:%s%s", id, g_hash_table_size(device_list), (device->cib_registered? " cib" : ""), (device->api_registered? " api" : "")); } } /*! * \internal * \brief Return the number of stonith levels registered for a node * * \param[in] tp Node's topology table entry * * \return Number of non-NULL levels in topology entry * \note This function is used only for log messages. */ static int count_active_levels(const stonith_topology_t *tp) { int lpc = 0; int count = 0; for (lpc = 0; lpc < ST__LEVEL_COUNT; lpc++) { if (tp->levels[lpc] != NULL) { count++; } } return count; } static void free_topology_entry(gpointer data) { stonith_topology_t *tp = data; int lpc = 0; for (lpc = 0; lpc < ST__LEVEL_COUNT; lpc++) { if (tp->levels[lpc] != NULL) { g_list_free_full(tp->levels[lpc], free); } } free(tp->target); free(tp->target_value); free(tp->target_pattern); free(tp->target_attribute); free(tp); } void free_topology_list(void) { if (topology != NULL) { g_hash_table_destroy(topology); topology = NULL; } } void init_topology_list(void) { if (topology == NULL) { topology = pcmk__strkey_table(NULL, free_topology_entry); } } char * stonith_level_key(const xmlNode *level, enum fenced_target_by mode) { if (mode == fenced_target_by_unknown) { mode = unpack_level_kind(level); } switch (mode) { case fenced_target_by_name: return pcmk__xe_get_copy(level, PCMK_XA_TARGET); case fenced_target_by_pattern: return pcmk__xe_get_copy(level, PCMK_XA_TARGET_PATTERN); case fenced_target_by_attribute: return pcmk__assert_asprintf("%s=%s", pcmk__xe_get(level, PCMK_XA_TARGET_ATTRIBUTE), pcmk__xe_get(level, PCMK_XA_TARGET_VALUE)); default: return pcmk__assert_asprintf("unknown-%s", pcmk__xe_id(level)); } } /*! * \internal * \brief Parse target identification from topology level XML * * \param[in] level Topology level XML to parse * * \return How to identify target of \p level */ static enum fenced_target_by unpack_level_kind(const xmlNode *level) { if (pcmk__xe_get(level, PCMK_XA_TARGET) != NULL) { return fenced_target_by_name; } if (pcmk__xe_get(level, PCMK_XA_TARGET_PATTERN) != NULL) { return fenced_target_by_pattern; } if ((pcmk__xe_get(level, PCMK_XA_TARGET_ATTRIBUTE) != NULL) && (pcmk__xe_get(level, PCMK_XA_TARGET_VALUE) != NULL)) { return fenced_target_by_attribute; } return fenced_target_by_unknown; } static stonith_key_value_t * parse_device_list(const char *devices) { int lpc = 0; int max = 0; int last = 0; stonith_key_value_t *output = NULL; if (devices == NULL) { return output; } max = strlen(devices); for (lpc = 0; lpc <= max; lpc++) { if (devices[lpc] == ',' || devices[lpc] == 0) { char *line = strndup(devices + last, lpc - last); output = stonith_key_value_add(output, NULL, line); free(line); last = lpc + 1; } } return output; } /*! * \internal * \brief Unpack essential information from topology request XML * * \param[in] xml Request XML to search * \param[out] mode If not NULL, where to store level kind * \param[out] target If not NULL, where to store representation of target * \param[out] id If not NULL, where to store level number * \param[out] desc If not NULL, where to store log-friendly level description * * \return Topology level XML from within \p xml, or NULL if not found * \note The caller is responsible for freeing \p *target and \p *desc if set. */ static xmlNode * unpack_level_request(xmlNode *xml, enum fenced_target_by *mode, char **target, int *id, char **desc) { enum fenced_target_by local_mode = fenced_target_by_unknown; char *local_target = NULL; int local_id = 0; /* The level element can be the top element or lower. If top level, don't * search by xpath, because it might give multiple hits if the XML is the * CIB. */ if ((xml != NULL) && !pcmk__xe_is(xml, PCMK_XE_FENCING_LEVEL)) { xml = pcmk__xpath_find_one(xml->doc, "//" PCMK_XE_FENCING_LEVEL, LOG_WARNING); } if (xml == NULL) { if (desc != NULL) { *desc = pcmk__assert_asprintf("missing"); } } else { local_mode = unpack_level_kind(xml); local_target = stonith_level_key(xml, local_mode); pcmk__xe_get_int(xml, PCMK_XA_INDEX, &local_id); if (desc != NULL) { *desc = pcmk__assert_asprintf("%s[%d]", local_target, local_id); } } if (mode != NULL) { *mode = local_mode; } if (id != NULL) { *id = local_id; } if (target != NULL) { *target = local_target; } else { free(local_target); } return xml; } /*! * \internal * \brief Register a fencing topology level for a target * * Given an XML request specifying the target name, level index, and device IDs * for the level, this will create an entry for the target in the global topology * table if one does not already exist, then append the specified device IDs to * the entry's device list for the specified level. * * \param[in] msg XML request for STONITH level registration * \param[out] desc If not NULL, set to string representation "TARGET[LEVEL]" * \param[out] result Where to set result of registration */ void fenced_register_level(xmlNode *msg, char **desc, pcmk__action_result_t *result) { int id = 0; xmlNode *level; enum fenced_target_by mode; char *target; stonith_topology_t *tp; stonith_key_value_t *dIter = NULL; stonith_key_value_t *devices = NULL; CRM_CHECK((msg != NULL) && (result != NULL), return); level = unpack_level_request(msg, &mode, &target, &id, desc); if (level == NULL) { fenced_set_protocol_error(result); return; } // Ensure an ID was given (even the client API adds an ID) if (pcmk__str_empty(pcmk__xe_id(level))) { pcmk__warn("Ignoring registration for topology level without ID"); free(target); - crm_log_xml_trace(level, "Bad level"); + pcmk__log_xml_trace(level, "Bad level"); pcmk__format_result(result, CRM_EX_INVALID_PARAM, PCMK_EXEC_INVALID, "Topology level is invalid without ID"); return; } // Ensure a valid target was specified if (mode == fenced_target_by_unknown) { pcmk__warn("Ignoring registration for topology level '%s' without " "valid target", pcmk__xe_id(level)); free(target); - crm_log_xml_trace(level, "Bad level"); + pcmk__log_xml_trace(level, "Bad level"); pcmk__format_result(result, CRM_EX_INVALID_PARAM, PCMK_EXEC_INVALID, "Invalid target for topology level '%s'", pcmk__xe_id(level)); return; } // Ensure level ID is in allowed range if ((id < ST__LEVEL_MIN) || (id > ST__LEVEL_MAX)) { pcmk__warn("Ignoring topology registration for %s with invalid level " "%d", target, id); free(target); - crm_log_xml_trace(level, "Bad level"); + pcmk__log_xml_trace(level, "Bad level"); pcmk__format_result(result, CRM_EX_INVALID_PARAM, PCMK_EXEC_INVALID, "Invalid level number '%s' for topology level '%s'", pcmk__s(pcmk__xe_get(level, PCMK_XA_INDEX), ""), pcmk__xe_id(level)); return; } /* Find or create topology table entry */ tp = g_hash_table_lookup(topology, target); if (tp == NULL) { tp = pcmk__assert_alloc(1, sizeof(stonith_topology_t)); tp->kind = mode; tp->target = target; tp->target_value = pcmk__xe_get_copy(level, PCMK_XA_TARGET_VALUE); tp->target_pattern = pcmk__xe_get_copy(level, PCMK_XA_TARGET_PATTERN); tp->target_attribute = pcmk__xe_get_copy(level, PCMK_XA_TARGET_ATTRIBUTE); g_hash_table_replace(topology, tp->target, tp); pcmk__trace("Added %s (%d) to the topology (%u active entries)", target, (int) mode, g_hash_table_size(topology)); } else { free(target); } if (tp->levels[id] != NULL) { pcmk__info("Adding to the existing %s[%d] topology entry", tp->target, id); } devices = parse_device_list(pcmk__xe_get(level, PCMK_XA_DEVICES)); for (dIter = devices; dIter; dIter = dIter->next) { const char *device = dIter->value; pcmk__trace("Adding device '%s' for %s[%d]", device, tp->target, id); tp->levels[id] = g_list_append(tp->levels[id], pcmk__str_copy(device)); } stonith_key_value_freeall(devices, 1, 1); { int nlevels = count_active_levels(tp); pcmk__info("Target %s has %d active fencing level%s", tp->target, nlevels, pcmk__plural_s(nlevels)); } pcmk__set_result(result, CRM_EX_OK, PCMK_EXEC_DONE, NULL); } /*! * \internal * \brief Unregister a fencing topology level for a target * * Given an XML request specifying the target name and level index (or 0 for all * levels), this will remove any corresponding entry for the target from the * global topology table. * * \param[in] msg XML request for STONITH level registration * \param[out] desc If not NULL, set to string representation "TARGET[LEVEL]" * \param[out] result Where to set result of unregistration */ void fenced_unregister_level(xmlNode *msg, char **desc, pcmk__action_result_t *result) { int id = -1; stonith_topology_t *tp; char *target; xmlNode *level = NULL; CRM_CHECK(result != NULL, return); level = unpack_level_request(msg, NULL, &target, &id, desc); if (level == NULL) { fenced_set_protocol_error(result); return; } // Ensure level ID is in allowed range if ((id < 0) || (id >= ST__LEVEL_COUNT)) { pcmk__warn("Ignoring topology unregistration for %s with invalid level " "%d", target, id); free(target); - crm_log_xml_trace(level, "Bad level"); + pcmk__log_xml_trace(level, "Bad level"); pcmk__format_result(result, CRM_EX_INVALID_PARAM, PCMK_EXEC_INVALID, "Invalid level number '%s' for topology level %s", pcmk__s(pcmk__xe_get(level, PCMK_XA_INDEX), ""), // Client API doesn't add ID to unregistration XML pcmk__s(pcmk__xe_id(level), "")); return; } tp = g_hash_table_lookup(topology, target); if (tp == NULL) { guint nentries = g_hash_table_size(topology); pcmk__info("No fencing topology found for %s (%d active %s)", target, nentries, pcmk__plural_alt(nentries, "entry", "entries")); } else if (id == 0 && g_hash_table_remove(topology, target)) { guint nentries = g_hash_table_size(topology); pcmk__info("Removed all fencing topology entries related to %s (%d " "active %s remaining)", target, nentries, pcmk__plural_alt(nentries, "entry", "entries")); } else if (tp->levels[id] != NULL) { guint nlevels; g_list_free_full(tp->levels[id], free); tp->levels[id] = NULL; nlevels = count_active_levels(tp); pcmk__info("Removed level %d from fencing topology for %s (%d " "active level%s remaining)", id, target, nlevels, pcmk__plural_s(nlevels)); } free(target); pcmk__set_result(result, CRM_EX_OK, PCMK_EXEC_DONE, NULL); } static char * list_to_string(GList *list, const char *delim, gboolean terminate_with_delim) { int max = g_list_length(list); size_t delim_len = delim?strlen(delim):0; size_t alloc_size = 1 + (max?((max-1+(terminate_with_delim?1:0))*delim_len):0); char *rv; GList *gIter; char *pos = NULL; const char *lead_delim = ""; for (gIter = list; gIter != NULL; gIter = gIter->next) { const char *value = (const char *) gIter->data; alloc_size += strlen(value); } rv = pcmk__assert_alloc(alloc_size, sizeof(char)); pos = rv; for (gIter = list; gIter != NULL; gIter = gIter->next) { const char *value = (const char *) gIter->data; pos = &pos[sprintf(pos, "%s%s", lead_delim, value)]; lead_delim = delim; } if (max && terminate_with_delim) { sprintf(pos, "%s", delim); } return rv; } /*! * \internal * \brief Execute a fence agent action directly (and asynchronously) * * Handle a STONITH_OP_EXEC API message by scheduling a requested agent action * directly on a specified device. Only list, monitor, and status actions are * expected to use this call, though it should work with any agent command. * * \param[in] msg Request XML specifying action * \param[out] result Where to store result of action * * \note If the action is monitor, the device must be registered via the API * (CIB registration is not sufficient), because monitor should not be * possible unless the device is "started" (API registered). */ static void execute_agent_action(xmlNode *msg, pcmk__action_result_t *result) { xmlNode *dev = pcmk__xpath_find_one(msg->doc, "//" PCMK__XE_ST_DEVICE_ID, LOG_ERR); xmlNode *op = pcmk__xpath_find_one(msg->doc, "//*[@" PCMK__XA_ST_DEVICE_ACTION "]", LOG_ERR); const char *id = pcmk__xe_get(dev, PCMK__XA_ST_DEVICE_ID); const char *action = pcmk__xe_get(op, PCMK__XA_ST_DEVICE_ACTION); async_command_t *cmd = NULL; stonith_device_t *device = NULL; if ((id == NULL) || (action == NULL)) { pcmk__info("Malformed API action request: device %s, action %s", pcmk__s(id, "not specified"), pcmk__s(action, "not specified")); fenced_set_protocol_error(result); return; } if (pcmk__str_eq(id, STONITH_WATCHDOG_ID, pcmk__str_none)) { // Watchdog agent actions are implemented internally if (stonith_watchdog_timeout_ms <= 0) { pcmk__set_result(result, CRM_EX_ERROR, PCMK_EXEC_NO_FENCE_DEVICE, "Watchdog fence device not configured"); return; } else if (pcmk__str_eq(action, PCMK_ACTION_LIST, pcmk__str_none)) { pcmk__set_result(result, CRM_EX_OK, PCMK_EXEC_DONE, NULL); pcmk__set_result_output(result, list_to_string(stonith_watchdog_targets, "\n", TRUE), NULL); return; } else if (pcmk__str_eq(action, PCMK_ACTION_MONITOR, pcmk__str_none)) { pcmk__set_result(result, CRM_EX_OK, PCMK_EXEC_DONE, NULL); return; } } device = g_hash_table_lookup(device_list, id); if (device == NULL) { pcmk__info("Ignoring API '%s' action request because device %s not " "found", action, id); pcmk__format_result(result, CRM_EX_ERROR, PCMK_EXEC_NO_FENCE_DEVICE, "'%s' not found", id); return; } else if (!device->api_registered && (strcmp(action, PCMK_ACTION_MONITOR) == 0)) { // Monitors may run only on "started" (API-registered) devices pcmk__info("Ignoring API '%s' action request because device %s not " "active", action, id); pcmk__format_result(result, CRM_EX_ERROR, PCMK_EXEC_NO_FENCE_DEVICE, "'%s' not active", id); return; } cmd = create_async_command(msg); if (cmd == NULL) { pcmk__log_xml_warn(msg, "invalid"); fenced_set_protocol_error(result); return; } schedule_stonith_command(cmd, device); pcmk__set_result(result, CRM_EX_OK, PCMK_EXEC_PENDING, NULL); } static void search_devices_record_result(struct device_search_s *search, const char *device, gboolean can_fence) { search->replies_received++; if (can_fence && device) { if (search->support_action_only != st_device_supports_none) { stonith_device_t *dev = g_hash_table_lookup(device_list, device); if (dev && !pcmk__is_set(dev->flags, search->support_action_only)) { return; } } search->capable = g_list_append(search->capable, pcmk__str_copy(device)); } if (search->replies_needed == search->replies_received) { guint ndevices = g_list_length(search->capable); pcmk__debug("Search found %d device%s that can perform '%s' targeting " "%s", ndevices, pcmk__plural_s(ndevices), pcmk__s(search->action, "unknown action"), pcmk__s(search->host, "any node")); search->callback(search->capable, search->user_data); free(search->host); free(search->action); free(search); } } /*! * \internal * \brief Check whether the local host is allowed to execute a fencing action * * \param[in] device Fence device to check * \param[in] action Fence action to check * \param[in] target Hostname of fence target * \param[in] allow_self Whether self-fencing is allowed for this operation * * \return TRUE if local host is allowed to execute action, FALSE otherwise */ static gboolean localhost_is_eligible(const stonith_device_t *device, const char *action, const char *target, gboolean allow_self) { gboolean localhost_is_target = pcmk__str_eq(target, fenced_get_local_node(), pcmk__str_casei); if ((device != NULL) && (action != NULL) && (device->on_target_actions != NULL) && (strstr((const char*) device->on_target_actions->str, action) != NULL)) { if (!localhost_is_target) { pcmk__trace("Operation '%s' using %s can only be executed for " "local host, not %s", action, device->id, target); return FALSE; } } else if (localhost_is_target && !allow_self) { pcmk__trace("'%s' operation does not support self-fencing", action); return FALSE; } return TRUE; } /*! * \internal * \brief Check if local node is allowed to execute (possibly remapped) action * * \param[in] device Fence device to check * \param[in] action Fence action to check * \param[in] target Node name of fence target * \param[in] allow_self Whether self-fencing is allowed for this operation * * \return true if local node is allowed to execute \p action or any actions it * might be remapped to, otherwise false */ static bool localhost_is_eligible_with_remap(const stonith_device_t *device, const char *action, const char *target, gboolean allow_self) { // Check exact action if (localhost_is_eligible(device, action, target, allow_self)) { return true; } // Check potential remaps if (pcmk__str_eq(action, PCMK_ACTION_REBOOT, pcmk__str_none)) { /* "reboot" might get remapped to "off" then "on", so even if reboot is * disallowed, return true if either of those is allowed. We'll report * the disallowed actions with the results. We never allow self-fencing * for remapped "on" actions because the target is off at that point. */ if (localhost_is_eligible(device, PCMK_ACTION_OFF, target, allow_self) || localhost_is_eligible(device, PCMK_ACTION_ON, target, FALSE)) { return true; } } return false; } /*! * \internal * \brief Check whether we can use a device's cached target list * * \param[in] dev Fencing device to check * * \return \c true if \p dev cached its targets less than a minute ago, * otherwise \c false */ static inline bool can_use_target_cache(const stonith_device_t *dev) { return (dev->targets != NULL) && (time(NULL) < (dev->targets_age + 60)); } static void can_fence_host_with_device(stonith_device_t *dev, struct device_search_s *search) { gboolean can = FALSE; const char *check_type = "Internal bug"; const char *target = NULL; const char *alias = NULL; const char *dev_id = "Unspecified device"; const char *action = (search == NULL)? NULL : search->action; CRM_CHECK((dev != NULL) && (action != NULL), goto search_report_results); if (dev->id != NULL) { dev_id = dev->id; } target = search->host; if (target == NULL) { can = TRUE; check_type = "No target"; goto search_report_results; } /* Answer immediately if the device does not support the action * or the local node is not allowed to perform it */ if (pcmk__str_eq(action, PCMK_ACTION_ON, pcmk__str_none) && !pcmk__is_set(dev->flags, st_device_supports_on)) { check_type = "Agent does not support 'on'"; goto search_report_results; } else if (!localhost_is_eligible_with_remap(dev, action, target, search->allow_self)) { check_type = "This node is not allowed to execute action"; goto search_report_results; } // Check eligibility as specified by pcmk_host_check check_type = target_list_type(dev); alias = g_hash_table_lookup(dev->aliases, target); if (pcmk__str_eq(check_type, PCMK_VALUE_NONE, pcmk__str_casei)) { can = TRUE; } else if (pcmk__str_eq(check_type, PCMK_VALUE_STATIC_LIST, pcmk__str_casei)) { if (pcmk__str_in_list(target, dev->targets, pcmk__str_casei)) { can = TRUE; } else if (g_hash_table_lookup(dev->params, PCMK_STONITH_HOST_MAP) && g_hash_table_lookup(dev->aliases, target)) { can = TRUE; } } else if (pcmk__str_eq(check_type, PCMK_VALUE_DYNAMIC_LIST, pcmk__str_casei)) { if (!can_use_target_cache(dev)) { int device_timeout = get_action_timeout(dev, PCMK_ACTION_LIST, search->per_device_timeout); if (device_timeout > search->per_device_timeout) { pcmk__notice("Since the pcmk_list_timeout (%ds) parameter of " "%s is larger than " PCMK_OPT_STONITH_TIMEOUT " " "(%ds), timeout may occur", device_timeout, dev_id, search->per_device_timeout); } pcmk__trace("Running '%s' to check whether %s is eligible to fence " "%s (%s)", check_type, dev_id, target, action); schedule_internal_command(__func__, dev, PCMK_ACTION_LIST, NULL, search->per_device_timeout, search, dynamic_list_search_cb); /* we'll respond to this search request async in the cb */ return; } if (pcmk__str_in_list(((alias == NULL)? target : alias), dev->targets, pcmk__str_casei)) { can = TRUE; } } else if (pcmk__str_eq(check_type, PCMK_VALUE_STATUS, pcmk__str_casei)) { int device_timeout = get_action_timeout(dev, check_type, search->per_device_timeout); if (device_timeout > search->per_device_timeout) { pcmk__notice("Since the pcmk_status_timeout (%ds) parameter of %s " "is larger than " PCMK_OPT_STONITH_TIMEOUT " (%ds), " "timeout may occur", device_timeout, dev_id, search->per_device_timeout); } pcmk__trace("Running '%s' to check whether %s is eligible to fence %s " "(%s)", check_type, dev_id, target, action); schedule_internal_command(__func__, dev, PCMK_ACTION_STATUS, target, search->per_device_timeout, search, status_search_cb); /* we'll respond to this search request async in the cb */ return; } else { pcmk__err("Invalid value for " PCMK_STONITH_HOST_CHECK ": %s", check_type); check_type = "Invalid " PCMK_STONITH_HOST_CHECK; } search_report_results: pcmk__info("%s is%s eligible to fence (%s) %s%s%s%s: %s", dev_id, (can? "" : " not"), pcmk__s(action, "unspecified action"), pcmk__s(target, "unspecified target"), ((alias != NULL)? " (as '" : ""), pcmk__s(alias, ""), ((alias != NULL)? "')" : ""), check_type); search_devices_record_result(search, ((dev == NULL)? NULL : dev_id), can); } static void search_devices(gpointer key, gpointer value, gpointer user_data) { stonith_device_t *dev = value; struct device_search_s *search = user_data; can_fence_host_with_device(dev, search); } #define DEFAULT_QUERY_TIMEOUT 20 static void get_capable_devices(const char *host, const char *action, int timeout, bool allow_self, void *user_data, void (*callback) (GList * devices, void *user_data), uint32_t support_action_only) { struct device_search_s *search; guint ndevices = g_hash_table_size(device_list); if (ndevices == 0) { callback(NULL, user_data); return; } search = pcmk__assert_alloc(1, sizeof(struct device_search_s)); search->host = pcmk__str_copy(host); search->action = pcmk__str_copy(action); search->per_device_timeout = timeout; search->allow_self = allow_self; search->callback = callback; search->user_data = user_data; search->support_action_only = support_action_only; /* We are guaranteed this many replies, even if a device is * unregistered while the search is in progress. */ search->replies_needed = ndevices; pcmk__debug("Searching %d device%s to see which can execute '%s' targeting " "%s", ndevices, pcmk__plural_s(ndevices), pcmk__s(search->action, "unknown action"), pcmk__s(search->host, "any node")); g_hash_table_foreach(device_list, search_devices, search); } struct st_query_data { xmlNode *reply; char *remote_peer; char *client_id; char *target; char *action; int call_options; }; /*! * \internal * \brief Add action-specific attributes to query reply XML * * \param[in,out] xml XML to add attributes to * \param[in] action Fence action * \param[in] device Fence device * \param[in] target Fence target */ static void add_action_specific_attributes(xmlNode *xml, const char *action, const stonith_device_t *device, const char *target) { int action_specific_timeout; int delay_max; int delay_base; CRM_CHECK(xml && action && device, return); // PCMK__XA_ST_REQUIRED is currently used only for unfencing if (is_action_required(action, device)) { pcmk__trace("Action '%s' is required using %s", action, device->id); pcmk__xe_set_int(xml, PCMK__XA_ST_REQUIRED, 1); } // pcmk__timeout if configured action_specific_timeout = get_action_timeout(device, action, 0); if (action_specific_timeout) { pcmk__trace("Action '%s' has timeout %ds using %s", action, action_specific_timeout, device->id); pcmk__xe_set_int(xml, PCMK__XA_ST_ACTION_TIMEOUT, action_specific_timeout); } delay_max = get_action_delay_max(device, action); if (delay_max > 0) { pcmk__trace("Action '%s' has maximum random delay %ds using %s", action, delay_max, device->id); pcmk__xe_set_int(xml, PCMK__XA_ST_DELAY_MAX, delay_max); } delay_base = get_action_delay_base(device, action, target); if (delay_base > 0) { pcmk__xe_set_int(xml, PCMK__XA_ST_DELAY_BASE, delay_base); } if ((delay_max > 0) && (delay_base == 0)) { pcmk__trace("Action '%s' has maximum random delay %ds using %s", action, delay_max, device->id); } else if ((delay_max == 0) && (delay_base > 0)) { pcmk__trace("Action '%s' has a static delay of %ds using %s", action, delay_base, device->id); } else if ((delay_max > 0) && (delay_base > 0)) { pcmk__trace("Action '%s' has a minimum delay of %ds and a randomly " "chosen maximum delay of %ds using %s", action, delay_base, delay_max, device->id); } } /*! * \internal * \brief Add "disallowed" attribute to query reply XML if appropriate * * \param[in,out] xml XML to add attribute to * \param[in] action Fence action * \param[in] device Fence device * \param[in] target Fence target * \param[in] allow_self Whether self-fencing is allowed */ static void add_disallowed(xmlNode *xml, const char *action, const stonith_device_t *device, const char *target, gboolean allow_self) { if (!localhost_is_eligible(device, action, target, allow_self)) { pcmk__trace("Action '%s' using %s is disallowed for local host", action, device->id); pcmk__xe_set_bool_attr(xml, PCMK__XA_ST_ACTION_DISALLOWED, true); } } /*! * \internal * \brief Add child element with action-specific values to query reply XML * * \param[in,out] xml XML to add attribute to * \param[in] action Fence action * \param[in] device Fence device * \param[in] target Fence target * \param[in] allow_self Whether self-fencing is allowed */ static void add_action_reply(xmlNode *xml, const char *action, const stonith_device_t *device, const char *target, gboolean allow_self) { xmlNode *child = pcmk__xe_create(xml, PCMK__XE_ST_DEVICE_ACTION); pcmk__xe_set(child, PCMK_XA_ID, action); add_action_specific_attributes(child, action, device, target); add_disallowed(child, action, device, target, allow_self); } /*! * \internal * \brief Send a reply to a CPG peer or IPC client * * \param[in] reply XML reply to send * \param[in] call_options Send synchronously if st_opt_sync_call is set * \param[in] remote_peer If not NULL, name of peer node to send CPG reply * \param[in,out] client If not NULL, client to send IPC reply */ static void stonith_send_reply(const xmlNode *reply, int call_options, const char *remote_peer, pcmk__client_t *client) { CRM_CHECK((reply != NULL) && ((remote_peer != NULL) || (client != NULL)), return); if (remote_peer == NULL) { do_local_reply(reply, client, call_options); } else { const pcmk__node_status_t *node = pcmk__get_node(0, remote_peer, NULL, pcmk__node_search_cluster_member); pcmk__cluster_send_message(node, pcmk_ipc_fenced, reply); } } static void stonith_query_capable_device_cb(GList * devices, void *user_data) { struct st_query_data *query = user_data; int available_devices = 0; xmlNode *wrapper = NULL; xmlNode *list = NULL; GList *lpc = NULL; pcmk__client_t *client = NULL; if (query->client_id != NULL) { client = pcmk__find_client_by_id(query->client_id); if ((client == NULL) && (query->remote_peer == NULL)) { pcmk__trace("Skipping reply to %s: no longer a client", query->client_id); goto done; } } // Pack the results into XML wrapper = pcmk__xe_create(query->reply, PCMK__XE_ST_CALLDATA); list = pcmk__xe_create(wrapper, __func__); pcmk__xe_set(list, PCMK__XA_ST_TARGET, query->target); for (lpc = devices; lpc != NULL; lpc = lpc->next) { stonith_device_t *device = g_hash_table_lookup(device_list, lpc->data); const char *action = query->action; xmlNode *dev = NULL; if (!device) { /* It is possible the device got unregistered while * determining who can fence the target */ continue; } available_devices++; dev = pcmk__xe_create(list, PCMK__XE_ST_DEVICE_ID); pcmk__xe_set(dev, PCMK_XA_ID, device->id); pcmk__xe_set(dev, PCMK__XA_NAMESPACE, device->namespace); pcmk__xe_set(dev, PCMK_XA_AGENT, device->agent); // Has had successful monitor, list, or status on this node pcmk__xe_set_int(dev, PCMK__XA_ST_MONITOR_VERIFIED, device->verified); pcmk__xe_set_int(dev, PCMK__XA_ST_DEVICE_SUPPORT_FLAGS, device->flags); /* If the originating fencer wants to reboot the node, and we have a * capable device that doesn't support "reboot", remap to "off" instead. */ if (!pcmk__is_set(device->flags, st_device_supports_reboot) && pcmk__str_eq(query->action, PCMK_ACTION_REBOOT, pcmk__str_none)) { pcmk__trace("%s doesn't support reboot, using values for off " "instead", device->id); action = PCMK_ACTION_OFF; } /* Add action-specific values if available */ add_action_specific_attributes(dev, action, device, query->target); if (pcmk__str_eq(query->action, PCMK_ACTION_REBOOT, pcmk__str_none)) { /* A "reboot" *might* get remapped to "off" then "on", so after * sending the "reboot"-specific values in the main element, we add * sub-elements for "off" and "on" values. * * We short-circuited earlier if "reboot", "off" and "on" are all * disallowed for the local host. However if only one or two are * disallowed, we send back the results and mark which ones are * disallowed. If "reboot" is disallowed, this might cause problems * with older fencer versions, which won't check for it. Older * versions will ignore "off" and "on", so they are not a problem. */ add_disallowed(dev, action, device, query->target, pcmk__is_set(query->call_options, st_opt_allow_self_fencing)); add_action_reply(dev, PCMK_ACTION_OFF, device, query->target, pcmk__is_set(query->call_options, st_opt_allow_self_fencing)); add_action_reply(dev, PCMK_ACTION_ON, device, query->target, FALSE); } /* A query without a target wants device parameters */ if (query->target == NULL) { xmlNode *attrs = pcmk__xe_create(dev, PCMK__XE_ATTRIBUTES); g_hash_table_foreach(device->params, hash2field, attrs); } } pcmk__xe_set_int(list, PCMK__XA_ST_AVAILABLE_DEVICES, available_devices); if (query->target) { pcmk__debug("Found %d matching device%s for target '%s'", available_devices, pcmk__plural_s(available_devices), query->target); } else { pcmk__debug("%d device%s installed", available_devices, pcmk__plural_s(available_devices)); } - crm_log_xml_trace(list, "query-result"); + pcmk__log_xml_trace(list, "query-result"); stonith_send_reply(query->reply, query->call_options, query->remote_peer, client); done: pcmk__xml_free(query->reply); free(query->remote_peer); free(query->client_id); free(query->target); free(query->action); free(query); g_list_free_full(devices, free); } /*! * \internal * \brief Log the result of an asynchronous command * * \param[in] cmd Command the result is for * \param[in] result Result of command * \param[in] pid Process ID of command, if available * \param[in] next Alternate device that will be tried if command failed * \param[in] op_merged Whether this command was merged with an earlier one */ static void log_async_result(const async_command_t *cmd, const pcmk__action_result_t *result, int pid, const char *next, bool op_merged) { int log_level = LOG_ERR; int output_log_level = PCMK__LOG_NEVER; guint devices_remaining = g_list_length(cmd->next_device_iter); GString *msg = g_string_sized_new(80); // Reasonable starting size // Choose log levels appropriately if we have a result if (pcmk__result_ok(result)) { log_level = (cmd->target == NULL)? LOG_DEBUG : LOG_NOTICE; if ((result->action_stdout != NULL) && !pcmk__str_eq(cmd->action, PCMK_ACTION_METADATA, pcmk__str_none)) { output_log_level = LOG_DEBUG; } next = NULL; } else { log_level = (cmd->target == NULL)? LOG_NOTICE : LOG_ERR; if ((result->action_stdout != NULL) && !pcmk__str_eq(cmd->action, PCMK_ACTION_METADATA, pcmk__str_none)) { output_log_level = LOG_WARNING; } } // Build the log message piece by piece pcmk__g_strcat(msg, "Operation '", cmd->action, "' ", NULL); if (pid != 0) { g_string_append_printf(msg, "[%d] ", pid); } if (cmd->target != NULL) { pcmk__g_strcat(msg, "targeting ", cmd->target, " ", NULL); } if (cmd->device != NULL) { pcmk__g_strcat(msg, "using ", cmd->device, " ", NULL); } // Add exit status or execution status as appropriate if (result->execution_status == PCMK_EXEC_DONE) { g_string_append_printf(msg, "returned %d", result->exit_status); } else { pcmk__g_strcat(msg, "could not be executed: ", pcmk_exec_status_str(result->execution_status), NULL); } // Add exit reason and next device if appropriate if (result->exit_reason != NULL) { pcmk__g_strcat(msg, " (", result->exit_reason, ")", NULL); } if (next != NULL) { pcmk__g_strcat(msg, ", retrying with ", next, NULL); } if (devices_remaining > 0) { g_string_append_printf(msg, " (%u device%s remaining)", (unsigned int) devices_remaining, pcmk__plural_s(devices_remaining)); } g_string_append_printf(msg, " " QB_XS " %scall %d from %s", (op_merged? "merged " : ""), cmd->id, cmd->client_name); // Log the result do_crm_log(log_level, "%s", msg->str); g_string_free(msg, TRUE); // Log the output (which may have multiple lines), if appropriate if (output_log_level != PCMK__LOG_NEVER) { char *prefix = pcmk__assert_asprintf("%s[%d]", cmd->device, pid); crm_log_output(output_log_level, prefix, result->action_stdout); free(prefix); } } /*! * \internal * \brief Reply to requester after asynchronous command completion * * \param[in] cmd Command that completed * \param[in] result Result of command * \param[in] pid Process ID of command, if available * \param[in] merged If true, command was merged with another, not executed */ static void send_async_reply(const async_command_t *cmd, const pcmk__action_result_t *result, int pid, bool merged) { xmlNode *reply = NULL; pcmk__client_t *client = NULL; CRM_CHECK((cmd != NULL) && (result != NULL), return); log_async_result(cmd, result, pid, NULL, merged); if (cmd->client != NULL) { client = pcmk__find_client_by_id(cmd->client); if ((client == NULL) && (cmd->origin == NULL)) { pcmk__trace("Skipping reply to %s: no longer a client", cmd->client); return; } } reply = construct_async_reply(cmd, result); if (merged) { pcmk__xe_set_bool_attr(reply, PCMK__XA_ST_OP_MERGED, true); } if (pcmk__is_fencing_action(cmd->action) && pcmk__str_eq(cmd->origin, cmd->target, pcmk__str_casei)) { /* The target was also the originator, so broadcast the result on its * behalf (since it will be unable to). */ pcmk__trace("Broadcast '%s' result for %s (target was also originator)", cmd->action, cmd->target); pcmk__xe_set(reply, PCMK__XA_SUBT, PCMK__VALUE_BROADCAST); pcmk__xe_set(reply, PCMK__XA_ST_OP, STONITH_OP_NOTIFY); pcmk__cluster_send_message(NULL, pcmk_ipc_fenced, reply); } else { // Reply only to the originator stonith_send_reply(reply, cmd->options, cmd->origin, client); } - crm_log_xml_trace(reply, "Reply"); + pcmk__log_xml_trace(reply, "Reply"); pcmk__xml_free(reply); } static void cancel_stonith_command(async_command_t * cmd) { stonith_device_t *device = cmd_device(cmd); if (device) { pcmk__trace("Cancel scheduled '%s' action using %s", cmd->action, device->id); device->pending_ops = g_list_remove(device->pending_ops, cmd); } } /*! * \internal * \brief Cancel and reply to any duplicates of a just-completed operation * * Check whether any fencing operations are scheduled to do the same thing as * one that just succeeded. If so, rather than performing the same operation * twice, return the result of this operation for all matching pending commands. * * \param[in,out] cmd Fencing operation that just succeeded * \param[in] result Result of \p cmd * \param[in] pid If nonzero, process ID of agent invocation (for logs) * * \note Duplicate merging will do the right thing for either type of remapped * reboot. If the executing fencer remapped an unsupported reboot to off, * then cmd->action will be "reboot" and will be merged with any other * reboot requests. If the originating fencer remapped a topology reboot * to off then on, we will get here once with cmd->action "off" and once * with "on", and they will be merged separately with similar requests. */ static void reply_to_duplicates(async_command_t *cmd, const pcmk__action_result_t *result, int pid) { GList *next = NULL; for (GList *iter = cmd_list; iter != NULL; iter = next) { async_command_t *cmd_other = iter->data; next = iter->next; // We might delete this entry, so grab next now if (cmd == cmd_other) { continue; } /* A pending operation matches if: * 1. The client connections are different. * 2. The target is the same. * 3. The fencing action is the same. * 4. The device scheduled to execute the action is the same. */ if (pcmk__str_eq(cmd->client, cmd_other->client, pcmk__str_casei) || !pcmk__str_eq(cmd->target, cmd_other->target, pcmk__str_casei) || !pcmk__str_eq(cmd->action, cmd_other->action, pcmk__str_none) || !pcmk__str_eq(cmd->device, cmd_other->device, pcmk__str_casei)) { continue; } pcmk__notice("Merging fencing action '%s'%s%s originating from client " "%s with identical fencing request from client %s", cmd_other->action, (cmd_other->target == NULL)? "" : " targeting ", pcmk__s(cmd_other->target, ""), cmd_other->client_name, cmd->client_name); // Stop tracking the duplicate, send its result, and cancel it cmd_list = g_list_remove_link(cmd_list, iter); send_async_reply(cmd_other, result, pid, true); cancel_stonith_command(cmd_other); free_async_command(cmd_other); g_list_free_1(iter); } } /*! * \internal * \brief Return the next required device (if any) for an operation * * \param[in,out] cmd Fencing operation that just succeeded * * \return Next device required for action if any, otherwise NULL */ static stonith_device_t * next_required_device(async_command_t *cmd) { for (GList *iter = cmd->next_device_iter; iter != NULL; iter = iter->next) { stonith_device_t *next_device = g_hash_table_lookup(device_list, iter->data); if (is_action_required(cmd->action, next_device)) { /* This is only called for successful actions, so it's OK to skip * non-required devices. */ cmd->next_device_iter = iter->next; return next_device; } } return NULL; } static void st_child_done(int pid, const pcmk__action_result_t *result, void *user_data) { async_command_t *cmd = user_data; stonith_device_t *device = NULL; stonith_device_t *next_device = NULL; CRM_CHECK(cmd != NULL, return); device = cmd_device(cmd); cmd->active_on = NULL; /* The device is ready to do something else now */ if (device) { if (!device->verified && pcmk__result_ok(result) && pcmk__strcase_any_of(cmd->action, PCMK_ACTION_LIST, PCMK_ACTION_MONITOR, PCMK_ACTION_STATUS, NULL)) { device->verified = TRUE; } mainloop_set_trigger(device->work); } if (pcmk__result_ok(result)) { next_device = next_required_device(cmd); } else if ((cmd->next_device_iter != NULL) && !is_action_required(cmd->action, device)) { /* if this device didn't work out, see if there are any others we can try. * if the failed device was 'required', we can't pick another device. */ next_device = g_hash_table_lookup(device_list, cmd->next_device_iter->data); cmd->next_device_iter = cmd->next_device_iter->next; } if (next_device == NULL) { send_async_reply(cmd, result, pid, false); if (pcmk__result_ok(result)) { reply_to_duplicates(cmd, result, pid); } free_async_command(cmd); } else { // This operation requires more fencing log_async_result(cmd, result, pid, next_device->id, false); schedule_stonith_command(cmd, next_device); } } static gint sort_device_priority(gconstpointer a, gconstpointer b) { const stonith_device_t *dev_a = a; const stonith_device_t *dev_b = b; if (dev_a->priority > dev_b->priority) { return -1; } else if (dev_a->priority < dev_b->priority) { return 1; } return 0; } static void stonith_fence_get_devices_cb(GList * devices, void *user_data) { async_command_t *cmd = user_data; stonith_device_t *device = NULL; guint ndevices = g_list_length(devices); pcmk__info("Found %d matching device%s for target '%s'", ndevices, pcmk__plural_s(ndevices), cmd->target); if (devices != NULL) { /* Order based on priority */ devices = g_list_sort(devices, sort_device_priority); device = g_hash_table_lookup(device_list, devices->data); } if (device == NULL) { // No device found pcmk__action_result_t result = PCMK__UNKNOWN_RESULT; pcmk__format_result(&result, CRM_EX_ERROR, PCMK_EXEC_NO_FENCE_DEVICE, "No device configured for target '%s'", cmd->target); send_async_reply(cmd, &result, 0, false); pcmk__reset_result(&result); free_async_command(cmd); g_list_free_full(devices, free); } else { // Device found, schedule it for fencing cmd->device_list = devices; cmd->next_device_iter = devices->next; schedule_stonith_command(cmd, device); } } /*! * \internal * \brief Execute a fence action via the local node * * \param[in] msg Fencing request * \param[out] result Where to store result of fence action */ static void fence_locally(xmlNode *msg, pcmk__action_result_t *result) { const char *device_id = NULL; stonith_device_t *device = NULL; async_command_t *cmd = NULL; xmlNode *dev = NULL; CRM_CHECK((msg != NULL) && (result != NULL), return); dev = pcmk__xpath_find_one(msg->doc, "//*[@" PCMK__XA_ST_TARGET "]", LOG_ERR); cmd = create_async_command(msg); if (cmd == NULL) { pcmk__log_xml_warn(msg, "invalid"); fenced_set_protocol_error(result); return; } device_id = pcmk__xe_get(dev, PCMK__XA_ST_DEVICE_ID); if (device_id != NULL) { device = g_hash_table_lookup(device_list, device_id); if (device == NULL) { pcmk__err("Requested device '%s' is not available", device_id); pcmk__format_result(result, CRM_EX_ERROR, PCMK_EXEC_NO_FENCE_DEVICE, "Requested device '%s' not found", device_id); return; } schedule_stonith_command(cmd, device); } else { const char *host = pcmk__xe_get(dev, PCMK__XA_ST_TARGET); if (pcmk__is_set(cmd->options, st_opt_cs_nodeid)) { int nodeid = 0; pcmk__node_status_t *node = NULL; pcmk__scan_min_int(host, &nodeid, 0); node = pcmk__search_node_caches(nodeid, NULL, NULL, pcmk__node_search_any |pcmk__node_search_cluster_cib); if (node != NULL) { host = node->name; } } /* If we get to here, then self-fencing is implicitly allowed */ get_capable_devices(host, cmd->action, cmd->default_timeout, TRUE, cmd, stonith_fence_get_devices_cb, fenced_support_flag(cmd->action)); } pcmk__set_result(result, CRM_EX_OK, PCMK_EXEC_PENDING, NULL); } /*! * \internal * \brief Build an XML reply for a fencing operation * * \param[in] request Request that reply is for * \param[in] data If not NULL, add to reply as call data * \param[in] result Full result of fencing operation * * \return Newly created XML reply * \note The caller is responsible for freeing the result. * \note This has some overlap with construct_async_reply(), but that copies * values from an async_command_t, whereas this one copies them from the * request. */ xmlNode * fenced_construct_reply(const xmlNode *request, xmlNode *data, const pcmk__action_result_t *result) { xmlNode *reply = NULL; reply = pcmk__xe_create(NULL, PCMK__XE_ST_REPLY); pcmk__xe_set(reply, PCMK__XA_ST_ORIGIN, __func__); pcmk__xe_set(reply, PCMK__XA_T, PCMK__VALUE_STONITH_NG); stonith__xe_set_result(reply, result); if (request == NULL) { /* Most likely, this is the result of a stonith operation that was * initiated before we came up. Unfortunately that means we lack enough * information to provide clients with a full result. * * @TODO Maybe synchronize this information at start-up? */ pcmk__warn("Missing request information for client notifications for " "operation with result '%s' (initiated before we came up?)", pcmk_exec_status_str(result->execution_status)); } else { const char *name = NULL; const char *value = NULL; // Attributes to copy from request to reply const char *names[] = { PCMK__XA_ST_OP, PCMK__XA_ST_CALLID, PCMK__XA_ST_CLIENTID, PCMK__XA_ST_CLIENTNAME, PCMK__XA_ST_REMOTE_OP, PCMK__XA_ST_CALLOPT, }; for (int lpc = 0; lpc < PCMK__NELEM(names); lpc++) { name = names[lpc]; value = pcmk__xe_get(request, name); pcmk__xe_set(reply, name, value); } if (data != NULL) { xmlNode *wrapper = pcmk__xe_create(reply, PCMK__XE_ST_CALLDATA); pcmk__xml_copy(wrapper, data); } } return reply; } /*! * \internal * \brief Build an XML reply to an asynchronous fencing command * * \param[in] cmd Fencing command that reply is for * \param[in] result Command result */ static xmlNode * construct_async_reply(const async_command_t *cmd, const pcmk__action_result_t *result) { xmlNode *reply = pcmk__xe_create(NULL, PCMK__XE_ST_REPLY); pcmk__xe_set(reply, PCMK__XA_ST_ORIGIN, __func__); pcmk__xe_set(reply, PCMK__XA_T, PCMK__VALUE_STONITH_NG); pcmk__xe_set(reply, PCMK__XA_ST_OP, cmd->op); pcmk__xe_set(reply, PCMK__XA_ST_DEVICE_ID, cmd->device); pcmk__xe_set(reply, PCMK__XA_ST_REMOTE_OP, cmd->remote_op_id); pcmk__xe_set(reply, PCMK__XA_ST_CLIENTID, cmd->client); pcmk__xe_set(reply, PCMK__XA_ST_CLIENTNAME, cmd->client_name); pcmk__xe_set(reply, PCMK__XA_ST_TARGET, cmd->target); pcmk__xe_set(reply, PCMK__XA_ST_DEVICE_ACTION, cmd->op); pcmk__xe_set(reply, PCMK__XA_ST_ORIGIN, cmd->origin); pcmk__xe_set_int(reply, PCMK__XA_ST_CALLID, cmd->id); pcmk__xe_set_int(reply, PCMK__XA_ST_CALLOPT, cmd->options); stonith__xe_set_result(reply, result); return reply; } bool fencing_peer_active(pcmk__node_status_t *peer) { return (peer != NULL) && (peer->name != NULL) && pcmk__is_set(peer->processes, crm_get_cluster_proc()); } void set_fencing_completed(remote_fencing_op_t *op) { struct timespec tv; qb_util_timespec_from_epoch_get(&tv); op->completed = tv.tv_sec; op->completed_nsec = tv.tv_nsec; } /*! * \internal * \brief Look for alternate node needed if local node shouldn't fence target * * \param[in] target Node that must be fenced * * \return Name of an alternate node that should fence \p target if any, * or NULL otherwise */ static const char * check_alternate_host(const char *target) { if (pcmk__str_eq(target, fenced_get_local_node(), pcmk__str_casei)) { GHashTableIter gIter; pcmk__node_status_t *entry = NULL; g_hash_table_iter_init(&gIter, pcmk__peer_cache); while (g_hash_table_iter_next(&gIter, NULL, (void **)&entry)) { if (fencing_peer_active(entry) && !pcmk__str_eq(entry->name, target, pcmk__str_casei)) { pcmk__notice("Forwarding self-fencing request to %s", entry->name); return entry->name; } } pcmk__warn("Will handle own fencing because no peer can"); } return NULL; } static void remove_relay_op(xmlNode * request) { xmlNode *dev = pcmk__xpath_find_one(request->doc, "//*[@" PCMK__XA_ST_DEVICE_ACTION "]", PCMK__LOG_TRACE); const char *relay_op_id = NULL; const char *op_id = NULL; const char *client_name = NULL; const char *target = NULL; remote_fencing_op_t *relay_op = NULL; if (dev) { target = pcmk__xe_get(dev, PCMK__XA_ST_TARGET); } relay_op_id = pcmk__xe_get(request, PCMK__XA_ST_REMOTE_OP_RELAY); op_id = pcmk__xe_get(request, PCMK__XA_ST_REMOTE_OP); client_name = pcmk__xe_get(request, PCMK__XA_ST_CLIENTNAME); /* Delete RELAY operation. */ if ((relay_op_id != NULL) && (target != NULL) && pcmk__str_eq(target, fenced_get_local_node(), pcmk__str_casei)) { relay_op = g_hash_table_lookup(stonith_remote_op_list, relay_op_id); if (relay_op) { GHashTableIter iter; remote_fencing_op_t *list_op = NULL; g_hash_table_iter_init(&iter, stonith_remote_op_list); /* If the operation to be deleted is registered as a duplicate, delete the registration. */ while (g_hash_table_iter_next(&iter, NULL, (void **)&list_op)) { GList *dup_iter = NULL; if (list_op != relay_op) { for (dup_iter = list_op->duplicates; dup_iter != NULL; dup_iter = dup_iter->next) { remote_fencing_op_t *other = dup_iter->data; if (other == relay_op) { other->duplicates = g_list_remove(other->duplicates, relay_op); break; } } } } pcmk__debug("Deleting relay op %s ('%s'%s%s for %s), replaced by " "op %s ('%s'%s%s for %s)", relay_op->id, relay_op->action, ((relay_op->target != NULL)? " targeting " : ""), pcmk__s(relay_op->target, ""), relay_op->client_name, op_id, relay_op->action, ((target != NULL)? " targeting " : ""), pcmk__s(target, ""), client_name); g_hash_table_remove(stonith_remote_op_list, relay_op_id); } } } /*! * \internal * \brief Check whether an API request was sent by a privileged user * * API commands related to fencing configuration may be done only by privileged * IPC users (i.e. root or hacluster), because all other users should go through * the CIB to have ACLs applied. If no client was given, this is a peer request, * which is always allowed. * * \param[in] c IPC client that sent request (or NULL if sent by CPG peer) * \param[in] op Requested API operation (for logging only) * * \return true if sender is peer or privileged client, otherwise false */ static inline bool is_privileged(const pcmk__client_t *c, const char *op) { if ((c == NULL) || pcmk__is_set(c->flags, pcmk__client_privileged)) { return true; } else { pcmk__warn("Rejecting IPC request '%s' from unprivileged client %s", pcmk__s(op, ""), pcmk__client_name(c)); return false; } } // CRM_OP_REGISTER static xmlNode * handle_register_request(pcmk__request_t *request) { xmlNode *reply = pcmk__xe_create(NULL, "reply"); pcmk__assert(request->ipc_client != NULL); pcmk__xe_set(reply, PCMK__XA_ST_OP, CRM_OP_REGISTER); pcmk__xe_set(reply, PCMK__XA_ST_CLIENTID, request->ipc_client->id); pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL); pcmk__set_request_flags(request, pcmk__request_reuse_options); return reply; } // STONITH_OP_EXEC static xmlNode * handle_agent_request(pcmk__request_t *request) { execute_agent_action(request->xml, &request->result); if (request->result.execution_status == PCMK_EXEC_PENDING) { return NULL; } return fenced_construct_reply(request->xml, NULL, &request->result); } // STONITH_OP_TIMEOUT_UPDATE static xmlNode * handle_update_timeout_request(pcmk__request_t *request) { const char *call_id = pcmk__xe_get(request->xml, PCMK__XA_ST_CALLID); const char *client_id = pcmk__xe_get(request->xml, PCMK__XA_ST_CLIENTID); int op_timeout = 0; pcmk__xe_get_int(request->xml, PCMK__XA_ST_TIMEOUT, &op_timeout); do_stonith_async_timeout_update(client_id, call_id, op_timeout); pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL); return NULL; } // STONITH_OP_QUERY static xmlNode * handle_query_request(pcmk__request_t *request) { int timeout = 0; xmlNode *dev = NULL; const char *action = NULL; const char *target = NULL; const char *client_id = pcmk__xe_get(request->xml, PCMK__XA_ST_CLIENTID); struct st_query_data *query = NULL; if (request->peer != NULL) { // Record it for the future notification create_remote_stonith_op(client_id, request->xml, TRUE); } /* Delete the DC node RELAY operation. */ remove_relay_op(request->xml); pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL); dev = pcmk__xpath_find_one(request->xml->doc, "//*[@" PCMK__XA_ST_DEVICE_ACTION "]", PCMK__LOG_NEVER); if (dev != NULL) { const char *device = pcmk__xe_get(dev, PCMK__XA_ST_DEVICE_ID); if (pcmk__str_eq(device, "manual_ack", pcmk__str_casei)) { return NULL; // No query or reply necessary } target = pcmk__xe_get(dev, PCMK__XA_ST_TARGET); action = pcmk__xe_get(dev, PCMK__XA_ST_DEVICE_ACTION); } - crm_log_xml_trace(request->xml, "Query"); + pcmk__log_xml_trace(request->xml, "Query"); query = pcmk__assert_alloc(1, sizeof(struct st_query_data)); query->reply = fenced_construct_reply(request->xml, NULL, &request->result); query->remote_peer = pcmk__str_copy(request->peer); query->client_id = pcmk__str_copy(client_id); query->target = pcmk__str_copy(target); query->action = pcmk__str_copy(action); query->call_options = request->call_options; pcmk__xe_get_int(request->xml, PCMK__XA_ST_TIMEOUT, &timeout); get_capable_devices(target, action, timeout, pcmk__is_set(query->call_options, st_opt_allow_self_fencing), query, stonith_query_capable_device_cb, st_device_supports_none); return NULL; } // STONITH_OP_NOTIFY static xmlNode * handle_notify_request(pcmk__request_t *request) { const char *flag_name = NULL; pcmk__assert(request->ipc_client != NULL); flag_name = pcmk__xe_get(request->xml, PCMK__XA_ST_NOTIFY_ACTIVATE); if (flag_name != NULL) { pcmk__debug("Enabling %s callbacks for client %s", flag_name, pcmk__request_origin(request)); pcmk__set_client_flags(request->ipc_client, get_stonith_flag(flag_name)); } flag_name = pcmk__xe_get(request->xml, PCMK__XA_ST_NOTIFY_DEACTIVATE); if (flag_name != NULL) { pcmk__debug("Disabling %s callbacks for client %s", flag_name, pcmk__request_origin(request)); pcmk__clear_client_flags(request->ipc_client, get_stonith_flag(flag_name)); } pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL); pcmk__set_request_flags(request, pcmk__request_reuse_options); return pcmk__ipc_create_ack(request->ipc_flags, PCMK__XE_ACK, NULL, CRM_EX_OK); } // STONITH_OP_RELAY static xmlNode * handle_relay_request(pcmk__request_t *request) { xmlNode *dev = pcmk__xpath_find_one(request->xml->doc, "//*[@" PCMK__XA_ST_TARGET "]", PCMK__LOG_TRACE); pcmk__notice("Received forwarded fencing request from %s %s to fence (%s) " "peer %s", pcmk__request_origin_type(request), pcmk__request_origin(request), pcmk__xe_get(dev, PCMK__XA_ST_DEVICE_ACTION), pcmk__xe_get(dev, PCMK__XA_ST_TARGET)); if (initiate_remote_stonith_op(NULL, request->xml, FALSE) == NULL) { fenced_set_protocol_error(&request->result); return fenced_construct_reply(request->xml, NULL, &request->result); } pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_PENDING, NULL); return NULL; } // STONITH_OP_FENCE static xmlNode * handle_fence_request(pcmk__request_t *request) { if (request->peer != NULL) { fence_locally(request->xml, &request->result); } else if (pcmk__is_set(request->call_options, st_opt_manual_ack)) { switch (fenced_handle_manual_confirmation(request->ipc_client, request->xml)) { case pcmk_rc_ok: pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL); break; case EINPROGRESS: pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_PENDING, NULL); break; default: fenced_set_protocol_error(&request->result); break; } } else { const char *alternate_host = NULL; xmlNode *dev = pcmk__xpath_find_one(request->xml->doc, "//*[@" PCMK__XA_ST_TARGET "]", PCMK__LOG_TRACE); const char *target = pcmk__xe_get(dev, PCMK__XA_ST_TARGET); const char *action = pcmk__xe_get(dev, PCMK__XA_ST_DEVICE_ACTION); const char *device = pcmk__xe_get(dev, PCMK__XA_ST_DEVICE_ID); if (request->ipc_client != NULL) { int tolerance = 0; pcmk__notice("Client %s wants to fence (%s) %s using %s", pcmk__request_origin(request), action, target, pcmk__s(device, "any device")); pcmk__xe_get_int(dev, PCMK__XA_ST_TOLERANCE, &tolerance); if (stonith_check_fence_tolerance(tolerance, target, action)) { pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL); return fenced_construct_reply(request->xml, NULL, &request->result); } alternate_host = check_alternate_host(target); } else { pcmk__notice("Peer %s wants to fence (%s) '%s' with device '%s'", request->peer, action, target, pcmk__s(device, "(any)")); } if (alternate_host != NULL) { const char *client_id = NULL; remote_fencing_op_t *op = NULL; pcmk__node_status_t *node = pcmk__get_node(0, alternate_host, NULL, pcmk__node_search_cluster_member); if (request->ipc_client->id == 0) { client_id = pcmk__xe_get(request->xml, PCMK__XA_ST_CLIENTID); } else { client_id = request->ipc_client->id; } /* Create a duplicate fencing operation to relay with the client ID. * When a query response is received, this operation should be * deleted to avoid keeping the duplicate around. */ op = create_remote_stonith_op(client_id, request->xml, FALSE); pcmk__xe_set(request->xml, PCMK__XA_ST_OP, STONITH_OP_RELAY); pcmk__xe_set(request->xml, PCMK__XA_ST_CLIENTID, request->ipc_client->id); pcmk__xe_set(request->xml, PCMK__XA_ST_REMOTE_OP, op->id); // @TODO On failure, fail request immediately, or maybe panic pcmk__cluster_send_message(node, pcmk_ipc_fenced, request->xml); pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_PENDING, NULL); } else if (initiate_remote_stonith_op(request->ipc_client, request->xml, FALSE) == NULL) { fenced_set_protocol_error(&request->result); } else { pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_PENDING, NULL); } } if (request->result.execution_status == PCMK_EXEC_PENDING) { return NULL; } return fenced_construct_reply(request->xml, NULL, &request->result); } // STONITH_OP_FENCE_HISTORY static xmlNode * handle_history_request(pcmk__request_t *request) { xmlNode *reply = NULL; xmlNode *data = NULL; stonith_fence_history(request->xml, &data, request->peer, request->call_options); pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL); if (!pcmk__is_set(request->call_options, st_opt_discard_reply)) { /* When the local node broadcasts its history, it sets * st_opt_discard_reply and doesn't need a reply. */ reply = fenced_construct_reply(request->xml, data, &request->result); } pcmk__xml_free(data); return reply; } // STONITH_OP_DEVICE_ADD static xmlNode * handle_device_add_request(pcmk__request_t *request) { const char *op = pcmk__xe_get(request->xml, PCMK__XA_ST_OP); xmlNode *dev = pcmk__xpath_find_one(request->xml->doc, "//" PCMK__XE_ST_DEVICE_ID, LOG_ERR); if (is_privileged(request->ipc_client, op)) { int rc = stonith_device_register(dev, FALSE); pcmk__set_result(&request->result, ((rc == pcmk_ok)? CRM_EX_OK : CRM_EX_ERROR), stonith__legacy2status(rc), ((rc == pcmk_ok)? NULL : pcmk_strerror(rc))); } else { pcmk__set_result(&request->result, CRM_EX_INSUFFICIENT_PRIV, PCMK_EXEC_INVALID, "Unprivileged users must register device via CIB"); } fenced_send_config_notification(op, &request->result, (dev == NULL)? NULL : pcmk__xe_id(dev)); return fenced_construct_reply(request->xml, NULL, &request->result); } // STONITH_OP_DEVICE_DEL static xmlNode * handle_device_delete_request(pcmk__request_t *request) { xmlNode *dev = pcmk__xpath_find_one(request->xml->doc, "//" PCMK__XE_ST_DEVICE_ID, LOG_ERR); const char *device_id = pcmk__xe_get(dev, PCMK_XA_ID); const char *op = pcmk__xe_get(request->xml, PCMK__XA_ST_OP); if (is_privileged(request->ipc_client, op)) { stonith_device_remove(device_id, false); pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL); } else { pcmk__set_result(&request->result, CRM_EX_INSUFFICIENT_PRIV, PCMK_EXEC_INVALID, "Unprivileged users must delete device via CIB"); } fenced_send_config_notification(op, &request->result, device_id); return fenced_construct_reply(request->xml, NULL, &request->result); } // STONITH_OP_LEVEL_ADD static xmlNode * handle_level_add_request(pcmk__request_t *request) { char *desc = NULL; const char *op = pcmk__xe_get(request->xml, PCMK__XA_ST_OP); if (is_privileged(request->ipc_client, op)) { fenced_register_level(request->xml, &desc, &request->result); } else { unpack_level_request(request->xml, NULL, NULL, NULL, &desc); pcmk__set_result(&request->result, CRM_EX_INSUFFICIENT_PRIV, PCMK_EXEC_INVALID, "Unprivileged users must add level via CIB"); } fenced_send_config_notification(op, &request->result, desc); free(desc); return fenced_construct_reply(request->xml, NULL, &request->result); } // STONITH_OP_LEVEL_DEL static xmlNode * handle_level_delete_request(pcmk__request_t *request) { char *desc = NULL; const char *op = pcmk__xe_get(request->xml, PCMK__XA_ST_OP); if (is_privileged(request->ipc_client, op)) { fenced_unregister_level(request->xml, &desc, &request->result); } else { unpack_level_request(request->xml, NULL, NULL, NULL, &desc); pcmk__set_result(&request->result, CRM_EX_INSUFFICIENT_PRIV, PCMK_EXEC_INVALID, "Unprivileged users must delete level via CIB"); } fenced_send_config_notification(op, &request->result, desc); free(desc); return fenced_construct_reply(request->xml, NULL, &request->result); } // CRM_OP_RM_NODE_CACHE static xmlNode * handle_cache_request(pcmk__request_t *request) { int node_id = 0; const char *name = NULL; pcmk__xe_get_int(request->xml, PCMK_XA_ID, &node_id); name = pcmk__xe_get(request->xml, PCMK_XA_UNAME); pcmk__cluster_forget_cluster_node(node_id, name); pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL); return NULL; } static xmlNode * handle_unknown_request(pcmk__request_t *request) { pcmk__err("Unknown IPC request %s from %s %s", request->op, pcmk__request_origin_type(request), pcmk__request_origin(request)); pcmk__format_result(&request->result, CRM_EX_PROTOCOL, PCMK_EXEC_INVALID, "Unknown IPC request type '%s' (bug?)", request->op); return fenced_construct_reply(request->xml, NULL, &request->result); } static void fenced_register_handlers(void) { pcmk__server_command_t handlers[] = { { CRM_OP_REGISTER, handle_register_request }, { STONITH_OP_EXEC, handle_agent_request }, { STONITH_OP_TIMEOUT_UPDATE, handle_update_timeout_request }, { STONITH_OP_QUERY, handle_query_request }, { STONITH_OP_NOTIFY, handle_notify_request }, { STONITH_OP_RELAY, handle_relay_request }, { STONITH_OP_FENCE, handle_fence_request }, { STONITH_OP_FENCE_HISTORY, handle_history_request }, { STONITH_OP_DEVICE_ADD, handle_device_add_request }, { STONITH_OP_DEVICE_DEL, handle_device_delete_request }, { STONITH_OP_LEVEL_ADD, handle_level_add_request }, { STONITH_OP_LEVEL_DEL, handle_level_delete_request }, { CRM_OP_RM_NODE_CACHE, handle_cache_request }, { NULL, handle_unknown_request }, }; fenced_handlers = pcmk__register_handlers(handlers); } void fenced_unregister_handlers(void) { if (fenced_handlers != NULL) { g_hash_table_destroy(fenced_handlers); fenced_handlers = NULL; } } static void handle_request(pcmk__request_t *request) { xmlNode *reply = NULL; const char *reason = NULL; if (fenced_handlers == NULL) { fenced_register_handlers(); } reply = pcmk__process_request(request, fenced_handlers); if (reply != NULL) { if (pcmk__is_set(request->flags, pcmk__request_reuse_options) && (request->ipc_client != NULL)) { /* Certain IPC-only commands must reuse the call options from the * original request rather than the ones set by stonith_send_reply() * -> do_local_reply(). */ pcmk__ipc_send_xml(request->ipc_client, request->ipc_id, reply, request->ipc_flags); request->ipc_client->request_id = 0; } else { stonith_send_reply(reply, request->call_options, request->peer, request->ipc_client); } pcmk__xml_free(reply); } reason = request->result.exit_reason; pcmk__debug("Processed %s request from %s %s: %s%s%s%s", request->op, pcmk__request_origin_type(request), pcmk__request_origin(request), pcmk_exec_status_str(request->result.execution_status), ((reason != NULL)? " (" : ""), pcmk__s(reason, ""), ((reason != NULL)? ")" : "")); } static void handle_reply(pcmk__client_t *client, xmlNode *request, const char *remote_peer) { // Copy, because request might be freed before we want to log this char *op = pcmk__xe_get_copy(request, PCMK__XA_ST_OP); if (pcmk__str_eq(op, STONITH_OP_QUERY, pcmk__str_none)) { process_remote_stonith_query(request); } else if (pcmk__str_any_of(op, STONITH_OP_NOTIFY, STONITH_OP_FENCE, NULL)) { fenced_process_fencing_reply(request); } else { pcmk__err("Ignoring unknown %s reply from %s %s", pcmk__s(op, "untyped"), ((client != NULL)? "client" : "peer"), ((client != NULL)? pcmk__client_name(client) : remote_peer)); pcmk__log_xml_warn(request, "UnknownOp"); free(op); return; } pcmk__debug("Processed %s reply from %s %s", op, ((client != NULL)? "client" : "peer"), ((client != NULL)? pcmk__client_name(client) : remote_peer)); free(op); } /*! * \internal * \brief Handle a message from an IPC client or CPG peer * * \param[in,out] client If not NULL, IPC client that sent message * \param[in] id If from IPC client, IPC message ID * \param[in] flags Message flags * \param[in,out] message Message XML * \param[in] remote_peer If not NULL, CPG peer that sent message */ void stonith_command(pcmk__client_t *client, uint32_t id, uint32_t flags, xmlNode *message, const char *remote_peer) { uint32_t call_options = st_opt_none; int rc = pcmk_rc_ok; bool is_reply = false; CRM_CHECK(message != NULL, return); if (pcmk__xpath_find_one(message->doc, "//" PCMK__XE_ST_REPLY, PCMK__LOG_NEVER) != NULL) { is_reply = true; } rc = pcmk__xe_get_flags(message, PCMK__XA_ST_CALLOPT, &call_options, st_opt_none); if (rc != pcmk_rc_ok) { pcmk__warn("Couldn't parse options from message: %s", pcmk_rc_str(rc)); } pcmk__debug("Processing %ssynchronous %s %s %u from %s %s", (pcmk__is_set(call_options, st_opt_sync_call)? "" : "a"), pcmk__xe_get(message, PCMK__XA_ST_OP), (is_reply? "reply" : "request"), id, ((client != NULL)? "client" : "peer"), ((client != NULL)? pcmk__client_name(client) : remote_peer)); if (pcmk__is_set(call_options, st_opt_sync_call)) { pcmk__assert((client == NULL) || (client->request_id == id)); } if (is_reply) { handle_reply(client, message, remote_peer); } else { pcmk__request_t request = { .ipc_client = client, .ipc_id = id, .ipc_flags = flags, .peer = remote_peer, .xml = message, .call_options = call_options, .result = PCMK__UNKNOWN_RESULT, }; request.op = pcmk__xe_get_copy(request.xml, PCMK__XA_ST_OP); CRM_CHECK(request.op != NULL, return); if (pcmk__is_set(request.call_options, st_opt_sync_call)) { pcmk__set_request_flags(&request, pcmk__request_sync); } handle_request(&request); pcmk__reset_request(&request); } } diff --git a/daemons/fenced/pacemaker-fenced.c b/daemons/fenced/pacemaker-fenced.c index e6c50d8879..fc53e4f586 100644 --- a/daemons/fenced/pacemaker-fenced.c +++ b/daemons/fenced/pacemaker-fenced.c @@ -1,671 +1,671 @@ /* * Copyright 2009-2025 the Pacemaker project contributors * * The version control history for this file may have further details. * * This source code is licensed under the GNU General Public License version 2 * or later (GPLv2+) WITHOUT ANY WARRANTY. */ #include #include #include #include #include #include #include #include #include #include #include // PRIu32, PRIx32 #include #include #include #include #include #include #include #include #include #include #include #include #define SUMMARY "daemon for executing fencing devices in a Pacemaker cluster" // @TODO This should be guint long long stonith_watchdog_timeout_ms = 0; GList *stonith_watchdog_targets = NULL; static GMainLoop *mainloop = NULL; gboolean stonith_shutdown_flag = FALSE; static qb_ipcs_service_t *ipcs = NULL; static pcmk__output_t *out = NULL; pcmk__supported_format_t formats[] = { PCMK__SUPPORTED_FORMAT_NONE, PCMK__SUPPORTED_FORMAT_TEXT, PCMK__SUPPORTED_FORMAT_XML, { NULL, NULL, NULL } }; static struct { gboolean stand_alone; gchar **log_files; } options; crm_exit_t exit_code = CRM_EX_OK; static void stonith_cleanup(void); static int32_t st_ipc_accept(qb_ipcs_connection_t * c, uid_t uid, gid_t gid) { if (stonith_shutdown_flag) { pcmk__info("Ignoring new client [%d] during shutdown", pcmk__client_pid(c)); return -ECONNREFUSED; } if (pcmk__new_client(c, uid, gid) == NULL) { return -ENOMEM; } return 0; } /* Exit code means? */ static int32_t st_ipc_dispatch(qb_ipcs_connection_t * qbc, void *data, size_t size) { uint32_t id = 0; uint32_t flags = 0; uint32_t call_options = st_opt_none; xmlNode *request = NULL; pcmk__client_t *c = pcmk__find_client(qbc); const char *op = NULL; int rc = pcmk_rc_ok; if (c == NULL) { pcmk__info("Invalid client: %p", qbc); return 0; } request = pcmk__client_data2xml(c, data, &id, &flags); if (request == NULL) { pcmk__ipc_send_ack(c, id, flags, PCMK__XE_NACK, NULL, CRM_EX_PROTOCOL); return 0; } op = pcmk__xe_get(request, PCMK__XA_CRM_TASK); if(pcmk__str_eq(op, CRM_OP_RM_NODE_CACHE, pcmk__str_casei)) { pcmk__xe_set(request, PCMK__XA_T, PCMK__VALUE_STONITH_NG); pcmk__xe_set(request, PCMK__XA_ST_OP, op); pcmk__xe_set(request, PCMK__XA_ST_CLIENTID, c->id); pcmk__xe_set(request, PCMK__XA_ST_CLIENTNAME, pcmk__client_name(c)); pcmk__xe_set(request, PCMK__XA_ST_CLIENTNODE, fenced_get_local_node()); pcmk__cluster_send_message(NULL, pcmk_ipc_fenced, request); pcmk__xml_free(request); return 0; } if (c->name == NULL) { const char *value = pcmk__xe_get(request, PCMK__XA_ST_CLIENTNAME); c->name = pcmk__assert_asprintf("%s.%u", pcmk__s(value, "unknown"), c->pid); } rc = pcmk__xe_get_flags(request, PCMK__XA_ST_CALLOPT, &call_options, st_opt_none); if (rc != pcmk_rc_ok) { pcmk__warn("Couldn't parse options from IPC request: %s", pcmk_rc_str(rc)); } pcmk__trace("Flags %#08" PRIx32 "/%#08x for command %" PRIu32 " from client %s", flags, call_options, id, pcmk__client_name(c)); if (pcmk__is_set(call_options, st_opt_sync_call)) { pcmk__assert(pcmk__is_set(flags, crm_ipc_client_response)); CRM_LOG_ASSERT(c->request_id == 0); /* This means the client has two synchronous events in-flight */ c->request_id = id; /* Reply only to the last one */ } pcmk__xe_set(request, PCMK__XA_ST_CLIENTID, c->id); pcmk__xe_set(request, PCMK__XA_ST_CLIENTNAME, pcmk__client_name(c)); pcmk__xe_set(request, PCMK__XA_ST_CLIENTNODE, fenced_get_local_node()); - crm_log_xml_trace(request, "ipc-received"); + pcmk__log_xml_trace(request, "ipc-received"); stonith_command(c, id, flags, request, NULL); pcmk__xml_free(request); return 0; } /* Error code means? */ static int32_t st_ipc_closed(qb_ipcs_connection_t * c) { pcmk__client_t *client = pcmk__find_client(c); if (client == NULL) { return 0; } pcmk__trace("Connection %p closed", c); pcmk__free_client(client); /* 0 means: yes, go ahead and destroy the connection */ return 0; } static void st_ipc_destroy(qb_ipcs_connection_t * c) { pcmk__trace("Connection %p destroyed", c); st_ipc_closed(c); } static void stonith_peer_callback(xmlNode * msg, void *private_data) { const char *remote_peer = pcmk__xe_get(msg, PCMK__XA_SRC); const char *op = pcmk__xe_get(msg, PCMK__XA_ST_OP); if (pcmk__str_eq(op, STONITH_OP_POKE, pcmk__str_none)) { return; } - crm_log_xml_trace(msg, "Peer[inbound]"); + pcmk__log_xml_trace(msg, "Peer[inbound]"); stonith_command(NULL, 0, 0, msg, remote_peer); } #if SUPPORT_COROSYNC static void handle_cpg_message(cpg_handle_t handle, const struct cpg_name *groupName, uint32_t nodeid, uint32_t pid, void *msg, size_t msg_len) { xmlNode *xml = NULL; const char *from = NULL; char *data = pcmk__cpg_message_data(handle, nodeid, pid, msg, &from); if(data == NULL) { return; } xml = pcmk__xml_parse(data); if (xml == NULL) { pcmk__err("Invalid XML: '%.120s'", data); free(data); return; } pcmk__xe_set(xml, PCMK__XA_SRC, from); stonith_peer_callback(xml, NULL); pcmk__xml_free(xml); free(data); } static void stonith_peer_cs_destroy(gpointer user_data) { pcmk__crit("Lost connection to cluster layer, shutting down"); stonith_shutdown(0); } #endif void do_local_reply(const xmlNode *notify_src, pcmk__client_t *client, int call_options) { /* send callback to originating child */ int local_rc = pcmk_rc_ok; int rid = 0; uint32_t ipc_flags = crm_ipc_server_event; if (pcmk__is_set(call_options, st_opt_sync_call)) { CRM_LOG_ASSERT(client->request_id); rid = client->request_id; client->request_id = 0; ipc_flags = crm_ipc_flags_none; } local_rc = pcmk__ipc_send_xml(client, rid, notify_src, ipc_flags); if (local_rc == pcmk_rc_ok) { pcmk__trace("Sent response %d to client %s", rid, pcmk__client_name(client)); } else { pcmk__warn("%synchronous reply to client %s failed: %s", (pcmk__is_set(call_options, st_opt_sync_call)? "S" : "As"), pcmk__client_name(client), pcmk_rc_str(local_rc)); } } uint64_t get_stonith_flag(const char *name) { if (pcmk__str_eq(name, PCMK__VALUE_ST_NOTIFY_FENCE, pcmk__str_none)) { return st_callback_notify_fence; } else if (pcmk__str_eq(name, STONITH_OP_DEVICE_ADD, pcmk__str_casei)) { return st_callback_device_add; } else if (pcmk__str_eq(name, STONITH_OP_DEVICE_DEL, pcmk__str_casei)) { return st_callback_device_del; } else if (pcmk__str_eq(name, PCMK__VALUE_ST_NOTIFY_HISTORY, pcmk__str_none)) { return st_callback_notify_history; } else if (pcmk__str_eq(name, PCMK__VALUE_ST_NOTIFY_HISTORY_SYNCED, pcmk__str_none)) { return st_callback_notify_history_synced; } return st_callback_unknown; } static void stonith_notify_client(gpointer key, gpointer value, gpointer user_data) { const xmlNode *update_msg = user_data; pcmk__client_t *client = value; const char *type = NULL; CRM_CHECK(client != NULL, return); CRM_CHECK(update_msg != NULL, return); type = pcmk__xe_get(update_msg, PCMK__XA_SUBT); CRM_CHECK(type != NULL, pcmk__log_xml_err(update_msg, "notify"); return); if (client->ipcs == NULL) { pcmk__trace("Skipping client with NULL channel"); return; } if (pcmk__is_set(client->flags, get_stonith_flag(type))) { int rc = pcmk__ipc_send_xml(client, 0, update_msg, crm_ipc_server_event); if (rc != pcmk_rc_ok) { pcmk__warn("%s notification of client %s failed: %s " QB_XS " id=%.8s rc=%d", type, pcmk__client_name(client), pcmk_rc_str(rc), client->id, rc); } else { pcmk__trace("Sent %s notification to client %s", type, pcmk__client_name(client)); } } } void do_stonith_async_timeout_update(const char *client_id, const char *call_id, int timeout) { pcmk__client_t *client = NULL; xmlNode *notify_data = NULL; if (!timeout || !call_id || !client_id) { return; } client = pcmk__find_client_by_id(client_id); if (!client) { return; } notify_data = pcmk__xe_create(NULL, PCMK__XE_ST_ASYNC_TIMEOUT_VALUE); pcmk__xe_set(notify_data, PCMK__XA_T, PCMK__VALUE_ST_ASYNC_TIMEOUT_VALUE); pcmk__xe_set(notify_data, PCMK__XA_ST_CALLID, call_id); pcmk__xe_set_int(notify_data, PCMK__XA_ST_TIMEOUT, timeout); pcmk__trace("timeout update is %d for client %s and call id %s", timeout, client_id, call_id); if (client) { pcmk__ipc_send_xml(client, 0, notify_data, crm_ipc_server_event); } pcmk__xml_free(notify_data); } /*! * \internal * \brief Notify relevant IPC clients of a fencing operation result * * \param[in] type Notification type * \param[in] result Result of fencing operation (assume success if NULL) * \param[in] data If not NULL, add to notification as call data */ void fenced_send_notification(const char *type, const pcmk__action_result_t *result, xmlNode *data) { /* TODO: Standardize the contents of data */ xmlNode *update_msg = pcmk__xe_create(NULL, PCMK__XE_NOTIFY); CRM_LOG_ASSERT(type != NULL); pcmk__xe_set(update_msg, PCMK__XA_T, PCMK__VALUE_ST_NOTIFY); pcmk__xe_set(update_msg, PCMK__XA_SUBT, type); pcmk__xe_set(update_msg, PCMK__XA_ST_OP, type); stonith__xe_set_result(update_msg, result); if (data != NULL) { xmlNode *wrapper = pcmk__xe_create(update_msg, PCMK__XE_ST_CALLDATA); pcmk__xml_copy(wrapper, data); } pcmk__trace("Notifying clients"); pcmk__foreach_ipc_client(stonith_notify_client, update_msg); pcmk__xml_free(update_msg); pcmk__trace("Notify complete"); } /*! * \internal * \brief Send notifications for a configuration change to subscribed clients * * \param[in] op Notification type (\c STONITH_OP_DEVICE_ADD, * \c STONITH_OP_DEVICE_DEL, \c STONITH_OP_LEVEL_ADD, or * \c STONITH_OP_LEVEL_DEL) * \param[in] result Operation result * \param[in] desc Description of what changed (either device ID or string * representation of level * ([])) */ void fenced_send_config_notification(const char *op, const pcmk__action_result_t *result, const char *desc) { xmlNode *notify_data = pcmk__xe_create(NULL, op); pcmk__xe_set(notify_data, PCMK__XA_ST_DEVICE_ID, desc); fenced_send_notification(op, result, notify_data); pcmk__xml_free(notify_data); } /*! * \internal * \brief Check whether a node does watchdog-fencing * * \param[in] node Name of node to check * * \return TRUE if node found in stonith_watchdog_targets * or stonith_watchdog_targets is empty indicating * all nodes are doing watchdog-fencing */ gboolean node_does_watchdog_fencing(const char *node) { return ((stonith_watchdog_targets == NULL) || pcmk__str_in_list(node, stonith_watchdog_targets, pcmk__str_casei)); } void stonith_shutdown(int nsig) { pcmk__info("Terminating with %d clients", pcmk__ipc_client_count()); stonith_shutdown_flag = TRUE; if (mainloop != NULL && g_main_loop_is_running(mainloop)) { g_main_loop_quit(mainloop); } } static void stonith_cleanup(void) { fenced_cib_cleanup(); if (ipcs) { qb_ipcs_destroy(ipcs); } pcmk__cluster_destroy_node_caches(); pcmk__client_cleanup(); free_stonith_remote_op_list(); free_topology_list(); free_device_list(); free_metadata_cache(); fenced_unregister_handlers(); } struct qb_ipcs_service_handlers ipc_callbacks = { .connection_accept = st_ipc_accept, .connection_created = NULL, .msg_process = st_ipc_dispatch, .connection_closed = st_ipc_closed, .connection_destroyed = st_ipc_destroy }; /*! * \internal * \brief Callback for peer status changes * * \param[in] type What changed * \param[in] node What peer had the change * \param[in] data Previous value of what changed */ static void st_peer_update_callback(enum pcmk__node_update type, pcmk__node_status_t *node, const void *data) { if ((type != pcmk__node_update_processes) && !pcmk__is_set(node->flags, pcmk__node_status_remote)) { /* * This is a hack until we can send to a nodeid and/or we fix node name lookups * These messages are ignored in stonith_peer_callback() */ xmlNode *query = pcmk__xe_create(NULL, PCMK__XE_STONITH_COMMAND); pcmk__xe_set(query, PCMK__XA_T, PCMK__VALUE_STONITH_NG); pcmk__xe_set(query, PCMK__XA_ST_OP, STONITH_OP_POKE); pcmk__debug("Broadcasting our uname because of node %" PRIu32, node->cluster_layer_id); pcmk__cluster_send_message(NULL, pcmk_ipc_fenced, query); pcmk__xml_free(query); } } /* @COMPAT Deprecated since 2.1.8. Use pcmk_list_fence_attrs() or * crm_resource --list-options=fencing instead of querying daemon metadata. * * NOTE: pcs (as of at least 0.11.8) uses this */ static int fencer_metadata(void) { const char *name = PCMK__SERVER_FENCED; const char *desc_short = N_("Instance attributes available for all " "\"stonith\"-class resources"); const char *desc_long = N_("Instance attributes available for all " "\"stonith\"-class resources and used by " "Pacemaker's fence daemon"); return pcmk__daemon_metadata(out, name, desc_short, desc_long, pcmk__opt_fencing); } static GOptionEntry entries[] = { { "stand-alone", 's', G_OPTION_FLAG_NONE, G_OPTION_ARG_NONE, &options.stand_alone, N_("Intended for use in regression testing only"), NULL }, { "logfile", 'l', G_OPTION_FLAG_NONE, G_OPTION_ARG_FILENAME_ARRAY, &options.log_files, N_("Send logs to the additional named logfile"), NULL }, { NULL } }; static GOptionContext * build_arg_context(pcmk__common_args_t *args, GOptionGroup **group) { GOptionContext *context = NULL; context = pcmk__build_arg_context(args, "text (default), xml", group, NULL); pcmk__add_main_args(context, entries); return context; } int main(int argc, char **argv) { int rc = pcmk_rc_ok; pcmk_cluster_t *cluster = NULL; crm_ipc_t *old_instance = NULL; GError *error = NULL; GOptionGroup *output_group = NULL; pcmk__common_args_t *args = pcmk__new_common_args(SUMMARY); gchar **processed_args = pcmk__cmdline_preproc(argv, "l"); GOptionContext *context = build_arg_context(args, &output_group); crm_log_preinit(NULL, argc, argv); pcmk__register_formats(output_group, formats); if (!g_option_context_parse_strv(context, &processed_args, &error)) { exit_code = CRM_EX_USAGE; goto done; } rc = pcmk__output_new(&out, args->output_ty, args->output_dest, argv); if (rc != pcmk_rc_ok) { exit_code = CRM_EX_ERROR; g_set_error(&error, PCMK__EXITC_ERROR, exit_code, "Error creating output format %s: %s", args->output_ty, pcmk_rc_str(rc)); goto done; } if (args->version) { out->version(out, false); goto done; } if ((g_strv_length(processed_args) >= 2) && pcmk__str_eq(processed_args[1], "metadata", pcmk__str_none)) { rc = fencer_metadata(); if (rc != pcmk_rc_ok) { exit_code = CRM_EX_FATAL; g_set_error(&error, PCMK__EXITC_ERROR, exit_code, "Unable to display metadata: %s", pcmk_rc_str(rc)); } goto done; } // Open additional log files pcmk__add_logfiles(options.log_files, out); crm_log_init(NULL, LOG_INFO + args->verbosity, TRUE, (args->verbosity > 0), argc, argv, FALSE); pcmk__notice("Starting Pacemaker fencer"); old_instance = crm_ipc_new("stonith-ng", 0); if (old_instance == NULL) { /* crm_ipc_new() will have already logged an error message with * pcmk__err() */ exit_code = CRM_EX_FATAL; goto done; } if (pcmk__connect_generic_ipc(old_instance) == pcmk_rc_ok) { // IPC endpoint already up crm_ipc_close(old_instance); crm_ipc_destroy(old_instance); pcmk__crit("Aborting start-up because another fencer instance is " "already active"); goto done; } else { // Not up or not authentic, we'll proceed either way crm_ipc_destroy(old_instance); old_instance = NULL; } mainloop_add_signal(SIGTERM, stonith_shutdown); pcmk__cluster_init_node_caches(); rc = fenced_scheduler_init(); if (rc != pcmk_rc_ok) { exit_code = CRM_EX_FATAL; g_set_error(&error, PCMK__EXITC_ERROR, exit_code, "Error initializing scheduler data: %s", pcmk_rc_str(rc)); goto done; } cluster = pcmk_cluster_new(); #if SUPPORT_COROSYNC if (pcmk_get_cluster_layer() == pcmk_cluster_layer_corosync) { pcmk_cluster_set_destroy_fn(cluster, stonith_peer_cs_destroy); pcmk_cpg_set_deliver_fn(cluster, handle_cpg_message); pcmk_cpg_set_confchg_fn(cluster, pcmk__cpg_confchg_cb); } #endif // SUPPORT_COROSYNC pcmk__cluster_set_status_callback(&st_peer_update_callback); if (pcmk_cluster_connect(cluster) != pcmk_rc_ok) { exit_code = CRM_EX_FATAL; pcmk__crit("Cannot sign in to the cluster... terminating"); goto done; } fenced_set_local_node(cluster->priv->node_name); if (!options.stand_alone) { setup_cib(); } init_device_list(); init_topology_list(); pcmk__serve_fenced_ipc(&ipcs, &ipc_callbacks); // Create the mainloop and run it... mainloop = g_main_loop_new(NULL, FALSE); pcmk__notice("Pacemaker fencer successfully started and accepting " "connections"); g_main_loop_run(mainloop); done: g_strfreev(processed_args); pcmk__free_arg_context(context); g_strfreev(options.log_files); stonith_cleanup(); pcmk_cluster_free(cluster); fenced_scheduler_cleanup(); pcmk__output_and_clear_error(&error, out); if (out != NULL) { out->finish(out, exit_code, true, NULL); pcmk__output_free(out); } pcmk__unregister_formats(); crm_exit(exit_code); } diff --git a/include/crm/common/logging_internal.h b/include/crm/common/logging_internal.h index cc44e0db0c..2b01ba0e03 100644 --- a/include/crm/common/logging_internal.h +++ b/include/crm/common/logging_internal.h @@ -1,403 +1,413 @@ /* * Copyright 2015-2025 the Pacemaker project contributors * * The version control history for this file may have further details. * * This source code is licensed under the GNU General Public License version 2 * or later (GPLv2+) WITHOUT ANY WARRANTY. */ #ifndef PCMK__CRM_COMMON_LOGGING_INTERNAL__H #define PCMK__CRM_COMMON_LOGGING_INTERNAL__H #include #include // pcmk__is_set() #include #include #ifdef __cplusplus extern "C" { #endif /* Define custom log priorities. * * syslog(3) uses int for priorities, but libqb's struct qb_log_callsite uses * uint8_t, so make sure they fit in the latter. */ #ifndef PCMK__LOG_TRACE /*! * \internal * \brief Log level for tracing (less importance than \c LOG_DEBUG messages) * * \note This value must stay the same as \c LOG_TRACE until the latter is * dropped. Be mindful of public API functions that may pass arbitrary * integer log levels as well. */ #define PCMK__LOG_TRACE (LOG_DEBUG + 1) #endif // PCMK__LOG_TRACE #ifndef PCMK__LOG_STDOUT /*! * \internal * \brief Request to print message to \c stdout instead of logging it * * Some callees print nothing when this is the log level. * * \note This value must stay the same as \c LOG_STDOUT until the latter is * dropped. Be mindful of public API functions that may pass arbitrary * integer log levels as well. */ #define PCMK__LOG_STDOUT 254 #endif // PCMK__LOG_STDOUT #ifndef PCMK__LOG_NEVER /*! * \internal * \brief Request not to print or log message anywhere * * \note This value must stay the same as \c LOG_NEVER until the latter is * dropped. Be mindful of public API functions that may pass arbitrary * integer log levels as well. */ #define PCMK__LOG_NEVER 255 #endif // PCMK__LOG_NEVER /*! * \internal * \brief Log a message at \c LOG_EMERG level * * \param[in] fmt \c printf() format string for log message * \param[in] args Format string arguments */ #define pcmk__emerg(fmt, args...) qb_log(LOG_EMERG, fmt, ##args) /*! * \internal * \brief Log a message at \c LOG_CRIT level * * \param[in] fmt \c printf() format string for log message * \param[in] args Format string arguments */ #define pcmk__crit(fmt, args...) qb_log(LOG_CRIT, fmt, ##args) /*! * \internal * \brief Log a message at \c LOG_ERR level * * \param[in] fmt \c printf() format string for log message * \param[in] args Format string arguments */ #define pcmk__err(fmt, args...) qb_log(LOG_ERR, fmt, ##args) /*! * \internal * \brief Log a message at \c LOG_WARN level * * \param[in] fmt \c printf() format string for log message * \param[in] args Format string arguments */ #define pcmk__warn(fmt, args...) qb_log(LOG_WARNING, fmt, ##args) /*! * \internal * \brief Log a message at \c LOG_NOTICE level * * \param[in] fmt \c printf() format string for log message * \param[in] args Format string arguments */ #define pcmk__notice(fmt, args...) qb_log(LOG_NOTICE, fmt, ##args) /*! * \internal * \brief Log a message at \c LOG_INFO level * * \param[in] fmt \c printf() format string for log message * \param[in] args Format string arguments */ #define pcmk__info(fmt, args...) qb_log(LOG_INFO, fmt, ##args) /*! * \internal * \brief Log a message at \c LOG_DEBUG level * * \param[in] fmt \c printf() format string for log message * \param[in] args Format string arguments */ #define pcmk__debug(fmt, args...) do_crm_log_unlikely(LOG_DEBUG, fmt, ##args) /*! * \internal * \brief Log a message at \c PCMK__LOG_TRACE level * * \param[in] fmt \c printf() format string for log message * \param[in] args Format string arguments */ #define pcmk__trace(fmt, args...) do_crm_log_unlikely(LOG_TRACE, fmt, ##args) /*! * \internal * \brief Log XML line-by-line in a formatted fashion at \c LOG_ERR level * * \param[in] prefix Prefix for each line * \param[in] xml XML to log */ #define pcmk__log_xml_err(xml, prefix) do_crm_log_xml(LOG_ERR, prefix, xml) /*! * \internal * \brief Log XML line-by-line in a formatted fashion at \c LOG_WARNING level * * \param[in] prefix Prefix for each line * \param[in] xml XML to log */ #define pcmk__log_xml_warn(xml, prefix) do_crm_log_xml(LOG_WARNING, prefix, xml) /*! * \internal * \brief Log XML line-by-line in a formatted fashion at \c LOG_NOTICE level * * \param[in] prefix Prefix for each line * \param[in] xml XML to log */ #define pcmk__log_xml_notice(xml, prefix) \ do_crm_log_xml(LOG_NOTICE, prefix, xml) /*! * \internal * \brief Log XML line-by-line in a formatted fashion at \c LOG_INFO level * * \param[in] prefix Prefix for each line * \param[in] xml XML to log */ #define pcmk__log_xml_info(xml, prefix) do_crm_log_xml(LOG_INFO, prefix, xml) /*! * \internal * \brief Log XML line-by-line in a formatted fashion at \c LOG_DEBUG level * * \param[in] prefix Prefix for each line * \param[in] xml XML to log */ #define pcmk__log_xml_debug(xml, prefix) do_crm_log_xml(LOG_DEBUG, prefix, xml) +/*! + * \internal + * \brief Log XML line-by-line in a formatted fashion at \c PCMK__LOG_TRACE level + * + * \param[in] prefix Prefix for each line + * \param[in] xml XML to log + */ +#define pcmk__log_xml_trace(xml, prefix) \ + do_crm_log_xml(PCMK__LOG_TRACE, prefix, xml) + /* Some warnings are too noisy when logged every time a given function is called * (for example, using a deprecated feature). As an alternative, we allow * warnings to be logged once per invocation of the calling program. Each of * those warnings needs a flag defined here. */ enum pcmk__warnings { pcmk__wo_blind = (1 << 0), pcmk__wo_record_pending = (1 << 1), pcmk__wo_require_all = (1 << 4), pcmk__wo_order_score = (1 << 5), pcmk__wo_group_order = (1 << 11), pcmk__wo_group_coloc = (1 << 12), pcmk__wo_set_ordering = (1 << 15), pcmk__wo_rdisc_enabled = (1 << 16), pcmk__wo_op_attr_expr = (1 << 19), pcmk__wo_clone_master_max = (1 << 23), pcmk__wo_clone_master_node_max = (1 << 24), pcmk__wo_master_role = (1 << 26), pcmk__wo_slave_role = (1 << 27), }; /*! * \internal * \brief Log a warning once per invocation of calling program * * \param[in] wo_flag enum pcmk__warnings value for this warning * \param[in] fmt... printf(3)-style format and arguments */ #define pcmk__warn_once(wo_flag, fmt...) do { \ if (!pcmk__is_set(pcmk__warnings, wo_flag)) { \ if (wo_flag == pcmk__wo_blind) { \ pcmk__warn(fmt); \ } else { \ pcmk__config_warn(fmt); \ } \ pcmk__warnings = pcmk__set_flags_as(__func__, __LINE__, \ PCMK__LOG_TRACE, \ "Warn-once", "logging", \ pcmk__warnings, \ (wo_flag), #wo_flag); \ } \ } while (0) typedef void (*pcmk__config_error_func) (void *ctx, const char *msg, ...) G_GNUC_PRINTF(2, 3); typedef void (*pcmk__config_warning_func) (void *ctx, const char *msg, ...) G_GNUC_PRINTF(2, 3); extern pcmk__config_error_func pcmk__config_error_handler; extern pcmk__config_warning_func pcmk__config_warning_handler; extern void *pcmk__config_error_context; extern void *pcmk__config_warning_context; void pcmk__set_config_error_handler(pcmk__config_error_func error_handler, void *error_context); void pcmk__set_config_warning_handler(pcmk__config_warning_func warning_handler, void *warning_context); /* Pacemaker library functions set this when a configuration error is found, * which turns on extra messages at the end of processing. */ extern bool pcmk__config_has_error; /* Pacemaker library functions set this when a configuration warning is found, * which turns on extra messages at the end of processing. */ extern bool pcmk__config_has_warning; /*! * \internal * \brief Log an error and make crm_verify return failure status * * \param[in] fmt... printf(3)-style format string and arguments */ #define pcmk__config_err(fmt...) do { \ pcmk__config_has_error = true; \ if (pcmk__config_error_handler == NULL) { \ pcmk__err(fmt); \ } else { \ pcmk__config_error_handler(pcmk__config_error_context, fmt); \ } \ } while (0) /*! * \internal * \brief Log a warning and make crm_verify return failure status * * \param[in] fmt... printf(3)-style format string and arguments */ #define pcmk__config_warn(fmt...) do { \ pcmk__config_has_warning = true; \ if (pcmk__config_warning_handler == NULL) { \ pcmk__warn(fmt); \ } else { \ pcmk__config_warning_handler(pcmk__config_warning_context, fmt);\ } \ } while (0) /*! * \internal * \brief Execute code depending on whether trace logging is enabled * * This is similar to \p do_crm_log_unlikely() except instead of logging, it * selects one of two code blocks to execute. * * \param[in] if_action Code block to execute if trace logging is enabled * \param[in] else_action Code block to execute if trace logging is not enabled * * \note Neither \p if_action nor \p else_action can contain a \p break or * \p continue statement. */ #define pcmk__if_tracing(if_action, else_action) do { \ static struct qb_log_callsite *trace_cs = NULL; \ \ if (trace_cs == NULL) { \ trace_cs = qb_log_callsite_get(__func__, __FILE__, \ "if_tracing", PCMK__LOG_TRACE, \ __LINE__, crm_trace_nonlog); \ } \ if (crm_is_callsite_active(trace_cs, PCMK__LOG_TRACE, \ crm_trace_nonlog)) { \ if_action; \ } else { \ else_action; \ } \ } while (0) /*! * \internal * \brief Log XML changes line-by-line in a formatted fashion * * \param[in] level Priority at which to log the messages * \param[in] xml XML to log * * \note This does nothing when \p level is \c PCMK__LOG_STDOUT or * \c PCMK__LOG_NEVER. */ #define pcmk__log_xml_changes(level, xml) do { \ uint8_t _level = pcmk__clip_log_level(level); \ static struct qb_log_callsite *xml_cs = NULL; \ \ switch (_level) { \ case PCMK__LOG_STDOUT: \ case PCMK__LOG_NEVER: \ break; \ default: \ if (xml_cs == NULL) { \ xml_cs = qb_log_callsite_get(__func__, __FILE__, \ "xml-changes", _level, \ __LINE__, 0); \ } \ if (crm_is_callsite_active(xml_cs, _level, 0)) { \ pcmk__log_xml_changes_as(__FILE__, __func__, __LINE__, \ 0, _level, xml); \ } \ break; \ } \ } while(0) /*! * \internal * \brief Log an XML patchset line-by-line in a formatted fashion * * \param[in] level Priority at which to log the messages * \param[in] patchset XML patchset to log * * \note This does nothing when \p level is \c PCMK__LOG_STDOUT or * \c PCMK__LOG_NEVER. */ #define pcmk__log_xml_patchset(level, patchset) do { \ uint8_t _level = pcmk__clip_log_level(level); \ static struct qb_log_callsite *xml_cs = NULL; \ \ switch (_level) { \ case PCMK__LOG_STDOUT: \ case PCMK__LOG_NEVER: \ break; \ default: \ if (xml_cs == NULL) { \ xml_cs = qb_log_callsite_get(__func__, __FILE__, \ "xml-patchset", _level, \ __LINE__, 0); \ } \ if (crm_is_callsite_active(xml_cs, _level, 0)) { \ pcmk__log_xml_patchset_as(__FILE__, __func__, __LINE__, \ 0, _level, patchset); \ } \ break; \ } \ } while(0) void pcmk__log_xml_changes_as(const char *file, const char *function, uint32_t line, uint32_t tags, uint8_t level, const xmlNode *xml); void pcmk__log_xml_patchset_as(const char *file, const char *function, uint32_t line, uint32_t tags, uint8_t level, const xmlNode *patchset); /*! * \internal * \brief Initialize logging for command line tools * * \param[in] name The name of the program * \param[in] verbosity How verbose to be in logging * * \note \p verbosity is not the same as the logging level (LOG_ERR, etc.). */ void pcmk__cli_init_logging(const char *name, unsigned int verbosity); int pcmk__add_logfile(const char *filename); void pcmk__add_logfiles(gchar **log_files, pcmk__output_t *out); void pcmk__free_common_logger(void); #ifdef __cplusplus } #endif #endif // PCMK__CRM_COMMON_LOGGING_INTERNAL__H diff --git a/lib/cib/cib_attrs.c b/lib/cib/cib_attrs.c index 87b81a9e44..31ec789cb5 100644 --- a/lib/cib/cib_attrs.c +++ b/lib/cib/cib_attrs.c @@ -1,671 +1,671 @@ /* * Copyright 2004-2025 the Pacemaker project contributors * * The version control history for this file may have further details. * * This source code is licensed under the GNU Lesser General Public License * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. */ #include #include #include #include #include #include #include #include #include #include #include // crm_create_nvpair_xml() #include #include #include #include static pcmk__output_t * new_output_object(const char *ty) { int rc = pcmk_rc_ok; pcmk__output_t *out = NULL; const char* argv[] = { "", NULL }; pcmk__supported_format_t formats[] = { PCMK__SUPPORTED_FORMAT_LOG, PCMK__SUPPORTED_FORMAT_TEXT, { NULL, NULL, NULL } }; pcmk__register_formats(NULL, formats); rc = pcmk__output_new(&out, ty, NULL, (char**)argv); if ((rc != pcmk_rc_ok) || (out == NULL)) { pcmk__err("Can't out due to internal error: %s", pcmk_rc_str(rc)); return NULL; } return out; } static int find_attr(cib_t *cib, const char *section, const char *node_uuid, const char *attr_set_type, const char *set_name, const char *attr_id, const char *attr_name, const char *user_name, xmlNode **result) { int rc = pcmk_rc_ok; const char *xpath_base = NULL; GString *xpath = NULL; xmlNode *xml_search = NULL; const char *set_type = NULL; const char *node_type = NULL; if (attr_set_type) { set_type = attr_set_type; } else { set_type = PCMK_XE_INSTANCE_ATTRIBUTES; } if (pcmk__str_eq(section, PCMK_XE_CRM_CONFIG, pcmk__str_casei)) { node_uuid = NULL; set_type = PCMK_XE_CLUSTER_PROPERTY_SET; } else if (pcmk__strcase_any_of(section, PCMK_XE_OP_DEFAULTS, PCMK_XE_RSC_DEFAULTS, NULL)) { node_uuid = NULL; set_type = PCMK_XE_META_ATTRIBUTES; } else if (pcmk__str_eq(section, PCMK_XE_TICKETS, pcmk__str_casei)) { node_uuid = NULL; section = PCMK_XE_STATUS; node_type = PCMK_XE_TICKETS; } else if (node_uuid == NULL) { return EINVAL; } xpath_base = pcmk_cib_xpath_for(section); if (xpath_base == NULL) { pcmk__warn("%s CIB section not known", section); return ENOMSG; } xpath = g_string_sized_new(1024); g_string_append(xpath, xpath_base); if (pcmk__str_eq(node_type, PCMK_XE_TICKETS, pcmk__str_casei)) { pcmk__g_strcat(xpath, "//", node_type, NULL); } else if (node_uuid) { const char *node_type = PCMK_XE_NODE; if (pcmk__str_eq(section, PCMK_XE_STATUS, pcmk__str_casei)) { node_type = PCMK__XE_NODE_STATE; set_type = PCMK__XE_TRANSIENT_ATTRIBUTES; } pcmk__g_strcat(xpath, "//", node_type, "[@" PCMK_XA_ID "='", node_uuid, "']", NULL); } pcmk__g_strcat(xpath, "//", set_type, NULL); if (set_name) { pcmk__g_strcat(xpath, "[@" PCMK_XA_ID "='", set_name, "']", NULL); } g_string_append(xpath, "//nvpair"); if (attr_id && attr_name) { pcmk__g_strcat(xpath, "[@" PCMK_XA_ID "='", attr_id, "' " "and @" PCMK_XA_NAME "='", attr_name, "']", NULL); } else if (attr_id) { pcmk__g_strcat(xpath, "[@" PCMK_XA_ID "='", attr_id, "']", NULL); } else if (attr_name) { pcmk__g_strcat(xpath, "[@" PCMK_XA_NAME "='", attr_name, "']", NULL); } rc = cib_internal_op(cib, PCMK__CIB_REQUEST_QUERY, NULL, (const char *) xpath->str, NULL, &xml_search, cib_sync_call|cib_xpath, user_name); rc = pcmk_legacy2rc(rc); if (rc != pcmk_rc_ok) { pcmk__trace("Query failed for attribute %s (section=%s, node=%s, " "set=%s, xpath=%s): %s", attr_name, section, pcmk__s(node_uuid, ""), pcmk__s(set_name, ""), xpath->str, pcmk_rc_str(rc)); } else { pcmk__log_xml_debug(xml_search, "Match"); } g_string_free(xpath, TRUE); *result = xml_search; return rc; } static int handle_multiples(pcmk__output_t *out, xmlNode *search, const char *attr_name) { if ((search != NULL) && (search->children != NULL)) { pcmk__warn_multiple_name_matches(out, search, attr_name); return ENOTUNIQ; } else { return pcmk_rc_ok; } } int cib__update_node_attr(pcmk__output_t *out, cib_t *cib, int call_options, const char *section, const char *node_uuid, const char *set_type, const char *set_name, const char *attr_id, const char *attr_name, const char *attr_value, const char *user_name, const char *node_type) { const char *tag = NULL; int rc = pcmk_rc_ok; xmlNode *xml_top = NULL; xmlNode *xml_obj = NULL; xmlNode *xml_search = NULL; char *local_attr_id = NULL; char *local_set_name = NULL; CRM_CHECK((out != NULL) && (cib != NULL) && (section != NULL) && ((attr_id != NULL) || (attr_name != NULL)) && (attr_value != NULL), return EINVAL); rc = find_attr(cib, section, node_uuid, set_type, set_name, attr_id, attr_name, user_name, &xml_search); if (rc == pcmk_rc_ok) { if (handle_multiples(out, xml_search, attr_name) == ENOTUNIQ) { pcmk__xml_free(xml_search); return ENOTUNIQ; } else { local_attr_id = pcmk__xe_get_copy(xml_search, PCMK_XA_ID); attr_id = local_attr_id; pcmk__xml_free(xml_search); goto do_modify; } } else if (rc != ENXIO) { pcmk__xml_free(xml_search); return rc; /* } else if(attr_id == NULL) { */ /* return EINVAL; */ } else { pcmk__xml_free(xml_search); pcmk__trace("%s does not exist, create it", attr_name); if (pcmk__str_eq(section, PCMK_XE_TICKETS, pcmk__str_casei)) { node_uuid = NULL; section = PCMK_XE_STATUS; node_type = PCMK_XE_TICKETS; xml_top = pcmk__xe_create(xml_obj, PCMK_XE_STATUS); xml_obj = pcmk__xe_create(xml_top, PCMK_XE_TICKETS); } else if (pcmk__str_eq(section, PCMK_XE_NODES, pcmk__str_casei)) { if (node_uuid == NULL) { return EINVAL; } if (pcmk__str_eq(node_type, PCMK_VALUE_REMOTE, pcmk__str_casei)) { xml_top = pcmk__xe_create(xml_obj, PCMK_XE_NODES); xml_obj = pcmk__xe_create(xml_top, PCMK_XE_NODE); pcmk__xe_set(xml_obj, PCMK_XA_TYPE, PCMK_VALUE_REMOTE); pcmk__xe_set(xml_obj, PCMK_XA_ID, node_uuid); pcmk__xe_set(xml_obj, PCMK_XA_UNAME, node_uuid); } else { tag = PCMK_XE_NODE; } } else if (pcmk__str_eq(section, PCMK_XE_STATUS, pcmk__str_casei)) { tag = PCMK__XE_TRANSIENT_ATTRIBUTES; if (node_uuid == NULL) { return EINVAL; } xml_top = pcmk__xe_create(xml_obj, PCMK__XE_NODE_STATE); pcmk__xe_set(xml_top, PCMK_XA_ID, node_uuid); xml_obj = xml_top; } else { tag = section; node_uuid = NULL; } if (set_name == NULL) { if (pcmk__str_eq(section, PCMK_XE_CRM_CONFIG, pcmk__str_casei)) { local_set_name = pcmk__str_copy(PCMK_VALUE_CIB_BOOTSTRAP_OPTIONS); } else if (pcmk__str_eq(node_type, PCMK_XE_TICKETS, pcmk__str_casei)) { local_set_name = pcmk__assert_asprintf("%s-%s", section, PCMK_XE_TICKETS); } else if (node_uuid) { local_set_name = pcmk__assert_asprintf("%s-%s", section, node_uuid); if (set_type) { char *tmp_set_name = local_set_name; local_set_name = pcmk__assert_asprintf("%s-%s", tmp_set_name, set_type); free(tmp_set_name); } } else { local_set_name = pcmk__assert_asprintf("%s-options", section); } set_name = local_set_name; } if (attr_id == NULL) { local_attr_id = pcmk__assert_asprintf("%s-%s", set_name, attr_name); pcmk__xml_sanitize_id(local_attr_id); attr_id = local_attr_id; } else if (attr_name == NULL) { attr_name = attr_id; } pcmk__trace("Creating %s/%s", section, tag); if (tag != NULL) { xml_obj = pcmk__xe_create(xml_obj, tag); pcmk__xe_set(xml_obj, PCMK_XA_ID, node_uuid); if (xml_top == NULL) { xml_top = xml_obj; } } if ((node_uuid == NULL) && !pcmk__str_eq(node_type, PCMK_XE_TICKETS, pcmk__str_casei)) { if (pcmk__str_eq(section, PCMK_XE_CRM_CONFIG, pcmk__str_casei)) { xml_obj = pcmk__xe_create(xml_obj, PCMK_XE_CLUSTER_PROPERTY_SET); } else { xml_obj = pcmk__xe_create(xml_obj, PCMK_XE_META_ATTRIBUTES); } } else if (set_type) { xml_obj = pcmk__xe_create(xml_obj, set_type); } else { xml_obj = pcmk__xe_create(xml_obj, PCMK_XE_INSTANCE_ATTRIBUTES); } pcmk__xe_set(xml_obj, PCMK_XA_ID, set_name); if (xml_top == NULL) { xml_top = xml_obj; } } do_modify: xml_obj = crm_create_nvpair_xml(xml_obj, attr_id, attr_name, attr_value); if (xml_top == NULL) { xml_top = xml_obj; } - crm_log_xml_trace(xml_top, "update_attr"); + pcmk__log_xml_trace(xml_top, "update_attr"); rc = cib_internal_op(cib, PCMK__CIB_REQUEST_MODIFY, NULL, section, xml_top, NULL, call_options, user_name); if (!pcmk__is_set(call_options, cib_sync_call) && (cib->variant != cib_file) && (rc >= 0)) { // For async call, positive rc is the call ID (file always synchronous) rc = pcmk_rc_ok; } else { rc = pcmk_legacy2rc(rc); } if (rc != pcmk_rc_ok) { out->err(out, "Error setting %s=%s (section=%s, set=%s): %s", attr_name, attr_value, section, pcmk__s(set_name, ""), pcmk_rc_str(rc)); pcmk__log_xml_info(xml_top, "Update"); } free(local_set_name); free(local_attr_id); pcmk__xml_free(xml_top); return rc; } int cib__get_node_attrs(pcmk__output_t *out, cib_t *cib, const char *section, const char *node_uuid, const char *set_type, const char *set_name, const char *attr_id, const char *attr_name, const char *user_name, xmlNode **result) { int rc = pcmk_rc_ok; pcmk__assert(result != NULL); CRM_CHECK(section != NULL, return EINVAL); *result = NULL; rc = find_attr(cib, section, node_uuid, set_type, set_name, attr_id, attr_name, user_name, result); if (rc != pcmk_rc_ok) { pcmk__trace("Query failed for attribute %s (section=%s node=%s " "set=%s): %s", pcmk__s(attr_name, "with unspecified name"), section, pcmk__s(set_name, ""), pcmk__s(node_uuid, ""), pcmk_rc_str(rc)); } return rc; } int cib__delete_node_attr(pcmk__output_t *out, cib_t *cib, int options, const char *section, const char *node_uuid, const char *set_type, const char *set_name, const char *attr_id, const char *attr_name, const char *attr_value, const char *user_name) { int rc = pcmk_rc_ok; xmlNode *xml_obj = NULL; xmlNode *xml_search = NULL; char *local_attr_id = NULL; CRM_CHECK(section != NULL, return EINVAL); CRM_CHECK(attr_name != NULL || attr_id != NULL, return EINVAL); if (attr_id == NULL) { rc = find_attr(cib, section, node_uuid, set_type, set_name, attr_id, attr_name, user_name, &xml_search); if (rc != pcmk_rc_ok || handle_multiples(out, xml_search, attr_name) == ENOTUNIQ) { pcmk__xml_free(xml_search); return rc; } else { local_attr_id = pcmk__xe_get_copy(xml_search, PCMK_XA_ID); attr_id = local_attr_id; pcmk__xml_free(xml_search); } } xml_obj = crm_create_nvpair_xml(NULL, attr_id, attr_name, attr_value); rc = cib_internal_op(cib, PCMK__CIB_REQUEST_DELETE, NULL, section, xml_obj, NULL, options, user_name); if (!pcmk__is_set(options, cib_sync_call) && (cib->variant != cib_file) && (rc >= 0)) { // For async call, positive rc is the call ID (file always synchronous) rc = pcmk_rc_ok; } else { rc = pcmk_legacy2rc(rc); } if (rc == pcmk_rc_ok) { out->info(out, "Deleted %s %s: id=%s%s%s%s%s", section, node_uuid ? "attribute" : "option", local_attr_id, set_name ? " set=" : "", set_name ? set_name : "", attr_name ? " name=" : "", attr_name ? attr_name : ""); } free(local_attr_id); pcmk__xml_free(xml_obj); return rc; } int find_nvpair_attr_delegate(cib_t *cib, const char *attr, const char *section, const char *node_uuid, const char *attr_set_type, const char *set_name, const char *attr_id, const char *attr_name, gboolean to_console, char **value, const char *user_name) { pcmk__output_t *out = NULL; xmlNode *xml_search = NULL; int rc = pcmk_ok; out = new_output_object(to_console ? "text" : "log"); if (out == NULL) { return pcmk_err_generic; } rc = find_attr(cib, section, node_uuid, attr_set_type, set_name, attr_id, attr_name, user_name, &xml_search); if (rc == pcmk_rc_ok) { rc = handle_multiples(out, xml_search, attr_name); if (rc == pcmk_rc_ok) { pcmk__str_update(value, pcmk__xe_get(xml_search, attr)); } } out->finish(out, CRM_EX_OK, true, NULL); pcmk__xml_free(xml_search); pcmk__output_free(out); return pcmk_rc2legacy(rc); } int update_attr_delegate(cib_t *cib, int call_options, const char *section, const char *node_uuid, const char *set_type, const char *set_name, const char *attr_id, const char *attr_name, const char *attr_value, gboolean to_console, const char *user_name, const char *node_type) { pcmk__output_t *out = NULL; int rc = pcmk_ok; out = new_output_object(to_console ? "text" : "log"); if (out == NULL) { return pcmk_err_generic; } rc = cib__update_node_attr(out, cib, call_options, section, node_uuid, set_type, set_name, attr_id, attr_name, attr_value, user_name, node_type); out->finish(out, CRM_EX_OK, true, NULL); pcmk__output_free(out); return pcmk_rc2legacy(rc); } int read_attr_delegate(cib_t *cib, const char *section, const char *node_uuid, const char *set_type, const char *set_name, const char *attr_id, const char *attr_name, char **attr_value, gboolean to_console, const char *user_name) { pcmk__output_t *out = NULL; xmlNode *result = NULL; int rc = pcmk_ok; out = new_output_object(to_console ? "text" : "log"); if (out == NULL) { return pcmk_err_generic; } rc = cib__get_node_attrs(out, cib, section, node_uuid, set_type, set_name, attr_id, attr_name, user_name, &result); if (rc == pcmk_rc_ok) { if (result->children == NULL) { pcmk__str_update(attr_value, pcmk__xe_get(result, PCMK_XA_VALUE)); } else { rc = ENOTUNIQ; } } out->finish(out, CRM_EX_OK, true, NULL); pcmk__xml_free(result); pcmk__output_free(out); return pcmk_rc2legacy(rc); } int delete_attr_delegate(cib_t *cib, int options, const char *section, const char *node_uuid, const char *set_type, const char *set_name, const char *attr_id, const char *attr_name, const char *attr_value, gboolean to_console, const char *user_name) { pcmk__output_t *out = NULL; int rc = pcmk_ok; out = new_output_object(to_console ? "text" : "log"); if (out == NULL) { return pcmk_err_generic; } rc = cib__delete_node_attr(out, cib, options, section, node_uuid, set_type, set_name, attr_id, attr_name, attr_value, user_name); out->finish(out, CRM_EX_OK, true, NULL); pcmk__output_free(out); return pcmk_rc2legacy(rc); } /*! * \internal * \brief Parse node UUID from search result * * \param[in] result XML search result * \param[out] uuid If non-NULL, where to store parsed UUID * \param[out] is_remote If non-NULL, set TRUE if result is remote node * * \return pcmk_ok if UUID was successfully parsed, -ENXIO otherwise */ static int get_uuid_from_result(const xmlNode *result, char **uuid, int *is_remote) { int rc = -ENXIO; const char *parsed_uuid = NULL; int parsed_is_remote = FALSE; if (result == NULL) { return rc; } /* If there are multiple results, the first is sufficient */ if (pcmk__xe_is(result, PCMK__XE_XPATH_QUERY)) { result = pcmk__xe_first_child(result, NULL, NULL, NULL); CRM_CHECK(result != NULL, return rc); } if (pcmk__xe_is(result, PCMK_XE_NODE)) { // Result is PCMK_XE_NODE element from PCMK_XE_NODES section if (pcmk__str_eq(pcmk__xe_get(result, PCMK_XA_TYPE), PCMK_VALUE_REMOTE, pcmk__str_casei)) { parsed_uuid = pcmk__xe_get(result, PCMK_XA_UNAME); parsed_is_remote = TRUE; } else { parsed_uuid = pcmk__xe_id(result); parsed_is_remote = FALSE; } } else if (pcmk__xe_is(result, PCMK_XE_PRIMITIVE)) { /* Result is for ocf:pacemaker:remote resource */ parsed_uuid = pcmk__xe_id(result); parsed_is_remote = TRUE; } else if (pcmk__xe_is(result, PCMK_XE_NVPAIR)) { /* Result is PCMK_META_REMOTE_NODE parameter of for guest * node */ parsed_uuid = pcmk__xe_get(result, PCMK_XA_VALUE); parsed_is_remote = TRUE; } else if (pcmk__xe_is(result, PCMK__XE_NODE_STATE)) { // Result is PCMK__XE_NODE_STATE element from PCMK_XE_STATUS section parsed_uuid = pcmk__xe_get(result, PCMK_XA_UNAME); if (pcmk__xe_attr_is_true(result, PCMK_XA_REMOTE_NODE)) { parsed_is_remote = TRUE; } } if (parsed_uuid) { if (uuid) { *uuid = strdup(parsed_uuid); } if (is_remote) { *is_remote = parsed_is_remote; } rc = pcmk_ok; } return rc; } /* Search string to find a node by name, as: * - cluster or remote node in nodes section * - remote node in resources section * - guest node in resources section * - orphaned remote node or bundle guest node in status section */ #define XPATH_UPPER_TRANS "ABCDEFGHIJKLMNOPQRSTUVWXYZ" #define XPATH_LOWER_TRANS "abcdefghijklmnopqrstuvwxyz" #define XPATH_NODE \ "/" PCMK_XE_CIB "/" PCMK_XE_CONFIGURATION "/" PCMK_XE_NODES \ "/" PCMK_XE_NODE "[translate(@" PCMK_XA_UNAME ",'" XPATH_UPPER_TRANS "','" XPATH_LOWER_TRANS "') ='%s']" \ "|/" PCMK_XE_CIB "/" PCMK_XE_CONFIGURATION "/" PCMK_XE_RESOURCES \ "/" PCMK_XE_PRIMITIVE \ "[@class='ocf'][@provider='pacemaker'][@type='remote'][translate(@id,'" XPATH_UPPER_TRANS "','" XPATH_LOWER_TRANS "') ='%s']" \ "|/" PCMK_XE_CIB "/" PCMK_XE_CONFIGURATION "/" PCMK_XE_RESOURCES \ "/" PCMK_XE_PRIMITIVE "/" PCMK_XE_META_ATTRIBUTES "/" PCMK_XE_NVPAIR \ "[@name='" PCMK_META_REMOTE_NODE "'][translate(@value,'" XPATH_UPPER_TRANS "','" XPATH_LOWER_TRANS "') ='%s']" \ "|/" PCMK_XE_CIB "/" PCMK_XE_STATUS "/" PCMK__XE_NODE_STATE \ "[@" PCMK_XA_REMOTE_NODE "='true'][translate(@" PCMK_XA_ID ",'" XPATH_UPPER_TRANS "','" XPATH_LOWER_TRANS "') ='%s']" int query_node_uuid(cib_t * the_cib, const char *uname, char **uuid, int *is_remote_node) { int rc = pcmk_ok; char *xpath_string; xmlNode *xml_search = NULL; char *host_lowercase = NULL; pcmk__assert(uname != NULL); host_lowercase = g_ascii_strdown(uname, -1); if (uuid) { *uuid = NULL; } if (is_remote_node) { *is_remote_node = FALSE; } xpath_string = pcmk__assert_asprintf(XPATH_NODE, host_lowercase, host_lowercase, host_lowercase, host_lowercase); if (cib_internal_op(the_cib, PCMK__CIB_REQUEST_QUERY, NULL, xpath_string, NULL, &xml_search, cib_sync_call|cib_xpath, NULL) == pcmk_ok) { rc = get_uuid_from_result(xml_search, uuid, is_remote_node); } else { rc = -ENXIO; } free(xpath_string); pcmk__xml_free(xml_search); g_free(host_lowercase); if (rc != pcmk_ok) { pcmk__debug("Could not map node name '%s' to a UUID: %s", uname, pcmk_strerror(rc)); } else { pcmk__info("Mapped node name '%s' to UUID %s", uname, ((uuid != NULL)? *uuid : "")); } return rc; } diff --git a/lib/cib/cib_file.c b/lib/cib/cib_file.c index db2c4e64b3..850581575b 100644 --- a/lib/cib/cib_file.c +++ b/lib/cib/cib_file.c @@ -1,1186 +1,1186 @@ /* * Original copyright 2004 International Business Machines * Later changes copyright 2008-2025 the Pacemaker project contributors * * The version control history for this file may have further details. * * This source code is licensed under the GNU Lesser General Public License * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define CIB_SERIES "cib" #define CIB_SERIES_MAX 100 #define CIB_SERIES_BZIP FALSE /* Must be false because archived copies are created with hard links */ #define CIB_LIVE_NAME CIB_SERIES ".xml" // key: client ID (const char *) -> value: client (cib_t *) static GHashTable *client_table = NULL; enum cib_file_flags { cib_file_flag_dirty = (1 << 0), cib_file_flag_live = (1 << 1), }; typedef struct cib_file_opaque_s { char *id; char *filename; uint32_t flags; // Group of enum cib_file_flags xmlNode *cib_xml; } cib_file_opaque_t; static int cib_file_process_commit_transaction(const char *op, int options, const char *section, xmlNode *req, xmlNode *input, xmlNode *existing_cib, xmlNode **result_cib, xmlNode **answer); /*! * \internal * \brief Add a CIB file client to client table * * \param[in] cib CIB client */ static void register_client(const cib_t *cib) { cib_file_opaque_t *private = cib->variant_opaque; if (client_table == NULL) { client_table = pcmk__strkey_table(NULL, NULL); } g_hash_table_insert(client_table, private->id, (gpointer) cib); } /*! * \internal * \brief Remove a CIB file client from client table * * \param[in] cib CIB client */ static void unregister_client(const cib_t *cib) { cib_file_opaque_t *private = cib->variant_opaque; if (client_table == NULL) { return; } g_hash_table_remove(client_table, private->id); /* @COMPAT: Add to crm_exit() when libcib and libcrmcommon are merged, * instead of destroying the client table when there are no more clients. */ if (g_hash_table_size(client_table) == 0) { g_hash_table_destroy(client_table); client_table = NULL; } } /*! * \internal * \brief Look up a CIB file client by its ID * * \param[in] client_id CIB client ID * * \return CIB client with matching ID if found, or \p NULL otherwise */ static cib_t * get_client(const char *client_id) { if (client_table == NULL) { return NULL; } return g_hash_table_lookup(client_table, (gpointer) client_id); } static const cib__op_fn_t cib_op_functions[] = { [cib__op_apply_patch] = cib_process_diff, [cib__op_bump] = cib_process_bump, [cib__op_commit_transact] = cib_file_process_commit_transaction, [cib__op_create] = cib_process_create, [cib__op_delete] = cib_process_delete, [cib__op_erase] = cib_process_erase, [cib__op_modify] = cib_process_modify, [cib__op_query] = cib_process_query, [cib__op_replace] = cib_process_replace, [cib__op_upgrade] = cib_process_upgrade, }; /* cib_file_backup() and cib_file_write_with_digest() need to chown the * written files only in limited circumstances, so these variables allow * that to be indicated without affecting external callers */ static uid_t cib_file_owner = 0; static uid_t cib_file_group = 0; static gboolean cib_do_chown = FALSE; #define cib_set_file_flags(cibfile, flags_to_set) do { \ (cibfile)->flags = pcmk__set_flags_as(__func__, __LINE__, \ PCMK__LOG_TRACE, "CIB file", \ cibfile->filename, \ (cibfile)->flags, \ (flags_to_set), \ #flags_to_set); \ } while (0) #define cib_clear_file_flags(cibfile, flags_to_clear) do { \ (cibfile)->flags = pcmk__clear_flags_as(__func__, __LINE__, \ PCMK__LOG_TRACE, \ "CIB file", \ cibfile->filename, \ (cibfile)->flags, \ (flags_to_clear), \ #flags_to_clear); \ } while (0) /*! * \internal * \brief Get the function that performs a given CIB file operation * * \param[in] operation Operation whose function to look up * * \return Function that performs \p operation for a CIB file client */ static cib__op_fn_t file_get_op_function(const cib__operation_t *operation) { enum cib__op_type type = operation->type; pcmk__assert(type >= 0); if (type >= PCMK__NELEM(cib_op_functions)) { return NULL; } return cib_op_functions[type]; } /*! * \internal * \brief Check whether a file is the live CIB * * \param[in] filename Name of file to check * * \return TRUE if file exists and its real path is same as live CIB's */ static gboolean cib_file_is_live(const char *filename) { gboolean same = FALSE; if (filename != NULL) { // Canonicalize file names for true comparison char *real_filename = NULL; if (pcmk__real_path(filename, &real_filename) == pcmk_rc_ok) { char *real_livename = NULL; if (pcmk__real_path(CRM_CONFIG_DIR "/" CIB_LIVE_NAME, &real_livename) == pcmk_rc_ok) { same = !strcmp(real_filename, real_livename); free(real_livename); } free(real_filename); } } return same; } static int cib_file_process_request(cib_t *cib, xmlNode *request, xmlNode **output) { int rc = pcmk_ok; const cib__operation_t *operation = NULL; cib__op_fn_t op_function = NULL; int call_id = 0; uint32_t call_options = cib_none; const char *op = pcmk__xe_get(request, PCMK__XA_CIB_OP); const char *section = pcmk__xe_get(request, PCMK__XA_CIB_SECTION); xmlNode *wrapper = pcmk__xe_first_child(request, PCMK__XE_CIB_CALLDATA, NULL, NULL); xmlNode *data = pcmk__xe_first_child(wrapper, NULL, NULL, NULL); bool changed = false; bool read_only = false; xmlNode *result_cib = NULL; xmlNode *cib_diff = NULL; cib_file_opaque_t *private = cib->variant_opaque; // We error checked these in callers cib__get_operation(op, &operation); op_function = file_get_op_function(operation); pcmk__xe_get_int(request, PCMK__XA_CIB_CALLID, &call_id); rc = pcmk__xe_get_flags(request, PCMK__XA_CIB_CALLOPT, &call_options, cib_none); if (rc != pcmk_rc_ok) { pcmk__warn("Couldn't parse options from request: %s", pcmk_rc_str(rc)); } read_only = !pcmk__is_set(operation->flags, cib__op_attr_modifies); // Mirror the logic in prepare_input() in the CIB manager if ((section != NULL) && pcmk__xe_is(data, PCMK_XE_CIB)) { data = pcmk_find_cib_element(data, section); } rc = cib_perform_op(cib, op, call_options, op_function, read_only, section, request, data, true, &changed, &private->cib_xml, &result_cib, &cib_diff, output); if (pcmk__is_set(call_options, cib_transaction)) { /* The rest of the logic applies only to the transaction as a whole, not * to individual requests. */ goto done; } if (rc == -pcmk_err_schema_validation) { // Show validation errors to stderr pcmk__validate_xml(result_cib, NULL, NULL, NULL); } else if ((rc == pcmk_ok) && !read_only) { pcmk__log_xml_patchset(LOG_DEBUG, cib_diff); if (result_cib != private->cib_xml) { pcmk__xml_free(private->cib_xml); private->cib_xml = result_cib; } cib_set_file_flags(private, cib_file_flag_dirty); } done: if ((result_cib != private->cib_xml) && (result_cib != *output)) { pcmk__xml_free(result_cib); } pcmk__xml_free(cib_diff); return rc; } static int cib_file_perform_op_delegate(cib_t *cib, const char *op, const char *host, const char *section, xmlNode *data, xmlNode **output_data, int call_options, const char *user_name) { int rc = pcmk_ok; xmlNode *request = NULL; xmlNode *output = NULL; cib_file_opaque_t *private = cib->variant_opaque; const cib__operation_t *operation = NULL; pcmk__info("Handling %s operation for %s as %s", pcmk__s(op, "invalid"), pcmk__s(section, "entire CIB"), pcmk__s(user_name, "default user")); if (output_data != NULL) { *output_data = NULL; } if (cib->state == cib_disconnected) { return -ENOTCONN; } rc = cib__get_operation(op, &operation); rc = pcmk_rc2legacy(rc); if (rc != pcmk_ok) { // @COMPAT: At compatibility break, use rc directly return -EPROTONOSUPPORT; } if (file_get_op_function(operation) == NULL) { // @COMPAT: At compatibility break, use EOPNOTSUPP pcmk__err("Operation %s is not supported by CIB file clients", op); return -EPROTONOSUPPORT; } cib__set_call_options(call_options, "file operation", cib_no_mtime); rc = cib__create_op(cib, op, host, section, data, call_options, user_name, NULL, &request); if (rc != pcmk_ok) { return rc; } pcmk__xe_set(request, PCMK__XA_ACL_TARGET, user_name); pcmk__xe_set(request, PCMK__XA_CIB_CLIENTID, private->id); if (pcmk__is_set(call_options, cib_transaction)) { rc = cib__extend_transaction(cib, request); goto done; } rc = cib_file_process_request(cib, request, &output); if ((output_data != NULL) && (output != NULL)) { if (output->doc == private->cib_xml->doc) { *output_data = pcmk__xml_copy(NULL, output); } else { *output_data = output; } } done: if ((output != NULL) && (output->doc != private->cib_xml->doc) && ((output_data == NULL) || (output != *output_data))) { pcmk__xml_free(output); } pcmk__xml_free(request); return rc; } /*! * \internal * \brief Read CIB from disk and validate it against XML schema * * \param[in] filename Name of file to read CIB from * \param[out] output Where to store the read CIB XML * * \return pcmk_ok on success, * -ENXIO if file does not exist (or stat() otherwise fails), or * -pcmk_err_schema_validation if XML doesn't parse or validate * \note If filename is the live CIB, this will *not* verify its digest, * though that functionality would be trivial to add here. * Also, this will *not* verify that the file is writable, * because some callers might not need to write. */ static int load_file_cib(const char *filename, xmlNode **output) { struct stat buf; xmlNode *root = NULL; /* Ensure file is readable */ if (strcmp(filename, "-") && (stat(filename, &buf) < 0)) { return -ENXIO; } /* Parse XML from file */ root = pcmk__xml_read(filename); if (root == NULL) { return -pcmk_err_schema_validation; } /* Add a status section if not already present */ if (pcmk__xe_first_child(root, PCMK_XE_STATUS, NULL, NULL) == NULL) { pcmk__xe_create(root, PCMK_XE_STATUS); } /* Validate XML against its specified schema */ if (!pcmk__configured_schema_validates(root)) { pcmk__xml_free(root); return -pcmk_err_schema_validation; } /* Remember the parsed XML for later use */ *output = root; return pcmk_ok; } static int cib_file_signon(cib_t *cib, const char *name, enum cib_conn_type type) { int rc = pcmk_ok; cib_file_opaque_t *private = cib->variant_opaque; if (private->filename == NULL) { rc = -EINVAL; } else { rc = load_file_cib(private->filename, &private->cib_xml); } if (rc == pcmk_ok) { pcmk__debug("Opened connection to local file '%s' for %s", private->filename, pcmk__s(name, "client")); cib->state = cib_connected_command; cib->type = cib_command; register_client(cib); } else { pcmk__info("Connection to local file '%s' for %s (client %s) failed: " "%s", private->filename, pcmk__s(name, "client"), private->id, pcmk_strerror(rc)); } return rc; } /*! * \internal * \brief Write out the in-memory CIB to a live CIB file * * \param[in] cib_root Root of XML tree to write * \param[in,out] path Full path to file to write * * \return 0 on success, -1 on failure */ static int cib_file_write_live(xmlNode *cib_root, char *path) { uid_t euid = geteuid(); uid_t daemon_uid = 0; gid_t daemon_gid = 0; char *sep = strrchr(path, '/'); const char *cib_dirname, *cib_filename; int rc = pcmk_rc_ok; /* Get the desired uid/gid */ rc = pcmk__daemon_user(&daemon_uid, &daemon_gid); if (rc != pcmk_rc_ok) { crm_perror(LOG_ERR, "Could not find user '" CRM_DAEMON_USER "': %s", pcmk_rc_str(rc)); return -1; } /* If we're root, we can change the ownership; * if we're daemon, anything we create will be OK; * otherwise, block access so we don't create wrong owner */ if ((euid != 0) && (euid != daemon_uid)) { crm_perror(LOG_ERR, "Must be root or " CRM_DAEMON_USER " to modify live CIB"); // @TODO Should this return -1 instead? return 0; } /* fancy footwork to separate dirname from filename * (we know the canonical name maps to the live CIB, * but the given name might be relative, or symlinked) */ if (sep == NULL) { /* no directory component specified */ cib_dirname = "./"; cib_filename = path; } else if (sep == path) { /* given name is in / */ cib_dirname = "/"; cib_filename = path + 1; } else { /* typical case; split given name into parts */ *sep = '\0'; cib_dirname = path; cib_filename = sep + 1; } /* if we're root, we want to update the file ownership */ if (euid == 0) { cib_file_owner = daemon_uid; cib_file_group = daemon_gid; cib_do_chown = TRUE; } /* write the file */ if (cib_file_write_with_digest(cib_root, cib_dirname, cib_filename) != pcmk_ok) { rc = -1; } /* turn off file ownership changes, for other callers */ if (euid == 0) { cib_do_chown = FALSE; } /* undo fancy stuff */ if ((sep != NULL) && (*sep == '\0')) { *sep = '/'; } return rc; } /*! * \internal * \brief Sign-off method for CIB file variants * * This will write the file to disk if needed, and free the in-memory CIB. If * the file is the live CIB, it will compute and write a signature as well. * * \param[in,out] cib CIB object to sign off * * \return pcmk_ok on success, pcmk_err_generic on failure * \todo This method should refuse to write the live CIB if the CIB manager is * running. */ static int cib_file_signoff(cib_t *cib) { int rc = pcmk_ok; cib_file_opaque_t *private = cib->variant_opaque; pcmk__debug("Disconnecting from the CIB manager"); cib->state = cib_disconnected; cib->type = cib_no_connection; unregister_client(cib); cib->cmds->end_transaction(cib, false, cib_none); /* If the in-memory CIB has been changed, write it to disk */ if (pcmk__is_set(private->flags, cib_file_flag_dirty)) { /* If this is the live CIB, write it out with a digest */ if (pcmk__is_set(private->flags, cib_file_flag_live)) { if (cib_file_write_live(private->cib_xml, private->filename) < 0) { rc = pcmk_err_generic; } /* Otherwise, it's a simple write */ } else { bool compress = pcmk__ends_with_ext(private->filename, ".bz2"); if (pcmk__xml_write_file(private->cib_xml, private->filename, compress) != pcmk_rc_ok) { rc = pcmk_err_generic; } } if (rc == pcmk_ok) { pcmk__info("Wrote CIB to %s", private->filename); cib_clear_file_flags(private, cib_file_flag_dirty); } else { pcmk__err("Could not write CIB to %s", private->filename); } } /* Free the in-memory CIB */ pcmk__xml_free(private->cib_xml); private->cib_xml = NULL; return rc; } static int cib_file_free(cib_t *cib) { int rc = pcmk_ok; if (cib->state != cib_disconnected) { rc = cib_file_signoff(cib); } if (rc == pcmk_ok) { cib_file_opaque_t *private = cib->variant_opaque; free(private->id); free(private->filename); free(private); free(cib->cmds); free(cib->user); free(cib); } else { fprintf(stderr, "Couldn't sign off: %d\n", rc); } return rc; } static int cib_file_register_notification(cib_t *cib, const char *callback, int enabled) { return -EPROTONOSUPPORT; } static int cib_file_set_connection_dnotify(cib_t *cib, void (*dnotify) (gpointer user_data)) { return -EPROTONOSUPPORT; } /*! * \internal * \brief Get the given CIB connection's unique client identifier * * \param[in] cib CIB connection * \param[out] async_id If not \p NULL, where to store asynchronous client ID * \param[out] sync_id If not \p NULL, where to store synchronous client ID * * \return Legacy Pacemaker return code * * \note This is the \p cib_file variant implementation of * \p cib_api_operations_t:client_id(). */ static int cib_file_client_id(const cib_t *cib, const char **async_id, const char **sync_id) { cib_file_opaque_t *private = cib->variant_opaque; if (async_id != NULL) { *async_id = private->id; } if (sync_id != NULL) { *sync_id = private->id; } return pcmk_ok; } cib_t * cib_file_new(const char *cib_location) { cib_t *cib = NULL; cib_file_opaque_t *private = NULL; char *filename = NULL; if (cib_location == NULL) { cib_location = getenv("CIB_file"); if (cib_location == NULL) { return NULL; // Shouldn't be possible if we were called internally } } cib = cib_new_variant(); if (cib == NULL) { return NULL; } filename = strdup(cib_location); if (filename == NULL) { free(cib); return NULL; } private = calloc(1, sizeof(cib_file_opaque_t)); if (private == NULL) { free(cib); free(filename); return NULL; } private->id = pcmk__generate_uuid(); private->filename = filename; cib->variant = cib_file; cib->variant_opaque = private; private->flags = 0; if (cib_file_is_live(cib_location)) { cib_set_file_flags(private, cib_file_flag_live); pcmk__trace("File %s detected as live CIB", cib_location); } /* assign variant specific ops */ cib->delegate_fn = cib_file_perform_op_delegate; cib->cmds->signon = cib_file_signon; cib->cmds->signoff = cib_file_signoff; cib->cmds->free = cib_file_free; cib->cmds->register_notification = cib_file_register_notification; cib->cmds->set_connection_dnotify = cib_file_set_connection_dnotify; cib->cmds->client_id = cib_file_client_id; return cib; } /*! * \internal * \brief Compare the calculated digest of an XML tree against a signature file * * \param[in] root Root of XML tree to compare * \param[in] sigfile Name of signature file containing digest to compare * * \return TRUE if digests match or signature file does not exist, else FALSE */ static gboolean cib_file_verify_digest(xmlNode *root, const char *sigfile) { gboolean passed = FALSE; char *expected; int rc = pcmk__file_contents(sigfile, &expected); switch (rc) { case pcmk_rc_ok: if (expected == NULL) { pcmk__err("On-disk digest at %s is empty", sigfile); return FALSE; } break; case ENOENT: pcmk__warn("No on-disk digest present at %s", sigfile); return TRUE; default: pcmk__err("Could not read on-disk digest from %s: %s", sigfile, pcmk_rc_str(rc)); return FALSE; } passed = pcmk__verify_digest(root, expected); free(expected); return passed; } /*! * \internal * \brief Read an XML tree from a file and verify its digest * * \param[in] filename Name of XML file to read * \param[in] sigfile Name of signature file containing digest to compare * \param[out] root If non-NULL, will be set to pointer to parsed XML tree * * \return 0 if file was successfully read, parsed and verified, otherwise: * -errno on stat() failure, * -pcmk_err_cib_corrupt if file size is 0 or XML is not parseable, or * -pcmk_err_cib_modified if digests do not match * \note If root is non-NULL, it is the caller's responsibility to free *root on * successful return. */ int cib_file_read_and_verify(const char *filename, const char *sigfile, xmlNode **root) { int s_res; struct stat buf; char *local_sigfile = NULL; xmlNode *local_root = NULL; pcmk__assert(filename != NULL); if (root) { *root = NULL; } /* Verify that file exists and its size is nonzero */ s_res = stat(filename, &buf); if (s_res < 0) { crm_perror(LOG_WARNING, "Could not verify cluster configuration file %s", filename); return -errno; } else if (buf.st_size == 0) { pcmk__warn("Cluster configuration file %s is corrupt (size is zero)", filename); return -pcmk_err_cib_corrupt; } /* Parse XML */ local_root = pcmk__xml_read(filename); if (local_root == NULL) { pcmk__warn("Cluster configuration file %s is corrupt (unparseable as " "XML)", filename); return -pcmk_err_cib_corrupt; } /* If sigfile is not specified, use original file name plus .sig */ if (sigfile == NULL) { sigfile = local_sigfile = pcmk__assert_asprintf("%s.sig", filename); } /* Verify that digests match */ if (cib_file_verify_digest(local_root, sigfile) == FALSE) { free(local_sigfile); pcmk__xml_free(local_root); return -pcmk_err_cib_modified; } free(local_sigfile); if (root) { *root = local_root; } else { pcmk__xml_free(local_root); } return pcmk_ok; } /*! * \internal * \brief Back up a CIB * * \param[in] cib_dirname Directory containing CIB file and backups * \param[in] cib_filename Name (relative to cib_dirname) of CIB file to back up * * \return 0 on success, -1 on error */ static int cib_file_backup(const char *cib_dirname, const char *cib_filename) { int rc = 0; unsigned int seq = 0U; char *cib_path = pcmk__assert_asprintf("%s/%s", cib_dirname, cib_filename); char *cib_digest = pcmk__assert_asprintf("%s.sig", cib_path); char *backup_path; char *backup_digest; // Determine backup and digest file names if (pcmk__read_series_sequence(cib_dirname, CIB_SERIES, &seq) != pcmk_rc_ok) { // @TODO maybe handle errors better ... seq = 0U; } backup_path = pcmk__series_filename(cib_dirname, CIB_SERIES, seq, CIB_SERIES_BZIP); backup_digest = pcmk__assert_asprintf("%s.sig", backup_path); /* Remove the old backups if they exist */ unlink(backup_path); unlink(backup_digest); /* Back up the CIB, by hard-linking it to the backup name */ if ((link(cib_path, backup_path) < 0) && (errno != ENOENT)) { crm_perror(LOG_ERR, "Could not archive %s by linking to %s", cib_path, backup_path); rc = -1; /* Back up the CIB signature similarly */ } else if ((link(cib_digest, backup_digest) < 0) && (errno != ENOENT)) { crm_perror(LOG_ERR, "Could not archive %s by linking to %s", cib_digest, backup_digest); rc = -1; /* Update the last counter and ensure everything is sync'd to media */ } else { pcmk__write_series_sequence(cib_dirname, CIB_SERIES, ++seq, CIB_SERIES_MAX); if (cib_do_chown) { int rc2; if ((chown(backup_path, cib_file_owner, cib_file_group) < 0) && (errno != ENOENT)) { crm_perror(LOG_ERR, "Could not set owner of %s", backup_path); rc = -1; } if ((chown(backup_digest, cib_file_owner, cib_file_group) < 0) && (errno != ENOENT)) { crm_perror(LOG_ERR, "Could not set owner of %s", backup_digest); rc = -1; } rc2 = pcmk__chown_series_sequence(cib_dirname, CIB_SERIES, cib_file_owner, cib_file_group); if (rc2 != pcmk_rc_ok) { pcmk__err("Could not set owner of sequence file in %s: %s", cib_dirname, pcmk_rc_str(rc2)); rc = -1; } } pcmk__sync_directory(cib_dirname); pcmk__info("Archived previous version as %s", backup_path); } free(cib_path); free(cib_digest); free(backup_path); free(backup_digest); return rc; } /*! * \internal * \brief Prepare CIB XML to be written to disk * * Set \c PCMK_XA_NUM_UPDATES to 0, set \c PCMK_XA_CIB_LAST_WRITTEN to the * current timestamp, and strip out the status section. * * \param[in,out] root Root of CIB XML tree * * \return void */ static void cib_file_prepare_xml(xmlNode *root) { xmlNode *cib_status_root = NULL; /* Always write out with num_updates=0 and current last-written timestamp */ pcmk__xe_set(root, PCMK_XA_NUM_UPDATES, "0"); pcmk__xe_add_last_written(root); /* Delete status section before writing to file, because * we discard it on startup anyway, and users get confused by it */ cib_status_root = pcmk__xe_first_child(root, PCMK_XE_STATUS, NULL, NULL); CRM_CHECK(cib_status_root != NULL, return); pcmk__xml_free(cib_status_root); } /*! * \internal * \brief Write CIB to disk, along with a signature file containing its digest * * \param[in,out] cib_root Root of XML tree to write * \param[in] cib_dirname Directory containing CIB and signature files * \param[in] cib_filename Name (relative to cib_dirname) of file to write * * \return pcmk_ok on success, * pcmk_err_cib_modified if existing cib_filename doesn't match digest, * pcmk_err_cib_backup if existing cib_filename couldn't be backed up, * or pcmk_err_cib_save if new cib_filename couldn't be saved */ int cib_file_write_with_digest(xmlNode *cib_root, const char *cib_dirname, const char *cib_filename) { int exit_rc = pcmk_ok; int rc, fd; char *digest = NULL; /* Detect CIB version for diagnostic purposes */ const char *epoch = pcmk__xe_get(cib_root, PCMK_XA_EPOCH); const char *admin_epoch = pcmk__xe_get(cib_root, PCMK_XA_ADMIN_EPOCH); /* Determine full CIB and signature pathnames */ char *cib_path = pcmk__assert_asprintf("%s/%s", cib_dirname, cib_filename); char *digest_path = pcmk__assert_asprintf("%s.sig", cib_path); /* Create temporary file name patterns for writing out CIB and signature */ char *tmp_cib = pcmk__assert_asprintf("%s/cib.XXXXXX", cib_dirname); char *tmp_digest = pcmk__assert_asprintf("%s/cib.XXXXXX", cib_dirname); /* Ensure the admin didn't modify the existing CIB underneath us */ pcmk__trace("Reading cluster configuration file %s", cib_path); rc = cib_file_read_and_verify(cib_path, NULL, NULL); if ((rc != pcmk_ok) && (rc != -ENOENT)) { pcmk__err("%s was manually modified while the cluster was active!", cib_path); exit_rc = pcmk_err_cib_modified; goto cleanup; } /* Back up the existing CIB */ if (cib_file_backup(cib_dirname, cib_filename) < 0) { exit_rc = pcmk_err_cib_backup; goto cleanup; } pcmk__debug("Writing CIB to disk"); umask(S_IWGRP | S_IWOTH | S_IROTH); cib_file_prepare_xml(cib_root); /* Write the CIB to a temporary file, so we can deploy (near) atomically */ fd = mkstemp(tmp_cib); if (fd < 0) { crm_perror(LOG_ERR, "Couldn't open temporary file %s for writing CIB", tmp_cib); exit_rc = pcmk_err_cib_save; goto cleanup; } /* Protect the temporary file */ if (fchmod(fd, S_IRUSR | S_IWUSR) < 0) { crm_perror(LOG_ERR, "Couldn't protect temporary file %s for writing CIB", tmp_cib); exit_rc = pcmk_err_cib_save; goto cleanup; } if (cib_do_chown && (fchown(fd, cib_file_owner, cib_file_group) < 0)) { crm_perror(LOG_ERR, "Couldn't protect temporary file %s for writing CIB", tmp_cib); exit_rc = pcmk_err_cib_save; goto cleanup; } /* Write out the CIB */ if (pcmk__xml_write_fd(cib_root, tmp_cib, fd) != pcmk_rc_ok) { pcmk__err("Changes couldn't be written to %s", tmp_cib); exit_rc = pcmk_err_cib_save; goto cleanup; } /* Calculate CIB digest */ digest = pcmk__digest_on_disk_cib(cib_root); pcmk__assert(digest != NULL); pcmk__info("Wrote version %s.%s.0 of the CIB to disk (digest: %s)", pcmk__s(admin_epoch, "0"), pcmk__s(epoch, "0"), digest); /* Write the CIB digest to a temporary file */ fd = mkstemp(tmp_digest); if (fd < 0) { crm_perror(LOG_ERR, "Could not create temporary file for CIB digest"); exit_rc = pcmk_err_cib_save; goto cleanup; } if (cib_do_chown && (fchown(fd, cib_file_owner, cib_file_group) < 0)) { crm_perror(LOG_ERR, "Couldn't protect temporary file %s for writing CIB", tmp_cib); exit_rc = pcmk_err_cib_save; close(fd); goto cleanup; } rc = pcmk__write_sync(fd, digest); if (rc != pcmk_rc_ok) { pcmk__err("Could not write digest to %s: %s", tmp_digest, pcmk_rc_str(rc)); exit_rc = pcmk_err_cib_save; close(fd); goto cleanup; } close(fd); pcmk__debug("Wrote digest %s to disk", digest); /* Verify that what we wrote is sane */ pcmk__info("Reading cluster configuration file %s (digest: %s)", tmp_cib, tmp_digest); rc = cib_file_read_and_verify(tmp_cib, tmp_digest, NULL); pcmk__assert(rc == 0); /* Rename temporary files to live, and sync directory changes to media */ pcmk__debug("Activating %s", tmp_cib); if (rename(tmp_cib, cib_path) < 0) { crm_perror(LOG_ERR, "Couldn't rename %s as %s", tmp_cib, cib_path); exit_rc = pcmk_err_cib_save; } if (rename(tmp_digest, digest_path) < 0) { crm_perror(LOG_ERR, "Couldn't rename %s as %s", tmp_digest, digest_path); exit_rc = pcmk_err_cib_save; } pcmk__sync_directory(cib_dirname); cleanup: free(cib_path); free(digest_path); free(digest); free(tmp_digest); free(tmp_cib); return exit_rc; } /*! * \internal * \brief Process requests in a CIB transaction * * Stop when a request fails or when all requests have been processed. * * \param[in,out] cib CIB client * \param[in,out] transaction CIB transaction * * \return Standard Pacemaker return code */ static int cib_file_process_transaction_requests(cib_t *cib, xmlNode *transaction) { cib_file_opaque_t *private = cib->variant_opaque; for (xmlNode *request = pcmk__xe_first_child(transaction, PCMK__XE_CIB_COMMAND, NULL, NULL); request != NULL; request = pcmk__xe_next(request, PCMK__XE_CIB_COMMAND)) { xmlNode *output = NULL; const char *op = pcmk__xe_get(request, PCMK__XA_CIB_OP); int rc = cib_file_process_request(cib, request, &output); rc = pcmk_legacy2rc(rc); if (rc != pcmk_rc_ok) { pcmk__err("Aborting transaction for CIB file client (%s) on file " "'%s' due to failed %s request: %s", private->id, private->filename, op, pcmk_rc_str(rc)); pcmk__log_xml_info(request, "Failed request"); return rc; } pcmk__trace("Applied %s request to transaction working CIB for CIB " "file client (%s) on file '%s'", op, private->id, private->filename); - crm_log_xml_trace(request, "Successful request"); + pcmk__log_xml_trace(request, "Successful request"); } return pcmk_rc_ok; } /*! * \internal * \brief Commit a given CIB file client's transaction to a working CIB copy * * \param[in,out] cib CIB file client * \param[in] transaction CIB transaction * \param[in,out] result_cib Where to store result CIB * * \return Standard Pacemaker return code * * \note The caller is responsible for replacing the \p cib argument's * \p private->cib_xml with \p result_cib on success, and for freeing * \p result_cib using \p pcmk__xml_free() on failure. */ static int cib_file_commit_transaction(cib_t *cib, xmlNode *transaction, xmlNode **result_cib) { int rc = pcmk_rc_ok; cib_file_opaque_t *private = cib->variant_opaque; xmlNode *saved_cib = private->cib_xml; CRM_CHECK(pcmk__xe_is(transaction, PCMK__XE_CIB_TRANSACTION), return pcmk_rc_no_transaction); /* *result_cib should be a copy of private->cib_xml (created by * cib_perform_op()). If not, make a copy now. Change tracking isn't * strictly required here because: * * Each request in the transaction will have changes tracked and ACLs * checked if appropriate. * * cib_perform_op() will infer changes for the commit request at the end. */ CRM_CHECK((*result_cib != NULL) && (*result_cib != private->cib_xml), *result_cib = pcmk__xml_copy(NULL, private->cib_xml)); pcmk__trace("Committing transaction for CIB file client (%s) on file '%s' " "to working CIB", private->id, private->filename); // Apply all changes to a working copy of the CIB private->cib_xml = *result_cib; rc = cib_file_process_transaction_requests(cib, transaction); pcmk__trace("Transaction commit %s for CIB file client (%s) on file '%s'", ((rc == pcmk_rc_ok)? "succeeded" : "failed"), private->id, private->filename); /* Some request types (for example, erase) may have freed private->cib_xml * (the working copy) and pointed it at a new XML object. In that case, it * follows that *result_cib (the working copy) was freed. * * Point *result_cib at the updated working copy stored in private->cib_xml. */ *result_cib = private->cib_xml; // Point private->cib_xml back to the unchanged original copy private->cib_xml = saved_cib; return rc; } static int cib_file_process_commit_transaction(const char *op, int options, const char *section, xmlNode *req, xmlNode *input, xmlNode *existing_cib, xmlNode **result_cib, xmlNode **answer) { int rc = pcmk_rc_ok; const char *client_id = pcmk__xe_get(req, PCMK__XA_CIB_CLIENTID); cib_t *cib = NULL; CRM_CHECK(client_id != NULL, return -EINVAL); cib = get_client(client_id); CRM_CHECK(cib != NULL, return -EINVAL); rc = cib_file_commit_transaction(cib, input, result_cib); if (rc != pcmk_rc_ok) { cib_file_opaque_t *private = cib->variant_opaque; pcmk__err("Could not commit transaction for CIB file client (%s) on " "file '%s': %s", private->id, private->filename, pcmk_rc_str(rc)); } return pcmk_rc2legacy(rc); } diff --git a/lib/cib/cib_native.c b/lib/cib/cib_native.c index a637798a5b..869130c082 100644 --- a/lib/cib/cib_native.c +++ b/lib/cib/cib_native.c @@ -1,486 +1,486 @@ /* * Copyright 2004 International Business Machines * Later changes copyright 2004-2025 the Pacemaker project contributors * * The version control history for this file may have further details. * * This source code is licensed under the GNU Lesser General Public License * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. */ #include #include #include #include #include #include #include #include #include #include #include #include #include typedef struct cib_native_opaque_s { char *token; crm_ipc_t *ipc; void (*dnotify_fn) (gpointer user_data); mainloop_io_t *source; } cib_native_opaque_t; static int cib_native_perform_op_delegate(cib_t *cib, const char *op, const char *host, const char *section, xmlNode *data, xmlNode **output_data, int call_options, const char *user_name) { int rc = pcmk_ok; int reply_id = 0; enum crm_ipc_flags ipc_flags = crm_ipc_flags_none; xmlNode *op_msg = NULL; xmlNode *op_reply = NULL; cib_native_opaque_t *native = cib->variant_opaque; if (cib->state == cib_disconnected) { return -ENOTCONN; } if (output_data != NULL) { *output_data = NULL; } if (op == NULL) { pcmk__err("No operation specified"); return -EINVAL; } if (call_options & cib_sync_call) { pcmk__set_ipc_flags(ipc_flags, "client", crm_ipc_client_response); } rc = cib__create_op(cib, op, host, section, data, call_options, user_name, NULL, &op_msg); if (rc != pcmk_ok) { return rc; } if (pcmk__is_set(call_options, cib_transaction)) { rc = cib__extend_transaction(cib, op_msg); goto done; } pcmk__trace("Sending %s message to the CIB manager (timeout=%ds)", op, cib->call_timeout); rc = crm_ipc_send(native->ipc, op_msg, ipc_flags, cib->call_timeout * 1000, &op_reply); if (rc < 0) { pcmk__err("Couldn't perform %s operation (timeout=%ds): %s (%d)", op, cib->call_timeout, pcmk_strerror(rc), rc); rc = -ECOMM; goto done; } - crm_log_xml_trace(op_reply, "Reply"); + pcmk__log_xml_trace(op_reply, "Reply"); if (!(call_options & cib_sync_call)) { pcmk__trace("Async call, returning %d", cib->call_id); CRM_CHECK(cib->call_id != 0, rc = -ENOMSG; goto done); rc = cib->call_id; goto done; } rc = pcmk_ok; pcmk__xe_get_int(op_reply, PCMK__XA_CIB_CALLID, &reply_id); if (reply_id == cib->call_id) { xmlNode *wrapper = pcmk__xe_first_child(op_reply, PCMK__XE_CIB_CALLDATA, NULL, NULL); xmlNode *tmp = pcmk__xe_first_child(wrapper, NULL, NULL, NULL); pcmk__trace("Synchronous reply %d received", reply_id); if (pcmk__xe_get_int(op_reply, PCMK__XA_CIB_RC, &rc) != pcmk_rc_ok) { rc = -EPROTO; } if (output_data == NULL || (call_options & cib_discard_reply)) { pcmk__trace("Discarding reply"); } else { *output_data = pcmk__xml_copy(NULL, tmp); } } else if (reply_id <= 0) { pcmk__err("Received bad reply: No id set"); pcmk__log_xml_err(op_reply, "Bad reply"); rc = -ENOMSG; goto done; } else { pcmk__err("Received bad reply: %d (wanted %d)", reply_id, cib->call_id); pcmk__log_xml_err(op_reply, "Old reply"); rc = -ENOMSG; goto done; } if (op_reply == NULL && cib->state == cib_disconnected) { rc = -ENOTCONN; } else if (rc == pcmk_ok && op_reply == NULL) { rc = -ETIME; } switch (rc) { case pcmk_ok: case -EPERM: break; /* This is an internal value that clients do not and should not care about */ case -pcmk_err_diff_resync: rc = pcmk_ok; break; /* These indicate internal problems */ case -EPROTO: case -ENOMSG: pcmk__err("Call failed: %s", pcmk_strerror(rc)); if (op_reply) { pcmk__log_xml_err(op_reply, "Invalid reply"); } break; default: if (!pcmk__str_eq(op, PCMK__CIB_REQUEST_QUERY, pcmk__str_none)) { pcmk__warn("Call failed: %s", pcmk_strerror(rc)); } } done: if (!crm_ipc_connected(native->ipc)) { pcmk__err("The CIB manager disconnected"); cib->state = cib_disconnected; } pcmk__xml_free(op_msg); pcmk__xml_free(op_reply); return rc; } static int cib_native_dispatch_internal(const char *buffer, ssize_t length, gpointer userdata) { const char *type = NULL; xmlNode *msg = NULL; cib_t *cib = userdata; pcmk__trace("dispatching %p", userdata); if (cib == NULL) { pcmk__err("No CIB!"); return 0; } msg = pcmk__xml_parse(buffer); if (msg == NULL) { pcmk__warn("Received a NULL message from the CIB manager"); return 0; } /* do callbacks */ type = pcmk__xe_get(msg, PCMK__XA_T); pcmk__trace("Activating %s callbacks...", type); crm_log_xml_explicit(msg, "cib-reply"); if (pcmk__str_eq(type, PCMK__VALUE_CIB, pcmk__str_none)) { cib_native_callback(cib, msg, 0, 0); } else if (pcmk__str_eq(type, PCMK__VALUE_CIB_NOTIFY, pcmk__str_none)) { g_list_foreach(cib->notify_list, cib_native_notify, msg); } else { pcmk__err("Unknown message type: %s", type); } pcmk__xml_free(msg); return 0; } static void cib_native_destroy(void *userdata) { cib_t *cib = userdata; cib_native_opaque_t *native = cib->variant_opaque; pcmk__trace("destroying %p", userdata); cib->state = cib_disconnected; native->source = NULL; native->ipc = NULL; if (native->dnotify_fn) { native->dnotify_fn(userdata); } } static int cib_native_signoff(cib_t *cib) { cib_native_opaque_t *native = cib->variant_opaque; pcmk__debug("Disconnecting from the CIB manager"); cib_free_notify(cib); remove_cib_op_callback(0, TRUE); if (native->source != NULL) { /* Attached to mainloop */ mainloop_del_ipc_client(native->source); native->source = NULL; native->ipc = NULL; } else if (native->ipc) { /* Not attached to mainloop */ crm_ipc_t *ipc = native->ipc; native->ipc = NULL; crm_ipc_close(ipc); crm_ipc_destroy(ipc); } cib->cmds->end_transaction(cib, false, cib_none); cib->state = cib_disconnected; cib->type = cib_no_connection; return pcmk_ok; } static int cib_native_signon(cib_t *cib, const char *name, enum cib_conn_type type) { int rc = pcmk_ok; const char *channel = NULL; cib_native_opaque_t *native = cib->variant_opaque; xmlNode *hello = NULL; struct ipc_client_callbacks cib_callbacks = { .dispatch = cib_native_dispatch_internal, .destroy = cib_native_destroy }; if (name == NULL) { name = pcmk__s(crm_system_name, "client"); } cib->call_timeout = PCMK__IPC_TIMEOUT; if (type == cib_command) { cib->state = cib_connected_command; channel = PCMK__SERVER_BASED_RW; } else if (type == cib_command_nonblocking) { cib->state = cib_connected_command; channel = PCMK__SERVER_BASED_SHM; } else if (type == cib_query) { cib->state = cib_connected_query; channel = PCMK__SERVER_BASED_RO; } else { return -ENOTCONN; } pcmk__trace("Connecting %s channel", channel); native->source = mainloop_add_ipc_client(channel, G_PRIORITY_HIGH, 512 * 1024, cib, &cib_callbacks); native->ipc = mainloop_get_ipc_client(native->source); if (rc != pcmk_ok || native->ipc == NULL || !crm_ipc_connected(native->ipc)) { pcmk__info("Could not connect to CIB manager for %s", name); rc = -ENOTCONN; } if (rc == pcmk_ok) { rc = cib__create_op(cib, CRM_OP_REGISTER, NULL, NULL, NULL, cib_sync_call, NULL, name, &hello); } if (rc == pcmk_ok) { xmlNode *reply = NULL; if (crm_ipc_send(native->ipc, hello, crm_ipc_client_response, -1, &reply) > 0) { const char *msg_type = pcmk__xe_get(reply, PCMK__XA_CIB_OP); - crm_log_xml_trace(reply, "reg-reply"); + pcmk__log_xml_trace(reply, "reg-reply"); if (!pcmk__str_eq(msg_type, CRM_OP_REGISTER, pcmk__str_casei)) { pcmk__info("Reply to CIB registration message has unknown type " "'%s'", msg_type); rc = -EPROTO; } else { native->token = pcmk__xe_get_copy(reply, PCMK__XA_CIB_CLIENTID); if (native->token == NULL) { rc = -EPROTO; } } pcmk__xml_free(reply); } else { rc = -ECOMM; } pcmk__xml_free(hello); } if (rc == pcmk_ok) { pcmk__info("Successfully connected to CIB manager for %s", name); return pcmk_ok; } pcmk__info("Connection to CIB manager for %s failed: %s", name, pcmk_strerror(rc)); cib_native_signoff(cib); return rc; } static int cib_native_free(cib_t *cib) { int rc = pcmk_ok; if (cib->state != cib_disconnected) { rc = cib_native_signoff(cib); } if (cib->state == cib_disconnected) { cib_native_opaque_t *native = cib->variant_opaque; free(native->token); free(cib->variant_opaque); free(cib->cmds); free(cib->user); free(cib); } return rc; } static int cib_native_register_notification(cib_t *cib, const char *callback, int enabled) { int rc = pcmk_ok; xmlNode *notify_msg = pcmk__xe_create(NULL, PCMK__XE_CIB_CALLBACK); cib_native_opaque_t *native = cib->variant_opaque; if (cib->state != cib_disconnected) { pcmk__xe_set(notify_msg, PCMK__XA_CIB_OP, PCMK__VALUE_CIB_NOTIFY); pcmk__xe_set(notify_msg, PCMK__XA_CIB_NOTIFY_TYPE, callback); pcmk__xe_set_int(notify_msg, PCMK__XA_CIB_NOTIFY_ACTIVATE, enabled); rc = crm_ipc_send(native->ipc, notify_msg, crm_ipc_client_response, 1000 * cib->call_timeout, NULL); if (rc <= 0) { pcmk__trace("Notification not registered: %d", rc); rc = -ECOMM; } } pcmk__xml_free(notify_msg); return rc; } static int cib_native_set_connection_dnotify(cib_t *cib, void (*dnotify) (gpointer user_data)) { cib_native_opaque_t *native = NULL; if (cib == NULL) { pcmk__err("No CIB!"); return FALSE; } native = cib->variant_opaque; native->dnotify_fn = dnotify; return pcmk_ok; } /*! * \internal * \brief Get the given CIB connection's unique client identifier * * These can be used to check whether this client requested the action that * triggered a CIB notification. * * \param[in] cib CIB connection * \param[out] async_id If not \p NULL, where to store asynchronous client ID * \param[out] sync_id If not \p NULL, where to store synchronous client ID * * \return Legacy Pacemaker return code (specifically, \p pcmk_ok) * * \note This is the \p cib_native variant implementation of * \p cib_api_operations_t:client_id(). * \note For \p cib_native objects, \p async_id and \p sync_id are the same. * \note The client ID is assigned during CIB sign-on. */ static int cib_native_client_id(const cib_t *cib, const char **async_id, const char **sync_id) { cib_native_opaque_t *native = cib->variant_opaque; if (async_id != NULL) { *async_id = native->token; } if (sync_id != NULL) { *sync_id = native->token; } return pcmk_ok; } cib_t * cib_native_new(void) { cib_native_opaque_t *native = NULL; cib_t *cib = cib_new_variant(); if (cib == NULL) { return NULL; } native = calloc(1, sizeof(cib_native_opaque_t)); if (native == NULL) { free(cib); return NULL; } cib->variant = cib_native; cib->variant_opaque = native; native->ipc = NULL; native->source = NULL; native->dnotify_fn = NULL; /* assign variant specific ops */ cib->delegate_fn = cib_native_perform_op_delegate; cib->cmds->signon = cib_native_signon; cib->cmds->signoff = cib_native_signoff; cib->cmds->free = cib_native_free; cib->cmds->register_notification = cib_native_register_notification; cib->cmds->set_connection_dnotify = cib_native_set_connection_dnotify; cib->cmds->client_id = cib_native_client_id; return cib; } diff --git a/lib/cib/cib_remote.c b/lib/cib/cib_remote.c index 3eaa223b58..f674daa3d9 100644 --- a/lib/cib/cib_remote.c +++ b/lib/cib/cib_remote.c @@ -1,662 +1,662 @@ /* * Copyright 2008-2025 the Pacemaker project contributors * * The version control history for this file may have further details. * * This source code is licensed under the GNU Lesser General Public License * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include // GnuTLS handshake timeout in seconds #define TLS_HANDSHAKE_TIMEOUT 5 static pcmk__tls_t *tls = NULL; #include typedef struct cib_remote_opaque_s { int port; char *server; char *user; char *passwd; gboolean encrypted; pcmk__remote_t command; pcmk__remote_t callback; pcmk__output_t *out; time_t start_time; int timeout_sec; } cib_remote_opaque_t; static int cib_remote_perform_op(cib_t *cib, const char *op, const char *host, const char *section, xmlNode *data, xmlNode **output_data, int call_options, const char *user_name) { int rc; int remaining_time = 0; time_t start_time; xmlNode *op_msg = NULL; xmlNode *op_reply = NULL; cib_remote_opaque_t *private = cib->variant_opaque; if (cib->state == cib_disconnected) { return -ENOTCONN; } if (output_data != NULL) { *output_data = NULL; } if (op == NULL) { pcmk__err("No operation specified"); return -EINVAL; } rc = cib__create_op(cib, op, host, section, data, call_options, user_name, NULL, &op_msg); if (rc != pcmk_ok) { return rc; } if (pcmk__is_set(call_options, cib_transaction)) { rc = cib__extend_transaction(cib, op_msg); pcmk__xml_free(op_msg); return rc; } pcmk__trace("Sending %s message to the CIB manager", op); if (!(call_options & cib_sync_call)) { pcmk__remote_send_xml(&private->callback, op_msg); } else { pcmk__remote_send_xml(&private->command, op_msg); } pcmk__xml_free(op_msg); if ((call_options & cib_discard_reply)) { pcmk__trace("Discarding reply"); return pcmk_ok; } else if (!(call_options & cib_sync_call)) { return cib->call_id; } pcmk__trace("Waiting for a synchronous reply"); start_time = time(NULL); remaining_time = cib->call_timeout ? cib->call_timeout : 60; rc = pcmk_rc_ok; while (remaining_time > 0 && (rc != ENOTCONN)) { int reply_id = -1; int msg_id = cib->call_id; rc = pcmk__read_remote_message(&private->command, remaining_time * 1000); op_reply = pcmk__remote_message_xml(&private->command); if (!op_reply) { break; } pcmk__xe_get_int(op_reply, PCMK__XA_CIB_CALLID, &reply_id); if (reply_id == msg_id) { break; } else if (reply_id < msg_id) { pcmk__debug("Received old reply: %d (wanted %d)", reply_id, msg_id); - crm_log_xml_trace(op_reply, "Old reply"); + pcmk__log_xml_trace(op_reply, "Old reply"); } else if ((reply_id - 10000) > msg_id) { /* wrap-around case */ pcmk__debug("Received old reply: %d (wanted %d)", reply_id, msg_id); - crm_log_xml_trace(op_reply, "Old reply"); + pcmk__log_xml_trace(op_reply, "Old reply"); } else { pcmk__err("Received a __future__ reply:" " %d (wanted %d)", reply_id, msg_id); } pcmk__xml_free(op_reply); op_reply = NULL; /* wasn't the right reply, try and read some more */ remaining_time = time(NULL) - start_time; } if (rc == ENOTCONN) { pcmk__err("Disconnected while waiting for reply"); return -ENOTCONN; } else if (op_reply == NULL) { pcmk__err("No reply message - empty"); return -ENOMSG; } pcmk__trace("Synchronous reply received"); /* Start processing the reply... */ if (pcmk__xe_get_int(op_reply, PCMK__XA_CIB_RC, &rc) != pcmk_rc_ok) { rc = -EPROTO; } if (rc == -pcmk_err_diff_resync) { /* This is an internal value that clients do not and should not care about */ rc = pcmk_ok; } if (rc == pcmk_ok || rc == -EPERM) { pcmk__log_xml_debug(op_reply, "passed"); } else { pcmk__err("Call failed: %s", pcmk_strerror(rc)); pcmk__log_xml_warn(op_reply, "failed"); } if (output_data == NULL) { /* do nothing more */ } else if (!(call_options & cib_discard_reply)) { xmlNode *wrapper = pcmk__xe_first_child(op_reply, PCMK__XE_CIB_CALLDATA, NULL, NULL); xmlNode *tmp = pcmk__xe_first_child(wrapper, NULL, NULL, NULL); if (tmp == NULL) { pcmk__trace("No output in reply to \"%s\" command %d", op, (cib->call_id - 1)); } else { *output_data = pcmk__xml_copy(NULL, tmp); } } pcmk__xml_free(op_reply); return rc; } static int cib_remote_callback_dispatch(gpointer user_data) { int rc; cib_t *cib = user_data; cib_remote_opaque_t *private = cib->variant_opaque; xmlNode *msg = NULL; const char *type = NULL; /* If start time is 0, we've previously handled a complete message and this * connection is being reused for a new message. Reset the start_time, * giving this new message timeout_sec from now to complete. */ if (private->start_time == 0) { private->start_time = time(NULL); } rc = pcmk__read_available_remote_data(&private->callback); switch (rc) { case pcmk_rc_ok: /* We have the whole message so process it */ break; case EAGAIN: /* Have we timed out? */ if (time(NULL) >= private->start_time + private->timeout_sec) { pcmk__info("Error reading from CIB manager connection: %s", pcmk_rc_str(ETIME)); return -1; } /* We haven't read the whole message yet */ return 0; default: /* Error */ pcmk__info("Error reading from CIB manager connection: %s", pcmk_rc_str(rc)); return -1; } msg = pcmk__remote_message_xml(&private->callback); if (msg == NULL) { private->start_time = 0; return 0; } type = pcmk__xe_get(msg, PCMK__XA_T); pcmk__trace("Activating %s callbacks...", type); if (pcmk__str_eq(type, PCMK__VALUE_CIB, pcmk__str_none)) { cib_native_callback(cib, msg, 0, 0); } else if (pcmk__str_eq(type, PCMK__VALUE_CIB_NOTIFY, pcmk__str_none)) { g_list_foreach(cib->notify_list, cib_native_notify, msg); } else { pcmk__err("Unknown message type: %s", type); } pcmk__xml_free(msg); private->start_time = 0; return 0; } static int cib_remote_command_dispatch(gpointer user_data) { int rc; cib_t *cib = user_data; cib_remote_opaque_t *private = cib->variant_opaque; /* See cib_remote_callback_dispatch */ if (private->start_time == 0) { private->start_time = time(NULL); } rc = pcmk__read_available_remote_data(&private->command); if (rc == EAGAIN) { /* Have we timed out? */ if (time(NULL) >= private->start_time + private->timeout_sec) { pcmk__info("Error reading from CIB manager connection: %s", pcmk_rc_str(ETIME)); return -1; } /* We haven't read the whole message yet */ return 0; } free(private->command.buffer); private->command.buffer = NULL; pcmk__err("Received late reply for remote cib connection, discarding"); if (rc != pcmk_rc_ok) { pcmk__info("Error reading from CIB manager connection: %s", pcmk_rc_str(rc)); return -1; } private->start_time = 0; return 0; } static int cib_tls_close(cib_t *cib) { cib_remote_opaque_t *private = cib->variant_opaque; if (private->encrypted) { if (private->command.tls_session) { gnutls_bye(private->command.tls_session, GNUTLS_SHUT_RDWR); gnutls_deinit(private->command.tls_session); } if (private->callback.tls_session) { gnutls_bye(private->callback.tls_session, GNUTLS_SHUT_RDWR); gnutls_deinit(private->callback.tls_session); } private->command.tls_session = NULL; private->callback.tls_session = NULL; pcmk__free_tls(tls); tls = NULL; } if (private->command.tcp_socket >= 0) { shutdown(private->command.tcp_socket, SHUT_RDWR); /* no more receptions */ close(private->command.tcp_socket); } if (private->callback.tcp_socket >= 0) { shutdown(private->callback.tcp_socket, SHUT_RDWR); /* no more receptions */ close(private->callback.tcp_socket); } private->command.tcp_socket = -1; private->callback.tcp_socket = -1; free(private->command.buffer); free(private->callback.buffer); private->command.buffer = NULL; private->callback.buffer = NULL; return 0; } static void cib_remote_connection_destroy(gpointer user_data) { pcmk__err("Connection destroyed"); cib_tls_close(user_data); } static int cib_tls_signon(cib_t *cib, pcmk__remote_t *connection, gboolean event_channel) { cib_remote_opaque_t *private = cib->variant_opaque; int rc; xmlNode *answer = NULL; xmlNode *login = NULL; static struct mainloop_fd_callbacks cib_fd_callbacks = { 0, }; cib_fd_callbacks.dispatch = event_channel ? cib_remote_callback_dispatch : cib_remote_command_dispatch; cib_fd_callbacks.destroy = cib_remote_connection_destroy; connection->tcp_socket = -1; connection->tls_session = NULL; rc = pcmk__connect_remote(private->server, private->port, 0, NULL, &(connection->tcp_socket), NULL, NULL); if (rc != pcmk_rc_ok) { pcmk__info("Remote connection to %s:%d failed: %s " QB_XS " rc=%d", private->server, private->port, pcmk_rc_str(rc), rc); return -ENOTCONN; } if (private->encrypted) { bool use_cert = pcmk__x509_enabled(); int tls_rc = GNUTLS_E_SUCCESS; rc = pcmk__init_tls(&tls, false, use_cert ? GNUTLS_CRD_CERTIFICATE : GNUTLS_CRD_ANON); if (rc != pcmk_rc_ok) { return -1; } /* bind the socket to GnuTls lib */ connection->tls_session = pcmk__new_tls_session(tls, connection->tcp_socket); if (connection->tls_session == NULL) { cib_tls_close(cib); return -1; } rc = pcmk__tls_client_handshake(connection, TLS_HANDSHAKE_TIMEOUT, &tls_rc); if (rc != pcmk_rc_ok) { const bool proto_err = (rc == EPROTO); pcmk__err("Remote CIB session creation for %s:%d failed: %s", private->server, private->port, (proto_err? gnutls_strerror(tls_rc) : pcmk_rc_str(rc))); gnutls_deinit(connection->tls_session); connection->tls_session = NULL; cib_tls_close(cib); return -1; } } /* Now that the handshake is done, see if any client TLS certificate is * close to its expiration date and log if so. If a TLS certificate is not * in use, this function will just return so we don't need to check for the * session type here. */ pcmk__tls_check_cert_expiration(connection->tls_session); /* login to server */ login = pcmk__xe_create(NULL, PCMK__XE_CIB_COMMAND); pcmk__xe_set(login, PCMK_XA_OP, "authenticate"); pcmk__xe_set(login, PCMK_XA_USER, private->user); pcmk__xe_set(login, PCMK__XA_PASSWORD, private->passwd); pcmk__xe_set(login, PCMK__XA_HIDDEN, PCMK__VALUE_PASSWORD); pcmk__remote_send_xml(connection, login); pcmk__xml_free(login); rc = pcmk_ok; if (pcmk__read_remote_message(connection, -1) == ENOTCONN) { rc = -ENOTCONN; } answer = pcmk__remote_message_xml(connection); - crm_log_xml_trace(answer, "Reply"); + pcmk__log_xml_trace(answer, "Reply"); if (answer == NULL) { rc = -EPROTO; } else { /* grab the token */ const char *msg_type = pcmk__xe_get(answer, PCMK__XA_CIB_OP); const char *tmp_ticket = pcmk__xe_get(answer, PCMK__XA_CIB_CLIENTID); if (!pcmk__str_eq(msg_type, CRM_OP_REGISTER, pcmk__str_casei)) { pcmk__err("Invalid registration message: %s", msg_type); rc = -EPROTO; } else if (tmp_ticket == NULL) { rc = -EPROTO; } else { connection->token = strdup(tmp_ticket); } } pcmk__xml_free(answer); answer = NULL; if (rc != 0) { cib_tls_close(cib); return rc; } pcmk__trace("remote client connection established"); private->timeout_sec = 60; connection->source = mainloop_add_fd("cib-remote", G_PRIORITY_HIGH, connection->tcp_socket, cib, &cib_fd_callbacks); return rc; } static int cib_remote_signon(cib_t *cib, const char *name, enum cib_conn_type type) { int rc = pcmk_ok; cib_remote_opaque_t *private = cib->variant_opaque; if (name == NULL) { name = pcmk__s(crm_system_name, "client"); } if (private->passwd == NULL) { if (private->out == NULL) { /* If no pcmk__output_t is set, just assume that a text prompt * is good enough. */ pcmk__text_prompt("Password", false, &(private->passwd)); } else { private->out->prompt("Password", false, &(private->passwd)); } } if (private->server == NULL || private->user == NULL) { rc = -EINVAL; goto done; } rc = cib_tls_signon(cib, &(private->command), FALSE); if (rc != pcmk_ok) { goto done; } rc = cib_tls_signon(cib, &(private->callback), TRUE); done: if (rc == pcmk_ok) { pcmk__info("Opened connection to %s:%d for %s", private->server, private->port, name); cib->state = cib_connected_command; cib->type = cib_command; } else { pcmk__info("Connection to %s:%d for %s failed: %s\n", private->server, private->port, name, pcmk_strerror(rc)); } return rc; } static int cib_remote_signoff(cib_t *cib) { int rc = pcmk_ok; pcmk__debug("Disconnecting from the CIB manager"); cib_tls_close(cib); cib->cmds->end_transaction(cib, false, cib_none); cib->state = cib_disconnected; cib->type = cib_no_connection; return rc; } static int cib_remote_free(cib_t *cib) { int rc = pcmk_ok; pcmk__warn("Freeing CIB"); if (cib->state != cib_disconnected) { rc = cib_remote_signoff(cib); if (rc == pcmk_ok) { cib_remote_opaque_t *private = cib->variant_opaque; free(private->server); free(private->user); free(private->passwd); free(cib->cmds); free(cib->user); free(private); free(cib); } } return rc; } static int cib_remote_register_notification(cib_t * cib, const char *callback, int enabled) { xmlNode *notify_msg = pcmk__xe_create(NULL, PCMK__XE_CIB_COMMAND); cib_remote_opaque_t *private = cib->variant_opaque; pcmk__xe_set(notify_msg, PCMK__XA_CIB_OP, PCMK__VALUE_CIB_NOTIFY); pcmk__xe_set(notify_msg, PCMK__XA_CIB_NOTIFY_TYPE, callback); pcmk__xe_set_int(notify_msg, PCMK__XA_CIB_NOTIFY_ACTIVATE, enabled); pcmk__remote_send_xml(&private->callback, notify_msg); pcmk__xml_free(notify_msg); return pcmk_ok; } static int cib_remote_set_connection_dnotify(cib_t * cib, void (*dnotify) (gpointer user_data)) { return -EPROTONOSUPPORT; } /*! * \internal * \brief Get the given CIB connection's unique client identifiers * * These can be used to check whether this client requested the action that * triggered a CIB notification. * * \param[in] cib CIB connection * \param[out] async_id If not \p NULL, where to store asynchronous client ID * \param[out] sync_id If not \p NULL, where to store synchronous client ID * * \return Legacy Pacemaker return code (specifically, \p pcmk_ok) * * \note This is the \p cib_remote variant implementation of * \p cib_api_operations_t:client_id(). * \note The client IDs are assigned during CIB sign-on. */ static int cib_remote_client_id(const cib_t *cib, const char **async_id, const char **sync_id) { cib_remote_opaque_t *private = cib->variant_opaque; if (async_id != NULL) { // private->callback is the channel for async requests *async_id = private->callback.token; } if (sync_id != NULL) { // private->command is the channel for sync requests *sync_id = private->command.token; } return pcmk_ok; } cib_t * cib_remote_new(const char *server, const char *user, const char *passwd, int port, gboolean encrypted) { cib_remote_opaque_t *private = NULL; cib_t *cib = cib_new_variant(); if (cib == NULL) { return NULL; } private = calloc(1, sizeof(cib_remote_opaque_t)); if (private == NULL) { free(cib); return NULL; } cib->variant = cib_remote; cib->variant_opaque = private; private->server = pcmk__str_copy(server); private->user = pcmk__str_copy(user); private->passwd = pcmk__str_copy(passwd); private->port = port; private->encrypted = encrypted; /* assign variant specific ops */ cib->delegate_fn = cib_remote_perform_op; cib->cmds->signon = cib_remote_signon; cib->cmds->signoff = cib_remote_signoff; cib->cmds->free = cib_remote_free; cib->cmds->register_notification = cib_remote_register_notification; cib->cmds->set_connection_dnotify = cib_remote_set_connection_dnotify; cib->cmds->client_id = cib_remote_client_id; return cib; } void cib__set_output(cib_t *cib, pcmk__output_t *out) { cib_remote_opaque_t *private; if (cib->variant != cib_remote) { return; } private = cib->variant_opaque; private->out = out; } diff --git a/lib/cib/cib_utils.c b/lib/cib/cib_utils.c index ace576eb65..587706d41d 100644 --- a/lib/cib/cib_utils.c +++ b/lib/cib/cib_utils.c @@ -1,968 +1,968 @@ /* * Original copyright 2004 International Business Machines * Later changes copyright 2008-2025 the Pacemaker project contributors * * The version control history for this file may have further details. * * This source code is licensed under the GNU Lesser General Public License * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. */ #include #include #include #include #include #include #include #include #include #include #include // pcmk_acl_required(), etc. #include #include // pcmk_unpack_nvpair_blocks() #include #include gboolean cib_version_details(xmlNode * cib, int *admin_epoch, int *epoch, int *updates) { *epoch = -1; *updates = -1; *admin_epoch = -1; if (cib == NULL) { return FALSE; } pcmk__xe_get_int(cib, PCMK_XA_EPOCH, epoch); pcmk__xe_get_int(cib, PCMK_XA_NUM_UPDATES, updates); pcmk__xe_get_int(cib, PCMK_XA_ADMIN_EPOCH, admin_epoch); return TRUE; } gboolean cib_diff_version_details(xmlNode * diff, int *admin_epoch, int *epoch, int *updates, int *_admin_epoch, int *_epoch, int *_updates) { int add[] = { 0, 0, 0 }; int del[] = { 0, 0, 0 }; pcmk__xml_patchset_versions(diff, del, add); *admin_epoch = add[0]; *epoch = add[1]; *updates = add[2]; *_admin_epoch = del[0]; *_epoch = del[1]; *_updates = del[2]; return TRUE; } /*! * \internal * \brief Get the XML patchset from a CIB diff notification * * \param[in] msg CIB diff notification * \param[out] patchset Where to store XML patchset * * \return Standard Pacemaker return code */ int cib__get_notify_patchset(const xmlNode *msg, const xmlNode **patchset) { int rc = pcmk_err_generic; xmlNode *wrapper = NULL; pcmk__assert(patchset != NULL); *patchset = NULL; if (msg == NULL) { pcmk__err("CIB diff notification received with no XML"); return ENOMSG; } if ((pcmk__xe_get_int(msg, PCMK__XA_CIB_RC, &rc) != pcmk_rc_ok) || (rc != pcmk_ok)) { pcmk__warn("Ignore failed CIB update: %s " QB_XS " rc=%d", pcmk_strerror(rc), rc); pcmk__log_xml_debug(msg, "failed"); return pcmk_legacy2rc(rc); } wrapper = pcmk__xe_first_child(msg, PCMK__XE_CIB_UPDATE_RESULT, NULL, NULL); *patchset = pcmk__xe_first_child(wrapper, NULL, NULL, NULL); if (*patchset == NULL) { pcmk__err("CIB diff notification received with no patchset"); return ENOMSG; } return pcmk_rc_ok; } /*! * \brief Create XML for a new (empty) CIB * * \param[in] cib_epoch What to use as \c PCMK_XA_EPOCH CIB attribute * * \return Newly created XML for empty CIB * * \note It is the caller's responsibility to free the result with * \c pcmk__xml_free(). */ xmlNode * createEmptyCib(int cib_epoch) { xmlNode *cib_root = NULL, *config = NULL; cib_root = pcmk__xe_create(NULL, PCMK_XE_CIB); pcmk__xe_set(cib_root, PCMK_XA_CRM_FEATURE_SET, CRM_FEATURE_SET); pcmk__xe_set(cib_root, PCMK_XA_VALIDATE_WITH, pcmk__highest_schema_name()); pcmk__xe_set_int(cib_root, PCMK_XA_EPOCH, cib_epoch); pcmk__xe_set_int(cib_root, PCMK_XA_NUM_UPDATES, 0); pcmk__xe_set_int(cib_root, PCMK_XA_ADMIN_EPOCH, 0); config = pcmk__xe_create(cib_root, PCMK_XE_CONFIGURATION); pcmk__xe_create(cib_root, PCMK_XE_STATUS); pcmk__xe_create(config, PCMK_XE_CRM_CONFIG); pcmk__xe_create(config, PCMK_XE_NODES); pcmk__xe_create(config, PCMK_XE_RESOURCES); pcmk__xe_create(config, PCMK_XE_CONSTRAINTS); #if PCMK__RESOURCE_STICKINESS_DEFAULT != 0 { xmlNode *rsc_defaults = pcmk__xe_create(config, PCMK_XE_RSC_DEFAULTS); xmlNode *meta = pcmk__xe_create(rsc_defaults, PCMK_XE_META_ATTRIBUTES); xmlNode *nvpair = pcmk__xe_create(meta, PCMK_XE_NVPAIR); pcmk__xe_set(meta, PCMK_XA_ID, "build-resource-defaults"); pcmk__xe_set(nvpair, PCMK_XA_ID, "build-" PCMK_META_RESOURCE_STICKINESS); pcmk__xe_set(nvpair, PCMK_XA_NAME, PCMK_META_RESOURCE_STICKINESS); pcmk__xe_set_int(nvpair, PCMK_XA_VALUE, PCMK__RESOURCE_STICKINESS_DEFAULT); } #endif return cib_root; } static bool cib_acl_enabled(xmlNode *xml, const char *user) { bool rc = false; if(pcmk_acl_required(user)) { const char *value = NULL; GHashTable *options = pcmk__strkey_table(free, free); cib_read_config(options, xml); value = pcmk__cluster_option(options, PCMK_OPT_ENABLE_ACL); rc = pcmk__is_true(value); g_hash_table_destroy(options); } pcmk__trace("CIB ACL is %s", (rc? "enabled" : "disabled")); return rc; } /*! * \internal * \brief Determine whether to perform operations on a scratch copy of the CIB * * \param[in] op CIB operation * \param[in] section CIB section * \param[in] call_options CIB call options * * \return \p true if we should make a copy of the CIB, or \p false otherwise */ static bool should_copy_cib(const char *op, const char *section, int call_options) { if (pcmk__is_set(call_options, cib_dryrun)) { // cib_dryrun implies a scratch copy by definition; no side effects return true; } if (pcmk__str_eq(op, PCMK__CIB_REQUEST_COMMIT_TRANSACT, pcmk__str_none)) { /* Commit-transaction must make a copy for atomicity. We must revert to * the original CIB if the entire transaction cannot be applied * successfully. */ return true; } if (pcmk__is_set(call_options, cib_transaction)) { /* If cib_transaction is set, then we're in the process of committing a * transaction. The commit-transaction request already made a scratch * copy, and we're accumulating changes in that copy. */ return false; } if (pcmk__str_eq(section, PCMK_XE_STATUS, pcmk__str_none)) { /* Copying large CIBs accounts for a huge percentage of our CIB usage, * and this avoids some of it. * * @TODO: Is this safe? See discussion at * https://github.com/ClusterLabs/pacemaker/pull/3094#discussion_r1211400690. */ return false; } // Default behavior is to operate on a scratch copy return true; } int cib_perform_op(cib_t *cib, const char *op, uint32_t call_options, cib__op_fn_t fn, bool is_query, const char *section, xmlNode *req, xmlNode *input, bool manage_counters, bool *config_changed, xmlNode **current_cib, xmlNode **result_cib, xmlNode **diff, xmlNode **output) { int rc = pcmk_ok; bool check_schema = true; bool make_copy = true; xmlNode *top = NULL; xmlNode *scratch = NULL; xmlNode *patchset_cib = NULL; xmlNode *local_diff = NULL; const char *user = pcmk__xe_get(req, PCMK__XA_CIB_USER); const bool enable_acl = cib_acl_enabled(*current_cib, user); bool with_digest = false; pcmk__trace("Begin %s%s%s op", (pcmk__is_set(call_options, cib_dryrun)? "dry run of " : ""), (is_query? "read-only " : ""), op); CRM_CHECK(output != NULL, return -ENOMSG); CRM_CHECK(current_cib != NULL, return -ENOMSG); CRM_CHECK(result_cib != NULL, return -ENOMSG); CRM_CHECK(config_changed != NULL, return -ENOMSG); if(output) { *output = NULL; } *result_cib = NULL; *config_changed = false; if (fn == NULL) { return -EINVAL; } if (is_query) { xmlNode *cib_ro = *current_cib; xmlNode *cib_filtered = NULL; if (enable_acl && xml_acl_filtered_copy(user, *current_cib, *current_cib, &cib_filtered)) { if (cib_filtered == NULL) { pcmk__debug("Pre-filtered the entire cib"); return -EACCES; } cib_ro = cib_filtered; - crm_log_xml_trace(cib_ro, "filtered"); + pcmk__log_xml_trace(cib_ro, "filtered"); } rc = (*fn) (op, call_options, section, req, input, cib_ro, result_cib, output); if(output == NULL || *output == NULL) { /* nothing */ } else if(cib_filtered == *output) { cib_filtered = NULL; /* Let them have this copy */ } else if (*output == *current_cib) { /* They already know not to free it */ } else if(cib_filtered && (*output)->doc == cib_filtered->doc) { /* We're about to free the document of which *output is a part */ *output = pcmk__xml_copy(NULL, *output); } else if ((*output)->doc == (*current_cib)->doc) { /* Give them a copy they can free */ *output = pcmk__xml_copy(NULL, *output); } pcmk__xml_free(cib_filtered); return rc; } make_copy = should_copy_cib(op, section, call_options); if (!make_copy) { /* Conditional on v2 patch style */ scratch = *current_cib; // Make a copy of the top-level element to store version details top = pcmk__xe_create(NULL, (const char *) scratch->name); pcmk__xe_copy_attrs(top, scratch, pcmk__xaf_none); patchset_cib = top; pcmk__xml_commit_changes(scratch->doc); pcmk__xml_doc_set_flags(scratch->doc, pcmk__xf_tracking); if (enable_acl) { pcmk__enable_acl(*current_cib, scratch, user); } rc = (*fn) (op, call_options, section, req, input, scratch, &scratch, output); /* If scratch points to a new object now (for example, after an erase * operation), then *current_cib should point to the same object. * * @TODO Enable tracking and ACLs and calculate changes? Change tracking * and unpacked ACLs didn't carry over to new object. */ *current_cib = scratch; } else { scratch = pcmk__xml_copy(NULL, *current_cib); patchset_cib = *current_cib; pcmk__xml_doc_set_flags(scratch->doc, pcmk__xf_tracking); if (enable_acl) { pcmk__enable_acl(*current_cib, scratch, user); } rc = (*fn) (op, call_options, section, req, input, *current_cib, &scratch, output); /* @TODO This appears to be a hack to determine whether scratch points * to a new object now, without saving the old pointer (which may be * invalid now) for comparison. Confirm this, and check more clearly. */ if (!pcmk__xml_doc_all_flags_set(scratch->doc, pcmk__xf_tracking)) { pcmk__trace("Inferring changes after %s op", op); pcmk__xml_commit_changes(scratch->doc); if (enable_acl) { pcmk__enable_acl(*current_cib, scratch, user); } pcmk__xml_mark_changes(*current_cib, scratch); } CRM_CHECK(*current_cib != scratch, return -EINVAL); } xml_acl_disable(scratch); /* Allow the system to make any additional changes */ if (rc == pcmk_ok && scratch == NULL) { rc = -EINVAL; goto done; } else if(rc == pcmk_ok && xml_acl_denied(scratch)) { pcmk__trace("ACL rejected part or all of the proposed changes"); rc = -EACCES; goto done; } else if (rc != pcmk_ok) { goto done; } /* If the CIB is from a file, we don't need to check that the feature set is * supported. All we care about in that case is the schema version, which * is checked elsewhere. */ if (scratch && (cib == NULL || cib->variant != cib_file)) { const char *new_version = pcmk__xe_get(scratch, PCMK_XA_CRM_FEATURE_SET); rc = pcmk__check_feature_set(new_version); if (rc != pcmk_rc_ok) { pcmk__err("Discarding update with feature set '%s' greater than " "our own '%s'", new_version, CRM_FEATURE_SET); rc = pcmk_rc2legacy(rc); goto done; } } if (patchset_cib != NULL) { int old = 0; int new = 0; pcmk__xe_get_int(scratch, PCMK_XA_ADMIN_EPOCH, &new); pcmk__xe_get_int(patchset_cib, PCMK_XA_ADMIN_EPOCH, &old); if (old > new) { pcmk__err("%s went backwards: %d -> %d (Opts: %#x)", PCMK_XA_ADMIN_EPOCH, old, new, call_options); pcmk__log_xml_warn(req, "Bad Op"); pcmk__log_xml_warn(input, "Bad Data"); rc = -pcmk_err_old_data; } else if (old == new) { pcmk__xe_get_int(scratch, PCMK_XA_EPOCH, &new); pcmk__xe_get_int(patchset_cib, PCMK_XA_EPOCH, &old); if (old > new) { pcmk__err("%s went backwards: %d -> %d (Opts: %#x)", PCMK_XA_EPOCH, old, new, call_options); pcmk__log_xml_warn(req, "Bad Op"); pcmk__log_xml_warn(input, "Bad Data"); rc = -pcmk_err_old_data; } } } pcmk__trace("Massaging CIB contents"); pcmk__strip_xml_text(scratch); if (make_copy) { static time_t expires = 0; time_t tm_now = time(NULL); if (expires < tm_now) { expires = tm_now + 60; /* Validate clients are correctly applying v2-style diffs at most once a minute */ with_digest = true; } } local_diff = xml_create_patchset(0, patchset_cib, scratch, config_changed, manage_counters); pcmk__log_xml_changes(PCMK__LOG_TRACE, scratch); pcmk__xml_commit_changes(scratch->doc); if(local_diff) { if (with_digest) { pcmk__xml_patchset_add_digest(local_diff, scratch); } pcmk__log_xml_patchset(LOG_INFO, local_diff); - crm_log_xml_trace(local_diff, "raw patch"); + pcmk__log_xml_trace(local_diff, "raw patch"); } if (make_copy && (local_diff != NULL)) { // Original to compare against doesn't exist pcmk__if_tracing( { // Validate the calculated patch set int test_rc = pcmk_ok; int format = 1; xmlNode *cib_copy = pcmk__xml_copy(NULL, patchset_cib); pcmk__xe_get_int(local_diff, PCMK_XA_FORMAT, &format); test_rc = xml_apply_patchset(cib_copy, local_diff, manage_counters); if (test_rc != pcmk_ok) { pcmk__xml_write_temp_file(cib_copy, "PatchApply:calculated", NULL); pcmk__xml_write_temp_file(patchset_cib, "PatchApply:input", NULL); pcmk__xml_write_temp_file(scratch, "PatchApply:actual", NULL); pcmk__xml_write_temp_file(local_diff, "PatchApply:diff", NULL); pcmk__err("v%d patchset error, patch failed to apply: %s " "(%d)", format, pcmk_rc_str(pcmk_legacy2rc(test_rc)), test_rc); } pcmk__xml_free(cib_copy); }, {} ); } if (pcmk__str_eq(section, PCMK_XE_STATUS, pcmk__str_casei)) { /* Throttle the amount of costly validation we perform due to status updates * a) we don't really care whats in the status section * b) we don't validate any of its contents at the moment anyway */ check_schema = false; } /* === scratch must not be modified after this point === * Exceptions, anything in: static filter_t filter[] = { { 0, PCMK_XA_CRM_DEBUG_ORIGIN }, { 0, PCMK_XA_CIB_LAST_WRITTEN }, { 0, PCMK_XA_UPDATE_ORIGIN }, { 0, PCMK_XA_UPDATE_CLIENT }, { 0, PCMK_XA_UPDATE_USER }, }; */ if (*config_changed && !pcmk__is_set(call_options, cib_no_mtime)) { const char *schema = pcmk__xe_get(scratch, PCMK_XA_VALIDATE_WITH); if (schema == NULL) { rc = -pcmk_err_cib_corrupt; } pcmk__xe_add_last_written(scratch); pcmk__warn_if_schema_deprecated(schema); /* Make values of origin, client, and user in scratch match * the ones in req (if the schema allows the attributes) */ if (pcmk__cmp_schemas_by_name(schema, "pacemaker-1.2") >= 0) { const char *origin = pcmk__xe_get(req, PCMK__XA_SRC); const char *client = pcmk__xe_get(req, PCMK__XA_CIB_CLIENTNAME); if (origin != NULL) { pcmk__xe_set(scratch, PCMK_XA_UPDATE_ORIGIN, origin); } else { pcmk__xe_remove_attr(scratch, PCMK_XA_UPDATE_ORIGIN); } if (client != NULL) { pcmk__xe_set(scratch, PCMK_XA_UPDATE_CLIENT, user); } else { pcmk__xe_remove_attr(scratch, PCMK_XA_UPDATE_CLIENT); } if (user != NULL) { pcmk__xe_set(scratch, PCMK_XA_UPDATE_USER, user); } else { pcmk__xe_remove_attr(scratch, PCMK_XA_UPDATE_USER); } } } pcmk__trace("Perform validation: %s", pcmk__btoa(check_schema)); if ((rc == pcmk_ok) && check_schema && !pcmk__configured_schema_validates(scratch)) { rc = -pcmk_err_schema_validation; } done: *result_cib = scratch; /* @TODO: This may not work correctly with !make_copy, since we don't * keep the original CIB. */ if ((rc != pcmk_ok) && cib_acl_enabled(patchset_cib, user) && xml_acl_filtered_copy(user, patchset_cib, scratch, result_cib)) { if (*result_cib == NULL) { pcmk__debug("Pre-filtered the entire cib result"); } pcmk__xml_free(scratch); } if(diff) { *diff = local_diff; } else { pcmk__xml_free(local_diff); } pcmk__xml_free(top); pcmk__trace("Done"); return rc; } int cib__create_op(cib_t *cib, const char *op, const char *host, const char *section, xmlNode *data, int call_options, const char *user_name, const char *client_name, xmlNode **op_msg) { CRM_CHECK((cib != NULL) && (op_msg != NULL), return -EPROTO); *op_msg = pcmk__xe_create(NULL, PCMK__XE_CIB_COMMAND); cib->call_id++; if (cib->call_id < 1) { cib->call_id = 1; } pcmk__xe_set(*op_msg, PCMK__XA_T, PCMK__VALUE_CIB); pcmk__xe_set(*op_msg, PCMK__XA_CIB_OP, op); pcmk__xe_set(*op_msg, PCMK__XA_CIB_HOST, host); pcmk__xe_set(*op_msg, PCMK__XA_CIB_SECTION, section); pcmk__xe_set(*op_msg, PCMK__XA_CIB_USER, user_name); pcmk__xe_set(*op_msg, PCMK__XA_CIB_CLIENTNAME, client_name); pcmk__xe_set_int(*op_msg, PCMK__XA_CIB_CALLID, cib->call_id); pcmk__trace("Sending call options: %.8lx, %d", (long) call_options, call_options); pcmk__xe_set_int(*op_msg, PCMK__XA_CIB_CALLOPT, call_options); if (data != NULL) { xmlNode *wrapper = pcmk__xe_create(*op_msg, PCMK__XE_CIB_CALLDATA); pcmk__xml_copy(wrapper, data); } return pcmk_ok; } /*! * \internal * \brief Check whether a CIB request is supported in a transaction * * \param[in] request CIB request * * \return Standard Pacemaker return code */ static int validate_transaction_request(const xmlNode *request) { const char *op = pcmk__xe_get(request, PCMK__XA_CIB_OP); const char *host = pcmk__xe_get(request, PCMK__XA_CIB_HOST); const cib__operation_t *operation = NULL; int rc = cib__get_operation(op, &operation); if (rc != pcmk_rc_ok) { // cib__get_operation() logs error return rc; } if (!pcmk__is_set(operation->flags, cib__op_attr_transaction)) { pcmk__err("Operation %s is not supported in CIB transactions", op); return EOPNOTSUPP; } if (host != NULL) { pcmk__err("Operation targeting a specific node (%s) is not supported " "in a CIB transaction", host); return EOPNOTSUPP; } return pcmk_rc_ok; } /*! * \internal * \brief Append a CIB request to a CIB transaction * * \param[in,out] cib CIB client whose transaction to extend * \param[in,out] request Request to add to transaction * * \return Legacy Pacemaker return code */ int cib__extend_transaction(cib_t *cib, xmlNode *request) { int rc = pcmk_rc_ok; pcmk__assert((cib != NULL) && (request != NULL)); rc = validate_transaction_request(request); if ((rc == pcmk_rc_ok) && (cib->transaction == NULL)) { rc = pcmk_rc_no_transaction; } if (rc == pcmk_rc_ok) { pcmk__xml_copy(cib->transaction, request); } else { const char *op = pcmk__xe_get(request, PCMK__XA_CIB_OP); const char *client_id = NULL; cib->cmds->client_id(cib, NULL, &client_id); pcmk__err("Failed to add '%s' operation to transaction for client %s: " "%s", op, pcmk__s(client_id, "(unidentified)"), pcmk_rc_str(rc)); pcmk__log_xml_info(request, "failed"); } return pcmk_rc2legacy(rc); } void cib_native_callback(cib_t * cib, xmlNode * msg, int call_id, int rc) { xmlNode *output = NULL; cib_callback_client_t *blob = NULL; if (msg != NULL) { xmlNode *wrapper = NULL; pcmk__xe_get_int(msg, PCMK__XA_CIB_RC, &rc); pcmk__xe_get_int(msg, PCMK__XA_CIB_CALLID, &call_id); wrapper = pcmk__xe_first_child(msg, PCMK__XE_CIB_CALLDATA, NULL, NULL); output = pcmk__xe_first_child(wrapper, NULL, NULL, NULL); } blob = cib__lookup_id(call_id); if (blob == NULL) { pcmk__trace("No callback found for call %d", call_id); } if (cib == NULL) { pcmk__debug("No cib object supplied"); } if (rc == -pcmk_err_diff_resync) { /* This is an internal value that clients do not and should not care about */ rc = pcmk_ok; } if (blob && blob->callback && (rc == pcmk_ok || blob->only_success == FALSE)) { pcmk__trace("Invoking callback %s for call %d", pcmk__s(blob->id, "without ID"), call_id); blob->callback(msg, call_id, rc, output, blob->user_data); } else if ((cib != NULL) && (rc != pcmk_ok)) { pcmk__warn("CIB command failed: %s", pcmk_strerror(rc)); pcmk__log_xml_debug(msg, "Failed CIB Update"); } /* This may free user_data, so do it after the callback */ if (blob) { remove_cib_op_callback(call_id, FALSE); } pcmk__trace("OP callback activated for %d", call_id); } void cib_native_notify(gpointer data, gpointer user_data) { xmlNode *msg = user_data; cib_notify_client_t *entry = data; const char *event = NULL; if (msg == NULL) { pcmk__warn("Skipping callback - NULL message"); return; } event = pcmk__xe_get(msg, PCMK__XA_SUBT); if (entry == NULL) { pcmk__warn("Skipping callback - NULL callback client"); return; } else if (entry->callback == NULL) { pcmk__warn("Skipping callback - NULL callback"); return; } else if (!pcmk__str_eq(entry->event, event, pcmk__str_casei)) { pcmk__trace("Skipping callback - event mismatch %p/%s vs. %s", entry, entry->event, event); return; } pcmk__trace("Invoking callback for %p/%s event...", entry, event); entry->callback(event, msg); pcmk__trace("Callback invoked..."); } gboolean cib_read_config(GHashTable * options, xmlNode * current_cib) { xmlNode *config = NULL; crm_time_t *now = NULL; if (options == NULL || current_cib == NULL) { return FALSE; } now = crm_time_new(NULL); g_hash_table_remove_all(options); config = pcmk_find_cib_element(current_cib, PCMK_XE_CRM_CONFIG); if (config) { pcmk_rule_input_t rule_input = { .now = now, }; pcmk_unpack_nvpair_blocks(config, PCMK_XE_CLUSTER_PROPERTY_SET, PCMK_VALUE_CIB_BOOTSTRAP_OPTIONS, &rule_input, options, NULL); } pcmk__validate_cluster_options(options); crm_time_free(now); return TRUE; } int cib_internal_op(cib_t * cib, const char *op, const char *host, const char *section, xmlNode * data, xmlNode ** output_data, int call_options, const char *user_name) { int (*delegate)(cib_t *cib, const char *op, const char *host, const char *section, xmlNode *data, xmlNode **output_data, int call_options, const char *user_name) = NULL; if (cib == NULL) { return -EINVAL; } delegate = cib->delegate_fn; if (delegate == NULL) { return -EPROTONOSUPPORT; } if (user_name == NULL) { user_name = getenv("CIB_user"); } return delegate(cib, op, host, section, data, output_data, call_options, user_name); } /*! * \brief Apply a CIB update patch to a given CIB * * \param[in] event CIB update patch * \param[in] input CIB to patch * \param[out] output Resulting CIB after patch * \param[in] level Log the patch at this log level (unless LOG_CRIT) * * \return Legacy Pacemaker return code * \note sbd calls this function */ int cib_apply_patch_event(xmlNode *event, xmlNode *input, xmlNode **output, int level) { int rc = pcmk_err_generic; xmlNode *wrapper = NULL; xmlNode *diff = NULL; pcmk__assert((event != NULL) && (input != NULL) && (output != NULL)); pcmk__xe_get_int(event, PCMK__XA_CIB_RC, &rc); wrapper = pcmk__xe_first_child(event, PCMK__XE_CIB_UPDATE_RESULT, NULL, NULL); diff = pcmk__xe_first_child(wrapper, NULL, NULL, NULL); if (rc < pcmk_ok || diff == NULL) { return rc; } if (level > LOG_CRIT) { pcmk__log_xml_patchset(level, diff); } if (input != NULL) { rc = cib_process_diff(NULL, cib_none, NULL, event, diff, input, output, NULL); if (rc != pcmk_ok) { pcmk__debug("Update didn't apply: %s (%d) %p", pcmk_strerror(rc), rc, *output); if (rc == -pcmk_err_old_data) { pcmk__trace("Masking error, we already have the supplied " "update"); return pcmk_ok; } pcmk__xml_free(*output); *output = NULL; return rc; } } return rc; } #define log_signon_query_err(out, fmt, args...) do { \ if (out != NULL) { \ out->err(out, fmt, ##args); \ } else { \ pcmk__err(fmt, ##args); \ } \ } while (0) int cib__signon_query(pcmk__output_t *out, cib_t **cib, xmlNode **cib_object) { int rc = pcmk_rc_ok; cib_t *cib_conn = NULL; pcmk__assert(cib_object != NULL); if (cib == NULL) { cib_conn = cib_new(); } else { if (*cib == NULL) { *cib = cib_new(); } cib_conn = *cib; } if (cib_conn == NULL) { return ENOMEM; } if (cib_conn->state == cib_disconnected) { rc = cib_conn->cmds->signon(cib_conn, crm_system_name, cib_command); rc = pcmk_legacy2rc(rc); } if (rc != pcmk_rc_ok) { log_signon_query_err(out, "Could not connect to the CIB: %s", pcmk_rc_str(rc)); goto done; } if (out != NULL) { out->transient(out, "Querying CIB..."); } rc = cib_conn->cmds->query(cib_conn, NULL, cib_object, cib_sync_call); rc = pcmk_legacy2rc(rc); if (rc != pcmk_rc_ok) { log_signon_query_err(out, "CIB query failed: %s", pcmk_rc_str(rc)); } done: if (cib == NULL) { cib__clean_up_connection(&cib_conn); } if ((rc == pcmk_rc_ok) && (*cib_object == NULL)) { return pcmk_rc_no_input; } return rc; } int cib__signon_attempts(cib_t *cib, enum cib_conn_type type, int attempts) { int rc = pcmk_rc_ok; pcmk__trace("Attempting connection to CIB manager (up to %d time%s)", attempts, pcmk__plural_s(attempts)); for (int remaining = attempts - 1; remaining >= 0; --remaining) { rc = cib->cmds->signon(cib, crm_system_name, type); if ((rc == pcmk_rc_ok) || (remaining == 0) || ((errno != EAGAIN) && (errno != EALREADY))) { break; } // Retry after soft error (interrupted by signal, etc.) pcmk__sleep_ms((attempts - remaining) * 500); pcmk__debug("Re-attempting connection to CIB manager (%d attempt%s " "remaining)", remaining, pcmk__plural_s(remaining)); } return rc; } int cib__clean_up_connection(cib_t **cib) { int rc; if (*cib == NULL) { return pcmk_rc_ok; } rc = (*cib)->cmds->signoff(*cib); cib_delete(*cib); *cib = NULL; return pcmk_legacy2rc(rc); } diff --git a/lib/cluster/election.c b/lib/cluster/election.c index a8ffb7c564..99e07d5ec5 100644 --- a/lib/cluster/election.c +++ b/lib/cluster/election.c @@ -1,750 +1,750 @@ /* * Copyright 2004-2025 the Pacemaker project contributors * * The version control history for this file may have further details. * * This source code is licensed under the GNU Lesser General Public License * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. */ #include #include #include #include #include #include #include #include #include "crmcluster_private.h" #define STORM_INTERVAL 2 /* in seconds */ struct pcmk__election { enum election_result state; // Current state of election guint count; // How many times local node has voted void (*cb)(pcmk_cluster_t *); // Function to call if election is won GHashTable *voted; // Key = node name, value = how node voted mainloop_timer_t *timeout; // When to abort if all votes not received int election_wins; // Track wins, for storm detection bool wrote_blackbox; // Write a storm blackbox at most once time_t expires; // When storm detection period ends time_t last_election_loss; // When dampening period ends }; static void election_complete(pcmk_cluster_t *cluster) { pcmk__assert((cluster != NULL) && (cluster->priv->election != NULL)); cluster->priv->election->state = election_won; if (cluster->priv->election->cb != NULL) { cluster->priv->election->cb(cluster); } election_reset(cluster); } static gboolean election_timer_cb(gpointer user_data) { pcmk_cluster_t *cluster = user_data; pcmk__info("Declaring local node as winner after election timed out"); election_complete(cluster); return FALSE; } /*! * \internal * \brief Get current state of an election * * \param[in] cluster Cluster with election * * \return Current state of \e */ enum election_result election_state(const pcmk_cluster_t *cluster) { if ((cluster == NULL) || (cluster->priv->election == NULL)) { return election_error; } return cluster->priv->election->state; } /* The local node will be declared the winner if missing votes are not received * within this time. The value is chosen to be the same as the default for the * election-timeout cluster option. */ #define ELECTION_TIMEOUT_MS 120000 /*! * \internal * \brief Track election state in a cluster * * Every node that wishes to participate in an election must initialize the * election once, typically at start-up. * * \param[in] cluster Cluster that election is for * \param[in] cb Function to call if local node wins election */ void election_init(pcmk_cluster_t *cluster, void (*cb)(pcmk_cluster_t *)) { const char *name = pcmk__s(crm_system_name, "election"); CRM_CHECK(cluster->priv->election == NULL, return); cluster->priv->election = pcmk__assert_alloc(1, sizeof(pcmk__election_t)); cluster->priv->election->cb = cb; cluster->priv->election->timeout = mainloop_timer_add(name, ELECTION_TIMEOUT_MS, FALSE, election_timer_cb, cluster); } /*! * \internal * \brief Disregard any previous vote by specified peer * * This discards any recorded vote from a specified peer. Election users should * call this whenever a voting peer becomes inactive. * * \param[in,out] cluster Cluster with election * \param[in] uname Name of peer to disregard */ void election_remove(pcmk_cluster_t *cluster, const char *uname) { if ((cluster != NULL) && (cluster->priv->election != NULL) && (uname != NULL) && (cluster->priv->election->voted != NULL)) { pcmk__trace("Discarding (no-)vote from lost peer %s", uname); g_hash_table_remove(cluster->priv->election->voted, uname); } } /*! * \internal * \brief Stop election timer and disregard all votes * * \param[in,out] cluster Cluster with election */ void election_reset(pcmk_cluster_t *cluster) { if ((cluster != NULL) && (cluster->priv->election != NULL)) { pcmk__trace("Resetting election"); mainloop_timer_stop(cluster->priv->election->timeout); if (cluster->priv->election->voted != NULL) { g_hash_table_destroy(cluster->priv->election->voted); cluster->priv->election->voted = NULL; } } } /*! * \internal * \brief Free an election object * * Free all memory associated with an election object, stopping its * election timer (if running). * * \param[in,out] cluster Cluster with election */ void election_fini(pcmk_cluster_t *cluster) { if ((cluster != NULL) && (cluster->priv->election != NULL)) { election_reset(cluster); pcmk__trace("Destroying election"); mainloop_timer_del(cluster->priv->election->timeout); free(cluster->priv->election); cluster->priv->election = NULL; } } static void election_timeout_start(pcmk_cluster_t *cluster) { mainloop_timer_start(cluster->priv->election->timeout); } /*! * \internal * \brief Stop an election's timer, if running * * \param[in,out] cluster Cluster with election */ void election_timeout_stop(pcmk_cluster_t *cluster) { if ((cluster != NULL) && (cluster->priv->election != NULL)) { mainloop_timer_stop(cluster->priv->election->timeout); } } /*! * \internal * \brief Change an election's timeout (restarting timer if running) * * \param[in,out] cluster Cluster with election * \param[in] period New timeout */ void election_timeout_set_period(pcmk_cluster_t *cluster, guint period) { CRM_CHECK((cluster != NULL) && (cluster->priv->election != NULL), return); mainloop_timer_set_period(cluster->priv->election->timeout, period); } static int get_uptime(struct timeval *output) { static time_t expires = 0; static struct rusage info; time_t tm_now = time(NULL); if (expires < tm_now) { int rc = 0; info.ru_utime.tv_sec = 0; info.ru_utime.tv_usec = 0; rc = getrusage(RUSAGE_SELF, &info); output->tv_sec = 0; output->tv_usec = 0; if (rc < 0) { crm_perror(LOG_ERR, "Could not calculate the current uptime"); expires = 0; return -1; } pcmk__debug("Current CPU usage is: %llds, %lldus", (long long) info.ru_utime.tv_sec, (long long) info.ru_utime.tv_usec); } expires = tm_now + STORM_INTERVAL; /* N seconds after the last _access_ */ output->tv_sec = info.ru_utime.tv_sec; output->tv_usec = info.ru_utime.tv_usec; return 1; } static int compare_age(struct timeval your_age) { struct timeval our_age; get_uptime(&our_age); /* If an error occurred, our_age will be compared as {0,0} */ if (our_age.tv_sec > your_age.tv_sec) { pcmk__debug("Win: %lld vs %lld (seconds)", (long long) our_age.tv_sec, (long long) your_age.tv_sec); return 1; } else if (our_age.tv_sec < your_age.tv_sec) { pcmk__debug("Lose: %lld vs %lld (seconds)", (long long) our_age.tv_sec, (long long) your_age.tv_sec); return -1; } else if (our_age.tv_usec > your_age.tv_usec) { pcmk__debug("Win: %lld.%06lld vs %lld.%06lld (usec)", (long long) our_age.tv_sec, (long long) our_age.tv_usec, (long long) your_age.tv_sec, (long long) your_age.tv_usec); return 1; } else if (our_age.tv_usec < your_age.tv_usec) { pcmk__debug("Lose: %lld.%06lld vs %lld.%06lld (usec)", (long long) our_age.tv_sec, (long long) our_age.tv_usec, (long long) your_age.tv_sec, (long long) your_age.tv_usec); return -1; } return 0; } /*! * \internal * \brief Start a new election by offering local node's candidacy * * Broadcast a "vote" election message containing the local node's ID, * (incremented) election counter, and uptime, and start the election timer. * * \param[in,out] cluster Cluster with election * * \note Any nodes agreeing to the candidacy will send a "no-vote" reply, and if * all active peers do so, or if the election times out, the local node * wins the election. (If we lose to any peer vote, we will stop the * timer, so a timeout means we did not lose -- either some peer did not * vote, or we did not call election_check() in time.) */ void election_vote(pcmk_cluster_t *cluster) { struct timeval age; xmlNode *vote = NULL; pcmk__node_status_t *our_node = NULL; const char *message_type = NULL; CRM_CHECK((cluster != NULL) && (cluster->priv->election != NULL), return); if (cluster->priv->node_name == NULL) { pcmk__err("Cannot start an election: Local node name unknown"); return; } our_node = pcmk__get_node(0, cluster->priv->node_name, NULL, pcmk__node_search_cluster_member); if (!pcmk__cluster_is_node_active(our_node)) { pcmk__trace("Cannot vote yet: local node not connected to cluster"); return; } election_reset(cluster); cluster->priv->election->state = election_in_progress; message_type = pcmk__server_message_type(cluster->priv->server); /* @COMPAT We use message_type as the sender and recipient system for * backward compatibility (see T566). */ vote = pcmk__new_request(cluster->priv->server, message_type, NULL, message_type, CRM_OP_VOTE, NULL); cluster->priv->election->count++; pcmk__xe_set(vote, PCMK__XA_ELECTION_OWNER, pcmk__cluster_get_xml_id(our_node)); pcmk__xe_set_int(vote, PCMK__XA_ELECTION_ID, cluster->priv->election->count); // Warning: PCMK__XA_ELECTION_AGE_NANO_SEC value is actually microseconds get_uptime(&age); pcmk__xe_set_timeval(vote, PCMK__XA_ELECTION_AGE_SEC, PCMK__XA_ELECTION_AGE_NANO_SEC, &age); pcmk__cluster_send_message(NULL, cluster->priv->server, vote); pcmk__xml_free(vote); pcmk__debug("Started election round %u", cluster->priv->election->count); election_timeout_start(cluster); return; } /*! * \internal * \brief Check whether local node has won an election * * If all known peers have sent no-vote messages, stop the election timer, set * the election state to won, and call any registered win callback. * * \param[in,out] cluster Cluster with election * * \return TRUE if local node has won, FALSE otherwise * \note If all known peers have sent no-vote messages, but the election owner * does not call this function, the election will not be won (and the * callback will not be called) until the election times out. * \note This should be called when election_count_vote() returns * \c election_in_progress. */ bool election_check(pcmk_cluster_t *cluster) { int voted_size = 0; int num_members = 0; CRM_CHECK((cluster != NULL) && (cluster->priv->election != NULL), return false); if (cluster->priv->election->voted == NULL) { pcmk__trace("Election check requested, but no votes received yet"); return FALSE; } voted_size = g_hash_table_size(cluster->priv->election->voted); num_members = pcmk__cluster_num_active_nodes(); /* in the case of #voted > #members, it is better to * wait for the timeout and give the cluster time to * stabilize */ if (voted_size >= num_members) { /* we won and everyone has voted */ election_timeout_stop(cluster); if (voted_size > num_members) { GHashTableIter gIter; const pcmk__node_status_t *node = NULL; char *key = NULL; pcmk__warn("Received too many votes in election"); g_hash_table_iter_init(&gIter, pcmk__peer_cache); while (g_hash_table_iter_next(&gIter, NULL, (gpointer *) & node)) { if (pcmk__cluster_is_node_active(node)) { pcmk__warn("* expected vote: %s", node->name); } } g_hash_table_iter_init(&gIter, cluster->priv->election->voted); while (g_hash_table_iter_next(&gIter, (gpointer *) & key, NULL)) { pcmk__warn("* actual vote: %s", key); } } pcmk__info("Election won by local node"); election_complete(cluster); return TRUE; } else { pcmk__debug("Election still waiting on %d of %d vote%s", (num_members - voted_size), num_members, pcmk__plural_s(num_members)); } return FALSE; } #define LOSS_DAMPEN 2 /* in seconds */ struct vote { const char *op; const char *from; const char *version; const char *election_owner; int election_id; struct timeval age; }; /*! * \internal * \brief Unpack an election message * * \param[in] message Election message XML * \param[out] vote Parsed fields from message * * \return TRUE if election message and election are valid, FALSE otherwise * \note The parsed struct's pointer members are valid only for the lifetime of * the message argument. */ static bool parse_election_message(const xmlNode *message, struct vote *vote) { CRM_CHECK(message && vote, return FALSE); vote->election_id = -1; vote->age.tv_sec = -1; vote->age.tv_usec = -1; vote->op = pcmk__xe_get(message, PCMK__XA_CRM_TASK); vote->from = pcmk__xe_get(message, PCMK__XA_SRC); vote->version = pcmk__xe_get(message, PCMK_XA_VERSION); vote->election_owner = pcmk__xe_get(message, PCMK__XA_ELECTION_OWNER); pcmk__xe_get_int(message, PCMK__XA_ELECTION_ID, &(vote->election_id)); if ((vote->op == NULL) || (vote->from == NULL) || (vote->version == NULL) || (vote->election_owner == NULL) || (vote->election_id < 0)) { pcmk__warn("Invalid %s message from %s", pcmk__s(vote->op, "election"), pcmk__s(vote->from, "unspecified node")); - crm_log_xml_trace(message, "bad-vote"); + pcmk__log_xml_trace(message, "bad-vote"); return FALSE; } // Op-specific validation if (pcmk__str_eq(vote->op, CRM_OP_VOTE, pcmk__str_none)) { /* Only vote ops have uptime. Warning: PCMK__XA_ELECTION_AGE_NANO_SEC value is in microseconds. */ if ((pcmk__xe_get_timeval(message, PCMK__XA_ELECTION_AGE_SEC, PCMK__XA_ELECTION_AGE_NANO_SEC, &(vote->age)) != pcmk_rc_ok) || (vote->age.tv_sec < 0) || (vote->age.tv_usec < 0)) { pcmk__warn("Cannot count election %s from %s because uptime is " "missing or invalid", vote->op, vote->from); return FALSE; } } else if (!pcmk__str_eq(vote->op, CRM_OP_NOVOTE, pcmk__str_none)) { pcmk__info("Cannot process election message from %s because %s is not " "a known election op", vote->from, vote->op); return FALSE; } /* If the membership cache is NULL, we REALLY shouldn't be voting -- * the question is how we managed to get here. */ if (pcmk__peer_cache == NULL) { pcmk__info("Cannot count election %s from %s becasue no peer " "information available", vote->op, vote->from); return FALSE; } return TRUE; } static void record_vote(pcmk_cluster_t *cluster, struct vote *vote) { pcmk__assert((vote->from != NULL) && (vote->op != NULL)); if (cluster->priv->election->voted == NULL) { cluster->priv->election->voted = pcmk__strkey_table(free, free); } pcmk__insert_dup(cluster->priv->election->voted, vote->from, vote->op); } static void send_no_vote(pcmk_cluster_t *cluster, pcmk__node_status_t *peer, struct vote *vote) { const char *message_type = NULL; xmlNode *novote = NULL; message_type = pcmk__server_message_type(cluster->priv->server); novote = pcmk__new_request(cluster->priv->server, message_type, vote->from, message_type, CRM_OP_NOVOTE, NULL); pcmk__xe_set(novote, PCMK__XA_ELECTION_OWNER, vote->election_owner); pcmk__xe_set_int(novote, PCMK__XA_ELECTION_ID, vote->election_id); pcmk__cluster_send_message(peer, cluster->priv->server, novote); pcmk__xml_free(novote); } /*! * \internal * \brief Process an election message (vote or no-vote) from a peer * * \param[in,out] cluster Cluster with election * \param[in] message Election message XML from peer * \param[in] can_win Whether local node is eligible to win * * \return Election state after new vote is considered * \note If the peer message is a vote, and we prefer the peer to win, this will * send a no-vote reply to the peer. * \note The situations "we lost to this vote" from "this is a late no-vote * after we've already lost" both return election_lost. If a caller needs * to distinguish them, it should save the current state before calling * this function, and then compare the result. */ enum election_result election_count_vote(pcmk_cluster_t *cluster, const xmlNode *message, bool can_win) { int log_level = LOG_INFO; gboolean done = FALSE; gboolean we_lose = FALSE; const char *reason = "unknown"; bool we_are_owner = FALSE; pcmk__node_status_t *our_node = NULL; pcmk__node_status_t *your_node = NULL; time_t tm_now = time(NULL); struct vote vote; CRM_CHECK((cluster != NULL) && (cluster->priv->election != NULL) && (message != NULL) && (cluster->priv->node_name != NULL), return election_error); if (!parse_election_message(message, &vote)) { return election_error; } your_node = pcmk__get_node(0, vote.from, NULL, pcmk__node_search_cluster_member); our_node = pcmk__get_node(0, cluster->priv->node_name, NULL, pcmk__node_search_cluster_member); we_are_owner = (our_node != NULL) && pcmk__str_eq(pcmk__cluster_get_xml_id(our_node), vote.election_owner, pcmk__str_none); if (!can_win) { reason = "Not eligible"; we_lose = TRUE; } else if (!pcmk__cluster_is_node_active(our_node)) { reason = "We are not part of the cluster"; log_level = LOG_ERR; we_lose = TRUE; } else if (we_are_owner && (vote.election_id != cluster->priv->election->count)) { log_level = PCMK__LOG_TRACE; reason = "Superseded"; done = TRUE; } else if (!pcmk__cluster_is_node_active(your_node)) { /* Possibly we cached the message in the FSA queue at a point that it wasn't */ reason = "Peer is not part of our cluster"; log_level = LOG_WARNING; done = TRUE; } else if (pcmk__str_eq(vote.op, CRM_OP_NOVOTE, pcmk__str_none) || pcmk__str_eq(vote.from, cluster->priv->node_name, pcmk__str_casei)) { /* Receiving our own broadcast vote, or a no-vote from peer, is a vote * for us to win */ if (!we_are_owner) { pcmk__warn("Cannot count election round %d %s from %s because we " "did not start election (node ID %s did)", vote.election_id, vote.op, vote.from, vote.election_owner); return election_error; } if (cluster->priv->election->state != election_in_progress) { // Should only happen if we already lost pcmk__debug("Not counting election round %d %s from %s because no " "election in progress", vote.election_id, vote.op, vote.from); return cluster->priv->election->state; } record_vote(cluster, &vote); reason = "Recorded"; done = TRUE; } else { // A peer vote requires a comparison to determine which node is better int age_result = compare_age(vote.age); int version_result = pcmk__compare_versions(vote.version, CRM_FEATURE_SET); if (version_result < 0) { reason = "Version"; we_lose = TRUE; } else if (version_result > 0) { reason = "Version"; } else if (age_result < 0) { reason = "Uptime"; we_lose = TRUE; } else if (age_result > 0) { reason = "Uptime"; } else if (strcasecmp(cluster->priv->node_name, vote.from) > 0) { reason = "Host name"; we_lose = TRUE; } else { reason = "Host name"; } } if (cluster->priv->election->expires < tm_now) { cluster->priv->election->election_wins = 0; cluster->priv->election->expires = tm_now + STORM_INTERVAL; } else if (done == FALSE && we_lose == FALSE) { int peers = 1 + g_hash_table_size(pcmk__peer_cache); /* If every node has to vote down every other node, thats N*(N-1) total elections * Allow some leeway before _really_ complaining */ cluster->priv->election->election_wins++; if (cluster->priv->election->election_wins > (peers * peers)) { pcmk__warn("Election storm detected: %d wins in %d seconds", cluster->priv->election->election_wins, STORM_INTERVAL); cluster->priv->election->election_wins = 0; cluster->priv->election->expires = tm_now + STORM_INTERVAL; if (!(cluster->priv->election->wrote_blackbox)) { /* It's questionable whether a black box (from every node in the * cluster) would be truly helpful in diagnosing an election * storm. It's also highly doubtful a production environment * would get multiple election storms from distinct causes, so * saving one blackbox per process lifetime should be * sufficient. Alternatives would be to save a timestamp of the * last blackbox write instead of a boolean, and write a new one * if some amount of time has passed; or to save a storm count, * write a blackbox on every Nth occurrence. */ crm_write_blackbox(0, NULL); cluster->priv->election->wrote_blackbox = true; } } } if (done) { do_crm_log(log_level + 1, "Processed election round %u %s (current round %d) " "from %s (%s)", vote.election_id, vote.op, cluster->priv->election->count, vote.from, reason); return cluster->priv->election->state; } else if (we_lose == FALSE) { /* We track the time of the last election loss to implement an election * dampening period, reducing the likelihood of an election storm. If * this node has lost within the dampening period, don't start a new * election, even if we win against a peer's vote -- the peer we lost to * should win again. * * @TODO This has a problem case: if an election winner immediately * leaves the cluster, and a new election is immediately called, all * nodes could lose, with no new winner elected. The ideal solution * would be to tie the election structure with the peer caches, which * would allow us to clear the dampening when the previous winner * leaves (and would allow other improvements as well). */ if ((cluster->priv->election->last_election_loss == 0) || ((tm_now - cluster->priv->election->last_election_loss) > (time_t) LOSS_DAMPEN)) { do_crm_log(log_level, "Election round %d (started by node ID %s) pass: " "%s from %s (%s)", vote.election_id, vote.election_owner, vote.op, vote.from, reason); cluster->priv->election->last_election_loss = 0; election_timeout_stop(cluster); /* Start a new election by voting down this, and other, peers */ cluster->priv->election->state = election_start; return cluster->priv->election->state; } else { char *loss_time = NULL; loss_time = ctime(&(cluster->priv->election->last_election_loss)); if (loss_time) { // Show only HH:MM:SS loss_time += 11; loss_time[8] = '\0'; } pcmk__info("Ignoring election round %d (started by node ID %s) " "pass vs %s because we lost less than %ds ago at %s", vote.election_id, vote.election_owner, vote.from, LOSS_DAMPEN, pcmk__s(loss_time, "unknown")); } } cluster->priv->election->last_election_loss = tm_now; do_crm_log(log_level, "Election round %d (started by node ID %s) lost: " "%s from %s (%s)", vote.election_id, vote.election_owner, vote.op, vote.from, reason); election_reset(cluster); send_no_vote(cluster, your_node, &vote); cluster->priv->election->state = election_lost; return cluster->priv->election->state; } /*! * \internal * \brief Reset any election dampening currently in effect * * \param[in,out] cluster Cluster with election */ void election_clear_dampening(pcmk_cluster_t *cluster) { if ((cluster != NULL) && (cluster->priv->election != NULL)) { cluster->priv->election->last_election_loss = 0; } } diff --git a/lib/common/ipc_client.c b/lib/common/ipc_client.c index bc4618b3ec..dcfe8f18af 100644 --- a/lib/common/ipc_client.c +++ b/lib/common/ipc_client.c @@ -1,1703 +1,1703 @@ /* * Copyright 2004-2025 the Pacemaker project contributors * * The version control history for this file may have further details. * * This source code is licensed under the GNU Lesser General Public License * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. */ #include #if defined(HAVE_UCRED) || defined(HAVE_SOCKPEERCRED) #include #elif defined(HAVE_GETPEERUCRED) #include #endif #include #include #include #include #include /* indirectly: pcmk_err_generic */ #include #include #include #include "crmcommon_private.h" static int is_ipc_provider_expected(qb_ipcc_connection_t *qb_ipc, int sock, uid_t refuid, gid_t refgid, pid_t *gotpid, uid_t *gotuid, gid_t *gotgid); /*! * \brief Create a new object for using Pacemaker daemon IPC * * \param[out] api Where to store new IPC object * \param[in] server Which Pacemaker daemon the object is for * * \return Standard Pacemaker result code * * \note The caller is responsible for freeing *api using pcmk_free_ipc_api(). * \note This is intended to supersede crm_ipc_new() but currently only * supports the controller, pacemakerd, and schedulerd IPC API. */ int pcmk_new_ipc_api(pcmk_ipc_api_t **api, enum pcmk_ipc_server server) { if (api == NULL) { return EINVAL; } *api = calloc(1, sizeof(pcmk_ipc_api_t)); if (*api == NULL) { return errno; } (*api)->server = server; if (pcmk_ipc_name(*api, false) == NULL) { pcmk_free_ipc_api(*api); *api = NULL; return EOPNOTSUPP; } (*api)->ipc_size_max = 0; // Set server methods and max_size (if not default) switch (server) { case pcmk_ipc_attrd: (*api)->cmds = pcmk__attrd_api_methods(); break; case pcmk_ipc_based: (*api)->ipc_size_max = 512 * 1024; // 512KB break; case pcmk_ipc_controld: (*api)->cmds = pcmk__controld_api_methods(); break; case pcmk_ipc_execd: break; case pcmk_ipc_fenced: break; case pcmk_ipc_pacemakerd: (*api)->cmds = pcmk__pacemakerd_api_methods(); break; case pcmk_ipc_schedulerd: (*api)->cmds = pcmk__schedulerd_api_methods(); // @TODO max_size could vary by client, maybe take as argument? (*api)->ipc_size_max = 5 * 1024 * 1024; // 5MB break; default: // pcmk_ipc_unknown pcmk_free_ipc_api(*api); *api = NULL; return EINVAL; } if ((*api)->cmds == NULL) { pcmk_free_ipc_api(*api); *api = NULL; return ENOMEM; } (*api)->ipc = crm_ipc_new(pcmk_ipc_name(*api, false), (*api)->ipc_size_max); if ((*api)->ipc == NULL) { pcmk_free_ipc_api(*api); *api = NULL; return ENOMEM; } // If daemon API has its own data to track, allocate it if ((*api)->cmds->new_data != NULL) { if ((*api)->cmds->new_data(*api) != pcmk_rc_ok) { pcmk_free_ipc_api(*api); *api = NULL; return ENOMEM; } } pcmk__trace("Created %s API IPC object", pcmk_ipc_name(*api, true)); return pcmk_rc_ok; } static void free_daemon_specific_data(pcmk_ipc_api_t *api) { if ((api != NULL) && (api->cmds != NULL)) { if ((api->cmds->free_data != NULL) && (api->api_data != NULL)) { api->cmds->free_data(api->api_data); api->api_data = NULL; } free(api->cmds); api->cmds = NULL; } } /*! * \internal * \brief Call an IPC API event callback, if one is registed * * \param[in,out] api IPC API connection * \param[in] event_type The type of event that occurred * \param[in] status Event status * \param[in,out] event_data Event-specific data */ void pcmk__call_ipc_callback(pcmk_ipc_api_t *api, enum pcmk_ipc_event event_type, crm_exit_t status, void *event_data) { if ((api != NULL) && (api->cb != NULL)) { api->cb(api, event_type, status, event_data, api->user_data); } } /*! * \internal * \brief Clean up after an IPC disconnect * * \param[in,out] user_data IPC API connection that disconnected * * \note This function can be used as a main loop IPC destroy callback. */ static void ipc_post_disconnect(gpointer user_data) { pcmk_ipc_api_t *api = user_data; pcmk__info("Disconnected from %s", pcmk_ipc_name(api, true)); // Perform any daemon-specific handling needed if ((api->cmds != NULL) && (api->cmds->post_disconnect != NULL)) { api->cmds->post_disconnect(api); } // Call client's registered event callback pcmk__call_ipc_callback(api, pcmk_ipc_event_disconnect, CRM_EX_DISCONNECT, NULL); /* If this is being called from a running main loop, mainloop_gio_destroy() * will free ipc and mainloop_io immediately after calling this function. * If this is called from a stopped main loop, these will leak, so the best * practice is to close the connection before stopping the main loop. */ api->ipc = NULL; api->mainloop_io = NULL; if (api->free_on_disconnect) { /* pcmk_free_ipc_api() has already been called, but did not free api * or api->cmds because this function needed them. Do that now. */ free_daemon_specific_data(api); pcmk__trace("Freeing IPC API object after disconnect"); free(api); } } /*! * \brief Free the contents of an IPC API object * * \param[in,out] api IPC API object to free */ void pcmk_free_ipc_api(pcmk_ipc_api_t *api) { bool free_on_disconnect = false; if (api == NULL) { return; } pcmk__debug("Releasing %s IPC API", pcmk_ipc_name(api, true)); if (api->ipc != NULL) { if (api->mainloop_io != NULL) { /* We need to keep the api pointer itself around, because it is the * user data for the IPC client destroy callback. That will be * triggered by the pcmk_disconnect_ipc() call below, but it might * happen later in the main loop (if still running). * * This flag tells the destroy callback to free the object. It can't * do that unconditionally, because the application might call this * function after a disconnect that happened by other means. */ free_on_disconnect = api->free_on_disconnect = true; } pcmk_disconnect_ipc(api); // Frees api if free_on_disconnect is true } if (!free_on_disconnect) { free_daemon_specific_data(api); pcmk__trace("Freeing IPC API object"); free(api); } } /*! * \brief Get the IPC name used with an IPC API connection * * \param[in] api IPC API connection * \param[in] for_log If true, return human-friendly name instead of IPC name * * \return IPC API's human-friendly or connection name, or if none is available, * "Pacemaker" if for_log is true and NULL if for_log is false */ const char * pcmk_ipc_name(const pcmk_ipc_api_t *api, bool for_log) { if (api == NULL) { return for_log? "Pacemaker" : NULL; } if (for_log) { const char *name = pcmk__server_log_name(api->server); return pcmk__s(name, "Pacemaker"); } switch (api->server) { // These servers do not have pcmk_ipc_api_t implementations yet case pcmk_ipc_based: case pcmk_ipc_execd: case pcmk_ipc_fenced: return NULL; default: return pcmk__server_ipc_name(api->server); } } /*! * \brief Check whether an IPC API connection is active * * \param[in,out] api IPC API connection * * \return true if IPC is connected, false otherwise */ bool pcmk_ipc_is_connected(pcmk_ipc_api_t *api) { return (api != NULL) && crm_ipc_connected(api->ipc); } /*! * \internal * \brief Call the daemon-specific API's dispatch function * * Perform daemon-specific handling of IPC reply dispatch. It is the daemon * method's responsibility to call the client's registered event callback, as * well as allocate and free any event data. * * \param[in,out] api IPC API connection * \param[in,out] message IPC reply XML to dispatch */ static bool call_api_dispatch(pcmk_ipc_api_t *api, xmlNode *message) { - crm_log_xml_trace(message, "ipc-received"); + pcmk__log_xml_trace(message, "ipc-received"); if ((api->cmds != NULL) && (api->cmds->dispatch != NULL)) { return api->cmds->dispatch(api, message); } return false; } /*! * \internal * \brief Dispatch previously read IPC data * * \param[in] buffer Data read from IPC * \param[in,out] api IPC object * * \return Standard Pacemaker return code. In particular: * * pcmk_rc_ok: There are no more messages expected from the server. Quit * reading. * EINPROGRESS: There are more messages expected from the server. Keep reading. * * All other values indicate an error. */ static int dispatch_ipc_data(const char *buffer, pcmk_ipc_api_t *api) { bool more = false; xmlNode *msg; if (buffer == NULL) { pcmk__warn("Empty message received from %s IPC", pcmk_ipc_name(api, true)); return ENOMSG; } msg = pcmk__xml_parse(buffer); if (msg == NULL) { pcmk__warn("Malformed message received from %s IPC", pcmk_ipc_name(api, true)); return EPROTO; } more = call_api_dispatch(api, msg); pcmk__xml_free(msg); if (more) { return EINPROGRESS; } else { return pcmk_rc_ok; } } /*! * \internal * \brief Dispatch data read from IPC source * * \param[in] buffer Data read from IPC * \param[in] length Number of bytes of data in buffer (ignored) * \param[in,out] user_data IPC object * * \return Always 0 (meaning connection is still required) * * \note This function can be used as a main loop IPC dispatch callback. */ static int dispatch_ipc_source_data(const char *buffer, ssize_t length, gpointer user_data) { pcmk_ipc_api_t *api = user_data; CRM_CHECK(api != NULL, return 0); dispatch_ipc_data(buffer, api); return 0; } /*! * \brief Check whether an IPC connection has data available (without main loop) * * \param[in] api IPC API connection * \param[in] timeout_ms If less than 0, poll indefinitely; if 0, poll once * and return immediately; otherwise, poll for up to * this many milliseconds * * \return Standard Pacemaker return code * * \note Callers of pcmk_connect_ipc() using pcmk_ipc_dispatch_poll should call * this function to check whether IPC data is available. Return values of * interest include pcmk_rc_ok meaning data is available, and EAGAIN * meaning no data is available; all other values indicate errors. * \todo This does not allow the caller to poll multiple file descriptors at * once. If there is demand for that, we could add a wrapper for * pcmk__ipc_fd(api->ipc), so the caller can call poll() themselves. */ int pcmk_poll_ipc(const pcmk_ipc_api_t *api, int timeout_ms) { int rc; struct pollfd pollfd = { 0, }; if ((api == NULL) || (api->dispatch_type != pcmk_ipc_dispatch_poll)) { return EINVAL; } rc = pcmk__ipc_fd(api->ipc, &(pollfd.fd)); if (rc != pcmk_rc_ok) { pcmk__debug("Could not obtain file descriptor for %s IPC: %s", pcmk_ipc_name(api, true), pcmk_rc_str(rc)); return rc; } pollfd.events = POLLIN; rc = poll(&pollfd, 1, timeout_ms); if (rc < 0) { /* Some UNIX systems return negative and set EAGAIN for failure to * allocate memory; standardize the return code in that case */ return (errno == EAGAIN)? ENOMEM : errno; } else if (rc == 0) { return EAGAIN; } return pcmk_rc_ok; } /*! * \brief Dispatch available messages on an IPC connection (without main loop) * * \param[in,out] api IPC API connection * * \return Standard Pacemaker return code * * \note Callers of pcmk_connect_ipc() using pcmk_ipc_dispatch_poll should call * this function when IPC data is available. */ void pcmk_dispatch_ipc(pcmk_ipc_api_t *api) { if (api == NULL) { return; } while (crm_ipc_ready(api->ipc) > 0) { if (crm_ipc_read(api->ipc) > 0) { dispatch_ipc_data(crm_ipc_buffer(api->ipc), api); } } } // \return Standard Pacemaker return code static int connect_with_main_loop(pcmk_ipc_api_t *api) { int rc; struct ipc_client_callbacks callbacks = { .dispatch = dispatch_ipc_source_data, .destroy = ipc_post_disconnect, }; rc = pcmk__add_mainloop_ipc(api->ipc, G_PRIORITY_DEFAULT, api, &callbacks, &(api->mainloop_io)); if (rc != pcmk_rc_ok) { return rc; } pcmk__debug("Connected to %s IPC (attached to main loop)", pcmk_ipc_name(api, true)); /* After this point, api->mainloop_io owns api->ipc, so api->ipc * should not be explicitly freed. */ return pcmk_rc_ok; } // \return Standard Pacemaker return code static int connect_without_main_loop(pcmk_ipc_api_t *api) { int rc = pcmk__connect_generic_ipc(api->ipc); if (rc != pcmk_rc_ok) { crm_ipc_close(api->ipc); } else { pcmk__debug("Connected to %s IPC (without main loop)", pcmk_ipc_name(api, true)); } return rc; } /*! * \internal * \brief Connect to a Pacemaker daemon via IPC (retrying after soft errors) * * \param[in,out] api IPC API instance * \param[in] dispatch_type How IPC replies should be dispatched * \param[in] attempts How many times to try (in case of soft error) * * \return Standard Pacemaker return code */ int pcmk__connect_ipc(pcmk_ipc_api_t *api, enum pcmk_ipc_dispatch dispatch_type, int attempts) { int rc = pcmk_rc_ok; if ((api == NULL) || (attempts < 1)) { return EINVAL; } if (api->ipc == NULL) { api->ipc = crm_ipc_new(pcmk_ipc_name(api, false), api->ipc_size_max); if (api->ipc == NULL) { return ENOMEM; } } if (crm_ipc_connected(api->ipc)) { pcmk__trace("Already connected to %s", pcmk_ipc_name(api, true)); return pcmk_rc_ok; } api->dispatch_type = dispatch_type; pcmk__debug("Attempting connection to %s (up to %d time%s)", pcmk_ipc_name(api, true), attempts, pcmk__plural_s(attempts)); for (int remaining = attempts - 1; remaining >= 0; --remaining) { switch (dispatch_type) { case pcmk_ipc_dispatch_main: rc = connect_with_main_loop(api); break; case pcmk_ipc_dispatch_sync: case pcmk_ipc_dispatch_poll: rc = connect_without_main_loop(api); break; } if ((remaining == 0) || ((rc != EAGAIN) && (rc != EALREADY))) { break; // Result is final } // Retry after soft error (interrupted by signal, etc.) pcmk__sleep_ms((attempts - remaining) * 500); pcmk__debug("Re-attempting connection to %s (%d attempt%s remaining)", pcmk_ipc_name(api, true), remaining, pcmk__plural_s(remaining)); } if (rc != pcmk_rc_ok) { return rc; } if ((api->cmds != NULL) && (api->cmds->post_connect != NULL)) { rc = api->cmds->post_connect(api); if (rc != pcmk_rc_ok) { crm_ipc_close(api->ipc); } } return rc; } /*! * \brief Connect to a Pacemaker daemon via IPC * * \param[in,out] api IPC API instance * \param[in] dispatch_type How IPC replies should be dispatched * * \return Standard Pacemaker return code */ int pcmk_connect_ipc(pcmk_ipc_api_t *api, enum pcmk_ipc_dispatch dispatch_type) { int rc = pcmk__connect_ipc(api, dispatch_type, 2); if (rc != pcmk_rc_ok) { pcmk__err("Connection to %s failed: %s", pcmk_ipc_name(api, true), pcmk_rc_str(rc)); } return rc; } /*! * \brief Disconnect an IPC API instance * * \param[in,out] api IPC API connection * * \return Standard Pacemaker return code * * \note If the connection is attached to a main loop, this function should be * called before quitting the main loop, to ensure that all memory is * freed. */ void pcmk_disconnect_ipc(pcmk_ipc_api_t *api) { if ((api == NULL) || (api->ipc == NULL)) { return; } switch (api->dispatch_type) { case pcmk_ipc_dispatch_main: { mainloop_io_t *mainloop_io = api->mainloop_io; // Make sure no code with access to api can use these again api->mainloop_io = NULL; api->ipc = NULL; mainloop_del_ipc_client(mainloop_io); // After this point api might have already been freed } break; case pcmk_ipc_dispatch_poll: case pcmk_ipc_dispatch_sync: { crm_ipc_t *ipc = api->ipc; // Make sure no code with access to api can use ipc again api->ipc = NULL; // This should always be the case already, but to be safe api->free_on_disconnect = false; crm_ipc_close(ipc); crm_ipc_destroy(ipc); ipc_post_disconnect(api); } break; } } /*! * \brief Register a callback for IPC API events * * \param[in,out] api IPC API connection * \param[in] callback Callback to register * \param[in] userdata Caller data to pass to callback * * \note This function may be called multiple times to update the callback * and/or user data. The caller remains responsible for freeing * userdata in any case (after the IPC is disconnected, if the * user data is still registered with the IPC). */ void pcmk_register_ipc_callback(pcmk_ipc_api_t *api, pcmk_ipc_callback_t cb, void *user_data) { if (api == NULL) { return; } api->cb = cb; api->user_data = user_data; } /*! * \internal * \brief Send an XML request across an IPC API connection * * \param[in,out] api IPC API connection * \param[in] request XML request to send * * \return Standard Pacemaker return code * * \note Daemon-specific IPC API functions should call this function to send * requests, because it handles different dispatch types appropriately. */ int pcmk__send_ipc_request(pcmk_ipc_api_t *api, const xmlNode *request) { int rc; xmlNode *reply = NULL; enum crm_ipc_flags flags = crm_ipc_flags_none; if ((api == NULL) || (api->ipc == NULL) || (request == NULL)) { return EINVAL; } - crm_log_xml_trace(request, "ipc-sent"); + pcmk__log_xml_trace(request, "ipc-sent"); // Synchronous dispatch requires waiting for a reply if ((api->dispatch_type == pcmk_ipc_dispatch_sync) && (api->cmds != NULL) && (api->cmds->reply_expected != NULL) && (api->cmds->reply_expected(api, request))) { flags = crm_ipc_client_response; } /* The 0 here means a default timeout of 5 seconds * * @TODO Maybe add a timeout_ms member to pcmk_ipc_api_t and a * pcmk_set_ipc_timeout() setter for it, then use it here. */ rc = crm_ipc_send(api->ipc, request, flags, 0, &reply); if (rc < 0) { return pcmk_legacy2rc(rc); } else if (rc == 0) { return ENODATA; } // With synchronous dispatch, we dispatch any reply now if (reply != NULL) { bool more = call_api_dispatch(api, reply); pcmk__xml_free(reply); while (more) { rc = crm_ipc_read(api->ipc); if (rc == -EAGAIN) { continue; } else if (rc == -ENOMSG || rc == pcmk_ok) { return pcmk_rc_ok; } else if (rc < 0) { return -rc; } rc = dispatch_ipc_data(crm_ipc_buffer(api->ipc), api); if (rc == pcmk_rc_ok) { more = false; } else if (rc == EINPROGRESS) { more = true; } else { continue; } } } return pcmk_rc_ok; } /*! * \internal * \brief Create the XML for an IPC request to purge a node from the peer cache * * \param[in] api IPC API connection * \param[in] node_name If not NULL, name of node to purge * \param[in] nodeid If not 0, node ID of node to purge * * \return Newly allocated IPC request XML * * \note The controller, fencer, and pacemakerd use the same request syntax, but * the attribute manager uses a different one. The CIB manager doesn't * have any syntax for it. The executor and scheduler don't connect to the * cluster layer and thus don't have or need any syntax for it. * * \todo Modify the attribute manager to accept the common syntax (as well * as its current one, for compatibility with older clients). Modify * the CIB manager to accept and honor the common syntax. Modify the * executor and scheduler to accept the syntax (immediately returning * success), just for consistency. Modify this function to use the * common syntax with all daemons if their version supports it. */ static xmlNode * create_purge_node_request(const pcmk_ipc_api_t *api, const char *node_name, uint32_t nodeid) { xmlNode *request = NULL; const char *client = crm_system_name? crm_system_name : "client"; switch (api->server) { case pcmk_ipc_attrd: request = pcmk__xe_create(NULL, __func__); pcmk__xe_set(request, PCMK__XA_T, PCMK__VALUE_ATTRD); pcmk__xe_set(request, PCMK__XA_SRC, crm_system_name); pcmk__xe_set(request, PCMK_XA_TASK, PCMK__ATTRD_CMD_PEER_REMOVE); pcmk__xe_set_bool_attr(request, PCMK__XA_REAP, true); pcmk__xe_set(request, PCMK__XA_ATTR_HOST, node_name); if (nodeid > 0) { pcmk__xe_set_int(request, PCMK__XA_ATTR_HOST_ID, nodeid); } break; case pcmk_ipc_controld: case pcmk_ipc_fenced: case pcmk_ipc_pacemakerd: request = pcmk__new_request(api->server, client, NULL, pcmk_ipc_name(api, false), CRM_OP_RM_NODE_CACHE, NULL); if (nodeid > 0) { pcmk__xe_set_ll(request, PCMK_XA_ID, (long long) nodeid); } pcmk__xe_set(request, PCMK_XA_UNAME, node_name); break; case pcmk_ipc_based: case pcmk_ipc_execd: case pcmk_ipc_schedulerd: break; default: // pcmk_ipc_unknown (shouldn't be possible) return NULL; } return request; } /*! * \brief Ask a Pacemaker daemon to purge a node from its peer cache * * \param[in,out] api IPC API connection * \param[in] node_name If not NULL, name of node to purge * \param[in] nodeid If not 0, node ID of node to purge * * \return Standard Pacemaker return code * * \note At least one of node_name or nodeid must be specified. */ int pcmk_ipc_purge_node(pcmk_ipc_api_t *api, const char *node_name, uint32_t nodeid) { int rc = 0; xmlNode *request = NULL; if (api == NULL) { return EINVAL; } if ((node_name == NULL) && (nodeid == 0)) { return EINVAL; } request = create_purge_node_request(api, node_name, nodeid); if (request == NULL) { return EOPNOTSUPP; } rc = pcmk__send_ipc_request(api, request); pcmk__xml_free(request); pcmk__debug("%s peer cache purge of node %s[%" PRIu32 "]: rc=%d", pcmk_ipc_name(api, true), pcmk__s(node_name, "(unnamed)"), nodeid, rc); return rc; } /* * Generic IPC API (to eventually be deprecated as public API and made internal) */ struct crm_ipc_s { struct pollfd pfd; unsigned int max_buf_size; // maximum bytes we can send or receive over IPC unsigned int buf_size; // size of allocated buffer int msg_size; int need_reply; char *buffer; char *server_name; // server IPC name being connected to qb_ipcc_connection_t *ipc; }; /*! * \brief Create a new (legacy) object for using Pacemaker daemon IPC * * \param[in] name IPC system name to connect to * \param[in] max_size Use a maximum IPC buffer size of at least this size * * \return Newly allocated IPC object on success, NULL otherwise * * \note The caller is responsible for freeing the result using * crm_ipc_destroy(). * \note This should be considered deprecated for use with daemons supported by * pcmk_new_ipc_api(). */ crm_ipc_t * crm_ipc_new(const char *name, size_t max_size) { crm_ipc_t *client = NULL; client = calloc(1, sizeof(crm_ipc_t)); if (client == NULL) { pcmk__err("Could not create IPC connection: %s", strerror(errno)); return NULL; } client->server_name = strdup(name); if (client->server_name == NULL) { pcmk__err("Could not create %s IPC connection: %s", name, strerror(errno)); free(client); return NULL; } client->buf_size = pcmk__ipc_buffer_size(max_size); client->buffer = malloc(client->buf_size); if (client->buffer == NULL) { pcmk__err("Could not create %s IPC connection: %s", name, strerror(errno)); free(client->server_name); free(client); return NULL; } /* Clients initiating connection pick the max buf size */ client->max_buf_size = client->buf_size; client->pfd.fd = -1; client->pfd.events = POLLIN; client->pfd.revents = 0; return client; } /*! * \internal * \brief Connect a generic (not daemon-specific) IPC object * * \param[in,out] ipc Generic IPC object to connect * * \return Standard Pacemaker return code */ int pcmk__connect_generic_ipc(crm_ipc_t *ipc) { uid_t cl_uid = 0; gid_t cl_gid = 0; pid_t found_pid = 0; uid_t found_uid = 0; gid_t found_gid = 0; int rc = pcmk_rc_ok; if (ipc == NULL) { return EINVAL; } ipc->need_reply = FALSE; ipc->ipc = qb_ipcc_connect(ipc->server_name, ipc->buf_size); if (ipc->ipc == NULL) { return errno; } rc = qb_ipcc_fd_get(ipc->ipc, &ipc->pfd.fd); if (rc < 0) { // -errno crm_ipc_close(ipc); return -rc; } rc = pcmk__daemon_user(&cl_uid, &cl_gid); if (rc != pcmk_rc_ok) { crm_ipc_close(ipc); return rc; } rc = is_ipc_provider_expected(ipc->ipc, ipc->pfd.fd, cl_uid, cl_gid, &found_pid, &found_uid, &found_gid); if (rc != pcmk_rc_ok) { if (rc == pcmk_rc_ipc_unauthorized) { pcmk__info("%s IPC provider authentication failed: process %lld " "has uid %lld (expected %lld) and gid %lld (expected " "%lld)", ipc->server_name, (long long) PCMK__SPECIAL_PID_AS_0(found_pid), (long long) found_uid, (long long) cl_uid, (long long) found_gid, (long long) cl_gid); } crm_ipc_close(ipc); return rc; } ipc->max_buf_size = qb_ipcc_get_buffer_size(ipc->ipc); if (ipc->max_buf_size > ipc->buf_size) { free(ipc->buffer); ipc->buffer = calloc(ipc->max_buf_size, sizeof(char)); if (ipc->buffer == NULL) { rc = errno; crm_ipc_close(ipc); return rc; } ipc->buf_size = ipc->max_buf_size; } return pcmk_rc_ok; } void crm_ipc_close(crm_ipc_t * client) { if (client) { if (client->ipc) { qb_ipcc_connection_t *ipc = client->ipc; client->ipc = NULL; qb_ipcc_disconnect(ipc); } } } void crm_ipc_destroy(crm_ipc_t * client) { if (client) { if (client->ipc && qb_ipcc_is_connected(client->ipc)) { pcmk__notice("Destroying active %s IPC connection", client->server_name); /* The next line is basically unsafe * * If this connection was attached to mainloop and mainloop is active, * the 'disconnected' callback will end up back here and we'll end * up free'ing the memory twice - something that can still happen * even without this if we destroy a connection and it closes before * we call exit */ /* crm_ipc_close(client); */ } else { pcmk__trace("Destroying inactive %s IPC connection", client->server_name); } free(client->buffer); free(client->server_name); free(client); } } /*! * \internal * \brief Get the file descriptor for a generic IPC object * * \param[in,out] ipc Generic IPC object to get file descriptor for * \param[out] fd Where to store file descriptor * * \return Standard Pacemaker return code */ int pcmk__ipc_fd(crm_ipc_t *ipc, int *fd) { if ((ipc == NULL) || (fd == NULL)) { return EINVAL; } if ((ipc->ipc == NULL) || (ipc->pfd.fd < 0)) { return ENOTCONN; } *fd = ipc->pfd.fd; return pcmk_rc_ok; } int crm_ipc_get_fd(crm_ipc_t * client) { int fd = -1; if (pcmk__ipc_fd(client, &fd) != pcmk_rc_ok) { pcmk__err("Could not obtain file descriptor for %s IPC", ((client == NULL)? "unspecified" : client->server_name)); errno = EINVAL; return -EINVAL; } return fd; } bool crm_ipc_connected(crm_ipc_t * client) { bool rc = FALSE; if (client == NULL) { pcmk__trace("No client"); return FALSE; } else if (client->ipc == NULL) { pcmk__trace("No connection"); return FALSE; } else if (client->pfd.fd < 0) { pcmk__trace("Bad descriptor"); return FALSE; } rc = qb_ipcc_is_connected(client->ipc); if (rc == FALSE) { client->pfd.fd = -EINVAL; } return rc; } /*! * \brief Check whether an IPC connection is ready to be read * * \param[in,out] client Connection to check * * \return Positive value if ready to be read, 0 if not ready, -errno on error */ int crm_ipc_ready(crm_ipc_t *client) { int rc; pcmk__assert(client != NULL); if (!crm_ipc_connected(client)) { return -ENOTCONN; } client->pfd.revents = 0; rc = poll(&(client->pfd), 1, 0); return (rc < 0)? -errno : rc; } // \return Standard Pacemaker return code static int crm_ipc_decompress(crm_ipc_t * client) { pcmk__ipc_header_t *header = (pcmk__ipc_header_t *)(void*)client->buffer; if (header->size_compressed) { int rc = 0; unsigned int size_u = 1 + header->size_uncompressed; /* never let buf size fall below our max size required for ipc reads. */ unsigned int new_buf_size = QB_MAX((sizeof(pcmk__ipc_header_t) + size_u), client->max_buf_size); char *uncompressed = pcmk__assert_alloc(1, new_buf_size); pcmk__trace("Decompressing message data %u bytes into %u bytes", header->size_compressed, size_u); rc = BZ2_bzBuffToBuffDecompress(uncompressed + sizeof(pcmk__ipc_header_t), &size_u, client->buffer + sizeof(pcmk__ipc_header_t), header->size_compressed, 1, 0); rc = pcmk__bzlib2rc(rc); if (rc != pcmk_rc_ok) { pcmk__err("Decompression failed: %s " QB_XS " rc=%d", pcmk_rc_str(rc), rc); free(uncompressed); return rc; } pcmk__assert(size_u == header->size_uncompressed); memcpy(uncompressed, client->buffer, sizeof(pcmk__ipc_header_t)); /* Preserve the header */ header = (pcmk__ipc_header_t *)(void*)uncompressed; free(client->buffer); client->buf_size = new_buf_size; client->buffer = uncompressed; } pcmk__assert(client->buffer[sizeof(pcmk__ipc_header_t) + header->size_uncompressed - 1] == 0); return pcmk_rc_ok; } long crm_ipc_read(crm_ipc_t * client) { pcmk__ipc_header_t *header = NULL; pcmk__assert((client != NULL) && (client->ipc != NULL) && (client->buffer != NULL)); client->buffer[0] = 0; client->msg_size = qb_ipcc_event_recv(client->ipc, client->buffer, client->buf_size, 0); if (client->msg_size >= 0) { int rc = crm_ipc_decompress(client); if (rc != pcmk_rc_ok) { return pcmk_rc2legacy(rc); } header = (pcmk__ipc_header_t *)(void*)client->buffer; if (!pcmk__valid_ipc_header(header)) { return -EBADMSG; } pcmk__trace("Received %s IPC event %d size=%u rc=%d text='%.100s'", client->server_name, header->qb.id, header->qb.size, client->msg_size, (client->buffer + sizeof(pcmk__ipc_header_t))); } else { pcmk__trace("No message received from %s IPC: %s", client->server_name, pcmk_strerror(client->msg_size)); if (client->msg_size == -EAGAIN) { return -EAGAIN; } } if (!crm_ipc_connected(client) || client->msg_size == -ENOTCONN) { pcmk__err("Connection to %s IPC failed", client->server_name); } if (header) { /* Data excluding the header */ return header->size_uncompressed; } return -ENOMSG; } const char * crm_ipc_buffer(crm_ipc_t * client) { pcmk__assert(client != NULL); return client->buffer + sizeof(pcmk__ipc_header_t); } uint32_t crm_ipc_buffer_flags(crm_ipc_t * client) { pcmk__ipc_header_t *header = NULL; pcmk__assert(client != NULL); if (client->buffer == NULL) { return 0; } header = (pcmk__ipc_header_t *)(void*)client->buffer; return header->flags; } const char * crm_ipc_name(crm_ipc_t * client) { pcmk__assert(client != NULL); return client->server_name; } // \return Standard Pacemaker return code static int internal_ipc_get_reply(crm_ipc_t *client, int request_id, int ms_timeout, ssize_t *bytes, xmlNode **reply) { pcmk__ipc_header_t *hdr = NULL; time_t timeout = 0; int32_t qb_timeout = -1; int rc = pcmk_rc_ok; if (ms_timeout > 0) { timeout = time(NULL) + 1 + pcmk__timeout_ms2s(ms_timeout); qb_timeout = 1000; } /* get the reply */ pcmk__trace("Expecting reply to %s IPC message %d", client->server_name, request_id); do { xmlNode *xml = NULL; *bytes = qb_ipcc_recv(client->ipc, client->buffer, client->buf_size, qb_timeout); if (*bytes <= 0) { if (!crm_ipc_connected(client)) { pcmk__err("%s IPC provider disconnected while waiting for " "message %d", client->server_name, request_id); break; } continue; } rc = crm_ipc_decompress(client); if (rc != pcmk_rc_ok) { return rc; } hdr = (pcmk__ipc_header_t *)(void*) client->buffer; if (hdr->qb.id == request_id) { /* Got the reply we were expecting. */ break; } xml = pcmk__xml_parse(crm_ipc_buffer(client)); if (hdr->qb.id < request_id) { pcmk__err("Discarding old reply %d (need %d)", hdr->qb.id, request_id); pcmk__log_xml_notice(xml, "OldIpcReply"); } else if (hdr->qb.id > request_id) { pcmk__err("Discarding newer reply %d (need %d)", hdr->qb.id, request_id); pcmk__log_xml_notice(xml, "ImpossibleReply"); pcmk__assert(hdr->qb.id <= request_id); } } while (time(NULL) < timeout || (timeout == 0 && *bytes == -EAGAIN)); if (*bytes > 0) { pcmk__trace("Received %zd-byte reply %" PRId32 " to %s IPC %d: %.100s", *bytes, hdr->qb.id, client->server_name, request_id, crm_ipc_buffer(client)); if (reply != NULL) { *reply = pcmk__xml_parse(crm_ipc_buffer(client)); } } else if (*bytes < 0) { rc = (int) -*bytes; // System errno pcmk__trace("No reply to %s IPC %d: %s " QB_XS " rc=%d", client->server_name, request_id, pcmk_rc_str(rc), rc); } /* If bytes == 0, we'll return that to crm_ipc_send which will interpret * that as pcmk_rc_ok, log that the IPC request failed (since we did not * give it a valid reply), and return that 0 to its callers. It's up to * the callers to take appropriate action after that. */ return rc; } /*! * \brief Send an IPC XML message * * \param[in,out] client Connection to IPC server * \param[in] message XML message to send * \param[in] flags Bitmask of crm_ipc_flags * \param[in] ms_timeout Give up if not sent within this much time * (5 seconds if 0, or no timeout if negative) * \param[out] reply Reply from server (or NULL if none) * * \return Negative errno on error, otherwise size of reply received in bytes * if reply was needed, otherwise number of bytes sent */ int crm_ipc_send(crm_ipc_t *client, const xmlNode *message, enum crm_ipc_flags flags, int32_t ms_timeout, xmlNode **reply) { int rc = 0; time_t timeout = 0; ssize_t qb_rc = 0; ssize_t bytes = 0; struct iovec *iov; static uint32_t id = 0; static int factor = 8; pcmk__ipc_header_t *header; if (client == NULL) { pcmk__notice("Can't send IPC request without connection (bug?): %.100s", message); return -ENOTCONN; } else if (!crm_ipc_connected(client)) { /* Don't even bother */ pcmk__notice("Can't send %s IPC requests: Connection closed", client->server_name); return -ENOTCONN; } if (ms_timeout == 0) { ms_timeout = 5000; } if (client->need_reply) { qb_rc = qb_ipcc_recv(client->ipc, client->buffer, client->buf_size, ms_timeout); if (qb_rc < 0) { pcmk__warn("Sending %s IPC disabled until pending reply received", client->server_name); return -EALREADY; } else { pcmk__notice("Sending %s IPC re-enabled after pending reply " "received", client->server_name); client->need_reply = FALSE; } } id++; CRM_LOG_ASSERT(id != 0); /* Crude wrap-around detection */ rc = pcmk__ipc_prepare_iov(id, message, client->max_buf_size, &iov, &bytes); if (rc != pcmk_rc_ok) { pcmk__warn("Couldn't prepare %s IPC request: %s " QB_XS " rc=%d", client->server_name, pcmk_rc_str(rc), rc); return pcmk_rc2legacy(rc); } header = iov[0].iov_base; pcmk__set_ipc_flags(header->flags, client->server_name, flags); if (pcmk__is_set(flags, crm_ipc_proxied)) { /* Don't look for a synchronous response */ pcmk__clear_ipc_flags(flags, "client", crm_ipc_client_response); } if(header->size_compressed) { if(factor < 10 && (client->max_buf_size / 10) < (bytes / factor)) { pcmk__notice("Compressed message exceeds %d0%% of configured IPC " "limit (%u bytes); consider setting PCMK_ipc_buffer " "to u or higher", factor, client->max_buf_size, (2 * client->max_buf_size)); factor++; } } pcmk__trace("Sending %s IPC request %d of %u bytes using %dms timeout", client->server_name, header->qb.id, header->qb.size, ms_timeout); /* Send the IPC request, respecting any timeout we were passed */ if (ms_timeout > 0) { timeout = time(NULL) + 1 + pcmk__timeout_ms2s(ms_timeout); } do { qb_rc = qb_ipcc_sendv(client->ipc, iov, 2); } while ((qb_rc == -EAGAIN) && ((timeout == 0) || (time(NULL) < timeout))); rc = (int) qb_rc; // Negative of system errno, or bytes sent if (qb_rc <= 0) { goto send_cleanup; } /* If we should not wait for a response, bail now */ if (!pcmk__is_set(flags, crm_ipc_client_response)) { pcmk__trace("Not waiting for reply to %s IPC request %d", client->server_name, header->qb.id); goto send_cleanup; } rc = internal_ipc_get_reply(client, header->qb.id, ms_timeout, &bytes, reply); if (rc == pcmk_rc_ok) { rc = (int) bytes; // Size of reply received } else { /* rc is either a positive system errno or a negative standard Pacemaker * return code. If it's an errno, we need to convert it back to a * negative number for comparison and return at the end of this function. */ rc = pcmk_rc2legacy(rc); if (ms_timeout > 0) { /* We didn't get the reply in time, so disable future sends for now. * The only alternative would be to close the connection since we * don't know how to detect and discard out-of-sequence replies. * * @TODO Implement out-of-sequence detection */ client->need_reply = TRUE; } } send_cleanup: if (!crm_ipc_connected(client)) { pcmk__notice("Couldn't send %s IPC request %d: Connection closed " QB_XS " rc=%d", client->server_name, header->qb.id, rc); } else if (rc == -ETIMEDOUT) { pcmk__warn("%s IPC request %d failed: %s after %dms " QB_XS " rc=%d", client->server_name, header->qb.id, pcmk_strerror(rc), ms_timeout, rc); crm_write_blackbox(0, NULL); } else if (rc <= 0) { pcmk__warn("%s IPC request %d failed: %s " QB_XS " rc=%d", client->server_name, header->qb.id, ((rc == 0)? "No bytes sent" : pcmk_strerror(rc)), rc); } pcmk_free_ipc_event(iov); return rc; } /*! * \brief Ensure an IPC provider has expected user or group * * \param[in] qb_ipc libqb client connection if available * \param[in] sock Connected Unix socket for IPC * \param[in] refuid Expected user ID * \param[in] refgid Expected group ID * \param[out] gotpid If not NULL, where to store provider's actual process ID * (or 1 on platforms where ID is not available) * \param[out] gotuid If not NULL, where to store provider's actual user ID * \param[out] gotgid If not NULL, where to store provider's actual group ID * * \return Standard Pacemaker return code * \note An actual user ID of 0 (root) will always be considered authorized, * regardless of the expected values provided. The caller can use the * output arguments to be stricter than this function. */ static int is_ipc_provider_expected(qb_ipcc_connection_t *qb_ipc, int sock, uid_t refuid, gid_t refgid, pid_t *gotpid, uid_t *gotuid, gid_t *gotgid) { int rc = EOPNOTSUPP; pid_t found_pid = 0; uid_t found_uid = 0; gid_t found_gid = 0; #ifdef HAVE_QB_IPCC_AUTH_GET if (qb_ipc != NULL) { rc = qb_ipcc_auth_get(qb_ipc, &found_pid, &found_uid, &found_gid); rc = -rc; // libqb returns 0 or -errno if (rc == pcmk_rc_ok) { goto found; } } #endif #ifdef HAVE_UCRED { struct ucred ucred; socklen_t ucred_len = sizeof(ucred); if (getsockopt(sock, SOL_SOCKET, SO_PEERCRED, &ucred, &ucred_len) < 0) { rc = errno; } else if (ucred_len != sizeof(ucred)) { rc = EOPNOTSUPP; } else { found_pid = ucred.pid; found_uid = ucred.uid; found_gid = ucred.gid; goto found; } } #endif #ifdef HAVE_SOCKPEERCRED { struct sockpeercred sockpeercred; socklen_t sockpeercred_len = sizeof(sockpeercred); if (getsockopt(sock, SOL_SOCKET, SO_PEERCRED, &sockpeercred, &sockpeercred_len) < 0) { rc = errno; } else if (sockpeercred_len != sizeof(sockpeercred)) { rc = EOPNOTSUPP; } else { found_pid = sockpeercred.pid; found_uid = sockpeercred.uid; found_gid = sockpeercred.gid; goto found; } } #endif #ifdef HAVE_GETPEEREID // For example, FreeBSD if (getpeereid(sock, &found_uid, &found_gid) < 0) { rc = errno; } else { found_pid = PCMK__SPECIAL_PID; goto found; } #endif #ifdef HAVE_GETPEERUCRED { ucred_t *ucred = NULL; if (getpeerucred(sock, &ucred) < 0) { rc = errno; } else { found_pid = ucred_getpid(ucred); found_uid = ucred_geteuid(ucred); found_gid = ucred_getegid(ucred); ucred_free(ucred); goto found; } } #endif return rc; // If we get here, nothing succeeded found: if (gotpid != NULL) { *gotpid = found_pid; } if (gotuid != NULL) { *gotuid = found_uid; } if (gotgid != NULL) { *gotgid = found_gid; } if ((found_uid != 0) && (found_uid != refuid) && (found_gid != refgid)) { return pcmk_rc_ipc_unauthorized; } return pcmk_rc_ok; } int crm_ipc_is_authentic_process(int sock, uid_t refuid, gid_t refgid, pid_t *gotpid, uid_t *gotuid, gid_t *gotgid) { int ret = is_ipc_provider_expected(NULL, sock, refuid, refgid, gotpid, gotuid, gotgid); /* The old function had some very odd return codes*/ if (ret == 0) { return 1; } else if (ret == pcmk_rc_ipc_unauthorized) { return 0; } else { return pcmk_rc2legacy(ret); } } int pcmk__ipc_is_authentic_process_active(const char *name, uid_t refuid, gid_t refgid, pid_t *gotpid) { static char last_asked_name[PATH_MAX / 2] = ""; /* log spam prevention */ int fd; int rc = pcmk_rc_ipc_unresponsive; int auth_rc = 0; int32_t qb_rc; pid_t found_pid = 0; uid_t found_uid = 0; gid_t found_gid = 0; qb_ipcc_connection_t *c; #ifdef HAVE_QB_IPCC_CONNECT_ASYNC struct pollfd pollfd = { 0, }; int poll_rc; c = qb_ipcc_connect_async(name, 0, &(pollfd.fd)); #else c = qb_ipcc_connect(name, 0); #endif if (c == NULL) { pcmk__info("Could not connect to %s IPC: %s", name, strerror(errno)); rc = pcmk_rc_ipc_unresponsive; goto bail; } #ifdef HAVE_QB_IPCC_CONNECT_ASYNC pollfd.events = POLLIN; do { poll_rc = poll(&pollfd, 1, 2000); } while ((poll_rc == -1) && (errno == EINTR)); /* If poll() failed, given that disconnect function is not registered yet, * qb_ipcc_disconnect() won't clean up the socket. In any case, call * qb_ipcc_connect_continue() here so that it may fail and do the cleanup * for us. */ if (qb_ipcc_connect_continue(c) != 0) { pcmk__info("Could not connect to %s IPC: %s", name, ((poll_rc == 0)? "timeout" :strerror(errno))); rc = pcmk_rc_ipc_unresponsive; c = NULL; // qb_ipcc_connect_continue cleaned up for us goto bail; } #endif qb_rc = qb_ipcc_fd_get(c, &fd); if (qb_rc != 0) { rc = (int) -qb_rc; // System errno pcmk__err("Could not get fd from %s IPC: %s " QB_XS " rc=%d", name, pcmk_rc_str(rc), rc); goto bail; } auth_rc = is_ipc_provider_expected(c, fd, refuid, refgid, &found_pid, &found_uid, &found_gid); if (auth_rc == pcmk_rc_ipc_unauthorized) { pcmk__err("Daemon (IPC %s) effectively blocked with unauthorized " "process %lld (uid: %lld, gid: %lld)", name, (long long) PCMK__SPECIAL_PID_AS_0(found_pid), (long long) found_uid, (long long) found_gid); rc = pcmk_rc_ipc_unauthorized; goto bail; } if (auth_rc != pcmk_rc_ok) { rc = auth_rc; pcmk__err("Could not get peer credentials from %s IPC: %s " QB_XS " rc=%d", name, pcmk_rc_str(rc), rc); goto bail; } if (gotpid != NULL) { *gotpid = found_pid; } rc = pcmk_rc_ok; if ((found_uid != refuid || found_gid != refgid) && strncmp(last_asked_name, name, sizeof(last_asked_name))) { if ((found_uid == 0) && (refuid != 0)) { pcmk__warn("Daemon (IPC %s) runs as root, whereas the expected " "credentials are %lld:%lld, hazard of violating the " "least privilege principle", name, (long long) refuid, (long long) refgid); } else { pcmk__notice("Daemon (IPC %s) runs as %lld:%lld, whereas the " "expected credentials are %lld:%lld, which may " "mean a different set of privileges than expected", name, (long long) found_uid, (long long) found_gid, (long long) refuid, (long long) refgid); } memccpy(last_asked_name, name, '\0', sizeof(last_asked_name)); } bail: if (c != NULL) { qb_ipcc_disconnect(c); } return rc; } // Deprecated functions kept only for backward API compatibility // LCOV_EXCL_START #include bool crm_ipc_connect(crm_ipc_t *client) { int rc = pcmk__connect_generic_ipc(client); if (rc == pcmk_rc_ok) { return true; } if ((client != NULL) && (client->ipc == NULL)) { errno = (rc > 0)? rc : ENOTCONN; pcmk__debug("Could not establish %s IPC connection: %s (%d)", client->server_name, pcmk_rc_str(errno), errno); } else if (rc == pcmk_rc_ipc_unauthorized) { pcmk__err("%s IPC provider authentication failed", (client == NULL)? "Pacemaker" : client->server_name); errno = ECONNABORTED; } else { pcmk__err("Could not verify authenticity of %s IPC provider", (client == NULL)? "Pacemaker" : client->server_name); errno = ENOTCONN; } return false; } // LCOV_EXCL_STOP // End deprecated API diff --git a/lib/common/ipc_server.c b/lib/common/ipc_server.c index 03e4e2bdf3..5b81d54801 100644 --- a/lib/common/ipc_server.c +++ b/lib/common/ipc_server.c @@ -1,1018 +1,1018 @@ /* * Copyright 2004-2025 the Pacemaker project contributors * * The version control history for this file may have further details. * * This source code is licensed under the GNU Lesser General Public License * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. */ #include #include #include #include #include #include #include #include #include #include #include "crmcommon_private.h" /* Evict clients whose event queue grows this large (by default) */ #define PCMK_IPC_DEFAULT_QUEUE_MAX 500 static GHashTable *client_connections = NULL; /*! * \internal * \brief Count IPC clients * * \return Number of active IPC client connections */ guint pcmk__ipc_client_count(void) { return client_connections? g_hash_table_size(client_connections) : 0; } /*! * \internal * \brief Execute a function for each active IPC client connection * * \param[in] func Function to call * \param[in,out] user_data Pointer to pass to function * * \note The parameters are the same as for g_hash_table_foreach(). */ void pcmk__foreach_ipc_client(GHFunc func, gpointer user_data) { if ((func != NULL) && (client_connections != NULL)) { g_hash_table_foreach(client_connections, func, user_data); } } pcmk__client_t * pcmk__find_client(const qb_ipcs_connection_t *c) { if (client_connections) { return g_hash_table_lookup(client_connections, c); } pcmk__trace("No client found for %p", c); return NULL; } pcmk__client_t * pcmk__find_client_by_id(const char *id) { if ((client_connections != NULL) && (id != NULL)) { gpointer key; pcmk__client_t *client = NULL; GHashTableIter iter; g_hash_table_iter_init(&iter, client_connections); while (g_hash_table_iter_next(&iter, &key, (gpointer *) & client)) { if (strcmp(client->id, id) == 0) { return client; } } } pcmk__trace("No client found with id='%s'", pcmk__s(id, "")); return NULL; } /*! * \internal * \brief Get a client identifier for use in log messages * * \param[in] c Client * * \return Client's name, client's ID, or a string literal, as available * \note This is intended to be used in format strings like "client %s". */ const char * pcmk__client_name(const pcmk__client_t *c) { if (c == NULL) { return "(unspecified)"; } else if (c->name != NULL) { return c->name; } else if (c->id != NULL) { return c->id; } else { return "(unidentified)"; } } void pcmk__client_cleanup(void) { if (client_connections != NULL) { int active = g_hash_table_size(client_connections); if (active > 0) { pcmk__warn("Exiting with %d active IPC client%s", active, pcmk__plural_s(active)); } g_hash_table_destroy(client_connections); client_connections = NULL; } } void pcmk__drop_all_clients(qb_ipcs_service_t *service) { qb_ipcs_connection_t *c = NULL; if (service == NULL) { return; } c = qb_ipcs_connection_first_get(service); while (c != NULL) { qb_ipcs_connection_t *last = c; c = qb_ipcs_connection_next_get(service, last); /* There really shouldn't be anyone connected at this point */ pcmk__notice("Disconnecting client %p, pid=%d...", last, pcmk__client_pid(last)); qb_ipcs_disconnect(last); qb_ipcs_connection_unref(last); } } /*! * \internal * \brief Allocate a new pcmk__client_t object based on an IPC connection * * \param[in] c IPC connection (NULL to allocate generic client) * \param[in] key Connection table key (NULL to use sane default) * \param[in] uid_client UID corresponding to c (ignored if c is NULL) * * \return Pointer to new pcmk__client_t (guaranteed not to be \c NULL) */ static pcmk__client_t * client_from_connection(qb_ipcs_connection_t *c, void *key, uid_t uid_client) { pcmk__client_t *client = pcmk__assert_alloc(1, sizeof(pcmk__client_t)); if (c) { client->user = pcmk__uid2username(uid_client); if (client->user == NULL) { client->user = pcmk__str_copy("#unprivileged"); pcmk__err("Unable to enforce ACLs for user ID %d, assuming " "unprivileged", uid_client); } client->ipcs = c; pcmk__set_client_flags(client, pcmk__client_ipc); client->pid = pcmk__client_pid(c); if (key == NULL) { key = c; } } client->id = pcmk__generate_uuid(); if (key == NULL) { key = client->id; } if (client_connections == NULL) { pcmk__trace("Creating IPC client table"); client_connections = g_hash_table_new(g_direct_hash, g_direct_equal); } g_hash_table_insert(client_connections, key, client); return client; } /*! * \brief Allocate a new pcmk__client_t object and generate its ID * * \param[in] key What to use as connections hash table key (NULL to use ID) * * \return Pointer to new pcmk__client_t (asserts on failure) */ pcmk__client_t * pcmk__new_unauth_client(void *key) { return client_from_connection(NULL, key, 0); } pcmk__client_t * pcmk__new_client(qb_ipcs_connection_t *c, uid_t uid_client, gid_t gid_client) { gid_t uid_cluster = 0; gid_t gid_cluster = 0; pcmk__client_t *client = NULL; CRM_CHECK(c != NULL, return NULL); if (pcmk__daemon_user(&uid_cluster, &gid_cluster) != pcmk_rc_ok) { static bool need_log = true; if (need_log) { pcmk__warn("Could not find user and group IDs for user " CRM_DAEMON_USER); need_log = false; } } if (uid_client != 0) { pcmk__trace("Giving group %u access to new IPC connection", gid_cluster); /* Passing -1 to chown(2) means don't change */ qb_ipcs_connection_auth_set(c, -1, gid_cluster, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP); } /* TODO: Do our own auth checking, return NULL if unauthorized */ client = client_from_connection(c, NULL, uid_client); if ((uid_client == 0) || (uid_client == uid_cluster)) { /* Remember when a connection came from root or hacluster */ pcmk__set_client_flags(client, pcmk__client_privileged); } pcmk__debug("New IPC client %s for PID %u with uid %d and gid %d", client->id, client->pid, uid_client, gid_client); return client; } static struct iovec * pcmk__new_ipc_event(void) { return (struct iovec *) pcmk__assert_alloc(2, sizeof(struct iovec)); } /*! * \brief Free an I/O vector created by pcmk__ipc_prepare_iov() * * \param[in,out] event I/O vector to free */ void pcmk_free_ipc_event(struct iovec *event) { if (event != NULL) { free(event[0].iov_base); free(event[1].iov_base); free(event); } } static void free_event(gpointer data) { pcmk_free_ipc_event((struct iovec *) data); } static void add_event(pcmk__client_t *c, struct iovec *iov) { if (c->event_queue == NULL) { c->event_queue = g_queue_new(); } g_queue_push_tail(c->event_queue, iov); } void pcmk__free_client(pcmk__client_t *c) { if (c == NULL) { return; } if (client_connections) { if (c->ipcs) { pcmk__trace("Destroying %p/%p (%u remaining)", c, c->ipcs, (g_hash_table_size(client_connections) - 1)); g_hash_table_remove(client_connections, c->ipcs); } else { pcmk__trace("Destroying remote connection %p (%u remaining)", c, (g_hash_table_size(client_connections) - 1)); g_hash_table_remove(client_connections, c->id); } } if (c->event_timer) { g_source_remove(c->event_timer); } if (c->event_queue) { pcmk__debug("Destroying %d events", g_queue_get_length(c->event_queue)); g_queue_free_full(c->event_queue, free_event); } free(c->id); free(c->name); free(c->user); if (c->remote) { if (c->remote->auth_timeout) { g_source_remove(c->remote->auth_timeout); } if (c->remote->tls_session != NULL) { /* @TODO Reduce duplication at callers. Put here everything * necessary to tear down and free tls_session. */ gnutls_deinit(c->remote->tls_session); } free(c->remote->buffer); free(c->remote); } free(c); } /*! * \internal * \brief Raise IPC eviction threshold for a client, if allowed * * \param[in,out] client Client to modify * \param[in] qmax New threshold */ void pcmk__set_client_queue_max(pcmk__client_t *client, const char *qmax) { int rc = pcmk_rc_ok; long long qmax_ll = 0LL; unsigned int orig_value = 0U; CRM_CHECK(client != NULL, return); orig_value = client->queue_max; if (pcmk__is_set(client->flags, pcmk__client_privileged)) { rc = pcmk__scan_ll(qmax, &qmax_ll, 0LL); if (rc == pcmk_rc_ok) { if ((qmax_ll <= 0LL) || (qmax_ll > UINT_MAX)) { rc = ERANGE; } else { client->queue_max = (unsigned int) qmax_ll; } } } else { rc = EACCES; } if (rc != pcmk_rc_ok) { pcmk__info("Could not set IPC threshold for client %s[%u] to %s: %s", pcmk__client_name(client), client->pid, pcmk__s(qmax, "default"), pcmk_rc_str(rc)); } else if (client->queue_max != orig_value) { pcmk__debug("IPC threshold for client %s[%u] is now %u (was %u)", pcmk__client_name(client), client->pid, client->queue_max, orig_value); } } int pcmk__client_pid(qb_ipcs_connection_t *c) { struct qb_ipcs_connection_stats stats; stats.client_pid = 0; qb_ipcs_connection_stats_get(c, &stats, 0); return stats.client_pid; } /*! * \internal * \brief Retrieve message XML from data read from client IPC * * \param[in,out] c IPC client connection * \param[in] data Data read from client connection * \param[out] id Where to store message ID from libqb header * \param[out] flags Where to store flags from libqb header * * \return Message XML on success, NULL otherwise */ xmlNode * pcmk__client_data2xml(pcmk__client_t *c, void *data, uint32_t *id, uint32_t *flags) { xmlNode *xml = NULL; char *uncompressed = NULL; char *text = ((char *)data) + sizeof(pcmk__ipc_header_t); pcmk__ipc_header_t *header = data; if (!pcmk__valid_ipc_header(header)) { return NULL; } if (id) { *id = ((struct qb_ipc_response_header *)data)->id; } if (flags) { *flags = header->flags; } if (pcmk__is_set(header->flags, crm_ipc_proxied)) { /* Mark this client as being the endpoint of a proxy connection. * Proxy connections responses are sent on the event channel, to avoid * blocking the controller serving as proxy. */ pcmk__set_client_flags(c, pcmk__client_proxied); } if (header->size_compressed) { int rc = 0; unsigned int size_u = 1 + header->size_uncompressed; uncompressed = pcmk__assert_alloc(1, size_u); pcmk__trace("Decompressing message data %u bytes into %u bytes", header->size_compressed, size_u); rc = BZ2_bzBuffToBuffDecompress(uncompressed, &size_u, text, header->size_compressed, 1, 0); text = uncompressed; rc = pcmk__bzlib2rc(rc); if (rc != pcmk_rc_ok) { pcmk__err("Decompression failed: %s " QB_XS " rc=%d", pcmk_rc_str(rc), rc); free(uncompressed); return NULL; } } pcmk__assert(text[header->size_uncompressed - 1] == 0); xml = pcmk__xml_parse(text); - crm_log_xml_trace(xml, "[IPC received]"); + pcmk__log_xml_trace(xml, "[IPC received]"); free(uncompressed); return xml; } static int crm_ipcs_flush_events(pcmk__client_t *c); static gboolean crm_ipcs_flush_events_cb(gpointer data) { pcmk__client_t *c = data; c->event_timer = 0; crm_ipcs_flush_events(c); return FALSE; } /*! * \internal * \brief Add progressive delay before next event queue flush * * \param[in,out] c Client connection to add delay to * \param[in] queue_len Current event queue length */ static inline void delay_next_flush(pcmk__client_t *c, unsigned int queue_len) { /* Delay a maximum of 1.5 seconds */ guint delay = (queue_len < 5)? (1000 + 100 * queue_len) : 1500; c->event_timer = pcmk__create_timer(delay, crm_ipcs_flush_events_cb, c); } /*! * \internal * \brief Send client any messages in its queue * * \param[in,out] c Client to flush * * \return Standard Pacemaker return value */ static int crm_ipcs_flush_events(pcmk__client_t *c) { int rc = pcmk_rc_ok; ssize_t qb_rc = 0; unsigned int sent = 0; unsigned int queue_len = 0; if (c == NULL) { return rc; } else if (c->event_timer) { /* There is already a timer, wait until it goes off */ pcmk__trace("Timer active for %p - %d", c->ipcs, c->event_timer); return rc; } if (c->event_queue) { queue_len = g_queue_get_length(c->event_queue); } while (sent < 100) { pcmk__ipc_header_t *header = NULL; struct iovec *event = NULL; if (c->event_queue) { // We don't pop unless send is successful event = g_queue_peek_head(c->event_queue); } if (event == NULL) { // Queue is empty break; } qb_rc = qb_ipcs_event_sendv(c->ipcs, event, 2); if (qb_rc < 0) { rc = (int) -qb_rc; break; } event = g_queue_pop_head(c->event_queue); sent++; header = event[0].iov_base; if (header->size_compressed) { pcmk__trace("Event %" PRId32 " to %p[%u] (%zd compressed bytes) " "sent", header->qb.id, c->ipcs, c->pid, qb_rc); } else { pcmk__trace("Event %" PRId32 " to %p[%u] (%zd bytes) sent: %.120s", header->qb.id, c->ipcs, c->pid, qb_rc, (const char *) event[1].iov_base); } pcmk_free_ipc_event(event); } queue_len -= sent; if (sent > 0 || queue_len) { pcmk__trace("Sent %u events (%u remaining) for %p[%d]: %s (%zd)", sent, queue_len, c->ipcs, c->pid, pcmk_rc_str(rc), qb_rc); } if (queue_len) { /* Allow clients to briefly fall behind on processing incoming messages, * but drop completely unresponsive clients so the connection doesn't * consume resources indefinitely. */ if (queue_len > QB_MAX(c->queue_max, PCMK_IPC_DEFAULT_QUEUE_MAX)) { if ((c->queue_backlog <= 1) || (queue_len < c->queue_backlog)) { /* Don't evict for a new or shrinking backlog */ pcmk__warn("Client with process ID %u has a backlog of %u " "messages " QB_XS " %p", c->pid, queue_len, c->ipcs); } else { pcmk__err("Evicting client with process ID %u due to backlog " "of %u messages " QB_XS " %p", c->pid, queue_len, c->ipcs); c->queue_backlog = 0; qb_ipcs_disconnect(c->ipcs); return rc; } } c->queue_backlog = queue_len; delay_next_flush(c, queue_len); } else { /* Event queue is empty, there is no backlog */ c->queue_backlog = 0; } return rc; } /*! * \internal * \brief Create an I/O vector for sending an IPC XML message * * \param[in] request Identifier for libqb response header * \param[in] message XML message to send * \param[in] max_send_size If 0, default IPC buffer size is used * \param[out] result Where to store prepared I/O vector * \param[out] bytes Size of prepared data in bytes * * \return Standard Pacemaker return code */ int pcmk__ipc_prepare_iov(uint32_t request, const xmlNode *message, uint32_t max_send_size, struct iovec **result, ssize_t *bytes) { struct iovec *iov; unsigned int total = 0; GString *buffer = NULL; pcmk__ipc_header_t *header = NULL; int rc = pcmk_rc_ok; if ((message == NULL) || (result == NULL)) { rc = EINVAL; goto done; } header = calloc(1, sizeof(pcmk__ipc_header_t)); if (header == NULL) { rc = ENOMEM; goto done; } buffer = g_string_sized_new(1024); pcmk__xml_string(message, 0, buffer, 0); if (max_send_size == 0) { max_send_size = crm_ipc_default_buffer_size(); } CRM_LOG_ASSERT(max_send_size != 0); *result = NULL; iov = pcmk__new_ipc_event(); iov[0].iov_len = sizeof(pcmk__ipc_header_t); iov[0].iov_base = header; header->version = PCMK__IPC_VERSION; header->size_uncompressed = 1 + buffer->len; total = iov[0].iov_len + header->size_uncompressed; if (total < max_send_size) { iov[1].iov_base = pcmk__str_copy(buffer->str); iov[1].iov_len = header->size_uncompressed; } else { static unsigned int biggest = 0; char *compressed = NULL; unsigned int new_size = 0; if (pcmk__compress(buffer->str, (unsigned int) header->size_uncompressed, (unsigned int) max_send_size, &compressed, &new_size) == pcmk_rc_ok) { pcmk__set_ipc_flags(header->flags, "send data", crm_ipc_compressed); header->size_compressed = new_size; iov[1].iov_len = header->size_compressed; iov[1].iov_base = compressed; biggest = QB_MAX(header->size_compressed, biggest); } else { - crm_log_xml_trace(message, "EMSGSIZE"); + pcmk__log_xml_trace(message, "EMSGSIZE"); biggest = QB_MAX(header->size_uncompressed, biggest); pcmk__err("Could not compress %u-byte message into less than IPC " "limit of %u bytes; set PCMK_ipc_buffer to higher value " "(%u bytes suggested)", header->size_uncompressed, max_send_size, (4 * biggest)); free(compressed); pcmk_free_ipc_event(iov); rc = EMSGSIZE; goto done; } } header->qb.size = iov[0].iov_len + iov[1].iov_len; header->qb.id = (int32_t)request; /* Replying to a specific request */ *result = iov; pcmk__assert(header->qb.size > 0); if (bytes != NULL) { *bytes = header->qb.size; } done: if (buffer != NULL) { g_string_free(buffer, TRUE); } return rc; } int pcmk__ipc_send_iov(pcmk__client_t *c, struct iovec *iov, uint32_t flags) { int rc = pcmk_rc_ok; static uint32_t id = 1; pcmk__ipc_header_t *header = iov[0].iov_base; if (c->flags & pcmk__client_proxied) { /* _ALL_ replies to proxied connections need to be sent as events */ if (!pcmk__is_set(flags, crm_ipc_server_event)) { /* The proxied flag lets us know this was originally meant to be a * response, even though we're sending it over the event channel. */ pcmk__set_ipc_flags(flags, "server event", crm_ipc_server_event |crm_ipc_proxied_relay_response); } } pcmk__set_ipc_flags(header->flags, "server event", flags); if (flags & crm_ipc_server_event) { header->qb.id = id++; /* We don't really use it, but doesn't hurt to set one */ if (flags & crm_ipc_server_free) { pcmk__trace("Sending the original to %p[%d]", c->ipcs, c->pid); add_event(c, iov); } else { struct iovec *iov_copy = pcmk__new_ipc_event(); pcmk__trace("Sending a copy to %p[%d]", c->ipcs, c->pid); iov_copy[0].iov_len = iov[0].iov_len; iov_copy[0].iov_base = malloc(iov[0].iov_len); memcpy(iov_copy[0].iov_base, iov[0].iov_base, iov[0].iov_len); iov_copy[1].iov_len = iov[1].iov_len; iov_copy[1].iov_base = malloc(iov[1].iov_len); memcpy(iov_copy[1].iov_base, iov[1].iov_base, iov[1].iov_len); add_event(c, iov_copy); } } else { ssize_t qb_rc; CRM_LOG_ASSERT(header->qb.id != 0); /* Replying to a specific request */ qb_rc = qb_ipcs_response_sendv(c->ipcs, iov, 2); if (qb_rc < header->qb.size) { if (qb_rc < 0) { rc = (int) -qb_rc; } pcmk__notice("Response %" PRId32 " to pid %u failed: %s " QB_XS " bytes=%" PRId32 " rc=%zd ipcs=%p", header->qb.id, c->pid, pcmk_rc_str(rc), header->qb.size, qb_rc, c->ipcs); } else { pcmk__trace("Response %" PRId32 " sent, %zd bytes to %p[%u]", header->qb.id, qb_rc, c->ipcs, c->pid); } if (flags & crm_ipc_server_free) { pcmk_free_ipc_event(iov); } } if (flags & crm_ipc_server_event) { rc = crm_ipcs_flush_events(c); } else { crm_ipcs_flush_events(c); } if ((rc == EPIPE) || (rc == ENOTCONN)) { pcmk__trace("Client %p disconnected", c->ipcs); } return rc; } int pcmk__ipc_send_xml(pcmk__client_t *c, uint32_t request, const xmlNode *message, uint32_t flags) { struct iovec *iov = NULL; int rc = pcmk_rc_ok; if (c == NULL) { return EINVAL; } rc = pcmk__ipc_prepare_iov(request, message, crm_ipc_default_buffer_size(), &iov, NULL); if (rc == pcmk_rc_ok) { pcmk__set_ipc_flags(flags, "send data", crm_ipc_server_free); rc = pcmk__ipc_send_iov(c, iov, flags); } else { pcmk_free_ipc_event(iov); pcmk__notice("IPC message to pid %d failed: %s " QB_XS " rc=%d", c->pid, pcmk_rc_str(rc), rc); } return rc; } /*! * \internal * \brief Create an acknowledgement with a status code to send to a client * * \param[in] function Calling function * \param[in] line Source file line within calling function * \param[in] flags IPC flags to use when sending * \param[in] tag Element name to use for acknowledgement * \param[in] ver IPC protocol version (can be NULL) * \param[in] status Exit status code to add to ack * * \return Newly created XML for ack * * \note The caller is responsible for freeing the return value with * \c pcmk__xml_free(). */ xmlNode * pcmk__ipc_create_ack_as(const char *function, int line, uint32_t flags, const char *tag, const char *ver, crm_exit_t status) { xmlNode *ack = NULL; if (pcmk__is_set(flags, crm_ipc_client_response)) { ack = pcmk__xe_create(NULL, tag); pcmk__xe_set(ack, PCMK_XA_FUNCTION, function); pcmk__xe_set_int(ack, PCMK__XA_LINE, line); pcmk__xe_set_int(ack, PCMK_XA_STATUS, (int) status); pcmk__xe_set(ack, PCMK__XA_IPC_PROTO_VERSION, ver); } return ack; } /*! * \internal * \brief Send an acknowledgement with a status code to a client * * \param[in] function Calling function * \param[in] line Source file line within calling function * \param[in] c Client to send ack to * \param[in] request Request ID being replied to * \param[in] flags IPC flags to use when sending * \param[in] tag Element name to use for acknowledgement * \param[in] ver IPC protocol version (can be NULL) * \param[in] status Status code to send with acknowledgement * * \return Standard Pacemaker return code */ int pcmk__ipc_send_ack_as(const char *function, int line, pcmk__client_t *c, uint32_t request, uint32_t flags, const char *tag, const char *ver, crm_exit_t status) { int rc = pcmk_rc_ok; xmlNode *ack = pcmk__ipc_create_ack_as(function, line, flags, tag, ver, status); if (ack != NULL) { pcmk__trace("Ack'ing IPC message from client %s as <%s status=%d>", pcmk__client_name(c), tag, status); - crm_log_xml_trace(ack, "sent-ack"); + pcmk__log_xml_trace(ack, "sent-ack"); c->request_id = 0; rc = pcmk__ipc_send_xml(c, request, ack, flags); pcmk__xml_free(ack); } return rc; } /*! * \internal * \brief Add an IPC server to the main loop for the CIB manager API * * \param[out] ipcs_ro New IPC server for read-only CIB manager API * \param[out] ipcs_rw New IPC server for read/write CIB manager API * \param[out] ipcs_shm New IPC server for shared-memory CIB manager API * \param[in] ro_cb IPC callbacks for read-only API * \param[in] rw_cb IPC callbacks for read/write and shared-memory APIs * * \note This function exits fatally if unable to create the servers. * \note There is no actual difference between the three IPC endpoints other * than their names. */ void pcmk__serve_based_ipc(qb_ipcs_service_t **ipcs_ro, qb_ipcs_service_t **ipcs_rw, qb_ipcs_service_t **ipcs_shm, struct qb_ipcs_service_handlers *ro_cb, struct qb_ipcs_service_handlers *rw_cb) { *ipcs_ro = mainloop_add_ipc_server(PCMK__SERVER_BASED_RO, QB_IPC_NATIVE, ro_cb); *ipcs_rw = mainloop_add_ipc_server(PCMK__SERVER_BASED_RW, QB_IPC_NATIVE, rw_cb); *ipcs_shm = mainloop_add_ipc_server(PCMK__SERVER_BASED_SHM, QB_IPC_SHM, rw_cb); if (*ipcs_ro == NULL || *ipcs_rw == NULL || *ipcs_shm == NULL) { pcmk__err("Failed to create the CIB manager: exiting and inhibiting " "respawn"); pcmk__warn("Verify pacemaker and pacemaker_remote are not both " "enabled"); crm_exit(CRM_EX_FATAL); } } /*! * \internal * \brief Destroy IPC servers for the CIB manager API * * \param[out] ipcs_ro IPC server for read-only the CIB manager API * \param[out] ipcs_rw IPC server for read/write the CIB manager API * \param[out] ipcs_shm IPC server for shared-memory the CIB manager API * * \note This is a convenience function for calling qb_ipcs_destroy() for each * argument. */ void pcmk__stop_based_ipc(qb_ipcs_service_t *ipcs_ro, qb_ipcs_service_t *ipcs_rw, qb_ipcs_service_t *ipcs_shm) { qb_ipcs_destroy(ipcs_ro); qb_ipcs_destroy(ipcs_rw); qb_ipcs_destroy(ipcs_shm); } /*! * \internal * \brief Add an IPC server to the main loop for the controller API * * \param[in] cb IPC callbacks * * \return Newly created IPC server */ qb_ipcs_service_t * pcmk__serve_controld_ipc(struct qb_ipcs_service_handlers *cb) { return mainloop_add_ipc_server(CRM_SYSTEM_CRMD, QB_IPC_NATIVE, cb); } /*! * \internal * \brief Add an IPC server to the main loop for the attribute manager API * * \param[out] ipcs Where to store newly created IPC server * \param[in] cb IPC callbacks * * \note This function exits fatally if unable to create the servers. */ void pcmk__serve_attrd_ipc(qb_ipcs_service_t **ipcs, struct qb_ipcs_service_handlers *cb) { *ipcs = mainloop_add_ipc_server(PCMK__VALUE_ATTRD, QB_IPC_NATIVE, cb); if (*ipcs == NULL) { pcmk__crit("Exiting fatally because unable to serve " PCMK__SERVER_ATTRD " IPC (verify pacemaker and pacemaker_remote are not both" " enabled)"); crm_exit(CRM_EX_FATAL); } } /*! * \internal * \brief Add an IPC server to the main loop for the fencer API * * \param[out] ipcs Where to store newly created IPC server * \param[in] cb IPC callbacks * * \note This function exits fatally if unable to create the servers. */ void pcmk__serve_fenced_ipc(qb_ipcs_service_t **ipcs, struct qb_ipcs_service_handlers *cb) { *ipcs = mainloop_add_ipc_server_with_prio("stonith-ng", QB_IPC_NATIVE, cb, QB_LOOP_HIGH); if (*ipcs == NULL) { pcmk__err("Failed to create fencer: exiting and inhibiting respawn"); pcmk__warn("Verify pacemaker and pacemaker_remote are not both " "enabled"); crm_exit(CRM_EX_FATAL); } } /*! * \internal * \brief Add an IPC server to the main loop for the pacemakerd API * * \param[out] ipcs Where to store newly created IPC server * \param[in] cb IPC callbacks * * \note This function exits with CRM_EX_OSERR if unable to create the servers. */ void pcmk__serve_pacemakerd_ipc(qb_ipcs_service_t **ipcs, struct qb_ipcs_service_handlers *cb) { *ipcs = mainloop_add_ipc_server(CRM_SYSTEM_MCP, QB_IPC_NATIVE, cb); if (*ipcs == NULL) { pcmk__err("Couldn't start pacemakerd IPC server"); pcmk__warn("Verify pacemaker and pacemaker_remote are not both " "enabled"); /* sub-daemons are observed by pacemakerd. Thus we exit CRM_EX_FATAL * if we want to prevent pacemakerd from restarting them. * With pacemakerd we leave the exit-code shown to e.g. systemd * to what it was prior to moving the code here from pacemakerd.c */ crm_exit(CRM_EX_OSERR); } } /*! * \internal * \brief Add an IPC server to the main loop for the scheduler API * * \param[in] cb IPC callbacks * * \return Newly created IPC server * \note This function exits fatally if unable to create the servers. */ qb_ipcs_service_t * pcmk__serve_schedulerd_ipc(struct qb_ipcs_service_handlers *cb) { return mainloop_add_ipc_server(CRM_SYSTEM_PENGINE, QB_IPC_NATIVE, cb); } diff --git a/lib/common/nvpair.c b/lib/common/nvpair.c index e0134b75b0..ae103ae510 100644 --- a/lib/common/nvpair.c +++ b/lib/common/nvpair.c @@ -1,729 +1,729 @@ /* * Copyright 2004-2025 the Pacemaker project contributors * * The version control history for this file may have further details. * * This source code is licensed under the GNU Lesser General Public License * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. */ #include #include #include // UINT32_MAX #include // PRIu32 #include #include #include #include // gchar, gint, etc. #include #include #include // pcmk_nvpair_t, etc. #include #include #include "crmcommon_private.h" /* * This file isolates handling of various kinds of name/value pairs: * * - pcmk_nvpair_t data type * - name=value strings * - XML nvpair elements () * - Instance attributes and meta-attributes (for resources and actions) */ // pcmk_nvpair_t handling /*! * \internal * \brief Allocate a new name/value pair * * \param[in] name New name (required) * \param[in] value New value * * \return Newly allocated name/value pair * \note The caller is responsible for freeing the result with * \c pcmk__free_nvpair(). */ static pcmk_nvpair_t * pcmk__new_nvpair(const char *name, const char *value) { pcmk_nvpair_t *nvpair = NULL; pcmk__assert(name); nvpair = pcmk__assert_alloc(1, sizeof(pcmk_nvpair_t)); nvpair->name = pcmk__str_copy(name); nvpair->value = pcmk__str_copy(value); return nvpair; } /*! * \internal * \brief Free a name/value pair * * \param[in,out] nvpair Name/value pair to free */ static void pcmk__free_nvpair(gpointer data) { if (data) { pcmk_nvpair_t *nvpair = data; free(nvpair->name); free(nvpair->value); free(nvpair); } } /*! * \brief Prepend a name/value pair to a list * * \param[in,out] nvpairs List to modify * \param[in] name New entry's name * \param[in] value New entry's value * * \return New head of list * \note The caller is responsible for freeing the list with * \c pcmk_free_nvpairs(). */ GSList * pcmk_prepend_nvpair(GSList *nvpairs, const char *name, const char *value) { return g_slist_prepend(nvpairs, pcmk__new_nvpair(name, value)); } /*! * \brief Free a list of name/value pairs * * \param[in,out] list List to free */ void pcmk_free_nvpairs(GSList *nvpairs) { g_slist_free_full(nvpairs, pcmk__free_nvpair); } // name=value string handling /*! * \internal * \brief Extract the name and value from a string formatted as "name=value" * * \param[in] input Input string, likely from the command line * \param[out] name Everything before the first \c '=' in the input string * \param[out] value Everything after the first \c '=' in the input string, * minus trailing newlines * * \return Standard Pacemaker return code * * \note On success, the caller is responsible for freeing \p *name and * \p *value using \c g_free(). On failure, nothing is allocated. */ int pcmk__scan_nvpair(const gchar *input, gchar **name, gchar **value) { gchar **nvpair = NULL; int rc = pcmk_rc_ok; pcmk__assert(input != NULL); pcmk__assert((name != NULL) && (*name == NULL)); pcmk__assert((value != NULL) && (*value == NULL)); nvpair = g_strsplit(input, "=", 2); /* Check whether nvpair is well-formed (short-circuits if input was split * into fewer than 2 tokens) */ if (pcmk__str_empty(nvpair[0]) || pcmk__str_empty(nvpair[1])) { rc = pcmk_rc_bad_nvpair; goto done; } *name = nvpair[0]; *value = nvpair[1]; pcmk__trim((char *) *value); // name and value took ownership nvpair[0] = NULL; nvpair[1] = NULL; done: g_strfreev(nvpair); return rc; } /*! * \internal * \brief Format a name/value pair. * * Units can optionally be provided for the value. Note that unlike most * formatting functions, this one returns the formatted string. It is * assumed that the most common use of this function will be to build up * a string to be output as part of other functions. * * \note The caller is responsible for freeing the return value after use. * * \param[in] name The name of the nvpair. * \param[in] value The value of the nvpair. * \param[in] units Optional units for the value, or NULL. * * \return Newly allocated string with name/value pair */ char * pcmk__format_nvpair(const char *name, const char *value, const char *units) { return pcmk__assert_asprintf("%s=\"%s%s\"", name, value, pcmk__s(units, "")); } /*! * \brief Safely add hash table entry to XML as attribute or name-value pair * * Suitable for \c g_hash_table_foreach(), this function takes a hash table key * and value, with an XML node passed as user data, and adds an XML attribute * with the specified name and value if it does not already exist. If the key * name starts with a digit, then it's not a valid XML attribute name. In that * case, this will instead add a child * to the XML. * * \param[in] key Key of hash table entry * \param[in] value Value of hash table entry * \param[in,out] user_data XML node */ void hash2smartfield(gpointer key, gpointer value, gpointer user_data) { /* @TODO Generate PCMK__XE_PARAM nodes for all keys that aren't valid XML * attribute names (not just those that start with digits), or possibly for * all keys to simplify parsing. * * Consider either deprecating as public API or exposing PCMK__XE_PARAM. * PCMK__XE_PARAM is currently private because it doesn't appear in any * output that Pacemaker generates. */ const char *name = key; const char *s_value = value; xmlNode *xml_node = user_data; if (isdigit(name[0])) { xmlNode *tmp = pcmk__xe_create(xml_node, PCMK__XE_PARAM); pcmk__xe_set(tmp, PCMK_XA_NAME, name); pcmk__xe_set(tmp, PCMK_XA_VALUE, s_value); } else if (pcmk__xe_get(xml_node, name) == NULL) { pcmk__xe_set(xml_node, name, s_value); pcmk__trace("dumped: %s=%s", name, s_value); } else { pcmk__trace("duplicate: %s=%s", name, s_value); } } /*! * \brief Set XML attribute based on hash table entry * * Suitable for \c g_hash_table_foreach(), this function takes a hash table key * and value, with an XML node passed as user data, and adds an XML attribute * with the specified name and value if it does not already exist. * * \param[in] key Key of hash table entry * \param[in] value Value of hash table entry * \param[in,out] user_data XML node */ void hash2field(gpointer key, gpointer value, gpointer user_data) { const char *name = key; const char *s_value = value; xmlNode *xml_node = user_data; if (pcmk__xe_get(xml_node, name) == NULL) { pcmk__xe_set(xml_node, name, s_value); } else { pcmk__trace("duplicate: %s=%s", name, s_value); } } /*! * \brief Set XML attribute based on hash table entry, as meta-attribute name * * Suitable for \c g_hash_table_foreach(), this function takes a hash table key * and value, with an XML node passed as user data, and adds an XML attribute * with the meta-attribute version of the specified name and value if it does * not already exist and if the name does not appear to be cluster-internal. * * \param[in] key Key of hash table entry * \param[in] value Value of hash table entry * \param[in,out] user_data XML node */ void hash2metafield(gpointer key, gpointer value, gpointer user_data) { char *crm_name = NULL; if (key == NULL || value == NULL) { return; } /* Filter out cluster-generated attributes that contain a '#' or ':' * (like fail-count and last-failure). */ for (crm_name = key; *crm_name; ++crm_name) { if ((*crm_name == '#') || (*crm_name == ':')) { return; } } crm_name = crm_meta_name(key); hash2field(crm_name, value, user_data); free(crm_name); } // nvpair handling /*! * \brief Create an XML name/value pair * * \param[in,out] parent If not \c NULL, make new XML node a child of this one * \param[in] id Set this as XML ID (or NULL to auto-generate) * \param[in] name Name to use * \param[in] value Value to use * * \return New XML object on success, \c NULL otherwise */ xmlNode * crm_create_nvpair_xml(xmlNode *parent, const char *id, const char *name, const char *value) { xmlNode *nvp; /* id can be NULL so we auto-generate one, and name can be NULL if this * will be used to delete a name/value pair by ID, but both can't be NULL */ CRM_CHECK(id || name, return NULL); nvp = pcmk__xe_create(parent, PCMK_XE_NVPAIR); if (id) { pcmk__xe_set(nvp, PCMK_XA_ID, id); } else { pcmk__xe_set_id(nvp, "%s-%s", pcmk__s(pcmk__xe_id(parent), PCMK_XE_NVPAIR), name); } pcmk__xe_set(nvp, PCMK_XA_NAME, name); pcmk__xe_set(nvp, PCMK_XA_VALUE, value); return nvp; } /*! * \brief Retrieve XML attributes as a hash table * * Given an XML element, this will look for any \ element child, * creating a hash table of (newly allocated string) name/value pairs taken * first from the attributes element's NAME=VALUE XML attributes, and then * from any \ children of attributes. * * \param[in] XML node to parse * * \return Hash table with name/value pairs * \note It is the caller's responsibility to free the result using * \c g_hash_table_destroy(). */ GHashTable * xml2list(const xmlNode *parent) { xmlNode *child = NULL; xmlAttrPtr pIter = NULL; xmlNode *nvpair_list = NULL; GHashTable *nvpair_hash = pcmk__strkey_table(free, free); CRM_CHECK(parent != NULL, return nvpair_hash); nvpair_list = pcmk__xe_first_child(parent, PCMK__XE_ATTRIBUTES, NULL, NULL); if (nvpair_list == NULL) { pcmk__trace("No attributes in %s", parent->name); - crm_log_xml_trace(parent, "No attributes for resource op"); + pcmk__log_xml_trace(parent, "No attributes for resource op"); } - crm_log_xml_trace(nvpair_list, "Unpacking"); + pcmk__log_xml_trace(nvpair_list, "Unpacking"); for (pIter = pcmk__xe_first_attr(nvpair_list); pIter != NULL; pIter = pIter->next) { const char *p_name = (const char *)pIter->name; const char *p_value = pcmk__xml_attr_value(pIter); pcmk__trace("Added %s=%s", p_name, p_value); pcmk__insert_dup(nvpair_hash, p_name, p_value); } for (child = pcmk__xe_first_child(nvpair_list, PCMK__XE_PARAM, NULL, NULL); child != NULL; child = pcmk__xe_next(child, PCMK__XE_PARAM)) { const char *key = pcmk__xe_get(child, PCMK_XA_NAME); const char *value = pcmk__xe_get(child, PCMK_XA_VALUE); pcmk__trace("Added %s=%s", key, value); if (key != NULL && value != NULL) { pcmk__insert_dup(nvpair_hash, key, value); } } return nvpair_hash; } /*! * \internal * \brief Unpack a single nvpair XML element into a hash table * * \param[in] nvpair XML nvpair element to unpack * \param[in,out] userdata Unpack data * * \return pcmk_rc_ok (to always proceed to next nvpair) */ static int unpack_nvpair(xmlNode *nvpair, void *userdata) { pcmk__nvpair_unpack_t *unpack_data = userdata; const char *name = NULL; const char *value = NULL; const char *old_value = NULL; const xmlNode *ref_nvpair = pcmk__xe_resolve_idref(nvpair, NULL); if (ref_nvpair == NULL) { /* Not possible with schema validation enabled (error already * logged) */ return pcmk_rc_ok; } name = pcmk__xe_get(ref_nvpair, PCMK_XA_NAME); value = pcmk__xe_get(ref_nvpair, PCMK_XA_VALUE); if ((name == NULL) || (value == NULL)) { return pcmk_rc_ok; // Not possible with schema validation enabled } old_value = g_hash_table_lookup(unpack_data->values, name); if (pcmk__str_eq(value, "#default", pcmk__str_casei)) { // @COMPAT Deprecated since 2.1.8 pcmk__config_warn("Support for setting meta-attributes (such as " "%s) to the explicit value '#default' is " "deprecated and will be removed in a future " "release", name); if (old_value != NULL) { g_hash_table_remove(unpack_data->values, name); } } else if ((old_value == NULL) || unpack_data->overwrite) { pcmk__trace("Setting %s=\"%s\" (was %s)", name, value, pcmk__s(old_value, "unset")); pcmk__insert_dup(unpack_data->values, name, value); } return pcmk_rc_ok; } /*! * \internal * \brief Unpack an XML block of nvpair elements into a hash table, * evaluated for any rule * * \param[in] data XML block to unpack * \param[in,out] user_data Unpack data * * \note This is suitable for use as a GList iterator function */ void pcmk__unpack_nvpair_block(gpointer data, gpointer user_data) { xmlNode *pair = data; pcmk__nvpair_unpack_t *unpack_data = user_data; xmlNode *rule_xml = NULL; pcmk__assert((pair != NULL) && (unpack_data != NULL) && (unpack_data->values != NULL)); rule_xml = pcmk__xe_first_child(pair, PCMK_XE_RULE, NULL, NULL); if ((rule_xml != NULL) && (pcmk_evaluate_rule(rule_xml, &(unpack_data->rule_input), unpack_data->next_change) != pcmk_rc_ok)) { return; } pcmk__trace("Adding name/value pairs from %s %s overwrite", pcmk__xe_id(pair), (unpack_data->overwrite? "with" : "without")); if (pcmk__xe_is(pair->children, PCMK__XE_ATTRIBUTES)) { pair = pair->children; } pcmk__xe_foreach_child(pair, PCMK_XE_NVPAIR, unpack_nvpair, unpack_data); } /*! * \brief Unpack nvpair blocks contained by an XML element into a hash table, * evaluated for any rules * * \param[in] xml XML element containing blocks of nvpair elements * \param[in] element_name If not NULL, only unpack blocks of this element * \param[in] first_id If not NULL, process block with this ID first * \param[in] rule_input Values used to evaluate rule criteria * \param[out] values Where to store extracted name/value pairs * \param[out] next_change If not NULL, set to when evaluation will next * change, if sooner than its current value */ void pcmk_unpack_nvpair_blocks(const xmlNode *xml, const char *element_name, const char *first_id, const pcmk_rule_input_t *rule_input, GHashTable *values, crm_time_t *next_change) { GList *blocks = pcmk__xe_dereference_children(xml, element_name); if (blocks != NULL) { pcmk__nvpair_unpack_t data = { .values = values, .first_id = first_id, .rule_input = { .now = NULL, }, .overwrite = false, .next_change = next_change, }; if (rule_input != NULL) { data.rule_input = *rule_input; } blocks = g_list_sort_with_data(blocks, pcmk__cmp_nvpair_blocks, &data); g_list_foreach(blocks, pcmk__unpack_nvpair_block, &data); g_list_free(blocks); } } // Meta-attribute handling /*! * \brief Get the environment variable equivalent of a meta-attribute name * * \param[in] attr_name Name of meta-attribute * * \return Newly allocated string for \p attr_name with "CRM_meta_" prefix and * underbars instead of dashes * \note This asserts on an invalid argument or memory allocation error, so * callers can assume the result is non-NULL. The caller is responsible * for freeing the result using free(). */ char * crm_meta_name(const char *attr_name) { char *env_name = NULL; pcmk__assert(!pcmk__str_empty(attr_name)); env_name = pcmk__assert_asprintf(CRM_META "_%s", attr_name); for (char *c = env_name; *c != '\0'; ++c) { if (*c == '-') { *c = '_'; } } return env_name; } /*! * \brief Get the value of a meta-attribute * * Get the value of a meta-attribute from a hash table whose keys are * meta-attribute environment variable names (as crm_meta_name() would * create, like pcmk__graph_action_t:params, not pcmk_resource_t:meta). * * \param[in] meta Hash table of meta-attributes * \param[in] attr_name Name of meta-attribute to get * * \return Value of given meta-attribute */ const char * crm_meta_value(GHashTable *meta, const char *attr_name) { if ((meta != NULL) && (attr_name != NULL)) { char *key = crm_meta_name(attr_name); const char *value = g_hash_table_lookup(meta, key); free(key); return value; } return NULL; } /*! * \internal * \brief Compare processing order of two XML blocks of name/value pairs * * \param[in] a First XML block to compare * \param[in] b Second XML block to compare * \param[in] user_data pcmk__nvpair_unpack_t with first_id (whether a * particular XML ID should have priority) and overwrite * (whether later-processed blocks will overwrite values * from earlier ones) set as desired * * \return Standard comparison return code (a negative value if \p a should sort * first, a positive value if \p b should sort first, and 0 if they * should sort equally) * \note This is suitable for use as a GList sorting function. */ gint pcmk__cmp_nvpair_blocks(gconstpointer a, gconstpointer b, gpointer user_data) { const xmlNode *pair_a = a; const xmlNode *pair_b = b; const pcmk__nvpair_unpack_t *unpack_data = user_data; int score_a = 0; int score_b = 0; int rc = pcmk_rc_ok; /* If we're overwriting values, we want to process blocks from * lowest priority to highest, so higher-priority values overwrite * lower-priority ones. If we're not overwriting values, we want to process * from highest priority to lowest. */ const gint a_is_higher = ((unpack_data != NULL) && unpack_data->overwrite)? 1 : -1; const gint b_is_higher = -a_is_higher; /* NULL values have lowest priority, regardless of the other's score * (it won't be possible in practice anyway, this is just a failsafe) */ if (a == NULL) { return (b == NULL)? 0 : b_is_higher; } else if (b == NULL) { return a_is_higher; } /* A particular XML ID can be specified as having highest priority * regardless of score (schema validation, if enabled, prevents two blocks * from having the same ID, so we can ignore handling that case * specifically) */ if ((unpack_data != NULL) && (unpack_data->first_id != NULL)) { if (pcmk__str_eq(pcmk__xe_id(pair_a), unpack_data->first_id, pcmk__str_none)) { return a_is_higher; } else if (pcmk__str_eq(pcmk__xe_id(pair_b), unpack_data->first_id, pcmk__str_none)) { return b_is_higher; } } // Otherwise, check the scores rc = pcmk__xe_get_score(pair_a, PCMK_XA_SCORE, &score_a, 0); if (rc != pcmk_rc_ok) { // Not possible with schema validation enabled pcmk__config_warn("Using 0 as %s score because '%s' " "is not a valid score: %s", pcmk__xe_id(pair_a), pcmk__xe_get(pair_a, PCMK_XA_SCORE), pcmk_rc_str(rc)); } rc = pcmk__xe_get_score(pair_b, PCMK_XA_SCORE, &score_b, 0); if (rc != pcmk_rc_ok) { // Not possible with schema validation enabled pcmk__config_warn("Using 0 as %s score because '%s' " "is not a valid score: %s", pcmk__xe_id(pair_b), pcmk__xe_get(pair_b, PCMK_XA_SCORE), pcmk_rc_str(rc)); } if (score_a < score_b) { return b_is_higher; } else if (score_a > score_b) { return a_is_higher; } return 0; } // Deprecated functions kept only for backward API compatibility // LCOV_EXCL_START #include static gint pcmk__compare_nvpair(gconstpointer a, gconstpointer b) { int rc = 0; const pcmk_nvpair_t *pair_a = a; const pcmk_nvpair_t *pair_b = b; pcmk__assert((pair_a != NULL) && (pair_a->name != NULL) && (pair_b != NULL) && (pair_b->name != NULL)); rc = strcmp(pair_a->name, pair_b->name); if (rc < 0) { return -1; } else if (rc > 0) { return 1; } return 0; } GSList * pcmk_sort_nvpairs(GSList *list) { return g_slist_sort(list, pcmk__compare_nvpair); } GSList * pcmk_xml_attrs2nvpairs(const xmlNode *xml) { GSList *result = NULL; for (xmlAttrPtr iter = pcmk__xe_first_attr(xml); iter != NULL; iter = iter->next) { result = pcmk_prepend_nvpair(result, (const char *) iter->name, (const char *) pcmk__xml_attr_value(iter)); } return result; } static void pcmk__nvpair_add_xml_attr(gpointer data, gpointer user_data) { pcmk_nvpair_t *pair = data; xmlNode *parent = user_data; pcmk__xe_set(parent, pair->name, pair->value); } void pcmk_nvpairs2xml_attrs(GSList *list, xmlNode *xml) { g_slist_foreach(list, pcmk__nvpair_add_xml_attr, xml); } void hash2nvpair(gpointer key, gpointer value, gpointer user_data) { const char *name = key; const char *s_value = value; xmlNode *xml_node = user_data; crm_create_nvpair_xml(xml_node, name, name, s_value); pcmk__trace("dumped: name=%s value=%s", name, s_value); } // LCOV_EXCL_STOP // End deprecated API diff --git a/lib/common/remote.c b/lib/common/remote.c index 509f8ff61b..8a33d20223 100644 --- a/lib/common/remote.c +++ b/lib/common/remote.c @@ -1,1034 +1,1034 @@ /* * Copyright 2008-2025 the Pacemaker project contributors * * The version control history for this file may have further details. * * This source code is licensed under the GNU Lesser General Public License * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include // PRIx32 #include #include #include #include #include #include #include #include /* Swab macros from linux/swab.h */ #ifdef HAVE_LINUX_SWAB_H # include #else /* * casts are necessary for constants, because we never know how for sure * how U/UL/ULL map to __u16, __u32, __u64. At least not in a portable way. */ #define __swab16(x) ((uint16_t)( \ (((uint16_t)(x) & (uint16_t)0x00ffU) << 8) | \ (((uint16_t)(x) & (uint16_t)0xff00U) >> 8))) #define __swab32(x) ((uint32_t)( \ (((uint32_t)(x) & (uint32_t)0x000000ffUL) << 24) | \ (((uint32_t)(x) & (uint32_t)0x0000ff00UL) << 8) | \ (((uint32_t)(x) & (uint32_t)0x00ff0000UL) >> 8) | \ (((uint32_t)(x) & (uint32_t)0xff000000UL) >> 24))) #define __swab64(x) ((uint64_t)( \ (((uint64_t)(x) & (uint64_t)0x00000000000000ffULL) << 56) | \ (((uint64_t)(x) & (uint64_t)0x000000000000ff00ULL) << 40) | \ (((uint64_t)(x) & (uint64_t)0x0000000000ff0000ULL) << 24) | \ (((uint64_t)(x) & (uint64_t)0x00000000ff000000ULL) << 8) | \ (((uint64_t)(x) & (uint64_t)0x000000ff00000000ULL) >> 8) | \ (((uint64_t)(x) & (uint64_t)0x0000ff0000000000ULL) >> 24) | \ (((uint64_t)(x) & (uint64_t)0x00ff000000000000ULL) >> 40) | \ (((uint64_t)(x) & (uint64_t)0xff00000000000000ULL) >> 56))) #endif #define REMOTE_MSG_VERSION 1 #define ENDIAN_LOCAL 0xBADADBBD struct remote_header_v0 { uint32_t endian; /* Detect messages from hosts with different endian-ness */ uint32_t version; uint64_t id; uint64_t flags; uint32_t size_total; uint32_t payload_offset; uint32_t payload_compressed; uint32_t payload_uncompressed; /* New fields get added here */ } __attribute__ ((packed)); /*! * \internal * \brief Retrieve remote message header, in local endianness * * Return a pointer to the header portion of a remote connection's message * buffer, converting the header to local endianness if needed. * * \param[in,out] remote Remote connection with new message * * \return Pointer to message header, localized if necessary */ static struct remote_header_v0 * localized_remote_header(pcmk__remote_t *remote) { struct remote_header_v0 *header = (struct remote_header_v0 *)remote->buffer; if(remote->buffer_offset < sizeof(struct remote_header_v0)) { return NULL; } else if(header->endian != ENDIAN_LOCAL) { uint32_t endian = __swab32(header->endian); CRM_LOG_ASSERT(endian == ENDIAN_LOCAL); if(endian != ENDIAN_LOCAL) { pcmk__err("Invalid message detected, endian mismatch: %" PRIx32 " is neither %" PRIx32 " nor the swab'd %" PRIx32, ENDIAN_LOCAL, header->endian, endian); return NULL; } header->id = __swab64(header->id); header->flags = __swab64(header->flags); header->endian = __swab32(header->endian); header->version = __swab32(header->version); header->size_total = __swab32(header->size_total); header->payload_offset = __swab32(header->payload_offset); header->payload_compressed = __swab32(header->payload_compressed); header->payload_uncompressed = __swab32(header->payload_uncompressed); } return header; } // \return Standard Pacemaker return code static int send_tls(gnutls_session_t session, struct iovec *iov) { const char *unsent = iov->iov_base; size_t unsent_len = iov->iov_len; ssize_t gnutls_rc; if (unsent == NULL) { return EINVAL; } pcmk__trace("Sending TLS message of %zu bytes", unsent_len); while (true) { gnutls_rc = gnutls_record_send(session, unsent, unsent_len); if (gnutls_rc == GNUTLS_E_INTERRUPTED || gnutls_rc == GNUTLS_E_AGAIN) { pcmk__trace("Retrying to send %zu bytes remaining", unsent_len); } else if (gnutls_rc < 0) { // Caller can log as error if necessary pcmk__info("TLS connection terminated: %s " QB_XS " rc=%zd", gnutls_strerror((int) gnutls_rc), gnutls_rc); return ECONNABORTED; } else if (gnutls_rc < unsent_len) { pcmk__trace("Sent %zd of %zu bytes remaining", gnutls_rc, unsent_len); unsent_len -= gnutls_rc; unsent += gnutls_rc; } else { pcmk__trace("Sent all %zd bytes remaining", gnutls_rc); break; } } return pcmk_rc_ok; } // \return Standard Pacemaker return code static int send_plaintext(int sock, struct iovec *iov) { const char *unsent = iov->iov_base; size_t unsent_len = iov->iov_len; if (unsent == NULL) { return EINVAL; } pcmk__debug("Sending plaintext message of %zu bytes to socket %d", unsent_len, sock); while (true) { ssize_t write_rc = write(sock, unsent, unsent_len); if (write_rc < 0) { int rc = errno; if ((rc == EINTR) || (rc == EAGAIN) || (rc == EWOULDBLOCK)) { pcmk__trace("Retrying to send %zu bytes remaining to socket %d", unsent_len, sock); continue; } // Caller can log as error if necessary pcmk__info("Could not send message: %s " QB_XS " rc=%d socket=%d", pcmk_rc_str(rc), rc, sock); return rc; } else if (write_rc < unsent_len) { pcmk__trace("Sent %zd of %zu bytes remaining", write_rc, unsent_len); unsent += write_rc; unsent_len -= write_rc; } else { pcmk__trace("Sent all %zd bytes remaining: %.100s", write_rc, (const char *) iov->iov_base); return pcmk_rc_ok; } } } // \return Standard Pacemaker return code static int remote_send_iovs(pcmk__remote_t *remote, struct iovec *iov, int iovs) { int rc = pcmk_rc_ok; for (int lpc = 0; (lpc < iovs) && (rc == pcmk_rc_ok); lpc++) { if (remote->tls_session) { rc = send_tls(remote->tls_session, &(iov[lpc])); continue; } if (remote->tcp_socket >= 0) { rc = send_plaintext(remote->tcp_socket, &(iov[lpc])); } else { rc = ESOCKTNOSUPPORT; } } return rc; } /*! * \internal * \brief Send an XML message over a Pacemaker Remote connection * * \param[in,out] remote Pacemaker Remote connection to use * \param[in] msg XML to send * * \return Standard Pacemaker return code */ int pcmk__remote_send_xml(pcmk__remote_t *remote, const xmlNode *msg) { int rc = pcmk_rc_ok; static uint64_t id = 0; GString *xml_text = NULL; struct iovec iov[2]; struct remote_header_v0 *header; CRM_CHECK((remote != NULL) && (msg != NULL), return EINVAL); xml_text = g_string_sized_new(1024); pcmk__xml_string(msg, 0, xml_text, 0); CRM_CHECK(xml_text->len > 0, g_string_free(xml_text, TRUE); return EINVAL); header = pcmk__assert_alloc(1, sizeof(struct remote_header_v0)); iov[0].iov_base = header; iov[0].iov_len = sizeof(struct remote_header_v0); iov[1].iov_len = 1 + xml_text->len; iov[1].iov_base = g_string_free(xml_text, FALSE); id++; header->id = id; header->endian = ENDIAN_LOCAL; header->version = REMOTE_MSG_VERSION; header->payload_offset = iov[0].iov_len; header->payload_uncompressed = iov[1].iov_len; header->size_total = iov[0].iov_len + iov[1].iov_len; rc = remote_send_iovs(remote, iov, 2); if (rc != pcmk_rc_ok) { pcmk__err("Could not send remote message: %s " QB_XS " rc=%d", pcmk_rc_str(rc), rc); } free(iov[0].iov_base); g_free((gchar *) iov[1].iov_base); return rc; } /*! * \internal * \brief Obtain the XML from the currently buffered remote connection message * * \param[in,out] remote Remote connection possibly with message available * * \return Newly allocated XML object corresponding to message data, or NULL * \note This effectively removes the message from the connection buffer. */ xmlNode * pcmk__remote_message_xml(pcmk__remote_t *remote) { xmlNode *xml = NULL; struct remote_header_v0 *header = localized_remote_header(remote); if (header == NULL) { return NULL; } /* Support compression on the receiving end now, in case we ever want to add it later */ if (header->payload_compressed) { int rc = 0; unsigned int size_u = 1 + header->payload_uncompressed; char *uncompressed = pcmk__assert_alloc(1, header->payload_offset + size_u); pcmk__trace("Decompressing message data %d bytes into %d bytes", header->payload_compressed, size_u); rc = BZ2_bzBuffToBuffDecompress(uncompressed + header->payload_offset, &size_u, remote->buffer + header->payload_offset, header->payload_compressed, 1, 0); rc = pcmk__bzlib2rc(rc); if (rc != pcmk_rc_ok && header->version > REMOTE_MSG_VERSION) { pcmk__warn("Couldn't decompress v%d message, we only understand " "v%d", header->version, REMOTE_MSG_VERSION); free(uncompressed); return NULL; } else if (rc != pcmk_rc_ok) { pcmk__err("Decompression failed: %s " QB_XS " rc=%d", pcmk_rc_str(rc), rc); free(uncompressed); return NULL; } pcmk__assert(size_u == header->payload_uncompressed); memcpy(uncompressed, remote->buffer, header->payload_offset); /* Preserve the header */ remote->buffer_size = header->payload_offset + size_u; free(remote->buffer); remote->buffer = uncompressed; header = localized_remote_header(remote); } /* take ownership of the buffer */ remote->buffer_offset = 0; CRM_LOG_ASSERT(remote->buffer[sizeof(struct remote_header_v0) + header->payload_uncompressed - 1] == 0); xml = pcmk__xml_parse(remote->buffer + header->payload_offset); if (xml == NULL && header->version > REMOTE_MSG_VERSION) { pcmk__warn("Couldn't parse v%d message, we only understand v%d", header->version, REMOTE_MSG_VERSION); } else if (xml == NULL) { pcmk__err("Couldn't parse: '%.120s'", remote->buffer + header->payload_offset); } - crm_log_xml_trace(xml, "[remote msg]"); + pcmk__log_xml_trace(xml, "[remote msg]"); return xml; } static int get_remote_socket(const pcmk__remote_t *remote) { if (remote->tls_session != NULL) { return pcmk__tls_get_client_sock(remote); } if (remote->tcp_socket >= 0) { return remote->tcp_socket; } pcmk__err("Remote connection type undetermined (bug?)"); return -1; } /*! * \internal * \brief Wait for a remote session to have data to read * * \param[in] remote Connection to check * \param[in] timeout_ms Maximum time (in ms) to wait * * \return Standard Pacemaker return code (of particular interest, pcmk_rc_ok if * there is data ready to be read, and ETIME if there is no data within * the specified timeout) */ int pcmk__remote_ready(const pcmk__remote_t *remote, int timeout_ms) { struct pollfd fds = { 0, }; int sock = -1; int rc = 0; time_t start; int timeout = timeout_ms; sock = get_remote_socket(remote); if (sock < 0) { pcmk__trace("No longer connected"); return ENOTCONN; } start = time(NULL); errno = 0; do { fds.fd = sock; fds.events = POLLIN; /* If we got an EINTR while polling, and we have a * specific timeout we are trying to honor, attempt * to adjust the timeout to the closest second. */ if (errno == EINTR && (timeout > 0)) { timeout = timeout_ms - ((time(NULL) - start) * 1000); if (timeout < 1000) { timeout = 1000; } } rc = poll(&fds, 1, timeout); } while (rc < 0 && errno == EINTR); if (rc < 0) { return errno; } return (rc == 0)? ETIME : pcmk_rc_ok; } /*! * \internal * \brief Read bytes from non-blocking remote connection * * \param[in,out] remote Remote connection to read * * \return Standard Pacemaker return code (of particular interest, pcmk_rc_ok if * a full message has been received, or EAGAIN for a partial message) * \note Use only with non-blocking sockets after polling the socket. * \note This function will return when the socket read buffer is empty or an * error is encountered. */ int pcmk__read_available_remote_data(pcmk__remote_t *remote) { int rc = pcmk_rc_ok; size_t read_len = sizeof(struct remote_header_v0); struct remote_header_v0 *header = localized_remote_header(remote); ssize_t read_rc; if(header) { /* Stop at the end of the current message */ read_len = header->size_total; } /* automatically grow the buffer when needed */ if(remote->buffer_size < read_len) { remote->buffer_size = 2 * read_len; pcmk__trace("Expanding buffer to %zu bytes", remote->buffer_size); remote->buffer = pcmk__realloc(remote->buffer, remote->buffer_size + 1); } if (remote->tls_session) { read_rc = gnutls_record_recv(remote->tls_session, remote->buffer + remote->buffer_offset, remote->buffer_size - remote->buffer_offset); if (read_rc == GNUTLS_E_INTERRUPTED) { rc = EINTR; } else if (read_rc == GNUTLS_E_AGAIN) { rc = EAGAIN; } else if (read_rc < 0) { pcmk__debug("TLS receive failed: %s (%zd)", gnutls_strerror((int) read_rc), read_rc); rc = EIO; } } else if (remote->tcp_socket >= 0) { read_rc = read(remote->tcp_socket, remote->buffer + remote->buffer_offset, remote->buffer_size - remote->buffer_offset); if (read_rc < 0) { rc = errno; } } else { pcmk__err("Remote connection type undetermined (bug?)"); return ESOCKTNOSUPPORT; } /* process any errors. */ if (read_rc > 0) { remote->buffer_offset += read_rc; /* always null terminate buffer, the +1 to alloc always allows for this. */ remote->buffer[remote->buffer_offset] = '\0'; pcmk__trace("Received %zd more bytes (%zu total)", read_rc, remote->buffer_offset); } else if (read_rc == 0) { pcmk__debug("End of remote data encountered after %zu bytes", remote->buffer_offset); return ENOTCONN; } else if ((rc == EINTR) || (rc == EAGAIN) || (rc == EWOULDBLOCK)) { pcmk__trace("No data available for non-blocking remote read: %s (%d)", pcmk_rc_str(rc), rc); } else { pcmk__debug("Error receiving remote data after %zu bytes: %s (%d)", remote->buffer_offset, pcmk_rc_str(rc), rc); return ENOTCONN; } header = localized_remote_header(remote); if(header) { if(remote->buffer_offset < header->size_total) { pcmk__trace("Read partial remote message (%zu of %" PRIu32 " bytes)", remote->buffer_offset, header->size_total); } else { pcmk__trace("Read full remote message of %zu bytes", remote->buffer_offset); return pcmk_rc_ok; } } return EAGAIN; } /*! * \internal * \brief Read one message from a remote connection * * \param[in,out] remote Remote connection to read * \param[in] timeout_ms Fail if message not read in this many milliseconds * (10s will be used if 0, and 60s if negative) * * \return Standard Pacemaker return code */ int pcmk__read_remote_message(pcmk__remote_t *remote, int timeout_ms) { int rc = pcmk_rc_ok; time_t start = time(NULL); int remaining_timeout = 0; if (timeout_ms == 0) { timeout_ms = 10000; } else if (timeout_ms < 0) { timeout_ms = 60000; } remaining_timeout = timeout_ms; while (remaining_timeout > 0) { pcmk__trace("Waiting for remote data (%d ms of %d ms timeout " "remaining)", remaining_timeout, timeout_ms); rc = pcmk__remote_ready(remote, remaining_timeout); if (rc == ETIME) { pcmk__err("Timed out (%d ms) while waiting for remote data", remaining_timeout); return rc; } else if (rc != pcmk_rc_ok) { pcmk__debug("Wait for remote data aborted (will retry): %s " QB_XS " rc=%d", pcmk_rc_str(rc), rc); } else { rc = pcmk__read_available_remote_data(remote); if (rc == pcmk_rc_ok) { return rc; } else if (rc == EAGAIN) { pcmk__trace("Waiting for more remote data"); } else { pcmk__debug("Could not receive remote data: %s " QB_XS " rc=%d", pcmk_rc_str(rc), rc); } } // Don't waste time retrying after fatal errors if ((rc == ENOTCONN) || (rc == ESOCKTNOSUPPORT)) { return rc; } remaining_timeout = timeout_ms - ((time(NULL) - start) * 1000); } return ETIME; } struct tcp_async_cb_data { int sock; int timeout_ms; time_t start; void *userdata; void (*callback) (void *userdata, int rc, int sock); }; // \return TRUE if timer should be rescheduled, FALSE otherwise static gboolean check_connect_finished(gpointer userdata) { struct tcp_async_cb_data *cb_data = userdata; int rc; fd_set rset, wset; struct timeval ts = { 0, }; if (cb_data->start == 0) { // Last connect() returned success immediately rc = pcmk_rc_ok; goto dispatch_done; } // If the socket is ready for reading or writing, the connect succeeded FD_ZERO(&rset); FD_SET(cb_data->sock, &rset); wset = rset; rc = select(cb_data->sock + 1, &rset, &wset, NULL, &ts); if (rc < 0) { // select() error rc = errno; if ((rc == EINTR) || (rc == EAGAIN)) { if ((time(NULL) - cb_data->start) < pcmk__timeout_ms2s(cb_data->timeout_ms)) { return TRUE; // There is time left, so reschedule timer } else { rc = ETIMEDOUT; } } pcmk__trace("Could not check socket %d for connection success: %s (%d)", cb_data->sock, pcmk_rc_str(rc), rc); } else if (rc == 0) { // select() timeout if ((time(NULL) - cb_data->start) < pcmk__timeout_ms2s(cb_data->timeout_ms)) { return TRUE; // There is time left, so reschedule timer } pcmk__debug("Timed out while waiting for socket %d connection success", cb_data->sock); rc = ETIMEDOUT; // select() returned number of file descriptors that are ready } else if (FD_ISSET(cb_data->sock, &rset) || FD_ISSET(cb_data->sock, &wset)) { // The socket is ready; check it for connection errors int error = 0; socklen_t len = sizeof(error); if (getsockopt(cb_data->sock, SOL_SOCKET, SO_ERROR, &error, &len) < 0) { rc = errno; pcmk__trace("Couldn't check socket %d for connection errors: %s " "(%d)", cb_data->sock, pcmk_rc_str(rc), rc); } else if (error != 0) { rc = error; pcmk__trace("Socket %d connected with error: %s (%d)", cb_data->sock, pcmk_rc_str(rc), rc); } else { rc = pcmk_rc_ok; } } else { // Should not be possible pcmk__trace("select() succeeded, but socket %d not in resulting " "read/write sets", cb_data->sock); rc = EAGAIN; } dispatch_done: if (rc == pcmk_rc_ok) { pcmk__trace("Socket %d is connected", cb_data->sock); } else { close(cb_data->sock); cb_data->sock = -1; } if (cb_data->callback) { cb_data->callback(cb_data->userdata, rc, cb_data->sock); } free(cb_data); return FALSE; // Do not reschedule timer } /*! * \internal * \brief Attempt to connect socket, calling callback when done * * Set a given socket non-blocking, then attempt to connect to it, * retrying periodically until success or a timeout is reached. * Call a caller-supplied callback function when completed. * * \param[in] sock Newly created socket * \param[in] addr Socket address information for connect * \param[in] addrlen Size of socket address information in bytes * \param[in] timeout_ms Fail if not connected within this much time * \param[out] timer_id If not NULL, store retry timer ID here * \param[in] userdata User data to pass to callback * \param[in] callback Function to call when connection attempt completes * * \return Standard Pacemaker return code */ static int connect_socket_retry(int sock, const struct sockaddr *addr, socklen_t addrlen, int timeout_ms, int *timer_id, void *userdata, void (*callback) (void *userdata, int rc, int sock)) { int rc = 0; int interval = 500; int timer; struct tcp_async_cb_data *cb_data = NULL; rc = pcmk__set_nonblocking(sock); if (rc != pcmk_rc_ok) { pcmk__warn("Could not set socket non-blocking: %s " QB_XS " rc=%d", pcmk_rc_str(rc), rc); return rc; } rc = connect(sock, addr, addrlen); if (rc < 0) { rc = errno; switch (rc) { case EINTR: case EINPROGRESS: case EAGAIN: break; default: pcmk__warn("Could not connect socket: %s " QB_XS " rc=%d", pcmk_rc_str(rc), rc); return rc; } } cb_data = pcmk__assert_alloc(1, sizeof(struct tcp_async_cb_data)); cb_data->userdata = userdata; cb_data->callback = callback; cb_data->sock = sock; cb_data->timeout_ms = timeout_ms; if (rc == 0) { /* The connect was successful immediately, we still return to mainloop * and let this callback get called later. This avoids the user of this api * to have to account for the fact the callback could be invoked within this * function before returning. */ cb_data->start = 0; interval = 1; } else { cb_data->start = time(NULL); } /* This timer function does a non-blocking poll on the socket to see if we * can use it. Once we can, the connect has completed. This method allows us * to connect without blocking the mainloop. * * @TODO Use a mainloop fd callback for this instead of polling. Something * about the way mainloop is currently polling prevents this from * working at the moment though. (See connect(2) regarding EINPROGRESS * for possible new handling needed.) */ pcmk__trace("Scheduling check in %dms for whether connect to fd %d " "finished", interval, sock); timer = pcmk__create_timer(interval, check_connect_finished, cb_data); if (timer_id) { *timer_id = timer; } return pcmk_rc_ok; } /*! * \internal * \brief Attempt once to connect socket and set it non-blocking * * \param[in] sock Newly created socket * \param[in] addr Socket address information for connect * \param[in] addrlen Size of socket address information in bytes * * \return Standard Pacemaker return code */ static int connect_socket_once(int sock, const struct sockaddr *addr, socklen_t addrlen) { int rc = connect(sock, addr, addrlen); if (rc < 0) { rc = errno; pcmk__warn("Could not connect socket: %s " QB_XS " rc=%d", pcmk_rc_str(rc), rc); return rc; } rc = pcmk__set_nonblocking(sock); if (rc != pcmk_rc_ok) { pcmk__warn("Could not set socket non-blocking: %s " QB_XS " rc=%d", pcmk_rc_str(rc), rc); return rc; } return pcmk_ok; } /*! * \internal * \brief Connect to server at specified TCP port * * \param[in] host Name of server to connect to * \param[in] port Server port to connect to * \param[in] timeout_ms If asynchronous, fail if not connected in this time * \param[out] timer_id If asynchronous and this is non-NULL, retry timer ID * will be put here (for ease of cancelling by caller) * \param[out] sock_fd Where to store socket file descriptor * \param[in] userdata If asynchronous, data to pass to callback * \param[in] callback If NULL, attempt a single synchronous connection, * otherwise retry asynchronously then call this * * \return Standard Pacemaker return code */ int pcmk__connect_remote(const char *host, int port, int timeout, int *timer_id, int *sock_fd, void *userdata, void (*callback) (void *userdata, int rc, int sock)) { char buffer[INET6_ADDRSTRLEN]; struct addrinfo *res = NULL; struct addrinfo *rp = NULL; struct addrinfo hints; const char *server = host; int rc; int sock = -1; CRM_CHECK((host != NULL) && (sock_fd != NULL), return EINVAL); // Get host's IP address(es) memset(&hints, 0, sizeof(struct addrinfo)); hints.ai_family = AF_UNSPEC; /* Allow IPv4 or IPv6 */ hints.ai_socktype = SOCK_STREAM; hints.ai_flags = AI_CANONNAME; rc = getaddrinfo(server, NULL, &hints, &res); rc = pcmk__gaierror2rc(rc); if (rc != pcmk_rc_ok) { pcmk__err("Unable to get IP address info for %s: %s", server, pcmk_rc_str(rc)); goto async_cleanup; } if (!res || !res->ai_addr) { pcmk__err("Unable to get IP address info for %s: no result", server); rc = ENOTCONN; goto async_cleanup; } // getaddrinfo() returns a list of host's addresses, try them in order for (rp = res; rp != NULL; rp = rp->ai_next) { struct sockaddr *addr = rp->ai_addr; if (!addr) { continue; } if (rp->ai_canonname) { server = res->ai_canonname; } pcmk__debug("Got canonical name %s for %s", server, host); sock = socket(rp->ai_family, SOCK_STREAM, IPPROTO_TCP); if (sock == -1) { rc = errno; pcmk__warn("Could not create socket for remote connection to " "%s:%d: %s " QB_XS " rc=%d", server, port, pcmk_rc_str(rc), rc); continue; } /* Set port appropriately for address family */ /* (void*) casts avoid false-positive compiler alignment warnings */ if (addr->sa_family == AF_INET6) { ((struct sockaddr_in6 *)(void*)addr)->sin6_port = htons(port); } else { ((struct sockaddr_in *)(void*)addr)->sin_port = htons(port); } memset(buffer, 0, PCMK__NELEM(buffer)); pcmk__sockaddr2str(addr, buffer); pcmk__info("Attempting remote connection to %s:%d", buffer, port); if (callback) { if (connect_socket_retry(sock, rp->ai_addr, rp->ai_addrlen, timeout, timer_id, userdata, callback) == pcmk_rc_ok) { goto async_cleanup; /* Success for now, we'll hear back later in the callback */ } } else if (connect_socket_once(sock, rp->ai_addr, rp->ai_addrlen) == pcmk_rc_ok) { break; /* Success */ } // Connect failed close(sock); sock = -1; rc = ENOTCONN; } async_cleanup: if (res) { freeaddrinfo(res); } *sock_fd = sock; return rc; } /*! * \internal * \brief Convert an IP address (IPv4 or IPv6) to a string for logging * * \param[in] sa Socket address for IP * \param[out] s Storage for at least INET6_ADDRSTRLEN bytes * * \note sa The socket address can be a pointer to struct sockaddr_in (IPv4), * struct sockaddr_in6 (IPv6) or struct sockaddr_storage (either), * as long as its sa_family member is set correctly. */ void pcmk__sockaddr2str(const void *sa, char *s) { switch (((const struct sockaddr *) sa)->sa_family) { case AF_INET: inet_ntop(AF_INET, &(((const struct sockaddr_in *) sa)->sin_addr), s, INET6_ADDRSTRLEN); break; case AF_INET6: inet_ntop(AF_INET6, &(((const struct sockaddr_in6 *) sa)->sin6_addr), s, INET6_ADDRSTRLEN); break; default: strcpy(s, ""); } } /*! * \internal * \brief Accept a client connection on a remote server socket * * \param[in] ssock Server socket file descriptor being listened on * \param[out] csock Where to put new client socket's file descriptor * * \return Standard Pacemaker return code */ int pcmk__accept_remote_connection(int ssock, int *csock) { int rc; struct sockaddr_storage addr; socklen_t laddr = sizeof(addr); char addr_str[INET6_ADDRSTRLEN]; #ifdef TCP_USER_TIMEOUT long sbd_timeout = 0; #endif /* accept the connection */ memset(&addr, 0, sizeof(addr)); *csock = accept(ssock, (struct sockaddr *)&addr, &laddr); if (*csock == -1) { rc = errno; pcmk__err("Could not accept remote client connection: %s " QB_XS " rc=%d", pcmk_rc_str(rc), rc); return rc; } pcmk__sockaddr2str(&addr, addr_str); pcmk__info("Accepted new remote client connection from %s", addr_str); rc = pcmk__set_nonblocking(*csock); if (rc != pcmk_rc_ok) { pcmk__err("Could not set socket non-blocking: %s " QB_XS " rc=%d", pcmk_rc_str(rc), rc); close(*csock); *csock = -1; return rc; } #ifdef TCP_USER_TIMEOUT sbd_timeout = pcmk__get_sbd_watchdog_timeout(); if (sbd_timeout > 0) { // Time to fail and retry before watchdog long half = sbd_timeout / 2; unsigned int optval = (half <= UINT_MAX)? half : UINT_MAX; rc = setsockopt(*csock, SOL_TCP, TCP_USER_TIMEOUT, &optval, sizeof(optval)); if (rc < 0) { rc = errno; pcmk__err("Could not set TCP timeout to %d ms on remote " "connection: %s " QB_XS " rc=%d", optval, pcmk_rc_str(rc), rc); close(*csock); *csock = -1; return rc; } } #endif return rc; } /*! * \brief Get the default remote connection TCP port on this host * * \return Remote connection TCP port number */ int crm_default_remote_port(void) { static int port = 0; if (port == 0) { const char *env = pcmk__env_option(PCMK__ENV_REMOTE_PORT); if (env) { errno = 0; port = strtol(env, NULL, 10); if (errno || (port < 1) || (port > 65535)) { pcmk__warn("Environment variable PCMK_" PCMK__ENV_REMOTE_PORT " has invalid value '%s', using %d instead", env, DEFAULT_REMOTE_PORT); port = DEFAULT_REMOTE_PORT; } } else { port = DEFAULT_REMOTE_PORT; } } return port; } diff --git a/lib/common/xml_element.c b/lib/common/xml_element.c index 34dfd34e60..1b296cd402 100644 --- a/lib/common/xml_element.c +++ b/lib/common/xml_element.c @@ -1,1862 +1,1862 @@ /* * Copyright 2004-2025 the Pacemaker project contributors * * The version control history for this file may have further details. * * This source code is licensed under the GNU Lesser General Public License * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. */ #include #include // va_start(), etc. #include // uint32_t #include // NULL, etc. #include // free(), etc. #include // strchr(), etc. #include // time_t, etc. #include // xmlNode, etc. #include // xmlValidateNameValue() #include // xmlChar #include #include // pcmk_rc_ok, etc. #include // pcmk_parse_score() #include #include "crmcommon_private.h" /*! * \internal * \brief Find first XML child element matching given criteria * * \param[in] parent XML element to search (can be \c NULL) * \param[in] node_name If not \c NULL, only match children of this type * \param[in] attr_n If not \c NULL, only match children with an attribute * of this name. * \param[in] attr_v If \p attr_n and this are not NULL, only match children * with an attribute named \p attr_n and this value * * \return Matching XML child element, or \c NULL if none found */ xmlNode * pcmk__xe_first_child(const xmlNode *parent, const char *node_name, const char *attr_n, const char *attr_v) { xmlNode *child = NULL; CRM_CHECK((attr_v == NULL) || (attr_n != NULL), return NULL); if (parent == NULL) { return NULL; } child = parent->children; while ((child != NULL) && (child->type != XML_ELEMENT_NODE)) { child = child->next; } for (; child != NULL; child = pcmk__xe_next(child, NULL)) { const char *value = NULL; if ((node_name != NULL) && !pcmk__xe_is(child, node_name)) { // Node name mismatch continue; } if (attr_n == NULL) { // No attribute match needed return child; } value = pcmk__xe_get(child, attr_n); if ((attr_v == NULL) && (value != NULL)) { // attr_v == NULL: Attribute attr_n must be set (to any value) return child; } if ((attr_v != NULL) && (pcmk__str_eq(value, attr_v, pcmk__str_none))) { // attr_v != NULL: Attribute attr_n must be set to value attr_v return child; } } if (attr_n == NULL) { pcmk__trace("%s XML has no child element of %s type", parent->name, pcmk__s(node_name, "any")); } else { pcmk__trace("%s XML has no child element of %s type with %s='%s'", parent->name, pcmk__s(node_name, "any"), attr_n, attr_v); } return NULL; } /*! * \internal * \brief Return next sibling element of an XML element * * \param[in] xml XML element to check * \param[in] element_name If not NULL, get next sibling with this element name * * \return Next desired sibling of \p xml (or NULL if none) */ xmlNode * pcmk__xe_next(const xmlNode *xml, const char *element_name) { for (xmlNode *next = (xml == NULL)? NULL : xml->next; next != NULL; next = next->next) { if ((next->type == XML_ELEMENT_NODE) && ((element_name == NULL) || pcmk__xe_is(next, element_name))) { return next; } } return NULL; } /*! * \internal * \brief Parse an integer score from an XML attribute * * \param[in] xml XML element with attribute to parse * \param[in] name Name of attribute to parse * \param[out] score Where to store parsed score (can be NULL to * just validate) * \param[in] default_score What to return if the attribute value is not * present or invalid * * \return Standard Pacemaker return code */ int pcmk__xe_get_score(const xmlNode *xml, const char *name, int *score, int default_score) { const char *value = NULL; CRM_CHECK((xml != NULL) && (name != NULL), return EINVAL); value = pcmk__xe_get(xml, name); return pcmk_parse_score(value, score, default_score); } /*! * \internal * \brief Set an XML attribute, expanding \c ++ and \c += where appropriate * * If \p target already has an attribute named \p name set to an integer value * and \p value is an addition assignment expression on \p name, then expand * \p value to an integer and set attribute \p name to the expanded value in * \p target. * * Otherwise, set attribute \p name on \p target using the literal \p value. * * The original attribute value in \p target and the number in an assignment * expression in \p value are parsed and added as scores (that is, their values * are capped at \c INFINITY and \c -INFINITY). For more details, refer to * \c pcmk_parse_score(). * * For example, suppose \p target has an attribute named \c "X" with value * \c "5", and that \p name is \c "X". * * If \p value is \c "X++", the new value of \c "X" in \p target is \c "6". * * If \p value is \c "X+=3", the new value of \c "X" in \p target is \c "8". * * If \p value is \c "val", the new value of \c "X" in \p target is \c "val". * * If \p value is \c "Y++", the new value of \c "X" in \p target is \c "Y++". * * \param[in,out] target XML node whose attribute to set * \param[in] name Name of the attribute to set * \param[in] value New value of attribute to set (if NULL, initial value * will be left unchanged) * * \return Standard Pacemaker return code (specifically, \c EINVAL on invalid * argument, or \c pcmk_rc_ok otherwise) */ int pcmk__xe_set_score(xmlNode *target, const char *name, const char *value) { const char *old_value = NULL; CRM_CHECK((target != NULL) && (name != NULL), return EINVAL); if (value == NULL) { // @TODO Maybe instead delete the attribute or set it to 0 return pcmk_rc_ok; } old_value = pcmk__xe_get(target, name); // If no previous value, skip to default case and set the value unexpanded. if (old_value != NULL) { const char *n = name; const char *v = value; // Stop at first character that differs between name and value for (; (*n == *v) && (*n != '\0'); n++, v++); // If value begins with name followed by a "++" or "+=" if ((*n == '\0') && (*v++ == '+') && ((*v == '+') || (*v == '='))) { int add = 1; int old_value_i = 0; int rc = pcmk_rc_ok; // If we're expanding ourselves, no previous value was set; use 0 if (old_value != value) { rc = pcmk_parse_score(old_value, &old_value_i, 0); if (rc != pcmk_rc_ok) { // @TODO This is inconsistent with old_value==NULL pcmk__trace("Using 0 before incrementing %s because '%s' " "is not a score", name, old_value); } } /* value="X++": new value of X is old_value + 1 * value="X+=Y": new value of X is old_value + Y (for some number Y) */ if (*v != '+') { rc = pcmk_parse_score(++v, &add, 0); if (rc != pcmk_rc_ok) { // @TODO We should probably skip expansion instead pcmk__trace("Not incrementing %s because '%s' does not " "have a valid increment", name, value); } } pcmk__xe_set_int(target, name, pcmk__add_scores(old_value_i, add)); return pcmk_rc_ok; } } // Default case: set the attribute unexpanded (with value treated literally) if (old_value != value) { pcmk__xe_set(target, name, value); } return pcmk_rc_ok; } /*! * \internal * \brief Copy XML attributes from a source element to a target element * * This is similar to \c xmlCopyPropList() except that attributes are marked * as dirty for change tracking purposes. * * \param[in,out] target XML element to receive copied attributes from \p src * \param[in] src XML element whose attributes to copy to \p target * \param[in] flags Group of enum pcmk__xa_flags * * \return Standard Pacemaker return code */ int pcmk__xe_copy_attrs(xmlNode *target, const xmlNode *src, uint32_t flags) { CRM_CHECK((src != NULL) && (target != NULL), return EINVAL); for (xmlAttr *attr = pcmk__xe_first_attr(src); attr != NULL; attr = attr->next) { const char *name = (const char *) attr->name; const char *value = pcmk__xml_attr_value(attr); if (pcmk__is_set(flags, pcmk__xaf_no_overwrite) && (pcmk__xe_get(target, name) != NULL)) { continue; } if (pcmk__is_set(flags, pcmk__xaf_score_update)) { pcmk__xe_set_score(target, name, value); } else { pcmk__xe_set(target, name, value); } } return pcmk_rc_ok; } /*! * \internal * \brief Compare two XML attributes by name * * \param[in] a First XML attribute to compare * \param[in] b Second XML attribute to compare * * \retval negative \c a->name is \c NULL or comes before \c b->name * lexicographically * \retval 0 \c a->name and \c b->name are equal * \retval positive \c b->name is \c NULL or comes before \c a->name * lexicographically */ static gint compare_xml_attr(gconstpointer a, gconstpointer b) { const xmlAttr *attr_a = a; const xmlAttr *attr_b = b; return pcmk__strcmp((const char *) attr_a->name, (const char *) attr_b->name, pcmk__str_none); } /*! * \internal * \brief Sort an XML element's attributes by name * * This does not consider ACLs and does not mark the attributes as deleted or * dirty. Upon return, all attributes still exist and are set to the same values * as before the call. The only thing that may change is the order of the * attribute list. * * \param[in,out] xml XML element whose attributes to sort */ void pcmk__xe_sort_attrs(xmlNode *xml) { GSList *attr_list = NULL; for (xmlAttr *iter = pcmk__xe_first_attr(xml); iter != NULL; iter = iter->next) { attr_list = g_slist_prepend(attr_list, iter); } attr_list = g_slist_sort(attr_list, compare_xml_attr); for (GSList *iter = attr_list; iter != NULL; iter = iter->next) { xmlNode *attr = iter->data; xmlUnlinkNode(attr); xmlAddChild(xml, attr); } g_slist_free(attr_list); } /*! * \internal * \brief Remove a named attribute from an XML element * * \param[in,out] element XML element to remove an attribute from * \param[in] name Name of attribute to remove */ void pcmk__xe_remove_attr(xmlNode *element, const char *name) { if (name != NULL) { pcmk__xa_remove(xmlHasProp(element, (const xmlChar *) name), false); } } /*! * \internal * \brief Remove a named attribute from an XML element * * This is a wrapper for \c pcmk__xe_remove_attr() for use with * \c pcmk__xml_tree_foreach(). * * \param[in,out] xml XML element to remove an attribute from * \param[in] user_data Name of attribute to remove * * \return \c true (to continue traversing the tree) * * \note This is compatible with \c pcmk__xml_tree_foreach(). */ bool pcmk__xe_remove_attr_cb(xmlNode *xml, void *user_data) { const char *name = user_data; pcmk__xe_remove_attr(xml, name); return true; } /*! * \internal * \brief Remove an XML element's attributes that match some criteria * * \param[in,out] element XML element to modify * \param[in] force If \c true, remove matching attributes immediately, * ignoring ACLs and change tracking * \param[in] match If not NULL, only remove attributes for which * this function returns true * \param[in,out] user_data Data to pass to \p match */ void pcmk__xe_remove_matching_attrs(xmlNode *element, bool force, bool (*match)(xmlAttrPtr, void *), void *user_data) { xmlAttrPtr next = NULL; for (xmlAttrPtr a = pcmk__xe_first_attr(element); a != NULL; a = next) { next = a->next; // Grab now because attribute might get removed if ((match == NULL) || match(a, user_data)) { if (pcmk__xa_remove(a, force) != pcmk_rc_ok) { return; } } } } /*! * \internal * \brief Create a new XML element under a given parent * * \param[in,out] parent XML element that will be the new element's parent * (\c NULL to create a new XML document with the new * node as root) * \param[in] name Name of new element * * \return Newly created XML element (guaranteed not to be \c NULL) */ xmlNode * pcmk__xe_create(xmlNode *parent, const char *name) { xmlNode *node = NULL; pcmk__assert(!pcmk__str_empty(name)); if (parent == NULL) { xmlDoc *doc = pcmk__xml_new_doc(); node = xmlNewDocRawNode(doc, NULL, (const xmlChar *) name, NULL); pcmk__mem_assert(node); xmlDocSetRootElement(doc, node); } else { node = xmlNewChild(parent, NULL, (const xmlChar *) name, NULL); pcmk__mem_assert(node); } pcmk__xml_new_private_data(node); return node; } /*! * \internal * \brief Set a formatted string as an XML node's content * * \param[in,out] node Node whose content to set * \param[in] format printf(3)-style format string * \param[in] ... Arguments for \p format * * \note This function escapes special characters. \c xmlNodeSetContent() does * not. */ G_GNUC_PRINTF(2, 3) void pcmk__xe_set_content(xmlNode *node, const char *format, ...) { if (node != NULL) { const char *content = NULL; char *buf = NULL; /* xmlNodeSetContent() frees node->children and replaces it with new * text. If this function is called for a node that already has a non- * text child, it's a bug. */ CRM_CHECK((node->children == NULL) || (node->children->type == XML_TEXT_NODE), return); if (strchr(format, '%') == NULL) { // Nothing to format content = format; } else { va_list ap; va_start(ap, format); if (pcmk__str_eq(format, "%s", pcmk__str_none)) { // No need to make a copy content = va_arg(ap, const char *); } else { pcmk__assert(vasprintf(&buf, format, ap) >= 0); content = buf; } va_end(ap); } xmlNodeSetContent(node, (const xmlChar *) content); free(buf); } } /*! * \internal * \brief Set a formatted string as an XML element's ID * * If the formatted string would not be a valid ID, it's first sanitized by * \c pcmk__xml_sanitize_id(). * * \param[in,out] node Node whose ID to set * \param[in] format printf(3)-style format string * \param[in] ... Arguments for \p format */ G_GNUC_PRINTF(2, 3) void pcmk__xe_set_id(xmlNode *node, const char *format, ...) { char *id = NULL; va_list ap; pcmk__assert(!pcmk__str_empty(format)); if (node == NULL) { return; } va_start(ap, format); pcmk__assert(vasprintf(&id, format, ap) >= 0); va_end(ap); if (!xmlValidateNameValue((const xmlChar *) id)) { pcmk__xml_sanitize_id(id); } pcmk__xe_set(node, PCMK_XA_ID, id); free(id); } /*! * \internal * \brief Add a "last written" attribute to an XML element, set to current time * * \param[in,out] xe XML element to add attribute to * * \return Value that was set, or NULL on error */ const char * pcmk__xe_add_last_written(xmlNode *xe) { char *now_s = pcmk__epoch2str(NULL, 0); pcmk__xe_set(xe, PCMK_XA_CIB_LAST_WRITTEN, pcmk__s(now_s, "Could not determine current time")); free(now_s); return pcmk__xe_get(xe, PCMK_XA_CIB_LAST_WRITTEN); } /*! * \internal * \brief Merge one XML tree into another * * Here, "merge" means: * 1. Copy attribute values from \p update to the target, overwriting in case of * conflict. * 2. Descend through \p update and the target in parallel. At each level, for * each child of \p update, look for a matching child of the target. * a. For each child, if a match is found, go to step 1, recursively merging * the child of \p update into the child of the target. * b. Otherwise, copy the child of \p update as a child of the target. * * A match is defined as the first child of the same type within the target, * with: * * the \c PCMK_XA_ID attribute matching, if set in \p update; otherwise, * * the \c PCMK_XA_ID_REF attribute matching, if set in \p update * * This function does not delete any elements or attributes from the target. It * may add elements or overwrite attributes, as described above. * * \param[in,out] parent If \p target is NULL and this is not, add or update * child of this XML node that matches \p update * \param[in,out] target If not NULL, update this XML * \param[in] update Make the desired XML match this (must not be \c NULL) * \param[in] flags Group of enum pcmk__xa_flags * * \note At least one of \p parent and \p target must be non-NULL. * \note This function is recursive. For the top-level call, \p parent is * \c NULL and \p target is not \c NULL. For recursive calls, \p target is * \c NULL and \p parent is not \c NULL. */ static void update_xe(xmlNode *parent, xmlNode *target, xmlNode *update, uint32_t flags) { // @TODO Try to refactor further, possibly using pcmk__xml_tree_foreach() const char *update_name = NULL; const char *update_id_attr = NULL; const char *update_id_val = NULL; char *trace_s = NULL; - crm_log_xml_trace(update, "update"); - crm_log_xml_trace(target, "target"); + pcmk__log_xml_trace(update, "update"); + pcmk__log_xml_trace(target, "target"); CRM_CHECK(update != NULL, goto done); if (update->type == XML_COMMENT_NODE) { pcmk__xc_update(parent, target, update); goto done; } update_name = (const char *) update->name; CRM_CHECK(update_name != NULL, goto done); CRM_CHECK((target != NULL) || (parent != NULL), goto done); update_id_val = pcmk__xe_id(update); if (update_id_val != NULL) { update_id_attr = PCMK_XA_ID; } else { update_id_val = pcmk__xe_get(update, PCMK_XA_ID_REF); if (update_id_val != NULL) { update_id_attr = PCMK_XA_ID_REF; } } pcmk__if_tracing( { if (update_id_attr != NULL) { trace_s = pcmk__assert_asprintf("<%s %s=%s/>", update_name, update_id_attr, update_id_val); } else { trace_s = pcmk__assert_asprintf("<%s/>", update_name); } }, {} ); if (target == NULL) { // Recursive call target = pcmk__xe_first_child(parent, update_name, update_id_attr, update_id_val); } if (target == NULL) { // Recursive call with no existing matching child target = pcmk__xe_create(parent, update_name); pcmk__trace("Added %s", pcmk__s(trace_s, update_name)); } else { // Either recursive call with match, or top-level call pcmk__trace("Found node %s to update", pcmk__s(trace_s, update_name)); } CRM_CHECK(pcmk__xe_is(target, (const char *) update->name), return); pcmk__xe_copy_attrs(target, update, flags); for (xmlNode *child = pcmk__xml_first_child(update); child != NULL; child = pcmk__xml_next(child)) { pcmk__trace("Updating child of %s", pcmk__s(trace_s, update_name)); update_xe(target, NULL, child, flags); } pcmk__trace("Finished with %s", pcmk__s(trace_s, update_name)); done: free(trace_s); } /*! * \internal * \brief Delete an XML subtree if it matches a search element * * A match is defined as follows: * * \p xml and \p user_data are both element nodes of the same type. * * If \p user_data has attributes set, \p xml has those attributes set to the * same values. (\p xml may have additional attributes set to arbitrary * values.) * * \param[in,out] xml XML subtree to delete upon match * \param[in] user_data Search element * * \return \c true to continue traversing the tree, or \c false to stop (because * \p xml was deleted) * * \note This is compatible with \c pcmk__xml_tree_foreach(). */ static bool delete_xe_if_matching(xmlNode *xml, void *user_data) { xmlNode *search = user_data; if (!pcmk__xe_is(search, (const char *) xml->name)) { // No match: either not both elements, or different element types return true; } for (const xmlAttr *attr = pcmk__xe_first_attr(search); attr != NULL; attr = attr->next) { const char *search_val = pcmk__xml_attr_value(attr); const char *xml_val = pcmk__xe_get(xml, (const char *) attr->name); if (!pcmk__str_eq(search_val, xml_val, pcmk__str_casei)) { // No match: an attr in xml doesn't match the attr in search return true; } } - crm_log_xml_trace(xml, "delete-match"); - crm_log_xml_trace(search, "delete-search"); + pcmk__log_xml_trace(xml, "delete-match"); + pcmk__log_xml_trace(search, "delete-search"); pcmk__xml_free(xml); // Found a match and deleted it; stop traversing tree return false; } /*! * \internal * \brief Search an XML tree depth-first and delete the first matching element * * This function does not attempt to match the tree root (\p xml). * * A match with a node \c node is defined as follows: * * \c node and \p search are both element nodes of the same type. * * If \p search has attributes set, \c node has those attributes set to the * same values. (\c node may have additional attributes set to arbitrary * values.) * * \param[in,out] xml XML subtree to search * \param[in] search Element to match against * * \return Standard Pacemaker return code (specifically, \c pcmk_rc_ok on * successful deletion and an error code otherwise) */ int pcmk__xe_delete_match(xmlNode *xml, xmlNode *search) { // See @COMPAT comment in pcmk__xe_replace_match() CRM_CHECK((xml != NULL) && (search != NULL), return EINVAL); for (xml = pcmk__xe_first_child(xml, NULL, NULL, NULL); xml != NULL; xml = pcmk__xe_next(xml, NULL)) { if (!pcmk__xml_tree_foreach(xml, delete_xe_if_matching, search)) { // Found and deleted an element return pcmk_rc_ok; } } // No match found in this subtree return ENXIO; } /*! * \internal * \brief Replace one XML node with a copy of another XML node * * This function handles change tracking and applies ACLs. * * \param[in,out] old XML node to replace * \param[in] new XML node to copy as replacement for \p old * * \note This frees \p old. */ static void replace_node(xmlNode *old, xmlNode *new) { // Pass old for its doc; it won't remain the parent of new new = pcmk__xml_copy(old, new); old = xmlReplaceNode(old, new); // old == NULL means memory allocation error pcmk__assert(old != NULL); // May be unnecessary but avoids slight changes to some test outputs pcmk__xml_tree_foreach(new, pcmk__xml_reset_node_flags, NULL); if (pcmk__xml_doc_all_flags_set(new->doc, pcmk__xf_tracking)) { // Replaced sections may have included relevant ACLs pcmk__apply_acl(new); } pcmk__xml_mark_changes(old, new); pcmk__xml_free_node(old); } /*! * \internal * \brief Replace one XML subtree with a copy of another if the two match * * A match is defined as follows: * * \p xml and \p user_data are both element nodes of the same type. * * If \p user_data has the \c PCMK_XA_ID attribute set, then \p xml has * \c PCMK_XA_ID set to the same value. * * \param[in,out] xml XML subtree to replace with \p user_data upon match * \param[in] user_data XML to replace \p xml with a copy of upon match * * \return \c true to continue traversing the tree, or \c false to stop (because * \p xml was replaced by \p user_data) * * \note This is compatible with \c pcmk__xml_tree_foreach(). */ static bool replace_xe_if_matching(xmlNode *xml, void *user_data) { xmlNode *replace = user_data; const char *xml_id = NULL; const char *replace_id = NULL; xml_id = pcmk__xe_id(xml); replace_id = pcmk__xe_id(replace); if (!pcmk__xe_is(replace, (const char *) xml->name)) { // No match: either not both elements, or different element types return true; } if ((replace_id != NULL) && !pcmk__str_eq(replace_id, xml_id, pcmk__str_none)) { // No match: ID was provided in replace and doesn't match xml's ID return true; } - crm_log_xml_trace(xml, "replace-match"); - crm_log_xml_trace(replace, "replace-with"); + pcmk__log_xml_trace(xml, "replace-match"); + pcmk__log_xml_trace(replace, "replace-with"); replace_node(xml, replace); // Found a match and replaced it; stop traversing tree return false; } /*! * \internal * \brief Search an XML tree depth-first and replace the first matching element * * This function does not attempt to match the tree root (\p xml). * * A match with a node \c node is defined as follows: * * \c node and \p replace are both element nodes of the same type. * * If \p replace has the \c PCMK_XA_ID attribute set, then \c node has * \c PCMK_XA_ID set to the same value. * * \param[in,out] xml XML tree to search * \param[in] replace XML to replace a matching element with a copy of * * \return Standard Pacemaker return code (specifically, \c pcmk_rc_ok on * successful replacement and an error code otherwise) */ int pcmk__xe_replace_match(xmlNode *xml, xmlNode *replace) { /* @COMPAT Some of this behavior (like not matching the tree root, which is * allowed by pcmk__xe_update_match()) is questionable for general use but * required for backward compatibility by cib_process_replace() and * cib_process_delete(). Behavior can change at a major version release if * desired. */ CRM_CHECK((xml != NULL) && (replace != NULL), return EINVAL); for (xml = pcmk__xe_first_child(xml, NULL, NULL, NULL); xml != NULL; xml = pcmk__xe_next(xml, NULL)) { if (!pcmk__xml_tree_foreach(xml, replace_xe_if_matching, replace)) { // Found and replaced an element return pcmk_rc_ok; } } // No match found in this subtree return ENXIO; } //! User data for \c update_xe_if_matching() struct update_data { xmlNode *update; //!< Update source uint32_t flags; //!< Group of enum pcmk__xa_flags }; /*! * \internal * \brief Update one XML subtree with another if the two match * * "Update" means to merge a source subtree into a target subtree (see * \c update_xe()). * * A match is defined as follows: * * \p xml and \p user_data->update are both element nodes of the same type. * * \p xml and \p user_data->update have the same \c PCMK_XA_ID attribute * value, or \c PCMK_XA_ID is unset in both * * \param[in,out] xml XML subtree to update with \p user_data->update * upon match * \param[in] user_data struct update_data object * * \return \c true to continue traversing the tree, or \c false to stop (because * \p xml was updated by \p user_data->update) * * \note This is compatible with \c pcmk__xml_tree_foreach(). */ static bool update_xe_if_matching(xmlNode *xml, void *user_data) { struct update_data *data = user_data; xmlNode *update = data->update; if (!pcmk__xe_is(update, (const char *) xml->name)) { // No match: either not both elements, or different element types return true; } if (!pcmk__str_eq(pcmk__xe_id(xml), pcmk__xe_id(update), pcmk__str_none)) { // No match: ID mismatch return true; } - crm_log_xml_trace(xml, "update-match"); - crm_log_xml_trace(update, "update-with"); + pcmk__log_xml_trace(xml, "update-match"); + pcmk__log_xml_trace(update, "update-with"); update_xe(NULL, xml, update, data->flags); // Found a match and replaced it; stop traversing tree return false; } /*! * \internal * \brief Search an XML tree depth-first and update the first matching element * * "Update" means to merge a source subtree into a target subtree (see * \c update_xe()). * * A match with a node \c node is defined as follows: * * \c node and \p update are both element nodes of the same type. * * \c node and \p update have the same \c PCMK_XA_ID attribute value, or * \c PCMK_XA_ID is unset in both * * \param[in,out] xml XML tree to search * \param[in] update XML to update a matching element with * \param[in] flags Group of enum pcmk__xa_flags * * \return Standard Pacemaker return code (specifically, \c pcmk_rc_ok on * successful update and an error code otherwise) */ int pcmk__xe_update_match(xmlNode *xml, xmlNode *update, uint32_t flags) { /* @COMPAT In pcmk__xe_delete_match() and pcmk__xe_replace_match(), we * compare IDs only if the equivalent of the update argument has an ID. * Here, we're stricter: we consider it a mismatch if only one element has * an ID attribute, or if both elements have IDs but they don't match. * * Perhaps we should align the behavior at a major version release. */ struct update_data data = { .update = update, .flags = flags, }; CRM_CHECK((xml != NULL) && (update != NULL), return EINVAL); if (!pcmk__xml_tree_foreach(xml, update_xe_if_matching, &data)) { // Found and updated an element return pcmk_rc_ok; } // No match found in this subtree return ENXIO; } void pcmk__xe_set_propv(xmlNodePtr node, va_list pairs) { while (true) { const char *name, *value; name = va_arg(pairs, const char *); if (name == NULL) { return; } value = va_arg(pairs, const char *); pcmk__xe_set(node, name, value); } } void pcmk__xe_set_props(xmlNodePtr node, ...) { va_list pairs; va_start(pairs, node); pcmk__xe_set_propv(node, pairs); va_end(pairs); } int pcmk__xe_foreach_child(xmlNode *xml, const char *child_element_name, int (*handler)(xmlNode *xml, void *userdata), void *userdata) { xmlNode *children = (xml? xml->children : NULL); pcmk__assert(handler != NULL); for (xmlNode *node = children; node != NULL; node = node->next) { if ((node->type == XML_ELEMENT_NODE) && ((child_element_name == NULL) || pcmk__xe_is(node, child_element_name))) { int rc = handler(node, userdata); if (rc != pcmk_rc_ok) { return rc; } } } return pcmk_rc_ok; } // XML attribute handling /*! * \internal * \brief Retrieve the value of an XML attribute * * \param[in] xml XML element whose attribute to get * \param[in] attr_name Attribute name * * \return Value of specified attribute (may be \c NULL) */ const char * pcmk__xe_get(const xmlNode *xml, const char *attr_name) { xmlAttr *attr = NULL; CRM_CHECK((xml != NULL) && (attr_name != NULL), return NULL); attr = xmlHasProp(xml, (const xmlChar *) attr_name); if ((attr == NULL) || (attr->children == NULL)) { return NULL; } return (const char *) attr->children->content; } /*! * \internal * \brief Set an XML attribute value * * This also performs change tracking if enabled and checks ACLs if enabled. * Upon ACL denial, the attribute is not updated. * * \param[in,out] xml XML element whose attribute to set * \param[in] attr_name Attribute name * \param[in] value Attribute value to set * * \return Standard Pacemaker return code * * \note This does nothing and returns \c pcmk_rc_ok if \p value is \c NULL. */ int pcmk__xe_set(xmlNode *xml, const char *attr_name, const char *value) { bool dirty = false; xmlAttr *attr = NULL; CRM_CHECK((xml != NULL) && (attr_name != NULL), return EINVAL); if (value == NULL) { return pcmk_rc_ok; } /* @TODO Can we return early if dirty is false? Or does anything rely on * "cleanly" resetting the value? If so, try to preserve existing behavior * for public API functions that depend on this. */ if (pcmk__xml_doc_all_flags_set(xml->doc, pcmk__xf_tracking)) { const char *old_value = pcmk__xe_get(xml, attr_name); if (!pcmk__str_eq(old_value, value, pcmk__str_none)) { dirty = true; } } if (dirty && !pcmk__check_acl(xml, attr_name, pcmk__xf_acl_create)) { pcmk__trace("Cannot add %s=%s to %s", attr_name, value, xml->name); return EACCES; } attr = xmlSetProp(xml, (const xmlChar *) attr_name, (const xmlChar *) value); // These should never be NULL -- out of memory? CRM_CHECK((attr != NULL) && (attr->children != NULL) && (attr->children->content != NULL), xmlRemoveProp(attr); return ENXIO); /* If the attribute already exists, this does nothing. Attribute values * don't get private data. */ pcmk__xml_new_private_data((xmlNode *) attr); if (dirty) { pcmk__mark_xml_attr_dirty(attr); } return pcmk_rc_ok; } /*! * \internal * \brief Retrieve a flag group from an XML attribute value * * This is like \c pcmk__xe_get() but returns the value as a \c uint32_t. * * \param[in] xml XML node to check * \param[in] name Attribute name to check (must not be NULL) * \param[out] dest Where to store flags (may be NULL to just * validate type) * \param[in] default_value What to use for missing or invalid value * * \return Standard Pacemaker return code */ int pcmk__xe_get_flags(const xmlNode *xml, const char *name, uint32_t *dest, uint32_t default_value) { const char *value = NULL; long long value_ll = 0LL; int rc = pcmk_rc_ok; if (dest != NULL) { *dest = default_value; } if (name == NULL) { return EINVAL; } if (xml == NULL) { return pcmk_rc_ok; } value = pcmk__xe_get(xml, name); if (value == NULL) { return pcmk_rc_ok; } rc = pcmk__scan_ll(value, &value_ll, default_value); if ((value_ll < 0) || (value_ll > UINT32_MAX)) { value_ll = default_value; if (rc == pcmk_rc_ok) { rc = pcmk_rc_bad_input; } } if (dest != NULL) { *dest = (uint32_t) value_ll; } return rc; } /*! * \internal * \brief Retrieve a \c guint value from an XML attribute * * This is like \c pcmk__xe_get() but returns the value as a \c guint. * * \param[in] xml XML element whose attribute to get * \param[in] attr Attribute name * \param[out] dest Where to store attribute value (unchanged on error) * * \return Standard Pacemaker return code */ int pcmk__xe_get_guint(const xmlNode *xml, const char *attr, guint *dest) { long long value_ll = 0; int rc = pcmk_rc_ok; CRM_CHECK((xml != NULL) && (attr != NULL) && (dest != NULL), return EINVAL); rc = pcmk__xe_get_ll(xml, attr, &value_ll); if (rc != pcmk_rc_ok) { return rc; } if ((value_ll < 0) || (value_ll > G_MAXUINT)) { return ERANGE; } *dest = (guint) value_ll; return pcmk_rc_ok; } /*! * \internal * \brief Set an XML attribute using a \c guint value * * This is like \c pcmk__xe_set() but takes a \c guint. * * \param[in,out] xml XML node to modify * \param[in] attr Attribute name * \param[in] value Attribute value to set */ void pcmk__xe_set_guint(xmlNode *xml, const char *attr, guint value) { char *value_s = NULL; CRM_CHECK((xml != NULL) && (attr != NULL), return); value_s = pcmk__assert_asprintf("%u", value); pcmk__xe_set(xml, attr, value_s); free(value_s); } /*! * \internal * \brief Retrieve an \c int value from an XML attribute * * This is like \c pcmk__xe_get() but returns the value as an \c int. * * \param[in] xml XML element whose attribute to get * \param[in] attr Attribute name * \param[out] dest Where to store element value (unchanged on error) * * \return Standard Pacemaker return code */ int pcmk__xe_get_int(const xmlNode *xml, const char *attr, int *dest) { long long value_ll = 0; int rc = pcmk_rc_ok; CRM_CHECK((xml != NULL) && (attr != NULL) && (dest != NULL), return EINVAL); rc = pcmk__xe_get_ll(xml, attr, &value_ll); if (rc != pcmk_rc_ok) { return rc; } if ((value_ll < INT_MIN) || (value_ll > INT_MAX)) { return ERANGE; } *dest = (int) value_ll; return pcmk_rc_ok; } /*! * \internal * \brief Set an XML attribute using an \c int value. * * This is like \c pcmk__xe_set() but takes an \c int. * * \param[in,out] xml XML node to modify * \param[in] attr Attribute name * \param[in] value Attribute value to set */ void pcmk__xe_set_int(xmlNode *xml, const char *attr, int value) { char *value_s = NULL; CRM_CHECK((xml != NULL) && (attr != NULL), return); value_s = pcmk__itoa(value); pcmk__xe_set(xml, attr, value_s); free(value_s); } /*! * \internal * \brief Retrieve a long long value from an XML attribute * * This is like \c pcmk__xe_get() but returns the value as a long long. * * \param[in] xml XML element whose attribute to get * \param[in] attr Attribute name * \param[out] dest Where to store element value (unchanged on error) * * \return Standard Pacemaker return code */ int pcmk__xe_get_ll(const xmlNode *xml, const char *attr, long long *dest) { const char *value = NULL; long long value_ll = 0; int rc = pcmk_rc_ok; CRM_CHECK((xml != NULL) && (attr != NULL) && (dest != NULL), return EINVAL); value = pcmk__xe_get(xml, attr); if (value == NULL) { return ENXIO; } rc = pcmk__scan_ll(value, &value_ll, PCMK__PARSE_INT_DEFAULT); if (rc != pcmk_rc_ok) { return rc; } *dest = value_ll; return pcmk_rc_ok; } /*! * \internal * \brief Set an XML attribute using a long long value * * This is like \c pcmk__xe_set() but takes a long long. * * \param[in,out] xml XML node to modify * \param[in] attr Attribute name * \param[in] value Attribute value to set * * \return Standard Pacemaker return code */ int pcmk__xe_set_ll(xmlNode *xml, const char *attr, long long value) { char *value_s = NULL; int rc = pcmk_rc_ok; CRM_CHECK((xml != NULL) && (attr != NULL), return EINVAL); value_s = pcmk__assert_asprintf("%lld", value); rc = pcmk__xe_set(xml, attr, value_s); free(value_s); return rc; } /*! * \internal * \brief Retrieve a \c time_t value from an XML attribute * * This is like \c pcmk__xe_get() but returns the value as a \c time_t. * * \param[in] xml XML element whose attribute to get * \param[in] attr Attribute name * \param[out] dest Where to store attribute value (unchanged on error) * * \return Standard Pacemaker return code */ int pcmk__xe_get_time(const xmlNode *xml, const char *attr, time_t *dest) { long long value_ll = 0; int rc = pcmk_rc_ok; CRM_CHECK((xml != NULL) && (attr != NULL) && (dest != NULL), return EINVAL); rc = pcmk__xe_get_ll(xml, attr, &value_ll); if (rc != pcmk_rc_ok) { return rc; } /* We don't do any bounds checking, since there are no constants provided * for the bounds of time_t, and calculating them isn't worth the effort. If * there are XML values beyond the native sizes, there will likely be worse * problems anyway. */ *dest = (time_t) value_ll; return pcmk_rc_ok; } /*! * \internal * \brief Set an XML attribute using a \c time_t value * * This is like \c pcmk__xe_set() but takes a \c time_t. * * \param[in,out] xml XML element whose attribute to set * \param[in] attr Attribute name * \param[in] value Attribute value to set (in seconds) */ void pcmk__xe_set_time(xmlNode *xml, const char *attr, time_t value) { // Could be inline, but keep it underneath pcmk__xe_get_time() CRM_CHECK((xml != NULL) && (attr != NULL), return); pcmk__xe_set_ll(xml, attr, (long long) value); } /*! * \internal * \brief Retrieve the values of XML second/microsecond attributes as time * * This is like \c pcmk__xe_get() but returns the value as a * struct timeval. * * \param[in] xml XML element whose attributes to get * \param[in] sec_attr Name of XML attribute for seconds * \param[in] usec_attr Name of XML attribute for microseconds * \param[out] dest Where to store result (unchanged on error) * * \return Standard Pacemaker return code */ int pcmk__xe_get_timeval(const xmlNode *xml, const char *sec_attr, const char *usec_attr, struct timeval *dest) { long long value_ll = 0; struct timeval result = { 0, 0 }; int rc = pcmk_rc_ok; // Could allow one of sec_attr and usec_attr to be NULL in the future CRM_CHECK((xml != NULL) && (sec_attr != NULL) && (usec_attr != NULL) && (dest != NULL), return EINVAL); // No bounds checking; see comment in pcmk__xe_get_time() // Parse seconds rc = pcmk__xe_get_time(xml, sec_attr, &(result.tv_sec)); if (rc != pcmk_rc_ok) { return rc; } // Parse microseconds rc = pcmk__xe_get_ll(xml, usec_attr, &value_ll); if (rc != pcmk_rc_ok) { return rc; } result.tv_usec = (suseconds_t) value_ll; *dest = result; return pcmk_rc_ok; } /*! * \internal * \brief Set XML attribute values for seconds and microseconds * * This is like \c pcmk__xe_set() but takes a struct timeval *. * * \param[in,out] xml XML element whose attributes to set * \param[in] sec_attr Name of XML attribute for seconds * \param[in] usec_attr Name of XML attribute for microseconds * \param[in] value Attribute values to set * * \note This does nothing if \p value is \c NULL. */ void pcmk__xe_set_timeval(xmlNode *xml, const char *sec_attr, const char *usec_attr, const struct timeval *value) { CRM_CHECK((xml != NULL) && (sec_attr != NULL) && (usec_attr != NULL), return); if (value == NULL) { return; } if (pcmk__xe_set_ll(xml, sec_attr, (long long) value->tv_sec) != pcmk_rc_ok) { return; } /* Seconds were added successfully. Ignore any errors adding microseconds. * * It would be nice to make this atomic: revert the seconds attribute if * adding the microseconds attribute fails. That's somewhat complicated due * to change tracking: the chain of parents is already marked dirty, etc. In * practice, microseconds should succeed if seconds succeeded, unless it's * due to memory allocation failure. Nothing checks the return values of * these setter functions at time of writing, anyway. */ pcmk__xe_set_ll(xml, usec_attr, (long long) value->tv_usec); } /*! * \internal * \brief Get a date/time object from an XML attribute value * * \param[in] xml XML with attribute to parse (from CIB) * \param[in] attr Name of attribute to parse * \param[out] t Where to create date/time object * (\p *t must be NULL initially) * * \return Standard Pacemaker return code * \note The caller is responsible for freeing \p *t using crm_time_free(). */ int pcmk__xe_get_datetime(const xmlNode *xml, const char *attr, crm_time_t **t) { const char *value = NULL; if ((t == NULL) || (*t != NULL) || (xml == NULL) || (attr == NULL)) { return EINVAL; } value = pcmk__xe_get(xml, attr); if (value != NULL) { *t = crm_time_new(value); if (*t == NULL) { return pcmk_rc_unpack_error; } } return pcmk_rc_ok; } /*! * \internal * \brief Add a boolean attribute to an XML node. * * \param[in,out] node XML node to add attributes to * \param[in] name XML attribute to create * \param[in] value Value to give to the attribute */ void pcmk__xe_set_bool_attr(xmlNodePtr node, const char *name, bool value) { pcmk__xe_set(node, name, pcmk__btoa(value)); } /*! * \internal * \brief Extract a boolean attribute's value from an XML element, with * error checking * * \param[in] node XML node to get attribute from * \param[in] name XML attribute to get * \param[out] value Destination for the value of the attribute * * \return EINVAL if \p name or \p value are NULL, ENODATA if \p node is * NULL or the attribute does not exist, pcmk_rc_unknown_format * if the attribute is not a boolean, and pcmk_rc_ok otherwise. * * \note \p value only has any meaning if the return value is pcmk_rc_ok. */ int pcmk__xe_get_bool_attr(const xmlNode *node, const char *name, bool *value) { const char *xml_value = NULL; if (node == NULL) { return ENODATA; } else if (name == NULL || value == NULL) { return EINVAL; } xml_value = pcmk__xe_get(node, name); if (xml_value == NULL) { return ENODATA; } return pcmk__parse_bool(xml_value, value); } /*! * \internal * \brief Extract a boolean attribute's value from an XML element * * \param[in] node XML node to get attribute from * \param[in] name XML attribute to get * * \return True if the given \p name is an attribute on \p node and has * the value \c PCMK_VALUE_TRUE, False in all other cases */ bool pcmk__xe_attr_is_true(const xmlNode *node, const char *name) { bool value = false; int rc; rc = pcmk__xe_get_bool_attr(node, name, &value); return rc == pcmk_rc_ok && value == true; } // Deprecated functions kept only for backward API compatibility // LCOV_EXCL_START #include // gboolean, GSList #include // pcmk_free_nvpairs() #include // pcmk_xml_attrs2nvpairs(), etc. #include // crm_xml_sanitize_id() #include xmlNode * expand_idref(xmlNode *input, xmlNode *top) { return pcmk__xe_resolve_idref(input, top); } const char * crm_xml_add(xmlNode *node, const char *name, const char *value) { bool dirty = FALSE; xmlAttr *attr = NULL; CRM_CHECK(node != NULL, return NULL); CRM_CHECK(name != NULL, return NULL); if (value == NULL) { return NULL; } if (pcmk__xml_doc_all_flags_set(node->doc, pcmk__xf_tracking)) { const char *old = pcmk__xe_get(node, name); if (old == NULL || value == NULL || strcmp(old, value) != 0) { dirty = TRUE; } } if (dirty && (pcmk__check_acl(node, name, pcmk__xf_acl_create) == FALSE)) { pcmk__trace("Cannot add %s=%s to %s", name, value, node->name); return NULL; } attr = xmlSetProp(node, (const xmlChar *) name, (const xmlChar *) value); /* If the attribute already exists, this does nothing. Attribute values * don't get private data. */ pcmk__xml_new_private_data((xmlNode *) attr); if (dirty) { pcmk__mark_xml_attr_dirty(attr); } CRM_CHECK(attr && attr->children && attr->children->content, return NULL); return (char *)attr->children->content; } void crm_xml_set_id(xmlNode *xml, const char *format, ...) { va_list ap; int len = 0; char *id = NULL; // Equivalent to pcmk__assert_asprintf() va_start(ap, format); len = vasprintf(&id, format, ap); va_end(ap); pcmk__assert(len > 0); crm_xml_sanitize_id(id); crm_xml_add(xml, PCMK_XA_ID, id); free(id); } xmlNode * sorted_xml(xmlNode *input, xmlNode *parent, gboolean recursive) { xmlNode *child = NULL; GSList *nvpairs = NULL; xmlNode *result = NULL; CRM_CHECK(input != NULL, return NULL); result = pcmk__xe_create(parent, (const char *) input->name); nvpairs = pcmk_xml_attrs2nvpairs(input); nvpairs = pcmk_sort_nvpairs(nvpairs); pcmk_nvpairs2xml_attrs(nvpairs, result); pcmk_free_nvpairs(nvpairs); for (child = pcmk__xe_first_child(input, NULL, NULL, NULL); child != NULL; child = pcmk__xe_next(child, NULL)) { if (recursive) { sorted_xml(child, result, recursive); } else { pcmk__xml_copy(result, child); } } return result; } const char * crm_element_value(const xmlNode *data, const char *name) { xmlAttr *attr = NULL; if (data == NULL) { pcmk__err("Couldn't find %s in NULL", pcmk__s(name, "")); CRM_LOG_ASSERT(data != NULL); return NULL; } else if (name == NULL) { pcmk__err("Couldn't find NULL in %s", data->name); return NULL; } attr = xmlHasProp(data, (const xmlChar *) name); if (!attr || !attr->children) { return NULL; } return (const char *) attr->children->content; } const char * crm_copy_xml_element(const xmlNode *obj1, xmlNode *obj2, const char *element) { const char *value = crm_element_value(obj1, element); crm_xml_add(obj2, element, value); return value; } int crm_element_value_ll(const xmlNode *data, const char *name, long long *dest) { const char *value = NULL; CRM_CHECK(dest != NULL, return -1); value = crm_element_value(data, name); if (value != NULL) { int rc = pcmk__scan_ll(value, dest, PCMK__PARSE_INT_DEFAULT); if (rc == pcmk_rc_ok) { return 0; } pcmk__warn("Using default for %s because '%s' is not a valid integer: " "%s", name, value, pcmk_rc_str(rc)); } return -1; } int crm_element_value_timeval(const xmlNode *xml, const char *name_sec, const char *name_usec, struct timeval *dest) { long long value_i = 0; CRM_CHECK(dest != NULL, return -EINVAL); dest->tv_sec = 0; dest->tv_usec = 0; if (xml == NULL) { return pcmk_ok; } // No bounds checking; see comment in pcmk__xe_get_time() // Parse seconds errno = 0; if (crm_element_value_ll(xml, name_sec, &value_i) < 0) { return -errno; } dest->tv_sec = (time_t) value_i; // Parse microseconds if (crm_element_value_ll(xml, name_usec, &value_i) < 0) { return -errno; } dest->tv_usec = (suseconds_t) value_i; return pcmk_ok; } int crm_element_value_epoch(const xmlNode *xml, const char *name, time_t *dest) { long long value_ll = 0; if (crm_element_value_ll(xml, name, &value_ll) < 0) { return -1; } // No bounds checking; see comment in pcmk__xe_get_time() *dest = (time_t) value_ll; return pcmk_ok; } int crm_element_value_ms(const xmlNode *data, const char *name, guint *dest) { const char *value = NULL; long long value_ll; int rc = pcmk_rc_ok; CRM_CHECK(dest != NULL, return -1); *dest = 0; value = crm_element_value(data, name); rc = pcmk__scan_ll(value, &value_ll, 0LL); if (rc != pcmk_rc_ok) { pcmk__warn("Using default for %s because '%s' is not valid " "milliseconds: %s", name, value, pcmk_rc_str(rc)); return -1; } if ((value_ll < 0) || (value_ll > G_MAXUINT)) { pcmk__warn("Using default for %s because '%s' is out of range", name, value); return -1; } *dest = (guint) value_ll; return pcmk_ok; } int crm_element_value_int(const xmlNode *data, const char *name, int *dest) { const char *value = NULL; CRM_CHECK(dest != NULL, return -1); value = crm_element_value(data, name); if (value) { long long value_ll; int rc = pcmk__scan_ll(value, &value_ll, 0LL); *dest = PCMK__PARSE_INT_DEFAULT; if (rc != pcmk_rc_ok) { pcmk__warn("Using default for %s because '%s' is not a valid " "integer: %s", name, value, pcmk_rc_str(rc)); } else if ((value_ll < INT_MIN) || (value_ll > INT_MAX)) { pcmk__warn("Using default for %s because '%s' is out of range", name, value); } else { *dest = (int) value_ll; return 0; } } return -1; } char * crm_element_value_copy(const xmlNode *data, const char *name) { CRM_CHECK((data != NULL) && (name != NULL), return NULL); return pcmk__str_copy(pcmk__xe_get(data, name)); } // Maximum size of null-terminated string representation of 64-bit integer // -9223372036854775808 #define LLSTRSIZE 21 const char * crm_xml_add_ll(xmlNode *xml, const char *name, long long value) { char s[LLSTRSIZE] = { '\0', }; if (snprintf(s, LLSTRSIZE, "%lld", (long long) value) == LLSTRSIZE) { return NULL; } return crm_xml_add(xml, name, s); } const char * crm_xml_add_timeval(xmlNode *xml, const char *name_sec, const char *name_usec, const struct timeval *value) { const char *added = NULL; if (xml && name_sec && value) { added = crm_xml_add_ll(xml, name_sec, (long long) value->tv_sec); if (added && name_usec) { // Any error is ignored (we successfully added seconds) crm_xml_add_ll(xml, name_usec, (long long) value->tv_usec); } } return added; } const char * crm_xml_add_ms(xmlNode *node, const char *name, guint ms) { char *number = pcmk__assert_asprintf("%u", ms); const char *added = crm_xml_add(node, name, number); free(number); return added; } const char * crm_xml_add_int(xmlNode *node, const char *name, int value) { char *number = pcmk__itoa(value); const char *added = crm_xml_add(node, name, number); free(number); return added; } // LCOV_EXCL_STOP // End deprecated API diff --git a/lib/common/xml_io.c b/lib/common/xml_io.c index 4eefbdf8ae..6df1210123 100644 --- a/lib/common/xml_io.c +++ b/lib/common/xml_io.c @@ -1,658 +1,658 @@ /* * Copyright 2004-2025 the Pacemaker project contributors * * The version control history for this file may have further details. * * This source code is licensed under the GNU Lesser General Public License * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. */ #include #include #include #include #include #include #include #include #include // xmlOutputBuffer* #include // xmlChar #include #include #include #include "crmcommon_private.h" /*! * \internal * \brief Decompress a bzip2-compressed file into a string buffer * * \param[in] filename Name of file to decompress * * \return Newly allocated string with the decompressed contents of \p filename, * or \c NULL on error. * * \note The caller is responsible for freeing the return value using \c free(). */ static char * decompress_file(const char *filename) { char *buffer = NULL; int rc = pcmk_rc_ok; size_t length = 0; BZFILE *bz_file = NULL; FILE *input = fopen(filename, "r"); if (input == NULL) { crm_perror(LOG_ERR, "Could not open %s for reading", filename); return NULL; } bz_file = BZ2_bzReadOpen(&rc, input, 0, 0, NULL, 0); rc = pcmk__bzlib2rc(rc); if (rc != pcmk_rc_ok) { pcmk__err("Could not prepare to read compressed %s: %s " QB_XS " rc=%d", filename, pcmk_rc_str(rc), rc); goto done; } do { int read_len = 0; buffer = pcmk__realloc(buffer, length + PCMK__BUFFER_SIZE + 1); read_len = BZ2_bzRead(&rc, bz_file, buffer + length, PCMK__BUFFER_SIZE); if ((rc == BZ_OK) || (rc == BZ_STREAM_END)) { pcmk__trace("Read %ld bytes from file: %d", (long) read_len, rc); length += read_len; } } while (rc == BZ_OK); rc = pcmk__bzlib2rc(rc); if (rc != pcmk_rc_ok) { rc = pcmk__bzlib2rc(rc); pcmk__err("Could not read compressed %s: %s " QB_XS " rc=%d", filename, pcmk_rc_str(rc), rc); free(buffer); buffer = NULL; } else { buffer[length] = '\0'; } done: BZ2_bzReadClose(&rc, bz_file); fclose(input); return buffer; } /*! * \internal * \brief Parse XML from a file * * \param[in] filename Name of file containing XML (\c NULL or \c "-" for * \c stdin); if \p filename ends in \c ".bz2", the file * will be decompressed using \c bzip2 * * \return XML tree parsed from the given file on success, otherwise \c NULL */ xmlNode * pcmk__xml_read(const char *filename) { bool use_stdin = pcmk__str_eq(filename, "-", pcmk__str_null_matches); xmlNode *xml = NULL; xmlDoc *output = NULL; xmlParserCtxt *ctxt = NULL; const xmlError *last_error = NULL; // Create a parser context ctxt = xmlNewParserCtxt(); CRM_CHECK(ctxt != NULL, return NULL); xmlCtxtResetLastError(ctxt); xmlSetGenericErrorFunc(ctxt, pcmk__log_xmllib_err); if (use_stdin) { output = xmlCtxtReadFd(ctxt, STDIN_FILENO, NULL, NULL, XML_PARSE_NOBLANKS); } else if (pcmk__ends_with_ext(filename, ".bz2")) { char *input = decompress_file(filename); if (input != NULL) { output = xmlCtxtReadDoc(ctxt, (const xmlChar *) input, NULL, NULL, XML_PARSE_NOBLANKS); free(input); } } else { output = xmlCtxtReadFile(ctxt, filename, NULL, XML_PARSE_NOBLANKS); } if (output != NULL) { pcmk__xml_new_private_data((xmlNode *) output); xml = xmlDocGetRootElement(output); if (xml != NULL) { /* @TODO Should we really be stripping out text? This seems like an * overly broad way to get rid of whitespace, if that's the goal. * Text nodes may be invalid in most or all Pacemaker inputs, but * stripping them in a generic "parse XML from file" function may * not be the best way to ignore them. */ pcmk__strip_xml_text(xml); } } last_error = xmlCtxtGetLastError(ctxt); if ((last_error != NULL) && (xml != NULL)) { pcmk__log_xml_debug(xml, "partial"); pcmk__xml_free(xml); xml = NULL; } xmlFreeParserCtxt(ctxt); return xml; } /*! * \internal * \brief Parse XML from a string * * \param[in] input String to parse * * \return XML tree parsed from the given string on success, otherwise \c NULL */ xmlNode * pcmk__xml_parse(const char *input) { xmlNode *xml = NULL; xmlDoc *output = NULL; xmlParserCtxt *ctxt = NULL; const xmlError *last_error = NULL; if (input == NULL) { return NULL; } ctxt = xmlNewParserCtxt(); if (ctxt == NULL) { return NULL; } xmlCtxtResetLastError(ctxt); xmlSetGenericErrorFunc(ctxt, pcmk__log_xmllib_err); output = xmlCtxtReadDoc(ctxt, (const xmlChar *) input, NULL, NULL, XML_PARSE_NOBLANKS); if (output != NULL) { pcmk__xml_new_private_data((xmlNode *) output); xml = xmlDocGetRootElement(output); } last_error = xmlCtxtGetLastError(ctxt); if ((last_error != NULL) && (xml != NULL)) { pcmk__log_xml_debug(xml, "partial"); pcmk__xml_free(xml); xml = NULL; } xmlFreeParserCtxt(ctxt); return xml; } /*! * \internal * \brief Append a string representation of an XML element to a buffer * * \param[in] data XML whose representation to append * \param[in] options Group of \p pcmk__xml_fmt_options flags * \param[in,out] buffer Where to append the content (must not be \p NULL) * \param[in] depth Current indentation level */ static void dump_xml_element(const xmlNode *data, uint32_t options, GString *buffer, int depth) { const bool pretty = pcmk__is_set(options, pcmk__xml_fmt_pretty); const bool filtered = pcmk__is_set(options, pcmk__xml_fmt_filtered); const int spaces = pretty? (2 * depth) : 0; for (int lpc = 0; lpc < spaces; lpc++) { g_string_append_c(buffer, ' '); } pcmk__g_strcat(buffer, "<", data->name, NULL); for (const xmlAttr *attr = pcmk__xe_first_attr(data); attr != NULL; attr = attr->next) { if (!filtered || !pcmk__xa_filterable((const char *) (attr->name))) { pcmk__dump_xml_attr(attr, buffer); } } if (data->children == NULL) { g_string_append(buffer, "/>"); } else { g_string_append_c(buffer, '>'); } if (pretty) { g_string_append_c(buffer, '\n'); } if (data->children) { for (const xmlNode *child = data->children; child != NULL; child = child->next) { pcmk__xml_string(child, options, buffer, depth + 1); } for (int lpc = 0; lpc < spaces; lpc++) { g_string_append_c(buffer, ' '); } pcmk__g_strcat(buffer, "name, ">", NULL); if (pretty) { g_string_append_c(buffer, '\n'); } } } /*! * \internal * \brief Append XML text content to a buffer * * \param[in] data XML whose content to append * \param[in] options Group of enum pcmk__xml_fmt_options * \param[in,out] buffer Where to append the content (must not be \p NULL) * \param[in] depth Current indentation level */ static void dump_xml_text(const xmlNode *data, uint32_t options, GString *buffer, int depth) { const bool pretty = pcmk__is_set(options, pcmk__xml_fmt_pretty); const int spaces = pretty? (2 * depth) : 0; const char *content = (const char *) data->content; gchar *content_esc = NULL; if (pcmk__xml_needs_escape(content, pcmk__xml_escape_text)) { content_esc = pcmk__xml_escape(content, pcmk__xml_escape_text); content = content_esc; } for (int lpc = 0; lpc < spaces; lpc++) { g_string_append_c(buffer, ' '); } g_string_append(buffer, content); if (pretty) { g_string_append_c(buffer, '\n'); } g_free(content_esc); } /*! * \internal * \brief Append XML CDATA content to a buffer * * \param[in] data XML whose content to append * \param[in] options Group of \p pcmk__xml_fmt_options flags * \param[in,out] buffer Where to append the content (must not be \p NULL) * \param[in] depth Current indentation level */ static void dump_xml_cdata(const xmlNode *data, uint32_t options, GString *buffer, int depth) { const bool pretty = pcmk__is_set(options, pcmk__xml_fmt_pretty); const int spaces = pretty? (2 * depth) : 0; for (int lpc = 0; lpc < spaces; lpc++) { g_string_append_c(buffer, ' '); } pcmk__g_strcat(buffer, "content, "]]>", NULL); if (pretty) { g_string_append_c(buffer, '\n'); } } /*! * \internal * \brief Append an XML comment to a buffer * * \param[in] data XML whose content to append * \param[in] options Group of \p pcmk__xml_fmt_options flags * \param[in,out] buffer Where to append the content (must not be \p NULL) * \param[in] depth Current indentation level */ static void dump_xml_comment(const xmlNode *data, uint32_t options, GString *buffer, int depth) { const bool pretty = pcmk__is_set(options, pcmk__xml_fmt_pretty); const int spaces = pretty? (2 * depth) : 0; for (int lpc = 0; lpc < spaces; lpc++) { g_string_append_c(buffer, ' '); } pcmk__g_strcat(buffer, "", NULL); if (pretty) { g_string_append_c(buffer, '\n'); } } /*! * \internal * \brief Create a string representation of an XML object * * libxml2's \c xmlNodeDumpOutput() doesn't allow filtering, doesn't escape * special characters thoroughly, and doesn't allow a const argument. * * \param[in] data XML to convert * \param[in] options Group of \p pcmk__xml_fmt_options flags * \param[in,out] buffer Where to store the text (must not be \p NULL) * \param[in] depth Current indentation level * * \todo Create a wrapper that doesn't require \p depth. Only used with * recursive calls currently. */ void pcmk__xml_string(const xmlNode *data, uint32_t options, GString *buffer, int depth) { if (data == NULL) { pcmk__trace("Nothing to dump"); return; } pcmk__assert(buffer != NULL); CRM_CHECK(depth >= 0, depth = 0); switch(data->type) { case XML_ELEMENT_NODE: /* Handle below */ dump_xml_element(data, options, buffer, depth); break; case XML_TEXT_NODE: if (pcmk__is_set(options, pcmk__xml_fmt_text)) { dump_xml_text(data, options, buffer, depth); } break; case XML_COMMENT_NODE: dump_xml_comment(data, options, buffer, depth); break; case XML_CDATA_SECTION_NODE: dump_xml_cdata(data, options, buffer, depth); break; default: pcmk__warn("Cannot convert XML %s node to text " QB_XS " type=%d", pcmk__xml_element_type_text(data->type), data->type); break; } } /*! * \internal * \brief Write a string to a file stream, compressed using \c bzip2 * * \param[in] text String to write * \param[in] filename Name of file being written (for logging only) * \param[in,out] stream Open file stream to write to * \param[out] bytes_out Number of bytes written (valid only on success) * * \return Standard Pacemaker return code */ static int write_compressed_stream(char *text, const char *filename, FILE *stream, unsigned int *bytes_out) { unsigned int bytes_in = 0; int rc = pcmk_rc_ok; // (5, 0, 0): (intermediate block size, silent, default workFactor) BZFILE *bz_file = BZ2_bzWriteOpen(&rc, stream, 5, 0, 0); rc = pcmk__bzlib2rc(rc); if (rc != pcmk_rc_ok) { pcmk__warn("Not compressing %s: could not prepare file stream: %s " QB_XS " rc=%d", filename, pcmk_rc_str(rc), rc); goto done; } BZ2_bzWrite(&rc, bz_file, text, strlen(text)); rc = pcmk__bzlib2rc(rc); if (rc != pcmk_rc_ok) { pcmk__warn("Not compressing %s: could not compress data: %s " QB_XS " rc=%d errno=%d", filename, pcmk_rc_str(rc), rc, errno); goto done; } BZ2_bzWriteClose(&rc, bz_file, 0, &bytes_in, bytes_out); bz_file = NULL; rc = pcmk__bzlib2rc(rc); if (rc != pcmk_rc_ok) { pcmk__warn("Not compressing %s: could not write compressed data: %s " QB_XS " rc=%d errno=%d", filename, pcmk_rc_str(rc), rc, errno); goto done; } pcmk__trace("Compressed XML for %s from %u bytes to %u", filename, bytes_in, *bytes_out); done: if (bz_file != NULL) { BZ2_bzWriteClose(&rc, bz_file, 0, NULL, NULL); } return rc; } /*! * \internal * \brief Write XML to a file stream * * \param[in] xml XML to write * \param[in] filename Name of file being written (for logging only) * \param[in,out] stream Open file stream corresponding to filename (closed * when this function returns) * \param[in] compress Whether to compress XML before writing * * \return Standard Pacemaker return code */ static int write_xml_stream(const xmlNode *xml, const char *filename, FILE *stream, bool compress) { GString *buffer = g_string_sized_new(1024); unsigned int bytes_out = 0; int rc = pcmk_rc_ok; pcmk__xml_string(xml, pcmk__xml_fmt_pretty, buffer, 0); CRM_CHECK(!pcmk__str_empty(buffer->str), pcmk__log_xml_info(xml, "dump-failed"); rc = pcmk_rc_error; goto done); - crm_log_xml_trace(xml, "writing"); + pcmk__log_xml_trace(xml, "writing"); if (compress && (write_compressed_stream(buffer->str, filename, stream, &bytes_out) == pcmk_rc_ok)) { goto done; } rc = fprintf(stream, "%s", buffer->str); if (rc < 0) { rc = EIO; crm_perror(LOG_ERR, "writing %s", filename); goto done; } bytes_out = (unsigned int) rc; rc = pcmk_rc_ok; done: if (fflush(stream) != 0) { rc = errno; crm_perror(LOG_ERR, "flushing %s", filename); } // Don't report error if the file does not support synchronization if ((fsync(fileno(stream)) < 0) && (errno != EROFS) && (errno != EINVAL)) { rc = errno; crm_perror(LOG_ERR, "synchronizing %s", filename); } fclose(stream); pcmk__trace("Saved %u bytes to %s as XML", bytes_out, filename); g_string_free(buffer, TRUE); return rc; } /*! * \internal * \brief Write XML to a file descriptor * * \param[in] xml XML to write * \param[in] filename Name of file being written (for logging only) * \param[in] fd Open file descriptor corresponding to \p filename * * \return Standard Pacemaker return code */ int pcmk__xml_write_fd(const xmlNode *xml, const char *filename, int fd) { FILE *stream = NULL; CRM_CHECK((xml != NULL) && (fd > 0), return EINVAL); stream = fdopen(fd, "w"); if (stream == NULL) { return errno; } return write_xml_stream(xml, pcmk__s(filename, "unnamed file"), stream, false); } /*! * \internal * \brief Write XML to a file * * \param[in] xml XML to write * \param[in] filename Name of file to write * \param[in] compress If \c true, compress XML before writing * * \return Standard Pacemaker return code */ int pcmk__xml_write_file(const xmlNode *xml, const char *filename, bool compress) { FILE *stream = NULL; CRM_CHECK((xml != NULL) && (filename != NULL), return EINVAL); stream = fopen(filename, "w"); if (stream == NULL) { return errno; } return write_xml_stream(xml, filename, stream, compress); } /*! * \internal * \brief Serialize XML (using libxml) into provided descriptor * * \param[in] fd File descriptor to (piece-wise) write to * \param[in] cur XML subtree to proceed * * \return a standard Pacemaker return code */ int pcmk__xml2fd(int fd, xmlNode *cur) { bool success; xmlOutputBuffer *fd_out = xmlOutputBufferCreateFd(fd, NULL); pcmk__mem_assert(fd_out); xmlNodeDumpOutput(fd_out, cur->doc, cur, 0, pcmk__xml_fmt_pretty, NULL); success = xmlOutputBufferWrite(fd_out, sizeof("\n") - 1, "\n") != -1; success = xmlOutputBufferClose(fd_out) != -1 && success; if (!success) { return EIO; } fsync(fd); return pcmk_rc_ok; } /*! * \internal * \brief Write XML to a file in a temporary directory * * \param[in] xml XML to write * \param[in] desc Description of \p xml * \param[in] filename Base name of file to write (\c NULL to create a name * based on a generated UUID) */ void pcmk__xml_write_temp_file(const xmlNode *xml, const char *desc, const char *filename) { char *path = NULL; char *uuid = NULL; CRM_CHECK((xml != NULL) && (desc != NULL), return); if (filename == NULL) { uuid = pcmk__generate_uuid(); filename = uuid; } path = pcmk__assert_asprintf("%s/%s", pcmk__get_tmpdir(), filename); pcmk__info("Saving %s to %s", desc, path); pcmk__xml_write_file(xml, filename, false); free(path); free(uuid); } // Deprecated functions kept only for backward API compatibility // LCOV_EXCL_START #include void save_xml_to_file(const xmlNode *xml, const char *desc, const char *filename) { char *f = NULL; if (filename == NULL) { char *uuid = pcmk__generate_uuid(); f = pcmk__assert_asprintf("%s/%s", pcmk__get_tmpdir(), uuid); filename = f; free(uuid); } pcmk__info("Saving %s to %s", desc, filename); pcmk__xml_write_file(xml, filename, false); free(f); } // LCOV_EXCL_STOP // End deprecated API diff --git a/lib/fencing/st_client.c b/lib/fencing/st_client.c index 9c3c4c1a29..c89939adc6 100644 --- a/lib/fencing/st_client.c +++ b/lib/fencing/st_client.c @@ -1,2756 +1,2756 @@ /* * Copyright 2004-2025 the Pacemaker project contributors * * The version control history for this file may have further details. * * This source code is licensed under the GNU Lesser General Public License * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. */ #include #include #include #include #include #include #include #include #include #include // xmlNode #include // xmlXPathObject, etc. #include #include #include #include // hash2field() #include #include #include "fencing_private.h" CRM_TRACE_INIT_DATA(stonith); // Used as stonith_t:st_private typedef struct stonith_private_s { char *token; crm_ipc_t *ipc; mainloop_io_t *source; GHashTable *stonith_op_callback_table; GList *notify_list; int notify_refcnt; bool notify_deletes; void (*op_callback) (stonith_t * st, stonith_callback_data_t * data); } stonith_private_t; // Used as stonith_event_t:opaque struct event_private { pcmk__action_result_t result; }; typedef struct stonith_notify_client_s { const char *event; const char *obj_id; /* implement one day */ const char *obj_type; /* implement one day */ void (*notify) (stonith_t * st, stonith_event_t * e); bool delete; } stonith_notify_client_t; typedef struct stonith_callback_client_s { void (*callback) (stonith_t * st, stonith_callback_data_t * data); const char *id; void *user_data; gboolean only_success; gboolean allow_timeout_updates; struct timer_rec_s *timer; } stonith_callback_client_t; struct notify_blob_s { stonith_t *stonith; xmlNode *xml; }; struct timer_rec_s { int call_id; int timeout; guint ref; stonith_t *stonith; }; typedef int (*stonith_op_t) (const char *, int, const char *, xmlNode *, xmlNode *, xmlNode *, xmlNode **, xmlNode **); bool stonith_dispatch(stonith_t * st); xmlNode *stonith_create_op(int call_id, const char *token, const char *op, xmlNode * data, int call_options); static int stonith_send_command(stonith_t *stonith, const char *op, xmlNode *data, xmlNode **output_data, int call_options, int timeout); static void stonith_connection_destroy(gpointer user_data); static void stonith_send_notification(gpointer data, gpointer user_data); static int stonith_api_del_notification(stonith_t *stonith, const char *event); /*! * \brief Get agent namespace by name * * \param[in] namespace_s Name of namespace as string * * \return Namespace as enum value */ enum stonith_namespace stonith_text2namespace(const char *namespace_s) { if (pcmk__str_eq(namespace_s, "any", pcmk__str_null_matches)) { return st_namespace_any; } else if (!strcmp(namespace_s, "redhat") || !strcmp(namespace_s, "stonith-ng")) { return st_namespace_rhcs; } else if (!strcmp(namespace_s, "internal")) { return st_namespace_internal; } else if (!strcmp(namespace_s, "heartbeat")) { return st_namespace_lha; } return st_namespace_invalid; } /*! * \brief Get agent namespace name * * \param[in] namespace Namespace as enum value * * \return Namespace name as string */ const char * stonith_namespace2text(enum stonith_namespace st_namespace) { switch (st_namespace) { case st_namespace_any: return "any"; case st_namespace_rhcs: return "stonith-ng"; case st_namespace_internal: return "internal"; case st_namespace_lha: return "heartbeat"; default: break; } return "unsupported"; } /*! * \brief Determine namespace of a fence agent * * \param[in] agent Fence agent type * \param[in] namespace_s Name of agent namespace as string, if known * * \return Namespace of specified agent, as enum value */ enum stonith_namespace stonith_get_namespace(const char *agent, const char *namespace_s) { if (pcmk__str_eq(namespace_s, "internal", pcmk__str_none)) { return st_namespace_internal; } if (stonith__agent_is_rhcs(agent)) { return st_namespace_rhcs; } #if HAVE_STONITH_STONITH_H if (stonith__agent_is_lha(agent)) { return st_namespace_lha; } #endif return st_namespace_invalid; } gboolean stonith__watchdog_fencing_enabled_for_node_api(stonith_t *st, const char *node) { gboolean rv = FALSE; stonith_t *stonith_api = st?st:stonith_api_new(); char *list = NULL; if(stonith_api) { if (stonith_api->state == stonith_disconnected) { int rc = stonith_api->cmds->connect(stonith_api, "stonith-api", NULL); if (rc != pcmk_ok) { pcmk__err("Failed connecting to Stonith-API for " "watchdog-fencing-query"); } } if (stonith_api->state != stonith_disconnected) { /* caveat!!! * this might fail when when stonithd is just updating the device-list * probably something we should fix as well for other api-calls */ int rc = stonith_api->cmds->list(stonith_api, st_opt_sync_call, STONITH_WATCHDOG_ID, &list, 0); if ((rc != pcmk_ok) || (list == NULL)) { /* due to the race described above it can happen that * we drop in here - so as not to make remote nodes * panic on that answer */ if (rc == -ENODEV) { pcmk__notice("Cluster does not have watchdog fencing " "device"); } else { pcmk__warn("Could not check for watchdog fencing device: %s", pcmk_strerror(rc)); } } else if (list[0] == '\0') { rv = TRUE; } else { GList *targets = stonith__parse_targets(list); rv = pcmk__str_in_list(node, targets, pcmk__str_casei); g_list_free_full(targets, free); } free(list); if (!st) { /* if we're provided the api we still might have done the * connection - but let's assume the caller won't bother */ stonith_api->cmds->disconnect(stonith_api); } } if (!st) { stonith_api_delete(stonith_api); } } else { pcmk__err("Stonith-API for watchdog-fencing-query couldn't be created"); } pcmk__trace("Pacemaker assumes node %s %sto do watchdog-fencing", node, (rv? "" : "not ")); return rv; } gboolean stonith__watchdog_fencing_enabled_for_node(const char *node) { return stonith__watchdog_fencing_enabled_for_node_api(NULL, node); } /* when cycling through the list we don't want to delete items so just mark them and when we know nobody is using the list loop over it to remove the marked items */ static void foreach_notify_entry (stonith_private_t *private, GFunc func, gpointer user_data) { private->notify_refcnt++; g_list_foreach(private->notify_list, func, user_data); private->notify_refcnt--; if ((private->notify_refcnt == 0) && private->notify_deletes) { GList *list_item = private->notify_list; private->notify_deletes = FALSE; while (list_item != NULL) { stonith_notify_client_t *list_client = list_item->data; GList *next = g_list_next(list_item); if (list_client->delete) { free(list_client); private->notify_list = g_list_delete_link(private->notify_list, list_item); } list_item = next; } } } static void stonith_connection_destroy(gpointer user_data) { stonith_t *stonith = user_data; stonith_private_t *native = NULL; struct notify_blob_s blob; pcmk__trace("Sending destroyed notification"); blob.stonith = stonith; blob.xml = pcmk__xe_create(NULL, PCMK__XE_NOTIFY); native = stonith->st_private; native->ipc = NULL; native->source = NULL; free(native->token); native->token = NULL; stonith->state = stonith_disconnected; pcmk__xe_set(blob.xml, PCMK__XA_T, PCMK__VALUE_ST_NOTIFY); pcmk__xe_set(blob.xml, PCMK__XA_SUBT, PCMK__VALUE_ST_NOTIFY_DISCONNECT); foreach_notify_entry(native, stonith_send_notification, &blob); pcmk__xml_free(blob.xml); } xmlNode * create_device_registration_xml(const char *id, enum stonith_namespace standard, const char *agent, const stonith_key_value_t *params, const char *rsc_provides) { xmlNode *data = pcmk__xe_create(NULL, PCMK__XE_ST_DEVICE_ID); xmlNode *args = pcmk__xe_create(data, PCMK__XE_ATTRIBUTES); #if HAVE_STONITH_STONITH_H if (standard == st_namespace_any) { standard = stonith_get_namespace(agent, NULL); } if (standard == st_namespace_lha) { hash2field((gpointer) "plugin", (gpointer) agent, args); agent = "fence_legacy"; } #endif pcmk__xe_set(data, PCMK_XA_ID, id); pcmk__xe_set(data, PCMK__XA_ST_ORIGIN, __func__); pcmk__xe_set(data, PCMK_XA_AGENT, agent); if ((standard != st_namespace_any) && (standard != st_namespace_invalid)) { pcmk__xe_set(data, PCMK__XA_NAMESPACE, stonith_namespace2text(standard)); } if (rsc_provides) { pcmk__xe_set(data, PCMK__XA_RSC_PROVIDES, rsc_provides); } for (; params; params = params->next) { hash2field((gpointer) params->key, (gpointer) params->value, args); } return data; } static int stonith_api_register_device(stonith_t *st, int call_options, const char *id, const char *namespace_s, const char *agent, const stonith_key_value_t *params) { int rc = 0; xmlNode *data = NULL; data = create_device_registration_xml(id, stonith_text2namespace(namespace_s), agent, params, NULL); rc = stonith_send_command(st, STONITH_OP_DEVICE_ADD, data, NULL, call_options, 0); pcmk__xml_free(data); return rc; } static int stonith_api_remove_device(stonith_t * st, int call_options, const char *name) { int rc = 0; xmlNode *data = NULL; data = pcmk__xe_create(NULL, PCMK__XE_ST_DEVICE_ID); pcmk__xe_set(data, PCMK__XA_ST_ORIGIN, __func__); pcmk__xe_set(data, PCMK_XA_ID, name); rc = stonith_send_command(st, STONITH_OP_DEVICE_DEL, data, NULL, call_options, 0); pcmk__xml_free(data); return rc; } static int stonith_api_remove_level_full(stonith_t *st, int options, const char *node, const char *pattern, const char *attr, const char *value, int level) { int rc = 0; xmlNode *data = NULL; CRM_CHECK(node || pattern || (attr && value), return -EINVAL); data = pcmk__xe_create(NULL, PCMK_XE_FENCING_LEVEL); pcmk__xe_set(data, PCMK__XA_ST_ORIGIN, __func__); if (node) { pcmk__xe_set(data, PCMK_XA_TARGET, node); } else if (pattern) { pcmk__xe_set(data, PCMK_XA_TARGET_PATTERN, pattern); } else { pcmk__xe_set(data, PCMK_XA_TARGET_ATTRIBUTE, attr); pcmk__xe_set(data, PCMK_XA_TARGET_VALUE, value); } pcmk__xe_set_int(data, PCMK_XA_INDEX, level); rc = stonith_send_command(st, STONITH_OP_LEVEL_DEL, data, NULL, options, 0); pcmk__xml_free(data); return rc; } static int stonith_api_remove_level(stonith_t * st, int options, const char *node, int level) { return stonith_api_remove_level_full(st, options, node, NULL, NULL, NULL, level); } /*! * \internal * \brief Create XML for fence topology level registration request * * \param[in] node If not NULL, target level by this node name * \param[in] pattern If not NULL, target by node name using this regex * \param[in] attr If not NULL, target by this node attribute * \param[in] value If not NULL, target by this node attribute value * \param[in] level Index number of level to register * \param[in] device_list List of devices in level * * \return Newly allocated XML tree on success, NULL otherwise * * \note The caller should set only one of node, pattern or attr/value. */ xmlNode * create_level_registration_xml(const char *node, const char *pattern, const char *attr, const char *value, int level, const stonith_key_value_t *device_list) { GString *list = NULL; xmlNode *data; CRM_CHECK(node || pattern || (attr && value), return NULL); data = pcmk__xe_create(NULL, PCMK_XE_FENCING_LEVEL); pcmk__xe_set(data, PCMK__XA_ST_ORIGIN, __func__); pcmk__xe_set_int(data, PCMK_XA_ID, level); pcmk__xe_set_int(data, PCMK_XA_INDEX, level); if (node) { pcmk__xe_set(data, PCMK_XA_TARGET, node); } else if (pattern) { pcmk__xe_set(data, PCMK_XA_TARGET_PATTERN, pattern); } else { pcmk__xe_set(data, PCMK_XA_TARGET_ATTRIBUTE, attr); pcmk__xe_set(data, PCMK_XA_TARGET_VALUE, value); } for (; device_list; device_list = device_list->next) { pcmk__add_separated_word(&list, 1024, device_list->value, ","); } if (list != NULL) { pcmk__xe_set(data, PCMK_XA_DEVICES, (const char *) list->str); g_string_free(list, TRUE); } return data; } static int stonith_api_register_level_full(stonith_t *st, int options, const char *node, const char *pattern, const char *attr, const char *value, int level, const stonith_key_value_t *device_list) { int rc = 0; xmlNode *data = create_level_registration_xml(node, pattern, attr, value, level, device_list); CRM_CHECK(data != NULL, return -EINVAL); rc = stonith_send_command(st, STONITH_OP_LEVEL_ADD, data, NULL, options, 0); pcmk__xml_free(data); return rc; } static int stonith_api_register_level(stonith_t * st, int options, const char *node, int level, const stonith_key_value_t * device_list) { return stonith_api_register_level_full(st, options, node, NULL, NULL, NULL, level, device_list); } static int stonith_api_device_list(stonith_t *stonith, int call_options, const char *namespace_s, stonith_key_value_t **devices, int timeout) { int count = 0; enum stonith_namespace ns = stonith_text2namespace(namespace_s); if (devices == NULL) { pcmk__err("Parameter error: stonith_api_device_list"); return -EFAULT; } #if HAVE_STONITH_STONITH_H // Include Linux-HA agents if requested if ((ns == st_namespace_any) || (ns == st_namespace_lha)) { count += stonith__list_lha_agents(devices); } #endif // Include Red Hat agents if requested if ((ns == st_namespace_any) || (ns == st_namespace_rhcs)) { count += stonith__list_rhcs_agents(devices); } return count; } // See stonith_api_operations_t:metadata() documentation static int stonith_api_device_metadata(stonith_t *stonith, int call_options, const char *agent, const char *namespace_s, char **output, int timeout_sec) { /* By executing meta-data directly, we can get it from stonith_admin when * the cluster is not running, which is important for higher-level tools. */ enum stonith_namespace ns = stonith_get_namespace(agent, namespace_s); if (timeout_sec <= 0) { timeout_sec = PCMK_DEFAULT_ACTION_TIMEOUT_MS; } pcmk__trace("Looking up metadata for %s agent %s", stonith_namespace2text(ns), agent); switch (ns) { case st_namespace_rhcs: return stonith__rhcs_metadata(agent, timeout_sec, output); #if HAVE_STONITH_STONITH_H case st_namespace_lha: return stonith__lha_metadata(agent, timeout_sec, output); #endif default: pcmk__err("Can't get fence agent '%s' meta-data: No such agent", agent); break; } return -ENODEV; } static int stonith_api_query(stonith_t * stonith, int call_options, const char *target, stonith_key_value_t ** devices, int timeout) { int rc = 0, lpc = 0, max = 0; xmlNode *data = NULL; xmlNode *output = NULL; xmlXPathObject *xpathObj = NULL; CRM_CHECK(devices != NULL, return -EINVAL); data = pcmk__xe_create(NULL, PCMK__XE_ST_DEVICE_ID); pcmk__xe_set(data, PCMK__XA_ST_ORIGIN, __func__); pcmk__xe_set(data, PCMK__XA_ST_TARGET, target); pcmk__xe_set(data, PCMK__XA_ST_DEVICE_ACTION, PCMK_ACTION_OFF); rc = stonith_send_command(stonith, STONITH_OP_QUERY, data, &output, call_options, timeout); if (rc < 0) { return rc; } xpathObj = pcmk__xpath_search(output->doc, "//*[@" PCMK_XA_AGENT "]"); if (xpathObj) { max = pcmk__xpath_num_results(xpathObj); for (lpc = 0; lpc < max; lpc++) { xmlNode *match = pcmk__xpath_result(xpathObj, lpc); CRM_LOG_ASSERT(match != NULL); if(match != NULL) { xmlChar *match_path = xmlGetNodePath(match); pcmk__info("//*[@" PCMK_XA_AGENT "][%d] = %s", lpc, match_path); free(match_path); *devices = stonith_key_value_add(*devices, NULL, pcmk__xe_get(match, PCMK_XA_ID)); } } xmlXPathFreeObject(xpathObj); } pcmk__xml_free(output); pcmk__xml_free(data); return max; } /*! * \internal * \brief Make a STONITH_OP_EXEC request * * \param[in,out] stonith Fencer connection * \param[in] call_options Bitmask of \c stonith_call_options * \param[in] id Fence device ID that request is for * \param[in] action Agent action to request (list, status, monitor) * \param[in] target Name of target node for requested action * \param[in] timeout_sec Error if not completed within this many seconds * \param[out] output Where to set agent output */ static int stonith_api_call(stonith_t *stonith, int call_options, const char *id, const char *action, const char *target, int timeout_sec, xmlNode **output) { int rc = 0; xmlNode *data = NULL; data = pcmk__xe_create(NULL, PCMK__XE_ST_DEVICE_ID); pcmk__xe_set(data, PCMK__XA_ST_ORIGIN, __func__); pcmk__xe_set(data, PCMK__XA_ST_DEVICE_ID, id); pcmk__xe_set(data, PCMK__XA_ST_DEVICE_ACTION, action); pcmk__xe_set(data, PCMK__XA_ST_TARGET, target); rc = stonith_send_command(stonith, STONITH_OP_EXEC, data, output, call_options, timeout_sec); pcmk__xml_free(data); return rc; } static int stonith_api_list(stonith_t * stonith, int call_options, const char *id, char **list_info, int timeout) { int rc; xmlNode *output = NULL; rc = stonith_api_call(stonith, call_options, id, PCMK_ACTION_LIST, NULL, timeout, &output); if (output && list_info) { const char *list_str; list_str = pcmk__xe_get(output, PCMK__XA_ST_OUTPUT); if (list_str) { *list_info = strdup(list_str); } } if (output) { pcmk__xml_free(output); } return rc; } static int stonith_api_monitor(stonith_t * stonith, int call_options, const char *id, int timeout) { return stonith_api_call(stonith, call_options, id, PCMK_ACTION_MONITOR, NULL, timeout, NULL); } static int stonith_api_status(stonith_t * stonith, int call_options, const char *id, const char *port, int timeout) { return stonith_api_call(stonith, call_options, id, PCMK_ACTION_STATUS, port, timeout, NULL); } static int stonith_api_fence_with_delay(stonith_t * stonith, int call_options, const char *node, const char *action, int timeout, int tolerance, int delay) { int rc = 0; xmlNode *data = NULL; data = pcmk__xe_create(NULL, __func__); pcmk__xe_set(data, PCMK__XA_ST_TARGET, node); pcmk__xe_set(data, PCMK__XA_ST_DEVICE_ACTION, action); pcmk__xe_set_int(data, PCMK__XA_ST_TIMEOUT, timeout); pcmk__xe_set_int(data, PCMK__XA_ST_TOLERANCE, tolerance); pcmk__xe_set_int(data, PCMK__XA_ST_DELAY, delay); rc = stonith_send_command(stonith, STONITH_OP_FENCE, data, NULL, call_options, timeout); pcmk__xml_free(data); return rc; } static int stonith_api_fence(stonith_t * stonith, int call_options, const char *node, const char *action, int timeout, int tolerance) { return stonith_api_fence_with_delay(stonith, call_options, node, action, timeout, tolerance, 0); } static int stonith_api_confirm(stonith_t * stonith, int call_options, const char *target) { stonith__set_call_options(call_options, target, st_opt_manual_ack); return stonith_api_fence(stonith, call_options, target, PCMK_ACTION_OFF, 0, 0); } static int stonith_api_history(stonith_t * stonith, int call_options, const char *node, stonith_history_t ** history, int timeout) { int rc = 0; xmlNode *data = NULL; xmlNode *output = NULL; stonith_history_t *last = NULL; *history = NULL; if (node) { data = pcmk__xe_create(NULL, __func__); pcmk__xe_set(data, PCMK__XA_ST_TARGET, node); } stonith__set_call_options(call_options, node, st_opt_sync_call); rc = stonith_send_command(stonith, STONITH_OP_FENCE_HISTORY, data, &output, call_options, timeout); pcmk__xml_free(data); if (rc == 0) { xmlNode *op = NULL; xmlNode *reply = pcmk__xpath_find_one(output->doc, "//" PCMK__XE_ST_HISTORY, PCMK__LOG_NEVER); for (op = pcmk__xe_first_child(reply, NULL, NULL, NULL); op != NULL; op = pcmk__xe_next(op, NULL)) { stonith_history_t *kvp = pcmk__assert_alloc(1, sizeof(stonith_history_t)); long long completed_nsec = 0LL; kvp->target = pcmk__xe_get_copy(op, PCMK__XA_ST_TARGET); kvp->action = pcmk__xe_get_copy(op, PCMK__XA_ST_DEVICE_ACTION); kvp->origin = pcmk__xe_get_copy(op, PCMK__XA_ST_ORIGIN); kvp->delegate = pcmk__xe_get_copy(op, PCMK__XA_ST_DELEGATE); kvp->client = pcmk__xe_get_copy(op, PCMK__XA_ST_CLIENTNAME); pcmk__xe_get_time(op, PCMK__XA_ST_DATE, &kvp->completed); pcmk__xe_get_ll(op, PCMK__XA_ST_DATE_NSEC, &completed_nsec); if ((completed_nsec >= LONG_MIN) && (completed_nsec <= LONG_MAX)) { kvp->completed_nsec = (long) completed_nsec; } pcmk__xe_get_int(op, PCMK__XA_ST_STATE, &kvp->state); kvp->exit_reason = pcmk__xe_get_copy(op, PCMK_XA_EXIT_REASON); if (last) { last->next = kvp; } else { *history = kvp; } last = kvp; } } pcmk__xml_free(output); return rc; } void stonith_history_free(stonith_history_t *history) { stonith_history_t *hp, *hp_old; for (hp = history; hp; hp_old = hp, hp = hp->next, free(hp_old)) { free(hp->target); free(hp->action); free(hp->origin); free(hp->delegate); free(hp->client); free(hp->exit_reason); } } static gint stonithlib_GCompareFunc(gconstpointer a, gconstpointer b) { int rc = 0; const stonith_notify_client_t *a_client = a; const stonith_notify_client_t *b_client = b; if (a_client->delete || b_client->delete) { /* make entries marked for deletion not findable */ return -1; } CRM_CHECK(a_client->event != NULL && b_client->event != NULL, return 0); rc = strcmp(a_client->event, b_client->event); if (rc == 0) { if (a_client->notify == NULL || b_client->notify == NULL) { return 0; } else if (a_client->notify == b_client->notify) { return 0; } else if (((long)a_client->notify) < ((long)b_client->notify)) { pcmk__err("callbacks for %s are not equal: %p vs. %p", a_client->event, a_client->notify, b_client->notify); return -1; } pcmk__err("callbacks for %s are not equal: %p vs. %p", a_client->event, a_client->notify, b_client->notify); return 1; } return rc; } xmlNode * stonith_create_op(int call_id, const char *token, const char *op, xmlNode * data, int call_options) { xmlNode *op_msg = NULL; CRM_CHECK(token != NULL, return NULL); op_msg = pcmk__xe_create(NULL, PCMK__XE_STONITH_COMMAND); pcmk__xe_set(op_msg, PCMK__XA_T, PCMK__VALUE_STONITH_NG); pcmk__xe_set(op_msg, PCMK__XA_ST_OP, op); pcmk__xe_set_int(op_msg, PCMK__XA_ST_CALLID, call_id); pcmk__trace("Sending call options: %.8lx, %d", (long) call_options, call_options); pcmk__xe_set_int(op_msg, PCMK__XA_ST_CALLOPT, call_options); if (data != NULL) { xmlNode *wrapper = pcmk__xe_create(op_msg, PCMK__XE_ST_CALLDATA); pcmk__xml_copy(wrapper, data); } return op_msg; } static void stonith_destroy_op_callback(gpointer data) { stonith_callback_client_t *blob = data; if (blob->timer && blob->timer->ref > 0) { g_source_remove(blob->timer->ref); } free(blob->timer); free(blob); } static int stonith_api_signoff(stonith_t * stonith) { stonith_private_t *native = stonith->st_private; pcmk__debug("Disconnecting from the fencer"); if (native->source != NULL) { /* Attached to mainloop */ mainloop_del_ipc_client(native->source); native->source = NULL; native->ipc = NULL; } else if (native->ipc) { /* Not attached to mainloop */ crm_ipc_t *ipc = native->ipc; native->ipc = NULL; crm_ipc_close(ipc); crm_ipc_destroy(ipc); } free(native->token); native->token = NULL; stonith->state = stonith_disconnected; return pcmk_ok; } static int stonith_api_del_callback(stonith_t * stonith, int call_id, bool all_callbacks) { stonith_private_t *private = stonith->st_private; if (all_callbacks) { private->op_callback = NULL; g_hash_table_destroy(private->stonith_op_callback_table); private->stonith_op_callback_table = pcmk__intkey_table(stonith_destroy_op_callback); } else if (call_id == 0) { private->op_callback = NULL; } else { pcmk__intkey_table_remove(private->stonith_op_callback_table, call_id); } return pcmk_ok; } /*! * \internal * \brief Invoke a (single) specified fence action callback * * \param[in,out] st Fencer API connection * \param[in] call_id If positive, call ID of completed fence action, * otherwise legacy return code for early failure * \param[in,out] result Full result for action * \param[in,out] userdata User data to pass to callback * \param[in] callback Fence action callback to invoke */ static void invoke_fence_action_callback(stonith_t *st, int call_id, pcmk__action_result_t *result, void *userdata, void (*callback) (stonith_t *st, stonith_callback_data_t *data)) { stonith_callback_data_t data = { 0, }; data.call_id = call_id; data.rc = pcmk_rc2legacy(stonith__result2rc(result)); data.userdata = userdata; data.opaque = (void *) result; callback(st, &data); } /*! * \internal * \brief Invoke any callbacks registered for a specified fence action result * * Given a fence action result from the fencer, invoke any callback registered * for that action, as well as any global callback registered. * * \param[in,out] stonith Fencer API connection * \param[in] msg If non-NULL, fencer reply * \param[in] call_id If \p msg is NULL, call ID of action that timed out */ static void invoke_registered_callbacks(stonith_t *stonith, const xmlNode *msg, int call_id) { stonith_private_t *private = NULL; stonith_callback_client_t *cb_info = NULL; pcmk__action_result_t result = PCMK__UNKNOWN_RESULT; CRM_CHECK(stonith != NULL, return); CRM_CHECK(stonith->st_private != NULL, return); private = stonith->st_private; if (msg == NULL) { // Fencer didn't reply in time pcmk__set_result(&result, CRM_EX_ERROR, PCMK_EXEC_TIMEOUT, "Fencer accepted request but did not reply in time"); CRM_LOG_ASSERT(call_id > 0); } else { // We have the fencer reply if ((pcmk__xe_get_int(msg, PCMK__XA_ST_CALLID, &call_id) != pcmk_rc_ok) || (call_id <= 0)) { pcmk__log_xml_warn(msg, "Bad fencer reply"); } stonith__xe_get_result(msg, &result); } if (call_id > 0) { cb_info = pcmk__intkey_table_lookup(private->stonith_op_callback_table, call_id); } if ((cb_info != NULL) && (cb_info->callback != NULL) && (pcmk__result_ok(&result) || !(cb_info->only_success))) { pcmk__trace("Invoking callback %s for call %d", pcmk__s(cb_info->id, "without ID"), call_id); invoke_fence_action_callback(stonith, call_id, &result, cb_info->user_data, cb_info->callback); } else if ((private->op_callback == NULL) && !pcmk__result_ok(&result)) { pcmk__warn("Fencing action without registered callback failed: %d " "(%s%s%s)", result.exit_status, pcmk_exec_status_str(result.execution_status), ((result.exit_reason != NULL)? ": " : ""), pcmk__s(result.exit_reason, "")); pcmk__log_xml_debug(msg, "Failed fence update"); } if (private->op_callback != NULL) { pcmk__trace("Invoking global callback for call %d", call_id); invoke_fence_action_callback(stonith, call_id, &result, NULL, private->op_callback); } if (cb_info != NULL) { stonith_api_del_callback(stonith, call_id, FALSE); } pcmk__reset_result(&result); } static gboolean stonith_async_timeout_handler(gpointer data) { struct timer_rec_s *timer = data; pcmk__err("Async call %d timed out after %dms", timer->call_id, timer->timeout); invoke_registered_callbacks(timer->stonith, NULL, timer->call_id); /* Always return TRUE, never remove the handler * We do that in stonith_del_callback() */ return TRUE; } static void set_callback_timeout(stonith_callback_client_t * callback, stonith_t * stonith, int call_id, int timeout) { struct timer_rec_s *async_timer = callback->timer; if (timeout <= 0) { return; } if (!async_timer) { async_timer = pcmk__assert_alloc(1, sizeof(struct timer_rec_s)); callback->timer = async_timer; } async_timer->stonith = stonith; async_timer->call_id = call_id; /* Allow a fair bit of grace to allow the server to tell us of a timeout * This is only a fallback */ async_timer->timeout = (timeout + 60) * 1000; if (async_timer->ref) { g_source_remove(async_timer->ref); } async_timer->ref = pcmk__create_timer(async_timer->timeout, stonith_async_timeout_handler, async_timer); } static void update_callback_timeout(int call_id, int timeout, stonith_t * st) { stonith_callback_client_t *callback = NULL; stonith_private_t *private = st->st_private; callback = pcmk__intkey_table_lookup(private->stonith_op_callback_table, call_id); if (!callback || !callback->allow_timeout_updates) { return; } set_callback_timeout(callback, st, call_id, timeout); } static int stonith_dispatch_internal(const char *buffer, ssize_t length, gpointer userdata) { const char *type = NULL; struct notify_blob_s blob; stonith_t *st = userdata; stonith_private_t *private = NULL; pcmk__assert(st != NULL); private = st->st_private; blob.stonith = st; blob.xml = pcmk__xml_parse(buffer); if (blob.xml == NULL) { pcmk__warn("Received malformed message from fencer: %s", buffer); return 0; } /* do callbacks */ type = pcmk__xe_get(blob.xml, PCMK__XA_T); pcmk__trace("Activating %s callbacks...", type); if (pcmk__str_eq(type, PCMK__VALUE_STONITH_NG, pcmk__str_none)) { invoke_registered_callbacks(st, blob.xml, 0); } else if (pcmk__str_eq(type, PCMK__VALUE_ST_NOTIFY, pcmk__str_none)) { foreach_notify_entry(private, stonith_send_notification, &blob); } else if (pcmk__str_eq(type, PCMK__VALUE_ST_ASYNC_TIMEOUT_VALUE, pcmk__str_none)) { int call_id = 0; int timeout = 0; pcmk__xe_get_int(blob.xml, PCMK__XA_ST_TIMEOUT, &timeout); pcmk__xe_get_int(blob.xml, PCMK__XA_ST_CALLID, &call_id); update_callback_timeout(call_id, timeout, st); } else { pcmk__err("Unknown message type: %s", type); pcmk__log_xml_warn(blob.xml, "BadReply"); } pcmk__xml_free(blob.xml); return 1; } static int stonith_api_signon(stonith_t * stonith, const char *name, int *stonith_fd) { int rc = pcmk_ok; stonith_private_t *native = NULL; const char *display_name = name? name : "client"; struct ipc_client_callbacks st_callbacks = { .dispatch = stonith_dispatch_internal, .destroy = stonith_connection_destroy }; CRM_CHECK(stonith != NULL, return -EINVAL); native = stonith->st_private; pcmk__assert(native != NULL); pcmk__debug("Attempting fencer connection by %s with%s mainloop", display_name, ((stonith_fd != 0)? "out" : "")); stonith->state = stonith_connected_command; if (stonith_fd) { /* No mainloop */ native->ipc = crm_ipc_new("stonith-ng", 0); if (native->ipc != NULL) { rc = pcmk__connect_generic_ipc(native->ipc); if (rc == pcmk_rc_ok) { rc = pcmk__ipc_fd(native->ipc, stonith_fd); if (rc != pcmk_rc_ok) { pcmk__debug("Couldn't get file descriptor for IPC: %s", pcmk_rc_str(rc)); } } if (rc != pcmk_rc_ok) { crm_ipc_close(native->ipc); crm_ipc_destroy(native->ipc); native->ipc = NULL; } } } else { /* With mainloop */ native->source = mainloop_add_ipc_client("stonith-ng", G_PRIORITY_MEDIUM, 0, stonith, &st_callbacks); native->ipc = mainloop_get_ipc_client(native->source); } if (native->ipc == NULL) { rc = -ENOTCONN; } else { xmlNode *reply = NULL; xmlNode *hello = pcmk__xe_create(NULL, PCMK__XE_STONITH_COMMAND); pcmk__xe_set(hello, PCMK__XA_T, PCMK__VALUE_STONITH_NG); pcmk__xe_set(hello, PCMK__XA_ST_OP, CRM_OP_REGISTER); pcmk__xe_set(hello, PCMK__XA_ST_CLIENTNAME, name); rc = crm_ipc_send(native->ipc, hello, crm_ipc_client_response, -1, &reply); if (rc < 0) { pcmk__debug("Couldn't register with the fencer: %s " QB_XS " rc=%d", pcmk_strerror(rc), rc); rc = -ECOMM; } else if (reply == NULL) { pcmk__debug("Couldn't register with the fencer: no reply"); rc = -EPROTO; } else { const char *msg_type = pcmk__xe_get(reply, PCMK__XA_ST_OP); native->token = pcmk__xe_get_copy(reply, PCMK__XA_ST_CLIENTID); if (!pcmk__str_eq(msg_type, CRM_OP_REGISTER, pcmk__str_none)) { pcmk__debug("Couldn't register with the fencer: invalid reply " "type '%s'", pcmk__s(msg_type, "(missing)")); pcmk__log_xml_debug(reply, "Invalid fencer reply"); rc = -EPROTO; } else if (native->token == NULL) { pcmk__debug("Couldn't register with the fencer: no token in " "reply"); pcmk__log_xml_debug(reply, "Invalid fencer reply"); rc = -EPROTO; } else { pcmk__debug("Connection to fencer by %s succeeded " "(registration token: %s)", display_name, native->token); rc = pcmk_ok; } } pcmk__xml_free(reply); pcmk__xml_free(hello); } if (rc != pcmk_ok) { pcmk__debug("Connection attempt to fencer by %s failed: %s " QB_XS " rc=%d", display_name, pcmk_strerror(rc), rc); stonith->cmds->disconnect(stonith); } return rc; } static int stonith_set_notification(stonith_t * stonith, const char *callback, int enabled) { int rc = pcmk_ok; xmlNode *notify_msg = pcmk__xe_create(NULL, __func__); stonith_private_t *native = stonith->st_private; if (stonith->state != stonith_disconnected) { pcmk__xe_set(notify_msg, PCMK__XA_ST_OP, STONITH_OP_NOTIFY); if (enabled) { pcmk__xe_set(notify_msg, PCMK__XA_ST_NOTIFY_ACTIVATE, callback); } else { pcmk__xe_set(notify_msg, PCMK__XA_ST_NOTIFY_DEACTIVATE, callback); } rc = crm_ipc_send(native->ipc, notify_msg, crm_ipc_client_response, -1, NULL); if (rc < 0) { crm_perror(LOG_DEBUG, "Couldn't register for fencing notifications: %d", rc); rc = -ECOMM; } else { rc = pcmk_ok; } } pcmk__xml_free(notify_msg); return rc; } static int stonith_api_add_notification(stonith_t * stonith, const char *event, void (*callback) (stonith_t * stonith, stonith_event_t * e)) { GList *list_item = NULL; stonith_notify_client_t *new_client = NULL; stonith_private_t *private = NULL; private = stonith->st_private; pcmk__trace("Adding callback for %s events (%u)", event, g_list_length(private->notify_list)); new_client = pcmk__assert_alloc(1, sizeof(stonith_notify_client_t)); new_client->event = event; new_client->notify = callback; list_item = g_list_find_custom(private->notify_list, new_client, stonithlib_GCompareFunc); if (list_item != NULL) { pcmk__warn("Callback already present"); free(new_client); return -ENOTUNIQ; } else { private->notify_list = g_list_append(private->notify_list, new_client); stonith_set_notification(stonith, event, 1); pcmk__trace("Callback added (%u)", g_list_length(private->notify_list)); } return pcmk_ok; } static void del_notify_entry(gpointer data, gpointer user_data) { stonith_notify_client_t *entry = data; stonith_t * stonith = user_data; if (!entry->delete) { pcmk__debug("Removing callback for %s events", entry->event); stonith_api_del_notification(stonith, entry->event); } } static int stonith_api_del_notification(stonith_t * stonith, const char *event) { GList *list_item = NULL; stonith_notify_client_t *new_client = NULL; stonith_private_t *private = stonith->st_private; if (event == NULL) { foreach_notify_entry(private, del_notify_entry, stonith); pcmk__trace("Removed callback"); return pcmk_ok; } pcmk__debug("Removing callback for %s events", event); new_client = pcmk__assert_alloc(1, sizeof(stonith_notify_client_t)); new_client->event = event; new_client->notify = NULL; list_item = g_list_find_custom(private->notify_list, new_client, stonithlib_GCompareFunc); stonith_set_notification(stonith, event, 0); if (list_item != NULL) { stonith_notify_client_t *list_client = list_item->data; if (private->notify_refcnt) { list_client->delete = TRUE; private->notify_deletes = TRUE; } else { private->notify_list = g_list_remove(private->notify_list, list_client); free(list_client); } pcmk__trace("Removed callback"); } else { pcmk__trace("Callback not present"); } free(new_client); return pcmk_ok; } static int stonith_api_add_callback(stonith_t * stonith, int call_id, int timeout, int options, void *user_data, const char *callback_name, void (*callback) (stonith_t * st, stonith_callback_data_t * data)) { stonith_callback_client_t *blob = NULL; stonith_private_t *private = NULL; CRM_CHECK(stonith != NULL, return -EINVAL); CRM_CHECK(stonith->st_private != NULL, return -EINVAL); private = stonith->st_private; if (call_id == 0) { // Add global callback private->op_callback = callback; } else if (call_id < 0) { // Call failed immediately, so call callback now if (!(options & st_opt_report_only_success)) { pcmk__action_result_t result = PCMK__UNKNOWN_RESULT; pcmk__trace("Call failed, calling %s: %s", callback_name, pcmk_strerror(call_id)); pcmk__set_result(&result, CRM_EX_ERROR, stonith__legacy2status(call_id), NULL); invoke_fence_action_callback(stonith, call_id, &result, user_data, callback); } else { pcmk__warn("Fencer call failed: %s", pcmk_strerror(call_id)); } return FALSE; } blob = pcmk__assert_alloc(1, sizeof(stonith_callback_client_t)); blob->id = callback_name; blob->only_success = (options & st_opt_report_only_success) ? TRUE : FALSE; blob->user_data = user_data; blob->callback = callback; blob->allow_timeout_updates = (options & st_opt_timeout_updates) ? TRUE : FALSE; if (timeout > 0) { set_callback_timeout(blob, stonith, call_id, timeout); } pcmk__intkey_table_insert(private->stonith_op_callback_table, call_id, blob); pcmk__trace("Added callback to %s for call %d", callback_name, call_id); return TRUE; } static void stonith_dump_pending_op(gpointer key, gpointer value, gpointer user_data) { int call = GPOINTER_TO_INT(key); stonith_callback_client_t *blob = value; pcmk__debug("Call %d (%s): pending", call, pcmk__s(blob->id, "no ID")); } void stonith_dump_pending_callbacks(stonith_t * stonith) { stonith_private_t *private = stonith->st_private; if (private->stonith_op_callback_table == NULL) { return; } return g_hash_table_foreach(private->stonith_op_callback_table, stonith_dump_pending_op, NULL); } /*! * \internal * \brief Get the data section of a fencer notification * * \param[in] msg Notification XML * \param[in] ntype Notification type */ static xmlNode * get_event_data_xml(xmlNode *msg, const char *ntype) { char *data_addr = pcmk__assert_asprintf("//%s", ntype); xmlNode *data = pcmk__xpath_find_one(msg->doc, data_addr, LOG_DEBUG); free(data_addr); return data; } /* */ static stonith_event_t * xml_to_event(xmlNode *msg) { stonith_event_t *event = pcmk__assert_alloc(1, sizeof(stonith_event_t)); struct event_private *event_private = NULL; event->opaque = pcmk__assert_alloc(1, sizeof(struct event_private)); event_private = (struct event_private *) event->opaque; - crm_log_xml_trace(msg, "stonith_notify"); + pcmk__log_xml_trace(msg, "stonith_notify"); // All notification types have the operation result and notification subtype stonith__xe_get_result(msg, &event_private->result); event->operation = pcmk__xe_get_copy(msg, PCMK__XA_ST_OP); // @COMPAT The API originally provided the result as a legacy return code event->result = pcmk_rc2legacy(stonith__result2rc(&event_private->result)); // Some notification subtypes have additional information if (pcmk__str_eq(event->operation, PCMK__VALUE_ST_NOTIFY_FENCE, pcmk__str_none)) { xmlNode *data = get_event_data_xml(msg, event->operation); if (data == NULL) { pcmk__err("No data for %s event", event->operation); pcmk__log_xml_notice(msg, "BadEvent"); } else { event->origin = pcmk__xe_get_copy(data, PCMK__XA_ST_ORIGIN); event->action = pcmk__xe_get_copy(data, PCMK__XA_ST_DEVICE_ACTION); event->target = pcmk__xe_get_copy(data, PCMK__XA_ST_TARGET); event->executioner = pcmk__xe_get_copy(data, PCMK__XA_ST_DELEGATE); event->id = pcmk__xe_get_copy(data, PCMK__XA_ST_REMOTE_OP); event->client_origin = pcmk__xe_get_copy(data, PCMK__XA_ST_CLIENTNAME); event->device = pcmk__xe_get_copy(data, PCMK__XA_ST_DEVICE_ID); } } else if (pcmk__str_any_of(event->operation, STONITH_OP_DEVICE_ADD, STONITH_OP_DEVICE_DEL, STONITH_OP_LEVEL_ADD, STONITH_OP_LEVEL_DEL, NULL)) { xmlNode *data = get_event_data_xml(msg, event->operation); if (data == NULL) { pcmk__err("No data for %s event", event->operation); pcmk__log_xml_notice(msg, "BadEvent"); } else { event->device = pcmk__xe_get_copy(data, PCMK__XA_ST_DEVICE_ID); } } return event; } static void event_free(stonith_event_t * event) { struct event_private *event_private = event->opaque; free(event->id); free(event->operation); free(event->origin); free(event->action); free(event->target); free(event->executioner); free(event->device); free(event->client_origin); pcmk__reset_result(&event_private->result); free(event->opaque); free(event); } static void stonith_send_notification(gpointer data, gpointer user_data) { struct notify_blob_s *blob = user_data; stonith_notify_client_t *entry = data; stonith_event_t *st_event = NULL; const char *event = NULL; if (blob->xml == NULL) { pcmk__warn("Skipping callback - NULL message"); return; } event = pcmk__xe_get(blob->xml, PCMK__XA_SUBT); if (entry == NULL) { pcmk__warn("Skipping callback - NULL callback client"); return; } else if (entry->delete) { pcmk__trace("Skipping callback - marked for deletion"); return; } else if (entry->notify == NULL) { pcmk__warn("Skipping callback - NULL callback"); return; } else if (!pcmk__str_eq(entry->event, event, pcmk__str_none)) { pcmk__trace("Skipping callback - event mismatch %p/%s vs. %s", entry, entry->event, event); return; } st_event = xml_to_event(blob->xml); pcmk__trace("Invoking callback for %p/%s event...", entry, event); entry->notify(blob->stonith, st_event); pcmk__trace("Callback invoked..."); event_free(st_event); } /*! * \internal * \brief Create and send an API request * * \param[in,out] stonith Stonith connection * \param[in] op API operation to request * \param[in] data Data to attach to request * \param[out] output_data If not NULL, will be set to reply if synchronous * \param[in] call_options Bitmask of stonith_call_options to use * \param[in] timeout Error if not completed within this many seconds * * \return pcmk_ok (for synchronous requests) or positive call ID * (for asynchronous requests) on success, -errno otherwise */ static int stonith_send_command(stonith_t * stonith, const char *op, xmlNode * data, xmlNode ** output_data, int call_options, int timeout) { int rc = 0; int reply_id = -1; xmlNode *op_msg = NULL; xmlNode *op_reply = NULL; stonith_private_t *native = NULL; pcmk__assert((stonith != NULL) && (stonith->st_private != NULL) && (op != NULL)); native = stonith->st_private; if (output_data != NULL) { *output_data = NULL; } if ((stonith->state == stonith_disconnected) || (native->token == NULL)) { return -ENOTCONN; } /* Increment the call ID, which must be positive to avoid conflicting with * error codes. This shouldn't be a problem unless the client mucked with * it or the counter wrapped around. */ stonith->call_id++; if (stonith->call_id < 1) { stonith->call_id = 1; } op_msg = stonith_create_op(stonith->call_id, native->token, op, data, call_options); if (op_msg == NULL) { return -EINVAL; } pcmk__xe_set_int(op_msg, PCMK__XA_ST_TIMEOUT, timeout); pcmk__trace("Sending %s message to fencer with timeout %ds", op, timeout); if (data) { const char *delay_s = pcmk__xe_get(data, PCMK__XA_ST_DELAY); if (delay_s) { pcmk__xe_set(op_msg, PCMK__XA_ST_DELAY, delay_s); } } { enum crm_ipc_flags ipc_flags = crm_ipc_flags_none; if (call_options & st_opt_sync_call) { pcmk__set_ipc_flags(ipc_flags, "stonith command", crm_ipc_client_response); } rc = crm_ipc_send(native->ipc, op_msg, ipc_flags, 1000 * (timeout + 60), &op_reply); } pcmk__xml_free(op_msg); if (rc < 0) { crm_perror(LOG_ERR, "Couldn't perform %s operation (timeout=%ds): %d", op, timeout, rc); rc = -ECOMM; goto done; } - crm_log_xml_trace(op_reply, "Reply"); + pcmk__log_xml_trace(op_reply, "Reply"); if (!(call_options & st_opt_sync_call)) { pcmk__trace("Async call %d, returning", stonith->call_id); pcmk__xml_free(op_reply); return stonith->call_id; } pcmk__xe_get_int(op_reply, PCMK__XA_ST_CALLID, &reply_id); if (reply_id == stonith->call_id) { pcmk__action_result_t result = PCMK__UNKNOWN_RESULT; pcmk__trace("Synchronous reply %d received", reply_id); stonith__xe_get_result(op_reply, &result); rc = pcmk_rc2legacy(stonith__result2rc(&result)); pcmk__reset_result(&result); if ((call_options & st_opt_discard_reply) || output_data == NULL) { pcmk__trace("Discarding reply"); } else { *output_data = op_reply; op_reply = NULL; /* Prevent subsequent free */ } } else if (reply_id <= 0) { pcmk__err("Received bad reply: No id set"); pcmk__log_xml_err(op_reply, "Bad reply"); pcmk__xml_free(op_reply); op_reply = NULL; rc = -ENOMSG; } else { pcmk__err("Received bad reply: %d (wanted %d)", reply_id, stonith->call_id); pcmk__log_xml_err(op_reply, "Old reply"); pcmk__xml_free(op_reply); op_reply = NULL; rc = -ENOMSG; } done: if (!crm_ipc_connected(native->ipc)) { pcmk__err("Fencer disconnected"); free(native->token); native->token = NULL; stonith->state = stonith_disconnected; } pcmk__xml_free(op_reply); return rc; } /* Not used with mainloop */ bool stonith_dispatch(stonith_t * st) { gboolean stay_connected = TRUE; stonith_private_t *private = NULL; pcmk__assert(st != NULL); private = st->st_private; while (crm_ipc_ready(private->ipc)) { if (crm_ipc_read(private->ipc) > 0) { const char *msg = crm_ipc_buffer(private->ipc); stonith_dispatch_internal(msg, strlen(msg), st); } if (!crm_ipc_connected(private->ipc)) { pcmk__err("Connection closed"); stay_connected = FALSE; } } return stay_connected; } static int stonith_api_free(stonith_t * stonith) { int rc = pcmk_ok; pcmk__trace("Destroying %p", stonith); if (stonith->state != stonith_disconnected) { pcmk__trace("Unregistering notifications and disconnecting %p first", stonith); stonith->cmds->remove_notification(stonith, NULL); rc = stonith->cmds->disconnect(stonith); } if (stonith->state == stonith_disconnected) { stonith_private_t *private = stonith->st_private; pcmk__trace("Removing %u callbacks", g_hash_table_size(private->stonith_op_callback_table)); g_hash_table_destroy(private->stonith_op_callback_table); pcmk__trace("Destroying %u notification clients", g_list_length(private->notify_list)); g_list_free_full(private->notify_list, free); free(stonith->st_private); free(stonith->cmds); free(stonith); } else { pcmk__err("Not free'ing active connection: %s (%d)", pcmk_strerror(rc), rc); } return rc; } void stonith_api_delete(stonith_t * stonith) { pcmk__trace("Destroying %p", stonith); if(stonith) { stonith->cmds->free(stonith); } } static gboolean is_stonith_param(gpointer key, gpointer value, gpointer user_data) { return pcmk_stonith_param(key); } int stonith__validate(stonith_t *st, int call_options, const char *rsc_id, const char *namespace_s, const char *agent, GHashTable *params, int timeout_sec, char **output, char **error_output) { int rc = pcmk_rc_ok; /* Use a dummy node name in case the agent requires a target. We assume the * actual target doesn't matter for validation purposes (if in practice, * that is incorrect, we will need to allow the caller to pass the target). */ const char *target = "node1"; char *host_arg = NULL; if (params != NULL) { host_arg = pcmk__str_copy(g_hash_table_lookup(params, PCMK_STONITH_HOST_ARGUMENT)); /* Remove special stonith params from the table before doing anything else */ g_hash_table_foreach_remove(params, is_stonith_param, NULL); } #if PCMK__ENABLE_CIBSECRETS rc = pcmk__substitute_secrets(rsc_id, params); if (rc != pcmk_rc_ok) { pcmk__warn("Could not replace secret parameters for validation of %s: " "%s", agent, pcmk_rc_str(rc)); // rc is standard return value, don't return it in this function } #endif if (output) { *output = NULL; } if (error_output) { *error_output = NULL; } if (timeout_sec <= 0) { timeout_sec = PCMK_DEFAULT_ACTION_TIMEOUT_MS; } switch (stonith_get_namespace(agent, namespace_s)) { case st_namespace_rhcs: rc = stonith__rhcs_validate(st, call_options, target, agent, params, host_arg, timeout_sec, output, error_output); rc = pcmk_legacy2rc(rc); break; #if HAVE_STONITH_STONITH_H case st_namespace_lha: rc = stonith__lha_validate(st, call_options, target, agent, params, timeout_sec, output, error_output); rc = pcmk_legacy2rc(rc); break; #endif case st_namespace_invalid: errno = ENOENT; rc = errno; if (error_output) { *error_output = pcmk__assert_asprintf("Agent %s not found", agent); } else { pcmk__err("Agent %s not found", agent); } break; default: errno = EOPNOTSUPP; rc = errno; if (error_output) { *error_output = pcmk__assert_asprintf("Agent %s does not " "support validation", agent); } else { pcmk__err("Agent %s does not support validation", agent); } break; } free(host_arg); return rc; } static int stonith_api_validate(stonith_t *st, int call_options, const char *rsc_id, const char *namespace_s, const char *agent, const stonith_key_value_t *params, int timeout_sec, char **output, char **error_output) { /* Validation should be done directly via the agent, so we can get it from * stonith_admin when the cluster is not running, which is important for * higher-level tools. */ int rc = pcmk_ok; GHashTable *params_table = pcmk__strkey_table(free, free); // Convert parameter list to a hash table for (; params; params = params->next) { if (!pcmk_stonith_param(params->key)) { pcmk__insert_dup(params_table, params->key, params->value); } } rc = stonith__validate(st, call_options, rsc_id, namespace_s, agent, params_table, timeout_sec, output, error_output); g_hash_table_destroy(params_table); return rc; } stonith_t * stonith_api_new(void) { stonith_t *new_stonith = NULL; stonith_private_t *private = NULL; new_stonith = calloc(1, sizeof(stonith_t)); if (new_stonith == NULL) { return NULL; } private = calloc(1, sizeof(stonith_private_t)); if (private == NULL) { free(new_stonith); return NULL; } new_stonith->st_private = private; private->stonith_op_callback_table = pcmk__intkey_table(stonith_destroy_op_callback); private->notify_list = NULL; private->notify_refcnt = 0; private->notify_deletes = FALSE; new_stonith->call_id = 1; new_stonith->state = stonith_disconnected; new_stonith->cmds = calloc(1, sizeof(stonith_api_operations_t)); if (new_stonith->cmds == NULL) { free(new_stonith->st_private); free(new_stonith); return NULL; } /* *INDENT-OFF* */ new_stonith->cmds->free = stonith_api_free; new_stonith->cmds->connect = stonith_api_signon; new_stonith->cmds->disconnect = stonith_api_signoff; new_stonith->cmds->list = stonith_api_list; new_stonith->cmds->monitor = stonith_api_monitor; new_stonith->cmds->status = stonith_api_status; new_stonith->cmds->fence = stonith_api_fence; new_stonith->cmds->fence_with_delay = stonith_api_fence_with_delay; new_stonith->cmds->confirm = stonith_api_confirm; new_stonith->cmds->history = stonith_api_history; new_stonith->cmds->list_agents = stonith_api_device_list; new_stonith->cmds->metadata = stonith_api_device_metadata; new_stonith->cmds->query = stonith_api_query; new_stonith->cmds->remove_device = stonith_api_remove_device; new_stonith->cmds->register_device = stonith_api_register_device; new_stonith->cmds->remove_level = stonith_api_remove_level; new_stonith->cmds->remove_level_full = stonith_api_remove_level_full; new_stonith->cmds->register_level = stonith_api_register_level; new_stonith->cmds->register_level_full = stonith_api_register_level_full; new_stonith->cmds->remove_callback = stonith_api_del_callback; new_stonith->cmds->register_callback = stonith_api_add_callback; new_stonith->cmds->remove_notification = stonith_api_del_notification; new_stonith->cmds->register_notification = stonith_api_add_notification; new_stonith->cmds->validate = stonith_api_validate; /* *INDENT-ON* */ return new_stonith; } /*! * \brief Make a blocking connection attempt to the fencer * * \param[in,out] st Fencer API object * \param[in] name Client name to use with fencer * \param[in] max_attempts Return error if this many attempts fail * * \return pcmk_ok on success, result of last attempt otherwise */ int stonith_api_connect_retry(stonith_t *st, const char *name, int max_attempts) { int rc = -EINVAL; // if max_attempts is not positive for (int attempt = 1; attempt <= max_attempts; attempt++) { rc = st->cmds->connect(st, name, NULL); if (rc == pcmk_ok) { return pcmk_ok; } else if (attempt < max_attempts) { pcmk__notice("Fencer connection attempt %d of %d failed (retrying " "in 2s): %s " QB_XS " rc=%d", attempt, max_attempts, pcmk_strerror(rc), rc); sleep(2); } } pcmk__notice("Could not connect to fencer: %s " QB_XS " rc=%d", pcmk_strerror(rc), rc); return rc; } stonith_key_value_t * stonith_key_value_add(stonith_key_value_t * head, const char *key, const char *value) { stonith_key_value_t *p, *end; p = pcmk__assert_alloc(1, sizeof(stonith_key_value_t)); p->key = pcmk__str_copy(key); p->value = pcmk__str_copy(value); end = head; while (end && end->next) { end = end->next; } if (end) { end->next = p; } else { head = p; } return head; } void stonith_key_value_freeall(stonith_key_value_t * head, int keys, int values) { stonith_key_value_t *p; while (head) { p = head->next; if (keys) { free(head->key); } if (values) { free(head->value); } free(head); head = p; } } #define api_log_open() openlog("stonith-api", LOG_CONS | LOG_NDELAY | LOG_PID, LOG_DAEMON) #define api_log(level, fmt, args...) syslog(level, "%s: "fmt, __func__, args) int stonith_api_kick(uint32_t nodeid, const char *uname, int timeout, bool off) { int rc = pcmk_ok; stonith_t *st = stonith_api_new(); const char *action = off? PCMK_ACTION_OFF : PCMK_ACTION_REBOOT; api_log_open(); if (st == NULL) { api_log(LOG_ERR, "API initialization failed, could not kick (%s) node %u/%s", action, nodeid, uname); return -EPROTO; } rc = st->cmds->connect(st, "stonith-api", NULL); if (rc != pcmk_ok) { api_log(LOG_ERR, "Connection failed, could not kick (%s) node %u/%s : %s (%d)", action, nodeid, uname, pcmk_strerror(rc), rc); } else { char *name = (uname == NULL)? pcmk__itoa(nodeid) : strdup(uname); int opts = 0; stonith__set_call_options(opts, name, st_opt_sync_call|st_opt_allow_self_fencing); if ((uname == NULL) && (nodeid > 0)) { stonith__set_call_options(opts, name, st_opt_cs_nodeid); } rc = st->cmds->fence(st, opts, name, action, timeout, 0); free(name); if (rc != pcmk_ok) { api_log(LOG_ERR, "Could not kick (%s) node %u/%s : %s (%d)", action, nodeid, uname, pcmk_strerror(rc), rc); } else { api_log(LOG_NOTICE, "Node %u/%s kicked: %s", nodeid, uname, action); } } stonith_api_delete(st); return rc; } time_t stonith_api_time(uint32_t nodeid, const char *uname, bool in_progress) { int rc = pcmk_ok; time_t when = 0; stonith_t *st = stonith_api_new(); stonith_history_t *history = NULL, *hp = NULL; if (st == NULL) { api_log(LOG_ERR, "Could not retrieve fence history for %u/%s: " "API initialization failed", nodeid, uname); return when; } rc = st->cmds->connect(st, "stonith-api", NULL); if (rc != pcmk_ok) { api_log(LOG_NOTICE, "Connection failed: %s (%d)", pcmk_strerror(rc), rc); } else { int entries = 0; int progress = 0; int completed = 0; int opts = 0; char *name = (uname == NULL)? pcmk__itoa(nodeid) : strdup(uname); stonith__set_call_options(opts, name, st_opt_sync_call); if ((uname == NULL) && (nodeid > 0)) { stonith__set_call_options(opts, name, st_opt_cs_nodeid); } rc = st->cmds->history(st, opts, name, &history, 120); free(name); for (hp = history; hp; hp = hp->next) { entries++; if (in_progress) { progress++; if (hp->state != st_done && hp->state != st_failed) { when = time(NULL); } } else if (hp->state == st_done) { completed++; if (hp->completed > when) { when = hp->completed; } } } stonith_history_free(history); if(rc == pcmk_ok) { api_log(LOG_INFO, "Found %d entries for %u/%s: %d in progress, %d completed", entries, nodeid, uname, progress, completed); } else { api_log(LOG_ERR, "Could not retrieve fence history for %u/%s: %s (%d)", nodeid, uname, pcmk_strerror(rc), rc); } } stonith_api_delete(st); if(when) { api_log(LOG_INFO, "Node %u/%s last kicked at: %ld", nodeid, uname, (long int)when); } return when; } bool stonith_agent_exists(const char *agent, int timeout) { stonith_t *st = NULL; stonith_key_value_t *devices = NULL; stonith_key_value_t *dIter = NULL; bool rc = FALSE; if (agent == NULL) { return rc; } st = stonith_api_new(); if (st == NULL) { pcmk__err("Could not list fence agents: API memory allocation failed"); return FALSE; } st->cmds->list_agents(st, st_opt_sync_call, NULL, &devices, timeout == 0 ? 120 : timeout); for (dIter = devices; dIter != NULL; dIter = dIter->next) { if (pcmk__str_eq(dIter->value, agent, pcmk__str_none)) { rc = TRUE; break; } } stonith_key_value_freeall(devices, 1, 1); stonith_api_delete(st); return rc; } const char * stonith_action_str(const char *action) { if (action == NULL) { return "fencing"; } else if (strcmp(action, PCMK_ACTION_ON) == 0) { return "unfencing"; } else if (strcmp(action, PCMK_ACTION_OFF) == 0) { return "turning off"; } else { return action; } } /*! * \internal * \brief Parse a target name from one line of a target list string * * \param[in] line One line of a target list string * \param[in] len String length of line * \param[in,out] output List to add newly allocated target name to */ static void parse_list_line(const char *line, int len, GList **output) { size_t i = 0; size_t entry_start = 0; /* Skip complaints about additional parameters device doesn't understand * * @TODO Document or eliminate the implied restriction of target names */ if (strstr(line, "invalid") || strstr(line, "variable")) { pcmk__debug("Skipping list output line: %s", line); return; } // Process line content, character by character for (i = 0; i <= len; i++) { if (isspace(line[i]) || (line[i] == ',') || (line[i] == ';') || (line[i] == '\0')) { // We've found a separator (i.e. the end of an entry) int rc = 0; char *entry = NULL; if (i == entry_start) { // Skip leading and sequential separators entry_start = i + 1; continue; } entry = pcmk__assert_alloc(i - entry_start + 1, sizeof(char)); /* Read entry, stopping at first separator * * @TODO Document or eliminate these character restrictions */ rc = sscanf(line + entry_start, "%[a-zA-Z0-9_-.]", entry); if (rc != 1) { pcmk__warn("Could not parse list output entry: %s " QB_XS " entry_start=%d position=%d", (line + entry_start), entry_start, i); free(entry); } else if (pcmk__strcase_any_of(entry, PCMK_ACTION_ON, PCMK_ACTION_OFF, NULL)) { /* Some agents print the target status in the list output, * though none are known now (the separate list-status command * is used for this, but it can also print "UNKNOWN"). To handle * this possibility, skip such entries. * * @TODO Document or eliminate the implied restriction of target * names. */ free(entry); } else { // We have a valid entry *output = g_list_append(*output, entry); } entry_start = i + 1; } } } /*! * \internal * \brief Parse a list of targets from a string * * \param[in] list_output Target list as a string * * \return List of target names * \note The target list string format is flexible, to allow for user-specified * lists such pcmk_host_list and the output of an agent's list action * (whether direct or via the API, which escapes newlines). There may be * multiple lines, separated by either a newline or an escaped newline * (backslash n). Each line may have one or more target names, separated * by any combination of whitespace, commas, and semi-colons. Lines * containing "invalid" or "variable" will be ignored entirely. Target * names "on" or "off" (case-insensitive) will be ignored. Target names * may contain only alphanumeric characters, underbars (_), dashes (-), * and dots (.) (if any other character occurs in the name, it and all * subsequent characters in the name will be ignored). * \note The caller is responsible for freeing the result with * g_list_free_full(result, free). */ GList * stonith__parse_targets(const char *target_spec) { GList *targets = NULL; if (target_spec != NULL) { size_t out_len = strlen(target_spec); size_t line_start = 0; // Starting index of line being processed for (size_t i = 0; i <= out_len; ++i) { if ((target_spec[i] == '\n') || (target_spec[i] == '\0') || ((target_spec[i] == '\\') && (target_spec[i + 1] == 'n'))) { // We've reached the end of one line of output int len = i - line_start; if (len > 0) { char *line = strndup(target_spec + line_start, len); line[len] = '\0'; // Because it might be a newline parse_list_line(line, len, &targets); free(line); } if (target_spec[i] == '\\') { ++i; // backslash-n takes up two positions } line_start = i + 1; } } } return targets; } /*! * \internal * \brief Check whether a fencing failure was followed by an equivalent success * * \param[in] event Fencing failure * \param[in] top_history Complete fencing history (must be sorted by * stonith__sort_history() beforehand) * * \return The name of the node that executed the fencing if a later successful * event exists, or NULL if no such event exists */ const char * stonith__later_succeeded(const stonith_history_t *event, const stonith_history_t *top_history) { const char *other = NULL; for (const stonith_history_t *prev_hp = top_history; prev_hp != NULL; prev_hp = prev_hp->next) { if (prev_hp == event) { break; } if ((prev_hp->state == st_done) && pcmk__str_eq(event->target, prev_hp->target, pcmk__str_casei) && pcmk__str_eq(event->action, prev_hp->action, pcmk__str_none) && ((event->completed < prev_hp->completed) || ((event->completed == prev_hp->completed) && (event->completed_nsec < prev_hp->completed_nsec)))) { if ((event->delegate == NULL) || pcmk__str_eq(event->delegate, prev_hp->delegate, pcmk__str_casei)) { // Prefer equivalent fencing by same executioner return prev_hp->delegate; } else if (other == NULL) { // Otherwise remember first successful executioner other = (prev_hp->delegate == NULL)? "some node" : prev_hp->delegate; } } } return other; } /*! * \internal * \brief Sort fencing history, pending first then by most recently completed * * \param[in,out] history List of stonith actions * * \return New head of sorted \p history */ stonith_history_t * stonith__sort_history(stonith_history_t *history) { stonith_history_t *new = NULL, *pending = NULL, *hp, *np, *tmp; for (hp = history; hp; ) { tmp = hp->next; if ((hp->state == st_done) || (hp->state == st_failed)) { /* sort into new */ if ((!new) || (hp->completed > new->completed) || ((hp->completed == new->completed) && (hp->completed_nsec > new->completed_nsec))) { hp->next = new; new = hp; } else { np = new; do { if ((!np->next) || (hp->completed > np->next->completed) || ((hp->completed == np->next->completed) && (hp->completed_nsec > np->next->completed_nsec))) { hp->next = np->next; np->next = hp; break; } np = np->next; } while (1); } } else { /* put into pending */ hp->next = pending; pending = hp; } hp = tmp; } /* pending actions don't have a completed-stamp so make them go front */ if (pending) { stonith_history_t *last_pending = pending; while (last_pending->next) { last_pending = last_pending->next; } last_pending->next = new; new = pending; } return new; } /*! * \brief Return string equivalent of an operation state value * * \param[in] state Fencing operation state value * * \return Human-friendly string equivalent of state */ const char * stonith_op_state_str(enum op_state state) { switch (state) { case st_query: return "querying"; case st_exec: return "executing"; case st_done: return "completed"; case st_duplicate: return "duplicate"; case st_failed: return "failed"; } return "unknown"; } stonith_history_t * stonith__first_matching_event(stonith_history_t *history, bool (*matching_fn)(stonith_history_t *, void *), void *user_data) { for (stonith_history_t *hp = history; hp; hp = hp->next) { if (matching_fn(hp, user_data)) { return hp; } } return NULL; } bool stonith__event_state_pending(stonith_history_t *history, void *user_data) { return history->state != st_failed && history->state != st_done; } bool stonith__event_state_eq(stonith_history_t *history, void *user_data) { return history->state == GPOINTER_TO_INT(user_data); } bool stonith__event_state_neq(stonith_history_t *history, void *user_data) { return history->state != GPOINTER_TO_INT(user_data); } void stonith__device_parameter_flags(uint32_t *device_flags, const char *device_name, xmlNode *metadata) { xmlXPathObject *xpath = NULL; int max = 0; int lpc = 0; CRM_CHECK((device_flags != NULL) && (metadata != NULL), return); xpath = pcmk__xpath_search(metadata->doc, "//" PCMK_XE_PARAMETER); max = pcmk__xpath_num_results(xpath); if (max == 0) { xmlXPathFreeObject(xpath); return; } for (lpc = 0; lpc < max; lpc++) { const char *parameter = NULL; xmlNode *match = pcmk__xpath_result(xpath, lpc); CRM_LOG_ASSERT(match != NULL); if (match == NULL) { continue; } parameter = pcmk__xe_get(match, PCMK_XA_NAME); if (pcmk__str_eq(parameter, "plug", pcmk__str_casei)) { stonith__set_device_flags(*device_flags, device_name, st_device_supports_parameter_plug); } else if (pcmk__str_eq(parameter, "port", pcmk__str_casei)) { stonith__set_device_flags(*device_flags, device_name, st_device_supports_parameter_port); } } xmlXPathFreeObject(xpath); } /*! * \internal * \brief Retrieve fence agent meta-data asynchronously * * \param[in] agent Agent to execute * \param[in] timeout_sec Error if not complete within this time * \param[in] callback Function to call with result (this will always be * called, whether by this function directly or * later via the main loop, and on success the * metadata will be in its result argument's * action_stdout) * \param[in,out] user_data User data to pass to callback * * \return Standard Pacemaker return code * \note The caller must use a main loop. This function is not a * stonith_api_operations_t method because it does not need a stonith_t * object and does not go through the fencer, but executes the agent * directly. */ int stonith__metadata_async(const char *agent, int timeout_sec, void (*callback)(int pid, const pcmk__action_result_t *result, void *user_data), void *user_data) { switch (stonith_get_namespace(agent, NULL)) { case st_namespace_rhcs: { stonith_action_t *action = NULL; int rc = pcmk_ok; action = stonith__action_create(agent, PCMK_ACTION_METADATA, NULL, 0, timeout_sec, NULL, NULL, NULL); rc = stonith__execute_async(action, user_data, callback, NULL); if (rc != pcmk_ok) { callback(0, stonith__action_result(action), user_data); stonith__destroy_action(action); } return pcmk_legacy2rc(rc); } #if HAVE_STONITH_STONITH_H case st_namespace_lha: // LHA metadata is simply synthesized, so simulate async { pcmk__action_result_t result = { .exit_status = CRM_EX_OK, .execution_status = PCMK_EXEC_DONE, .exit_reason = NULL, .action_stdout = NULL, .action_stderr = NULL, }; stonith__lha_metadata(agent, timeout_sec, &result.action_stdout); callback(0, &result, user_data); pcmk__reset_result(&result); return pcmk_rc_ok; } #endif default: { pcmk__action_result_t result = { .exit_status = CRM_EX_NOSUCH, .execution_status = PCMK_EXEC_ERROR_HARD, .exit_reason = pcmk__assert_asprintf("No such agent '%s'", agent), .action_stdout = NULL, .action_stderr = NULL, }; callback(0, &result, user_data); pcmk__reset_result(&result); return ENOENT; } } } /*! * \internal * \brief Return the exit status from an async action callback * * \param[in] data Callback data * * \return Exit status from callback data */ int stonith__exit_status(const stonith_callback_data_t *data) { if ((data == NULL) || (data->opaque == NULL)) { return CRM_EX_ERROR; } return ((pcmk__action_result_t *) data->opaque)->exit_status; } /*! * \internal * \brief Return the execution status from an async action callback * * \param[in] data Callback data * * \return Execution status from callback data */ int stonith__execution_status(const stonith_callback_data_t *data) { if ((data == NULL) || (data->opaque == NULL)) { return PCMK_EXEC_UNKNOWN; } return ((pcmk__action_result_t *) data->opaque)->execution_status; } /*! * \internal * \brief Return the exit reason from an async action callback * * \param[in] data Callback data * * \return Exit reason from callback data */ const char * stonith__exit_reason(const stonith_callback_data_t *data) { if ((data == NULL) || (data->opaque == NULL)) { return NULL; } return ((pcmk__action_result_t *) data->opaque)->exit_reason; } /*! * \internal * \brief Return the exit status from an event notification * * \param[in] event Event * * \return Exit status from event */ int stonith__event_exit_status(const stonith_event_t *event) { if ((event == NULL) || (event->opaque == NULL)) { return CRM_EX_ERROR; } else { struct event_private *event_private = event->opaque; return event_private->result.exit_status; } } /*! * \internal * \brief Return the execution status from an event notification * * \param[in] event Event * * \return Execution status from event */ int stonith__event_execution_status(const stonith_event_t *event) { if ((event == NULL) || (event->opaque == NULL)) { return PCMK_EXEC_UNKNOWN; } else { struct event_private *event_private = event->opaque; return event_private->result.execution_status; } } /*! * \internal * \brief Return the exit reason from an event notification * * \param[in] event Event * * \return Exit reason from event */ const char * stonith__event_exit_reason(const stonith_event_t *event) { if ((event == NULL) || (event->opaque == NULL)) { return NULL; } else { struct event_private *event_private = event->opaque; return event_private->result.exit_reason; } } /*! * \internal * \brief Return a human-friendly description of a fencing event * * \param[in] event Event to describe * * \return Newly allocated string with description of \p event * \note The caller is responsible for freeing the return value. * This function asserts on memory errors and never returns NULL. */ char * stonith__event_description(const stonith_event_t *event) { // Use somewhat readable defaults const char *origin = pcmk__s(event->client_origin, "a client"); const char *origin_node = pcmk__s(event->origin, "a node"); const char *executioner = pcmk__s(event->executioner, "the cluster"); const char *device = pcmk__s(event->device, "unknown"); const char *action = pcmk__s(event->action, event->operation); const char *target = pcmk__s(event->target, "no node"); const char *reason = stonith__event_exit_reason(event); const char *status; if (action == NULL) { action = "(unknown)"; } if (stonith__event_execution_status(event) != PCMK_EXEC_DONE) { status = pcmk_exec_status_str(stonith__event_execution_status(event)); } else if (stonith__event_exit_status(event) != CRM_EX_OK) { status = pcmk_exec_status_str(PCMK_EXEC_ERROR); } else { status = crm_exit_str(CRM_EX_OK); } if (pcmk__str_eq(event->operation, PCMK__VALUE_ST_NOTIFY_HISTORY, pcmk__str_none)) { return pcmk__assert_asprintf("Fencing history may have changed"); } else if (pcmk__str_eq(event->operation, STONITH_OP_DEVICE_ADD, pcmk__str_none)) { return pcmk__assert_asprintf("A fencing device (%s) was added", device); } else if (pcmk__str_eq(event->operation, STONITH_OP_DEVICE_DEL, pcmk__str_none)) { return pcmk__assert_asprintf("A fencing device (%s) was removed", device); } else if (pcmk__str_eq(event->operation, STONITH_OP_LEVEL_ADD, pcmk__str_none)) { return pcmk__assert_asprintf("A fencing topology level (%s) was added", device); } else if (pcmk__str_eq(event->operation, STONITH_OP_LEVEL_DEL, pcmk__str_none)) { return pcmk__assert_asprintf("A fencing topology level (%s) was " "removed", device); } // event->operation should be PCMK__VALUE_ST_NOTIFY_FENCE at this point return pcmk__assert_asprintf("Operation %s of %s by %s for %s@%s: %s%s%s%s " "(ref=%s)", action, target, executioner, origin, origin_node, status, ((reason == NULL)? "" : " ("), pcmk__s(reason, ""), ((reason == NULL)? "" : ")"), pcmk__s(event->id, "(none)")); } diff --git a/lib/lrmd/lrmd_client.c b/lib/lrmd/lrmd_client.c index 777c693dbd..fa8ff81d71 100644 --- a/lib/lrmd/lrmd_client.c +++ b/lib/lrmd/lrmd_client.c @@ -1,2680 +1,2680 @@ /* * Copyright 2012-2025 the Pacemaker project contributors * * The version control history for this file may have further details. * * This source code is licensed under the GNU Lesser General Public License * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. */ #include #include #include #include #include // uint32_t, uint64_t #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include // hash2smartfield(), xml2list() #include #include #include #include #include #include // stonith__* #include #include #include #include #include #include #define MAX_TLS_RECV_WAIT 10000 CRM_TRACE_INIT_DATA(lrmd); static int lrmd_api_disconnect(lrmd_t * lrmd); static int lrmd_api_is_connected(lrmd_t * lrmd); /* IPC proxy functions */ int lrmd_internal_proxy_send(lrmd_t * lrmd, xmlNode *msg); static void lrmd_internal_proxy_dispatch(lrmd_t *lrmd, xmlNode *msg); void lrmd_internal_set_proxy_callback(lrmd_t * lrmd, void *userdata, void (*callback)(lrmd_t *lrmd, void *userdata, xmlNode *msg)); // GnuTLS client handshake timeout in seconds #define TLS_HANDSHAKE_TIMEOUT 5 static void lrmd_tls_disconnect(lrmd_t * lrmd); static int global_remote_msg_id = 0; static void lrmd_tls_connection_destroy(gpointer userdata); static int add_tls_to_mainloop(lrmd_t *lrmd, bool do_api_handshake); typedef struct lrmd_private_s { uint64_t type; char *token; mainloop_io_t *source; /* IPC parameters */ crm_ipc_t *ipc; pcmk__remote_t *remote; /* Extra TLS parameters */ char *remote_nodename; char *server; int port; pcmk__tls_t *tls; /* while the async connection is occurring, this is the id * of the connection timeout timer. */ int async_timer; int sock; /* since tls requires a round trip across the network for a * request/reply, there are times where we just want to be able * to send a request from the client and not wait around (or even care * about) what the reply is. */ int expected_late_replies; GList *pending_notify; crm_trigger_t *process_notify; crm_trigger_t *handshake_trigger; lrmd_event_callback callback; /* Internal IPC proxy msg passing for remote guests */ void (*proxy_callback)(lrmd_t *lrmd, void *userdata, xmlNode *msg); void *proxy_callback_userdata; char *peer_version; } lrmd_private_t; static int process_lrmd_handshake_reply(xmlNode *reply, lrmd_private_t *native); static void report_async_connection_result(lrmd_t * lrmd, int rc); static lrmd_list_t * lrmd_list_add(lrmd_list_t * head, const char *value) { lrmd_list_t *p, *end; p = pcmk__assert_alloc(1, sizeof(lrmd_list_t)); p->val = strdup(value); end = head; while (end && end->next) { end = end->next; } if (end) { end->next = p; } else { head = p; } return head; } void lrmd_list_freeall(lrmd_list_t * head) { lrmd_list_t *p; while (head) { char *val = (char *)head->val; p = head->next; free(val); free(head); head = p; } } lrmd_key_value_t * lrmd_key_value_add(lrmd_key_value_t * head, const char *key, const char *value) { lrmd_key_value_t *p, *end; p = pcmk__assert_alloc(1, sizeof(lrmd_key_value_t)); p->key = strdup(key); p->value = strdup(value); end = head; while (end && end->next) { end = end->next; } if (end) { end->next = p; } else { head = p; } return head; } void lrmd_key_value_freeall(lrmd_key_value_t * head) { lrmd_key_value_t *p; while (head) { p = head->next; free(head->key); free(head->value); free(head); head = p; } } /*! * \brief Create a new lrmd_event_data_t object * * \param[in] rsc_id ID of resource involved in event * \param[in] task Action name * \param[in] interval_ms Action interval * * \return Newly allocated and initialized lrmd_event_data_t * \note This functions asserts on memory errors, so the return value is * guaranteed to be non-NULL. The caller is responsible for freeing the * result with lrmd_free_event(). */ lrmd_event_data_t * lrmd_new_event(const char *rsc_id, const char *task, guint interval_ms) { lrmd_event_data_t *event = pcmk__assert_alloc(1, sizeof(lrmd_event_data_t)); // lrmd_event_data_t has (const char *) members that lrmd_free_event() frees event->rsc_id = pcmk__str_copy(rsc_id); event->op_type = pcmk__str_copy(task); event->interval_ms = interval_ms; return event; } lrmd_event_data_t * lrmd_copy_event(lrmd_event_data_t * event) { lrmd_event_data_t *copy = NULL; copy = pcmk__assert_alloc(1, sizeof(lrmd_event_data_t)); copy->type = event->type; // lrmd_event_data_t has (const char *) members that lrmd_free_event() frees copy->rsc_id = pcmk__str_copy(event->rsc_id); copy->op_type = pcmk__str_copy(event->op_type); copy->user_data = pcmk__str_copy(event->user_data); copy->output = pcmk__str_copy(event->output); copy->remote_nodename = pcmk__str_copy(event->remote_nodename); copy->exit_reason = pcmk__str_copy(event->exit_reason); copy->call_id = event->call_id; copy->timeout = event->timeout; copy->interval_ms = event->interval_ms; copy->start_delay = event->start_delay; copy->rsc_deleted = event->rsc_deleted; copy->rc = event->rc; copy->op_status = event->op_status; copy->t_run = event->t_run; copy->t_rcchange = event->t_rcchange; copy->exec_time = event->exec_time; copy->queue_time = event->queue_time; copy->connection_rc = event->connection_rc; copy->params = pcmk__str_table_dup(event->params); return copy; } /*! * \brief Free an executor event * * \param[in,out] Executor event object to free */ void lrmd_free_event(lrmd_event_data_t *event) { if (event == NULL) { return; } // @TODO Why are these const char *? free((void *) event->rsc_id); free((void *) event->op_type); free((void *) event->user_data); free((void *) event->remote_nodename); lrmd__reset_result(event); if (event->params != NULL) { g_hash_table_destroy(event->params); } free(event); } static void lrmd_dispatch_internal(gpointer data, gpointer user_data) { xmlNode *msg = data; lrmd_t *lrmd = user_data; const char *type; const char *proxy_session = pcmk__xe_get(msg, PCMK__XA_LRMD_IPC_SESSION); lrmd_private_t *native = lrmd->lrmd_private; lrmd_event_data_t event = { 0, }; if (proxy_session != NULL) { /* this is proxy business */ lrmd_internal_proxy_dispatch(lrmd, msg); return; } else if (!native->callback) { /* no callback set */ pcmk__trace("notify event received but client has not set callback"); return; } event.remote_nodename = native->remote_nodename; type = pcmk__xe_get(msg, PCMK__XA_LRMD_OP); pcmk__xe_get_int(msg, PCMK__XA_LRMD_CALLID, &event.call_id); event.rsc_id = pcmk__xe_get(msg, PCMK__XA_LRMD_RSC_ID); if (pcmk__str_eq(type, LRMD_OP_RSC_REG, pcmk__str_none)) { event.type = lrmd_event_register; } else if (pcmk__str_eq(type, LRMD_OP_RSC_UNREG, pcmk__str_none)) { event.type = lrmd_event_unregister; } else if (pcmk__str_eq(type, LRMD_OP_RSC_EXEC, pcmk__str_none)) { int rc = 0; int exec_time = 0; int queue_time = 0; pcmk__xe_get_int(msg, PCMK__XA_LRMD_TIMEOUT, &event.timeout); pcmk__xe_get_guint(msg, PCMK__XA_LRMD_RSC_INTERVAL, &event.interval_ms); pcmk__xe_get_int(msg, PCMK__XA_LRMD_RSC_START_DELAY, &event.start_delay); pcmk__xe_get_int(msg, PCMK__XA_LRMD_EXEC_RC, &rc); event.rc = (enum ocf_exitcode) rc; pcmk__xe_get_int(msg, PCMK__XA_LRMD_EXEC_OP_STATUS, &event.op_status); pcmk__xe_get_int(msg, PCMK__XA_LRMD_RSC_DELETED, &event.rsc_deleted); pcmk__xe_get_time(msg, PCMK__XA_LRMD_RUN_TIME, &event.t_run); pcmk__xe_get_time(msg, PCMK__XA_LRMD_RCCHANGE_TIME, &event.t_rcchange); pcmk__xe_get_int(msg, PCMK__XA_LRMD_EXEC_TIME, &exec_time); CRM_LOG_ASSERT(exec_time >= 0); event.exec_time = QB_MAX(0, exec_time); pcmk__xe_get_int(msg, PCMK__XA_LRMD_QUEUE_TIME, &queue_time); CRM_LOG_ASSERT(queue_time >= 0); event.queue_time = QB_MAX(0, queue_time); event.op_type = pcmk__xe_get(msg, PCMK__XA_LRMD_RSC_ACTION); event.user_data = pcmk__xe_get(msg, PCMK__XA_LRMD_RSC_USERDATA_STR); event.type = lrmd_event_exec_complete; /* output and exit_reason may be freed by a callback */ event.output = pcmk__xe_get_copy(msg, PCMK__XA_LRMD_RSC_OUTPUT); lrmd__set_result(&event, event.rc, event.op_status, pcmk__xe_get(msg, PCMK__XA_LRMD_RSC_EXIT_REASON)); event.params = xml2list(msg); } else if (pcmk__str_eq(type, LRMD_OP_NEW_CLIENT, pcmk__str_none)) { event.type = lrmd_event_new_client; } else if (pcmk__str_eq(type, LRMD_OP_POKE, pcmk__str_none)) { event.type = lrmd_event_poke; } else { return; } pcmk__trace("op %s notify event received", type); native->callback(&event); if (event.params) { g_hash_table_destroy(event.params); } lrmd__reset_result(&event); } // \return Always 0, to indicate that IPC mainloop source should be kept static int lrmd_ipc_dispatch(const char *buffer, ssize_t length, gpointer userdata) { lrmd_t *lrmd = userdata; lrmd_private_t *native = lrmd->lrmd_private; if (native->callback != NULL) { xmlNode *msg = pcmk__xml_parse(buffer); lrmd_dispatch_internal(msg, lrmd); pcmk__xml_free(msg); } return 0; } static void lrmd_free_xml(gpointer userdata) { pcmk__xml_free((xmlNode *) userdata); } static bool remote_executor_connected(lrmd_t * lrmd) { lrmd_private_t *native = lrmd->lrmd_private; return (native->remote->tls_session != NULL); } static void handle_remote_msg(xmlNode *xml, lrmd_t *lrmd) { lrmd_private_t *native = lrmd->lrmd_private; const char *msg_type = NULL; msg_type = pcmk__xe_get(xml, PCMK__XA_LRMD_REMOTE_MSG_TYPE); if (pcmk__str_eq(msg_type, "notify", pcmk__str_casei)) { lrmd_dispatch_internal(xml, lrmd); } else if (pcmk__str_eq(msg_type, "reply", pcmk__str_casei)) { const char *op = pcmk__xe_get(xml, PCMK__XA_LRMD_OP); if (native->expected_late_replies > 0) { native->expected_late_replies--; /* The register op message we get as a response to lrmd_handshake_async * is a reply, so we have to handle that here. */ if (pcmk__str_eq(op, "register", pcmk__str_casei)) { int rc = process_lrmd_handshake_reply(xml, native); report_async_connection_result(lrmd, pcmk_rc2legacy(rc)); } } else { int reply_id = 0; pcmk__xe_get_int(xml, PCMK__XA_LRMD_CALLID, &reply_id); /* if this happens, we want to know about it */ pcmk__err("Got outdated Pacemaker Remote reply %d", reply_id); } } } /*! * \internal * \brief Notify trigger handler * * \param[in,out] userdata API connection * * \return Always return G_SOURCE_CONTINUE to leave this trigger handler in the * mainloop */ static int process_pending_notifies(gpointer userdata) { lrmd_t *lrmd = userdata; lrmd_private_t *native = lrmd->lrmd_private; if (native->pending_notify == NULL) { return G_SOURCE_CONTINUE; } pcmk__trace("Processing pending notifies"); g_list_foreach(native->pending_notify, lrmd_dispatch_internal, lrmd); g_list_free_full(native->pending_notify, lrmd_free_xml); native->pending_notify = NULL; return G_SOURCE_CONTINUE; } /*! * \internal * \brief TLS dispatch function for file descriptor sources * * \param[in,out] userdata API connection * * \return -1 on error to remove the source from the mainloop, or 0 otherwise * to leave it in the mainloop */ static int lrmd_tls_dispatch(gpointer userdata) { lrmd_t *lrmd = userdata; lrmd_private_t *native = lrmd->lrmd_private; xmlNode *xml = NULL; int rc = pcmk_rc_ok; if (!remote_executor_connected(lrmd)) { pcmk__trace("TLS dispatch triggered after disconnect"); return -1; } pcmk__trace("TLS dispatch triggered"); rc = pcmk__remote_ready(native->remote, 0); if (rc == pcmk_rc_ok) { rc = pcmk__read_remote_message(native->remote, -1); } if (rc != pcmk_rc_ok && rc != ETIME) { pcmk__info("Lost %s executor connection while reading data", pcmk__s(native->remote_nodename, "local")); lrmd_tls_disconnect(lrmd); return -1; } /* If rc is ETIME, there was nothing to read but we may already have a * full message in the buffer */ xml = pcmk__remote_message_xml(native->remote); if (xml == NULL) { return 0; } handle_remote_msg(xml, lrmd); pcmk__xml_free(xml); return 0; } /* Not used with mainloop */ int lrmd_poll(lrmd_t * lrmd, int timeout) { lrmd_private_t *native = lrmd->lrmd_private; switch (native->type) { case pcmk__client_ipc: return crm_ipc_ready(native->ipc); case pcmk__client_tls: if (native->pending_notify) { return 1; } else { int rc = pcmk__remote_ready(native->remote, 0); switch (rc) { case pcmk_rc_ok: return 1; case ETIME: return 0; default: return pcmk_rc2legacy(rc); } } default: pcmk__err("Unsupported executor connection type (bug?): %d", native->type); return -EPROTONOSUPPORT; } } /* Not used with mainloop */ bool lrmd_dispatch(lrmd_t * lrmd) { lrmd_private_t *private = NULL; pcmk__assert(lrmd != NULL); private = lrmd->lrmd_private; switch (private->type) { case pcmk__client_ipc: while (crm_ipc_ready(private->ipc)) { if (crm_ipc_read(private->ipc) > 0) { const char *msg = crm_ipc_buffer(private->ipc); lrmd_ipc_dispatch(msg, strlen(msg), lrmd); } } break; case pcmk__client_tls: lrmd_tls_dispatch(lrmd); break; default: pcmk__err("Unsupported executor connection type (bug?): %d", private->type); } if (lrmd_api_is_connected(lrmd) == FALSE) { pcmk__err("Connection closed"); return FALSE; } return TRUE; } static xmlNode * lrmd_create_op(const char *token, const char *op, xmlNode *data, int timeout, enum lrmd_call_options options) { xmlNode *op_msg = NULL; CRM_CHECK(token != NULL, return NULL); op_msg = pcmk__xe_create(NULL, PCMK__XE_LRMD_COMMAND); pcmk__xe_set(op_msg, PCMK__XA_T, PCMK__VALUE_LRMD); pcmk__xe_set(op_msg, PCMK__XA_LRMD_OP, op); pcmk__xe_set_int(op_msg, PCMK__XA_LRMD_TIMEOUT, timeout); pcmk__xe_set_int(op_msg, PCMK__XA_LRMD_CALLOPT, options); if (data != NULL) { xmlNode *wrapper = pcmk__xe_create(op_msg, PCMK__XE_LRMD_CALLDATA); pcmk__xml_copy(wrapper, data); } pcmk__trace("Created executor %s command with call options %.8lx (%d)", op, (long) options, options); return op_msg; } static void lrmd_ipc_connection_destroy(gpointer userdata) { lrmd_t *lrmd = userdata; lrmd_private_t *native = lrmd->lrmd_private; switch (native->type) { case pcmk__client_ipc: pcmk__info("Disconnected from local executor"); break; case pcmk__client_tls: pcmk__info("Disconnected from remote executor on %s", native->remote_nodename); break; default: pcmk__err("Unsupported executor connection type %d (bug?)", native->type); } /* Prevent these from being cleaned up in lrmd_api_disconnect() */ native->ipc = NULL; native->source = NULL; if (native->callback) { lrmd_event_data_t event = { 0, }; event.type = lrmd_event_disconnect; event.remote_nodename = native->remote_nodename; native->callback(&event); } } static void lrmd_tls_connection_destroy(gpointer userdata) { lrmd_t *lrmd = userdata; lrmd_private_t *native = lrmd->lrmd_private; pcmk__info("TLS connection destroyed"); if (native->remote->tls_session) { gnutls_bye(native->remote->tls_session, GNUTLS_SHUT_RDWR); gnutls_deinit(native->remote->tls_session); native->remote->tls_session = NULL; } if (native->tls) { pcmk__free_tls(native->tls); native->tls = NULL; } if (native->sock >= 0) { close(native->sock); } if (native->process_notify) { mainloop_destroy_trigger(native->process_notify); native->process_notify = NULL; } if (native->pending_notify) { g_list_free_full(native->pending_notify, lrmd_free_xml); native->pending_notify = NULL; } if (native->handshake_trigger != NULL) { mainloop_destroy_trigger(native->handshake_trigger); native->handshake_trigger = NULL; } free(native->remote->buffer); free(native->remote->start_state); native->remote->buffer = NULL; native->remote->start_state = NULL; native->source = 0; native->sock = -1; if (native->callback) { lrmd_event_data_t event = { 0, }; event.remote_nodename = native->remote_nodename; event.type = lrmd_event_disconnect; native->callback(&event); } return; } // \return Standard Pacemaker return code int lrmd__remote_send_xml(pcmk__remote_t *session, xmlNode *msg, uint32_t id, const char *msg_type) { pcmk__xe_set_int(msg, PCMK__XA_LRMD_REMOTE_MSG_ID, id); pcmk__xe_set(msg, PCMK__XA_LRMD_REMOTE_MSG_TYPE, msg_type); return pcmk__remote_send_xml(session, msg); } // \return Standard Pacemaker return code static int read_remote_reply(lrmd_t *lrmd, int total_timeout, int expected_reply_id, xmlNode **reply) { lrmd_private_t *native = lrmd->lrmd_private; time_t start = time(NULL); const char *msg_type = NULL; int reply_id = 0; int remaining_timeout = 0; int rc = pcmk_rc_ok; /* A timeout of 0 here makes no sense. We have to wait a period of time * for the response to come back. If -1 or 0, default to 10 seconds. */ if (total_timeout <= 0 || total_timeout > MAX_TLS_RECV_WAIT) { total_timeout = MAX_TLS_RECV_WAIT; } for (*reply = NULL; *reply == NULL; ) { *reply = pcmk__remote_message_xml(native->remote); if (*reply == NULL) { /* read some more off the tls buffer if we still have time left. */ if (remaining_timeout) { remaining_timeout = total_timeout - ((time(NULL) - start) * 1000); } else { remaining_timeout = total_timeout; } if (remaining_timeout <= 0) { return ETIME; } rc = pcmk__read_remote_message(native->remote, remaining_timeout); if (rc != pcmk_rc_ok) { return rc; } *reply = pcmk__remote_message_xml(native->remote); if (*reply == NULL) { return ENOMSG; } } pcmk__xe_get_int(*reply, PCMK__XA_LRMD_REMOTE_MSG_ID, &reply_id); msg_type = pcmk__xe_get(*reply, PCMK__XA_LRMD_REMOTE_MSG_TYPE); if (!msg_type) { pcmk__err("Empty msg type received while waiting for reply"); pcmk__xml_free(*reply); *reply = NULL; } else if (pcmk__str_eq(msg_type, "notify", pcmk__str_casei)) { /* got a notify while waiting for reply, trigger the notify to be processed later */ pcmk__info("queueing notify"); native->pending_notify = g_list_append(native->pending_notify, *reply); if (native->process_notify) { pcmk__info("notify trigger set"); mainloop_set_trigger(native->process_notify); } *reply = NULL; } else if (!pcmk__str_eq(msg_type, "reply", pcmk__str_casei)) { /* msg isn't a reply, make some noise */ pcmk__err("Expected a reply, got %s", msg_type); pcmk__xml_free(*reply); *reply = NULL; } else if (reply_id != expected_reply_id) { if (native->expected_late_replies > 0) { native->expected_late_replies--; } else { pcmk__err("Got outdated reply, expected id %d got id %d", expected_reply_id, reply_id); } pcmk__xml_free(*reply); *reply = NULL; } } if (native->remote->buffer && native->process_notify) { mainloop_set_trigger(native->process_notify); } return rc; } // \return Standard Pacemaker return code static int send_remote_message(lrmd_t *lrmd, xmlNode *msg) { int rc = pcmk_rc_ok; lrmd_private_t *native = lrmd->lrmd_private; global_remote_msg_id++; if (global_remote_msg_id <= 0) { global_remote_msg_id = 1; } rc = lrmd__remote_send_xml(native->remote, msg, global_remote_msg_id, "request"); if (rc != pcmk_rc_ok) { pcmk__err("Disconnecting because TLS message could not be sent to " "Pacemaker Remote: %s", pcmk_rc_str(rc)); lrmd_tls_disconnect(lrmd); } return rc; } static int lrmd_tls_send_recv(lrmd_t * lrmd, xmlNode * msg, int timeout, xmlNode ** reply) { int rc = 0; xmlNode *xml = NULL; if (!remote_executor_connected(lrmd)) { return -ENOTCONN; } rc = send_remote_message(lrmd, msg); if (rc != pcmk_rc_ok) { return pcmk_rc2legacy(rc); } rc = read_remote_reply(lrmd, timeout, global_remote_msg_id, &xml); if (rc != pcmk_rc_ok) { pcmk__err("Disconnecting remote after request %d reply not received: " "%s " QB_XS " rc=%d timeout=%dms", global_remote_msg_id, pcmk_rc_str(rc), rc, timeout); lrmd_tls_disconnect(lrmd); } if (reply) { *reply = xml; } else { pcmk__xml_free(xml); } return pcmk_rc2legacy(rc); } static int lrmd_send_xml(lrmd_t * lrmd, xmlNode * msg, int timeout, xmlNode ** reply) { int rc = pcmk_ok; lrmd_private_t *native = lrmd->lrmd_private; switch (native->type) { case pcmk__client_ipc: rc = crm_ipc_send(native->ipc, msg, crm_ipc_client_response, timeout, reply); break; case pcmk__client_tls: rc = lrmd_tls_send_recv(lrmd, msg, timeout, reply); break; default: pcmk__err("Unsupported executor connection type (bug?): %d", native->type); rc = -EPROTONOSUPPORT; } return rc; } static int lrmd_send_xml_no_reply(lrmd_t * lrmd, xmlNode * msg) { int rc = pcmk_ok; lrmd_private_t *native = lrmd->lrmd_private; switch (native->type) { case pcmk__client_ipc: rc = crm_ipc_send(native->ipc, msg, crm_ipc_flags_none, 0, NULL); break; case pcmk__client_tls: rc = send_remote_message(lrmd, msg); if (rc == pcmk_rc_ok) { /* we don't want to wait around for the reply, but * since the request/reply protocol needs to behave the same * as libqb, a reply will eventually come later anyway. */ native->expected_late_replies++; } rc = pcmk_rc2legacy(rc); break; default: pcmk__err("Unsupported executor connection type (bug?): %d", native->type); rc = -EPROTONOSUPPORT; } return rc; } static int lrmd_api_is_connected(lrmd_t * lrmd) { lrmd_private_t *native = lrmd->lrmd_private; switch (native->type) { case pcmk__client_ipc: return crm_ipc_connected(native->ipc); case pcmk__client_tls: return remote_executor_connected(lrmd); default: pcmk__err("Unsupported executor connection type (bug?): %d", native->type); return 0; } } /*! * \internal * \brief Send a prepared API command to the executor * * \param[in,out] lrmd Existing connection to the executor * \param[in] op Name of API command to send * \param[in] data Command data XML to add to the sent command * \param[out] output_data If expecting a reply, it will be stored here * \param[in] timeout Timeout in milliseconds (if 0, defaults to * a sensible value per the type of connection, * standard vs. pacemaker remote); * also propagated to the command XML * \param[in] call_options Call options to pass to server when sending * \param[in] expect_reply If true, wait for a reply from the server; * must be true for IPC (as opposed to TLS) clients * * \return pcmk_ok on success, -errno on error */ static int lrmd_send_command(lrmd_t *lrmd, const char *op, xmlNode *data, xmlNode **output_data, int timeout, enum lrmd_call_options options, bool expect_reply) { int rc = pcmk_ok; lrmd_private_t *native = lrmd->lrmd_private; xmlNode *op_msg = NULL; xmlNode *op_reply = NULL; if (!lrmd_api_is_connected(lrmd)) { return -ENOTCONN; } if (op == NULL) { pcmk__err("No operation specified"); return -EINVAL; } CRM_LOG_ASSERT(native->token != NULL); pcmk__trace("Sending %s op to executor", op); op_msg = lrmd_create_op(native->token, op, data, timeout, options); if (op_msg == NULL) { return -EINVAL; } if (expect_reply) { rc = lrmd_send_xml(lrmd, op_msg, timeout, &op_reply); } else { rc = lrmd_send_xml_no_reply(lrmd, op_msg); goto done; } if (rc < 0) { crm_perror(LOG_ERR, "Couldn't perform %s operation (timeout=%d): %d", op, timeout, rc); goto done; } else if (op_reply == NULL) { rc = -ENOMSG; goto done; } rc = pcmk_ok; pcmk__trace("%s op reply received", op); if (pcmk__xe_get_int(op_reply, PCMK__XA_LRMD_RC, &rc) != pcmk_rc_ok) { rc = -ENOMSG; goto done; } - crm_log_xml_trace(op_reply, "Reply"); + pcmk__log_xml_trace(op_reply, "Reply"); if (output_data) { *output_data = op_reply; op_reply = NULL; /* Prevent subsequent free */ } done: if (lrmd_api_is_connected(lrmd) == FALSE) { pcmk__err("Executor disconnected"); } pcmk__xml_free(op_msg); pcmk__xml_free(op_reply); return rc; } static int lrmd_api_poke_connection(lrmd_t * lrmd) { int rc; lrmd_private_t *native = lrmd->lrmd_private; xmlNode *data = pcmk__xe_create(NULL, PCMK__XE_LRMD_RSC); pcmk__xe_set(data, PCMK__XA_LRMD_ORIGIN, __func__); rc = lrmd_send_command(lrmd, LRMD_OP_POKE, data, NULL, 0, 0, (native->type == pcmk__client_ipc)); pcmk__xml_free(data); return rc < 0 ? rc : pcmk_ok; } // \return Standard Pacemaker return code int lrmd__validate_remote_settings(lrmd_t *lrmd, GHashTable *hash) { int rc = pcmk_rc_ok; const char *value; lrmd_private_t *native = lrmd->lrmd_private; xmlNode *data = pcmk__xe_create(NULL, PCMK__XA_LRMD_OP); pcmk__xe_set(data, PCMK__XA_LRMD_ORIGIN, __func__); value = g_hash_table_lookup(hash, PCMK_OPT_STONITH_WATCHDOG_TIMEOUT); if ((value) && (stonith__watchdog_fencing_enabled_for_node(native->remote_nodename))) { pcmk__xe_set(data, PCMK__XA_LRMD_WATCHDOG, value); } rc = lrmd_send_command(lrmd, LRMD_OP_CHECK, data, NULL, 0, 0, (native->type == pcmk__client_ipc)); pcmk__xml_free(data); return (rc < 0)? pcmk_legacy2rc(rc) : pcmk_rc_ok; } static xmlNode * lrmd_handshake_hello_msg(const char *name, bool is_proxy) { xmlNode *hello = pcmk__xe_create(NULL, PCMK__XE_LRMD_COMMAND); pcmk__xe_set(hello, PCMK__XA_T, PCMK__VALUE_LRMD); pcmk__xe_set(hello, PCMK__XA_LRMD_OP, CRM_OP_REGISTER); pcmk__xe_set(hello, PCMK__XA_LRMD_CLIENTNAME, name); pcmk__xe_set(hello, PCMK__XA_LRMD_PROTOCOL_VERSION, LRMD_PROTOCOL_VERSION); /* advertise that we are a proxy provider */ if (is_proxy) { pcmk__xe_set_bool_attr(hello, PCMK__XA_LRMD_IS_IPC_PROVIDER, true); } return hello; } static int process_lrmd_handshake_reply(xmlNode *reply, lrmd_private_t *native) { int rc = pcmk_rc_ok; const char *version = pcmk__xe_get(reply, PCMK__XA_LRMD_PROTOCOL_VERSION); const char *msg_type = pcmk__xe_get(reply, PCMK__XA_LRMD_OP); const char *tmp_ticket = pcmk__xe_get(reply, PCMK__XA_LRMD_CLIENTID); const char *start_state = pcmk__xe_get(reply, PCMK__XA_NODE_START_STATE); pcmk__xe_get_int(reply, PCMK__XA_LRMD_RC, &rc); rc = pcmk_legacy2rc(rc); /* The remote executor may add its uptime to the XML reply, which is useful * in handling transient attributes when the connection to the remote node * unexpectedly drops. If no parameter is given, just default to -1. */ native->remote->uptime = -1; pcmk__xe_get_time(reply, PCMK__XA_UPTIME, &native->remote->uptime); if (start_state) { native->remote->start_state = strdup(start_state); } if (rc == EPROTO) { pcmk__err("Executor protocol version mismatch between client " "(" LRMD_PROTOCOL_VERSION ") and server (%s)", version); pcmk__log_xml_err(reply, "Protocol Error"); } else if (!pcmk__str_eq(msg_type, CRM_OP_REGISTER, pcmk__str_casei)) { pcmk__err("Invalid registration message: %s", msg_type); pcmk__log_xml_err(reply, "Bad reply"); rc = EPROTO; } else if (tmp_ticket == NULL) { pcmk__err("No registration token provided"); pcmk__log_xml_err(reply, "Bad reply"); rc = EPROTO; } else { pcmk__trace("Obtained registration token: %s", tmp_ticket); native->token = strdup(tmp_ticket); native->peer_version = strdup(version?version:"1.0"); /* Included since 1.1 */ rc = pcmk_rc_ok; } return rc; } static int lrmd_handshake(lrmd_t * lrmd, const char *name) { int rc = pcmk_rc_ok; lrmd_private_t *native = lrmd->lrmd_private; xmlNode *reply = NULL; xmlNode *hello = lrmd_handshake_hello_msg(name, native->proxy_callback != NULL); rc = lrmd_send_xml(lrmd, hello, -1, &reply); if (rc < 0) { crm_perror(LOG_DEBUG, "Couldn't complete registration with the executor API: %d", rc); rc = ECOMM; } else if (reply == NULL) { pcmk__err("Did not receive registration reply"); rc = EPROTO; } else { rc = process_lrmd_handshake_reply(reply, native); } pcmk__xml_free(reply); pcmk__xml_free(hello); if (rc != pcmk_rc_ok) { lrmd_api_disconnect(lrmd); } return rc; } static int lrmd_handshake_async(lrmd_t * lrmd, const char *name) { int rc = pcmk_rc_ok; lrmd_private_t *native = lrmd->lrmd_private; xmlNode *hello = lrmd_handshake_hello_msg(name, native->proxy_callback != NULL); rc = send_remote_message(lrmd, hello); if (rc == pcmk_rc_ok) { native->expected_late_replies++; } else { lrmd_api_disconnect(lrmd); } pcmk__xml_free(hello); return rc; } static int lrmd_ipc_connect(lrmd_t * lrmd, int *fd) { int rc = pcmk_ok; lrmd_private_t *native = lrmd->lrmd_private; struct ipc_client_callbacks lrmd_callbacks = { .dispatch = lrmd_ipc_dispatch, .destroy = lrmd_ipc_connection_destroy }; pcmk__info("Connecting to executor"); if (fd) { /* No mainloop */ native->ipc = crm_ipc_new(CRM_SYSTEM_LRMD, 0); if (native->ipc != NULL) { rc = pcmk__connect_generic_ipc(native->ipc); if (rc == pcmk_rc_ok) { rc = pcmk__ipc_fd(native->ipc, fd); } if (rc != pcmk_rc_ok) { pcmk__err("Connection to executor failed: %s", pcmk_rc_str(rc)); rc = -ENOTCONN; } } } else { native->source = mainloop_add_ipc_client(CRM_SYSTEM_LRMD, G_PRIORITY_HIGH, 0, lrmd, &lrmd_callbacks); native->ipc = mainloop_get_ipc_client(native->source); } if (native->ipc == NULL) { pcmk__debug("Could not connect to the executor API"); rc = -ENOTCONN; } return rc; } static void copy_gnutls_datum(gnutls_datum_t *dest, gnutls_datum_t *source) { pcmk__assert((dest != NULL) && (source != NULL) && (source->data != NULL)); dest->data = gnutls_malloc(source->size); pcmk__mem_assert(dest->data); memcpy(dest->data, source->data, source->size); dest->size = source->size; } static void clear_gnutls_datum(gnutls_datum_t *datum) { gnutls_free(datum->data); datum->data = NULL; datum->size = 0; } #define KEY_READ_LEN 256 // Chunk size for reading key from file // \return Standard Pacemaker return code static int read_gnutls_key(const char *location, gnutls_datum_t *key) { FILE *stream = NULL; size_t buf_len = KEY_READ_LEN; if ((location == NULL) || (key == NULL)) { return EINVAL; } stream = fopen(location, "r"); if (stream == NULL) { return errno; } key->data = gnutls_malloc(buf_len); key->size = 0; while (!feof(stream)) { int next = fgetc(stream); if (next == EOF) { if (!feof(stream)) { pcmk__warn("Pacemaker Remote key read was partially successful " "(copy in memory may be corrupted)"); } break; } if (key->size == buf_len) { buf_len = key->size + KEY_READ_LEN; key->data = gnutls_realloc(key->data, buf_len); pcmk__assert(key->data); } key->data[key->size++] = (unsigned char) next; } fclose(stream); if (key->size == 0) { clear_gnutls_datum(key); return ENOKEY; } return pcmk_rc_ok; } // Cache the most recently used Pacemaker Remote authentication key struct key_cache_s { time_t updated; // When cached key was read (valid for 1 minute) const char *location; // Where cached key was read from gnutls_datum_t key; // Cached key }; static bool key_is_cached(struct key_cache_s *key_cache) { return key_cache->updated != 0; } static bool key_cache_expired(struct key_cache_s *key_cache) { return (time(NULL) - key_cache->updated) >= 60; } static void clear_key_cache(struct key_cache_s *key_cache) { clear_gnutls_datum(&(key_cache->key)); if ((key_cache->updated != 0) || (key_cache->location != NULL)) { key_cache->updated = 0; key_cache->location = NULL; pcmk__debug("Cleared Pacemaker Remote key cache"); } } static void get_cached_key(struct key_cache_s *key_cache, gnutls_datum_t *key) { copy_gnutls_datum(key, &(key_cache->key)); pcmk__debug("Using cached Pacemaker Remote key from %s", pcmk__s(key_cache->location, "unknown location")); } static void cache_key(struct key_cache_s *key_cache, gnutls_datum_t *key, const char *location) { key_cache->updated = time(NULL); key_cache->location = location; copy_gnutls_datum(&(key_cache->key), key); pcmk__debug("Using (and cacheing) Pacemaker Remote key from %s", pcmk__s(location, "unknown location")); } /*! * \internal * \brief Get Pacemaker Remote authentication key from file or cache * * \param[in] location Path to key file to try (this memory must * persist across all calls of this function) * \param[out] key Key from location or cache * * \return Standard Pacemaker return code */ static int get_remote_key(const char *location, gnutls_datum_t *key) { static struct key_cache_s key_cache = { 0, }; int rc = pcmk_rc_ok; if ((location == NULL) || (key == NULL)) { return EINVAL; } if (key_is_cached(&key_cache)) { if (key_cache_expired(&key_cache)) { clear_key_cache(&key_cache); } else { get_cached_key(&key_cache, key); return pcmk_rc_ok; } } rc = read_gnutls_key(location, key); if (rc != pcmk_rc_ok) { return rc; } cache_key(&key_cache, key, location); return pcmk_rc_ok; } /*! * \internal * \brief Initialize the Pacemaker Remote authentication key * * Try loading the Pacemaker Remote authentication key from cache if available, * otherwise from these locations, in order of preference: * * - The value of the PCMK_authkey_location environment variable, if set * - The Pacemaker default key file location * * \param[out] key Where to store key * * \return Standard Pacemaker return code */ int lrmd__init_remote_key(gnutls_datum_t *key) { static const char *env_location = NULL; static bool need_env = true; int rc = pcmk_rc_ok; if (need_env) { env_location = pcmk__env_option(PCMK__ENV_AUTHKEY_LOCATION); need_env = false; } // Try location in environment variable, if set if (env_location != NULL) { rc = get_remote_key(env_location, key); if (rc == pcmk_rc_ok) { return pcmk_rc_ok; } pcmk__warn("Could not read Pacemaker Remote key from %s: %s", env_location, pcmk_rc_str(rc)); return ENOKEY; } // Try default location, if environment wasn't explicitly set to it rc = get_remote_key(DEFAULT_REMOTE_KEY_LOCATION, key); if (rc == pcmk_rc_ok) { return pcmk_rc_ok; } pcmk__warn("Could not read Pacemaker Remote key from default location " DEFAULT_REMOTE_KEY_LOCATION ": %s", pcmk_rc_str(rc)); return ENOKEY; } static void report_async_connection_result(lrmd_t * lrmd, int rc) { lrmd_private_t *native = lrmd->lrmd_private; if (native->callback) { lrmd_event_data_t event = { 0, }; event.type = lrmd_event_connect; event.remote_nodename = native->remote_nodename; event.connection_rc = rc; native->callback(&event); } } static void tls_handshake_failed(lrmd_t *lrmd, int tls_rc, int rc) { lrmd_private_t *native = lrmd->lrmd_private; pcmk__warn("Disconnecting after TLS handshake with Pacemaker Remote server " "%s:%d failed: %s", native->server, native->port, ((rc == EPROTO)? gnutls_strerror(tls_rc) : pcmk_rc_str(rc))); report_async_connection_result(lrmd, pcmk_rc2legacy(rc)); gnutls_deinit(native->remote->tls_session); native->remote->tls_session = NULL; lrmd_tls_connection_destroy(lrmd); } static void tls_handshake_succeeded(lrmd_t *lrmd) { int rc = pcmk_rc_ok; lrmd_private_t *native = lrmd->lrmd_private; /* Now that the handshake is done, see if any client TLS certificate is * close to its expiration date and log if so. If a TLS certificate is not * in use, this function will just return so we don't need to check for the * session type here. */ pcmk__tls_check_cert_expiration(native->remote->tls_session); pcmk__info("TLS connection to Pacemaker Remote server %s:%d succeeded", native->server, native->port); rc = add_tls_to_mainloop(lrmd, true); /* If add_tls_to_mainloop failed, report that right now. Otherwise, we have * to wait until we read the async reply to report anything. */ if (rc != pcmk_rc_ok) { report_async_connection_result(lrmd, pcmk_rc2legacy(rc)); } } /*! * \internal * \brief Perform a TLS client handshake with a Pacemaker Remote server * * \param[in] lrmd Newly established Pacemaker Remote executor connection * * \return Standard Pacemaker return code */ static int tls_client_handshake(lrmd_t *lrmd) { lrmd_private_t *native = lrmd->lrmd_private; int tls_rc = GNUTLS_E_SUCCESS; int rc = pcmk__tls_client_handshake(native->remote, TLS_HANDSHAKE_TIMEOUT, &tls_rc); if (rc != pcmk_rc_ok) { tls_handshake_failed(lrmd, tls_rc, rc); } return rc; } /*! * \internal * \brief Add trigger and file descriptor mainloop sources for TLS * * \param[in,out] lrmd API connection with established TLS session * \param[in] do_api_handshake Whether to perform executor handshake * * \return Standard Pacemaker return code */ static int add_tls_to_mainloop(lrmd_t *lrmd, bool do_api_handshake) { lrmd_private_t *native = lrmd->lrmd_private; int rc = pcmk_rc_ok; char *name = pcmk__assert_asprintf("pacemaker-remote-%s:%d", native->server, native->port); struct mainloop_fd_callbacks tls_fd_callbacks = { .dispatch = lrmd_tls_dispatch, .destroy = lrmd_tls_connection_destroy, }; native->process_notify = mainloop_add_trigger(G_PRIORITY_HIGH, process_pending_notifies, lrmd); native->source = mainloop_add_fd(name, G_PRIORITY_HIGH, native->sock, lrmd, &tls_fd_callbacks); /* Async connections lose the client name provided by the API caller, so we * have to use our generated name here to perform the executor handshake. * * @TODO Keep track of the caller-provided name. Perhaps we should be using * that name in this function instead of generating one anyway. */ if (do_api_handshake) { rc = lrmd_handshake_async(lrmd, name); } free(name); return rc; } struct handshake_data_s { lrmd_t *lrmd; time_t start_time; int timeout_sec; }; static gboolean try_handshake_cb(gpointer user_data) { struct handshake_data_s *hs = user_data; lrmd_t *lrmd = hs->lrmd; lrmd_private_t *native = lrmd->lrmd_private; pcmk__remote_t *remote = native->remote; int rc = pcmk_rc_ok; int tls_rc = GNUTLS_E_SUCCESS; if (time(NULL) >= hs->start_time + hs->timeout_sec) { rc = ETIME; tls_handshake_failed(lrmd, GNUTLS_E_TIMEDOUT, rc); free(hs); return 0; } rc = pcmk__tls_client_try_handshake(remote, &tls_rc); if (rc == pcmk_rc_ok) { tls_handshake_succeeded(lrmd); free(hs); return 0; } else if (rc == EAGAIN) { mainloop_set_trigger(native->handshake_trigger); return 1; } else { rc = EKEYREJECTED; tls_handshake_failed(lrmd, tls_rc, rc); free(hs); return 0; } } static void lrmd_tcp_connect_cb(void *userdata, int rc, int sock) { lrmd_t *lrmd = userdata; lrmd_private_t *native = lrmd->lrmd_private; int tls_rc = GNUTLS_E_SUCCESS; bool use_cert = pcmk__x509_enabled(); native->async_timer = 0; if (rc != pcmk_rc_ok) { lrmd_tls_connection_destroy(lrmd); pcmk__info("Could not connect to Pacemaker Remote at %s:%d: %s " QB_XS " rc=%d", native->server, native->port, pcmk_rc_str(rc), rc); report_async_connection_result(lrmd, pcmk_rc2legacy(rc)); return; } /* The TCP connection was successful, so establish the TLS connection. */ native->sock = sock; if (native->tls == NULL) { rc = pcmk__init_tls(&native->tls, false, use_cert ? GNUTLS_CRD_CERTIFICATE : GNUTLS_CRD_PSK); if (rc != pcmk_rc_ok) { lrmd_tls_connection_destroy(lrmd); report_async_connection_result(lrmd, pcmk_rc2legacy(rc)); return; } } if (!use_cert) { gnutls_datum_t psk_key = { NULL, 0 }; rc = lrmd__init_remote_key(&psk_key); if (rc != pcmk_rc_ok) { pcmk__info("Could not connect to Pacemaker Remote at %s:%d: %s " QB_XS " rc=%d", native->server, native->port, pcmk_rc_str(rc), rc); lrmd_tls_connection_destroy(lrmd); report_async_connection_result(lrmd, pcmk_rc2legacy(rc)); return; } pcmk__tls_add_psk_key(native->tls, &psk_key); gnutls_free(psk_key.data); } native->remote->tls_session = pcmk__new_tls_session(native->tls, sock); if (native->remote->tls_session == NULL) { lrmd_tls_connection_destroy(lrmd); report_async_connection_result(lrmd, -EPROTO); return; } /* If the TLS handshake immediately succeeds or fails, we can handle that * now without having to deal with mainloops and retries. Otherwise, add a * trigger to keep trying until we get a result (or it times out). */ rc = pcmk__tls_client_try_handshake(native->remote, &tls_rc); if (rc == EAGAIN) { struct handshake_data_s *hs = NULL; if (native->handshake_trigger != NULL) { return; } hs = pcmk__assert_alloc(1, sizeof(struct handshake_data_s)); hs->lrmd = lrmd; hs->start_time = time(NULL); hs->timeout_sec = TLS_HANDSHAKE_TIMEOUT; native->handshake_trigger = mainloop_add_trigger(G_PRIORITY_LOW, try_handshake_cb, hs); mainloop_set_trigger(native->handshake_trigger); } else if (rc == pcmk_rc_ok) { tls_handshake_succeeded(lrmd); } else { tls_handshake_failed(lrmd, tls_rc, rc); } } static int lrmd_tls_connect_async(lrmd_t * lrmd, int timeout /*ms */ ) { int rc = pcmk_rc_ok; int timer_id = 0; lrmd_private_t *native = lrmd->lrmd_private; native->sock = -1; rc = pcmk__connect_remote(native->server, native->port, timeout, &timer_id, &(native->sock), lrmd, lrmd_tcp_connect_cb); if (rc != pcmk_rc_ok) { pcmk__warn("Pacemaker Remote connection to %s:%d failed: %s " QB_XS " rc=%d", native->server, native->port, pcmk_rc_str(rc), rc); return rc; } native->async_timer = timer_id; return rc; } static int lrmd_tls_connect(lrmd_t * lrmd, int *fd) { int rc = pcmk_rc_ok; bool use_cert = pcmk__x509_enabled(); lrmd_private_t *native = lrmd->lrmd_private; native->sock = -1; rc = pcmk__connect_remote(native->server, native->port, 0, NULL, &(native->sock), NULL, NULL); if (rc != pcmk_rc_ok) { pcmk__warn("Pacemaker Remote connection to %s:%d failed: %s " QB_XS " rc=%d", native->server, native->port, pcmk_rc_str(rc), rc); lrmd_tls_connection_destroy(lrmd); return ENOTCONN; } if (native->tls == NULL) { rc = pcmk__init_tls(&native->tls, false, use_cert ? GNUTLS_CRD_CERTIFICATE : GNUTLS_CRD_PSK); if (rc != pcmk_rc_ok) { lrmd_tls_connection_destroy(lrmd); return rc; } } if (!use_cert) { gnutls_datum_t psk_key = { NULL, 0 }; rc = lrmd__init_remote_key(&psk_key); if (rc != pcmk_rc_ok) { lrmd_tls_connection_destroy(lrmd); return rc; } pcmk__tls_add_psk_key(native->tls, &psk_key); gnutls_free(psk_key.data); } native->remote->tls_session = pcmk__new_tls_session(native->tls, native->sock); if (native->remote->tls_session == NULL) { lrmd_tls_connection_destroy(lrmd); return EPROTO; } if (tls_client_handshake(lrmd) != pcmk_rc_ok) { return EKEYREJECTED; } pcmk__info("Client TLS connection established with Pacemaker Remote server " "%s:%d", native->server, native->port); if (fd) { *fd = native->sock; } else { rc = add_tls_to_mainloop(lrmd, false); } return rc; } static int lrmd_api_connect(lrmd_t * lrmd, const char *name, int *fd) { int rc = -ENOTCONN; lrmd_private_t *native = lrmd->lrmd_private; switch (native->type) { case pcmk__client_ipc: rc = lrmd_ipc_connect(lrmd, fd); break; case pcmk__client_tls: rc = lrmd_tls_connect(lrmd, fd); rc = pcmk_rc2legacy(rc); break; default: pcmk__err("Unsupported executor connection type (bug?): %d", native->type); rc = -EPROTONOSUPPORT; } if (rc == pcmk_ok) { rc = lrmd_handshake(lrmd, name); rc = pcmk_rc2legacy(rc); } return rc; } static int lrmd_api_connect_async(lrmd_t * lrmd, const char *name, int timeout) { int rc = pcmk_ok; lrmd_private_t *native = lrmd->lrmd_private; CRM_CHECK(native && native->callback, return -EINVAL); switch (native->type) { case pcmk__client_ipc: /* fake async connection with ipc. it should be fast * enough that we gain very little from async */ rc = lrmd_api_connect(lrmd, name, NULL); if (!rc) { report_async_connection_result(lrmd, rc); } break; case pcmk__client_tls: rc = lrmd_tls_connect_async(lrmd, timeout); rc = pcmk_rc2legacy(rc); break; default: pcmk__err("Unsupported executor connection type (bug?): %d", native->type); rc = -EPROTONOSUPPORT; } return rc; } static void lrmd_ipc_disconnect(lrmd_t * lrmd) { lrmd_private_t *native = lrmd->lrmd_private; if (native->source != NULL) { /* Attached to mainloop */ mainloop_del_ipc_client(native->source); native->source = NULL; native->ipc = NULL; } else if (native->ipc) { /* Not attached to mainloop */ crm_ipc_t *ipc = native->ipc; native->ipc = NULL; crm_ipc_close(ipc); crm_ipc_destroy(ipc); } } static void lrmd_tls_disconnect(lrmd_t * lrmd) { lrmd_private_t *native = lrmd->lrmd_private; if (native->remote->tls_session) { gnutls_bye(native->remote->tls_session, GNUTLS_SHUT_RDWR); gnutls_deinit(native->remote->tls_session); native->remote->tls_session = NULL; } if (native->async_timer) { g_source_remove(native->async_timer); native->async_timer = 0; } if (native->source != NULL) { /* Attached to mainloop */ mainloop_del_ipc_client(native->source); native->source = NULL; } else if (native->sock >= 0) { close(native->sock); native->sock = -1; } if (native->pending_notify) { g_list_free_full(native->pending_notify, lrmd_free_xml); native->pending_notify = NULL; } } static int lrmd_api_disconnect(lrmd_t * lrmd) { lrmd_private_t *native = lrmd->lrmd_private; int rc = pcmk_ok; switch (native->type) { case pcmk__client_ipc: pcmk__debug("Disconnecting from local executor"); lrmd_ipc_disconnect(lrmd); break; case pcmk__client_tls: pcmk__debug("Disconnecting from remote executor on %s", native->remote_nodename); lrmd_tls_disconnect(lrmd); break; default: pcmk__err("Unsupported executor connection type (bug?): %d", native->type); rc = -EPROTONOSUPPORT; } free(native->token); native->token = NULL; free(native->peer_version); native->peer_version = NULL; return rc; } static int lrmd_api_register_rsc(lrmd_t * lrmd, const char *rsc_id, const char *class, const char *provider, const char *type, enum lrmd_call_options options) { int rc = pcmk_ok; xmlNode *data = NULL; if (!class || !type || !rsc_id) { return -EINVAL; } if (pcmk__is_set(pcmk_get_ra_caps(class), pcmk_ra_cap_provider) && (provider == NULL)) { return -EINVAL; } data = pcmk__xe_create(NULL, PCMK__XE_LRMD_RSC); pcmk__xe_set(data, PCMK__XA_LRMD_ORIGIN, __func__); pcmk__xe_set(data, PCMK__XA_LRMD_RSC_ID, rsc_id); pcmk__xe_set(data, PCMK__XA_LRMD_CLASS, class); pcmk__xe_set(data, PCMK__XA_LRMD_PROVIDER, provider); pcmk__xe_set(data, PCMK__XA_LRMD_TYPE, type); rc = lrmd_send_command(lrmd, LRMD_OP_RSC_REG, data, NULL, 0, options, true); pcmk__xml_free(data); return rc; } static int lrmd_api_unregister_rsc(lrmd_t * lrmd, const char *rsc_id, enum lrmd_call_options options) { int rc = pcmk_ok; xmlNode *data = pcmk__xe_create(NULL, PCMK__XE_LRMD_RSC); pcmk__xe_set(data, PCMK__XA_LRMD_ORIGIN, __func__); pcmk__xe_set(data, PCMK__XA_LRMD_RSC_ID, rsc_id); rc = lrmd_send_command(lrmd, LRMD_OP_RSC_UNREG, data, NULL, 0, options, true); pcmk__xml_free(data); return rc; } lrmd_rsc_info_t * lrmd_new_rsc_info(const char *rsc_id, const char *standard, const char *provider, const char *type) { lrmd_rsc_info_t *rsc_info = pcmk__assert_alloc(1, sizeof(lrmd_rsc_info_t)); rsc_info->id = pcmk__str_copy(rsc_id); rsc_info->standard = pcmk__str_copy(standard); rsc_info->provider = pcmk__str_copy(provider); rsc_info->type = pcmk__str_copy(type); return rsc_info; } lrmd_rsc_info_t * lrmd_copy_rsc_info(lrmd_rsc_info_t * rsc_info) { return lrmd_new_rsc_info(rsc_info->id, rsc_info->standard, rsc_info->provider, rsc_info->type); } void lrmd_free_rsc_info(lrmd_rsc_info_t * rsc_info) { if (!rsc_info) { return; } free(rsc_info->id); free(rsc_info->type); free(rsc_info->standard); free(rsc_info->provider); free(rsc_info); } static lrmd_rsc_info_t * lrmd_api_get_rsc_info(lrmd_t * lrmd, const char *rsc_id, enum lrmd_call_options options) { lrmd_rsc_info_t *rsc_info = NULL; xmlNode *data = pcmk__xe_create(NULL, PCMK__XE_LRMD_RSC); xmlNode *output = NULL; const char *class = NULL; const char *provider = NULL; const char *type = NULL; pcmk__xe_set(data, PCMK__XA_LRMD_ORIGIN, __func__); pcmk__xe_set(data, PCMK__XA_LRMD_RSC_ID, rsc_id); lrmd_send_command(lrmd, LRMD_OP_RSC_INFO, data, &output, 0, options, true); pcmk__xml_free(data); if (!output) { return NULL; } class = pcmk__xe_get(output, PCMK__XA_LRMD_CLASS); provider = pcmk__xe_get(output, PCMK__XA_LRMD_PROVIDER); type = pcmk__xe_get(output, PCMK__XA_LRMD_TYPE); if (!class || !type) { pcmk__xml_free(output); return NULL; } else if (pcmk__is_set(pcmk_get_ra_caps(class), pcmk_ra_cap_provider) && (provider == NULL)) { pcmk__xml_free(output); return NULL; } rsc_info = lrmd_new_rsc_info(rsc_id, class, provider, type); pcmk__xml_free(output); return rsc_info; } void lrmd_free_op_info(lrmd_op_info_t *op_info) { if (op_info) { free(op_info->rsc_id); free(op_info->action); free(op_info->interval_ms_s); free(op_info->timeout_ms_s); free(op_info); } } static int lrmd_api_get_recurring_ops(lrmd_t *lrmd, const char *rsc_id, int timeout_ms, enum lrmd_call_options options, GList **output) { xmlNode *data = NULL; xmlNode *output_xml = NULL; int rc = pcmk_ok; if (output == NULL) { return -EINVAL; } *output = NULL; // Send request if (rsc_id) { data = pcmk__xe_create(NULL, PCMK__XE_LRMD_RSC); pcmk__xe_set(data, PCMK__XA_LRMD_ORIGIN, __func__); pcmk__xe_set(data, PCMK__XA_LRMD_RSC_ID, rsc_id); } rc = lrmd_send_command(lrmd, LRMD_OP_GET_RECURRING, data, &output_xml, timeout_ms, options, true); if (data) { pcmk__xml_free(data); } // Process reply if ((rc != pcmk_ok) || (output_xml == NULL)) { return rc; } for (const xmlNode *rsc_xml = pcmk__xe_first_child(output_xml, PCMK__XE_LRMD_RSC, NULL, NULL); (rsc_xml != NULL) && (rc == pcmk_ok); rsc_xml = pcmk__xe_next(rsc_xml, PCMK__XE_LRMD_RSC)) { rsc_id = pcmk__xe_get(rsc_xml, PCMK__XA_LRMD_RSC_ID); if (rsc_id == NULL) { pcmk__err("Could not parse recurring operation information from " "executor"); continue; } for (const xmlNode *op_xml = pcmk__xe_first_child(rsc_xml, PCMK__XE_LRMD_RSC_OP, NULL, NULL); op_xml != NULL; op_xml = pcmk__xe_next(op_xml, PCMK__XE_LRMD_RSC_OP)) { lrmd_op_info_t *op_info = calloc(1, sizeof(lrmd_op_info_t)); if (op_info == NULL) { rc = -ENOMEM; break; } op_info->rsc_id = strdup(rsc_id); op_info->action = pcmk__xe_get_copy(op_xml, PCMK__XA_LRMD_RSC_ACTION); op_info->interval_ms_s = pcmk__xe_get_copy(op_xml, PCMK__XA_LRMD_RSC_INTERVAL); op_info->timeout_ms_s = pcmk__xe_get_copy(op_xml, PCMK__XA_LRMD_TIMEOUT); *output = g_list_prepend(*output, op_info); } } pcmk__xml_free(output_xml); return rc; } static void lrmd_api_set_callback(lrmd_t * lrmd, lrmd_event_callback callback) { lrmd_private_t *native = lrmd->lrmd_private; native->callback = callback; } void lrmd_internal_set_proxy_callback(lrmd_t * lrmd, void *userdata, void (*callback)(lrmd_t *lrmd, void *userdata, xmlNode *msg)) { lrmd_private_t *native = lrmd->lrmd_private; native->proxy_callback = callback; native->proxy_callback_userdata = userdata; } void lrmd_internal_proxy_dispatch(lrmd_t *lrmd, xmlNode *msg) { lrmd_private_t *native = lrmd->lrmd_private; if (native->proxy_callback) { - crm_log_xml_trace(msg, "PROXY_INBOUND"); + pcmk__log_xml_trace(msg, "PROXY_INBOUND"); native->proxy_callback(lrmd, native->proxy_callback_userdata, msg); } } int lrmd_internal_proxy_send(lrmd_t * lrmd, xmlNode *msg) { if (lrmd == NULL) { return -ENOTCONN; } pcmk__xe_set(msg, PCMK__XA_LRMD_OP, CRM_OP_IPC_FWD); - crm_log_xml_trace(msg, "PROXY_OUTBOUND"); + pcmk__log_xml_trace(msg, "PROXY_OUTBOUND"); return lrmd_send_xml_no_reply(lrmd, msg); } static int stonith_get_metadata(const char *provider, const char *type, char **output) { int rc = pcmk_ok; stonith_t *stonith_api = stonith_api_new(); if (stonith_api == NULL) { pcmk__err("Could not get fence agent meta-data: API memory allocation " "failed"); return -ENOMEM; } rc = stonith_api->cmds->metadata(stonith_api, st_opt_sync_call, type, provider, output, 0); if ((rc == pcmk_ok) && (*output == NULL)) { rc = -EIO; } stonith_api->cmds->free(stonith_api); return rc; } static int lrmd_api_get_metadata(lrmd_t *lrmd, const char *standard, const char *provider, const char *type, char **output, enum lrmd_call_options options) { return lrmd->cmds->get_metadata_params(lrmd, standard, provider, type, output, options, NULL); } static int lrmd_api_get_metadata_params(lrmd_t *lrmd, const char *standard, const char *provider, const char *type, char **output, enum lrmd_call_options options, lrmd_key_value_t *params) { svc_action_t *action = NULL; GHashTable *params_table = NULL; if (!standard || !type) { lrmd_key_value_freeall(params); return -EINVAL; } if (pcmk__str_eq(standard, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_casei)) { lrmd_key_value_freeall(params); return stonith_get_metadata(provider, type, output); } params_table = pcmk__strkey_table(free, free); for (const lrmd_key_value_t *param = params; param; param = param->next) { pcmk__insert_dup(params_table, param->key, param->value); } action = services__create_resource_action(type, standard, provider, type, PCMK_ACTION_META_DATA, 0, PCMK_DEFAULT_ACTION_TIMEOUT_MS, params_table, 0); lrmd_key_value_freeall(params); if (action == NULL) { return -ENOMEM; } if (action->rc != PCMK_OCF_UNKNOWN) { services_action_free(action); return -EINVAL; } if (!services_action_sync(action)) { pcmk__err("Failed to retrieve meta-data for %s:%s:%s", standard, provider, type); services_action_free(action); return -EIO; } if (!action->stdout_data) { pcmk__err("Failed to receive meta-data for %s:%s:%s", standard, provider, type); services_action_free(action); return -EIO; } *output = strdup(action->stdout_data); services_action_free(action); return pcmk_ok; } static int lrmd_api_exec(lrmd_t *lrmd, const char *rsc_id, const char *action, const char *userdata, guint interval_ms, int timeout, /* ms */ int start_delay, /* ms */ enum lrmd_call_options options, lrmd_key_value_t * params) { int rc = pcmk_ok; xmlNode *data = pcmk__xe_create(NULL, PCMK__XE_LRMD_RSC); xmlNode *args = pcmk__xe_create(data, PCMK__XE_ATTRIBUTES); lrmd_key_value_t *tmp = NULL; pcmk__xe_set(data, PCMK__XA_LRMD_ORIGIN, __func__); pcmk__xe_set(data, PCMK__XA_LRMD_RSC_ID, rsc_id); pcmk__xe_set(data, PCMK__XA_LRMD_RSC_ACTION, action); pcmk__xe_set(data, PCMK__XA_LRMD_RSC_USERDATA_STR, userdata); pcmk__xe_set_guint(data, PCMK__XA_LRMD_RSC_INTERVAL, interval_ms); pcmk__xe_set_int(data, PCMK__XA_LRMD_TIMEOUT, timeout); pcmk__xe_set_int(data, PCMK__XA_LRMD_RSC_START_DELAY, start_delay); for (tmp = params; tmp; tmp = tmp->next) { hash2smartfield((gpointer) tmp->key, (gpointer) tmp->value, args); } rc = lrmd_send_command(lrmd, LRMD_OP_RSC_EXEC, data, NULL, timeout, options, true); pcmk__xml_free(data); lrmd_key_value_freeall(params); return rc; } /* timeout is in ms */ static int lrmd_api_exec_alert(lrmd_t *lrmd, const char *alert_id, const char *alert_path, int timeout, lrmd_key_value_t *params) { int rc = pcmk_ok; xmlNode *data = pcmk__xe_create(NULL, PCMK__XE_LRMD_ALERT); xmlNode *args = pcmk__xe_create(data, PCMK__XE_ATTRIBUTES); lrmd_key_value_t *tmp = NULL; pcmk__xe_set(data, PCMK__XA_LRMD_ORIGIN, __func__); pcmk__xe_set(data, PCMK__XA_LRMD_ALERT_ID, alert_id); pcmk__xe_set(data, PCMK__XA_LRMD_ALERT_PATH, alert_path); pcmk__xe_set_int(data, PCMK__XA_LRMD_TIMEOUT, timeout); for (tmp = params; tmp; tmp = tmp->next) { hash2smartfield((gpointer) tmp->key, (gpointer) tmp->value, args); } rc = lrmd_send_command(lrmd, LRMD_OP_ALERT_EXEC, data, NULL, timeout, lrmd_opt_notify_orig_only, true); pcmk__xml_free(data); lrmd_key_value_freeall(params); return rc; } static int lrmd_api_cancel(lrmd_t *lrmd, const char *rsc_id, const char *action, guint interval_ms) { int rc = pcmk_ok; xmlNode *data = pcmk__xe_create(NULL, PCMK__XE_LRMD_RSC); pcmk__xe_set(data, PCMK__XA_LRMD_ORIGIN, __func__); pcmk__xe_set(data, PCMK__XA_LRMD_RSC_ACTION, action); pcmk__xe_set(data, PCMK__XA_LRMD_RSC_ID, rsc_id); pcmk__xe_set_guint(data, PCMK__XA_LRMD_RSC_INTERVAL, interval_ms); rc = lrmd_send_command(lrmd, LRMD_OP_RSC_CANCEL, data, NULL, 0, 0, true); pcmk__xml_free(data); return rc; } static int list_stonith_agents(lrmd_list_t ** resources) { int rc = 0; stonith_t *stonith_api = stonith_api_new(); stonith_key_value_t *stonith_resources = NULL; stonith_key_value_t *dIter = NULL; if (stonith_api == NULL) { pcmk__err("Could not list fence agents: API memory allocation failed"); return -ENOMEM; } stonith_api->cmds->list_agents(stonith_api, st_opt_sync_call, NULL, &stonith_resources, 0); stonith_api->cmds->free(stonith_api); for (dIter = stonith_resources; dIter; dIter = dIter->next) { rc++; if (resources) { *resources = lrmd_list_add(*resources, dIter->value); } } stonith_key_value_freeall(stonith_resources, 1, 0); return rc; } static int lrmd_api_list_agents(lrmd_t * lrmd, lrmd_list_t ** resources, const char *class, const char *provider) { int rc = 0; int stonith_count = 0; // Initially, whether to include stonith devices if (pcmk__str_eq(class, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_casei)) { stonith_count = 1; } else { GList *gIter = NULL; GList *agents = resources_list_agents(class, provider); for (gIter = agents; gIter != NULL; gIter = gIter->next) { *resources = lrmd_list_add(*resources, (const char *)gIter->data); rc++; } g_list_free_full(agents, free); if (!class) { stonith_count = 1; } } if (stonith_count) { // Now, if stonith devices are included, how many there are stonith_count = list_stonith_agents(resources); if (stonith_count > 0) { rc += stonith_count; } } if (rc == 0) { pcmk__notice("No agents found for class %s", class); rc = -EPROTONOSUPPORT; } return rc; } static bool does_provider_have_agent(const char *agent, const char *provider, const char *class) { bool found = false; GList *agents = NULL; GList *gIter2 = NULL; agents = resources_list_agents(class, provider); for (gIter2 = agents; gIter2 != NULL; gIter2 = gIter2->next) { if (pcmk__str_eq(agent, gIter2->data, pcmk__str_casei)) { found = true; } } g_list_free_full(agents, free); return found; } static int lrmd_api_list_ocf_providers(lrmd_t * lrmd, const char *agent, lrmd_list_t ** providers) { int rc = pcmk_ok; char *provider = NULL; GList *ocf_providers = NULL; GList *gIter = NULL; ocf_providers = resources_list_providers(PCMK_RESOURCE_CLASS_OCF); for (gIter = ocf_providers; gIter != NULL; gIter = gIter->next) { provider = gIter->data; if (!agent || does_provider_have_agent(agent, provider, PCMK_RESOURCE_CLASS_OCF)) { *providers = lrmd_list_add(*providers, (const char *)gIter->data); rc++; } } g_list_free_full(ocf_providers, free); return rc; } static int lrmd_api_list_standards(lrmd_t * lrmd, lrmd_list_t ** supported) { int rc = 0; GList *standards = NULL; GList *gIter = NULL; standards = resources_list_standards(); for (gIter = standards; gIter != NULL; gIter = gIter->next) { *supported = lrmd_list_add(*supported, (const char *)gIter->data); rc++; } if (list_stonith_agents(NULL) > 0) { *supported = lrmd_list_add(*supported, PCMK_RESOURCE_CLASS_STONITH); rc++; } g_list_free_full(standards, free); return rc; } /*! * \internal * \brief Create an executor API object * * \param[out] api Will be set to newly created API object (it is the * caller's responsibility to free this value with * lrmd_api_delete() if this function succeeds) * \param[in] nodename If the object will be used for a remote connection, * the node name to use in cluster for remote executor * \param[in] server If the object will be used for a remote connection, * the resolvable host name to connect to * \param[in] port If the object will be used for a remote connection, * port number on \p server to connect to * * \return Standard Pacemaker return code * \note If the caller leaves one of \p nodename or \p server NULL, the other's * value will be used for both. If the caller leaves both NULL, an API * object will be created for a local executor connection. */ int lrmd__new(lrmd_t **api, const char *nodename, const char *server, int port) { lrmd_private_t *pvt = NULL; if (api == NULL) { return EINVAL; } *api = NULL; // Allocate all memory needed *api = calloc(1, sizeof(lrmd_t)); if (*api == NULL) { return ENOMEM; } pvt = calloc(1, sizeof(lrmd_private_t)); if (pvt == NULL) { lrmd_api_delete(*api); *api = NULL; return ENOMEM; } (*api)->lrmd_private = pvt; // @TODO Do we need to do this for local connections? pvt->remote = calloc(1, sizeof(pcmk__remote_t)); (*api)->cmds = calloc(1, sizeof(lrmd_api_operations_t)); if ((pvt->remote == NULL) || ((*api)->cmds == NULL)) { lrmd_api_delete(*api); *api = NULL; return ENOMEM; } // Set methods (*api)->cmds->connect = lrmd_api_connect; (*api)->cmds->connect_async = lrmd_api_connect_async; (*api)->cmds->is_connected = lrmd_api_is_connected; (*api)->cmds->poke_connection = lrmd_api_poke_connection; (*api)->cmds->disconnect = lrmd_api_disconnect; (*api)->cmds->register_rsc = lrmd_api_register_rsc; (*api)->cmds->unregister_rsc = lrmd_api_unregister_rsc; (*api)->cmds->get_rsc_info = lrmd_api_get_rsc_info; (*api)->cmds->get_recurring_ops = lrmd_api_get_recurring_ops; (*api)->cmds->set_callback = lrmd_api_set_callback; (*api)->cmds->get_metadata = lrmd_api_get_metadata; (*api)->cmds->exec = lrmd_api_exec; (*api)->cmds->cancel = lrmd_api_cancel; (*api)->cmds->list_agents = lrmd_api_list_agents; (*api)->cmds->list_ocf_providers = lrmd_api_list_ocf_providers; (*api)->cmds->list_standards = lrmd_api_list_standards; (*api)->cmds->exec_alert = lrmd_api_exec_alert; (*api)->cmds->get_metadata_params = lrmd_api_get_metadata_params; if ((nodename == NULL) && (server == NULL)) { pvt->type = pcmk__client_ipc; } else { if (nodename == NULL) { nodename = server; } else if (server == NULL) { server = nodename; } pvt->type = pcmk__client_tls; pvt->remote_nodename = strdup(nodename); pvt->server = strdup(server); if ((pvt->remote_nodename == NULL) || (pvt->server == NULL)) { lrmd_api_delete(*api); *api = NULL; return ENOMEM; } pvt->port = port; if (pvt->port == 0) { pvt->port = crm_default_remote_port(); } } return pcmk_rc_ok; } lrmd_t * lrmd_api_new(void) { lrmd_t *api = NULL; pcmk__assert(lrmd__new(&api, NULL, NULL, 0) == pcmk_rc_ok); return api; } lrmd_t * lrmd_remote_api_new(const char *nodename, const char *server, int port) { lrmd_t *api = NULL; pcmk__assert(lrmd__new(&api, nodename, server, port) == pcmk_rc_ok); return api; } void lrmd_api_delete(lrmd_t * lrmd) { if (lrmd == NULL) { return; } if (lrmd->cmds != NULL) { // Never NULL, but make static analysis happy if (lrmd->cmds->disconnect != NULL) { // Also never really NULL lrmd->cmds->disconnect(lrmd); // No-op if already disconnected } free(lrmd->cmds); } if (lrmd->lrmd_private != NULL) { lrmd_private_t *native = lrmd->lrmd_private; free(native->server); free(native->remote_nodename); free(native->remote); free(native->token); free(native->peer_version); free(lrmd->lrmd_private); } free(lrmd); } struct metadata_cb { void (*callback)(int pid, const pcmk__action_result_t *result, void *user_data); void *user_data; }; /*! * \internal * \brief Process asynchronous metadata completion * * \param[in,out] action Metadata action that completed */ static void metadata_complete(svc_action_t *action) { struct metadata_cb *metadata_cb = (struct metadata_cb *) action->cb_data; pcmk__action_result_t result = PCMK__UNKNOWN_RESULT; services__copy_result(action, &result); pcmk__set_result_output(&result, action->stdout_data, action->stderr_data); metadata_cb->callback(0, &result, metadata_cb->user_data); result.action_stdout = NULL; // Prevent free, because action owns it result.action_stderr = NULL; // Prevent free, because action owns it pcmk__reset_result(&result); free(metadata_cb); } /*! * \internal * \brief Retrieve agent metadata asynchronously * * \param[in] rsc Resource agent specification * \param[in] callback Function to call with result (this will always be * called, whether by this function directly or later * via the main loop, and on success the metadata will * be in its result argument's action_stdout) * \param[in,out] user_data User data to pass to callback * * \return Standard Pacemaker return code * \note This function is not a lrmd_api_operations_t method because it does not * need an lrmd_t object and does not go through the executor, but * executes the agent directly. */ int lrmd__metadata_async(const lrmd_rsc_info_t *rsc, void (*callback)(int pid, const pcmk__action_result_t *result, void *user_data), void *user_data) { svc_action_t *action = NULL; struct metadata_cb *metadata_cb = NULL; pcmk__action_result_t result = PCMK__UNKNOWN_RESULT; CRM_CHECK(callback != NULL, return EINVAL); if ((rsc == NULL) || (rsc->standard == NULL) || (rsc->type == NULL)) { pcmk__set_result(&result, PCMK_OCF_NOT_CONFIGURED, PCMK_EXEC_ERROR_FATAL, "Invalid resource specification"); callback(0, &result, user_data); pcmk__reset_result(&result); return EINVAL; } if (strcmp(rsc->standard, PCMK_RESOURCE_CLASS_STONITH) == 0) { return stonith__metadata_async(rsc->type, pcmk__timeout_ms2s(PCMK_DEFAULT_ACTION_TIMEOUT_MS), callback, user_data); } action = services__create_resource_action(pcmk__s(rsc->id, rsc->type), rsc->standard, rsc->provider, rsc->type, PCMK_ACTION_META_DATA, 0, PCMK_DEFAULT_ACTION_TIMEOUT_MS, NULL, 0); if (action == NULL) { pcmk__set_result(&result, PCMK_OCF_UNKNOWN_ERROR, PCMK_EXEC_ERROR, "Out of memory"); callback(0, &result, user_data); pcmk__reset_result(&result); return ENOMEM; } if (action->rc != PCMK_OCF_UNKNOWN) { services__copy_result(action, &result); callback(0, &result, user_data); pcmk__reset_result(&result); services_action_free(action); return EINVAL; } action->cb_data = calloc(1, sizeof(struct metadata_cb)); if (action->cb_data == NULL) { services_action_free(action); pcmk__set_result(&result, PCMK_OCF_UNKNOWN_ERROR, PCMK_EXEC_ERROR, "Out of memory"); callback(0, &result, user_data); pcmk__reset_result(&result); return ENOMEM; } metadata_cb = (struct metadata_cb *) action->cb_data; metadata_cb->callback = callback; metadata_cb->user_data = user_data; if (!services_action_async(action, metadata_complete)) { services_action_free(action); return pcmk_rc_error; // @TODO Derive from action->rc and ->status } // The services library has taken responsibility for action return pcmk_rc_ok; } /*! * \internal * \brief Set the result of an executor event * * \param[in,out] event Executor event to set * \param[in] rc OCF exit status of event * \param[in] op_status Executor status of event * \param[in] exit_reason Human-friendly description of event */ void lrmd__set_result(lrmd_event_data_t *event, enum ocf_exitcode rc, int op_status, const char *exit_reason) { if (event == NULL) { return; } event->rc = rc; event->op_status = op_status; // lrmd_event_data_t has (const char *) members that lrmd_free_event() frees pcmk__str_update((char **) &event->exit_reason, exit_reason); } /*! * \internal * \brief Clear an executor event's exit reason, output, and error output * * \param[in,out] event Executor event to reset */ void lrmd__reset_result(lrmd_event_data_t *event) { if (event == NULL) { return; } free((void *) event->exit_reason); event->exit_reason = NULL; free((void *) event->output); event->output = NULL; } /*! * \internal * \brief Get the uptime of a remote resource connection * * When the cluster connects to a remote resource, part of that resource's * handshake includes the uptime of the remote resource's connection. This * uptime is stored in the lrmd_t object. * * \return The connection's uptime, or -1 if unknown */ time_t lrmd__uptime(lrmd_t *lrmd) { lrmd_private_t *native = lrmd->lrmd_private; if (native->remote == NULL) { return -1; } else { return native->remote->uptime; } } const char * lrmd__node_start_state(lrmd_t *lrmd) { lrmd_private_t *native = lrmd->lrmd_private; if (native->remote == NULL) { return NULL; } else { return native->remote->start_state; } } diff --git a/lib/pacemaker/pcmk_graph_consumer.c b/lib/pacemaker/pcmk_graph_consumer.c index 51ed65e719..5a4b0636b6 100644 --- a/lib/pacemaker/pcmk_graph_consumer.c +++ b/lib/pacemaker/pcmk_graph_consumer.c @@ -1,870 +1,870 @@ /* * Copyright 2004-2025 the Pacemaker project contributors * * The version control history for this file may have further details. * * This source code is licensed under the GNU Lesser General Public License * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. */ #include #include #include #include #include // crm_meta_value() #include // PCMK_SCORE_INFINITY #include #include #include #include /* * Functions for freeing transition graph objects */ /*! * \internal * \brief Free a transition graph action object * * \param[in,out] user_data Action to free */ static void free_graph_action(gpointer user_data) { pcmk__graph_action_t *action = user_data; if (action->timer != 0) { pcmk__warn("Cancelling timer for graph action %d", action->id); g_source_remove(action->timer); } if (action->params != NULL) { g_hash_table_destroy(action->params); } pcmk__xml_free(action->xml); free(action); } /*! * \internal * \brief Free a transition graph synapse object * * \param[in,out] user_data Synapse to free */ static void free_graph_synapse(gpointer user_data) { pcmk__graph_synapse_t *synapse = user_data; g_list_free_full(synapse->actions, free_graph_action); g_list_free_full(synapse->inputs, free_graph_action); free(synapse); } /*! * \internal * \brief Free a transition graph object * * \param[in,out] graph Transition graph to free */ void pcmk__free_graph(pcmk__graph_t *graph) { if (graph != NULL) { g_list_free_full(graph->synapses, free_graph_synapse); free(graph->source); free(graph->failed_stop_offset); free(graph->failed_start_offset); free(graph); } } /* * Functions for updating graph */ /*! * \internal * \brief Update synapse after completed prerequisite * * A synapse is ready to be executed once all its prerequisite actions (inputs) * complete. Given a completed action, check whether it is an input for a given * synapse, and if so, mark the input as confirmed, and mark the synapse as * ready if appropriate. * * \param[in,out] synapse Transition graph synapse to update * \param[in] action_id ID of an action that completed * * \note The only substantial effect here is confirming synapse inputs. * should_fire_synapse() will recalculate pcmk__synapse_ready, so the only * thing that uses the pcmk__synapse_ready from here is * synapse_state_str(). */ static void update_synapse_ready(pcmk__graph_synapse_t *synapse, int action_id) { if (pcmk__is_set(synapse->flags, pcmk__synapse_ready)) { return; // All inputs have already been confirmed } // Presume ready until proven otherwise pcmk__set_synapse_flags(synapse, pcmk__synapse_ready); for (GList *lpc = synapse->inputs; lpc != NULL; lpc = lpc->next) { pcmk__graph_action_t *prereq = (pcmk__graph_action_t *) lpc->data; if (prereq->id == action_id) { pcmk__trace("Confirming input %d of synapse %d", action_id, synapse->id); pcmk__set_graph_action_flags(prereq, pcmk__graph_action_confirmed); } else if (!pcmk__is_set(prereq->flags, pcmk__graph_action_confirmed)) { pcmk__clear_synapse_flags(synapse, pcmk__synapse_ready); pcmk__trace("Synapse %d still not ready after action %d", synapse->id, action_id); } } if (pcmk__is_set(synapse->flags, pcmk__synapse_ready)) { pcmk__trace("Synapse %d is now ready to execute", synapse->id); } } /*! * \internal * \brief Update action and synapse confirmation after action completion * * \param[in,out] synapse Transition graph synapse that action belongs to * \param[in] action_id ID of action that completed */ static void update_synapse_confirmed(pcmk__graph_synapse_t *synapse, int action_id) { bool all_confirmed = true; for (GList *lpc = synapse->actions; lpc != NULL; lpc = lpc->next) { pcmk__graph_action_t *action = (pcmk__graph_action_t *) lpc->data; if (action->id == action_id) { pcmk__trace("Confirmed action %d of synapse %d", action_id, synapse->id); pcmk__set_graph_action_flags(action, pcmk__graph_action_confirmed); } else if (all_confirmed && !pcmk__is_set(action->flags, pcmk__graph_action_confirmed)) { all_confirmed = false; pcmk__trace("Synapse %d still not confirmed after action %d", synapse->id, action_id); } } if (all_confirmed && !pcmk__is_set(synapse->flags, pcmk__synapse_confirmed)) { pcmk__trace("Confirmed synapse %d", synapse->id); pcmk__set_synapse_flags(synapse, pcmk__synapse_confirmed); } } /*! * \internal * \brief Update the transition graph with a completed action result * * \param[in,out] graph Transition graph to update * \param[in] action Action that completed */ void pcmk__update_graph(pcmk__graph_t *graph, const pcmk__graph_action_t *action) { for (GList *lpc = graph->synapses; lpc != NULL; lpc = lpc->next) { pcmk__graph_synapse_t *synapse = (pcmk__graph_synapse_t *) lpc->data; if (pcmk__any_flags_set(synapse->flags, pcmk__synapse_confirmed|pcmk__synapse_failed)) { continue; // This synapse already completed } else if (pcmk__is_set(synapse->flags, pcmk__synapse_executed)) { update_synapse_confirmed(synapse, action->id); } else if (!pcmk__is_set(action->flags, pcmk__graph_action_failed) || (synapse->priority == PCMK_SCORE_INFINITY)) { update_synapse_ready(synapse, action->id); } } } /* * Functions for executing graph */ /* A transition graph consists of various types of actions. The library caller * registers execution functions for each action type, which will be stored * here. */ static pcmk__graph_functions_t *graph_fns = NULL; /*! * \internal * \brief Set transition graph execution functions * * \param[in] Execution functions to use */ void pcmk__set_graph_functions(pcmk__graph_functions_t *fns) { pcmk__assert((fns != NULL) && (fns->rsc != NULL) && (fns->cluster != NULL) && (fns->pseudo != NULL) && (fns->fence != NULL)); pcmk__debug("Setting custom functions for executing transition graphs"); graph_fns = fns; } /*! * \internal * \brief Check whether a graph synapse is ready to be executed * * \param[in,out] graph Transition graph that synapse is part of * \param[in,out] synapse Synapse to check * * \return true if synapse is ready, false otherwise */ static bool should_fire_synapse(pcmk__graph_t *graph, pcmk__graph_synapse_t *synapse) { GList *lpc = NULL; pcmk__set_synapse_flags(synapse, pcmk__synapse_ready); for (lpc = synapse->inputs; lpc != NULL; lpc = lpc->next) { pcmk__graph_action_t *prereq = (pcmk__graph_action_t *) lpc->data; if (!(pcmk__is_set(prereq->flags, pcmk__graph_action_confirmed))) { pcmk__trace("Input %d for synapse %d not yet confirmed", prereq->id, synapse->id); pcmk__clear_synapse_flags(synapse, pcmk__synapse_ready); break; } else if (pcmk__is_set(prereq->flags, pcmk__graph_action_failed)) { pcmk__trace("Input %d for synapse %d confirmed but failed", prereq->id, synapse->id); pcmk__clear_synapse_flags(synapse, pcmk__synapse_ready); break; } } if (pcmk__is_set(synapse->flags, pcmk__synapse_ready)) { pcmk__trace("Synapse %d is ready to execute", synapse->id); } else { return false; } for (lpc = synapse->actions; lpc != NULL; lpc = lpc->next) { pcmk__graph_action_t *a = (pcmk__graph_action_t *) lpc->data; if (a->type == pcmk__pseudo_graph_action) { /* None of the below applies to pseudo ops */ } else if (synapse->priority < graph->abort_priority) { pcmk__trace("Skipping synapse %d: priority %d is less than abort " "priority %d", synapse->id, synapse->priority, graph->abort_priority); graph->skipped++; return false; } else if (graph_fns->allowed && !(graph_fns->allowed(graph, a))) { pcmk__trace("Deferring synapse %d: not allowed", synapse->id); return false; } } return true; } /*! * \internal * \brief Initiate an action from a transition graph * * \param[in,out] graph Transition graph containing action * \param[in,out] action Action to execute * * \return Standard Pacemaker return code */ static int initiate_action(pcmk__graph_t *graph, pcmk__graph_action_t *action) { const char *id = pcmk__xe_id(action->xml); CRM_CHECK(id != NULL, return EINVAL); CRM_CHECK(!pcmk__is_set(action->flags, pcmk__graph_action_executed), return pcmk_rc_already); pcmk__set_graph_action_flags(action, pcmk__graph_action_executed); switch (action->type) { case pcmk__pseudo_graph_action: pcmk__trace("Executing pseudo-action %d (%s)", action->id, id); return graph_fns->pseudo(graph, action); case pcmk__rsc_graph_action: pcmk__trace("Executing resource action %d (%s)", action->id, id); return graph_fns->rsc(graph, action); case pcmk__cluster_graph_action: if (pcmk__str_eq(pcmk__xe_get(action->xml, PCMK_XA_OPERATION), PCMK_ACTION_STONITH, pcmk__str_none)) { pcmk__trace("Executing fencing action %d (%s)", action->id, id); return graph_fns->fence(graph, action); } pcmk__trace("Executing cluster action %d (%s)", action->id, id); return graph_fns->cluster(graph, action); default: pcmk__err("Unsupported graph action type <%s " PCMK_XA_ID "='%s'> " "(bug?)", action->xml->name, id); return EINVAL; } } /*! * \internal * \brief Execute a graph synapse * * \param[in,out] graph Transition graph with synapse to execute * \param[in,out] synapse Synapse to execute * * \return Standard Pacemaker return value */ static int fire_synapse(pcmk__graph_t *graph, pcmk__graph_synapse_t *synapse) { pcmk__set_synapse_flags(synapse, pcmk__synapse_executed); for (GList *lpc = synapse->actions; lpc != NULL; lpc = lpc->next) { pcmk__graph_action_t *action = (pcmk__graph_action_t *) lpc->data; int rc = initiate_action(graph, action); if (rc != pcmk_rc_ok) { pcmk__err("Failed initiating <%s " PCMK_XA_ID "=%d> in synapse %d: " "%s", action->xml->name, action->id, synapse->id, pcmk_rc_str(rc)); pcmk__set_synapse_flags(synapse, pcmk__synapse_confirmed); pcmk__set_graph_action_flags(action, pcmk__graph_action_confirmed |pcmk__graph_action_failed); return pcmk_rc_error; } } return pcmk_rc_ok; } /*! * \internal * \brief Dummy graph method that can be used with simulations * * \param[in,out] graph Transition graph containing action * \param[in,out] action Graph action to be initiated * * \return Standard Pacemaker return code * \note If the PE_fail environment variable is set to the action ID, * then the graph action will be marked as failed. */ static int pseudo_action_dummy(pcmk__graph_t *graph, pcmk__graph_action_t *action) { static int fail = -1; if (fail < 0) { long long fail_ll; if ((pcmk__scan_ll(getenv("PE_fail"), &fail_ll, 0LL) == pcmk_rc_ok) && (fail_ll > 0LL) && (fail_ll <= INT_MAX)) { fail = (int) fail_ll; } else { fail = 0; } } if (action->id == fail) { pcmk__err("Dummy event handler: pretending action %d failed", action->id); pcmk__set_graph_action_flags(action, pcmk__graph_action_failed); graph->abort_priority = PCMK_SCORE_INFINITY; } else { pcmk__trace("Dummy event handler: action %d initiated", action->id); } pcmk__set_graph_action_flags(action, pcmk__graph_action_confirmed); pcmk__update_graph(graph, action); return pcmk_rc_ok; } static pcmk__graph_functions_t default_fns = { pseudo_action_dummy, pseudo_action_dummy, pseudo_action_dummy, pseudo_action_dummy }; /*! * \internal * \brief Execute all actions in a transition graph * * \param[in,out] graph Transition graph to execute * * \return Status of transition after execution */ enum pcmk__graph_status pcmk__execute_graph(pcmk__graph_t *graph) { GList *lpc = NULL; int log_level = LOG_DEBUG; enum pcmk__graph_status pass_result = pcmk__graph_active; const char *status = "In progress"; if (graph_fns == NULL) { graph_fns = &default_fns; } if (graph == NULL) { return pcmk__graph_complete; } graph->fired = 0; graph->pending = 0; graph->skipped = 0; graph->completed = 0; graph->incomplete = 0; // Count completed and in-flight synapses for (lpc = graph->synapses; lpc != NULL; lpc = lpc->next) { pcmk__graph_synapse_t *synapse = (pcmk__graph_synapse_t *) lpc->data; if (pcmk__is_set(synapse->flags, pcmk__synapse_confirmed)) { graph->completed++; } else if (!pcmk__is_set(synapse->flags, pcmk__synapse_failed) && pcmk__is_set(synapse->flags, pcmk__synapse_executed)) { graph->pending++; } } pcmk__trace("Executing graph %d (%d synapses already completed, %d " "pending)", graph->id, graph->completed, graph->pending); // Execute any synapses that are ready for (lpc = graph->synapses; lpc != NULL; lpc = lpc->next) { pcmk__graph_synapse_t *synapse = (pcmk__graph_synapse_t *) lpc->data; if ((graph->batch_limit > 0) && (graph->pending >= graph->batch_limit)) { pcmk__debug("Throttling graph execution: batch limit (%d) reached", graph->batch_limit); break; } else if (pcmk__is_set(synapse->flags, pcmk__synapse_failed)) { graph->skipped++; continue; } else if (pcmk__any_flags_set(synapse->flags, pcmk__synapse_confirmed |pcmk__synapse_executed)) { continue; // Already handled } else if (should_fire_synapse(graph, synapse)) { graph->fired++; if (fire_synapse(graph, synapse) != pcmk_rc_ok) { pcmk__err("Synapse %d failed to fire", synapse->id); log_level = LOG_ERR; graph->abort_priority = PCMK_SCORE_INFINITY; graph->incomplete++; graph->fired--; } if (!(pcmk__is_set(synapse->flags, pcmk__synapse_confirmed))) { graph->pending++; } } else { pcmk__trace("Synapse %d cannot fire", synapse->id); graph->incomplete++; } } if ((graph->pending == 0) && (graph->fired == 0)) { graph->complete = true; if ((graph->incomplete != 0) && (graph->abort_priority <= 0)) { log_level = LOG_WARNING; pass_result = pcmk__graph_terminated; status = "Terminated"; } else if (graph->skipped != 0) { log_level = LOG_NOTICE; pass_result = pcmk__graph_complete; status = "Stopped"; } else { log_level = LOG_NOTICE; pass_result = pcmk__graph_complete; status = "Complete"; } } else if (graph->fired == 0) { pass_result = pcmk__graph_pending; } do_crm_log(log_level, "Transition %d (Complete=%d, Pending=%d," " Fired=%d, Skipped=%d, Incomplete=%d, Source=%s): %s", graph->id, graph->completed, graph->pending, graph->fired, graph->skipped, graph->incomplete, graph->source, status); return pass_result; } /* * Functions for unpacking transition graph XML into structs */ /*! * \internal * \brief Unpack a transition graph action from XML * * \param[in] parent Synapse that action is part of * \param[in] xml_action Action XML to unparse * * \return Newly allocated action on success, or NULL otherwise */ static pcmk__graph_action_t * unpack_action(pcmk__graph_synapse_t *parent, xmlNode *xml_action) { enum pcmk__graph_action_type action_type; pcmk__graph_action_t *action = NULL; const char *value = pcmk__xe_id(xml_action); if (value == NULL) { pcmk__err("Ignoring transition graph action without " PCMK_XA_ID " (bug?)"); - crm_log_xml_trace(xml_action, "invalid"); + pcmk__log_xml_trace(xml_action, "invalid"); return NULL; } if (pcmk__xe_is(xml_action, PCMK__XE_RSC_OP)) { action_type = pcmk__rsc_graph_action; } else if (pcmk__xe_is(xml_action, PCMK__XE_PSEUDO_EVENT)) { action_type = pcmk__pseudo_graph_action; } else if (pcmk__xe_is(xml_action, PCMK__XE_CRM_EVENT)) { action_type = pcmk__cluster_graph_action; } else { pcmk__err("Ignoring transition graph action of unknown type '%s' " "(bug?)", xml_action->name); - crm_log_xml_trace(xml_action, "invalid"); + pcmk__log_xml_trace(xml_action, "invalid"); return NULL; } action = calloc(1, sizeof(pcmk__graph_action_t)); if (action == NULL) { crm_perror(LOG_CRIT, "Cannot unpack transition graph action"); - crm_log_xml_trace(xml_action, "lost"); + pcmk__log_xml_trace(xml_action, "lost"); return NULL; } pcmk__scan_min_int(value, &(action->id), -1); action->type = pcmk__rsc_graph_action; action->xml = pcmk__xml_copy(NULL, xml_action); action->synapse = parent; action->type = action_type; action->params = xml2list(action->xml); value = crm_meta_value(action->params, PCMK_META_TIMEOUT); pcmk__scan_min_int(value, &(action->timeout), 0); /* Take PCMK_META_START_DELAY into account for the timeout of the action * timer */ value = crm_meta_value(action->params, PCMK_META_START_DELAY); { int start_delay; pcmk__scan_min_int(value, &start_delay, 0); action->timeout += start_delay; } if (pcmk__guint_from_hash(action->params, CRM_META "_" PCMK_META_INTERVAL, 0, &(action->interval_ms)) != pcmk_rc_ok) { action->interval_ms = 0; } pcmk__trace("Action %d has timer set to %dms", action->id, action->timeout); return action; } /*! * \internal * \brief Unpack transition graph synapse from XML * * \param[in,out] new_graph Transition graph that synapse is part of * \param[in] xml_synapse Synapse XML * * \return Newly allocated synapse on success, or NULL otherwise */ static pcmk__graph_synapse_t * unpack_synapse(pcmk__graph_t *new_graph, const xmlNode *xml_synapse) { const char *value = NULL; xmlNode *action_set = NULL; pcmk__graph_synapse_t *new_synapse = NULL; pcmk__trace("Unpacking synapse %s", pcmk__xe_id(xml_synapse)); new_synapse = calloc(1, sizeof(pcmk__graph_synapse_t)); if (new_synapse == NULL) { return NULL; } pcmk__scan_min_int(pcmk__xe_id(xml_synapse), &(new_synapse->id), 0); value = pcmk__xe_get(xml_synapse, PCMK__XA_PRIORITY); pcmk__scan_min_int(value, &(new_synapse->priority), 0); CRM_CHECK(new_synapse->id >= 0, free_graph_synapse((gpointer) new_synapse); return NULL); new_graph->num_synapses++; pcmk__trace("Unpacking synapse %s action sets", pcmk__xe_get(xml_synapse, PCMK_XA_ID)); for (action_set = pcmk__xe_first_child(xml_synapse, PCMK__XE_ACTION_SET, NULL, NULL); action_set != NULL; action_set = pcmk__xe_next(action_set, PCMK__XE_ACTION_SET)) { for (xmlNode *action = pcmk__xe_first_child(action_set, NULL, NULL, NULL); action != NULL; action = pcmk__xe_next(action, NULL)) { pcmk__graph_action_t *new_action = unpack_action(new_synapse, action); if (new_action == NULL) { continue; } pcmk__trace("Adding action %d to synapse %d", new_action->id, new_synapse->id); new_graph->num_actions++; new_synapse->actions = g_list_append(new_synapse->actions, new_action); } } pcmk__trace("Unpacking synapse %s inputs", pcmk__xe_id(xml_synapse)); for (xmlNode *inputs = pcmk__xe_first_child(xml_synapse, PCMK__XE_INPUTS, NULL, NULL); inputs != NULL; inputs = pcmk__xe_next(inputs, PCMK__XE_INPUTS)) { for (xmlNode *trigger = pcmk__xe_first_child(inputs, PCMK__XE_TRIGGER, NULL, NULL); trigger != NULL; trigger = pcmk__xe_next(trigger, PCMK__XE_TRIGGER)) { for (xmlNode *input = pcmk__xe_first_child(trigger, NULL, NULL, NULL); input != NULL; input = pcmk__xe_next(input, NULL)) { pcmk__graph_action_t *new_input = unpack_action(new_synapse, input); if (new_input == NULL) { continue; } pcmk__trace("Adding input %d to synapse %d", new_input->id, new_synapse->id); new_synapse->inputs = g_list_append(new_synapse->inputs, new_input); } } } return new_synapse; } /*! * \internal * \brief Unpack transition graph XML * * \param[in] xml_graph Transition graph XML to unpack * \param[in] reference Where the XML came from (for logging) * * \return Newly allocated transition graph on success, NULL otherwise * \note The caller is responsible for freeing the return value using * pcmk__free_graph(). * \note The XML is expected to be structured like: ... ... */ pcmk__graph_t * pcmk__unpack_graph(const xmlNode *xml_graph, const char *reference) { pcmk__graph_t *new_graph = NULL; new_graph = calloc(1, sizeof(pcmk__graph_t)); if (new_graph == NULL) { return NULL; } new_graph->source = strdup(pcmk__s(reference, "unknown")); if (new_graph->source == NULL) { pcmk__free_graph(new_graph); return NULL; } new_graph->completion_action = pcmk__graph_done; // Parse top-level attributes from PCMK__XE_TRANSITION_GRAPH if (xml_graph != NULL) { const char *buf = pcmk__xe_get(xml_graph, "transition_id"); CRM_CHECK(buf != NULL, pcmk__free_graph(new_graph); return NULL); pcmk__scan_min_int(buf, &(new_graph->id), 1); buf = pcmk__xe_get(xml_graph, PCMK_OPT_CLUSTER_DELAY); CRM_CHECK(buf != NULL, pcmk__free_graph(new_graph); return NULL); pcmk_parse_interval_spec(buf, &(new_graph->network_delay)); buf = pcmk__xe_get(xml_graph, PCMK_OPT_STONITH_TIMEOUT); if (buf == NULL) { new_graph->stonith_timeout = new_graph->network_delay; } else { pcmk_parse_interval_spec(buf, &(new_graph->stonith_timeout)); } // Use 0 (dynamic limit) as default/invalid, -1 (no limit) as minimum buf = pcmk__xe_get(xml_graph, PCMK_OPT_BATCH_LIMIT); if ((buf == NULL) || (pcmk__scan_min_int(buf, &(new_graph->batch_limit), -1) != pcmk_rc_ok)) { new_graph->batch_limit = 0; } buf = pcmk__xe_get(xml_graph, PCMK_OPT_MIGRATION_LIMIT); pcmk__scan_min_int(buf, &(new_graph->migration_limit), -1); new_graph->failed_stop_offset = pcmk__xe_get_copy(xml_graph, PCMK__XA_FAILED_STOP_OFFSET); new_graph->failed_start_offset = pcmk__xe_get_copy(xml_graph, PCMK__XA_FAILED_START_OFFSET); pcmk__xe_get_time(xml_graph, "recheck-by", &(new_graph->recheck_by)); } // Unpack each child element for (const xmlNode *synapse_xml = pcmk__xe_first_child(xml_graph, PCMK__XE_SYNAPSE, NULL, NULL); synapse_xml != NULL; synapse_xml = pcmk__xe_next(synapse_xml, PCMK__XE_SYNAPSE)) { pcmk__graph_synapse_t *new_synapse = unpack_synapse(new_graph, synapse_xml); if (new_synapse != NULL) { new_graph->synapses = g_list_append(new_graph->synapses, new_synapse); } } pcmk__debug("Unpacked transition %d from %s: %d actions in %d synapses", new_graph->id, new_graph->source, new_graph->num_actions, new_graph->num_synapses); return new_graph; } /* * Other transition graph utilities */ /*! * \internal * \brief Synthesize an executor event from a graph action * * \param[in] resource If not NULL, use greater call ID than in this XML * \param[in] action Graph action * \param[in] status What to use as event execution status * \param[in] rc What to use as event exit status * \param[in] exit_reason What to use as event exit reason * * \return Newly allocated executor event on success, or NULL otherwise */ lrmd_event_data_t * pcmk__event_from_graph_action(const xmlNode *resource, const pcmk__graph_action_t *action, int status, int rc, const char *exit_reason) { lrmd_event_data_t *op = NULL; GHashTableIter iter; const char *name = NULL; const char *value = NULL; xmlNode *action_resource = NULL; CRM_CHECK(action != NULL, return NULL); CRM_CHECK(action->type == pcmk__rsc_graph_action, return NULL); action_resource = pcmk__xe_first_child(action->xml, PCMK_XE_PRIMITIVE, NULL, NULL); CRM_CHECK(action_resource != NULL, pcmk__log_xml_warn(action->xml, "invalid"); return NULL); op = lrmd_new_event(pcmk__xe_id(action_resource), pcmk__xe_get(action->xml, PCMK_XA_OPERATION), action->interval_ms); lrmd__set_result(op, rc, status, exit_reason); op->t_run = time(NULL); op->t_rcchange = op->t_run; op->params = pcmk__strkey_table(free, free); g_hash_table_iter_init(&iter, action->params); while (g_hash_table_iter_next(&iter, (void **)&name, (void **)&value)) { pcmk__insert_dup(op->params, name, value); } for (xmlNode *xop = pcmk__xe_first_child(resource, NULL, NULL, NULL); xop != NULL; xop = pcmk__xe_next(xop, NULL)) { int tmp = 0; pcmk__xe_get_int(xop, PCMK__XA_CALL_ID, &tmp); pcmk__debug("Got call_id=%d for %s", tmp, pcmk__xe_id(resource)); if (tmp > op->call_id) { op->call_id = tmp; } } op->call_id++; return op; } diff --git a/lib/pacemaker/pcmk_graph_producer.c b/lib/pacemaker/pcmk_graph_producer.c index c72236ff82..4d6ed1f43f 100644 --- a/lib/pacemaker/pcmk_graph_producer.c +++ b/lib/pacemaker/pcmk_graph_producer.c @@ -1,1095 +1,1095 @@ /* * Copyright 2004-2025 the Pacemaker project contributors * * The version control history for this file may have further details. * * This source code is licensed under the GNU General Public License version 2 * or later (GPLv2+) WITHOUT ANY WARRANTY. */ #include #include #include #include #include // hash2field(), etc. #include #include #include #include "libpacemaker_private.h" // Convenience macros for logging action properties #define action_type_str(flags) \ (pcmk__is_set((flags), pcmk__action_pseudo)? "pseudo-action" : "action") #define action_optional_str(flags) \ (pcmk__is_set((flags), pcmk__action_optional)? "optional" : "required") #define action_runnable_str(flags) \ (pcmk__is_set((flags), pcmk__action_runnable)? "runnable" : "unrunnable") #define action_node_str(a) \ (((a)->node == NULL)? "no node" : (a)->node->priv->name) /*! * \internal * \brief Add an XML node tag for a specified ID * * \param[in] id Node UUID to add * \param[in,out] xml Parent XML tag to add to */ static xmlNode* add_node_to_xml_by_id(const char *id, xmlNode *xml) { xmlNode *node_xml; node_xml = pcmk__xe_create(xml, PCMK_XE_NODE); pcmk__xe_set(node_xml, PCMK_XA_ID, id); return node_xml; } /*! * \internal * \brief Add an XML node tag for a specified node * * \param[in] node Node to add * \param[in,out] xml XML to add node to */ static void add_node_to_xml(const pcmk_node_t *node, void *xml) { add_node_to_xml_by_id(node->priv->id, (xmlNode *) xml); } /*! * \internal * \brief Count (optionally add to XML) nodes needing maintenance state update * * \param[in,out] xml Parent XML tag to add to, if any * \param[in] scheduler Scheduler data * * \return Count of nodes added * \note Only Pacemaker Remote nodes are considered currently */ static int add_maintenance_nodes(xmlNode *xml, const pcmk_scheduler_t *scheduler) { xmlNode *maintenance = NULL; int count = 0; if (xml != NULL) { maintenance = pcmk__xe_create(xml, PCMK__XE_MAINTENANCE); } for (const GList *iter = scheduler->nodes; iter != NULL; iter = iter->next) { const pcmk_node_t *node = iter->data; if (!pcmk__is_pacemaker_remote_node(node)) { continue; } if ((node->details->maintenance && !pcmk__is_set(node->priv->flags, pcmk__node_remote_maint)) || (!node->details->maintenance && pcmk__is_set(node->priv->flags, pcmk__node_remote_maint))) { if (maintenance != NULL) { pcmk__xe_set(add_node_to_xml_by_id(node->priv->id, maintenance), PCMK__XA_NODE_IN_MAINTENANCE, (node->details->maintenance? "1" : "0")); } count++; } } pcmk__trace("%s %d nodes in need of maintenance mode update in state", ((maintenance == NULL)? "Counted" : "Added"), count); return count; } /*! * \internal * \brief Add pseudo action with nodes needing maintenance state update * * \param[in,out] scheduler Scheduler data */ static void add_maintenance_update(pcmk_scheduler_t *scheduler) { pcmk_action_t *action = NULL; if (add_maintenance_nodes(NULL, scheduler) != 0) { action = get_pseudo_op(PCMK_ACTION_MAINTENANCE_NODES, scheduler); pcmk__set_action_flags(action, pcmk__action_always_in_graph); } } /*! * \internal * \brief Add XML with nodes that an action is expected to bring down * * If a specified action is expected to bring any nodes down, add an XML block * with their UUIDs. When a node is lost, this allows the controller to * determine whether it was expected. * * \param[in,out] xml Parent XML tag to add to * \param[in] action Action to check for downed nodes */ static void add_downed_nodes(xmlNode *xml, const pcmk_action_t *action) { CRM_CHECK((xml != NULL) && (action != NULL) && (action->node != NULL), return); if (pcmk__str_eq(action->task, PCMK_ACTION_DO_SHUTDOWN, pcmk__str_none)) { /* Shutdown makes the action's node down */ xmlNode *downed = pcmk__xe_create(xml, PCMK__XE_DOWNED); add_node_to_xml_by_id(action->node->priv->id, downed); } else if (pcmk__str_eq(action->task, PCMK_ACTION_STONITH, pcmk__str_none)) { /* Fencing makes the action's node and any hosted guest nodes down */ const char *fence = g_hash_table_lookup(action->meta, PCMK__META_STONITH_ACTION); if (pcmk__is_fencing_action(fence)) { xmlNode *downed = pcmk__xe_create(xml, PCMK__XE_DOWNED); add_node_to_xml_by_id(action->node->priv->id, downed); pe_foreach_guest_node(action->node->priv->scheduler, action->node, add_node_to_xml, downed); } } else if ((action->rsc != NULL) && pcmk__is_set(action->rsc->flags, pcmk__rsc_is_remote_connection) && pcmk__str_eq(action->task, PCMK_ACTION_STOP, pcmk__str_none)) { /* Stopping a remote connection resource makes connected node down, * unless it's part of a migration */ GList *iter; pcmk_action_t *input; bool migrating = false; for (iter = action->actions_before; iter != NULL; iter = iter->next) { input = ((pcmk__related_action_t *) iter->data)->action; if ((input->rsc != NULL) && pcmk__str_eq(action->rsc->id, input->rsc->id, pcmk__str_none) && pcmk__str_eq(input->task, PCMK_ACTION_MIGRATE_FROM, pcmk__str_none)) { migrating = true; break; } } if (!migrating) { xmlNode *downed = pcmk__xe_create(xml, PCMK__XE_DOWNED); add_node_to_xml_by_id(action->rsc->id, downed); } } } /*! * \internal * \brief Create a transition graph operation key for a clone action * * \param[in] action Clone action * \param[in] interval_ms Action interval in milliseconds * * \return Newly allocated string with transition graph operation key */ static char * clone_op_key(const pcmk_action_t *action, guint interval_ms) { if (pcmk__str_eq(action->task, PCMK_ACTION_NOTIFY, pcmk__str_none)) { const char *n_type = g_hash_table_lookup(action->meta, "notify_type"); const char *n_task = g_hash_table_lookup(action->meta, "notify_operation"); return pcmk__notify_key(action->rsc->priv->history_id, n_type, n_task); } return pcmk__op_key(action->rsc->priv->history_id, pcmk__s(action->cancel_task, action->task), interval_ms); } /*! * \internal * \brief Add node details to transition graph action XML * * \param[in] action Scheduled action * \param[in,out] xml Transition graph action XML for \p action */ static void add_node_details(const pcmk_action_t *action, xmlNode *xml) { pcmk_node_t *router_node = pcmk__connection_host_for_action(action); pcmk__xe_set(xml, PCMK__META_ON_NODE, action->node->priv->name); pcmk__xe_set(xml, PCMK__META_ON_NODE_UUID, action->node->priv->id); if (router_node != NULL) { pcmk__xe_set(xml, PCMK__XA_ROUTER_NODE, router_node->priv->name); } } /*! * \internal * \brief Add resource details to transition graph action XML * * \param[in] action Scheduled action * \param[in,out] action_xml Transition graph action XML for \p action */ static void add_resource_details(const pcmk_action_t *action, xmlNode *action_xml) { xmlNode *rsc_xml = NULL; const char *attr_list[] = { PCMK_XA_CLASS, PCMK_XA_PROVIDER, PCMK_XA_TYPE, }; /* If a resource is locked to a node via PCMK_OPT_SHUTDOWN_LOCK, mark its * actions so the controller can preserve the lock when the action * completes. */ if (pcmk__action_locks_rsc_to_node(action)) { pcmk__xe_set_time(action_xml, PCMK_OPT_SHUTDOWN_LOCK, action->rsc->priv->lock_time); } // List affected resource rsc_xml = pcmk__xe_create(action_xml, (const char *) action->rsc->priv->xml->name); if (pcmk__is_set(action->rsc->flags, pcmk__rsc_removed) && (action->rsc->priv->history_id != NULL)) { /* Use the numbered instance name here, because if there is more * than one instance on a node, we need to make sure the command * goes to the right one. * * This is important even for anonymous clones, because the clone's * unique meta-attribute might have just been toggled from on to * off. */ pcmk__debug("Using orphan clone name %s instead of history ID %s", action->rsc->id, action->rsc->priv->history_id); pcmk__xe_set(rsc_xml, PCMK_XA_ID, action->rsc->priv->history_id); pcmk__xe_set(rsc_xml, PCMK__XA_LONG_ID, action->rsc->id); } else if (!pcmk__is_set(action->rsc->flags, pcmk__rsc_unique)) { const char *xml_id = pcmk__xe_id(action->rsc->priv->xml); pcmk__debug("Using anonymous clone name %s for %s (aka %s)", xml_id, action->rsc->id, action->rsc->priv->history_id); /* ID is what we'd like client to use * LONG_ID is what they might know it as instead * * LONG_ID is only strictly needed /here/ during the * transition period until all nodes in the cluster * are running the new software /and/ have rebooted * once (meaning that they've only ever spoken to a DC * supporting this feature). (@TODO The effect of removing this on * regression tests suggests that it is still needed for unique clones) * * If anyone toggles the unique flag to 'on', the * 'instance free' name will correspond to an orphan * and fall into the clause above instead */ pcmk__xe_set(rsc_xml, PCMK_XA_ID, xml_id); if ((action->rsc->priv->history_id != NULL) && !pcmk__str_eq(xml_id, action->rsc->priv->history_id, pcmk__str_none)) { pcmk__xe_set(rsc_xml, PCMK__XA_LONG_ID, action->rsc->priv->history_id); } else { pcmk__xe_set(rsc_xml, PCMK__XA_LONG_ID, action->rsc->id); } } else { pcmk__assert(action->rsc->priv->history_id == NULL); pcmk__xe_set(rsc_xml, PCMK_XA_ID, action->rsc->id); } for (int lpc = 0; lpc < PCMK__NELEM(attr_list); lpc++) { pcmk__xe_set(rsc_xml, attr_list[lpc], g_hash_table_lookup(action->rsc->priv->meta, attr_list[lpc])); } } /*! * \internal * \brief Add action attributes to transition graph action XML * * \param[in,out] action Scheduled action * \param[in,out] action_xml Transition graph action XML for \p action */ static void add_action_attributes(pcmk_action_t *action, xmlNode *action_xml) { xmlNode *args_xml = NULL; pcmk_resource_t *rsc = action->rsc; /* We create free-standing XML to start, so we can sort the attributes * before adding it to action_xml, which keeps the scheduler regression * test graphs comparable. */ args_xml = pcmk__xe_create(action_xml, PCMK__XE_ATTRIBUTES); pcmk__xe_set(args_xml, PCMK_XA_CRM_FEATURE_SET, CRM_FEATURE_SET); g_hash_table_foreach(action->extra, hash2field, args_xml); if ((rsc != NULL) && (action->node != NULL)) { // Get the resource instance attributes, evaluated properly for node GHashTable *params = pe_rsc_params(rsc, action->node, rsc->priv->scheduler); pcmk__substitute_remote_addr(rsc, params); g_hash_table_foreach(params, hash2smartfield, args_xml); } else if ((rsc != NULL) && (rsc->priv->variant <= pcmk__rsc_variant_primitive)) { GHashTable *params = pe_rsc_params(rsc, NULL, rsc->priv->scheduler); g_hash_table_foreach(params, hash2smartfield, args_xml); } g_hash_table_foreach(action->meta, hash2metafield, args_xml); if (rsc != NULL) { pcmk_resource_t *parent = rsc; while (parent != NULL) { parent->priv->cmds->add_graph_meta(parent, args_xml); parent = parent->priv->parent; } pcmk__add_guest_meta_to_xml(args_xml, action); } pcmk__xe_sort_attrs(args_xml); } /*! * \internal * \brief Create the transition graph XML for a scheduled action * * \param[in,out] parent Parent XML element to add action to * \param[in,out] action Scheduled action * \param[in] skip_details If false, add action details as sub-elements * \param[in] scheduler Scheduler data */ static void create_graph_action(xmlNode *parent, pcmk_action_t *action, bool skip_details, const pcmk_scheduler_t *scheduler) { bool needs_node_info = true; bool needs_maintenance_info = false; xmlNode *action_xml = NULL; if ((action == NULL) || (scheduler == NULL)) { return; } // Create the top-level element based on task if (pcmk__str_eq(action->task, PCMK_ACTION_STONITH, pcmk__str_none)) { /* All fences need node info; guest node fences are pseudo-events */ if (pcmk__is_set(action->flags, pcmk__action_pseudo)) { action_xml = pcmk__xe_create(parent, PCMK__XE_PSEUDO_EVENT); } else { action_xml = pcmk__xe_create(parent, PCMK__XE_CRM_EVENT); } } else if (pcmk__str_any_of(action->task, PCMK_ACTION_DO_SHUTDOWN, PCMK_ACTION_CLEAR_FAILCOUNT, NULL)) { action_xml = pcmk__xe_create(parent, PCMK__XE_CRM_EVENT); } else if (pcmk__str_eq(action->task, PCMK_ACTION_LRM_DELETE, pcmk__str_none)) { // CIB-only clean-up for shutdown locks action_xml = pcmk__xe_create(parent, PCMK__XE_CRM_EVENT); pcmk__xe_set(action_xml, PCMK__XA_MODE, PCMK__VALUE_CIB); } else if (pcmk__is_set(action->flags, pcmk__action_pseudo)) { if (pcmk__str_eq(action->task, PCMK_ACTION_MAINTENANCE_NODES, pcmk__str_none)) { needs_maintenance_info = true; } action_xml = pcmk__xe_create(parent, PCMK__XE_PSEUDO_EVENT); needs_node_info = false; } else { action_xml = pcmk__xe_create(parent, PCMK__XE_RSC_OP); } pcmk__xe_set_int(action_xml, PCMK_XA_ID, action->id); pcmk__xe_set(action_xml, PCMK_XA_OPERATION, action->task); if ((action->rsc != NULL) && (action->rsc->priv->history_id != NULL)) { char *clone_key = NULL; guint interval_ms; if (pcmk__guint_from_hash(action->meta, PCMK_META_INTERVAL, 0, &interval_ms) != pcmk_rc_ok) { interval_ms = 0; } clone_key = clone_op_key(action, interval_ms); pcmk__xe_set(action_xml, PCMK__XA_OPERATION_KEY, clone_key); pcmk__xe_set(action_xml, "internal_" PCMK__XA_OPERATION_KEY, action->uuid); free(clone_key); } else { pcmk__xe_set(action_xml, PCMK__XA_OPERATION_KEY, action->uuid); } if (needs_node_info && (action->node != NULL)) { add_node_details(action, action_xml); pcmk__insert_dup(action->meta, PCMK__META_ON_NODE, action->node->priv->name); pcmk__insert_dup(action->meta, PCMK__META_ON_NODE_UUID, action->node->priv->id); } if (skip_details) { return; } if ((action->rsc != NULL) && !pcmk__is_set(action->flags, pcmk__action_pseudo)) { // This is a real resource action, so add resource details add_resource_details(action, action_xml); } /* List any attributes in effect */ add_action_attributes(action, action_xml); /* List any nodes this action is expected to make down */ if (needs_node_info && (action->node != NULL)) { add_downed_nodes(action_xml, action); } if (needs_maintenance_info) { add_maintenance_nodes(action_xml, scheduler); } } /*! * \internal * \brief Check whether an action should be added to the transition graph * * \param[in,out] action Action to check * * \return true if action should be added to graph, otherwise false */ static bool should_add_action_to_graph(pcmk_action_t *action) { if (!pcmk__is_set(action->flags, pcmk__action_runnable)) { pcmk__trace("Ignoring action %s (%d): unrunnable", action->uuid, action->id); return false; } if (pcmk__is_set(action->flags, pcmk__action_optional) && !pcmk__is_set(action->flags, pcmk__action_always_in_graph)) { pcmk__trace("Ignoring action %s (%d): optional", action->uuid, action->id); return false; } /* Actions for unmanaged resources should be excluded from the graph, * with the exception of monitors and cancellation of recurring monitors. */ if ((action->rsc != NULL) && !pcmk__is_set(action->rsc->flags, pcmk__rsc_managed) && !pcmk__str_eq(action->task, PCMK_ACTION_MONITOR, pcmk__str_none)) { const char *interval_ms_s; /* A cancellation of a recurring monitor will get here because the task * is cancel rather than monitor, but the interval can still be used to * recognize it. The interval has been normalized to milliseconds by * this point, so a string comparison is sufficient. */ interval_ms_s = g_hash_table_lookup(action->meta, PCMK_META_INTERVAL); if (pcmk__str_eq(interval_ms_s, "0", pcmk__str_null_matches)) { pcmk__trace("Ignoring action %s (%d): for unmanaged resource (%s)", action->uuid, action->id, action->rsc->id); return false; } } /* Always add pseudo-actions, fence actions, and shutdown actions (already * determined to be required and runnable by this point) */ if (pcmk__is_set(action->flags, pcmk__action_pseudo) || pcmk__strcase_any_of(action->task, PCMK_ACTION_STONITH, PCMK_ACTION_DO_SHUTDOWN, NULL)) { return true; } if (action->node == NULL) { pcmk__sched_err(action->scheduler, "Skipping action %s (%d) " "because it was not assigned to a node (bug?)", action->uuid, action->id); pcmk__log_action("Unassigned", action, false); return false; } if (pcmk__is_set(action->flags, pcmk__action_on_dc)) { pcmk__trace("Action %s (%d) should be dumped: " "can run on DC instead of %s", action->uuid, action->id, pcmk__node_name(action->node)); } else if (pcmk__is_guest_or_bundle_node(action->node) && !pcmk__is_set(action->node->priv->flags, pcmk__node_remote_reset)) { pcmk__trace("Action %s (%d) should be dumped: " "assuming will be runnable on guest %s", action->uuid, action->id, pcmk__node_name(action->node)); } else if (!action->node->details->online) { pcmk__sched_err(action->scheduler, "Skipping action %s (%d) " "because it was scheduled for offline node (bug?)", action->uuid, action->id); pcmk__log_action("Offline node", action, false); return false; } else if (action->node->details->unclean) { pcmk__sched_err(action->scheduler, "Skipping action %s (%d) " "because it was scheduled for unclean node (bug?)", action->uuid, action->id); pcmk__log_action("Unclean node", action, false); return false; } return true; } /*! * \internal * \brief Check whether an ordering's flags can change an action * * \param[in] ordering Ordering to check * * \return true if ordering has flags that can change an action, false otherwise */ static bool ordering_can_change_actions(const pcmk__related_action_t *ordering) { return pcmk__any_flags_set(ordering->flags, ~(pcmk__ar_then_implies_first_graphed |pcmk__ar_first_implies_then_graphed |pcmk__ar_ordered)); } /*! * \internal * \brief Check whether an action input should be in the transition graph * * \param[in] action Action to check * \param[in,out] input Action input to check * * \return true if input should be in graph, false otherwise * \note This function may not only check an input, but disable it under certian * circumstances (load or anti-colocation orderings that are not needed). */ static bool should_add_input_to_graph(const pcmk_action_t *action, pcmk__related_action_t *input) { if (input->graphed) { return true; } if (input->flags == pcmk__ar_none) { pcmk__trace("Ignoring %s (%d) input %s (%d): " "ordering disabled", action->uuid, action->id, input->action->uuid, input->action->id); return false; } else if (!pcmk__is_set(input->action->flags, pcmk__action_runnable) && !ordering_can_change_actions(input)) { pcmk__trace("Ignoring %s (%d) input %s (%d): " "optional and input unrunnable", action->uuid, action->id, input->action->uuid, input->action->id); return false; } else if (!pcmk__is_set(input->action->flags, pcmk__action_runnable) && pcmk__is_set(input->flags, pcmk__ar_min_runnable)) { pcmk__trace("Ignoring %s (%d) input %s (%d): " "minimum number of instances required but input unrunnable", action->uuid, action->id, input->action->uuid, input->action->id); return false; } else if (pcmk__is_set(input->flags, pcmk__ar_unmigratable_then_blocks) && !pcmk__is_set(input->action->flags, pcmk__action_runnable)) { pcmk__trace("Ignoring %s (%d) input %s (%d): " "input blocked if 'then' unmigratable", action->uuid, action->id, input->action->uuid, input->action->id); return false; } else if (pcmk__is_set(input->flags, pcmk__ar_if_first_unmigratable) && pcmk__is_set(input->action->flags, pcmk__action_migratable)) { pcmk__trace("Ignoring %s (%d) input %s (%d): ordering applies " "only if input is unmigratable, but it is migratable", action->uuid, action->id, input->action->uuid, input->action->id); return false; } else if ((input->flags == pcmk__ar_ordered) && pcmk__is_set(input->action->flags, pcmk__action_migratable) && pcmk__ends_with(input->action->uuid, "_stop_0")) { pcmk__trace("Ignoring %s (%d) input %s (%d): " "optional but stop in migration", action->uuid, action->id, input->action->uuid, input->action->id); return false; } else if (input->flags == pcmk__ar_if_on_same_node_or_target) { pcmk_node_t *input_node = input->action->node; if ((action->rsc != NULL) && pcmk__str_eq(action->task, PCMK_ACTION_MIGRATE_TO, pcmk__str_none)) { pcmk_node_t *assigned = action->rsc->priv->assigned_node; /* For load_stopped -> migrate_to orderings, we care about where * the resource has been assigned, not where migrate_to will be * executed. */ if (!pcmk__same_node(input_node, assigned)) { pcmk__trace("Ignoring %s (%d) input %s (%d): " "migration target %s is not same as input node %s", action->uuid, action->id, input->action->uuid, input->action->id, pcmk__node_name(assigned), pcmk__node_name(input_node)); input->flags = pcmk__ar_none; return false; } } else if (!pcmk__same_node(input_node, action->node)) { pcmk__trace("Ignoring %s (%d) input %s (%d): " "not on same node (%s vs %s)", action->uuid, action->id, input->action->uuid, input->action->id, pcmk__node_name(action->node), pcmk__node_name(input_node)); input->flags = pcmk__ar_none; return false; } else if (pcmk__is_set(input->action->flags, pcmk__action_optional)) { pcmk__trace("Ignoring %s (%d) input %s (%d): " "ordering optional", action->uuid, action->id, input->action->uuid, input->action->id); input->flags = pcmk__ar_none; return false; } } else if (input->flags == pcmk__ar_if_required_on_same_node) { if (input->action->node && action->node && !pcmk__same_node(input->action->node, action->node)) { pcmk__trace("Ignoring %s (%d) input %s (%d): " "not on same node (%s vs %s)", action->uuid, action->id, input->action->uuid, input->action->id, pcmk__node_name(action->node), pcmk__node_name(input->action->node)); input->flags = pcmk__ar_none; return false; } else if (pcmk__is_set(input->action->flags, pcmk__action_optional)) { pcmk__trace("Ignoring %s (%d) input %s (%d): optional", action->uuid, action->id, input->action->uuid, input->action->id); input->flags = pcmk__ar_none; return false; } } else if (input->action->rsc && input->action->rsc != action->rsc && pcmk__is_set(input->action->rsc->flags, pcmk__rsc_failed) && !pcmk__is_set(input->action->rsc->flags, pcmk__rsc_managed) && pcmk__ends_with(input->action->uuid, "_stop_0") && pcmk__is_clone(action->rsc)) { pcmk__warn("Ignoring requirement that %s complete before %s: unmanaged " "failed resources cannot prevent clone shutdown", input->action->uuid, action->uuid); return false; } else if (pcmk__is_set(input->action->flags, pcmk__action_optional) && !pcmk__any_flags_set(input->action->flags, pcmk__action_always_in_graph |pcmk__action_added_to_graph) && !should_add_action_to_graph(input->action)) { pcmk__trace("Ignoring %s (%d) input %s (%d): " "input optional", action->uuid, action->id, input->action->uuid, input->action->id); return false; } pcmk__trace("%s (%d) input %s %s (%d) on %s should be dumped: %s %s %#.6x", action->uuid, action->id, action_type_str(input->action->flags), input->action->uuid, input->action->id, action_node_str(input->action), action_runnable_str(input->action->flags), action_optional_str(input->action->flags), input->flags); return true; } /*! * \internal * \brief Check whether an ordering creates an ordering loop * * \param[in] init_action "First" action in ordering * \param[in] action Callers should always set this the same as * \p init_action (this function may use a different * value for recursive calls) * \param[in,out] input Action wrapper for "then" action in ordering * * \return true if the ordering creates a loop, otherwise false */ bool pcmk__graph_has_loop(const pcmk_action_t *init_action, const pcmk_action_t *action, pcmk__related_action_t *input) { bool has_loop = false; if (pcmk__is_set(input->action->flags, pcmk__action_detect_loop)) { pcmk__trace("Breaking tracking loop: %s@%s -> %s@%s (%#.6x)", input->action->uuid, pcmk__node_name(input->action->node), action->uuid, pcmk__node_name(action->node), input->flags); return false; } // Don't need to check inputs that won't be used if (!should_add_input_to_graph(action, input)) { return false; } if (input->action == init_action) { pcmk__debug("Input loop found in %s@%s ->...-> %s@%s", action->uuid, pcmk__node_name(action->node), init_action->uuid, pcmk__node_name(init_action->node)); return true; } pcmk__set_action_flags(input->action, pcmk__action_detect_loop); pcmk__trace("Checking inputs of action %s@%s input %s@%s (%#.6x)" "for graph loop with %s@%s ", action->uuid, pcmk__node_name(action->node), input->action->uuid, pcmk__node_name(input->action->node), input->flags, init_action->uuid, pcmk__node_name(init_action->node)); // Recursively check input itself for loops for (GList *iter = input->action->actions_before; iter != NULL; iter = iter->next) { if (pcmk__graph_has_loop(init_action, input->action, (pcmk__related_action_t *) iter->data)) { // Recursive call already logged a debug message has_loop = true; break; } } pcmk__clear_action_flags(input->action, pcmk__action_detect_loop); if (!has_loop) { pcmk__trace("No input loop found in %s@%s -> %s@%s (%#.6x)", input->action->uuid, pcmk__node_name(input->action->node), action->uuid, pcmk__node_name(action->node), input->flags); } return has_loop; } /*! * \internal * \brief Create a synapse XML element for a transition graph * * \param[in] action Action that synapse is for * \param[in,out] scheduler Scheduler data containing graph * * \return Newly added XML element for new graph synapse */ static xmlNode * create_graph_synapse(const pcmk_action_t *action, pcmk_scheduler_t *scheduler) { int synapse_priority = 0; xmlNode *syn = pcmk__xe_create(scheduler->priv->graph, PCMK__XE_SYNAPSE); pcmk__xe_set_int(syn, PCMK_XA_ID, scheduler->priv->synapse_count++); if (action->rsc != NULL) { synapse_priority = action->rsc->priv->priority; } if (action->priority > synapse_priority) { synapse_priority = action->priority; } if (synapse_priority > 0) { pcmk__xe_set_int(syn, PCMK__XA_PRIORITY, synapse_priority); } return syn; } /*! * \internal * \brief Add an action to the transition graph XML if appropriate * * \param[in,out] data Action to possibly add * \param[in,out] user_data Scheduler data * * \note This will de-duplicate the action inputs, meaning that the * pcmk__related_action_t:type flags can no longer be relied on to retain * their original settings. That means this MUST be called after * pcmk__apply_orderings() is complete, and nothing after this should rely * on those type flags. (For example, some code looks for type equal to * some flag rather than whether the flag is set, and some code looks for * particular combinations of flags -- such code must be done before * pcmk__create_graph().) */ static void add_action_to_graph(gpointer data, gpointer user_data) { pcmk_action_t *action = (pcmk_action_t *) data; pcmk_scheduler_t *scheduler = (pcmk_scheduler_t *) user_data; xmlNode *syn = NULL; xmlNode *set = NULL; xmlNode *in = NULL; /* If we haven't already, de-duplicate inputs (even if we won't be adding * the action to the graph, so that crm_simulate's dot graphs don't have * duplicates). */ if (!pcmk__is_set(action->flags, pcmk__action_inputs_deduplicated)) { pcmk__deduplicate_action_inputs(action); pcmk__set_action_flags(action, pcmk__action_inputs_deduplicated); } if (pcmk__is_set(action->flags, pcmk__action_added_to_graph) || !should_add_action_to_graph(action)) { return; // Already added, or shouldn't be } pcmk__set_action_flags(action, pcmk__action_added_to_graph); pcmk__trace("Adding action %d (%s%s%s) to graph", action->id, action->uuid, ((action->node != NULL)? " on " : ""), pcmk__node_name(action->node)); syn = create_graph_synapse(action, scheduler); set = pcmk__xe_create(syn, PCMK__XE_ACTION_SET); in = pcmk__xe_create(syn, PCMK__XE_INPUTS); create_graph_action(set, action, false, scheduler); for (GList *lpc = action->actions_before; lpc != NULL; lpc = lpc->next) { pcmk__related_action_t *input = lpc->data; if (should_add_input_to_graph(action, input)) { xmlNode *input_xml = pcmk__xe_create(in, PCMK__XE_TRIGGER); input->graphed = true; create_graph_action(input_xml, input->action, true, scheduler); } } } static int transition_id = 0; /*! * \internal * \brief Log a message after calculating a transition * * \param[in] scheduler Scheduler data * \param[in] filename Where transition input is stored */ void pcmk__log_transition_summary(const pcmk_scheduler_t *scheduler, const char *filename) { if (pcmk__is_set(scheduler->flags, pcmk__sched_processing_error) || pcmk__config_has_error) { pcmk__err("Calculated transition %d (with errors)%s%s", transition_id, ((filename != NULL)? ", saving inputs in " : ""), ((filename != NULL)? filename : "")); } else if (pcmk__is_set(scheduler->flags, pcmk__sched_processing_warning) || pcmk__config_has_warning) { pcmk__warn("Calculated transition %d (with warnings)%s%s", transition_id, ((filename != NULL)? ", saving inputs in " : ""), pcmk__s(filename, "")); } else { pcmk__notice("Calculated transition %d%s%s", transition_id, ((filename != NULL)? ", saving inputs in " : ""), pcmk__s(filename, "")); } if (pcmk__config_has_error) { pcmk__notice("Configuration errors found during scheduler processing, " "please run \"crm_verify -L\" to identify issues"); } } /*! * \internal * \brief Add a resource's actions to the transition graph * * \param[in,out] rsc Resource whose actions should be added */ void pcmk__add_rsc_actions_to_graph(pcmk_resource_t *rsc) { GList *iter = NULL; pcmk__assert(rsc != NULL); pcmk__rsc_trace(rsc, "Adding actions for %s to graph", rsc->id); // First add the resource's own actions g_list_foreach(rsc->priv->actions, add_action_to_graph, rsc->priv->scheduler); // Then recursively add its children's actions (appropriate to variant) for (iter = rsc->priv->children; iter != NULL; iter = iter->next) { pcmk_resource_t *child_rsc = (pcmk_resource_t *) iter->data; child_rsc->priv->cmds->add_actions_to_graph(child_rsc); } } /*! * \internal * \brief Create a transition graph with all cluster actions needed * * \param[in,out] scheduler Scheduler data */ void pcmk__create_graph(pcmk_scheduler_t *scheduler) { GList *iter = NULL; const char *value = NULL; long long limit = 0LL; GHashTable *config_hash = scheduler->priv->options; int rc = pcmk_rc_ok; transition_id++; pcmk__trace("Creating transition graph %d", transition_id); scheduler->priv->graph = pcmk__xe_create(NULL, PCMK__XE_TRANSITION_GRAPH); value = pcmk__cluster_option(config_hash, PCMK_OPT_CLUSTER_DELAY); pcmk__xe_set(scheduler->priv->graph, PCMK_OPT_CLUSTER_DELAY, value); value = pcmk__cluster_option(config_hash, PCMK_OPT_STONITH_TIMEOUT); pcmk__xe_set(scheduler->priv->graph, PCMK_OPT_STONITH_TIMEOUT, value); pcmk__xe_set(scheduler->priv->graph, PCMK__XA_FAILED_STOP_OFFSET, PCMK_VALUE_INFINITY); if (pcmk__is_set(scheduler->flags, pcmk__sched_start_failure_fatal)) { pcmk__xe_set(scheduler->priv->graph, PCMK__XA_FAILED_START_OFFSET, PCMK_VALUE_INFINITY); } else { pcmk__xe_set(scheduler->priv->graph, PCMK__XA_FAILED_START_OFFSET, "1"); } value = pcmk__cluster_option(config_hash, PCMK_OPT_BATCH_LIMIT); pcmk__xe_set(scheduler->priv->graph, PCMK_OPT_BATCH_LIMIT, value); pcmk__xe_set_int(scheduler->priv->graph, "transition_id", transition_id); value = pcmk__cluster_option(config_hash, PCMK_OPT_MIGRATION_LIMIT); rc = pcmk__scan_ll(value, &limit, 0LL); if (rc != pcmk_rc_ok) { pcmk__warn("Ignoring invalid value '%s' for " PCMK_OPT_MIGRATION_LIMIT ": %s", value, pcmk_rc_str(rc)); } else if (limit > 0) { pcmk__xe_set(scheduler->priv->graph, PCMK_OPT_MIGRATION_LIMIT, value); } if (scheduler->priv->recheck_by > 0) { pcmk__xe_set_time(scheduler->priv->graph, "recheck-by", scheduler->priv->recheck_by); } /* The following code will de-duplicate action inputs, so nothing past this * should rely on the action input type flags retaining their original * values. */ // Add resource actions to graph for (iter = scheduler->priv->resources; iter != NULL; iter = iter->next) { pcmk_resource_t *rsc = (pcmk_resource_t *) iter->data; pcmk__rsc_trace(rsc, "Processing actions for %s", rsc->id); rsc->priv->cmds->add_actions_to_graph(rsc); } // Add pseudo-action for list of nodes with maintenance state update add_maintenance_update(scheduler); // Add non-resource (node) actions for (iter = scheduler->priv->actions; iter != NULL; iter = iter->next) { pcmk_action_t *action = (pcmk_action_t *) iter->data; if ((action->rsc != NULL) && (action->node != NULL) && action->node->details->shutdown && !pcmk__is_set(action->rsc->flags, pcmk__rsc_maintenance) && !pcmk__any_flags_set(action->flags, pcmk__action_optional|pcmk__action_runnable) && pcmk__str_eq(action->task, PCMK_ACTION_STOP, pcmk__str_none)) { /* Eventually we should just ignore the 'fence' case, but for now * it's the best way to detect (in CTS) when CIB resource updates * are being lost. */ if (pcmk__is_set(scheduler->flags, pcmk__sched_quorate) || (scheduler->no_quorum_policy == pcmk_no_quorum_ignore)) { const bool unclean = action->node->details->unclean; const bool managed = pcmk__is_set(action->rsc->flags, pcmk__rsc_managed); const bool failed = pcmk__is_set(action->rsc->flags, pcmk__rsc_failed); pcmk__crit("Cannot %s %s because of %s:%s%s (%s)", (unclean? "fence" : "shut down"), pcmk__node_name(action->node), action->rsc->id, (managed? " blocked" : " unmanaged"), (failed? " failed" : ""), action->uuid); } } add_action_to_graph((gpointer) action, (gpointer) scheduler); } - crm_log_xml_trace(scheduler->priv->graph, "graph"); + pcmk__log_xml_trace(scheduler->priv->graph, "graph"); } diff --git a/lib/pacemaker/pcmk_sched_colocation.c b/lib/pacemaker/pcmk_sched_colocation.c index 4f269889ba..538ff9c977 100644 --- a/lib/pacemaker/pcmk_sched_colocation.c +++ b/lib/pacemaker/pcmk_sched_colocation.c @@ -1,2041 +1,2041 @@ /* * Copyright 2004-2025 the Pacemaker project contributors * * The version control history for this file may have further details. * * This source code is licensed under the GNU General Public License version 2 * or later (GPLv2+) WITHOUT ANY WARRANTY. */ #include #include #include #include #include #include #include // PCMK_SCORE_INFINITY #include #include #include "crm/common/util.h" #include "crm/common/xml_internal.h" #include "crm/common/xml.h" #include "libpacemaker_private.h" // Used to temporarily mark a node as unusable #define INFINITY_HACK (PCMK_SCORE_INFINITY * -100) /*! * \internal * \brief Get the value of a colocation's node attribute * * \param[in] node Node on which to look up the attribute * \param[in] attr Name of attribute to look up * \param[in] rsc Resource on whose behalf to look up the attribute * * \return Value of \p attr on \p node or on the host of \p node, as appropriate */ const char * pcmk__colocation_node_attr(const pcmk_node_t *node, const char *attr, const pcmk_resource_t *rsc) { const char *target = NULL; /* A resource colocated with a bundle or its primitive can't run on the * bundle node itself (where only the primitive, if any, can run). Instead, * we treat it as a colocation with the bundle's containers, so always look * up colocation node attributes on the container host. */ if (pcmk__is_bundle_node(node) && pcmk__is_bundled(rsc) && (pe__const_top_resource(rsc, false) == pe__bundled_resource(rsc))) { target = PCMK_VALUE_HOST; } else if (rsc != NULL) { target = g_hash_table_lookup(rsc->priv->meta, PCMK_META_CONTAINER_ATTRIBUTE_TARGET); } return pcmk__node_attr(node, attr, target, pcmk__rsc_node_assigned); } /*! * \internal * \brief Compare two colocations according to priority * * Compare two colocations according to the order in which they should be * considered, based on either their dependent resources or their primary * resources -- preferring (in order): * * Colocation that is not \c NULL * * Colocation whose resource has higher priority * * Colocation whose resource is of a higher-level variant * (bundle > clone > group > primitive) * * Colocation whose resource is promotable, if both are clones * * Colocation whose resource has lower ID in lexicographic order * * \param[in] colocation1 First colocation to compare * \param[in] colocation2 Second colocation to compare * \param[in] dependent If \c true, compare colocations by dependent * priority; otherwise compare them by primary priority * * \return A negative number if \p colocation1 should be considered first, * a positive number if \p colocation2 should be considered first, * or 0 if order doesn't matter */ static gint cmp_colocation_priority(const pcmk__colocation_t *colocation1, const pcmk__colocation_t *colocation2, bool dependent) { const pcmk_resource_t *rsc1 = NULL; const pcmk_resource_t *rsc2 = NULL; if (colocation1 == NULL) { return 1; } if (colocation2 == NULL) { return -1; } if (dependent) { rsc1 = colocation1->dependent; rsc2 = colocation2->dependent; pcmk__assert(colocation1->primary != NULL); } else { rsc1 = colocation1->primary; rsc2 = colocation2->primary; pcmk__assert(colocation1->dependent != NULL); } pcmk__assert((rsc1 != NULL) && (rsc2 != NULL)); if (rsc1->priv->priority > rsc2->priv->priority) { return -1; } if (rsc1->priv->priority < rsc2->priv->priority) { return 1; } // Process clones before primitives and groups if (rsc1->priv->variant > rsc2->priv->variant) { return -1; } if (rsc1->priv->variant < rsc2->priv->variant) { return 1; } /* @COMPAT scheduler <2.0.0: Process promotable clones before nonpromotable * clones (probably unnecessary, but avoids having to update regression * tests) */ if (pcmk__is_clone(rsc1)) { if (pcmk__is_set(rsc1->flags, pcmk__rsc_promotable) && !pcmk__is_set(rsc2->flags, pcmk__rsc_promotable)) { return -1; } if (!pcmk__is_set(rsc1->flags, pcmk__rsc_promotable) && pcmk__is_set(rsc2->flags, pcmk__rsc_promotable)) { return 1; } } return strcmp(rsc1->id, rsc2->id); } /*! * \internal * \brief Compare two colocations according to priority based on dependents * * Compare two colocations according to the order in which they should be * considered, based on their dependent resources -- preferring (in order): * * Colocation that is not \c NULL * * Colocation whose resource has higher priority * * Colocation whose resource is of a higher-level variant * (bundle > clone > group > primitive) * * Colocation whose resource is promotable, if both are clones * * Colocation whose resource has lower ID in lexicographic order * * \param[in] a First colocation to compare * \param[in] b Second colocation to compare * * \return A negative number if \p a should be considered first, * a positive number if \p b should be considered first, * or 0 if order doesn't matter */ static gint cmp_dependent_priority(gconstpointer a, gconstpointer b) { return cmp_colocation_priority(a, b, true); } /*! * \internal * \brief Compare two colocations according to priority based on primaries * * Compare two colocations according to the order in which they should be * considered, based on their primary resources -- preferring (in order): * * Colocation that is not \c NULL * * Colocation whose primary has higher priority * * Colocation whose primary is of a higher-level variant * (bundle > clone > group > primitive) * * Colocation whose primary is promotable, if both are clones * * Colocation whose primary has lower ID in lexicographic order * * \param[in] a First colocation to compare * \param[in] b Second colocation to compare * * \return A negative number if \p a should be considered first, * a positive number if \p b should be considered first, * or 0 if order doesn't matter */ static gint cmp_primary_priority(gconstpointer a, gconstpointer b) { return cmp_colocation_priority(a, b, false); } /*! * \internal * \brief Add a "this with" colocation constraint to a sorted list * * \param[in,out] list List of constraints to add \p colocation to * \param[in] colocation Colocation constraint to add to \p list * \param[in] rsc Resource whose colocations we're getting (for * logging only) * * \note The list will be sorted using cmp_primary_priority(). */ void pcmk__add_this_with(GList **list, const pcmk__colocation_t *colocation, const pcmk_resource_t *rsc) { pcmk__assert((list != NULL) && (colocation != NULL) && (rsc != NULL)); pcmk__rsc_trace(rsc, "Adding colocation %s (%s with %s using %s @%s) to " "'this with' list for %s", colocation->id, colocation->dependent->id, colocation->primary->id, colocation->node_attribute, pcmk_readable_score(colocation->score), rsc->id); *list = g_list_insert_sorted(*list, (gpointer) colocation, cmp_primary_priority); } /*! * \internal * \brief Add a list of "this with" colocation constraints to a list * * \param[in,out] list List of constraints to add \p addition to * \param[in] addition List of colocation constraints to add to \p list * \param[in] rsc Resource whose colocations we're getting (for * logging only) * * \note The lists must be pre-sorted by cmp_primary_priority(). */ void pcmk__add_this_with_list(GList **list, GList *addition, const pcmk_resource_t *rsc) { pcmk__assert((list != NULL) && (rsc != NULL)); pcmk__if_tracing( {}, // Always add each colocation individually if tracing { if (*list == NULL) { // Trivial case for efficiency if not tracing *list = g_list_copy(addition); return; } } ); for (const GList *iter = addition; iter != NULL; iter = iter->next) { pcmk__add_this_with(list, addition->data, rsc); } } /*! * \internal * \brief Add a "with this" colocation constraint to a sorted list * * \param[in,out] list List of constraints to add \p colocation to * \param[in] colocation Colocation constraint to add to \p list * \param[in] rsc Resource whose colocations we're getting (for * logging only) * * \note The list will be sorted using cmp_dependent_priority(). */ void pcmk__add_with_this(GList **list, const pcmk__colocation_t *colocation, const pcmk_resource_t *rsc) { pcmk__assert((list != NULL) && (colocation != NULL) && (rsc != NULL)); pcmk__rsc_trace(rsc, "Adding colocation %s (%s with %s using %s @%s) to " "'with this' list for %s", colocation->id, colocation->dependent->id, colocation->primary->id, colocation->node_attribute, pcmk_readable_score(colocation->score), rsc->id); *list = g_list_insert_sorted(*list, (gpointer) colocation, cmp_dependent_priority); } /*! * \internal * \brief Add a list of "with this" colocation constraints to a list * * \param[in,out] list List of constraints to add \p addition to * \param[in] addition List of colocation constraints to add to \p list * \param[in] rsc Resource whose colocations we're getting (for * logging only) * * \note The lists must be pre-sorted by cmp_dependent_priority(). */ void pcmk__add_with_this_list(GList **list, GList *addition, const pcmk_resource_t *rsc) { pcmk__assert((list != NULL) && (rsc != NULL)); pcmk__if_tracing( {}, // Always add each colocation individually if tracing { if (*list == NULL) { // Trivial case for efficiency if not tracing *list = g_list_copy(addition); return; } } ); for (const GList *iter = addition; iter != NULL; iter = iter->next) { pcmk__add_with_this(list, addition->data, rsc); } } /*! * \internal * \brief Add orderings necessary for an anti-colocation constraint * * \param[in,out] first_rsc One resource in an anti-colocation * \param[in] first_role Anti-colocation role of \p first_rsc * \param[in] then_rsc Other resource in the anti-colocation * \param[in] then_role Anti-colocation role of \p then_rsc */ static void anti_colocation_order(pcmk_resource_t *first_rsc, int first_role, pcmk_resource_t *then_rsc, int then_role) { const char *first_tasks[] = { NULL, NULL }; const char *then_tasks[] = { NULL, NULL }; /* Actions to make first_rsc lose first_role */ if (first_role == pcmk_role_promoted) { first_tasks[0] = PCMK_ACTION_DEMOTE; } else { first_tasks[0] = PCMK_ACTION_STOP; if (first_role == pcmk_role_unpromoted) { first_tasks[1] = PCMK_ACTION_PROMOTE; } } /* Actions to make then_rsc gain then_role */ if (then_role == pcmk_role_promoted) { then_tasks[0] = PCMK_ACTION_PROMOTE; } else { then_tasks[0] = PCMK_ACTION_START; if (then_role == pcmk_role_unpromoted) { then_tasks[1] = PCMK_ACTION_DEMOTE; } } for (int first_lpc = 0; (first_lpc <= 1) && (first_tasks[first_lpc] != NULL); first_lpc++) { for (int then_lpc = 0; (then_lpc <= 1) && (then_tasks[then_lpc] != NULL); then_lpc++) { pcmk__order_resource_actions(first_rsc, first_tasks[first_lpc], then_rsc, then_tasks[then_lpc], pcmk__ar_if_required_on_same_node); } } } /*! * \internal * \brief Add a new colocation constraint to scheduler data * * \param[in] id XML ID for this constraint * \param[in] node_attr Colocate by this attribute (NULL for #uname) * \param[in] score Constraint score * \param[in,out] dependent Resource to be colocated * \param[in,out] primary Resource to colocate \p dependent with * \param[in] dependent_role_spec If not NULL, only \p dependent instances * with this role should be colocated * \param[in] primary_role_spec If not NULL, only \p primary instances * with this role should be colocated * \param[in] flags Group of enum pcmk__coloc_flags */ void pcmk__new_colocation(const char *id, const char *node_attr, int score, pcmk_resource_t *dependent, pcmk_resource_t *primary, const char *dependent_role_spec, const char *primary_role_spec, uint32_t flags) { pcmk__colocation_t *new_con = NULL; enum rsc_role_e dependent_role = pcmk_role_unknown; enum rsc_role_e primary_role = pcmk_role_unknown; CRM_CHECK(id != NULL, return); if ((dependent == NULL) || (primary == NULL)) { pcmk__config_err("Ignoring colocation '%s' because resource " "does not exist", id); return; } if ((pcmk__parse_constraint_role(id, dependent_role_spec, &dependent_role) != pcmk_rc_ok) || (pcmk__parse_constraint_role(id, primary_role_spec, &primary_role) != pcmk_rc_ok)) { // Not possible with schema validation enabled (error already logged) return; } if (score == 0) { pcmk__rsc_trace(dependent, "Ignoring colocation '%s' (%s with %s) because score is 0", id, dependent->id, primary->id); return; } new_con = pcmk__assert_alloc(1, sizeof(pcmk__colocation_t)); new_con->id = id; new_con->dependent = dependent; new_con->primary = primary; new_con->score = score; new_con->dependent_role = dependent_role; new_con->primary_role = primary_role; new_con->node_attribute = pcmk__s(node_attr, CRM_ATTR_UNAME); new_con->flags = flags; pcmk__add_this_with(&(dependent->priv->this_with_colocations), new_con, dependent); pcmk__add_with_this(&(primary->priv->with_this_colocations), new_con, primary); dependent->priv->scheduler->priv->colocation_constraints = g_list_prepend(dependent->priv->scheduler->priv->colocation_constraints, new_con); if (score <= -PCMK_SCORE_INFINITY) { anti_colocation_order(dependent, new_con->dependent_role, primary, new_con->primary_role); anti_colocation_order(primary, new_con->primary_role, dependent, new_con->dependent_role); } } /*! * \internal * \brief Return the boolean influence corresponding to configuration * * \param[in] coloc_id Colocation XML ID (for error logging) * \param[in] rsc Resource involved in constraint (for default) * \param[in] influence_s String value of \c PCMK_XA_INFLUENCE option * * \return \c pcmk__coloc_influence if string evaluates true, or string is * \c NULL or invalid and resource's \c PCMK_META_CRITICAL option * evaluates true, otherwise \c pcmk__coloc_none */ static uint32_t unpack_influence(const char *coloc_id, const pcmk_resource_t *rsc, const char *influence_s) { if (influence_s != NULL) { bool influence = false; if (pcmk__parse_bool(influence_s, &influence) == pcmk_rc_ok) { return (influence? pcmk__coloc_influence : pcmk__coloc_none); } pcmk__config_err("Constraint '%s' has invalid value for " PCMK_XA_INFLUENCE " (using default)", coloc_id); } if (pcmk__is_set(rsc->flags, pcmk__rsc_critical)) { return pcmk__coloc_influence; } return pcmk__coloc_none; } static void unpack_colocation_set(xmlNode *set, int score, const char *coloc_id, const char *influence_s, pcmk_scheduler_t *scheduler) { xmlNode *xml_rsc = NULL; pcmk_resource_t *other = NULL; pcmk_resource_t *resource = NULL; const char *set_id = pcmk__xe_id(set); const char *role = pcmk__xe_get(set, PCMK_XA_ROLE); bool with_previous = false; int local_score = score; bool sequential = false; uint32_t flags = pcmk__coloc_none; const char *xml_rsc_id = NULL; const char *score_s = pcmk__xe_get(set, PCMK_XA_SCORE); if (score_s != NULL) { int rc = pcmk_parse_score(score_s, &local_score, 0); if (rc != pcmk_rc_ok) { // Not possible with schema validation enabled pcmk__config_err("Ignoring colocation '%s' for set '%s' " "because '%s' is not a valid score", coloc_id, set_id, score_s); return; } } if (local_score == 0) { pcmk__trace("Ignoring colocation '%s' for set '%s' because score is 0", coloc_id, set_id); return; } /* @COMPAT The deprecated PCMK__XA_ORDERING attribute specifies whether * resources in a positive-score set are colocated with the previous or next * resource. */ if (pcmk__str_eq(pcmk__xe_get(set, PCMK__XA_ORDERING), PCMK__VALUE_GROUP, pcmk__str_null_matches|pcmk__str_casei)) { with_previous = true; } else { pcmk__warn_once(pcmk__wo_set_ordering, "Support for '" PCMK__XA_ORDERING "' other than" " '" PCMK__VALUE_GROUP "' in " PCMK_XE_RESOURCE_SET " (such as %s) is deprecated and will be removed in a" " future release", set_id); } if ((pcmk__xe_get_bool_attr(set, PCMK_XA_SEQUENTIAL, &sequential) == pcmk_rc_ok) && !sequential) { return; } if (local_score > 0) { for (xml_rsc = pcmk__xe_first_child(set, PCMK_XE_RESOURCE_REF, NULL, NULL); xml_rsc != NULL; xml_rsc = pcmk__xe_next(xml_rsc, PCMK_XE_RESOURCE_REF)) { xml_rsc_id = pcmk__xe_id(xml_rsc); resource = pcmk__find_constraint_resource(scheduler->priv->resources, xml_rsc_id); if (resource == NULL) { // Should be possible only with validation disabled pcmk__config_err("Ignoring %s and later resources in set %s: " "No such resource", xml_rsc_id, set_id); return; } if (other != NULL) { flags = pcmk__coloc_explicit | unpack_influence(coloc_id, resource, influence_s); if (with_previous) { pcmk__rsc_trace(resource, "Colocating %s with %s in set %s", resource->id, other->id, set_id); pcmk__new_colocation(set_id, NULL, local_score, resource, other, role, role, flags); } else { pcmk__rsc_trace(resource, "Colocating %s with %s in set %s", other->id, resource->id, set_id); pcmk__new_colocation(set_id, NULL, local_score, other, resource, role, role, flags); } } other = resource; } } else { /* Anti-colocating with every prior resource is * the only way to ensure the intuitive result * (i.e. that no one in the set can run with anyone else in the set) */ for (xml_rsc = pcmk__xe_first_child(set, PCMK_XE_RESOURCE_REF, NULL, NULL); xml_rsc != NULL; xml_rsc = pcmk__xe_next(xml_rsc, PCMK_XE_RESOURCE_REF)) { xmlNode *xml_rsc_with = NULL; xml_rsc_id = pcmk__xe_id(xml_rsc); resource = pcmk__find_constraint_resource(scheduler->priv->resources, xml_rsc_id); if (resource == NULL) { // Should be possible only with validation disabled pcmk__config_err("Ignoring %s and later resources in set %s: " "No such resource", xml_rsc_id, set_id); return; } flags = pcmk__coloc_explicit | unpack_influence(coloc_id, resource, influence_s); for (xml_rsc_with = pcmk__xe_first_child(set, PCMK_XE_RESOURCE_REF, NULL, NULL); xml_rsc_with != NULL; xml_rsc_with = pcmk__xe_next(xml_rsc_with, PCMK_XE_RESOURCE_REF)) { xml_rsc_id = pcmk__xe_id(xml_rsc_with); if (pcmk__str_eq(resource->id, xml_rsc_id, pcmk__str_none)) { break; } other = pcmk__find_constraint_resource(scheduler->priv->resources, xml_rsc_id); pcmk__assert(other != NULL); // We already processed it pcmk__new_colocation(set_id, NULL, local_score, resource, other, role, role, flags); } } } } /*! * \internal * \brief Colocate two resource sets relative to each other * * \param[in] id Colocation XML ID * \param[in] set1 Dependent set * \param[in] set2 Primary set * \param[in] score Colocation score * \param[in] influence_s Value of colocation's \c PCMK_XA_INFLUENCE * attribute * \param[in,out] scheduler Scheduler data */ static void colocate_rsc_sets(const char *id, const xmlNode *set1, const xmlNode *set2, int score, const char *influence_s, pcmk_scheduler_t *scheduler) { xmlNode *xml_rsc = NULL; pcmk_resource_t *rsc_1 = NULL; pcmk_resource_t *rsc_2 = NULL; const char *xml_rsc_id = NULL; const char *role_1 = pcmk__xe_get(set1, PCMK_XA_ROLE); const char *role_2 = pcmk__xe_get(set2, PCMK_XA_ROLE); int rc = pcmk_rc_ok; bool sequential = false; uint32_t flags = pcmk__coloc_none; if (score == 0) { pcmk__trace("Ignoring colocation '%s' between sets %s and %s because " "score is 0", id, pcmk__xe_id(set1), pcmk__xe_id(set2)); return; } rc = pcmk__xe_get_bool_attr(set1, PCMK_XA_SEQUENTIAL, &sequential); if ((rc != pcmk_rc_ok) || sequential) { // Get the first one xml_rsc = pcmk__xe_first_child(set1, PCMK_XE_RESOURCE_REF, NULL, NULL); if (xml_rsc != NULL) { xml_rsc_id = pcmk__xe_id(xml_rsc); rsc_1 = pcmk__find_constraint_resource(scheduler->priv->resources, xml_rsc_id); if (rsc_1 == NULL) { // Should be possible only with validation disabled pcmk__config_err("Ignoring colocation of set %s with set %s " "because first resource %s not found", pcmk__xe_id(set1), pcmk__xe_id(set2), xml_rsc_id); return; } } } rc = pcmk__xe_get_bool_attr(set2, PCMK_XA_SEQUENTIAL, &sequential); if ((rc != pcmk_rc_ok) || sequential) { // Get the last one for (xml_rsc = pcmk__xe_first_child(set2, PCMK_XE_RESOURCE_REF, NULL, NULL); xml_rsc != NULL; xml_rsc = pcmk__xe_next(xml_rsc, PCMK_XE_RESOURCE_REF)) { xml_rsc_id = pcmk__xe_id(xml_rsc); } rsc_2 = pcmk__find_constraint_resource(scheduler->priv->resources, xml_rsc_id); if (rsc_2 == NULL) { // Should be possible only with validation disabled pcmk__config_err("Ignoring colocation of set %s with set %s " "because last resource %s not found", pcmk__xe_id(set1), pcmk__xe_id(set2), xml_rsc_id); return; } } if ((rsc_1 != NULL) && (rsc_2 != NULL)) { // Both sets are sequential flags = pcmk__coloc_explicit | unpack_influence(id, rsc_1, influence_s); pcmk__new_colocation(id, NULL, score, rsc_1, rsc_2, role_1, role_2, flags); } else if (rsc_1 != NULL) { // Only set1 is sequential flags = pcmk__coloc_explicit | unpack_influence(id, rsc_1, influence_s); for (xml_rsc = pcmk__xe_first_child(set2, PCMK_XE_RESOURCE_REF, NULL, NULL); xml_rsc != NULL; xml_rsc = pcmk__xe_next(xml_rsc, PCMK_XE_RESOURCE_REF)) { xml_rsc_id = pcmk__xe_id(xml_rsc); rsc_2 = pcmk__find_constraint_resource(scheduler->priv->resources, xml_rsc_id); if (rsc_2 == NULL) { // Should be possible only with validation disabled pcmk__config_err("Ignoring set %s colocation with resource %s " "in set %s: No such resource", pcmk__xe_id(set1), xml_rsc_id, pcmk__xe_id(set2)); continue; } pcmk__new_colocation(id, NULL, score, rsc_1, rsc_2, role_1, role_2, flags); } } else if (rsc_2 != NULL) { // Only set2 is sequential for (xml_rsc = pcmk__xe_first_child(set1, PCMK_XE_RESOURCE_REF, NULL, NULL); xml_rsc != NULL; xml_rsc = pcmk__xe_next(xml_rsc, PCMK_XE_RESOURCE_REF)) { xml_rsc_id = pcmk__xe_id(xml_rsc); rsc_1 = pcmk__find_constraint_resource(scheduler->priv->resources, xml_rsc_id); if (rsc_1 == NULL) { // Should be possible only with validation disabled pcmk__config_err("Ignoring colocation of set %s resource %s " "with set %s: No such resource", pcmk__xe_id(set1), xml_rsc_id, pcmk__xe_id(set2)); continue; } flags = pcmk__coloc_explicit | unpack_influence(id, rsc_1, influence_s); pcmk__new_colocation(id, NULL, score, rsc_1, rsc_2, role_1, role_2, flags); } } else { // Neither set is sequential for (xml_rsc = pcmk__xe_first_child(set1, PCMK_XE_RESOURCE_REF, NULL, NULL); xml_rsc != NULL; xml_rsc = pcmk__xe_next(xml_rsc, PCMK_XE_RESOURCE_REF)) { xmlNode *xml_rsc_2 = NULL; xml_rsc_id = pcmk__xe_id(xml_rsc); rsc_1 = pcmk__find_constraint_resource(scheduler->priv->resources, xml_rsc_id); if (rsc_1 == NULL) { // Should be possible only with validation disabled pcmk__config_err("Ignoring colocation of set %s resource %s " "with set %s: No such resource", pcmk__xe_id(set1), xml_rsc_id, pcmk__xe_id(set2)); continue; } flags = pcmk__coloc_explicit | unpack_influence(id, rsc_1, influence_s); for (xml_rsc_2 = pcmk__xe_first_child(set2, PCMK_XE_RESOURCE_REF, NULL, NULL); xml_rsc_2 != NULL; xml_rsc_2 = pcmk__xe_next(xml_rsc_2, PCMK_XE_RESOURCE_REF)) { xml_rsc_id = pcmk__xe_id(xml_rsc_2); rsc_2 = pcmk__find_constraint_resource(scheduler->priv->resources, xml_rsc_id); if (rsc_2 == NULL) { // Should be possible only with validation disabled pcmk__config_err("Ignoring colocation of set %s resource " "%s with set %s resource %s: No such " "resource", pcmk__xe_id(set1), pcmk__xe_id(xml_rsc), pcmk__xe_id(set2), xml_rsc_id); continue; } pcmk__new_colocation(id, NULL, score, rsc_1, rsc_2, role_1, role_2, flags); } } } } /*! * \internal * \brief Unpack a colocation constraint that contains no resource sets * * \param[in] xml_obj Colocation constraint XML * \param[in] id Colocation constraint XML ID (non-NULL) * \param[in] score Integer score parsed from score attribute * \param[in] influence_s Colocation constraint's influence attribute value * \param[in,out] scheduler Scheduler data */ static void unpack_simple_colocation(const xmlNode *xml_obj, const char *id, int score, const char *influence_s, pcmk_scheduler_t *scheduler) { uint32_t flags = pcmk__coloc_none; const char *dependent_id = pcmk__xe_get(xml_obj, PCMK_XA_RSC); const char *primary_id = pcmk__xe_get(xml_obj, PCMK_XA_WITH_RSC); const char *dependent_role = pcmk__xe_get(xml_obj, PCMK_XA_RSC_ROLE); const char *primary_role = pcmk__xe_get(xml_obj, PCMK_XA_WITH_RSC_ROLE); const char *attr = pcmk__xe_get(xml_obj, PCMK_XA_NODE_ATTRIBUTE); pcmk_resource_t *primary = NULL; pcmk_resource_t *dependent = NULL; primary = pcmk__find_constraint_resource(scheduler->priv->resources, primary_id); dependent = pcmk__find_constraint_resource(scheduler->priv->resources, dependent_id); if (dependent == NULL) { pcmk__config_err("Ignoring constraint '%s' because resource '%s' " "does not exist", id, dependent_id); return; } else if (primary == NULL) { pcmk__config_err("Ignoring constraint '%s' because resource '%s' " "does not exist", id, primary_id); return; } if (pcmk__xe_attr_is_true(xml_obj, PCMK_XA_SYMMETRICAL)) { pcmk__config_warn("The colocation constraint " "'" PCMK_XA_SYMMETRICAL "' attribute has been " "removed"); } flags = pcmk__coloc_explicit | unpack_influence(id, dependent, influence_s); pcmk__new_colocation(id, attr, score, dependent, primary, dependent_role, primary_role, flags); } // \return Standard Pacemaker return code static int unpack_colocation_tags(xmlNode *xml_obj, xmlNode **expanded_xml, pcmk_scheduler_t *scheduler) { const char *id = NULL; const char *dependent_id = NULL; const char *primary_id = NULL; const char *dependent_role = NULL; const char *primary_role = NULL; pcmk_resource_t *dependent = NULL; pcmk_resource_t *primary = NULL; pcmk__idref_t *dependent_tag = NULL; pcmk__idref_t *primary_tag = NULL; xmlNode *dependent_set = NULL; xmlNode *primary_set = NULL; bool any_sets = false; *expanded_xml = NULL; CRM_CHECK(xml_obj != NULL, return EINVAL); id = pcmk__xe_id(xml_obj); if (id == NULL) { pcmk__config_err("Ignoring <%s> constraint without " PCMK_XA_ID, xml_obj->name); return pcmk_rc_unpack_error; } // Check whether there are any resource sets with template or tag references *expanded_xml = pcmk__expand_tags_in_sets(xml_obj, scheduler); if (*expanded_xml != NULL) { - crm_log_xml_trace(*expanded_xml, "Expanded " PCMK_XE_RSC_COLOCATION); + pcmk__log_xml_trace(*expanded_xml, "Expanded " PCMK_XE_RSC_COLOCATION); return pcmk_rc_ok; } dependent_id = pcmk__xe_get(xml_obj, PCMK_XA_RSC); primary_id = pcmk__xe_get(xml_obj, PCMK_XA_WITH_RSC); if ((dependent_id == NULL) || (primary_id == NULL)) { return pcmk_rc_ok; } if (!pcmk__valid_resource_or_tag(scheduler, dependent_id, &dependent, &dependent_tag)) { pcmk__config_err("Ignoring constraint '%s' because '%s' is not a " "valid resource or tag", id, dependent_id); return pcmk_rc_unpack_error; } if (!pcmk__valid_resource_or_tag(scheduler, primary_id, &primary, &primary_tag)) { pcmk__config_err("Ignoring constraint '%s' because '%s' is not a " "valid resource or tag", id, primary_id); return pcmk_rc_unpack_error; } if ((dependent != NULL) && (primary != NULL)) { /* Neither side references any template/tag. */ return pcmk_rc_ok; } if ((dependent_tag != NULL) && (primary_tag != NULL)) { // A colocation constraint between two templates/tags makes no sense pcmk__config_err("Ignoring constraint '%s' because two templates or " "tags cannot be colocated", id); return pcmk_rc_unpack_error; } dependent_role = pcmk__xe_get(xml_obj, PCMK_XA_RSC_ROLE); primary_role = pcmk__xe_get(xml_obj, PCMK_XA_WITH_RSC_ROLE); *expanded_xml = pcmk__xml_copy(NULL, xml_obj); /* Convert dependent's template/tag reference into constraint * PCMK_XE_RESOURCE_SET */ if (!pcmk__tag_to_set(*expanded_xml, &dependent_set, PCMK_XA_RSC, true, scheduler)) { pcmk__xml_free(*expanded_xml); *expanded_xml = NULL; return pcmk_rc_unpack_error; } if (dependent_set != NULL) { if (dependent_role != NULL) { /* Move PCMK_XA_RSC_ROLE into converted PCMK_XE_RESOURCE_SET as * PCMK_XA_ROLE */ pcmk__xe_set(dependent_set, PCMK_XA_ROLE, dependent_role); pcmk__xe_remove_attr(*expanded_xml, PCMK_XA_RSC_ROLE); } any_sets = true; } /* Convert primary's template/tag reference into constraint * PCMK_XE_RESOURCE_SET */ if (!pcmk__tag_to_set(*expanded_xml, &primary_set, PCMK_XA_WITH_RSC, true, scheduler)) { pcmk__xml_free(*expanded_xml); *expanded_xml = NULL; return pcmk_rc_unpack_error; } if (primary_set != NULL) { if (primary_role != NULL) { /* Move PCMK_XA_WITH_RSC_ROLE into converted PCMK_XE_RESOURCE_SET as * PCMK_XA_ROLE */ pcmk__xe_set(primary_set, PCMK_XA_ROLE, primary_role); pcmk__xe_remove_attr(*expanded_xml, PCMK_XA_WITH_RSC_ROLE); } any_sets = true; } if (any_sets) { - crm_log_xml_trace(*expanded_xml, "Expanded " PCMK_XE_RSC_COLOCATION); + pcmk__log_xml_trace(*expanded_xml, "Expanded " PCMK_XE_RSC_COLOCATION); } else { pcmk__xml_free(*expanded_xml); *expanded_xml = NULL; } return pcmk_rc_ok; } /*! * \internal * \brief Parse a colocation constraint from XML into scheduler data * * \param[in,out] xml_obj Colocation constraint XML to unpack * \param[in,out] scheduler Scheduler data to add constraint to */ void pcmk__unpack_colocation(xmlNode *xml_obj, pcmk_scheduler_t *scheduler) { int score_i = 0; xmlNode *set = NULL; xmlNode *last = NULL; xmlNode *orig_xml = NULL; xmlNode *expanded_xml = NULL; const char *id = pcmk__xe_get(xml_obj, PCMK_XA_ID); const char *score = NULL; const char *influence_s = NULL; if (pcmk__str_empty(id)) { pcmk__config_err("Ignoring " PCMK_XE_RSC_COLOCATION " without " CRM_ATTR_ID); return; } if (unpack_colocation_tags(xml_obj, &expanded_xml, scheduler) != pcmk_rc_ok) { return; } if (expanded_xml != NULL) { orig_xml = xml_obj; xml_obj = expanded_xml; } score = pcmk__xe_get(xml_obj, PCMK_XA_SCORE); if (score != NULL) { int rc = pcmk_parse_score(score, &score_i, 0); if (rc != pcmk_rc_ok) { // Not possible with schema validation enabled pcmk__config_err("Ignoring colocation %s because '%s' " "is not a valid score", id, score); return; } } influence_s = pcmk__xe_get(xml_obj, PCMK_XA_INFLUENCE); for (set = pcmk__xe_first_child(xml_obj, PCMK_XE_RESOURCE_SET, NULL, NULL); set != NULL; set = pcmk__xe_next(set, PCMK_XE_RESOURCE_SET)) { set = pcmk__xe_resolve_idref(set, scheduler->input); if (set == NULL) { // Configuration error, message already logged if (expanded_xml != NULL) { pcmk__xml_free(expanded_xml); } return; } if (pcmk__str_empty(pcmk__xe_id(set))) { pcmk__config_err("Ignoring " PCMK_XE_RESOURCE_SET " without " CRM_ATTR_ID); continue; } unpack_colocation_set(set, score_i, id, influence_s, scheduler); if (last != NULL) { colocate_rsc_sets(id, last, set, score_i, influence_s, scheduler); } last = set; } if (expanded_xml) { pcmk__xml_free(expanded_xml); xml_obj = orig_xml; } if (last == NULL) { unpack_simple_colocation(xml_obj, id, score_i, influence_s, scheduler); } } /*! * \internal * \brief Check whether colocation's dependent preferences should be considered * * \param[in] colocation Colocation constraint * \param[in] rsc Primary instance (normally this will be * colocation->primary, which NULL will be treated as, * but for clones or bundles with multiple instances * this can be a particular instance) * * \return true if colocation influence should be effective, otherwise false */ bool pcmk__colocation_has_influence(const pcmk__colocation_t *colocation, const pcmk_resource_t *rsc) { if (rsc == NULL) { rsc = colocation->primary; } /* A bundle replica colocates its remote connection with its container, * using a finite score so that the container can run on Pacemaker Remote * nodes. * * Moving a connection is lightweight and does not interrupt the service, * while moving a container is heavyweight and does interrupt the service, * so don't move a clean, active container based solely on the preferences * of its connection. * * This also avoids problematic scenarios where two containers want to * perpetually swap places. */ if (pcmk__is_set(colocation->dependent->flags, pcmk__rsc_remote_nesting_allowed) && !pcmk__is_set(rsc->flags, pcmk__rsc_failed) && pcmk__list_of_1(rsc->priv->active_nodes)) { return false; } /* The dependent in a colocation influences the primary's location * if the PCMK_XA_INFLUENCE option is true or the primary is not yet active. */ return pcmk__is_set(colocation->flags, pcmk__coloc_influence) || (rsc->priv->active_nodes == NULL); } /*! * \internal * \brief Make actions of a given type unrunnable for a given resource * * \param[in,out] rsc Resource whose actions should be blocked * \param[in] task Name of action to block * \param[in] reason Unrunnable start action causing the block */ static void mark_action_blocked(pcmk_resource_t *rsc, const char *task, const pcmk_resource_t *reason) { GList *iter = NULL; char *reason_text = pcmk__assert_asprintf("colocation with %s", reason->id); for (iter = rsc->priv->actions; iter != NULL; iter = iter->next) { pcmk_action_t *action = iter->data; if (pcmk__is_set(action->flags, pcmk__action_runnable) && pcmk__str_eq(action->task, task, pcmk__str_none)) { pcmk__clear_action_flags(action, pcmk__action_runnable); pe_action_set_reason(action, reason_text, false); pcmk__block_colocation_dependents(action); pcmk__update_action_for_orderings(action, rsc->priv->scheduler); } } // If parent resource can't perform an action, neither can any children for (iter = rsc->priv->children; iter != NULL; iter = iter->next) { mark_action_blocked((pcmk_resource_t *) (iter->data), task, reason); } free(reason_text); } /*! * \internal * \brief If an action is unrunnable, block any relevant dependent actions * * If a given action is an unrunnable start or promote, block the start or * promote actions of resources colocated with it, as appropriate to the * colocations' configured roles. * * \param[in,out] action Action to check */ void pcmk__block_colocation_dependents(pcmk_action_t *action) { GList *iter = NULL; GList *colocations = NULL; pcmk_resource_t *rsc = NULL; bool is_start = false; if (pcmk__is_set(action->flags, pcmk__action_runnable)) { return; // Only unrunnable actions block dependents } is_start = pcmk__str_eq(action->task, PCMK_ACTION_START, pcmk__str_none); if (!is_start && !pcmk__str_eq(action->task, PCMK_ACTION_PROMOTE, pcmk__str_none)) { return; // Only unrunnable starts and promotes block dependents } pcmk__assert(action->rsc != NULL); // Start and promote are resource actions /* If this resource is part of a collective resource, dependents are blocked * only if all instances of the collective are unrunnable, so check the * collective resource. */ rsc = uber_parent(action->rsc); if (rsc->priv->parent != NULL) { rsc = rsc->priv->parent; // Bundle } // Colocation fails only if entire primary can't reach desired role for (iter = rsc->priv->children; iter != NULL; iter = iter->next) { pcmk_resource_t *child = iter->data; pcmk_action_t *child_action = NULL; child_action = find_first_action(child->priv->actions, NULL, action->task, NULL); if ((child_action == NULL) || pcmk__is_set(child_action->flags, pcmk__action_runnable)) { pcmk__trace("Not blocking %s colocation dependents because at " "least %s has runnable %s", rsc->id, child->id, action->task); return; // At least one child can reach desired role } } pcmk__trace("Blocking %s colocation dependents due to unrunnable %s %s", rsc->id, action->rsc->id, action->task); // Check each colocation where this resource is primary colocations = pcmk__with_this_colocations(rsc); for (iter = colocations; iter != NULL; iter = iter->next) { pcmk__colocation_t *colocation = iter->data; if (colocation->score < PCMK_SCORE_INFINITY) { continue; // Only mandatory colocations block dependent } /* If the primary can't start, the dependent can't reach its colocated * role, regardless of what the primary or dependent colocation role is. * * If the primary can't be promoted, the dependent can't reach its * colocated role if the primary's colocation role is promoted. */ if (!is_start && (colocation->primary_role != pcmk_role_promoted)) { continue; } // Block the dependent from reaching its colocated role if (colocation->dependent_role == pcmk_role_promoted) { mark_action_blocked(colocation->dependent, PCMK_ACTION_PROMOTE, action->rsc); } else { mark_action_blocked(colocation->dependent, PCMK_ACTION_START, action->rsc); } } g_list_free(colocations); } /*! * \internal * \brief Get the resource to use for role comparisons * * A bundle replica includes a container and possibly an instance of the bundled * resource. The dependent in a "with bundle" colocation is colocated with a * particular bundle container. However, if the colocation includes a role, then * the role must be checked on the bundled resource instance inside the * container. The container itself will never be promoted; the bundled resource * may be. * * If the given resource is a bundle replica container, return the resource * inside it, if any. Otherwise, return the resource itself. * * \param[in] rsc Resource to check * * \return Resource to use for role comparisons */ static const pcmk_resource_t * get_resource_for_role(const pcmk_resource_t *rsc) { if (pcmk__is_set(rsc->flags, pcmk__rsc_replica_container)) { const pcmk_resource_t *child = pe__get_rsc_in_container(rsc); if (child != NULL) { return child; } } return rsc; } /*! * \internal * \brief Determine how a colocation constraint should affect a resource * * Colocation constraints have different effects at different points in the * scheduler sequence. Initially, they affect a resource's location; once that * is determined, then for promotable clones they can affect a resource * instance's role; after both are determined, the constraints no longer matter. * Given a specific colocation constraint, check what has been done so far to * determine what should be affected at the current point in the scheduler. * * \param[in] dependent Dependent resource in colocation * \param[in] primary Primary resource in colocation * \param[in] colocation Colocation constraint * \param[in] preview If true, pretend resources have already been assigned * * \return How colocation constraint should be applied at this point */ enum pcmk__coloc_affects pcmk__colocation_affects(const pcmk_resource_t *dependent, const pcmk_resource_t *primary, const pcmk__colocation_t *colocation, bool preview) { const pcmk_resource_t *dependent_role_rsc = NULL; const pcmk_resource_t *primary_role_rsc = NULL; pcmk__assert((dependent != NULL) && (primary != NULL) && (colocation != NULL)); if (!preview && pcmk__is_set(primary->flags, pcmk__rsc_unassigned)) { // Primary resource has not been assigned yet, so we can't do anything return pcmk__coloc_affects_nothing; } dependent_role_rsc = get_resource_for_role(dependent); primary_role_rsc = get_resource_for_role(primary); if ((colocation->dependent_role >= pcmk_role_unpromoted) && (dependent_role_rsc->priv->parent != NULL) && pcmk__is_set(dependent_role_rsc->priv->parent->flags, pcmk__rsc_promotable) && !pcmk__is_set(dependent_role_rsc->flags, pcmk__rsc_unassigned)) { /* This is a colocation by role, and the dependent is a promotable clone * that has already been assigned, so the colocation should now affect * the role. */ return pcmk__coloc_affects_role; } if (!preview && !pcmk__is_set(dependent->flags, pcmk__rsc_unassigned)) { /* The dependent resource has already been through assignment, so the * constraint no longer matters. */ return pcmk__coloc_affects_nothing; } if ((colocation->dependent_role != pcmk_role_unknown) && (colocation->dependent_role != dependent_role_rsc->priv->next_role)) { pcmk__trace("Skipping %scolocation '%s': dependent limited to %s role " "but %s next role is %s", ((colocation->score < 0)? "anti-" : ""), colocation->id, pcmk_role_text(colocation->dependent_role), dependent_role_rsc->id, pcmk_role_text(dependent_role_rsc->priv->next_role)); return pcmk__coloc_affects_nothing; } if ((colocation->primary_role != pcmk_role_unknown) && (colocation->primary_role != primary_role_rsc->priv->next_role)) { pcmk__trace("Skipping %scolocation '%s': primary limited to %s role " "but %s next role is %s", ((colocation->score < 0)? "anti-" : ""), colocation->id, pcmk_role_text(colocation->primary_role), primary_role_rsc->id, pcmk_role_text(primary_role_rsc->priv->next_role)); return pcmk__coloc_affects_nothing; } return pcmk__coloc_affects_location; } /*! * \internal * \brief Apply colocation to dependent for assignment purposes * * Update the allowed node scores of the dependent resource in a colocation, * for the purposes of assigning it to a node. * * \param[in,out] dependent Dependent resource in colocation * \param[in] primary Primary resource in colocation * \param[in] colocation Colocation constraint */ void pcmk__apply_coloc_to_scores(pcmk_resource_t *dependent, const pcmk_resource_t *primary, const pcmk__colocation_t *colocation) { const char *attr = colocation->node_attribute; const char *value = NULL; GHashTable *work = NULL; GHashTableIter iter; pcmk_node_t *node = NULL; if (primary->priv->assigned_node != NULL) { value = pcmk__colocation_node_attr(primary->priv->assigned_node, attr, primary); } else if (colocation->score < 0) { // Nothing to do (anti-colocation with something that is not running) return; } work = pcmk__copy_node_table(dependent->priv->allowed_nodes); g_hash_table_iter_init(&iter, work); while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) { if (primary->priv->assigned_node == NULL) { node->assign->score = pcmk__add_scores(-colocation->score, node->assign->score); pcmk__rsc_trace(dependent, "Applied %s to %s score on %s (now %s after " "subtracting %s because primary %s inactive)", colocation->id, dependent->id, pcmk__node_name(node), pcmk_readable_score(node->assign->score), pcmk_readable_score(colocation->score), primary->id); continue; } if (pcmk__str_eq(pcmk__colocation_node_attr(node, attr, dependent), value, pcmk__str_casei)) { /* Add colocation score only if optional (or minus infinity). A * mandatory colocation is a requirement rather than a preference, * so we don't need to consider it for relative assignment purposes. * The resource will simply be forbidden from running on the node if * the primary isn't active there (via the condition above). */ if (colocation->score < PCMK_SCORE_INFINITY) { node->assign->score = pcmk__add_scores(colocation->score, node->assign->score); pcmk__rsc_trace(dependent, "Applied %s to %s score on %s (now %s after " "adding %s)", colocation->id, dependent->id, pcmk__node_name(node), pcmk_readable_score(node->assign->score), pcmk_readable_score(colocation->score)); } continue; } if (colocation->score >= PCMK_SCORE_INFINITY) { /* Only mandatory colocations are relevant when the colocation * attribute doesn't match, because an attribute not matching is not * a negative preference -- the colocation is simply relevant only * where it matches. */ node->assign->score = -PCMK_SCORE_INFINITY; pcmk__rsc_trace(dependent, "Banned %s from %s because colocation %s attribute %s " "does not match", dependent->id, pcmk__node_name(node), colocation->id, attr); } } if ((colocation->score <= -PCMK_SCORE_INFINITY) || (colocation->score >= PCMK_SCORE_INFINITY) || pcmk__any_node_available(work)) { g_hash_table_destroy(dependent->priv->allowed_nodes); dependent->priv->allowed_nodes = work; work = NULL; } else { pcmk__rsc_info(dependent, "%s: Rolling back scores from %s (no available nodes)", dependent->id, primary->id); } if (work != NULL) { g_hash_table_destroy(work); } } /*! * \internal * \brief Apply colocation to dependent for role purposes * * Update the priority of the dependent resource in a colocation, for the * purposes of selecting its role * * \param[in,out] dependent Dependent resource in colocation * \param[in] primary Primary resource in colocation * \param[in] colocation Colocation constraint * * \return The score added to the dependent's priority */ int pcmk__apply_coloc_to_priority(pcmk_resource_t *dependent, const pcmk_resource_t *primary, const pcmk__colocation_t *colocation) { const char *dependent_value = NULL; const char *primary_value = NULL; const char *attr = colocation->node_attribute; int score_multiplier = 1; int priority_delta = 0; const pcmk_node_t *primary_node = NULL; const pcmk_node_t *dependent_node = NULL; pcmk__assert((dependent != NULL) && (primary != NULL) && (colocation != NULL)); primary_node = primary->priv->assigned_node; dependent_node = dependent->priv->assigned_node; if (dependent_node == NULL) { return 0; } if ((primary_node != NULL) && (colocation->primary_role != pcmk_role_unknown)) { /* Colocation applies only if the primary's next role matches. * * If primary_node == NULL, we want to proceed past this block, so that * dependent_node is marked ineligible for promotion. * * @TODO Why ignore a mandatory colocation in this case when we apply * its negation in the mismatched value case? */ const pcmk_resource_t *role_rsc = get_resource_for_role(primary); if (colocation->primary_role != role_rsc->priv->next_role) { return 0; } } dependent_value = pcmk__colocation_node_attr(dependent_node, attr, dependent); primary_value = pcmk__colocation_node_attr(primary_node, attr, primary); if (!pcmk__str_eq(dependent_value, primary_value, pcmk__str_casei)) { if ((colocation->score == PCMK_SCORE_INFINITY) && (colocation->dependent_role == pcmk_role_promoted)) { /* For a mandatory promoted-role colocation, mark the dependent node * ineligible to promote the dependent if its attribute value * doesn't match the primary node's */ score_multiplier = -1; } else { // Otherwise, ignore the colocation if attribute values don't match return 0; } } else if (colocation->dependent_role == pcmk_role_unpromoted) { /* Node attribute values matched, so we want to avoid promoting the * dependent on this node */ score_multiplier = -1; } priority_delta = score_multiplier * colocation->score; dependent->priv->priority = pcmk__add_scores(priority_delta, dependent->priv->priority); pcmk__rsc_trace(dependent, "Applied %s to %s promotion priority (now %s after %s %d)", colocation->id, dependent->id, pcmk_readable_score(dependent->priv->priority), ((score_multiplier == 1)? "adding" : "subtracting"), colocation->score); return priority_delta; } /*! * \internal * \brief Find score of highest-scored node that matches colocation attribute * * \param[in] colocation Colocation constraint being applied * \param[in,out] rsc Resource whose allowed nodes should be searched * \param[in] attr Colocation attribute name (must not be NULL) * \param[in] value Colocation attribute value to require */ static int best_node_score_matching_attr(const pcmk__colocation_t *colocation, pcmk_resource_t *rsc, const char *attr, const char *value) { GHashTable *allowed_nodes_orig = NULL; GHashTableIter iter; pcmk_node_t *node = NULL; int best_score = -PCMK_SCORE_INFINITY; const char *best_node = NULL; if ((colocation != NULL) && (rsc == colocation->dependent) && pcmk__is_set(colocation->flags, pcmk__coloc_explicit) && pcmk__is_group(rsc->priv->parent) && (rsc != rsc->priv->parent->priv->children->data)) { /* The resource is a user-configured colocation's explicit dependent, * and a group member other than the first, which means the group's * location constraint scores were not applied to it (see * pcmk__group_apply_location()). Explicitly consider those scores now. * * @TODO This does leave one suboptimal case: if the group itself or * another member other than the first is explicitly colocated with * the same primary, the primary will count the group's location scores * multiple times. This is much less likely than a single member being * explicitly colocated, so it's an acceptable tradeoff for now. */ allowed_nodes_orig = rsc->priv->allowed_nodes; rsc->priv->allowed_nodes = pcmk__copy_node_table(allowed_nodes_orig); for (GList *loc_iter = rsc->priv->scheduler->priv->location_constraints; loc_iter != NULL; loc_iter = loc_iter->next) { pcmk__location_t *location = loc_iter->data; if (location->rsc == rsc->priv->parent) { rsc->priv->cmds->apply_location(rsc, location); } } } // Find best allowed node with matching attribute g_hash_table_iter_init(&iter, rsc->priv->allowed_nodes); while (g_hash_table_iter_next(&iter, NULL, (void **) &node)) { if ((node->assign->score > best_score) && pcmk__node_available(node, false, false) && pcmk__str_eq(value, pcmk__colocation_node_attr(node, attr, rsc), pcmk__str_casei)) { best_score = node->assign->score; best_node = node->priv->name; } } if (!pcmk__str_eq(attr, CRM_ATTR_UNAME, pcmk__str_none)) { if (best_node == NULL) { pcmk__info("No allowed node for %s matches node attribute %s=%s", rsc->id, attr, value); } else { pcmk__info("Allowed node %s for %s had best score (%d) of those " "matching node attribute %s=%s", best_node, rsc->id, best_score, attr, value); } } if (allowed_nodes_orig != NULL) { g_hash_table_destroy(rsc->priv->allowed_nodes); rsc->priv->allowed_nodes = allowed_nodes_orig; } return best_score; } /*! * \internal * \brief Check whether a resource is allowed only on a single node * * \param[in] rsc Resource to check * * \return \c true if \p rsc is allowed only on one node, otherwise \c false */ static bool allowed_on_one(const pcmk_resource_t *rsc) { GHashTableIter iter; pcmk_node_t *allowed_node = NULL; int allowed_nodes = 0; g_hash_table_iter_init(&iter, rsc->priv->allowed_nodes); while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &allowed_node)) { if ((allowed_node->assign->score >= 0) && (++allowed_nodes > 1)) { pcmk__rsc_trace(rsc, "%s is allowed on multiple nodes", rsc->id); return false; } } pcmk__rsc_trace(rsc, "%s is allowed %s", rsc->id, ((allowed_nodes == 1)? "on a single node" : "nowhere")); return (allowed_nodes == 1); } /*! * \internal * \brief Add resource's colocation matches to current node assignment scores * * For each node in a given table, if any of a given resource's allowed nodes * have a matching value for the colocation attribute, add the highest of those * nodes' scores to the node's score. * * \param[in,out] nodes Table of nodes with assignment scores so far * \param[in,out] source_rsc Resource whose node scores to add * \param[in] target_rsc Resource on whose behalf to update \p nodes * \param[in] colocation Original colocation constraint (used to get * configured primary resource's stickiness, and * to get colocation node attribute; pass NULL to * ignore stickiness and use default attribute) * \param[in] factor Factor by which to multiply scores being added * \param[in] only_positive Whether to add only positive scores */ static void add_node_scores_matching_attr(GHashTable *nodes, pcmk_resource_t *source_rsc, const pcmk_resource_t *target_rsc, const pcmk__colocation_t *colocation, float factor, bool only_positive) { GHashTableIter iter; pcmk_node_t *node = NULL; const char *attr = colocation->node_attribute; // Iterate through each node g_hash_table_iter_init(&iter, nodes); while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) { float delta_f = 0; int delta = 0; int score = 0; int new_score = 0; const char *value = pcmk__colocation_node_attr(node, attr, target_rsc); score = best_node_score_matching_attr(colocation, source_rsc, attr, value); if ((factor < 0) && (score < 0)) { /* If the dependent is anti-colocated, we generally don't want the * primary to prefer nodes that the dependent avoids. That could * lead to unnecessary shuffling of the primary when the dependent * hits its migration threshold somewhere, for example. * * However, there are cases when it is desirable. If the dependent * can't run anywhere but where the primary is, it would be * worthwhile to move the primary for the sake of keeping the * dependent active. * * We can't know that exactly at this point since we don't know * where the primary will be assigned, but we can limit considering * the preference to when the dependent is allowed only on one node. * This is less than ideal for multiple reasons: * * - the dependent could be allowed on more than one node but have * anti-colocation primaries on each; * - the dependent could be a clone or bundle with multiple * instances, and the dependent as a whole is allowed on multiple * nodes but some instance still can't run * - the dependent has considered node-specific criteria such as * location constraints and stickiness by this point, but might * have other factors that end up disallowing a node * * but the alternative is making the primary move when it doesn't * need to. * * We also consider the primary's stickiness and influence, so the * user has some say in the matter. (This is the configured primary, * not a particular instance of the primary, but that doesn't matter * unless stickiness uses a rule to vary by node, and that seems * acceptable to ignore.) */ if ((colocation->primary->priv->stickiness >= -score) || !pcmk__colocation_has_influence(colocation, NULL) || !allowed_on_one(colocation->dependent)) { pcmk__trace("%s: Filtering %d + %f * %d " "(double negative disallowed)", pcmk__node_name(node), node->assign->score, factor, score); continue; } } if (node->assign->score == INFINITY_HACK) { pcmk__trace("%s: Filtering %d + %f * %d (node was marked unusable)", pcmk__node_name(node), node->assign->score, factor, score); continue; } delta_f = factor * score; // Round the number; see http://c-faq.com/fp/round.html delta = (int) ((delta_f < 0)? (delta_f - 0.5) : (delta_f + 0.5)); /* Small factors can obliterate the small scores that are often actually * used in configurations. If the score and factor are nonzero, ensure * that the result is nonzero as well. */ if ((delta == 0) && (score != 0)) { if (factor > 0.0) { delta = 1; } else if (factor < 0.0) { delta = -1; } } new_score = pcmk__add_scores(delta, node->assign->score); if (only_positive && (new_score < 0) && (node->assign->score > 0)) { pcmk__trace("%s: Filtering %d + %f * %d = %d " "(negative disallowed, marking node unusable)", pcmk__node_name(node), node->assign->score, factor, score, new_score); node->assign->score = INFINITY_HACK; continue; } if (only_positive && (new_score < 0) && (node->assign->score == 0)) { pcmk__trace("%s: Filtering %d + %f * %d = %d (negative disallowed)", pcmk__node_name(node), node->assign->score, factor, score, new_score); continue; } pcmk__trace("%s: %d + %f * %d = %d", pcmk__node_name(node), node->assign->score, factor, score, new_score); node->assign->score = new_score; } } /*! * \internal * \brief Update nodes with scores of colocated resources' nodes * * Given a table of nodes and a resource, update the nodes' scores with the * scores of the best nodes matching the attribute used for each of the * resource's relevant colocations. * * \param[in,out] source_rsc Resource whose node scores to add * \param[in] target_rsc Resource on whose behalf to update \p *nodes * \param[in] log_id Resource ID for logs (if \c NULL, use * \p source_rsc ID) * \param[in,out] nodes Nodes to update (set initial contents to \c NULL * to copy allowed nodes from \p source_rsc) * \param[in] colocation Original colocation constraint (used to get * configured primary resource's stickiness, and * to get colocation node attribute; if \c NULL, * source_rsc's own matching node scores * will not be added, and \p *nodes must be \c NULL * as well) * \param[in] factor Incorporate scores multiplied by this factor * \param[in] flags Bitmask of enum pcmk__coloc_select values * * \note \c NULL \p target_rsc, \c NULL \p *nodes, \c NULL \p colocation, and * the \c pcmk__coloc_select_this_with flag are used together (and only by * \c cmp_resources()). * \note The caller remains responsible for freeing \p *nodes. * \note This is the shared implementation of * \c pcmk__assignment_methods_t:add_colocated_node_scores(). */ void pcmk__add_colocated_node_scores(pcmk_resource_t *source_rsc, const pcmk_resource_t *target_rsc, const char *log_id, GHashTable **nodes, const pcmk__colocation_t *colocation, float factor, uint32_t flags) { GHashTable *work = NULL; pcmk__assert((source_rsc != NULL) && (nodes != NULL) && ((colocation != NULL) || ((target_rsc == NULL) && (*nodes == NULL)))); if (log_id == NULL) { log_id = source_rsc->id; } // Avoid infinite recursion if (pcmk__is_set(source_rsc->flags, pcmk__rsc_updating_nodes)) { pcmk__rsc_info(source_rsc, "%s: Breaking dependency loop at %s", log_id, source_rsc->id); return; } pcmk__set_rsc_flags(source_rsc, pcmk__rsc_updating_nodes); if (*nodes == NULL) { work = pcmk__copy_node_table(source_rsc->priv->allowed_nodes); target_rsc = source_rsc; } else { const bool pos = pcmk__is_set(flags, pcmk__coloc_select_nonnegative); pcmk__rsc_trace(source_rsc, "%s: Merging %s scores from %s (at %.6f)", log_id, (pos? "positive" : "all"), source_rsc->id, factor); work = pcmk__copy_node_table(*nodes); add_node_scores_matching_attr(work, source_rsc, target_rsc, colocation, factor, pos); } if (work == NULL) { pcmk__clear_rsc_flags(source_rsc, pcmk__rsc_updating_nodes); return; } if (pcmk__any_node_available(work)) { GList *colocations = NULL; if (pcmk__is_set(flags, pcmk__coloc_select_this_with)) { colocations = pcmk__this_with_colocations(source_rsc); pcmk__rsc_trace(source_rsc, "Checking additional %d optional '%s with' " "constraints", g_list_length(colocations), source_rsc->id); } else { colocations = pcmk__with_this_colocations(source_rsc); pcmk__rsc_trace(source_rsc, "Checking additional %d optional 'with %s' " "constraints", g_list_length(colocations), source_rsc->id); } flags |= pcmk__coloc_select_active; for (GList *iter = colocations; iter != NULL; iter = iter->next) { pcmk__colocation_t *constraint = iter->data; pcmk_resource_t *other = NULL; float other_factor = factor * constraint->score / (float) PCMK_SCORE_INFINITY; if (pcmk__is_set(flags, pcmk__coloc_select_this_with)) { other = constraint->primary; } else if (!pcmk__colocation_has_influence(constraint, NULL)) { continue; } else { other = constraint->dependent; } pcmk__rsc_trace(source_rsc, "Optionally merging score of '%s' constraint " "(%s with %s)", constraint->id, constraint->dependent->id, constraint->primary->id); other->priv->cmds->add_colocated_node_scores(other, target_rsc, log_id, &work, constraint, other_factor, flags); pe__show_node_scores(true, NULL, log_id, work, source_rsc->priv->scheduler); } g_list_free(colocations); } else if (pcmk__is_set(flags, pcmk__coloc_select_active)) { pcmk__rsc_info(source_rsc, "%s: Rolling back optional scores from %s", log_id, source_rsc->id); g_hash_table_destroy(work); pcmk__clear_rsc_flags(source_rsc, pcmk__rsc_updating_nodes); return; } if (pcmk__is_set(flags, pcmk__coloc_select_nonnegative)) { pcmk_node_t *node = NULL; GHashTableIter iter; g_hash_table_iter_init(&iter, work); while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) { if (node->assign->score == INFINITY_HACK) { node->assign->score = 1; } } } if (*nodes != NULL) { g_hash_table_destroy(*nodes); } *nodes = work; pcmk__clear_rsc_flags(source_rsc, pcmk__rsc_updating_nodes); } /*! * \internal * \brief Apply a "with this" colocation to a resource's allowed node scores * * \param[in,out] data Colocation to apply * \param[in,out] user_data Resource being assigned */ void pcmk__add_dependent_scores(gpointer data, gpointer user_data) { pcmk__colocation_t *colocation = data; pcmk_resource_t *primary = user_data; pcmk_resource_t *dependent = colocation->dependent; const float factor = colocation->score / (float) PCMK_SCORE_INFINITY; uint32_t flags = pcmk__coloc_select_active; if (!pcmk__colocation_has_influence(colocation, NULL)) { return; } if (pcmk__is_clone(primary)) { flags |= pcmk__coloc_select_nonnegative; } pcmk__rsc_trace(primary, "%s: Incorporating attenuated %s assignment scores due " "to colocation %s", primary->id, dependent->id, colocation->id); dependent->priv->cmds->add_colocated_node_scores(dependent, primary, dependent->id, &(primary->priv->allowed_nodes), colocation, factor, flags); } /*! * \internal * \brief Exclude nodes from a dependent's node table if not in a given list * * Given a dependent resource in a colocation and a list of nodes where the * primary resource will run, set a node's score to \c -INFINITY in the * dependent's node table if not found in the primary nodes list. * * \param[in,out] dependent Dependent resource * \param[in] primary Primary resource (for logging only) * \param[in] colocation Colocation constraint (for logging only) * \param[in] primary_nodes List of nodes where the primary will have * unblocked instances in a suitable role * \param[in] merge_scores If \c true and a node is found in both \p table * and \p list, add the node's score in \p list to * the node's score in \p table */ void pcmk__colocation_intersect_nodes(pcmk_resource_t *dependent, const pcmk_resource_t *primary, const pcmk__colocation_t *colocation, const GList *primary_nodes, bool merge_scores) { GHashTableIter iter; pcmk_node_t *dependent_node = NULL; pcmk__assert((dependent != NULL) && (primary != NULL) && (colocation != NULL)); g_hash_table_iter_init(&iter, dependent->priv->allowed_nodes); while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &dependent_node)) { const pcmk_node_t *primary_node = NULL; primary_node = pe_find_node_id(primary_nodes, dependent_node->priv->id); if (primary_node == NULL) { dependent_node->assign->score = -PCMK_SCORE_INFINITY; pcmk__rsc_trace(dependent, "Banning %s from %s (no primary instance) for %s", dependent->id, pcmk__node_name(dependent_node), colocation->id); } else if (merge_scores) { dependent_node->assign->score = pcmk__add_scores(dependent_node->assign->score, primary_node->assign->score); pcmk__rsc_trace(dependent, "Added %s's score %s to %s's score for %s (now %d) " "for colocation %s", primary->id, pcmk_readable_score(primary_node->assign->score), dependent->id, pcmk__node_name(dependent_node), dependent_node->assign->score, colocation->id); } } } /*! * \internal * \brief Get all colocations affecting a resource as the primary * * \param[in] rsc Resource to get colocations for * * \return Newly allocated list of colocations affecting \p rsc as primary * * \note This is a convenience wrapper for the with_this_colocations() method. */ GList * pcmk__with_this_colocations(const pcmk_resource_t *rsc) { GList *list = NULL; rsc->priv->cmds->with_this_colocations(rsc, rsc, &list); return list; } /*! * \internal * \brief Get all colocations affecting a resource as the dependent * * \param[in] rsc Resource to get colocations for * * \return Newly allocated list of colocations affecting \p rsc as dependent * * \note This is a convenience wrapper for the this_with_colocations() method. */ GList * pcmk__this_with_colocations(const pcmk_resource_t *rsc) { GList *list = NULL; rsc->priv->cmds->this_with_colocations(rsc, rsc, &list); return list; } diff --git a/lib/pacemaker/pcmk_sched_location.c b/lib/pacemaker/pcmk_sched_location.c index 43a1d27ffe..45c9281bf9 100644 --- a/lib/pacemaker/pcmk_sched_location.c +++ b/lib/pacemaker/pcmk_sched_location.c @@ -1,784 +1,784 @@ /* * Copyright 2004-2025 the Pacemaker project contributors * * The version control history for this file may have further details. * * This source code is licensed under the GNU General Public License version 2 * or later (GPLv2+) WITHOUT ANY WARRANTY. */ #include #include #include #include #include #include // pcmk_parse_score(), etc. #include #include #include "libpacemaker_private.h" /*! * \internal * \brief Parse a role configuration for a location constraint * * \param[in] role_spec Role specification * \param[out] role Where to store parsed role * * \return true if role specification is valid, otherwise false */ static bool parse_location_role(const char *role_spec, enum rsc_role_e *role) { if (role_spec == NULL) { *role = pcmk_role_unknown; return true; } *role = pcmk_parse_role(role_spec); switch (*role) { case pcmk_role_unknown: return false; case pcmk_role_started: case pcmk_role_unpromoted: /* Any promotable clone instance cannot be promoted without being in * the unpromoted role first. Therefore, any constraint for the * started or unpromoted role applies to every role. */ *role = pcmk_role_unknown; break; default: break; } return true; } /*! * \internal * \brief Get the score attribute name (if any) used for a rule * * \param[in] rule_xml Rule XML * \param[out] allocated If the score attribute name needs to be allocated, * this will be set to the non-const equivalent of the * return value (should be set to NULL when passed) * \param[in] rule_input Values used to evaluate rule criteria * * \return Score attribute name used for rule, or NULL if none * \note The caller is responsible for freeing \p *allocated if it is non-NULL. */ static const char * score_attribute_name(const xmlNode *rule_xml, char **allocated, const pcmk_rule_input_t *rule_input) { const char *name = NULL; name = pcmk__xe_get(rule_xml, PCMK_XA_SCORE_ATTRIBUTE); if (name == NULL) { return NULL; } /* A score attribute name may use submatches extracted from a * resource ID regular expression. For example, if score-attribute is * "loc-\1", rsc-pattern is "ip-(.*)", and the resource ID is "ip-db", then * the score attribute name is "loc-db". */ if ((rule_input->rsc_id != NULL) && (rule_input->rsc_id_nmatches > 0)) { *allocated = pcmk__replace_submatches(name, rule_input->rsc_id, rule_input->rsc_id_submatches, rule_input->rsc_id_nmatches); if (*allocated != NULL) { name = *allocated; } } return name; } /*! * \internal * \brief Parse a score from a rule without a score attribute * * \param[in] rule_xml Rule XML * \param[out] score Where to store parsed score * * \return Standard Pacemaker return code */ static int score_from_rule(const xmlNode *rule_xml, int *score) { int rc = pcmk_rc_ok; const char *score_s = pcmk__xe_get(rule_xml, PCMK_XA_SCORE); if (score_s == NULL) { // Not possible with schema validation enabled pcmk__config_err("Ignoring location constraint rule %s because " "neither " PCMK_XA_SCORE " nor " PCMK_XA_SCORE_ATTRIBUTE " was specified", pcmk__xe_id(rule_xml)); return pcmk_rc_unpack_error; } rc = pcmk_parse_score(score_s, score, 0); if (rc != pcmk_rc_ok) { // Not possible with schema validation enabled pcmk__config_err("Ignoring location constraint rule %s because " "'%s' is not a valid " PCMK_XA_SCORE ": %s", pcmk__xe_id(rule_xml), score_s, pcmk_rc_str(rc)); return pcmk_rc_unpack_error; } return pcmk_rc_ok; } /*! * \internal * \brief Get a rule score from a node attribute * * \param[in] constraint_id Location constraint ID (for logging only) * \param[in] attr_name Name of node attribute with score * \param[in] node Node to get attribute for * \param[in] rsc Resource being located * \param[out] score Where to store parsed score * * \return Standard Pacemaker return code (pcmk_rc_ok if a valid score was * parsed, ENXIO if the node attribute was unset, and some other value * if the node attribute value was invalid) */ static int score_from_attr(const char *constraint_id, const char *attr_name, const pcmk_node_t *node, const pcmk_resource_t *rsc, int *score) { int rc = pcmk_rc_ok; const char *target = NULL; const char *score_s = NULL; target = g_hash_table_lookup(rsc->priv->meta, PCMK_META_CONTAINER_ATTRIBUTE_TARGET); score_s = pcmk__node_attr(node, attr_name, target, pcmk__rsc_node_current); if (pcmk__str_empty(score_s)) { pcmk__info("Ignoring location %s for %s on %s because it has no node " "attribute %s", constraint_id, rsc->id, pcmk__node_name(node), attr_name); return ENXIO; } rc = pcmk_parse_score(score_s, score, 0); if (rc != pcmk_rc_ok) { pcmk__warn("Ignoring location %s for node %s because node attribute " "%s value '%s' is not a valid score: %s", constraint_id, pcmk__node_name(node), attr_name, score_s, pcmk_rc_str(rc)); return rc; } return pcmk_rc_ok; } /*! * \internal * \brief Generate a location constraint from a rule * * \param[in,out] rsc Resource that constraint is for * \param[in] rule_xml Rule XML (sub-element of location constraint) * \param[in] discovery Value of \c PCMK_XA_RESOURCE_DISCOVERY for * constraint * \param[out] next_change Where to set when rule evaluation will change * \param[in,out] rule_input Values used to evaluate rule criteria * (node-specific values will be overwritten by * this function) * \param[in] constraint_id ID of location constraint (for logging only) * * \return true if rule is valid, otherwise false */ static bool generate_location_rule(pcmk_resource_t *rsc, xmlNode *rule_xml, const char *discovery, crm_time_t *next_change, pcmk_rule_input_t *rule_input, const char *constraint_id) { const char *rule_id = NULL; const char *score_attr = NULL; const char *boolean = NULL; const char *role_spec = NULL; GList *iter = NULL; int score = 0; char *local_score_attr = NULL; pcmk__location_t *location_rule = NULL; enum rsc_role_e role = pcmk_role_unknown; enum pcmk__combine combine = pcmk__combine_unknown; rule_xml = pcmk__xe_resolve_idref(rule_xml, rsc->priv->scheduler->input); if (rule_xml == NULL) { return false; // Error already logged } rule_id = pcmk__xe_get(rule_xml, PCMK_XA_ID); if (rule_id == NULL) { pcmk__config_err("Ignoring location constraint '%s' because its rule " "has no " PCMK_XA_ID, constraint_id); return false; } boolean = pcmk__xe_get(rule_xml, PCMK_XA_BOOLEAN_OP); role_spec = pcmk__xe_get(rule_xml, PCMK_XA_ROLE); if (parse_location_role(role_spec, &role)) { pcmk__trace("Setting rule %s role filter to %s", rule_id, role_spec); } else { pcmk__config_err("Ignoring location constraint '%s' because rule '%s' " "has invalid " PCMK_XA_ROLE " '%s'", constraint_id, rule_id, role_spec); return false; } combine = pcmk__parse_combine(boolean); switch (combine) { case pcmk__combine_and: case pcmk__combine_or: break; default: // Not possible with schema validation enabled pcmk__config_err("Ignoring location constraint '%s' because rule " "'%s' has invalid " PCMK_XA_BOOLEAN_OP " '%s'", constraint_id, rule_id, boolean); return false; } /* Users may configure the rule with either a score or the name of a * node attribute whose value should be used as the constraint score for * that node. */ score_attr = score_attribute_name(rule_xml, &local_score_attr, rule_input); if ((score_attr == NULL) && (score_from_rule(rule_xml, &score) != pcmk_rc_ok)) { return false; } location_rule = pcmk__new_location(rule_id, rsc, 0, discovery, NULL); CRM_CHECK(location_rule != NULL, return NULL); location_rule->role_filter = role; for (iter = rsc->priv->scheduler->nodes; iter != NULL; iter = iter->next) { pcmk_node_t *node = iter->data; pcmk_node_t *local = NULL; rule_input->node_attrs = node->priv->attrs; rule_input->rsc_params = pe_rsc_params(rsc, node, rsc->priv->scheduler); if (pcmk_evaluate_rule(rule_xml, rule_input, next_change) != pcmk_rc_ok) { continue; } if ((score_attr != NULL) && (score_from_attr(constraint_id, score_attr, node, rsc, &score) != pcmk_rc_ok)) { continue; // Message already logged } local = pe__copy_node(node); location_rule->nodes = g_list_prepend(location_rule->nodes, local); local->assign->score = score; pcmk__rsc_trace(rsc, "Location %s score for %s on %s is %s via rule %s", constraint_id, rsc->id, pcmk__node_name(node), pcmk_readable_score(score), rule_id); } free(local_score_attr); if (location_rule->nodes == NULL) { pcmk__trace("No matching nodes for location constraint rule %s", rule_id); } else { pcmk__trace("Location constraint rule %s matched %u nodes", rule_id, g_list_length(location_rule->nodes)); } return true; } static void unpack_rsc_location(xmlNode *xml_obj, pcmk_resource_t *rsc, const char *role_spec, const char *score, char *rsc_id_match, int rsc_id_nmatches, regmatch_t *rsc_id_submatches) { const char *rsc_id = pcmk__xe_get(xml_obj, PCMK_XA_RSC); const char *id = pcmk__xe_get(xml_obj, PCMK_XA_ID); const char *node = pcmk__xe_get(xml_obj, PCMK_XA_NODE); const char *discovery = pcmk__xe_get(xml_obj, PCMK_XA_RESOURCE_DISCOVERY); if (rsc == NULL) { pcmk__config_warn("Ignoring constraint '%s' because resource '%s' " "does not exist", id, rsc_id); return; } if (score == NULL) { score = pcmk__xe_get(xml_obj, PCMK_XA_SCORE); } if ((node != NULL) && (score != NULL)) { int score_i = 0; int rc = pcmk_rc_ok; pcmk_node_t *match = pcmk_find_node(rsc->priv->scheduler, node); enum rsc_role_e role = pcmk_role_unknown; pcmk__location_t *location = NULL; if (match == NULL) { pcmk__info("Ignoring location constraint %s because '%s' is not a " "known node", pcmk__s(id, "without ID"), node); return; } rc = pcmk_parse_score(score, &score_i, 0); if (rc != pcmk_rc_ok) { // Not possible with schema validation enabled pcmk__config_err("Ignoring location constraint %s " "because '%s' is not a valid score", id, score); return; } if (role_spec == NULL) { role_spec = pcmk__xe_get(xml_obj, PCMK_XA_ROLE); } if (parse_location_role(role_spec, &role)) { pcmk__trace("Setting location constraint %s role filter: %s", id, role_spec); } else { // Not possible with schema validation enabled pcmk__config_err("Ignoring location constraint %s " "because '%s' is not a valid " PCMK_XA_ROLE, id, role_spec); return; } location = pcmk__new_location(id, rsc, score_i, discovery, match); if (location == NULL) { return; // Error already logged } location->role_filter = role; } else { crm_time_t *next_change = crm_time_new_undefined(); xmlNode *rule_xml = pcmk__xe_first_child(xml_obj, PCMK_XE_RULE, NULL, NULL); pcmk_rule_input_t rule_input = { .now = rsc->priv->scheduler->priv->now, .rsc_meta = rsc->priv->meta, .rsc_id = rsc_id_match, .rsc_id_submatches = rsc_id_submatches, .rsc_id_nmatches = rsc_id_nmatches, }; generate_location_rule(rsc, rule_xml, discovery, next_change, &rule_input, id); /* If there is a point in the future when the evaluation of a rule will * change, make sure the scheduler is re-run by that time. */ if (crm_time_is_defined(next_change)) { time_t t = (time_t) crm_time_get_seconds_since_epoch(next_change); pcmk__update_recheck_time(t, rsc->priv->scheduler, "location rule evaluation"); } crm_time_free(next_change); } } static void unpack_simple_location(xmlNode *xml_obj, pcmk_scheduler_t *scheduler) { const char *id = pcmk__xe_get(xml_obj, PCMK_XA_ID); const char *value = pcmk__xe_get(xml_obj, PCMK_XA_RSC); if (value) { pcmk_resource_t *rsc; rsc = pcmk__find_constraint_resource(scheduler->priv->resources, value); unpack_rsc_location(xml_obj, rsc, NULL, NULL, NULL, 0, NULL); } value = pcmk__xe_get(xml_obj, PCMK_XA_RSC_PATTERN); if (value) { regex_t regex; bool invert = false; if (value[0] == '!') { value++; invert = true; } if (regcomp(®ex, value, REG_EXTENDED) != 0) { pcmk__config_err("Ignoring constraint '%s' because " PCMK_XA_RSC_PATTERN " has invalid value '%s'", id, value); return; } for (GList *iter = scheduler->priv->resources; iter != NULL; iter = iter->next) { pcmk_resource_t *r = iter->data; int nregs = 0; regmatch_t *pmatch = NULL; int status; if (regex.re_nsub > 0) { nregs = regex.re_nsub + 1; } else { nregs = 1; } pmatch = pcmk__assert_alloc(nregs, sizeof(regmatch_t)); status = regexec(®ex, r->id, nregs, pmatch, 0); if (!invert && (status == 0)) { pcmk__debug("'%s' matched '%s' for %s", r->id, value, id); unpack_rsc_location(xml_obj, r, NULL, NULL, r->id, nregs, pmatch); } else if (invert && (status != 0)) { pcmk__debug("'%s' is an inverted match of '%s' for %s", r->id, value, id); unpack_rsc_location(xml_obj, r, NULL, NULL, NULL, 0, NULL); } else { pcmk__trace("'%s' does not match '%s' for %s", r->id, value, id); } free(pmatch); } // @TODO Maybe log a notice if we did not match any resources regfree(®ex); } } // \return Standard Pacemaker return code static int unpack_location_tags(xmlNode *xml_obj, xmlNode **expanded_xml, pcmk_scheduler_t *scheduler) { const char *id = NULL; const char *rsc_id = NULL; const char *state = NULL; pcmk_resource_t *rsc = NULL; pcmk__idref_t *tag = NULL; xmlNode *rsc_set = NULL; *expanded_xml = NULL; CRM_CHECK(xml_obj != NULL, return EINVAL); id = pcmk__xe_id(xml_obj); if (id == NULL) { pcmk__config_err("Ignoring <%s> constraint without " PCMK_XA_ID, xml_obj->name); return pcmk_rc_unpack_error; } // Check whether there are any resource sets with template or tag references *expanded_xml = pcmk__expand_tags_in_sets(xml_obj, scheduler); if (*expanded_xml != NULL) { - crm_log_xml_trace(*expanded_xml, "Expanded " PCMK_XE_RSC_LOCATION); + pcmk__log_xml_trace(*expanded_xml, "Expanded " PCMK_XE_RSC_LOCATION); return pcmk_rc_ok; } rsc_id = pcmk__xe_get(xml_obj, PCMK_XA_RSC); if (rsc_id == NULL) { return pcmk_rc_ok; } if (!pcmk__valid_resource_or_tag(scheduler, rsc_id, &rsc, &tag)) { pcmk__config_err("Ignoring constraint '%s' because '%s' is not a " "valid resource or tag", id, rsc_id); return pcmk_rc_unpack_error; } else if (rsc != NULL) { // No template is referenced return pcmk_rc_ok; } state = pcmk__xe_get(xml_obj, PCMK_XA_ROLE); *expanded_xml = pcmk__xml_copy(NULL, xml_obj); /* Convert any template or tag reference into constraint * PCMK_XE_RESOURCE_SET */ if (!pcmk__tag_to_set(*expanded_xml, &rsc_set, PCMK_XA_RSC, false, scheduler)) { pcmk__xml_free(*expanded_xml); *expanded_xml = NULL; return pcmk_rc_unpack_error; } if (rsc_set != NULL) { if (state != NULL) { /* Move PCMK_XA_RSC_ROLE into converted PCMK_XE_RESOURCE_SET as * PCMK_XA_ROLE attribute */ pcmk__xe_set(rsc_set, PCMK_XA_ROLE, state); pcmk__xe_remove_attr(*expanded_xml, PCMK_XA_ROLE); } - crm_log_xml_trace(*expanded_xml, "Expanded " PCMK_XE_RSC_LOCATION); + pcmk__log_xml_trace(*expanded_xml, "Expanded " PCMK_XE_RSC_LOCATION); } else { // No sets pcmk__xml_free(*expanded_xml); *expanded_xml = NULL; } return pcmk_rc_ok; } // \return Standard Pacemaker return code static int unpack_location_set(xmlNode *location, xmlNode *set, pcmk_scheduler_t *scheduler) { xmlNode *xml_rsc = NULL; pcmk_resource_t *resource = NULL; const char *set_id; const char *role; const char *local_score; CRM_CHECK(set != NULL, return EINVAL); set_id = pcmk__xe_id(set); if (set_id == NULL) { pcmk__config_err("Ignoring " PCMK_XE_RESOURCE_SET " without " PCMK_XA_ID " in constraint '%s'", pcmk__s(pcmk__xe_id(location), "(missing ID)")); return pcmk_rc_unpack_error; } role = pcmk__xe_get(set, PCMK_XA_ROLE); local_score = pcmk__xe_get(set, PCMK_XA_SCORE); for (xml_rsc = pcmk__xe_first_child(set, PCMK_XE_RESOURCE_REF, NULL, NULL); xml_rsc != NULL; xml_rsc = pcmk__xe_next(xml_rsc, PCMK_XE_RESOURCE_REF)) { resource = pcmk__find_constraint_resource(scheduler->priv->resources, pcmk__xe_id(xml_rsc)); if (resource == NULL) { pcmk__config_err("%s: No resource found for %s", set_id, pcmk__xe_id(xml_rsc)); return pcmk_rc_unpack_error; } unpack_rsc_location(location, resource, role, local_score, NULL, 0, NULL); } return pcmk_rc_ok; } void pcmk__unpack_location(xmlNode *xml_obj, pcmk_scheduler_t *scheduler) { xmlNode *set = NULL; bool any_sets = false; xmlNode *orig_xml = NULL; xmlNode *expanded_xml = NULL; if (unpack_location_tags(xml_obj, &expanded_xml, scheduler) != pcmk_rc_ok) { return; } if (expanded_xml) { orig_xml = xml_obj; xml_obj = expanded_xml; } for (set = pcmk__xe_first_child(xml_obj, PCMK_XE_RESOURCE_SET, NULL, NULL); set != NULL; set = pcmk__xe_next(set, PCMK_XE_RESOURCE_SET)) { any_sets = true; set = pcmk__xe_resolve_idref(set, scheduler->input); if ((set == NULL) // Configuration error, message already logged || (unpack_location_set(xml_obj, set, scheduler) != pcmk_rc_ok)) { if (expanded_xml) { pcmk__xml_free(expanded_xml); } return; } } if (expanded_xml) { pcmk__xml_free(expanded_xml); xml_obj = orig_xml; } if (!any_sets) { unpack_simple_location(xml_obj, scheduler); } } /*! * \internal * \brief Add a new location constraint to scheduler data * * \param[in] id XML ID of location constraint * \param[in,out] rsc Resource in location constraint * \param[in] node_score Constraint score * \param[in] probe_mode When resource should be probed on node * \param[in] node Node in constraint (or NULL if rule-based) * * \return Newly allocated location constraint on success, otherwise NULL * \note The result will be added to the cluster (via \p rsc) and should not be * freed separately. */ pcmk__location_t * pcmk__new_location(const char *id, pcmk_resource_t *rsc, int node_score, const char *probe_mode, pcmk_node_t *node) { pcmk__location_t *new_con = NULL; CRM_CHECK((node != NULL) || (node_score == 0), return NULL); if (id == NULL) { pcmk__config_err("Invalid constraint: no ID specified"); return NULL; } if (rsc == NULL) { pcmk__config_err("Invalid constraint %s: no resource specified", id); return NULL; } new_con = pcmk__assert_alloc(1, sizeof(pcmk__location_t)); new_con->id = pcmk__str_copy(id); new_con->rsc = rsc; new_con->nodes = NULL; new_con->role_filter = pcmk_role_unknown; if (pcmk__str_eq(probe_mode, PCMK_VALUE_ALWAYS, pcmk__str_null_matches|pcmk__str_casei)) { new_con->probe_mode = pcmk__probe_always; } else if (pcmk__str_eq(probe_mode, PCMK_VALUE_NEVER, pcmk__str_casei)) { new_con->probe_mode = pcmk__probe_never; } else if (pcmk__str_eq(probe_mode, PCMK_VALUE_EXCLUSIVE, pcmk__str_casei)) { new_con->probe_mode = pcmk__probe_exclusive; pcmk__set_rsc_flags(rsc, pcmk__rsc_exclusive_probes); } else { pcmk__config_err("Invalid " PCMK_XA_RESOURCE_DISCOVERY " value %s " "in location constraint", probe_mode); } if (node != NULL) { pcmk_node_t *copy = pe__copy_node(node); copy->assign->score = node_score; new_con->nodes = g_list_prepend(NULL, copy); } rsc->priv->scheduler->priv->location_constraints = g_list_prepend(rsc->priv->scheduler->priv->location_constraints, new_con); rsc->priv->location_constraints = g_list_prepend(rsc->priv->location_constraints, new_con); return new_con; } /*! * \internal * \brief Apply all location constraints * * \param[in,out] scheduler Scheduler data */ void pcmk__apply_locations(pcmk_scheduler_t *scheduler) { for (GList *iter = scheduler->priv->location_constraints; iter != NULL; iter = iter->next) { pcmk__location_t *location = iter->data; location->rsc->priv->cmds->apply_location(location->rsc, location); } } /*! * \internal * \brief Apply a location constraint to a resource's allowed node scores * * \param[in,out] rsc Resource to apply constraint to * \param[in,out] location Location constraint to apply * * \note This does not consider the resource's children, so the resource's * apply_location() method should be used instead in most cases. */ void pcmk__apply_location(pcmk_resource_t *rsc, pcmk__location_t *location) { bool need_role = false; pcmk__assert((rsc != NULL) && (location != NULL)); // If a role was specified, ensure constraint is applicable need_role = (location->role_filter > pcmk_role_unknown); if (need_role && (location->role_filter != rsc->priv->next_role)) { pcmk__rsc_trace(rsc, "Not applying %s to %s because role will be %s not %s", location->id, rsc->id, pcmk_role_text(rsc->priv->next_role), pcmk_role_text(location->role_filter)); return; } if (location->nodes == NULL) { pcmk__rsc_trace(rsc, "Not applying %s to %s because no nodes match", location->id, rsc->id); return; } for (GList *iter = location->nodes; iter != NULL; iter = iter->next) { pcmk_node_t *node = iter->data; pcmk_node_t *allowed_node = NULL; allowed_node = g_hash_table_lookup(rsc->priv->allowed_nodes, node->priv->id); pcmk__rsc_trace(rsc, "Applying %s%s%s to %s score on %s: %c %s", location->id, (need_role? " for role " : ""), (need_role? pcmk_role_text(location->role_filter) : ""), rsc->id, pcmk__node_name(node), ((allowed_node == NULL)? '=' : '+'), pcmk_readable_score(node->assign->score)); if (allowed_node == NULL) { allowed_node = pe__copy_node(node); g_hash_table_insert(rsc->priv->allowed_nodes, (gpointer) allowed_node->priv->id, allowed_node); } else { allowed_node->assign->score = pcmk__add_scores(allowed_node->assign->score, node->assign->score); } if (allowed_node->assign->probe_mode < location->probe_mode) { if (location->probe_mode == pcmk__probe_exclusive) { pcmk__set_rsc_flags(rsc, pcmk__rsc_exclusive_probes); } /* exclusive > never > always... always is default */ allowed_node->assign->probe_mode = location->probe_mode; } } } diff --git a/lib/pacemaker/pcmk_sched_ordering.c b/lib/pacemaker/pcmk_sched_ordering.c index e3900e20b3..1e58d142b3 100644 --- a/lib/pacemaker/pcmk_sched_ordering.c +++ b/lib/pacemaker/pcmk_sched_ordering.c @@ -1,1504 +1,1504 @@ /* * Copyright 2004-2025 the Pacemaker project contributors * * The version control history for this file may have further details. * * This source code is licensed under the GNU General Public License version 2 * or later (GPLv2+) WITHOUT ANY WARRANTY. */ #include #include // PRIx32 #include // bool, true, false #include #include #include // pcmk_parse_score() #include #include "libpacemaker_private.h" enum pe_order_kind { pe_order_kind_optional, pe_order_kind_mandatory, pe_order_kind_serialize, }; enum ordering_symmetry { ordering_asymmetric, // the only relation in an asymmetric ordering ordering_symmetric, // the normal relation in a symmetric ordering ordering_symmetric_inverse, // the inverse relation in a symmetric ordering }; // @TODO de-functionize this for readability and possibly better log messages #define EXPAND_CONSTRAINT_IDREF(__set, __rsc, __name) do { \ __rsc = pcmk__find_constraint_resource(scheduler->priv->resources, \ __name); \ if (__rsc == NULL) { \ pcmk__config_err("%s: No resource found for %s", __set, __name);\ return pcmk_rc_unpack_error; \ } \ } while (0) static const char * invert_action(const char *action) { if (pcmk__str_eq(action, PCMK_ACTION_START, pcmk__str_none)) { return PCMK_ACTION_STOP; } else if (pcmk__str_eq(action, PCMK_ACTION_STOP, pcmk__str_none)) { return PCMK_ACTION_START; } else if (pcmk__str_eq(action, PCMK_ACTION_PROMOTE, pcmk__str_none)) { return PCMK_ACTION_DEMOTE; } else if (pcmk__str_eq(action, PCMK_ACTION_DEMOTE, pcmk__str_none)) { return PCMK_ACTION_PROMOTE; } else if (pcmk__str_eq(action, PCMK_ACTION_PROMOTED, pcmk__str_none)) { return PCMK_ACTION_DEMOTED; } else if (pcmk__str_eq(action, PCMK_ACTION_DEMOTED, pcmk__str_none)) { return PCMK_ACTION_PROMOTED; } else if (pcmk__str_eq(action, PCMK_ACTION_RUNNING, pcmk__str_none)) { return PCMK_ACTION_STOPPED; } else if (pcmk__str_eq(action, PCMK_ACTION_STOPPED, pcmk__str_none)) { return PCMK_ACTION_RUNNING; } pcmk__config_warn("Unknown action '%s' specified in order constraint", action); return NULL; } static enum pe_order_kind get_ordering_type(const xmlNode *xml_obj) { enum pe_order_kind kind_e = pe_order_kind_mandatory; const char *kind = pcmk__xe_get(xml_obj, PCMK_XA_KIND); if (kind == NULL) { const char *score = pcmk__xe_get(xml_obj, PCMK_XA_SCORE); kind_e = pe_order_kind_mandatory; if (score) { // @COMPAT deprecated informally since 1.0.7, formally since 2.0.1 int score_i = 0; (void) pcmk_parse_score(score, &score_i, 0); if (score_i == 0) { kind_e = pe_order_kind_optional; } pcmk__warn_once(pcmk__wo_order_score, "Support for '" PCMK_XA_SCORE "' in " PCMK_XE_RSC_ORDER " is deprecated and will be " "removed in a future release " "(use '" PCMK_XA_KIND "' instead)"); } } else if (pcmk__str_eq(kind, PCMK_VALUE_MANDATORY, pcmk__str_none)) { kind_e = pe_order_kind_mandatory; } else if (pcmk__str_eq(kind, PCMK_VALUE_OPTIONAL, pcmk__str_none)) { kind_e = pe_order_kind_optional; } else if (pcmk__str_eq(kind, PCMK_VALUE_SERIALIZE, pcmk__str_none)) { kind_e = pe_order_kind_serialize; } else { pcmk__config_err("Resetting '" PCMK_XA_KIND "' for constraint %s to " "'" PCMK_VALUE_MANDATORY "' because '%s' is not valid", pcmk__s(pcmk__xe_id(xml_obj), "missing ID"), kind); } return kind_e; } /*! * \internal * \brief Get ordering symmetry from XML * * \param[in] xml_obj Ordering XML * \param[in] parent_kind Default ordering kind * \param[in] parent_symmetrical_s Parent element's \c PCMK_XA_SYMMETRICAL * setting, if any * * \retval ordering_symmetric Ordering is symmetric * \retval ordering_asymmetric Ordering is asymmetric */ static enum ordering_symmetry get_ordering_symmetry(const xmlNode *xml_obj, enum pe_order_kind parent_kind, const char *parent_symmetrical_s) { int rc = pcmk_rc_ok; bool symmetric = false; enum pe_order_kind kind = parent_kind; // Default to parent's kind // Check ordering XML for explicit kind if ((pcmk__xe_get(xml_obj, PCMK_XA_KIND) != NULL) || (pcmk__xe_get(xml_obj, PCMK_XA_SCORE) != NULL)) { kind = get_ordering_type(xml_obj); } // Check ordering XML (and parent) for explicit PCMK_XA_SYMMETRICAL setting rc = pcmk__xe_get_bool_attr(xml_obj, PCMK_XA_SYMMETRICAL, &symmetric); if (rc != pcmk_rc_ok && parent_symmetrical_s != NULL) { symmetric = pcmk__is_true(parent_symmetrical_s); rc = pcmk_rc_ok; } if (rc == pcmk_rc_ok) { if (symmetric) { if (kind == pe_order_kind_serialize) { pcmk__config_warn("Ignoring " PCMK_XA_SYMMETRICAL " for '%s' because not valid with " PCMK_XA_KIND " of '" PCMK_VALUE_SERIALIZE "'", pcmk__xe_id(xml_obj)); } else { return ordering_symmetric; } } return ordering_asymmetric; } // Use default symmetry if (kind == pe_order_kind_serialize) { return ordering_asymmetric; } return ordering_symmetric; } /*! * \internal * \brief Get ordering flags appropriate to ordering kind * * \param[in] kind Ordering kind * \param[in] first Action name for 'first' action * \param[in] symmetry This ordering's symmetry role * * \return Minimal ordering flags appropriate to \p kind */ static uint32_t ordering_flags_for_kind(enum pe_order_kind kind, const char *first, enum ordering_symmetry symmetry) { uint32_t flags = pcmk__ar_none; // so we trace-log all flags set switch (kind) { case pe_order_kind_optional: pcmk__set_relation_flags(flags, pcmk__ar_ordered); break; case pe_order_kind_serialize: /* This flag is not used anywhere directly but means the relation * will not match an equality comparison against pcmk__ar_none or * pcmk__ar_ordered. */ pcmk__set_relation_flags(flags, pcmk__ar_serialize); break; case pe_order_kind_mandatory: pcmk__set_relation_flags(flags, pcmk__ar_ordered); switch (symmetry) { case ordering_asymmetric: pcmk__set_relation_flags(flags, pcmk__ar_asymmetric); break; case ordering_symmetric: pcmk__set_relation_flags(flags, pcmk__ar_first_implies_then); if (pcmk__is_up_action(first)) { pcmk__set_relation_flags(flags, pcmk__ar_unrunnable_first_blocks); } break; case ordering_symmetric_inverse: pcmk__set_relation_flags(flags, pcmk__ar_then_implies_first); break; } break; } return flags; } /*! * \internal * \brief Find resource corresponding to ID specified in ordering * * \param[in] xml Ordering XML * \param[in] resource_attr XML attribute name for resource ID * \param[in] scheduler Scheduler data * * \return Resource corresponding to \p id, or NULL if none */ static pcmk_resource_t * get_ordering_resource(const xmlNode *xml, const char *resource_attr, const pcmk_scheduler_t *scheduler) { pcmk_resource_t *rsc = NULL; const char *rsc_id = pcmk__xe_get(xml, resource_attr); if (rsc_id == NULL) { pcmk__config_err("Ignoring constraint '%s' without %s", pcmk__xe_id(xml), resource_attr); return NULL; } rsc = pcmk__find_constraint_resource(scheduler->priv->resources, rsc_id); if (rsc == NULL) { pcmk__config_err("Ignoring constraint '%s' because resource '%s' " "does not exist", pcmk__xe_id(xml), rsc_id); return NULL; } return rsc; } /*! * \internal * \brief Determine minimum number of 'first' instances required in ordering * * \param[in] rsc 'First' resource in ordering * \param[in] xml Ordering XML * * \return Minimum 'first' instances required (or 0 if not applicable) */ static int get_minimum_first_instances(const pcmk_resource_t *rsc, const xmlNode *xml) { const char *clone_min = NULL; bool require_all = false; if (!pcmk__is_clone(rsc)) { return 0; } clone_min = g_hash_table_lookup(rsc->priv->meta, PCMK_META_CLONE_MIN); if (clone_min != NULL) { int clone_min_int = 0; pcmk__scan_min_int(clone_min, &clone_min_int, 0); return clone_min_int; } /* @COMPAT 1.1.13: * PCMK_XA_REQUIRE_ALL=PCMK_VALUE_FALSE is deprecated equivalent of * PCMK_META_CLONE_MIN=1 */ if (pcmk__xe_get_bool_attr(xml, PCMK_XA_REQUIRE_ALL, &require_all) != ENODATA) { pcmk__warn_once(pcmk__wo_require_all, "Support for " PCMK_XA_REQUIRE_ALL " in ordering " "constraints is deprecated and will be removed in a " "future release (use " PCMK_META_CLONE_MIN " clone " "meta-attribute instead)"); if (!require_all) { return 1; } } return 0; } /*! * \internal * \brief Create orderings for a constraint with \c PCMK_META_CLONE_MIN > 0 * * \param[in] id Ordering ID * \param[in,out] rsc_first 'First' resource in ordering (a clone) * \param[in] action_first 'First' action in ordering * \param[in] rsc_then 'Then' resource in ordering * \param[in] action_then 'Then' action in ordering * \param[in] flags Ordering flags * \param[in] clone_min Minimum required instances of 'first' */ static void clone_min_ordering(const char *id, pcmk_resource_t *rsc_first, const char *action_first, pcmk_resource_t *rsc_then, const char *action_then, uint32_t flags, int clone_min) { // Create a pseudo-action for when the minimum instances are active char *task = pcmk__assert_asprintf(PCMK_ACTION_CLONE_ONE_OR_MORE ":%s", id); pcmk_action_t *clone_min_met = get_pseudo_op(task, rsc_first->priv->scheduler); free(task); /* Require the pseudo-action to have the required number of actions to be * considered runnable before allowing the pseudo-action to be runnable. */ clone_min_met->required_runnable_before = clone_min; // Order the actions for each clone instance before the pseudo-action for (GList *iter = rsc_first->priv->children; iter != NULL; iter = iter->next) { pcmk_resource_t *child = iter->data; pcmk__new_ordering(child, pcmk__op_key(child->id, action_first, 0), NULL, NULL, NULL, clone_min_met, pcmk__ar_min_runnable |pcmk__ar_first_implies_then_graphed, rsc_first->priv->scheduler); } // Order "then" action after the pseudo-action (if runnable) pcmk__new_ordering(NULL, NULL, clone_min_met, rsc_then, pcmk__op_key(rsc_then->id, action_then, 0), NULL, flags|pcmk__ar_unrunnable_first_blocks, rsc_first->priv->scheduler); } /*! * \internal * \brief Create new ordering for inverse of symmetric constraint * * \param[in] id Ordering ID (for logging only) * \param[in] kind Ordering kind * \param[in] rsc_first 'First' resource in ordering (a clone) * \param[in] action_first 'First' action in ordering * \param[in,out] rsc_then 'Then' resource in ordering * \param[in] action_then 'Then' action in ordering */ static void inverse_ordering(const char *id, enum pe_order_kind kind, pcmk_resource_t *rsc_first, const char *action_first, pcmk_resource_t *rsc_then, const char *action_then) { uint32_t flags; const char *inverted_first = invert_action(action_first); const char *inverted_then = invert_action(action_then); if ((inverted_then == NULL) || (inverted_first == NULL)) { pcmk__config_warn("Cannot invert constraint '%s' " "(please specify inverse manually)", id); return; } // Order inverted actions flags = ordering_flags_for_kind(kind, inverted_first, ordering_symmetric_inverse); pcmk__order_resource_actions(rsc_then, inverted_then, rsc_first, inverted_first, flags); } static void unpack_simple_rsc_order(xmlNode *xml_obj, pcmk_scheduler_t *scheduler) { pcmk_resource_t *rsc_then = NULL; pcmk_resource_t *rsc_first = NULL; int min_required_before = 0; enum pe_order_kind kind = pe_order_kind_mandatory; uint32_t flags = pcmk__ar_none; enum ordering_symmetry symmetry; const char *action_then = NULL; const char *action_first = NULL; const char *id = NULL; CRM_CHECK(xml_obj != NULL, return); id = pcmk__xe_get(xml_obj, PCMK_XA_ID); if (id == NULL) { pcmk__config_err("Ignoring <%s> constraint without " PCMK_XA_ID, xml_obj->name); return; } rsc_first = get_ordering_resource(xml_obj, PCMK_XA_FIRST, scheduler); if (rsc_first == NULL) { return; } rsc_then = get_ordering_resource(xml_obj, PCMK_XA_THEN, scheduler); if (rsc_then == NULL) { return; } action_first = pcmk__xe_get(xml_obj, PCMK_XA_FIRST_ACTION); if (action_first == NULL) { action_first = PCMK_ACTION_START; } action_then = pcmk__xe_get(xml_obj, PCMK_XA_THEN_ACTION); if (action_then == NULL) { action_then = action_first; } kind = get_ordering_type(xml_obj); symmetry = get_ordering_symmetry(xml_obj, kind, NULL); flags = ordering_flags_for_kind(kind, action_first, symmetry); /* If there is a minimum number of instances that must be runnable before * the 'then' action is runnable, we use a pseudo-action for convenience: * minimum number of clone instances have runnable actions -> * pseudo-action is runnable -> dependency is runnable. */ min_required_before = get_minimum_first_instances(rsc_first, xml_obj); if (min_required_before > 0) { clone_min_ordering(id, rsc_first, action_first, rsc_then, action_then, flags, min_required_before); } else { pcmk__order_resource_actions(rsc_first, action_first, rsc_then, action_then, flags); } if (symmetry == ordering_symmetric) { inverse_ordering(id, kind, rsc_first, action_first, rsc_then, action_then); } } /*! * \internal * \brief Create a new ordering between two actions * * \param[in,out] first_rsc Resource for 'first' action (if NULL and * \p first_action is a resource action, that * resource will be used) * \param[in,out] first_action_task Action key for 'first' action (if NULL and * \p first_action is not NULL, its UUID will * be used) * \param[in,out] first_action 'first' action (if NULL, \p first_rsc and * \p first_action_task must be set) * * \param[in] then_rsc Resource for 'then' action (if NULL and * \p then_action is a resource action, that * resource will be used) * \param[in,out] then_action_task Action key for 'then' action (if NULL and * \p then_action is not NULL, its UUID will * be used) * \param[in] then_action 'then' action (if NULL, \p then_rsc and * \p then_action_task must be set) * * \param[in] flags Group of enum pcmk__action_relation_flags * \param[in,out] sched Scheduler data to add ordering to * * \note This function takes ownership of first_action_task and * then_action_task, which do not need to be freed by the caller. */ void pcmk__new_ordering(pcmk_resource_t *first_rsc, char *first_action_task, pcmk_action_t *first_action, pcmk_resource_t *then_rsc, char *then_action_task, pcmk_action_t *then_action, uint32_t flags, pcmk_scheduler_t *sched) { pcmk__action_relation_t *order = NULL; // One of action or resource must be specified for each side CRM_CHECK(((first_action != NULL) || (first_rsc != NULL)) && ((then_action != NULL) || (then_rsc != NULL)), free(first_action_task); free(then_action_task); return); if ((first_rsc == NULL) && (first_action != NULL)) { first_rsc = first_action->rsc; } if ((then_rsc == NULL) && (then_action != NULL)) { then_rsc = then_action->rsc; } order = pcmk__assert_alloc(1, sizeof(pcmk__action_relation_t)); order->id = sched->priv->next_ordering_id++; order->flags = flags; order->rsc1 = first_rsc; order->rsc2 = then_rsc; order->action1 = first_action; order->action2 = then_action; order->task1 = first_action_task; order->task2 = then_action_task; if ((order->task1 == NULL) && (first_action != NULL)) { order->task1 = strdup(first_action->uuid); } if ((order->task2 == NULL) && (then_action != NULL)) { order->task2 = strdup(then_action->uuid); } if ((order->rsc1 == NULL) && (first_action != NULL)) { order->rsc1 = first_action->rsc; } if ((order->rsc2 == NULL) && (then_action != NULL)) { order->rsc2 = then_action->rsc; } pcmk__rsc_trace(first_rsc, "Created ordering %d for %s then %s", (sched->priv->next_ordering_id - 1), pcmk__s(order->task1, "an underspecified action"), pcmk__s(order->task2, "an underspecified action")); sched->priv->ordering_constraints = g_list_prepend(sched->priv->ordering_constraints, order); pcmk__order_migration_equivalents(order); } /*! * \brief Unpack a set in an ordering constraint * * \param[in] set Set XML to unpack * \param[in] parent_kind \c PCMK_XE_RSC_ORDER XML \c PCMK_XA_KIND * attribute * \param[in] parent_symmetrical_s \c PCMK_XE_RSC_ORDER XML * \c PCMK_XA_SYMMETRICAL attribute * \param[in,out] scheduler Scheduler data * * \return Standard Pacemaker return code */ static int unpack_order_set(const xmlNode *set, enum pe_order_kind parent_kind, const char *parent_symmetrical_s, pcmk_scheduler_t *scheduler) { GList *set_iter = NULL; GList *resources = NULL; pcmk_resource_t *last = NULL; pcmk_resource_t *resource = NULL; int local_kind = parent_kind; bool sequential = false; uint32_t flags = pcmk__ar_ordered; enum ordering_symmetry symmetry; char *key = NULL; const char *id = pcmk__xe_id(set); const char *action = pcmk__xe_get(set, PCMK_XA_ACTION); const char *sequential_s = pcmk__xe_get(set, PCMK_XA_SEQUENTIAL); const char *kind_s = pcmk__xe_get(set, PCMK_XA_KIND); if (action == NULL) { action = PCMK_ACTION_START; } if (kind_s) { local_kind = get_ordering_type(set); } if (sequential_s == NULL) { sequential_s = "1"; } sequential = pcmk__is_true(sequential_s); symmetry = get_ordering_symmetry(set, parent_kind, parent_symmetrical_s); flags = ordering_flags_for_kind(local_kind, action, symmetry); for (const xmlNode *xml_rsc = pcmk__xe_first_child(set, PCMK_XE_RESOURCE_REF, NULL, NULL); xml_rsc != NULL; xml_rsc = pcmk__xe_next(xml_rsc, PCMK_XE_RESOURCE_REF)) { EXPAND_CONSTRAINT_IDREF(id, resource, pcmk__xe_id(xml_rsc)); resources = g_list_append(resources, resource); } if (pcmk__list_of_1(resources)) { pcmk__trace("Single set: %s", id); goto done; } set_iter = resources; while (set_iter != NULL) { resource = (pcmk_resource_t *) set_iter->data; set_iter = set_iter->next; key = pcmk__op_key(resource->id, action, 0); if (local_kind == pe_order_kind_serialize) { /* Serialize before everything that comes after */ for (GList *iter = set_iter; iter != NULL; iter = iter->next) { pcmk_resource_t *then_rsc = iter->data; char *then_key = pcmk__op_key(then_rsc->id, action, 0); pcmk__new_ordering(resource, strdup(key), NULL, then_rsc, then_key, NULL, flags, scheduler); } } else if (sequential) { if (last != NULL) { pcmk__order_resource_actions(last, action, resource, action, flags); } last = resource; } free(key); } if (symmetry == ordering_asymmetric) { goto done; } last = NULL; action = invert_action(action); flags = ordering_flags_for_kind(local_kind, action, ordering_symmetric_inverse); set_iter = resources; while (set_iter != NULL) { resource = (pcmk_resource_t *) set_iter->data; set_iter = set_iter->next; if (sequential) { if (last != NULL) { pcmk__order_resource_actions(resource, action, last, action, flags); } last = resource; } } done: g_list_free(resources); return pcmk_rc_ok; } /*! * \brief Order two resource sets relative to each other * * \param[in] id Ordering ID (for logging) * \param[in] set1 First listed set * \param[in] set2 Second listed set * \param[in] kind Ordering kind * \param[in,out] scheduler Scheduler data * \param[in] symmetry Which ordering symmetry applies to this relation * * \return Standard Pacemaker return code */ static int order_rsc_sets(const char *id, const xmlNode *set1, const xmlNode *set2, enum pe_order_kind kind, pcmk_scheduler_t *scheduler, enum ordering_symmetry symmetry) { const xmlNode *xml_rsc = NULL; const xmlNode *xml_rsc_2 = NULL; pcmk_resource_t *rsc_1 = NULL; pcmk_resource_t *rsc_2 = NULL; const char *action_1 = pcmk__xe_get(set1, PCMK_XA_ACTION); const char *action_2 = pcmk__xe_get(set2, PCMK_XA_ACTION); uint32_t flags = pcmk__ar_none; bool require_all = true; (void) pcmk__xe_get_bool_attr(set1, PCMK_XA_REQUIRE_ALL, &require_all); if (action_1 == NULL) { action_1 = PCMK_ACTION_START; } if (action_2 == NULL) { action_2 = PCMK_ACTION_START; } if (symmetry == ordering_symmetric_inverse) { action_1 = invert_action(action_1); action_2 = invert_action(action_2); } if (pcmk__str_eq(PCMK_ACTION_STOP, action_1, pcmk__str_none) || pcmk__str_eq(PCMK_ACTION_DEMOTE, action_1, pcmk__str_none)) { /* Assuming: A -> ( B || C) -> D * The one-or-more logic only applies during the start/promote phase. * During shutdown neither B nor can shutdown until D is down, so simply * turn require_all back on. */ require_all = true; } flags = ordering_flags_for_kind(kind, action_1, symmetry); /* If we have an unordered set1, whether it is sequential or not is * irrelevant in regards to set2. */ if (!require_all) { char *task = pcmk__assert_asprintf(PCMK_ACTION_ONE_OR_MORE ":%s", pcmk__xe_id(set1)); pcmk_action_t *unordered_action = get_pseudo_op(task, scheduler); free(task); unordered_action->required_runnable_before = 1; for (xml_rsc = pcmk__xe_first_child(set1, PCMK_XE_RESOURCE_REF, NULL, NULL); xml_rsc != NULL; xml_rsc = pcmk__xe_next(xml_rsc, PCMK_XE_RESOURCE_REF)) { EXPAND_CONSTRAINT_IDREF(id, rsc_1, pcmk__xe_id(xml_rsc)); /* Add an ordering constraint between every element in set1 and the * pseudo action. If any action in set1 is runnable the pseudo * action will be runnable. */ pcmk__new_ordering(rsc_1, pcmk__op_key(rsc_1->id, action_1, 0), NULL, NULL, NULL, unordered_action, pcmk__ar_min_runnable |pcmk__ar_first_implies_then_graphed, scheduler); } for (xml_rsc_2 = pcmk__xe_first_child(set2, PCMK_XE_RESOURCE_REF, NULL, NULL); xml_rsc_2 != NULL; xml_rsc_2 = pcmk__xe_next(xml_rsc_2, PCMK_XE_RESOURCE_REF)) { EXPAND_CONSTRAINT_IDREF(id, rsc_2, pcmk__xe_id(xml_rsc_2)); /* Add an ordering constraint between the pseudo-action and every * element in set2. If the pseudo-action is runnable, every action * in set2 will be runnable. */ pcmk__new_ordering(NULL, NULL, unordered_action, rsc_2, pcmk__op_key(rsc_2->id, action_2, 0), NULL, flags|pcmk__ar_unrunnable_first_blocks, scheduler); } return pcmk_rc_ok; } if (pcmk__xe_attr_is_true(set1, PCMK_XA_SEQUENTIAL)) { if (symmetry == ordering_symmetric_inverse) { // Get the first one xml_rsc = pcmk__xe_first_child(set1, PCMK_XE_RESOURCE_REF, NULL, NULL); if (xml_rsc != NULL) { EXPAND_CONSTRAINT_IDREF(id, rsc_1, pcmk__xe_id(xml_rsc)); } } else { // Get the last one const char *rid = NULL; for (xml_rsc = pcmk__xe_first_child(set1, PCMK_XE_RESOURCE_REF, NULL, NULL); xml_rsc != NULL; xml_rsc = pcmk__xe_next(xml_rsc, PCMK_XE_RESOURCE_REF)) { rid = pcmk__xe_id(xml_rsc); } EXPAND_CONSTRAINT_IDREF(id, rsc_1, rid); } } if (pcmk__xe_attr_is_true(set2, PCMK_XA_SEQUENTIAL)) { if (symmetry == ordering_symmetric_inverse) { // Get the last one const char *rid = NULL; for (xml_rsc = pcmk__xe_first_child(set2, PCMK_XE_RESOURCE_REF, NULL, NULL); xml_rsc != NULL; xml_rsc = pcmk__xe_next(xml_rsc, PCMK_XE_RESOURCE_REF)) { rid = pcmk__xe_id(xml_rsc); } EXPAND_CONSTRAINT_IDREF(id, rsc_2, rid); } else { // Get the first one xml_rsc = pcmk__xe_first_child(set2, PCMK_XE_RESOURCE_REF, NULL, NULL); if (xml_rsc != NULL) { EXPAND_CONSTRAINT_IDREF(id, rsc_2, pcmk__xe_id(xml_rsc)); } } } if ((rsc_1 != NULL) && (rsc_2 != NULL)) { pcmk__order_resource_actions(rsc_1, action_1, rsc_2, action_2, flags); } else if (rsc_1 != NULL) { for (xml_rsc = pcmk__xe_first_child(set2, PCMK_XE_RESOURCE_REF, NULL, NULL); xml_rsc != NULL; xml_rsc = pcmk__xe_next(xml_rsc, PCMK_XE_RESOURCE_REF)) { EXPAND_CONSTRAINT_IDREF(id, rsc_2, pcmk__xe_id(xml_rsc)); pcmk__order_resource_actions(rsc_1, action_1, rsc_2, action_2, flags); } } else if (rsc_2 != NULL) { for (xml_rsc = pcmk__xe_first_child(set1, PCMK_XE_RESOURCE_REF, NULL, NULL); xml_rsc != NULL; xml_rsc = pcmk__xe_next(xml_rsc, PCMK_XE_RESOURCE_REF)) { EXPAND_CONSTRAINT_IDREF(id, rsc_1, pcmk__xe_id(xml_rsc)); pcmk__order_resource_actions(rsc_1, action_1, rsc_2, action_2, flags); } } else { for (xml_rsc = pcmk__xe_first_child(set1, PCMK_XE_RESOURCE_REF, NULL, NULL); xml_rsc != NULL; xml_rsc = pcmk__xe_next(xml_rsc, PCMK_XE_RESOURCE_REF)) { EXPAND_CONSTRAINT_IDREF(id, rsc_1, pcmk__xe_id(xml_rsc)); for (xmlNode *xml_rsc_2 = pcmk__xe_first_child(set2, PCMK_XE_RESOURCE_REF, NULL, NULL); xml_rsc_2 != NULL; xml_rsc_2 = pcmk__xe_next(xml_rsc_2, PCMK_XE_RESOURCE_REF)) { EXPAND_CONSTRAINT_IDREF(id, rsc_2, pcmk__xe_id(xml_rsc_2)); pcmk__order_resource_actions(rsc_1, action_1, rsc_2, action_2, flags); } } } return pcmk_rc_ok; } /*! * \internal * \brief If an ordering constraint uses resource tags, expand them * * \param[in,out] xml_obj Ordering constraint XML * \param[out] expanded_xml Equivalent XML with tags expanded * \param[in] scheduler Scheduler data * * \return Standard Pacemaker return code (specifically, pcmk_rc_ok on success, * and pcmk_rc_unpack_error on invalid configuration) */ static int unpack_order_tags(xmlNode *xml_obj, xmlNode **expanded_xml, const pcmk_scheduler_t *scheduler) { const char *id_first = NULL; const char *id_then = NULL; const char *action_first = NULL; const char *action_then = NULL; pcmk_resource_t *rsc_first = NULL; pcmk_resource_t *rsc_then = NULL; pcmk__idref_t *tag_first = NULL; pcmk__idref_t *tag_then = NULL; xmlNode *rsc_set_first = NULL; xmlNode *rsc_set_then = NULL; bool any_sets = false; // Check whether there are any resource sets with template or tag references *expanded_xml = pcmk__expand_tags_in_sets(xml_obj, scheduler); if (*expanded_xml != NULL) { - crm_log_xml_trace(*expanded_xml, "Expanded " PCMK_XE_RSC_ORDER); + pcmk__log_xml_trace(*expanded_xml, "Expanded " PCMK_XE_RSC_ORDER); return pcmk_rc_ok; } id_first = pcmk__xe_get(xml_obj, PCMK_XA_FIRST); id_then = pcmk__xe_get(xml_obj, PCMK_XA_THEN); if ((id_first == NULL) || (id_then == NULL)) { return pcmk_rc_ok; } if (!pcmk__valid_resource_or_tag(scheduler, id_first, &rsc_first, &tag_first)) { pcmk__config_err("Ignoring constraint '%s' because '%s' is not a " "valid resource or tag", pcmk__xe_id(xml_obj), id_first); return pcmk_rc_unpack_error; } if (!pcmk__valid_resource_or_tag(scheduler, id_then, &rsc_then, &tag_then)) { pcmk__config_err("Ignoring constraint '%s' because '%s' is not a " "valid resource or tag", pcmk__xe_id(xml_obj), id_then); return pcmk_rc_unpack_error; } if ((rsc_first != NULL) && (rsc_then != NULL)) { // Neither side references a template or tag return pcmk_rc_ok; } action_first = pcmk__xe_get(xml_obj, PCMK_XA_FIRST_ACTION); action_then = pcmk__xe_get(xml_obj, PCMK_XA_THEN_ACTION); *expanded_xml = pcmk__xml_copy(NULL, xml_obj); /* Convert template/tag reference in PCMK_XA_FIRST into constraint * PCMK_XE_RESOURCE_SET */ if (!pcmk__tag_to_set(*expanded_xml, &rsc_set_first, PCMK_XA_FIRST, true, scheduler)) { pcmk__xml_free(*expanded_xml); *expanded_xml = NULL; return pcmk_rc_unpack_error; } if (rsc_set_first != NULL) { if (action_first != NULL) { /* Move PCMK_XA_FIRST_ACTION into converted PCMK_XE_RESOURCE_SET as * PCMK_XA_ACTION */ pcmk__xe_set(rsc_set_first, PCMK_XA_ACTION, action_first); pcmk__xe_remove_attr(*expanded_xml, PCMK_XA_FIRST_ACTION); } any_sets = true; } /* Convert template/tag reference in PCMK_XA_THEN into constraint * PCMK_XE_RESOURCE_SET */ if (!pcmk__tag_to_set(*expanded_xml, &rsc_set_then, PCMK_XA_THEN, true, scheduler)) { pcmk__xml_free(*expanded_xml); *expanded_xml = NULL; return pcmk_rc_unpack_error; } if (rsc_set_then != NULL) { if (action_then != NULL) { /* Move PCMK_XA_THEN_ACTION into converted PCMK_XE_RESOURCE_SET as * PCMK_XA_ACTION */ pcmk__xe_set(rsc_set_then, PCMK_XA_ACTION, action_then); pcmk__xe_remove_attr(*expanded_xml, PCMK_XA_THEN_ACTION); } any_sets = true; } if (any_sets) { - crm_log_xml_trace(*expanded_xml, "Expanded " PCMK_XE_RSC_ORDER); + pcmk__log_xml_trace(*expanded_xml, "Expanded " PCMK_XE_RSC_ORDER); } else { pcmk__xml_free(*expanded_xml); *expanded_xml = NULL; } return pcmk_rc_ok; } /*! * \internal * \brief Unpack ordering constraint XML * * \param[in,out] xml_obj Ordering constraint XML to unpack * \param[in,out] scheduler Scheduler data */ void pcmk__unpack_ordering(xmlNode *xml_obj, pcmk_scheduler_t *scheduler) { xmlNode *set = NULL; xmlNode *last = NULL; xmlNode *orig_xml = NULL; xmlNode *expanded_xml = NULL; const char *id = pcmk__xe_get(xml_obj, PCMK_XA_ID); const char *invert = pcmk__xe_get(xml_obj, PCMK_XA_SYMMETRICAL); enum pe_order_kind kind = get_ordering_type(xml_obj); enum ordering_symmetry symmetry = get_ordering_symmetry(xml_obj, kind, NULL); // Expand any resource tags in the constraint XML if (unpack_order_tags(xml_obj, &expanded_xml, scheduler) != pcmk_rc_ok) { return; } if (expanded_xml != NULL) { orig_xml = xml_obj; xml_obj = expanded_xml; } // If the constraint has resource sets, unpack them for (set = pcmk__xe_first_child(xml_obj, PCMK_XE_RESOURCE_SET, NULL, NULL); set != NULL; set = pcmk__xe_next(set, PCMK_XE_RESOURCE_SET)) { set = pcmk__xe_resolve_idref(set, scheduler->input); if ((set == NULL) // Configuration error, message already logged || (unpack_order_set(set, kind, invert, scheduler) != pcmk_rc_ok)) { if (expanded_xml != NULL) { pcmk__xml_free(expanded_xml); } return; } if (last != NULL) { if (order_rsc_sets(id, last, set, kind, scheduler, symmetry) != pcmk_rc_ok) { if (expanded_xml != NULL) { pcmk__xml_free(expanded_xml); } return; } if ((symmetry == ordering_symmetric) && (order_rsc_sets(id, set, last, kind, scheduler, ordering_symmetric_inverse) != pcmk_rc_ok)) { if (expanded_xml != NULL) { pcmk__xml_free(expanded_xml); } return; } } last = set; } if (expanded_xml) { pcmk__xml_free(expanded_xml); xml_obj = orig_xml; } // If the constraint has no resource sets, unpack it as a simple ordering if (last == NULL) { return unpack_simple_rsc_order(xml_obj, scheduler); } } static bool ordering_is_invalid(pcmk_action_t *action, pcmk__related_action_t *input) { /* Prevent user-defined ordering constraints between resources * running in a guest node and the resource that defines that node. */ if (!pcmk__is_set(input->flags, pcmk__ar_guest_allowed) && (input->action->rsc != NULL) && pcmk__rsc_corresponds_to_guest(action->rsc, input->action->node)) { pcmk__config_warn("Invalid ordering constraint between %s and %s", input->action->rsc->id, action->rsc->id); return true; } /* If there's an order like * "rscB_stop node2"-> "load_stopped_node2" -> "rscA_migrate_to node1" * * then rscA is being migrated from node1 to node2, while rscB is being * migrated from node2 to node1. If there would be a graph loop, * break the order "load_stopped_node2" -> "rscA_migrate_to node1". */ if ((input->flags == pcmk__ar_if_on_same_node_or_target) && (action->rsc != NULL) && pcmk__str_eq(action->task, PCMK_ACTION_MIGRATE_TO, pcmk__str_none) && pcmk__graph_has_loop(action, action, input)) { return true; } return false; } void pcmk__disable_invalid_orderings(pcmk_scheduler_t *scheduler) { for (GList *iter = scheduler->priv->actions; iter != NULL; iter = iter->next) { pcmk_action_t *action = (pcmk_action_t *) iter->data; pcmk__related_action_t *input = NULL; for (GList *input_iter = action->actions_before; input_iter != NULL; input_iter = input_iter->next) { input = input_iter->data; if (ordering_is_invalid(action, input)) { input->flags = pcmk__ar_none; } } } } /*! * \internal * \brief Order stops on a node before the node's shutdown * * \param[in,out] node Node being shut down * \param[in] shutdown_op Shutdown action for node */ void pcmk__order_stops_before_shutdown(pcmk_node_t *node, pcmk_action_t *shutdown_op) { for (GList *iter = node->priv->scheduler->priv->actions; iter != NULL; iter = iter->next) { pcmk_action_t *action = (pcmk_action_t *) iter->data; // Only stops on the node shutting down are relevant if (!pcmk__same_node(action->node, node) || !pcmk__str_eq(action->task, PCMK_ACTION_STOP, pcmk__str_none)) { continue; } // Resources and nodes in maintenance mode won't be touched if (pcmk__is_set(action->rsc->flags, pcmk__rsc_maintenance)) { pcmk__rsc_trace(action->rsc, "Not ordering %s before shutdown of %s because " "resource in maintenance mode", action->uuid, pcmk__node_name(node)); continue; } else if (node->details->maintenance) { pcmk__rsc_trace(action->rsc, "Not ordering %s before shutdown of %s because " "node in maintenance mode", action->uuid, pcmk__node_name(node)); continue; } /* Don't touch a resource that is unmanaged or blocked, to avoid * blocking the shutdown (though if another action depends on this one, * we may still end up blocking) * * @TODO This "if" looks wrong, create a regression test for these cases */ if (!pcmk__any_flags_set(action->rsc->flags, pcmk__rsc_managed|pcmk__rsc_blocked)) { pcmk__rsc_trace(action->rsc, "Not ordering %s before shutdown of %s because " "resource is unmanaged or blocked", action->uuid, pcmk__node_name(node)); continue; } pcmk__rsc_trace(action->rsc, "Ordering %s before shutdown of %s", action->uuid, pcmk__node_name(node)); pcmk__clear_action_flags(action, pcmk__action_optional); pcmk__new_ordering(action->rsc, NULL, action, NULL, strdup(PCMK_ACTION_DO_SHUTDOWN), shutdown_op, pcmk__ar_ordered|pcmk__ar_unrunnable_first_blocks, node->priv->scheduler); } } /*! * \brief Find resource actions matching directly or as child * * \param[in] rsc Resource to check * \param[in] original_key Action key to search for (possibly referencing * parent of \rsc) * * \return Newly allocated list of matching actions * \note It is the caller's responsibility to free the result with g_list_free() */ static GList * find_actions_by_task(const pcmk_resource_t *rsc, const char *original_key) { // Search under given task key directly GList *list = find_actions(rsc->priv->actions, original_key, NULL); if (list == NULL) { // Search again using this resource's ID char *key = NULL; char *task = NULL; guint interval_ms = 0; CRM_CHECK(parse_op_key(original_key, NULL, &task, &interval_ms), return NULL); key = pcmk__op_key(rsc->id, task, interval_ms); list = find_actions(rsc->priv->actions, key, NULL); free(key); free(task); } return list; } /*! * \internal * \brief Order relevant resource actions after a given action * * \param[in,out] first_action Action to order after (or NULL if none runnable) * \param[in] rsc Resource whose actions should be ordered * \param[in,out] order Ordering constraint being applied */ static void order_resource_actions_after(pcmk_action_t *first_action, const pcmk_resource_t *rsc, pcmk__action_relation_t *order) { GList *then_actions = NULL; uint32_t flags = pcmk__ar_none; CRM_CHECK((rsc != NULL) && (order != NULL), return); flags = order->flags; pcmk__rsc_trace(rsc, "Applying ordering %d for 'then' resource %s", order->id, rsc->id); if (order->action2 != NULL) { then_actions = g_list_prepend(NULL, order->action2); } else { then_actions = find_actions_by_task(rsc, order->task2); } if (then_actions == NULL) { pcmk__rsc_trace(rsc, "Ignoring ordering %d: no %s actions found for %s", order->id, order->task2, rsc->id); return; } if ((first_action != NULL) && (first_action->rsc == rsc) && pcmk__is_set(first_action->flags, pcmk__action_migration_abort)) { pcmk__rsc_trace(rsc, "Detected dangling migration ordering (%s then %s %s)", first_action->uuid, order->task2, rsc->id); pcmk__clear_relation_flags(flags, pcmk__ar_first_implies_then); } if ((first_action == NULL) && !pcmk__is_set(flags, pcmk__ar_first_implies_then)) { pcmk__rsc_debug(rsc, "Ignoring ordering %d for %s: No first action found", order->id, rsc->id); g_list_free(then_actions); return; } for (GList *iter = then_actions; iter != NULL; iter = iter->next) { pcmk_action_t *then_action_iter = (pcmk_action_t *) iter->data; if (first_action != NULL) { order_actions(first_action, then_action_iter, flags); } else { pcmk__clear_action_flags(then_action_iter, pcmk__action_runnable); pcmk__warn("%s of %s is unrunnable because there is no %s of %s to " "order it after", then_action_iter->task, rsc->id, order->task1, order->rsc1->id); } } g_list_free(then_actions); } static void rsc_order_first(pcmk_resource_t *first_rsc, pcmk__action_relation_t *order) { GList *first_actions = NULL; pcmk_action_t *first_action = order->action1; pcmk_resource_t *then_rsc = order->rsc2; pcmk__assert(first_rsc != NULL); pcmk__rsc_trace(first_rsc, "Applying ordering constraint %d (first: %s)", order->id, first_rsc->id); if (first_action != NULL) { first_actions = g_list_prepend(NULL, first_action); } else { first_actions = find_actions_by_task(first_rsc, order->task1); } if ((first_actions == NULL) && (first_rsc == then_rsc)) { pcmk__rsc_trace(first_rsc, "Ignoring constraint %d: first (%s for %s) not found", order->id, order->task1, first_rsc->id); } else if (first_actions == NULL) { char *key = NULL; char *op_type = NULL; guint interval_ms = 0; enum rsc_role_e first_role; parse_op_key(order->task1, NULL, &op_type, &interval_ms); key = pcmk__op_key(first_rsc->id, op_type, interval_ms); first_role = first_rsc->priv->fns->state(first_rsc, true); if ((first_role == pcmk_role_stopped) && pcmk__str_eq(op_type, PCMK_ACTION_STOP, pcmk__str_none)) { free(key); pcmk__rsc_trace(first_rsc, "Ignoring constraint %d: first (%s for %s) " "not found", order->id, order->task1, first_rsc->id); } else if ((first_role == pcmk_role_unpromoted) && pcmk__str_eq(op_type, PCMK_ACTION_DEMOTE, pcmk__str_none)) { free(key); pcmk__rsc_trace(first_rsc, "Ignoring constraint %d: first (%s for %s) " "not found", order->id, order->task1, first_rsc->id); } else { pcmk__rsc_trace(first_rsc, "Creating first (%s for %s) for constraint %d ", order->task1, first_rsc->id, order->id); first_action = custom_action(first_rsc, key, op_type, NULL, TRUE, first_rsc->priv->scheduler); first_actions = g_list_prepend(NULL, first_action); } free(op_type); } if (then_rsc == NULL) { if (order->action2 == NULL) { pcmk__rsc_trace(first_rsc, "Ignoring constraint %d: then not found", order->id); return; } then_rsc = order->action2->rsc; } for (GList *iter = first_actions; iter != NULL; iter = iter->next) { first_action = iter->data; if (then_rsc == NULL) { order_actions(first_action, order->action2, order->flags); } else { order_resource_actions_after(first_action, then_rsc, order); } } g_list_free(first_actions); } // GFunc to call pcmk__block_colocation_dependents() static void block_colocation_dependents(gpointer data, gpointer user_data) { pcmk__block_colocation_dependents(data); } // GFunc to call pcmk__update_action_for_orderings() static void update_action_for_orderings(gpointer data, gpointer user_data) { pcmk__update_action_for_orderings((pcmk_action_t *) data, (pcmk_scheduler_t *) user_data); } /*! * \internal * \brief Apply all ordering constraints * * \param[in,out] sched Scheduler data */ void pcmk__apply_orderings(pcmk_scheduler_t *sched) { pcmk__trace("Applying ordering constraints"); /* Ordering constraints need to be processed in the order they were created. * rsc_order_first() and order_resource_actions_after() require the relevant * actions to already exist in some cases, but rsc_order_first() will create * the 'first' action in certain cases. Thus calling rsc_order_first() can * change the behavior of later-created orderings. * * Also, g_list_append() should be avoided for performance reasons, so we * prepend orderings when creating them and reverse the list here. * * @TODO This is brittle and should be carefully redesigned so that the * order of creation doesn't matter, and the reverse becomes unneeded. */ sched->priv->ordering_constraints = g_list_reverse(sched->priv->ordering_constraints); for (GList *iter = sched->priv->ordering_constraints; iter != NULL; iter = iter->next) { pcmk__action_relation_t *order = iter->data; pcmk_resource_t *rsc = order->rsc1; if (rsc != NULL) { rsc_order_first(rsc, order); continue; } rsc = order->rsc2; if (rsc != NULL) { order_resource_actions_after(order->action1, rsc, order); } else { pcmk__trace("Applying ordering constraint %d (non-resource " "actions)", order->id); order_actions(order->action1, order->action2, order->flags); } } g_list_foreach(sched->priv->actions, block_colocation_dependents, NULL); pcmk__trace("Ordering probes"); pcmk__order_probes(sched); pcmk__trace("Updating %u actions", g_list_length(sched->priv->actions)); g_list_foreach(sched->priv->actions, update_action_for_orderings, sched); pcmk__disable_invalid_orderings(sched); } /*! * \internal * \brief Order a given action after each action in a given list * * \param[in,out] after "After" action * \param[in,out] list List of "before" actions */ void pcmk__order_after_each(pcmk_action_t *after, GList *list) { const char *after_desc = (after->task == NULL)? after->uuid : after->task; for (GList *iter = list; iter != NULL; iter = iter->next) { pcmk_action_t *before = (pcmk_action_t *) iter->data; const char *before_desc = before->task? before->task : before->uuid; pcmk__debug("Ordering %s on %s before %s on %s", before_desc, pcmk__node_name(before->node), after_desc, pcmk__node_name(after->node)); order_actions(before, after, pcmk__ar_ordered); } } /*! * \internal * \brief Order promotions and demotions for restarts of a clone or bundle * * \param[in,out] rsc Clone or bundle to order */ void pcmk__promotable_restart_ordering(pcmk_resource_t *rsc) { // Order start and promote after all instances are stopped pcmk__order_resource_actions(rsc, PCMK_ACTION_STOPPED, rsc, PCMK_ACTION_START, pcmk__ar_ordered); pcmk__order_resource_actions(rsc, PCMK_ACTION_STOPPED, rsc, PCMK_ACTION_PROMOTE, pcmk__ar_ordered); // Order stop, start, and promote after all instances are demoted pcmk__order_resource_actions(rsc, PCMK_ACTION_DEMOTED, rsc, PCMK_ACTION_STOP, pcmk__ar_ordered); pcmk__order_resource_actions(rsc, PCMK_ACTION_DEMOTED, rsc, PCMK_ACTION_START, pcmk__ar_ordered); pcmk__order_resource_actions(rsc, PCMK_ACTION_DEMOTED, rsc, PCMK_ACTION_PROMOTE, pcmk__ar_ordered); // Order promote after all instances are started pcmk__order_resource_actions(rsc, PCMK_ACTION_RUNNING, rsc, PCMK_ACTION_PROMOTE, pcmk__ar_ordered); // Order demote after all instances are demoted pcmk__order_resource_actions(rsc, PCMK_ACTION_DEMOTE, rsc, PCMK_ACTION_DEMOTED, pcmk__ar_ordered); } diff --git a/lib/pacemaker/pcmk_sched_tickets.c b/lib/pacemaker/pcmk_sched_tickets.c index 0a9287cb22..aaf1bba77b 100644 --- a/lib/pacemaker/pcmk_sched_tickets.c +++ b/lib/pacemaker/pcmk_sched_tickets.c @@ -1,524 +1,524 @@ /* * Copyright 2004-2025 the Pacemaker project contributors * * The version control history for this file may have further details. * * This source code is licensed under the GNU General Public License version 2 * or later (GPLv2+) WITHOUT ANY WARRANTY. */ #include #include #include #include #include #include #include // PCMK_SCORE_INFINITY #include #include #include "libpacemaker_private.h" enum loss_ticket_policy { loss_ticket_stop, loss_ticket_demote, loss_ticket_fence, loss_ticket_freeze }; typedef struct { const char *id; pcmk_resource_t *rsc; pcmk__ticket_t *ticket; enum loss_ticket_policy loss_policy; int role; } rsc_ticket_t; /*! * \brief Check whether a ticket constraint matches a resource by role * * \param[in] rsc_ticket Ticket constraint * \param[in] rsc Resource to compare with ticket * * \param[in] true if constraint has no role or resource's role matches * constraint's, otherwise false */ static bool ticket_role_matches(const pcmk_resource_t *rsc, const rsc_ticket_t *rsc_ticket) { if ((rsc_ticket->role == pcmk_role_unknown) || (rsc_ticket->role == rsc->priv->orig_role)) { return true; } pcmk__rsc_trace(rsc, "Skipping constraint: \"%s\" state filter", pcmk_role_text(rsc_ticket->role)); return false; } /*! * \brief Create location constraints and fencing as needed for a ticket * * \param[in,out] rsc Resource affected by ticket * \param[in] rsc_ticket Ticket */ static void constraints_for_ticket(pcmk_resource_t *rsc, const rsc_ticket_t *rsc_ticket) { GList *iter = NULL; CRM_CHECK((rsc != NULL) && (rsc_ticket != NULL), return); if (pcmk__is_set(rsc_ticket->ticket->flags, pcmk__ticket_granted) && !pcmk__is_set(rsc_ticket->ticket->flags, pcmk__ticket_standby)) { return; } if (rsc->priv->children != NULL) { pcmk__rsc_trace(rsc, "Processing ticket dependencies from %s", rsc->id); for (iter = rsc->priv->children; iter != NULL; iter = iter->next) { constraints_for_ticket((pcmk_resource_t *) iter->data, rsc_ticket); } return; } pcmk__rsc_trace(rsc, "%s: Processing ticket dependency on %s (%s, %s)", rsc->id, rsc_ticket->ticket->id, rsc_ticket->id, pcmk_role_text(rsc_ticket->role)); if (!pcmk__is_set(rsc_ticket->ticket->flags, pcmk__ticket_granted) && (rsc->priv->active_nodes != NULL)) { switch (rsc_ticket->loss_policy) { case loss_ticket_stop: resource_location(rsc, NULL, -PCMK_SCORE_INFINITY, "__loss_of_ticket__", rsc->priv->scheduler); break; case loss_ticket_demote: // Promotion score will be set to -INFINITY in promotion_order() if (rsc_ticket->role != pcmk_role_promoted) { resource_location(rsc, NULL, -PCMK_SCORE_INFINITY, "__loss_of_ticket__", rsc->priv->scheduler); } break; case loss_ticket_fence: if (!ticket_role_matches(rsc, rsc_ticket)) { return; } resource_location(rsc, NULL, -PCMK_SCORE_INFINITY, "__loss_of_ticket__", rsc->priv->scheduler); for (iter = rsc->priv->active_nodes; iter != NULL; iter = iter->next) { pe_fence_node(rsc->priv->scheduler, (pcmk_node_t *) iter->data, "deadman ticket was lost", FALSE); } break; case loss_ticket_freeze: if (!ticket_role_matches(rsc, rsc_ticket)) { return; } if (rsc->priv->active_nodes != NULL) { pcmk__clear_rsc_flags(rsc, pcmk__rsc_managed); pcmk__set_rsc_flags(rsc, pcmk__rsc_blocked); } break; } } else if (!pcmk__is_set(rsc_ticket->ticket->flags, pcmk__ticket_granted)) { if ((rsc_ticket->role != pcmk_role_promoted) || (rsc_ticket->loss_policy == loss_ticket_stop)) { resource_location(rsc, NULL, -PCMK_SCORE_INFINITY, "__no_ticket__", rsc->priv->scheduler); } } else if (pcmk__is_set(rsc_ticket->ticket->flags, pcmk__ticket_standby)) { if ((rsc_ticket->role != pcmk_role_promoted) || (rsc_ticket->loss_policy == loss_ticket_stop)) { resource_location(rsc, NULL, -PCMK_SCORE_INFINITY, "__ticket_standby__", rsc->priv->scheduler); } } } static void rsc_ticket_new(const char *id, pcmk_resource_t *rsc, pcmk__ticket_t *ticket, const char *role_spec, const char *loss_policy) { rsc_ticket_t *new_rsc_ticket = NULL; enum rsc_role_e role = pcmk_role_unknown; if (rsc == NULL) { pcmk__config_err("Ignoring ticket '%s' because resource " "does not exist", id); return; } if (pcmk__parse_constraint_role(id, role_spec, &role) != pcmk_rc_ok) { // Not possible with schema validation enabled (error already logged) return; } new_rsc_ticket = pcmk__assert_alloc(1, sizeof(rsc_ticket_t)); new_rsc_ticket->id = id; new_rsc_ticket->ticket = ticket; new_rsc_ticket->rsc = rsc; new_rsc_ticket->role = role; if (pcmk__str_eq(loss_policy, PCMK_VALUE_FENCE, pcmk__str_casei)) { if (pcmk__is_set(rsc->priv->scheduler->flags, pcmk__sched_fencing_enabled)) { new_rsc_ticket->loss_policy = loss_ticket_fence; } else { pcmk__config_err("Resetting '" PCMK_XA_LOSS_POLICY "' " "for ticket '%s' to '" PCMK_VALUE_STOP "' " "because fencing is not configured", ticket->id); loss_policy = PCMK_VALUE_STOP; } } if (new_rsc_ticket->loss_policy == loss_ticket_fence) { pcmk__debug("On loss of ticket '%s': Fence the nodes running %s (%s)", new_rsc_ticket->ticket->id, new_rsc_ticket->rsc->id, pcmk_role_text(new_rsc_ticket->role)); } else if (pcmk__str_eq(loss_policy, PCMK_VALUE_FREEZE, pcmk__str_casei)) { pcmk__debug("On loss of ticket '%s': Freeze %s (%s)", new_rsc_ticket->ticket->id, new_rsc_ticket->rsc->id, pcmk_role_text(new_rsc_ticket->role)); new_rsc_ticket->loss_policy = loss_ticket_freeze; } else if (pcmk__str_eq(loss_policy, PCMK_VALUE_DEMOTE, pcmk__str_casei)) { pcmk__debug("On loss of ticket '%s': Demote %s (%s)", new_rsc_ticket->ticket->id, new_rsc_ticket->rsc->id, pcmk_role_text(new_rsc_ticket->role)); new_rsc_ticket->loss_policy = loss_ticket_demote; } else if (pcmk__str_eq(loss_policy, PCMK_VALUE_STOP, pcmk__str_casei)) { pcmk__debug("On loss of ticket '%s': Stop %s (%s)", new_rsc_ticket->ticket->id, new_rsc_ticket->rsc->id, pcmk_role_text(new_rsc_ticket->role)); new_rsc_ticket->loss_policy = loss_ticket_stop; } else { if (new_rsc_ticket->role == pcmk_role_promoted) { pcmk__debug("On loss of ticket '%s': Default to demote %s (%s)", new_rsc_ticket->ticket->id, new_rsc_ticket->rsc->id, pcmk_role_text(new_rsc_ticket->role)); new_rsc_ticket->loss_policy = loss_ticket_demote; } else { pcmk__debug("On loss of ticket '%s': Default to stop %s (%s)", new_rsc_ticket->ticket->id, new_rsc_ticket->rsc->id, pcmk_role_text(new_rsc_ticket->role)); new_rsc_ticket->loss_policy = loss_ticket_stop; } } pcmk__rsc_trace(rsc, "%s (%s) ==> %s", rsc->id, pcmk_role_text(new_rsc_ticket->role), ticket->id); rsc->priv->ticket_constraints = g_list_append(rsc->priv->ticket_constraints, new_rsc_ticket); if (!pcmk__is_set(new_rsc_ticket->ticket->flags, pcmk__ticket_granted) || pcmk__is_set(new_rsc_ticket->ticket->flags, pcmk__ticket_standby)) { constraints_for_ticket(rsc, new_rsc_ticket); } } // \return Standard Pacemaker return code static int unpack_rsc_ticket_set(xmlNode *set, pcmk__ticket_t *ticket, const char *loss_policy, pcmk_scheduler_t *scheduler) { const char *set_id = NULL; const char *role = NULL; CRM_CHECK(set != NULL, return EINVAL); CRM_CHECK(ticket != NULL, return EINVAL); set_id = pcmk__xe_id(set); if (set_id == NULL) { pcmk__config_err("Ignoring <" PCMK_XE_RESOURCE_SET "> without " PCMK_XA_ID); return pcmk_rc_unpack_error; } role = pcmk__xe_get(set, PCMK_XA_ROLE); for (xmlNode *xml_rsc = pcmk__xe_first_child(set, PCMK_XE_RESOURCE_REF, NULL, NULL); xml_rsc != NULL; xml_rsc = pcmk__xe_next(xml_rsc, PCMK_XE_RESOURCE_REF)) { pcmk_resource_t *resource = NULL; resource = pcmk__find_constraint_resource(scheduler->priv->resources, pcmk__xe_id(xml_rsc)); if (resource == NULL) { pcmk__config_err("%s: No resource found for %s", set_id, pcmk__xe_id(xml_rsc)); return pcmk_rc_unpack_error; } pcmk__rsc_trace(resource, "Resource '%s' depends on ticket '%s'", resource->id, ticket->id); rsc_ticket_new(set_id, resource, ticket, role, loss_policy); } return pcmk_rc_ok; } static void unpack_simple_rsc_ticket(xmlNode *xml_obj, pcmk_scheduler_t *scheduler) { const char *id = NULL; const char *ticket_str = pcmk__xe_get(xml_obj, PCMK_XA_TICKET); const char *loss_policy = pcmk__xe_get(xml_obj, PCMK_XA_LOSS_POLICY); pcmk__ticket_t *ticket = NULL; const char *rsc_id = pcmk__xe_get(xml_obj, PCMK_XA_RSC); const char *state = pcmk__xe_get(xml_obj, PCMK_XA_RSC_ROLE); pcmk_resource_t *rsc = NULL; CRM_CHECK(xml_obj != NULL, return); id = pcmk__xe_id(xml_obj); if (id == NULL) { pcmk__config_err("Ignoring <%s> constraint without " PCMK_XA_ID, xml_obj->name); return; } if (ticket_str == NULL) { pcmk__config_err("Ignoring constraint '%s' without ticket specified", id); return; } else { ticket = g_hash_table_lookup(scheduler->priv->ticket_constraints, ticket_str); } if (ticket == NULL) { pcmk__config_err("Ignoring constraint '%s' because ticket '%s' " "does not exist", id, ticket_str); return; } if (rsc_id == NULL) { pcmk__config_err("Ignoring constraint '%s' without resource", id); return; } else { rsc = pcmk__find_constraint_resource(scheduler->priv->resources, rsc_id); } if (rsc == NULL) { pcmk__config_err("Ignoring constraint '%s' because resource '%s' " "does not exist", id, rsc_id); return; } rsc_ticket_new(id, rsc, ticket, state, loss_policy); } // \return Standard Pacemaker return code static int unpack_rsc_ticket_tags(xmlNode *xml_obj, xmlNode **expanded_xml, pcmk_scheduler_t *scheduler) { const char *id = NULL; const char *rsc_id = NULL; const char *state = NULL; pcmk_resource_t *rsc = NULL; pcmk__idref_t *tag = NULL; xmlNode *rsc_set = NULL; *expanded_xml = NULL; CRM_CHECK(xml_obj != NULL, return EINVAL); id = pcmk__xe_id(xml_obj); if (id == NULL) { pcmk__config_err("Ignoring <%s> constraint without " PCMK_XA_ID, xml_obj->name); return pcmk_rc_unpack_error; } // Check whether there are any resource sets with template or tag references *expanded_xml = pcmk__expand_tags_in_sets(xml_obj, scheduler); if (*expanded_xml != NULL) { - crm_log_xml_trace(*expanded_xml, "Expanded rsc_ticket"); + pcmk__log_xml_trace(*expanded_xml, "Expanded rsc_ticket"); return pcmk_rc_ok; } rsc_id = pcmk__xe_get(xml_obj, PCMK_XA_RSC); if (rsc_id == NULL) { return pcmk_rc_ok; } if (!pcmk__valid_resource_or_tag(scheduler, rsc_id, &rsc, &tag)) { pcmk__config_err("Ignoring constraint '%s' because '%s' is not a " "valid resource or tag", id, rsc_id); return pcmk_rc_unpack_error; } else if (rsc != NULL) { // No template or tag is referenced return pcmk_rc_ok; } state = pcmk__xe_get(xml_obj, PCMK_XA_RSC_ROLE); *expanded_xml = pcmk__xml_copy(NULL, xml_obj); /* Convert any template or tag reference in "rsc" into ticket * PCMK_XE_RESOURCE_SET */ if (!pcmk__tag_to_set(*expanded_xml, &rsc_set, PCMK_XA_RSC, false, scheduler)) { pcmk__xml_free(*expanded_xml); *expanded_xml = NULL; return pcmk_rc_unpack_error; } if (rsc_set != NULL) { if (state != NULL) { /* Move PCMK_XA_RSC_ROLE into converted PCMK_XE_RESOURCE_SET as a * PCMK_XA_ROLE attribute */ pcmk__xe_set(rsc_set, PCMK_XA_ROLE, state); pcmk__xe_remove_attr(*expanded_xml, PCMK_XA_RSC_ROLE); } } else { pcmk__xml_free(*expanded_xml); *expanded_xml = NULL; } return pcmk_rc_ok; } void pcmk__unpack_rsc_ticket(xmlNode *xml_obj, pcmk_scheduler_t *scheduler) { xmlNode *set = NULL; bool any_sets = false; const char *id = NULL; const char *ticket_str = NULL; pcmk__ticket_t *ticket = NULL; xmlNode *orig_xml = NULL; xmlNode *expanded_xml = NULL; CRM_CHECK(xml_obj != NULL, return); id = pcmk__xe_id(xml_obj); if (id == NULL) { pcmk__config_err("Ignoring <%s> constraint without " PCMK_XA_ID, xml_obj->name); return; } if (scheduler->priv->ticket_constraints == NULL) { scheduler->priv->ticket_constraints = pcmk__strkey_table(free, destroy_ticket); } ticket_str = pcmk__xe_get(xml_obj, PCMK_XA_TICKET); if (ticket_str == NULL) { pcmk__config_err("Ignoring constraint '%s' without ticket", id); return; } else { ticket = g_hash_table_lookup(scheduler->priv->ticket_constraints, ticket_str); } if (ticket == NULL) { ticket = ticket_new(ticket_str, scheduler); if (ticket == NULL) { return; } } if (unpack_rsc_ticket_tags(xml_obj, &expanded_xml, scheduler) != pcmk_rc_ok) { return; } if (expanded_xml != NULL) { orig_xml = xml_obj; xml_obj = expanded_xml; } for (set = pcmk__xe_first_child(xml_obj, PCMK_XE_RESOURCE_SET, NULL, NULL); set != NULL; set = pcmk__xe_next(set, PCMK_XE_RESOURCE_SET)) { const char *loss_policy = NULL; any_sets = true; set = pcmk__xe_resolve_idref(set, scheduler->input); loss_policy = pcmk__xe_get(xml_obj, PCMK_XA_LOSS_POLICY); if ((set == NULL) // Configuration error, message already logged || (unpack_rsc_ticket_set(set, ticket, loss_policy, scheduler) != pcmk_rc_ok)) { if (expanded_xml != NULL) { pcmk__xml_free(expanded_xml); } return; } } if (expanded_xml) { pcmk__xml_free(expanded_xml); xml_obj = orig_xml; } if (!any_sets) { unpack_simple_rsc_ticket(xml_obj, scheduler); } } /*! * \internal * \brief Ban resource from a node if it doesn't have a promotion ticket * * If a resource has tickets for the promoted role, and the ticket is either not * granted or set to standby, then ban the resource from all nodes. * * \param[in,out] rsc Resource to check */ void pcmk__require_promotion_tickets(pcmk_resource_t *rsc) { for (GList *item = rsc->priv->ticket_constraints; item != NULL; item = item->next) { rsc_ticket_t *rsc_ticket = (rsc_ticket_t *) item->data; if ((rsc_ticket->role == pcmk_role_promoted) && (!pcmk__is_set(rsc_ticket->ticket->flags, pcmk__ticket_granted) || pcmk__is_set(rsc_ticket->ticket->flags, pcmk__ticket_standby))) { resource_location(rsc, NULL, -PCMK_SCORE_INFINITY, "__stateful_without_ticket__", rsc->priv->scheduler); } } } diff --git a/lib/pengine/complex.c b/lib/pengine/complex.c index 80e3c1ce7f..30e8fa75ce 100644 --- a/lib/pengine/complex.c +++ b/lib/pengine/complex.c @@ -1,1264 +1,1264 @@ /* * Copyright 2004-2025 the Pacemaker project contributors * * The version control history for this file may have further details. * * This source code is licensed under the GNU Lesser General Public License * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. */ #include #include // xmlNode #include #include // pcmk_parse_score() #include #include #include #include "pe_status_private.h" void populate_hash(xmlNode * nvpair_list, GHashTable * hash, const char **attrs, int attrs_length); static pcmk_node_t *active_node(const pcmk_resource_t *rsc, unsigned int *count_all, unsigned int *count_clean); static pcmk__rsc_methods_t resource_class_functions[] = { { native_unpack, native_find_rsc, native_active, native_resource_state, native_location, native_free, pe__count_common, pe__native_is_filtered, active_node, pe__primitive_max_per_node, }, { group_unpack, native_find_rsc, group_active, group_resource_state, native_location, group_free, pe__count_common, pe__group_is_filtered, active_node, pe__group_max_per_node, }, { clone_unpack, native_find_rsc, clone_active, clone_resource_state, native_location, clone_free, pe__count_common, pe__clone_is_filtered, active_node, pe__clone_max_per_node, }, { pe__unpack_bundle, native_find_rsc, pe__bundle_active, pe__bundle_resource_state, native_location, pe__free_bundle, pe__count_bundle, pe__bundle_is_filtered, pe__bundle_active_node, pe__bundle_max_per_node, } }; static enum pcmk__rsc_variant get_resource_type(const char *name) { if (pcmk__str_eq(name, PCMK_XE_PRIMITIVE, pcmk__str_casei)) { return pcmk__rsc_variant_primitive; } else if (pcmk__str_eq(name, PCMK_XE_GROUP, pcmk__str_casei)) { return pcmk__rsc_variant_group; } else if (pcmk__str_eq(name, PCMK_XE_CLONE, pcmk__str_casei)) { return pcmk__rsc_variant_clone; } else if (pcmk__str_eq(name, PCMK_XE_BUNDLE, pcmk__str_casei)) { return pcmk__rsc_variant_bundle; } return pcmk__rsc_variant_unknown; } /*! * \internal * \brief Insert a meta-attribute if not already present * * \param[in] key Meta-attribute name * \param[in] value Meta-attribute value to add if not already present * \param[in,out] table Meta-attribute hash table to insert into * * \note This is like pcmk__insert_meta() except it won't overwrite existing * values. */ static void dup_attr(gpointer key, gpointer value, gpointer user_data) { GHashTable *table = user_data; CRM_CHECK((key != NULL) && (table != NULL), return); if (pcmk__str_eq((const char *) value, "#default", pcmk__str_casei)) { // @COMPAT Deprecated since 2.1.8 pcmk__config_warn("Support for setting meta-attributes (such as %s) to " "the explicit value '#default' is deprecated and " "will be removed in a future release", (const char *) key); } else if ((value != NULL) && (g_hash_table_lookup(table, key) == NULL)) { pcmk__insert_dup(table, (const char *) key, (const char *) value); } } static void expand_parents_fixed_nvpairs(pcmk_resource_t *rsc, const pcmk_rule_input_t *rule_input, GHashTable *meta_hash, pcmk_scheduler_t *scheduler) { GHashTable *parent_orig_meta = pcmk__strkey_table(free, free); pcmk_resource_t *p = rsc->priv->parent; if (p == NULL) { return ; } /* Search all parent resources, get the fixed value of * PCMK_XE_META_ATTRIBUTES set only in the original xml, and stack it in the * hash table. The fixed value of the lower parent resource takes precedence * and is not overwritten. */ while(p != NULL) { /* A hash table for comparison is generated, including the id-ref. */ pe__unpack_dataset_nvpairs(p->priv->xml, PCMK_XE_META_ATTRIBUTES, rule_input, parent_orig_meta, NULL, scheduler); p = p->priv->parent; } if (parent_orig_meta != NULL) { // This will not overwrite any values already existing for child g_hash_table_foreach(parent_orig_meta, dup_attr, meta_hash); } if (parent_orig_meta != NULL) { g_hash_table_destroy(parent_orig_meta); } return ; } /* * \brief Get fully evaluated resource meta-attributes * * \param[in,out] meta_hash Where to store evaluated meta-attributes * \param[in] rsc Resource to get meta-attributes for * \param[in] node Ignored * \param[in,out] scheduler Scheduler data */ void get_meta_attributes(GHashTable * meta_hash, pcmk_resource_t * rsc, pcmk_node_t *node, pcmk_scheduler_t *scheduler) { const pcmk_rule_input_t rule_input = { .now = scheduler->priv->now, .rsc_standard = pcmk__xe_get(rsc->priv->xml, PCMK_XA_CLASS), .rsc_provider = pcmk__xe_get(rsc->priv->xml, PCMK_XA_PROVIDER), .rsc_agent = pcmk__xe_get(rsc->priv->xml, PCMK_XA_TYPE) }; for (xmlAttrPtr a = pcmk__xe_first_attr(rsc->priv->xml); a != NULL; a = a->next) { if (a->children != NULL) { dup_attr((gpointer) a->name, (gpointer) a->children->content, meta_hash); } } pe__unpack_dataset_nvpairs(rsc->priv->xml, PCMK_XE_META_ATTRIBUTES, &rule_input, meta_hash, NULL, scheduler); /* Set the PCMK_XE_META_ATTRIBUTES explicitly set in the parent resource to * the hash table of the child resource. If it is already explicitly set as * a child, it will not be overwritten. */ if (rsc->priv->parent != NULL) { expand_parents_fixed_nvpairs(rsc, &rule_input, meta_hash, scheduler); } /* check the defaults */ pe__unpack_dataset_nvpairs(scheduler->priv->rsc_defaults, PCMK_XE_META_ATTRIBUTES, &rule_input, meta_hash, NULL, scheduler); /* If there is PCMK_XE_META_ATTRIBUTES that the parent resource has not * explicitly set, set a value that is not set from PCMK_XE_RSC_DEFAULTS * either. The values already set up to this point will not be overwritten. */ if (rsc->priv->parent != NULL) { g_hash_table_foreach(rsc->priv->parent->priv->meta, dup_attr, meta_hash); } } /*! * \brief Get final values of a resource's instance attributes * * \param[in,out] instance_attrs Where to store the instance attributes * \param[in] rsc Resource to get instance attributes for * \param[in] node If not NULL, evaluate rules for this node * \param[in,out] scheduler Scheduler data */ void get_rsc_attributes(GHashTable *instance_attrs, const pcmk_resource_t *rsc, const pcmk_node_t *node, pcmk_scheduler_t *scheduler) { pcmk_rule_input_t rule_input = { .now = NULL, }; CRM_CHECK((instance_attrs != NULL) && (rsc != NULL) && (scheduler != NULL), return); rule_input.now = scheduler->priv->now; if (node != NULL) { rule_input.node_attrs = node->priv->attrs; } // Evaluate resource's own values, then its ancestors' values pe__unpack_dataset_nvpairs(rsc->priv->xml, PCMK_XE_INSTANCE_ATTRIBUTES, &rule_input, instance_attrs, NULL, scheduler); if (rsc->priv->parent != NULL) { get_rsc_attributes(instance_attrs, rsc->priv->parent, node, scheduler); } } static char * template_op_key(xmlNode * op) { const char *name = pcmk__xe_get(op, PCMK_XA_NAME); const char *role = pcmk__xe_get(op, PCMK_XA_ROLE); char *key = NULL; if ((role == NULL) || pcmk__strcase_any_of(role, PCMK_ROLE_STARTED, PCMK_ROLE_UNPROMOTED, PCMK__ROLE_UNPROMOTED_LEGACY, NULL)) { role = PCMK__ROLE_UNKNOWN; } key = pcmk__assert_asprintf("%s-%s", name, role); return key; } static gboolean unpack_template(xmlNode *xml_obj, xmlNode **expanded_xml, pcmk_scheduler_t *scheduler) { xmlNode *cib_resources = NULL; xmlNode *template = NULL; xmlNode *new_xml = NULL; xmlNode *child_xml = NULL; xmlNode *rsc_ops = NULL; xmlNode *template_ops = NULL; const char *template_ref = NULL; const char *id = NULL; if (xml_obj == NULL) { pcmk__config_err("No resource object for template unpacking"); return FALSE; } template_ref = pcmk__xe_get(xml_obj, PCMK_XA_TEMPLATE); if (template_ref == NULL) { return TRUE; } id = pcmk__xe_id(xml_obj); if (id == NULL) { pcmk__config_err("'%s' object must have a id", xml_obj->name); return FALSE; } if (pcmk__str_eq(template_ref, id, pcmk__str_none)) { pcmk__config_err("The resource object '%s' should not reference itself", id); return FALSE; } cib_resources = pcmk__xpath_find_one(scheduler->input->doc, "//" PCMK_XE_RESOURCES, PCMK__LOG_TRACE); if (cib_resources == NULL) { pcmk__config_err("No resources configured"); return FALSE; } template = pcmk__xe_first_child(cib_resources, PCMK_XE_TEMPLATE, PCMK_XA_ID, template_ref); if (template == NULL) { pcmk__config_err("No template named '%s'", template_ref); return FALSE; } new_xml = pcmk__xml_copy(NULL, template); xmlNodeSetName(new_xml, xml_obj->name); pcmk__xe_set(new_xml, PCMK_XA_ID, id); pcmk__xe_set(new_xml, PCMK__META_CLONE, pcmk__xe_get(xml_obj, PCMK__META_CLONE)); template_ops = pcmk__xe_first_child(new_xml, PCMK_XE_OPERATIONS, NULL, NULL); for (child_xml = pcmk__xe_first_child(xml_obj, NULL, NULL, NULL); child_xml != NULL; child_xml = pcmk__xe_next(child_xml, NULL)) { xmlNode *new_child = pcmk__xml_copy(new_xml, child_xml); if (pcmk__xe_is(new_child, PCMK_XE_OPERATIONS)) { rsc_ops = new_child; } } if (template_ops && rsc_ops) { xmlNode *op = NULL; GHashTable *rsc_ops_hash = pcmk__strkey_table(free, NULL); for (op = pcmk__xe_first_child(rsc_ops, NULL, NULL, NULL); op != NULL; op = pcmk__xe_next(op, NULL)) { char *key = template_op_key(op); g_hash_table_insert(rsc_ops_hash, key, op); } for (op = pcmk__xe_first_child(template_ops, NULL, NULL, NULL); op != NULL; op = pcmk__xe_next(op, NULL)) { char *key = template_op_key(op); if (g_hash_table_lookup(rsc_ops_hash, key) == NULL) { pcmk__xml_copy(rsc_ops, op); } free(key); } if (rsc_ops_hash) { g_hash_table_destroy(rsc_ops_hash); } pcmk__xml_free(template_ops); } /*pcmk__xml_free(*expanded_xml); */ *expanded_xml = new_xml; #if 0 /* Disable multi-level templates for now */ if (!unpack_template(new_xml, expanded_xml, scheduler)) { pcmk__xml_free(*expanded_xml); *expanded_xml = NULL; return FALSE; } #endif return TRUE; } static gboolean add_template_rsc(xmlNode *xml_obj, pcmk_scheduler_t *scheduler) { const char *template_ref = NULL; const char *id = NULL; if (xml_obj == NULL) { pcmk__config_err("No resource object for processing resource list " "of template"); return FALSE; } template_ref = pcmk__xe_get(xml_obj, PCMK_XA_TEMPLATE); if (template_ref == NULL) { return TRUE; } id = pcmk__xe_id(xml_obj); if (id == NULL) { pcmk__config_err("'%s' object must have a id", xml_obj->name); return FALSE; } if (pcmk__str_eq(template_ref, id, pcmk__str_none)) { pcmk__config_err("The resource object '%s' should not reference itself", id); return FALSE; } pcmk__add_idref(scheduler->priv->templates, template_ref, id); return TRUE; } /*! * \internal * \brief Check whether a clone or instance being unpacked is globally unique * * \param[in] rsc Clone or clone instance to check * * \return \c true if \p rsc is globally unique according to its * meta-attributes, otherwise \c false */ static bool detect_unique(const pcmk_resource_t *rsc) { const char *value = g_hash_table_lookup(rsc->priv->meta, PCMK_META_GLOBALLY_UNIQUE); if (value == NULL) { // Default to true if clone-node-max > 1 value = g_hash_table_lookup(rsc->priv->meta, PCMK_META_CLONE_NODE_MAX); if (value != NULL) { int node_max = 1; if ((pcmk__scan_min_int(value, &node_max, 0) == pcmk_rc_ok) && (node_max > 1)) { return true; } } return false; } return pcmk__is_true(value); } static void free_params_table(gpointer data) { g_hash_table_destroy((GHashTable *) data); } /*! * \brief Get a table of resource parameters * * \param[in,out] rsc Resource to query * \param[in] node Node for evaluating rules (NULL for defaults) * \param[in,out] scheduler Scheduler data * * \return Hash table containing resource parameter names and values * (or NULL if \p rsc or \p scheduler is NULL) * \note The returned table will be destroyed when the resource is freed, so * callers should not destroy it. */ GHashTable * pe_rsc_params(pcmk_resource_t *rsc, const pcmk_node_t *node, pcmk_scheduler_t *scheduler) { GHashTable *params_on_node = NULL; /* A NULL node is used to request the resource's default parameters * (not evaluated for node), but we always want something non-NULL * as a hash table key. */ const char *node_name = ""; // Sanity check if ((rsc == NULL) || (scheduler == NULL)) { return NULL; } if ((node != NULL) && (node->priv->name != NULL)) { node_name = node->priv->name; } // Find the parameter table for given node if (rsc->priv->parameter_cache == NULL) { rsc->priv->parameter_cache = pcmk__strikey_table(free, free_params_table); } else { params_on_node = g_hash_table_lookup(rsc->priv->parameter_cache, node_name); } // If none exists yet, create one with parameters evaluated for node if (params_on_node == NULL) { params_on_node = pcmk__strkey_table(free, free); get_rsc_attributes(params_on_node, rsc, node, scheduler); g_hash_table_insert(rsc->priv->parameter_cache, strdup(node_name), params_on_node); } return params_on_node; } /*! * \internal * \brief Unpack a resource's \c PCMK_META_REQUIRES meta-attribute * * \param[in,out] rsc Resource being unpacked * \param[in] value Value of \c PCMK_META_REQUIRES meta-attribute * \param[in] is_default Whether \p value was selected by default */ static void unpack_requires(pcmk_resource_t *rsc, const char *value, bool is_default) { const pcmk_scheduler_t *scheduler = rsc->priv->scheduler; if (pcmk__str_eq(value, PCMK_VALUE_NOTHING, pcmk__str_casei)) { } else if (pcmk__str_eq(value, PCMK_VALUE_QUORUM, pcmk__str_casei)) { pcmk__set_rsc_flags(rsc, pcmk__rsc_needs_quorum); } else if (pcmk__str_eq(value, PCMK_VALUE_FENCING, pcmk__str_casei)) { pcmk__set_rsc_flags(rsc, pcmk__rsc_needs_fencing); if (!pcmk__is_set(scheduler->flags, pcmk__sched_fencing_enabled)) { pcmk__config_warn("%s requires fencing but fencing is disabled", rsc->id); } } else if (pcmk__str_eq(value, PCMK_VALUE_UNFENCING, pcmk__str_casei)) { if (pcmk__is_set(rsc->flags, pcmk__rsc_fence_device)) { pcmk__config_warn("Resetting \"" PCMK_META_REQUIRES "\" for %s " "to \"" PCMK_VALUE_QUORUM "\" because fencing " "devices cannot require unfencing", rsc->id); unpack_requires(rsc, PCMK_VALUE_QUORUM, true); return; } if (!pcmk__is_set(scheduler->flags, pcmk__sched_fencing_enabled)) { pcmk__config_warn("Resetting \"" PCMK_META_REQUIRES "\" for %s " "to \"" PCMK_VALUE_QUORUM "\" because fencing is " "disabled", rsc->id); unpack_requires(rsc, PCMK_VALUE_QUORUM, true); return; } pcmk__set_rsc_flags(rsc, pcmk__rsc_needs_fencing|pcmk__rsc_needs_unfencing); } else { const char *orig_value = value; if (pcmk__is_set(rsc->flags, pcmk__rsc_fence_device)) { value = PCMK_VALUE_QUORUM; } else if (pcmk__is_primitive(rsc) && xml_contains_remote_node(rsc->priv->xml)) { value = PCMK_VALUE_QUORUM; } else if (pcmk__is_set(scheduler->flags, pcmk__sched_enable_unfencing)) { value = PCMK_VALUE_UNFENCING; } else if (pcmk__is_set(scheduler->flags, pcmk__sched_fencing_enabled)) { value = PCMK_VALUE_FENCING; } else if (scheduler->no_quorum_policy == pcmk_no_quorum_ignore) { value = PCMK_VALUE_NOTHING; } else { value = PCMK_VALUE_QUORUM; } if (orig_value != NULL) { pcmk__config_err("Resetting '" PCMK_META_REQUIRES "' for %s " "to '%s' because '%s' is not valid", rsc->id, value, orig_value); } unpack_requires(rsc, value, true); return; } pcmk__rsc_trace(rsc, "\tRequired to start: %s%s", value, (is_default? " (default)" : "")); } /*! * \internal * \brief Parse resource priority from meta-attribute * * \param[in,out] rsc Resource being unpacked */ static void unpack_priority(pcmk_resource_t *rsc) { const char *value = g_hash_table_lookup(rsc->priv->meta, PCMK_META_PRIORITY); int rc = pcmk_parse_score(value, &(rsc->priv->priority), 0); if (rc != pcmk_rc_ok) { pcmk__config_warn("Using default (0) for resource %s " PCMK_META_PRIORITY " because '%s' is not a valid value: %s", rsc->id, value, pcmk_rc_str(rc)); } } /*! * \internal * \brief Parse resource stickiness from meta-attribute * * \param[in,out] rsc Resource being unpacked */ static void unpack_stickiness(pcmk_resource_t *rsc) { const char *value = g_hash_table_lookup(rsc->priv->meta, PCMK_META_RESOURCE_STICKINESS); if (pcmk__str_eq(value, PCMK_VALUE_DEFAULT, pcmk__str_casei)) { // @COMPAT Deprecated since 2.1.8 pcmk__config_warn("Support for setting " PCMK_META_RESOURCE_STICKINESS " to the explicit value '" PCMK_VALUE_DEFAULT "' is deprecated and will be removed in a " "future release (just leave it unset)"); } else { int rc = pcmk_parse_score(value, &(rsc->priv->stickiness), 0); if (rc != pcmk_rc_ok) { pcmk__config_warn("Using default (0) for resource %s " PCMK_META_RESOURCE_STICKINESS " because '%s' is not a valid value: %s", rsc->id, value, pcmk_rc_str(rc)); } } } /*! * \internal * \brief Parse resource migration threshold from meta-attribute * * \param[in,out] rsc Resource being unpacked */ static void unpack_migration_threshold(pcmk_resource_t *rsc) { const char *value = g_hash_table_lookup(rsc->priv->meta, PCMK_META_MIGRATION_THRESHOLD); if (pcmk__str_eq(value, PCMK_VALUE_DEFAULT, pcmk__str_casei)) { // @COMPAT Deprecated since 2.1.8 pcmk__config_warn("Support for setting " PCMK_META_MIGRATION_THRESHOLD " to the explicit value '" PCMK_VALUE_DEFAULT "' is deprecated and will be removed in a " "future release (just leave it unset)"); rsc->priv->ban_after_failures = PCMK_SCORE_INFINITY; } else { int rc = pcmk_parse_score(value, &(rsc->priv->ban_after_failures), PCMK_SCORE_INFINITY); if ((rc != pcmk_rc_ok) || (rsc->priv->ban_after_failures < 0)) { pcmk__config_warn("Using default (" PCMK_VALUE_INFINITY ") for resource %s meta-attribute " PCMK_META_MIGRATION_THRESHOLD " because '%s' is not a valid value: %s", rsc->id, value, pcmk_rc_str(rc)); rsc->priv->ban_after_failures = PCMK_SCORE_INFINITY; } } } /*! * \internal * \brief Unpack configuration XML for a given resource * * Unpack the XML object containing a resource's configuration into a new * \c pcmk_resource_t object. * * \param[in] xml_obj XML node containing the resource's configuration * \param[out] rsc Where to store the unpacked resource information * \param[in] parent Resource's parent, if any * \param[in,out] scheduler Scheduler data * * \return Standard Pacemaker return code * \note If pcmk_rc_ok is returned, \p *rsc is guaranteed to be non-NULL, and * the caller is responsible for freeing it using its variant-specific * free() method. Otherwise, \p *rsc is guaranteed to be NULL. */ int pe__unpack_resource(xmlNode *xml_obj, pcmk_resource_t **rsc, pcmk_resource_t *parent, pcmk_scheduler_t *scheduler) { xmlNode *expanded_xml = NULL; xmlNode *ops = NULL; const char *value = NULL; const char *id = NULL; bool guest_node = false; bool remote_node = false; pcmk__resource_private_t *rsc_private = NULL; pcmk_rule_input_t rule_input = { .now = NULL, }; CRM_CHECK(rsc != NULL, return EINVAL); CRM_CHECK((xml_obj != NULL) && (scheduler != NULL), *rsc = NULL; return EINVAL); rule_input.now = scheduler->priv->now; - crm_log_xml_trace(xml_obj, "[raw XML]"); + pcmk__log_xml_trace(xml_obj, "[raw XML]"); id = pcmk__xe_get(xml_obj, PCMK_XA_ID); if (id == NULL) { pcmk__config_err("Ignoring <%s> configuration without " PCMK_XA_ID, xml_obj->name); return pcmk_rc_unpack_error; } if (unpack_template(xml_obj, &expanded_xml, scheduler) == FALSE) { return pcmk_rc_unpack_error; } *rsc = calloc(1, sizeof(pcmk_resource_t)); if (*rsc == NULL) { pcmk__sched_err(scheduler, "Unable to allocate memory for resource '%s'", id); return ENOMEM; } (*rsc)->priv = calloc(1, sizeof(pcmk__resource_private_t)); if ((*rsc)->priv == NULL) { pcmk__sched_err(scheduler, "Unable to allocate memory for resource '%s'", id); free(*rsc); return ENOMEM; } rsc_private = (*rsc)->priv; rsc_private->scheduler = scheduler; if (expanded_xml) { - crm_log_xml_trace(expanded_xml, "[expanded XML]"); + pcmk__log_xml_trace(expanded_xml, "[expanded XML]"); rsc_private->xml = expanded_xml; rsc_private->orig_xml = xml_obj; } else { rsc_private->xml = xml_obj; rsc_private->orig_xml = NULL; } /* Do not use xml_obj from here on, use (*rsc)->xml in case templates are involved */ rsc_private->parent = parent; ops = pcmk__xe_first_child(rsc_private->xml, PCMK_XE_OPERATIONS, NULL, NULL); rsc_private->ops_xml = pcmk__xe_resolve_idref(ops, scheduler->input); rsc_private->variant = get_resource_type((const char *) rsc_private->xml->name); if (rsc_private->variant == pcmk__rsc_variant_unknown) { pcmk__config_err("Ignoring resource '%s' of unknown type '%s'", id, rsc_private->xml->name); common_free(*rsc); *rsc = NULL; return pcmk_rc_unpack_error; } rsc_private->meta = pcmk__strkey_table(free, free); rsc_private->utilization = pcmk__strkey_table(free, free); rsc_private->probed_nodes = pcmk__strkey_table(NULL, pcmk__free_node_copy); rsc_private->allowed_nodes = pcmk__strkey_table(NULL, pcmk__free_node_copy); value = pcmk__xe_get(rsc_private->xml, PCMK__META_CLONE); if (value) { (*rsc)->id = pcmk__assert_asprintf("%s:%s", id, value); pcmk__insert_meta(rsc_private, PCMK__META_CLONE, value); } else { (*rsc)->id = strdup(id); } rsc_private->fns = &resource_class_functions[rsc_private->variant]; get_meta_attributes(rsc_private->meta, *rsc, NULL, scheduler); (*rsc)->flags = 0; pcmk__set_rsc_flags(*rsc, pcmk__rsc_unassigned); if (!pcmk__is_set(scheduler->flags, pcmk__sched_in_maintenance)) { pcmk__set_rsc_flags(*rsc, pcmk__rsc_managed); } rsc_private->orig_role = pcmk_role_stopped; rsc_private->next_role = pcmk_role_unknown; unpack_priority(*rsc); value = g_hash_table_lookup(rsc_private->meta, PCMK_META_CRITICAL); if ((value == NULL) || pcmk__is_true(value)) { pcmk__set_rsc_flags(*rsc, pcmk__rsc_critical); } value = g_hash_table_lookup(rsc_private->meta, PCMK_META_NOTIFY); if (pcmk__is_true(value)) { pcmk__set_rsc_flags(*rsc, pcmk__rsc_notify); } if (xml_contains_remote_node(rsc_private->xml)) { pcmk__set_rsc_flags(*rsc, pcmk__rsc_is_remote_connection); if (g_hash_table_lookup(rsc_private->meta, PCMK__META_CONTAINER)) { guest_node = true; } else { remote_node = true; } } value = g_hash_table_lookup(rsc_private->meta, PCMK_META_ALLOW_MIGRATE); if (pcmk__is_true(value)) { pcmk__set_rsc_flags(*rsc, pcmk__rsc_migratable); } else if ((value == NULL) && remote_node) { /* By default, we want remote nodes to be able * to float around the cluster without having to stop all the * resources within the remote-node before moving. Allowing * migration support enables this feature. If this ever causes * problems, migration support can be explicitly turned off with * PCMK_META_ALLOW_MIGRATE=false. */ pcmk__set_rsc_flags(*rsc, pcmk__rsc_migratable); } value = g_hash_table_lookup(rsc_private->meta, PCMK_META_IS_MANAGED); if (value != NULL) { if (pcmk__str_eq(PCMK_VALUE_DEFAULT, value, pcmk__str_casei)) { // @COMPAT Deprecated since 2.1.8 pcmk__config_warn("Support for setting " PCMK_META_IS_MANAGED " to the explicit value '" PCMK_VALUE_DEFAULT "' is deprecated and will be removed in a " "future release (just leave it unset)"); } else if (pcmk__is_true(value)) { pcmk__set_rsc_flags(*rsc, pcmk__rsc_managed); } else { pcmk__clear_rsc_flags(*rsc, pcmk__rsc_managed); } } value = g_hash_table_lookup(rsc_private->meta, PCMK_META_MAINTENANCE); if (pcmk__is_true(value)) { pcmk__clear_rsc_flags(*rsc, pcmk__rsc_managed); pcmk__set_rsc_flags(*rsc, pcmk__rsc_maintenance); } if (pcmk__is_set(scheduler->flags, pcmk__sched_in_maintenance)) { pcmk__clear_rsc_flags(*rsc, pcmk__rsc_managed); pcmk__set_rsc_flags(*rsc, pcmk__rsc_maintenance); } if (pcmk__is_clone(pe__const_top_resource(*rsc, false))) { if (detect_unique(*rsc)) { pcmk__set_rsc_flags(*rsc, pcmk__rsc_unique); } if (pcmk__is_true(g_hash_table_lookup((*rsc)->priv->meta, PCMK_META_PROMOTABLE))) { pcmk__set_rsc_flags(*rsc, pcmk__rsc_promotable); } } else { pcmk__set_rsc_flags(*rsc, pcmk__rsc_unique); } value = g_hash_table_lookup(rsc_private->meta, PCMK_META_MULTIPLE_ACTIVE); if (pcmk__str_eq(value, PCMK_VALUE_STOP_ONLY, pcmk__str_casei)) { rsc_private->multiply_active_policy = pcmk__multiply_active_stop; pcmk__rsc_trace(*rsc, "%s multiple running resource recovery: stop only", (*rsc)->id); } else if (pcmk__str_eq(value, PCMK_VALUE_BLOCK, pcmk__str_casei)) { rsc_private->multiply_active_policy = pcmk__multiply_active_block; pcmk__rsc_trace(*rsc, "%s multiple running resource recovery: block", (*rsc)->id); } else if (pcmk__str_eq(value, PCMK_VALUE_STOP_UNEXPECTED, pcmk__str_casei)) { rsc_private->multiply_active_policy = pcmk__multiply_active_unexpected; pcmk__rsc_trace(*rsc, "%s multiple running resource recovery: " "stop unexpected instances", (*rsc)->id); } else { // PCMK_VALUE_STOP_START if (!pcmk__str_eq(value, PCMK_VALUE_STOP_START, pcmk__str_casei|pcmk__str_null_matches)) { pcmk__config_warn("%s is not a valid value for " PCMK_META_MULTIPLE_ACTIVE ", using default of " "\"" PCMK_VALUE_STOP_START "\"", value); } rsc_private->multiply_active_policy = pcmk__multiply_active_restart; pcmk__rsc_trace(*rsc, "%s multiple running resource recovery: stop/start", (*rsc)->id); } unpack_stickiness(*rsc); unpack_migration_threshold(*rsc); if (pcmk__str_eq(pcmk__xe_get(rsc_private->xml, PCMK_XA_CLASS), PCMK_RESOURCE_CLASS_STONITH, pcmk__str_casei)) { pcmk__set_scheduler_flags(scheduler, pcmk__sched_have_fencing); pcmk__set_rsc_flags(*rsc, pcmk__rsc_fence_device); } value = g_hash_table_lookup(rsc_private->meta, PCMK_META_REQUIRES); unpack_requires(*rsc, value, false); value = g_hash_table_lookup(rsc_private->meta, PCMK_META_FAILURE_TIMEOUT); if (value != NULL) { pcmk_parse_interval_spec(value, &(rsc_private->failure_expiration_ms)); } if (remote_node) { GHashTable *params = pe_rsc_params(*rsc, NULL, scheduler); /* Grabbing the value now means that any rules based on node attributes * will evaluate to false, so such rules should not be used with * PCMK_REMOTE_RA_RECONNECT_INTERVAL. * * @TODO Evaluate per node before using */ value = g_hash_table_lookup(params, PCMK_REMOTE_RA_RECONNECT_INTERVAL); if (value) { /* reconnect delay works by setting failure_timeout and preventing the * connection from starting until the failure is cleared. */ pcmk_parse_interval_spec(value, &(rsc_private->remote_reconnect_ms)); /* We want to override any default failure_timeout in use when remote * PCMK_REMOTE_RA_RECONNECT_INTERVAL is in use. */ rsc_private->failure_expiration_ms = rsc_private->remote_reconnect_ms; } } get_target_role(*rsc, &(rsc_private->next_role)); pcmk__rsc_trace(*rsc, "%s desired next state: %s", (*rsc)->id, (rsc_private->next_role == pcmk_role_unknown)? "default" : pcmk_role_text(rsc_private->next_role)); if (!rsc_private->fns->unpack(*rsc)) { pcmk__free_resource(*rsc); *rsc = NULL; return pcmk_rc_unpack_error; } if (pcmk__is_set(scheduler->flags, pcmk__sched_symmetric_cluster)) { // This tag must stay exactly the same because it is tested elsewhere resource_location(*rsc, NULL, 0, "symmetric_default", scheduler); } else if (guest_node) { /* remote resources tied to a container resource must always be allowed * to opt-in to the cluster. Whether the connection resource is actually * allowed to be placed on a node is dependent on the container resource */ resource_location(*rsc, NULL, 0, "remote_connection_default", scheduler); } if (pcmk__is_set((*rsc)->flags, pcmk__rsc_notify)) { pcmk__rsc_trace(*rsc, "%s action notification: required", (*rsc)->id); } else { pcmk__rsc_trace(*rsc, "%s action notification: not required", (*rsc)->id); } pe__unpack_dataset_nvpairs(rsc_private->xml, PCMK_XE_UTILIZATION, &rule_input, rsc_private->utilization, NULL, scheduler); if (expanded_xml) { if (add_template_rsc(xml_obj, scheduler) == FALSE) { pcmk__free_resource(*rsc); *rsc = NULL; return pcmk_rc_unpack_error; } } return pcmk_rc_ok; } gboolean is_parent(pcmk_resource_t *child, pcmk_resource_t *rsc) { pcmk_resource_t *parent = child; if (parent == NULL || rsc == NULL) { return FALSE; } while (parent->priv->parent != NULL) { if (parent->priv->parent == rsc) { return TRUE; } parent = parent->priv->parent; } return FALSE; } pcmk_resource_t * uber_parent(pcmk_resource_t *rsc) { pcmk_resource_t *parent = rsc; if (parent == NULL) { return NULL; } while ((parent->priv->parent != NULL) && !pcmk__is_bundle(parent->priv->parent)) { parent = parent->priv->parent; } return parent; } /*! * \internal * \brief Get the topmost parent of a resource as a const pointer * * \param[in] rsc Resource to check * \param[in] include_bundle If true, go all the way to bundle * * \return \p NULL if \p rsc is NULL, \p rsc if \p rsc has no parent, * the bundle if \p rsc is bundled and \p include_bundle is true, * otherwise the topmost parent of \p rsc up to a clone */ const pcmk_resource_t * pe__const_top_resource(const pcmk_resource_t *rsc, bool include_bundle) { const pcmk_resource_t *parent = rsc; if (parent == NULL) { return NULL; } while (parent->priv->parent != NULL) { if (!include_bundle && pcmk__is_bundle(parent->priv->parent)) { break; } parent = parent->priv->parent; } return parent; } void common_free(pcmk_resource_t * rsc) { if (rsc == NULL) { return; } pcmk__rsc_trace(rsc, "Freeing %s", rsc->id); if (rsc->priv->parameter_cache != NULL) { g_hash_table_destroy(rsc->priv->parameter_cache); } if ((rsc->priv->parent == NULL) && pcmk__is_set(rsc->flags, pcmk__rsc_removed)) { pcmk__xml_free(rsc->priv->xml); rsc->priv->xml = NULL; pcmk__xml_free(rsc->priv->orig_xml); rsc->priv->orig_xml = NULL; } else if (rsc->priv->orig_xml != NULL) { // rsc->private->xml was expanded from a template pcmk__xml_free(rsc->priv->xml); rsc->priv->xml = NULL; } free(rsc->id); free(rsc->priv->variant_opaque); free(rsc->priv->history_id); free(rsc->priv->pending_action); pcmk__free_node_copy(rsc->priv->assigned_node); g_list_free(rsc->priv->actions); g_list_free(rsc->priv->active_nodes); g_list_free(rsc->priv->launched); g_list_free(rsc->priv->dangling_migration_sources); g_list_free(rsc->priv->with_this_colocations); g_list_free(rsc->priv->this_with_colocations); g_list_free(rsc->priv->location_constraints); g_list_free(rsc->priv->ticket_constraints); if (rsc->priv->meta != NULL) { g_hash_table_destroy(rsc->priv->meta); } if (rsc->priv->utilization != NULL) { g_hash_table_destroy(rsc->priv->utilization); } if (rsc->priv->probed_nodes != NULL) { g_hash_table_destroy(rsc->priv->probed_nodes); } if (rsc->priv->allowed_nodes != NULL) { g_hash_table_destroy(rsc->priv->allowed_nodes); } free(rsc->priv); free(rsc); } /*! * \internal * \brief Count a node and update most preferred to it as appropriate * * \param[in] rsc An active resource * \param[in] node A node that \p rsc is active on * \param[in,out] active This will be set to \p node if \p node is more * preferred than the current value * \param[in,out] count_all If not NULL, this will be incremented * \param[in,out] count_clean If not NULL, this will be incremented if \p node * is online and clean * * \return true if the count should continue, or false if sufficiently known */ bool pe__count_active_node(const pcmk_resource_t *rsc, pcmk_node_t *node, pcmk_node_t **active, unsigned int *count_all, unsigned int *count_clean) { bool keep_looking = false; bool is_happy = false; CRM_CHECK((rsc != NULL) && (node != NULL) && (active != NULL), return false); is_happy = node->details->online && !node->details->unclean; if (count_all != NULL) { ++*count_all; } if ((count_clean != NULL) && is_happy) { ++*count_clean; } if ((count_all != NULL) || (count_clean != NULL)) { keep_looking = true; // We're counting, so go through entire list } if (rsc->priv->partial_migration_source != NULL) { if (pcmk__same_node(node, rsc->priv->partial_migration_source)) { *active = node; // This is the migration source } else { keep_looking = true; } } else if (!pcmk__is_set(rsc->flags, pcmk__rsc_needs_fencing)) { if (is_happy && ((*active == NULL) || !(*active)->details->online || (*active)->details->unclean)) { *active = node; // This is the first clean node } else { keep_looking = true; } } if (*active == NULL) { *active = node; // This is the first node checked } return keep_looking; } // Shared implementation of pcmk__rsc_methods_t:active_node() static pcmk_node_t * active_node(const pcmk_resource_t *rsc, unsigned int *count_all, unsigned int *count_clean) { pcmk_node_t *active = NULL; if (count_all != NULL) { *count_all = 0; } if (count_clean != NULL) { *count_clean = 0; } if (rsc == NULL) { return NULL; } for (GList *iter = rsc->priv->active_nodes; iter != NULL; iter = iter->next) { if (!pe__count_active_node(rsc, (pcmk_node_t *) iter->data, &active, count_all, count_clean)) { break; // Don't waste time iterating if we don't have to } } return active; } /*! * \brief * \internal Find and count active nodes according to \c PCMK_META_REQUIRES * * \param[in] rsc Resource to check * \param[out] count If not NULL, will be set to count of active nodes * * \return An active node (or NULL if resource is not active anywhere) * * \note This is a convenience wrapper for active_node() where the count of all * active nodes or only clean active nodes is desired according to the * \c PCMK_META_REQUIRES meta-attribute. */ pcmk_node_t * pe__find_active_requires(const pcmk_resource_t *rsc, unsigned int *count) { if (rsc == NULL) { if (count != NULL) { *count = 0; } return NULL; } if (pcmk__is_set(rsc->flags, pcmk__rsc_needs_fencing)) { return rsc->priv->fns->active_node(rsc, count, NULL); } else { return rsc->priv->fns->active_node(rsc, NULL, count); } } void pe__count_common(pcmk_resource_t *rsc) { if (rsc->priv->children != NULL) { for (GList *item = rsc->priv->children; item != NULL; item = item->next) { pcmk_resource_t *child = item->data; child->priv->fns->count(item->data); } } else if (!pcmk__is_set(rsc->flags, pcmk__rsc_removed) || (rsc->priv->orig_role > pcmk_role_stopped)) { rsc->priv->scheduler->priv->ninstances++; if (pe__resource_is_disabled(rsc)) { rsc->priv->scheduler->priv->disabled_resources++; } if (pcmk__is_set(rsc->flags, pcmk__rsc_blocked)) { rsc->priv->scheduler->priv->blocked_resources++; } } } /*! * \internal * \brief Update a resource's next role * * \param[in,out] rsc Resource to be updated * \param[in] role Resource's new next role * \param[in] why Human-friendly reason why role is changing (for logs) */ void pe__set_next_role(pcmk_resource_t *rsc, enum rsc_role_e role, const char *why) { pcmk__assert((rsc != NULL) && (why != NULL)); if (rsc->priv->next_role != role) { pcmk__rsc_trace(rsc, "Resetting next role for %s from %s to %s (%s)", rsc->id, pcmk_role_text(rsc->priv->next_role), pcmk_role_text(role), why); rsc->priv->next_role = role; } }