diff --git a/daemons/controld/controld_fencing.c b/daemons/controld/controld_fencing.c index 004f3b1ebc..178f162e0e 100644 --- a/daemons/controld/controld_fencing.c +++ b/daemons/controld/controld_fencing.c @@ -1,1115 +1,1116 @@ /* * Copyright 2004-2024 the Pacemaker project contributors * * The version control history for this file may have further details. * * This source code is licensed under the GNU General Public License version 2 * or later (GPLv2+) WITHOUT ANY WARRANTY. */ #include #include #include #include #include #include #include static void tengine_stonith_history_synced(stonith_t *st, stonith_event_t *st_event); /* * stonith failure counting * * We don't want to get stuck in a permanent fencing loop. Keep track of the * number of fencing failures for each target node, and the most we'll restart a * transition for. */ struct st_fail_rec { int count; }; static bool fence_reaction_panic = false; static unsigned long int stonith_max_attempts = 10; static GHashTable *stonith_failures = NULL; /*! * \internal * \brief Update max fencing attempts before giving up * * \param[in] value New max fencing attempts */ static void update_stonith_max_attempts(const char *value) { stonith_max_attempts = char2score(value); if (stonith_max_attempts < 1UL) { stonith_max_attempts = 10UL; } } /*! * \internal * \brief Configure reaction to notification of local node being fenced * * \param[in] reaction_s Reaction type */ static void set_fence_reaction(const char *reaction_s) { if (pcmk__str_eq(reaction_s, "panic", pcmk__str_casei)) { fence_reaction_panic = true; } else { if (!pcmk__str_eq(reaction_s, "stop", pcmk__str_casei)) { crm_warn("Invalid value '%s' for %s, using 'stop'", reaction_s, PCMK_OPT_FENCE_REACTION); } fence_reaction_panic = false; } } /*! * \internal * \brief Configure fencing options based on the CIB * * \param[in,out] options Name/value pairs for configured options */ void controld_configure_fencing(GHashTable *options) { const char *value = NULL; value = g_hash_table_lookup(options, PCMK_OPT_FENCE_REACTION); set_fence_reaction(value); value = g_hash_table_lookup(options, PCMK_OPT_STONITH_MAX_ATTEMPTS); update_stonith_max_attempts(value); } static gboolean too_many_st_failures(const char *target) { GHashTableIter iter; const char *key = NULL; struct st_fail_rec *value = NULL; if (stonith_failures == NULL) { return FALSE; } if (target == NULL) { g_hash_table_iter_init(&iter, stonith_failures); while (g_hash_table_iter_next(&iter, (gpointer *) &key, (gpointer *) &value)) { if (value->count >= stonith_max_attempts) { target = (const char*)key; goto too_many; } } } else { value = g_hash_table_lookup(stonith_failures, target); if ((value != NULL) && (value->count >= stonith_max_attempts)) { goto too_many; } } return FALSE; too_many: crm_warn("Too many failures (%d) to fence %s, giving up", value->count, target); return TRUE; } /*! * \internal * \brief Reset a stonith fail count * * \param[in] target Name of node to reset, or NULL for all */ void st_fail_count_reset(const char *target) { if (stonith_failures == NULL) { return; } if (target) { struct st_fail_rec *rec = NULL; rec = g_hash_table_lookup(stonith_failures, target); if (rec) { rec->count = 0; } } else { GHashTableIter iter; const char *key = NULL; struct st_fail_rec *rec = NULL; g_hash_table_iter_init(&iter, stonith_failures); while (g_hash_table_iter_next(&iter, (gpointer *) &key, (gpointer *) &rec)) { rec->count = 0; } } } static void st_fail_count_increment(const char *target) { struct st_fail_rec *rec = NULL; if (stonith_failures == NULL) { stonith_failures = pcmk__strkey_table(free, free); } rec = g_hash_table_lookup(stonith_failures, target); if (rec) { rec->count++; } else { rec = malloc(sizeof(struct st_fail_rec)); if(rec == NULL) { return; } rec->count = 1; g_hash_table_insert(stonith_failures, strdup(target), rec); } } /* end stonith fail count functions */ static void cib_fencing_updated(xmlNode *msg, int call_id, int rc, xmlNode *output, void *user_data) { if (rc < pcmk_ok) { crm_err("Fencing update %d for %s: failed - %s (%d)", call_id, (char *)user_data, pcmk_strerror(rc), rc); crm_log_xml_warn(msg, "Failed update"); abort_transition(INFINITY, pcmk__graph_shutdown, "CIB update failed", NULL); } else { crm_info("Fencing update %d for %s: complete", call_id, (char *)user_data); } } static void send_stonith_update(pcmk__graph_action_t *action, const char *target, const char *uuid) { int rc = pcmk_ok; crm_node_t *peer = NULL; /* We (usually) rely on the membership layer to do node_update_cluster, * and the peer status callback to do node_update_peer, because the node * might have already rejoined before we get the stonith result here. */ int flags = node_update_join | node_update_expected; /* zero out the node-status & remove all LRM status info */ xmlNode *node_state = NULL; CRM_CHECK(target != NULL, return); CRM_CHECK(uuid != NULL, return); /* Make sure the membership and join caches are accurate. * Try getting any existing node cache entry also by node uuid in case it * doesn't have an uname yet. */ peer = pcmk__get_node(0, target, uuid, pcmk__node_search_any); CRM_CHECK(peer != NULL, return); if (peer->state == NULL) { /* Usually, we rely on the membership layer to update the cluster state * in the CIB. However, if the node has never been seen, do it here, so * the node is not considered unclean. */ flags |= node_update_cluster; } if (peer->uuid == NULL) { crm_info("Recording uuid '%s' for node '%s'", uuid, target); peer->uuid = strdup(uuid); } crmd_peer_down(peer, TRUE); /* Generate a node state update for the CIB */ node_state = create_node_state_update(peer, flags, NULL, __func__); /* we have to mark whether or not remote nodes have already been fenced */ if (peer->flags & crm_remote_node) { char *now_s = pcmk__ttoa(time(NULL)); crm_xml_add(node_state, XML_NODE_IS_FENCED, now_s); free(now_s); } /* Force our known ID */ crm_xml_add(node_state, PCMK_XA_ID, uuid); rc = controld_globals.cib_conn->cmds->modify(controld_globals.cib_conn, XML_CIB_TAG_STATUS, node_state, cib_scope_local |cib_can_create); /* Delay processing the trigger until the update completes */ crm_debug("Sending fencing update %d for %s", rc, target); fsa_register_cib_callback(rc, strdup(target), cib_fencing_updated); // Make sure it sticks /* controld_globals.cib_conn->cmds->bump_epoch(controld_globals.cib_conn, * cib_scope_local); */ controld_delete_node_state(peer->uname, controld_section_all, cib_scope_local); free_xml(node_state); return; } /*! * \internal * \brief Abort transition due to stonith failure * * \param[in] abort_action Whether to restart or stop transition * \param[in] target Don't restart if this (NULL for any) has too many failures * \param[in] reason Log this stonith action XML as abort reason (or NULL) */ static void abort_for_stonith_failure(enum pcmk__graph_next abort_action, const char *target, const xmlNode *reason) { /* If stonith repeatedly fails, we eventually give up on starting a new * transition for that reason. */ if ((abort_action != pcmk__graph_wait) && too_many_st_failures(target)) { abort_action = pcmk__graph_wait; } abort_transition(INFINITY, abort_action, "Stonith failed", reason); } /* * stonith cleanup list * * If the DC is shot, proper notifications might not go out. * The stonith cleanup list allows the cluster to (re-)send * notifications once a new DC is elected. */ static GList *stonith_cleanup_list = NULL; /*! * \internal * \brief Add a node to the stonith cleanup list * * \param[in] target Name of node to add */ void add_stonith_cleanup(const char *target) { stonith_cleanup_list = g_list_append(stonith_cleanup_list, strdup(target)); } /*! * \internal * \brief Remove a node from the stonith cleanup list * * \param[in] Name of node to remove */ void remove_stonith_cleanup(const char *target) { GList *iter = stonith_cleanup_list; while (iter != NULL) { GList *tmp = iter; char *iter_name = tmp->data; iter = iter->next; if (pcmk__str_eq(target, iter_name, pcmk__str_casei)) { crm_trace("Removing %s from the cleanup list", iter_name); stonith_cleanup_list = g_list_delete_link(stonith_cleanup_list, tmp); free(iter_name); } } } /*! * \internal * \brief Purge all entries from the stonith cleanup list */ void purge_stonith_cleanup(void) { if (stonith_cleanup_list) { GList *iter = NULL; for (iter = stonith_cleanup_list; iter != NULL; iter = iter->next) { char *target = iter->data; crm_info("Purging %s from stonith cleanup list", target); free(target); } g_list_free(stonith_cleanup_list); stonith_cleanup_list = NULL; } } /*! * \internal * \brief Send stonith updates for all entries in cleanup list, then purge it */ void execute_stonith_cleanup(void) { GList *iter; for (iter = stonith_cleanup_list; iter != NULL; iter = iter->next) { char *target = iter->data; crm_node_t *target_node = pcmk__get_node(0, target, NULL, pcmk__node_search_cluster); const char *uuid = crm_peer_uuid(target_node); crm_notice("Marking %s, target of a previous stonith action, as clean", target); send_stonith_update(NULL, target, uuid); free(target); } g_list_free(stonith_cleanup_list); stonith_cleanup_list = NULL; } /* end stonith cleanup list functions */ /* stonith API client * * Functions that need to interact directly with the fencer via its API */ static stonith_t *stonith_api = NULL; static mainloop_timer_t *controld_fencer_connect_timer = NULL; static char *te_client_id = NULL; static gboolean fail_incompletable_stonith(pcmk__graph_t *graph) { GList *lpc = NULL; const char *task = NULL; xmlNode *last_action = NULL; if (graph == NULL) { return FALSE; } for (lpc = graph->synapses; lpc != NULL; lpc = lpc->next) { GList *lpc2 = NULL; pcmk__graph_synapse_t *synapse = (pcmk__graph_synapse_t *) lpc->data; if (pcmk_is_set(synapse->flags, pcmk__synapse_confirmed)) { continue; } for (lpc2 = synapse->actions; lpc2 != NULL; lpc2 = lpc2->next) { pcmk__graph_action_t *action = (pcmk__graph_action_t *) lpc2->data; if ((action->type != pcmk__cluster_graph_action) || pcmk_is_set(action->flags, pcmk__graph_action_confirmed)) { continue; } task = crm_element_value(action->xml, PCMK_XA_OPERATION); if (pcmk__str_eq(task, PCMK_ACTION_STONITH, pcmk__str_casei)) { pcmk__set_graph_action_flags(action, pcmk__graph_action_failed); last_action = action->xml; pcmk__update_graph(graph, action); crm_notice("Failing action %d (%s): fencer terminated", action->id, ID(action->xml)); } } } if (last_action != NULL) { crm_warn("Fencer failure resulted in unrunnable actions"); abort_for_stonith_failure(pcmk__graph_restart, NULL, last_action); return TRUE; } return FALSE; } static void tengine_stonith_connection_destroy(stonith_t *st, stonith_event_t *e) { te_cleanup_stonith_history_sync(st, FALSE); if (pcmk_is_set(controld_globals.fsa_input_register, R_ST_REQUIRED)) { crm_err("Lost fencer connection (will attempt to reconnect)"); if (!mainloop_timer_running(controld_fencer_connect_timer)) { mainloop_timer_start(controld_fencer_connect_timer); } } else { crm_info("Disconnected from fencer"); } if (stonith_api) { /* the client API won't properly reconnect notifications * if they are still in the table - so remove them */ if (stonith_api->state != stonith_disconnected) { stonith_api->cmds->disconnect(st); } stonith_api->cmds->remove_notification(stonith_api, NULL); } if (AM_I_DC) { fail_incompletable_stonith(controld_globals.transition_graph); trigger_graph(); } } /*! * \internal * \brief Handle an event notification from the fencing API * * \param[in] st Fencing API connection (ignored) * \param[in] event Fencing API event notification */ static void handle_fence_notification(stonith_t *st, stonith_event_t *event) { bool succeeded = true; const char *executioner = "the cluster"; const char *client = "a client"; const char *reason = NULL; int exec_status; if (te_client_id == NULL) { te_client_id = crm_strdup_printf("%s.%lu", crm_system_name, (unsigned long) getpid()); } if (event == NULL) { crm_err("Notify data not found"); return; } if (event->executioner != NULL) { executioner = event->executioner; } if (event->client_origin != NULL) { client = event->client_origin; } exec_status = stonith__event_execution_status(event); if ((stonith__event_exit_status(event) != CRM_EX_OK) || (exec_status != PCMK_EXEC_DONE)) { succeeded = false; if (exec_status == PCMK_EXEC_DONE) { exec_status = PCMK_EXEC_ERROR; } } reason = stonith__event_exit_reason(event); crmd_alert_fencing_op(event); if (pcmk__str_eq(PCMK_ACTION_ON, event->action, pcmk__str_none)) { // Unfencing doesn't need special handling, just a log message if (succeeded) { crm_notice("%s was unfenced by %s at the request of %s@%s", event->target, executioner, client, event->origin); } else { crm_err("Unfencing of %s by %s failed (%s%s%s) with exit status %d", event->target, executioner, pcmk_exec_status_str(exec_status), ((reason == NULL)? "" : ": "), ((reason == NULL)? "" : reason), stonith__event_exit_status(event)); } return; } if (succeeded && pcmk__str_eq(event->target, controld_globals.our_nodename, pcmk__str_casei)) { /* We were notified of our own fencing. Most likely, either fencing was * misconfigured, or fabric fencing that doesn't cut cluster * communication is in use. * * Either way, shutting down the local host is a good idea, to require * administrator intervention. Also, other nodes would otherwise likely * set our status to lost because of the fencing callback and discard * our subsequent election votes as "not part of our cluster". */ crm_crit("We were allegedly just fenced by %s for %s!", executioner, event->origin); // Dumps blackbox if enabled if (fence_reaction_panic) { pcmk__panic(__func__); } else { crm_exit(CRM_EX_FATAL); } return; // Should never get here } /* Update the count of fencing failures for this target, in case we become * DC later. The current DC has already updated its fail count in * tengine_stonith_callback(). */ if (!AM_I_DC) { if (succeeded) { st_fail_count_reset(event->target); } else { st_fail_count_increment(event->target); } } crm_notice("Peer %s was%s terminated (%s) by %s on behalf of %s@%s: " "%s%s%s%s " CRM_XS " event=%s", event->target, (succeeded? "" : " not"), event->action, executioner, client, event->origin, (succeeded? "OK" : pcmk_exec_status_str(exec_status)), ((reason == NULL)? "" : " ("), ((reason == NULL)? "" : reason), ((reason == NULL)? "" : ")"), event->id); if (succeeded) { crm_node_t *peer = pcmk__search_node_caches(0, event->target, pcmk__node_search_any |pcmk__node_search_known); const char *uuid = NULL; if (peer == NULL) { return; } uuid = crm_peer_uuid(peer); if (AM_I_DC) { /* The DC always sends updates */ send_stonith_update(NULL, event->target, uuid); /* @TODO Ideally, at this point, we'd check whether the fenced node * hosted any guest nodes, and call remote_node_down() for them. * Unfortunately, the controller doesn't have a simple, reliable way * to map hosts to guests. It might be possible to track this in the * peer cache via crm_remote_peer_cache_refresh(). For now, we rely * on the scheduler creating fence pseudo-events for the guests. */ if (!pcmk__str_eq(client, te_client_id, pcmk__str_casei)) { /* Abort the current transition if it wasn't the cluster that * initiated fencing. */ crm_info("External fencing operation from %s fenced %s", client, event->target); abort_transition(INFINITY, pcmk__graph_restart, "External Fencing Operation", NULL); } } else if (pcmk__str_eq(controld_globals.dc_name, event->target, pcmk__str_null_matches|pcmk__str_casei) && !pcmk_is_set(peer->flags, crm_remote_node)) { // Assume the target was our DC if we don't currently have one if (controld_globals.dc_name != NULL) { crm_notice("Fencing target %s was our DC", event->target); } else { crm_notice("Fencing target %s may have been our DC", event->target); } /* Given the CIB resyncing that occurs around elections, * have one node update the CIB now and, if the new DC is different, * have them do so too after the election */ if (pcmk__str_eq(event->executioner, controld_globals.our_nodename, pcmk__str_casei)) { send_stonith_update(NULL, event->target, uuid); } add_stonith_cleanup(event->target); } /* If the target is a remote node, and we host its connection, * immediately fail all monitors so it can be recovered quickly. * The connection won't necessarily drop when a remote node is fenced, * so the failure might not otherwise be detected until the next poke. */ if (pcmk_is_set(peer->flags, crm_remote_node)) { remote_ra_fail(event->target); } crmd_peer_down(peer, TRUE); } } /*! * \brief Connect to fencer * * \param[in] user_data If NULL, retry failures now, otherwise retry in mainloop timer * * \return G_SOURCE_REMOVE on success, G_SOURCE_CONTINUE to retry * \note If user_data is NULL, this will wait 2s between attempts, for up to * 30 attempts, meaning the controller could be blocked as long as 58s. */ gboolean controld_timer_fencer_connect(gpointer user_data) { int rc = pcmk_ok; if (stonith_api == NULL) { stonith_api = stonith_api_new(); if (stonith_api == NULL) { crm_err("Could not connect to fencer: API memory allocation failed"); return G_SOURCE_REMOVE; } } if (stonith_api->state != stonith_disconnected) { crm_trace("Already connected to fencer, no need to retry"); return G_SOURCE_REMOVE; } if (user_data == NULL) { // Blocking (retry failures now until successful) rc = stonith_api_connect_retry(stonith_api, crm_system_name, 30); if (rc != pcmk_ok) { crm_err("Could not connect to fencer in 30 attempts: %s " CRM_XS " rc=%d", pcmk_strerror(rc), rc); } } else { // Non-blocking (retry failures later in main loop) rc = stonith_api->cmds->connect(stonith_api, crm_system_name, NULL); if (controld_fencer_connect_timer == NULL) { controld_fencer_connect_timer = mainloop_timer_add("controld_fencer_connect", 1000, TRUE, controld_timer_fencer_connect, GINT_TO_POINTER(TRUE)); } if (rc != pcmk_ok) { if (pcmk_is_set(controld_globals.fsa_input_register, R_ST_REQUIRED)) { crm_notice("Fencer connection failed (will retry): %s " CRM_XS " rc=%d", pcmk_strerror(rc), rc); if (!mainloop_timer_running(controld_fencer_connect_timer)) { mainloop_timer_start(controld_fencer_connect_timer); } return G_SOURCE_CONTINUE; } else { crm_info("Fencer connection failed (ignoring because no longer required): %s " CRM_XS " rc=%d", pcmk_strerror(rc), rc); } return G_SOURCE_REMOVE; } } if (rc == pcmk_ok) { stonith_api->cmds->register_notification(stonith_api, T_STONITH_NOTIFY_DISCONNECT, tengine_stonith_connection_destroy); stonith_api->cmds->register_notification(stonith_api, T_STONITH_NOTIFY_FENCE, handle_fence_notification); stonith_api->cmds->register_notification(stonith_api, T_STONITH_NOTIFY_HISTORY_SYNCED, tengine_stonith_history_synced); te_trigger_stonith_history_sync(TRUE); crm_notice("Fencer successfully connected"); } return G_SOURCE_REMOVE; } void controld_disconnect_fencer(bool destroy) { if (stonith_api) { // Prevent fencer connection from coming up again controld_clear_fsa_input_flags(R_ST_REQUIRED); if (stonith_api->state != stonith_disconnected) { stonith_api->cmds->disconnect(stonith_api); } stonith_api->cmds->remove_notification(stonith_api, NULL); } if (destroy) { if (stonith_api) { stonith_api->cmds->free(stonith_api); stonith_api = NULL; } if (controld_fencer_connect_timer) { mainloop_timer_del(controld_fencer_connect_timer); controld_fencer_connect_timer = NULL; } if (te_client_id) { free(te_client_id); te_client_id = NULL; } } } static gboolean do_stonith_history_sync(gpointer user_data) { if (stonith_api && (stonith_api->state != stonith_disconnected)) { stonith_history_t *history = NULL; te_cleanup_stonith_history_sync(stonith_api, FALSE); stonith_api->cmds->history(stonith_api, st_opt_sync_call | st_opt_broadcast, NULL, &history, 5); stonith_history_free(history); return TRUE; } else { crm_info("Skip triggering stonith history-sync as stonith is disconnected"); return FALSE; } } static void tengine_stonith_callback(stonith_t *stonith, stonith_callback_data_t *data) { char *uuid = NULL; int stonith_id = -1; int transition_id = -1; pcmk__graph_action_t *action = NULL; const char *target = NULL; if ((data == NULL) || (data->userdata == NULL)) { crm_err("Ignoring fence operation %d result: " "No transition key given (bug?)", ((data == NULL)? -1 : data->call_id)); return; } if (!AM_I_DC) { const char *reason = stonith__exit_reason(data); if (reason == NULL) { reason = pcmk_exec_status_str(stonith__execution_status(data)); } crm_notice("Result of fence operation %d: %d (%s) " CRM_XS " key=%s", data->call_id, stonith__exit_status(data), reason, (const char *) data->userdata); return; } CRM_CHECK(decode_transition_key(data->userdata, &uuid, &transition_id, &stonith_id, NULL), goto bail); if (controld_globals.transition_graph->complete || (stonith_id < 0) || !pcmk__str_eq(uuid, controld_globals.te_uuid, pcmk__str_none) || (controld_globals.transition_graph->id != transition_id)) { crm_info("Ignoring fence operation %d result: " "Not from current transition " CRM_XS " complete=%s action=%d uuid=%s (vs %s) transition=%d (vs %d)", data->call_id, pcmk__btoa(controld_globals.transition_graph->complete), stonith_id, uuid, controld_globals.te_uuid, transition_id, controld_globals.transition_graph->id); goto bail; } action = controld_get_action(stonith_id); if (action == NULL) { crm_err("Ignoring fence operation %d result: " "Action %d not found in transition graph (bug?) " CRM_XS " uuid=%s transition=%d", data->call_id, stonith_id, uuid, transition_id); goto bail; } target = crm_element_value(action->xml, PCMK__META_ON_NODE); if (target == NULL) { crm_err("Ignoring fence operation %d result: No target given (bug?)", data->call_id); goto bail; } stop_te_timer(action); if (stonith__exit_status(data) == CRM_EX_OK) { - const char *uuid = crm_element_value(action->xml, XML_LRM_ATTR_TARGET_UUID); + const char *uuid = crm_element_value(action->xml, + PCMK__META_ON_NODE_UUID); const char *op = crm_meta_value(action->params, "stonith_action"); crm_info("Fence operation %d for %s succeeded", data->call_id, target); if (!(pcmk_is_set(action->flags, pcmk__graph_action_confirmed))) { te_action_confirmed(action, NULL); if (pcmk__str_eq(PCMK_ACTION_ON, op, pcmk__str_casei)) { const char *value = NULL; char *now = pcmk__ttoa(time(NULL)); gboolean is_remote_node = FALSE; /* This check is not 100% reliable, since this node is not * guaranteed to have the remote node cached. However, it * doesn't have to be reliable, since the attribute manager can * learn a node's "remoteness" by other means sooner or later. * This allows it to learn more quickly if this node does have * the information. */ if (g_hash_table_lookup(crm_remote_peer_cache, uuid) != NULL) { is_remote_node = TRUE; } update_attrd(target, CRM_ATTR_UNFENCED, now, NULL, is_remote_node); free(now); value = crm_meta_value(action->params, PCMK__META_DIGESTS_ALL); update_attrd(target, CRM_ATTR_DIGESTS_ALL, value, NULL, is_remote_node); value = crm_meta_value(action->params, PCMK__META_DIGESTS_SECURE); update_attrd(target, CRM_ATTR_DIGESTS_SECURE, value, NULL, is_remote_node); } else if (!(pcmk_is_set(action->flags, pcmk__graph_action_sent_update))) { send_stonith_update(action, target, uuid); pcmk__set_graph_action_flags(action, pcmk__graph_action_sent_update); } } st_fail_count_reset(target); } else { enum pcmk__graph_next abort_action = pcmk__graph_restart; int status = stonith__execution_status(data); const char *reason = stonith__exit_reason(data); if (reason == NULL) { if (status == PCMK_EXEC_DONE) { reason = "Agent returned error"; } else { reason = pcmk_exec_status_str(status); } } pcmk__set_graph_action_flags(action, pcmk__graph_action_failed); /* If no fence devices were available, there's no use in immediately * checking again, so don't start a new transition in that case. */ if (status == PCMK_EXEC_NO_FENCE_DEVICE) { crm_warn("Fence operation %d for %s failed: %s " "(aborting transition and giving up for now)", data->call_id, target, reason); abort_action = pcmk__graph_wait; } else { crm_notice("Fence operation %d for %s failed: %s " "(aborting transition)", data->call_id, target, reason); } /* Increment the fail count now, so abort_for_stonith_failure() can * check it. Non-DC nodes will increment it in * handle_fence_notification(). */ st_fail_count_increment(target); abort_for_stonith_failure(abort_action, target, NULL); } pcmk__update_graph(controld_globals.transition_graph, action); trigger_graph(); bail: free(data->userdata); free(uuid); return; } static int fence_with_delay(const char *target, const char *type, int delay) { uint32_t options = st_opt_none; // Group of enum stonith_call_options int timeout_sec = (int) (controld_globals.transition_graph->stonith_timeout / 1000); if (crmd_join_phase_count(crm_join_confirmed) == 1) { stonith__set_call_options(options, target, st_opt_allow_suicide); } return stonith_api->cmds->fence_with_delay(stonith_api, options, target, type, timeout_sec, 0, delay); } /*! * \internal * \brief Execute a fencing action from a transition graph * * \param[in] graph Transition graph being executed (ignored) * \param[in] action Fencing action to execute * * \return Standard Pacemaker return code */ int controld_execute_fence_action(pcmk__graph_t *graph, pcmk__graph_action_t *action) { int rc = 0; const char *id = ID(action->xml); - const char *uuid = crm_element_value(action->xml, XML_LRM_ATTR_TARGET_UUID); + const char *uuid = crm_element_value(action->xml, PCMK__META_ON_NODE_UUID); const char *target = crm_element_value(action->xml, PCMK__META_ON_NODE); const char *type = crm_meta_value(action->params, "stonith_action"); char *transition_key = NULL; const char *priority_delay = NULL; int delay_i = 0; gboolean invalid_action = FALSE; int stonith_timeout = (int) (controld_globals.transition_graph->stonith_timeout / 1000); CRM_CHECK(id != NULL, invalid_action = TRUE); CRM_CHECK(uuid != NULL, invalid_action = TRUE); CRM_CHECK(type != NULL, invalid_action = TRUE); CRM_CHECK(target != NULL, invalid_action = TRUE); if (invalid_action) { crm_log_xml_warn(action->xml, "BadAction"); return EPROTO; } priority_delay = crm_meta_value(action->params, PCMK_OPT_PRIORITY_FENCING_DELAY); crm_notice("Requesting fencing (%s) targeting node %s " CRM_XS " action=%s timeout=%i%s%s", type, target, id, stonith_timeout, priority_delay ? " priority_delay=" : "", priority_delay ? priority_delay : ""); /* Passing NULL means block until we can connect... */ controld_timer_fencer_connect(NULL); pcmk__scan_min_int(priority_delay, &delay_i, 0); rc = fence_with_delay(target, type, delay_i); transition_key = pcmk__transition_key(controld_globals.transition_graph->id, action->id, 0, controld_globals.te_uuid), stonith_api->cmds->register_callback(stonith_api, rc, (stonith_timeout + (delay_i > 0 ? delay_i : 0)), st_opt_timeout_updates, transition_key, "tengine_stonith_callback", tengine_stonith_callback); return pcmk_rc_ok; } bool controld_verify_stonith_watchdog_timeout(const char *value) { long st_timeout = value? crm_get_msec(value) : 0; const char *our_nodename = controld_globals.our_nodename; gboolean rv = TRUE; if (st_timeout == 0 || (stonith_api && (stonith_api->state != stonith_disconnected) && stonith__watchdog_fencing_enabled_for_node_api(stonith_api, our_nodename))) { rv = pcmk__valid_sbd_timeout(value); } return rv; } /* end stonith API client functions */ /* * stonith history synchronization * * Each node's fencer keeps track of a cluster-wide fencing history. When a node * joins or leaves, we need to synchronize the history across all nodes. */ static crm_trigger_t *stonith_history_sync_trigger = NULL; static mainloop_timer_t *stonith_history_sync_timer_short = NULL; static mainloop_timer_t *stonith_history_sync_timer_long = NULL; void te_cleanup_stonith_history_sync(stonith_t *st, bool free_timers) { if (free_timers) { mainloop_timer_del(stonith_history_sync_timer_short); stonith_history_sync_timer_short = NULL; mainloop_timer_del(stonith_history_sync_timer_long); stonith_history_sync_timer_long = NULL; } else { mainloop_timer_stop(stonith_history_sync_timer_short); mainloop_timer_stop(stonith_history_sync_timer_long); } if (st) { st->cmds->remove_notification(st, T_STONITH_NOTIFY_HISTORY_SYNCED); } } static void tengine_stonith_history_synced(stonith_t *st, stonith_event_t *st_event) { te_cleanup_stonith_history_sync(st, FALSE); crm_debug("Fence-history synced - cancel all timers"); } static gboolean stonith_history_sync_set_trigger(gpointer user_data) { mainloop_set_trigger(stonith_history_sync_trigger); return FALSE; } void te_trigger_stonith_history_sync(bool long_timeout) { /* trigger a sync in 5s to give more nodes the * chance to show up so that we don't create * unnecessary stonith-history-sync traffic * * the long timeout of 30s is there as a fallback * so that after a successful connection to fenced * we will wait for 30s for the DC to trigger a * history-sync * if this doesn't happen we trigger a sync locally * (e.g. fenced segfaults and is restarted by pacemakerd) */ /* as we are finally checking the stonith-connection * in do_stonith_history_sync we should be fine * leaving stonith_history_sync_time & stonith_history_sync_trigger * around */ if (stonith_history_sync_trigger == NULL) { stonith_history_sync_trigger = mainloop_add_trigger(G_PRIORITY_LOW, do_stonith_history_sync, NULL); } if (long_timeout) { if(stonith_history_sync_timer_long == NULL) { stonith_history_sync_timer_long = mainloop_timer_add("history_sync_long", 30000, FALSE, stonith_history_sync_set_trigger, NULL); } crm_info("Fence history will be synchronized cluster-wide within 30 seconds"); mainloop_timer_start(stonith_history_sync_timer_long); } else { if(stonith_history_sync_timer_short == NULL) { stonith_history_sync_timer_short = mainloop_timer_add("history_sync_short", 5000, FALSE, stonith_history_sync_set_trigger, NULL); } crm_info("Fence history will be synchronized cluster-wide within 5 seconds"); mainloop_timer_start(stonith_history_sync_timer_short); } } /* end stonith history synchronization functions */ diff --git a/daemons/controld/controld_te_actions.c b/daemons/controld/controld_te_actions.c index 2310b080c7..22fcfa1f2a 100644 --- a/daemons/controld/controld_te_actions.c +++ b/daemons/controld/controld_te_actions.c @@ -1,756 +1,757 @@ /* * Copyright 2004-2024 the Pacemaker project contributors * * The version control history for this file may have further details. * * This source code is licensed under the GNU General Public License version 2 * or later (GPLv2+) WITHOUT ANY WARRANTY. */ #include #include #include #include #include // lrmd_event_data_t, lrmd_free_event() #include #include #include #include #include static GHashTable *te_targets = NULL; void send_rsc_command(pcmk__graph_action_t *action); static void te_update_job_count(pcmk__graph_action_t *action, int offset); static void te_start_action_timer(const pcmk__graph_t *graph, pcmk__graph_action_t *action) { action->timer = g_timeout_add(action->timeout + graph->network_delay, action_timer_callback, (void *) action); CRM_ASSERT(action->timer != 0); } /*! * \internal * \brief Execute a graph pseudo-action * * \param[in,out] graph Transition graph being executed * \param[in,out] pseudo Pseudo-action to execute * * \return Standard Pacemaker return code */ static int execute_pseudo_action(pcmk__graph_t *graph, pcmk__graph_action_t *pseudo) { const char *task = crm_element_value(pseudo->xml, PCMK_XA_OPERATION); /* send to peers as well? */ if (pcmk__str_eq(task, PCMK_ACTION_MAINTENANCE_NODES, pcmk__str_casei)) { GHashTableIter iter; crm_node_t *node = NULL; g_hash_table_iter_init(&iter, crm_peer_cache); while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &node)) { xmlNode *cmd = NULL; if (pcmk__str_eq(controld_globals.our_nodename, node->uname, pcmk__str_casei)) { continue; } cmd = create_request(task, pseudo->xml, node->uname, CRM_SYSTEM_CRMD, CRM_SYSTEM_TENGINE, NULL); send_cluster_message(node, crm_msg_crmd, cmd, FALSE); free_xml(cmd); } remote_ra_process_maintenance_nodes(pseudo->xml); } else { /* Check action for Pacemaker Remote node side effects */ remote_ra_process_pseudo(pseudo->xml); } crm_debug("Pseudo-action %d (%s) fired and confirmed", pseudo->id, crm_element_value(pseudo->xml, PCMK__XA_OPERATION_KEY)); te_action_confirmed(pseudo, graph); return pcmk_rc_ok; } static int get_target_rc(pcmk__graph_action_t *action) { int exit_status; pcmk__scan_min_int(crm_meta_value(action->params, XML_ATTR_TE_TARGET_RC), &exit_status, 0); return exit_status; } /*! * \internal * \brief Execute a cluster action from a transition graph * * \param[in,out] graph Transition graph being executed * \param[in,out] action Cluster action to execute * * \return Standard Pacemaker return code */ static int execute_cluster_action(pcmk__graph_t *graph, pcmk__graph_action_t *action) { char *counter = NULL; xmlNode *cmd = NULL; gboolean is_local = FALSE; const char *id = NULL; const char *task = NULL; const char *value = NULL; const char *on_node = NULL; const char *router_node = NULL; gboolean rc = TRUE; gboolean no_wait = FALSE; id = ID(action->xml); CRM_CHECK(!pcmk__str_empty(id), return EPROTO); task = crm_element_value(action->xml, PCMK_XA_OPERATION); CRM_CHECK(!pcmk__str_empty(task), return EPROTO); on_node = crm_element_value(action->xml, PCMK__META_ON_NODE); CRM_CHECK(!pcmk__str_empty(on_node), return pcmk_rc_node_unknown); router_node = crm_element_value(action->xml, XML_LRM_ATTR_ROUTER_NODE); if (router_node == NULL) { router_node = on_node; if (pcmk__str_eq(task, PCMK_ACTION_LRM_DELETE, pcmk__str_none)) { const char *mode = crm_element_value(action->xml, PCMK__XA_MODE); if (pcmk__str_eq(mode, XML_TAG_CIB, pcmk__str_none)) { router_node = controld_globals.our_nodename; } } } if (pcmk__str_eq(router_node, controld_globals.our_nodename, pcmk__str_casei)) { is_local = TRUE; } value = crm_meta_value(action->params, XML_ATTR_TE_NOWAIT); if (crm_is_true(value)) { no_wait = TRUE; } crm_info("Handling controller request '%s' (%s on %s)%s%s", id, task, on_node, (is_local? " locally" : ""), (no_wait? " without waiting" : "")); if (is_local && pcmk__str_eq(task, PCMK_ACTION_DO_SHUTDOWN, pcmk__str_none)) { /* defer until everything else completes */ crm_info("Controller request '%s' is a local shutdown", id); graph->completion_action = pcmk__graph_shutdown; graph->abort_reason = "local shutdown"; te_action_confirmed(action, graph); return pcmk_rc_ok; } else if (pcmk__str_eq(task, PCMK_ACTION_DO_SHUTDOWN, pcmk__str_none)) { crm_node_t *peer = pcmk__get_node(0, router_node, NULL, pcmk__node_search_cluster); pcmk__update_peer_expected(__func__, peer, CRMD_JOINSTATE_DOWN); } cmd = create_request(task, action->xml, router_node, CRM_SYSTEM_CRMD, CRM_SYSTEM_TENGINE, NULL); counter = pcmk__transition_key(controld_globals.transition_graph->id, action->id, get_target_rc(action), controld_globals.te_uuid); crm_xml_add(cmd, PCMK__XA_TRANSITION_KEY, counter); rc = send_cluster_message(pcmk__get_node(0, router_node, NULL, pcmk__node_search_cluster), crm_msg_crmd, cmd, TRUE); free(counter); free_xml(cmd); if (rc == FALSE) { crm_err("Action %d failed: send", action->id); return ECOMM; } else if (no_wait) { te_action_confirmed(action, graph); } else { if (action->timeout <= 0) { crm_err("Action %d: %s on %s had an invalid timeout (%dms). Using %ums instead", action->id, task, on_node, action->timeout, graph->network_delay); action->timeout = (int) graph->network_delay; } te_start_action_timer(graph, action); } return pcmk_rc_ok; } /*! * \internal * \brief Synthesize an executor event for a resource action timeout * * \param[in] action Resource action that timed out * \param[in] target_rc Expected result of action that timed out * * Synthesize an executor event for a resource action timeout. (If the executor * gets a timeout while waiting for a resource action to complete, that will be * reported via the usual callback. This timeout means we didn't hear from the * executor itself or the controller that relayed the action to the executor.) * * \return Newly created executor event for result of \p action * \note The caller is responsible for freeing the return value using * lrmd_free_event(). */ static lrmd_event_data_t * synthesize_timeout_event(const pcmk__graph_action_t *action, int target_rc) { lrmd_event_data_t *op = NULL; const char *target = crm_element_value(action->xml, PCMK__META_ON_NODE); const char *reason = NULL; char *dynamic_reason = NULL; if (pcmk__str_eq(target, get_local_node_name(), pcmk__str_casei)) { reason = "Local executor did not return result in time"; } else { const char *router_node = NULL; router_node = crm_element_value(action->xml, XML_LRM_ATTR_ROUTER_NODE); if (router_node == NULL) { router_node = target; } dynamic_reason = crm_strdup_printf("Controller on %s did not return " "result in time", router_node); reason = dynamic_reason; } op = pcmk__event_from_graph_action(NULL, action, PCMK_EXEC_TIMEOUT, PCMK_OCF_UNKNOWN_ERROR, reason); op->call_id = -1; op->user_data = pcmk__transition_key(controld_globals.transition_graph->id, action->id, target_rc, controld_globals.te_uuid); free(dynamic_reason); return op; } static void controld_record_action_event(pcmk__graph_action_t *action, lrmd_event_data_t *op) { cib_t *cib_conn = controld_globals.cib_conn; xmlNode *state = NULL; xmlNode *rsc = NULL; xmlNode *action_rsc = NULL; int rc = pcmk_ok; const char *rsc_id = NULL; const char *target = crm_element_value(action->xml, PCMK__META_ON_NODE); const char *task_uuid = crm_element_value(action->xml, PCMK__XA_OPERATION_KEY); - const char *target_uuid = crm_element_value(action->xml, XML_LRM_ATTR_TARGET_UUID); + const char *target_uuid = crm_element_value(action->xml, + PCMK__META_ON_NODE_UUID); int target_rc = get_target_rc(action); action_rsc = find_xml_node(action->xml, XML_CIB_TAG_RESOURCE, TRUE); if (action_rsc == NULL) { return; } rsc_id = ID(action_rsc); CRM_CHECK(rsc_id != NULL, crm_log_xml_err(action->xml, "Bad:action"); return); /* update the CIB */ state = create_xml_node(NULL, XML_CIB_TAG_STATE); crm_xml_add(state, PCMK_XA_ID, target_uuid); crm_xml_add(state, PCMK_XA_UNAME, target); rsc = create_xml_node(state, XML_CIB_TAG_LRM); crm_xml_add(rsc, PCMK_XA_ID, target_uuid); rsc = create_xml_node(rsc, XML_LRM_TAG_RESOURCES); rsc = create_xml_node(rsc, XML_LRM_TAG_RESOURCE); crm_xml_add(rsc, PCMK_XA_ID, rsc_id); crm_copy_xml_element(action_rsc, rsc, PCMK_XA_TYPE); crm_copy_xml_element(action_rsc, rsc, PCMK_XA_CLASS); crm_copy_xml_element(action_rsc, rsc, PCMK_XA_PROVIDER); pcmk__create_history_xml(rsc, op, CRM_FEATURE_SET, target_rc, target, __func__); rc = cib_conn->cmds->modify(cib_conn, XML_CIB_TAG_STATUS, state, cib_scope_local); fsa_register_cib_callback(rc, NULL, cib_action_updated); free_xml(state); crm_trace("Sent CIB update (call ID %d) for synthesized event of action %d (%s on %s)", rc, action->id, task_uuid, target); pcmk__set_graph_action_flags(action, pcmk__graph_action_sent_update); } void controld_record_action_timeout(pcmk__graph_action_t *action) { lrmd_event_data_t *op = NULL; const char *target = crm_element_value(action->xml, PCMK__META_ON_NODE); const char *task_uuid = crm_element_value(action->xml, PCMK__XA_OPERATION_KEY); int target_rc = get_target_rc(action); crm_warn("%s %d: %s on %s timed out", action->xml->name, action->id, task_uuid, target); op = synthesize_timeout_event(action, target_rc); controld_record_action_event(action, op); lrmd_free_event(op); } /*! * \internal * \brief Execute a resource action from a transition graph * * \param[in,out] graph Transition graph being executed * \param[in,out] action Resource action to execute * * \return Standard Pacemaker return code */ static int execute_rsc_action(pcmk__graph_t *graph, pcmk__graph_action_t *action) { /* never overwrite stop actions in the CIB with * anything other than completed results * * Writing pending stops makes it look like the * resource is running again */ xmlNode *cmd = NULL; xmlNode *rsc_op = NULL; gboolean rc = TRUE; gboolean no_wait = FALSE; gboolean is_local = FALSE; char *counter = NULL; const char *task = NULL; const char *value = NULL; const char *on_node = NULL; const char *router_node = NULL; const char *task_uuid = NULL; CRM_ASSERT(action != NULL); CRM_ASSERT(action->xml != NULL); pcmk__clear_graph_action_flags(action, pcmk__graph_action_executed); on_node = crm_element_value(action->xml, PCMK__META_ON_NODE); CRM_CHECK(!pcmk__str_empty(on_node), crm_err("Corrupted command(id=%s) %s: no node", ID(action->xml), pcmk__s(task, "without task")); return pcmk_rc_node_unknown); rsc_op = action->xml; task = crm_element_value(rsc_op, PCMK_XA_OPERATION); task_uuid = crm_element_value(action->xml, PCMK__XA_OPERATION_KEY); router_node = crm_element_value(rsc_op, XML_LRM_ATTR_ROUTER_NODE); if (!router_node) { router_node = on_node; } counter = pcmk__transition_key(controld_globals.transition_graph->id, action->id, get_target_rc(action), controld_globals.te_uuid); crm_xml_add(rsc_op, PCMK__XA_TRANSITION_KEY, counter); if (pcmk__str_eq(router_node, controld_globals.our_nodename, pcmk__str_casei)) { is_local = TRUE; } value = crm_meta_value(action->params, XML_ATTR_TE_NOWAIT); if (crm_is_true(value)) { no_wait = TRUE; } crm_notice("Initiating %s operation %s%s on %s%s "CRM_XS" action %d", task, task_uuid, (is_local? " locally" : ""), on_node, (no_wait? " without waiting" : ""), action->id); cmd = create_request(CRM_OP_INVOKE_LRM, rsc_op, router_node, CRM_SYSTEM_LRMD, CRM_SYSTEM_TENGINE, NULL); if (is_local) { /* shortcut local resource commands */ ha_msg_input_t data = { .msg = cmd, .xml = rsc_op, }; fsa_data_t msg = { .id = 0, .data = &data, .data_type = fsa_dt_ha_msg, .fsa_input = I_NULL, .fsa_cause = C_FSA_INTERNAL, .actions = A_LRM_INVOKE, .origin = __func__, }; do_lrm_invoke(A_LRM_INVOKE, C_FSA_INTERNAL, controld_globals.fsa_state, I_NULL, &msg); } else { rc = send_cluster_message(pcmk__get_node(0, router_node, NULL, pcmk__node_search_cluster), crm_msg_lrmd, cmd, TRUE); } free(counter); free_xml(cmd); pcmk__set_graph_action_flags(action, pcmk__graph_action_executed); if (rc == FALSE) { crm_err("Action %d failed: send", action->id); return ECOMM; } else if (no_wait) { /* Just mark confirmed. Don't bump the job count only to immediately * decrement it. */ crm_info("Action %d confirmed - no wait", action->id); pcmk__set_graph_action_flags(action, pcmk__graph_action_confirmed); pcmk__update_graph(controld_globals.transition_graph, action); trigger_graph(); } else if (pcmk_is_set(action->flags, pcmk__graph_action_confirmed)) { crm_debug("Action %d: %s %s on %s(timeout %dms) was already confirmed.", action->id, task, task_uuid, on_node, action->timeout); } else { if (action->timeout <= 0) { crm_err("Action %d: %s %s on %s had an invalid timeout (%dms). Using %ums instead", action->id, task, task_uuid, on_node, action->timeout, graph->network_delay); action->timeout = (int) graph->network_delay; } te_update_job_count(action, 1); te_start_action_timer(graph, action); } return pcmk_rc_ok; } struct te_peer_s { char *name; int jobs; int migrate_jobs; }; static void te_peer_free(gpointer p) { struct te_peer_s *peer = p; free(peer->name); free(peer); } void te_reset_job_counts(void) { GHashTableIter iter; struct te_peer_s *peer = NULL; if(te_targets == NULL) { te_targets = pcmk__strkey_table(NULL, te_peer_free); } g_hash_table_iter_init(&iter, te_targets); while (g_hash_table_iter_next(&iter, NULL, (gpointer *) & peer)) { peer->jobs = 0; peer->migrate_jobs = 0; } } static void te_update_job_count_on(const char *target, int offset, bool migrate) { struct te_peer_s *r = NULL; if(target == NULL || te_targets == NULL) { return; } r = g_hash_table_lookup(te_targets, target); if(r == NULL) { r = calloc(1, sizeof(struct te_peer_s)); r->name = strdup(target); g_hash_table_insert(te_targets, r->name, r); } r->jobs += offset; if(migrate) { r->migrate_jobs += offset; } crm_trace("jobs[%s] = %d", target, r->jobs); } static void te_update_job_count(pcmk__graph_action_t *action, int offset) { const char *task = crm_element_value(action->xml, PCMK_XA_OPERATION); const char *target = crm_element_value(action->xml, PCMK__META_ON_NODE); if ((action->type != pcmk__rsc_graph_action) || (target == NULL)) { /* No limit on these */ return; } /* if we have a router node, this means the action is performing * on a remote node. For now, we count all actions occurring on a * remote node against the job list on the cluster node hosting * the connection resources */ target = crm_element_value(action->xml, XML_LRM_ATTR_ROUTER_NODE); if ((target == NULL) && pcmk__strcase_any_of(task, PCMK_ACTION_MIGRATE_TO, PCMK_ACTION_MIGRATE_FROM, NULL)) { const char *t1 = crm_meta_value(action->params, XML_LRM_ATTR_MIGRATE_SOURCE); const char *t2 = crm_meta_value(action->params, XML_LRM_ATTR_MIGRATE_TARGET); te_update_job_count_on(t1, offset, TRUE); te_update_job_count_on(t2, offset, TRUE); return; } else if (target == NULL) { target = crm_element_value(action->xml, PCMK__META_ON_NODE); } te_update_job_count_on(target, offset, FALSE); } /*! * \internal * \brief Check whether a graph action is allowed to be executed on a node * * \param[in] graph Transition graph being executed * \param[in] action Graph action being executed * \param[in] target Name of node where action should be executed * * \return true if action is allowed, otherwise false */ static bool allowed_on_node(const pcmk__graph_t *graph, const pcmk__graph_action_t *action, const char *target) { int limit = 0; struct te_peer_s *r = NULL; const char *task = crm_element_value(action->xml, PCMK_XA_OPERATION); const char *id = crm_element_value(action->xml, PCMK__XA_OPERATION_KEY); if(target == NULL) { /* No limit on these */ return true; } else if(te_targets == NULL) { return false; } r = g_hash_table_lookup(te_targets, target); limit = throttle_get_job_limit(target); if(r == NULL) { r = calloc(1, sizeof(struct te_peer_s)); r->name = strdup(target); g_hash_table_insert(te_targets, r->name, r); } if(limit <= r->jobs) { crm_trace("Peer %s is over their job limit of %d (%d): deferring %s", target, limit, r->jobs, id); return false; } else if(graph->migration_limit > 0 && r->migrate_jobs >= graph->migration_limit) { if (pcmk__strcase_any_of(task, PCMK_ACTION_MIGRATE_TO, PCMK_ACTION_MIGRATE_FROM, NULL)) { crm_trace("Peer %s is over their migration job limit of %d (%d): deferring %s", target, graph->migration_limit, r->migrate_jobs, id); return false; } } crm_trace("Peer %s has not hit their limit yet. current jobs = %d limit= %d limit", target, r->jobs, limit); return true; } /*! * \internal * \brief Check whether a graph action is allowed to be executed * * \param[in] graph Transition graph being executed * \param[in] action Graph action being executed * * \return true if action is allowed, otherwise false */ static bool graph_action_allowed(pcmk__graph_t *graph, pcmk__graph_action_t *action) { const char *target = NULL; const char *task = crm_element_value(action->xml, PCMK_XA_OPERATION); if (action->type != pcmk__rsc_graph_action) { /* No limit on these */ return true; } /* if we have a router node, this means the action is performing * on a remote node. For now, we count all actions occurring on a * remote node against the job list on the cluster node hosting * the connection resources */ target = crm_element_value(action->xml, XML_LRM_ATTR_ROUTER_NODE); if ((target == NULL) && pcmk__strcase_any_of(task, PCMK_ACTION_MIGRATE_TO, PCMK_ACTION_MIGRATE_FROM, NULL)) { target = crm_meta_value(action->params, XML_LRM_ATTR_MIGRATE_SOURCE); if (!allowed_on_node(graph, action, target)) { return false; } target = crm_meta_value(action->params, XML_LRM_ATTR_MIGRATE_TARGET); } else if (target == NULL) { target = crm_element_value(action->xml, PCMK__META_ON_NODE); } return allowed_on_node(graph, action, target); } /*! * \brief Confirm a graph action (and optionally update graph) * * \param[in,out] action Action to confirm * \param[in,out] graph Update and trigger this graph (if non-NULL) */ void te_action_confirmed(pcmk__graph_action_t *action, pcmk__graph_t *graph) { if (!pcmk_is_set(action->flags, pcmk__graph_action_confirmed)) { if ((action->type == pcmk__rsc_graph_action) && (crm_element_value(action->xml, PCMK__META_ON_NODE) != NULL)) { te_update_job_count(action, -1); } pcmk__set_graph_action_flags(action, pcmk__graph_action_confirmed); } if (graph) { pcmk__update_graph(graph, action); trigger_graph(); } } static pcmk__graph_functions_t te_graph_fns = { execute_pseudo_action, execute_rsc_action, execute_cluster_action, controld_execute_fence_action, graph_action_allowed, }; /* * \internal * \brief Register the transitioner's graph functions with \p libpacemaker */ void controld_register_graph_functions(void) { pcmk__set_graph_functions(&te_graph_fns); } void notify_crmd(pcmk__graph_t *graph) { const char *type = "unknown"; enum crmd_fsa_input event = I_NULL; crm_debug("Processing transition completion in state %s", fsa_state2string(controld_globals.fsa_state)); CRM_CHECK(graph->complete, graph->complete = true); switch (graph->completion_action) { case pcmk__graph_wait: type = "stop"; if (controld_globals.fsa_state == S_TRANSITION_ENGINE) { event = I_TE_SUCCESS; } break; case pcmk__graph_done: type = "done"; if (controld_globals.fsa_state == S_TRANSITION_ENGINE) { event = I_TE_SUCCESS; } break; case pcmk__graph_restart: type = "restart"; if (controld_globals.fsa_state == S_TRANSITION_ENGINE) { if (controld_get_period_transition_timer() > 0) { controld_stop_transition_timer(); controld_start_transition_timer(); } else { event = I_PE_CALC; } } else if (controld_globals.fsa_state == S_POLICY_ENGINE) { controld_set_fsa_action_flags(A_PE_INVOKE); controld_trigger_fsa(); } break; case pcmk__graph_shutdown: type = "shutdown"; if (pcmk_is_set(controld_globals.fsa_input_register, R_SHUTDOWN)) { event = I_STOP; } else { crm_err("We didn't ask to be shut down, yet the scheduler is telling us to"); event = I_TERMINATE; } } crm_debug("Transition %d status: %s - %s", graph->id, type, pcmk__s(graph->abort_reason, "unspecified reason")); graph->abort_reason = NULL; graph->completion_action = pcmk__graph_done; if (event != I_NULL) { register_fsa_input(C_FSA_INTERNAL, event, NULL); } else { controld_trigger_fsa(); } } diff --git a/daemons/controld/controld_te_events.c b/daemons/controld/controld_te_events.c index 762eed2677..6fa59f00ec 100644 --- a/daemons/controld/controld_te_events.c +++ b/daemons/controld/controld_te_events.c @@ -1,611 +1,612 @@ /* * Copyright 2004-2024 the Pacemaker project contributors * * The version control history for this file may have further details. * * This source code is licensed under the GNU General Public License version 2 * or later (GPLv2+) WITHOUT ANY WARRANTY. */ #include #include #include #include #include #include #include #include #include /*! * \internal * \brief Action numbers of outside events processed in current update diff * * This table is to be used as a set. It should be empty when the transitioner * begins processing a CIB update diff. It ensures that if there are multiple * events (for example, "_last_0" and "_last_failure_0") for the same action, * only one of them updates the failcount. Events that originate outside the * cluster can't be confirmed, since they're not in the transition graph. */ static GHashTable *outside_events = NULL; /*! * \internal * \brief Empty the hash table containing action numbers of outside events */ void controld_remove_all_outside_events(void) { if (outside_events != NULL) { g_hash_table_remove_all(outside_events); } } /*! * \internal * \brief Destroy the hash table containing action numbers of outside events */ void controld_destroy_outside_events_table(void) { if (outside_events != NULL) { g_hash_table_destroy(outside_events); outside_events = NULL; } } /*! * \internal * \brief Add an outside event's action number to a set * * \return Standard Pacemaker return code. Specifically, \p pcmk_rc_ok if the * event was not already in the set, or \p pcmk_rc_already otherwise. */ static int record_outside_event(gint action_num) { if (outside_events == NULL) { outside_events = g_hash_table_new(NULL, NULL); } if (g_hash_table_add(outside_events, GINT_TO_POINTER(action_num))) { return pcmk_rc_ok; } return pcmk_rc_already; } gboolean fail_incompletable_actions(pcmk__graph_t *graph, const char *down_node) { const char *target_uuid = NULL; const char *router = NULL; const char *router_uuid = NULL; xmlNode *last_action = NULL; GList *gIter = NULL; GList *gIter2 = NULL; if (graph == NULL || graph->complete) { return FALSE; } gIter = graph->synapses; for (; gIter != NULL; gIter = gIter->next) { pcmk__graph_synapse_t *synapse = (pcmk__graph_synapse_t *) gIter->data; if (pcmk_any_flags_set(synapse->flags, pcmk__synapse_confirmed|pcmk__synapse_failed)) { /* We've already been here */ continue; } gIter2 = synapse->actions; for (; gIter2 != NULL; gIter2 = gIter2->next) { pcmk__graph_action_t *action = (pcmk__graph_action_t *) gIter2->data; if ((action->type == pcmk__pseudo_graph_action) || pcmk_is_set(action->flags, pcmk__graph_action_confirmed)) { continue; } else if (action->type == pcmk__cluster_graph_action) { const char *task = crm_element_value(action->xml, PCMK_XA_OPERATION); if (pcmk__str_eq(task, PCMK_ACTION_STONITH, pcmk__str_casei)) { continue; } } - target_uuid = crm_element_value(action->xml, XML_LRM_ATTR_TARGET_UUID); + target_uuid = crm_element_value(action->xml, + PCMK__META_ON_NODE_UUID); router = crm_element_value(action->xml, XML_LRM_ATTR_ROUTER_NODE); if (router) { crm_node_t *node = pcmk__get_node(0, router, NULL, pcmk__node_search_cluster); if (node) { router_uuid = node->uuid; } } if (pcmk__str_eq(target_uuid, down_node, pcmk__str_casei) || pcmk__str_eq(router_uuid, down_node, pcmk__str_casei)) { pcmk__set_graph_action_flags(action, pcmk__graph_action_failed); pcmk__set_synapse_flags(synapse, pcmk__synapse_failed); last_action = action->xml; stop_te_timer(action); pcmk__update_graph(graph, action); if (pcmk_is_set(synapse->flags, pcmk__synapse_executed)) { crm_notice("Action %d (%s) was pending on %s (offline)", action->id, crm_element_value(action->xml, PCMK__XA_OPERATION_KEY), down_node); } else { crm_info("Action %d (%s) is scheduled for %s (offline)", action->id, crm_element_value(action->xml, PCMK__XA_OPERATION_KEY), down_node); } } } } if (last_action != NULL) { crm_info("Node %s shutdown resulted in un-runnable actions", down_node); abort_transition(INFINITY, pcmk__graph_restart, "Node failure", last_action); return TRUE; } return FALSE; } /*! * \internal * \brief Update failure-related node attributes if warranted * * \param[in] event XML describing operation that (maybe) failed * \param[in] event_node_uuid Node that event occurred on * \param[in] rc Actual operation return code * \param[in] target_rc Expected operation return code * \param[in] do_update If TRUE, do update regardless of operation type * \param[in] ignore_failures If TRUE, update last failure but not fail count * * \return TRUE if this was not a direct nack, success or lrm status refresh */ static gboolean update_failcount(const xmlNode *event, const char *event_node_uuid, int rc, int target_rc, gboolean do_update, gboolean ignore_failures) { guint interval_ms = 0; char *task = NULL; char *rsc_id = NULL; const char *value = NULL; const char *id = crm_element_value(event, PCMK__XA_OPERATION_KEY); const char *on_uname = crm_peer_uname(event_node_uuid); const char *origin = crm_element_value(event, PCMK_XA_CRM_DEBUG_ORIGIN); // Nothing needs to be done for success or status refresh if (rc == target_rc) { return FALSE; } else if (pcmk__str_eq(origin, "build_active_RAs", pcmk__str_casei)) { crm_debug("No update for %s (rc=%d) on %s: Old failure from lrm status refresh", id, rc, on_uname); return FALSE; } /* Sanity check */ CRM_CHECK(on_uname != NULL, return TRUE); CRM_CHECK(parse_op_key(id, &rsc_id, &task, &interval_ms), crm_err("Couldn't parse: %s", ID(event)); goto bail); /* Decide whether update is necessary and what value to use */ if ((interval_ms > 0) || pcmk__str_eq(task, PCMK_ACTION_PROMOTE, pcmk__str_none) || pcmk__str_eq(task, PCMK_ACTION_DEMOTE, pcmk__str_none)) { do_update = TRUE; } else if (pcmk__str_eq(task, PCMK_ACTION_START, pcmk__str_none)) { do_update = TRUE; value = pcmk__s(controld_globals.transition_graph->failed_start_offset, CRM_INFINITY_S); } else if (pcmk__str_eq(task, PCMK_ACTION_STOP, pcmk__str_none)) { do_update = TRUE; value = pcmk__s(controld_globals.transition_graph->failed_stop_offset, CRM_INFINITY_S); } if (do_update) { pcmk__attrd_query_pair_t *fail_pair = NULL; pcmk__attrd_query_pair_t *last_pair = NULL; char *fail_name = NULL; char *last_name = NULL; GList *attrs = NULL; uint32_t opts = pcmk__node_attr_none; char *now = pcmk__ttoa(time(NULL)); // Fail count will be either incremented or set to infinity if (!pcmk_str_is_infinity(value)) { value = PCMK_XA_VALUE "++"; } if (g_hash_table_lookup(crm_remote_peer_cache, event_node_uuid)) { opts |= pcmk__node_attr_remote; } crm_info("Updating %s for %s on %s after failed %s: rc=%d (update=%s, time=%s)", (ignore_failures? "last failure" : "failcount"), rsc_id, on_uname, task, rc, value, now); /* Update the fail count, if we're not ignoring failures */ if (!ignore_failures) { fail_pair = calloc(1, sizeof(pcmk__attrd_query_pair_t)); CRM_ASSERT(fail_pair != NULL); fail_name = pcmk__failcount_name(rsc_id, task, interval_ms); fail_pair->name = fail_name; fail_pair->value = value; fail_pair->node = on_uname; attrs = g_list_prepend(attrs, fail_pair); } /* Update the last failure time (even if we're ignoring failures, * so that failure can still be detected and shown, e.g. by crm_mon) */ last_pair = calloc(1, sizeof(pcmk__attrd_query_pair_t)); CRM_ASSERT(last_pair != NULL); last_name = pcmk__lastfailure_name(rsc_id, task, interval_ms); last_pair->name = last_name; last_pair->value = now; last_pair->node = on_uname; attrs = g_list_prepend(attrs, last_pair); update_attrd_list(attrs, opts); free(fail_name); free(fail_pair); free(last_name); free(last_pair); g_list_free(attrs); free(now); } bail: free(rsc_id); free(task); return TRUE; } pcmk__graph_action_t * controld_get_action(int id) { for (GList *item = controld_globals.transition_graph->synapses; item != NULL; item = item->next) { pcmk__graph_synapse_t *synapse = (pcmk__graph_synapse_t *) item->data; for (GList *item2 = synapse->actions; item2; item2 = item2->next) { pcmk__graph_action_t *action = (pcmk__graph_action_t *) item2->data; if (action->id == id) { return action; } } } return NULL; } pcmk__graph_action_t * get_cancel_action(const char *id, const char *node) { GList *gIter = NULL; GList *gIter2 = NULL; gIter = controld_globals.transition_graph->synapses; for (; gIter != NULL; gIter = gIter->next) { pcmk__graph_synapse_t *synapse = (pcmk__graph_synapse_t *) gIter->data; gIter2 = synapse->actions; for (; gIter2 != NULL; gIter2 = gIter2->next) { const char *task = NULL; const char *target = NULL; pcmk__graph_action_t *action = (pcmk__graph_action_t *) gIter2->data; task = crm_element_value(action->xml, PCMK_XA_OPERATION); if (!pcmk__str_eq(PCMK_ACTION_CANCEL, task, pcmk__str_casei)) { continue; } task = crm_element_value(action->xml, PCMK__XA_OPERATION_KEY); if (!pcmk__str_eq(task, id, pcmk__str_casei)) { crm_trace("Wrong key %s for %s on %s", task, id, node); continue; } - target = crm_element_value(action->xml, XML_LRM_ATTR_TARGET_UUID); + target = crm_element_value(action->xml, PCMK__META_ON_NODE_UUID); if (node && !pcmk__str_eq(target, node, pcmk__str_casei)) { crm_trace("Wrong node %s for %s on %s", target, id, node); continue; } crm_trace("Found %s on %s", id, node); return action; } } return NULL; } bool confirm_cancel_action(const char *id, const char *node_id) { const char *op_key = NULL; const char *node_name = NULL; pcmk__graph_action_t *cancel = get_cancel_action(id, node_id); if (cancel == NULL) { return FALSE; } op_key = crm_element_value(cancel->xml, PCMK__XA_OPERATION_KEY); node_name = crm_element_value(cancel->xml, PCMK__META_ON_NODE); stop_te_timer(cancel); te_action_confirmed(cancel, controld_globals.transition_graph); crm_info("Cancellation of %s on %s confirmed (action %d)", op_key, node_name, cancel->id); return TRUE; } /* downed nodes are listed like: ... */ #define XPATH_DOWNED "//" XML_GRAPH_TAG_DOWNED \ "/" XML_CIB_TAG_NODE "[@" PCMK_XA_ID "='%s']" /*! * \brief Find a transition event that would have made a specified node down * * \param[in] target UUID of node to match * * \return Matching event if found, NULL otherwise */ pcmk__graph_action_t * match_down_event(const char *target) { pcmk__graph_action_t *match = NULL; xmlXPathObjectPtr xpath_ret = NULL; GList *gIter, *gIter2; char *xpath = crm_strdup_printf(XPATH_DOWNED, target); for (gIter = controld_globals.transition_graph->synapses; gIter != NULL && match == NULL; gIter = gIter->next) { for (gIter2 = ((pcmk__graph_synapse_t * ) gIter->data)->actions; gIter2 != NULL && match == NULL; gIter2 = gIter2->next) { match = (pcmk__graph_action_t *) gIter2->data; if (pcmk_is_set(match->flags, pcmk__graph_action_executed)) { xpath_ret = xpath_search(match->xml, xpath); if (numXpathResults(xpath_ret) < 1) { match = NULL; } freeXpathObject(xpath_ret); } else { // Only actions that were actually started can match match = NULL; } } } free(xpath); if (match != NULL) { crm_debug("Shutdown action %d (%s) found for node %s", match->id, crm_element_value(match->xml, PCMK__XA_OPERATION_KEY), target); } else { crm_debug("No reason to expect node %s to be down", target); } return match; } void process_graph_event(xmlNode *event, const char *event_node) { int rc = -1; // Actual result int target_rc = -1; // Expected result int status = -1; // Executor status int callid = -1; // Executor call ID int transition_num = -1; // Transition number int action_num = -1; // Action number within transition char *update_te_uuid = NULL; bool ignore_failures = FALSE; const char *id = NULL; const char *desc = NULL; const char *magic = NULL; const char *uname = NULL; CRM_ASSERT(event != NULL); /* */ magic = crm_element_value(event, PCMK__XA_TRANSITION_KEY); if (magic == NULL) { /* non-change */ return; } crm_element_value_int(event, XML_LRM_ATTR_OPSTATUS, &status); if (status == PCMK_EXEC_PENDING) { return; } id = crm_element_value(event, PCMK__XA_OPERATION_KEY); crm_element_value_int(event, XML_LRM_ATTR_RC, &rc); crm_element_value_int(event, XML_LRM_ATTR_CALLID, &callid); rc = pcmk__effective_rc(rc); if (decode_transition_key(magic, &update_te_uuid, &transition_num, &action_num, &target_rc) == FALSE) { // decode_transition_key() already logged the bad key crm_err("Can't process action %s result: Incompatible versions? " CRM_XS " call-id=%d", id, callid); abort_transition(INFINITY, pcmk__graph_restart, "Bad event", event); return; } if (transition_num == -1) { // E.g. crm_resource --fail if (record_outside_event(action_num) != pcmk_rc_ok) { crm_debug("Outside event with transition key '%s' has already been " "processed", magic); goto bail; } desc = "initiated outside of the cluster"; abort_transition(INFINITY, pcmk__graph_restart, "Unexpected event", event); } else if ((action_num < 0) || !pcmk__str_eq(update_te_uuid, controld_globals.te_uuid, pcmk__str_none)) { desc = "initiated by a different DC"; abort_transition(INFINITY, pcmk__graph_restart, "Foreign event", event); } else if ((controld_globals.transition_graph->id != transition_num) || controld_globals.transition_graph->complete) { // Action is not from currently active transition guint interval_ms = 0; if (parse_op_key(id, NULL, NULL, &interval_ms) && (interval_ms != 0)) { /* Recurring actions have the transition number they were first * scheduled in. */ if (status == PCMK_EXEC_CANCELLED) { confirm_cancel_action(id, get_node_id(event)); goto bail; } desc = "arrived after initial scheduling"; abort_transition(INFINITY, pcmk__graph_restart, "Change in recurring result", event); } else if (controld_globals.transition_graph->id != transition_num) { desc = "arrived really late"; abort_transition(INFINITY, pcmk__graph_restart, "Old event", event); } else { desc = "arrived late"; abort_transition(INFINITY, pcmk__graph_restart, "Inactive graph", event); } } else { // Event is result of an action from currently active transition pcmk__graph_action_t *action = controld_get_action(action_num); if (action == NULL) { // Should never happen desc = "unknown"; abort_transition(INFINITY, pcmk__graph_restart, "Unknown event", event); } else if (pcmk_is_set(action->flags, pcmk__graph_action_confirmed)) { /* Nothing further needs to be done if the action has already been * confirmed. This can happen e.g. when processing both an * "xxx_last_0" or "xxx_last_failure_0" record as well as the main * history record, which would otherwise result in incorrectly * bumping the fail count twice. */ crm_log_xml_debug(event, "Event already confirmed:"); goto bail; } else { /* An action result needs to be confirmed. * (This is the only case where desc == NULL.) */ if (pcmk__str_eq(crm_meta_value(action->params, PCMK_META_ON_FAIL), "ignore", pcmk__str_casei)) { ignore_failures = TRUE; } else if (rc != target_rc) { pcmk__set_graph_action_flags(action, pcmk__graph_action_failed); } stop_te_timer(action); te_action_confirmed(action, controld_globals.transition_graph); if (pcmk_is_set(action->flags, pcmk__graph_action_failed)) { abort_transition(action->synapse->priority + 1, pcmk__graph_restart, "Event failed", event); } } } if (id == NULL) { id = "unknown action"; } uname = crm_element_value(event, PCMK__META_ON_NODE); if (uname == NULL) { uname = "unknown node"; } if (status == PCMK_EXEC_INVALID) { // We couldn't attempt the action crm_info("Transition %d action %d (%s on %s): %s", transition_num, action_num, id, uname, pcmk_exec_status_str(status)); } else if (desc && update_failcount(event, event_node, rc, target_rc, (transition_num == -1), FALSE)) { crm_notice("Transition %d action %d (%s on %s): expected '%s' but got '%s' " CRM_XS " target-rc=%d rc=%d call-id=%d event='%s'", transition_num, action_num, id, uname, services_ocf_exitcode_str(target_rc), services_ocf_exitcode_str(rc), target_rc, rc, callid, desc); } else if (desc) { crm_info("Transition %d action %d (%s on %s): %s " CRM_XS " rc=%d target-rc=%d call-id=%d", transition_num, action_num, id, uname, desc, rc, target_rc, callid); } else if (rc == target_rc) { crm_info("Transition %d action %d (%s on %s) confirmed: %s " CRM_XS " rc=%d call-id=%d", transition_num, action_num, id, uname, services_ocf_exitcode_str(rc), rc, callid); } else { update_failcount(event, event_node, rc, target_rc, (transition_num == -1), ignore_failures); crm_notice("Transition %d action %d (%s on %s): expected '%s' but got '%s' " CRM_XS " target-rc=%d rc=%d call-id=%d", transition_num, action_num, id, uname, services_ocf_exitcode_str(target_rc), services_ocf_exitcode_str(rc), target_rc, rc, callid); } bail: free(update_te_uuid); } diff --git a/include/crm/common/options_internal.h b/include/crm/common/options_internal.h index f59a76e41e..22d3b6723a 100644 --- a/include/crm/common/options_internal.h +++ b/include/crm/common/options_internal.h @@ -1,179 +1,180 @@ /* * Copyright 2006-2024 the Pacemaker project contributors * * The version control history for this file may have further details. * * This source code is licensed under the GNU Lesser General Public License * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. */ #ifndef PCMK__OPTIONS_INTERNAL__H # define PCMK__OPTIONS_INTERNAL__H # ifndef PCMK__CONFIG_H # define PCMK__CONFIG_H # include // _Noreturn # endif # include // GHashTable # include // bool #include // pcmk_parse_interval_spec() _Noreturn void pcmk__cli_help(char cmd); /* * Environment variable option handling */ const char *pcmk__env_option(const char *option); void pcmk__set_env_option(const char *option, const char *value, bool compat); bool pcmk__env_option_enabled(const char *daemon, const char *option); /* * Cluster option handling */ typedef struct pcmk__cluster_option_s { const char *name; const char *alt_name; const char *type; const char *values; const char *default_value; bool (*is_valid)(const char *); const char *description_short; const char *description_long; } pcmk__cluster_option_t; const char *pcmk__cluster_option(GHashTable *options, const pcmk__cluster_option_t *option_list, int len, const char *name); gchar *pcmk__format_option_metadata(const char *name, const char *desc_short, const char *desc_long, pcmk__cluster_option_t *option_list, int len); void pcmk__validate_cluster_options(GHashTable *options, pcmk__cluster_option_t *option_list, int len); bool pcmk__valid_interval_spec(const char *value); bool pcmk__valid_boolean(const char *value); bool pcmk__valid_int(const char *value); bool pcmk__valid_positive_int(const char *value); bool pcmk__valid_no_quorum_policy(const char *value); bool pcmk__valid_percentage(const char *value); bool pcmk__valid_script(const char *value); // from watchdog.c long pcmk__get_sbd_timeout(void); bool pcmk__get_sbd_sync_resource_startup(void); long pcmk__auto_watchdog_timeout(void); bool pcmk__valid_sbd_timeout(const char *value); // Constants for environment variable names #define PCMK__ENV_AUTHKEY_LOCATION "authkey_location" #define PCMK__ENV_BLACKBOX "blackbox" #define PCMK__ENV_CALLGRIND_ENABLED "callgrind_enabled" #define PCMK__ENV_CLUSTER_TYPE "cluster_type" #define PCMK__ENV_DEBUG "debug" #define PCMK__ENV_DH_MAX_BITS "dh_max_bits" #define PCMK__ENV_DH_MIN_BITS "dh_min_bits" #define PCMK__ENV_FAIL_FAST "fail_fast" #define PCMK__ENV_IPC_BUFFER "ipc_buffer" #define PCMK__ENV_IPC_TYPE "ipc_type" #define PCMK__ENV_LOGFACILITY "logfacility" #define PCMK__ENV_LOGFILE "logfile" #define PCMK__ENV_LOGFILE_MODE "logfile_mode" #define PCMK__ENV_LOGPRIORITY "logpriority" #define PCMK__ENV_NODE_ACTION_LIMIT "node_action_limit" #define PCMK__ENV_NODE_START_STATE "node_start_state" #define PCMK__ENV_PANIC_ACTION "panic_action" #define PCMK__ENV_REMOTE_ADDRESS "remote_address" #define PCMK__ENV_REMOTE_SCHEMA_DIR "remote_schema_directory" #define PCMK__ENV_REMOTE_PID1 "remote_pid1" #define PCMK__ENV_REMOTE_PORT "remote_port" #define PCMK__ENV_RESPAWNED "respawned" #define PCMK__ENV_SCHEMA_DIRECTORY "schema_directory" #define PCMK__ENV_SERVICE "service" #define PCMK__ENV_STDERR "stderr" #define PCMK__ENV_TLS_PRIORITIES "tls_priorities" #define PCMK__ENV_TRACE_BLACKBOX "trace_blackbox" #define PCMK__ENV_TRACE_FILES "trace_files" #define PCMK__ENV_TRACE_FORMATS "trace_formats" #define PCMK__ENV_TRACE_FUNCTIONS "trace_functions" #define PCMK__ENV_TRACE_TAGS "trace_tags" #define PCMK__ENV_VALGRIND_ENABLED "valgrind_enabled" // @COMPAT Drop at 3.0.0; default is plenty #define PCMK__ENV_CIB_TIMEOUT "cib_timeout" // @COMPAT Drop at 3.0.0; likely last used in 1.1.24 #define PCMK__ENV_MCP "mcp" // @COMPAT Drop at 3.0.0; added unused in 1.1.9 #define PCMK__ENV_QUORUM_TYPE "quorum_type" /* @COMPAT Drop at 3.0.0; added to debug shutdown issues when Pacemaker is * managed by systemd, but no longer useful. */ #define PCMK__ENV_SHUTDOWN_DELAY "shutdown_delay" // @COMPAT Deprecated since 2.1.0 #define PCMK__OPT_REMOVE_AFTER_STOP "remove-after-stop" // Constants for meta-attribute names #define PCMK__META_CLONE_INSTANCE_NUM "clone" #define PCMK__META_CONTAINER "container" #define PCMK__META_DIGESTS_ALL "digests-all" #define PCMK__META_DIGESTS_SECURE "digests-secure" #define PCMK__META_INTERNAL_RSC "internal_rsc" #define PCMK__META_ON_NODE "on_node" +#define PCMK__META_ON_NODE_UUID "on_node_uuid" #define PCMK__META_PHYSICAL_HOST "physical-host" /* @TODO Plug these in. Currently, they're never set. These are op attrs for use * with https://projects.clusterlabs.org/T382. */ #define PCMK__META_CLEAR_FAILURE_OP "clear_failure_op" #define PCMK__META_CLEAR_FAILURE_INTERVAL "clear_failure_interval" // @COMPAT Deprecated meta-attribute since 2.1.0 #define PCMK__META_CAN_FAIL "can_fail" // @COMPAT Deprecated alias for PCMK__META_PROMOTED_MAX since 2.0.0 #define PCMK__META_PROMOTED_MAX_LEGACY "master-max" // @COMPAT Deprecated alias for PCMK__META_PROMOTED_NODE_MAX since 2.0.0 #define PCMK__META_PROMOTED_NODE_MAX_LEGACY "master-node-max" // @COMPAT Deprecated meta-attribute since 2.0.0 #define PCMK__META_RESTART_TYPE "restart-type" // @COMPAT Deprecated meta-attribute since 2.0.0 #define PCMK__META_ROLE_AFTER_FAILURE "role_after_failure" // Constants for enumerated values for various options #define PCMK__VALUE_CLUSTER "cluster" #define PCMK__VALUE_CUSTOM "custom" #define PCMK__VALUE_FENCING "fencing" #define PCMK__VALUE_GREEN "green" #define PCMK__VALUE_LOCAL "local" #define PCMK__VALUE_MIGRATE_ON_RED "migrate-on-red" #define PCMK__VALUE_NONE "none" #define PCMK__VALUE_NOTHING "nothing" #define PCMK__VALUE_ONLY_GREEN "only-green" #define PCMK__VALUE_PROGRESSIVE "progressive" #define PCMK__VALUE_QUORUM "quorum" #define PCMK__VALUE_RED "red" #define PCMK__VALUE_REQUEST "request" #define PCMK__VALUE_RESPONSE "response" #define PCMK__VALUE_UNFENCING "unfencing" #define PCMK__VALUE_YELLOW "yellow" #endif // PCMK__OPTIONS_INTERNAL__H diff --git a/lib/common/actions.c b/lib/common/actions.c index 837cad0abb..0f044a5eac 100644 --- a/lib/common/actions.c +++ b/lib/common/actions.c @@ -1,532 +1,532 @@ /* * Copyright 2004-2024 the Pacemaker project contributors * * The version control history for this file may have further details. * * This source code is licensed under the GNU Lesser General Public License * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. */ #include #ifndef _GNU_SOURCE # define _GNU_SOURCE #endif #include #include #include #include #include #include #include #include #include #include #include /*! * \brief Generate an operation key (RESOURCE_ACTION_INTERVAL) * * \param[in] rsc_id ID of resource being operated on * \param[in] op_type Operation name * \param[in] interval_ms Operation interval * * \return Newly allocated memory containing operation key as string * * \note This function asserts on errors, so it will never return NULL. * The caller is responsible for freeing the result with free(). */ char * pcmk__op_key(const char *rsc_id, const char *op_type, guint interval_ms) { CRM_ASSERT(rsc_id != NULL); CRM_ASSERT(op_type != NULL); return crm_strdup_printf(PCMK__OP_FMT, rsc_id, op_type, interval_ms); } static inline gboolean convert_interval(const char *s, guint *interval_ms) { unsigned long l; errno = 0; l = strtoul(s, NULL, 10); if (errno != 0) { return FALSE; } *interval_ms = (guint) l; return TRUE; } /*! * \internal * \brief Check for underbar-separated substring match * * \param[in] key Overall string being checked * \param[in] position Match before underbar at this \p key index * \param[in] matches Substrings to match (may contain underbars) * * \return \p key index of underbar before any matching substring, * or 0 if none */ static size_t match_before(const char *key, size_t position, const char **matches) { for (int i = 0; matches[i] != NULL; ++i) { const size_t match_len = strlen(matches[i]); // Must have at least X_MATCH before position if (position > (match_len + 1)) { const size_t possible = position - match_len - 1; if ((key[possible] == '_') && (strncmp(key + possible + 1, matches[i], match_len) == 0)) { return possible; } } } return 0; } gboolean parse_op_key(const char *key, char **rsc_id, char **op_type, guint *interval_ms) { guint local_interval_ms = 0; const size_t key_len = (key == NULL)? 0 : strlen(key); // Operation keys must be formatted as RSC_ACTION_INTERVAL size_t action_underbar = 0; // Index in key of underbar before ACTION size_t interval_underbar = 0; // Index in key of underbar before INTERVAL size_t possible = 0; /* Underbar was a poor choice of separator since both RSC and ACTION can * contain underbars. Here, list action names and name prefixes that can. */ const char *actions_with_underbars[] = { PCMK_ACTION_MIGRATE_FROM, PCMK_ACTION_MIGRATE_TO, NULL }; const char *action_prefixes_with_underbars[] = { "pre_" PCMK_ACTION_NOTIFY, "post_" PCMK_ACTION_NOTIFY, "confirmed-pre_" PCMK_ACTION_NOTIFY, "confirmed-post_" PCMK_ACTION_NOTIFY, NULL, }; // Initialize output variables in case of early return if (rsc_id) { *rsc_id = NULL; } if (op_type) { *op_type = NULL; } if (interval_ms) { *interval_ms = 0; } // RSC_ACTION_INTERVAL implies a minimum of 5 characters if (key_len < 5) { return FALSE; } // Find, parse, and validate interval interval_underbar = key_len - 2; while ((interval_underbar > 2) && (key[interval_underbar] != '_')) { --interval_underbar; } if ((interval_underbar == 2) || !convert_interval(key + interval_underbar + 1, &local_interval_ms)) { return FALSE; } // Find the base (OCF) action name, disregarding prefixes action_underbar = match_before(key, interval_underbar, actions_with_underbars); if (action_underbar == 0) { action_underbar = interval_underbar - 2; while ((action_underbar > 0) && (key[action_underbar] != '_')) { --action_underbar; } if (action_underbar == 0) { return FALSE; } } possible = match_before(key, action_underbar, action_prefixes_with_underbars); if (possible != 0) { action_underbar = possible; } // Set output variables if (rsc_id != NULL) { *rsc_id = strndup(key, action_underbar); CRM_ASSERT(*rsc_id != NULL); } if (op_type != NULL) { *op_type = strndup(key + action_underbar + 1, interval_underbar - action_underbar - 1); CRM_ASSERT(*op_type != NULL); } if (interval_ms != NULL) { *interval_ms = local_interval_ms; } return TRUE; } char * pcmk__notify_key(const char *rsc_id, const char *notify_type, const char *op_type) { CRM_CHECK(rsc_id != NULL, return NULL); CRM_CHECK(op_type != NULL, return NULL); CRM_CHECK(notify_type != NULL, return NULL); return crm_strdup_printf("%s_%s_notify_%s_0", rsc_id, notify_type, op_type); } /*! * \brief Parse a transition magic string into its constituent parts * * \param[in] magic Magic string to parse (must be non-NULL) * \param[out] uuid If non-NULL, where to store copy of parsed UUID * \param[out] transition_id If non-NULL, where to store parsed transition ID * \param[out] action_id If non-NULL, where to store parsed action ID * \param[out] op_status If non-NULL, where to store parsed result status * \param[out] op_rc If non-NULL, where to store parsed actual rc * \param[out] target_rc If non-NULL, where to stored parsed target rc * * \return TRUE if key was valid, FALSE otherwise * \note If uuid is supplied and this returns TRUE, the caller is responsible * for freeing the memory for *uuid using free(). */ gboolean decode_transition_magic(const char *magic, char **uuid, int *transition_id, int *action_id, int *op_status, int *op_rc, int *target_rc) { int res = 0; char *key = NULL; gboolean result = TRUE; int local_op_status = -1; int local_op_rc = -1; CRM_CHECK(magic != NULL, return FALSE); #ifdef HAVE_SSCANF_M res = sscanf(magic, "%d:%d;%ms", &local_op_status, &local_op_rc, &key); #else key = calloc(1, strlen(magic) - 3); // magic must have >=4 other characters CRM_ASSERT(key); res = sscanf(magic, "%d:%d;%s", &local_op_status, &local_op_rc, key); #endif if (res == EOF) { crm_err("Could not decode transition information '%s': %s", magic, pcmk_rc_str(errno)); result = FALSE; } else if (res < 3) { crm_warn("Transition information '%s' incomplete (%d of 3 expected items)", magic, res); result = FALSE; } else { if (op_status) { *op_status = local_op_status; } if (op_rc) { *op_rc = local_op_rc; } result = decode_transition_key(key, uuid, transition_id, action_id, target_rc); } free(key); return result; } char * pcmk__transition_key(int transition_id, int action_id, int target_rc, const char *node) { CRM_CHECK(node != NULL, return NULL); return crm_strdup_printf("%d:%d:%d:%-*s", action_id, transition_id, target_rc, 36, node); } /*! * \brief Parse a transition key into its constituent parts * * \param[in] key Transition key to parse (must be non-NULL) * \param[out] uuid If non-NULL, where to store copy of parsed UUID * \param[out] transition_id If non-NULL, where to store parsed transition ID * \param[out] action_id If non-NULL, where to store parsed action ID * \param[out] target_rc If non-NULL, where to stored parsed target rc * * \return TRUE if key was valid, FALSE otherwise * \note If uuid is supplied and this returns TRUE, the caller is responsible * for freeing the memory for *uuid using free(). */ gboolean decode_transition_key(const char *key, char **uuid, int *transition_id, int *action_id, int *target_rc) { int local_transition_id = -1; int local_action_id = -1; int local_target_rc = -1; char local_uuid[37] = { '\0' }; // Initialize any supplied output arguments if (uuid) { *uuid = NULL; } if (transition_id) { *transition_id = -1; } if (action_id) { *action_id = -1; } if (target_rc) { *target_rc = -1; } CRM_CHECK(key != NULL, return FALSE); if (sscanf(key, "%d:%d:%d:%36s", &local_action_id, &local_transition_id, &local_target_rc, local_uuid) != 4) { crm_err("Invalid transition key '%s'", key); return FALSE; } if (strlen(local_uuid) != 36) { crm_warn("Invalid UUID '%s' in transition key '%s'", local_uuid, key); } if (uuid) { *uuid = strdup(local_uuid); CRM_ASSERT(*uuid); } if (transition_id) { *transition_id = local_transition_id; } if (action_id) { *action_id = local_action_id; } if (target_rc) { *target_rc = local_target_rc; } return TRUE; } // Return true if a is an attribute that should be filtered static bool should_filter_for_digest(xmlAttrPtr a, void *user_data) { if (strncmp((const char *) a->name, CRM_META "_", sizeof(CRM_META " ") - 1) == 0) { return true; } return pcmk__str_any_of((const char *) a->name, PCMK_XA_ID, PCMK_XA_CRM_FEATURE_SET, XML_LRM_ATTR_OP_DIGEST, PCMK__META_ON_NODE, - XML_LRM_ATTR_TARGET_UUID, + PCMK__META_ON_NODE_UUID, "pcmk_external_ip", NULL); } /*! * \internal * \brief Remove XML attributes not needed for operation digest * * \param[in,out] param_set XML with operation parameters */ void pcmk__filter_op_for_digest(xmlNode *param_set) { char *key = NULL; char *timeout = NULL; guint interval_ms = 0; if (param_set == NULL) { return; } /* Timeout is useful for recurring operation digests, so grab it before * removing meta-attributes */ key = crm_meta_name(PCMK_META_INTERVAL); if (crm_element_value_ms(param_set, key, &interval_ms) != pcmk_ok) { interval_ms = 0; } free(key); key = NULL; if (interval_ms != 0) { key = crm_meta_name(PCMK_META_TIMEOUT); timeout = crm_element_value_copy(param_set, key); } // Remove all CRM_meta_* attributes and certain other attributes pcmk__xe_remove_matching_attrs(param_set, should_filter_for_digest, NULL); // Add timeout back for recurring operation digests if (timeout != NULL) { crm_xml_add(param_set, key, timeout); } free(timeout); free(key); } int rsc_op_expected_rc(const lrmd_event_data_t *op) { int rc = 0; if (op && op->user_data) { decode_transition_key(op->user_data, NULL, NULL, NULL, &rc); } return rc; } gboolean did_rsc_op_fail(lrmd_event_data_t * op, int target_rc) { switch (op->op_status) { case PCMK_EXEC_CANCELLED: case PCMK_EXEC_PENDING: return FALSE; case PCMK_EXEC_NOT_SUPPORTED: case PCMK_EXEC_TIMEOUT: case PCMK_EXEC_ERROR: case PCMK_EXEC_NOT_CONNECTED: case PCMK_EXEC_NO_FENCE_DEVICE: case PCMK_EXEC_NO_SECRETS: case PCMK_EXEC_INVALID: return TRUE; default: if (target_rc != op->rc) { return TRUE; } } return FALSE; } /*! * \brief Create a CIB XML element for an operation * * \param[in,out] parent If not NULL, make new XML node a child of this * \param[in] prefix Generate an ID using this prefix * \param[in] task Operation task to set * \param[in] interval_spec Operation interval to set * \param[in] timeout If not NULL, operation timeout to set * * \return New XML object on success, NULL otherwise */ xmlNode * crm_create_op_xml(xmlNode *parent, const char *prefix, const char *task, const char *interval_spec, const char *timeout) { xmlNode *xml_op; CRM_CHECK(prefix && task && interval_spec, return NULL); xml_op = create_xml_node(parent, PCMK_XE_OP); crm_xml_set_id(xml_op, "%s-%s-%s", prefix, task, interval_spec); crm_xml_add(xml_op, PCMK_META_INTERVAL, interval_spec); crm_xml_add(xml_op, PCMK_XA_NAME, task); if (timeout) { crm_xml_add(xml_op, PCMK_META_TIMEOUT, timeout); } return xml_op; } /*! * \brief Check whether an operation requires resource agent meta-data * * \param[in] rsc_class Resource agent class (or NULL to skip class check) * \param[in] op Operation action (or NULL to skip op check) * * \return true if operation needs meta-data, false otherwise * \note At least one of rsc_class and op must be specified. */ bool crm_op_needs_metadata(const char *rsc_class, const char *op) { /* Agent metadata is used to determine whether an agent reload is possible, * so if this op is not relevant to that feature, we don't need metadata. */ CRM_CHECK((rsc_class != NULL) || (op != NULL), return false); if ((rsc_class != NULL) && !pcmk_is_set(pcmk_get_ra_caps(rsc_class), pcmk_ra_cap_params)) { // Metadata is needed only for resource classes that use parameters return false; } if (op == NULL) { return true; } // Metadata is needed only for these actions return pcmk__str_any_of(op, PCMK_ACTION_START, PCMK_ACTION_MONITOR, PCMK_ACTION_PROMOTE, PCMK_ACTION_DEMOTE, PCMK_ACTION_RELOAD, PCMK_ACTION_RELOAD_AGENT, PCMK_ACTION_MIGRATE_TO, PCMK_ACTION_MIGRATE_FROM, PCMK_ACTION_NOTIFY, NULL); } /*! * \internal * \brief Check whether an action name is for a fencing action * * \param[in] action Action name to check * * \return true if \p action is "off", "reboot", or "poweroff", otherwise false */ bool pcmk__is_fencing_action(const char *action) { return pcmk__str_any_of(action, PCMK_ACTION_OFF, PCMK_ACTION_REBOOT, "poweroff", NULL); } bool pcmk_is_probe(const char *task, guint interval) { if (task == NULL) { return false; } return (interval == 0) && pcmk__str_eq(task, PCMK_ACTION_MONITOR, pcmk__str_none); } bool pcmk_xe_is_probe(const xmlNode *xml_op) { const char *task = crm_element_value(xml_op, PCMK_XA_OPERATION); const char *interval_ms_s = crm_element_value(xml_op, PCMK_META_INTERVAL); int interval_ms; pcmk__scan_min_int(interval_ms_s, &interval_ms, 0); return pcmk_is_probe(task, interval_ms); } bool pcmk_xe_mask_probe_failure(const xmlNode *xml_op) { int status = PCMK_EXEC_UNKNOWN; int rc = PCMK_OCF_OK; if (!pcmk_xe_is_probe(xml_op)) { return false; } crm_element_value_int(xml_op, XML_LRM_ATTR_OPSTATUS, &status); crm_element_value_int(xml_op, XML_LRM_ATTR_RC, &rc); return rc == PCMK_OCF_NOT_INSTALLED || rc == PCMK_OCF_INVALID_PARAM || status == PCMK_EXEC_NOT_INSTALLED; } diff --git a/lib/pacemaker/pcmk_graph_producer.c b/lib/pacemaker/pcmk_graph_producer.c index 6410e6baa7..e92283824a 100644 --- a/lib/pacemaker/pcmk_graph_producer.c +++ b/lib/pacemaker/pcmk_graph_producer.c @@ -1,1096 +1,1096 @@ /* * Copyright 2004-2024 the Pacemaker project contributors * * The version control history for this file may have further details. * * This source code is licensed under the GNU General Public License version 2 * or later (GPLv2+) WITHOUT ANY WARRANTY. */ #include #include #include #include #include #include #include #include #include "libpacemaker_private.h" // Convenience macros for logging action properties #define action_type_str(flags) \ (pcmk_is_set((flags), pcmk_action_pseudo)? "pseudo-action" : "action") #define action_optional_str(flags) \ (pcmk_is_set((flags), pcmk_action_optional)? "optional" : "required") #define action_runnable_str(flags) \ (pcmk_is_set((flags), pcmk_action_runnable)? "runnable" : "unrunnable") #define action_node_str(a) \ (((a)->node == NULL)? "no node" : (a)->node->details->uname) /*! * \internal * \brief Add an XML node tag for a specified ID * * \param[in] id Node UUID to add * \param[in,out] xml Parent XML tag to add to */ static xmlNode* add_node_to_xml_by_id(const char *id, xmlNode *xml) { xmlNode *node_xml; node_xml = create_xml_node(xml, XML_CIB_TAG_NODE); crm_xml_add(node_xml, PCMK_XA_ID, id); return node_xml; } /*! * \internal * \brief Add an XML node tag for a specified node * * \param[in] node Node to add * \param[in,out] xml XML to add node to */ static void add_node_to_xml(const pcmk_node_t *node, void *xml) { add_node_to_xml_by_id(node->details->id, (xmlNode *) xml); } /*! * \internal * \brief Count (optionally add to XML) nodes needing maintenance state update * * \param[in,out] xml Parent XML tag to add to, if any * \param[in] scheduler Scheduler data * * \return Count of nodes added * \note Only Pacemaker Remote nodes are considered currently */ static int add_maintenance_nodes(xmlNode *xml, const pcmk_scheduler_t *scheduler) { xmlNode *maintenance = NULL; int count = 0; if (xml != NULL) { maintenance = create_xml_node(xml, XML_GRAPH_TAG_MAINTENANCE); } for (const GList *iter = scheduler->nodes; iter != NULL; iter = iter->next) { const pcmk_node_t *node = iter->data; if (pe__is_guest_or_remote_node(node) && (node->details->maintenance != node->details->remote_maintenance)) { if (maintenance != NULL) { crm_xml_add(add_node_to_xml_by_id(node->details->id, maintenance), XML_NODE_IS_MAINTENANCE, (node->details->maintenance? "1" : "0")); } count++; } } crm_trace("%s %d nodes in need of maintenance mode update in state", ((maintenance == NULL)? "Counted" : "Added"), count); return count; } /*! * \internal * \brief Add pseudo action with nodes needing maintenance state update * * \param[in,out] scheduler Scheduler data */ static void add_maintenance_update(pcmk_scheduler_t *scheduler) { pcmk_action_t *action = NULL; if (add_maintenance_nodes(NULL, scheduler) != 0) { action = get_pseudo_op(PCMK_ACTION_MAINTENANCE_NODES, scheduler); pcmk__set_action_flags(action, pcmk_action_always_in_graph); } } /*! * \internal * \brief Add XML with nodes that an action is expected to bring down * * If a specified action is expected to bring any nodes down, add an XML block * with their UUIDs. When a node is lost, this allows the controller to * determine whether it was expected. * * \param[in,out] xml Parent XML tag to add to * \param[in] action Action to check for downed nodes */ static void add_downed_nodes(xmlNode *xml, const pcmk_action_t *action) { CRM_CHECK((xml != NULL) && (action != NULL) && (action->node != NULL), return); if (pcmk__str_eq(action->task, PCMK_ACTION_DO_SHUTDOWN, pcmk__str_none)) { /* Shutdown makes the action's node down */ xmlNode *downed = create_xml_node(xml, XML_GRAPH_TAG_DOWNED); add_node_to_xml_by_id(action->node->details->id, downed); } else if (pcmk__str_eq(action->task, PCMK_ACTION_STONITH, pcmk__str_none)) { /* Fencing makes the action's node and any hosted guest nodes down */ const char *fence = g_hash_table_lookup(action->meta, "stonith_action"); if (pcmk__is_fencing_action(fence)) { xmlNode *downed = create_xml_node(xml, XML_GRAPH_TAG_DOWNED); add_node_to_xml_by_id(action->node->details->id, downed); pe_foreach_guest_node(action->node->details->data_set, action->node, add_node_to_xml, downed); } } else if (action->rsc && action->rsc->is_remote_node && pcmk__str_eq(action->task, PCMK_ACTION_STOP, pcmk__str_none)) { /* Stopping a remote connection resource makes connected node down, * unless it's part of a migration */ GList *iter; pcmk_action_t *input; bool migrating = false; for (iter = action->actions_before; iter != NULL; iter = iter->next) { input = ((pcmk__related_action_t *) iter->data)->action; if ((input->rsc != NULL) && pcmk__str_eq(action->rsc->id, input->rsc->id, pcmk__str_none) && pcmk__str_eq(input->task, PCMK_ACTION_MIGRATE_FROM, pcmk__str_none)) { migrating = true; break; } } if (!migrating) { xmlNode *downed = create_xml_node(xml, XML_GRAPH_TAG_DOWNED); add_node_to_xml_by_id(action->rsc->id, downed); } } } /*! * \internal * \brief Create a transition graph operation key for a clone action * * \param[in] action Clone action * \param[in] interval_ms Action interval in milliseconds * * \return Newly allocated string with transition graph operation key */ static char * clone_op_key(const pcmk_action_t *action, guint interval_ms) { if (pcmk__str_eq(action->task, PCMK_ACTION_NOTIFY, pcmk__str_none)) { const char *n_type = g_hash_table_lookup(action->meta, "notify_type"); const char *n_task = g_hash_table_lookup(action->meta, "notify_operation"); CRM_LOG_ASSERT((n_type != NULL) && (n_task != NULL)); return pcmk__notify_key(action->rsc->clone_name, n_type, n_task); } else if (action->cancel_task != NULL) { return pcmk__op_key(action->rsc->clone_name, action->cancel_task, interval_ms); } else { return pcmk__op_key(action->rsc->clone_name, action->task, interval_ms); } } /*! * \internal * \brief Add node details to transition graph action XML * * \param[in] action Scheduled action * \param[in,out] xml Transition graph action XML for \p action */ static void add_node_details(const pcmk_action_t *action, xmlNode *xml) { pcmk_node_t *router_node = pcmk__connection_host_for_action(action); crm_xml_add(xml, PCMK__META_ON_NODE, action->node->details->uname); - crm_xml_add(xml, XML_LRM_ATTR_TARGET_UUID, action->node->details->id); + crm_xml_add(xml, PCMK__META_ON_NODE_UUID, action->node->details->id); if (router_node != NULL) { crm_xml_add(xml, XML_LRM_ATTR_ROUTER_NODE, router_node->details->uname); } } /*! * \internal * \brief Add resource details to transition graph action XML * * \param[in] action Scheduled action * \param[in,out] action_xml Transition graph action XML for \p action */ static void add_resource_details(const pcmk_action_t *action, xmlNode *action_xml) { xmlNode *rsc_xml = NULL; const char *attr_list[] = { PCMK_XA_CLASS, PCMK_XA_PROVIDER, PCMK_XA_TYPE, }; /* If a resource is locked to a node via PCMK_OPT_SHUTDOWN_LOCK, mark its * actions so the controller can preserve the lock when the action * completes. */ if (pcmk__action_locks_rsc_to_node(action)) { crm_xml_add_ll(action_xml, PCMK_OPT_SHUTDOWN_LOCK, (long long) action->rsc->lock_time); } // List affected resource rsc_xml = create_xml_node(action_xml, (const char *) action->rsc->xml->name); if (pcmk_is_set(action->rsc->flags, pcmk_rsc_removed) && (action->rsc->clone_name != NULL)) { /* Use the numbered instance name here, because if there is more * than one instance on a node, we need to make sure the command * goes to the right one. * * This is important even for anonymous clones, because the clone's * unique meta-attribute might have just been toggled from on to * off. */ crm_debug("Using orphan clone name %s instead of %s", action->rsc->id, action->rsc->clone_name); crm_xml_add(rsc_xml, PCMK_XA_ID, action->rsc->clone_name); crm_xml_add(rsc_xml, PCMK__XA_LONG_ID, action->rsc->id); } else if (!pcmk_is_set(action->rsc->flags, pcmk_rsc_unique)) { const char *xml_id = ID(action->rsc->xml); crm_debug("Using anonymous clone name %s for %s (aka %s)", xml_id, action->rsc->id, action->rsc->clone_name); /* ID is what we'd like client to use * LONG_ID is what they might know it as instead * * LONG_ID is only strictly needed /here/ during the * transition period until all nodes in the cluster * are running the new software /and/ have rebooted * once (meaning that they've only ever spoken to a DC * supporting this feature). * * If anyone toggles the unique flag to 'on', the * 'instance free' name will correspond to an orphan * and fall into the clause above instead */ crm_xml_add(rsc_xml, PCMK_XA_ID, xml_id); if ((action->rsc->clone_name != NULL) && !pcmk__str_eq(xml_id, action->rsc->clone_name, pcmk__str_none)) { crm_xml_add(rsc_xml, PCMK__XA_LONG_ID, action->rsc->clone_name); } else { crm_xml_add(rsc_xml, PCMK__XA_LONG_ID, action->rsc->id); } } else { CRM_ASSERT(action->rsc->clone_name == NULL); crm_xml_add(rsc_xml, PCMK_XA_ID, action->rsc->id); } for (int lpc = 0; lpc < PCMK__NELEM(attr_list); lpc++) { crm_xml_add(rsc_xml, attr_list[lpc], g_hash_table_lookup(action->rsc->meta, attr_list[lpc])); } } /*! * \internal * \brief Add action attributes to transition graph action XML * * \param[in,out] action Scheduled action * \param[in,out] action_xml Transition graph action XML for \p action */ static void add_action_attributes(pcmk_action_t *action, xmlNode *action_xml) { xmlNode *args_xml = NULL; /* We create free-standing XML to start, so we can sort the attributes * before adding it to action_xml, which keeps the scheduler regression * test graphs comparable. */ args_xml = create_xml_node(NULL, XML_TAG_ATTRS); crm_xml_add(args_xml, PCMK_XA_CRM_FEATURE_SET, CRM_FEATURE_SET); g_hash_table_foreach(action->extra, hash2field, args_xml); if ((action->rsc != NULL) && (action->node != NULL)) { // Get the resource instance attributes, evaluated properly for node GHashTable *params = pe_rsc_params(action->rsc, action->node, action->rsc->cluster); pcmk__substitute_remote_addr(action->rsc, params); g_hash_table_foreach(params, hash2smartfield, args_xml); } else if ((action->rsc != NULL) && (action->rsc->variant <= pcmk_rsc_variant_primitive)) { GHashTable *params = pe_rsc_params(action->rsc, NULL, action->rsc->cluster); g_hash_table_foreach(params, hash2smartfield, args_xml); } g_hash_table_foreach(action->meta, hash2metafield, args_xml); if (action->rsc != NULL) { pcmk_resource_t *parent = action->rsc; while (parent != NULL) { parent->cmds->add_graph_meta(parent, args_xml); parent = parent->parent; } pcmk__add_guest_meta_to_xml(args_xml, action); } else if (pcmk__str_eq(action->task, PCMK_ACTION_STONITH, pcmk__str_none) && (action->node != NULL)) { /* Pass the node's attributes as meta-attributes. * * @TODO: Determine whether it is still necessary to do this. It was * added in 33d99707, probably for the libfence-based implementation in * c9a90bd, which is no longer used. */ g_hash_table_foreach(action->node->details->attrs, hash2metafield, args_xml); } sorted_xml(args_xml, action_xml, FALSE); free_xml(args_xml); } /*! * \internal * \brief Create the transition graph XML for a scheduled action * * \param[in,out] parent Parent XML element to add action to * \param[in,out] action Scheduled action * \param[in] skip_details If false, add action details as sub-elements * \param[in] scheduler Scheduler data */ static void create_graph_action(xmlNode *parent, pcmk_action_t *action, bool skip_details, const pcmk_scheduler_t *scheduler) { bool needs_node_info = true; bool needs_maintenance_info = false; xmlNode *action_xml = NULL; if ((action == NULL) || (scheduler == NULL)) { return; } // Create the top-level element based on task if (pcmk__str_eq(action->task, PCMK_ACTION_STONITH, pcmk__str_none)) { /* All fences need node info; guest node fences are pseudo-events */ if (pcmk_is_set(action->flags, pcmk_action_pseudo)) { action_xml = create_xml_node(parent, XML_GRAPH_TAG_PSEUDO_EVENT); } else { action_xml = create_xml_node(parent, XML_GRAPH_TAG_CRM_EVENT); } } else if (pcmk__str_any_of(action->task, PCMK_ACTION_DO_SHUTDOWN, PCMK_ACTION_CLEAR_FAILCOUNT, NULL)) { action_xml = create_xml_node(parent, XML_GRAPH_TAG_CRM_EVENT); } else if (pcmk__str_eq(action->task, PCMK_ACTION_LRM_DELETE, pcmk__str_none)) { // CIB-only clean-up for shutdown locks action_xml = create_xml_node(parent, XML_GRAPH_TAG_CRM_EVENT); crm_xml_add(action_xml, PCMK__XA_MODE, XML_TAG_CIB); } else if (pcmk_is_set(action->flags, pcmk_action_pseudo)) { if (pcmk__str_eq(action->task, PCMK_ACTION_MAINTENANCE_NODES, pcmk__str_none)) { needs_maintenance_info = true; } action_xml = create_xml_node(parent, XML_GRAPH_TAG_PSEUDO_EVENT); needs_node_info = false; } else { action_xml = create_xml_node(parent, XML_GRAPH_TAG_RSC_OP); } crm_xml_add_int(action_xml, PCMK_XA_ID, action->id); crm_xml_add(action_xml, PCMK_XA_OPERATION, action->task); if ((action->rsc != NULL) && (action->rsc->clone_name != NULL)) { char *clone_key = NULL; guint interval_ms; if (pcmk__guint_from_hash(action->meta, PCMK_META_INTERVAL, 0, &interval_ms) != pcmk_rc_ok) { interval_ms = 0; } clone_key = clone_op_key(action, interval_ms); crm_xml_add(action_xml, PCMK__XA_OPERATION_KEY, clone_key); crm_xml_add(action_xml, "internal_" PCMK__XA_OPERATION_KEY, action->uuid); free(clone_key); } else { crm_xml_add(action_xml, PCMK__XA_OPERATION_KEY, action->uuid); } if (needs_node_info && (action->node != NULL)) { add_node_details(action, action_xml); g_hash_table_insert(action->meta, strdup(PCMK__META_ON_NODE), strdup(action->node->details->uname)); - g_hash_table_insert(action->meta, strdup(XML_LRM_ATTR_TARGET_UUID), + g_hash_table_insert(action->meta, strdup(PCMK__META_ON_NODE_UUID), strdup(action->node->details->id)); } if (skip_details) { return; } if ((action->rsc != NULL) && !pcmk_is_set(action->flags, pcmk_action_pseudo)) { // This is a real resource action, so add resource details add_resource_details(action, action_xml); } /* List any attributes in effect */ add_action_attributes(action, action_xml); /* List any nodes this action is expected to make down */ if (needs_node_info && (action->node != NULL)) { add_downed_nodes(action_xml, action); } if (needs_maintenance_info) { add_maintenance_nodes(action_xml, scheduler); } } /*! * \internal * \brief Check whether an action should be added to the transition graph * * \param[in] action Action to check * * \return true if action should be added to graph, otherwise false */ static bool should_add_action_to_graph(const pcmk_action_t *action) { if (!pcmk_is_set(action->flags, pcmk_action_runnable)) { crm_trace("Ignoring action %s (%d): unrunnable", action->uuid, action->id); return false; } if (pcmk_is_set(action->flags, pcmk_action_optional) && !pcmk_is_set(action->flags, pcmk_action_always_in_graph)) { crm_trace("Ignoring action %s (%d): optional", action->uuid, action->id); return false; } /* Actions for unmanaged resources should be excluded from the graph, * with the exception of monitors and cancellation of recurring monitors. */ if ((action->rsc != NULL) && !pcmk_is_set(action->rsc->flags, pcmk_rsc_managed) && !pcmk__str_eq(action->task, PCMK_ACTION_MONITOR, pcmk__str_none)) { const char *interval_ms_s; /* A cancellation of a recurring monitor will get here because the task * is cancel rather than monitor, but the interval can still be used to * recognize it. The interval has been normalized to milliseconds by * this point, so a string comparison is sufficient. */ interval_ms_s = g_hash_table_lookup(action->meta, PCMK_META_INTERVAL); if (pcmk__str_eq(interval_ms_s, "0", pcmk__str_null_matches)) { crm_trace("Ignoring action %s (%d): for unmanaged resource (%s)", action->uuid, action->id, action->rsc->id); return false; } } /* Always add pseudo-actions, fence actions, and shutdown actions (already * determined to be required and runnable by this point) */ if (pcmk_is_set(action->flags, pcmk_action_pseudo) || pcmk__strcase_any_of(action->task, PCMK_ACTION_STONITH, PCMK_ACTION_DO_SHUTDOWN, NULL)) { return true; } if (action->node == NULL) { pcmk__sched_err("Skipping action %s (%d) " "because it was not assigned to a node (bug?)", action->uuid, action->id); pcmk__log_action("Unassigned", action, false); return false; } if (pcmk_is_set(action->flags, pcmk_action_on_dc)) { crm_trace("Action %s (%d) should be dumped: " "can run on DC instead of %s", action->uuid, action->id, pe__node_name(action->node)); } else if (pe__is_guest_node(action->node) && !action->node->details->remote_requires_reset) { crm_trace("Action %s (%d) should be dumped: " "assuming will be runnable on guest %s", action->uuid, action->id, pe__node_name(action->node)); } else if (!action->node->details->online) { pcmk__sched_err("Skipping action %s (%d) " "because it was scheduled for offline node (bug?)", action->uuid, action->id); pcmk__log_action("Offline node", action, false); return false; } else if (action->node->details->unclean) { pcmk__sched_err("Skipping action %s (%d) " "because it was scheduled for unclean node (bug?)", action->uuid, action->id); pcmk__log_action("Unclean node", action, false); return false; } return true; } /*! * \internal * \brief Check whether an ordering's flags can change an action * * \param[in] ordering Ordering to check * * \return true if ordering has flags that can change an action, false otherwise */ static bool ordering_can_change_actions(const pcmk__related_action_t *ordering) { return pcmk_any_flags_set(ordering->type, ~(pcmk__ar_then_implies_first_graphed |pcmk__ar_first_implies_then_graphed |pcmk__ar_ordered)); } /*! * \internal * \brief Check whether an action input should be in the transition graph * * \param[in] action Action to check * \param[in,out] input Action input to check * * \return true if input should be in graph, false otherwise * \note This function may not only check an input, but disable it under certian * circumstances (load or anti-colocation orderings that are not needed). */ static bool should_add_input_to_graph(const pcmk_action_t *action, pcmk__related_action_t *input) { if (input->state == pe_link_dumped) { return true; } if ((uint32_t) input->type == pcmk__ar_none) { crm_trace("Ignoring %s (%d) input %s (%d): " "ordering disabled", action->uuid, action->id, input->action->uuid, input->action->id); return false; } else if (!pcmk_is_set(input->action->flags, pcmk_action_runnable) && !ordering_can_change_actions(input)) { crm_trace("Ignoring %s (%d) input %s (%d): " "optional and input unrunnable", action->uuid, action->id, input->action->uuid, input->action->id); return false; } else if (!pcmk_is_set(input->action->flags, pcmk_action_runnable) && pcmk_is_set(input->type, pcmk__ar_min_runnable)) { crm_trace("Ignoring %s (%d) input %s (%d): " "minimum number of instances required but input unrunnable", action->uuid, action->id, input->action->uuid, input->action->id); return false; } else if (pcmk_is_set(input->type, pcmk__ar_unmigratable_then_blocks) && !pcmk_is_set(input->action->flags, pcmk_action_runnable)) { crm_trace("Ignoring %s (%d) input %s (%d): " "input blocked if 'then' unmigratable", action->uuid, action->id, input->action->uuid, input->action->id); return false; } else if (pcmk_is_set(input->type, pcmk__ar_if_first_unmigratable) && pcmk_is_set(input->action->flags, pcmk_action_migratable)) { crm_trace("Ignoring %s (%d) input %s (%d): ordering applies " "only if input is unmigratable, but it is migratable", action->uuid, action->id, input->action->uuid, input->action->id); return false; } else if (((uint32_t) input->type == pcmk__ar_ordered) && pcmk_is_set(input->action->flags, pcmk_action_migratable) && pcmk__ends_with(input->action->uuid, "_stop_0")) { crm_trace("Ignoring %s (%d) input %s (%d): " "optional but stop in migration", action->uuid, action->id, input->action->uuid, input->action->id); return false; } else if ((uint32_t) input->type == pcmk__ar_if_on_same_node_or_target) { pcmk_node_t *input_node = input->action->node; if ((action->rsc != NULL) && pcmk__str_eq(action->task, PCMK_ACTION_MIGRATE_TO, pcmk__str_none)) { pcmk_node_t *assigned = action->rsc->allocated_to; /* For load_stopped -> migrate_to orderings, we care about where * the resource has been assigned, not where migrate_to will be * executed. */ if (!pe__same_node(input_node, assigned)) { crm_trace("Ignoring %s (%d) input %s (%d): " "migration target %s is not same as input node %s", action->uuid, action->id, input->action->uuid, input->action->id, (assigned? assigned->details->uname : ""), (input_node? input_node->details->uname : "")); input->type = (enum pe_ordering) pcmk__ar_none; return false; } } else if (!pe__same_node(input_node, action->node)) { crm_trace("Ignoring %s (%d) input %s (%d): " "not on same node (%s vs %s)", action->uuid, action->id, input->action->uuid, input->action->id, (action->node? action->node->details->uname : ""), (input_node? input_node->details->uname : "")); input->type = (enum pe_ordering) pcmk__ar_none; return false; } else if (pcmk_is_set(input->action->flags, pcmk_action_optional)) { crm_trace("Ignoring %s (%d) input %s (%d): " "ordering optional", action->uuid, action->id, input->action->uuid, input->action->id); input->type = (enum pe_ordering) pcmk__ar_none; return false; } } else if ((uint32_t) input->type == pcmk__ar_if_required_on_same_node) { if (input->action->node && action->node && !pe__same_node(input->action->node, action->node)) { crm_trace("Ignoring %s (%d) input %s (%d): " "not on same node (%s vs %s)", action->uuid, action->id, input->action->uuid, input->action->id, pe__node_name(action->node), pe__node_name(input->action->node)); input->type = (enum pe_ordering) pcmk__ar_none; return false; } else if (pcmk_is_set(input->action->flags, pcmk_action_optional)) { crm_trace("Ignoring %s (%d) input %s (%d): optional", action->uuid, action->id, input->action->uuid, input->action->id); input->type = (enum pe_ordering) pcmk__ar_none; return false; } } else if (input->action->rsc && input->action->rsc != action->rsc && pcmk_is_set(input->action->rsc->flags, pcmk_rsc_failed) && !pcmk_is_set(input->action->rsc->flags, pcmk_rsc_managed) && pcmk__ends_with(input->action->uuid, "_stop_0") && action->rsc && pe_rsc_is_clone(action->rsc)) { crm_warn("Ignoring requirement that %s complete before %s:" " unmanaged failed resources cannot prevent clone shutdown", input->action->uuid, action->uuid); return false; } else if (pcmk_is_set(input->action->flags, pcmk_action_optional) && !pcmk_any_flags_set(input->action->flags, pcmk_action_always_in_graph |pcmk_action_added_to_graph) && !should_add_action_to_graph(input->action)) { crm_trace("Ignoring %s (%d) input %s (%d): " "input optional", action->uuid, action->id, input->action->uuid, input->action->id); return false; } crm_trace("%s (%d) input %s %s (%d) on %s should be dumped: %s %s %#.6x", action->uuid, action->id, action_type_str(input->action->flags), input->action->uuid, input->action->id, action_node_str(input->action), action_runnable_str(input->action->flags), action_optional_str(input->action->flags), input->type); return true; } /*! * \internal * \brief Check whether an ordering creates an ordering loop * * \param[in] init_action "First" action in ordering * \param[in] action Callers should always set this the same as * \p init_action (this function may use a different * value for recursive calls) * \param[in,out] input Action wrapper for "then" action in ordering * * \return true if the ordering creates a loop, otherwise false */ bool pcmk__graph_has_loop(const pcmk_action_t *init_action, const pcmk_action_t *action, pcmk__related_action_t *input) { bool has_loop = false; if (pcmk_is_set(input->action->flags, pcmk_action_detect_loop)) { crm_trace("Breaking tracking loop: %s@%s -> %s@%s (%#.6x)", input->action->uuid, input->action->node? input->action->node->details->uname : "", action->uuid, action->node? action->node->details->uname : "", input->type); return false; } // Don't need to check inputs that won't be used if (!should_add_input_to_graph(action, input)) { return false; } if (input->action == init_action) { crm_debug("Input loop found in %s@%s ->...-> %s@%s", action->uuid, action->node? action->node->details->uname : "", init_action->uuid, init_action->node? init_action->node->details->uname : ""); return true; } pcmk__set_action_flags(input->action, pcmk_action_detect_loop); crm_trace("Checking inputs of action %s@%s input %s@%s (%#.6x)" "for graph loop with %s@%s ", action->uuid, action->node? action->node->details->uname : "", input->action->uuid, input->action->node? input->action->node->details->uname : "", input->type, init_action->uuid, init_action->node? init_action->node->details->uname : ""); // Recursively check input itself for loops for (GList *iter = input->action->actions_before; iter != NULL; iter = iter->next) { if (pcmk__graph_has_loop(init_action, input->action, (pcmk__related_action_t *) iter->data)) { // Recursive call already logged a debug message has_loop = true; break; } } pcmk__clear_action_flags(input->action, pcmk_action_detect_loop); if (!has_loop) { crm_trace("No input loop found in %s@%s -> %s@%s (%#.6x)", input->action->uuid, input->action->node? input->action->node->details->uname : "", action->uuid, action->node? action->node->details->uname : "", input->type); } return has_loop; } /*! * \internal * \brief Create a synapse XML element for a transition graph * * \param[in] action Action that synapse is for * \param[in,out] scheduler Scheduler data containing graph * * \return Newly added XML element for new graph synapse */ static xmlNode * create_graph_synapse(const pcmk_action_t *action, pcmk_scheduler_t *scheduler) { int synapse_priority = 0; xmlNode *syn = create_xml_node(scheduler->graph, "synapse"); crm_xml_add_int(syn, PCMK_XA_ID, scheduler->num_synapse); scheduler->num_synapse++; if (action->rsc != NULL) { synapse_priority = action->rsc->priority; } if (action->priority > synapse_priority) { synapse_priority = action->priority; } if (synapse_priority > 0) { crm_xml_add_int(syn, PCMK__XA_PRIORITY, synapse_priority); } return syn; } /*! * \internal * \brief Add an action to the transition graph XML if appropriate * * \param[in,out] data Action to possibly add * \param[in,out] user_data Scheduler data * * \note This will de-duplicate the action inputs, meaning that the * pcmk__related_action_t:type flags can no longer be relied on to retain * their original settings. That means this MUST be called after * pcmk__apply_orderings() is complete, and nothing after this should rely * on those type flags. (For example, some code looks for type equal to * some flag rather than whether the flag is set, and some code looks for * particular combinations of flags -- such code must be done before * pcmk__create_graph().) */ static void add_action_to_graph(gpointer data, gpointer user_data) { pcmk_action_t *action = (pcmk_action_t *) data; pcmk_scheduler_t *scheduler = (pcmk_scheduler_t *) user_data; xmlNode *syn = NULL; xmlNode *set = NULL; xmlNode *in = NULL; /* If we haven't already, de-duplicate inputs (even if we won't be adding * the action to the graph, so that crm_simulate's dot graphs don't have * duplicates). */ if (!pcmk_is_set(action->flags, pcmk_action_inputs_deduplicated)) { pcmk__deduplicate_action_inputs(action); pcmk__set_action_flags(action, pcmk_action_inputs_deduplicated); } if (pcmk_is_set(action->flags, pcmk_action_added_to_graph) || !should_add_action_to_graph(action)) { return; // Already added, or shouldn't be } pcmk__set_action_flags(action, pcmk_action_added_to_graph); crm_trace("Adding action %d (%s%s%s) to graph", action->id, action->uuid, ((action->node == NULL)? "" : " on "), ((action->node == NULL)? "" : action->node->details->uname)); syn = create_graph_synapse(action, scheduler); set = create_xml_node(syn, "action_set"); in = create_xml_node(syn, "inputs"); create_graph_action(set, action, false, scheduler); for (GList *lpc = action->actions_before; lpc != NULL; lpc = lpc->next) { pcmk__related_action_t *input = lpc->data; if (should_add_input_to_graph(action, input)) { xmlNode *input_xml = create_xml_node(in, "trigger"); input->state = pe_link_dumped; create_graph_action(input_xml, input->action, true, scheduler); } } } static int transition_id = -1; /*! * \internal * \brief Log a message after calculating a transition * * \param[in] filename Where transition input is stored */ void pcmk__log_transition_summary(const char *filename) { if (was_processing_error || crm_config_error) { crm_err("Calculated transition %d (with errors)%s%s", transition_id, (filename == NULL)? "" : ", saving inputs in ", (filename == NULL)? "" : filename); } else if (was_processing_warning || crm_config_warning) { crm_warn("Calculated transition %d (with warnings)%s%s", transition_id, (filename == NULL)? "" : ", saving inputs in ", (filename == NULL)? "" : filename); } else { crm_notice("Calculated transition %d%s%s", transition_id, (filename == NULL)? "" : ", saving inputs in ", (filename == NULL)? "" : filename); } if (crm_config_error) { crm_notice("Configuration errors found during scheduler processing," " please run \"crm_verify -L\" to identify issues"); } } /*! * \internal * \brief Add a resource's actions to the transition graph * * \param[in,out] rsc Resource whose actions should be added */ void pcmk__add_rsc_actions_to_graph(pcmk_resource_t *rsc) { GList *iter = NULL; CRM_ASSERT(rsc != NULL); pcmk__rsc_trace(rsc, "Adding actions for %s to graph", rsc->id); // First add the resource's own actions g_list_foreach(rsc->actions, add_action_to_graph, rsc->cluster); // Then recursively add its children's actions (appropriate to variant) for (iter = rsc->children; iter != NULL; iter = iter->next) { pcmk_resource_t *child_rsc = (pcmk_resource_t *) iter->data; child_rsc->cmds->add_actions_to_graph(child_rsc); } } /*! * \internal * \brief Create a transition graph with all cluster actions needed * * \param[in,out] scheduler Scheduler data */ void pcmk__create_graph(pcmk_scheduler_t *scheduler) { GList *iter = NULL; const char *value = NULL; long long limit = 0LL; transition_id++; crm_trace("Creating transition graph %d", transition_id); scheduler->graph = create_xml_node(NULL, XML_TAG_GRAPH); value = pe_pref(scheduler->config_hash, PCMK_OPT_CLUSTER_DELAY); crm_xml_add(scheduler->graph, PCMK_OPT_CLUSTER_DELAY, value); value = pe_pref(scheduler->config_hash, PCMK_OPT_STONITH_TIMEOUT); crm_xml_add(scheduler->graph, PCMK_OPT_STONITH_TIMEOUT, value); crm_xml_add(scheduler->graph, "failed-stop-offset", "INFINITY"); if (pcmk_is_set(scheduler->flags, pcmk_sched_start_failure_fatal)) { crm_xml_add(scheduler->graph, "failed-start-offset", "INFINITY"); } else { crm_xml_add(scheduler->graph, "failed-start-offset", "1"); } value = pe_pref(scheduler->config_hash, PCMK_OPT_BATCH_LIMIT); crm_xml_add(scheduler->graph, PCMK_OPT_BATCH_LIMIT, value); crm_xml_add_int(scheduler->graph, "transition_id", transition_id); value = pe_pref(scheduler->config_hash, PCMK_OPT_MIGRATION_LIMIT); if ((pcmk__scan_ll(value, &limit, 0LL) == pcmk_rc_ok) && (limit > 0)) { crm_xml_add(scheduler->graph, PCMK_OPT_MIGRATION_LIMIT, value); } if (scheduler->recheck_by > 0) { char *recheck_epoch = NULL; recheck_epoch = crm_strdup_printf("%llu", (long long) scheduler->recheck_by); crm_xml_add(scheduler->graph, "recheck-by", recheck_epoch); free(recheck_epoch); } /* The following code will de-duplicate action inputs, so nothing past this * should rely on the action input type flags retaining their original * values. */ // Add resource actions to graph for (iter = scheduler->resources; iter != NULL; iter = iter->next) { pcmk_resource_t *rsc = (pcmk_resource_t *) iter->data; pcmk__rsc_trace(rsc, "Processing actions for %s", rsc->id); rsc->cmds->add_actions_to_graph(rsc); } // Add pseudo-action for list of nodes with maintenance state update add_maintenance_update(scheduler); // Add non-resource (node) actions for (iter = scheduler->actions; iter != NULL; iter = iter->next) { pcmk_action_t *action = (pcmk_action_t *) iter->data; if ((action->rsc != NULL) && (action->node != NULL) && action->node->details->shutdown && !pcmk_is_set(action->rsc->flags, pcmk_rsc_maintenance) && !pcmk_any_flags_set(action->flags, pcmk_action_optional|pcmk_action_runnable) && pcmk__str_eq(action->task, PCMK_ACTION_STOP, pcmk__str_none)) { /* Eventually we should just ignore the 'fence' case, but for now * it's the best way to detect (in CTS) when CIB resource updates * are being lost. */ if (pcmk_is_set(scheduler->flags, pcmk_sched_quorate) || (scheduler->no_quorum_policy == pcmk_no_quorum_ignore)) { const bool managed = pcmk_is_set(action->rsc->flags, pcmk_rsc_managed); const bool failed = pcmk_is_set(action->rsc->flags, pcmk_rsc_failed); crm_crit("Cannot %s %s because of %s:%s%s (%s)", action->node->details->unclean? "fence" : "shut down", pe__node_name(action->node), action->rsc->id, (managed? " blocked" : " unmanaged"), (failed? " failed" : ""), action->uuid); } } add_action_to_graph((gpointer) action, (gpointer) scheduler); } crm_log_xml_trace(scheduler->graph, "graph"); } diff --git a/lib/pacemaker/pcmk_simulate.c b/lib/pacemaker/pcmk_simulate.c index cb3bfabec6..43f44cb4d3 100644 --- a/lib/pacemaker/pcmk_simulate.c +++ b/lib/pacemaker/pcmk_simulate.c @@ -1,1005 +1,1005 @@ /* * Copyright 2021-2024 the Pacemaker project contributors * * The version control history for this file may have further details. * * This source code is licensed under the GNU Lesser General Public License * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. */ #include #include #include #include #include #include #include #include #include #include #include #include "libpacemaker_private.h" static pcmk__output_t *out = NULL; static cib_t *fake_cib = NULL; static GList *fake_resource_list = NULL; static const GList *fake_op_fail_list = NULL; static void set_effective_date(pcmk_scheduler_t *scheduler, bool print_original, const char *use_date); /*! * \internal * \brief Create an action name for use in a dot graph * * \param[in] action Action to create name for * \param[in] verbose If true, add action ID to name * * \return Newly allocated string with action name * \note It is the caller's responsibility to free the result. */ static char * create_action_name(const pcmk_action_t *action, bool verbose) { char *action_name = NULL; const char *prefix = ""; const char *action_host = NULL; const char *clone_name = NULL; const char *task = action->task; if (action->node != NULL) { action_host = action->node->details->uname; } else if (!pcmk_is_set(action->flags, pcmk_action_pseudo)) { action_host = ""; } if (pcmk__str_eq(action->task, PCMK_ACTION_CANCEL, pcmk__str_none)) { prefix = "Cancel "; task = action->cancel_task; } if (action->rsc != NULL) { clone_name = action->rsc->clone_name; } if (clone_name != NULL) { char *key = NULL; guint interval_ms = 0; if (pcmk__guint_from_hash(action->meta, PCMK_META_INTERVAL, 0, &interval_ms) != pcmk_rc_ok) { interval_ms = 0; } if (pcmk__strcase_any_of(action->task, PCMK_ACTION_NOTIFY, PCMK_ACTION_NOTIFIED, NULL)) { const char *n_type = g_hash_table_lookup(action->meta, "notify_key_type"); const char *n_task = g_hash_table_lookup(action->meta, "notify_key_operation"); CRM_ASSERT(n_type != NULL); CRM_ASSERT(n_task != NULL); key = pcmk__notify_key(clone_name, n_type, n_task); } else { key = pcmk__op_key(clone_name, task, interval_ms); } if (action_host != NULL) { action_name = crm_strdup_printf("%s%s %s", prefix, key, action_host); } else { action_name = crm_strdup_printf("%s%s", prefix, key); } free(key); } else if (pcmk__str_eq(action->task, PCMK_ACTION_STONITH, pcmk__str_none)) { const char *op = g_hash_table_lookup(action->meta, "stonith_action"); action_name = crm_strdup_printf("%s%s '%s' %s", prefix, action->task, op, action_host); } else if (action->rsc && action_host) { action_name = crm_strdup_printf("%s%s %s", prefix, action->uuid, action_host); } else if (action_host) { action_name = crm_strdup_printf("%s%s %s", prefix, action->task, action_host); } else { action_name = crm_strdup_printf("%s", action->uuid); } if (verbose) { char *with_id = crm_strdup_printf("%s (%d)", action_name, action->id); free(action_name); action_name = with_id; } return action_name; } /*! * \internal * \brief Display the status of a cluster * * \param[in,out] scheduler Scheduler data * \param[in] show_opts How to modify display (as pcmk_show_opt_e flags) * \param[in] section_opts Sections to display (as pcmk_section_e flags) * \param[in] title What to use as list title * \param[in] print_spacer Whether to display a spacer first */ static void print_cluster_status(pcmk_scheduler_t *scheduler, uint32_t show_opts, uint32_t section_opts, const char *title, bool print_spacer) { pcmk__output_t *out = scheduler->priv; GList *all = NULL; crm_exit_t stonith_rc = 0; enum pcmk_pacemakerd_state state = pcmk_pacemakerd_state_invalid; section_opts |= pcmk_section_nodes | pcmk_section_resources; show_opts |= pcmk_show_inactive_rscs | pcmk_show_failed_detail; all = g_list_prepend(all, (gpointer) "*"); PCMK__OUTPUT_SPACER_IF(out, print_spacer); out->begin_list(out, NULL, NULL, "%s", title); out->message(out, "cluster-status", scheduler, state, stonith_rc, NULL, false, section_opts, show_opts, NULL, all, all); out->end_list(out); g_list_free(all); } /*! * \internal * \brief Display a summary of all actions scheduled in a transition * * \param[in,out] scheduler Scheduler data (fully scheduled) * \param[in] print_spacer Whether to display a spacer first */ static void print_transition_summary(pcmk_scheduler_t *scheduler, bool print_spacer) { pcmk__output_t *out = scheduler->priv; PCMK__OUTPUT_SPACER_IF(out, print_spacer); out->begin_list(out, NULL, NULL, "Transition Summary"); pcmk__output_actions(scheduler); out->end_list(out); } /*! * \internal * \brief Reset scheduler input, output, date, and flags * * \param[in,out] scheduler Scheduler data * \param[in] input What to set as cluster input * \param[in] out What to set as cluster output object * \param[in] use_date What to set as cluster's current timestamp * \param[in] flags Group of enum pcmk_scheduler_flags to set */ static void reset(pcmk_scheduler_t *scheduler, xmlNodePtr input, pcmk__output_t *out, const char *use_date, unsigned int flags) { scheduler->input = input; scheduler->priv = out; set_effective_date(scheduler, true, use_date); if (pcmk_is_set(flags, pcmk_sim_sanitized)) { pcmk__set_scheduler_flags(scheduler, pcmk_sched_sanitized); } if (pcmk_is_set(flags, pcmk_sim_show_scores)) { pcmk__set_scheduler_flags(scheduler, pcmk_sched_output_scores); } if (pcmk_is_set(flags, pcmk_sim_show_utilization)) { pcmk__set_scheduler_flags(scheduler, pcmk_sched_show_utilization); } } /*! * \brief Write out a file in dot(1) format describing the actions that will * be taken by the scheduler in response to an input CIB file. * * \param[in,out] scheduler Scheduler data * \param[in] dot_file The filename to write * \param[in] all_actions Write all actions, even those that are optional * or are on unmanaged resources * \param[in] verbose Add extra information, such as action IDs, to the * output * * \return Standard Pacemaker return code */ static int write_sim_dotfile(pcmk_scheduler_t *scheduler, const char *dot_file, bool all_actions, bool verbose) { GList *iter = NULL; FILE *dot_strm = fopen(dot_file, "w"); if (dot_strm == NULL) { return errno; } fprintf(dot_strm, " digraph \"g\" {\n"); for (iter = scheduler->actions; iter != NULL; iter = iter->next) { pcmk_action_t *action = (pcmk_action_t *) iter->data; const char *style = "dashed"; const char *font = "black"; const char *color = "black"; char *action_name = create_action_name(action, verbose); if (pcmk_is_set(action->flags, pcmk_action_pseudo)) { font = "orange"; } if (pcmk_is_set(action->flags, pcmk_action_added_to_graph)) { style = "bold"; color = "green"; } else if ((action->rsc != NULL) && !pcmk_is_set(action->rsc->flags, pcmk_rsc_managed)) { color = "red"; font = "purple"; if (!all_actions) { goto do_not_write; } } else if (pcmk_is_set(action->flags, pcmk_action_optional)) { color = "blue"; if (!all_actions) { goto do_not_write; } } else { color = "red"; CRM_LOG_ASSERT(!pcmk_is_set(action->flags, pcmk_action_runnable)); } pcmk__set_action_flags(action, pcmk_action_added_to_graph); fprintf(dot_strm, "\"%s\" [ style=%s color=\"%s\" fontcolor=\"%s\"]\n", action_name, style, color, font); do_not_write: free(action_name); } for (iter = scheduler->actions; iter != NULL; iter = iter->next) { pcmk_action_t *action = (pcmk_action_t *) iter->data; for (GList *before_iter = action->actions_before; before_iter != NULL; before_iter = before_iter->next) { pcmk__related_action_t *before = before_iter->data; char *before_name = NULL; char *after_name = NULL; const char *style = "dashed"; bool optional = true; if (before->state == pe_link_dumped) { optional = false; style = "bold"; } else if ((uint32_t) before->type == pcmk__ar_none) { continue; } else if (pcmk_is_set(before->action->flags, pcmk_action_added_to_graph) && pcmk_is_set(action->flags, pcmk_action_added_to_graph) && (uint32_t) before->type != pcmk__ar_if_on_same_node_or_target) { optional = false; } if (all_actions || !optional) { before_name = create_action_name(before->action, verbose); after_name = create_action_name(action, verbose); fprintf(dot_strm, "\"%s\" -> \"%s\" [ style = %s]\n", before_name, after_name, style); free(before_name); free(after_name); } } } fprintf(dot_strm, "}\n"); fflush(dot_strm); fclose(dot_strm); return pcmk_rc_ok; } /*! * \brief Profile the configuration updates and scheduler actions in a single * CIB file, printing the profiling timings. * * \note \p scheduler->priv must have been set to a valid \p pcmk__output_t * object before this function is called. * * \param[in] xml_file The CIB file to profile * \param[in] repeat Number of times to run * \param[in,out] scheduler Scheduler data * \param[in] use_date The date to set the cluster's time to (may be NULL) */ static void profile_file(const char *xml_file, long long repeat, pcmk_scheduler_t *scheduler, const char *use_date) { pcmk__output_t *out = scheduler->priv; xmlNode *cib_object = NULL; clock_t start = 0; clock_t end; unsigned long long scheduler_flags = pcmk_sched_no_compat; CRM_ASSERT(out != NULL); cib_object = filename2xml(xml_file); start = clock(); if (pcmk_find_cib_element(cib_object, XML_CIB_TAG_STATUS) == NULL) { create_xml_node(cib_object, XML_CIB_TAG_STATUS); } if (cli_config_update(&cib_object, NULL, FALSE) == FALSE) { free_xml(cib_object); return; } if (validate_xml(cib_object, NULL, FALSE) != TRUE) { free_xml(cib_object); return; } if (pcmk_is_set(scheduler->flags, pcmk_sched_output_scores)) { scheduler_flags |= pcmk_sched_output_scores; } if (pcmk_is_set(scheduler->flags, pcmk_sched_show_utilization)) { scheduler_flags |= pcmk_sched_show_utilization; } for (int i = 0; i < repeat; ++i) { xmlNode *input = (repeat == 1)? cib_object : copy_xml(cib_object); scheduler->input = input; set_effective_date(scheduler, false, use_date); pcmk__schedule_actions(input, scheduler_flags, scheduler); pe_reset_working_set(scheduler); } end = clock(); out->message(out, "profile", xml_file, start, end); } void pcmk__profile_dir(const char *dir, long long repeat, pcmk_scheduler_t *scheduler, const char *use_date) { pcmk__output_t *out = scheduler->priv; struct dirent **namelist; int file_num = scandir(dir, &namelist, 0, alphasort); CRM_ASSERT(out != NULL); if (file_num > 0) { struct stat prop; char buffer[FILENAME_MAX]; out->begin_list(out, NULL, NULL, "Timings"); while (file_num--) { if ('.' == namelist[file_num]->d_name[0]) { free(namelist[file_num]); continue; } else if (!pcmk__ends_with_ext(namelist[file_num]->d_name, ".xml")) { free(namelist[file_num]); continue; } snprintf(buffer, sizeof(buffer), "%s/%s", dir, namelist[file_num]->d_name); if (stat(buffer, &prop) == 0 && S_ISREG(prop.st_mode)) { profile_file(buffer, repeat, scheduler, use_date); } free(namelist[file_num]); } free(namelist); out->end_list(out); } } /*! * \brief Set the date of the cluster, either to the value given by * \p use_date, or to the "execution-date" value in the CIB. * * \note \p scheduler->priv must have been set to a valid \p pcmk__output_t * object before this function is called. * * \param[in,out] scheduler Scheduler data * \param[in] print_original If \p true, the "execution-date" should * also be printed * \param[in] use_date The date to set the cluster's time to * (may be NULL) */ static void set_effective_date(pcmk_scheduler_t *scheduler, bool print_original, const char *use_date) { pcmk__output_t *out = scheduler->priv; time_t original_date = 0; CRM_ASSERT(out != NULL); crm_element_value_epoch(scheduler->input, "execution-date", &original_date); if (use_date) { scheduler->now = crm_time_new(use_date); out->info(out, "Setting effective cluster time: %s", use_date); crm_time_log(LOG_NOTICE, "Pretending 'now' is", scheduler->now, crm_time_log_date | crm_time_log_timeofday); } else if (original_date != 0) { scheduler->now = pcmk__copy_timet(original_date); if (print_original) { char *when = crm_time_as_string(scheduler->now, crm_time_log_date|crm_time_log_timeofday); out->info(out, "Using the original execution date of: %s", when); free(when); } } } /*! * \internal * \brief Simulate successfully executing a pseudo-action in a graph * * \param[in,out] graph Graph to update with pseudo-action result * \param[in,out] action Pseudo-action to simulate executing * * \return Standard Pacemaker return code */ static int simulate_pseudo_action(pcmk__graph_t *graph, pcmk__graph_action_t *action) { const char *node = crm_element_value(action->xml, PCMK__META_ON_NODE); const char *task = crm_element_value(action->xml, PCMK__XA_OPERATION_KEY); pcmk__set_graph_action_flags(action, pcmk__graph_action_confirmed); out->message(out, "inject-pseudo-action", node, task); pcmk__update_graph(graph, action); return pcmk_rc_ok; } /*! * \internal * \brief Simulate executing a resource action in a graph * * \param[in,out] graph Graph to update with resource action result * \param[in,out] action Resource action to simulate executing * * \return Standard Pacemaker return code */ static int simulate_resource_action(pcmk__graph_t *graph, pcmk__graph_action_t *action) { int rc; lrmd_event_data_t *op = NULL; int target_outcome = PCMK_OCF_OK; const char *rtype = NULL; const char *rclass = NULL; const char *resource = NULL; const char *rprovider = NULL; const char *resource_config_name = NULL; const char *operation = crm_element_value(action->xml, PCMK_XA_OPERATION); const char *target_rc_s = crm_meta_value(action->params, XML_ATTR_TE_TARGET_RC); xmlNode *cib_node = NULL; xmlNode *cib_resource = NULL; xmlNode *action_rsc = first_named_child(action->xml, XML_CIB_TAG_RESOURCE); char *node = crm_element_value_copy(action->xml, PCMK__META_ON_NODE); char *uuid = NULL; const char *router_node = crm_element_value(action->xml, XML_LRM_ATTR_ROUTER_NODE); // Certain actions don't need to be displayed or history entries if (pcmk__str_eq(operation, CRM_OP_REPROBE, pcmk__str_none)) { crm_debug("No history injection for %s op on %s", operation, node); goto done; // Confirm action and update graph } if (action_rsc == NULL) { // Shouldn't be possible crm_log_xml_err(action->xml, "Bad"); free(node); return EPROTO; } /* A resource might be known by different names in the configuration and in * the action (for example, a clone instance). Grab the configuration name * (which is preferred when writing history), and if necessary, the instance * name. */ resource_config_name = crm_element_value(action_rsc, PCMK_XA_ID); if (resource_config_name == NULL) { // Shouldn't be possible crm_log_xml_err(action->xml, "No ID"); free(node); return EPROTO; } resource = resource_config_name; if (pe_find_resource(fake_resource_list, resource) == NULL) { const char *longname = crm_element_value(action_rsc, PCMK__XA_LONG_ID); if ((longname != NULL) && (pe_find_resource(fake_resource_list, longname) != NULL)) { resource = longname; } } // Certain actions need to be displayed but don't need history entries if (pcmk__strcase_any_of(operation, PCMK_ACTION_DELETE, PCMK_ACTION_META_DATA, NULL)) { out->message(out, "inject-rsc-action", resource, operation, node, (guint) 0); goto done; // Confirm action and update graph } rclass = crm_element_value(action_rsc, PCMK_XA_CLASS); rtype = crm_element_value(action_rsc, PCMK_XA_TYPE); rprovider = crm_element_value(action_rsc, PCMK_XA_PROVIDER); pcmk__scan_min_int(target_rc_s, &target_outcome, 0); CRM_ASSERT(fake_cib->cmds->query(fake_cib, NULL, NULL, cib_sync_call|cib_scope_local) == pcmk_ok); // Ensure the action node is in the CIB - uuid = crm_element_value_copy(action->xml, XML_LRM_ATTR_TARGET_UUID); + uuid = crm_element_value_copy(action->xml, PCMK__META_ON_NODE_UUID); cib_node = pcmk__inject_node(fake_cib, node, ((router_node == NULL)? uuid: node)); free(uuid); CRM_ASSERT(cib_node != NULL); // Add a history entry for the action cib_resource = pcmk__inject_resource_history(out, cib_node, resource, resource_config_name, rclass, rtype, rprovider); if (cib_resource == NULL) { crm_err("Could not simulate action %d history for resource %s", action->id, resource); free(node); free_xml(cib_node); return EINVAL; } // Simulate and display an executor event for the action result op = pcmk__event_from_graph_action(cib_resource, action, PCMK_EXEC_DONE, target_outcome, "User-injected result"); out->message(out, "inject-rsc-action", resource, op->op_type, node, op->interval_ms); // Check whether action is in a list of desired simulated failures for (const GList *iter = fake_op_fail_list; iter != NULL; iter = iter->next) { const char *spec = (const char *) iter->data; char *key = NULL; const char *match_name = NULL; // Allow user to specify anonymous clone with or without instance number key = crm_strdup_printf(PCMK__OP_FMT "@%s=", resource, op->op_type, op->interval_ms, node); if (strncasecmp(key, spec, strlen(key)) == 0) { match_name = resource; } free(key); // If not found, try the resource's name in the configuration if ((match_name == NULL) && (strcmp(resource, resource_config_name) != 0)) { key = crm_strdup_printf(PCMK__OP_FMT "@%s=", resource_config_name, op->op_type, op->interval_ms, node); if (strncasecmp(key, spec, strlen(key)) == 0) { match_name = resource_config_name; } free(key); } if (match_name == NULL) { continue; // This failed action entry doesn't match } // ${match_name}_${task}_${interval_in_ms}@${node}=${rc} rc = sscanf(spec, "%*[^=]=%d", (int *) &op->rc); if (rc != 1) { out->err(out, "Invalid failed operation '%s' " "(result code must be integer)", spec); continue; // Keep checking other list entries } out->info(out, "Pretending action %d failed with rc=%d", action->id, op->rc); pcmk__set_graph_action_flags(action, pcmk__graph_action_failed); graph->abort_priority = INFINITY; pcmk__inject_failcount(out, cib_node, match_name, op->op_type, op->interval_ms, op->rc); break; } pcmk__inject_action_result(cib_resource, op, target_outcome); lrmd_free_event(op); rc = fake_cib->cmds->modify(fake_cib, XML_CIB_TAG_STATUS, cib_node, cib_sync_call|cib_scope_local); CRM_ASSERT(rc == pcmk_ok); done: free(node); free_xml(cib_node); pcmk__set_graph_action_flags(action, pcmk__graph_action_confirmed); pcmk__update_graph(graph, action); return pcmk_rc_ok; } /*! * \internal * \brief Simulate successfully executing a cluster action * * \param[in,out] graph Graph to update with action result * \param[in,out] action Cluster action to simulate * * \return Standard Pacemaker return code */ static int simulate_cluster_action(pcmk__graph_t *graph, pcmk__graph_action_t *action) { const char *node = crm_element_value(action->xml, PCMK__META_ON_NODE); const char *task = crm_element_value(action->xml, PCMK_XA_OPERATION); xmlNode *rsc = first_named_child(action->xml, XML_CIB_TAG_RESOURCE); pcmk__set_graph_action_flags(action, pcmk__graph_action_confirmed); out->message(out, "inject-cluster-action", node, task, rsc); pcmk__update_graph(graph, action); return pcmk_rc_ok; } /*! * \internal * \brief Simulate successfully executing a fencing action * * \param[in,out] graph Graph to update with action result * \param[in,out] action Fencing action to simulate * * \return Standard Pacemaker return code */ static int simulate_fencing_action(pcmk__graph_t *graph, pcmk__graph_action_t *action) { const char *op = crm_meta_value(action->params, "stonith_action"); char *target = crm_element_value_copy(action->xml, PCMK__META_ON_NODE); out->message(out, "inject-fencing-action", target, op); if (!pcmk__str_eq(op, PCMK_ACTION_ON, pcmk__str_casei)) { int rc = pcmk_ok; GString *xpath = g_string_sized_new(512); // Set node state to offline xmlNode *cib_node = pcmk__inject_node_state_change(fake_cib, target, false); CRM_ASSERT(cib_node != NULL); crm_xml_add(cib_node, PCMK_XA_CRM_DEBUG_ORIGIN, __func__); rc = fake_cib->cmds->replace(fake_cib, XML_CIB_TAG_STATUS, cib_node, cib_sync_call|cib_scope_local); CRM_ASSERT(rc == pcmk_ok); // Simulate controller clearing node's resource history and attributes pcmk__g_strcat(xpath, "//" XML_CIB_TAG_STATE "[@" PCMK_XA_UNAME "='", target, "']/" XML_CIB_TAG_LRM, NULL); fake_cib->cmds->remove(fake_cib, (const char *) xpath->str, NULL, cib_xpath|cib_sync_call|cib_scope_local); g_string_truncate(xpath, 0); pcmk__g_strcat(xpath, "//" XML_CIB_TAG_STATE "[@" PCMK_XA_UNAME "='", target, "']" "/" XML_TAG_TRANSIENT_NODEATTRS, NULL); fake_cib->cmds->remove(fake_cib, (const char *) xpath->str, NULL, cib_xpath|cib_sync_call|cib_scope_local); free_xml(cib_node); g_string_free(xpath, TRUE); } pcmk__set_graph_action_flags(action, pcmk__graph_action_confirmed); pcmk__update_graph(graph, action); free(target); return pcmk_rc_ok; } enum pcmk__graph_status pcmk__simulate_transition(pcmk_scheduler_t *scheduler, cib_t *cib, const GList *op_fail_list) { pcmk__graph_t *transition = NULL; enum pcmk__graph_status graph_rc; pcmk__graph_functions_t simulation_fns = { simulate_pseudo_action, simulate_resource_action, simulate_cluster_action, simulate_fencing_action, }; out = scheduler->priv; fake_cib = cib; fake_op_fail_list = op_fail_list; if (!out->is_quiet(out)) { out->begin_list(out, NULL, NULL, "Executing Cluster Transition"); } pcmk__set_graph_functions(&simulation_fns); transition = pcmk__unpack_graph(scheduler->graph, crm_system_name); pcmk__log_graph(LOG_DEBUG, transition); fake_resource_list = scheduler->resources; do { graph_rc = pcmk__execute_graph(transition); } while (graph_rc == pcmk__graph_active); fake_resource_list = NULL; if (graph_rc != pcmk__graph_complete) { out->err(out, "Transition failed: %s", pcmk__graph_status2text(graph_rc)); pcmk__log_graph(LOG_ERR, transition); out->err(out, "An invalid transition was produced"); } pcmk__free_graph(transition); if (!out->is_quiet(out)) { // If not quiet, we'll need the resulting CIB for later display xmlNode *cib_object = NULL; int rc = fake_cib->cmds->query(fake_cib, NULL, &cib_object, cib_sync_call|cib_scope_local); CRM_ASSERT(rc == pcmk_ok); pe_reset_working_set(scheduler); scheduler->input = cib_object; out->end_list(out); } return graph_rc; } int pcmk__simulate(pcmk_scheduler_t *scheduler, pcmk__output_t *out, const pcmk_injections_t *injections, unsigned int flags, uint32_t section_opts, const char *use_date, const char *input_file, const char *graph_file, const char *dot_file) { int printed = pcmk_rc_no_output; int rc = pcmk_rc_ok; xmlNodePtr input = NULL; cib_t *cib = NULL; rc = cib__signon_query(out, &cib, &input); if (rc != pcmk_rc_ok) { goto simulate_done; } reset(scheduler, input, out, use_date, flags); cluster_status(scheduler); if ((cib->variant == cib_native) && pcmk_is_set(section_opts, pcmk_section_times)) { if (pcmk__our_nodename == NULL) { // Currently used only in the times section pcmk__query_node_name(out, 0, &pcmk__our_nodename, 0); } scheduler->localhost = pcmk__our_nodename; } if (!out->is_quiet(out)) { const bool show_pending = pcmk_is_set(flags, pcmk_sim_show_pending); if (pcmk_is_set(scheduler->flags, pcmk_sched_in_maintenance)) { printed = out->message(out, "maint-mode", scheduler->flags); } if (scheduler->disabled_resources || scheduler->blocked_resources) { PCMK__OUTPUT_SPACER_IF(out, printed == pcmk_rc_ok); printed = out->info(out, "%d of %d resource instances DISABLED and " "%d BLOCKED from further action due to failure", scheduler->disabled_resources, scheduler->ninstances, scheduler->blocked_resources); } /* Most formatted output headers use caps for each word, but this one * only has the first word capitalized for compatibility with pcs. */ print_cluster_status(scheduler, (show_pending? pcmk_show_pending : 0), section_opts, "Current cluster status", (printed == pcmk_rc_ok)); printed = pcmk_rc_ok; } // If the user requested any injections, handle them if ((injections->node_down != NULL) || (injections->node_fail != NULL) || (injections->node_up != NULL) || (injections->op_inject != NULL) || (injections->ticket_activate != NULL) || (injections->ticket_grant != NULL) || (injections->ticket_revoke != NULL) || (injections->ticket_standby != NULL) || (injections->watchdog != NULL)) { PCMK__OUTPUT_SPACER_IF(out, printed == pcmk_rc_ok); pcmk__inject_scheduler_input(scheduler, cib, injections); printed = pcmk_rc_ok; rc = cib->cmds->query(cib, NULL, &input, cib_sync_call); if (rc != pcmk_rc_ok) { rc = pcmk_legacy2rc(rc); goto simulate_done; } cleanup_calculations(scheduler); reset(scheduler, input, out, use_date, flags); cluster_status(scheduler); } if (input_file != NULL) { rc = write_xml_file(input, input_file, FALSE); if (rc < 0) { rc = pcmk_legacy2rc(rc); goto simulate_done; } } if (pcmk_any_flags_set(flags, pcmk_sim_process | pcmk_sim_simulate)) { pcmk__output_t *logger_out = NULL; unsigned long long scheduler_flags = pcmk_sched_no_compat; if (pcmk_is_set(scheduler->flags, pcmk_sched_output_scores)) { scheduler_flags |= pcmk_sched_output_scores; } if (pcmk_is_set(scheduler->flags, pcmk_sched_show_utilization)) { scheduler_flags |= pcmk_sched_show_utilization; } if (pcmk_all_flags_set(scheduler->flags, pcmk_sched_output_scores |pcmk_sched_show_utilization)) { PCMK__OUTPUT_SPACER_IF(out, printed == pcmk_rc_ok); out->begin_list(out, NULL, NULL, "Assignment Scores and Utilization Information"); printed = pcmk_rc_ok; } else if (pcmk_is_set(scheduler->flags, pcmk_sched_output_scores)) { PCMK__OUTPUT_SPACER_IF(out, printed == pcmk_rc_ok); out->begin_list(out, NULL, NULL, "Assignment Scores"); printed = pcmk_rc_ok; } else if (pcmk_is_set(scheduler->flags, pcmk_sched_show_utilization)) { PCMK__OUTPUT_SPACER_IF(out, printed == pcmk_rc_ok); out->begin_list(out, NULL, NULL, "Utilization Information"); printed = pcmk_rc_ok; } else { rc = pcmk__log_output_new(&logger_out); if (rc != pcmk_rc_ok) { goto simulate_done; } pe__register_messages(logger_out); pcmk__register_lib_messages(logger_out); scheduler->priv = logger_out; } pcmk__schedule_actions(input, scheduler_flags, scheduler); if (logger_out == NULL) { out->end_list(out); } else { logger_out->finish(logger_out, CRM_EX_OK, true, NULL); pcmk__output_free(logger_out); scheduler->priv = out; } input = NULL; /* Don't try and free it twice */ if (graph_file != NULL) { rc = write_xml_file(scheduler->graph, graph_file, FALSE); if (rc < 0) { rc = pcmk_rc_graph_error; goto simulate_done; } } if (dot_file != NULL) { rc = write_sim_dotfile(scheduler, dot_file, pcmk_is_set(flags, pcmk_sim_all_actions), pcmk_is_set(flags, pcmk_sim_verbose)); if (rc != pcmk_rc_ok) { rc = pcmk_rc_dot_error; goto simulate_done; } } if (!out->is_quiet(out)) { print_transition_summary(scheduler, printed == pcmk_rc_ok); } } rc = pcmk_rc_ok; if (!pcmk_is_set(flags, pcmk_sim_simulate)) { goto simulate_done; } PCMK__OUTPUT_SPACER_IF(out, printed == pcmk_rc_ok); if (pcmk__simulate_transition(scheduler, cib, injections->op_fail) != pcmk__graph_complete) { rc = pcmk_rc_invalid_transition; } if (out->is_quiet(out)) { goto simulate_done; } set_effective_date(scheduler, true, use_date); if (pcmk_is_set(flags, pcmk_sim_show_scores)) { pcmk__set_scheduler_flags(scheduler, pcmk_sched_output_scores); } if (pcmk_is_set(flags, pcmk_sim_show_utilization)) { pcmk__set_scheduler_flags(scheduler, pcmk_sched_show_utilization); } cluster_status(scheduler); print_cluster_status(scheduler, 0, section_opts, "Revised Cluster Status", true); simulate_done: cib__clean_up_connection(&cib); return rc; } int pcmk_simulate(xmlNodePtr *xml, pcmk_scheduler_t *scheduler, const pcmk_injections_t *injections, unsigned int flags, unsigned int section_opts, const char *use_date, const char *input_file, const char *graph_file, const char *dot_file) { pcmk__output_t *out = NULL; int rc = pcmk_rc_ok; rc = pcmk__xml_output_new(&out, xml); if (rc != pcmk_rc_ok) { return rc; } pe__register_messages(out); pcmk__register_lib_messages(out); rc = pcmk__simulate(scheduler, out, injections, flags, section_opts, use_date, input_file, graph_file, dot_file); pcmk__xml_output_finish(out, pcmk_rc2exitc(rc), xml); return rc; } diff --git a/lib/pengine/pe_actions.c b/lib/pengine/pe_actions.c index 3173f25938..f399673dc6 100644 --- a/lib/pengine/pe_actions.c +++ b/lib/pengine/pe_actions.c @@ -1,1891 +1,1892 @@ /* * Copyright 2004-2024 the Pacemaker project contributors * * The version control history for this file may have further details. * * This source code is licensed under the GNU Lesser General Public License * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. */ #include #include #include #include #include #include #include #include #include "pe_status_private.h" static void unpack_operation(pcmk_action_t *action, const xmlNode *xml_obj, guint interval_ms); static void add_singleton(pcmk_scheduler_t *scheduler, pcmk_action_t *action) { if (scheduler->singletons == NULL) { scheduler->singletons = pcmk__strkey_table(NULL, NULL); } g_hash_table_insert(scheduler->singletons, action->uuid, action); } static pcmk_action_t * lookup_singleton(pcmk_scheduler_t *scheduler, const char *action_uuid) { if (scheduler->singletons == NULL) { return NULL; } return g_hash_table_lookup(scheduler->singletons, action_uuid); } /*! * \internal * \brief Find an existing action that matches arguments * * \param[in] key Action key to match * \param[in] rsc Resource to match (if any) * \param[in] node Node to match (if any) * \param[in] scheduler Scheduler data * * \return Existing action that matches arguments (or NULL if none) */ static pcmk_action_t * find_existing_action(const char *key, const pcmk_resource_t *rsc, const pcmk_node_t *node, const pcmk_scheduler_t *scheduler) { GList *matches = NULL; pcmk_action_t *action = NULL; /* When rsc is NULL, it would be quicker to check scheduler->singletons, * but checking all scheduler->actions takes the node into account. */ matches = find_actions(((rsc == NULL)? scheduler->actions : rsc->actions), key, node); if (matches == NULL) { return NULL; } CRM_LOG_ASSERT(!pcmk__list_of_multiple(matches)); action = matches->data; g_list_free(matches); return action; } /*! * \internal * \brief Find the XML configuration corresponding to a specific action key * * \param[in] rsc Resource to find action configuration for * \param[in] key "RSC_ACTION_INTERVAL" of action to find * \param[in] include_disabled If false, do not return disabled actions * * \return XML configuration of desired action if any, otherwise NULL */ static xmlNode * find_exact_action_config(const pcmk_resource_t *rsc, const char *action_name, guint interval_ms, bool include_disabled) { for (xmlNode *operation = first_named_child(rsc->ops_xml, PCMK_XE_OP); operation != NULL; operation = crm_next_same_xml(operation)) { bool enabled = false; const char *config_name = NULL; const char *interval_spec = NULL; guint tmp_ms = 0U; // @TODO This does not consider meta-attributes, rules, defaults, etc. if (!include_disabled && (pcmk__xe_get_bool_attr(operation, PCMK_META_ENABLED, &enabled) == pcmk_rc_ok) && !enabled) { continue; } interval_spec = crm_element_value(operation, PCMK_META_INTERVAL); pcmk_parse_interval_spec(interval_spec, &tmp_ms); if (tmp_ms != interval_ms) { continue; } config_name = crm_element_value(operation, PCMK_XA_NAME); if (pcmk__str_eq(action_name, config_name, pcmk__str_none)) { return operation; } } return NULL; } /*! * \internal * \brief Find the XML configuration of a resource action * * \param[in] rsc Resource to find action configuration for * \param[in] action_name Action name to search for * \param[in] interval_ms Action interval (in milliseconds) to search for * \param[in] include_disabled If false, do not return disabled actions * * \return XML configuration of desired action if any, otherwise NULL */ xmlNode * pcmk__find_action_config(const pcmk_resource_t *rsc, const char *action_name, guint interval_ms, bool include_disabled) { xmlNode *action_config = NULL; // Try requested action first action_config = find_exact_action_config(rsc, action_name, interval_ms, include_disabled); // For migrate_to and migrate_from actions, retry with "migrate" // @TODO This should be either documented or deprecated if ((action_config == NULL) && pcmk__str_any_of(action_name, PCMK_ACTION_MIGRATE_TO, PCMK_ACTION_MIGRATE_FROM, NULL)) { action_config = find_exact_action_config(rsc, "migrate", 0, include_disabled); } return action_config; } /*! * \internal * \brief Create a new action object * * \param[in] key Action key * \param[in] task Action name * \param[in,out] rsc Resource that action is for (if any) * \param[in] node Node that action is on (if any) * \param[in] optional Whether action should be considered optional * \param[in,out] scheduler Scheduler data * * \return Newly allocated action * \note This function takes ownership of \p key. It is the caller's * responsibility to free the return value with pe_free_action(). */ static pcmk_action_t * new_action(char *key, const char *task, pcmk_resource_t *rsc, const pcmk_node_t *node, bool optional, pcmk_scheduler_t *scheduler) { pcmk_action_t *action = calloc(1, sizeof(pcmk_action_t)); CRM_ASSERT(action != NULL); action->rsc = rsc; action->task = strdup(task); CRM_ASSERT(action->task != NULL); action->uuid = key; if (node) { action->node = pe__copy_node(node); } if (pcmk__str_eq(task, PCMK_ACTION_LRM_DELETE, pcmk__str_casei)) { // Resource history deletion for a node can be done on the DC pcmk__set_action_flags(action, pcmk_action_on_dc); } pcmk__set_action_flags(action, pcmk_action_runnable); if (optional) { pcmk__set_action_flags(action, pcmk_action_optional); } else { pcmk__clear_action_flags(action, pcmk_action_optional); } if (rsc == NULL) { action->meta = pcmk__strkey_table(free, free); } else { guint interval_ms = 0; parse_op_key(key, NULL, NULL, &interval_ms); action->op_entry = pcmk__find_action_config(rsc, task, interval_ms, true); /* If the given key is for one of the many notification pseudo-actions * (pre_notify_promote, etc.), the actual action name is "notify" */ if ((action->op_entry == NULL) && (strstr(key, "_notify_") != NULL)) { action->op_entry = find_exact_action_config(rsc, PCMK_ACTION_NOTIFY, 0, true); } unpack_operation(action, action->op_entry, interval_ms); } pcmk__rsc_trace(rsc, "Created %s action %d (%s): %s for %s on %s", (optional? "optional" : "required"), scheduler->action_id, key, task, ((rsc == NULL)? "no resource" : rsc->id), pe__node_name(node)); action->id = scheduler->action_id++; scheduler->actions = g_list_prepend(scheduler->actions, action); if (rsc == NULL) { add_singleton(scheduler, action); } else { rsc->actions = g_list_prepend(rsc->actions, action); } return action; } /*! * \internal * \brief Unpack a resource's action-specific instance parameters * * \param[in] action_xml XML of action's configuration in CIB (if any) * \param[in,out] node_attrs Table of node attributes (for rule evaluation) * \param[in,out] scheduler Cluster working set (for rule evaluation) * * \return Newly allocated hash table of action-specific instance parameters */ GHashTable * pcmk__unpack_action_rsc_params(const xmlNode *action_xml, GHashTable *node_attrs, pcmk_scheduler_t *scheduler) { GHashTable *params = pcmk__strkey_table(free, free); pe_rule_eval_data_t rule_data = { .node_hash = node_attrs, .role = pcmk_role_unknown, .now = scheduler->now, .match_data = NULL, .rsc_data = NULL, .op_data = NULL }; pe__unpack_dataset_nvpairs(action_xml, XML_TAG_ATTR_SETS, &rule_data, params, NULL, FALSE, scheduler); return params; } /*! * \internal * \brief Update an action's optional flag * * \param[in,out] action Action to update * \param[in] optional Requested optional status */ static void update_action_optional(pcmk_action_t *action, gboolean optional) { // Force a non-recurring action to be optional if its resource is unmanaged if ((action->rsc != NULL) && (action->node != NULL) && !pcmk_is_set(action->flags, pcmk_action_pseudo) && !pcmk_is_set(action->rsc->flags, pcmk_rsc_managed) && (g_hash_table_lookup(action->meta, PCMK_META_INTERVAL) == NULL)) { pcmk__rsc_debug(action->rsc, "%s on %s is optional (%s is unmanaged)", action->uuid, pe__node_name(action->node), action->rsc->id); pcmk__set_action_flags(action, pcmk_action_optional); // We shouldn't clear runnable here because ... something // Otherwise require the action if requested } else if (!optional) { pcmk__clear_action_flags(action, pcmk_action_optional); } } static enum pe_quorum_policy effective_quorum_policy(pcmk_resource_t *rsc, pcmk_scheduler_t *scheduler) { enum pe_quorum_policy policy = scheduler->no_quorum_policy; if (pcmk_is_set(scheduler->flags, pcmk_sched_quorate)) { policy = pcmk_no_quorum_ignore; } else if (scheduler->no_quorum_policy == pcmk_no_quorum_demote) { switch (rsc->role) { case pcmk_role_promoted: case pcmk_role_unpromoted: if (rsc->next_role > pcmk_role_unpromoted) { pe__set_next_role(rsc, pcmk_role_unpromoted, PCMK_OPT_NO_QUORUM_POLICY "=demote"); } policy = pcmk_no_quorum_ignore; break; default: policy = pcmk_no_quorum_stop; break; } } return policy; } /*! * \internal * \brief Update a resource action's runnable flag * * \param[in,out] action Action to update * \param[in,out] scheduler Scheduler data * * \note This may also schedule fencing if a stop is unrunnable. */ static void update_resource_action_runnable(pcmk_action_t *action, pcmk_scheduler_t *scheduler) { if (pcmk_is_set(action->flags, pcmk_action_pseudo)) { return; } if (action->node == NULL) { pcmk__rsc_trace(action->rsc, "%s is unrunnable (unallocated)", action->uuid); pcmk__clear_action_flags(action, pcmk_action_runnable); } else if (!pcmk_is_set(action->flags, pcmk_action_on_dc) && !(action->node->details->online) && (!pe__is_guest_node(action->node) || action->node->details->remote_requires_reset)) { pcmk__clear_action_flags(action, pcmk_action_runnable); do_crm_log(LOG_WARNING, "%s on %s is unrunnable (node is offline)", action->uuid, pe__node_name(action->node)); if (pcmk_is_set(action->rsc->flags, pcmk_rsc_managed) && pcmk__str_eq(action->task, PCMK_ACTION_STOP, pcmk__str_casei) && !(action->node->details->unclean)) { pe_fence_node(scheduler, action->node, "stop is unrunnable", false); } } else if (!pcmk_is_set(action->flags, pcmk_action_on_dc) && action->node->details->pending) { pcmk__clear_action_flags(action, pcmk_action_runnable); do_crm_log(LOG_WARNING, "Action %s on %s is unrunnable (node is pending)", action->uuid, pe__node_name(action->node)); } else if (action->needs == pcmk_requires_nothing) { pe_action_set_reason(action, NULL, TRUE); if (pe__is_guest_node(action->node) && !pe_can_fence(scheduler, action->node)) { /* An action that requires nothing usually does not require any * fencing in order to be runnable. However, there is an exception: * such an action cannot be completed if it is on a guest node whose * host is unclean and cannot be fenced. */ pcmk__rsc_debug(action->rsc, "%s on %s is unrunnable " "(node's host cannot be fenced)", action->uuid, pe__node_name(action->node)); pcmk__clear_action_flags(action, pcmk_action_runnable); } else { pcmk__rsc_trace(action->rsc, "%s on %s does not require fencing or quorum", action->uuid, pe__node_name(action->node)); pcmk__set_action_flags(action, pcmk_action_runnable); } } else { switch (effective_quorum_policy(action->rsc, scheduler)) { case pcmk_no_quorum_stop: pcmk__rsc_debug(action->rsc, "%s on %s is unrunnable (no quorum)", action->uuid, pe__node_name(action->node)); pcmk__clear_action_flags(action, pcmk_action_runnable); pe_action_set_reason(action, "no quorum", true); break; case pcmk_no_quorum_freeze: if (!action->rsc->fns->active(action->rsc, TRUE) || (action->rsc->next_role > action->rsc->role)) { pcmk__rsc_debug(action->rsc, "%s on %s is unrunnable (no quorum)", action->uuid, pe__node_name(action->node)); pcmk__clear_action_flags(action, pcmk_action_runnable); pe_action_set_reason(action, "quorum freeze", true); } break; default: //pe_action_set_reason(action, NULL, TRUE); pcmk__set_action_flags(action, pcmk_action_runnable); break; } } } /*! * \internal * \brief Update a resource object's flags for a new action on it * * \param[in,out] rsc Resource that action is for (if any) * \param[in] action New action */ static void update_resource_flags_for_action(pcmk_resource_t *rsc, const pcmk_action_t *action) { /* @COMPAT pcmk_rsc_starting and pcmk_rsc_stopping are deprecated and unused * within Pacemaker, and will eventually be removed */ if (pcmk__str_eq(action->task, PCMK_ACTION_STOP, pcmk__str_casei)) { pcmk__set_rsc_flags(rsc, pcmk_rsc_stopping); } else if (pcmk__str_eq(action->task, PCMK_ACTION_START, pcmk__str_casei)) { if (pcmk_is_set(action->flags, pcmk_action_runnable)) { pcmk__set_rsc_flags(rsc, pcmk_rsc_starting); } else { pcmk__clear_rsc_flags(rsc, pcmk_rsc_starting); } } } static bool valid_stop_on_fail(const char *value) { return !pcmk__strcase_any_of(value, "standby", "demote", "stop", NULL); } /*! * \internal * \brief Validate (and possibly reset) resource action's on_fail meta-attribute * * \param[in] rsc Resource that action is for * \param[in] action_name Action name * \param[in] action_config Action configuration XML from CIB (if any) * \param[in,out] meta Table of action meta-attributes */ static void validate_on_fail(const pcmk_resource_t *rsc, const char *action_name, const xmlNode *action_config, GHashTable *meta) { const char *name = NULL; const char *role = NULL; const char *interval_spec = NULL; const char *value = g_hash_table_lookup(meta, PCMK_META_ON_FAIL); char *key = NULL; char *new_value = NULL; guint interval_ms = 0U; // Stop actions can only use certain on-fail values if (pcmk__str_eq(action_name, PCMK_ACTION_STOP, pcmk__str_none) && !valid_stop_on_fail(value)) { pcmk__config_err("Resetting '" PCMK_META_ON_FAIL "' for %s stop " "action to default value because '%s' is not " "allowed for stop", rsc->id, value); g_hash_table_remove(meta, PCMK_META_ON_FAIL); return; } /* Demote actions default on-fail to the on-fail value for the first * recurring monitor for the promoted role (if any). */ if (pcmk__str_eq(action_name, PCMK_ACTION_DEMOTE, pcmk__str_none) && (value == NULL)) { /* @TODO This does not consider promote options set in a meta-attribute * block (which may have rules that need to be evaluated) rather than * XML properties. */ for (xmlNode *operation = first_named_child(rsc->ops_xml, PCMK_XE_OP); operation != NULL; operation = crm_next_same_xml(operation)) { bool enabled = false; const char *promote_on_fail = NULL; /* We only care about explicit on-fail (if promote uses default, so * can demote) */ promote_on_fail = crm_element_value(operation, PCMK_META_ON_FAIL); if (promote_on_fail == NULL) { continue; } // We only care about recurring monitors for the promoted role name = crm_element_value(operation, PCMK_XA_NAME); role = crm_element_value(operation, "role"); if (!pcmk__str_eq(name, PCMK_ACTION_MONITOR, pcmk__str_none) || !pcmk__strcase_any_of(role, PCMK__ROLE_PROMOTED, PCMK__ROLE_PROMOTED_LEGACY, NULL)) { continue; } interval_spec = crm_element_value(operation, PCMK_META_INTERVAL); pcmk_parse_interval_spec(interval_spec, &interval_ms); if (interval_ms == 0U) { continue; } // We only care about enabled monitors if ((pcmk__xe_get_bool_attr(operation, PCMK_META_ENABLED, &enabled) == pcmk_rc_ok) && !enabled) { continue; } // Demote actions can't default to on-fail="demote" if (pcmk__str_eq(promote_on_fail, "demote", pcmk__str_casei)) { continue; } // Use value from first applicable promote action found key = strdup(PCMK_META_ON_FAIL); new_value = strdup(promote_on_fail); CRM_ASSERT((key != NULL) && (new_value != NULL)); g_hash_table_insert(meta, key, new_value); } return; } if (pcmk__str_eq(action_name, PCMK_ACTION_LRM_DELETE, pcmk__str_none) && !pcmk__str_eq(value, "ignore", pcmk__str_casei)) { key = strdup(PCMK_META_ON_FAIL); new_value = strdup("ignore"); CRM_ASSERT((key != NULL) && (new_value != NULL)); g_hash_table_insert(meta, key, new_value); return; } // on-fail="demote" is allowed only for certain actions if (pcmk__str_eq(value, "demote", pcmk__str_casei)) { name = crm_element_value(action_config, PCMK_XA_NAME); role = crm_element_value(action_config, "role"); interval_spec = crm_element_value(action_config, PCMK_META_INTERVAL); pcmk_parse_interval_spec(interval_spec, &interval_ms); if (!pcmk__str_eq(name, PCMK_ACTION_PROMOTE, pcmk__str_none) && ((interval_ms == 0U) || !pcmk__str_eq(name, PCMK_ACTION_MONITOR, pcmk__str_none) || !pcmk__strcase_any_of(role, PCMK__ROLE_PROMOTED, PCMK__ROLE_PROMOTED_LEGACY, NULL))) { pcmk__config_err("Resetting '" PCMK_META_ON_FAIL "' for %s %s " "action to default value because 'demote' is not " "allowed for it", rsc->id, name); g_hash_table_remove(meta, PCMK_META_ON_FAIL); return; } } } static int unpack_timeout(const char *value) { int timeout_ms = crm_get_msec(value); if (timeout_ms < 0) { timeout_ms = PCMK_DEFAULT_ACTION_TIMEOUT_MS; } return timeout_ms; } // true if value contains valid, non-NULL interval origin for recurring op static bool unpack_interval_origin(const char *value, const xmlNode *xml_obj, guint interval_ms, const crm_time_t *now, long long *start_delay) { long long result = 0; guint interval_sec = interval_ms / 1000; crm_time_t *origin = NULL; // Ignore unspecified values and non-recurring operations if ((value == NULL) || (interval_ms == 0) || (now == NULL)) { return false; } // Parse interval origin from text origin = crm_time_new(value); if (origin == NULL) { pcmk__config_err("Ignoring '" PCMK_META_INTERVAL_ORIGIN "' for " "operation '%s' because '%s' is not valid", (ID(xml_obj)? ID(xml_obj) : "(missing ID)"), value); return false; } // Get seconds since origin (negative if origin is in the future) result = crm_time_get_seconds(now) - crm_time_get_seconds(origin); crm_time_free(origin); // Calculate seconds from closest interval to now result = result % interval_sec; // Calculate seconds remaining until next interval result = ((result <= 0)? 0 : interval_sec) - result; crm_info("Calculated a start delay of %llds for operation '%s'", result, (ID(xml_obj)? ID(xml_obj) : "(unspecified)")); if (start_delay != NULL) { *start_delay = result * 1000; // milliseconds } return true; } static int unpack_start_delay(const char *value, GHashTable *meta) { int start_delay = 0; if (value != NULL) { start_delay = crm_get_msec(value); if (start_delay < 0) { start_delay = 0; } if (meta) { g_hash_table_replace(meta, strdup(PCMK_META_START_DELAY), pcmk__itoa(start_delay)); } } return start_delay; } /*! * \internal * \brief Find a resource's most frequent recurring monitor * * \param[in] rsc Resource to check * * \return Operation XML configured for most frequent recurring monitor for * \p rsc (if any) */ static xmlNode * most_frequent_monitor(const pcmk_resource_t *rsc) { guint min_interval_ms = G_MAXUINT; xmlNode *op = NULL; for (xmlNode *operation = first_named_child(rsc->ops_xml, PCMK_XE_OP); operation != NULL; operation = crm_next_same_xml(operation)) { bool enabled = false; guint interval_ms = 0U; const char *interval_spec = crm_element_value(operation, PCMK_META_INTERVAL); // We only care about enabled recurring monitors if (!pcmk__str_eq(crm_element_value(operation, PCMK_XA_NAME), PCMK_ACTION_MONITOR, pcmk__str_none)) { continue; } pcmk_parse_interval_spec(interval_spec, &interval_ms); if (interval_ms == 0U) { continue; } // @TODO This does not consider meta-attributes, rules, defaults, etc. if ((pcmk__xe_get_bool_attr(operation, PCMK_META_ENABLED, &enabled) == pcmk_rc_ok) && !enabled) { continue; } if (interval_ms < min_interval_ms) { min_interval_ms = interval_ms; op = operation; } } return op; } /*! * \internal * \brief Unpack action meta-attributes * * \param[in,out] rsc Resource that action is for * \param[in] node Node that action is on * \param[in] action_name Action name * \param[in] interval_ms Action interval (in milliseconds) * \param[in] action_config Action XML configuration from CIB (if any) * * Unpack a resource action's meta-attributes (normalizing the interval, * timeout, and start delay values as integer milliseconds) from its CIB XML * configuration (including defaults). * * \return Newly allocated hash table with normalized action meta-attributes */ GHashTable * pcmk__unpack_action_meta(pcmk_resource_t *rsc, const pcmk_node_t *node, const char *action_name, guint interval_ms, const xmlNode *action_config) { GHashTable *meta = NULL; char *name = NULL; char *value = NULL; const char *timeout_spec = NULL; const char *str = NULL; pe_rsc_eval_data_t rsc_rule_data = { .standard = crm_element_value(rsc->xml, PCMK_XA_CLASS), .provider = crm_element_value(rsc->xml, PCMK_XA_PROVIDER), .agent = crm_element_value(rsc->xml, PCMK_XA_TYPE), }; pe_op_eval_data_t op_rule_data = { .op_name = action_name, .interval = interval_ms, }; pe_rule_eval_data_t rule_data = { .node_hash = (node == NULL)? NULL : node->details->attrs, .role = pcmk_role_unknown, .now = rsc->cluster->now, .match_data = NULL, .rsc_data = &rsc_rule_data, .op_data = &op_rule_data, }; meta = pcmk__strkey_table(free, free); // Cluster-wide pe__unpack_dataset_nvpairs(rsc->cluster->op_defaults, XML_TAG_META_SETS, &rule_data, meta, NULL, FALSE, rsc->cluster); // Derive default timeout for probes from recurring monitor timeouts if (pcmk_is_probe(action_name, interval_ms)) { xmlNode *min_interval_mon = most_frequent_monitor(rsc); if (min_interval_mon != NULL) { /* @TODO This does not consider timeouts set in meta_attributes * blocks (which may also have rules that need to be evaluated). */ timeout_spec = crm_element_value(min_interval_mon, PCMK_META_TIMEOUT); if (timeout_spec != NULL) { pcmk__rsc_trace(rsc, "Setting default timeout for %s probe to " "most frequent monitor's timeout '%s'", rsc->id, timeout_spec); name = strdup(PCMK_META_TIMEOUT); value = strdup(timeout_spec); CRM_ASSERT((name != NULL) && (value != NULL)); g_hash_table_insert(meta, name, value); } } } if (action_config != NULL) { // take precedence over defaults pe__unpack_dataset_nvpairs(action_config, XML_TAG_META_SETS, &rule_data, meta, NULL, TRUE, rsc->cluster); /* Anything set as an XML property has highest precedence. * This ensures we use the name and interval from the tag. * (See below for the only exception, fence device start/probe timeout.) */ for (xmlAttrPtr attr = action_config->properties; attr != NULL; attr = attr->next) { name = strdup((const char *) attr->name); value = strdup(pcmk__xml_attr_value(attr)); CRM_ASSERT((name != NULL) && (value != NULL)); g_hash_table_insert(meta, name, value); } } g_hash_table_remove(meta, PCMK_XA_ID); // Normalize interval to milliseconds if (interval_ms > 0) { name = strdup(PCMK_META_INTERVAL); CRM_ASSERT(name != NULL); value = crm_strdup_printf("%u", interval_ms); g_hash_table_insert(meta, name, value); } else { g_hash_table_remove(meta, PCMK_META_INTERVAL); } /* Timeout order of precedence (highest to lowest): * 1. pcmk_monitor_timeout resource parameter (only for starts and probes * when rsc has pcmk_ra_cap_fence_params; this gets used for recurring * monitors via the executor instead) * 2. timeout configured in (with taking precedence over * ) * 3. timeout configured in * 4. PCMK_DEFAULT_ACTION_TIMEOUT_MS */ // Check for pcmk_monitor_timeout if (pcmk_is_set(pcmk_get_ra_caps(rsc_rule_data.standard), pcmk_ra_cap_fence_params) && (pcmk__str_eq(action_name, PCMK_ACTION_START, pcmk__str_none) || pcmk_is_probe(action_name, interval_ms))) { GHashTable *params = pe_rsc_params(rsc, node, rsc->cluster); timeout_spec = g_hash_table_lookup(params, "pcmk_monitor_timeout"); if (timeout_spec != NULL) { pcmk__rsc_trace(rsc, "Setting timeout for %s %s to " "pcmk_monitor_timeout (%s)", rsc->id, action_name, timeout_spec); name = strdup(PCMK_META_TIMEOUT); value = strdup(timeout_spec); CRM_ASSERT((name != NULL) && (value != NULL)); g_hash_table_insert(meta, name, value); } } // Normalize timeout to positive milliseconds name = strdup(PCMK_META_TIMEOUT); CRM_ASSERT(name != NULL); timeout_spec = g_hash_table_lookup(meta, PCMK_META_TIMEOUT); g_hash_table_insert(meta, name, pcmk__itoa(unpack_timeout(timeout_spec))); // Ensure on-fail has a valid value validate_on_fail(rsc, action_name, action_config, meta); // Normalize PCMK_META_START_DELAY str = g_hash_table_lookup(meta, PCMK_META_START_DELAY); if (str != NULL) { unpack_start_delay(str, meta); } else { long long start_delay = 0; str = g_hash_table_lookup(meta, PCMK_META_INTERVAL_ORIGIN); if (unpack_interval_origin(str, action_config, interval_ms, rsc->cluster->now, &start_delay)) { name = strdup(PCMK_META_START_DELAY); CRM_ASSERT(name != NULL); g_hash_table_insert(meta, name, crm_strdup_printf("%lld", start_delay)); } } return meta; } /*! * \internal * \brief Determine an action's quorum and fencing dependency * * \param[in] rsc Resource that action is for * \param[in] action_name Name of action being unpacked * * \return Quorum and fencing dependency appropriate to action */ enum rsc_start_requirement pcmk__action_requires(const pcmk_resource_t *rsc, const char *action_name) { const char *value = NULL; enum rsc_start_requirement requires = pcmk_requires_nothing; CRM_CHECK((rsc != NULL) && (action_name != NULL), return requires); if (!pcmk__strcase_any_of(action_name, PCMK_ACTION_START, PCMK_ACTION_PROMOTE, NULL)) { value = "nothing (not start or promote)"; } else if (pcmk_is_set(rsc->flags, pcmk_rsc_needs_fencing)) { requires = pcmk_requires_fencing; value = "fencing"; } else if (pcmk_is_set(rsc->flags, pcmk_rsc_needs_quorum)) { requires = pcmk_requires_quorum; value = "quorum"; } else { value = "nothing"; } pcmk__rsc_trace(rsc, "%s of %s requires %s", action_name, rsc->id, value); return requires; } /*! * \internal * \brief Parse action failure response from a user-provided string * * \param[in] rsc Resource that action is for * \param[in] action_name Name of action * \param[in] interval_ms Action interval (in milliseconds) * \param[in] value User-provided configuration value for on-fail * * \return Action failure response parsed from \p text */ enum action_fail_response pcmk__parse_on_fail(const pcmk_resource_t *rsc, const char *action_name, guint interval_ms, const char *value) { const char *desc = NULL; bool needs_remote_reset = false; enum action_fail_response on_fail = pcmk_on_fail_ignore; if (value == NULL) { // Use default } else if (pcmk__str_eq(value, "block", pcmk__str_casei)) { on_fail = pcmk_on_fail_block; desc = "block"; } else if (pcmk__str_eq(value, "fence", pcmk__str_casei)) { if (pcmk_is_set(rsc->cluster->flags, pcmk_sched_fencing_enabled)) { on_fail = pcmk_on_fail_fence_node; desc = "node fencing"; } else { pcmk__config_err("Resetting '" PCMK_META_ON_FAIL "' for " "%s of %s to 'stop' because 'fence' is not " "valid when fencing is disabled", action_name, rsc->id); on_fail = pcmk_on_fail_stop; desc = "stop resource"; } } else if (pcmk__str_eq(value, "standby", pcmk__str_casei)) { on_fail = pcmk_on_fail_standby_node; desc = "node standby"; } else if (pcmk__strcase_any_of(value, "ignore", PCMK__VALUE_NOTHING, NULL)) { desc = "ignore"; } else if (pcmk__str_eq(value, "migrate", pcmk__str_casei)) { on_fail = pcmk_on_fail_ban; desc = "force migration"; } else if (pcmk__str_eq(value, "stop", pcmk__str_casei)) { on_fail = pcmk_on_fail_stop; desc = "stop resource"; } else if (pcmk__str_eq(value, "restart", pcmk__str_casei)) { on_fail = pcmk_on_fail_restart; desc = "restart (and possibly migrate)"; } else if (pcmk__str_eq(value, "restart-container", pcmk__str_casei)) { if (rsc->container == NULL) { pcmk__rsc_debug(rsc, "Using default " PCMK_META_ON_FAIL " for %s " "of %s because it does not have a container", action_name, rsc->id); } else { on_fail = pcmk_on_fail_restart_container; desc = "restart container (and possibly migrate)"; } } else if (pcmk__str_eq(value, "demote", pcmk__str_casei)) { on_fail = pcmk_on_fail_demote; desc = "demote instance"; } else { pcmk__config_err("Using default '" PCMK_META_ON_FAIL "' for " "%s of %s because '%s' is not valid", action_name, rsc->id, value); } /* Remote node connections are handled specially. Failures that result * in dropping an active connection must result in fencing. The only * failures that don't are probes and starts. The user can explicitly set * on-fail="fence" to fence after start failures. */ if (pe__resource_is_remote_conn(rsc) && !pcmk_is_probe(action_name, interval_ms) && !pcmk__str_eq(action_name, PCMK_ACTION_START, pcmk__str_none)) { needs_remote_reset = true; if (!pcmk_is_set(rsc->flags, pcmk_rsc_managed)) { desc = NULL; // Force default for unmanaged connections } } if (desc != NULL) { // Explicit value used, default not needed } else if (rsc->container != NULL) { on_fail = pcmk_on_fail_restart_container; desc = "restart container (and possibly migrate) (default)"; } else if (needs_remote_reset) { if (pcmk_is_set(rsc->flags, pcmk_rsc_managed)) { if (pcmk_is_set(rsc->cluster->flags, pcmk_sched_fencing_enabled)) { desc = "fence remote node (default)"; } else { desc = "recover remote node connection (default)"; } on_fail = pcmk_on_fail_reset_remote; } else { on_fail = pcmk_on_fail_stop; desc = "stop unmanaged remote node (enforcing default)"; } } else if (pcmk__str_eq(action_name, PCMK_ACTION_STOP, pcmk__str_none)) { if (pcmk_is_set(rsc->cluster->flags, pcmk_sched_fencing_enabled)) { on_fail = pcmk_on_fail_fence_node; desc = "resource fence (default)"; } else { on_fail = pcmk_on_fail_block; desc = "resource block (default)"; } } else { on_fail = pcmk_on_fail_restart; desc = "restart (and possibly migrate) (default)"; } pcmk__rsc_trace(rsc, "Failure handling for %s-interval %s of %s: %s", pcmk__readable_interval(interval_ms), action_name, rsc->id, desc); return on_fail; } /*! * \internal * \brief Determine a resource's role after failure of an action * * \param[in] rsc Resource that action is for * \param[in] action_name Action name * \param[in] on_fail Failure handling for action * \param[in] meta Unpacked action meta-attributes * * \return Resource role that results from failure of action */ enum rsc_role_e pcmk__role_after_failure(const pcmk_resource_t *rsc, const char *action_name, enum action_fail_response on_fail, GHashTable *meta) { const char *value = NULL; enum rsc_role_e role = pcmk_role_unknown; // Set default for role after failure specially in certain circumstances switch (on_fail) { case pcmk_on_fail_stop: role = pcmk_role_stopped; break; case pcmk_on_fail_reset_remote: if (rsc->remote_reconnect_ms != 0) { role = pcmk_role_stopped; } break; default: break; } // @COMPAT Check for explicitly configured role (deprecated) value = g_hash_table_lookup(meta, PCMK__META_ROLE_AFTER_FAILURE); if (value != NULL) { pcmk__warn_once(pcmk__wo_role_after, "Support for " PCMK__META_ROLE_AFTER_FAILURE " is " "deprecated and will be removed in a future release"); if (role == pcmk_role_unknown) { role = text2role(value); if (role == pcmk_role_unknown) { pcmk__config_err("Ignoring invalid value %s for " PCMK__META_ROLE_AFTER_FAILURE, value); } } } if (role == pcmk_role_unknown) { // Use default if (pcmk__str_eq(action_name, PCMK_ACTION_PROMOTE, pcmk__str_none)) { role = pcmk_role_unpromoted; } else { role = pcmk_role_started; } } pcmk__rsc_trace(rsc, "Role after %s %s failure is: %s", rsc->id, action_name, role2text(role)); return role; } /*! * \internal * \brief Unpack action configuration * * Unpack a resource action's meta-attributes (normalizing the interval, * timeout, and start delay values as integer milliseconds), requirements, and * failure policy from its CIB XML configuration (including defaults). * * \param[in,out] action Resource action to unpack into * \param[in] xml_obj Action configuration XML (NULL for defaults only) * \param[in] interval_ms How frequently to perform the operation */ static void unpack_operation(pcmk_action_t *action, const xmlNode *xml_obj, guint interval_ms) { const char *value = NULL; action->meta = pcmk__unpack_action_meta(action->rsc, action->node, action->task, interval_ms, xml_obj); action->needs = pcmk__action_requires(action->rsc, action->task); value = g_hash_table_lookup(action->meta, PCMK_META_ON_FAIL); action->on_fail = pcmk__parse_on_fail(action->rsc, action->task, interval_ms, value); action->fail_role = pcmk__role_after_failure(action->rsc, action->task, action->on_fail, action->meta); } /*! * \brief Create or update an action object * * \param[in,out] rsc Resource that action is for (if any) * \param[in,out] key Action key (must be non-NULL) * \param[in] task Action name (must be non-NULL) * \param[in] on_node Node that action is on (if any) * \param[in] optional Whether action should be considered optional * \param[in,out] scheduler Scheduler data * * \return Action object corresponding to arguments (guaranteed not to be * \c NULL) * \note This function takes ownership of (and might free) \p key, and * \p scheduler takes ownership of the returned action (the caller should * not free it). */ pcmk_action_t * custom_action(pcmk_resource_t *rsc, char *key, const char *task, const pcmk_node_t *on_node, gboolean optional, pcmk_scheduler_t *scheduler) { pcmk_action_t *action = NULL; CRM_ASSERT((key != NULL) && (task != NULL) && (scheduler != NULL)); action = find_existing_action(key, rsc, on_node, scheduler); if (action == NULL) { action = new_action(key, task, rsc, on_node, optional, scheduler); } else { free(key); } update_action_optional(action, optional); if (rsc != NULL) { if ((action->node != NULL) && (action->op_entry != NULL) && !pcmk_is_set(action->flags, pcmk_action_attrs_evaluated)) { GHashTable *attrs = action->node->details->attrs; if (action->extra != NULL) { g_hash_table_destroy(action->extra); } action->extra = pcmk__unpack_action_rsc_params(action->op_entry, attrs, scheduler); pcmk__set_action_flags(action, pcmk_action_attrs_evaluated); } update_resource_action_runnable(action, scheduler); update_resource_flags_for_action(rsc, action); } if (action->extra == NULL) { action->extra = pcmk__strkey_table(free, free); } return action; } pcmk_action_t * get_pseudo_op(const char *name, pcmk_scheduler_t *scheduler) { pcmk_action_t *op = lookup_singleton(scheduler, name); if (op == NULL) { op = custom_action(NULL, strdup(name), name, NULL, TRUE, scheduler); pcmk__set_action_flags(op, pcmk_action_pseudo|pcmk_action_runnable); } return op; } static GList * find_unfencing_devices(GList *candidates, GList *matches) { for (GList *gIter = candidates; gIter != NULL; gIter = gIter->next) { pcmk_resource_t *candidate = gIter->data; if (candidate->children != NULL) { matches = find_unfencing_devices(candidate->children, matches); } else if (!pcmk_is_set(candidate->flags, pcmk_rsc_fence_device)) { continue; } else if (pcmk_is_set(candidate->flags, pcmk_rsc_needs_unfencing)) { matches = g_list_prepend(matches, candidate); } else if (pcmk__str_eq(g_hash_table_lookup(candidate->meta, PCMK_STONITH_PROVIDES), PCMK__VALUE_UNFENCING, pcmk__str_casei)) { matches = g_list_prepend(matches, candidate); } } return matches; } static int node_priority_fencing_delay(const pcmk_node_t *node, const pcmk_scheduler_t *scheduler) { int member_count = 0; int online_count = 0; int top_priority = 0; int lowest_priority = 0; GList *gIter = NULL; // PCMK_OPT_PRIORITY_FENCING_DELAY is disabled if (scheduler->priority_fencing_delay <= 0) { return 0; } /* No need to request a delay if the fencing target is not a normal cluster * member, for example if it's a remote node or a guest node. */ if (node->details->type != pcmk_node_variant_cluster) { return 0; } // No need to request a delay if the fencing target is in our partition if (node->details->online) { return 0; } for (gIter = scheduler->nodes; gIter != NULL; gIter = gIter->next) { pcmk_node_t *n = gIter->data; if (n->details->type != pcmk_node_variant_cluster) { continue; } member_count ++; if (n->details->online) { online_count++; } if (member_count == 1 || n->details->priority > top_priority) { top_priority = n->details->priority; } if (member_count == 1 || n->details->priority < lowest_priority) { lowest_priority = n->details->priority; } } // No need to delay if we have more than half of the cluster members if (online_count > member_count / 2) { return 0; } /* All the nodes have equal priority. * Any configured corresponding `pcmk_delay_base/max` will be applied. */ if (lowest_priority == top_priority) { return 0; } if (node->details->priority < top_priority) { return 0; } return scheduler->priority_fencing_delay; } pcmk_action_t * pe_fence_op(pcmk_node_t *node, const char *op, bool optional, const char *reason, bool priority_delay, pcmk_scheduler_t *scheduler) { char *op_key = NULL; pcmk_action_t *stonith_op = NULL; if(op == NULL) { op = scheduler->stonith_action; } op_key = crm_strdup_printf("%s-%s-%s", PCMK_ACTION_STONITH, node->details->uname, op); stonith_op = lookup_singleton(scheduler, op_key); if(stonith_op == NULL) { stonith_op = custom_action(NULL, op_key, PCMK_ACTION_STONITH, node, TRUE, scheduler); add_hash_param(stonith_op->meta, PCMK__META_ON_NODE, node->details->uname); - add_hash_param(stonith_op->meta, XML_LRM_ATTR_TARGET_UUID, node->details->id); + add_hash_param(stonith_op->meta, PCMK__META_ON_NODE_UUID, + node->details->id); add_hash_param(stonith_op->meta, "stonith_action", op); if (pcmk_is_set(scheduler->flags, pcmk_sched_enable_unfencing)) { /* Extra work to detect device changes */ GString *digests_all = g_string_sized_new(1024); GString *digests_secure = g_string_sized_new(1024); GList *matches = find_unfencing_devices(scheduler->resources, NULL); char *key = NULL; char *value = NULL; for (GList *gIter = matches; gIter != NULL; gIter = gIter->next) { pcmk_resource_t *match = gIter->data; const char *agent = g_hash_table_lookup(match->meta, PCMK_XA_TYPE); pcmk__op_digest_t *data = NULL; data = pe__compare_fencing_digest(match, agent, node, scheduler); if (data->rc == pcmk__digest_mismatch) { optional = FALSE; crm_notice("Unfencing node %s because the definition of " "%s changed", pe__node_name(node), match->id); if (!pcmk__is_daemon && scheduler->priv != NULL) { pcmk__output_t *out = scheduler->priv; out->info(out, "notice: Unfencing node %s because the " "definition of %s changed", pe__node_name(node), match->id); } } pcmk__g_strcat(digests_all, match->id, ":", agent, ":", data->digest_all_calc, ",", NULL); pcmk__g_strcat(digests_secure, match->id, ":", agent, ":", data->digest_secure_calc, ",", NULL); } key = strdup(PCMK__META_DIGESTS_ALL); value = strdup((const char *) digests_all->str); CRM_ASSERT((key != NULL) && (value != NULL)); g_hash_table_insert(stonith_op->meta, key, value); g_string_free(digests_all, TRUE); key = strdup(PCMK__META_DIGESTS_SECURE); value = strdup((const char *) digests_secure->str); CRM_ASSERT((key != NULL) && (value != NULL)); g_hash_table_insert(stonith_op->meta, key, value); g_string_free(digests_secure, TRUE); } } else { free(op_key); } if (scheduler->priority_fencing_delay > 0 /* It's a suitable case where PCMK_OPT_PRIORITY_FENCING_DELAY * applies. At least add PCMK_OPT_PRIORITY_FENCING_DELAY field as * an indicator. */ && (priority_delay /* The priority delay needs to be recalculated if this function has * been called by schedule_fencing_and_shutdowns() after node * priority has already been calculated by native_add_running(). */ || g_hash_table_lookup(stonith_op->meta, PCMK_OPT_PRIORITY_FENCING_DELAY) != NULL)) { /* Add PCMK_OPT_PRIORITY_FENCING_DELAY to the fencing op even if * it's 0 for the targeting node. So that it takes precedence over * any possible `pcmk_delay_base/max`. */ char *delay_s = pcmk__itoa(node_priority_fencing_delay(node, scheduler)); g_hash_table_insert(stonith_op->meta, strdup(PCMK_OPT_PRIORITY_FENCING_DELAY), delay_s); } if(optional == FALSE && pe_can_fence(scheduler, node)) { pcmk__clear_action_flags(stonith_op, pcmk_action_optional); pe_action_set_reason(stonith_op, reason, false); } else if(reason && stonith_op->reason == NULL) { stonith_op->reason = strdup(reason); } return stonith_op; } void pe_free_action(pcmk_action_t *action) { if (action == NULL) { return; } g_list_free_full(action->actions_before, free); g_list_free_full(action->actions_after, free); if (action->extra) { g_hash_table_destroy(action->extra); } if (action->meta) { g_hash_table_destroy(action->meta); } free(action->cancel_task); free(action->reason); free(action->task); free(action->uuid); free(action->node); free(action); } int pe_get_configured_timeout(pcmk_resource_t *rsc, const char *action, pcmk_scheduler_t *scheduler) { xmlNode *child = NULL; GHashTable *action_meta = NULL; const char *timeout_spec = NULL; int timeout_ms = 0; pe_rule_eval_data_t rule_data = { .node_hash = NULL, .role = pcmk_role_unknown, .now = scheduler->now, .match_data = NULL, .rsc_data = NULL, .op_data = NULL }; for (child = first_named_child(rsc->ops_xml, PCMK_XE_OP); child != NULL; child = crm_next_same_xml(child)) { if (pcmk__str_eq(action, crm_element_value(child, PCMK_XA_NAME), pcmk__str_casei)) { timeout_spec = crm_element_value(child, PCMK_META_TIMEOUT); break; } } if (timeout_spec == NULL && scheduler->op_defaults) { action_meta = pcmk__strkey_table(free, free); pe__unpack_dataset_nvpairs(scheduler->op_defaults, XML_TAG_META_SETS, &rule_data, action_meta, NULL, FALSE, scheduler); timeout_spec = g_hash_table_lookup(action_meta, PCMK_META_TIMEOUT); } // @TODO check meta-attributes // @TODO maybe use min-interval monitor timeout as default for monitors timeout_ms = crm_get_msec(timeout_spec); if (timeout_ms < 0) { timeout_ms = PCMK_DEFAULT_ACTION_TIMEOUT_MS; } if (action_meta != NULL) { g_hash_table_destroy(action_meta); } return timeout_ms; } enum action_tasks get_complex_task(const pcmk_resource_t *rsc, const char *name) { enum action_tasks task = text2task(name); if ((rsc != NULL) && (rsc->variant == pcmk_rsc_variant_primitive)) { switch (task) { case pcmk_action_stopped: case pcmk_action_started: case pcmk_action_demoted: case pcmk_action_promoted: crm_trace("Folding %s back into its atomic counterpart for %s", name, rsc->id); --task; break; default: break; } } return task; } /*! * \internal * \brief Find first matching action in a list * * \param[in] input List of actions to search * \param[in] uuid If not NULL, action must have this UUID * \param[in] task If not NULL, action must have this action name * \param[in] on_node If not NULL, action must be on this node * * \return First action in list that matches criteria, or NULL if none */ pcmk_action_t * find_first_action(const GList *input, const char *uuid, const char *task, const pcmk_node_t *on_node) { CRM_CHECK(uuid || task, return NULL); for (const GList *gIter = input; gIter != NULL; gIter = gIter->next) { pcmk_action_t *action = (pcmk_action_t *) gIter->data; if (uuid != NULL && !pcmk__str_eq(uuid, action->uuid, pcmk__str_casei)) { continue; } else if (task != NULL && !pcmk__str_eq(task, action->task, pcmk__str_casei)) { continue; } else if (on_node == NULL) { return action; } else if (action->node == NULL) { continue; } else if (on_node->details == action->node->details) { return action; } } return NULL; } GList * find_actions(GList *input, const char *key, const pcmk_node_t *on_node) { GList *gIter = input; GList *result = NULL; CRM_CHECK(key != NULL, return NULL); for (; gIter != NULL; gIter = gIter->next) { pcmk_action_t *action = (pcmk_action_t *) gIter->data; if (!pcmk__str_eq(key, action->uuid, pcmk__str_casei)) { continue; } else if (on_node == NULL) { crm_trace("Action %s matches (ignoring node)", key); result = g_list_prepend(result, action); } else if (action->node == NULL) { crm_trace("Action %s matches (unallocated, assigning to %s)", key, pe__node_name(on_node)); action->node = pe__copy_node(on_node); result = g_list_prepend(result, action); } else if (on_node->details == action->node->details) { crm_trace("Action %s on %s matches", key, pe__node_name(on_node)); result = g_list_prepend(result, action); } } return result; } GList * find_actions_exact(GList *input, const char *key, const pcmk_node_t *on_node) { GList *result = NULL; CRM_CHECK(key != NULL, return NULL); if (on_node == NULL) { return NULL; } for (GList *gIter = input; gIter != NULL; gIter = gIter->next) { pcmk_action_t *action = (pcmk_action_t *) gIter->data; if ((action->node != NULL) && pcmk__str_eq(key, action->uuid, pcmk__str_casei) && pcmk__str_eq(on_node->details->id, action->node->details->id, pcmk__str_casei)) { crm_trace("Action %s on %s matches", key, pe__node_name(on_node)); result = g_list_prepend(result, action); } } return result; } /*! * \brief Find all actions of given type for a resource * * \param[in] rsc Resource to search * \param[in] node Find only actions scheduled on this node * \param[in] task Action name to search for * \param[in] require_node If TRUE, NULL node or action node will not match * * \return List of actions found (or NULL if none) * \note If node is not NULL and require_node is FALSE, matching actions * without a node will be assigned to node. */ GList * pe__resource_actions(const pcmk_resource_t *rsc, const pcmk_node_t *node, const char *task, bool require_node) { GList *result = NULL; char *key = pcmk__op_key(rsc->id, task, 0); if (require_node) { result = find_actions_exact(rsc->actions, key, node); } else { result = find_actions(rsc->actions, key, node); } free(key); return result; } /*! * \internal * \brief Create an action reason string based on the action itself * * \param[in] action Action to create reason string for * \param[in] flag Action flag that was cleared * * \return Newly allocated string suitable for use as action reason * \note It is the caller's responsibility to free() the result. */ char * pe__action2reason(const pcmk_action_t *action, enum pe_action_flags flag) { const char *change = NULL; switch (flag) { case pcmk_action_runnable: change = "unrunnable"; break; case pcmk_action_migratable: change = "unmigrateable"; break; case pcmk_action_optional: change = "required"; break; default: // Bug: caller passed unsupported flag CRM_CHECK(change != NULL, change = ""); break; } return crm_strdup_printf("%s%s%s %s", change, (action->rsc == NULL)? "" : " ", (action->rsc == NULL)? "" : action->rsc->id, action->task); } void pe_action_set_reason(pcmk_action_t *action, const char *reason, bool overwrite) { if (action->reason != NULL && overwrite) { pcmk__rsc_trace(action->rsc, "Changing %s reason from '%s' to '%s'", action->uuid, action->reason, pcmk__s(reason, "(none)")); } else if (action->reason == NULL) { pcmk__rsc_trace(action->rsc, "Set %s reason to '%s'", action->uuid, pcmk__s(reason, "(none)")); } else { // crm_assert(action->reason != NULL && !overwrite); return; } pcmk__str_update(&action->reason, reason); } /*! * \internal * \brief Create an action to clear a resource's history from CIB * * \param[in,out] rsc Resource to clear * \param[in] node Node to clear history on */ void pe__clear_resource_history(pcmk_resource_t *rsc, const pcmk_node_t *node) { CRM_ASSERT((rsc != NULL) && (node != NULL)); custom_action(rsc, pcmk__op_key(rsc->id, PCMK_ACTION_LRM_DELETE, 0), PCMK_ACTION_LRM_DELETE, node, FALSE, rsc->cluster); } #define sort_return(an_int, why) do { \ free(a_uuid); \ free(b_uuid); \ crm_trace("%s (%d) %c %s (%d) : %s", \ a_xml_id, a_call_id, an_int>0?'>':an_int<0?'<':'=', \ b_xml_id, b_call_id, why); \ return an_int; \ } while(0) int pe__is_newer_op(const xmlNode *xml_a, const xmlNode *xml_b, bool same_node_default) { int a_call_id = -1; int b_call_id = -1; char *a_uuid = NULL; char *b_uuid = NULL; const char *a_xml_id = crm_element_value(xml_a, PCMK_XA_ID); const char *b_xml_id = crm_element_value(xml_b, PCMK_XA_ID); const char *a_node = crm_element_value(xml_a, PCMK__META_ON_NODE); const char *b_node = crm_element_value(xml_b, PCMK__META_ON_NODE); bool same_node = true; /* @COMPAT The on_node attribute was added to last_failure as of 1.1.13 (via * 8b3ca1c) and the other entries as of 1.1.12 (via 0b07b5c). * * In case that any of the lrm_rsc_op entries doesn't have on_node * attribute, we need to explicitly tell whether the two operations are on * the same node. */ if (a_node == NULL || b_node == NULL) { same_node = same_node_default; } else { same_node = pcmk__str_eq(a_node, b_node, pcmk__str_casei); } if (same_node && pcmk__str_eq(a_xml_id, b_xml_id, pcmk__str_none)) { /* We have duplicate lrm_rsc_op entries in the status * section which is unlikely to be a good thing * - we can handle it easily enough, but we need to get * to the bottom of why it's happening. */ pcmk__config_err("Duplicate lrm_rsc_op entries named %s", a_xml_id); sort_return(0, "duplicate"); } crm_element_value_int(xml_a, XML_LRM_ATTR_CALLID, &a_call_id); crm_element_value_int(xml_b, XML_LRM_ATTR_CALLID, &b_call_id); if (a_call_id == -1 && b_call_id == -1) { /* both are pending ops so it doesn't matter since * stops are never pending */ sort_return(0, "pending"); } else if (same_node && a_call_id >= 0 && a_call_id < b_call_id) { sort_return(-1, "call id"); } else if (same_node && b_call_id >= 0 && a_call_id > b_call_id) { sort_return(1, "call id"); } else if (a_call_id >= 0 && b_call_id >= 0 && (!same_node || a_call_id == b_call_id)) { /* * The op and last_failed_op are the same * Order on last-rc-change */ time_t last_a = -1; time_t last_b = -1; crm_element_value_epoch(xml_a, XML_RSC_OP_LAST_CHANGE, &last_a); crm_element_value_epoch(xml_b, XML_RSC_OP_LAST_CHANGE, &last_b); crm_trace("rc-change: %lld vs %lld", (long long) last_a, (long long) last_b); if (last_a >= 0 && last_a < last_b) { sort_return(-1, "rc-change"); } else if (last_b >= 0 && last_a > last_b) { sort_return(1, "rc-change"); } sort_return(0, "rc-change"); } else { /* One of the inputs is a pending operation. * Attempt to use PCMK__XA_TRANSITION_MAGIC to determine its age relative * to the other. */ int a_id = -1; int b_id = -1; const char *a_magic = crm_element_value(xml_a, PCMK__XA_TRANSITION_MAGIC); const char *b_magic = crm_element_value(xml_b, PCMK__XA_TRANSITION_MAGIC); CRM_CHECK(a_magic != NULL && b_magic != NULL, sort_return(0, "No magic")); if (!decode_transition_magic(a_magic, &a_uuid, &a_id, NULL, NULL, NULL, NULL)) { sort_return(0, "bad magic a"); } if (!decode_transition_magic(b_magic, &b_uuid, &b_id, NULL, NULL, NULL, NULL)) { sort_return(0, "bad magic b"); } /* try to determine the relative age of the operation... * some pending operations (e.g. a start) may have been superseded * by a subsequent stop * * [a|b]_id == -1 means it's a shutdown operation and _always_ comes last */ if (!pcmk__str_eq(a_uuid, b_uuid, pcmk__str_casei) || a_id == b_id) { /* * some of the logic in here may be redundant... * * if the UUID from the TE doesn't match then one better * be a pending operation. * pending operations don't survive between elections and joins * because we query the LRM directly */ if (b_call_id == -1) { sort_return(-1, "transition + call"); } else if (a_call_id == -1) { sort_return(1, "transition + call"); } } else if ((a_id >= 0 && a_id < b_id) || b_id == -1) { sort_return(-1, "transition"); } else if ((b_id >= 0 && a_id > b_id) || a_id == -1) { sort_return(1, "transition"); } } /* we should never end up here */ CRM_CHECK(FALSE, sort_return(0, "default")); } gint sort_op_by_callid(gconstpointer a, gconstpointer b) { const xmlNode *xml_a = a; const xmlNode *xml_b = b; return pe__is_newer_op(xml_a, xml_b, true); } /*! * \internal * \brief Create a new pseudo-action for a resource * * \param[in,out] rsc Resource to create action for * \param[in] task Action name * \param[in] optional Whether action should be considered optional * \param[in] runnable Whethe action should be considered runnable * * \return New action object corresponding to arguments */ pcmk_action_t * pe__new_rsc_pseudo_action(pcmk_resource_t *rsc, const char *task, bool optional, bool runnable) { pcmk_action_t *action = NULL; CRM_ASSERT((rsc != NULL) && (task != NULL)); action = custom_action(rsc, pcmk__op_key(rsc->id, task, 0), task, NULL, optional, rsc->cluster); pcmk__set_action_flags(action, pcmk_action_pseudo); if (runnable) { pcmk__set_action_flags(action, pcmk_action_runnable); } return action; } /*! * \internal * \brief Add the expected result to an action * * \param[in,out] action Action to add expected result to * \param[in] expected_result Expected result to add * * \note This is more efficient than calling add_hash_param(). */ void pe__add_action_expected_result(pcmk_action_t *action, int expected_result) { char *name = NULL; CRM_ASSERT((action != NULL) && (action->meta != NULL)); name = strdup(XML_ATTR_TE_TARGET_RC); CRM_ASSERT (name != NULL); g_hash_table_insert(action->meta, name, pcmk__itoa(expected_result)); }