diff --git a/daemons/controld/controld_execd.c b/daemons/controld/controld_execd.c index d46d99c40c..86e64c3732 100644 --- a/daemons/controld/controld_execd.c +++ b/daemons/controld/controld_execd.c @@ -1,2447 +1,2447 @@ /* * Copyright 2004-2024 the Pacemaker project contributors * * The version control history for this file may have further details. * * This source code is licensed under the GNU General Public License version 2 * or later (GPLv2+) WITHOUT ANY WARRANTY. */ #include #include #include #include #include #include #include // lrmd_event_data_t, lrmd_rsc_info_t, etc. #include #include #include #include #include #include #include #define START_DELAY_THRESHOLD 5 * 60 * 1000 #define MAX_LRM_REG_FAILS 30 struct delete_event_s { int rc; const char *rsc; lrm_state_t *lrm_state; }; static gboolean is_rsc_active(lrm_state_t * lrm_state, const char *rsc_id); static gboolean build_active_RAs(lrm_state_t * lrm_state, xmlNode * rsc_list); static gboolean stop_recurring_actions(gpointer key, gpointer value, gpointer user_data); static lrmd_event_data_t *construct_op(const lrm_state_t *lrm_state, const xmlNode *rsc_op, const char *rsc_id, const char *operation); static void do_lrm_rsc_op(lrm_state_t *lrm_state, lrmd_rsc_info_t *rsc, xmlNode *msg, struct ra_metadata_s *md); static gboolean lrm_state_verify_stopped(lrm_state_t * lrm_state, enum crmd_fsa_state cur_state, int log_level); static void lrm_connection_destroy(void) { if (pcmk_is_set(controld_globals.fsa_input_register, R_LRM_CONNECTED)) { crm_crit("Lost connection to local executor"); register_fsa_input(C_FSA_INTERNAL, I_ERROR, NULL); controld_clear_fsa_input_flags(R_LRM_CONNECTED); } } static char * make_stop_id(const char *rsc, int call_id) { return crm_strdup_printf("%s:%d", rsc, call_id); } static void copy_instance_keys(gpointer key, gpointer value, gpointer user_data) { if (strstr(key, CRM_META "_") == NULL) { g_hash_table_replace(user_data, strdup((const char *)key), strdup((const char *)value)); } } static void copy_meta_keys(gpointer key, gpointer value, gpointer user_data) { if (strstr(key, CRM_META "_") != NULL) { g_hash_table_replace(user_data, strdup((const char *)key), strdup((const char *)value)); } } /*! * \internal * \brief Remove a recurring operation from a resource's history * * \param[in,out] history Resource history to modify * \param[in] op Operation to remove * * \return TRUE if the operation was found and removed, FALSE otherwise */ static gboolean history_remove_recurring_op(rsc_history_t *history, const lrmd_event_data_t *op) { GList *iter; for (iter = history->recurring_op_list; iter != NULL; iter = iter->next) { lrmd_event_data_t *existing = iter->data; if ((op->interval_ms == existing->interval_ms) && pcmk__str_eq(op->rsc_id, existing->rsc_id, pcmk__str_none) && pcmk__str_eq(op->op_type, existing->op_type, pcmk__str_casei)) { history->recurring_op_list = g_list_delete_link(history->recurring_op_list, iter); lrmd_free_event(existing); return TRUE; } } return FALSE; } /*! * \internal * \brief Free all recurring operations in resource history * * \param[in,out] history Resource history to modify */ static void history_free_recurring_ops(rsc_history_t *history) { GList *iter; for (iter = history->recurring_op_list; iter != NULL; iter = iter->next) { lrmd_free_event(iter->data); } g_list_free(history->recurring_op_list); history->recurring_op_list = NULL; } /*! * \internal * \brief Free resource history * * \param[in,out] history Resource history to free */ void history_free(gpointer data) { rsc_history_t *history = (rsc_history_t*)data; if (history->stop_params) { g_hash_table_destroy(history->stop_params); } /* Don't need to free history->rsc.id because it's set to history->id */ free(history->rsc.type); free(history->rsc.standard); free(history->rsc.provider); lrmd_free_event(history->failed); lrmd_free_event(history->last); free(history->id); history_free_recurring_ops(history); free(history); } static void update_history_cache(lrm_state_t * lrm_state, lrmd_rsc_info_t * rsc, lrmd_event_data_t * op) { int target_rc = 0; rsc_history_t *entry = NULL; if (op->rsc_deleted) { crm_debug("Purged history for '%s' after %s", op->rsc_id, op->op_type); controld_delete_resource_history(op->rsc_id, lrm_state->node_name, NULL, crmd_cib_smart_opt()); return; } if (pcmk__str_eq(op->op_type, PCMK_ACTION_NOTIFY, pcmk__str_casei)) { return; } crm_debug("Updating history for '%s' with %s op", op->rsc_id, op->op_type); entry = g_hash_table_lookup(lrm_state->resource_history, op->rsc_id); if (entry == NULL && rsc) { entry = calloc(1, sizeof(rsc_history_t)); entry->id = strdup(op->rsc_id); g_hash_table_insert(lrm_state->resource_history, entry->id, entry); entry->rsc.id = entry->id; entry->rsc.type = strdup(rsc->type); entry->rsc.standard = strdup(rsc->standard); pcmk__str_update(&entry->rsc.provider, rsc->provider); } else if (entry == NULL) { crm_info("Resource %s no longer exists, not updating cache", op->rsc_id); return; } entry->last_callid = op->call_id; target_rc = rsc_op_expected_rc(op); if (op->op_status == PCMK_EXEC_CANCELLED) { if (op->interval_ms > 0) { crm_trace("Removing cancelled recurring op: " PCMK__OP_FMT, op->rsc_id, op->op_type, op->interval_ms); history_remove_recurring_op(entry, op); return; } else { crm_trace("Skipping " PCMK__OP_FMT " rc=%d, status=%d", op->rsc_id, op->op_type, op->interval_ms, op->rc, op->op_status); } } else if (did_rsc_op_fail(op, target_rc)) { /* Store failed monitors here, otherwise the block below will cause them * to be forgotten when a stop happens. */ if (entry->failed) { lrmd_free_event(entry->failed); } entry->failed = lrmd_copy_event(op); } else if (op->interval_ms == 0) { if (entry->last) { lrmd_free_event(entry->last); } entry->last = lrmd_copy_event(op); if (op->params && pcmk__strcase_any_of(op->op_type, PCMK_ACTION_START, PCMK_ACTION_RELOAD, PCMK_ACTION_RELOAD_AGENT, PCMK_ACTION_MONITOR, NULL)) { if (entry->stop_params) { g_hash_table_destroy(entry->stop_params); } entry->stop_params = pcmk__strkey_table(free, free); g_hash_table_foreach(op->params, copy_instance_keys, entry->stop_params); } } if (op->interval_ms > 0) { /* Ensure there are no duplicates */ history_remove_recurring_op(entry, op); crm_trace("Adding recurring op: " PCMK__OP_FMT, op->rsc_id, op->op_type, op->interval_ms); entry->recurring_op_list = g_list_prepend(entry->recurring_op_list, lrmd_copy_event(op)); } else if ((entry->recurring_op_list != NULL) && !pcmk__str_eq(op->op_type, PCMK_ACTION_MONITOR, pcmk__str_casei)) { crm_trace("Dropping %d recurring ops because of: " PCMK__OP_FMT, g_list_length(entry->recurring_op_list), op->rsc_id, op->op_type, op->interval_ms); history_free_recurring_ops(entry); } } /*! * \internal * \brief Send a direct OK ack for a resource task * * \param[in] lrm_state LRM connection * \param[in] input Input message being ack'ed * \param[in] rsc_id ID of affected resource * \param[in] rsc Affected resource (if available) * \param[in] task Operation task being ack'ed * \param[in] ack_host Name of host to send ack to * \param[in] ack_sys IPC system name to ack */ static void send_task_ok_ack(const lrm_state_t *lrm_state, const ha_msg_input_t *input, const char *rsc_id, const lrmd_rsc_info_t *rsc, const char *task, const char *ack_host, const char *ack_sys) { lrmd_event_data_t *op = construct_op(lrm_state, input->xml, rsc_id, task); lrmd__set_result(op, PCMK_OCF_OK, PCMK_EXEC_DONE, NULL); controld_ack_event_directly(ack_host, ack_sys, rsc, op, rsc_id); lrmd_free_event(op); } static inline const char * op_node_name(lrmd_event_data_t *op) { return pcmk__s(op->remote_nodename, controld_globals.our_nodename); } void lrm_op_callback(lrmd_event_data_t * op) { CRM_CHECK(op != NULL, return); switch (op->type) { case lrmd_event_disconnect: if (op->remote_nodename == NULL) { /* If this is the local executor IPC connection, set the right * bits in the controller when the connection goes down. */ lrm_connection_destroy(); } break; case lrmd_event_exec_complete: { lrm_state_t *lrm_state = lrm_state_find(op_node_name(op)); CRM_ASSERT(lrm_state != NULL); process_lrm_event(lrm_state, op, NULL, NULL); } break; default: break; } } static void try_local_executor_connect(long long action, fsa_data_t *msg_data, lrm_state_t *lrm_state) { int rc = pcmk_rc_ok; crm_debug("Connecting to the local executor"); // If we can connect, great rc = controld_connect_local_executor(lrm_state); if (rc == pcmk_rc_ok) { controld_set_fsa_input_flags(R_LRM_CONNECTED); crm_info("Connection to the local executor established"); return; } // Otherwise, if we can try again, set a timer to do so if (lrm_state->num_lrm_register_fails < MAX_LRM_REG_FAILS) { crm_warn("Failed to connect to the local executor %d time%s " "(%d max): %s", lrm_state->num_lrm_register_fails, pcmk__plural_s(lrm_state->num_lrm_register_fails), MAX_LRM_REG_FAILS, pcmk_rc_str(rc)); controld_start_wait_timer(); crmd_fsa_stall(FALSE); return; } // Otherwise give up crm_err("Failed to connect to the executor the max allowed " "%d time%s: %s", lrm_state->num_lrm_register_fails, pcmk__plural_s(lrm_state->num_lrm_register_fails), pcmk_rc_str(rc)); register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL); } /* A_LRM_CONNECT */ void do_lrm_control(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data) { /* This only pertains to local executor connections. Remote connections are * handled as resources within the scheduler. Connecting and disconnecting * from remote executor instances is handled differently. */ lrm_state_t *lrm_state = NULL; if (controld_globals.our_nodename == NULL) { return; /* Nothing to do */ } lrm_state = lrm_state_find_or_create(controld_globals.our_nodename); if (lrm_state == NULL) { register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL); return; } if (action & A_LRM_DISCONNECT) { if (lrm_state_verify_stopped(lrm_state, cur_state, LOG_INFO) == FALSE) { if (action == A_LRM_DISCONNECT) { crmd_fsa_stall(FALSE); return; } } controld_clear_fsa_input_flags(R_LRM_CONNECTED); lrm_state_disconnect(lrm_state); lrm_state_reset_tables(lrm_state, FALSE); } if (action & A_LRM_CONNECT) { try_local_executor_connect(action, msg_data, lrm_state); } if (action & ~(A_LRM_CONNECT | A_LRM_DISCONNECT)) { crm_err("Unexpected action %s in %s", fsa_action2string(action), __func__); } } static gboolean lrm_state_verify_stopped(lrm_state_t * lrm_state, enum crmd_fsa_state cur_state, int log_level) { int counter = 0; gboolean rc = TRUE; const char *when = "lrm disconnect"; GHashTableIter gIter; const char *key = NULL; rsc_history_t *entry = NULL; active_op_t *pending = NULL; crm_debug("Checking for active resources before exit"); if (cur_state == S_TERMINATE) { log_level = LOG_ERR; when = "shutdown"; } else if (pcmk_is_set(controld_globals.fsa_input_register, R_SHUTDOWN)) { when = "shutdown... waiting"; } if ((lrm_state->active_ops != NULL) && lrm_state_is_connected(lrm_state)) { guint removed = g_hash_table_foreach_remove(lrm_state->active_ops, stop_recurring_actions, lrm_state); guint nremaining = g_hash_table_size(lrm_state->active_ops); if (removed || nremaining) { crm_notice("Stopped %u recurring operation%s at %s (%u remaining)", removed, pcmk__plural_s(removed), when, nremaining); } } if (lrm_state->active_ops != NULL) { g_hash_table_iter_init(&gIter, lrm_state->active_ops); while (g_hash_table_iter_next(&gIter, NULL, (void **)&pending)) { /* Ignore recurring actions in the shutdown calculations */ if (pending->interval_ms == 0) { counter++; } } } if (counter > 0) { do_crm_log(log_level, "%d pending executor operation%s at %s", counter, pcmk__plural_s(counter), when); if ((cur_state == S_TERMINATE) || !pcmk_is_set(controld_globals.fsa_input_register, R_SENT_RSC_STOP)) { g_hash_table_iter_init(&gIter, lrm_state->active_ops); while (g_hash_table_iter_next(&gIter, (gpointer*)&key, (gpointer*)&pending)) { do_crm_log(log_level, "Pending action: %s (%s)", key, pending->op_key); } } else { rc = FALSE; } return rc; } if (lrm_state->resource_history == NULL) { return rc; } if (pcmk_is_set(controld_globals.fsa_input_register, R_SHUTDOWN)) { /* At this point we're not waiting, we're just shutting down */ when = "shutdown"; } counter = 0; g_hash_table_iter_init(&gIter, lrm_state->resource_history); while (g_hash_table_iter_next(&gIter, NULL, (gpointer*)&entry)) { if (is_rsc_active(lrm_state, entry->id) == FALSE) { continue; } counter++; if (log_level == LOG_ERR) { crm_info("Found %s active at %s", entry->id, when); } else { crm_trace("Found %s active at %s", entry->id, when); } if (lrm_state->active_ops != NULL) { GHashTableIter hIter; g_hash_table_iter_init(&hIter, lrm_state->active_ops); while (g_hash_table_iter_next(&hIter, (gpointer*)&key, (gpointer*)&pending)) { if (pcmk__str_eq(entry->id, pending->rsc_id, pcmk__str_none)) { crm_notice("%sction %s (%s) incomplete at %s", pending->interval_ms == 0 ? "A" : "Recurring a", key, pending->op_key, when); } } } } if (counter) { crm_err("%d resource%s active at %s", counter, (counter == 1)? " was" : "s were", when); } return rc; } static gboolean is_rsc_active(lrm_state_t * lrm_state, const char *rsc_id) { rsc_history_t *entry = NULL; entry = g_hash_table_lookup(lrm_state->resource_history, rsc_id); if (entry == NULL || entry->last == NULL) { return FALSE; } crm_trace("Processing %s: %s.%d=%d", rsc_id, entry->last->op_type, entry->last->interval_ms, entry->last->rc); if ((entry->last->rc == PCMK_OCF_OK) && pcmk__str_eq(entry->last->op_type, PCMK_ACTION_STOP, pcmk__str_casei)) { return FALSE; } else if (entry->last->rc == PCMK_OCF_OK && pcmk__str_eq(entry->last->op_type, PCMK_ACTION_MIGRATE_TO, pcmk__str_casei)) { // A stricter check is too complex ... leave that to the scheduler return FALSE; } else if (entry->last->rc == PCMK_OCF_NOT_RUNNING) { return FALSE; } else if ((entry->last->interval_ms == 0) && (entry->last->rc == PCMK_OCF_NOT_CONFIGURED)) { /* Badly configured resources can't be reliably stopped */ return FALSE; } return TRUE; } static gboolean build_active_RAs(lrm_state_t * lrm_state, xmlNode * rsc_list) { GHashTableIter iter; rsc_history_t *entry = NULL; g_hash_table_iter_init(&iter, lrm_state->resource_history); while (g_hash_table_iter_next(&iter, NULL, (void **)&entry)) { GList *gIter = NULL; xmlNode *xml_rsc = create_xml_node(rsc_list, XML_LRM_TAG_RESOURCE); crm_xml_add(xml_rsc, PCMK_XA_ID, entry->id); crm_xml_add(xml_rsc, PCMK_XA_TYPE, entry->rsc.type); crm_xml_add(xml_rsc, PCMK_XA_CLASS, entry->rsc.standard); crm_xml_add(xml_rsc, PCMK_XA_PROVIDER, entry->rsc.provider); if (entry->last && entry->last->params) { static const char *name = CRM_META "_" PCMK__META_CONTAINER; const char *container = g_hash_table_lookup(entry->last->params, name); if (container) { crm_trace("Resource %s is a part of container resource %s", entry->id, container); crm_xml_add(xml_rsc, PCMK__META_CONTAINER, container); } } controld_add_resource_history_xml(xml_rsc, &(entry->rsc), entry->failed, lrm_state->node_name); controld_add_resource_history_xml(xml_rsc, &(entry->rsc), entry->last, lrm_state->node_name); for (gIter = entry->recurring_op_list; gIter != NULL; gIter = gIter->next) { controld_add_resource_history_xml(xml_rsc, &(entry->rsc), gIter->data, lrm_state->node_name); } } return FALSE; } xmlNode * controld_query_executor_state(void) { xmlNode *xml_state = NULL; xmlNode *xml_data = NULL; xmlNode *rsc_list = NULL; crm_node_t *peer = NULL; lrm_state_t *lrm_state = lrm_state_find(controld_globals.our_nodename); if (!lrm_state) { crm_err("Could not find executor state for node %s", controld_globals.our_nodename); return NULL; } peer = pcmk__get_node(0, lrm_state->node_name, NULL, pcmk__node_search_any); CRM_CHECK(peer != NULL, return NULL); xml_state = create_node_state_update(peer, node_update_cluster|node_update_peer, NULL, __func__); if (xml_state == NULL) { return NULL; } xml_data = create_xml_node(xml_state, XML_CIB_TAG_LRM); crm_xml_add(xml_data, PCMK_XA_ID, peer->uuid); rsc_list = create_xml_node(xml_data, XML_LRM_TAG_RESOURCES); /* Build a list of active (not always running) resources */ build_active_RAs(lrm_state, rsc_list); crm_log_xml_trace(xml_state, "Current executor state"); return xml_state; } /*! * \internal * \brief Map standard Pacemaker return code to operation status and OCF code * * \param[out] event Executor event whose status and return code should be set * \param[in] rc Standard Pacemaker return code */ void controld_rc2event(lrmd_event_data_t *event, int rc) { /* This is called for cleanup requests from controller peers/clients, not * for resource actions, so no exit reason is needed. */ switch (rc) { case pcmk_rc_ok: lrmd__set_result(event, PCMK_OCF_OK, PCMK_EXEC_DONE, NULL); break; case EACCES: lrmd__set_result(event, PCMK_OCF_INSUFFICIENT_PRIV, PCMK_EXEC_ERROR, NULL); break; default: lrmd__set_result(event, PCMK_OCF_UNKNOWN_ERROR, PCMK_EXEC_ERROR, NULL); break; } } /*! * \internal * \brief Trigger a new transition after CIB status was deleted * * If a CIB status delete was not expected (as part of the transition graph), * trigger a new transition by updating the (arbitrary) "last-lrm-refresh" * cluster property. * * \param[in] from_sys IPC name that requested the delete * \param[in] rsc_id Resource whose status was deleted (for logging only) */ void controld_trigger_delete_refresh(const char *from_sys, const char *rsc_id) { if (!pcmk__str_eq(from_sys, CRM_SYSTEM_TENGINE, pcmk__str_casei)) { char *now_s = crm_strdup_printf("%lld", (long long) time(NULL)); crm_debug("Triggering a refresh after %s cleaned %s", from_sys, rsc_id); cib__update_node_attr(controld_globals.logger_out, controld_globals.cib_conn, cib_none, XML_CIB_TAG_CRMCONFIG, NULL, NULL, NULL, NULL, "last-lrm-refresh", now_s, NULL, NULL); free(now_s); } } static void notify_deleted(lrm_state_t * lrm_state, ha_msg_input_t * input, const char *rsc_id, int rc) { lrmd_event_data_t *op = NULL; const char *from_sys = crm_element_value(input->msg, PCMK__XA_CRM_SYS_FROM); const char *from_host = crm_element_value(input->msg, PCMK__XA_SRC); crm_info("Notifying %s on %s that %s was%s deleted", from_sys, (from_host? from_host : "localhost"), rsc_id, ((rc == pcmk_ok)? "" : " not")); op = construct_op(lrm_state, input->xml, rsc_id, PCMK_ACTION_DELETE); controld_rc2event(op, pcmk_legacy2rc(rc)); controld_ack_event_directly(from_host, from_sys, NULL, op, rsc_id); lrmd_free_event(op); controld_trigger_delete_refresh(from_sys, rsc_id); } static gboolean lrm_remove_deleted_rsc(gpointer key, gpointer value, gpointer user_data) { struct delete_event_s *event = user_data; struct pending_deletion_op_s *op = value; if (pcmk__str_eq(event->rsc, op->rsc, pcmk__str_none)) { notify_deleted(event->lrm_state, op->input, event->rsc, event->rc); return TRUE; } return FALSE; } static gboolean lrm_remove_deleted_op(gpointer key, gpointer value, gpointer user_data) { const char *rsc = user_data; active_op_t *pending = value; if (pcmk__str_eq(rsc, pending->rsc_id, pcmk__str_none)) { crm_info("Removing op %s:%d for deleted resource %s", pending->op_key, pending->call_id, rsc); return TRUE; } return FALSE; } static void delete_rsc_entry(lrm_state_t *lrm_state, ha_msg_input_t *input, const char *rsc_id, GHashTableIter *rsc_iter, int rc, const char *user_name, bool from_cib) { struct delete_event_s event; CRM_CHECK(rsc_id != NULL, return); if (rc == pcmk_ok) { char *rsc_id_copy = strdup(rsc_id); if (rsc_iter) { g_hash_table_iter_remove(rsc_iter); } else { g_hash_table_remove(lrm_state->resource_history, rsc_id_copy); } if (from_cib) { controld_delete_resource_history(rsc_id_copy, lrm_state->node_name, user_name, crmd_cib_smart_opt()); } g_hash_table_foreach_remove(lrm_state->active_ops, lrm_remove_deleted_op, rsc_id_copy); free(rsc_id_copy); } if (input) { notify_deleted(lrm_state, input, rsc_id, rc); } event.rc = rc; event.rsc = rsc_id; event.lrm_state = lrm_state; g_hash_table_foreach_remove(lrm_state->deletion_ops, lrm_remove_deleted_rsc, &event); } static inline gboolean last_failed_matches_op(rsc_history_t *entry, const char *op, guint interval_ms) { if (entry == NULL) { return FALSE; } if (op == NULL) { return TRUE; } return (pcmk__str_eq(op, entry->failed->op_type, pcmk__str_casei) && (interval_ms == entry->failed->interval_ms)); } /*! * \internal * \brief Clear a resource's last failure * * Erase a resource's last failure on a particular node from both the * LRM resource history in the CIB, and the resource history remembered * for the LRM state. * * \param[in] rsc_id Resource name * \param[in] node_name Node name * \param[in] operation If specified, only clear if matching this operation * \param[in] interval_ms If operation is specified, it has this interval */ void lrm_clear_last_failure(const char *rsc_id, const char *node_name, const char *operation, guint interval_ms) { lrm_state_t *lrm_state = lrm_state_find(node_name); if (lrm_state == NULL) { return; } if (lrm_state->resource_history != NULL) { rsc_history_t *entry = g_hash_table_lookup(lrm_state->resource_history, rsc_id); if (last_failed_matches_op(entry, operation, interval_ms)) { lrmd_free_event(entry->failed); entry->failed = NULL; } } } /* Returns: gboolean - cancellation is in progress */ static gboolean cancel_op(lrm_state_t * lrm_state, const char *rsc_id, const char *key, int op, gboolean remove) { int rc = pcmk_ok; char *local_key = NULL; active_op_t *pending = NULL; CRM_CHECK(op != 0, return FALSE); CRM_CHECK(rsc_id != NULL, return FALSE); if (key == NULL) { local_key = make_stop_id(rsc_id, op); key = local_key; } pending = g_hash_table_lookup(lrm_state->active_ops, key); if (pending) { if (remove && !pcmk_is_set(pending->flags, active_op_remove)) { controld_set_active_op_flags(pending, active_op_remove); crm_debug("Scheduling %s for removal", key); } if (pcmk_is_set(pending->flags, active_op_cancelled)) { crm_debug("Operation %s already cancelled", key); free(local_key); return FALSE; } controld_set_active_op_flags(pending, active_op_cancelled); } else { crm_info("No pending op found for %s", key); free(local_key); return FALSE; } crm_debug("Cancelling op %d for %s (%s)", op, rsc_id, key); rc = lrm_state_cancel(lrm_state, pending->rsc_id, pending->op_type, pending->interval_ms); if (rc == pcmk_ok) { crm_debug("Op %d for %s (%s): cancelled", op, rsc_id, key); free(local_key); return TRUE; } crm_debug("Op %d for %s (%s): Nothing to cancel", op, rsc_id, key); /* The caller needs to make sure the entry is * removed from the active operations list * * Usually by returning TRUE inside the worker function * supplied to g_hash_table_foreach_remove() * * Not removing the entry from active operations will block * the node from shutting down */ free(local_key); return FALSE; } struct cancel_data { gboolean done; gboolean remove; const char *key; lrmd_rsc_info_t *rsc; lrm_state_t *lrm_state; }; static gboolean cancel_action_by_key(gpointer key, gpointer value, gpointer user_data) { gboolean remove = FALSE; struct cancel_data *data = user_data; active_op_t *op = value; if (pcmk__str_eq(op->op_key, data->key, pcmk__str_none)) { data->done = TRUE; remove = !cancel_op(data->lrm_state, data->rsc->id, key, op->call_id, data->remove); } return remove; } static gboolean cancel_op_key(lrm_state_t * lrm_state, lrmd_rsc_info_t * rsc, const char *key, gboolean remove) { guint removed = 0; struct cancel_data data; CRM_CHECK(rsc != NULL, return FALSE); CRM_CHECK(key != NULL, return FALSE); data.key = key; data.rsc = rsc; data.done = FALSE; data.remove = remove; data.lrm_state = lrm_state; removed = g_hash_table_foreach_remove(lrm_state->active_ops, cancel_action_by_key, &data); crm_trace("Removed %u op cache entries, new size: %u", removed, g_hash_table_size(lrm_state->active_ops)); return data.done; } /*! * \internal * \brief Retrieve resource information from LRM * * \param[in,out] lrm_state Executor connection state to use * \param[in] rsc_xml XML containing resource configuration * \param[in] do_create If true, register resource if not already * \param[out] rsc_info Where to store information obtained from executor * * \retval pcmk_ok Success (and rsc_info holds newly allocated result) * \retval -EINVAL Required information is missing from arguments * \retval -ENOTCONN No active connection to LRM * \retval -ENODEV Resource not found * \retval -errno Error communicating with executor when registering resource * * \note Caller is responsible for freeing result on success. */ static int get_lrm_resource(lrm_state_t *lrm_state, const xmlNode *rsc_xml, gboolean do_create, lrmd_rsc_info_t **rsc_info) { const char *id = ID(rsc_xml); CRM_CHECK(lrm_state && rsc_xml && rsc_info, return -EINVAL); CRM_CHECK(id, return -EINVAL); if (lrm_state_is_connected(lrm_state) == FALSE) { return -ENOTCONN; } crm_trace("Retrieving resource information for %s from the executor", id); *rsc_info = lrm_state_get_rsc_info(lrm_state, id, 0); // If resource isn't known by ID, try clone name, if provided if (!*rsc_info) { const char *long_id = crm_element_value(rsc_xml, PCMK__XA_LONG_ID); if (long_id) { *rsc_info = lrm_state_get_rsc_info(lrm_state, long_id, 0); } } if ((*rsc_info == NULL) && do_create) { const char *class = crm_element_value(rsc_xml, PCMK_XA_CLASS); const char *provider = crm_element_value(rsc_xml, PCMK_XA_PROVIDER); const char *type = crm_element_value(rsc_xml, PCMK_XA_TYPE); int rc; crm_trace("Registering resource %s with the executor", id); rc = lrm_state_register_rsc(lrm_state, id, class, provider, type, lrmd_opt_drop_recurring); if (rc != pcmk_ok) { fsa_data_t *msg_data = NULL; crm_err("Could not register resource %s with the executor on %s: %s " CRM_XS " rc=%d", id, lrm_state->node_name, pcmk_strerror(rc), rc); /* Register this as an internal error if this involves the local * executor. Otherwise, we're likely dealing with an unresponsive * remote node, which is not an FSA failure. */ if (lrm_state_is_local(lrm_state) == TRUE) { register_fsa_error(C_FSA_INTERNAL, I_FAIL, NULL); } return rc; } *rsc_info = lrm_state_get_rsc_info(lrm_state, id, 0); } return *rsc_info? pcmk_ok : -ENODEV; } static void delete_resource(lrm_state_t *lrm_state, const char *id, lrmd_rsc_info_t *rsc, GHashTableIter *iter, const char *sys, const char *user, ha_msg_input_t *request, bool unregister, bool from_cib) { int rc = pcmk_ok; crm_info("Removing resource %s from executor for %s%s%s", id, sys, (user? " as " : ""), (user? user : "")); if (rsc && unregister) { rc = lrm_state_unregister_rsc(lrm_state, id, 0); } if (rc == pcmk_ok) { crm_trace("Resource %s deleted from executor", id); } else if (rc == -EINPROGRESS) { crm_info("Deletion of resource '%s' from executor is pending", id); if (request) { struct pending_deletion_op_s *op = NULL; char *ref = crm_element_value_copy(request->msg, PCMK_XA_REFERENCE); op = calloc(1, sizeof(struct pending_deletion_op_s)); op->rsc = strdup(rsc->id); op->input = copy_ha_msg_input(request); g_hash_table_insert(lrm_state->deletion_ops, ref, op); } return; } else { crm_warn("Could not delete '%s' from executor for %s%s%s: %s " CRM_XS " rc=%d", id, sys, (user? " as " : ""), (user? user : ""), pcmk_strerror(rc), rc); } delete_rsc_entry(lrm_state, request, id, iter, rc, user, from_cib); } static int get_fake_call_id(lrm_state_t *lrm_state, const char *rsc_id) { int call_id = 999999999; rsc_history_t *entry = NULL; if(lrm_state) { entry = g_hash_table_lookup(lrm_state->resource_history, rsc_id); } /* Make sure the call id is greater than the last successful operation, * otherwise the failure will not result in a possible recovery of the resource * as it could appear the failure occurred before the successful start */ if (entry) { call_id = entry->last_callid + 1; } if (call_id < 0) { call_id = 1; } return call_id; } static void fake_op_status(lrm_state_t *lrm_state, lrmd_event_data_t *op, int op_status, enum ocf_exitcode op_exitcode, const char *exit_reason) { op->call_id = get_fake_call_id(lrm_state, op->rsc_id); op->t_run = time(NULL); op->t_rcchange = op->t_run; lrmd__set_result(op, op_exitcode, op_status, exit_reason); } static void force_reprobe(lrm_state_t *lrm_state, const char *from_sys, const char *from_host, const char *user_name, gboolean is_remote_node, bool reprobe_all_nodes) { GHashTableIter gIter; rsc_history_t *entry = NULL; crm_info("Clearing resource history on node %s", lrm_state->node_name); g_hash_table_iter_init(&gIter, lrm_state->resource_history); while (g_hash_table_iter_next(&gIter, NULL, (void **)&entry)) { /* only unregister the resource during a reprobe if it is not a remote connection * resource. otherwise unregistering the connection will terminate remote-node * membership */ bool unregister = true; if (is_remote_lrmd_ra(NULL, NULL, entry->id)) { unregister = false; if (reprobe_all_nodes) { lrm_state_t *remote_lrm_state = lrm_state_find(entry->id); if (remote_lrm_state != NULL) { /* If reprobing all nodes, be sure to reprobe the remote * node before clearing its connection resource */ force_reprobe(remote_lrm_state, from_sys, from_host, user_name, TRUE, reprobe_all_nodes); } } } /* Don't delete from the CIB, since we'll delete the whole node's LRM * state from the CIB soon */ delete_resource(lrm_state, entry->id, &entry->rsc, &gIter, from_sys, user_name, NULL, unregister, false); } /* Now delete the copy in the CIB */ controld_delete_node_state(lrm_state->node_name, controld_section_lrm, cib_scope_local); // @COMPAT DCs < 1.1.14 need this deleted (in case it was explicitly false) update_attrd(lrm_state->node_name, CRM_OP_PROBED, NULL, user_name, is_remote_node); } /*! * \internal * \brief Fail a requested action without actually executing it * * For an action that can't be executed, process it similarly to an actual * execution result, with specified error status (except for notify actions, * which will always be treated as successful). * * \param[in,out] lrm_state Executor connection that action is for * \param[in] action Action XML from request * \param[in] rc Desired return code to use * \param[in] op_status Desired operation status to use * \param[in] exit_reason Human-friendly detail, if error */ static void synthesize_lrmd_failure(lrm_state_t *lrm_state, const xmlNode *action, int op_status, enum ocf_exitcode rc, const char *exit_reason) { lrmd_event_data_t *op = NULL; const char *operation = crm_element_value(action, PCMK_XA_OPERATION); const char *target_node = crm_element_value(action, PCMK__META_ON_NODE); xmlNode *xml_rsc = find_xml_node(action, XML_CIB_TAG_RESOURCE, TRUE); if ((xml_rsc == NULL) || (ID(xml_rsc) == NULL)) { /* @TODO Should we do something else, like direct ack? */ crm_info("Can't fake %s failure (%d) on %s without resource configuration", crm_element_value(action, PCMK__XA_OPERATION_KEY), rc, target_node); return; } else if(operation == NULL) { /* This probably came from crm_resource -C, nothing to do */ crm_info("Can't fake %s failure (%d) on %s without operation", ID(xml_rsc), rc, target_node); return; } op = construct_op(lrm_state, action, ID(xml_rsc), operation); if (pcmk__str_eq(operation, PCMK_ACTION_NOTIFY, pcmk__str_casei)) { // Notifications can't fail fake_op_status(lrm_state, op, PCMK_EXEC_DONE, PCMK_OCF_OK, NULL); } else { fake_op_status(lrm_state, op, op_status, rc, exit_reason); } crm_info("Faking " PCMK__OP_FMT " result (%d) on %s", op->rsc_id, op->op_type, op->interval_ms, op->rc, target_node); // Process the result as if it came from the LRM process_lrm_event(lrm_state, op, NULL, action); lrmd_free_event(op); } /*! * \internal * \brief Get target of an LRM operation (replacing \p NULL with local node * name) * * \param[in] xml LRM operation data XML * * \return LRM operation target node name (local node or Pacemaker Remote node) */ static const char * lrm_op_target(const xmlNode *xml) { const char *target = NULL; if (xml) { target = crm_element_value(xml, PCMK__META_ON_NODE); } if (target == NULL) { target = controld_globals.our_nodename; } return target; } static void fail_lrm_resource(xmlNode *xml, lrm_state_t *lrm_state, const char *user_name, const char *from_host, const char *from_sys) { lrmd_event_data_t *op = NULL; lrmd_rsc_info_t *rsc = NULL; xmlNode *xml_rsc = find_xml_node(xml, XML_CIB_TAG_RESOURCE, TRUE); CRM_CHECK(xml_rsc != NULL, return); /* The executor simply executes operations and reports the results, without * any concept of success or failure, so to fail a resource, we must fake * what a failure looks like. * * To do this, we create a fake executor operation event for the resource, * and pass that event to the executor client callback so it will be * processed as if it came from the executor. */ op = construct_op(lrm_state, xml, ID(xml_rsc), "asyncmon"); free((char*) op->user_data); op->user_data = NULL; op->interval_ms = 0; if (user_name && !pcmk__is_privileged(user_name)) { crm_err("%s does not have permission to fail %s", user_name, ID(xml_rsc)); fake_op_status(lrm_state, op, PCMK_EXEC_ERROR, PCMK_OCF_INSUFFICIENT_PRIV, "Unprivileged user cannot fail resources"); controld_ack_event_directly(from_host, from_sys, NULL, op, ID(xml_rsc)); lrmd_free_event(op); return; } if (get_lrm_resource(lrm_state, xml_rsc, TRUE, &rsc) == pcmk_ok) { crm_info("Failing resource %s...", rsc->id); fake_op_status(lrm_state, op, PCMK_EXEC_DONE, PCMK_OCF_UNKNOWN_ERROR, "Simulated failure"); process_lrm_event(lrm_state, op, NULL, xml); op->rc = PCMK_OCF_OK; // The request to fail the resource succeeded lrmd_free_rsc_info(rsc); } else { crm_info("Cannot find/create resource in order to fail it..."); crm_log_xml_warn(xml, "bad input"); fake_op_status(lrm_state, op, PCMK_EXEC_ERROR, PCMK_OCF_UNKNOWN_ERROR, "Cannot fail unknown resource"); } controld_ack_event_directly(from_host, from_sys, NULL, op, ID(xml_rsc)); lrmd_free_event(op); } static void handle_reprobe_op(lrm_state_t *lrm_state, const char *from_sys, const char *from_host, const char *user_name, gboolean is_remote_node, bool reprobe_all_nodes) { crm_notice("Forcing the status of all resources to be redetected"); force_reprobe(lrm_state, from_sys, from_host, user_name, is_remote_node, reprobe_all_nodes); if (!pcmk__strcase_any_of(from_sys, CRM_SYSTEM_PENGINE, CRM_SYSTEM_TENGINE, NULL)) { xmlNode *reply = create_request(CRM_OP_INVOKE_LRM, NULL, from_host, from_sys, CRM_SYSTEM_LRMD, controld_globals.our_uuid); crm_debug("ACK'ing re-probe from %s (%s)", from_sys, from_host); if (relay_message(reply, TRUE) == FALSE) { crm_log_xml_err(reply, "Unable to route reply"); } free_xml(reply); } } static bool do_lrm_cancel(ha_msg_input_t *input, lrm_state_t *lrm_state, lrmd_rsc_info_t *rsc, const char *from_host, const char *from_sys) { char *op_key = NULL; char *meta_key = NULL; int call = 0; const char *call_id = NULL; const char *op_task = NULL; guint interval_ms = 0; gboolean in_progress = FALSE; xmlNode *params = find_xml_node(input->xml, XML_TAG_ATTRS, TRUE); CRM_CHECK(params != NULL, return FALSE); meta_key = crm_meta_name(PCMK_XA_OPERATION); op_task = crm_element_value(params, meta_key); free(meta_key); CRM_CHECK(op_task != NULL, return FALSE); meta_key = crm_meta_name(PCMK_META_INTERVAL); if (crm_element_value_ms(params, meta_key, &interval_ms) != pcmk_ok) { free(meta_key); return FALSE; } free(meta_key); op_key = pcmk__op_key(rsc->id, op_task, interval_ms); meta_key = crm_meta_name(XML_LRM_ATTR_CALLID); call_id = crm_element_value(params, meta_key); free(meta_key); crm_debug("Scheduler requested op %s (call=%s) be cancelled", op_key, (call_id? call_id : "NA")); pcmk__scan_min_int(call_id, &call, 0); if (call == 0) { // Normal case when the scheduler cancels a recurring op in_progress = cancel_op_key(lrm_state, rsc, op_key, TRUE); } else { // Normal case when the scheduler cancels an orphan op in_progress = cancel_op(lrm_state, rsc->id, NULL, call, TRUE); } // Acknowledge cancellation operation if for a remote connection resource if (!in_progress || is_remote_lrmd_ra(NULL, NULL, rsc->id)) { char *op_id = make_stop_id(rsc->id, call); if (is_remote_lrmd_ra(NULL, NULL, rsc->id) == FALSE) { crm_info("Nothing known about operation %d for %s", call, op_key); } controld_delete_action_history_by_key(rsc->id, lrm_state->node_name, op_key, call); send_task_ok_ack(lrm_state, input, rsc->id, rsc, op_task, from_host, from_sys); /* needed at least for cancellation of a remote operation */ if (lrm_state->active_ops != NULL) { g_hash_table_remove(lrm_state->active_ops, op_id); } free(op_id); } else { /* No ack is needed since abcdaa8, but peers with older versions * in a rolling upgrade need one. We didn't bump the feature set * at that commit, so we can only compare against the previous * CRM version (3.0.8). If any peers have feature set 3.0.9 but * not abcdaa8, they will time out waiting for the ack (no * released versions of Pacemaker are affected). */ const char *peer_version = crm_element_value(params, PCMK_XA_CRM_FEATURE_SET); if (compare_version(peer_version, "3.0.8") <= 0) { crm_info("Sending compatibility ack for %s cancellation to %s (CRM version %s)", op_key, from_host, peer_version); send_task_ok_ack(lrm_state, input, rsc->id, rsc, op_task, from_host, from_sys); } } free(op_key); return TRUE; } static void do_lrm_delete(ha_msg_input_t *input, lrm_state_t *lrm_state, lrmd_rsc_info_t *rsc, const char *from_sys, const char *from_host, bool crm_rsc_delete, const char *user_name) { bool unregister = true; int cib_rc = controld_delete_resource_history(rsc->id, lrm_state->node_name, user_name, cib_dryrun|cib_sync_call); if (cib_rc != pcmk_rc_ok) { lrmd_event_data_t *op = NULL; op = construct_op(lrm_state, input->xml, rsc->id, PCMK_ACTION_DELETE); /* These are resource clean-ups, not actions, so no exit reason is * needed. */ lrmd__set_result(op, pcmk_rc2ocf(cib_rc), PCMK_EXEC_ERROR, NULL); controld_ack_event_directly(from_host, from_sys, NULL, op, rsc->id); lrmd_free_event(op); return; } if (crm_rsc_delete && is_remote_lrmd_ra(NULL, NULL, rsc->id)) { unregister = false; } delete_resource(lrm_state, rsc->id, rsc, NULL, from_sys, user_name, input, unregister, true); } // User data for asynchronous metadata execution struct metadata_cb_data { lrmd_rsc_info_t *rsc; // Copy of resource information xmlNode *input_xml; // Copy of FSA input XML }; static struct metadata_cb_data * new_metadata_cb_data(lrmd_rsc_info_t *rsc, xmlNode *input_xml) { struct metadata_cb_data *data = NULL; data = calloc(1, sizeof(struct metadata_cb_data)); CRM_ASSERT(data != NULL); data->input_xml = copy_xml(input_xml); data->rsc = lrmd_copy_rsc_info(rsc); return data; } static void free_metadata_cb_data(struct metadata_cb_data *data) { lrmd_free_rsc_info(data->rsc); free_xml(data->input_xml); free(data); } /*! * \internal * \brief Execute an action after metadata has been retrieved * * \param[in] pid Ignored * \param[in] result Result of metadata action * \param[in] user_data Metadata callback data */ static void metadata_complete(int pid, const pcmk__action_result_t *result, void *user_data) { struct metadata_cb_data *data = (struct metadata_cb_data *) user_data; struct ra_metadata_s *md = NULL; lrm_state_t *lrm_state = lrm_state_find(lrm_op_target(data->input_xml)); if ((lrm_state != NULL) && pcmk__result_ok(result)) { md = controld_cache_metadata(lrm_state->metadata_cache, data->rsc, result->action_stdout); } if (!pcmk_is_set(controld_globals.fsa_input_register, R_HA_DISCONNECTED)) { do_lrm_rsc_op(lrm_state, data->rsc, data->input_xml, md); } free_metadata_cb_data(data); } /* A_LRM_INVOKE */ void do_lrm_invoke(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data) { lrm_state_t *lrm_state = NULL; const char *crm_op = NULL; const char *from_sys = NULL; const char *from_host = NULL; const char *operation = NULL; ha_msg_input_t *input = fsa_typed_data(fsa_dt_ha_msg); const char *user_name = NULL; const char *target_node = lrm_op_target(input->xml); gboolean is_remote_node = FALSE; bool crm_rsc_delete = FALSE; // Message routed to the local node is targeting a specific, non-local node is_remote_node = !pcmk__str_eq(target_node, controld_globals.our_nodename, pcmk__str_casei); lrm_state = lrm_state_find(target_node); if ((lrm_state == NULL) && is_remote_node) { crm_err("Failing action because local node has never had connection to remote node %s", target_node); synthesize_lrmd_failure(NULL, input->xml, PCMK_EXEC_NOT_CONNECTED, PCMK_OCF_UNKNOWN_ERROR, "Local node has no connection to remote"); return; } CRM_ASSERT(lrm_state != NULL); user_name = pcmk__update_acl_user(input->msg, PCMK__XA_CRM_USER, NULL); crm_op = crm_element_value(input->msg, PCMK__XA_CRM_TASK); from_sys = crm_element_value(input->msg, PCMK__XA_CRM_SYS_FROM); if (!pcmk__str_eq(from_sys, CRM_SYSTEM_TENGINE, pcmk__str_none)) { from_host = crm_element_value(input->msg, PCMK__XA_SRC); } if (pcmk__str_eq(crm_op, PCMK_ACTION_LRM_DELETE, pcmk__str_none)) { if (!pcmk__str_eq(from_sys, CRM_SYSTEM_TENGINE, pcmk__str_none)) { crm_rsc_delete = TRUE; // from crm_resource } operation = PCMK_ACTION_DELETE; } else if (input->xml != NULL) { operation = crm_element_value(input->xml, PCMK_XA_OPERATION); } CRM_CHECK(!pcmk__str_empty(crm_op) || !pcmk__str_empty(operation), return); crm_trace("'%s' execution request from %s as %s user", pcmk__s(crm_op, operation), pcmk__s(from_sys, "unknown subsystem"), pcmk__s(user_name, "current")); if (pcmk__str_eq(crm_op, CRM_OP_LRM_FAIL, pcmk__str_none)) { fail_lrm_resource(input->xml, lrm_state, user_name, from_host, from_sys); } else if (pcmk__str_eq(crm_op, CRM_OP_LRM_REFRESH, pcmk__str_none)) { /* @COMPAT This can only be sent by crm_resource --refresh on a * Pacemaker Remote node running Pacemaker 1.1.9, which is extremely * unlikely. It previously would cause the controller to re-write its * resource history to the CIB. Just ignore it. */ crm_notice("Ignoring refresh request from Pacemaker Remote 1.1.9 node"); // @COMPAT DCs <1.1.14 in a rolling upgrade might schedule this op } else if (pcmk__str_eq(operation, CRM_OP_PROBED, pcmk__str_none)) { update_attrd(lrm_state->node_name, CRM_OP_PROBED, PCMK_VALUE_TRUE, user_name, is_remote_node); } else if (pcmk__str_eq(crm_op, CRM_OP_REPROBE, pcmk__str_none) || pcmk__str_eq(operation, CRM_OP_REPROBE, pcmk__str_none)) { const char *raw_target = NULL; if (input->xml != NULL) { // For CRM_OP_REPROBE, a NULL target means we're targeting all nodes raw_target = crm_element_value(input->xml, PCMK__META_ON_NODE); } handle_reprobe_op(lrm_state, from_sys, from_host, user_name, is_remote_node, (raw_target == NULL)); } else if (operation != NULL) { lrmd_rsc_info_t *rsc = NULL; xmlNode *xml_rsc = find_xml_node(input->xml, XML_CIB_TAG_RESOURCE, TRUE); gboolean create_rsc = !pcmk__str_eq(operation, PCMK_ACTION_DELETE, pcmk__str_none); int rc; // We can't return anything meaningful without a resource ID CRM_CHECK(xml_rsc && ID(xml_rsc), return); rc = get_lrm_resource(lrm_state, xml_rsc, create_rsc, &rsc); if (rc == -ENOTCONN) { synthesize_lrmd_failure(lrm_state, input->xml, PCMK_EXEC_NOT_CONNECTED, PCMK_OCF_UNKNOWN_ERROR, "Not connected to remote executor"); return; } else if ((rc < 0) && !create_rsc) { /* Delete of malformed or nonexistent resource * (deleting something that does not exist is a success) */ crm_notice("Not registering resource '%s' for a %s event " CRM_XS " get-rc=%d (%s) transition-key=%s", ID(xml_rsc), operation, rc, pcmk_strerror(rc), ID(input->xml)); delete_rsc_entry(lrm_state, input, ID(xml_rsc), NULL, pcmk_ok, user_name, true); return; } else if (rc == -EINVAL) { // Resource operation on malformed resource crm_err("Invalid resource definition for %s", ID(xml_rsc)); crm_log_xml_warn(input->msg, "invalid resource"); synthesize_lrmd_failure(lrm_state, input->xml, PCMK_EXEC_ERROR, PCMK_OCF_NOT_CONFIGURED, // fatal error "Invalid resource definition"); return; } else if (rc < 0) { // Error communicating with the executor crm_err("Could not register resource '%s' with executor: %s " CRM_XS " rc=%d", ID(xml_rsc), pcmk_strerror(rc), rc); crm_log_xml_warn(input->msg, "failed registration"); synthesize_lrmd_failure(lrm_state, input->xml, PCMK_EXEC_ERROR, PCMK_OCF_INVALID_PARAM, // hard error "Could not register resource with executor"); return; } if (pcmk__str_eq(operation, PCMK_ACTION_CANCEL, pcmk__str_none)) { if (!do_lrm_cancel(input, lrm_state, rsc, from_host, from_sys)) { crm_log_xml_warn(input->xml, "Bad command"); } } else if (pcmk__str_eq(operation, PCMK_ACTION_DELETE, pcmk__str_none)) { do_lrm_delete(input, lrm_state, rsc, from_sys, from_host, crm_rsc_delete, user_name); } else { struct ra_metadata_s *md = NULL; /* Getting metadata from cache is OK except for start actions -- * always refresh from the agent for those, in case the resource * agent was updated. * * @TODO Only refresh metadata for starts if the agent actually * changed (using something like inotify, or a hash or modification * time of the agent executable). */ if (strcmp(operation, PCMK_ACTION_START) != 0) { md = controld_get_rsc_metadata(lrm_state, rsc, controld_metadata_from_cache); } if ((md == NULL) && crm_op_needs_metadata(rsc->standard, operation)) { /* Most likely, we'll need the agent metadata to record the * pending operation and the operation result. Get it now rather * than wait until then, so the metadata action doesn't eat into * the real action's timeout. * * @TODO Metadata is retrieved via direct execution of the * agent, which has a couple of related issues: the executor * should execute agents, not the controller; and metadata for * Pacemaker Remote nodes should be collected on those nodes, * not locally. */ struct metadata_cb_data *data = NULL; data = new_metadata_cb_data(rsc, input->xml); crm_info("Retrieving metadata for %s (%s%s%s:%s) asynchronously", rsc->id, rsc->standard, ((rsc->provider == NULL)? "" : ":"), ((rsc->provider == NULL)? "" : rsc->provider), rsc->type); (void) lrmd__metadata_async(rsc, metadata_complete, (void *) data); } else { do_lrm_rsc_op(lrm_state, rsc, input->xml, md); } } lrmd_free_rsc_info(rsc); } else { crm_err("Invalid execution request: unknown command '%s' (bug?)", crm_op); register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL); } } static lrmd_event_data_t * construct_op(const lrm_state_t *lrm_state, const xmlNode *rsc_op, const char *rsc_id, const char *operation) { lrmd_event_data_t *op = NULL; const char *op_delay = NULL; const char *op_timeout = NULL; GHashTable *params = NULL; xmlNode *primitive = NULL; const char *class = NULL; const char *transition = NULL; CRM_ASSERT(rsc_id && operation); op = lrmd_new_event(rsc_id, operation, 0); op->type = lrmd_event_exec_complete; op->timeout = 0; op->start_delay = 0; lrmd__set_result(op, PCMK_OCF_UNKNOWN, PCMK_EXEC_PENDING, NULL); if (rsc_op == NULL) { CRM_LOG_ASSERT(pcmk__str_eq(operation, PCMK_ACTION_STOP, pcmk__str_casei)); op->user_data = NULL; /* the stop_all_resources() case * by definition there is no DC (or they'd be shutting * us down). * So we should put our version here. */ op->params = pcmk__strkey_table(free, free); g_hash_table_insert(op->params, strdup(PCMK_XA_CRM_FEATURE_SET), strdup(CRM_FEATURE_SET)); crm_trace("Constructed %s op for %s", operation, rsc_id); return op; } params = xml2list(rsc_op); - g_hash_table_remove(params, CRM_META "_op_target_rc"); + g_hash_table_remove(params, CRM_META "_" PCMK__META_OP_TARGET_RC); op_delay = crm_meta_value(params, PCMK_META_START_DELAY); pcmk__scan_min_int(op_delay, &op->start_delay, 0); op_timeout = crm_meta_value(params, PCMK_META_TIMEOUT); pcmk__scan_min_int(op_timeout, &op->timeout, 0); if (pcmk__guint_from_hash(params, CRM_META "_" PCMK_META_INTERVAL, 0, &(op->interval_ms)) != pcmk_rc_ok) { op->interval_ms = 0; } /* Use pcmk_monitor_timeout instead of meta timeout for stonith recurring monitor, if set */ primitive = find_xml_node(rsc_op, XML_CIB_TAG_RESOURCE, FALSE); class = crm_element_value(primitive, PCMK_XA_CLASS); if (pcmk_is_set(pcmk_get_ra_caps(class), pcmk_ra_cap_fence_params) && pcmk__str_eq(operation, PCMK_ACTION_MONITOR, pcmk__str_casei) && (op->interval_ms > 0)) { op_timeout = g_hash_table_lookup(params, "pcmk_monitor_timeout"); if (op_timeout != NULL) { op->timeout = crm_get_msec(op_timeout); } } if (!pcmk__str_eq(operation, PCMK_ACTION_STOP, pcmk__str_casei)) { op->params = params; } else { rsc_history_t *entry = NULL; if (lrm_state) { entry = g_hash_table_lookup(lrm_state->resource_history, rsc_id); } /* If we do not have stop parameters cached, use * whatever we are given */ if (!entry || !entry->stop_params) { op->params = params; } else { /* Copy the cached parameter list so that we stop the resource * with the old attributes, not the new ones */ op->params = pcmk__strkey_table(free, free); g_hash_table_foreach(params, copy_meta_keys, op->params); g_hash_table_foreach(entry->stop_params, copy_instance_keys, op->params); g_hash_table_destroy(params); params = NULL; } } /* sanity */ if (op->timeout <= 0) { op->timeout = op->interval_ms; } if (op->start_delay < 0) { op->start_delay = 0; } transition = crm_element_value(rsc_op, PCMK__XA_TRANSITION_KEY); CRM_CHECK(transition != NULL, return op); op->user_data = strdup(transition); if (op->interval_ms != 0) { if (pcmk__strcase_any_of(operation, PCMK_ACTION_START, PCMK_ACTION_STOP, NULL)) { crm_err("Start and Stop actions cannot have an interval: %u", op->interval_ms); op->interval_ms = 0; } } crm_trace("Constructed %s op for %s: interval=%u", operation, rsc_id, op->interval_ms); return op; } /*! * \internal * \brief Send a (synthesized) event result * * Reply with a synthesized event result directly, as opposed to going through * the executor. * * \param[in] to_host Host to send result to * \param[in] to_sys IPC name to send result (NULL for transition engine) * \param[in] rsc Type information about resource the result is for * \param[in,out] op Event with result to send * \param[in] rsc_id ID of resource the result is for */ void controld_ack_event_directly(const char *to_host, const char *to_sys, const lrmd_rsc_info_t *rsc, lrmd_event_data_t *op, const char *rsc_id) { xmlNode *reply = NULL; xmlNode *update, *iter; crm_node_t *peer = NULL; CRM_CHECK(op != NULL, return); if (op->rsc_id == NULL) { CRM_ASSERT(rsc_id != NULL); op->rsc_id = strdup(rsc_id); } if (to_sys == NULL) { to_sys = CRM_SYSTEM_TENGINE; } peer = pcmk__get_node(0, controld_globals.our_nodename, NULL, pcmk__node_search_cluster); update = create_node_state_update(peer, node_update_none, NULL, __func__); iter = create_xml_node(update, XML_CIB_TAG_LRM); crm_xml_add(iter, PCMK_XA_ID, controld_globals.our_uuid); iter = create_xml_node(iter, XML_LRM_TAG_RESOURCES); iter = create_xml_node(iter, XML_LRM_TAG_RESOURCE); crm_xml_add(iter, PCMK_XA_ID, op->rsc_id); controld_add_resource_history_xml(iter, rsc, op, controld_globals.our_nodename); reply = create_request(CRM_OP_INVOKE_LRM, update, to_host, to_sys, CRM_SYSTEM_LRMD, NULL); crm_log_xml_trace(update, "[direct ACK]"); crm_debug("ACK'ing resource op " PCMK__OP_FMT " from %s: %s", op->rsc_id, op->op_type, op->interval_ms, op->user_data, crm_element_value(reply, PCMK_XA_REFERENCE)); if (relay_message(reply, TRUE) == FALSE) { crm_log_xml_err(reply, "Unable to route reply"); } free_xml(update); free_xml(reply); } gboolean verify_stopped(enum crmd_fsa_state cur_state, int log_level) { gboolean res = TRUE; GList *lrm_state_list = lrm_state_get_list(); GList *state_entry; for (state_entry = lrm_state_list; state_entry != NULL; state_entry = state_entry->next) { lrm_state_t *lrm_state = state_entry->data; if (!lrm_state_verify_stopped(lrm_state, cur_state, log_level)) { /* keep iterating through all even when false is returned */ res = FALSE; } } controld_set_fsa_input_flags(R_SENT_RSC_STOP); g_list_free(lrm_state_list); lrm_state_list = NULL; return res; } struct stop_recurring_action_s { lrmd_rsc_info_t *rsc; lrm_state_t *lrm_state; }; static gboolean stop_recurring_action_by_rsc(gpointer key, gpointer value, gpointer user_data) { gboolean remove = FALSE; struct stop_recurring_action_s *event = user_data; active_op_t *op = value; if ((op->interval_ms != 0) && pcmk__str_eq(op->rsc_id, event->rsc->id, pcmk__str_none)) { crm_debug("Cancelling op %d for %s (%s)", op->call_id, op->rsc_id, (char*)key); remove = !cancel_op(event->lrm_state, event->rsc->id, key, op->call_id, FALSE); } return remove; } static gboolean stop_recurring_actions(gpointer key, gpointer value, gpointer user_data) { gboolean remove = FALSE; lrm_state_t *lrm_state = user_data; active_op_t *op = value; if (op->interval_ms != 0) { crm_info("Cancelling op %d for %s (%s)", op->call_id, op->rsc_id, (const char *) key); remove = !cancel_op(lrm_state, op->rsc_id, key, op->call_id, FALSE); } return remove; } /*! * \internal * \brief Check whether recurring actions should be cancelled before an action * * \param[in] rsc_id Resource that action is for * \param[in] action Action being performed * \param[in] interval_ms Operation interval of \p action (in milliseconds) * * \return true if recurring actions should be cancelled, otherwise false */ static bool should_cancel_recurring(const char *rsc_id, const char *action, guint interval_ms) { if (is_remote_lrmd_ra(NULL, NULL, rsc_id) && (interval_ms == 0) && (strcmp(action, PCMK_ACTION_MIGRATE_TO) == 0)) { /* Don't stop monitoring a migrating Pacemaker Remote connection * resource until the entire migration has completed. We must detect if * the connection is unexpectedly severed, even during a migration. */ return false; } // Cancel recurring actions before changing resource state return (interval_ms == 0) && !pcmk__str_any_of(action, PCMK_ACTION_MONITOR, PCMK_ACTION_NOTIFY, NULL); } /*! * \internal * \brief Check whether an action should not be performed at this time * * \param[in] operation Action to be performed * * \return Readable description of why action should not be performed, * or NULL if it should be performed */ static const char * should_nack_action(const char *action) { if (pcmk_is_set(controld_globals.fsa_input_register, R_SHUTDOWN) && pcmk__str_eq(action, PCMK_ACTION_START, pcmk__str_none)) { register_fsa_input(C_SHUTDOWN, I_SHUTDOWN, NULL); return "Not attempting start due to shutdown in progress"; } switch (controld_globals.fsa_state) { case S_NOT_DC: case S_POLICY_ENGINE: // Recalculating case S_TRANSITION_ENGINE: break; default: if (!pcmk__str_eq(action, PCMK_ACTION_STOP, pcmk__str_none)) { return "Controller cannot attempt actions at this time"; } break; } return NULL; } static void do_lrm_rsc_op(lrm_state_t *lrm_state, lrmd_rsc_info_t *rsc, xmlNode *msg, struct ra_metadata_s *md) { int rc; int call_id = 0; char *op_id = NULL; lrmd_event_data_t *op = NULL; fsa_data_t *msg_data = NULL; const char *transition = NULL; const char *operation = NULL; const char *nack_reason = NULL; CRM_CHECK((rsc != NULL) && (msg != NULL), return); operation = crm_element_value(msg, PCMK_XA_OPERATION); CRM_CHECK(!pcmk__str_empty(operation), return); transition = crm_element_value(msg, PCMK__XA_TRANSITION_KEY); if (pcmk__str_empty(transition)) { crm_log_xml_err(msg, "Missing transition number"); } if (lrm_state == NULL) { // This shouldn't be possible, but provide a failsafe just in case crm_err("Cannot execute %s of %s: No executor connection " CRM_XS " transition_key=%s", operation, rsc->id, pcmk__s(transition, "")); synthesize_lrmd_failure(NULL, msg, PCMK_EXEC_INVALID, PCMK_OCF_UNKNOWN_ERROR, "No executor connection"); return; } if (pcmk__str_any_of(operation, PCMK_ACTION_RELOAD, PCMK_ACTION_RELOAD_AGENT, NULL)) { /* Pre-2.1.0 DCs will schedule reload actions only, and 2.1.0+ DCs * will schedule reload-agent actions only. In either case, we need * to map that to whatever the resource agent actually supports. * Default to the OCF 1.1 name. */ if ((md != NULL) && pcmk_is_set(md->ra_flags, ra_supports_legacy_reload)) { operation = PCMK_ACTION_RELOAD; } else { operation = PCMK_ACTION_RELOAD_AGENT; } } op = construct_op(lrm_state, msg, rsc->id, operation); CRM_CHECK(op != NULL, return); if (should_cancel_recurring(rsc->id, operation, op->interval_ms)) { guint removed = 0; struct stop_recurring_action_s data; data.rsc = rsc; data.lrm_state = lrm_state; removed = g_hash_table_foreach_remove(lrm_state->active_ops, stop_recurring_action_by_rsc, &data); if (removed) { crm_debug("Stopped %u recurring operation%s in preparation for " PCMK__OP_FMT, removed, pcmk__plural_s(removed), rsc->id, operation, op->interval_ms); } } /* now do the op */ crm_notice("Requesting local execution of %s operation for %s on %s " CRM_XS " transition_key=%s op_key=" PCMK__OP_FMT, pcmk__readable_action(op->op_type, op->interval_ms), rsc->id, lrm_state->node_name, pcmk__s(transition, ""), rsc->id, operation, op->interval_ms); nack_reason = should_nack_action(operation); if (nack_reason != NULL) { crm_notice("Discarding attempt to perform action %s on %s in state %s " "(shutdown=%s)", operation, rsc->id, fsa_state2string(controld_globals.fsa_state), pcmk__btoa(pcmk_is_set(controld_globals.fsa_input_register, R_SHUTDOWN))); lrmd__set_result(op, PCMK_OCF_UNKNOWN_ERROR, PCMK_EXEC_INVALID, nack_reason); controld_ack_event_directly(NULL, NULL, rsc, op, rsc->id); lrmd_free_event(op); free(op_id); return; } controld_record_pending_op(lrm_state->node_name, rsc, op); op_id = pcmk__op_key(rsc->id, op->op_type, op->interval_ms); if (op->interval_ms > 0) { /* cancel it so we can then restart it without conflict */ cancel_op_key(lrm_state, rsc, op_id, FALSE); } rc = controld_execute_resource_agent(lrm_state, rsc->id, op->op_type, op->user_data, op->interval_ms, op->timeout, op->start_delay, op->params, &call_id); if (rc == pcmk_rc_ok) { /* record all operations so we can wait * for them to complete during shutdown */ char *call_id_s = make_stop_id(rsc->id, call_id); active_op_t *pending = NULL; pending = calloc(1, sizeof(active_op_t)); crm_trace("Recording pending op: %d - %s %s", call_id, op_id, call_id_s); pending->call_id = call_id; pending->interval_ms = op->interval_ms; pending->op_type = strdup(operation); pending->op_key = strdup(op_id); pending->rsc_id = strdup(rsc->id); pending->start_time = time(NULL); pcmk__str_update(&pending->user_data, op->user_data); if (crm_element_value_epoch(msg, PCMK_OPT_SHUTDOWN_LOCK, &(pending->lock_time)) != pcmk_ok) { pending->lock_time = 0; } g_hash_table_replace(lrm_state->active_ops, call_id_s, pending); if ((op->interval_ms > 0) && (op->start_delay > START_DELAY_THRESHOLD)) { int target_rc = PCMK_OCF_OK; crm_info("Faking confirmation of %s: execution postponed for over 5 minutes", op_id); decode_transition_key(op->user_data, NULL, NULL, NULL, &target_rc); lrmd__set_result(op, target_rc, PCMK_EXEC_DONE, NULL); controld_ack_event_directly(NULL, NULL, rsc, op, rsc->id); } pending->params = op->params; op->params = NULL; } else if (lrm_state_is_local(lrm_state)) { crm_err("Could not initiate %s action for resource %s locally: %s " CRM_XS " rc=%d", operation, rsc->id, pcmk_rc_str(rc), rc); fake_op_status(lrm_state, op, PCMK_EXEC_NOT_CONNECTED, PCMK_OCF_UNKNOWN_ERROR, pcmk_rc_str(rc)); process_lrm_event(lrm_state, op, NULL, NULL); register_fsa_error(C_FSA_INTERNAL, I_FAIL, NULL); } else { crm_err("Could not initiate %s action for resource %s remotely on %s: " "%s " CRM_XS " rc=%d", operation, rsc->id, lrm_state->node_name, pcmk_rc_str(rc), rc); fake_op_status(lrm_state, op, PCMK_EXEC_NOT_CONNECTED, PCMK_OCF_UNKNOWN_ERROR, pcmk_rc_str(rc)); process_lrm_event(lrm_state, op, NULL, NULL); } free(op_id); lrmd_free_event(op); } void do_lrm_event(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input cur_input, fsa_data_t * msg_data) { CRM_CHECK(FALSE, return); } static char * unescape_newlines(const char *string) { char *pch = NULL; char *ret = NULL; static const char *escaped_newline = "\\n"; if (!string) { return NULL; } ret = strdup(string); pch = strstr(ret, escaped_newline); while (pch != NULL) { /* Replace newline escape pattern with actual newline (and a space so we * don't have to shuffle the rest of the buffer) */ pch[0] = '\n'; pch[1] = ' '; pch = strstr(pch, escaped_newline); } return ret; } static bool did_lrm_rsc_op_fail(lrm_state_t *lrm_state, const char * rsc_id, const char * op_type, guint interval_ms) { rsc_history_t *entry = NULL; CRM_CHECK(lrm_state != NULL, return FALSE); CRM_CHECK(rsc_id != NULL, return FALSE); CRM_CHECK(op_type != NULL, return FALSE); entry = g_hash_table_lookup(lrm_state->resource_history, rsc_id); if (entry == NULL || entry->failed == NULL) { return FALSE; } if (pcmk__str_eq(entry->failed->rsc_id, rsc_id, pcmk__str_none) && pcmk__str_eq(entry->failed->op_type, op_type, pcmk__str_casei) && entry->failed->interval_ms == interval_ms) { return TRUE; } return FALSE; } /*! * \internal * \brief Log the result of an executor action (actual or synthesized) * * \param[in] op Executor action to log result for * \param[in] op_key Operation key for action * \param[in] node_name Name of node action was performed on, if known * \param[in] confirmed Whether to log that graph action was confirmed */ static void log_executor_event(const lrmd_event_data_t *op, const char *op_key, const char *node_name, gboolean confirmed) { int log_level = LOG_ERR; GString *str = g_string_sized_new(100); // reasonable starting size pcmk__g_strcat(str, "Result of ", pcmk__readable_action(op->op_type, op->interval_ms), " operation for ", op->rsc_id, NULL); if (node_name != NULL) { pcmk__g_strcat(str, " on ", node_name, NULL); } switch (op->op_status) { case PCMK_EXEC_DONE: log_level = LOG_NOTICE; pcmk__g_strcat(str, ": ", services_ocf_exitcode_str(op->rc), NULL); break; case PCMK_EXEC_TIMEOUT: pcmk__g_strcat(str, ": ", pcmk_exec_status_str(op->op_status), " after ", pcmk__readable_interval(op->timeout), NULL); break; case PCMK_EXEC_CANCELLED: log_level = LOG_INFO; /* order of __attribute__ and Fall through comment is IMPORTANT! * do not change it without proper testing with both clang and gcc * in multiple versions. * the clang check allows to build with all versions of clang. * the has_c_attribute check is to workaround a bug in clang version * in rhel7. has_attribute would happily return "YES SIR WE GOT IT" * and fail the build the next line. */ #ifdef __clang__ #ifdef __has_c_attribute #if __has_attribute(fallthrough) __attribute__((fallthrough)); #endif #endif #endif // Fall through default: pcmk__g_strcat(str, ": ", pcmk_exec_status_str(op->op_status), NULL); } if ((op->exit_reason != NULL) && ((op->op_status != PCMK_EXEC_DONE) || (op->rc != PCMK_OCF_OK))) { pcmk__g_strcat(str, " (", op->exit_reason, ")", NULL); } g_string_append(str, " " CRM_XS); g_string_append_printf(str, " graph action %sconfirmed; call=%d key=%s", (confirmed? "" : "un"), op->call_id, op_key); if (op->op_status == PCMK_EXEC_DONE) { g_string_append_printf(str, " rc=%d", op->rc); } do_crm_log(log_level, "%s", str->str); g_string_free(str, TRUE); /* The services library has already logged the output at info or debug * level, so just raise to notice if it looks like a failure. */ if ((op->output != NULL) && (op->rc != PCMK_OCF_OK)) { char *prefix = crm_strdup_printf(PCMK__OP_FMT "@%s output", op->rsc_id, op->op_type, op->interval_ms, node_name); crm_log_output(LOG_NOTICE, prefix, op->output); free(prefix); } } void process_lrm_event(lrm_state_t *lrm_state, lrmd_event_data_t *op, active_op_t *pending, const xmlNode *action_xml) { char *op_id = NULL; char *op_key = NULL; gboolean remove = FALSE; gboolean removed = FALSE; bool need_direct_ack = FALSE; lrmd_rsc_info_t *rsc = NULL; const char *node_name = NULL; CRM_CHECK(op != NULL, return); CRM_CHECK(op->rsc_id != NULL, return); // Remap new status codes for older DCs if (compare_version(controld_globals.dc_version, "3.2.0") < 0) { switch (op->op_status) { case PCMK_EXEC_NOT_CONNECTED: lrmd__set_result(op, PCMK_OCF_CONNECTION_DIED, PCMK_EXEC_ERROR, op->exit_reason); break; case PCMK_EXEC_INVALID: lrmd__set_result(op, CRM_DIRECT_NACK_RC, PCMK_EXEC_ERROR, op->exit_reason); break; default: break; } } op_id = make_stop_id(op->rsc_id, op->call_id); op_key = pcmk__op_key(op->rsc_id, op->op_type, op->interval_ms); // Get resource info if available (from executor state or action XML) if (lrm_state) { rsc = lrm_state_get_rsc_info(lrm_state, op->rsc_id, 0); } if ((rsc == NULL) && action_xml) { xmlNode *xml = find_xml_node(action_xml, XML_CIB_TAG_RESOURCE, TRUE); const char *standard = crm_element_value(xml, PCMK_XA_CLASS); const char *provider = crm_element_value(xml, PCMK_XA_PROVIDER); const char *type = crm_element_value(xml, PCMK_XA_TYPE); if (standard && type) { crm_info("%s agent information not cached, using %s%s%s:%s from action XML", op->rsc_id, standard, (provider? ":" : ""), (provider? provider : ""), type); rsc = lrmd_new_rsc_info(op->rsc_id, standard, provider, type); } else { crm_err("Can't process %s result because %s agent information not cached or in XML", op_key, op->rsc_id); } } // Get node name if available (from executor state or action XML) if (lrm_state) { node_name = lrm_state->node_name; } else if (action_xml) { node_name = crm_element_value(action_xml, PCMK__META_ON_NODE); } if(pending == NULL) { remove = TRUE; if (lrm_state) { pending = g_hash_table_lookup(lrm_state->active_ops, op_id); } } if (op->op_status == PCMK_EXEC_ERROR) { switch(op->rc) { case PCMK_OCF_NOT_RUNNING: case PCMK_OCF_RUNNING_PROMOTED: case PCMK_OCF_DEGRADED: case PCMK_OCF_DEGRADED_PROMOTED: // Leave it to the TE/scheduler to decide if this is an error op->op_status = PCMK_EXEC_DONE; break; default: /* Nothing to do */ break; } } if (op->op_status != PCMK_EXEC_CANCELLED) { /* We might not record the result, so directly acknowledge it to the * originator instead, so it doesn't time out waiting for the result * (especially important if part of a transition). */ need_direct_ack = TRUE; if (controld_action_is_recordable(op->op_type)) { if (node_name && rsc) { // We should record the result, and happily, we can time_t lock_time = (pending == NULL)? 0 : pending->lock_time; controld_update_resource_history(node_name, rsc, op, lock_time); need_direct_ack = FALSE; } else if (op->rsc_deleted) { /* We shouldn't record the result (likely the resource was * refreshed, cleaned, or removed while this operation was * in flight). */ crm_notice("Not recording %s result in CIB because " "resource information was removed since it was initiated", op_key); } else { /* This shouldn't be possible; the executor didn't consider the * resource deleted, but we couldn't find resource or node * information. */ crm_err("Unable to record %s result in CIB: %s", op_key, (node_name? "No resource information" : "No node name")); } } } else if (op->interval_ms == 0) { /* A non-recurring operation was cancelled. Most likely, the * never-initiated action was removed from the executor's pending * operations list upon resource removal. */ need_direct_ack = TRUE; } else if (pending == NULL) { /* This recurring operation was cancelled, but was not pending. No * transition actions are waiting on it, nothing needs to be done. */ } else if (op->user_data == NULL) { /* This recurring operation was cancelled and pending, but we don't * have a transition key. This should never happen. */ crm_err("Recurring operation %s was cancelled without transition information", op_key); } else if (pcmk_is_set(pending->flags, active_op_remove)) { /* This recurring operation was cancelled (by us) and pending, and we * have been waiting for it to finish. */ if (lrm_state) { controld_delete_action_history(op); } /* Directly acknowledge failed recurring actions here. The above call to * controld_delete_action_history() will not erase any corresponding * last_failure entry, which means that the DC won't confirm the * cancellation via process_op_deletion(), and the transition would * otherwise wait for the action timer to pop. */ if (did_lrm_rsc_op_fail(lrm_state, pending->rsc_id, pending->op_type, pending->interval_ms)) { need_direct_ack = TRUE; } } else if (op->rsc_deleted) { /* This recurring operation was cancelled (but not by us, and the * executor does not have resource information, likely due to resource * cleanup, refresh, or removal) and pending. */ crm_debug("Recurring op %s was cancelled due to resource deletion", op_key); need_direct_ack = TRUE; } else { /* This recurring operation was cancelled (but not by us, likely by the * executor before stopping the resource) and pending. We don't need to * do anything special. */ } if (need_direct_ack) { controld_ack_event_directly(NULL, NULL, NULL, op, op->rsc_id); } if(remove == FALSE) { /* The caller will do this afterwards, but keep the logging consistent */ removed = TRUE; } else if (lrm_state && ((op->interval_ms == 0) || (op->op_status == PCMK_EXEC_CANCELLED))) { gboolean found = g_hash_table_remove(lrm_state->active_ops, op_id); if (op->interval_ms != 0) { removed = TRUE; } else if (found) { removed = TRUE; crm_trace("Op %s (call=%d, stop-id=%s, remaining=%u): Confirmed", op_key, op->call_id, op_id, g_hash_table_size(lrm_state->active_ops)); } } log_executor_event(op, op_key, node_name, removed); if (lrm_state) { if (!pcmk__str_eq(op->op_type, PCMK_ACTION_META_DATA, pcmk__str_casei)) { crmd_alert_resource_op(lrm_state->node_name, op); } else if (rsc && (op->rc == PCMK_OCF_OK)) { char *metadata = unescape_newlines(op->output); controld_cache_metadata(lrm_state->metadata_cache, rsc, metadata); free(metadata); } } if (op->rsc_deleted) { crm_info("Deletion of resource '%s' complete after %s", op->rsc_id, op_key); if (lrm_state) { delete_rsc_entry(lrm_state, NULL, op->rsc_id, NULL, pcmk_ok, NULL, true); } } /* If a shutdown was escalated while operations were pending, * then the FSA will be stalled right now... allow it to continue */ controld_trigger_fsa(); if (lrm_state && rsc) { update_history_cache(lrm_state, rsc, op); } lrmd_free_rsc_info(rsc); free(op_key); free(op_id); } diff --git a/daemons/controld/controld_te_actions.c b/daemons/controld/controld_te_actions.c index 556d4a3496..bb89b04193 100644 --- a/daemons/controld/controld_te_actions.c +++ b/daemons/controld/controld_te_actions.c @@ -1,757 +1,757 @@ /* * Copyright 2004-2024 the Pacemaker project contributors * * The version control history for this file may have further details. * * This source code is licensed under the GNU General Public License version 2 * or later (GPLv2+) WITHOUT ANY WARRANTY. */ #include #include #include #include #include // lrmd_event_data_t, lrmd_free_event() #include #include #include #include #include static GHashTable *te_targets = NULL; void send_rsc_command(pcmk__graph_action_t *action); static void te_update_job_count(pcmk__graph_action_t *action, int offset); static void te_start_action_timer(const pcmk__graph_t *graph, pcmk__graph_action_t *action) { action->timer = g_timeout_add(action->timeout + graph->network_delay, action_timer_callback, (void *) action); CRM_ASSERT(action->timer != 0); } /*! * \internal * \brief Execute a graph pseudo-action * * \param[in,out] graph Transition graph being executed * \param[in,out] pseudo Pseudo-action to execute * * \return Standard Pacemaker return code */ static int execute_pseudo_action(pcmk__graph_t *graph, pcmk__graph_action_t *pseudo) { const char *task = crm_element_value(pseudo->xml, PCMK_XA_OPERATION); /* send to peers as well? */ if (pcmk__str_eq(task, PCMK_ACTION_MAINTENANCE_NODES, pcmk__str_casei)) { GHashTableIter iter; crm_node_t *node = NULL; g_hash_table_iter_init(&iter, crm_peer_cache); while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &node)) { xmlNode *cmd = NULL; if (pcmk__str_eq(controld_globals.our_nodename, node->uname, pcmk__str_casei)) { continue; } cmd = create_request(task, pseudo->xml, node->uname, CRM_SYSTEM_CRMD, CRM_SYSTEM_TENGINE, NULL); send_cluster_message(node, crm_msg_crmd, cmd, FALSE); free_xml(cmd); } remote_ra_process_maintenance_nodes(pseudo->xml); } else { /* Check action for Pacemaker Remote node side effects */ remote_ra_process_pseudo(pseudo->xml); } crm_debug("Pseudo-action %d (%s) fired and confirmed", pseudo->id, crm_element_value(pseudo->xml, PCMK__XA_OPERATION_KEY)); te_action_confirmed(pseudo, graph); return pcmk_rc_ok; } static int get_target_rc(pcmk__graph_action_t *action) { int exit_status; - pcmk__scan_min_int(crm_meta_value(action->params, XML_ATTR_TE_TARGET_RC), + pcmk__scan_min_int(crm_meta_value(action->params, PCMK__META_OP_TARGET_RC), &exit_status, 0); return exit_status; } /*! * \internal * \brief Execute a cluster action from a transition graph * * \param[in,out] graph Transition graph being executed * \param[in,out] action Cluster action to execute * * \return Standard Pacemaker return code */ static int execute_cluster_action(pcmk__graph_t *graph, pcmk__graph_action_t *action) { char *counter = NULL; xmlNode *cmd = NULL; gboolean is_local = FALSE; const char *id = NULL; const char *task = NULL; const char *value = NULL; const char *on_node = NULL; const char *router_node = NULL; gboolean rc = TRUE; gboolean no_wait = FALSE; id = ID(action->xml); CRM_CHECK(!pcmk__str_empty(id), return EPROTO); task = crm_element_value(action->xml, PCMK_XA_OPERATION); CRM_CHECK(!pcmk__str_empty(task), return EPROTO); on_node = crm_element_value(action->xml, PCMK__META_ON_NODE); CRM_CHECK(!pcmk__str_empty(on_node), return pcmk_rc_node_unknown); router_node = crm_element_value(action->xml, XML_LRM_ATTR_ROUTER_NODE); if (router_node == NULL) { router_node = on_node; if (pcmk__str_eq(task, PCMK_ACTION_LRM_DELETE, pcmk__str_none)) { const char *mode = crm_element_value(action->xml, PCMK__XA_MODE); if (pcmk__str_eq(mode, XML_TAG_CIB, pcmk__str_none)) { router_node = controld_globals.our_nodename; } } } if (pcmk__str_eq(router_node, controld_globals.our_nodename, pcmk__str_casei)) { is_local = TRUE; } value = crm_meta_value(action->params, PCMK__META_OP_NO_WAIT); if (crm_is_true(value)) { no_wait = TRUE; } crm_info("Handling controller request '%s' (%s on %s)%s%s", id, task, on_node, (is_local? " locally" : ""), (no_wait? " without waiting" : "")); if (is_local && pcmk__str_eq(task, PCMK_ACTION_DO_SHUTDOWN, pcmk__str_none)) { /* defer until everything else completes */ crm_info("Controller request '%s' is a local shutdown", id); graph->completion_action = pcmk__graph_shutdown; graph->abort_reason = "local shutdown"; te_action_confirmed(action, graph); return pcmk_rc_ok; } else if (pcmk__str_eq(task, PCMK_ACTION_DO_SHUTDOWN, pcmk__str_none)) { crm_node_t *peer = pcmk__get_node(0, router_node, NULL, pcmk__node_search_cluster); pcmk__update_peer_expected(__func__, peer, CRMD_JOINSTATE_DOWN); } cmd = create_request(task, action->xml, router_node, CRM_SYSTEM_CRMD, CRM_SYSTEM_TENGINE, NULL); counter = pcmk__transition_key(controld_globals.transition_graph->id, action->id, get_target_rc(action), controld_globals.te_uuid); crm_xml_add(cmd, PCMK__XA_TRANSITION_KEY, counter); rc = send_cluster_message(pcmk__get_node(0, router_node, NULL, pcmk__node_search_cluster), crm_msg_crmd, cmd, TRUE); free(counter); free_xml(cmd); if (rc == FALSE) { crm_err("Action %d failed: send", action->id); return ECOMM; } else if (no_wait) { te_action_confirmed(action, graph); } else { if (action->timeout <= 0) { crm_err("Action %d: %s on %s had an invalid timeout (%dms). Using %ums instead", action->id, task, on_node, action->timeout, graph->network_delay); action->timeout = (int) graph->network_delay; } te_start_action_timer(graph, action); } return pcmk_rc_ok; } /*! * \internal * \brief Synthesize an executor event for a resource action timeout * * \param[in] action Resource action that timed out * \param[in] target_rc Expected result of action that timed out * * Synthesize an executor event for a resource action timeout. (If the executor * gets a timeout while waiting for a resource action to complete, that will be * reported via the usual callback. This timeout means we didn't hear from the * executor itself or the controller that relayed the action to the executor.) * * \return Newly created executor event for result of \p action * \note The caller is responsible for freeing the return value using * lrmd_free_event(). */ static lrmd_event_data_t * synthesize_timeout_event(const pcmk__graph_action_t *action, int target_rc) { lrmd_event_data_t *op = NULL; const char *target = crm_element_value(action->xml, PCMK__META_ON_NODE); const char *reason = NULL; char *dynamic_reason = NULL; if (pcmk__str_eq(target, get_local_node_name(), pcmk__str_casei)) { reason = "Local executor did not return result in time"; } else { const char *router_node = NULL; router_node = crm_element_value(action->xml, XML_LRM_ATTR_ROUTER_NODE); if (router_node == NULL) { router_node = target; } dynamic_reason = crm_strdup_printf("Controller on %s did not return " "result in time", router_node); reason = dynamic_reason; } op = pcmk__event_from_graph_action(NULL, action, PCMK_EXEC_TIMEOUT, PCMK_OCF_UNKNOWN_ERROR, reason); op->call_id = -1; op->user_data = pcmk__transition_key(controld_globals.transition_graph->id, action->id, target_rc, controld_globals.te_uuid); free(dynamic_reason); return op; } static void controld_record_action_event(pcmk__graph_action_t *action, lrmd_event_data_t *op) { cib_t *cib_conn = controld_globals.cib_conn; xmlNode *state = NULL; xmlNode *rsc = NULL; xmlNode *action_rsc = NULL; int rc = pcmk_ok; const char *rsc_id = NULL; const char *target = crm_element_value(action->xml, PCMK__META_ON_NODE); const char *task_uuid = crm_element_value(action->xml, PCMK__XA_OPERATION_KEY); const char *target_uuid = crm_element_value(action->xml, PCMK__META_ON_NODE_UUID); int target_rc = get_target_rc(action); action_rsc = find_xml_node(action->xml, XML_CIB_TAG_RESOURCE, TRUE); if (action_rsc == NULL) { return; } rsc_id = ID(action_rsc); CRM_CHECK(rsc_id != NULL, crm_log_xml_err(action->xml, "Bad:action"); return); /* update the CIB */ state = create_xml_node(NULL, XML_CIB_TAG_STATE); crm_xml_add(state, PCMK_XA_ID, target_uuid); crm_xml_add(state, PCMK_XA_UNAME, target); rsc = create_xml_node(state, XML_CIB_TAG_LRM); crm_xml_add(rsc, PCMK_XA_ID, target_uuid); rsc = create_xml_node(rsc, XML_LRM_TAG_RESOURCES); rsc = create_xml_node(rsc, XML_LRM_TAG_RESOURCE); crm_xml_add(rsc, PCMK_XA_ID, rsc_id); crm_copy_xml_element(action_rsc, rsc, PCMK_XA_TYPE); crm_copy_xml_element(action_rsc, rsc, PCMK_XA_CLASS); crm_copy_xml_element(action_rsc, rsc, PCMK_XA_PROVIDER); pcmk__create_history_xml(rsc, op, CRM_FEATURE_SET, target_rc, target, __func__); rc = cib_conn->cmds->modify(cib_conn, XML_CIB_TAG_STATUS, state, cib_scope_local); fsa_register_cib_callback(rc, NULL, cib_action_updated); free_xml(state); crm_trace("Sent CIB update (call ID %d) for synthesized event of action %d (%s on %s)", rc, action->id, task_uuid, target); pcmk__set_graph_action_flags(action, pcmk__graph_action_sent_update); } void controld_record_action_timeout(pcmk__graph_action_t *action) { lrmd_event_data_t *op = NULL; const char *target = crm_element_value(action->xml, PCMK__META_ON_NODE); const char *task_uuid = crm_element_value(action->xml, PCMK__XA_OPERATION_KEY); int target_rc = get_target_rc(action); crm_warn("%s %d: %s on %s timed out", action->xml->name, action->id, task_uuid, target); op = synthesize_timeout_event(action, target_rc); controld_record_action_event(action, op); lrmd_free_event(op); } /*! * \internal * \brief Execute a resource action from a transition graph * * \param[in,out] graph Transition graph being executed * \param[in,out] action Resource action to execute * * \return Standard Pacemaker return code */ static int execute_rsc_action(pcmk__graph_t *graph, pcmk__graph_action_t *action) { /* never overwrite stop actions in the CIB with * anything other than completed results * * Writing pending stops makes it look like the * resource is running again */ xmlNode *cmd = NULL; xmlNode *rsc_op = NULL; gboolean rc = TRUE; gboolean no_wait = FALSE; gboolean is_local = FALSE; char *counter = NULL; const char *task = NULL; const char *value = NULL; const char *on_node = NULL; const char *router_node = NULL; const char *task_uuid = NULL; CRM_ASSERT(action != NULL); CRM_ASSERT(action->xml != NULL); pcmk__clear_graph_action_flags(action, pcmk__graph_action_executed); on_node = crm_element_value(action->xml, PCMK__META_ON_NODE); CRM_CHECK(!pcmk__str_empty(on_node), crm_err("Corrupted command(id=%s) %s: no node", ID(action->xml), pcmk__s(task, "without task")); return pcmk_rc_node_unknown); rsc_op = action->xml; task = crm_element_value(rsc_op, PCMK_XA_OPERATION); task_uuid = crm_element_value(action->xml, PCMK__XA_OPERATION_KEY); router_node = crm_element_value(rsc_op, XML_LRM_ATTR_ROUTER_NODE); if (!router_node) { router_node = on_node; } counter = pcmk__transition_key(controld_globals.transition_graph->id, action->id, get_target_rc(action), controld_globals.te_uuid); crm_xml_add(rsc_op, PCMK__XA_TRANSITION_KEY, counter); if (pcmk__str_eq(router_node, controld_globals.our_nodename, pcmk__str_casei)) { is_local = TRUE; } value = crm_meta_value(action->params, PCMK__META_OP_NO_WAIT); if (crm_is_true(value)) { no_wait = TRUE; } crm_notice("Initiating %s operation %s%s on %s%s "CRM_XS" action %d", task, task_uuid, (is_local? " locally" : ""), on_node, (no_wait? " without waiting" : ""), action->id); cmd = create_request(CRM_OP_INVOKE_LRM, rsc_op, router_node, CRM_SYSTEM_LRMD, CRM_SYSTEM_TENGINE, NULL); if (is_local) { /* shortcut local resource commands */ ha_msg_input_t data = { .msg = cmd, .xml = rsc_op, }; fsa_data_t msg = { .id = 0, .data = &data, .data_type = fsa_dt_ha_msg, .fsa_input = I_NULL, .fsa_cause = C_FSA_INTERNAL, .actions = A_LRM_INVOKE, .origin = __func__, }; do_lrm_invoke(A_LRM_INVOKE, C_FSA_INTERNAL, controld_globals.fsa_state, I_NULL, &msg); } else { rc = send_cluster_message(pcmk__get_node(0, router_node, NULL, pcmk__node_search_cluster), crm_msg_lrmd, cmd, TRUE); } free(counter); free_xml(cmd); pcmk__set_graph_action_flags(action, pcmk__graph_action_executed); if (rc == FALSE) { crm_err("Action %d failed: send", action->id); return ECOMM; } else if (no_wait) { /* Just mark confirmed. Don't bump the job count only to immediately * decrement it. */ crm_info("Action %d confirmed - no wait", action->id); pcmk__set_graph_action_flags(action, pcmk__graph_action_confirmed); pcmk__update_graph(controld_globals.transition_graph, action); trigger_graph(); } else if (pcmk_is_set(action->flags, pcmk__graph_action_confirmed)) { crm_debug("Action %d: %s %s on %s(timeout %dms) was already confirmed.", action->id, task, task_uuid, on_node, action->timeout); } else { if (action->timeout <= 0) { crm_err("Action %d: %s %s on %s had an invalid timeout (%dms). Using %ums instead", action->id, task, task_uuid, on_node, action->timeout, graph->network_delay); action->timeout = (int) graph->network_delay; } te_update_job_count(action, 1); te_start_action_timer(graph, action); } return pcmk_rc_ok; } struct te_peer_s { char *name; int jobs; int migrate_jobs; }; static void te_peer_free(gpointer p) { struct te_peer_s *peer = p; free(peer->name); free(peer); } void te_reset_job_counts(void) { GHashTableIter iter; struct te_peer_s *peer = NULL; if(te_targets == NULL) { te_targets = pcmk__strkey_table(NULL, te_peer_free); } g_hash_table_iter_init(&iter, te_targets); while (g_hash_table_iter_next(&iter, NULL, (gpointer *) & peer)) { peer->jobs = 0; peer->migrate_jobs = 0; } } static void te_update_job_count_on(const char *target, int offset, bool migrate) { struct te_peer_s *r = NULL; if(target == NULL || te_targets == NULL) { return; } r = g_hash_table_lookup(te_targets, target); if(r == NULL) { r = calloc(1, sizeof(struct te_peer_s)); r->name = strdup(target); g_hash_table_insert(te_targets, r->name, r); } r->jobs += offset; if(migrate) { r->migrate_jobs += offset; } crm_trace("jobs[%s] = %d", target, r->jobs); } static void te_update_job_count(pcmk__graph_action_t *action, int offset) { const char *task = crm_element_value(action->xml, PCMK_XA_OPERATION); const char *target = crm_element_value(action->xml, PCMK__META_ON_NODE); if ((action->type != pcmk__rsc_graph_action) || (target == NULL)) { /* No limit on these */ return; } /* if we have a router node, this means the action is performing * on a remote node. For now, we count all actions occurring on a * remote node against the job list on the cluster node hosting * the connection resources */ target = crm_element_value(action->xml, XML_LRM_ATTR_ROUTER_NODE); if ((target == NULL) && pcmk__strcase_any_of(task, PCMK_ACTION_MIGRATE_TO, PCMK_ACTION_MIGRATE_FROM, NULL)) { const char *t1 = crm_meta_value(action->params, XML_LRM_ATTR_MIGRATE_SOURCE); const char *t2 = crm_meta_value(action->params, XML_LRM_ATTR_MIGRATE_TARGET); te_update_job_count_on(t1, offset, TRUE); te_update_job_count_on(t2, offset, TRUE); return; } else if (target == NULL) { target = crm_element_value(action->xml, PCMK__META_ON_NODE); } te_update_job_count_on(target, offset, FALSE); } /*! * \internal * \brief Check whether a graph action is allowed to be executed on a node * * \param[in] graph Transition graph being executed * \param[in] action Graph action being executed * \param[in] target Name of node where action should be executed * * \return true if action is allowed, otherwise false */ static bool allowed_on_node(const pcmk__graph_t *graph, const pcmk__graph_action_t *action, const char *target) { int limit = 0; struct te_peer_s *r = NULL; const char *task = crm_element_value(action->xml, PCMK_XA_OPERATION); const char *id = crm_element_value(action->xml, PCMK__XA_OPERATION_KEY); if(target == NULL) { /* No limit on these */ return true; } else if(te_targets == NULL) { return false; } r = g_hash_table_lookup(te_targets, target); limit = throttle_get_job_limit(target); if(r == NULL) { r = calloc(1, sizeof(struct te_peer_s)); r->name = strdup(target); g_hash_table_insert(te_targets, r->name, r); } if(limit <= r->jobs) { crm_trace("Peer %s is over their job limit of %d (%d): deferring %s", target, limit, r->jobs, id); return false; } else if(graph->migration_limit > 0 && r->migrate_jobs >= graph->migration_limit) { if (pcmk__strcase_any_of(task, PCMK_ACTION_MIGRATE_TO, PCMK_ACTION_MIGRATE_FROM, NULL)) { crm_trace("Peer %s is over their migration job limit of %d (%d): deferring %s", target, graph->migration_limit, r->migrate_jobs, id); return false; } } crm_trace("Peer %s has not hit their limit yet. current jobs = %d limit= %d limit", target, r->jobs, limit); return true; } /*! * \internal * \brief Check whether a graph action is allowed to be executed * * \param[in] graph Transition graph being executed * \param[in] action Graph action being executed * * \return true if action is allowed, otherwise false */ static bool graph_action_allowed(pcmk__graph_t *graph, pcmk__graph_action_t *action) { const char *target = NULL; const char *task = crm_element_value(action->xml, PCMK_XA_OPERATION); if (action->type != pcmk__rsc_graph_action) { /* No limit on these */ return true; } /* if we have a router node, this means the action is performing * on a remote node. For now, we count all actions occurring on a * remote node against the job list on the cluster node hosting * the connection resources */ target = crm_element_value(action->xml, XML_LRM_ATTR_ROUTER_NODE); if ((target == NULL) && pcmk__strcase_any_of(task, PCMK_ACTION_MIGRATE_TO, PCMK_ACTION_MIGRATE_FROM, NULL)) { target = crm_meta_value(action->params, XML_LRM_ATTR_MIGRATE_SOURCE); if (!allowed_on_node(graph, action, target)) { return false; } target = crm_meta_value(action->params, XML_LRM_ATTR_MIGRATE_TARGET); } else if (target == NULL) { target = crm_element_value(action->xml, PCMK__META_ON_NODE); } return allowed_on_node(graph, action, target); } /*! * \brief Confirm a graph action (and optionally update graph) * * \param[in,out] action Action to confirm * \param[in,out] graph Update and trigger this graph (if non-NULL) */ void te_action_confirmed(pcmk__graph_action_t *action, pcmk__graph_t *graph) { if (!pcmk_is_set(action->flags, pcmk__graph_action_confirmed)) { if ((action->type == pcmk__rsc_graph_action) && (crm_element_value(action->xml, PCMK__META_ON_NODE) != NULL)) { te_update_job_count(action, -1); } pcmk__set_graph_action_flags(action, pcmk__graph_action_confirmed); } if (graph) { pcmk__update_graph(graph, action); trigger_graph(); } } static pcmk__graph_functions_t te_graph_fns = { execute_pseudo_action, execute_rsc_action, execute_cluster_action, controld_execute_fence_action, graph_action_allowed, }; /* * \internal * \brief Register the transitioner's graph functions with \p libpacemaker */ void controld_register_graph_functions(void) { pcmk__set_graph_functions(&te_graph_fns); } void notify_crmd(pcmk__graph_t *graph) { const char *type = "unknown"; enum crmd_fsa_input event = I_NULL; crm_debug("Processing transition completion in state %s", fsa_state2string(controld_globals.fsa_state)); CRM_CHECK(graph->complete, graph->complete = true); switch (graph->completion_action) { case pcmk__graph_wait: type = "stop"; if (controld_globals.fsa_state == S_TRANSITION_ENGINE) { event = I_TE_SUCCESS; } break; case pcmk__graph_done: type = "done"; if (controld_globals.fsa_state == S_TRANSITION_ENGINE) { event = I_TE_SUCCESS; } break; case pcmk__graph_restart: type = "restart"; if (controld_globals.fsa_state == S_TRANSITION_ENGINE) { if (controld_get_period_transition_timer() > 0) { controld_stop_transition_timer(); controld_start_transition_timer(); } else { event = I_PE_CALC; } } else if (controld_globals.fsa_state == S_POLICY_ENGINE) { controld_set_fsa_action_flags(A_PE_INVOKE); controld_trigger_fsa(); } break; case pcmk__graph_shutdown: type = "shutdown"; if (pcmk_is_set(controld_globals.fsa_input_register, R_SHUTDOWN)) { event = I_STOP; } else { crm_err("We didn't ask to be shut down, yet the scheduler is telling us to"); event = I_TERMINATE; } } crm_debug("Transition %d status: %s - %s", graph->id, type, pcmk__s(graph->abort_reason, "unspecified reason")); graph->abort_reason = NULL; graph->completion_action = pcmk__graph_done; if (event != I_NULL) { register_fsa_input(C_FSA_INTERNAL, event, NULL); } else { controld_trigger_fsa(); } } diff --git a/include/crm/common/options_internal.h b/include/crm/common/options_internal.h index 6ab4881808..fcff2cb92b 100644 --- a/include/crm/common/options_internal.h +++ b/include/crm/common/options_internal.h @@ -1,181 +1,182 @@ /* * Copyright 2006-2024 the Pacemaker project contributors * * The version control history for this file may have further details. * * This source code is licensed under the GNU Lesser General Public License * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. */ #ifndef PCMK__OPTIONS_INTERNAL__H # define PCMK__OPTIONS_INTERNAL__H # ifndef PCMK__CONFIG_H # define PCMK__CONFIG_H # include // _Noreturn # endif # include // GHashTable # include // bool #include // pcmk_parse_interval_spec() _Noreturn void pcmk__cli_help(char cmd); /* * Environment variable option handling */ const char *pcmk__env_option(const char *option); void pcmk__set_env_option(const char *option, const char *value, bool compat); bool pcmk__env_option_enabled(const char *daemon, const char *option); /* * Cluster option handling */ typedef struct pcmk__cluster_option_s { const char *name; const char *alt_name; const char *type; const char *values; const char *default_value; bool (*is_valid)(const char *); const char *description_short; const char *description_long; } pcmk__cluster_option_t; const char *pcmk__cluster_option(GHashTable *options, const pcmk__cluster_option_t *option_list, int len, const char *name); gchar *pcmk__format_option_metadata(const char *name, const char *desc_short, const char *desc_long, pcmk__cluster_option_t *option_list, int len); void pcmk__validate_cluster_options(GHashTable *options, pcmk__cluster_option_t *option_list, int len); bool pcmk__valid_interval_spec(const char *value); bool pcmk__valid_boolean(const char *value); bool pcmk__valid_int(const char *value); bool pcmk__valid_positive_int(const char *value); bool pcmk__valid_no_quorum_policy(const char *value); bool pcmk__valid_percentage(const char *value); bool pcmk__valid_script(const char *value); // from watchdog.c long pcmk__get_sbd_timeout(void); bool pcmk__get_sbd_sync_resource_startup(void); long pcmk__auto_watchdog_timeout(void); bool pcmk__valid_sbd_timeout(const char *value); // Constants for environment variable names #define PCMK__ENV_AUTHKEY_LOCATION "authkey_location" #define PCMK__ENV_BLACKBOX "blackbox" #define PCMK__ENV_CALLGRIND_ENABLED "callgrind_enabled" #define PCMK__ENV_CLUSTER_TYPE "cluster_type" #define PCMK__ENV_DEBUG "debug" #define PCMK__ENV_DH_MAX_BITS "dh_max_bits" #define PCMK__ENV_DH_MIN_BITS "dh_min_bits" #define PCMK__ENV_FAIL_FAST "fail_fast" #define PCMK__ENV_IPC_BUFFER "ipc_buffer" #define PCMK__ENV_IPC_TYPE "ipc_type" #define PCMK__ENV_LOGFACILITY "logfacility" #define PCMK__ENV_LOGFILE "logfile" #define PCMK__ENV_LOGFILE_MODE "logfile_mode" #define PCMK__ENV_LOGPRIORITY "logpriority" #define PCMK__ENV_NODE_ACTION_LIMIT "node_action_limit" #define PCMK__ENV_NODE_START_STATE "node_start_state" #define PCMK__ENV_PANIC_ACTION "panic_action" #define PCMK__ENV_REMOTE_ADDRESS "remote_address" #define PCMK__ENV_REMOTE_SCHEMA_DIR "remote_schema_directory" #define PCMK__ENV_REMOTE_PID1 "remote_pid1" #define PCMK__ENV_REMOTE_PORT "remote_port" #define PCMK__ENV_RESPAWNED "respawned" #define PCMK__ENV_SCHEMA_DIRECTORY "schema_directory" #define PCMK__ENV_SERVICE "service" #define PCMK__ENV_STDERR "stderr" #define PCMK__ENV_TLS_PRIORITIES "tls_priorities" #define PCMK__ENV_TRACE_BLACKBOX "trace_blackbox" #define PCMK__ENV_TRACE_FILES "trace_files" #define PCMK__ENV_TRACE_FORMATS "trace_formats" #define PCMK__ENV_TRACE_FUNCTIONS "trace_functions" #define PCMK__ENV_TRACE_TAGS "trace_tags" #define PCMK__ENV_VALGRIND_ENABLED "valgrind_enabled" // @COMPAT Drop at 3.0.0; default is plenty #define PCMK__ENV_CIB_TIMEOUT "cib_timeout" // @COMPAT Drop at 3.0.0; likely last used in 1.1.24 #define PCMK__ENV_MCP "mcp" // @COMPAT Drop at 3.0.0; added unused in 1.1.9 #define PCMK__ENV_QUORUM_TYPE "quorum_type" /* @COMPAT Drop at 3.0.0; added to debug shutdown issues when Pacemaker is * managed by systemd, but no longer useful. */ #define PCMK__ENV_SHUTDOWN_DELAY "shutdown_delay" // @COMPAT Deprecated since 2.1.0 #define PCMK__OPT_REMOVE_AFTER_STOP "remove-after-stop" // Constants for meta-attribute names #define PCMK__META_CLONE_INSTANCE_NUM "clone" #define PCMK__META_CONTAINER "container" #define PCMK__META_DIGESTS_ALL "digests-all" #define PCMK__META_DIGESTS_SECURE "digests-secure" #define PCMK__META_INTERNAL_RSC "internal_rsc" #define PCMK__META_ON_NODE "on_node" #define PCMK__META_ON_NODE_UUID "on_node_uuid" #define PCMK__META_OP_NO_WAIT "op_no_wait" +#define PCMK__META_OP_TARGET_RC "op_target_rc" #define PCMK__META_PHYSICAL_HOST "physical-host" /* @TODO Plug these in. Currently, they're never set. These are op attrs for use * with https://projects.clusterlabs.org/T382. */ #define PCMK__META_CLEAR_FAILURE_OP "clear_failure_op" #define PCMK__META_CLEAR_FAILURE_INTERVAL "clear_failure_interval" // @COMPAT Deprecated meta-attribute since 2.1.0 #define PCMK__META_CAN_FAIL "can_fail" // @COMPAT Deprecated alias for PCMK__META_PROMOTED_MAX since 2.0.0 #define PCMK__META_PROMOTED_MAX_LEGACY "master-max" // @COMPAT Deprecated alias for PCMK__META_PROMOTED_NODE_MAX since 2.0.0 #define PCMK__META_PROMOTED_NODE_MAX_LEGACY "master-node-max" // @COMPAT Deprecated meta-attribute since 2.0.0 #define PCMK__META_RESTART_TYPE "restart-type" // @COMPAT Deprecated meta-attribute since 2.0.0 #define PCMK__META_ROLE_AFTER_FAILURE "role_after_failure" // Constants for enumerated values for various options #define PCMK__VALUE_CLUSTER "cluster" #define PCMK__VALUE_CUSTOM "custom" #define PCMK__VALUE_FENCING "fencing" #define PCMK__VALUE_GREEN "green" #define PCMK__VALUE_LOCAL "local" #define PCMK__VALUE_MIGRATE_ON_RED "migrate-on-red" #define PCMK__VALUE_NONE "none" #define PCMK__VALUE_NOTHING "nothing" #define PCMK__VALUE_ONLY_GREEN "only-green" #define PCMK__VALUE_PROGRESSIVE "progressive" #define PCMK__VALUE_QUORUM "quorum" #define PCMK__VALUE_RED "red" #define PCMK__VALUE_REQUEST "request" #define PCMK__VALUE_RESPONSE "response" #define PCMK__VALUE_UNFENCING "unfencing" #define PCMK__VALUE_YELLOW "yellow" #endif // PCMK__OPTIONS_INTERNAL__H diff --git a/lib/pacemaker/pcmk_sched_resource.c b/lib/pacemaker/pcmk_sched_resource.c index ab14471bef..2ab20edb65 100644 --- a/lib/pacemaker/pcmk_sched_resource.c +++ b/lib/pacemaker/pcmk_sched_resource.c @@ -1,774 +1,774 @@ /* * Copyright 2014-2024 the Pacemaker project contributors * * The version control history for this file may have further details. * * This source code is licensed under the GNU General Public License version 2 * or later (GPLv2+) WITHOUT ANY WARRANTY. */ #include #include #include #include #include #include "libpacemaker_private.h" // Resource assignment methods by resource variant static pcmk_assignment_methods_t assignment_methods[] = { { pcmk__primitive_assign, pcmk__primitive_create_actions, pcmk__probe_rsc_on_node, pcmk__primitive_internal_constraints, pcmk__primitive_apply_coloc_score, pcmk__colocated_resources, pcmk__with_primitive_colocations, pcmk__primitive_with_colocations, pcmk__add_colocated_node_scores, pcmk__apply_location, pcmk__primitive_action_flags, pcmk__update_ordered_actions, pcmk__output_resource_actions, pcmk__add_rsc_actions_to_graph, pcmk__primitive_add_graph_meta, pcmk__primitive_add_utilization, pcmk__primitive_shutdown_lock, }, { pcmk__group_assign, pcmk__group_create_actions, pcmk__probe_rsc_on_node, pcmk__group_internal_constraints, pcmk__group_apply_coloc_score, pcmk__group_colocated_resources, pcmk__with_group_colocations, pcmk__group_with_colocations, pcmk__group_add_colocated_node_scores, pcmk__group_apply_location, pcmk__group_action_flags, pcmk__group_update_ordered_actions, pcmk__output_resource_actions, pcmk__add_rsc_actions_to_graph, pcmk__noop_add_graph_meta, pcmk__group_add_utilization, pcmk__group_shutdown_lock, }, { pcmk__clone_assign, pcmk__clone_create_actions, pcmk__clone_create_probe, pcmk__clone_internal_constraints, pcmk__clone_apply_coloc_score, pcmk__colocated_resources, pcmk__with_clone_colocations, pcmk__clone_with_colocations, pcmk__add_colocated_node_scores, pcmk__clone_apply_location, pcmk__clone_action_flags, pcmk__instance_update_ordered_actions, pcmk__output_resource_actions, pcmk__clone_add_actions_to_graph, pcmk__clone_add_graph_meta, pcmk__clone_add_utilization, pcmk__clone_shutdown_lock, }, { pcmk__bundle_assign, pcmk__bundle_create_actions, pcmk__bundle_create_probe, pcmk__bundle_internal_constraints, pcmk__bundle_apply_coloc_score, pcmk__colocated_resources, pcmk__with_bundle_colocations, pcmk__bundle_with_colocations, pcmk__add_colocated_node_scores, pcmk__bundle_apply_location, pcmk__bundle_action_flags, pcmk__instance_update_ordered_actions, pcmk__output_bundle_actions, pcmk__bundle_add_actions_to_graph, pcmk__noop_add_graph_meta, pcmk__bundle_add_utilization, pcmk__bundle_shutdown_lock, } }; /*! * \internal * \brief Check whether a resource's agent standard, provider, or type changed * * \param[in,out] rsc Resource to check * \param[in,out] node Node needing unfencing if agent changed * \param[in] rsc_entry XML with previously known agent information * \param[in] active_on_node Whether \p rsc is active on \p node * * \return true if agent for \p rsc changed, otherwise false */ bool pcmk__rsc_agent_changed(pcmk_resource_t *rsc, pcmk_node_t *node, const xmlNode *rsc_entry, bool active_on_node) { bool changed = false; const char *attr_list[] = { PCMK_XA_TYPE, PCMK_XA_CLASS, PCMK_XA_PROVIDER, }; for (int i = 0; i < PCMK__NELEM(attr_list); i++) { const char *value = crm_element_value(rsc->xml, attr_list[i]); const char *old_value = crm_element_value(rsc_entry, attr_list[i]); if (!pcmk__str_eq(value, old_value, pcmk__str_none)) { changed = true; trigger_unfencing(rsc, node, "Device definition changed", NULL, rsc->cluster); if (active_on_node) { crm_notice("Forcing restart of %s on %s " "because %s changed from '%s' to '%s'", rsc->id, pe__node_name(node), attr_list[i], pcmk__s(old_value, ""), pcmk__s(value, "")); } } } if (changed && active_on_node) { // Make sure the resource is restarted custom_action(rsc, stop_key(rsc), PCMK_ACTION_STOP, node, FALSE, rsc->cluster); pcmk__set_rsc_flags(rsc, pcmk_rsc_start_pending); } return changed; } /*! * \internal * \brief Add resource (and any matching children) to list if it matches ID * * \param[in] result List to add resource to * \param[in] rsc Resource to check * \param[in] id ID to match * * \return (Possibly new) head of list */ static GList * add_rsc_if_matching(GList *result, pcmk_resource_t *rsc, const char *id) { if ((strcmp(rsc->id, id) == 0) || ((rsc->clone_name != NULL) && (strcmp(rsc->clone_name, id) == 0))) { result = g_list_prepend(result, rsc); } for (GList *iter = rsc->children; iter != NULL; iter = iter->next) { pcmk_resource_t *child = (pcmk_resource_t *) iter->data; result = add_rsc_if_matching(result, child, id); } return result; } /*! * \internal * \brief Find all resources matching a given ID by either ID or clone name * * \param[in] id Resource ID to check * \param[in] scheduler Scheduler data * * \return List of all resources that match \p id * \note The caller is responsible for freeing the return value with * g_list_free(). */ GList * pcmk__rscs_matching_id(const char *id, const pcmk_scheduler_t *scheduler) { GList *result = NULL; CRM_CHECK((id != NULL) && (scheduler != NULL), return NULL); for (GList *iter = scheduler->resources; iter != NULL; iter = iter->next) { result = add_rsc_if_matching(result, (pcmk_resource_t *) iter->data, id); } return result; } /*! * \internal * \brief Set the variant-appropriate assignment methods for a resource * * \param[in,out] data Resource to set assignment methods for * \param[in] user_data Ignored */ static void set_assignment_methods_for_rsc(gpointer data, gpointer user_data) { pcmk_resource_t *rsc = data; rsc->cmds = &assignment_methods[rsc->variant]; g_list_foreach(rsc->children, set_assignment_methods_for_rsc, NULL); } /*! * \internal * \brief Set the variant-appropriate assignment methods for all resources * * \param[in,out] scheduler Scheduler data */ void pcmk__set_assignment_methods(pcmk_scheduler_t *scheduler) { g_list_foreach(scheduler->resources, set_assignment_methods_for_rsc, NULL); } /*! * \internal * \brief Wrapper for colocated_resources() method for readability * * \param[in] rsc Resource to add to colocated list * \param[in] orig_rsc Resource originally requested * \param[in,out] list Pointer to list to add to * * \return (Possibly new) head of list */ static inline void add_colocated_resources(const pcmk_resource_t *rsc, const pcmk_resource_t *orig_rsc, GList **list) { *list = rsc->cmds->colocated_resources(rsc, orig_rsc, *list); } // Shared implementation of pcmk_assignment_methods_t:colocated_resources() GList * pcmk__colocated_resources(const pcmk_resource_t *rsc, const pcmk_resource_t *orig_rsc, GList *colocated_rscs) { const GList *iter = NULL; GList *colocations = NULL; if (orig_rsc == NULL) { orig_rsc = rsc; } if ((rsc == NULL) || (g_list_find(colocated_rscs, rsc) != NULL)) { return colocated_rscs; } pcmk__rsc_trace(orig_rsc, "%s is in colocation chain with %s", rsc->id, orig_rsc->id); colocated_rscs = g_list_prepend(colocated_rscs, (gpointer) rsc); // Follow colocations where this resource is the dependent resource colocations = pcmk__this_with_colocations(rsc); for (iter = colocations; iter != NULL; iter = iter->next) { const pcmk__colocation_t *constraint = iter->data; const pcmk_resource_t *primary = constraint->primary; if (primary == orig_rsc) { continue; // Break colocation loop } if ((constraint->score == INFINITY) && (pcmk__colocation_affects(rsc, primary, constraint, true) == pcmk__coloc_affects_location)) { add_colocated_resources(primary, orig_rsc, &colocated_rscs); } } g_list_free(colocations); // Follow colocations where this resource is the primary resource colocations = pcmk__with_this_colocations(rsc); for (iter = colocations; iter != NULL; iter = iter->next) { const pcmk__colocation_t *constraint = iter->data; const pcmk_resource_t *dependent = constraint->dependent; if (dependent == orig_rsc) { continue; // Break colocation loop } if (pe_rsc_is_clone(rsc) && !pe_rsc_is_clone(dependent)) { continue; // We can't be sure whether dependent will be colocated } if ((constraint->score == INFINITY) && (pcmk__colocation_affects(dependent, rsc, constraint, true) == pcmk__coloc_affects_location)) { add_colocated_resources(dependent, orig_rsc, &colocated_rscs); } } g_list_free(colocations); return colocated_rscs; } // No-op function for variants that don't need to implement add_graph_meta() void pcmk__noop_add_graph_meta(const pcmk_resource_t *rsc, xmlNode *xml) { } /*! * \internal * \brief Output a summary of scheduled actions for a resource * * \param[in,out] rsc Resource to output actions for */ void pcmk__output_resource_actions(pcmk_resource_t *rsc) { pcmk_node_t *next = NULL; pcmk_node_t *current = NULL; pcmk__output_t *out = NULL; CRM_ASSERT(rsc != NULL); out = rsc->cluster->priv; if (rsc->children != NULL) { for (GList *iter = rsc->children; iter != NULL; iter = iter->next) { pcmk_resource_t *child = (pcmk_resource_t *) iter->data; child->cmds->output_actions(child); } return; } next = rsc->allocated_to; if (rsc->running_on) { current = pe__current_node(rsc); if (rsc->role == pcmk_role_stopped) { /* This can occur when resources are being recovered because * the current role can change in pcmk__primitive_create_actions() */ rsc->role = pcmk_role_started; } } if ((current == NULL) && pcmk_is_set(rsc->flags, pcmk_rsc_removed)) { /* Don't log stopped orphans */ return; } out->message(out, "rsc-action", rsc, current, next); } /*! * \internal * \brief Add a resource to a node's list of assigned resources * * \param[in,out] node Node to add resource to * \param[in] rsc Resource to add */ static inline void add_assigned_resource(pcmk_node_t *node, pcmk_resource_t *rsc) { node->details->allocated_rsc = g_list_prepend(node->details->allocated_rsc, rsc); } /*! * \internal * \brief Assign a specified resource (of any variant) to a node * * Assign a specified resource and its children (if any) to a specified node, if * the node can run the resource (or unconditionally, if \p force is true). Mark * the resources as no longer provisional. * * If a resource can't be assigned (or \p node is \c NULL), unassign any * previous assignment. If \p stop_if_fail is \c true, set next role to stopped * and update any existing actions scheduled for the resource. * * \param[in,out] rsc Resource to assign * \param[in,out] node Node to assign \p rsc to * \param[in] force If true, assign to \p node even if unavailable * \param[in] stop_if_fail If \c true and either \p rsc can't be assigned * or \p chosen is \c NULL, set next role to * stopped and update existing actions (if \p rsc * is not a primitive, this applies to its * primitive descendants instead) * * \return \c true if the assignment of \p rsc changed, or \c false otherwise * * \note Assigning a resource to the NULL node using this function is different * from calling pcmk__unassign_resource(), in that it may also update any * actions created for the resource. * \note The \c pcmk_assignment_methods_t:assign() method is preferred, unless * a resource should be assigned to the \c NULL node or every resource in * a tree should be assigned to the same node. * \note If \p stop_if_fail is \c false, then \c pcmk__unassign_resource() can * completely undo the assignment. A successful assignment can be either * undone or left alone as final. A failed assignment has the same effect * as calling pcmk__unassign_resource(); there are no side effects on * roles or actions. */ bool pcmk__assign_resource(pcmk_resource_t *rsc, pcmk_node_t *node, bool force, bool stop_if_fail) { bool changed = false; CRM_ASSERT(rsc != NULL); if (rsc->children != NULL) { for (GList *iter = rsc->children; iter != NULL; iter = iter->next) { pcmk_resource_t *child_rsc = iter->data; changed |= pcmk__assign_resource(child_rsc, node, force, stop_if_fail); } return changed; } // Assigning a primitive if (!force && (node != NULL) && ((node->weight < 0) // Allow graph to assume that guest node connections will come up || (!pcmk__node_available(node, true, false) && !pe__is_guest_node(node)))) { pcmk__rsc_debug(rsc, "All nodes for resource %s are unavailable, unclean or " "shutting down (%s can%s run resources, with score %s)", rsc->id, pe__node_name(node), (pcmk__node_available(node, true, false)? "" : "not"), pcmk_readable_score(node->weight)); if (stop_if_fail) { pe__set_next_role(rsc, pcmk_role_stopped, "node availability"); } node = NULL; } if (rsc->allocated_to != NULL) { changed = !pe__same_node(rsc->allocated_to, node); } else { changed = (node != NULL); } pcmk__unassign_resource(rsc); pcmk__clear_rsc_flags(rsc, pcmk_rsc_unassigned); if (node == NULL) { char *rc_stopped = NULL; pcmk__rsc_debug(rsc, "Could not assign %s to a node", rsc->id); if (!stop_if_fail) { return changed; } pe__set_next_role(rsc, pcmk_role_stopped, "unable to assign"); for (GList *iter = rsc->actions; iter != NULL; iter = iter->next) { pcmk_action_t *op = (pcmk_action_t *) iter->data; pcmk__rsc_debug(rsc, "Updating %s for %s assignment failure", op->uuid, rsc->id); if (pcmk__str_eq(op->task, PCMK_ACTION_STOP, pcmk__str_none)) { pcmk__clear_action_flags(op, pcmk_action_optional); } else if (pcmk__str_eq(op->task, PCMK_ACTION_START, pcmk__str_none)) { pcmk__clear_action_flags(op, pcmk_action_runnable); } else { // Cancel recurring actions, unless for stopped state const char *interval_ms_s = NULL; const char *target_rc_s = NULL; interval_ms_s = g_hash_table_lookup(op->meta, PCMK_META_INTERVAL); target_rc_s = g_hash_table_lookup(op->meta, - XML_ATTR_TE_TARGET_RC); + PCMK__META_OP_TARGET_RC); if (rc_stopped == NULL) { rc_stopped = pcmk__itoa(PCMK_OCF_NOT_RUNNING); } if (!pcmk__str_eq(interval_ms_s, "0", pcmk__str_null_matches) && !pcmk__str_eq(rc_stopped, target_rc_s, pcmk__str_none)) { pcmk__clear_action_flags(op, pcmk_action_runnable); } } } free(rc_stopped); return changed; } pcmk__rsc_debug(rsc, "Assigning %s to %s", rsc->id, pe__node_name(node)); rsc->allocated_to = pe__copy_node(node); add_assigned_resource(node, rsc); node->details->num_resources++; node->count++; pcmk__consume_node_capacity(node->details->utilization, rsc); if (pcmk_is_set(rsc->cluster->flags, pcmk_sched_show_utilization)) { pcmk__output_t *out = rsc->cluster->priv; out->message(out, "resource-util", rsc, node, __func__); } return changed; } /*! * \internal * \brief Remove any node assignment from a specified resource and its children * * If a specified resource has been assigned to a node, remove that assignment * and mark the resource as provisional again. * * \param[in,out] rsc Resource to unassign * * \note This function is called recursively on \p rsc and its children. */ void pcmk__unassign_resource(pcmk_resource_t *rsc) { pcmk_node_t *old = rsc->allocated_to; if (old == NULL) { crm_info("Unassigning %s", rsc->id); } else { crm_info("Unassigning %s from %s", rsc->id, pe__node_name(old)); } pcmk__set_rsc_flags(rsc, pcmk_rsc_unassigned); if (rsc->children == NULL) { if (old == NULL) { return; } rsc->allocated_to = NULL; /* We're going to free the pcmk_node_t, but its details member is shared * and will remain, so update that appropriately first. */ old->details->allocated_rsc = g_list_remove(old->details->allocated_rsc, rsc); old->details->num_resources--; pcmk__release_node_capacity(old->details->utilization, rsc); free(old); return; } for (GList *iter = rsc->children; iter != NULL; iter = iter->next) { pcmk__unassign_resource((pcmk_resource_t *) iter->data); } } /*! * \internal * \brief Check whether a resource has reached its migration threshold on a node * * \param[in,out] rsc Resource to check * \param[in] node Node to check * \param[out] failed If threshold has been reached, this will be set to * resource that failed (possibly a parent of \p rsc) * * \return true if the migration threshold has been reached, false otherwise */ bool pcmk__threshold_reached(pcmk_resource_t *rsc, const pcmk_node_t *node, pcmk_resource_t **failed) { int fail_count, remaining_tries; pcmk_resource_t *rsc_to_ban = rsc; // Migration threshold of 0 means never force away if (rsc->migration_threshold == 0) { return false; } // If we're ignoring failures, also ignore the migration threshold if (pcmk_is_set(rsc->flags, pcmk_rsc_ignore_failure)) { return false; } // If there are no failures, there's no need to force away fail_count = pe_get_failcount(node, rsc, NULL, pcmk__fc_effective|pcmk__fc_fillers, NULL); if (fail_count <= 0) { return false; } // If failed resource is anonymous clone instance, we'll force clone away if (!pcmk_is_set(rsc->flags, pcmk_rsc_unique)) { rsc_to_ban = uber_parent(rsc); } // How many more times recovery will be tried on this node remaining_tries = rsc->migration_threshold - fail_count; if (remaining_tries <= 0) { pcmk__sched_warn("%s cannot run on %s due to reaching migration " "threshold (clean up resource to allow again)" CRM_XS " failures=%d " PCMK_META_MIGRATION_THRESHOLD "=%d", rsc_to_ban->id, pe__node_name(node), fail_count, rsc->migration_threshold); if (failed != NULL) { *failed = rsc_to_ban; } return true; } crm_info("%s can fail %d more time%s on " "%s before reaching migration threshold (%d)", rsc_to_ban->id, remaining_tries, pcmk__plural_s(remaining_tries), pe__node_name(node), rsc->migration_threshold); return false; } /*! * \internal * \brief Get a node's score * * \param[in] node Node with ID to check * \param[in] nodes List of nodes to look for \p node score in * * \return Node's score, or -INFINITY if not found */ static int get_node_score(const pcmk_node_t *node, GHashTable *nodes) { pcmk_node_t *found_node = NULL; if ((node != NULL) && (nodes != NULL)) { found_node = g_hash_table_lookup(nodes, node->details->id); } return (found_node == NULL)? -INFINITY : found_node->weight; } /*! * \internal * \brief Compare two resources according to which should be assigned first * * \param[in] a First resource to compare * \param[in] b Second resource to compare * \param[in] data Sorted list of all nodes in cluster * * \return -1 if \p a should be assigned before \b, 0 if they are equal, * or +1 if \p a should be assigned after \b */ static gint cmp_resources(gconstpointer a, gconstpointer b, gpointer data) { /* GLib insists that this function require gconstpointer arguments, but we * make a small, temporary change to each argument (setting the * pe_rsc_merging flag) during comparison */ pcmk_resource_t *resource1 = (pcmk_resource_t *) a; pcmk_resource_t *resource2 = (pcmk_resource_t *) b; const GList *nodes = data; int rc = 0; int r1_score = -INFINITY; int r2_score = -INFINITY; pcmk_node_t *r1_node = NULL; pcmk_node_t *r2_node = NULL; GHashTable *r1_nodes = NULL; GHashTable *r2_nodes = NULL; const char *reason = NULL; // Resources with highest priority should be assigned first reason = "priority"; r1_score = resource1->priority; r2_score = resource2->priority; if (r1_score > r2_score) { rc = -1; goto done; } if (r1_score < r2_score) { rc = 1; goto done; } // We need nodes to make any other useful comparisons reason = "no node list"; if (nodes == NULL) { goto done; } // Calculate and log node scores resource1->cmds->add_colocated_node_scores(resource1, NULL, resource1->id, &r1_nodes, NULL, 1, pcmk__coloc_select_this_with); resource2->cmds->add_colocated_node_scores(resource2, NULL, resource2->id, &r2_nodes, NULL, 1, pcmk__coloc_select_this_with); pe__show_node_scores(true, NULL, resource1->id, r1_nodes, resource1->cluster); pe__show_node_scores(true, NULL, resource2->id, r2_nodes, resource2->cluster); // The resource with highest score on its current node goes first reason = "current location"; if (resource1->running_on != NULL) { r1_node = pe__current_node(resource1); } if (resource2->running_on != NULL) { r2_node = pe__current_node(resource2); } r1_score = get_node_score(r1_node, r1_nodes); r2_score = get_node_score(r2_node, r2_nodes); if (r1_score > r2_score) { rc = -1; goto done; } if (r1_score < r2_score) { rc = 1; goto done; } // Otherwise a higher score on any node will do reason = "score"; for (const GList *iter = nodes; iter != NULL; iter = iter->next) { const pcmk_node_t *node = (const pcmk_node_t *) iter->data; r1_score = get_node_score(node, r1_nodes); r2_score = get_node_score(node, r2_nodes); if (r1_score > r2_score) { rc = -1; goto done; } if (r1_score < r2_score) { rc = 1; goto done; } } done: crm_trace("%s (%d)%s%s %c %s (%d)%s%s: %s", resource1->id, r1_score, ((r1_node == NULL)? "" : " on "), ((r1_node == NULL)? "" : r1_node->details->id), ((rc < 0)? '>' : ((rc > 0)? '<' : '=')), resource2->id, r2_score, ((r2_node == NULL)? "" : " on "), ((r2_node == NULL)? "" : r2_node->details->id), reason); if (r1_nodes != NULL) { g_hash_table_destroy(r1_nodes); } if (r2_nodes != NULL) { g_hash_table_destroy(r2_nodes); } return rc; } /*! * \internal * \brief Sort resources in the order they should be assigned to nodes * * \param[in,out] scheduler Scheduler data */ void pcmk__sort_resources(pcmk_scheduler_t *scheduler) { GList *nodes = g_list_copy(scheduler->nodes); nodes = pcmk__sort_nodes(nodes, NULL); scheduler->resources = g_list_sort_with_data(scheduler->resources, cmp_resources, nodes); g_list_free(nodes); } diff --git a/lib/pacemaker/pcmk_simulate.c b/lib/pacemaker/pcmk_simulate.c index 43f44cb4d3..00187501f6 100644 --- a/lib/pacemaker/pcmk_simulate.c +++ b/lib/pacemaker/pcmk_simulate.c @@ -1,1005 +1,1005 @@ /* * Copyright 2021-2024 the Pacemaker project contributors * * The version control history for this file may have further details. * * This source code is licensed under the GNU Lesser General Public License * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. */ #include #include #include #include #include #include #include #include #include #include #include #include "libpacemaker_private.h" static pcmk__output_t *out = NULL; static cib_t *fake_cib = NULL; static GList *fake_resource_list = NULL; static const GList *fake_op_fail_list = NULL; static void set_effective_date(pcmk_scheduler_t *scheduler, bool print_original, const char *use_date); /*! * \internal * \brief Create an action name for use in a dot graph * * \param[in] action Action to create name for * \param[in] verbose If true, add action ID to name * * \return Newly allocated string with action name * \note It is the caller's responsibility to free the result. */ static char * create_action_name(const pcmk_action_t *action, bool verbose) { char *action_name = NULL; const char *prefix = ""; const char *action_host = NULL; const char *clone_name = NULL; const char *task = action->task; if (action->node != NULL) { action_host = action->node->details->uname; } else if (!pcmk_is_set(action->flags, pcmk_action_pseudo)) { action_host = ""; } if (pcmk__str_eq(action->task, PCMK_ACTION_CANCEL, pcmk__str_none)) { prefix = "Cancel "; task = action->cancel_task; } if (action->rsc != NULL) { clone_name = action->rsc->clone_name; } if (clone_name != NULL) { char *key = NULL; guint interval_ms = 0; if (pcmk__guint_from_hash(action->meta, PCMK_META_INTERVAL, 0, &interval_ms) != pcmk_rc_ok) { interval_ms = 0; } if (pcmk__strcase_any_of(action->task, PCMK_ACTION_NOTIFY, PCMK_ACTION_NOTIFIED, NULL)) { const char *n_type = g_hash_table_lookup(action->meta, "notify_key_type"); const char *n_task = g_hash_table_lookup(action->meta, "notify_key_operation"); CRM_ASSERT(n_type != NULL); CRM_ASSERT(n_task != NULL); key = pcmk__notify_key(clone_name, n_type, n_task); } else { key = pcmk__op_key(clone_name, task, interval_ms); } if (action_host != NULL) { action_name = crm_strdup_printf("%s%s %s", prefix, key, action_host); } else { action_name = crm_strdup_printf("%s%s", prefix, key); } free(key); } else if (pcmk__str_eq(action->task, PCMK_ACTION_STONITH, pcmk__str_none)) { const char *op = g_hash_table_lookup(action->meta, "stonith_action"); action_name = crm_strdup_printf("%s%s '%s' %s", prefix, action->task, op, action_host); } else if (action->rsc && action_host) { action_name = crm_strdup_printf("%s%s %s", prefix, action->uuid, action_host); } else if (action_host) { action_name = crm_strdup_printf("%s%s %s", prefix, action->task, action_host); } else { action_name = crm_strdup_printf("%s", action->uuid); } if (verbose) { char *with_id = crm_strdup_printf("%s (%d)", action_name, action->id); free(action_name); action_name = with_id; } return action_name; } /*! * \internal * \brief Display the status of a cluster * * \param[in,out] scheduler Scheduler data * \param[in] show_opts How to modify display (as pcmk_show_opt_e flags) * \param[in] section_opts Sections to display (as pcmk_section_e flags) * \param[in] title What to use as list title * \param[in] print_spacer Whether to display a spacer first */ static void print_cluster_status(pcmk_scheduler_t *scheduler, uint32_t show_opts, uint32_t section_opts, const char *title, bool print_spacer) { pcmk__output_t *out = scheduler->priv; GList *all = NULL; crm_exit_t stonith_rc = 0; enum pcmk_pacemakerd_state state = pcmk_pacemakerd_state_invalid; section_opts |= pcmk_section_nodes | pcmk_section_resources; show_opts |= pcmk_show_inactive_rscs | pcmk_show_failed_detail; all = g_list_prepend(all, (gpointer) "*"); PCMK__OUTPUT_SPACER_IF(out, print_spacer); out->begin_list(out, NULL, NULL, "%s", title); out->message(out, "cluster-status", scheduler, state, stonith_rc, NULL, false, section_opts, show_opts, NULL, all, all); out->end_list(out); g_list_free(all); } /*! * \internal * \brief Display a summary of all actions scheduled in a transition * * \param[in,out] scheduler Scheduler data (fully scheduled) * \param[in] print_spacer Whether to display a spacer first */ static void print_transition_summary(pcmk_scheduler_t *scheduler, bool print_spacer) { pcmk__output_t *out = scheduler->priv; PCMK__OUTPUT_SPACER_IF(out, print_spacer); out->begin_list(out, NULL, NULL, "Transition Summary"); pcmk__output_actions(scheduler); out->end_list(out); } /*! * \internal * \brief Reset scheduler input, output, date, and flags * * \param[in,out] scheduler Scheduler data * \param[in] input What to set as cluster input * \param[in] out What to set as cluster output object * \param[in] use_date What to set as cluster's current timestamp * \param[in] flags Group of enum pcmk_scheduler_flags to set */ static void reset(pcmk_scheduler_t *scheduler, xmlNodePtr input, pcmk__output_t *out, const char *use_date, unsigned int flags) { scheduler->input = input; scheduler->priv = out; set_effective_date(scheduler, true, use_date); if (pcmk_is_set(flags, pcmk_sim_sanitized)) { pcmk__set_scheduler_flags(scheduler, pcmk_sched_sanitized); } if (pcmk_is_set(flags, pcmk_sim_show_scores)) { pcmk__set_scheduler_flags(scheduler, pcmk_sched_output_scores); } if (pcmk_is_set(flags, pcmk_sim_show_utilization)) { pcmk__set_scheduler_flags(scheduler, pcmk_sched_show_utilization); } } /*! * \brief Write out a file in dot(1) format describing the actions that will * be taken by the scheduler in response to an input CIB file. * * \param[in,out] scheduler Scheduler data * \param[in] dot_file The filename to write * \param[in] all_actions Write all actions, even those that are optional * or are on unmanaged resources * \param[in] verbose Add extra information, such as action IDs, to the * output * * \return Standard Pacemaker return code */ static int write_sim_dotfile(pcmk_scheduler_t *scheduler, const char *dot_file, bool all_actions, bool verbose) { GList *iter = NULL; FILE *dot_strm = fopen(dot_file, "w"); if (dot_strm == NULL) { return errno; } fprintf(dot_strm, " digraph \"g\" {\n"); for (iter = scheduler->actions; iter != NULL; iter = iter->next) { pcmk_action_t *action = (pcmk_action_t *) iter->data; const char *style = "dashed"; const char *font = "black"; const char *color = "black"; char *action_name = create_action_name(action, verbose); if (pcmk_is_set(action->flags, pcmk_action_pseudo)) { font = "orange"; } if (pcmk_is_set(action->flags, pcmk_action_added_to_graph)) { style = "bold"; color = "green"; } else if ((action->rsc != NULL) && !pcmk_is_set(action->rsc->flags, pcmk_rsc_managed)) { color = "red"; font = "purple"; if (!all_actions) { goto do_not_write; } } else if (pcmk_is_set(action->flags, pcmk_action_optional)) { color = "blue"; if (!all_actions) { goto do_not_write; } } else { color = "red"; CRM_LOG_ASSERT(!pcmk_is_set(action->flags, pcmk_action_runnable)); } pcmk__set_action_flags(action, pcmk_action_added_to_graph); fprintf(dot_strm, "\"%s\" [ style=%s color=\"%s\" fontcolor=\"%s\"]\n", action_name, style, color, font); do_not_write: free(action_name); } for (iter = scheduler->actions; iter != NULL; iter = iter->next) { pcmk_action_t *action = (pcmk_action_t *) iter->data; for (GList *before_iter = action->actions_before; before_iter != NULL; before_iter = before_iter->next) { pcmk__related_action_t *before = before_iter->data; char *before_name = NULL; char *after_name = NULL; const char *style = "dashed"; bool optional = true; if (before->state == pe_link_dumped) { optional = false; style = "bold"; } else if ((uint32_t) before->type == pcmk__ar_none) { continue; } else if (pcmk_is_set(before->action->flags, pcmk_action_added_to_graph) && pcmk_is_set(action->flags, pcmk_action_added_to_graph) && (uint32_t) before->type != pcmk__ar_if_on_same_node_or_target) { optional = false; } if (all_actions || !optional) { before_name = create_action_name(before->action, verbose); after_name = create_action_name(action, verbose); fprintf(dot_strm, "\"%s\" -> \"%s\" [ style = %s]\n", before_name, after_name, style); free(before_name); free(after_name); } } } fprintf(dot_strm, "}\n"); fflush(dot_strm); fclose(dot_strm); return pcmk_rc_ok; } /*! * \brief Profile the configuration updates and scheduler actions in a single * CIB file, printing the profiling timings. * * \note \p scheduler->priv must have been set to a valid \p pcmk__output_t * object before this function is called. * * \param[in] xml_file The CIB file to profile * \param[in] repeat Number of times to run * \param[in,out] scheduler Scheduler data * \param[in] use_date The date to set the cluster's time to (may be NULL) */ static void profile_file(const char *xml_file, long long repeat, pcmk_scheduler_t *scheduler, const char *use_date) { pcmk__output_t *out = scheduler->priv; xmlNode *cib_object = NULL; clock_t start = 0; clock_t end; unsigned long long scheduler_flags = pcmk_sched_no_compat; CRM_ASSERT(out != NULL); cib_object = filename2xml(xml_file); start = clock(); if (pcmk_find_cib_element(cib_object, XML_CIB_TAG_STATUS) == NULL) { create_xml_node(cib_object, XML_CIB_TAG_STATUS); } if (cli_config_update(&cib_object, NULL, FALSE) == FALSE) { free_xml(cib_object); return; } if (validate_xml(cib_object, NULL, FALSE) != TRUE) { free_xml(cib_object); return; } if (pcmk_is_set(scheduler->flags, pcmk_sched_output_scores)) { scheduler_flags |= pcmk_sched_output_scores; } if (pcmk_is_set(scheduler->flags, pcmk_sched_show_utilization)) { scheduler_flags |= pcmk_sched_show_utilization; } for (int i = 0; i < repeat; ++i) { xmlNode *input = (repeat == 1)? cib_object : copy_xml(cib_object); scheduler->input = input; set_effective_date(scheduler, false, use_date); pcmk__schedule_actions(input, scheduler_flags, scheduler); pe_reset_working_set(scheduler); } end = clock(); out->message(out, "profile", xml_file, start, end); } void pcmk__profile_dir(const char *dir, long long repeat, pcmk_scheduler_t *scheduler, const char *use_date) { pcmk__output_t *out = scheduler->priv; struct dirent **namelist; int file_num = scandir(dir, &namelist, 0, alphasort); CRM_ASSERT(out != NULL); if (file_num > 0) { struct stat prop; char buffer[FILENAME_MAX]; out->begin_list(out, NULL, NULL, "Timings"); while (file_num--) { if ('.' == namelist[file_num]->d_name[0]) { free(namelist[file_num]); continue; } else if (!pcmk__ends_with_ext(namelist[file_num]->d_name, ".xml")) { free(namelist[file_num]); continue; } snprintf(buffer, sizeof(buffer), "%s/%s", dir, namelist[file_num]->d_name); if (stat(buffer, &prop) == 0 && S_ISREG(prop.st_mode)) { profile_file(buffer, repeat, scheduler, use_date); } free(namelist[file_num]); } free(namelist); out->end_list(out); } } /*! * \brief Set the date of the cluster, either to the value given by * \p use_date, or to the "execution-date" value in the CIB. * * \note \p scheduler->priv must have been set to a valid \p pcmk__output_t * object before this function is called. * * \param[in,out] scheduler Scheduler data * \param[in] print_original If \p true, the "execution-date" should * also be printed * \param[in] use_date The date to set the cluster's time to * (may be NULL) */ static void set_effective_date(pcmk_scheduler_t *scheduler, bool print_original, const char *use_date) { pcmk__output_t *out = scheduler->priv; time_t original_date = 0; CRM_ASSERT(out != NULL); crm_element_value_epoch(scheduler->input, "execution-date", &original_date); if (use_date) { scheduler->now = crm_time_new(use_date); out->info(out, "Setting effective cluster time: %s", use_date); crm_time_log(LOG_NOTICE, "Pretending 'now' is", scheduler->now, crm_time_log_date | crm_time_log_timeofday); } else if (original_date != 0) { scheduler->now = pcmk__copy_timet(original_date); if (print_original) { char *when = crm_time_as_string(scheduler->now, crm_time_log_date|crm_time_log_timeofday); out->info(out, "Using the original execution date of: %s", when); free(when); } } } /*! * \internal * \brief Simulate successfully executing a pseudo-action in a graph * * \param[in,out] graph Graph to update with pseudo-action result * \param[in,out] action Pseudo-action to simulate executing * * \return Standard Pacemaker return code */ static int simulate_pseudo_action(pcmk__graph_t *graph, pcmk__graph_action_t *action) { const char *node = crm_element_value(action->xml, PCMK__META_ON_NODE); const char *task = crm_element_value(action->xml, PCMK__XA_OPERATION_KEY); pcmk__set_graph_action_flags(action, pcmk__graph_action_confirmed); out->message(out, "inject-pseudo-action", node, task); pcmk__update_graph(graph, action); return pcmk_rc_ok; } /*! * \internal * \brief Simulate executing a resource action in a graph * * \param[in,out] graph Graph to update with resource action result * \param[in,out] action Resource action to simulate executing * * \return Standard Pacemaker return code */ static int simulate_resource_action(pcmk__graph_t *graph, pcmk__graph_action_t *action) { int rc; lrmd_event_data_t *op = NULL; int target_outcome = PCMK_OCF_OK; const char *rtype = NULL; const char *rclass = NULL; const char *resource = NULL; const char *rprovider = NULL; const char *resource_config_name = NULL; const char *operation = crm_element_value(action->xml, PCMK_XA_OPERATION); const char *target_rc_s = crm_meta_value(action->params, - XML_ATTR_TE_TARGET_RC); + PCMK__META_OP_TARGET_RC); xmlNode *cib_node = NULL; xmlNode *cib_resource = NULL; xmlNode *action_rsc = first_named_child(action->xml, XML_CIB_TAG_RESOURCE); char *node = crm_element_value_copy(action->xml, PCMK__META_ON_NODE); char *uuid = NULL; const char *router_node = crm_element_value(action->xml, XML_LRM_ATTR_ROUTER_NODE); // Certain actions don't need to be displayed or history entries if (pcmk__str_eq(operation, CRM_OP_REPROBE, pcmk__str_none)) { crm_debug("No history injection for %s op on %s", operation, node); goto done; // Confirm action and update graph } if (action_rsc == NULL) { // Shouldn't be possible crm_log_xml_err(action->xml, "Bad"); free(node); return EPROTO; } /* A resource might be known by different names in the configuration and in * the action (for example, a clone instance). Grab the configuration name * (which is preferred when writing history), and if necessary, the instance * name. */ resource_config_name = crm_element_value(action_rsc, PCMK_XA_ID); if (resource_config_name == NULL) { // Shouldn't be possible crm_log_xml_err(action->xml, "No ID"); free(node); return EPROTO; } resource = resource_config_name; if (pe_find_resource(fake_resource_list, resource) == NULL) { const char *longname = crm_element_value(action_rsc, PCMK__XA_LONG_ID); if ((longname != NULL) && (pe_find_resource(fake_resource_list, longname) != NULL)) { resource = longname; } } // Certain actions need to be displayed but don't need history entries if (pcmk__strcase_any_of(operation, PCMK_ACTION_DELETE, PCMK_ACTION_META_DATA, NULL)) { out->message(out, "inject-rsc-action", resource, operation, node, (guint) 0); goto done; // Confirm action and update graph } rclass = crm_element_value(action_rsc, PCMK_XA_CLASS); rtype = crm_element_value(action_rsc, PCMK_XA_TYPE); rprovider = crm_element_value(action_rsc, PCMK_XA_PROVIDER); pcmk__scan_min_int(target_rc_s, &target_outcome, 0); CRM_ASSERT(fake_cib->cmds->query(fake_cib, NULL, NULL, cib_sync_call|cib_scope_local) == pcmk_ok); // Ensure the action node is in the CIB uuid = crm_element_value_copy(action->xml, PCMK__META_ON_NODE_UUID); cib_node = pcmk__inject_node(fake_cib, node, ((router_node == NULL)? uuid: node)); free(uuid); CRM_ASSERT(cib_node != NULL); // Add a history entry for the action cib_resource = pcmk__inject_resource_history(out, cib_node, resource, resource_config_name, rclass, rtype, rprovider); if (cib_resource == NULL) { crm_err("Could not simulate action %d history for resource %s", action->id, resource); free(node); free_xml(cib_node); return EINVAL; } // Simulate and display an executor event for the action result op = pcmk__event_from_graph_action(cib_resource, action, PCMK_EXEC_DONE, target_outcome, "User-injected result"); out->message(out, "inject-rsc-action", resource, op->op_type, node, op->interval_ms); // Check whether action is in a list of desired simulated failures for (const GList *iter = fake_op_fail_list; iter != NULL; iter = iter->next) { const char *spec = (const char *) iter->data; char *key = NULL; const char *match_name = NULL; // Allow user to specify anonymous clone with or without instance number key = crm_strdup_printf(PCMK__OP_FMT "@%s=", resource, op->op_type, op->interval_ms, node); if (strncasecmp(key, spec, strlen(key)) == 0) { match_name = resource; } free(key); // If not found, try the resource's name in the configuration if ((match_name == NULL) && (strcmp(resource, resource_config_name) != 0)) { key = crm_strdup_printf(PCMK__OP_FMT "@%s=", resource_config_name, op->op_type, op->interval_ms, node); if (strncasecmp(key, spec, strlen(key)) == 0) { match_name = resource_config_name; } free(key); } if (match_name == NULL) { continue; // This failed action entry doesn't match } // ${match_name}_${task}_${interval_in_ms}@${node}=${rc} rc = sscanf(spec, "%*[^=]=%d", (int *) &op->rc); if (rc != 1) { out->err(out, "Invalid failed operation '%s' " "(result code must be integer)", spec); continue; // Keep checking other list entries } out->info(out, "Pretending action %d failed with rc=%d", action->id, op->rc); pcmk__set_graph_action_flags(action, pcmk__graph_action_failed); graph->abort_priority = INFINITY; pcmk__inject_failcount(out, cib_node, match_name, op->op_type, op->interval_ms, op->rc); break; } pcmk__inject_action_result(cib_resource, op, target_outcome); lrmd_free_event(op); rc = fake_cib->cmds->modify(fake_cib, XML_CIB_TAG_STATUS, cib_node, cib_sync_call|cib_scope_local); CRM_ASSERT(rc == pcmk_ok); done: free(node); free_xml(cib_node); pcmk__set_graph_action_flags(action, pcmk__graph_action_confirmed); pcmk__update_graph(graph, action); return pcmk_rc_ok; } /*! * \internal * \brief Simulate successfully executing a cluster action * * \param[in,out] graph Graph to update with action result * \param[in,out] action Cluster action to simulate * * \return Standard Pacemaker return code */ static int simulate_cluster_action(pcmk__graph_t *graph, pcmk__graph_action_t *action) { const char *node = crm_element_value(action->xml, PCMK__META_ON_NODE); const char *task = crm_element_value(action->xml, PCMK_XA_OPERATION); xmlNode *rsc = first_named_child(action->xml, XML_CIB_TAG_RESOURCE); pcmk__set_graph_action_flags(action, pcmk__graph_action_confirmed); out->message(out, "inject-cluster-action", node, task, rsc); pcmk__update_graph(graph, action); return pcmk_rc_ok; } /*! * \internal * \brief Simulate successfully executing a fencing action * * \param[in,out] graph Graph to update with action result * \param[in,out] action Fencing action to simulate * * \return Standard Pacemaker return code */ static int simulate_fencing_action(pcmk__graph_t *graph, pcmk__graph_action_t *action) { const char *op = crm_meta_value(action->params, "stonith_action"); char *target = crm_element_value_copy(action->xml, PCMK__META_ON_NODE); out->message(out, "inject-fencing-action", target, op); if (!pcmk__str_eq(op, PCMK_ACTION_ON, pcmk__str_casei)) { int rc = pcmk_ok; GString *xpath = g_string_sized_new(512); // Set node state to offline xmlNode *cib_node = pcmk__inject_node_state_change(fake_cib, target, false); CRM_ASSERT(cib_node != NULL); crm_xml_add(cib_node, PCMK_XA_CRM_DEBUG_ORIGIN, __func__); rc = fake_cib->cmds->replace(fake_cib, XML_CIB_TAG_STATUS, cib_node, cib_sync_call|cib_scope_local); CRM_ASSERT(rc == pcmk_ok); // Simulate controller clearing node's resource history and attributes pcmk__g_strcat(xpath, "//" XML_CIB_TAG_STATE "[@" PCMK_XA_UNAME "='", target, "']/" XML_CIB_TAG_LRM, NULL); fake_cib->cmds->remove(fake_cib, (const char *) xpath->str, NULL, cib_xpath|cib_sync_call|cib_scope_local); g_string_truncate(xpath, 0); pcmk__g_strcat(xpath, "//" XML_CIB_TAG_STATE "[@" PCMK_XA_UNAME "='", target, "']" "/" XML_TAG_TRANSIENT_NODEATTRS, NULL); fake_cib->cmds->remove(fake_cib, (const char *) xpath->str, NULL, cib_xpath|cib_sync_call|cib_scope_local); free_xml(cib_node); g_string_free(xpath, TRUE); } pcmk__set_graph_action_flags(action, pcmk__graph_action_confirmed); pcmk__update_graph(graph, action); free(target); return pcmk_rc_ok; } enum pcmk__graph_status pcmk__simulate_transition(pcmk_scheduler_t *scheduler, cib_t *cib, const GList *op_fail_list) { pcmk__graph_t *transition = NULL; enum pcmk__graph_status graph_rc; pcmk__graph_functions_t simulation_fns = { simulate_pseudo_action, simulate_resource_action, simulate_cluster_action, simulate_fencing_action, }; out = scheduler->priv; fake_cib = cib; fake_op_fail_list = op_fail_list; if (!out->is_quiet(out)) { out->begin_list(out, NULL, NULL, "Executing Cluster Transition"); } pcmk__set_graph_functions(&simulation_fns); transition = pcmk__unpack_graph(scheduler->graph, crm_system_name); pcmk__log_graph(LOG_DEBUG, transition); fake_resource_list = scheduler->resources; do { graph_rc = pcmk__execute_graph(transition); } while (graph_rc == pcmk__graph_active); fake_resource_list = NULL; if (graph_rc != pcmk__graph_complete) { out->err(out, "Transition failed: %s", pcmk__graph_status2text(graph_rc)); pcmk__log_graph(LOG_ERR, transition); out->err(out, "An invalid transition was produced"); } pcmk__free_graph(transition); if (!out->is_quiet(out)) { // If not quiet, we'll need the resulting CIB for later display xmlNode *cib_object = NULL; int rc = fake_cib->cmds->query(fake_cib, NULL, &cib_object, cib_sync_call|cib_scope_local); CRM_ASSERT(rc == pcmk_ok); pe_reset_working_set(scheduler); scheduler->input = cib_object; out->end_list(out); } return graph_rc; } int pcmk__simulate(pcmk_scheduler_t *scheduler, pcmk__output_t *out, const pcmk_injections_t *injections, unsigned int flags, uint32_t section_opts, const char *use_date, const char *input_file, const char *graph_file, const char *dot_file) { int printed = pcmk_rc_no_output; int rc = pcmk_rc_ok; xmlNodePtr input = NULL; cib_t *cib = NULL; rc = cib__signon_query(out, &cib, &input); if (rc != pcmk_rc_ok) { goto simulate_done; } reset(scheduler, input, out, use_date, flags); cluster_status(scheduler); if ((cib->variant == cib_native) && pcmk_is_set(section_opts, pcmk_section_times)) { if (pcmk__our_nodename == NULL) { // Currently used only in the times section pcmk__query_node_name(out, 0, &pcmk__our_nodename, 0); } scheduler->localhost = pcmk__our_nodename; } if (!out->is_quiet(out)) { const bool show_pending = pcmk_is_set(flags, pcmk_sim_show_pending); if (pcmk_is_set(scheduler->flags, pcmk_sched_in_maintenance)) { printed = out->message(out, "maint-mode", scheduler->flags); } if (scheduler->disabled_resources || scheduler->blocked_resources) { PCMK__OUTPUT_SPACER_IF(out, printed == pcmk_rc_ok); printed = out->info(out, "%d of %d resource instances DISABLED and " "%d BLOCKED from further action due to failure", scheduler->disabled_resources, scheduler->ninstances, scheduler->blocked_resources); } /* Most formatted output headers use caps for each word, but this one * only has the first word capitalized for compatibility with pcs. */ print_cluster_status(scheduler, (show_pending? pcmk_show_pending : 0), section_opts, "Current cluster status", (printed == pcmk_rc_ok)); printed = pcmk_rc_ok; } // If the user requested any injections, handle them if ((injections->node_down != NULL) || (injections->node_fail != NULL) || (injections->node_up != NULL) || (injections->op_inject != NULL) || (injections->ticket_activate != NULL) || (injections->ticket_grant != NULL) || (injections->ticket_revoke != NULL) || (injections->ticket_standby != NULL) || (injections->watchdog != NULL)) { PCMK__OUTPUT_SPACER_IF(out, printed == pcmk_rc_ok); pcmk__inject_scheduler_input(scheduler, cib, injections); printed = pcmk_rc_ok; rc = cib->cmds->query(cib, NULL, &input, cib_sync_call); if (rc != pcmk_rc_ok) { rc = pcmk_legacy2rc(rc); goto simulate_done; } cleanup_calculations(scheduler); reset(scheduler, input, out, use_date, flags); cluster_status(scheduler); } if (input_file != NULL) { rc = write_xml_file(input, input_file, FALSE); if (rc < 0) { rc = pcmk_legacy2rc(rc); goto simulate_done; } } if (pcmk_any_flags_set(flags, pcmk_sim_process | pcmk_sim_simulate)) { pcmk__output_t *logger_out = NULL; unsigned long long scheduler_flags = pcmk_sched_no_compat; if (pcmk_is_set(scheduler->flags, pcmk_sched_output_scores)) { scheduler_flags |= pcmk_sched_output_scores; } if (pcmk_is_set(scheduler->flags, pcmk_sched_show_utilization)) { scheduler_flags |= pcmk_sched_show_utilization; } if (pcmk_all_flags_set(scheduler->flags, pcmk_sched_output_scores |pcmk_sched_show_utilization)) { PCMK__OUTPUT_SPACER_IF(out, printed == pcmk_rc_ok); out->begin_list(out, NULL, NULL, "Assignment Scores and Utilization Information"); printed = pcmk_rc_ok; } else if (pcmk_is_set(scheduler->flags, pcmk_sched_output_scores)) { PCMK__OUTPUT_SPACER_IF(out, printed == pcmk_rc_ok); out->begin_list(out, NULL, NULL, "Assignment Scores"); printed = pcmk_rc_ok; } else if (pcmk_is_set(scheduler->flags, pcmk_sched_show_utilization)) { PCMK__OUTPUT_SPACER_IF(out, printed == pcmk_rc_ok); out->begin_list(out, NULL, NULL, "Utilization Information"); printed = pcmk_rc_ok; } else { rc = pcmk__log_output_new(&logger_out); if (rc != pcmk_rc_ok) { goto simulate_done; } pe__register_messages(logger_out); pcmk__register_lib_messages(logger_out); scheduler->priv = logger_out; } pcmk__schedule_actions(input, scheduler_flags, scheduler); if (logger_out == NULL) { out->end_list(out); } else { logger_out->finish(logger_out, CRM_EX_OK, true, NULL); pcmk__output_free(logger_out); scheduler->priv = out; } input = NULL; /* Don't try and free it twice */ if (graph_file != NULL) { rc = write_xml_file(scheduler->graph, graph_file, FALSE); if (rc < 0) { rc = pcmk_rc_graph_error; goto simulate_done; } } if (dot_file != NULL) { rc = write_sim_dotfile(scheduler, dot_file, pcmk_is_set(flags, pcmk_sim_all_actions), pcmk_is_set(flags, pcmk_sim_verbose)); if (rc != pcmk_rc_ok) { rc = pcmk_rc_dot_error; goto simulate_done; } } if (!out->is_quiet(out)) { print_transition_summary(scheduler, printed == pcmk_rc_ok); } } rc = pcmk_rc_ok; if (!pcmk_is_set(flags, pcmk_sim_simulate)) { goto simulate_done; } PCMK__OUTPUT_SPACER_IF(out, printed == pcmk_rc_ok); if (pcmk__simulate_transition(scheduler, cib, injections->op_fail) != pcmk__graph_complete) { rc = pcmk_rc_invalid_transition; } if (out->is_quiet(out)) { goto simulate_done; } set_effective_date(scheduler, true, use_date); if (pcmk_is_set(flags, pcmk_sim_show_scores)) { pcmk__set_scheduler_flags(scheduler, pcmk_sched_output_scores); } if (pcmk_is_set(flags, pcmk_sim_show_utilization)) { pcmk__set_scheduler_flags(scheduler, pcmk_sched_show_utilization); } cluster_status(scheduler); print_cluster_status(scheduler, 0, section_opts, "Revised Cluster Status", true); simulate_done: cib__clean_up_connection(&cib); return rc; } int pcmk_simulate(xmlNodePtr *xml, pcmk_scheduler_t *scheduler, const pcmk_injections_t *injections, unsigned int flags, unsigned int section_opts, const char *use_date, const char *input_file, const char *graph_file, const char *dot_file) { pcmk__output_t *out = NULL; int rc = pcmk_rc_ok; rc = pcmk__xml_output_new(&out, xml); if (rc != pcmk_rc_ok) { return rc; } pe__register_messages(out); pcmk__register_lib_messages(out); rc = pcmk__simulate(scheduler, out, injections, flags, section_opts, use_date, input_file, graph_file, dot_file); pcmk__xml_output_finish(out, pcmk_rc2exitc(rc), xml); return rc; } diff --git a/lib/pengine/pe_actions.c b/lib/pengine/pe_actions.c index f399673dc6..7ee66358fd 100644 --- a/lib/pengine/pe_actions.c +++ b/lib/pengine/pe_actions.c @@ -1,1892 +1,1892 @@ /* * Copyright 2004-2024 the Pacemaker project contributors * * The version control history for this file may have further details. * * This source code is licensed under the GNU Lesser General Public License * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. */ #include #include #include #include #include #include #include #include #include "pe_status_private.h" static void unpack_operation(pcmk_action_t *action, const xmlNode *xml_obj, guint interval_ms); static void add_singleton(pcmk_scheduler_t *scheduler, pcmk_action_t *action) { if (scheduler->singletons == NULL) { scheduler->singletons = pcmk__strkey_table(NULL, NULL); } g_hash_table_insert(scheduler->singletons, action->uuid, action); } static pcmk_action_t * lookup_singleton(pcmk_scheduler_t *scheduler, const char *action_uuid) { if (scheduler->singletons == NULL) { return NULL; } return g_hash_table_lookup(scheduler->singletons, action_uuid); } /*! * \internal * \brief Find an existing action that matches arguments * * \param[in] key Action key to match * \param[in] rsc Resource to match (if any) * \param[in] node Node to match (if any) * \param[in] scheduler Scheduler data * * \return Existing action that matches arguments (or NULL if none) */ static pcmk_action_t * find_existing_action(const char *key, const pcmk_resource_t *rsc, const pcmk_node_t *node, const pcmk_scheduler_t *scheduler) { GList *matches = NULL; pcmk_action_t *action = NULL; /* When rsc is NULL, it would be quicker to check scheduler->singletons, * but checking all scheduler->actions takes the node into account. */ matches = find_actions(((rsc == NULL)? scheduler->actions : rsc->actions), key, node); if (matches == NULL) { return NULL; } CRM_LOG_ASSERT(!pcmk__list_of_multiple(matches)); action = matches->data; g_list_free(matches); return action; } /*! * \internal * \brief Find the XML configuration corresponding to a specific action key * * \param[in] rsc Resource to find action configuration for * \param[in] key "RSC_ACTION_INTERVAL" of action to find * \param[in] include_disabled If false, do not return disabled actions * * \return XML configuration of desired action if any, otherwise NULL */ static xmlNode * find_exact_action_config(const pcmk_resource_t *rsc, const char *action_name, guint interval_ms, bool include_disabled) { for (xmlNode *operation = first_named_child(rsc->ops_xml, PCMK_XE_OP); operation != NULL; operation = crm_next_same_xml(operation)) { bool enabled = false; const char *config_name = NULL; const char *interval_spec = NULL; guint tmp_ms = 0U; // @TODO This does not consider meta-attributes, rules, defaults, etc. if (!include_disabled && (pcmk__xe_get_bool_attr(operation, PCMK_META_ENABLED, &enabled) == pcmk_rc_ok) && !enabled) { continue; } interval_spec = crm_element_value(operation, PCMK_META_INTERVAL); pcmk_parse_interval_spec(interval_spec, &tmp_ms); if (tmp_ms != interval_ms) { continue; } config_name = crm_element_value(operation, PCMK_XA_NAME); if (pcmk__str_eq(action_name, config_name, pcmk__str_none)) { return operation; } } return NULL; } /*! * \internal * \brief Find the XML configuration of a resource action * * \param[in] rsc Resource to find action configuration for * \param[in] action_name Action name to search for * \param[in] interval_ms Action interval (in milliseconds) to search for * \param[in] include_disabled If false, do not return disabled actions * * \return XML configuration of desired action if any, otherwise NULL */ xmlNode * pcmk__find_action_config(const pcmk_resource_t *rsc, const char *action_name, guint interval_ms, bool include_disabled) { xmlNode *action_config = NULL; // Try requested action first action_config = find_exact_action_config(rsc, action_name, interval_ms, include_disabled); // For migrate_to and migrate_from actions, retry with "migrate" // @TODO This should be either documented or deprecated if ((action_config == NULL) && pcmk__str_any_of(action_name, PCMK_ACTION_MIGRATE_TO, PCMK_ACTION_MIGRATE_FROM, NULL)) { action_config = find_exact_action_config(rsc, "migrate", 0, include_disabled); } return action_config; } /*! * \internal * \brief Create a new action object * * \param[in] key Action key * \param[in] task Action name * \param[in,out] rsc Resource that action is for (if any) * \param[in] node Node that action is on (if any) * \param[in] optional Whether action should be considered optional * \param[in,out] scheduler Scheduler data * * \return Newly allocated action * \note This function takes ownership of \p key. It is the caller's * responsibility to free the return value with pe_free_action(). */ static pcmk_action_t * new_action(char *key, const char *task, pcmk_resource_t *rsc, const pcmk_node_t *node, bool optional, pcmk_scheduler_t *scheduler) { pcmk_action_t *action = calloc(1, sizeof(pcmk_action_t)); CRM_ASSERT(action != NULL); action->rsc = rsc; action->task = strdup(task); CRM_ASSERT(action->task != NULL); action->uuid = key; if (node) { action->node = pe__copy_node(node); } if (pcmk__str_eq(task, PCMK_ACTION_LRM_DELETE, pcmk__str_casei)) { // Resource history deletion for a node can be done on the DC pcmk__set_action_flags(action, pcmk_action_on_dc); } pcmk__set_action_flags(action, pcmk_action_runnable); if (optional) { pcmk__set_action_flags(action, pcmk_action_optional); } else { pcmk__clear_action_flags(action, pcmk_action_optional); } if (rsc == NULL) { action->meta = pcmk__strkey_table(free, free); } else { guint interval_ms = 0; parse_op_key(key, NULL, NULL, &interval_ms); action->op_entry = pcmk__find_action_config(rsc, task, interval_ms, true); /* If the given key is for one of the many notification pseudo-actions * (pre_notify_promote, etc.), the actual action name is "notify" */ if ((action->op_entry == NULL) && (strstr(key, "_notify_") != NULL)) { action->op_entry = find_exact_action_config(rsc, PCMK_ACTION_NOTIFY, 0, true); } unpack_operation(action, action->op_entry, interval_ms); } pcmk__rsc_trace(rsc, "Created %s action %d (%s): %s for %s on %s", (optional? "optional" : "required"), scheduler->action_id, key, task, ((rsc == NULL)? "no resource" : rsc->id), pe__node_name(node)); action->id = scheduler->action_id++; scheduler->actions = g_list_prepend(scheduler->actions, action); if (rsc == NULL) { add_singleton(scheduler, action); } else { rsc->actions = g_list_prepend(rsc->actions, action); } return action; } /*! * \internal * \brief Unpack a resource's action-specific instance parameters * * \param[in] action_xml XML of action's configuration in CIB (if any) * \param[in,out] node_attrs Table of node attributes (for rule evaluation) * \param[in,out] scheduler Cluster working set (for rule evaluation) * * \return Newly allocated hash table of action-specific instance parameters */ GHashTable * pcmk__unpack_action_rsc_params(const xmlNode *action_xml, GHashTable *node_attrs, pcmk_scheduler_t *scheduler) { GHashTable *params = pcmk__strkey_table(free, free); pe_rule_eval_data_t rule_data = { .node_hash = node_attrs, .role = pcmk_role_unknown, .now = scheduler->now, .match_data = NULL, .rsc_data = NULL, .op_data = NULL }; pe__unpack_dataset_nvpairs(action_xml, XML_TAG_ATTR_SETS, &rule_data, params, NULL, FALSE, scheduler); return params; } /*! * \internal * \brief Update an action's optional flag * * \param[in,out] action Action to update * \param[in] optional Requested optional status */ static void update_action_optional(pcmk_action_t *action, gboolean optional) { // Force a non-recurring action to be optional if its resource is unmanaged if ((action->rsc != NULL) && (action->node != NULL) && !pcmk_is_set(action->flags, pcmk_action_pseudo) && !pcmk_is_set(action->rsc->flags, pcmk_rsc_managed) && (g_hash_table_lookup(action->meta, PCMK_META_INTERVAL) == NULL)) { pcmk__rsc_debug(action->rsc, "%s on %s is optional (%s is unmanaged)", action->uuid, pe__node_name(action->node), action->rsc->id); pcmk__set_action_flags(action, pcmk_action_optional); // We shouldn't clear runnable here because ... something // Otherwise require the action if requested } else if (!optional) { pcmk__clear_action_flags(action, pcmk_action_optional); } } static enum pe_quorum_policy effective_quorum_policy(pcmk_resource_t *rsc, pcmk_scheduler_t *scheduler) { enum pe_quorum_policy policy = scheduler->no_quorum_policy; if (pcmk_is_set(scheduler->flags, pcmk_sched_quorate)) { policy = pcmk_no_quorum_ignore; } else if (scheduler->no_quorum_policy == pcmk_no_quorum_demote) { switch (rsc->role) { case pcmk_role_promoted: case pcmk_role_unpromoted: if (rsc->next_role > pcmk_role_unpromoted) { pe__set_next_role(rsc, pcmk_role_unpromoted, PCMK_OPT_NO_QUORUM_POLICY "=demote"); } policy = pcmk_no_quorum_ignore; break; default: policy = pcmk_no_quorum_stop; break; } } return policy; } /*! * \internal * \brief Update a resource action's runnable flag * * \param[in,out] action Action to update * \param[in,out] scheduler Scheduler data * * \note This may also schedule fencing if a stop is unrunnable. */ static void update_resource_action_runnable(pcmk_action_t *action, pcmk_scheduler_t *scheduler) { if (pcmk_is_set(action->flags, pcmk_action_pseudo)) { return; } if (action->node == NULL) { pcmk__rsc_trace(action->rsc, "%s is unrunnable (unallocated)", action->uuid); pcmk__clear_action_flags(action, pcmk_action_runnable); } else if (!pcmk_is_set(action->flags, pcmk_action_on_dc) && !(action->node->details->online) && (!pe__is_guest_node(action->node) || action->node->details->remote_requires_reset)) { pcmk__clear_action_flags(action, pcmk_action_runnable); do_crm_log(LOG_WARNING, "%s on %s is unrunnable (node is offline)", action->uuid, pe__node_name(action->node)); if (pcmk_is_set(action->rsc->flags, pcmk_rsc_managed) && pcmk__str_eq(action->task, PCMK_ACTION_STOP, pcmk__str_casei) && !(action->node->details->unclean)) { pe_fence_node(scheduler, action->node, "stop is unrunnable", false); } } else if (!pcmk_is_set(action->flags, pcmk_action_on_dc) && action->node->details->pending) { pcmk__clear_action_flags(action, pcmk_action_runnable); do_crm_log(LOG_WARNING, "Action %s on %s is unrunnable (node is pending)", action->uuid, pe__node_name(action->node)); } else if (action->needs == pcmk_requires_nothing) { pe_action_set_reason(action, NULL, TRUE); if (pe__is_guest_node(action->node) && !pe_can_fence(scheduler, action->node)) { /* An action that requires nothing usually does not require any * fencing in order to be runnable. However, there is an exception: * such an action cannot be completed if it is on a guest node whose * host is unclean and cannot be fenced. */ pcmk__rsc_debug(action->rsc, "%s on %s is unrunnable " "(node's host cannot be fenced)", action->uuid, pe__node_name(action->node)); pcmk__clear_action_flags(action, pcmk_action_runnable); } else { pcmk__rsc_trace(action->rsc, "%s on %s does not require fencing or quorum", action->uuid, pe__node_name(action->node)); pcmk__set_action_flags(action, pcmk_action_runnable); } } else { switch (effective_quorum_policy(action->rsc, scheduler)) { case pcmk_no_quorum_stop: pcmk__rsc_debug(action->rsc, "%s on %s is unrunnable (no quorum)", action->uuid, pe__node_name(action->node)); pcmk__clear_action_flags(action, pcmk_action_runnable); pe_action_set_reason(action, "no quorum", true); break; case pcmk_no_quorum_freeze: if (!action->rsc->fns->active(action->rsc, TRUE) || (action->rsc->next_role > action->rsc->role)) { pcmk__rsc_debug(action->rsc, "%s on %s is unrunnable (no quorum)", action->uuid, pe__node_name(action->node)); pcmk__clear_action_flags(action, pcmk_action_runnable); pe_action_set_reason(action, "quorum freeze", true); } break; default: //pe_action_set_reason(action, NULL, TRUE); pcmk__set_action_flags(action, pcmk_action_runnable); break; } } } /*! * \internal * \brief Update a resource object's flags for a new action on it * * \param[in,out] rsc Resource that action is for (if any) * \param[in] action New action */ static void update_resource_flags_for_action(pcmk_resource_t *rsc, const pcmk_action_t *action) { /* @COMPAT pcmk_rsc_starting and pcmk_rsc_stopping are deprecated and unused * within Pacemaker, and will eventually be removed */ if (pcmk__str_eq(action->task, PCMK_ACTION_STOP, pcmk__str_casei)) { pcmk__set_rsc_flags(rsc, pcmk_rsc_stopping); } else if (pcmk__str_eq(action->task, PCMK_ACTION_START, pcmk__str_casei)) { if (pcmk_is_set(action->flags, pcmk_action_runnable)) { pcmk__set_rsc_flags(rsc, pcmk_rsc_starting); } else { pcmk__clear_rsc_flags(rsc, pcmk_rsc_starting); } } } static bool valid_stop_on_fail(const char *value) { return !pcmk__strcase_any_of(value, "standby", "demote", "stop", NULL); } /*! * \internal * \brief Validate (and possibly reset) resource action's on_fail meta-attribute * * \param[in] rsc Resource that action is for * \param[in] action_name Action name * \param[in] action_config Action configuration XML from CIB (if any) * \param[in,out] meta Table of action meta-attributes */ static void validate_on_fail(const pcmk_resource_t *rsc, const char *action_name, const xmlNode *action_config, GHashTable *meta) { const char *name = NULL; const char *role = NULL; const char *interval_spec = NULL; const char *value = g_hash_table_lookup(meta, PCMK_META_ON_FAIL); char *key = NULL; char *new_value = NULL; guint interval_ms = 0U; // Stop actions can only use certain on-fail values if (pcmk__str_eq(action_name, PCMK_ACTION_STOP, pcmk__str_none) && !valid_stop_on_fail(value)) { pcmk__config_err("Resetting '" PCMK_META_ON_FAIL "' for %s stop " "action to default value because '%s' is not " "allowed for stop", rsc->id, value); g_hash_table_remove(meta, PCMK_META_ON_FAIL); return; } /* Demote actions default on-fail to the on-fail value for the first * recurring monitor for the promoted role (if any). */ if (pcmk__str_eq(action_name, PCMK_ACTION_DEMOTE, pcmk__str_none) && (value == NULL)) { /* @TODO This does not consider promote options set in a meta-attribute * block (which may have rules that need to be evaluated) rather than * XML properties. */ for (xmlNode *operation = first_named_child(rsc->ops_xml, PCMK_XE_OP); operation != NULL; operation = crm_next_same_xml(operation)) { bool enabled = false; const char *promote_on_fail = NULL; /* We only care about explicit on-fail (if promote uses default, so * can demote) */ promote_on_fail = crm_element_value(operation, PCMK_META_ON_FAIL); if (promote_on_fail == NULL) { continue; } // We only care about recurring monitors for the promoted role name = crm_element_value(operation, PCMK_XA_NAME); role = crm_element_value(operation, "role"); if (!pcmk__str_eq(name, PCMK_ACTION_MONITOR, pcmk__str_none) || !pcmk__strcase_any_of(role, PCMK__ROLE_PROMOTED, PCMK__ROLE_PROMOTED_LEGACY, NULL)) { continue; } interval_spec = crm_element_value(operation, PCMK_META_INTERVAL); pcmk_parse_interval_spec(interval_spec, &interval_ms); if (interval_ms == 0U) { continue; } // We only care about enabled monitors if ((pcmk__xe_get_bool_attr(operation, PCMK_META_ENABLED, &enabled) == pcmk_rc_ok) && !enabled) { continue; } // Demote actions can't default to on-fail="demote" if (pcmk__str_eq(promote_on_fail, "demote", pcmk__str_casei)) { continue; } // Use value from first applicable promote action found key = strdup(PCMK_META_ON_FAIL); new_value = strdup(promote_on_fail); CRM_ASSERT((key != NULL) && (new_value != NULL)); g_hash_table_insert(meta, key, new_value); } return; } if (pcmk__str_eq(action_name, PCMK_ACTION_LRM_DELETE, pcmk__str_none) && !pcmk__str_eq(value, "ignore", pcmk__str_casei)) { key = strdup(PCMK_META_ON_FAIL); new_value = strdup("ignore"); CRM_ASSERT((key != NULL) && (new_value != NULL)); g_hash_table_insert(meta, key, new_value); return; } // on-fail="demote" is allowed only for certain actions if (pcmk__str_eq(value, "demote", pcmk__str_casei)) { name = crm_element_value(action_config, PCMK_XA_NAME); role = crm_element_value(action_config, "role"); interval_spec = crm_element_value(action_config, PCMK_META_INTERVAL); pcmk_parse_interval_spec(interval_spec, &interval_ms); if (!pcmk__str_eq(name, PCMK_ACTION_PROMOTE, pcmk__str_none) && ((interval_ms == 0U) || !pcmk__str_eq(name, PCMK_ACTION_MONITOR, pcmk__str_none) || !pcmk__strcase_any_of(role, PCMK__ROLE_PROMOTED, PCMK__ROLE_PROMOTED_LEGACY, NULL))) { pcmk__config_err("Resetting '" PCMK_META_ON_FAIL "' for %s %s " "action to default value because 'demote' is not " "allowed for it", rsc->id, name); g_hash_table_remove(meta, PCMK_META_ON_FAIL); return; } } } static int unpack_timeout(const char *value) { int timeout_ms = crm_get_msec(value); if (timeout_ms < 0) { timeout_ms = PCMK_DEFAULT_ACTION_TIMEOUT_MS; } return timeout_ms; } // true if value contains valid, non-NULL interval origin for recurring op static bool unpack_interval_origin(const char *value, const xmlNode *xml_obj, guint interval_ms, const crm_time_t *now, long long *start_delay) { long long result = 0; guint interval_sec = interval_ms / 1000; crm_time_t *origin = NULL; // Ignore unspecified values and non-recurring operations if ((value == NULL) || (interval_ms == 0) || (now == NULL)) { return false; } // Parse interval origin from text origin = crm_time_new(value); if (origin == NULL) { pcmk__config_err("Ignoring '" PCMK_META_INTERVAL_ORIGIN "' for " "operation '%s' because '%s' is not valid", (ID(xml_obj)? ID(xml_obj) : "(missing ID)"), value); return false; } // Get seconds since origin (negative if origin is in the future) result = crm_time_get_seconds(now) - crm_time_get_seconds(origin); crm_time_free(origin); // Calculate seconds from closest interval to now result = result % interval_sec; // Calculate seconds remaining until next interval result = ((result <= 0)? 0 : interval_sec) - result; crm_info("Calculated a start delay of %llds for operation '%s'", result, (ID(xml_obj)? ID(xml_obj) : "(unspecified)")); if (start_delay != NULL) { *start_delay = result * 1000; // milliseconds } return true; } static int unpack_start_delay(const char *value, GHashTable *meta) { int start_delay = 0; if (value != NULL) { start_delay = crm_get_msec(value); if (start_delay < 0) { start_delay = 0; } if (meta) { g_hash_table_replace(meta, strdup(PCMK_META_START_DELAY), pcmk__itoa(start_delay)); } } return start_delay; } /*! * \internal * \brief Find a resource's most frequent recurring monitor * * \param[in] rsc Resource to check * * \return Operation XML configured for most frequent recurring monitor for * \p rsc (if any) */ static xmlNode * most_frequent_monitor(const pcmk_resource_t *rsc) { guint min_interval_ms = G_MAXUINT; xmlNode *op = NULL; for (xmlNode *operation = first_named_child(rsc->ops_xml, PCMK_XE_OP); operation != NULL; operation = crm_next_same_xml(operation)) { bool enabled = false; guint interval_ms = 0U; const char *interval_spec = crm_element_value(operation, PCMK_META_INTERVAL); // We only care about enabled recurring monitors if (!pcmk__str_eq(crm_element_value(operation, PCMK_XA_NAME), PCMK_ACTION_MONITOR, pcmk__str_none)) { continue; } pcmk_parse_interval_spec(interval_spec, &interval_ms); if (interval_ms == 0U) { continue; } // @TODO This does not consider meta-attributes, rules, defaults, etc. if ((pcmk__xe_get_bool_attr(operation, PCMK_META_ENABLED, &enabled) == pcmk_rc_ok) && !enabled) { continue; } if (interval_ms < min_interval_ms) { min_interval_ms = interval_ms; op = operation; } } return op; } /*! * \internal * \brief Unpack action meta-attributes * * \param[in,out] rsc Resource that action is for * \param[in] node Node that action is on * \param[in] action_name Action name * \param[in] interval_ms Action interval (in milliseconds) * \param[in] action_config Action XML configuration from CIB (if any) * * Unpack a resource action's meta-attributes (normalizing the interval, * timeout, and start delay values as integer milliseconds) from its CIB XML * configuration (including defaults). * * \return Newly allocated hash table with normalized action meta-attributes */ GHashTable * pcmk__unpack_action_meta(pcmk_resource_t *rsc, const pcmk_node_t *node, const char *action_name, guint interval_ms, const xmlNode *action_config) { GHashTable *meta = NULL; char *name = NULL; char *value = NULL; const char *timeout_spec = NULL; const char *str = NULL; pe_rsc_eval_data_t rsc_rule_data = { .standard = crm_element_value(rsc->xml, PCMK_XA_CLASS), .provider = crm_element_value(rsc->xml, PCMK_XA_PROVIDER), .agent = crm_element_value(rsc->xml, PCMK_XA_TYPE), }; pe_op_eval_data_t op_rule_data = { .op_name = action_name, .interval = interval_ms, }; pe_rule_eval_data_t rule_data = { .node_hash = (node == NULL)? NULL : node->details->attrs, .role = pcmk_role_unknown, .now = rsc->cluster->now, .match_data = NULL, .rsc_data = &rsc_rule_data, .op_data = &op_rule_data, }; meta = pcmk__strkey_table(free, free); // Cluster-wide pe__unpack_dataset_nvpairs(rsc->cluster->op_defaults, XML_TAG_META_SETS, &rule_data, meta, NULL, FALSE, rsc->cluster); // Derive default timeout for probes from recurring monitor timeouts if (pcmk_is_probe(action_name, interval_ms)) { xmlNode *min_interval_mon = most_frequent_monitor(rsc); if (min_interval_mon != NULL) { /* @TODO This does not consider timeouts set in meta_attributes * blocks (which may also have rules that need to be evaluated). */ timeout_spec = crm_element_value(min_interval_mon, PCMK_META_TIMEOUT); if (timeout_spec != NULL) { pcmk__rsc_trace(rsc, "Setting default timeout for %s probe to " "most frequent monitor's timeout '%s'", rsc->id, timeout_spec); name = strdup(PCMK_META_TIMEOUT); value = strdup(timeout_spec); CRM_ASSERT((name != NULL) && (value != NULL)); g_hash_table_insert(meta, name, value); } } } if (action_config != NULL) { // take precedence over defaults pe__unpack_dataset_nvpairs(action_config, XML_TAG_META_SETS, &rule_data, meta, NULL, TRUE, rsc->cluster); /* Anything set as an XML property has highest precedence. * This ensures we use the name and interval from the tag. * (See below for the only exception, fence device start/probe timeout.) */ for (xmlAttrPtr attr = action_config->properties; attr != NULL; attr = attr->next) { name = strdup((const char *) attr->name); value = strdup(pcmk__xml_attr_value(attr)); CRM_ASSERT((name != NULL) && (value != NULL)); g_hash_table_insert(meta, name, value); } } g_hash_table_remove(meta, PCMK_XA_ID); // Normalize interval to milliseconds if (interval_ms > 0) { name = strdup(PCMK_META_INTERVAL); CRM_ASSERT(name != NULL); value = crm_strdup_printf("%u", interval_ms); g_hash_table_insert(meta, name, value); } else { g_hash_table_remove(meta, PCMK_META_INTERVAL); } /* Timeout order of precedence (highest to lowest): * 1. pcmk_monitor_timeout resource parameter (only for starts and probes * when rsc has pcmk_ra_cap_fence_params; this gets used for recurring * monitors via the executor instead) * 2. timeout configured in (with taking precedence over * ) * 3. timeout configured in * 4. PCMK_DEFAULT_ACTION_TIMEOUT_MS */ // Check for pcmk_monitor_timeout if (pcmk_is_set(pcmk_get_ra_caps(rsc_rule_data.standard), pcmk_ra_cap_fence_params) && (pcmk__str_eq(action_name, PCMK_ACTION_START, pcmk__str_none) || pcmk_is_probe(action_name, interval_ms))) { GHashTable *params = pe_rsc_params(rsc, node, rsc->cluster); timeout_spec = g_hash_table_lookup(params, "pcmk_monitor_timeout"); if (timeout_spec != NULL) { pcmk__rsc_trace(rsc, "Setting timeout for %s %s to " "pcmk_monitor_timeout (%s)", rsc->id, action_name, timeout_spec); name = strdup(PCMK_META_TIMEOUT); value = strdup(timeout_spec); CRM_ASSERT((name != NULL) && (value != NULL)); g_hash_table_insert(meta, name, value); } } // Normalize timeout to positive milliseconds name = strdup(PCMK_META_TIMEOUT); CRM_ASSERT(name != NULL); timeout_spec = g_hash_table_lookup(meta, PCMK_META_TIMEOUT); g_hash_table_insert(meta, name, pcmk__itoa(unpack_timeout(timeout_spec))); // Ensure on-fail has a valid value validate_on_fail(rsc, action_name, action_config, meta); // Normalize PCMK_META_START_DELAY str = g_hash_table_lookup(meta, PCMK_META_START_DELAY); if (str != NULL) { unpack_start_delay(str, meta); } else { long long start_delay = 0; str = g_hash_table_lookup(meta, PCMK_META_INTERVAL_ORIGIN); if (unpack_interval_origin(str, action_config, interval_ms, rsc->cluster->now, &start_delay)) { name = strdup(PCMK_META_START_DELAY); CRM_ASSERT(name != NULL); g_hash_table_insert(meta, name, crm_strdup_printf("%lld", start_delay)); } } return meta; } /*! * \internal * \brief Determine an action's quorum and fencing dependency * * \param[in] rsc Resource that action is for * \param[in] action_name Name of action being unpacked * * \return Quorum and fencing dependency appropriate to action */ enum rsc_start_requirement pcmk__action_requires(const pcmk_resource_t *rsc, const char *action_name) { const char *value = NULL; enum rsc_start_requirement requires = pcmk_requires_nothing; CRM_CHECK((rsc != NULL) && (action_name != NULL), return requires); if (!pcmk__strcase_any_of(action_name, PCMK_ACTION_START, PCMK_ACTION_PROMOTE, NULL)) { value = "nothing (not start or promote)"; } else if (pcmk_is_set(rsc->flags, pcmk_rsc_needs_fencing)) { requires = pcmk_requires_fencing; value = "fencing"; } else if (pcmk_is_set(rsc->flags, pcmk_rsc_needs_quorum)) { requires = pcmk_requires_quorum; value = "quorum"; } else { value = "nothing"; } pcmk__rsc_trace(rsc, "%s of %s requires %s", action_name, rsc->id, value); return requires; } /*! * \internal * \brief Parse action failure response from a user-provided string * * \param[in] rsc Resource that action is for * \param[in] action_name Name of action * \param[in] interval_ms Action interval (in milliseconds) * \param[in] value User-provided configuration value for on-fail * * \return Action failure response parsed from \p text */ enum action_fail_response pcmk__parse_on_fail(const pcmk_resource_t *rsc, const char *action_name, guint interval_ms, const char *value) { const char *desc = NULL; bool needs_remote_reset = false; enum action_fail_response on_fail = pcmk_on_fail_ignore; if (value == NULL) { // Use default } else if (pcmk__str_eq(value, "block", pcmk__str_casei)) { on_fail = pcmk_on_fail_block; desc = "block"; } else if (pcmk__str_eq(value, "fence", pcmk__str_casei)) { if (pcmk_is_set(rsc->cluster->flags, pcmk_sched_fencing_enabled)) { on_fail = pcmk_on_fail_fence_node; desc = "node fencing"; } else { pcmk__config_err("Resetting '" PCMK_META_ON_FAIL "' for " "%s of %s to 'stop' because 'fence' is not " "valid when fencing is disabled", action_name, rsc->id); on_fail = pcmk_on_fail_stop; desc = "stop resource"; } } else if (pcmk__str_eq(value, "standby", pcmk__str_casei)) { on_fail = pcmk_on_fail_standby_node; desc = "node standby"; } else if (pcmk__strcase_any_of(value, "ignore", PCMK__VALUE_NOTHING, NULL)) { desc = "ignore"; } else if (pcmk__str_eq(value, "migrate", pcmk__str_casei)) { on_fail = pcmk_on_fail_ban; desc = "force migration"; } else if (pcmk__str_eq(value, "stop", pcmk__str_casei)) { on_fail = pcmk_on_fail_stop; desc = "stop resource"; } else if (pcmk__str_eq(value, "restart", pcmk__str_casei)) { on_fail = pcmk_on_fail_restart; desc = "restart (and possibly migrate)"; } else if (pcmk__str_eq(value, "restart-container", pcmk__str_casei)) { if (rsc->container == NULL) { pcmk__rsc_debug(rsc, "Using default " PCMK_META_ON_FAIL " for %s " "of %s because it does not have a container", action_name, rsc->id); } else { on_fail = pcmk_on_fail_restart_container; desc = "restart container (and possibly migrate)"; } } else if (pcmk__str_eq(value, "demote", pcmk__str_casei)) { on_fail = pcmk_on_fail_demote; desc = "demote instance"; } else { pcmk__config_err("Using default '" PCMK_META_ON_FAIL "' for " "%s of %s because '%s' is not valid", action_name, rsc->id, value); } /* Remote node connections are handled specially. Failures that result * in dropping an active connection must result in fencing. The only * failures that don't are probes and starts. The user can explicitly set * on-fail="fence" to fence after start failures. */ if (pe__resource_is_remote_conn(rsc) && !pcmk_is_probe(action_name, interval_ms) && !pcmk__str_eq(action_name, PCMK_ACTION_START, pcmk__str_none)) { needs_remote_reset = true; if (!pcmk_is_set(rsc->flags, pcmk_rsc_managed)) { desc = NULL; // Force default for unmanaged connections } } if (desc != NULL) { // Explicit value used, default not needed } else if (rsc->container != NULL) { on_fail = pcmk_on_fail_restart_container; desc = "restart container (and possibly migrate) (default)"; } else if (needs_remote_reset) { if (pcmk_is_set(rsc->flags, pcmk_rsc_managed)) { if (pcmk_is_set(rsc->cluster->flags, pcmk_sched_fencing_enabled)) { desc = "fence remote node (default)"; } else { desc = "recover remote node connection (default)"; } on_fail = pcmk_on_fail_reset_remote; } else { on_fail = pcmk_on_fail_stop; desc = "stop unmanaged remote node (enforcing default)"; } } else if (pcmk__str_eq(action_name, PCMK_ACTION_STOP, pcmk__str_none)) { if (pcmk_is_set(rsc->cluster->flags, pcmk_sched_fencing_enabled)) { on_fail = pcmk_on_fail_fence_node; desc = "resource fence (default)"; } else { on_fail = pcmk_on_fail_block; desc = "resource block (default)"; } } else { on_fail = pcmk_on_fail_restart; desc = "restart (and possibly migrate) (default)"; } pcmk__rsc_trace(rsc, "Failure handling for %s-interval %s of %s: %s", pcmk__readable_interval(interval_ms), action_name, rsc->id, desc); return on_fail; } /*! * \internal * \brief Determine a resource's role after failure of an action * * \param[in] rsc Resource that action is for * \param[in] action_name Action name * \param[in] on_fail Failure handling for action * \param[in] meta Unpacked action meta-attributes * * \return Resource role that results from failure of action */ enum rsc_role_e pcmk__role_after_failure(const pcmk_resource_t *rsc, const char *action_name, enum action_fail_response on_fail, GHashTable *meta) { const char *value = NULL; enum rsc_role_e role = pcmk_role_unknown; // Set default for role after failure specially in certain circumstances switch (on_fail) { case pcmk_on_fail_stop: role = pcmk_role_stopped; break; case pcmk_on_fail_reset_remote: if (rsc->remote_reconnect_ms != 0) { role = pcmk_role_stopped; } break; default: break; } // @COMPAT Check for explicitly configured role (deprecated) value = g_hash_table_lookup(meta, PCMK__META_ROLE_AFTER_FAILURE); if (value != NULL) { pcmk__warn_once(pcmk__wo_role_after, "Support for " PCMK__META_ROLE_AFTER_FAILURE " is " "deprecated and will be removed in a future release"); if (role == pcmk_role_unknown) { role = text2role(value); if (role == pcmk_role_unknown) { pcmk__config_err("Ignoring invalid value %s for " PCMK__META_ROLE_AFTER_FAILURE, value); } } } if (role == pcmk_role_unknown) { // Use default if (pcmk__str_eq(action_name, PCMK_ACTION_PROMOTE, pcmk__str_none)) { role = pcmk_role_unpromoted; } else { role = pcmk_role_started; } } pcmk__rsc_trace(rsc, "Role after %s %s failure is: %s", rsc->id, action_name, role2text(role)); return role; } /*! * \internal * \brief Unpack action configuration * * Unpack a resource action's meta-attributes (normalizing the interval, * timeout, and start delay values as integer milliseconds), requirements, and * failure policy from its CIB XML configuration (including defaults). * * \param[in,out] action Resource action to unpack into * \param[in] xml_obj Action configuration XML (NULL for defaults only) * \param[in] interval_ms How frequently to perform the operation */ static void unpack_operation(pcmk_action_t *action, const xmlNode *xml_obj, guint interval_ms) { const char *value = NULL; action->meta = pcmk__unpack_action_meta(action->rsc, action->node, action->task, interval_ms, xml_obj); action->needs = pcmk__action_requires(action->rsc, action->task); value = g_hash_table_lookup(action->meta, PCMK_META_ON_FAIL); action->on_fail = pcmk__parse_on_fail(action->rsc, action->task, interval_ms, value); action->fail_role = pcmk__role_after_failure(action->rsc, action->task, action->on_fail, action->meta); } /*! * \brief Create or update an action object * * \param[in,out] rsc Resource that action is for (if any) * \param[in,out] key Action key (must be non-NULL) * \param[in] task Action name (must be non-NULL) * \param[in] on_node Node that action is on (if any) * \param[in] optional Whether action should be considered optional * \param[in,out] scheduler Scheduler data * * \return Action object corresponding to arguments (guaranteed not to be * \c NULL) * \note This function takes ownership of (and might free) \p key, and * \p scheduler takes ownership of the returned action (the caller should * not free it). */ pcmk_action_t * custom_action(pcmk_resource_t *rsc, char *key, const char *task, const pcmk_node_t *on_node, gboolean optional, pcmk_scheduler_t *scheduler) { pcmk_action_t *action = NULL; CRM_ASSERT((key != NULL) && (task != NULL) && (scheduler != NULL)); action = find_existing_action(key, rsc, on_node, scheduler); if (action == NULL) { action = new_action(key, task, rsc, on_node, optional, scheduler); } else { free(key); } update_action_optional(action, optional); if (rsc != NULL) { if ((action->node != NULL) && (action->op_entry != NULL) && !pcmk_is_set(action->flags, pcmk_action_attrs_evaluated)) { GHashTable *attrs = action->node->details->attrs; if (action->extra != NULL) { g_hash_table_destroy(action->extra); } action->extra = pcmk__unpack_action_rsc_params(action->op_entry, attrs, scheduler); pcmk__set_action_flags(action, pcmk_action_attrs_evaluated); } update_resource_action_runnable(action, scheduler); update_resource_flags_for_action(rsc, action); } if (action->extra == NULL) { action->extra = pcmk__strkey_table(free, free); } return action; } pcmk_action_t * get_pseudo_op(const char *name, pcmk_scheduler_t *scheduler) { pcmk_action_t *op = lookup_singleton(scheduler, name); if (op == NULL) { op = custom_action(NULL, strdup(name), name, NULL, TRUE, scheduler); pcmk__set_action_flags(op, pcmk_action_pseudo|pcmk_action_runnable); } return op; } static GList * find_unfencing_devices(GList *candidates, GList *matches) { for (GList *gIter = candidates; gIter != NULL; gIter = gIter->next) { pcmk_resource_t *candidate = gIter->data; if (candidate->children != NULL) { matches = find_unfencing_devices(candidate->children, matches); } else if (!pcmk_is_set(candidate->flags, pcmk_rsc_fence_device)) { continue; } else if (pcmk_is_set(candidate->flags, pcmk_rsc_needs_unfencing)) { matches = g_list_prepend(matches, candidate); } else if (pcmk__str_eq(g_hash_table_lookup(candidate->meta, PCMK_STONITH_PROVIDES), PCMK__VALUE_UNFENCING, pcmk__str_casei)) { matches = g_list_prepend(matches, candidate); } } return matches; } static int node_priority_fencing_delay(const pcmk_node_t *node, const pcmk_scheduler_t *scheduler) { int member_count = 0; int online_count = 0; int top_priority = 0; int lowest_priority = 0; GList *gIter = NULL; // PCMK_OPT_PRIORITY_FENCING_DELAY is disabled if (scheduler->priority_fencing_delay <= 0) { return 0; } /* No need to request a delay if the fencing target is not a normal cluster * member, for example if it's a remote node or a guest node. */ if (node->details->type != pcmk_node_variant_cluster) { return 0; } // No need to request a delay if the fencing target is in our partition if (node->details->online) { return 0; } for (gIter = scheduler->nodes; gIter != NULL; gIter = gIter->next) { pcmk_node_t *n = gIter->data; if (n->details->type != pcmk_node_variant_cluster) { continue; } member_count ++; if (n->details->online) { online_count++; } if (member_count == 1 || n->details->priority > top_priority) { top_priority = n->details->priority; } if (member_count == 1 || n->details->priority < lowest_priority) { lowest_priority = n->details->priority; } } // No need to delay if we have more than half of the cluster members if (online_count > member_count / 2) { return 0; } /* All the nodes have equal priority. * Any configured corresponding `pcmk_delay_base/max` will be applied. */ if (lowest_priority == top_priority) { return 0; } if (node->details->priority < top_priority) { return 0; } return scheduler->priority_fencing_delay; } pcmk_action_t * pe_fence_op(pcmk_node_t *node, const char *op, bool optional, const char *reason, bool priority_delay, pcmk_scheduler_t *scheduler) { char *op_key = NULL; pcmk_action_t *stonith_op = NULL; if(op == NULL) { op = scheduler->stonith_action; } op_key = crm_strdup_printf("%s-%s-%s", PCMK_ACTION_STONITH, node->details->uname, op); stonith_op = lookup_singleton(scheduler, op_key); if(stonith_op == NULL) { stonith_op = custom_action(NULL, op_key, PCMK_ACTION_STONITH, node, TRUE, scheduler); add_hash_param(stonith_op->meta, PCMK__META_ON_NODE, node->details->uname); add_hash_param(stonith_op->meta, PCMK__META_ON_NODE_UUID, node->details->id); add_hash_param(stonith_op->meta, "stonith_action", op); if (pcmk_is_set(scheduler->flags, pcmk_sched_enable_unfencing)) { /* Extra work to detect device changes */ GString *digests_all = g_string_sized_new(1024); GString *digests_secure = g_string_sized_new(1024); GList *matches = find_unfencing_devices(scheduler->resources, NULL); char *key = NULL; char *value = NULL; for (GList *gIter = matches; gIter != NULL; gIter = gIter->next) { pcmk_resource_t *match = gIter->data; const char *agent = g_hash_table_lookup(match->meta, PCMK_XA_TYPE); pcmk__op_digest_t *data = NULL; data = pe__compare_fencing_digest(match, agent, node, scheduler); if (data->rc == pcmk__digest_mismatch) { optional = FALSE; crm_notice("Unfencing node %s because the definition of " "%s changed", pe__node_name(node), match->id); if (!pcmk__is_daemon && scheduler->priv != NULL) { pcmk__output_t *out = scheduler->priv; out->info(out, "notice: Unfencing node %s because the " "definition of %s changed", pe__node_name(node), match->id); } } pcmk__g_strcat(digests_all, match->id, ":", agent, ":", data->digest_all_calc, ",", NULL); pcmk__g_strcat(digests_secure, match->id, ":", agent, ":", data->digest_secure_calc, ",", NULL); } key = strdup(PCMK__META_DIGESTS_ALL); value = strdup((const char *) digests_all->str); CRM_ASSERT((key != NULL) && (value != NULL)); g_hash_table_insert(stonith_op->meta, key, value); g_string_free(digests_all, TRUE); key = strdup(PCMK__META_DIGESTS_SECURE); value = strdup((const char *) digests_secure->str); CRM_ASSERT((key != NULL) && (value != NULL)); g_hash_table_insert(stonith_op->meta, key, value); g_string_free(digests_secure, TRUE); } } else { free(op_key); } if (scheduler->priority_fencing_delay > 0 /* It's a suitable case where PCMK_OPT_PRIORITY_FENCING_DELAY * applies. At least add PCMK_OPT_PRIORITY_FENCING_DELAY field as * an indicator. */ && (priority_delay /* The priority delay needs to be recalculated if this function has * been called by schedule_fencing_and_shutdowns() after node * priority has already been calculated by native_add_running(). */ || g_hash_table_lookup(stonith_op->meta, PCMK_OPT_PRIORITY_FENCING_DELAY) != NULL)) { /* Add PCMK_OPT_PRIORITY_FENCING_DELAY to the fencing op even if * it's 0 for the targeting node. So that it takes precedence over * any possible `pcmk_delay_base/max`. */ char *delay_s = pcmk__itoa(node_priority_fencing_delay(node, scheduler)); g_hash_table_insert(stonith_op->meta, strdup(PCMK_OPT_PRIORITY_FENCING_DELAY), delay_s); } if(optional == FALSE && pe_can_fence(scheduler, node)) { pcmk__clear_action_flags(stonith_op, pcmk_action_optional); pe_action_set_reason(stonith_op, reason, false); } else if(reason && stonith_op->reason == NULL) { stonith_op->reason = strdup(reason); } return stonith_op; } void pe_free_action(pcmk_action_t *action) { if (action == NULL) { return; } g_list_free_full(action->actions_before, free); g_list_free_full(action->actions_after, free); if (action->extra) { g_hash_table_destroy(action->extra); } if (action->meta) { g_hash_table_destroy(action->meta); } free(action->cancel_task); free(action->reason); free(action->task); free(action->uuid); free(action->node); free(action); } int pe_get_configured_timeout(pcmk_resource_t *rsc, const char *action, pcmk_scheduler_t *scheduler) { xmlNode *child = NULL; GHashTable *action_meta = NULL; const char *timeout_spec = NULL; int timeout_ms = 0; pe_rule_eval_data_t rule_data = { .node_hash = NULL, .role = pcmk_role_unknown, .now = scheduler->now, .match_data = NULL, .rsc_data = NULL, .op_data = NULL }; for (child = first_named_child(rsc->ops_xml, PCMK_XE_OP); child != NULL; child = crm_next_same_xml(child)) { if (pcmk__str_eq(action, crm_element_value(child, PCMK_XA_NAME), pcmk__str_casei)) { timeout_spec = crm_element_value(child, PCMK_META_TIMEOUT); break; } } if (timeout_spec == NULL && scheduler->op_defaults) { action_meta = pcmk__strkey_table(free, free); pe__unpack_dataset_nvpairs(scheduler->op_defaults, XML_TAG_META_SETS, &rule_data, action_meta, NULL, FALSE, scheduler); timeout_spec = g_hash_table_lookup(action_meta, PCMK_META_TIMEOUT); } // @TODO check meta-attributes // @TODO maybe use min-interval monitor timeout as default for monitors timeout_ms = crm_get_msec(timeout_spec); if (timeout_ms < 0) { timeout_ms = PCMK_DEFAULT_ACTION_TIMEOUT_MS; } if (action_meta != NULL) { g_hash_table_destroy(action_meta); } return timeout_ms; } enum action_tasks get_complex_task(const pcmk_resource_t *rsc, const char *name) { enum action_tasks task = text2task(name); if ((rsc != NULL) && (rsc->variant == pcmk_rsc_variant_primitive)) { switch (task) { case pcmk_action_stopped: case pcmk_action_started: case pcmk_action_demoted: case pcmk_action_promoted: crm_trace("Folding %s back into its atomic counterpart for %s", name, rsc->id); --task; break; default: break; } } return task; } /*! * \internal * \brief Find first matching action in a list * * \param[in] input List of actions to search * \param[in] uuid If not NULL, action must have this UUID * \param[in] task If not NULL, action must have this action name * \param[in] on_node If not NULL, action must be on this node * * \return First action in list that matches criteria, or NULL if none */ pcmk_action_t * find_first_action(const GList *input, const char *uuid, const char *task, const pcmk_node_t *on_node) { CRM_CHECK(uuid || task, return NULL); for (const GList *gIter = input; gIter != NULL; gIter = gIter->next) { pcmk_action_t *action = (pcmk_action_t *) gIter->data; if (uuid != NULL && !pcmk__str_eq(uuid, action->uuid, pcmk__str_casei)) { continue; } else if (task != NULL && !pcmk__str_eq(task, action->task, pcmk__str_casei)) { continue; } else if (on_node == NULL) { return action; } else if (action->node == NULL) { continue; } else if (on_node->details == action->node->details) { return action; } } return NULL; } GList * find_actions(GList *input, const char *key, const pcmk_node_t *on_node) { GList *gIter = input; GList *result = NULL; CRM_CHECK(key != NULL, return NULL); for (; gIter != NULL; gIter = gIter->next) { pcmk_action_t *action = (pcmk_action_t *) gIter->data; if (!pcmk__str_eq(key, action->uuid, pcmk__str_casei)) { continue; } else if (on_node == NULL) { crm_trace("Action %s matches (ignoring node)", key); result = g_list_prepend(result, action); } else if (action->node == NULL) { crm_trace("Action %s matches (unallocated, assigning to %s)", key, pe__node_name(on_node)); action->node = pe__copy_node(on_node); result = g_list_prepend(result, action); } else if (on_node->details == action->node->details) { crm_trace("Action %s on %s matches", key, pe__node_name(on_node)); result = g_list_prepend(result, action); } } return result; } GList * find_actions_exact(GList *input, const char *key, const pcmk_node_t *on_node) { GList *result = NULL; CRM_CHECK(key != NULL, return NULL); if (on_node == NULL) { return NULL; } for (GList *gIter = input; gIter != NULL; gIter = gIter->next) { pcmk_action_t *action = (pcmk_action_t *) gIter->data; if ((action->node != NULL) && pcmk__str_eq(key, action->uuid, pcmk__str_casei) && pcmk__str_eq(on_node->details->id, action->node->details->id, pcmk__str_casei)) { crm_trace("Action %s on %s matches", key, pe__node_name(on_node)); result = g_list_prepend(result, action); } } return result; } /*! * \brief Find all actions of given type for a resource * * \param[in] rsc Resource to search * \param[in] node Find only actions scheduled on this node * \param[in] task Action name to search for * \param[in] require_node If TRUE, NULL node or action node will not match * * \return List of actions found (or NULL if none) * \note If node is not NULL and require_node is FALSE, matching actions * without a node will be assigned to node. */ GList * pe__resource_actions(const pcmk_resource_t *rsc, const pcmk_node_t *node, const char *task, bool require_node) { GList *result = NULL; char *key = pcmk__op_key(rsc->id, task, 0); if (require_node) { result = find_actions_exact(rsc->actions, key, node); } else { result = find_actions(rsc->actions, key, node); } free(key); return result; } /*! * \internal * \brief Create an action reason string based on the action itself * * \param[in] action Action to create reason string for * \param[in] flag Action flag that was cleared * * \return Newly allocated string suitable for use as action reason * \note It is the caller's responsibility to free() the result. */ char * pe__action2reason(const pcmk_action_t *action, enum pe_action_flags flag) { const char *change = NULL; switch (flag) { case pcmk_action_runnable: change = "unrunnable"; break; case pcmk_action_migratable: change = "unmigrateable"; break; case pcmk_action_optional: change = "required"; break; default: // Bug: caller passed unsupported flag CRM_CHECK(change != NULL, change = ""); break; } return crm_strdup_printf("%s%s%s %s", change, (action->rsc == NULL)? "" : " ", (action->rsc == NULL)? "" : action->rsc->id, action->task); } void pe_action_set_reason(pcmk_action_t *action, const char *reason, bool overwrite) { if (action->reason != NULL && overwrite) { pcmk__rsc_trace(action->rsc, "Changing %s reason from '%s' to '%s'", action->uuid, action->reason, pcmk__s(reason, "(none)")); } else if (action->reason == NULL) { pcmk__rsc_trace(action->rsc, "Set %s reason to '%s'", action->uuid, pcmk__s(reason, "(none)")); } else { // crm_assert(action->reason != NULL && !overwrite); return; } pcmk__str_update(&action->reason, reason); } /*! * \internal * \brief Create an action to clear a resource's history from CIB * * \param[in,out] rsc Resource to clear * \param[in] node Node to clear history on */ void pe__clear_resource_history(pcmk_resource_t *rsc, const pcmk_node_t *node) { CRM_ASSERT((rsc != NULL) && (node != NULL)); custom_action(rsc, pcmk__op_key(rsc->id, PCMK_ACTION_LRM_DELETE, 0), PCMK_ACTION_LRM_DELETE, node, FALSE, rsc->cluster); } #define sort_return(an_int, why) do { \ free(a_uuid); \ free(b_uuid); \ crm_trace("%s (%d) %c %s (%d) : %s", \ a_xml_id, a_call_id, an_int>0?'>':an_int<0?'<':'=', \ b_xml_id, b_call_id, why); \ return an_int; \ } while(0) int pe__is_newer_op(const xmlNode *xml_a, const xmlNode *xml_b, bool same_node_default) { int a_call_id = -1; int b_call_id = -1; char *a_uuid = NULL; char *b_uuid = NULL; const char *a_xml_id = crm_element_value(xml_a, PCMK_XA_ID); const char *b_xml_id = crm_element_value(xml_b, PCMK_XA_ID); const char *a_node = crm_element_value(xml_a, PCMK__META_ON_NODE); const char *b_node = crm_element_value(xml_b, PCMK__META_ON_NODE); bool same_node = true; /* @COMPAT The on_node attribute was added to last_failure as of 1.1.13 (via * 8b3ca1c) and the other entries as of 1.1.12 (via 0b07b5c). * * In case that any of the lrm_rsc_op entries doesn't have on_node * attribute, we need to explicitly tell whether the two operations are on * the same node. */ if (a_node == NULL || b_node == NULL) { same_node = same_node_default; } else { same_node = pcmk__str_eq(a_node, b_node, pcmk__str_casei); } if (same_node && pcmk__str_eq(a_xml_id, b_xml_id, pcmk__str_none)) { /* We have duplicate lrm_rsc_op entries in the status * section which is unlikely to be a good thing * - we can handle it easily enough, but we need to get * to the bottom of why it's happening. */ pcmk__config_err("Duplicate lrm_rsc_op entries named %s", a_xml_id); sort_return(0, "duplicate"); } crm_element_value_int(xml_a, XML_LRM_ATTR_CALLID, &a_call_id); crm_element_value_int(xml_b, XML_LRM_ATTR_CALLID, &b_call_id); if (a_call_id == -1 && b_call_id == -1) { /* both are pending ops so it doesn't matter since * stops are never pending */ sort_return(0, "pending"); } else if (same_node && a_call_id >= 0 && a_call_id < b_call_id) { sort_return(-1, "call id"); } else if (same_node && b_call_id >= 0 && a_call_id > b_call_id) { sort_return(1, "call id"); } else if (a_call_id >= 0 && b_call_id >= 0 && (!same_node || a_call_id == b_call_id)) { /* * The op and last_failed_op are the same * Order on last-rc-change */ time_t last_a = -1; time_t last_b = -1; crm_element_value_epoch(xml_a, XML_RSC_OP_LAST_CHANGE, &last_a); crm_element_value_epoch(xml_b, XML_RSC_OP_LAST_CHANGE, &last_b); crm_trace("rc-change: %lld vs %lld", (long long) last_a, (long long) last_b); if (last_a >= 0 && last_a < last_b) { sort_return(-1, "rc-change"); } else if (last_b >= 0 && last_a > last_b) { sort_return(1, "rc-change"); } sort_return(0, "rc-change"); } else { /* One of the inputs is a pending operation. * Attempt to use PCMK__XA_TRANSITION_MAGIC to determine its age relative * to the other. */ int a_id = -1; int b_id = -1; const char *a_magic = crm_element_value(xml_a, PCMK__XA_TRANSITION_MAGIC); const char *b_magic = crm_element_value(xml_b, PCMK__XA_TRANSITION_MAGIC); CRM_CHECK(a_magic != NULL && b_magic != NULL, sort_return(0, "No magic")); if (!decode_transition_magic(a_magic, &a_uuid, &a_id, NULL, NULL, NULL, NULL)) { sort_return(0, "bad magic a"); } if (!decode_transition_magic(b_magic, &b_uuid, &b_id, NULL, NULL, NULL, NULL)) { sort_return(0, "bad magic b"); } /* try to determine the relative age of the operation... * some pending operations (e.g. a start) may have been superseded * by a subsequent stop * * [a|b]_id == -1 means it's a shutdown operation and _always_ comes last */ if (!pcmk__str_eq(a_uuid, b_uuid, pcmk__str_casei) || a_id == b_id) { /* * some of the logic in here may be redundant... * * if the UUID from the TE doesn't match then one better * be a pending operation. * pending operations don't survive between elections and joins * because we query the LRM directly */ if (b_call_id == -1) { sort_return(-1, "transition + call"); } else if (a_call_id == -1) { sort_return(1, "transition + call"); } } else if ((a_id >= 0 && a_id < b_id) || b_id == -1) { sort_return(-1, "transition"); } else if ((b_id >= 0 && a_id > b_id) || a_id == -1) { sort_return(1, "transition"); } } /* we should never end up here */ CRM_CHECK(FALSE, sort_return(0, "default")); } gint sort_op_by_callid(gconstpointer a, gconstpointer b) { const xmlNode *xml_a = a; const xmlNode *xml_b = b; return pe__is_newer_op(xml_a, xml_b, true); } /*! * \internal * \brief Create a new pseudo-action for a resource * * \param[in,out] rsc Resource to create action for * \param[in] task Action name * \param[in] optional Whether action should be considered optional * \param[in] runnable Whethe action should be considered runnable * * \return New action object corresponding to arguments */ pcmk_action_t * pe__new_rsc_pseudo_action(pcmk_resource_t *rsc, const char *task, bool optional, bool runnable) { pcmk_action_t *action = NULL; CRM_ASSERT((rsc != NULL) && (task != NULL)); action = custom_action(rsc, pcmk__op_key(rsc->id, task, 0), task, NULL, optional, rsc->cluster); pcmk__set_action_flags(action, pcmk_action_pseudo); if (runnable) { pcmk__set_action_flags(action, pcmk_action_runnable); } return action; } /*! * \internal * \brief Add the expected result to an action * * \param[in,out] action Action to add expected result to * \param[in] expected_result Expected result to add * * \note This is more efficient than calling add_hash_param(). */ void pe__add_action_expected_result(pcmk_action_t *action, int expected_result) { char *name = NULL; CRM_ASSERT((action != NULL) && (action->meta != NULL)); - name = strdup(XML_ATTR_TE_TARGET_RC); + name = strdup(PCMK__META_OP_TARGET_RC); CRM_ASSERT (name != NULL); g_hash_table_insert(action->meta, name, pcmk__itoa(expected_result)); }