diff --git a/crmd/te_utils.c b/crmd/te_utils.c index 47880dd58a..00ea3d93ba 100644 --- a/crmd/te_utils.c +++ b/crmd/te_utils.c @@ -1,497 +1,528 @@ /* * Copyright (C) 2004 Andrew Beekhof * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public * License as published by the Free Software Foundation; either * version 2 of the License, or (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ #include #include #include #include #include #include #include #include #include #include crm_trigger_t *stonith_reconnect = NULL; GListPtr stonith_cleanup_list = NULL; static gboolean fail_incompletable_stonith(crm_graph_t * graph) { GListPtr lpc = NULL; const char *task = NULL; xmlNode *last_action = NULL; if (graph == NULL) { return FALSE; } for (lpc = graph->synapses; lpc != NULL; lpc = lpc->next) { GListPtr lpc2 = NULL; synapse_t *synapse = (synapse_t *) lpc->data; if (synapse->confirmed) { continue; } for (lpc2 = synapse->actions; lpc2 != NULL; lpc2 = lpc2->next) { crm_action_t *action = (crm_action_t *) lpc2->data; if (action->type != action_type_crm || action->confirmed) { continue; } task = crm_element_value(action->xml, XML_LRM_ATTR_TASK); if (task && safe_str_eq(task, CRM_OP_FENCE)) { action->failed = TRUE; last_action = action->xml; update_graph(graph, action); crm_notice("Failing action %d (%s): STONITHd terminated", action->id, ID(action->xml)); } } } if (last_action != NULL) { crm_warn("STONITHd failure resulted in un-runnable actions"); abort_transition(INFINITY, tg_restart, "Stonith failure", last_action); return TRUE; } return FALSE; } static void tengine_stonith_connection_destroy(stonith_t * st, stonith_event_t * e) { if (is_set(fsa_input_register, R_ST_REQUIRED)) { crm_crit("Fencing daemon connection failed"); mainloop_set_trigger(stonith_reconnect); } else { crm_info("Fencing daemon disconnected"); } /* cbchan will be garbage at this point, arrange for it to be reset */ if(stonith_api) { stonith_api->state = stonith_disconnected; } if (AM_I_DC) { fail_incompletable_stonith(transition_graph); trigger_graph(); } } #if SUPPORT_CMAN # include #endif char *te_client_id = NULL; #ifdef HAVE_SYS_REBOOT_H # include # include #endif static void tengine_stonith_notify(stonith_t * st, stonith_event_t * st_event) { if(te_client_id == NULL) { te_client_id = g_strdup_printf("%s.%d", crm_system_name, getpid()); } if (st_event == NULL) { crm_err("Notify data not found"); return; } if (st_event->result == pcmk_ok && crm_str_eq(st_event->target, fsa_our_uname, TRUE)) { crm_crit("We were alegedly just fenced by %s for %s with %s!", st_event->executioner, st_event->origin, st_event->device); /* Dumps blackbox if enabled */ qb_log_fini(); /* Try to get the above log message to disk - somehow */ /* Get out ASAP and do not come back up. * * Triggering a reboot is also not the worst idea either since * the rest of the cluster thinks we're safely down */ #ifdef RB_HALT_SYSTEM reboot(RB_HALT_SYSTEM); #endif /* * If reboot() fails or is not supported, coming back up will * probably lead to a situation where the other nodes set our * status to 'lost' because of the fencing callback and will * discard subsequent election votes with: * * Election 87 (current: 5171, owner: 103): Processed vote from east-03 (Peer is not part of our cluster) * * So just stay dead, something is seriously messed up anyway. * */ exit(100); /* None of our wrappers since we already called qb_log_fini() */ return; } if (st_event->result == pcmk_ok && safe_str_eq(st_event->operation, T_STONITH_NOTIFY_FENCE)) { st_fail_count_reset(st_event->target); } crm_notice("Peer %s was%s terminated (%s) by %s for %s: %s (ref=%s) by client %s", st_event->target, st_event->result == pcmk_ok ? "" : " not", st_event->action, st_event->executioner ? st_event->executioner : "", st_event->origin, pcmk_strerror(st_event->result), st_event->id, st_event->client_origin ? st_event->client_origin : ""); #if SUPPORT_CMAN if (st_event->result == pcmk_ok && is_cman_cluster()) { int local_rc = 0; int confirm = 0; char *target_copy = strdup(st_event->target); /* In case fenced hasn't noticed yet * * Any fencing that has been inititated will be completed by way of the fence_pcmk redirect */ local_rc = fenced_external(target_copy); if (local_rc != 0) { crm_err("Could not notify CMAN that '%s' is now fenced: %d", st_event->target, local_rc); } else { crm_notice("Notified CMAN that '%s' is now fenced", st_event->target); } /* In case fenced is already trying to shoot it */ confirm = open("/var/run/cluster/fenced_override", O_NONBLOCK|O_WRONLY); if (confirm) { int ignore = 0; int len = strlen(target_copy); errno = 0; local_rc = write(confirm, target_copy, len); ignore = write(confirm, "\n", 1); if(ignore < 0 && errno == EBADF) { crm_trace("CMAN not expecting %s to be fenced (yet)", st_event->target); } else if (local_rc < len) { crm_perror(LOG_ERR, "Confirmation of CMAN fencing event for '%s' failed: %d", st_event->target, local_rc); } else { fsync(confirm); crm_notice("Confirmed CMAN fencing event for '%s'", st_event->target); } close(confirm); } free(target_copy); } #endif if (st_event->result == pcmk_ok) { crm_node_t *peer = crm_find_peer_full(0, st_event->target, CRM_GET_PEER_REMOTE | CRM_GET_PEER_CLUSTER); const char *uuid = NULL; gboolean we_are_executioner = safe_str_eq(st_event->executioner, fsa_our_uname); if (peer == NULL) { return; } uuid = crm_peer_uuid(peer); crm_trace("target=%s dc=%s", st_event->target, fsa_our_dc); if(AM_I_DC) { /* The DC always sends updates */ send_stonith_update(NULL, st_event->target, uuid); if (st_event->client_origin && safe_str_neq(st_event->client_origin, te_client_id)) { /* Abort the current transition graph if it wasn't us * that invoked stonith to fence someone */ crm_info("External fencing operation from %s fenced %s", st_event->client_origin, st_event->target); abort_transition(INFINITY, tg_restart, "External Fencing Operation", NULL); } /* Assume it was our leader if we dont currently have one */ } else if (fsa_our_dc == NULL || safe_str_eq(fsa_our_dc, st_event->target)) { crm_notice("Target %s our leader %s (recorded: %s)", fsa_our_dc ? "was" : "may have been", st_event->target, fsa_our_dc ? fsa_our_dc : ""); /* Given the CIB resyncing that occurs around elections, * have one node update the CIB now and, if the new DC is different, * have them do so too after the election */ if (we_are_executioner) { send_stonith_update(NULL, st_event->target, uuid); } stonith_cleanup_list = g_list_append(stonith_cleanup_list, strdup(st_event->target)); } crmd_peer_down(peer, TRUE); } } gboolean te_connect_stonith(gpointer user_data) { int lpc = 0; int rc = pcmk_ok; if (stonith_api == NULL) { stonith_api = stonith_api_new(); } if (stonith_api->state != stonith_disconnected) { crm_trace("Still connected"); return TRUE; } for (lpc = 0; lpc < 30; lpc++) { crm_debug("Attempting connection to fencing daemon..."); sleep(1); rc = stonith_api->cmds->connect(stonith_api, crm_system_name, NULL); if (rc == pcmk_ok) { break; } if (user_data != NULL) { crm_err("Sign-in failed: triggered a retry"); mainloop_set_trigger(stonith_reconnect); return TRUE; } crm_err("Sign-in failed: pausing and trying again in 2s..."); sleep(1); } CRM_CHECK(rc == pcmk_ok, return TRUE); /* If not, we failed 30 times... just get out */ stonith_api->cmds->register_notification(stonith_api, T_STONITH_NOTIFY_DISCONNECT, tengine_stonith_connection_destroy); stonith_api->cmds->register_notification(stonith_api, T_STONITH_NOTIFY_FENCE, tengine_stonith_notify); crm_trace("Connected"); return TRUE; } gboolean stop_te_timer(crm_action_timer_t * timer) { const char *timer_desc = "action timer"; if (timer == NULL) { return FALSE; } if (timer->reason == timeout_abort) { timer_desc = "global timer"; crm_trace("Stopping %s", timer_desc); } if (timer->source_id != 0) { crm_trace("Stopping %s", timer_desc); g_source_remove(timer->source_id); timer->source_id = 0; } else { crm_trace("%s was already stopped", timer_desc); return FALSE; } return TRUE; } gboolean te_graph_trigger(gpointer user_data) { enum transition_status graph_rc = -1; if (transition_graph == NULL) { crm_debug("Nothing to do"); return TRUE; } crm_trace("Invoking graph %d in state %s", transition_graph->id, fsa_state2string(fsa_state)); switch (fsa_state) { case S_STARTING: case S_PENDING: case S_NOT_DC: case S_HALT: case S_ILLEGAL: case S_STOPPING: case S_TERMINATE: return TRUE; break; default: break; } if (transition_graph->complete == FALSE) { int limit = transition_graph->batch_limit; transition_graph->batch_limit = throttle_get_total_job_limit(limit); graph_rc = run_graph(transition_graph); transition_graph->batch_limit = limit; /* Restore the configured value */ print_graph(LOG_DEBUG_3, transition_graph); if (graph_rc == transition_active) { crm_trace("Transition not yet complete"); return TRUE; } else if (graph_rc == transition_pending) { crm_trace("Transition not yet complete - no actions fired"); return TRUE; } if (graph_rc != transition_complete) { crm_warn("Transition failed: %s", transition_status(graph_rc)); print_graph(LOG_NOTICE, transition_graph); } } crm_debug("Transition %d is now complete", transition_graph->id); transition_graph->complete = TRUE; notify_crmd(transition_graph); return TRUE; } void trigger_graph_processing(const char *fn, int line) { crm_trace("%s:%d - Triggered graph processing", fn, line); mainloop_set_trigger(transition_trigger); } void abort_transition_graph(int abort_priority, enum transition_action abort_action, const char *abort_text, xmlNode * reason, const char *fn, int line) { - const char *magic = NULL; + int add[3]; + int del[3]; + int level = LOG_INFO; + xmlNode *diff = NULL; + xmlNode *change = NULL; CRM_CHECK(transition_graph != NULL, return); - if (reason) { - int diff_add_updates = 0; - int diff_add_epoch = 0; - int diff_add_admin_epoch = 0; - - int diff_del_updates = 0; - int diff_del_epoch = 0; - int diff_del_admin_epoch = 0; - - const char *uname = ""; - xmlNode *search = reason; - xmlNode *diff = get_xpath_object("//" F_CIB_UPDATE_RESULT "//diff", reason, LOG_DEBUG_2); - - magic = crm_element_value(reason, XML_ATTR_TRANSITION_MAGIC); - - while(search) { - const char *kind = TYPE(search); - - if (safe_str_eq(XML_CIB_TAG_STATE, kind) - || safe_str_eq(XML_CIB_TAG_NODE, kind)) { - - uname = crm_peer_uname(ID(search)); - break; - } - search = search->parent; - } - - if (diff) { - cib_diff_version_details(diff, - &diff_add_admin_epoch, &diff_add_epoch, &diff_add_updates, - &diff_del_admin_epoch, &diff_del_epoch, &diff_del_updates); - if (crm_str_eq(TYPE(reason), XML_CIB_TAG_NVPAIR, TRUE)) { - crm_info - ("%s:%d - Triggered transition abort (complete=%d, node=%s, tag=%s, id=%s, name=%s, value=%s, magic=%s, cib=%d.%d.%d) : %s", - fn, line, transition_graph->complete, uname, TYPE(reason), ID(reason), NAME(reason), - VALUE(reason), magic ? magic : "NA", diff_add_admin_epoch, diff_add_epoch, - diff_add_updates, abort_text); - } else { - crm_info - ("%s:%d - Triggered transition abort (complete=%d, node=%s, tag=%s, id=%s, magic=%s, cib=%d.%d.%d) : %s", - fn, line, transition_graph->complete, uname, TYPE(reason), ID(reason), - magic ? magic : "NA", diff_add_admin_epoch, diff_add_epoch, diff_add_updates, - abort_text); - } - - } else { - crm_info - ("%s:%d - Triggered transition abort (complete=%d, node=%s, tag=%s, id=%s, magic=%s) : %s", - fn, line, transition_graph->complete, uname, TYPE(reason), ID(reason), - magic ? magic : "NA", abort_text); - } - - } else { - crm_info("%s:%d - Triggered transition abort (complete=%d) : %s", - fn, line, transition_graph->complete, abort_text); - } - switch (fsa_state) { case S_STARTING: case S_PENDING: case S_NOT_DC: case S_HALT: case S_ILLEGAL: case S_STOPPING: case S_TERMINATE: - crm_info("Abort suppressed: state=%s (complete=%d)", - fsa_state2string(fsa_state), transition_graph->complete); + crm_info("Abort %s suppressed: state=%s (complete=%d)", + abort_text, fsa_state2string(fsa_state), transition_graph->complete); return; default: break; } - if (magic == NULL && reason != NULL) { - crm_log_xml_debug(reason, "Cause"); - } - /* Make sure any queued calculations are discarded ASAP */ free(fsa_pe_ref); fsa_pe_ref = NULL; + if (transition_graph->complete == FALSE) { + if(update_abort_priority(transition_graph, abort_priority, abort_action, abort_text)) { + level = LOG_NOTICE; + } + } + + if(reason) { + xmlNode *search = NULL; + + for(search = reason; search; search = search->parent) { + if (safe_str_eq(XML_TAG_DIFF, TYPE(search))) { + diff = search; + break; + } + } + + if(diff) { + xml_patch_versions(diff, add, del); + for(search = reason; search; search = search->parent) { + if (safe_str_eq(XML_DIFF_CHANGE, TYPE(search))) { + change = search; + break; + } + } + } + } + + if(reason == NULL) { + do_crm_log(level, "Transition aborted: %s (source=%s:%d, %d)", + abort_text, fn, line, transition_graph->complete); + + } else if(change == NULL) { + char *local_path = xml_get_path(reason); + + do_crm_log(level, "Transition aborted by %s.%s: %s (cib=%d.%d.%d, source=%s:%d, path=%s, %d)", + TYPE(reason), ID(reason), abort_text, add[0], add[1], add[2], fn, line, local_path, transition_graph->complete); + free(local_path); + + } else { + const char *kind = NULL; + const char *op = crm_element_value(change, XML_DIFF_OP); + const char *path = crm_element_value(change, XML_DIFF_PATH); + + if(change == reason) { + if(strcmp(op, "create") == 0) { + reason = reason->children; + + } else if(strcmp(op, "modify") == 0) { + reason = first_named_child(reason, XML_DIFF_RESULT); + if(reason) { + reason = reason->children; + } + } + } + + kind = TYPE(reason); + if(strcmp(op, "delete") == 0) { + const char *shortpath = strrchr(path, '/'); + + do_crm_log(level, "Transition aborted by deletion of %s: %s (cib=%d.%d.%d, source=%s:%d, path=%s, %d)", + shortpath?shortpath:path, abort_text, add[0], add[1], add[2], fn, line, path, transition_graph->complete); + + } else if (safe_str_eq(XML_CIB_TAG_NVPAIR, kind)) { + do_crm_log(level, "Transition aborted by %s=%s '%s': %s (cib=%d.%d.%d, source=%s:%d, path=%s, %d)", + crm_element_value(reason, XML_NVPAIR_ATTR_NAME), + crm_element_value(reason, XML_NVPAIR_ATTR_VALUE), + op, abort_text, add[0], add[1], add[2], fn, line, path, transition_graph->complete); + + } else if (safe_str_eq(XML_LRM_TAG_RSC_OP, kind)) { + const char *magic = crm_element_value(reason, XML_ATTR_TRANSITION_MAGIC); + + do_crm_log(level, "Transition aborted by %s '%s' on %s: %s (magic=%s, cib=%d.%d.%d, source=%s:%d, %d)", + crm_element_value(reason, XML_LRM_ATTR_TASK_KEY), op, + crm_element_value(reason, XML_LRM_ATTR_TARGET), abort_text, + magic, add[0], add[1], add[2], fn, line, transition_graph->complete); + + } else if (safe_str_eq(XML_CIB_TAG_STATE, kind) + || safe_str_eq(XML_CIB_TAG_NODE, kind)) { + const char *uname = crm_peer_uname(ID(reason)); + + do_crm_log(level, "Transition aborted by %s '%s' on %s: %s (cib=%d.%d.%d, source=%s:%d, %d)", + kind, op, uname, abort_text, + add[0], add[1], add[2], fn, line, transition_graph->complete); + + } else { + do_crm_log(level, "Transition aborted by %s.%s '%s': %s (cib=%d.%d.%d, source=%s:%d, path=%s, %d)", + TYPE(reason), ID(reason), op?op:"change", abort_text, add[0], add[1], add[2], fn, line, path, transition_graph->complete); + } + } + if (transition_graph->complete) { if (transition_timer->period_ms > 0) { crm_timer_stop(transition_timer); crm_timer_start(transition_timer); } else { register_fsa_input(C_FSA_INTERNAL, I_PE_CALC, NULL); } return; } - update_abort_priority(transition_graph, abort_priority, abort_action, abort_text); - mainloop_set_trigger(transition_trigger); } diff --git a/include/crm/transition.h b/include/crm/transition.h index 4e96b3abb4..ef420207c9 100644 --- a/include/crm/transition.h +++ b/include/crm/transition.h @@ -1,146 +1,146 @@ /* * Copyright (C) 2004 Andrew Beekhof * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public * License as published by the Free Software Foundation; either * version 2 of the License, or (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ #include #include #include typedef enum { action_type_pseudo, action_type_rsc, action_type_crm } action_type_e; typedef struct te_timer_s crm_action_timer_t; typedef struct synapse_s { int id; int priority; gboolean ready; gboolean failed; gboolean executed; gboolean confirmed; GListPtr actions; /* crm_action_t* */ GListPtr inputs; /* crm_action_t* */ } synapse_t; typedef struct crm_action_s { int id; int timeout; int interval; GHashTable *params; action_type_e type; crm_action_timer_t *timer; synapse_t *synapse; gboolean sent_update; /* sent to the CIB */ gboolean executed; /* sent to the CRM */ gboolean confirmed; gboolean failed; gboolean can_fail; xmlNode *xml; } crm_action_t; enum timer_reason { timeout_action, timeout_action_warn, timeout_abort, }; struct te_timer_s { int source_id; int timeout; enum timer_reason reason; crm_action_t *action; }; /* order matters here */ enum transition_action { tg_done, tg_stop, tg_restart, tg_shutdown, }; typedef struct crm_graph_s { int id; char *source; int abort_priority; gboolean complete; const char *abort_reason; enum transition_action completion_action; int num_actions; int num_synapses; int batch_limit; int network_delay; int stonith_timeout; int transition_timeout; int fired; int pending; int skipped; int completed; int incomplete; GListPtr synapses; /* synpase_t* */ int migration_limit; } crm_graph_t; typedef struct crm_graph_functions_s { gboolean(*pseudo) (crm_graph_t * graph, crm_action_t * action); gboolean(*rsc) (crm_graph_t * graph, crm_action_t * action); gboolean(*crmd) (crm_graph_t * graph, crm_action_t * action); gboolean(*stonith) (crm_graph_t * graph, crm_action_t * action); gboolean(*allowed) (crm_graph_t * graph, crm_action_t * action); } crm_graph_functions_t; enum transition_status { transition_active, transition_pending, /* active but no actions performed this time */ transition_complete, transition_stopped, transition_terminated, transition_action_failed, transition_failed, }; void set_default_graph_functions(void); void set_graph_functions(crm_graph_functions_t * fns); crm_graph_t *unpack_graph(xmlNode * xml_graph, const char *reference); int run_graph(crm_graph_t * graph); gboolean update_graph(crm_graph_t * graph, crm_action_t * action); void destroy_graph(crm_graph_t * graph); const char *transition_status(enum transition_status state); void print_graph(unsigned int log_level, crm_graph_t * graph); void print_action(int log_level, const char *prefix, crm_action_t * action); -void update_abort_priority(crm_graph_t * graph, int priority, +bool update_abort_priority(crm_graph_t * graph, int priority, enum transition_action action, const char *abort_reason); const char *actiontype2text(action_type_e type); lrmd_event_data_t *convert_graph_action(xmlNode * resource, crm_action_t * action, int status, int rc); diff --git a/lib/transition/utils.c b/lib/transition/utils.c index 0b1a123b4d..c1b546778b 100644 --- a/lib/transition/utils.c +++ b/lib/transition/utils.c @@ -1,284 +1,290 @@ /* * Copyright (C) 2004 Andrew Beekhof * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ #include #include #include #include #include /* #include */ /* */ extern crm_graph_functions_t *graph_fns; static gboolean pseudo_action_dummy(crm_graph_t * graph, crm_action_t * action) { static int fail = -1; if (fail < 0) { char *fail_s = getenv("PE_fail"); if (fail_s) { fail = crm_int_helper(fail_s, NULL); } else { fail = 0; } } crm_trace("Dummy event handler: action %d executed", action->id); if (action->id == fail) { crm_err("Dummy event handler: pretending action %d failed", action->id); action->failed = TRUE; graph->abort_priority = INFINITY; } action->confirmed = TRUE; update_graph(graph, action); return TRUE; } crm_graph_functions_t default_fns = { pseudo_action_dummy, pseudo_action_dummy, pseudo_action_dummy, pseudo_action_dummy }; void set_default_graph_functions(void) { graph_fns = &default_fns; } void set_graph_functions(crm_graph_functions_t * fns) { crm_info("Setting custom graph functions"); graph_fns = fns; CRM_ASSERT(graph_fns != NULL); CRM_ASSERT(graph_fns->rsc != NULL); CRM_ASSERT(graph_fns->crmd != NULL); CRM_ASSERT(graph_fns->pseudo != NULL); CRM_ASSERT(graph_fns->stonith != NULL); } const char * transition_status(enum transition_status state) { switch (state) { case transition_active: return "active"; case transition_pending: return "pending"; case transition_complete: return "complete"; case transition_stopped: return "stopped"; case transition_terminated: return "terminated"; case transition_action_failed: return "failed (action)"; case transition_failed: return "failed"; } return "unknown"; } const char * actiontype2text(action_type_e type) { switch (type) { case action_type_pseudo: return "pseudo"; case action_type_rsc: return "rsc"; case action_type_crm: return "crm"; } return ""; } static crm_action_t * find_action(crm_graph_t * graph, int id) { GListPtr sIter = NULL; if (graph == NULL) { return NULL; } for (sIter = graph->synapses; sIter != NULL; sIter = sIter->next) { GListPtr aIter = NULL; synapse_t *synapse = (synapse_t *) sIter->data; for (aIter = synapse->actions; aIter != NULL; aIter = aIter->next) { crm_action_t *action = (crm_action_t *) aIter->data; if (action->id == id) { return action; } } } return NULL; } static void print_synapse(unsigned int log_level, crm_graph_t * graph, synapse_t * synapse) { GListPtr lpc = NULL; char *pending = NULL; const char *state = "Pending"; if (synapse->failed) { state = "Failed"; } else if (synapse->confirmed) { state = "Completed"; } else if (synapse->executed) { state = "In-flight"; } else if (synapse->ready) { state = "Ready"; } if (synapse->executed == FALSE) { for (lpc = synapse->inputs; lpc != NULL; lpc = lpc->next) { crm_action_t *input = (crm_action_t *) lpc->data; const char *id_string = crm_element_value(input->xml, XML_ATTR_ID); if (input->failed) { pending = add_list_element(pending, id_string); } else if (input->confirmed) { /* Confirmed, skip */ } else if (find_action(graph, input->id)) { /* In-flight or pending */ pending = add_list_element(pending, id_string); } } } for (lpc = synapse->actions; lpc != NULL; lpc = lpc->next) { crm_action_t *action = (crm_action_t *) lpc->data; const char *key = crm_element_value(action->xml, XML_LRM_ATTR_TASK_KEY); const char *host = crm_element_value(action->xml, XML_LRM_ATTR_TARGET); char *desc = g_strdup_printf("%s %s op %s", state, actiontype2text(action->type), key); do_crm_log(log_level, "[Action %4d]: %-50s on %s (priority: %d, waiting: %s)", action->id, desc, host ? host : "N/A", synapse->priority, pending ? pending : "none"); g_free(desc); } if (synapse->executed == FALSE) { for (lpc = synapse->inputs; lpc != NULL; lpc = lpc->next) { crm_action_t *input = (crm_action_t *) lpc->data; const char *key = crm_element_value(input->xml, XML_LRM_ATTR_TASK_KEY); const char *host = crm_element_value(input->xml, XML_LRM_ATTR_TARGET); if (find_action(graph, input->id) == NULL) { if (host == NULL) { do_crm_log(log_level, " * [Input %2d]: Unresolved dependancy %s op %s", input->id, actiontype2text(input->type), key); } else { do_crm_log(log_level, " * [Input %2d]: Unresolved dependancy %s op %s on %s", input->id, actiontype2text(input->type), key, host); } } } } free(pending); } void print_action(int log_level, const char *prefix, crm_action_t * action) { print_synapse(log_level, NULL, action->synapse); } void print_graph(unsigned int log_level, crm_graph_t * graph) { GListPtr lpc = NULL; if (graph == NULL || graph->num_actions == 0) { if (log_level > LOG_DEBUG) { crm_debug("Empty transition graph"); } return; } do_crm_log(log_level, "Graph %d with %d actions:" " batch-limit=%d jobs, network-delay=%dms", graph->id, graph->num_actions, graph->num_synapses, graph->batch_limit, graph->network_delay); for (lpc = graph->synapses; lpc != NULL; lpc = lpc->next) { synapse_t *synapse = (synapse_t *) lpc->data; print_synapse(log_level, graph, synapse); } } static const char * abort2text(enum transition_action abort_action) { switch (abort_action) { case tg_done: return "done"; case tg_stop: return "stop"; case tg_restart: return "restart"; case tg_shutdown: return "shutdown"; } return "unknown"; } -void +bool update_abort_priority(crm_graph_t * graph, int priority, enum transition_action action, const char *abort_reason) { + bool change = FALSE; + if (graph == NULL) { - return; + return change; } if (graph->abort_priority < priority) { crm_debug("Abort priority upgraded from %d to %d", graph->abort_priority, priority); graph->abort_priority = priority; if (graph->abort_reason != NULL) { - crm_debug("'%s' abort superceeded", graph->abort_reason); + crm_debug("'%s' abort superceeded by %s", graph->abort_reason, abort_reason); } graph->abort_reason = abort_reason; + change = TRUE; } if (graph->completion_action < action) { - crm_debug("Abort action %s superceeded by %s", - abort2text(graph->completion_action), abort2text(action)); + crm_debug("Abort action %s superceeded by %s: %s", + abort2text(graph->completion_action), abort2text(action), abort_reason); graph->completion_action = action; + change = TRUE; } + + return change; }