diff --git a/crmd/te_events.c b/crmd/te_events.c index 6475141bdd..6e4b96eba1 100644 --- a/crmd/te_events.c +++ b/crmd/te_events.c @@ -1,505 +1,506 @@ /* * Copyright (C) 2004 Andrew Beekhof * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public * License as published by the Free Software Foundation; either * version 2 of the License, or (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ #include #include #include #include #include #include #include #include #include #include char *failed_stop_offset = NULL; char *failed_start_offset = NULL; int match_graph_event(int action_id, xmlNode *event, const char *event_node, int op_status, int op_rc, int target_rc); gboolean fail_incompletable_actions(crm_graph_t *graph, const char *down_node) { const char *target = NULL; xmlNode *last_action = NULL; GListPtr gIter = NULL; GListPtr gIter2 = NULL; if(graph == NULL || graph->complete) { return FALSE; } gIter = graph->synapses; for(; gIter != NULL; gIter = gIter->next) { synapse_t *synapse = (synapse_t*)gIter->data; if (synapse->confirmed) { continue; } gIter2 = synapse->actions; for(; gIter2 != NULL; gIter2 = gIter2->next) { crm_action_t *action = (crm_action_t*)gIter2->data; if(action->type == action_type_pseudo || action->confirmed) { continue; } else if(action->type == action_type_crm) { const char *task = crm_element_value(action->xml, XML_LRM_ATTR_TASK); if(safe_str_eq(task, CRM_OP_FENCE)) { continue; } } target = crm_element_value(action->xml, XML_LRM_ATTR_TARGET_UUID); if(safe_str_eq(target, down_node)) { action->failed = TRUE; + synapse->failed = TRUE; last_action = action->xml; update_graph(graph, action); crm_notice("Action %d (%s) is scheduled for %s (offline)", action->id, ID(action->xml), down_node); } } } if(last_action != NULL) { crm_warn("Node %s shutdown resulted in un-runnable actions", down_node); abort_transition(INFINITY, tg_restart, "Node failure", last_action); return TRUE; } return FALSE; } static gboolean update_failcount(xmlNode *event, const char *event_node, int rc, int target_rc, gboolean do_update) { int interval = 0; char *task = NULL; char *rsc_id = NULL; char *attr_name = NULL; const char *value = NULL; const char *id = crm_element_value(event, XML_LRM_ATTR_TASK_KEY); const char *on_uname = get_uname(event_node); if(rc == 99) { /* this is an internal code for "we're busy, try again" */ return FALSE; } else if(rc == target_rc) { return FALSE; } if(failed_stop_offset == NULL) { failed_stop_offset = crm_strdup(INFINITY_S); } if(failed_start_offset == NULL) { failed_start_offset = crm_strdup(INFINITY_S); } CRM_CHECK(on_uname != NULL, return TRUE); CRM_CHECK(parse_op_key(id, &rsc_id, &task, &interval), crm_err("Couldn't parse: %s", ID(event)); goto bail); CRM_CHECK(task != NULL, goto bail); CRM_CHECK(rsc_id != NULL, goto bail); if(do_update || interval > 0) { do_update = TRUE; } else if(safe_str_eq(task, CRMD_ACTION_START)) { do_update = TRUE; value = failed_start_offset; } else if(safe_str_eq(task, CRMD_ACTION_STOP)) { do_update = TRUE; value = failed_stop_offset; } else if(safe_str_eq(task, CRMD_ACTION_STOP)) { do_update = TRUE; value = failed_stop_offset; } else if(safe_str_eq(task, CRMD_ACTION_PROMOTE)) { do_update = TRUE; } else if(safe_str_eq(task, CRMD_ACTION_DEMOTE)) { do_update = TRUE; } if(value == NULL || safe_str_neq(value, INFINITY_S)) { value = XML_NVPAIR_ATTR_VALUE"++"; } if(do_update) { char *now = crm_itoa(time(NULL)); crm_warn("Updating failcount for %s on %s after failed %s:" " rc=%d (update=%s, time=%s)", rsc_id, on_uname, task, rc, value, now); attr_name = crm_concat("fail-count", rsc_id, '-'); update_attrd(on_uname, attr_name, value, NULL); crm_free(attr_name); attr_name = crm_concat("last-failure", rsc_id, '-'); update_attrd(on_uname, attr_name, now, NULL); crm_free(attr_name); crm_free(now); } bail: crm_free(rsc_id); crm_free(task); return TRUE; } static int status_from_rc(crm_action_t *action, int orig_status, int rc, int target_rc) { int status = orig_status; if(target_rc == rc) { crm_debug_2("Target rc: == %d", rc); if(status != LRM_OP_DONE) { crm_debug_2("Re-mapping op status to" " LRM_OP_DONE for rc=%d", rc); status = LRM_OP_DONE; } } else { status = LRM_OP_ERROR; } /* 99 is the code we use for direct nack's */ if(rc != 99 && status != LRM_OP_DONE) { const char *task, *uname; task = crm_element_value(action->xml, XML_LRM_ATTR_TASK_KEY); uname = crm_element_value(action->xml, XML_LRM_ATTR_TARGET); crm_warn("Action %d (%s) on %s failed (target: %d vs. rc: %d): %s", action->id, task, uname, target_rc, rc, op_status2text(status)); } return status; } /* * returns the ID of the action if a match is found * returns -1 if a match was not found * returns -2 if a match was found but the action failed (and was * not allowed to) */ int match_graph_event(int action_id, xmlNode *event, const char *event_node, int op_status, int op_rc, int target_rc) { const char *target = NULL; const char *allow_fail = NULL; const char *this_event = NULL; crm_action_t *action = NULL; action = get_action(action_id, FALSE); if(action == NULL) { return -1; } op_status = status_from_rc(action, op_status, op_rc, target_rc); if(op_status != LRM_OP_DONE) { update_failcount(event, event_node, op_rc, target_rc, FALSE); } /* Process OP status */ switch(op_status) { case LRM_OP_PENDING: crm_debug("Ignoring pending operation"); return action->id; break; case LRM_OP_DONE: break; case LRM_OP_ERROR: case LRM_OP_TIMEOUT: case LRM_OP_NOTSUPPORTED: action->failed = TRUE; break; case LRM_OP_CANCELLED: /* do nothing?? */ crm_err("Dont know what to do for cancelled ops yet"); break; default: action->failed = TRUE; crm_err("Unsupported action result: %d", op_status); } /* stop this event's timer if it had one */ stop_te_timer(action->timer); action->confirmed = TRUE; update_graph(transition_graph, action); trigger_graph(); if(action->failed) { allow_fail = crm_meta_value(action->params, XML_ATTR_TE_ALLOWFAIL); if(crm_is_true(allow_fail)) { action->failed = FALSE; } } if(action->failed) { abort_transition(action->synapse->priority+1, tg_restart, "Event failed", event); } this_event = crm_element_value(event, XML_LRM_ATTR_TASK_KEY); target = crm_element_value(action->xml, XML_LRM_ATTR_TARGET); te_log_action(LOG_INFO, "Action %s (%d) confirmed on %s (rc=%d)", crm_str(this_event), action->id, crm_str(target), op_status); return action->id; } crm_action_t * get_action(int id, gboolean confirmed) { GListPtr gIter = NULL; GListPtr gIter2 = NULL; gIter = transition_graph->synapses; for(; gIter != NULL; gIter = gIter->next) { synapse_t *synapse = (synapse_t*) gIter->data; gIter2 = synapse->actions; for(; gIter2 != NULL; gIter2 = gIter2->next) { crm_action_t *action = (crm_action_t*)gIter2->data; if(action->id == id) { if(confirmed) { stop_te_timer(action->timer); action->confirmed = TRUE; } return action; } } } return NULL; } crm_action_t * get_cancel_action(const char *id, const char *node) { const char *task = NULL; const char *target = NULL; GListPtr gIter = NULL; GListPtr gIter2 = NULL; gIter = transition_graph->synapses; for(; gIter != NULL; gIter = gIter->next) { synapse_t *synapse = (synapse_t*) gIter->data; gIter2 = synapse->actions; for(; gIter2 != NULL; gIter2 = gIter2->next) { crm_action_t *action = (crm_action_t*)gIter2->data; task = crm_element_value(action->xml, XML_LRM_ATTR_TASK); if(safe_str_neq(CRMD_ACTION_CANCEL, task)) { continue; } task = crm_element_value(action->xml, XML_LRM_ATTR_TASK_KEY); if(safe_str_neq(task, id)) { continue; } target = crm_element_value(action->xml, XML_LRM_ATTR_TARGET_UUID); if(safe_str_neq(target, node)) { continue; } return action; } } return NULL; } crm_action_t * match_down_event(int id, const char *target, const char *filter) { const char *this_action = NULL; const char *this_node = NULL; crm_action_t *match = NULL; GListPtr gIter = NULL; GListPtr gIter2 = NULL; gIter = transition_graph->synapses; for(; gIter != NULL; gIter = gIter->next) { synapse_t *synapse = (synapse_t*)gIter->data; /* lookup event */ gIter2 = synapse->actions; for(; gIter2 != NULL; gIter2 = gIter2->next) { crm_action_t *action = (crm_action_t*)gIter2->data; if(id > 0 && action->id == id) { match = action; break; } this_action = crm_element_value( action->xml, XML_LRM_ATTR_TASK); if(action->type != action_type_crm) { continue; } else if(safe_str_eq(this_action, CRM_OP_LRM_REFRESH)){ continue; } else if(filter != NULL && safe_str_neq(this_action, filter)) { continue; } this_node = crm_element_value( action->xml, XML_LRM_ATTR_TARGET_UUID); if(this_node == NULL) { crm_log_xml_err(action->xml, "No node uuid"); } if(safe_str_neq(this_node, target)) { crm_debug("Action %d : Node mismatch: %s", action->id, this_node); continue; } match = action; id = action->id; break; } if(match != NULL) { /* stop this event's timer if it had one */ break; } } if(match != NULL) { /* stop this event's timer if it had one */ crm_debug("Match found for action %d: %s on %s", id, crm_element_value(match->xml, XML_LRM_ATTR_TASK_KEY), target); } else if(id > 0) { crm_err("No match for action %d", id); } else { crm_warn("No match for shutdown action on %s", target); } return match; } gboolean process_graph_event(xmlNode *event, const char *event_node) { int rc = -1; int status = -1; int action = -1; int target_rc = -1; int transition_num = -1; char *update_te_uuid = NULL; gboolean stop_early = FALSE; gboolean passed = FALSE; const char *id = NULL; const char *magic = NULL; CRM_ASSERT(event != NULL); id = crm_element_value(event, XML_LRM_ATTR_TASK_KEY); magic = crm_element_value(event, XML_ATTR_TRANSITION_MAGIC); if(magic == NULL) { /* non-change */ return FALSE; } CRM_CHECK(decode_transition_magic( magic, &update_te_uuid, &transition_num, &action, &status, &rc, &target_rc), crm_err("Invalid event %s detected", id); abort_transition(INFINITY, tg_restart,"Bad event", event); return FALSE; ); if(status == LRM_OP_PENDING) { goto bail; } if(transition_num == -1) { crm_err("Action %s (%s) initiated outside of a transition", id, magic); abort_transition(INFINITY, tg_restart,"Unexpected event",event); } else if(action < 0 || crm_str_eq(update_te_uuid, te_uuid, TRUE) == FALSE) { crm_info("Action %s/%d (%s) initiated by a different transitioner", id, action, magic); abort_transition(INFINITY, tg_restart,"Foreign event", event); stop_early = TRUE; /* This could be an lrm status refresh */ } else if(transition_graph->id != transition_num) { crm_info("Detected action %s from a different transition:" " %d vs. %d", id, transition_num, transition_graph->id); abort_transition(INFINITY, tg_restart,"Old event", event); stop_early = TRUE; /* This could be an lrm status refresh */ } else if(transition_graph->complete) { crm_info("Action %s arrived after a completed transition", id); abort_transition(INFINITY, tg_restart, "Inactive graph", event); } else if(match_graph_event( action, event, event_node, status, rc, target_rc) < 0) { crm_err("Unknown graph action %s", id); abort_transition(INFINITY, tg_restart, "Unknown event", event); } else { passed = TRUE; crm_debug_2("Processed update to %s: %s", id, magic); } if(passed == FALSE) { if(update_failcount(event, event_node, rc, target_rc, transition_num == -1)) { /* Turns out this wasn't an lrm status refresh update aferall */ stop_early = FALSE; } } bail: crm_free(update_te_uuid); return stop_early; } diff --git a/include/crm/transition.h b/include/crm/transition.h index 85e709186c..5ebf894fef 100644 --- a/include/crm/transition.h +++ b/include/crm/transition.h @@ -1,160 +1,161 @@ /* * Copyright (C) 2004 Andrew Beekhof * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public * License as published by the Free Software Foundation; either * version 2 of the License, or (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ #include #include #include typedef enum { action_type_pseudo, action_type_rsc, action_type_crm } action_type_e; typedef struct te_timer_s crm_action_timer_t; typedef struct synapse_s { int id; int priority; gboolean ready; + gboolean failed; gboolean executed; gboolean confirmed; GListPtr actions; /* crm_action_t* */ GListPtr inputs; /* crm_action_t* */ } synapse_t; typedef struct crm_action_s { int id; int timeout; int interval; GHashTable *params; action_type_e type; crm_action_timer_t *timer; synapse_t *synapse; gboolean sent_update; /* sent to the CIB */ gboolean executed; /* sent to the CRM */ gboolean confirmed; gboolean failed; gboolean can_fail; xmlNode *xml; } crm_action_t; enum timer_reason { timeout_action, timeout_action_warn, timeout_abort, }; struct te_timer_s { int source_id; int timeout; enum timer_reason reason; crm_action_t *action; }; /* order matters here */ enum transition_action { tg_done, tg_stop, tg_restart, tg_shutdown, }; typedef struct crm_graph_s { int id; char *source; int abort_priority; gboolean complete; const char *abort_reason; enum transition_action completion_action; int num_actions; int num_synapses; int batch_limit; int network_delay; int stonith_timeout; int transition_timeout; int fired; int pending; int skipped; int completed; int incomplete; GListPtr synapses; /* synpase_t* */ } crm_graph_t; typedef struct crm_graph_functions_s { gboolean (*pseudo)(crm_graph_t *graph, crm_action_t *action); gboolean (*rsc)(crm_graph_t *graph, crm_action_t *action); gboolean (*crmd)(crm_graph_t *graph, crm_action_t *action); gboolean (*stonith)(crm_graph_t *graph, crm_action_t *action); } crm_graph_functions_t; enum transition_status { transition_active, transition_pending, /* active but no actions performed this time */ transition_complete, transition_stopped, transition_terminated, transition_action_failed, transition_failed, }; extern void set_default_graph_functions(void); extern void set_graph_functions(crm_graph_functions_t *fns); extern crm_graph_t *unpack_graph(xmlNode *xml_graph, const char *reference); extern int run_graph(crm_graph_t *graph); extern gboolean update_graph(crm_graph_t *graph, crm_action_t *action); extern void destroy_graph(crm_graph_t *graph); extern const char *transition_status(enum transition_status state); extern void print_graph(unsigned int log_level, crm_graph_t *graph); extern void print_action( int log_level, const char *prefix, crm_action_t *action); extern void update_abort_priority( crm_graph_t *graph, int priority, enum transition_action action, const char *abort_reason); extern const char *actiontype2text(action_type_e type); #ifdef TESTING # define te_log_action(log_level, fmt, args...) { \ do_crm_log(log_level, fmt, ##args); \ fprintf(stderr, fmt"\n", ##args); \ } #else # define te_log_action(log_level, fmt, args...) do_crm_log(log_level, fmt, ##args) #endif #include extern lrm_op_t *convert_graph_action(xmlNode *resource, crm_action_t *action, int status, int rc); diff --git a/lib/transition/graph.c b/lib/transition/graph.c index 31d957e602..8a11c25d29 100644 --- a/lib/transition/graph.c +++ b/lib/transition/graph.c @@ -1,334 +1,338 @@ /* * Copyright (C) 2004 Andrew Beekhof * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ #include #include #include #include #include /* #include */ /* */ crm_graph_functions_t *graph_fns = NULL; static gboolean update_synapse_ready(synapse_t *synapse, int action_id) { GListPtr lpc = NULL; gboolean updates = FALSE; CRM_CHECK(synapse->executed == FALSE, return FALSE); CRM_CHECK(synapse->confirmed == FALSE, return FALSE); synapse->ready = TRUE; for(lpc = synapse->inputs; lpc != NULL; lpc = lpc->next) { crm_action_t *prereq = (crm_action_t*)lpc->data; crm_debug_3("Processing input %d", prereq->id); if(prereq->id == action_id) { crm_debug_2("Marking input %d of synapse %d confirmed", action_id, synapse->id); prereq->confirmed = TRUE; updates = TRUE; } else if(prereq->confirmed == FALSE) { synapse->ready = FALSE; } } if(updates) { crm_debug_2("Updated synapse %d", synapse->id); } return updates; } static gboolean update_synapse_confirmed(synapse_t *synapse, int action_id) { GListPtr lpc = NULL; gboolean updates = FALSE; gboolean is_confirmed = TRUE; CRM_CHECK(synapse->executed, return FALSE); CRM_CHECK(synapse->confirmed == FALSE, return TRUE); is_confirmed = TRUE; for(lpc = synapse->actions; lpc != NULL; lpc = lpc->next) { crm_action_t *action = (crm_action_t*)lpc->data; crm_debug_3("Processing action %d", action->id); if(action->id == action_id) { crm_debug_2("Confirmed: Action %d of Synapse %d", action_id, synapse->id); action->confirmed = TRUE; updates = TRUE; } else if(action->confirmed == FALSE) { is_confirmed = FALSE; crm_debug_3("Synapse %d still not confirmed after action %d", synapse->id, action_id); } } if(is_confirmed && synapse->confirmed == FALSE) { crm_debug_2("Confirmed: Synapse %d", synapse->id); synapse->confirmed = TRUE; updates = TRUE; } if(updates) { crm_debug_3("Updated synapse %d", synapse->id); } return updates; } gboolean update_graph(crm_graph_t *graph, crm_action_t *action) { gboolean rc = FALSE; gboolean updates = FALSE; GListPtr lpc = NULL; for(lpc = graph->synapses; lpc != NULL; lpc = lpc->next) { synapse_t *synapse = (synapse_t*)lpc->data; - if (synapse->confirmed) { + if (synapse->confirmed || synapse->failed) { crm_debug_2("Synapse complete"); } else if (synapse->executed) { crm_debug_2("Synapse executed"); rc = update_synapse_confirmed(synapse, action->id); } else if(action->failed == FALSE || synapse->priority == INFINITY) { rc = update_synapse_ready(synapse, action->id); } updates = updates || rc; } if(updates) { crm_debug_2("Updated graph with completed action %d", action->id); } return updates; } static gboolean should_fire_synapse(synapse_t *synapse) { GListPtr lpc = NULL; CRM_CHECK(synapse->executed == FALSE, return FALSE); CRM_CHECK(synapse->confirmed == FALSE, return FALSE); crm_debug_3("Checking pre-reqs for %d", synapse->id); /* lookup prereqs */ synapse->ready = TRUE; for(lpc = synapse->inputs; lpc != NULL; lpc = lpc->next) { crm_action_t *prereq = (crm_action_t*)lpc->data; crm_debug_3("Processing input %d", prereq->id); if(prereq->confirmed == FALSE) { crm_debug_3("Inputs for synapse %d not satisfied", synapse->id); synapse->ready = FALSE; break; } } return synapse->ready; } static gboolean initiate_action(crm_graph_t *graph, crm_action_t *action) { const char *id = NULL; CRM_CHECK(action->executed == FALSE, return FALSE); id = ID(action->xml); CRM_CHECK(id != NULL, return FALSE); action->executed = TRUE; if(action->type == action_type_pseudo){ crm_debug_2("Executing pseudo-event: %d", action->id); return graph_fns->pseudo(graph, action); } else if(action->type == action_type_rsc) { crm_debug_2("Executing rsc-event: %d", action->id); return graph_fns->rsc(graph, action); } else if(action->type == action_type_crm) { const char *task = NULL; task = crm_element_value(action->xml, XML_LRM_ATTR_TASK); CRM_CHECK(task != NULL, return FALSE); if(safe_str_eq(task, CRM_OP_FENCE)) { crm_debug_2("Executing STONITH-event: %d", action->id); return graph_fns->stonith(graph, action); } crm_debug_2("Executing crm-event: %d", action->id); return graph_fns->crmd(graph, action); } te_log_action(LOG_ERR, "Failed on unsupported command type: %s (id=%s)", crm_element_name(action->xml), id); return FALSE; } static gboolean fire_synapse(crm_graph_t *graph, synapse_t *synapse) { GListPtr lpc = NULL; CRM_CHECK(synapse != NULL, return FALSE); CRM_CHECK(synapse->ready, return FALSE); CRM_CHECK(synapse->confirmed == FALSE, return TRUE); crm_debug_2("Synapse %d fired", synapse->id); synapse->executed = TRUE; for(lpc = synapse->actions; lpc != NULL; lpc = lpc->next) { crm_action_t *action = (crm_action_t*)lpc->data; /* allow some leeway */ gboolean passed = FALSE; /* Invoke the action and start the timer */ passed = initiate_action(graph, action); if(passed == FALSE) { crm_err("Failed initiating <%s id=%d> in synapse %d", crm_element_name(action->xml), action->id, synapse->id); synapse->confirmed = TRUE; action->confirmed = TRUE; action->failed = TRUE; return FALSE; } } return TRUE; } int run_graph(crm_graph_t *graph) { GListPtr lpc = NULL; int stat_log_level = LOG_DEBUG; int pass_result = transition_active; const char *status = "In-progress"; if(graph_fns == NULL) { set_default_graph_functions(); } if(graph == NULL) { return transition_complete; } graph->fired = 0; graph->pending = 0; graph->skipped = 0; graph->completed = 0; graph->incomplete = 0; crm_debug_2("Entering graph %d callback", graph->id); /* Pre-calculate the number of completed and in-flight operations */ for(lpc = graph->synapses; lpc != NULL; lpc = lpc->next) { synapse_t *synapse = (synapse_t*)lpc->data; if (synapse->confirmed) { crm_debug_3("Synapse %d complete", synapse->id); graph->completed++; } else if(synapse->executed) { crm_debug_2("Synapse %d: confirmation pending", synapse->id); graph->pending++; } } /* Now check if there is work to do */ for(lpc = graph->synapses; lpc != NULL; lpc = lpc->next) { synapse_t *synapse = (synapse_t*)lpc->data; if(graph->batch_limit > 0 && graph->pending >= graph->batch_limit) { crm_debug("Throttling output: batch limit (%d) reached", graph->batch_limit); break; + } else if (synapse->failed) { + graph->skipped++; + continue; + } else if (synapse->confirmed || synapse->executed) { /* Already handled */ continue; } if(synapse->priority < graph->abort_priority) { crm_debug_2("Skipping synapse %d: aborting", synapse->id); graph->skipped++; } else if(should_fire_synapse(synapse)) { crm_debug_2("Synapse %d fired", synapse->id); graph->fired++; CRM_CHECK(fire_synapse(graph, synapse), stat_log_level = LOG_ERR; graph->abort_priority = INFINITY; graph->incomplete++; graph->fired--); if (synapse->confirmed == FALSE) { graph->pending++; } } else { crm_debug_2("Synapse %d cannot fire", synapse->id); graph->incomplete++; } } if(graph->pending == 0 && graph->fired == 0) { graph->complete = TRUE; stat_log_level = LOG_NOTICE; pass_result = transition_complete; status = "Complete"; if(graph->incomplete != 0 && graph->abort_priority <= 0) { stat_log_level = LOG_WARNING; pass_result = transition_terminated; status = "Terminated"; } else if(graph->skipped != 0) { status = "Stopped"; } } else if(graph->fired == 0) { pass_result = transition_pending; } do_crm_log(stat_log_level+1, "===================================================="); do_crm_log(stat_log_level, "Transition %d (Complete=%d, Pending=%d," " Fired=%d, Skipped=%d, Incomplete=%d, Source=%s): %s", graph->id, graph->completed, graph->pending, graph->fired, graph->skipped, graph->incomplete, graph->source, status); return pass_result; }