diff --git a/crm/crmd/fsa.c b/crm/crmd/fsa.c index 994f9667e2..2c78c169ce 100644 --- a/crm/crmd/fsa.c +++ b/crm/crmd/fsa.c @@ -1,1098 +1,1102 @@ /* * Copyright (C) 2004 Andrew Beekhof * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include long long do_state_transition(long long actions, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_state next_state, enum crmd_fsa_input current_input, void *data); #ifdef DOT_FSA_ACTIONS # ifdef FSA_TRACE # define IF_FSA_ACTION(x,y) \ if(is_set(actions,x)) { \ CRM_DEBUG("Invoking action %s (%.16llx)", \ fsa_action2string(x), x); \ last_action = x; \ actions = clear_bit(actions, x); \ next_input = y(x, cause, cur_state, last_input, data); \ if( (x & O_DC_TICKLE) == 0 && next_input != I_DC_HEARTBEAT ) \ fprintf(dot_strm, \ "\t// %s:\t%s\t(data? %s)\t(result=%s)\n", \ fsa_input2string(cur_input), \ fsa_action2string(x), \ data==NULL?"no":"yes", \ fsa_input2string(next_input)); \ fflush(dot_strm); \ CRM_DEBUG("Result of action %s was %s", \ fsa_action2string(x), fsa_input2string(next_input)); \ } # else # define IF_FSA_ACTION(x,y) \ if(is_set(actions,x)) { \ last_action = x; \ actions = clear_bit(actions, x); \ next_input = y(x, cause, cur_state, last_input, data); \ if( (x & O_DC_TICKLE) == 0 && next_input != I_DC_HEARTBEAT ) \ fprintf(dot_strm, \ "\t// %s:\t%s\t(data? %s)\t(result=%s)\n", \ fsa_input2string(cur_input), \ fsa_action2string(x), \ data==NULL?"no":"yes", \ fsa_input2string(next_input)); \ fflush(dot_strm); \ } # endif #else # ifdef FSA_TRACE # define IF_FSA_ACTION(x,y) \ if(is_set(actions,x)) { \ CRM_DEBUG("Invoking action %s (%.16llx)", \ fsa_action2string(x), x); \ last_action = x; \ actions = clear_bit(actions, x); \ next_input = y(x, cause, cur_state, last_input, data); \ CRM_DEBUG("Result of action %s was %s", \ fsa_action2string(x), fsa_input2string(next_input)); \ } # else # define IF_FSA_ACTION(x,y) \ if(is_set(actions,x)) { \ last_action = x; \ actions = clear_bit(actions, x); \ next_input = y(x, cause, cur_state, last_input, data); \ } # endif #endif #define ELSEIF_FSA_ACTION(x,y) else IF_FSA_ACTION(x,y) const char *dot_intro = "digraph \"g\" {\n" " size = \"30,30\"\n" " graph [\n" " fontsize = \"12\"\n" " fontname = \"Times-Roman\"\n" " fontcolor = \"black\"\n" " bb = \"0,0,398.922306,478.927856\"\n" " color = \"black\"\n" " ]\n" " node [\n" " fontsize = \"12\"\n" " fontname = \"Times-Roman\"\n" " fontcolor = \"black\"\n" " shape = \"ellipse\"\n" " color = \"black\"\n" " ]\n" " edge [\n" " fontsize = \"12\"\n" " fontname = \"Times-Roman\"\n" " fontcolor = \"black\"\n" " color = \"black\"\n" " ]\n" "// special nodes\n" " \"S_PENDING\" \n" " [\n" " color = \"blue\"\n" " fontcolor = \"blue\"\n" " ]\n" " \"S_TERMINATE\" \n" " [\n" " color = \"red\"\n" " fontcolor = \"red\"\n" " ]\n" "\n" "// DC only nodes\n" " \"S_RECOVERY_DC\" [ fontcolor = \"green\" ]\n" " \"S_INTEGRATION\" [ fontcolor = \"green\" ]\n" " \"S_POLICY_ENGINE\" [ fontcolor = \"green\" ]\n" " \"S_TRANSITION_ENGINE\" [ fontcolor = \"green\" ]\n" " \"S_RELEASE_DC\" [ fontcolor = \"green\" ]\n" " \"S_IDLE\" [ fontcolor = \"green\" ]\n"; static FILE *dot_strm = NULL; enum crmd_fsa_state fsa_state; oc_node_list_t *fsa_membership_copy; ll_cluster_t *fsa_cluster_conn; ll_lrm_t *fsa_lrm_conn; long long fsa_input_register; long long fsa_actions = A_NOTHING; const char *fsa_our_uname; fsa_timer_t *election_trigger = NULL; /* */ fsa_timer_t *election_timeout = NULL; /* */ fsa_timer_t *shutdown_escalation_timmer = NULL; /* */ fsa_timer_t *integration_timer = NULL; fsa_timer_t *dc_heartbeat = NULL; long long toggle_bit(long long action_list, long long action) { // CRM_DEBUG("Toggling bit %.16llx", action); action_list ^= action; // CRM_DEBUG("Result %.16llx", action_list & action); return action_list; } long long clear_bit(long long action_list, long long action) { // CRM_DEBUG("Clearing bit\t%.16llx", action); // ensure its set action_list |= action; // then toggle action_list = action_list ^ action; return action_list; } long long set_bit(long long action_list, long long action) { // CRM_DEBUG("Adding bit\t%.16llx", action); action_list |= action; return action_list; } void toggle_bit_inplace(long long *action_list, long long action) { *action_list = toggle_bit(*action_list, action); } void clear_bit_inplace(long long *action_list, long long action) { *action_list = clear_bit(*action_list, action); } void set_bit_inplace(long long *action_list, long long action) { *action_list = set_bit(*action_list, action); } gboolean is_set(long long action_list, long long action) { // CRM_DEBUG("Checking bit\t%.16llx", action); return ((action_list & action) == action); } gboolean startTimer(fsa_timer_t *timer) { if(((int)timer->source_id) < 0) { timer->source_id = Gmain_timeout_add(timer->period_ms, timer->callback, (void*)timer); /* CRM_DEBUG("#!!#!!# Started %s timer (%d)", fsa_input2string(timer->fsa_input), timer->source_id); */ } else { cl_log(LOG_INFO, "#!!#!!# Timer %s already running (%d)", fsa_input2string(timer->fsa_input), timer->source_id); return FALSE; } return TRUE; } gboolean stopTimer(fsa_timer_t *timer) { if(((int)timer->source_id) > 0) { /* CRM_DEBUG("#!!#!!# Stopping %s timer (%d)", fsa_input2string(timer->fsa_input), timer->source_id); */ g_source_remove(timer->source_id); timer->source_id = -2; } else { cl_log(LOG_INFO, "#!!#!!# Timer %s already stopped (%d)", fsa_input2string(timer->fsa_input), timer->source_id); return FALSE; } return TRUE; } enum crmd_fsa_state s_crmd_fsa(enum crmd_fsa_cause cause, enum crmd_fsa_input initial_input, void *data) { long long actions = fsa_actions; long long new_actions = A_NOTHING; long long last_action = A_NOTHING; enum crmd_fsa_input last_input = initial_input; enum crmd_fsa_input cur_input; enum crmd_fsa_input next_input; enum crmd_fsa_state last_state, cur_state, next_state, starting_state; FNIN(); starting_state = fsa_state; cur_input = initial_input; next_input = initial_input; last_state = starting_state; cur_state = starting_state; next_state = starting_state; #ifdef FSA_TRACE CRM_DEBUG("FSA invoked with Cause: %s\n\tState: %s, Input: %s", fsa_cause2string(cause), fsa_state2string(cur_state), fsa_input2string(cur_input)); #endif if(dot_strm == NULL) { dot_strm = fopen("/tmp/live.dot", "w"); fprintf(dot_strm, "%s", dot_intro); } /* * Process actions in order of priority but do only one * action at a time to avoid complicating the ordering. * * Actions may result in a new I_ event, these are added to * (not replace) existing actions before the next iteration. * */ while(next_input != I_NULL || actions != A_NOTHING) { if(next_input == I_WAIT_FOR_EVENT) { /* we may be waiting for an a-sync task to "happen" * and until it does, we cant do anything else * * Re-add the last action */ actions |= last_action; cl_log(LOG_INFO, "Wait until something else happens"); break; } #ifdef FSA_TRACE CRM_DEBUG("FSA while loop:\tState: %s, Input: %s", fsa_state2string(cur_state), fsa_input2string(cur_input)); #endif /* update input variables */ cur_input = next_input; if(cur_input != I_NULL) { last_input = cur_input; } /* get the next batch of actions */ new_actions = crmd_fsa_actions[cur_input][cur_state]; if(new_actions != A_NOTHING) { #ifdef FSA_TRACE CRM_DEBUG("Adding actions %.16llx", new_actions); #endif actions |= new_actions; } /* logging : *before* the state is changed */ IF_FSA_ACTION(A_ERROR, do_log) ELSEIF_FSA_ACTION(A_WARN, do_log) ELSEIF_FSA_ACTION(A_LOG, do_log) /* update state variables */ next_state = crmd_fsa_state[cur_input][cur_state]; last_state = cur_state; cur_state = next_state; fsa_state = next_state; /* start doing things... */ /* * Hook for change of state. * Allows actions to be added or removed when entering a state */ if(last_state != cur_state){ actions = do_state_transition(actions, cause, last_state, cur_state, last_input, data); } /* this is always run, some inputs/states may make various * actions irrelevant/invalid */ actions = clear_flags(actions, cause, cur_state, cur_input); /* regular action processing in order of action priority * * Make sure all actions that connect to required systems * are performed first */ if(actions == A_NOTHING) { cl_log(LOG_INFO, "Nothing to do"); next_input = I_NULL; /* // check registers, see if anything is pending if(is_set(fsa_input_register, R_SHUTDOWN)) { CRM_DEBUG("(Re-)invoking shutdown"); next_input = I_SHUTDOWN; } else if(is_set(fsa_input_register, R_INVOKE_PE)) { CRM_DEBUG("Invoke the PE somehow"); } */ } /* get out of here NOW! before anything worse happens */ ELSEIF_FSA_ACTION(A_EXIT_1, do_exit) ELSEIF_FSA_ACTION(A_STARTUP, do_startup) ELSEIF_FSA_ACTION(A_CIB_START, do_cib_control) ELSEIF_FSA_ACTION(A_HA_CONNECT, do_ha_control) ELSEIF_FSA_ACTION(A_LRM_CONNECT,do_lrm_control) ELSEIF_FSA_ACTION(A_CCM_CONNECT,do_ccm_control) ELSEIF_FSA_ACTION(A_ANNOUNCE, do_announce) /* sub-system start */ ELSEIF_FSA_ACTION(A_PE_START, do_pe_control) ELSEIF_FSA_ACTION(A_TE_START, do_te_control) /* sub-system restart */ ELSEIF_FSA_ACTION(O_CIB_RESTART,do_cib_control) ELSEIF_FSA_ACTION(O_PE_RESTART, do_pe_control) ELSEIF_FSA_ACTION(O_TE_RESTART, do_te_control) ELSEIF_FSA_ACTION(A_STARTED, do_started) /* DC Timer */ ELSEIF_FSA_ACTION(O_DC_TIMER_RESTART, do_dc_timer_control) ELSEIF_FSA_ACTION(A_DC_TIMER_STOP, do_dc_timer_control) ELSEIF_FSA_ACTION(A_DC_TIMER_START, do_dc_timer_control) /* * Highest priority actions */ ELSEIF_FSA_ACTION(A_TE_COPYTO, do_te_copyto) ELSEIF_FSA_ACTION(A_SHUTDOWN_REQ, do_shutdown_req) ELSEIF_FSA_ACTION(A_MSG_ROUTE, do_msg_route) ELSEIF_FSA_ACTION(A_RECOVER, do_recover) ELSEIF_FSA_ACTION(A_ELECTION_VOTE, do_election_vote) ELSEIF_FSA_ACTION(A_ELECT_TIMER_START, do_election_timer_ctrl) ELSEIF_FSA_ACTION(A_ELECT_TIMER_STOP, do_election_timer_ctrl) ELSEIF_FSA_ACTION(A_ELECTION_COUNT, do_election_count_vote) ELSEIF_FSA_ACTION(A_ELECTION_TIMEOUT, do_election_timer_ctrl) /* * "Get this over with" actions */ ELSEIF_FSA_ACTION(A_MSG_STORE, do_msg_store) ELSEIF_FSA_ACTION(A_NODE_BLOCK, do_node_block) /* * High priority actions * Update the cache first */ ELSEIF_FSA_ACTION(A_CCM_UPDATE_CACHE, do_ccm_update_cache) ELSEIF_FSA_ACTION(A_CCM_EVENT, do_ccm_event) /* * Medium priority actions */ ELSEIF_FSA_ACTION(A_DC_TAKEOVER, do_dc_takeover) ELSEIF_FSA_ACTION(A_DC_RELEASE, do_dc_release) ELSEIF_FSA_ACTION(A_JOIN_WELCOME_ALL, do_send_welcome) ELSEIF_FSA_ACTION(A_JOIN_WELCOME, do_send_welcome) ELSEIF_FSA_ACTION(A_JOIN_ACK, do_ack_welcome) ELSEIF_FSA_ACTION(A_JOIN_PROCESS_ACK, do_process_welcome_ack) /* * Low(er) priority actions * Make sure the CIB is always updated before invoking the * PE, and the PE before the TE */ ELSEIF_FSA_ACTION(A_UPDATE_NODESTATUS, do_lrm_invoke) ELSEIF_FSA_ACTION(A_CIB_INVOKE_LOCAL, do_cib_invoke) ELSEIF_FSA_ACTION(A_CIB_INVOKE, do_cib_invoke) ELSEIF_FSA_ACTION(A_CIB_BUMPGEN, do_cib_invoke) ELSEIF_FSA_ACTION(A_LRM_INVOKE, do_lrm_invoke) ELSEIF_FSA_ACTION(A_LRM_EVENT, do_lrm_event) ELSEIF_FSA_ACTION(A_TE_CANCEL, do_te_invoke) ELSEIF_FSA_ACTION(A_PE_INVOKE, do_pe_invoke) ELSEIF_FSA_ACTION(A_TE_INVOKE, do_te_invoke) /* sub-system stop */ ELSEIF_FSA_ACTION(A_PE_STOP, do_pe_control) ELSEIF_FSA_ACTION(A_TE_STOP, do_te_control) ELSEIF_FSA_ACTION(A_DC_RELEASED, do_dc_release) ELSEIF_FSA_ACTION(A_HA_DISCONNECT, do_ha_control) ELSEIF_FSA_ACTION(A_CCM_DISCONNECT, do_ccm_control) ELSEIF_FSA_ACTION(A_LRM_DISCONNECT, do_lrm_control) ELSEIF_FSA_ACTION(A_CIB_STOP, do_cib_control) /* time to go now... */ /* Some of these can probably be consolidated */ ELSEIF_FSA_ACTION(A_SHUTDOWN, do_shutdown) ELSEIF_FSA_ACTION(A_STOP, do_stop) /* exit gracefully */ ELSEIF_FSA_ACTION(A_EXIT_0, do_exit) // ELSEIF_FSA_ACTION(A_, do_) else if((actions & A_MSG_PROCESS) != 0 || is_message()) { xmlNodePtr stored_msg = NULL; fsa_message_queue_t msg = get_message(); if(is_message() == FALSE) { actions = clear_bit(actions, A_MSG_PROCESS); } if(msg == NULL || msg->message == NULL) { cl_log(LOG_ERR, "Invalid stored message"); continue; } /* * This is where we should clean up old messages * The problem is that we dont always know the * type of the data (and therefore the correct way * to free it). A wrapper is probably required. */ data = msg->message; #ifdef DOT_FSA_ACTIONS fprintf(dot_strm, "\t// %s:\t%s\t(data? %s)", fsa_input2string(cur_input), fsa_action2string(A_MSG_PROCESS), stored_msg==NULL?"no":"yes"); fflush(dot_strm); #endif #ifdef FSA_TRACE CRM_DEBUG("Invoking action %s (%.16llx)", fsa_action2string(A_MSG_PROCESS), A_MSG_PROCESS); #endif stored_msg = (xmlNodePtr)data; #ifdef FSA_TRACE xml_message_debug(stored_msg,"FSA processing message"); #endif next_input = handle_message(stored_msg); #ifdef DOT_FSA_ACTIONS fprintf(dot_strm, "\t(result=%s)\n", fsa_input2string(next_input)); #endif CRM_DEBUG("Result of action %s was %s", fsa_action2string(A_MSG_PROCESS), fsa_input2string(next_input)); /* Error checking and reporting */ } else if(cur_input != I_NULL && is_set(actions, A_NOTHING)) { cl_log(LOG_WARNING, "No action specified for input,state (%s,%s)", fsa_input2string(cur_input), fsa_state2string(cur_state)); next_input = I_NULL; } else if(cur_input == I_NULL && is_set(actions, A_NOTHING)) { #ifdef FSA_TRACE cl_log(LOG_INFO, "Nothing left to do"); #endif } else { cl_log(LOG_ERR, "Action %s (0x%llx) not supported ", fsa_action2string(actions), actions); next_input = I_ERROR; } + + if(is_message()) { + actions |= A_MSG_PROCESS; + } } #ifdef FSA_TRACE CRM_DEBUG("################# Exiting the FSA (%s) ##################", fsa_state2string(fsa_state)); #endif #ifdef DOT_FSA_ACTIONS fprintf(dot_strm, "\t// ### Exiting the FSA (%s)\n", fsa_state2string(fsa_state)); fflush(dot_strm); #endif // cleanup inputs? fsa_actions = actions; FNRET(fsa_state); } /* A_NODE_BLOCK */ enum crmd_fsa_input do_node_block(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, void *data) { xmlNodePtr xml_message = (xmlNodePtr)data; const char *host_from = xmlGetProp(xml_message, XML_ATTR_HOSTFROM); FNIN(); (void)host_from; FNRET(I_NULL); } const char * fsa_input2string(int input) { const char *inputAsText = NULL; switch(input){ case I_NULL: inputAsText = "I_NULL"; break; case I_CCM_EVENT: inputAsText = "I_CCM_EVENT"; break; case I_CIB_OP: inputAsText = "I_CIB_OP"; break; case I_CIB_UPDATE: inputAsText = "I_CIB_UPDATE"; break; case I_DC_TIMEOUT: inputAsText = "I_DC_TIMEOUT"; break; case I_ELECTION: inputAsText = "I_ELECTION"; break; case I_RELEASE_DC: inputAsText = "I_RELEASE_DC"; break; case I_ELECTION_DC: inputAsText = "I_ELECTION_DC"; break; case I_ERROR: inputAsText = "I_ERROR"; break; case I_FAIL: inputAsText = "I_FAIL"; break; case I_INTEGRATION_TIMEOUT: inputAsText = "I_INTEGRATION_TIMEOUT"; break; case I_NODE_JOIN: inputAsText = "I_NODE_JOIN"; break; case I_NODE_LEFT: inputAsText = "I_NODE_LEFT"; break; case I_NODE_LEAVING: inputAsText = "I_NODE_LEAVING"; break; case I_NOT_DC: inputAsText = "I_NOT_DC"; break; case I_RECOVERED: inputAsText = "I_RECOVERED"; break; case I_RELEASE_FAIL: inputAsText = "I_RELEASE_FAIL"; break; case I_RELEASE_SUCCESS: inputAsText = "I_RELEASE_SUCCESS"; break; case I_RESTART: inputAsText = "I_RESTART"; break; case I_REQUEST: inputAsText = "I_REQUEST"; break; case I_ROUTER: inputAsText = "I_ROUTER"; break; case I_SHUTDOWN: inputAsText = "I_SHUTDOWN"; break; /* case I_SHUTDOWN_REQ: */ /* inputAsText = "I_SHUTDOWN_REQ"; */ /* break; */ case I_STARTUP: inputAsText = "I_STARTUP"; break; case I_SUCCESS: inputAsText = "I_SUCCESS"; break; case I_TERMINATE: inputAsText = "I_TERMINATE"; break; case I_WELCOME: inputAsText = "I_WELCOME"; break; case I_WELCOME_ACK: inputAsText = "I_WELCOME_ACK"; break; case I_DC_HEARTBEAT: inputAsText = "I_DC_HEARTBEAT"; break; case I_WAIT_FOR_EVENT: inputAsText = "I_WAIT_FOR_EVENT"; break; case I_ILLEGAL: inputAsText = "I_ILLEGAL"; break; } if(inputAsText == NULL) { cl_log(LOG_ERR, "Input %d is unknown", input); inputAsText = ""; } return inputAsText; } const char * fsa_state2string(int state) { const char *stateAsText = NULL; switch(state){ case S_IDLE: stateAsText = "S_IDLE"; break; case S_ELECTION: stateAsText = "S_ELECTION"; break; case S_INTEGRATION: stateAsText = "S_INTEGRATION"; break; case S_NOT_DC: stateAsText = "S_NOT_DC"; break; case S_POLICY_ENGINE: stateAsText = "S_POLICY_ENGINE"; break; case S_RECOVERY: stateAsText = "S_RECOVERY"; break; case S_RECOVERY_DC: stateAsText = "S_RECOVERY_DC"; break; case S_RELEASE_DC: stateAsText = "S_RELEASE_DC"; break; case S_PENDING: stateAsText = "S_PENDING"; break; case S_STOPPING: stateAsText = "S_STOPPING"; break; case S_TERMINATE: stateAsText = "S_TERMINATE"; break; case S_TRANSITION_ENGINE: stateAsText = "S_TRANSITION_ENGINE"; break; case S_ILLEGAL: stateAsText = "S_ILLEGAL"; break; } if(stateAsText == NULL) { cl_log(LOG_ERR, "State %d is unknown", state); stateAsText = ""; } return stateAsText; } const char * fsa_cause2string(int cause) { const char *causeAsText = NULL; switch(cause){ case C_UNKNOWN: causeAsText = "C_UNKNOWN"; break; case C_STARTUP: causeAsText = "C_STARTUP"; break; case C_IPC_MESSAGE: causeAsText = "C_IPC_MESSAGE"; break; case C_HA_MESSAGE: causeAsText = "C_HA_MESSAGE"; break; case C_CCM_CALLBACK: causeAsText = "C_CCM_CALLBACK"; break; case C_TIMER_POPPED: causeAsText = "C_TIMER_POPPED"; break; case C_SHUTDOWN: causeAsText = "C_SHUTDOWN"; break; case C_HEARTBEAT_FAILED: causeAsText = "C_HEARTBEAT_FAILED"; break; case C_SUBSYSTEM_CONNECT: causeAsText = "C_SUBSYSTEM_CONNECT"; break; case C_ILLEGAL: causeAsText = "C_ILLEGAL"; break; } if(causeAsText == NULL) { cl_log(LOG_ERR, "Cause %d is unknown", cause); causeAsText = ""; } return causeAsText; } const char * fsa_action2string(long long action) { const char *actionAsText = NULL; switch(action){ case A_NOTHING: actionAsText = "A_NOTHING"; break; case O_SHUTDOWN: actionAsText = "O_SHUTDOWN"; break; case O_RELEASE: actionAsText = "O_RELEASE"; break; case A_STARTUP: actionAsText = "A_STARTUP"; break; case A_STARTED: actionAsText = "A_STARTED"; break; case A_HA_CONNECT: actionAsText = "A_HA_CONNECT"; break; case A_HA_DISCONNECT: actionAsText = "A_HA_DISCONNECT"; break; case A_LRM_CONNECT: actionAsText = "A_LRM_CONNECT"; break; case A_LRM_DISCONNECT: actionAsText = "A_LRM_DISCONNECT"; break; case O_DC_TIMER_RESTART: actionAsText = "O_DC_TIMER_RESTART"; break; case A_DC_TIMER_STOP: actionAsText = "A_DC_TIMER_STOP"; break; case A_DC_TIMER_START: actionAsText = "A_DC_TIMER_START"; break; case A_ELECTION_COUNT: actionAsText = "A_ELECTION_COUNT"; break; case A_ELECTION_TIMEOUT: actionAsText = "A_ELECTION_TIMEOUT"; break; case A_ELECT_TIMER_START: actionAsText = "A_ELECT_TIMER_START"; break; case A_ELECT_TIMER_STOP: actionAsText = "A_ELECT_TIMER_STOP"; break; case A_ELECTION_VOTE: actionAsText = "A_ELECTION_VOTE"; break; case A_ANNOUNCE: actionAsText = "A_ANNOUNCE"; break; case A_JOIN_ACK: actionAsText = "A_JOIN_ACK"; break; case A_JOIN_WELCOME: actionAsText = "A_JOIN_WELCOME"; break; case A_JOIN_WELCOME_ALL: actionAsText = "A_JOIN_WELCOME_ALL"; break; case A_JOIN_PROCESS_ACK: actionAsText = "A_JOIN_PROCESS_ACK"; break; case A_MSG_PROCESS: actionAsText = "A_MSG_PROCESS"; break; case A_MSG_ROUTE: actionAsText = "A_MSG_ROUTE"; break; case A_MSG_STORE: actionAsText = "A_MSG_STORE"; break; case A_RECOVER: actionAsText = "A_RECOVER"; break; case A_DC_RELEASE: actionAsText = "A_DC_RELEASE"; break; case A_DC_RELEASED: actionAsText = "A_DC_RELEASED"; break; case A_DC_TAKEOVER: actionAsText = "A_DC_TAKEOVER"; break; case A_SHUTDOWN: actionAsText = "A_SHUTDOWN"; break; case A_SHUTDOWN_REQ: actionAsText = "A_SHUTDOWN_REQ"; break; case A_STOP: actionAsText = "A_STOP "; break; case A_EXIT_0: actionAsText = "A_EXIT_0"; break; case A_EXIT_1: actionAsText = "A_EXIT_1"; break; case A_CCM_CONNECT: actionAsText = "A_CCM_CONNECT"; break; case A_CCM_DISCONNECT: actionAsText = "A_CCM_DISCONNECT"; break; case A_CCM_EVENT: actionAsText = "A_CCM_EVENT"; break; case A_CCM_UPDATE_CACHE: actionAsText = "A_CCM_UPDATE_CACHE"; break; case A_CIB_BUMPGEN: actionAsText = "A_CIB_BUMPGEN"; break; case A_CIB_INVOKE: actionAsText = "A_CIB_INVOKE"; break; case O_CIB_RESTART: actionAsText = "O_CIB_RESTART"; break; case A_CIB_START: actionAsText = "A_CIB_START"; break; case A_CIB_STOP: actionAsText = "A_CIB_STOP"; break; case A_TE_INVOKE: actionAsText = "A_TE_INVOKE"; break; case O_TE_RESTART: actionAsText = "O_TE_RESTART"; break; case A_TE_START: actionAsText = "A_TE_START"; break; case A_TE_STOP: actionAsText = "A_TE_STOP"; break; case A_TE_COPYTO: actionAsText = "A_TE_COPYTO"; break; case A_PE_INVOKE: actionAsText = "A_PE_INVOKE"; break; case O_PE_RESTART: actionAsText = "O_PE_RESTART"; break; case A_PE_START: actionAsText = "A_PE_START"; break; case A_PE_STOP: actionAsText = "A_PE_STOP"; break; case A_NODE_BLOCK: actionAsText = "A_NODE_BLOCK"; break; case A_UPDATE_NODESTATUS: actionAsText = "A_UPDATE_NODESTATUS"; break; case A_LOG: actionAsText = "A_LOG "; break; case A_ERROR: actionAsText = "A_ERROR "; break; case A_WARN: actionAsText = "A_WARN "; break; } if(actionAsText == NULL) { cl_log(LOG_ERR, "Action %.16llx is unknown", action); actionAsText = ""; } return actionAsText; } long long do_state_transition(long long actions, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_state next_state, enum crmd_fsa_input current_input, void *data) { long long tmp = A_NOTHING; if(current_input != I_NULL && (current_input != I_DC_HEARTBEAT || cur_state != S_NOT_DC)){ const char *state_from = fsa_state2string(cur_state); const char *state_to = fsa_state2string(next_state); const char *input = fsa_input2string(current_input); time_t now = time(NULL); fprintf(dot_strm, "\t\"%s\" -> \"%s\" [ label =\"%s\" ] // %s", state_from, state_to, input, asctime(localtime(&now))); fflush(dot_strm); } switch(next_state) { case S_PENDING: case S_NOT_DC: if(is_set(fsa_input_register, R_SHUTDOWN)){ tmp = set_bit(actions, A_SHUTDOWN_REQ); } tmp = clear_bit(actions, A_RECOVER); break; case S_RECOVERY_DC: case S_RECOVERY: tmp = set_bit(actions, A_RECOVER); break; default: tmp = clear_bit(actions, A_RECOVER); break; } if(tmp != actions) { cl_log(LOG_INFO, "Action b4 %.16llx ", actions); cl_log(LOG_INFO, "Action after %.16llx ", tmp); actions = tmp; } return actions; } long long clear_flags(long long actions, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input cur_input) { if(is_set(fsa_input_register, R_SHUTDOWN)){ clear_bit_inplace(&actions, A_DC_TIMER_START); } switch(cur_state) { case S_IDLE: break; case S_ELECTION: break; case S_INTEGRATION: break; case S_NOT_DC: break; case S_POLICY_ENGINE: break; case S_RECOVERY: break; case S_RECOVERY_DC: break; case S_RELEASE_DC: break; case S_PENDING: break; case S_STOPPING: break; case S_TERMINATE: break; case S_TRANSITION_ENGINE: break; case S_ILLEGAL: break; } return actions; } diff --git a/crm/crmd/subsystems.c b/crm/crmd/subsystems.c index 8509d4ed1a..06fe248df5 100644 --- a/crm/crmd/subsystems.c +++ b/crm/crmd/subsystems.c @@ -1,1176 +1,1183 @@ /* * Copyright (C) 2004 Andrew Beekhof * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #include #include #include #include #include #include // for access #include #include #include // for calls to open #include // for calls to open #include // for calls to open #include // for getpwuid #include // for initgroups #include // for getrlimit #include // for getrlimit #include #include #include #include #include #include #include #include #include #include #include #define CLIENT_EXIT_WAIT 10 static gboolean stop_subsystem (struct crm_subsystem_s *centry); static gboolean start_subsystem(struct crm_subsystem_s *centry); static gboolean run_command (struct crm_subsystem_s *centry, const char *options, gboolean update_pid); xmlNodePtr do_lrm_query(void); GHashTable *xml2list(xmlNodePtr parent, const char **attr_path, int depth); gboolean lrm_dispatch(int fd, gpointer user_data); void do_update_resource(lrm_rsc_t *rsc, int status, int rc, const char *op_type); struct crm_subsystem_s *cib_subsystem = NULL; struct crm_subsystem_s *te_subsystem = NULL; struct crm_subsystem_s *pe_subsystem = NULL; void cleanup_subsystem(struct crm_subsystem_s *the_subsystem) { int pid_status = -1; the_subsystem->ipc = NULL; clear_bit_inplace(&fsa_input_register, the_subsystem->flag); /* Forcing client to die */ kill(the_subsystem->pid, -9); // cleanup the ps entry waitpid(the_subsystem->pid, &pid_status, WNOHANG); the_subsystem->pid = -1; } /* A_CIB_STOP, A_CIB_START, A_CIB_RESTART, */ enum crmd_fsa_input do_cib_control(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, void *data) { enum crmd_fsa_input result = I_NULL; struct crm_subsystem_s *this_subsys = cib_subsystem; long long stop_actions = A_CIB_STOP; long long start_actions = A_CIB_START; FNIN(); if(action & stop_actions) { // dont do anything, its embedded now } if(action & start_actions) { if(cur_state != S_STOPPING) { if(startCib(CIB_FILENAME) == FALSE) result = I_FAIL; } else { cl_log(LOG_INFO, "Ignoring request to start %s after shutdown", this_subsys->command); } } FNRET(result); } /* A_CIB_INVOKE, A_CIB_BUMPGEN, A_UPDATE_NODESTATUS */ enum crmd_fsa_input do_cib_invoke(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, void *data) { xmlNodePtr cib_msg = NULL; xmlNodePtr answer = NULL; xmlNodePtr tmp1 = NULL; xmlNodePtr tmp2 = NULL; xmlNodePtr new_options = NULL; const char *req_from; const char *section = NULL; FNIN(); if(data != NULL) cib_msg = (xmlNodePtr)data; if(action & A_CIB_INVOKE) { const char *op = get_xml_attr(cib_msg, XML_TAG_OPTIONS, XML_ATTR_OP, TRUE); if(safe_str_eq(op, CRM_OPERATION_SHUTDOWN_REQ)){ // create update section tmp2 = create_xml_node(NULL, XML_CIB_TAG_STATE); req_from = xmlGetProp(cib_msg, XML_ATTR_HOSTFROM); set_xml_property_copy(tmp1, "id", req_from); set_xml_property_copy(tmp1, "exp_state", "shutdown"); // create fragment tmp1 = create_cib_fragment(tmp2, NULL); // add to cib_msg add_node_copy(cib_msg, tmp1); free_xml(tmp2); free_xml(tmp1); } set_xml_property_copy(cib_msg, XML_ATTR_SYSTO, "cib"); answer = process_cib_message(cib_msg, TRUE); if(relay_message(answer, TRUE) == FALSE) { cl_log(LOG_ERR, "Confused what to do with cib result"); xml_message_debug(answer, "Couldnt route: "); } if(op != NULL && AM_I_DC && (strcmp(op, CRM_OPERATION_CREATE) == 0 || strcmp(op, CRM_OPERATION_UPDATE) == 0 || strcmp(op, CRM_OPERATION_DELETE) == 0 || strcmp(op, CRM_OPERATION_REPLACE) == 0 || strcmp(op, CRM_OPERATION_WELCOME) == 0 || strcmp(op, CRM_OPERATION_SHUTDOWN_REQ) == 0 || strcmp(op, CRM_OPERATION_ERASE) == 0)) { FNRET(I_CIB_UPDATE); } if(op == NULL) { xml_message_debug(cib_msg, "Invalid CIB Message"); } // check the answer, see if we are interested in it also #if 0 if(interested in reply) { put_message(answer); FNRET(I_REQUEST); } #endif free_xml(answer); /* experimental */ } else if(action & A_CIB_INVOKE_LOCAL) { answer = process_cib_message(cib_msg, TRUE); put_message(answer); FNRET(I_REQUEST); } else if(action & A_CIB_BUMPGEN) { // check if the response was ok before next bit section = get_xml_attr(cib_msg, XML_TAG_OPTIONS, XML_ATTR_FILTER_TYPE, FALSE); /* set the section so that we dont always send the * whole thing */ if(section != NULL) { new_options = set_xml_attr(NULL, XML_TAG_OPTIONS, XML_ATTR_FILTER_TYPE, section, TRUE); } answer = process_cib_request(CRM_OPERATION_BUMP, new_options, NULL); free_xml(new_options); if(answer == NULL) { cl_log(LOG_ERR, "Result of BUMP in %s was NULL", __FUNCTION__); FNRET(I_FAIL); } send_request(NULL, answer, CRM_OPERATION_REPLACE, NULL, CRM_SYSTEM_CRMD); free_xml(answer); } else { cl_log(LOG_ERR, "Unexpected action %s in %s", fsa_action2string(action), __FUNCTION__); } FNRET(I_NULL); } /* A_PE_START, A_PE_STOP, A_TE_RESTART */ enum crmd_fsa_input do_pe_control(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, void *data) { enum crmd_fsa_input result = I_NULL; struct crm_subsystem_s *this_subsys = pe_subsystem; long long stop_actions = A_PE_STOP; long long start_actions = A_PE_START; FNIN(); if(action & stop_actions) { if(stop_subsystem(this_subsys) == FALSE) result = I_FAIL; else if(this_subsys->pid > 0){ int lpc = CLIENT_EXIT_WAIT; int pid_status = -1; while(lpc-- > 0 && this_subsys->pid > 0 && CL_PID_EXISTS(this_subsys->pid)) { sleep(1); waitpid(this_subsys->pid, &pid_status, WNOHANG); } if(CL_PID_EXISTS(this_subsys->pid)) { cl_log(LOG_ERR, "Process %s is still active with pid=%d", this_subsys->command, this_subsys->pid); result = I_FAIL; } } cleanup_subsystem(this_subsys); } if(action & start_actions) { if(cur_state != S_STOPPING) { if(start_subsystem(this_subsys) == FALSE) { result = I_FAIL; cleanup_subsystem(this_subsys); } } else { cl_log(LOG_INFO, "Ignoring request to start %s while shutting down", this_subsys->command); } } FNRET(result); } /* A_PE_INVOKE */ enum crmd_fsa_input do_pe_invoke(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, void *data) { FNIN(); stopTimer(integration_timer); if(is_set(fsa_input_register, R_PE_CONNECTED) == FALSE){ cl_log(LOG_INFO, "Waiting for the PE to connect"); FNRET(I_WAIT_FOR_EVENT); } xmlNodePtr local_cib = get_cib_copy(); CRM_DEBUG("Invoking %s with %p", CRM_SYSTEM_PENGINE, local_cib); send_request(NULL, local_cib, "pecalc", NULL, CRM_SYSTEM_PENGINE); FNRET(I_NULL); } /* A_TE_START, A_TE_STOP, A_TE_RESTART */ enum crmd_fsa_input do_te_control(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, void *data) { enum crmd_fsa_input result = I_NULL; struct crm_subsystem_s *this_subsys = te_subsystem; long long stop_actions = A_TE_STOP; long long start_actions = A_TE_START; FNIN(); /* if(action & stop_actions && cur_state != S_STOPPING */ /* && is_set(fsa_input_register, R_TE_PEND)) { */ /* result = I_WAIT_FOR_EVENT; */ /* FNRET(result); */ /* } */ if(action & stop_actions) { if(stop_subsystem(this_subsys) == FALSE) result = I_FAIL; else if(this_subsys->pid > 0){ int lpc = CLIENT_EXIT_WAIT; int pid_status = -1; while(lpc-- > 0 && this_subsys->pid > 0 && CL_PID_EXISTS(this_subsys->pid)) { sleep(1); waitpid(this_subsys->pid, &pid_status, WNOHANG); } if(CL_PID_EXISTS(this_subsys->pid)) { cl_log(LOG_ERR, "Process %s is still active with pid=%d", this_subsys->command, this_subsys->pid); result = I_FAIL; } } cleanup_subsystem(this_subsys); } if(action & start_actions) { if(cur_state != S_STOPPING) { if(start_subsystem(this_subsys) == FALSE) { result = I_FAIL; cleanup_subsystem(this_subsys); } } else { cl_log(LOG_INFO, "Ignoring request to start %s while shutting down", this_subsys->command); } } FNRET(result); } /* A_TE_COPYTO */ enum crmd_fsa_input do_te_copyto(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, void *data) { xmlNodePtr message = copy_xml_node_recursive((xmlNodePtr)data); xmlNodePtr opts = find_xml_node(message, "options"); const char *true_op = xmlGetProp(opts, XML_ATTR_OP); FNIN(); set_xml_property_copy(opts, XML_ATTR_OP, "event"); set_xml_property_copy(message, XML_ATTR_SYSTO, CRM_SYSTEM_TENGINE); set_xml_property_copy(opts, "true_op", true_op); relay_message(message, FALSE); free_xml(message); FNRET(I_NULL); } static xmlNodePtr te_last_input = NULL; /* A_TE_INVOKE, A_TE_CANCEL */ enum crmd_fsa_input do_te_invoke(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, void *data) { xmlNodePtr graph = NULL; xmlNodePtr msg = (xmlNodePtr)data; FNIN(); if(is_set(fsa_input_register, R_TE_CONNECTED) == FALSE){ cl_log(LOG_INFO, "Waiting for the TE to connect"); if(data != NULL) { free_xml(te_last_input); te_last_input = copy_xml_node_recursive(msg); } FNRET(I_WAIT_FOR_EVENT); } if(msg == NULL) { msg = te_last_input; } else { free_xml(te_last_input); } if(action & A_TE_INVOKE) { graph = find_xml_node(msg, "transition_graph"); if(graph == NULL) { FNRET(I_FAIL); } send_request(NULL, graph, "transition", NULL, CRM_SYSTEM_TENGINE); } else { send_request(NULL, graph, "abort", NULL, CRM_SYSTEM_TENGINE); } FNRET(I_NULL); } gboolean crmd_client_connect(IPC_Channel *client_channel, gpointer user_data) { FNIN(); CRM_DEBUG("A client tried to connect... and there was much rejoicing."); if (client_channel == NULL) { cl_log(LOG_ERR, "Channel was NULL"); } else if (client_channel->ch_status == IPC_DISCONNECT) { cl_log(LOG_ERR, "Channel was disconnected"); } else { crmd_client_t *blank_client = (crmd_client_t *)cl_malloc(sizeof(crmd_client_t)); if (blank_client == NULL) { cl_log(LOG_ERR, "Could not allocate memory for a blank crmd_client_t"); FNRET(FALSE); } client_channel->ops->set_recv_qlen(client_channel, 100); client_channel->ops->set_send_qlen(client_channel, 100); blank_client->client_channel = client_channel; blank_client->sub_sys = NULL; blank_client->uuid = NULL; blank_client->table_key = NULL; CRM_DEBUG("Adding IPC Channel to main thread."); blank_client->client_source = G_main_add_IPC_Channel(G_PRIORITY_LOW, client_channel, FALSE, crmd_ipc_input_callback, blank_client, default_ipc_input_destroy); } FNRET(TRUE); } static gboolean stop_subsystem(struct crm_subsystem_s* centry) { cl_log(LOG_INFO, "Stopping sub-system \"%s\"", centry->name); if (centry->pid <= 0) { cl_log(LOG_ERR, "OOPS! client %s not running yet", centry->command); } else { cl_log(LOG_INFO, "Sending quit message to %s.", centry->name); send_request(NULL, NULL, "quit", NULL, centry->name); } return TRUE; } static gboolean start_subsystem(struct crm_subsystem_s* centry) { cl_log(LOG_INFO, "Starting sub-system \"%s\"", centry->command); if (centry->pid != 0) { cl_log(LOG_ERR, "OOPS! client %s already running as pid %d" , centry->command, (int) centry->pid); } return run_command(centry, "-r", TRUE); } static gboolean run_command(struct crm_subsystem_s *centry, const char *options, gboolean update_pid) { pid_t pid; struct stat buf; int s_res,size; char *cmd_with_options = NULL; /* * We need to ensure that the exec will succeed before * we bother forking. We don't want to respawn something that * won't exec in the first place. */ if (access(centry->path, F_OK|X_OK) != 0) { cl_perror("Cannot (access) exec %s", centry->path); return FALSE; } s_res = stat(centry->command, &buf); if(s_res != 0) { cl_perror("Cannot (stat) exec %s", centry->command); return FALSE; } /* We need to fork so we can make child procs not real time */ switch(pid=fork()) { case -1: cl_log(LOG_ERR , "start_a_child_client: Cannot fork."); return FALSE; default: /* Parent */ #if 0 NewTrackedProc(pid, 1, PT_LOGVERBOSE , centry, &ManagedChildTrackOps); #else if(update_pid) centry->pid = pid; #endif return TRUE; case 0: /* Child */ break; } /* Child process: start the managed child */ cl_make_normaltime(); setpgid(0,0); /* Limit peak resource usage, maximize success chances */ if (centry->shortrcount > 0) { alarm(0); sleep(1); } size = strlen(options); size += strlen(centry->command); size += 2; // ' ' + \0 cmd_with_options = cl_malloc((1+size)*sizeof(char)); sprintf(cmd_with_options, "%s %s", centry->command, options); cmd_with_options[size] = 0; cl_log(LOG_INFO, "Executing \"%s\" (pid %d)", cmd_with_options, (int) getpid()); if(CL_SIGINTERRUPT(SIGALRM, 0) < 0) { cl_perror("Cannot set interrupt for child process %s", cmd_with_options); }else{ const char * devnull = "/dev/null"; unsigned int j; struct rlimit oflimits; CL_SIGNAL(SIGCHLD, SIG_DFL); alarm(0); CL_IGNORE_SIG(SIGALRM); /* A precautionary measure */ getrlimit(RLIMIT_NOFILE, &oflimits); for (j=0; j < oflimits.rlim_cur; ++j) { close(j); } (void)devnull; (void)open(devnull, O_RDONLY); /* Stdin: fd 0 */ (void)open(devnull, O_WRONLY); /* Stdout: fd 1 */ (void)open(devnull, O_WRONLY); /* Stderr: fd 2 */ (void)execl("/bin/sh", "sh", "-c", cmd_with_options, (const char *)NULL); /* Should not happen */ cl_perror("Cannot exec %s", cmd_with_options); } /* Suppress respawning */ exit(100); // never reached return TRUE; } /* A_LRM_CONNECT */ enum crmd_fsa_input do_lrm_control(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, void *data) { enum crmd_fsa_input failed = I_NULL;//I_FAIL; int ret = HA_OK; FNIN(); if(action & A_LRM_DISCONNECT) { fsa_lrm_conn->lrm_ops->signoff(fsa_lrm_conn); } if(action & A_LRM_CONNECT) { CRM_DEBUG("LRM: connect..."); fsa_lrm_conn = ll_lrm_new("lrm"); if(NULL == fsa_lrm_conn) { return failed; } CRM_DEBUG("LRM: sigon..."); ret = fsa_lrm_conn->lrm_ops->signon(fsa_lrm_conn, "crmd"); if(ret != HA_OK) { cl_log(LOG_ERR, "Failed to sign on to the LRM"); return failed; } CRM_DEBUG("LRM: set_lrm_callback..."); ret = fsa_lrm_conn->lrm_ops->set_lrm_callback(fsa_lrm_conn, lrm_op_callback, lrm_monitor_callback); if(ret != HA_OK) { cl_log(LOG_ERR, "Failed to set LRM callbacks"); return failed; } /* TODO: create a destroy handler that causes * some recovery to happen */ G_main_add_fd(G_PRIORITY_LOW, fsa_lrm_conn->lrm_ops->inputfd(fsa_lrm_conn), FALSE, lrm_dispatch, fsa_lrm_conn, default_ipc_input_destroy); } if(action & ~(A_LRM_CONNECT|A_LRM_DISCONNECT)) { cl_log(LOG_ERR, "Unexpected action %s in %s", fsa_action2string(action), __FUNCTION__); } FNRET(I_NULL); } gboolean lrm_dispatch(int fd, gpointer user_data) { ll_lrm_t *lrm = (ll_lrm_t*)user_data; lrm->lrm_ops->rcvmsg(lrm, FALSE); return TRUE; } xmlNodePtr do_lrm_query(void) { GList* lrm_list = NULL; GList* element = NULL; GList* op_list = NULL; xmlNodePtr agent = NULL; xmlNodePtr data = create_xml_node(NULL, "lrm"); xmlNodePtr agent_list = create_xml_node(data, "lrm_agents"); xmlNodePtr rsc_list; char *rsc_type = NULL; state_flag_t cur_state = 0; const char *this_op = NULL; GList* node = NULL; lrm_list = fsa_lrm_conn->lrm_ops->get_ra_supported(fsa_lrm_conn); if (NULL != lrm_list) { GList* element = g_list_first(lrm_list); while (NULL != element) { rsc_type = (char*)element->data; agent = create_xml_node(agent_list, "lrm_agent"); set_xml_property_copy(agent, "class", rsc_type); /* we dont have these yet */ set_xml_property_copy(agent, "type", NULL); set_xml_property_copy(agent, "version", NULL); element = g_list_next(element); } } g_list_free(lrm_list); lrm_list = fsa_lrm_conn->lrm_ops->get_all_rscs(fsa_lrm_conn); rsc_list = create_xml_node(data, "lrm_resources"); if (NULL != lrm_list) { element = g_list_first(lrm_list); } while (NULL != element) { lrm_rsc_t *the_rsc = (lrm_rsc_t*)element->data; /* const char* ra_type; */ /* GHashTable* params; */ xmlNodePtr xml_rsc = create_xml_node(rsc_list, "rsc_state"); set_xml_property_copy(xml_rsc, "id", the_rsc->id); set_xml_property_copy(xml_rsc, "rsc_id", the_rsc->name); set_xml_property_copy(xml_rsc, "node_id",fsa_our_uname); CRM_DEBUG("get_cur_state..."); op_list = the_rsc->ops->get_cur_state(the_rsc, &cur_state); CRM_DEBUG("\tcurrent state:%s\n", cur_state==LRM_RSC_IDLE?"Idel":"Busy"); node = g_list_first(op_list); while(NULL != node){ lrm_op_t* op = (lrm_op_t*)node->data; this_op = op->op_type; if(this_op == NULL || strcmp(this_op, "status") != 0){ const char *status_text = ""; switch(op->status) { case LRM_OP_DONE: status_text = "done"; break; case LRM_OP_CANCELLED: status_text = "cancelled"; break; case LRM_OP_TIMEOUT: status_text = "timeout"; break; case LRM_OP_NOTSUPPORTED: status_text = "not suported"; break; case LRM_OP_ERROR: status_text = "error"; break; } set_xml_property_copy(xml_rsc, "op_result", status_text); set_xml_property_copy(xml_rsc, "rsc_op", this_op); // we only want the last one break; } node = g_list_next(node); } element = g_list_next(element); } if (NULL != lrm_list) { g_list_free(lrm_list); } return data; } /* A_LRM_INVOKE */ enum crmd_fsa_input do_lrm_invoke(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, void *data) { enum crmd_fsa_input next_input = I_NULL; xmlNodePtr fragment, tmp1; xmlNodePtr msg; const char *rsc_path[] = { "msg_data", "rsc_op", "resource", "instance_attributes", "parameters" }; const char *operation = NULL; rsc_id_t rid; const char *id_from_cib = NULL; const char *crm_op = NULL; lrm_rsc_t *rsc = NULL; lrm_mon_t* mon = NULL; lrm_op_t* op = NULL; FNIN(); #ifdef USE_FAKE_LRM msg = (xmlNodePtr)data; operation = get_xml_attr_nested(msg, rsc_path, DIMOF(rsc_path) -3, "task", TRUE); id_from_cib = get_xml_attr_nested(msg, rsc_path, DIMOF(rsc_path) -2, "id", TRUE); crm_op = get_xml_attr(msg, XML_TAG_OPTIONS, XML_ATTR_OP, TRUE); if(safe_str_eq(crm_op, "rsc_op")) { CRM_DEBUG("performing op %s...", operation); + xmlNodePtr update = NULL; + xmlNodePtr state = create_xml_node(NULL, XML_CIB_TAG_STATE); + xmlNodePtr iter = create_xml_node(state, "lrm"); + iter = create_xml_node(iter, "lrm_resources"); - xmlNodePtr iter = create_xml_node(NULL, "lrm_resource"); + iter = create_xml_node(iter, "lrm_resource"); + set_xml_property_copy(iter, "id", fsa_our_uname); set_xml_property_copy(iter, XML_ATTR_ID, id_from_cib); set_xml_property_copy(iter, "last_op", operation); if(safe_str_eq(operation, "start")){ set_xml_property_copy(iter, "op_status", "started"); } else { set_xml_property_copy(iter, "op_status", "stopped"); } set_xml_property_copy(iter, "op_code", "0"); set_xml_property_copy(iter, "op_node", fsa_our_uname); - send_request(NULL, iter, "event", - NULL, CRM_SYSTEM_TENGINE); + update = create_cib_fragment(state, NULL); + + send_request(NULL, update, "update", + NULL, CRM_SYSTEM_DCIB); } FNRET(I_NULL); #endif if(action & A_UPDATE_NODESTATUS) { xmlNodePtr data = do_lrm_query(); set_xml_property_copy(data, "replace_lrm", "true"); tmp1 = create_xml_node(NULL, XML_CIB_TAG_STATE); set_xml_property_copy(tmp1, XML_ATTR_ID, fsa_our_uname); fragment = create_cib_fragment(tmp1, NULL); set_xml_property_copy(data, "replace_lrm", "true"); add_node_copy(tmp1, data); send_request(NULL, fragment, CRM_OPERATION_UPDATE, NULL, CRM_SYSTEM_DC); free_xml(fragment); free_xml(tmp1); free_xml(data); FNRET(next_input); } cl_log(LOG_WARNING, "Action %s (%.16llx) only kind of supported\n", fsa_action2string(action), action); msg = (xmlNodePtr)data; operation = get_xml_attr_nested(msg, rsc_path, DIMOF(rsc_path) -3, XML_ATTR_OP, TRUE); id_from_cib = get_xml_attr_nested(msg, rsc_path, DIMOF(rsc_path) -2, "id", TRUE); // only the first 16 chars are used by the LRM strncpy(rid, id_from_cib, 16); crm_op = get_xml_attr(msg, XML_TAG_OPTIONS, XML_ATTR_OP, TRUE); rsc = fsa_lrm_conn->lrm_ops->get_rsc(fsa_lrm_conn, rid); if(crm_op != NULL && strcmp(crm_op, "lrm_query") == 0) { xmlNodePtr data, tmp1, tmp2, reply; tmp1 = create_xml_node(NULL, XML_CIB_TAG_STATE); set_xml_property_copy(tmp1, XML_ATTR_ID, fsa_our_uname); data = create_cib_fragment(tmp1, NULL); tmp2 = do_lrm_query(); add_node_copy(tmp1, tmp2); reply = create_reply(msg, data); relay_message(reply, TRUE); free_xml(data); free_xml(reply); free_xml(tmp2); free_xml(tmp1); } else if(operation != NULL && strcmp(operation, "monitor") == 0) { if(rsc == NULL) { cl_log(LOG_ERR, "Could not find resource to monitor"); FNRET(I_FAIL); } mon = g_new(lrm_mon_t, 1); mon->op_type = "status"; mon->params = NULL; mon->timeout = 0; mon->user_data = rsc; mon->mode = LRM_MONITOR_SET; mon->interval = 2; mon->target = 1; rsc->ops->set_monitor(rsc,mon); mon = g_new(lrm_mon_t, 1); } else if(operation != NULL) { if(rsc == NULL) { // add it to the list CRM_DEBUG("add_rsc..."); fsa_lrm_conn->lrm_ops->add_rsc( fsa_lrm_conn, rid, get_xml_attr_nested(msg, rsc_path, DIMOF(rsc_path) -2, "class", TRUE), get_xml_attr_nested(msg, rsc_path, DIMOF(rsc_path) -2, "type", TRUE), NULL); rsc = fsa_lrm_conn->lrm_ops->get_rsc( fsa_lrm_conn, rid); } if(rsc == NULL) { cl_log(LOG_ERR, "Could not add resource to LRM"); FNRET(I_FAIL); } // now do the op CRM_DEBUG("performing op %s...", operation); op = g_new(lrm_op_t, 1); op->op_type = operation; op->params = xml2list(msg, rsc_path, DIMOF(rsc_path)); op->timeout = 0; op->user_data = rsc; rsc->ops->perform_op(rsc, op); } FNRET(next_input); } GHashTable * xml2list(xmlNodePtr parent, const char**attr_path, int depth) { xmlNodePtr node_iter = NULL; GHashTable *nvpair_hash = g_hash_table_new(&g_str_hash, &g_str_equal); xmlNodePtr nvpair_list = find_xml_node_nested(parent, attr_path, depth); if(nvpair_list != NULL){ node_iter = nvpair_list->children; while(node_iter != NULL) { const char *key = xmlGetProp(node_iter, "name"); const char *value = xmlGetProp(node_iter, "value"); CRM_DEBUG("Added %s=%s", key, value); g_hash_table_insert (nvpair_hash, cl_strdup(key), cl_strdup(value)); node_iter = node_iter->next; } } return nvpair_hash; } void do_update_resource(lrm_rsc_t *rsc, int status, int rc, const char *op_type) { /* */ xmlNodePtr update, iter; char *tmp = NULL; xmlNodePtr fragment, tmp1; update = create_xml_node(NULL, "node_state"); set_xml_property_copy(update, XML_ATTR_ID, fsa_our_uname); iter = create_xml_node(update, "lrm"); iter = create_xml_node(iter, "lrm_resources"); iter = create_xml_node(iter, "lrm_resource"); set_xml_property_copy(iter, XML_ATTR_ID, rsc->id); set_xml_property_copy(iter, "last_op", op_type); tmp = crm_itoa(status); set_xml_property_copy(iter, "op_status", tmp); cl_free(tmp); tmp = crm_itoa(rc); set_xml_property_copy(iter, "op_code", tmp); cl_free(tmp); set_xml_property_copy(iter, "op_node", fsa_our_uname); tmp1 = create_xml_node(NULL, XML_CIB_TAG_STATE); set_xml_property_copy(tmp1, XML_ATTR_ID, fsa_our_uname); add_node_copy(tmp1, update); fragment = create_cib_fragment(tmp1, NULL); send_request(NULL, fragment, CRM_OPERATION_UPDATE, NULL, CRM_SYSTEM_DCIB); free_xml(fragment); free_xml(update); free_xml(tmp1); } enum crmd_fsa_input do_lrm_event(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input cur_input, void *data) { FNIN(); if(cause == C_LRM_MONITOR_CALLBACK) { lrm_mon_t* monitor = (lrm_mon_t*)data; lrm_rsc_t* rsc = monitor->rsc; switch(monitor->status) { case LRM_OP_DONE: CRM_DEBUG("An LRM monitor operation passed"); FNRET(I_NULL); break; case LRM_OP_CANCELLED: case LRM_OP_TIMEOUT: case LRM_OP_NOTSUPPORTED: case LRM_OP_ERROR: cl_log(LOG_ERR, "An LRM monitor operation failed" " or was aborted"); do_update_resource(rsc, monitor->status, monitor->rc, monitor->op_type); break; } } else if(cause == C_LRM_OP_CALLBACK) { lrm_op_t* op = (lrm_op_t*)data; lrm_rsc_t* rsc = op->rsc; switch(op->status) { case LRM_OP_CANCELLED: case LRM_OP_TIMEOUT: case LRM_OP_NOTSUPPORTED: case LRM_OP_ERROR: cl_log(LOG_ERR, "An LRM operation failed" " or was aborted"); // keep going case LRM_OP_DONE: do_update_resource(rsc, op->status, op->rc, op->op_type); break; } } else { FNRET(I_FAIL); } FNRET(I_NULL); } diff --git a/crm/tengine/tengine.c b/crm/tengine/tengine.c index cbf5522336..8f7e462eda 100644 --- a/crm/tengine/tengine.c +++ b/crm/tengine/tengine.c @@ -1,543 +1,556 @@ #include #include #include #include #include #include GSListPtr graph = NULL; IPC_Channel *crm_ch = NULL; typedef struct action_list_s { int index; int index_max; GSListPtr actions; } action_list_t; gboolean initialize_graph(void); gboolean unpack_graph(xmlNodePtr xml_graph); gboolean extract_event(xmlNodePtr msg); gboolean initiate_transition(void); gboolean initiate_action(action_list_t *list); gboolean process_graph_event(const char *event_node, const char *event_rsc, const char *event_action, const char *event_status, const char *event_rc); void send_success(void); void send_abort(void); gboolean process_fake_event(xmlNodePtr msg); gboolean initialize_graph(void) { while(g_slist_length(graph) > 0) { action_list_t *action_list = g_slist_nth_data(graph, 0); while(g_slist_length(action_list->actions) > 0) { xmlNodePtr action = g_slist_nth_data(action_list->actions, 0); action_list->actions = g_slist_remove(action_list->actions, action); free_xml(action); } graph = g_slist_remove(graph, action_list); cl_free(action_list); } graph = NULL; return TRUE; } gboolean unpack_graph(xmlNodePtr xml_graph) { /* */ xmlNodePtr xml_action_list = xml_graph?xml_graph->children:NULL; if(xml_action_list == NULL) { // nothing to do return FALSE; } while(xml_action_list != NULL) { xmlNodePtr xml_obj = xml_action_list; xmlNodePtr xml_action = xml_obj->children; action_list_t *action_list = (action_list_t*) cl_malloc(sizeof(action_list_t)); xml_action_list = xml_action_list->next; action_list->index = -1; action_list->index_max = 0; action_list->actions = NULL; while(xml_action != NULL) { xmlNodePtr action = copy_xml_node_recursive(xml_action); action_list->actions = g_slist_append(action_list->actions, action); action_list->index_max++; xml_action = xml_action->next; } graph = g_slist_append(graph, action_list); } return TRUE; } gboolean process_fake_event(xmlNodePtr msg) { xmlNodePtr data = find_xml_node(msg, "lrm_resource"); CRM_DEBUG("Processing fake LRM event..."); const char *last_op = "start"; if(safe_str_eq("stopped", xmlGetProp(data, "last_op"))) { last_op = "stop"; } CRM_DEBUG("Fake LRM event... %s, %s, %s, %s, %s", xmlGetProp(data, "op_node"), xmlGetProp(data, "id"), last_op, xmlGetProp(data, "op_status"), xmlGetProp(data, "op_code")); return process_graph_event(xmlGetProp(data, "op_node"), xmlGetProp(data, "id"), xmlGetProp(data, "last_op"), xmlGetProp(data, "op_status"), xmlGetProp(data, "op_code")); } gboolean extract_event(xmlNodePtr msg) { gboolean abort = FALSE; xmlNodePtr iter = NULL; const char *section = NULL; const char *event_action = NULL; const char *event_node = NULL; const char *event_rsc = NULL; const char *event_status = NULL; const char *event_rc = NULL; -/* +/* +[cib fragment] +... */ xml_message_debug(msg, "TE Event"); iter = find_xml_node(msg, XML_TAG_FRAGMENT); section = xmlGetProp(iter, "section"); if(safe_str_neq(section, XML_CIB_TAG_STATUS)) { // these too are never expected - send_abort(); return FALSE; - } - iter = find_xml_node(msg, XML_TAG_CIB); + iter = find_xml_node(iter, XML_TAG_CIB); iter = get_object_root(XML_CIB_TAG_STATUS, iter); iter = iter->children; - + while(abort == FALSE && iter != NULL) { xmlNodePtr node_state = iter; xmlNodePtr child = iter->children; const char *state = xmlGetProp(node_state, "state"); iter = iter->next; if(state != NULL && child == NULL) { /* node state update, * possibly from a shutdown we requested */ + CRM_DEBUG("state no child"); + event_status = state; event_node = xmlGetProp(node_state, XML_ATTR_ID); if(safe_str_eq(event_status, "down")) { event_action = "shutdown"; } abort = !process_graph_event(event_node, event_rsc, event_action, event_status, event_rc); } else if(state != NULL && child != NULL) { - /* this is a complex eventand could not be completely + /* this is a complex event and could not be completely * due to any request we made */ - send_abort(); abort = TRUE; - + CRM_DEBUG("state and child"); + } else { + CRM_DEBUG("child no state"); child = find_xml_node(node_state, "lrm"); child = find_xml_node(child, "lrm_resources"); - child = child->children; + + if(child != NULL) { + child = child->children; + } else { + abort = TRUE; + } while(abort == FALSE && child != NULL) { event_action = xmlGetProp(child, "last_op"); event_node = xmlGetProp(child, "op_node"); event_rsc = xmlGetProp(child, "id"); event_status = xmlGetProp(child, "op_status"); event_rc = xmlGetProp(child, "op_code"); abort = !process_graph_event(event_node, event_rsc, event_action, event_status, event_rc); child = child->next; } } } return !abort; } gboolean process_graph_event(const char *event_node, const char *event_rsc, const char *event_action, const char *event_status, const char *event_rc) { int lpc; xmlNodePtr action = NULL; // or xmlNodePtr next_action = NULL; action_list_t *matched_action_list = NULL; // Find the action corresponding to this event slist_iter( action_list, action_list_t, graph, lpc, action = g_slist_nth_data(action_list->actions, action_list->index); + + if(action == NULL) { + continue; + } /* */ const char *this_action = xmlGetProp(action, "task"); const char *this_node = xmlGetProp(action, "on_node"); const char *this_rsc = xmlGetProp(action->children, "id"); if(safe_str_neq(this_node, event_node)) { continue; } else if(safe_str_neq(this_action, event_action)) { continue; } else if(safe_str_eq(action->name, "rsc_op") && safe_str_eq(this_rsc, event_rsc)) { matched_action_list = action_list; } else if(safe_str_eq(action->name, "crm_event")) { matched_action_list = action_list; } ); // for the moment all actions succeed if(matched_action_list == NULL) { // unexpected event, trigger a pe-recompute // possibly do this only for certain types of actions send_abort(); return FALSE; } gboolean more_to_do = FALSE; while(matched_action_list->index < matched_action_list->index_max) { next_action = g_slist_nth_data(matched_action_list->actions, matched_action_list->index); gboolean failed = initiate_action(matched_action_list); if(failed) { send_abort(); } else if(matched_action_list->index > matched_action_list->index_max) { /* last action in that list, check if there are * anymore actions at all */ slist_iter( action_list, action_list_t, graph, lpc, if(action_list->index <= action_list->index_max){ more_to_do = TRUE; break; } ); } else { more_to_do = TRUE; } } if(more_to_do == FALSE) { // indicate to the CRMd that we're done xmlNodePtr options = create_xml_node(NULL, "options"); set_xml_property_copy(options, XML_ATTR_OP, "te_complete"); send_ipc_request(crm_ch, options, NULL, NULL, "dc", "tengine", NULL, NULL); free_xml(options); return TRUE; } // else wait for the next event return FALSE; } gboolean initiate_transition(void) { int lpc; gboolean anything = FALSE; FNIN(); slist_iter( action_list, action_list_t, graph, lpc, if(initiate_action(action_list) == FALSE) { anything = TRUE; } ); FNRET(anything); } gboolean initiate_action(action_list_t *list) { gboolean was_error = FALSE; xmlNodePtr xml_action = NULL; const char *on_node = NULL; const char *id = NULL; const char *runnable = NULL; const char *optional = NULL; const char *task = NULL; while(TRUE) { list->index++; xml_action = g_slist_nth_data(list->actions, list->index); if(xml_action == NULL) { cl_log(LOG_INFO, "No tasks left on this list"); list->index = list->index_max + 1; return was_error; } on_node = xmlGetProp(xml_action, "on_node"); id = xmlGetProp(xml_action, "id"); runnable = xmlGetProp(xml_action, "runnable"); optional = xmlGetProp(xml_action, "optional"); task = xmlGetProp(xml_action, "task"); cl_log(LOG_INFO, "Invoking action %s (id=%s) on %s", task, id, on_node); if(safe_str_eq(optional, "true")) { cl_log(LOG_INFO, "Skipping optional command"); } else if(safe_str_eq(runnable, "false")) { cl_log(LOG_ERR, "Skipping un-runnable command"); return !was_error; } else if(id == NULL || strlen(id) == 0 || on_node == NULL || strlen(on_node) == 0 || task == NULL || strlen(task) == 0) { // error cl_log(LOG_ERR, "Command: \"%s (id=%s) on %s\" was corrupted.", task, id, on_node); return !was_error; // } else if(safe_str_eq(xml_action->name, "pseduo_event")){ } else if(safe_str_eq(xml_action->name, "crm_event")){ /* */ xmlNodePtr options = create_xml_node(NULL, "options"); set_xml_property_copy(options, XML_ATTR_OP, task); send_ipc_request(crm_ch, options, NULL, on_node, "crmd", "tengine", NULL, NULL); free_xml(options); return was_error; } else if(safe_str_eq(xml_action->name, "rsc_op")){ /* ... */ xmlNodePtr options = create_xml_node(NULL, "options"); xmlNodePtr data = create_xml_node(NULL, "msg_data"); xmlNodePtr rsc_op = create_xml_node(data, "rsc_op"); set_xml_property_copy(options, XML_ATTR_OP, "rsc_op"); set_xml_property_copy(rsc_op, "id", id); set_xml_property_copy(rsc_op, "task", task); set_xml_property_copy(rsc_op, "on_node", on_node); add_node_copy(rsc_op, xml_action->children); send_ipc_request(crm_ch, options, data, on_node, "lrmd", "tengine", NULL, NULL); free_xml(options); free_xml(data); return was_error; } else { // error cl_log(LOG_ERR, "Action %s is not (yet?) supported", xml_action->name); } } return !was_error; } gboolean process_te_message(xmlNodePtr msg, IPC_Channel *sender) { const char *op = get_xml_attr (msg, XML_TAG_OPTIONS, XML_ATTR_OP, TRUE); const char *sys_to = xmlGetProp(msg, XML_ATTR_SYSTO); cl_log(LOG_DEBUG, "Processing %s message", op); if(op == NULL){ // error } else if(sys_to == NULL || strcmp(sys_to, "tengine") != 0) { CRM_DEBUG("Bad sys-to %s", sys_to); return FALSE; } else if(strcmp(op, "transition") == 0) { CRM_DEBUG("Initializing graph..."); initialize_graph(); xmlNodePtr graph = find_xml_node(msg, "transition_graph"); CRM_DEBUG("Unpacking graph..."); unpack_graph(graph); CRM_DEBUG("Initiating transition..."); if(initiate_transition() == FALSE) { // nothing to be done.. means we're done. cl_log(LOG_INFO, "No actions to be taken..." " transition compelte."); send_success(); } CRM_DEBUG("Processing complete..."); } else if(strcmp(op, "event") == 0) { const char *true_op = get_xml_attr (msg, XML_TAG_OPTIONS, "true_op", TRUE); if(true_op == NULL) { -#ifdef USE_FAKE_LRM - process_fake_event(msg); -#else - // error -#endif - } else if(strcmp(op, CRM_OPERATION_CREATE) == 0 - || strcmp(op, CRM_OPERATION_DELETE) == 0 - || strcmp(op, CRM_OPERATION_REPLACE) == 0 - || strcmp(op, CRM_OPERATION_WELCOME) == 0 - || strcmp(op, CRM_OPERATION_SHUTDOWN_REQ) == 0 - || strcmp(op, CRM_OPERATION_ERASE) == 0) { + cl_log(LOG_ERR, + "Illegal update, the original operation must be specified"); + send_abort(); + + } else if(strcmp(true_op, CRM_OPERATION_CREATE) == 0 + || strcmp(true_op, CRM_OPERATION_DELETE) == 0 + || strcmp(true_op, CRM_OPERATION_REPLACE) == 0 + || strcmp(true_op, CRM_OPERATION_WELCOME) == 0 + || strcmp(true_op, CRM_OPERATION_SHUTDOWN_REQ) == 0 + || strcmp(true_op, CRM_OPERATION_ERASE) == 0) { // these are always unexpected, trigger the PE send_abort(); - } else if(strcmp(op, CRM_OPERATION_UPDATE) == 0) { + } else if(strcmp(true_op, CRM_OPERATION_UPDATE) == 0) { // this may not be un-expected - extract_event(msg); + if(extract_event(msg) == FALSE){ + send_abort(); + } } else { cl_log(LOG_ERR, "Did not expect copy of action %s", op); } } else if(strcmp(op, "abort") == 0) { initialize_graph(); } else if(strcmp(op, "quit") == 0) { cl_log(LOG_WARNING, "Received quit message, terminating"); exit(0); } return TRUE; } void send_abort(void) { xmlNodePtr options = create_xml_node(NULL, "options"); CRM_DEBUG("Sending \"abort\" message"); set_xml_property_copy(options, XML_ATTR_OP, "te_abort"); send_ipc_request(crm_ch, options, NULL, NULL, "dc", "tengine", NULL, NULL); free_xml(options); } void send_success(void) { xmlNodePtr options = create_xml_node(NULL, "options"); CRM_DEBUG("Sending \"complete\" message"); set_xml_property_copy(options, XML_ATTR_OP, "te_complete"); send_ipc_request(crm_ch, options, NULL, NULL, "dc", "tengine", NULL, NULL); free_xml(options); }