diff --git a/crmd/lrm.c b/crmd/lrm.c index 9c0118ee95..27ca39e8e2 100644 --- a/crmd/lrm.c +++ b/crmd/lrm.c @@ -1,1899 +1,1901 @@ /* * Copyright (C) 2004 Andrew Beekhof * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #include #include #include #include #include #include #include /* for access */ #include #include #include #include #include #include #include #include #include #include #include struct recurring_op_s { char *rsc_id; char *op_key; int call_id; int interval; gboolean remove; gboolean cancelled; }; char *make_stop_id(const char *rsc, int call_id); void cib_rsc_callback(xmlNode *msg, int call_id, int rc, xmlNode *output, void *user_data); gboolean build_operation_update( xmlNode *rsc_list, lrm_op_t *op, const char *src, int lpc); gboolean build_active_RAs(xmlNode *rsc_list); gboolean is_rsc_active(const char *rsc_id); void do_update_resource(lrm_op_t *op); gboolean process_lrm_event(lrm_op_t *op); void do_lrm_rsc_op(lrm_rsc_t *rsc, const char *operation, xmlNode *msg, xmlNode *request); lrm_op_t *construct_op( xmlNode *rsc_op, const char *rsc_id, const char *operation); void send_direct_ack(const char *to_host, const char *to_sys, lrm_op_t* op, const char *rsc_id); void free_recurring_op(gpointer value); GHashTable *meta_hash = NULL; GHashTable *resources = NULL; GHashTable *pending_ops = NULL; GCHSource *lrm_source = NULL; int num_lrm_register_fails = 0; int max_lrm_register_fails = 30; /* A_LRM_CONNECT */ void do_lrm_control(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t *msg_data) { int ret = HA_OK; if(action & A_LRM_DISCONNECT) { if(verify_stopped(cur_state, LOG_INFO) == FALSE) { crmd_fsa_stall(NULL); return; } if(lrm_source) { crm_debug("Removing LRM connection from MainLoop"); if(G_main_del_IPC_Channel(lrm_source) == FALSE) { crm_err("Could not remove LRM connection" " from MainLoop"); } lrm_source = NULL; } if(fsa_lrm_conn) { fsa_lrm_conn->lrm_ops->signoff(fsa_lrm_conn); crm_info("Disconnected from the LRM"); clear_bit_inplace(fsa_input_register, R_LRM_CONNECTED); } /* TODO: Clean up the hashtable */ } if(action & A_LRM_CONNECT) { ret = HA_OK; pending_ops = g_hash_table_new_full( g_str_hash, g_str_equal, g_hash_destroy_str, free_recurring_op); resources = g_hash_table_new_full( g_str_hash, g_str_equal, g_hash_destroy_str, g_hash_destroy_str); if(NULL == fsa_lrm_conn) { register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL); ret = HA_FAIL; } if(ret == HA_OK) { crm_debug("Connecting to the LRM"); ret = fsa_lrm_conn->lrm_ops->signon( fsa_lrm_conn, CRM_SYSTEM_CRMD); } if(ret != HA_OK) { if(++num_lrm_register_fails < max_lrm_register_fails) { crm_warn("Failed to sign on to the LRM %d" " (%d max) times", num_lrm_register_fails, max_lrm_register_fails); crm_timer_start(wait_timer); crmd_fsa_stall(NULL); return; } } if(ret == HA_OK) { crm_debug_4("LRM: set_lrm_callback..."); ret = fsa_lrm_conn->lrm_ops->set_lrm_callback( fsa_lrm_conn, lrm_op_callback); if(ret != HA_OK) { crm_err("Failed to set LRM callbacks"); } } if(ret != HA_OK) { crm_err("Failed to sign on to the LRM %d" " (max) times", num_lrm_register_fails); register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL); return; } /* TODO: create a destroy handler that causes * some recovery to happen */ lrm_source = G_main_add_IPC_Channel( G_PRIORITY_LOW, fsa_lrm_conn->lrm_ops->ipcchan(fsa_lrm_conn), FALSE, lrm_dispatch, fsa_lrm_conn, default_ipc_connection_destroy); set_bit_inplace(fsa_input_register, R_LRM_CONNECTED); crm_debug("LRM connection established"); } if(action & ~(A_LRM_CONNECT|A_LRM_DISCONNECT)) { crm_err("Unexpected action %s in %s", fsa_action2string(action), __FUNCTION__); } } static void ghash_print_pending(gpointer key, gpointer value, gpointer user_data) { const char *stop_id = key; int *log_level = user_data; struct recurring_op_s *pending = value; do_crm_log(*log_level, "Pending action: %s (%s)", stop_id, pending->op_key); } static void ghash_print_pending_for_rsc(gpointer key, gpointer value, gpointer user_data) { const char *stop_id = key; char *rsc = user_data; struct recurring_op_s *pending = value; if(safe_str_eq(rsc, pending->rsc_id)) { do_crm_log(LOG_NOTICE, "%sction %s (%s) incomplete at shutdown", pending->interval==0?"A":"Recurring a", stop_id, pending->op_key); } } static void ghash_count_pending(gpointer key, gpointer value, gpointer user_data) { int *counter = user_data; struct recurring_op_s *pending = value; if(pending->interval > 0) { /* Ignore recurring actions in the shutdown calculations */ return; } (*counter)++; } gboolean verify_stopped(enum crmd_fsa_state cur_state, int log_level) { int counter = 0; gboolean rc = TRUE; GListPtr lrm_list = NULL; crm_debug("Checking for active resources before exit"); if(cur_state == S_TERMINATE) { log_level = LOG_ERR; } g_hash_table_foreach(pending_ops, ghash_count_pending, &counter); if(counter > 0) { rc = FALSE; do_crm_log(log_level, "%d pending LRM operations at shutdown%s", g_hash_table_size(pending_ops), cur_state == S_TERMINATE?"":"... waiting"); if(cur_state == S_TERMINATE || !is_set(fsa_input_register, R_SENT_RSC_STOP)) { g_hash_table_foreach( pending_ops, ghash_print_pending, &log_level); } goto bail; } if(lrm_source != NULL && fsa_lrm_conn != NULL) { lrm_list = fsa_lrm_conn->lrm_ops->get_all_rscs(fsa_lrm_conn); } slist_iter( rsc_id, char, lrm_list, lpc, if(is_rsc_active(rsc_id) == FALSE) { continue; } crm_err("Resource %s was active at shutdown." " You may ignore this error if it is unmanaged.", rsc_id); g_hash_table_foreach( pending_ops, ghash_print_pending_for_rsc, rsc_id); ); bail: set_bit_inplace(fsa_input_register, R_SENT_RSC_STOP); if(cur_state == S_TERMINATE) { rc = TRUE; } return rc; } static const char * get_rsc_metadata(const char *type, const char *class, const char *provider) { int len = 0; char *key = NULL; char *metadata = NULL; if(meta_hash == NULL) { meta_hash = g_hash_table_new_full( g_str_hash, g_str_equal, g_hash_destroy_str, g_hash_destroy_str); } CRM_CHECK(type != NULL, return NULL); CRM_CHECK(class != NULL, return NULL); if(provider == NULL) { provider = "heartbeat"; } len = strlen(type) + strlen(class) + strlen(provider) + 4; crm_malloc0(key, len); sprintf(key, "%s::%s:%s", type, class, provider); key[len-1] = 0; metadata = g_hash_table_lookup(meta_hash, key); if(metadata) { crm_debug_2("Returning cached metadata for %s", key); goto out; } crm_debug("Retreiving metadata for %s", key); metadata = fsa_lrm_conn->lrm_ops->get_rsc_type_metadata( fsa_lrm_conn, class, type, provider); if(metadata) { /* copy the metadata because the LRM likes using * g_alloc instead of cl_malloc */ char *m_copy = crm_strdup(metadata); g_hash_table_insert(meta_hash, key, m_copy); key = NULL; /* prevent it from being free'd */ g_free(metadata); metadata = m_copy; } else { crm_warn("No metadata found for %s", key); } out: crm_free(key); return metadata; } static GListPtr get_rsc_restart_list(lrm_rsc_t *rsc, lrm_op_t *op) { gboolean supported = FALSE; GListPtr restart_list = NULL; const char *value = NULL; const char *metadata_str = get_rsc_metadata( rsc->type, rsc->class, rsc->provider); xmlNode *params = NULL; xmlNode *actions = NULL; xmlNode *metadata = NULL; if(metadata_str == NULL) { return NULL; } metadata = string2xml(metadata_str); if(metadata == NULL) { crm_err("Metadata for %s::%s:%s is not valid XML", rsc->provider, rsc->class, rsc->type); return NULL; } actions = find_xml_node(metadata, "actions", TRUE); xml_child_iter_filter( actions, action, "action", value = crm_element_value(action, "name"); if(safe_str_eq("reload", value)) { supported = TRUE; break; } ); if(supported == FALSE) { goto cleanup; } params = find_xml_node(metadata, "parameters", TRUE); xml_child_iter_filter( params, param, "parameter", value = crm_element_value(param, "unique"); if(crm_is_true(value)) { value = crm_element_value(param, "name"); crm_debug("Attr %s is not reloadable", value); restart_list = g_list_append( restart_list, crm_strdup(value)); } ); cleanup: free_xml(metadata); return restart_list; } static void append_restart_list(xmlNode *update, lrm_op_t *op, const char *version) { int len = 0; char *list = NULL; char *digest = NULL; lrm_rsc_t *rsc = NULL; const char *value = NULL; gboolean non_empty = FALSE; xmlNode *restart = NULL; GListPtr restart_list = NULL; if(op->interval > 0) { /* monitors are not reloadable */ return; } else if(safe_str_neq(CRMD_ACTION_START, op->op_type)) { /* only starts are potentially reloadable */ return; } else if(compare_version("1.0.8", version) > 0) { crm_debug("Caller version %s does not support reloads", version); return; } rsc = fsa_lrm_conn->lrm_ops->get_rsc(fsa_lrm_conn, op->rsc_id); if(rsc == NULL) { crm_info("Resource %s no longer in the LRM", op->rsc_id); return; } restart_list = get_rsc_restart_list(rsc, op); if(restart_list == NULL) { crm_debug("Resource %s does not support reloads", op->rsc_id); return; } restart = create_xml_node(NULL, XML_TAG_PARAMS); slist_iter(param, const char, restart_list, lpc, int start = len; value = g_hash_table_lookup(op->params, param); if(value != NULL) { non_empty = TRUE; crm_xml_add(restart, param, value); } len += strlen(param) + 2; crm_realloc(list, len+1); sprintf(list+start, " %s ", param); ); digest = calculate_xml_digest(restart, TRUE, FALSE); crm_xml_add(update, XML_LRM_ATTR_OP_RESTART, list); crm_xml_add(update, XML_LRM_ATTR_RESTART_DIGEST, digest); crm_debug("%s : %s", digest, list); if(non_empty) { crm_log_xml_debug(restart, "restart digest source"); } slist_destroy(char, child, restart_list, crm_free(child)); free_xml(restart); crm_free(digest); crm_free(list); } gboolean build_operation_update( xmlNode *xml_rsc, lrm_op_t *op, const char *src, int lpc) { char *magic = NULL; const char *task = NULL; xmlNode *xml_op = NULL; char *op_id = NULL; char *local_user_data = NULL; const char *caller_version = NULL; char *digest = NULL; xmlNode *args_xml = NULL; xmlNode *args_parent = NULL; CRM_DEV_ASSERT(op != NULL); if(crm_assert_failed) { return FALSE; } crm_debug_2("%s: Updating resouce %s after %s %s op", src, op->rsc_id, op_status2text(op->op_status), op->op_type); if(op->op_status == LRM_OP_CANCELLED) { crm_debug_3("Ignoring cancelled op"); return TRUE; } if(AM_I_DC) { caller_version = CRM_FEATURE_SET; } else if(fsa_our_dc_version != NULL) { caller_version = fsa_our_dc_version; } else { /* there is a small risk in formerly mixed clusters that * it will be sub-optimal. * however with our upgrade policy, the update we send * should still be completely supported anyway */ caller_version = g_hash_table_lookup( op->params, XML_ATTR_CRM_VERSION); crm_warn("Falling back to operation originator version: %s", caller_version); } crm_debug_3("DC version: %s", caller_version); task = op->op_type; /* remap the task name under various scenarios * this makes life easier for the PE when its trying determin the current state */ if(crm_str_eq(task, "reload", TRUE)) { if(op->op_status == LRM_OP_DONE) { task = CRMD_ACTION_START; } else { task = CRMD_ACTION_STATUS; } } else if(crm_str_eq(task, CRMD_ACTION_MIGRATE, TRUE)) { /* if the migrate_from fails it will have enough info to do the right thing */ if(op->op_status == LRM_OP_DONE) { task = CRMD_ACTION_STOP; } else { task = CRMD_ACTION_STATUS; } } else if(op->op_status == LRM_OP_DONE && crm_str_eq(task, CRMD_ACTION_MIGRATED, TRUE)) { task = CRMD_ACTION_START; } if(safe_str_eq(task, CRMD_ACTION_NOTIFY)) { const char *n_type = g_hash_table_lookup( op->params, crm_meta_name("notify_type")); const char *n_task = g_hash_table_lookup( op->params, crm_meta_name("notify_operation")); #if CRM_DEPRECATED_SINCE_2_0_5 if(n_type == NULL) { n_type = g_hash_table_lookup(op->params, "notify_type"); } if(n_task == NULL) { n_task = g_hash_table_lookup(op->params, "notify_operation"); } #endif CRM_DEV_ASSERT(n_type != NULL); CRM_DEV_ASSERT(n_task != NULL); op_id = generate_notify_key(op->rsc_id, n_type, n_task); /* these are not yet allowed to fail */ op->op_status = LRM_OP_DONE; op->rc = 0; } else { op_id = generate_op_key(op->rsc_id, task, op->interval); } xml_op = find_entity(xml_rsc, XML_LRM_TAG_RSC_OP, op_id); if(xml_op != NULL) { crm_log_xml(LOG_DEBUG, "Replacing existing entry", xml_op); } else { xml_op = create_xml_node(xml_rsc, XML_LRM_TAG_RSC_OP); } crm_xml_add(xml_op, XML_ATTR_ID, op_id); crm_free(op_id); crm_xml_add(xml_op, XML_LRM_ATTR_TASK, task); crm_xml_add(xml_op, XML_ATTR_ORIGIN, src); if(op->user_data == NULL) { char *id = crm_itoa(op->call_id); crm_debug("Generating fake transition key for:" " %s_%s_%d %d from %s", op->rsc_id, op->op_type, op->interval, op->call_id, op->app_name); local_user_data = generate_transition_key(-1, 0, 0, id); op->user_data = local_user_data; crm_free(id); } if(compare_version("1.0.3", caller_version) > 0) { magic = generate_transition_magic_v202( op->user_data, op->op_status); } else { magic = generate_transition_magic( op->user_data, op->op_status, op->rc); } crm_xml_add(xml_op, XML_ATTR_TRANSITION_KEY, op->user_data); crm_xml_add(xml_op, XML_ATTR_TRANSITION_MAGIC, magic); crm_free(magic); switch(op->op_status) { case LRM_OP_PENDING: break; case LRM_OP_CANCELLED: crm_err("What to do here"); break; case LRM_OP_ERROR: case LRM_OP_TIMEOUT: case LRM_OP_NOTSUPPORTED: crm_debug_2("Resource action %s/%s %s: %d", op->rsc_id, task, op_status2text(op->op_status), op->rc); break; case LRM_OP_DONE: break; } crm_xml_add_int(xml_op, XML_LRM_ATTR_CALLID, op->call_id); /* set these on 'xml_rsc' too to make life easy for the PE */ crm_xml_add(xml_op, XML_ATTR_CRM_VERSION, caller_version); crm_xml_add_int(xml_op, XML_LRM_ATTR_RC, op->rc); crm_xml_add_int(xml_op, XML_LRM_ATTR_OPSTATUS, op->op_status); crm_xml_add_int(xml_op, XML_LRM_ATTR_INTERVAL, op->interval); if(compare_version("2.1", caller_version) <= 0) { if(op->t_run || op->t_rcchange || op->exec_time || op->queue_time) { crm_debug_2("Timing data (%s_%s_%d): last=%lu change=%lu exec=%lu queue=%lu", op->rsc_id, op->op_type, op->interval, op->t_run, op->t_rcchange, op->exec_time, op->queue_time); crm_xml_add_int(xml_op, "last_run", op->t_run); crm_xml_add_int(xml_op, "last_rc_change", op->t_rcchange); crm_xml_add_int(xml_op, "exec_time", op->exec_time); crm_xml_add_int(xml_op, "queue_time", op->queue_time); } } /* this will enable us to later determin that the * resource's parameters have changed and we should force * a restart */ args_parent = NULL; #if CRM_DEPRECATED_SINCE_2_0_4 if(compare_version("1.0.4", caller_version) > 0) { args_parent = xml_op; } #endif args_xml = create_xml_node(args_parent, XML_TAG_PARAMS); g_hash_table_foreach(op->params, hash2field, args_xml); filter_action_parameters(args_xml, caller_version); digest = calculate_xml_digest(args_xml, TRUE, FALSE); if(op->interval == 0 && safe_str_neq(task, CRMD_ACTION_STOP)) { crm_debug("Calculated digest %s for %s (%s)\n", digest, ID(xml_op), crm_element_value(xml_op, XML_ATTR_TRANSITION_MAGIC)); crm_log_xml(LOG_DEBUG, "digest:source", args_xml); } crm_xml_add(xml_op, XML_LRM_ATTR_OP_DIGEST, digest); crm_free(digest); if(args_parent == NULL) { free_xml(args_xml); } append_restart_list(xml_op, op, caller_version); if(op->op_status != LRM_OP_DONE && crm_str_eq(op->op_type, CRMD_ACTION_MIGRATED, TRUE)) { const char *host = g_hash_table_lookup( op->params, crm_meta_name("migrate_source_uuid")); crm_xml_add(xml_op, CRMD_ACTION_MIGRATED, host); } if(local_user_data) { crm_free(local_user_data); op->user_data = NULL; } return TRUE; } gboolean is_rsc_active(const char *rsc_id) { GList *op_list = NULL; gboolean active = FALSE; lrm_rsc_t *the_rsc = NULL; state_flag_t cur_state = 0; int max_call_id = -1; if(fsa_lrm_conn == NULL) { return FALSE; } the_rsc = fsa_lrm_conn->lrm_ops->get_rsc(fsa_lrm_conn, rsc_id); crm_debug_3("Processing lrm_rsc_t entry %s", rsc_id); if(the_rsc == NULL) { crm_err("NULL resource returned from the LRM"); return FALSE; } op_list = the_rsc->ops->get_cur_state(the_rsc, &cur_state); crm_debug_3("\tcurrent state:%s",cur_state==LRM_RSC_IDLE?"Idle":"Busy"); slist_iter( op, lrm_op_t, op_list, llpc, crm_debug_2("Processing op %s_%d (%d) for %s (status=%d, rc=%d)", op->op_type, op->interval, op->call_id, the_rsc->id, op->op_status, op->rc); CRM_ASSERT(max_call_id <= op->call_id); if(op->rc == EXECRA_OK && safe_str_eq(op->op_type, CRMD_ACTION_STOP)) { active = FALSE; } else if(op->rc == EXECRA_OK && safe_str_eq(op->op_type, CRMD_ACTION_MIGRATE)) { /* a stricter check is too complex... * leave that to the PE */ active = FALSE; } else if(op->rc == EXECRA_NOT_RUNNING) { active = FALSE; } else { active = TRUE; } max_call_id = op->call_id; lrm_free_op(op); ); g_list_free(op_list); lrm_free_rsc(the_rsc); return active; } gboolean build_active_RAs(xmlNode *rsc_list) { GList *op_list = NULL; GList *lrm_list = NULL; gboolean found_op = FALSE; state_flag_t cur_state = 0; if(fsa_lrm_conn == NULL) { return FALSE; } lrm_list = fsa_lrm_conn->lrm_ops->get_all_rscs(fsa_lrm_conn); slist_iter( rid, char, lrm_list, lpc, lrm_rsc_t *the_rsc = fsa_lrm_conn->lrm_ops->get_rsc(fsa_lrm_conn, rid); xmlNode *xml_rsc = create_xml_node( rsc_list, XML_LRM_TAG_RESOURCE); int max_call_id = -1; crm_debug_2("Processing lrm_rsc_t entry %s", rid); if(the_rsc == NULL) { crm_err("NULL resource returned from the LRM"); continue; } crm_xml_add(xml_rsc, XML_ATTR_ID, the_rsc->id); crm_xml_add(xml_rsc, XML_ATTR_TYPE, the_rsc->type); crm_xml_add(xml_rsc, XML_AGENT_ATTR_CLASS, the_rsc->class); crm_xml_add(xml_rsc, XML_AGENT_ATTR_PROVIDER,the_rsc->provider); op_list = the_rsc->ops->get_cur_state(the_rsc, &cur_state); crm_debug_2("\tcurrent state:%s", cur_state==LRM_RSC_IDLE?"Idle":"Busy"); slist_iter( op, lrm_op_t, op_list, llpc, crm_debug_2("Processing op %s for %s (status=%d, rc=%d)", op->op_type, the_rsc->id, op->op_status, op->rc); if(max_call_id < op->call_id) { build_operation_update( xml_rsc, op, __FUNCTION__, llpc); } else if(max_call_id > op->call_id) { crm_err("Bad call_id in list=%d. Previous call_id=%d", op->call_id, max_call_id); } else { crm_warn("lrm->get_cur_state() returned" " duplicate entries for call_id=%d", op->call_id); } max_call_id = op->call_id; found_op = TRUE; lrm_free_op(op); ); if(found_op == FALSE && g_list_length(op_list) != 0) { crm_err("Could not properly determin last op" " for %s from %d entries", the_rsc->id, g_list_length(op_list)); } g_list_free(op_list); lrm_free_rsc(the_rsc); ); g_list_free(lrm_list); return TRUE; } xmlNode* do_lrm_query(gboolean is_replace) { gboolean shut_down = FALSE; xmlNode *xml_result= NULL; xmlNode *xml_state = NULL; xmlNode *xml_data = NULL; xmlNode *rsc_list = NULL; const char *exp_state = CRMD_STATE_ACTIVE; if(is_set(fsa_input_register, R_SHUTDOWN)) { exp_state = CRMD_STATE_INACTIVE; shut_down = TRUE; } xml_state = create_node_state( fsa_our_uname, ACTIVESTATUS, XML_BOOLEAN_TRUE, ONLINESTATUS, CRMD_JOINSTATE_MEMBER, exp_state, !shut_down, __FUNCTION__); xml_data = create_xml_node(xml_state, XML_CIB_TAG_LRM); crm_xml_add(xml_data, XML_ATTR_ID, fsa_our_uuid); rsc_list = create_xml_node(xml_data, XML_LRM_TAG_RESOURCES); /* Build a list of active (not always running) resources */ build_active_RAs(rsc_list); if(is_replace) { crm_xml_add(xml_state, XML_CIB_ATTR_REPLACE, XML_CIB_TAG_LRM); } xml_result = create_cib_fragment(xml_state, XML_CIB_TAG_STATUS); free_xml(xml_state); crm_log_xml_debug_3(xml_state, "Current state of the LRM"); return xml_result; } static void delete_rsc_entry(const char *rsc_id) { xmlNode *xml_top = NULL; xmlNode *xml_tmp = NULL; /* * Remove the rsc from the CIB * * Avoids refreshing the entire LRM section of this host */ CRM_CHECK(rsc_id != NULL, return); xml_top = create_xml_node(NULL, XML_CIB_TAG_STATE); crm_xml_add(xml_top, XML_ATTR_ID, fsa_our_uuid); xml_tmp = create_xml_node(xml_top, XML_CIB_TAG_LRM); crm_xml_add(xml_tmp, XML_ATTR_ID, fsa_our_uuid); xml_tmp = create_xml_node(xml_tmp, XML_LRM_TAG_RESOURCES); xml_tmp = create_xml_node(xml_tmp, XML_LRM_TAG_RESOURCE); crm_xml_add(xml_tmp, XML_ATTR_ID, rsc_id); crm_debug("sync: Sending delete op for %s", rsc_id); fsa_cib_conn->cmds->delete_absolute(fsa_cib_conn, XML_CIB_TAG_STATUS, xml_top, NULL, cib_quorum_override); /* crm_log_xml_err(xml_top, "op:cancel"); */ free_xml(xml_top); } static void delete_op_entry(lrm_op_t *op, const char *rsc_id, const char *key, int call_id) { xmlNode *xml_top = NULL; /* * Remove the op from the CIB * * Avoids refreshing the entire LRM section of this host */ if(op != NULL) { xml_top = create_xml_node(NULL, XML_LRM_TAG_RSC_OP); crm_xml_add_int(xml_top, XML_LRM_ATTR_CALLID, op->call_id); crm_xml_add(xml_top, XML_ATTR_TRANSITION_KEY, op->user_data); crm_debug("async: Sending delete op for %s_%s_%d (call=%d)", op->rsc_id, op->op_type, op->interval, op->call_id); fsa_cib_conn->cmds->delete(fsa_cib_conn, XML_CIB_TAG_STATUS, xml_top, NULL, cib_quorum_override); } else if (rsc_id != NULL && key != NULL) { xmlNode *xml_tmp = NULL; xml_top = create_xml_node(NULL, XML_CIB_TAG_STATE); crm_xml_add(xml_top, XML_ATTR_ID, fsa_our_uuid); xml_tmp = create_xml_node(xml_top, XML_CIB_TAG_LRM); crm_xml_add(xml_tmp, XML_ATTR_ID, fsa_our_uuid); xml_tmp = create_xml_node(xml_tmp, XML_LRM_TAG_RESOURCES); xml_tmp = create_xml_node(xml_tmp, XML_LRM_TAG_RESOURCE); crm_xml_add(xml_tmp, XML_ATTR_ID, rsc_id); xml_tmp = create_xml_node(xml_tmp, XML_LRM_TAG_RSC_OP); crm_xml_add(xml_tmp, XML_ATTR_ID, key); if(call_id > 0) { crm_xml_add_int(xml_tmp, XML_LRM_ATTR_CALLID, call_id); } crm_debug("sync: Sending delete op for %s (call=%d)", key, call_id); fsa_cib_conn->cmds->delete_absolute(fsa_cib_conn, XML_CIB_TAG_STATUS, xml_top, NULL, cib_quorum_override); } else { crm_err("Not enough information to delete op entry: rsc=%p key=%p", rsc_id, key); return; } crm_log_xml_debug_2(xml_top, "op:cancel"); free_xml(xml_top); } static gboolean cancel_op(lrm_rsc_t *rsc, const char *key, int op, gboolean remove) { int rc = HA_OK; struct recurring_op_s *pending = NULL; CRM_CHECK(op != 0, return FALSE); CRM_CHECK(rsc != NULL, return FALSE); if(key == NULL) { key = make_stop_id(rsc->id, op); } pending = g_hash_table_lookup(pending_ops, key); if(pending) { if(remove && pending->remove == FALSE) { pending->remove = TRUE; crm_debug("Scheduling %s for removal", key); } if(pending->cancelled) { crm_debug("Operation %s already cancelled", key); return TRUE; } pending->cancelled = TRUE; } else { crm_info("No pending op found for %s", key); } crm_debug("Cancelling op %d for %s (%s)", op, rsc->id, key); rc = rsc->ops->cancel_op(rsc, op); if(rc != HA_OK) { crm_debug("Op %d for %s (%s): Nothing to cancel", op, rsc->id, key); /* The caller needs to make sure the entry is * removed from the pending_ops list * * Usually by returning TRUE inside the worker function * supplied to g_hash_table_foreach_remove() * * Not removing the entry from pending_ops will block * the node from shutting down */ return FALSE; } return TRUE; } struct cancel_data { gboolean done; gboolean remove; const char *key; lrm_rsc_t *rsc; }; static gboolean cancel_action_by_key(gpointer key, gpointer value, gpointer user_data) { struct cancel_data *data = user_data; struct recurring_op_s *op = (struct recurring_op_s*)value; if(safe_str_eq(op->op_key, data->key)) { data->done = TRUE; if (cancel_op(data->rsc, key, op->call_id, data->remove) == FALSE) { return TRUE; } } return FALSE; } static gboolean cancel_op_key(lrm_rsc_t *rsc, const char *key, gboolean remove) { struct cancel_data data; CRM_CHECK(rsc != NULL, return FALSE); CRM_CHECK(key != NULL, return FALSE); data.key = key; data.rsc = rsc; data.done = FALSE; data.remove = remove; g_hash_table_foreach_remove(pending_ops, cancel_action_by_key, &data); return data.done; } static lrm_rsc_t * get_lrm_resource(xmlNode *resource, xmlNode *op_msg, gboolean do_create) { char rid[64]; lrm_rsc_t *rsc = NULL; const char *short_id = ID(resource); const char *long_id = crm_element_value(resource, XML_ATTR_ID_LONG); crm_debug_2("Retrieving %s from the LRM.", short_id); CRM_CHECK(short_id != NULL, return NULL); if(rsc == NULL) { /* check if its already there (short name) */ strncpy(rid, short_id, 64); rid[63] = 0; rsc = fsa_lrm_conn->lrm_ops->get_rsc(fsa_lrm_conn, rid); } if(rsc == NULL && long_id != NULL) { /* try the long name instead */ strncpy(rid, long_id, 64); rid[63] = 0; rsc = fsa_lrm_conn->lrm_ops->get_rsc(fsa_lrm_conn, rid); } if(rsc == NULL && do_create) { /* add it to the LRM */ const char *type = crm_element_value(resource, XML_ATTR_TYPE); const char *class = crm_element_value(resource, XML_AGENT_ATTR_CLASS); const char *provider = crm_element_value(resource, XML_AGENT_ATTR_PROVIDER); GHashTable *params = xml2list(op_msg); CRM_CHECK(class != NULL, return NULL); CRM_CHECK(type != NULL, return NULL); crm_debug("Adding rsc %s before operation", short_id); strncpy(rid, short_id, 64); rid[63] = 0; #if CRM_DEPRECATED_SINCE_2_0_3 if(op_msg != NULL) { if(g_hash_table_lookup( params, XML_ATTR_CRM_VERSION) == NULL) { g_hash_table_destroy(params); params = xml2list_202(op_msg); } } #endif if(g_hash_table_size(params) == 0) { crm_log_xml_warn(op_msg, "EmptyParams"); } fsa_lrm_conn->lrm_ops->add_rsc( fsa_lrm_conn, rid, class, type, provider, params); rsc = fsa_lrm_conn->lrm_ops->get_rsc(fsa_lrm_conn, rid); g_hash_table_destroy(params); if(rsc == NULL) { fsa_data_t *msg_data = NULL; crm_err("Could not add resource %s to LRM", rid); register_fsa_error(C_FSA_INTERNAL, I_FAIL, NULL); } } return rsc; } static gboolean lrm_remove_deleted_op( gpointer key, gpointer value, gpointer user_data) { const char *rsc = user_data; struct recurring_op_s *pending = value; if(safe_str_eq(rsc, pending->rsc_id)) { crm_info("Removing op %s:%d for deleted resource %s", pending->op_key, pending->call_id, rsc); return TRUE; } return FALSE; } /* A_LRM_INVOKE */ void do_lrm_invoke(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t *msg_data) { gboolean done = FALSE; gboolean create_rsc = TRUE; const char *crm_op = NULL; const char *from_sys = NULL; const char *from_host = NULL; const char *operation = NULL; ha_msg_input_t *input = fsa_typed_data(fsa_dt_ha_msg); crm_op = crm_element_value(input->msg, F_CRM_TASK); from_sys = crm_element_value(input->msg, F_CRM_SYS_FROM); if(safe_str_neq(from_sys, CRM_SYSTEM_TENGINE)) { from_host = crm_element_value(input->msg, F_CRM_HOST_FROM); } crm_debug_2("LRM command from: %s", from_sys); if(safe_str_eq(crm_op, CRM_OP_LRM_DELETE)) { operation = CRMD_ACTION_DELETE; } else if(safe_str_eq(operation, CRM_OP_LRM_REFRESH)) { crm_op = CRM_OP_LRM_REFRESH; } else if(safe_str_eq(crm_op, CRM_OP_LRM_FAIL)) { crm_info("Failing resource..."); operation = "fail"; } else if(input->xml != NULL) { operation = crm_element_value(input->xml, XML_LRM_ATTR_TASK); } if(safe_str_eq(crm_op, CRM_OP_LRM_REFRESH)) { enum cib_errors rc = cib_ok; xmlNode *fragment = do_lrm_query(TRUE); crm_info("Forcing a local LRM refresh"); fsa_cib_update(XML_CIB_TAG_STATUS, fragment, cib_quorum_override, rc); free_xml(fragment); } else if(safe_str_eq(crm_op, CRM_OP_LRM_QUERY)) { xmlNode *data = do_lrm_query(FALSE); xmlNode *reply = create_reply(input->msg, data); if(relay_message(reply, TRUE) == FALSE) { crm_err("Unable to route reply"); crm_log_xml(LOG_ERR, "reply", reply); } free_xml(reply); free_xml(data); } else if(safe_str_eq(operation, CRM_OP_PROBED) || safe_str_eq(crm_op, CRM_OP_REPROBE)) { + int cib_options = cib_inhibit_notify; const char *probed = XML_BOOLEAN_TRUE; if(safe_str_eq(crm_op, CRM_OP_REPROBE)) { + cib_options = cib_none; probed = XML_BOOLEAN_FALSE; } update_attr(fsa_cib_conn, cib_inhibit_notify, XML_CIB_TAG_STATUS, fsa_our_uuid, NULL, NULL, CRM_OP_PROBED, probed, FALSE); } else if(operation != NULL) { lrm_rsc_t *rsc = NULL; xmlNode *params = NULL; xmlNode *xml_rsc = find_xml_node( input->xml, XML_CIB_TAG_RESOURCE, TRUE); CRM_CHECK(xml_rsc != NULL, return); /* only the first 16 chars are used by the LRM */ params = find_xml_node(input->xml, XML_TAG_ATTRS, TRUE); if(safe_str_eq(operation, CRMD_ACTION_DELETE)) { create_rsc = FALSE; } rsc = get_lrm_resource(xml_rsc, input->xml, create_rsc); if(rsc == NULL && create_rsc) { crm_err("Invalid resource definition"); crm_log_xml_warn(input->msg, "bad input"); } else if(rsc == NULL) { lrm_op_t* op = NULL; crm_err("Not creating resource for a %s event: %s", operation, ID(input->xml)); crm_log_xml_warn(input->msg, "bad input"); op = construct_op(input->xml, ID(xml_rsc), operation); op->op_status = LRM_OP_DONE; op->rc = EXECRA_OK; CRM_ASSERT(op != NULL); send_direct_ack(from_host, from_sys, op, ID(xml_rsc)); free_lrm_op(op); } else if(safe_str_eq(operation, CRMD_ACTION_CANCEL)) { lrm_op_t* op = NULL; char *op_key = NULL; int call = 0; const char *call_id = NULL; const char *op_task = NULL; const char *op_interval = NULL; CRM_CHECK(params != NULL, crm_log_xml_warn(input->xml, "Bad command"); return); op_interval = crm_element_value(params, crm_meta_name("interval")); op_task = crm_element_value(params, crm_meta_name(XML_LRM_ATTR_TASK)); call_id = crm_element_value(params, crm_meta_name(XML_LRM_ATTR_CALLID)); #if CRM_DEPRECATED_SINCE_2_0_5 if(op_interval == NULL) { op_interval = crm_element_value(params, "interval"); } if(op_task == NULL) { op_task = crm_element_value(params, XML_LRM_ATTR_TASK); if(op_task == NULL) { op_task = crm_element_value(params, "task"); } } #endif CRM_CHECK(op_task != NULL, crm_log_xml_warn(input->xml, "Bad command"); return); CRM_CHECK(op_interval != NULL, crm_log_xml_warn(input->xml, "Bad command"); return); op = construct_op(input->xml, rsc->id, op_task); CRM_ASSERT(op != NULL); op_key = generate_op_key( rsc->id,op_task,crm_parse_int(op_interval,"0")); crm_debug("PE requested op %s (call=%s) be cancelled", op_key, call_id?call_id:"NA"); call = crm_parse_int(call_id, "0"); if(call == 0) { /* the normal case when the PE cancels a recurring op */ done = cancel_op_key(rsc, op_key, TRUE); } else { /* the normal case when the PE cancels an orphan op */ done = cancel_op(rsc, NULL, call, TRUE); } if(done == FALSE) { crm_debug("Nothing known about operation %d for %s", call, op_key); delete_op_entry(NULL, rsc->id, op_key, call); /* needed?? surely not otherwise the cancel_op_(_key) wouldn't * have failed in the first place */ g_hash_table_remove(pending_ops, op_key); } op->rc = EXECRA_OK; op->op_status = LRM_OP_DONE; send_direct_ack(from_host, from_sys, op, rsc->id); crm_free(op_key); free_lrm_op(op); } else if(safe_str_eq(operation, CRMD_ACTION_DELETE)) { int rc = HA_OK; lrm_op_t* op = NULL; CRM_ASSERT(rsc != NULL); op = construct_op(input->xml, rsc->id, operation); CRM_ASSERT(op != NULL); op->op_status = LRM_OP_DONE; op->rc = EXECRA_OK; crm_info("Removing resource %s from the LRM", rsc->id); rc = fsa_lrm_conn->lrm_ops->delete_rsc(fsa_lrm_conn, rsc->id); if(rc != HA_OK) { crm_err("Failed to remove resource %s", rsc->id); op->op_status = LRM_OP_ERROR; op->rc = EXECRA_UNKNOWN_ERROR; } delete_rsc_entry(rsc->id); send_direct_ack(from_host, from_sys, op, rsc->id); free_lrm_op(op); g_hash_table_foreach_remove(pending_ops, lrm_remove_deleted_op, rsc->id); if(safe_str_neq(from_sys, CRM_SYSTEM_TENGINE)) { /* this isn't expected - trigger a new transition */ time_t now = time(NULL); char *now_s = crm_itoa(now); crm_debug("Triggering a refresh after %s deleted %s from the LRM", from_sys, rsc->id); update_attr(fsa_cib_conn, cib_none, XML_CIB_TAG_CRMCONFIG, NULL, NULL, NULL, "last-lrm-refresh", now_s, FALSE); crm_free(now_s); } } else if(rsc != NULL) { do_lrm_rsc_op(rsc, operation, input->xml, input->msg); } lrm_free_rsc(rsc); } else { crm_err("Operation was neither a lrm_query, nor a rsc op. %s", crm_str(crm_op)); register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL); } } lrm_op_t * construct_op(xmlNode *rsc_op, const char *rsc_id, const char *operation) { lrm_op_t *op = NULL; const char *op_delay = NULL; const char *op_timeout = NULL; const char *op_interval = NULL; const char *transition = NULL; CRM_DEV_ASSERT(rsc_id != NULL); crm_malloc0(op, sizeof(lrm_op_t)); op->op_type = crm_strdup(operation); op->op_status = LRM_OP_PENDING; op->rc = -1; op->rsc_id = crm_strdup(rsc_id); op->interval = 0; op->timeout = 0; op->start_delay = 0; op->copyparams = 0; op->app_name = crm_strdup(CRM_SYSTEM_CRMD); if(rsc_op == NULL) { CRM_DEV_ASSERT(safe_str_eq(CRMD_ACTION_STOP, operation)); op->user_data = NULL; op->user_data_len = 0; /* the stop_all_resources() case * by definition there is no DC (or they'd be shutting * us down). * So we should put our version here. */ op->params = g_hash_table_new_full( g_str_hash, g_str_equal, g_hash_destroy_str, g_hash_destroy_str); g_hash_table_insert(op->params, crm_strdup(XML_ATTR_CRM_VERSION), crm_strdup(CRM_FEATURE_SET)); crm_debug_2("Constructed %s op for %s", operation, rsc_id); return op; } op->params = xml2list(rsc_op); #if CRM_DEPRECATED_SINCE_2_0_3 if(g_hash_table_lookup(op->params, XML_ATTR_CRM_VERSION) == NULL) { g_hash_table_destroy(op->params); op->params = xml2list_202(rsc_op); } #endif if(op->params == NULL) { CRM_DEV_ASSERT(safe_str_eq(CRMD_ACTION_STOP, operation)); } op_delay = g_hash_table_lookup(op->params, crm_meta_name("start_delay")); op_timeout = g_hash_table_lookup(op->params, crm_meta_name("timeout")); op_interval = g_hash_table_lookup(op->params, crm_meta_name("interval")); #if CRM_DEPRECATED_SINCE_2_0_5 if(op_delay == NULL) { op_delay = g_hash_table_lookup(op->params, "start_delay"); } if(op_timeout == NULL) { op_timeout = g_hash_table_lookup(op->params, "timeout"); } if(op_interval == NULL) { op_interval = g_hash_table_lookup(op->params, "interval"); } #endif op->interval = crm_parse_int(op_interval, "0"); op->timeout = crm_parse_int(op_timeout, "0"); op->start_delay = crm_parse_int(op_delay, "0"); /* sanity */ if(op->interval < 0) { op->interval = 0; } if(op->timeout < 0) { op->timeout = 0; } if(op->start_delay < 0) { op->start_delay = 0; } transition = crm_element_value(rsc_op, XML_ATTR_TRANSITION_KEY); CRM_CHECK(transition != NULL, return op); op->user_data = crm_strdup(transition); op->user_data_len = 1+strlen(op->user_data); if(op->interval != 0) { if(safe_str_eq(operation, CRMD_ACTION_START) || safe_str_eq(operation, CRMD_ACTION_STOP)) { crm_err("Start and Stop actions cannot have an interval"); op->interval = 0; } } /* reset the resource's parameters? */ if(op->interval == 0) { if(safe_str_eq(CRMD_ACTION_START, operation) || safe_str_eq(CRMD_ACTION_STATUS, operation)) { op->copyparams = 1; } } crm_debug_2("Constructed %s op for %s: interval=%d", operation, rsc_id, op->interval); return op; } void send_direct_ack(const char *to_host, const char *to_sys, lrm_op_t* op, const char *rsc_id) { xmlNode *reply = NULL; xmlNode *update, *iter; xmlNode *fragment; CRM_DEV_ASSERT(op != NULL); if(crm_assert_failed) { return; } if(op->rsc_id == NULL) { CRM_DEV_ASSERT(rsc_id != NULL); op->rsc_id = crm_strdup(rsc_id); } if(to_sys == NULL) { to_sys = CRM_SYSTEM_TENGINE; } update = create_node_state( fsa_our_uname, NULL, NULL, NULL, NULL, NULL, FALSE, __FUNCTION__); iter = create_xml_node(update, XML_CIB_TAG_LRM); crm_xml_add(iter, XML_ATTR_ID, fsa_our_uuid); iter = create_xml_node(iter, XML_LRM_TAG_RESOURCES); iter = create_xml_node(iter, XML_LRM_TAG_RESOURCE); crm_xml_add(iter, XML_ATTR_ID, op->rsc_id); build_operation_update(iter, op, __FUNCTION__, 0); fragment = create_cib_fragment(update, XML_CIB_TAG_STATUS); reply = create_request(CRM_OP_INVOKE_LRM, fragment, to_host, to_sys, CRM_SYSTEM_LRMD, NULL); crm_log_xml_debug_2(update, "ACK Update"); crm_info("ACK'ing resource op %s_%s_%d from %s: %s", op->rsc_id, op->op_type, op->interval, op->user_data, crm_element_value(reply, XML_ATTR_REFERENCE)); if(relay_message(reply, TRUE) == FALSE) { crm_log_xml(LOG_ERR, "Unable to route reply", reply); } free_xml(fragment); free_xml(update); free_xml(reply); } static gboolean stop_recurring_action_by_rsc(gpointer key, gpointer value, gpointer user_data) { lrm_rsc_t *rsc = user_data; struct recurring_op_s *op = (struct recurring_op_s*)value; if(op->interval != 0 && safe_str_eq(op->rsc_id, rsc->id)) { if (cancel_op(rsc, key, op->call_id, FALSE) == FALSE) { return TRUE; } } return FALSE; } void do_lrm_rsc_op(lrm_rsc_t *rsc, const char *operation, xmlNode *msg, xmlNode *request) { int call_id = 0; char *op_id = NULL; lrm_op_t* op = NULL; fsa_data_t *msg_data = NULL; const char *transition = NULL; CRM_CHECK(rsc != NULL, return); if(msg != NULL) { transition = crm_element_value(msg, XML_ATTR_TRANSITION_KEY); if(transition == NULL) { crm_log_xml_err(msg, "Missing transition number"); } } op = construct_op(msg, rsc->id, operation); /* stop the monitor before stopping the resource */ if(crm_str_eq(operation, CRMD_ACTION_STOP, TRUE) || crm_str_eq(operation, CRMD_ACTION_DEMOTE, TRUE) || crm_str_eq(operation, CRMD_ACTION_PROMOTE, TRUE) || crm_str_eq(operation, CRMD_ACTION_MIGRATE, TRUE)) { g_hash_table_foreach_remove(pending_ops, stop_recurring_action_by_rsc, rsc); } /* now do the op */ crm_info("Performing op=%s_%s_%d key=%s)", rsc->id, operation, op->interval, transition); if(fsa_state != S_NOT_DC && fsa_state != S_TRANSITION_ENGINE) { if(safe_str_neq(operation, "fail") && safe_str_neq(operation, CRMD_ACTION_STOP)) { crm_info("Discarding attempt to perform action %s on %s" " in state %s", operation, rsc->id, fsa_state2string(fsa_state)); op->rc = 99; op->op_status = LRM_OP_ERROR; send_direct_ack(NULL, NULL, op, rsc->id); free_lrm_op(op); crm_free(op_id); return; } } op_id = generate_op_key(rsc->id, op->op_type, op->interval); if(op->interval > 0) { /* cancel it so we can then restart it without conflict */ cancel_op_key(rsc, op_id, FALSE); op->target_rc = CHANGED; } else { op->target_rc = EVERYTIME; } g_hash_table_replace(resources,crm_strdup(rsc->id), crm_strdup(op_id)); call_id = rsc->ops->perform_op(rsc, op); if(call_id <= 0) { crm_err("Operation %s on %s failed: %d", operation, rsc->id, call_id); register_fsa_error(C_FSA_INTERNAL, I_FAIL, NULL); } else { /* record all operations so we can wait * for them to complete during shutdown */ char *call_id_s = make_stop_id(rsc->id, call_id); struct recurring_op_s *pending = NULL; crm_malloc0(pending, sizeof(struct recurring_op_s)); crm_debug("Recording pending op: %d - %s %s", call_id, op_id, call_id_s); pending->call_id = call_id; pending->interval = op->interval; pending->op_key = crm_strdup(op_id); pending->rsc_id = crm_strdup(rsc->id); g_hash_table_replace(pending_ops, call_id_s, pending); } crm_free(op_id); free_lrm_op(op); return; } void free_recurring_op(gpointer value) { struct recurring_op_s *op = (struct recurring_op_s*)value; crm_free(op->rsc_id); crm_free(op->op_key); crm_free(op); } void free_lrm_op(lrm_op_t *op) { g_hash_table_destroy(op->params); crm_free(op->user_data); crm_free(op->output); crm_free(op->rsc_id); crm_free(op->op_type); crm_free(op->app_name); crm_free(op); } static void dup_attr(gpointer key, gpointer value, gpointer user_data) { g_hash_table_replace(user_data, crm_strdup(key), crm_strdup(value)); } lrm_op_t * copy_lrm_op(const lrm_op_t *op) { lrm_op_t *op_copy = NULL; CRM_DEV_ASSERT(op != NULL); if(crm_assert_failed) { return NULL; } CRM_ASSERT(op->rsc_id != NULL); crm_malloc0(op_copy, sizeof(lrm_op_t)); op_copy->op_type = crm_strdup(op->op_type); /* input fields */ op_copy->params = g_hash_table_new_full( g_str_hash, g_str_equal, g_hash_destroy_str, g_hash_destroy_str); if(op->params != NULL) { g_hash_table_foreach(op->params, dup_attr, op_copy->params); } op_copy->timeout = op->timeout; op_copy->interval = op->interval; op_copy->target_rc = op->target_rc; /* in the CRM, this is always a string */ if(op->user_data != NULL) { op_copy->user_data = crm_strdup(op->user_data); } /* output fields */ op_copy->op_status = op->op_status; op_copy->rc = op->rc; op_copy->call_id = op->call_id; op_copy->output = NULL; op_copy->rsc_id = crm_strdup(op->rsc_id); if(op->app_name != NULL) { op_copy->app_name = crm_strdup(op->app_name); } if(op->output != NULL) { op_copy->output = crm_strdup(op->output); } return op_copy; } lrm_rsc_t * copy_lrm_rsc(const lrm_rsc_t *rsc) { lrm_rsc_t *rsc_copy = NULL; if(rsc == NULL) { return NULL; } crm_malloc0(rsc_copy, sizeof(lrm_rsc_t)); rsc_copy->id = crm_strdup(rsc->id); rsc_copy->type = crm_strdup(rsc->type); rsc_copy->class = NULL; rsc_copy->provider = NULL; if(rsc->class != NULL) { rsc_copy->class = crm_strdup(rsc->class); } if(rsc->provider != NULL) { rsc_copy->provider = crm_strdup(rsc->provider); } /* GHashTable* params; */ rsc_copy->params = NULL; rsc_copy->ops = NULL; return rsc_copy; } void cib_rsc_callback(xmlNode *msg, int call_id, int rc, xmlNode *output, void *user_data) { switch(rc) { case cib_ok: case cib_diff_failed: case cib_diff_resync: crm_debug("Resource update %d complete: rc=%d", call_id, rc); break; default: crm_err("Resource update %d failed: (rc=%d) %s", call_id, rc, cib_error2string(rc)); } } void do_update_resource(lrm_op_t* op) { /* */ int rc = cib_ok; lrm_rsc_t *rsc = NULL; xmlNode *update, *iter; CRM_CHECK(op != NULL, return); update = create_node_state( fsa_our_uname, NULL, NULL, NULL, NULL, NULL, FALSE, __FUNCTION__); iter = create_xml_node(update, XML_CIB_TAG_LRM); crm_xml_add(iter, XML_ATTR_ID, fsa_our_uuid); iter = create_xml_node(iter, XML_LRM_TAG_RESOURCES); iter = create_xml_node(iter, XML_LRM_TAG_RESOURCE); crm_xml_add(iter, XML_ATTR_ID, op->rsc_id); rsc = fsa_lrm_conn->lrm_ops->get_rsc(fsa_lrm_conn, op->rsc_id); CRM_CHECK(rsc->type != NULL, crm_err("Resource %s has no value for type", op->rsc_id)); CRM_CHECK(rsc->class != NULL, crm_err("Resource %s has no value for class", op->rsc_id)); crm_xml_add(iter, XML_ATTR_TYPE, rsc->type); crm_xml_add(iter, XML_AGENT_ATTR_CLASS, rsc->class); crm_xml_add(iter, XML_AGENT_ATTR_PROVIDER,rsc->provider); lrm_free_rsc(rsc); build_operation_update(iter, op, __FUNCTION__, 0); /* make it an asyncronous call and be done with it * * Best case: * the resource state will be discovered during * the next signup or election. * * Bad case: * we are shutting down and there is no DC at the time, * but then why were we shutting down then anyway? * (probably because of an internal error) * * Worst case: * we get shot for having resources "running" when the really weren't * * the alternative however means blocking here for too long, which * isnt acceptable */ fsa_cib_update(XML_CIB_TAG_STATUS, update, cib_quorum_override, rc); /* the return code is a call number, not an error code */ crm_debug("Sent resource state update message: %d", rc); fsa_cib_conn->cmds->register_callback( fsa_cib_conn, rc, 60, FALSE, NULL, "cib_rsc_callback", cib_rsc_callback); free_xml(update); } void do_lrm_event(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input cur_input, fsa_data_t *msg_data) { CRM_CHECK(FALSE, return); } gboolean process_lrm_event(lrm_op_t *op) { char *op_id = NULL; char *op_key = NULL; int log_level = LOG_ERR; struct recurring_op_s *pending = NULL; CRM_CHECK(op != NULL, return FALSE); CRM_CHECK(op->rsc_id != NULL, return FALSE); op_key = generate_op_key(op->rsc_id, op->op_type, op->interval); switch(op->op_status) { case LRM_OP_ERROR: case LRM_OP_PENDING: case LRM_OP_NOTSUPPORTED: break; case LRM_OP_CANCELLED: log_level = LOG_INFO; break; case LRM_OP_DONE: log_level = LOG_INFO; break; case LRM_OP_TIMEOUT: log_level = LOG_DEBUG_3; crm_err("LRM operation %s (%d) %s (timeout=%dms)", op_key, op->call_id, op_status2text(op->op_status), op->timeout); break; default: crm_err("Mapping unknown status (%d) to ERROR", op->op_status); op->op_status = LRM_OP_ERROR; } if(op->op_status == LRM_OP_ERROR && (op->rc == EXECRA_RUNNING_MASTER || op->rc == EXECRA_NOT_RUNNING)) { /* Leave it up to the TE/PE to decide if this is an error */ op->op_status = LRM_OP_DONE; log_level = LOG_INFO; } do_crm_log(log_level, "LRM operation %s (call=%d, rc=%d) %s %s", op_key, op->call_id, op->rc, op_status2text(op->op_status), op->op_status==LRM_OP_ERROR?execra_code2string(op->rc):""); if(op->op_status == LRM_OP_ERROR && op->output != NULL) { crm_info("Result: %s", op->output); } op_id = make_stop_id(op->rsc_id, op->call_id); pending = g_hash_table_lookup(pending_ops, op_id); if(op->op_status != LRM_OP_CANCELLED) { do_update_resource(op); if(op->interval != 0) { goto out; } } else if(op->interval == 0) { /* no known valid reason for this to happen */ crm_err("Op %s (call=%d): Cancelled", op_key, op->call_id); } else if(pending == NULL) { crm_err("Op %s (call=%d): No 'pending' entry", op_key, op->call_id); } else if(op->user_data == NULL) { crm_err("Op %s (call=%d): No user data", op_key, op->call_id); } else if(pending->remove) { delete_op_entry(op, op->rsc_id, op_key, op->call_id); } else { crm_debug("Op %s (call=%d): no delete event required", op_key, op->call_id); } if(g_hash_table_remove(pending_ops, op_id)) { crm_debug("Op %s (call=%d, stop_id=%s): Confirmed", op_key, op->call_id, op_id); } out: crm_free(op_key); crm_free(op_id); return TRUE; } char * make_stop_id(const char *rsc, int call_id) { char *op_id = NULL; crm_malloc0(op_id, strlen(rsc) + 34); if(op_id != NULL) { snprintf(op_id, strlen(rsc) + 34, "%s:%d", rsc, call_id); } return op_id; } diff --git a/cts/CTSlab.py.in b/cts/CTSlab.py.in index 46a1d2b59c..9757d933e5 100755 --- a/cts/CTSlab.py.in +++ b/cts/CTSlab.py.in @@ -1,841 +1,841 @@ #!@PYTHON@ '''CTS: Cluster Testing System: Lab environment module ''' __copyright__=''' Copyright (C) 2001,2005 Alan Robertson Licensed under the GNU GPL. ''' # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License # as published by the Free Software Foundation; either version 2 # of the License, or (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. from UserDict import UserDict import sys, time, types, string, syslog, random, os, string, signal, traceback from CTS import ClusterManager from CM_hb import HeartbeatCM from CTStests import BSC_AddResource from socket import gethostbyname_ex tests = None cm = None old_handler = None DefaultFacility = "daemon" def sig_handler(signum, frame) : if cm != None: cm.log("Interrupted by signal %d"%signum) if signum == 10 and tests != None : tests.summarize() if signum == 15 : sys.exit(1) class ResetMechanism: def reset(self, node): raise ValueError("Abstract class member (reset)") class Stonith(ResetMechanism): def __init__(self, sttype="external/ssh", pName=None, pValue=None , path="@sbindir@/stonith"): self.pathname=path self.configName=pName self.configValue=pValue self.stonithtype=sttype def reset(self, node): if self.configValue == None : config=node else: config=self.configValue cmdstring = "%s -t '%s' -p '%s' '%s' 2>/dev/null" % (self.pathname , self.stonithtype, config, node) return (os.system(cmdstring) == 0) class Stonithd(ResetMechanism): def __init__(self, nodes, sttype = 'external/ssh'): self.sttype = sttype self.nodes = nodes self.query_cmd_pat = '@libdir@/heartbeat/stonithdtest/apitest 0 %s 20000 0' self.reset_cmd_pat = '@libdir@/heartbeat/stonithdtest/apitest 1 %s 20000 0' self.poweron_cmd_pat = '@libdir@/heartbeat/stonithdtest/apitest 2 %s 20000 0' self.poweroff_cmd_pat= '@libdir@/heartbeat/stonithdtest/apitest 3 %s 20000 0' self.lrmd_add_pat = '@libdir@/heartbeat/lrmadmin -A %s stonith ' + sttype + ' NULL hostlist=%s' self.lrmd_start_pat = '@libdir@/heartbeat/lrmadmin -E %s start 0 0 EVERYTIME' self.lrmd_stop_pat = '@libdir@/heartbeat/lrmadmin -E %s stop 0 0 EVERYTIME' self.lrmd_del_pat = '@libdir@/heartbeat/lrmadmin -D %s' self.rsc_id = 'my_stonithd_id' self.command = "@SSH@ -l root -n -x" self.command_noblock = "@SSH@ -f -l root -n -x" self.stonithd_started_nodes = [] self.fail_reason = '' def _remote_exec(self, node, cmnd): return (os.system("%s %s %s > /dev/null" % (self.command, node, cmnd)) == 0) def _remote_readlines(self, node, cmnd): f = os.popen("%s %s %s" % (self.command, node, cmnd)) return f.readlines() def _stonithd_started(self, node): return node in self.stonithd_started_nodes def _start_stonithd(self, node, hosts): hostlist = string.join(hosts, ',') lrmd_add_cmd = self.lrmd_add_pat % (self.rsc_id, hostlist) ret = self._remote_exec(node, lrmd_add_cmd) if not ret:return ret lrmd_start_cmd = self.lrmd_start_pat % self.rsc_id ret = self._remote_exec(node, lrmd_start_cmd) if not ret:return ret self.stonithd_started_nodes.append(node) return 1 def _stop_stonithd(self, node): lrmd_stop_cmd = self.lrmd_stop_pat % self.rsc_id ret = self._remote_exec(node, lrmd_stop_cmd) if not ret:return ret lrmd_del_cmd = self.lrmd_del_pat % self.rsc_id ret = self._remote_exec(node, lrmd_del_cmd) if not ret:return ret self.stonithd_started_nodes.remove(node) return 1 def _do_stonith(self, init_node, target_node, action): stonithd_started = self._stonithd_started(init_node) if not stonithd_started: ret = self._start_stonithd(init_node, [target_node]) if not ret: self.fail_reason = "failed to start stonithd on node %s" % init_node return ret command = "" if action == "RESET": command = self.reset_cmd_pat % target_node elif action == "POWEROFF": command = self.poweroff_cmd_pat % target_node elif action == "POWERON": command = self.poweron_cmd_pat % target_node else: self.fail_reason = "unknown opration type %s" % action return 0 lines = self._remote_readlines(init_node, command) result = "".join(lines) if not stonithd_started: self._stop_stonithd(init_node) index = result.find("result=0") if index == -1: self.fail_reason = "unexpected stonithd status: %s" % result return 0 return 1 # Should we randomly choose a node as init_node here if init_node not specified? def reset(self, init_node, target_node): return self._do_stonith(init_node, target_node, "RESET") def poweron(self, init_node, target_node): return self._do_stonith(init_node, target_node, "POWERON") def poweroff(self, init_node, target_node): return self._do_stonith(init_node, target_node, "POWEROFF") class Logger: TimeFormat = "%b %d %H:%M:%S\t" def __call__(self, lines): raise ValueError("Abstract class member (__call__)") def write(self, line): return self(line.rstrip()) def writelines(self, lines): for s in lines: self.write(s) return 1 def flush(self): return 1 def isatty(self): return None class SysLog(Logger): # http://docs.python.org/lib/module-syslog.html defaultsource="CTS" map = { "kernel": syslog.LOG_KERN, "user": syslog.LOG_USER, "mail": syslog.LOG_MAIL, "daemon": syslog.LOG_DAEMON, "auth": syslog.LOG_AUTH, "lpr": syslog.LOG_LPR, "news": syslog.LOG_NEWS, "uucp": syslog.LOG_UUCP, "cron": syslog.LOG_CRON, "local0": syslog.LOG_LOCAL0, "local1": syslog.LOG_LOCAL1, "local2": syslog.LOG_LOCAL2, "local3": syslog.LOG_LOCAL3, "local4": syslog.LOG_LOCAL4, "local5": syslog.LOG_LOCAL5, "local6": syslog.LOG_LOCAL6, "local7": syslog.LOG_LOCAL7, } def __init__(self, labinfo): if labinfo.has_key("syslogsource"): self.source=labinfo["syslogsource"] else: self.source=SysLog.defaultsource if labinfo.has_key("SyslogFacility"): self.facility=labinfo["SyslogFacility"] else: self.facility=DefaultFacility if SysLog.map.has_key(self.facility): self.facility=SysLog.map[self.facility] syslog.openlog(self.source, 0, self.facility) def setfacility(self, facility): self.facility = facility if SysLog.map.has_key(self.facility): self.facility=SysLog.map[self.facility] syslog.closelog() syslog.openlog(self.source, 0, self.facility) def __call__(self, lines): if isinstance(lines, types.StringType): syslog.syslog(lines) else: for line in lines: syslog.syslog(line) def name(self): return "Syslog" class StdErrLog(Logger): def __init__(self, labinfo): pass def __call__(self, lines): t = time.strftime(Logger.TimeFormat, time.localtime(time.time())) if isinstance(lines, types.StringType): sys.__stderr__.writelines([t, lines, "\n"]) else: for line in lines: sys.__stderr__.writelines([t, line, "\n"]) sys.__stderr__.flush() def name(self): return "StdErrLog" class FileLog(Logger): def __init__(self, labinfo, filename=None): if filename == None: filename=labinfo["LogFileName"] self.logfile=filename import os self.hostname = os.uname()[1]+" " self.source = "CTS: " def __call__(self, lines): fd = open(self.logfile, "a") t = time.strftime(Logger.TimeFormat, time.localtime(time.time())) if isinstance(lines, types.StringType): fd.writelines([t, self.hostname, self.source, lines, "\n"]) else: for line in lines: fd.writelines([t, self.hostname, self.source, line, "\n"]) fd.close() def name(self): return "FileLog" class CtsLab(UserDict): '''This class defines the Lab Environment for the Cluster Test System. It defines those things which are expected to change from test environment to test environment for the same cluster manager. It is where you define the set of nodes that are in your test lab what kind of reset mechanism you use, etc. This class is derived from a UserDict because we hold many different parameters of different kinds, and this provides provide a uniform and extensible interface useful for any kind of communication between the user/administrator/tester and CTS. At this point in time, it is the intent of this class to model static configuration and/or environmental data about the environment which doesn't change as the tests proceed. Well-known names (keys) are an important concept in this class. The HasMinimalKeys member function knows the minimal set of well-known names for the class. The following names are standard (well-known) at this time: nodes An array of the nodes in the cluster reset A ResetMechanism object logger An array of objects that log strings... CMclass The type of ClusterManager we are running (This is a class object, not a class instance) RandSeed Random seed. It is a triple of bytes. (optional) HAdir Base directory for HA installation The CTS code ignores names it doesn't know about/need. The individual tests have access to this information, and it is perfectly acceptable to provide hints, tweaks, fine-tuning directions or other information to the tests through this mechanism. ''' def __init__(self, nodes): self.data = {} self["nodes"] = nodes self.MinimalKeys=["nodes", "reset", "logger", "CMclass", "HAdir"] def HasMinimalKeys(self): 'Return TRUE if our object has the minimal set of keys/values in it' result = 1 for key in self.MinimalKeys: if not self.has_key(key): result = None return result def SupplyDefaults(self): if not self.has_key("logger"): self["logger"] = (SysLog(self), StdErrLog(self)) if not self.has_key("reset"): self["reset"] = Stonith() if not self.has_key("CMclass"): self["CMclass"] = HeartbeatCM if not self.has_key("LogFileName"): self["LogFileName"] = "@HA_VARLOGDIR@/ha-log" if not self.has_key("logrestartcmd"): self["logrestartcmd"] = "@INITDIR@/syslog restart" if not self.has_key("logfacility"): LogFacility = DefaultFacility # # Now set up our random number generator... # self.RandomGen = random.Random() # Get a random seed for the random number generator. if self.has_key("RandSeed"): randseed = self["RandSeed"] self.log("Random seed is: " + str(randseed)) self.RandomGen.seed(str(randseed)) else: randseed = int(time.time()) self.log("Random seed is: " + str(randseed)) self.RandomGen.seed(str(randseed)) def log(self, args): "Log using each of the supplied logging methods" for logfcn in self._logfunctions: logfcn(string.strip(args)) def debug(self, args): "Log using each of the supplied logging methods" for logfcn in self._logfunctions: if logfcn.name() != "StdErrLog": logfcn("debug: %s" % string.strip(args)) def __setitem__(self, key, value): '''Since this function gets called whenever we modify the dictionary (object), we can (and do) validate those keys that we know how to validate. For the most part, we know how to validate the "MinimalKeys" elements. ''' # # List of nodes in the system # if key == "nodes": self.Nodes = {} for node in value: # I don't think I need the IP address, etc. but this validates # the node name against /etc/hosts and/or DNS, so it's a # GoodThing(tm). try: self.Nodes[node] = gethostbyname_ex(node) except: print node+" not found in DNS... aborting" raise # # Reset Mechanism # elif key == "reset": if not issubclass(value.__class__, ResetMechanism): raise ValueError("'reset' Value must be a subclass" " of ResetMechanism") # # List of Logging Mechanism(s) # elif key == "logger": if len(value) < 1: raise ValueError("Must have at least one logging mechanism") for logger in value: if not callable(logger): raise ValueError("'logger' elements must be callable") self._logfunctions = value # # Cluster Manager Class # elif key == "CMclass": if not issubclass(value, ClusterManager): raise ValueError("'CMclass' must be a subclass of" " ClusterManager") # # Initial Random seed... # #elif key == "RandSeed": # if len(value) != 3: # raise ValueError("'Randseed' must be a 3-element list/tuple") # for elem in value: # if not isinstance(elem, types.IntType): # raise ValueError("'Randseed' list must all be ints") self.data[key] = value def IsValidNode(self, node): 'Return TRUE if the given node is valid' return self.Nodes.has_key(node) def __CheckNode(self, node): "Raise a ValueError if the given node isn't valid" if not self.IsValidNode(node): raise ValueError("Invalid node [%s] in CheckNode" % node) def RandomNode(self): '''Choose a random node from the cluster''' return self.RandomGen.choice(self["nodes"]) def ResetNode(self, node): "Reset a node, (normally) using a hardware mechanism" self.__CheckNode(node) return self["reset"].reset(node) def ResetNode2(self, init_node, target_node, reasons): self.__CheckNode(target_node) stonithd = Stonithd(self["nodes"]) ret = stonithd.reset(init_node, target_node) if not ret: reasons.append(stonithd.fail_reason) return ret def usage(arg): print "Illegal argument " + arg print "usage: " + sys.argv[0] \ + " --directory config-directory" \ + " -D config-directory" \ + " --logfile system-logfile-name" \ + " --trunc (truncate logfile before starting)" \ + " -L system-logfile-name" \ + " --limit-nodes maxnumnodes" \ + " --xmit-loss lost-rate(0.0-1.0)" \ + " --recv-loss lost-rate(0.0-1.0)" \ + " --suppressmonitoring" \ + " --syslog-facility syslog-facility" \ + " --facility syslog-facility" \ + " --choose testcase-name" \ + " --test-ip-base ip" \ + " --oprofile \"whitespace separated list of nodes to oprofile\"" \ + " (-2 |"\ + " -v2 |"\ + " --crm |"\ + " --classic)"\ + " (--populate-resources | -r)" \ + " --resource-can-stop" \ + " --stonith (1 | 0 | yes | no)" \ + " --stonith-type type" \ + " --stonith-args name=value" \ + " --standby (1 | 0 | yes | no)" \ + " --fencing (1 | 0 | yes | no)" \ + " --suppress_cib_writes (1 | 0 | yes | no)" \ + " -lstests" \ + " --seed" \ + " [number-of-iterations]" sys.exit(1) # # A little test code... # if __name__ == '__main__': from CTSaudits import AuditList from CTStests import TestList,RandomTests from CTS import Scenario, InitClusterManager, PingFest, PacketLoss, BasicSanityCheck import CM_hb HAdir = "@sysconfdir@/ha.d" LogFile = "@HA_VARLOGDIR@/ha-log-"+DefaultFacility DoStonith = 1 DoStandby = 1 DoFencing = 1 NumIter = 500 SuppressMonitoring = None Version = 1 CIBfilename = None CIBResource = 0 ClobberCIB = 0 LimitNodes = 0 TestCase = None LogFacility = None TruncateLog = 0 ResCanStop = 0 XmitLoss = "0.0" RecvLoss = "0.0" IPBase = "127.0.0.10" SuppressCib = 1 DoBSC = 0 ListTests = 0 HaveSeed = 0 oprofile = None - warn_inactive = 1 + warn_inactive = 0 StonithType = "ssh" StonithParams = None StonithParams = "hostlist=dynamic".split('=') # # The values of the rest of the parameters are now properly derived from # the configuration files. # # Stonith is configurable because it's slow, I have a few machines which # don't reboot very reliably, and it can mild damage to your machine if # you're using a real power switch. # # Standby is configurable because the test is very heartbeat specific # and I haven't written the code to set it properly yet. Patches are # being accepted... # Set the signal handler signal.signal(15, sig_handler) signal.signal(10, sig_handler) # Process arguments... skipthis=None args=sys.argv[1:] for i in range(0, len(args)): if skipthis: skipthis=None continue elif args[i] == "-D" or args[i] == "--directory": skipthis=1 HAdir = args[i+1] elif args[i] == "-l" or args[i] == "--limit-nodes": skipthis=1 LimitNodes = int(args[i+1]) elif args[i] == "-r" or args[i] == "--populate-resources": CIBResource = 1 elif args[i] == "-L" or args[i] == "--logfile": skipthis=1 LogFile = args[i+1] elif args[i] == "--test-ip-base": skipthis=1 IPBase = args[i+1] elif args[i] == "--oprofile": skipthis=1 oprofile = args[i+1].split(' ') elif args[i] == "--trunc": TruncateLog=1 elif args[i] == "-v2": Version=2 elif args[i] == "-lstests": ListTests=1 elif args[i] == "--stonith": skipthis=1 if args[i+1] == "1" or args[i+1] == "yes": DoStonith=1 elif args[i+1] == "0" or args[i+1] == "no": DoStonith=0 else: usage(args[i+1]) elif args[i] == "--stonith-type": StonithType = args[i+1] skipthis=1 elif args[i] == "--stonith-args": StonithParams = args[i+1].split('=') skipthis=1 elif args[i] == "--suppress-cib-writes": skipthis=1 if args[i+1] == "1" or args[i+1] == "yes": SuppressCib=1 elif args[i+1] == "0" or args[i+1] == "no": SuppressCib=0 else: usage(args[i+1]) elif args[i] == "--bsc": DoBSC=1 elif args[i] == "--standby": skipthis=1 if args[i+1] == "1" or args[i+1] == "yes": DoStandby=1 elif args[i+1] == "0" or args[i+1] == "no": DoStandby=0 else: usage(args[i+1]) elif args[i] == "--fencing": skipthis=1 if args[i+1] == "1" or args[i+1] == "yes": DoFencing=1 elif args[i+1] == "0" or args[i+1] == "no": DoFencing=0 else: usage(args[i+1]) elif args[i] == "--suppressmonitoring": SuppressMonitoring = 1 elif args[i] == "--resource-can-stop": ResCanStop = 1 elif args[i] == "-3" or args[i] == "--ais": Version = 3 elif args[i] == "-2" or args[i] == "--crm": Version = 2 elif args[i] == "-1" or args[i] == "--classic": Version = 1 elif args[i] == "--clobber-cib" or args[i] == "-c": ClobberCIB = 1 elif args[i] == "--cib-filename": skipthis=1 CIBfilename = args[i+1] elif args[i] == "--xmit-loss": try: float(args[i+1]) except ValueError: print ("--xmit-loss parameter should be float") usage(args[i+1]) skipthis=1 XmitLoss = args[i+1] elif args[i] == "--recv-loss": try: float(args[i+1]) except ValueError: print ("--recv-loss parameter should be float") usage(args[i+1]) skipthis=1 RecvLoss = args[i+1] elif args[i] == "--choose": skipthis=1 TestCase = args[i+1] elif args[i] == "--syslog-facility" or args[i] == "--facility": skipthis=1 LogFacility = args[i+1] elif args[i] == "--seed": skipthis=1 Seed=args[i+1] HaveSeed = 1 - elif args[i] == "--no-inactive": - warn_inactive = 0 + elif args[i] == "--warn-inactive": + warn_inactive = 1 else: NumIter=int(args[i]) if not oprofile: oprofile = [] # # This reading of HBconfig here is ugly, and I suppose ought to # be done by the Cluster manager. This would probably mean moving the # list of cluster nodes into the ClusterManager class. A good thought # for our Copious Spare Time in the future... # config = CM_hb.HBConfig(HAdir) node_list = config.Parameters["node"] if DoBSC: NumIter = 2 Version = 2 while len(node_list) > 1: node_list.pop(len(node_list)-1) if LogFacility == None: if config.Parameters.has_key("logfacility"): LogFacility = config.Parameters["logfacility"][0] else: LogFacility = DefaultFacility if LimitNodes > 0: if len(node_list) > LimitNodes: print("Limiting the number of nodes configured=%d (max=%d)" %(len(node_list), LimitNodes)) while len(node_list) > LimitNodes: node_list.pop(len(node_list)-1) if StonithParams[0] == "hostlist": StonithParams[1] = string.join(node_list, " ") # alt_list = [] # for node in node_list: # alt_list.append(string.lower(node)) # node_list = alt_list Environment = CtsLab(node_list) Environment["HAdir"] = HAdir Environment["ClobberCIB"] = ClobberCIB Environment["CIBfilename"] = CIBfilename Environment["CIBResource"] = CIBResource Environment["LogFileName"] = LogFile Environment["DoStonith"] = DoStonith Environment["SyslogFacility"] = LogFacility Environment["DoStandby"] = DoStandby Environment["DoFencing"] = DoFencing Environment["ResCanStop"] = ResCanStop Environment["SuppressMonitoring"] = SuppressMonitoring Environment["XmitLoss"] = XmitLoss Environment["RecvLoss"] = RecvLoss Environment["IPBase"] = IPBase Environment["SuppressCib"] = SuppressCib Environment["DoBSC"] = 0 Environment["use_logd"] = 0 Environment["logfacility"] = LogFacility Environment["oprofile"] = oprofile Environment["warn-inactive"] = warn_inactive if config.Parameters.has_key("use_logd"): Environment["use_logd"] = 1 if Version == 2: from CM_LinuxHAv2 import LinuxHAv2 Environment['CMclass']=LinuxHAv2 if Version == 3: from CM_ais import crm_ais Environment['CMclass'] = crm_ais Environment["DoStonith"] = 0 Environment["DoFencing"] = 0 Environment["use_logd"] = 0 if HaveSeed: Environment["RandSeed"] = Seed Environment["reset"] = Stonith(sttype=StonithType, pName=StonithParams[0], pValue=StonithParams[1]) if DoBSC: Environment["DoBSC"] = 1 Environment["ClobberCIB"] = 1 Environment["CIBResource"] = 0 Environment["logger"] = (FileLog(Environment), StdErrLog(Environment)) scenario = Scenario([ BasicSanityCheck(Environment) ]) else: scenario = Scenario( [ InitClusterManager(Environment), PacketLoss(Environment)]) Environment.SupplyDefaults() # Your basic start up the world type of test scenario... #scenario = Scenario( #[ InitClusterManager(Environment) #, PingFest(Environment)]) # Create the Cluster Manager object cm = Environment['CMclass'](Environment) if TruncateLog: cm.log("Truncating %s" % LogFile) lf = open(LogFile, "w"); if lf != None: lf.truncate(0) lf.close() cm.log(">>>>>>>>>>>>>>>> BEGINNING " + repr(NumIter) + " TESTS ") cm.log("Version: %d" % Version) cm.log("HA configuration directory: " + Environment["HAdir"]) cm.log("System log files: " + Environment["LogFileName"]) cm.log("Enable Stonith: %d" % Environment["DoStonith"]) cm.log("Enable Fencing: %d" % Environment["DoFencing"]) cm.log("Enable Standby: %d" % Environment["DoStandby"]) cm.log("Enable Resources: %d" % Environment["CIBResource"]) if Environment.has_key("SuppressMonitoring") \ and Environment["SuppressMonitoring"]: cm.log("Resource Monitoring is disabled") cm.ns.WaitForAllNodesToComeUp(config.Parameters["node"]) cm.log("Cluster nodes: ") for node in config.Parameters["node"]: (rc, lines) = cm.rsh.remote_py(node, "os", "system", "@sbindir@/crm_uuid") if not lines: cm.log(" * %s: __undefined_uuid__" % node) else: out=lines[0] out = out[:-1] cm.log(" * %s: %s" % (node, out)) Audits = AuditList(cm) Tests = [] if Environment["DoBSC"]: test = BSC_AddResource(cm) Tests.append(test) elif TestCase != None: for test in TestList(cm): if test.name == TestCase: Tests.append(test) if Tests == []: usage("--choose: No applicable/valid tests chosen") else: Tests = TestList(cm) if ListTests == 1 : cm.log("Total %d tests"%len(Tests)) for test in Tests : cm.log(str(test.name)); sys.exit(0) tests = RandomTests(scenario, cm, Tests, Audits) Environment.RandomTests = tests try : overall, detailed = tests.run(NumIter) except : cm.Env.log("Exception by %s" % sys.exc_info()[0]) for logmethod in Environment["logger"]: traceback.print_exc(50, logmethod) tests.summarize() if tests.Stats["failure"] > 0: sys.exit(tests.Stats["failure"]) elif tests.Stats["success"] != NumIter: cm.Env.log("No failure count but success != requested iterations") sys.exit(1) diff --git a/lib/crm/common/membership.c b/lib/crm/common/membership.c index 253303bbb4..249f41e542 100644 --- a/lib/crm/common/membership.c +++ b/lib/crm/common/membership.c @@ -1,392 +1,390 @@ /* * Copyright (C) 2004 Andrew Beekhof * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #include #ifndef _GNU_SOURCE # define _GNU_SOURCE #endif #include #include #include #include #include #include #include #include #include struct quorum_count_s { guint votes_max; guint votes_active; guint votes_total; guint nodes_max; guint nodes_total; }; GHashTable *crm_peer_cache = NULL; unsigned long long crm_peer_seq = 0; unsigned long long crm_max_peers = 0; struct quorum_count_s quorum_stats; gboolean crm_have_quorum = FALSE; gboolean crm_is_member_active(const crm_node_t *node) { if(safe_str_eq(node->state, CRM_NODE_MEMBER)) { return TRUE; } return FALSE; } static gboolean crm_reap_dead_member( gpointer key, gpointer value, gpointer user_data) { if(crm_is_member_active(value) == FALSE) { return TRUE; } return FALSE; } guint reap_crm_membership(void) { /* remove all dead members */ return g_hash_table_foreach_remove( crm_peer_cache, crm_reap_dead_member, NULL); } static void crm_count_member( gpointer key, gpointer value, gpointer user_data) { guint *count = user_data; if(crm_is_member_active(value)) { *count = *count + 1; } } guint crm_active_members(void) { guint count = 0; g_hash_table_foreach(crm_peer_cache, crm_count_member, &count); return count; } struct peer_count_s { uint32_t peer; guint count; }; static void crm_count_peer( gpointer key, gpointer value, gpointer user_data) { crm_node_t *node = value; struct peer_count_s *search = user_data; if(crm_is_member_active(node) && (node->processes & search->peer)) { search->count = search->count + 1; } } guint crm_active_peers(uint32_t peer) { struct peer_count_s search; search.count = 0; search.peer = peer; g_hash_table_foreach(crm_peer_cache, crm_count_peer, &search); return search.count; } void destroy_crm_node(gpointer data) { crm_node_t *node = data; crm_info("Destroying entry for node %u", node->id); crm_free(node->addr); crm_free(node->uname); crm_free(node->state); crm_free(node); } void crm_peer_init(void) { - crm_warn("Set these options via openais.conf"); - quorum_stats.votes_max = 2; quorum_stats.votes_active = 0; quorum_stats.votes_total = 0; quorum_stats.nodes_max = 1; quorum_stats.nodes_total = 0; crm_peer_destroy(); if(crm_peer_cache == NULL) { crm_peer_cache = g_hash_table_new_full( g_str_hash, g_str_equal, NULL, destroy_crm_node); } } void crm_peer_destroy(void) { if(crm_peer_cache != NULL) { g_hash_table_destroy(crm_peer_cache); crm_peer_cache = NULL; } } crm_node_t *crm_update_peer( unsigned int id, unsigned long long born, int32_t votes, uint32_t children, const char *uuid, const char *uname, const char *addr, const char *state) { crm_node_t *node = NULL; CRM_CHECK(uname != NULL, return NULL); CRM_ASSERT(crm_peer_cache != NULL); node = g_hash_table_lookup(crm_peer_cache, uname); if(node == NULL) { crm_info("Creating entry for node %s/%u/%llu", uname, id, born); CRM_CHECK(id >= 0, return NULL); CRM_CHECK(uuid != NULL, return NULL); crm_malloc0(node, sizeof(crm_node_t)); node->id = id; node->born = 0; node->processes = 0; node->uuid = crm_strdup(uuid); node->uname = crm_strdup(uname); node->votes = votes; node->addr = NULL; node->state = crm_strdup("unknown"); g_hash_table_insert(crm_peer_cache, node->uname, node); node = g_hash_table_lookup(crm_peer_cache, uname); CRM_ASSERT(node != NULL); } if(votes > 0 && node->votes != votes) { node->votes = votes; crm_info("Node %s now has %d votes", node->uname, votes); } if(id > 0 && id != node->id) { node->id = id; crm_info("Node %s now has id %u", node->uname, id); } if(children > 0 && children != node->processes) { crm_info("Node %s now has children: %.32x (%u)", node->uname, children, children); node->processes = children; } if(state != NULL) { if(node->state == NULL || crm_str_eq(node->state, state, FALSE) == FALSE) { crm_free(node->state); node->state = crm_strdup(state); crm_info("Node %s is now: %s", node->uname, state); if(crm_is_member_active(node)) { node->born = born; } else { node->born = -1; } } } if(addr != NULL) { if(node->addr == NULL || crm_str_eq(node->addr, addr, FALSE) == FALSE) { crm_free(node->addr); node->addr = crm_strdup(addr); crm_info("Node %s now has address: %s", node->uname, addr); } } return node; } crm_node_t *crm_update_ais_node(xmlNode *member, long long seq) { const char *addr = crm_element_value(member, "addr"); const char *uname = crm_element_value(member, "uname"); const char *state = crm_element_value(member, "state"); const char *id_s = crm_element_value(member, "id"); const char *votes_s = crm_element_value(member, "votes"); const char *procs_s = crm_element_value(member, "processes"); int votes = crm_int_helper(votes_s, NULL); unsigned int id = crm_int_helper(id_s, NULL); unsigned int procs = crm_int_helper(procs_s, NULL); return crm_update_peer(id, seq, votes, procs, uname, uname, addr, state); } #if SUPPORT_HEARTBEAT crm_node_t *crm_update_ccm_node( const oc_ev_membership_t *oc, int offset, const char *state) { crm_node_t *node = NULL; const char *uuid = NULL; CRM_CHECK(oc->m_array[offset].node_uname != NULL, return NULL); uuid = get_uuid(oc->m_array[offset].node_uname); node = crm_update_peer(oc->m_array[offset].node_id, oc->m_array[offset].node_born_on, -1, 0, uuid, oc->m_array[offset].node_uname, NULL, state); if(safe_str_eq(CRM_NODE_ACTIVE, state)) { crm_update_peer_proc( oc->m_array[offset].node_uname, crm_proc_ais, ONLINESTATUS); } return node; } #endif void crm_update_peer_proc(const char *uname, uint32_t flag, const char *status) { crm_node_t *node = NULL; gboolean changed = FALSE; CRM_ASSERT(crm_peer_cache != NULL); CRM_CHECK(uname != NULL, return); node = g_hash_table_lookup(crm_peer_cache, uname); CRM_CHECK(node != NULL, crm_err("Could not set %s.%s to %s", uname, peer2text(flag), status); return); if(safe_str_eq(status, ONLINESTATUS)) { if((node->processes & flag) == 0) { set_bit_inplace(node->processes, flag); changed = TRUE; } } else if(node->processes & flag) { clear_bit_inplace(node->processes, flag); changed = TRUE; } if(changed) { crm_info("%s.%s is now %s", uname, peer2text(flag), status); } } static void crm_count_quorum( gpointer key, gpointer value, gpointer user_data) { crm_node_t *node = value; quorum_stats.nodes_total += 1; quorum_stats.votes_total += node->votes; if(crm_is_member_active(node)) { quorum_stats.votes_active = quorum_stats.votes_active + node->votes; } } gboolean crm_calculate_quorum(void) { unsigned int limit = 0; gboolean quorate = TRUE; quorum_stats.votes_total = 0; quorum_stats.nodes_total = 0; quorum_stats.votes_active = 0; g_hash_table_foreach(crm_peer_cache, crm_count_quorum, NULL); if(quorum_stats.votes_total > quorum_stats.votes_max) { crm_info("Known quorum votes: %u -> %u", quorum_stats.votes_max, quorum_stats.votes_total); quorum_stats.votes_max = quorum_stats.votes_total; } if(quorum_stats.nodes_total > quorum_stats.nodes_max) { crm_info("Known quorum nodes: %u -> %u", quorum_stats.nodes_max, quorum_stats.nodes_total); quorum_stats.nodes_max = quorum_stats.nodes_total; } limit = (quorum_stats.votes_max + 2) / 2; if(quorum_stats.votes_active < limit) { quorate = FALSE; } crm_debug("known: %u, available: %u, limit: %u, active: %u: %s", quorum_stats.votes_max, quorum_stats.votes_total, limit, quorum_stats.votes_active, quorate?"true":"false"); if(quorate != crm_have_quorum) { crm_notice("Membership %llu: quorum %s", crm_peer_seq, quorate?"attained":"lost"); } else { crm_debug("Membership %llu: quorum %s", crm_peer_seq, quorate?"retained":"lost"); } crm_have_quorum = quorate; return quorate; } /* Code appropriated (with permission) from cman/daemon/commands.c under GPLv2 */ #if 0 static int calculate_quorum(int allow_decrease, int max_expected, unsigned int *ret_total_votes) { struct list *nodelist; struct cluster_node *node; unsigned int total_votes = 0; unsigned int highest_expected = 0; unsigned int newquorum, q1, q2; unsigned int total_nodes = 0; list_iterate(nodelist, &cluster_members_list) { node = list_item(nodelist, struct cluster_node); if (node->state == NODESTATE_MEMBER) { highest_expected = max(highest_expected, node->expected_votes); total_votes += node->votes; total_nodes++; } } if (quorum_device && quorum_device->state == NODESTATE_MEMBER) total_votes += quorum_device->votes; if (max_expected > 0) highest_expected = max_expected; /* This quorum calculation is taken from the OpenVMS Cluster Systems * manual, but, then, you guessed that didn't you */ q1 = (highest_expected + 2) / 2; q2 = (total_votes + 2) / 2; newquorum = max(q1, q2); /* Normally quorum never decreases but the system administrator can * force it down by setting expected votes to a maximum value */ if (!allow_decrease) newquorum = max(quorum, newquorum); /* The special two_node mode allows each of the two nodes to retain * quorum if the other fails. Only one of the two should live past * fencing (as both nodes try to fence each other in split-brain.) * Also: if there are more than two nodes, force us inquorate to avoid * any damage or confusion. */ if (two_node && total_nodes <= 2) newquorum = 1; if (ret_total_votes) *ret_total_votes = total_votes; return newquorum; } #endif diff --git a/lib/crm/pengine/utils.c b/lib/crm/pengine/utils.c index 43477999d8..ee0b6d92b0 100644 --- a/lib/crm/pengine/utils.c +++ b/lib/crm/pengine/utils.c @@ -1,1278 +1,1313 @@ /* * Copyright (C) 2004 Andrew Beekhof * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #include #include #include #include #include #include #include #include void print_str_str(gpointer key, gpointer value, gpointer user_data); gboolean ghash_free_str_str(gpointer key, gpointer value, gpointer user_data); void unpack_operation( action_t *action, xmlNode *xml_obj, pe_working_set_t* data_set); void pe_free_shallow(GListPtr alist) { pe_free_shallow_adv(alist, TRUE); } void pe_free_shallow_adv(GListPtr alist, gboolean with_data) { GListPtr item; GListPtr item_next = alist; if(with_data == FALSE && alist != NULL) { g_list_free(alist); return; } while(item_next != NULL) { item = item_next; item_next = item_next->next; if(with_data) { /* crm_debug_5("freeing %p", item->data); */ crm_free(item->data); } item->data = NULL; item->next = NULL; g_list_free_1(item); } } node_t * node_copy(node_t *this_node) { node_t *new_node = NULL; CRM_CHECK(this_node != NULL, return NULL); crm_malloc0(new_node, sizeof(node_t)); CRM_ASSERT(new_node != NULL); crm_debug_5("Copying %p (%s) to %p", this_node, this_node->details->uname, new_node); new_node->weight = this_node->weight; new_node->fixed = this_node->fixed; new_node->details = this_node->details; return new_node; } /* are the contents of list1 and list2 equal * nodes with weight < 0 are ignored if filter == TRUE * * slow but linear * */ gboolean node_list_eq(GListPtr list1, GListPtr list2, gboolean filter) { node_t *other_node; GListPtr lhs = list1; GListPtr rhs = list2; slist_iter( node, node_t, lhs, lpc, if(node == NULL || (filter && node->weight < 0)) { continue; } other_node = (node_t*) pe_find_node_id(rhs, node->details->id); if(other_node == NULL || other_node->weight < 0) { return FALSE; } ); lhs = list2; rhs = list1; slist_iter( node, node_t, lhs, lpc, if(node == NULL || (filter && node->weight < 0)) { continue; } other_node = (node_t*) pe_find_node_id(rhs, node->details->id); if(other_node == NULL || other_node->weight < 0) { return FALSE; } ); return TRUE; } -/* the intersection of list1 and list2 - */ +/* any node in list1 or list2 and not in the other gets a score of -INFINITY */ +GListPtr +node_list_exclude(GListPtr list1, GListPtr list2) +{ + node_t *other_node = NULL; + GListPtr result = NULL; + + result = node_list_dup(list1, FALSE, FALSE); + + slist_iter( + node, node_t, result, lpc, + + other_node = pe_find_node_id(list2, node->details->id); + + if(other_node == NULL) { + node->weight = -INFINITY; + } else { + node->weight = merge_weights(node->weight, other_node->weight); + } + ); + + slist_iter( + node, node_t, list2, lpc, + + other_node = pe_find_node_id(result, node->details->id); + + if(other_node == NULL) { + node_t *new_node = node_copy(node); + new_node->weight = -INFINITY; + result = g_list_append(result, new_node); + } + ); + + return result; +} + +/* the intersection of list1 and list2 */ GListPtr node_list_and(GListPtr list1, GListPtr list2, gboolean filter) { GListPtr result = NULL; unsigned lpc = 0; for(lpc = 0; lpc < g_list_length(list1); lpc++) { node_t *node = (node_t*)g_list_nth_data(list1, lpc); node_t *other_node = pe_find_node_id(list2, node->details->id); node_t *new_node = NULL; if(other_node != NULL) { new_node = node_copy(node); } if(new_node != NULL) { crm_debug_4("%s: %d + %d", node->details->uname, other_node->weight, new_node->weight); new_node->weight = merge_weights( new_node->weight, other_node->weight); crm_debug_3("New node weight for %s: %d", new_node->details->uname, new_node->weight); if(filter && new_node->weight < 0) { crm_free(new_node); new_node = NULL; } } if(new_node != NULL) { result = g_list_append(result, new_node); } } return result; } /* list1 - list2 */ GListPtr node_list_minus(GListPtr list1, GListPtr list2, gboolean filter) { GListPtr result = NULL; slist_iter( node, node_t, list1, lpc, node_t *other_node = pe_find_node_id(list2, node->details->id); node_t *new_node = NULL; if(node == NULL || other_node != NULL || (filter && node->weight < 0)) { continue; } new_node = node_copy(node); result = g_list_append(result, new_node); ); crm_debug_3("Minus result len: %d", g_list_length(result)); return result; } /* list1 + list2 - (intersection of list1 and list2) */ GListPtr node_list_xor(GListPtr list1, GListPtr list2, gboolean filter) { GListPtr result = NULL; slist_iter( node, node_t, list1, lpc, node_t *new_node = NULL; node_t *other_node = (node_t*) pe_find_node_id(list2, node->details->id); if(node == NULL || other_node != NULL || (filter && node->weight < 0)) { continue; } new_node = node_copy(node); result = g_list_append(result, new_node); ); slist_iter( node, node_t, list2, lpc, node_t *new_node = NULL; node_t *other_node = (node_t*) pe_find_node_id(list1, node->details->id); if(node == NULL || other_node != NULL || (filter && node->weight < 0)) { continue; } new_node = node_copy(node); result = g_list_append(result, new_node); ); crm_debug_3("Xor result len: %d", g_list_length(result)); return result; } GListPtr node_list_or(GListPtr list1, GListPtr list2, gboolean filter) { node_t *other_node = NULL; GListPtr result = NULL; gboolean needs_filter = FALSE; result = node_list_dup(list1, FALSE, filter); slist_iter( node, node_t, list2, lpc, if(node == NULL) { continue; } other_node = (node_t*)pe_find_node_id( result, node->details->id); if(other_node != NULL) { crm_debug_4("%s + %s: %d + %d", node->details->uname, other_node->details->uname, node->weight, other_node->weight); other_node->weight = merge_weights( other_node->weight, node->weight); if(filter && node->weight < 0) { needs_filter = TRUE; } } else if(filter == FALSE || node->weight >= 0) { node_t *new_node = node_copy(node); result = g_list_append(result, new_node); } ); /* not the neatest way, but the most expedient for now */ if(filter && needs_filter) { GListPtr old_result = result; result = node_list_dup(old_result, FALSE, filter); pe_free_shallow_adv(old_result, TRUE); } return result; } GListPtr node_list_dup(GListPtr list1, gboolean reset, gboolean filter) { GListPtr result = NULL; slist_iter( this_node, node_t, list1, lpc, node_t *new_node = NULL; if(filter && this_node->weight < 0) { continue; } new_node = node_copy(this_node); if(reset) { new_node->weight = 0; } if(new_node != NULL) { result = g_list_append(result, new_node); } ); return result; } void dump_node_scores(int level, resource_t *rsc, const char *comment, GListPtr nodes) { GListPtr list = nodes; if(rsc) { list = rsc->allowed_nodes; } slist_iter( node, node_t, list, lpc, if(rsc) { do_crm_log(level, "%s: %s allocation score on %s: %d", comment, rsc->id, node->details->uname, node->weight); } else { do_crm_log(level, "%s: %s = %d", comment, node->details->uname, node->weight); } ); if(rsc && rsc->children) { slist_iter( child, resource_t, rsc->children, lpc, dump_node_scores(level, child, comment, nodes); ); } } gint sort_rsc_index(gconstpointer a, gconstpointer b) { const resource_t *resource1 = (const resource_t*)a; const resource_t *resource2 = (const resource_t*)b; if(a == NULL && b == NULL) { return 0; } if(a == NULL) { return 1; } if(b == NULL) { return -1; } if(resource1->sort_index > resource2->sort_index) { return -1; } if(resource1->sort_index < resource2->sort_index) { return 1; } return 0; } gint sort_rsc_priority(gconstpointer a, gconstpointer b) { const resource_t *resource1 = (const resource_t*)a; const resource_t *resource2 = (const resource_t*)b; if(a == NULL && b == NULL) { return 0; } if(a == NULL) { return 1; } if(b == NULL) { return -1; } if(resource1->priority > resource2->priority) { return -1; } if(resource1->priority < resource2->priority) { return 1; } return 0; } action_t * custom_action(resource_t *rsc, char *key, const char *task, node_t *on_node, gboolean optional, gboolean save_action, pe_working_set_t *data_set) { action_t *action = NULL; GListPtr possible_matches = NULL; CRM_CHECK(key != NULL, return NULL); CRM_CHECK(task != NULL, return NULL); if(save_action && rsc != NULL) { possible_matches = find_actions(rsc->actions, key, on_node); } if(possible_matches != NULL) { crm_free(key); if(g_list_length(possible_matches) > 1) { pe_warn("Action %s for %s on %s exists %d times", task, rsc?rsc->id:"", on_node?on_node->details->uname:"", g_list_length(possible_matches)); } action = g_list_nth_data(possible_matches, 0); crm_debug_4("Found existing action (%d) %s for %s on %s", action->id, task, rsc?rsc->id:"", on_node?on_node->details->uname:""); g_list_free(possible_matches); } if(action == NULL) { if(save_action) { crm_debug_2("Creating%s action %d: %s for %s on %s", optional?"":" manditory", data_set->action_id, key, rsc?rsc->id:"", on_node?on_node->details->uname:""); } crm_malloc0(action, sizeof(action_t)); if(save_action) { action->id = data_set->action_id++; } else { action->id = 0; } action->rsc = rsc; CRM_ASSERT(task != NULL); action->task = crm_strdup(task); action->node = on_node; action->uuid = key; action->actions_before = NULL; action->actions_after = NULL; action->failure_is_fatal = TRUE; action->pseudo = FALSE; action->dumped = FALSE; action->runnable = TRUE; action->processed = FALSE; action->optional = optional; action->seen_count = 0; action->extra = g_hash_table_new_full( g_str_hash, g_str_equal, g_hash_destroy_str, g_hash_destroy_str); action->meta = g_hash_table_new_full( g_str_hash, g_str_equal, g_hash_destroy_str, g_hash_destroy_str); if(save_action) { data_set->actions = g_list_append( data_set->actions, action); } if(rsc != NULL) { action->op_entry = find_rsc_op_entry(rsc, key); unpack_operation( action, action->op_entry, data_set); if(save_action) { rsc->actions = g_list_append( rsc->actions, action); } } if(save_action) { crm_debug_4("Action %d created", action->id); } } if(optional == FALSE && action->optional) { crm_debug_2("Action %d (%s) marked manditory", action->id, action->uuid); action->optional = FALSE; } if(rsc != NULL) { enum action_tasks a_task = text2task(action->task); int warn_level = LOG_DEBUG_3; if(save_action) { warn_level = LOG_WARNING; } if(action->node != NULL && action->op_entry != NULL) { unpack_instance_attributes( action->op_entry, XML_TAG_ATTR_SETS, action->node->details->attrs, action->extra, NULL, FALSE, data_set->now); } if(action->pseudo) { /* leave untouched */ } else if(action->node == NULL) { action->runnable = FALSE; } else if(is_not_set(rsc->flags, pe_rsc_managed)) { do_crm_log(warn_level, "Action %s (unmanaged)", action->uuid); action->optional = TRUE; /* action->runnable = FALSE; */ } else if(action->node->details->online == FALSE) { action->runnable = FALSE; do_crm_log(warn_level, "Action %s on %s is unrunnable (offline)", action->uuid, action->node->details->uname); if(is_set(action->rsc->flags, pe_rsc_managed) && save_action && a_task == stop_rsc) { do_crm_log(warn_level, "Marking node %s unclean", action->node->details->uname); action->node->details->unclean = TRUE; } } else if(action->needs == rsc_req_nothing) { crm_debug_3("Action %s doesnt require anything", action->uuid); action->runnable = TRUE; #if 0 /* * No point checking this * - if we dont have quorum we cant stonith anyway */ } else if(action->needs == rsc_req_stonith) { crm_debug_3("Action %s requires only stonith", action->uuid); action->runnable = TRUE; #endif } else if(data_set->have_quorum == FALSE && data_set->no_quorum_policy == no_quorum_stop) { action->runnable = FALSE; crm_debug("%s\t%s (cancelled : quorum)", action->node->details->uname, action->uuid); } else if(data_set->have_quorum == FALSE && data_set->no_quorum_policy == no_quorum_freeze) { crm_debug_3("Check resource is already active"); if(rsc->fns->active(rsc, TRUE) == FALSE) { action->runnable = FALSE; crm_debug("%s\t%s (cancelled : quorum freeze)", action->node->details->uname, action->uuid); } } else { crm_debug_3("Action %s is runnable", action->uuid); action->runnable = TRUE; } if(save_action) { switch(a_task) { case stop_rsc: set_bit(rsc->flags, pe_rsc_stopping); break; case start_rsc: clear_bit(rsc->flags, pe_rsc_starting); if(action->runnable) { set_bit(rsc->flags, pe_rsc_starting); } break; default: break; } } } return action; } void unpack_operation( action_t *action, xmlNode *xml_obj, pe_working_set_t* data_set) { int value_i = 0; int start_delay = 0; char *value_ms = NULL; const char *class = NULL; const char *value = NULL; const char *field = NULL; CRM_CHECK(action->rsc != NULL, return); class = g_hash_table_lookup(action->rsc->meta, "class"); if(xml_obj != NULL) { value = crm_element_value(xml_obj, "prereq"); } if(value == NULL && safe_str_eq(action->task, CRMD_ACTION_START)) { value = g_hash_table_lookup(action->rsc->meta, "start_prereq"); } if(value == NULL && safe_str_neq(action->task, CRMD_ACTION_START)) { /* todo: integrate stop as an option? */ action->needs = rsc_req_nothing; value = "nothing (default)"; } else if(safe_str_eq(value, "nothing")) { action->needs = rsc_req_nothing; } else if(safe_str_eq(value, "quorum")) { action->needs = rsc_req_quorum; } else if(safe_str_eq(value, "fencing")) { action->needs = rsc_req_stonith; } else if(data_set->no_quorum_policy == no_quorum_ignore || safe_str_eq(class, "stonith")) { action->needs = rsc_req_nothing; value = "nothing (default)"; } else if(data_set->no_quorum_policy == no_quorum_freeze && data_set->stonith_enabled) { action->needs = rsc_req_stonith; value = "fencing (default)"; } else { action->needs = rsc_req_quorum; value = "quorum (default)"; } if(safe_str_eq(class, "stonith")) { if(action->needs == rsc_req_stonith) { crm_config_err("Stonith resources (eg. %s) cannot require" " fencing to start", action->rsc->id); } action->needs = rsc_req_nothing; value = "nothing (fencing override)"; } crm_debug_3("\tAction %s requires: %s", action->task, value); value = NULL; if(xml_obj != NULL) { value = crm_element_value(xml_obj, "on_fail"); } if(value == NULL && safe_str_eq(action->task, CRMD_ACTION_STOP)) { value = g_hash_table_lookup( action->rsc->meta, "on_stopfail"); if(value != NULL) { #if CRM_DEPRECATED_SINCE_2_0_2 crm_config_err("The \"on_stopfail\" attribute used in" " %s has been deprecated since 2.0.2", action->rsc->id); #else crm_config_err("The \"on_stopfail\" attribute used in" " %s has been deprecated since 2.0.2" " and is now disabled", action->rsc->id); value = NULL; #endif crm_config_err("Please use specify the \"on_fail\"" " attribute on the \"stop\" operation" " instead"); } } if(value == NULL) { } else if(safe_str_eq(value, "block")) { action->on_fail = action_fail_block; } else if(safe_str_eq(value, "fence")) { action->on_fail = action_fail_fence; value = "node fencing"; if(data_set->stonith_enabled == FALSE) { crm_config_err("Specifying on_fail=fence and" " stonith-enabled=false makes no sense"); action->on_fail = action_fail_stop; action->fail_role = RSC_ROLE_STOPPED; value = "stop resource"; } } else if(safe_str_eq(value, "ignore") || safe_str_eq(value, "nothing")) { action->on_fail = action_fail_ignore; value = "ignore"; } else if(safe_str_eq(value, "migrate")) { action->on_fail = action_fail_migrate; value = "force migration"; } else if(safe_str_eq(value, "stop")) { action->on_fail = action_fail_stop; action->fail_role = RSC_ROLE_STOPPED; value = "stop resource"; } else if(safe_str_eq(value, "restart")) { action->on_fail = action_fail_recover; value = "restart (and possibly migrate)"; } else { pe_err("Resource %s: Unknown failure type (%s)", action->rsc->id, value); value = NULL; } /* defaults */ if(value == NULL && safe_str_eq(action->task, CRMD_ACTION_STOP)) { if(data_set->stonith_enabled) { action->on_fail = action_fail_fence; value = "resource fence (default)"; } else { action->on_fail = action_fail_block; value = "resource block (default)"; } } else if(value == NULL && safe_str_eq(action->task, CRMD_ACTION_MIGRATED)) { action->on_fail = action_migrate_failure; value = "atomic migration recovery (default)"; } else if(value == NULL) { action->on_fail = action_fail_recover; value = "restart (and possibly migrate) (default)"; } crm_debug_3("\t%s failure handling: %s", action->task, value); value = NULL; if(xml_obj != NULL) { value = crm_element_value(xml_obj, "role_after_failure"); } if(value != NULL && action->fail_role == RSC_ROLE_UNKNOWN) { action->fail_role = text2role(value); } /* defaults */ if(action->fail_role == RSC_ROLE_UNKNOWN) { if(safe_str_eq(action->task, CRMD_ACTION_PROMOTE)) { action->fail_role = RSC_ROLE_SLAVE; } else { action->fail_role = RSC_ROLE_STARTED; } } crm_debug_3("\t%s failure results in: %s", action->task, role2text(action->fail_role)); if(xml_obj != NULL) { xml_prop_iter(xml_obj, p_name, p_value, if(p_value != NULL) { g_hash_table_insert(action->meta, crm_strdup(p_name), crm_strdup(p_value)); } ); unpack_instance_attributes(xml_obj, XML_TAG_META_SETS, NULL, action->meta, NULL, FALSE, data_set->now); unpack_instance_attributes(xml_obj, XML_TAG_ATTR_SETS, NULL, action->meta, NULL, FALSE, data_set->now); } field = XML_LRM_ATTR_INTERVAL; value = g_hash_table_lookup(action->meta, field); if(value != NULL) { value_i = crm_get_msec(value); CRM_CHECK(value_i >= 0, value_i = 0); value_ms = crm_itoa(value_i); g_hash_table_replace(action->meta, crm_strdup(field), value_ms); } field = "start_delay"; value = g_hash_table_lookup(action->meta, field); if(value != NULL) { value_i = crm_get_msec(value); if(value_i < 0) { value_i = 0; } start_delay = value_i; value_ms = crm_itoa(value_i); g_hash_table_replace(action->meta, crm_strdup(field), value_ms); } field = "timeout"; value = g_hash_table_lookup(action->meta, field); if(value == NULL) { value = pe_pref( data_set->config_hash, "default-action-timeout"); } value_i = crm_get_msec(value); if(value_i < 0) { value_i = 0; } value_i += start_delay; value_ms = crm_itoa(value_i); g_hash_table_replace(action->meta, crm_strdup(field), value_ms); } xmlNode * find_rsc_op_entry(resource_t *rsc, const char *key) { int number = 0; const char *name = NULL; const char *value = NULL; const char *interval = NULL; char *match_key = NULL; xmlNode *op = NULL; xml_child_iter_filter( rsc->ops_xml, operation, "op", name = crm_element_value(operation, "name"); interval = crm_element_value(operation, XML_LRM_ATTR_INTERVAL); value = crm_element_value(operation, "disabled"); if(crm_is_true(value)) { crm_debug_2("%s disabled", ID(operation)); continue; } number = crm_get_msec(interval); if(number < 0) { continue; } match_key = generate_op_key(rsc->id, name, number); if(safe_str_eq(key, match_key)) { op = operation; } crm_free(match_key); if(op != NULL) { return op; } ); crm_debug_3("No match for %s", key); return op; } void print_node(const char *pre_text, node_t *node, gboolean details) { if(node == NULL) { crm_debug_4("%s%s: ", pre_text==NULL?"":pre_text, pre_text==NULL?"":": "); return; } crm_debug_4("%s%s%sNode %s: (weight=%d, fixed=%s)", pre_text==NULL?"":pre_text, pre_text==NULL?"":": ", node->details==NULL?"error ":node->details->online?"":"Unavailable/Unclean ", node->details->uname, node->weight, node->fixed?"True":"False"); if(details && node != NULL && node->details != NULL) { char *pe_mutable = crm_strdup("\t\t"); crm_debug_4("\t\t===Node Attributes"); g_hash_table_foreach(node->details->attrs, print_str_str, pe_mutable); crm_free(pe_mutable); crm_debug_4("\t\t=== Resources"); slist_iter( rsc, resource_t, node->details->running_rsc, lpc, print_resource(LOG_DEBUG_4, "\t\t", rsc, FALSE); ); } } /* * Used by the HashTable for-loop */ void print_str_str(gpointer key, gpointer value, gpointer user_data) { crm_debug_4("%s%s %s ==> %s", user_data==NULL?"":(char*)user_data, user_data==NULL?"":": ", (char*)key, (char*)value); } void print_resource( int log_level, const char *pre_text, resource_t *rsc, gboolean details) { long options = pe_print_log; if(rsc == NULL) { do_crm_log(log_level-1, "%s%s: ", pre_text==NULL?"":pre_text, pre_text==NULL?"":": "); return; } if(details) { options |= pe_print_details; } rsc->fns->print(rsc, pre_text, options, &log_level); } void pe_free_action(action_t *action) { if(action == NULL) { return; } pe_free_shallow(action->actions_before);/* action_warpper_t* */ pe_free_shallow(action->actions_after); /* action_warpper_t* */ g_hash_table_destroy(action->extra); g_hash_table_destroy(action->meta); crm_free(action->task); crm_free(action->uuid); crm_free(action); } GListPtr find_recurring_actions(GListPtr input, node_t *not_on_node) { const char *value = NULL; GListPtr result = NULL; CRM_CHECK(input != NULL, return NULL); slist_iter( action, action_t, input, lpc, value = g_hash_table_lookup(action->meta, XML_LRM_ATTR_INTERVAL); if(value == NULL) { /* skip */ } else if(safe_str_eq(value, "0")) { /* skip */ } else if(safe_str_eq(CRMD_ACTION_CANCEL, action->task)) { /* skip */ } else if(not_on_node == NULL) { crm_debug_5("(null) Found: %s", action->uuid); result = g_list_append(result, action); } else if(action->node == NULL) { /* skip */ } else if(action->node->details != not_on_node->details) { crm_debug_5("Found: %s", action->uuid); result = g_list_append(result, action); } ); return result; } GListPtr find_actions(GListPtr input, const char *key, node_t *on_node) { GListPtr result = NULL; CRM_CHECK(key != NULL, return NULL); slist_iter( action, action_t, input, lpc, crm_debug_5("Matching %s against %s", key, action->uuid); if(safe_str_neq(key, action->uuid)) { continue; } else if(on_node == NULL) { result = g_list_append(result, action); } else if(action->node == NULL) { /* skip */ crm_debug_2("While looking for %s action on %s, " "found an unallocated one. Assigning" " it to the requested node...", key, on_node->details->uname); action->node = on_node; result = g_list_append(result, action); } else if(safe_str_eq(on_node->details->id, action->node->details->id)) { result = g_list_append(result, action); } ); return result; } GListPtr find_actions_exact(GListPtr input, const char *key, node_t *on_node) { GListPtr result = NULL; CRM_CHECK(key != NULL, return NULL); slist_iter( action, action_t, input, lpc, crm_debug_5("Matching %s against %s", key, action->uuid); if(safe_str_neq(key, action->uuid)) { crm_debug_3("Key mismatch: %s vs. %s", key, action->uuid); continue; } else if(on_node == NULL || action->node == NULL) { crm_debug_3("on_node=%p, action->node=%p", on_node, action->node); continue; } else if(safe_str_eq(on_node->details->id, action->node->details->id)) { result = g_list_append(result, action); } crm_debug_2("Node mismatch: %s vs. %s", on_node->details->id, action->node->details->id); ); return result; } void set_id(xmlNode * xml_obj, const char *prefix, int child) { int id_len = 0; gboolean use_prefix = TRUE; gboolean use_child = TRUE; char *new_id = NULL; const char *id = crm_element_value(xml_obj, XML_ATTR_ID); id_len = 1 + strlen(id); if(child > 999) { pe_err("Are you insane?!?" " The CRM does not support > 1000 children per resource"); return; } else if(child < 0) { use_child = FALSE; } else { id_len += 4; /* child */ } if(prefix == NULL || safe_str_eq(id, prefix)) { use_prefix = FALSE; } else { id_len += (1 + strlen(prefix)); } crm_malloc0(new_id, id_len); if(use_child) { snprintf(new_id, id_len, "%s%s%s:%d", use_prefix?prefix:"", use_prefix?":":"", id, child); } else { snprintf(new_id, id_len, "%s%s%s", use_prefix?prefix:"", use_prefix?":":"", id); } crm_xml_add(xml_obj, XML_ATTR_ID, new_id); crm_free(new_id); } static void resource_node_score(resource_t *rsc, node_t *node, int score, const char *tag) { node_t *match = NULL; if(rsc->children) { slist_iter( child_rsc, resource_t, rsc->children, lpc, resource_node_score(child_rsc, node, score, tag); ); } crm_debug_2("Setting %s for %s on %s: %d", tag, rsc->id, node->details->uname, score); match = pe_find_node_id(rsc->allowed_nodes, node->details->id); if(match == NULL) { match = node_copy(node); match->weight = 0; rsc->allowed_nodes = g_list_append(rsc->allowed_nodes, match); } match->weight = merge_weights(match->weight, score); } void resource_location(resource_t *rsc, node_t *node, int score, const char *tag, pe_working_set_t *data_set) { if(node != NULL) { resource_node_score(rsc, node, score, tag); } else if(data_set != NULL) { slist_iter( node, node_t, data_set->nodes, lpc, resource_node_score(rsc, node, score, tag); ); } else { slist_iter( node, node_t, rsc->allowed_nodes, lpc, resource_node_score(rsc, node, score, tag); ); } if(node == NULL && score == -INFINITY) { if(rsc->allocated_to) { crm_info("Deallocating %s from %s", rsc->id, rsc->allocated_to->details->uname); crm_free(rsc->allocated_to); rsc->allocated_to = NULL; } } } #define sort_return(an_int) crm_free(a_uuid); crm_free(b_uuid); return an_int gint sort_op_by_callid(gconstpointer a, gconstpointer b) { char *a_uuid = NULL; char *b_uuid = NULL; const xmlNode *xml_a = a; const xmlNode *xml_b = b; const char *a_xml_id = crm_element_value_const(xml_a, XML_ATTR_ID); const char *b_xml_id = crm_element_value_const(xml_b, XML_ATTR_ID); const char *a_task_id = crm_element_value_const(xml_a, XML_LRM_ATTR_CALLID); const char *b_task_id = crm_element_value_const(xml_b, XML_LRM_ATTR_CALLID); const char *a_key = crm_element_value_const(xml_a, XML_ATTR_TRANSITION_MAGIC); const char *b_key = crm_element_value_const(xml_b, XML_ATTR_TRANSITION_MAGIC); int dummy = -1; int a_id = -1; int b_id = -1; int a_rc = -1; int b_rc = -1; int a_status = -1; int b_status = -1; int a_call_id = -1; int b_call_id = -1; if(safe_str_eq(a_xml_id, b_xml_id)) { /* We have duplicate lrm_rsc_op entries in the status * section which is unliklely to be a good thing * - we can handle it easily enough, but we need to get * to the bottom of why its happening. */ pe_err("Duplicate lrm_rsc_op entries named %s", a_xml_id); sort_return(0); } CRM_CHECK(a_task_id != NULL && b_task_id != NULL, sort_return(0)); a_call_id = crm_parse_int(a_task_id, NULL); b_call_id = crm_parse_int(b_task_id, NULL); if(a_call_id == -1 && b_call_id == -1) { /* both are pending ops so it doesnt matter since * stops are never pending */ sort_return(0); } else if(a_call_id >= 0 && a_call_id < b_call_id) { crm_debug_4("%s (%d) < %s (%d) : call id", a_xml_id, a_call_id, b_xml_id, b_call_id); sort_return(-1); } else if(b_call_id >= 0 && a_call_id > b_call_id) { crm_debug_4("%s (%d) > %s (%d) : call id", a_xml_id, a_call_id, b_xml_id, b_call_id); sort_return(1); } crm_debug_5("%s (%d) == %s (%d) : continuing", a_xml_id, a_call_id, b_xml_id, b_call_id); /* now process pending ops */ CRM_CHECK(a_key != NULL && b_key != NULL, sort_return(0)); CRM_CHECK(decode_transition_magic( a_key, &a_uuid, &a_id, &dummy, &a_status, &a_rc, &dummy), sort_return(0)); CRM_CHECK(decode_transition_magic( b_key, &b_uuid, &b_id, &dummy, &b_status, &b_rc, &dummy), sort_return(0)); /* try and determin the relative age of the operation... * some pending operations (ie. a start) may have been supuerceeded * by a subsequent stop * * [a|b]_id == -1 means its a shutdown operation and _always_ comes last */ if(safe_str_neq(a_uuid, b_uuid) || a_id == b_id) { /* * some of the logic in here may be redundant... * * if the UUID from the TE doesnt match then one better * be a pending operation. * pending operations dont survive between elections and joins * because we query the LRM directly */ CRM_CHECK(a_call_id == -1 || b_call_id == -1, sort_return(0)); CRM_CHECK(a_call_id >= 0 || b_call_id >= 0, sort_return(0)); if(b_call_id == -1) { crm_debug_2("%s (%d) < %s (%d) : transition + call id", a_xml_id, a_call_id, b_xml_id, b_call_id); sort_return(-1); } if(a_call_id == -1) { crm_debug_2("%s (%d) > %s (%d) : transition + call id", a_xml_id, a_call_id, b_xml_id, b_call_id); sort_return(1); } } else if((a_id >= 0 && a_id < b_id) || b_id == -1) { crm_debug_3("%s (%d) < %s (%d) : transition", a_xml_id, a_id, b_xml_id, b_id); sort_return(-1); } else if((b_id >= 0 && a_id > b_id) || a_id == -1) { crm_debug_3("%s (%d) > %s (%d) : transition", a_xml_id, a_id, b_xml_id, b_id); sort_return(1); } /* we should never end up here */ crm_err("%s (%d:%d:%s) ?? %s (%d:%d:%s) : default", a_xml_id, a_call_id, a_id, a_uuid, b_xml_id, b_call_id, b_id, b_uuid); CRM_CHECK(FALSE, sort_return(0)); } time_t get_timet_now(pe_working_set_t *data_set) { time_t now = 0; if(data_set && data_set->now) { now = data_set->now->tm_now; } if(now == 0) { /* eventually we should convert data_set->now into time_tm * for now, its only triggered by PE regression tests */ now = time(NULL); crm_crit("Defaulting to 'now'"); if(data_set && data_set->now) { data_set->now->tm_now = now; } } return now; } int get_failcount(node_t *node, resource_t *rsc, int *last_failure, pe_working_set_t *data_set) { int last = 0; int fail_count = 0; resource_t *failed = rsc; char *fail_attr = crm_concat("fail-count", rsc->id, '-'); const char *value = g_hash_table_lookup(node->details->attrs, fail_attr); if(is_not_set(rsc->flags, pe_rsc_unique)) { failed = uber_parent(rsc); } if(value != NULL) { fail_count = char2score(value); crm_info("%s has failed %d on %s", rsc->id, fail_count, node->details->uname); } crm_free(fail_attr); fail_attr = crm_concat("last-failure", rsc->id, '-'); value = g_hash_table_lookup(node->details->attrs, fail_attr); if(value != NULL && rsc->failure_timeout) { last = crm_parse_int(value, NULL); if(last_failure) { *last_failure = last; } if(last > 0) { time_t now = get_timet_now(data_set); if(now > (last + rsc->failure_timeout)) { crm_notice("Failcount for %s on %s has expired (limit was %ds)", failed->id, node->details->uname, rsc->failure_timeout); fail_count = 0; } } } crm_free(fail_attr); return fail_count; } diff --git a/lib/crm/pengine/utils.h b/lib/crm/pengine/utils.h index 85ef507e6d..73fe342f00 100644 --- a/lib/crm/pengine/utils.h +++ b/lib/crm/pengine/utils.h @@ -1,125 +1,126 @@ /* * Copyright (C) 2004 Andrew Beekhof * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #ifndef PE_UTILS__H #define PE_UTILS__H #include #include extern node_t *node_copy(node_t *this_node) ; extern time_t get_timet_now(pe_working_set_t *data_set); extern int get_failcount(node_t *node, resource_t *rsc, int *last_failure, pe_working_set_t *data_set); /* Binary like operators for lists of nodes */ +extern GListPtr node_list_exclude(GListPtr list1, GListPtr list2); extern GListPtr node_list_dup(GListPtr list1, gboolean reset, gboolean filter); extern GListPtr node_list_and(GListPtr list1, GListPtr list2, gboolean filter); extern GListPtr node_list_xor(GListPtr list1, GListPtr list2, gboolean filter); extern GListPtr node_list_minus(GListPtr list1,GListPtr list2,gboolean filter); extern gboolean node_list_eq(GListPtr list1, GListPtr list2, gboolean filter); extern GListPtr node_list_or(GListPtr list1, GListPtr list2, gboolean filter); extern void pe_free_shallow(GListPtr alist); extern void pe_free_shallow_adv(GListPtr alist, gboolean with_data); /* For creating the transition graph */ extern xmlNode *action2xml(action_t *action, gboolean as_input); /* Printing functions for debug */ extern void print_node( const char *pre_text, node_t *node, gboolean details); extern void print_resource( int log_level, const char *pre_text, resource_t *rsc, gboolean details); extern void dump_node_scores(int level, resource_t *rsc, const char *comment, GListPtr nodes); /* Sorting functions */ extern gint sort_rsc_priority(gconstpointer a, gconstpointer b); extern gint sort_rsc_index(gconstpointer a, gconstpointer b); extern xmlNode *find_rsc_op_entry(resource_t *rsc, const char *key); extern action_t *custom_action( resource_t *rsc, char *key, const char *task, node_t *on_node, gboolean optional, gboolean foo, pe_working_set_t *data_set); #define delete_key(rsc) generate_op_key(rsc->id, CRMD_ACTION_DELETE, 0) #define delete_action(rsc, node, optional) custom_action( \ rsc, delete_key(rsc), CRMD_ACTION_DELETE, node, \ optional, TRUE, data_set); #define stopped_key(rsc) generate_op_key(rsc->id, CRMD_ACTION_STOPPED, 0) #define stopped_action(rsc, node, optional) custom_action( \ rsc, stopped_key(rsc), CRMD_ACTION_STOPPED, node, \ optional, TRUE, data_set); #define stop_key(rsc) generate_op_key(rsc->id, CRMD_ACTION_STOP, 0) #define stop_action(rsc, node, optional) custom_action( \ rsc, stop_key(rsc), CRMD_ACTION_STOP, node, \ optional, TRUE, data_set); #define start_key(rsc) generate_op_key(rsc->id, CRMD_ACTION_START, 0) #define start_action(rsc, node, optional) custom_action( \ rsc, start_key(rsc), CRMD_ACTION_START, node, \ optional, TRUE, data_set) #define started_key(rsc) generate_op_key(rsc->id, CRMD_ACTION_STARTED, 0) #define started_action(rsc, node, optional) custom_action( \ rsc, started_key(rsc), CRMD_ACTION_STARTED, node, \ optional, TRUE, data_set) #define promote_key(rsc) generate_op_key(rsc->id, CRMD_ACTION_PROMOTE, 0) #define promote_action(rsc, node, optional) custom_action( \ rsc, promote_key(rsc), CRMD_ACTION_PROMOTE, node, \ optional, TRUE, data_set) #define promoted_key(rsc) generate_op_key(rsc->id, CRMD_ACTION_PROMOTED, 0) #define promoted_action(rsc, node, optional) custom_action( \ rsc, promoted_key(rsc), CRMD_ACTION_PROMOTED, node, \ optional, TRUE, data_set) #define demote_key(rsc) generate_op_key(rsc->id, CRMD_ACTION_DEMOTE, 0) #define demote_action(rsc, node, optional) custom_action( \ rsc, demote_key(rsc), CRMD_ACTION_DEMOTE, node, \ optional, TRUE, data_set) #define demoted_key(rsc) generate_op_key(rsc->id, CRMD_ACTION_DEMOTED, 0) #define demoted_action(rsc, node, optional) custom_action( \ rsc, demoted_key(rsc), CRMD_ACTION_DEMOTED, node, \ optional, TRUE, data_set) extern GListPtr find_actions(GListPtr input, const char *key, node_t *on_node); extern GListPtr find_actions_exact( GListPtr input, const char *key, node_t *on_node); extern GListPtr find_recurring_actions(GListPtr input, node_t *not_on_node); extern void set_id(xmlNode *xml_obj, const char *prefix, int child); extern void pe_free_action(action_t *action); extern void resource_location(resource_t *rsc, node_t *node, int score, const char *tag, pe_working_set_t *data_set); extern gint sort_op_by_callid(gconstpointer a, gconstpointer b); #endif diff --git a/pengine/clone.c b/pengine/clone.c index 686063e64e..29ef72c3ed 100644 --- a/pengine/clone.c +++ b/pengine/clone.c @@ -1,1475 +1,1475 @@ /* * Copyright (C) 2004 Andrew Beekhof * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #include #include #include #include #include #define VARIANT_CLONE 1 #include gint sort_clone_instance(gconstpointer a, gconstpointer b); void clone_create_notifications( resource_t *rsc, action_t *action, action_t *action_complete, pe_working_set_t *data_set); void child_stopping_constraints( clone_variant_data_t *clone_data, resource_t *self, resource_t *child, resource_t *last, pe_working_set_t *data_set); void child_starting_constraints( clone_variant_data_t *clone_data, resource_t *self, resource_t *child, resource_t *last, pe_working_set_t *data_set); static node_t * parent_node_instance(const resource_t *rsc, node_t *node) { node_t *ret = NULL; if(node != NULL) { ret = pe_find_node_id( rsc->parent->allowed_nodes, node->details->id); } return ret; } static gboolean did_fail(const resource_t *rsc) { if(is_set(rsc->flags, pe_rsc_failed)) { return TRUE; } slist_iter( child_rsc, resource_t, rsc->children, lpc, if(did_fail(child_rsc)) { return TRUE; } ); return FALSE; } gint sort_clone_instance(gconstpointer a, gconstpointer b) { int level = LOG_DEBUG_3; node_t *node1 = NULL; node_t *node2 = NULL; gboolean can1 = TRUE; gboolean can2 = TRUE; const resource_t *resource1 = (const resource_t*)a; const resource_t *resource2 = (const resource_t*)b; CRM_ASSERT(resource1 != NULL); CRM_ASSERT(resource2 != NULL); /* allocation order: * - active instances * - instances running on nodes with the least copies * - active instances on nodes that cant support them or are to be fenced * - failed instances * - inactive instances */ do_crm_log(level+1, "%s ? %s", resource1->id, resource2->id); if(resource1->running_on && resource2->running_on) { if(g_list_length(resource1->running_on) < g_list_length(resource2->running_on)) { do_crm_log(level, "%s < %s: running_on", resource1->id, resource2->id); return -1; } else if(g_list_length(resource1->running_on) > g_list_length(resource2->running_on)) { do_crm_log(level, "%s > %s: running_on", resource1->id, resource2->id); return 1; } } if(resource1->running_on) { node1 = resource1->running_on->data; } if(resource2->running_on) { node2 = resource2->running_on->data; } if(node1) { node_t *match = pe_find_node_id(resource1->allowed_nodes, node1->details->id); if(match->weight < 0) { do_crm_log(level, "%s: current location is unavailable", resource1->id); node1 = NULL; can1 = FALSE; } } if(node2) { node_t *match = pe_find_node_id(resource2->allowed_nodes, node2->details->id); if(match->weight < 0) { do_crm_log(level, "%s: current location is unavailable", resource2->id); node2 = NULL; can2 = FALSE; } } if(can1 != can2) { if(can1) { do_crm_log(level, "%s < %s: availability of current location", resource1->id, resource2->id); return -1; } do_crm_log(level, "%s > %s: availability of current location", resource1->id, resource2->id); return 1; } if(resource1->priority < resource2->priority) { do_crm_log(level, "%s < %s: priority", resource1->id, resource2->id); return 1; } else if(resource1->priority > resource2->priority) { do_crm_log(level, "%s > %s: priority", resource1->id, resource2->id); return -1; } if(node1 == NULL && node2 == NULL) { do_crm_log(level, "%s == %s: not active", resource1->id, resource2->id); return 0; } if(node1 != node2) { if(node1 == NULL) { do_crm_log(level, "%s > %s: active", resource1->id, resource2->id); return 1; } else if(node2 == NULL) { do_crm_log(level, "%s < %s: active", resource1->id, resource2->id); return -1; } } can1 = can_run_resources(node1); can2 = can_run_resources(node2); if(can1 != can2) { if(can1) { do_crm_log(level, "%s < %s: can", resource1->id, resource2->id); return -1; } do_crm_log(level, "%s > %s: can", resource1->id, resource2->id); return 1; } node1 = parent_node_instance(resource1, node1); node2 = parent_node_instance(resource2, node2); if(node1 != NULL && node2 == NULL) { do_crm_log(level, "%s < %s: not allowed", resource1->id, resource2->id); return -1; } else if(node1 == NULL && node2 != NULL) { do_crm_log(level, "%s > %s: not allowed", resource1->id, resource2->id); return 1; } if(node1 == NULL) { do_crm_log(level, "%s == %s: not allowed", resource1->id, resource2->id); return 0; } if(node1->count < node2->count) { do_crm_log(level, "%s < %s: count", resource1->id, resource2->id); return -1; } else if(node1->count > node2->count) { do_crm_log(level, "%s > %s: count", resource1->id, resource2->id); return 1; } if(node1->weight < node2->weight) { do_crm_log(level, "%s < %s: node score", resource1->id, resource2->id); return 1; } else if(node1->weight > node2->weight) { do_crm_log(level, "%s > %s: node score", resource1->id, resource2->id); return -1; } can1 = did_fail(resource1); can2 = did_fail(resource2); if(can1 != can2) { if(can1) { do_crm_log(level, "%s > %s: failed", resource1->id, resource2->id); return 1; } do_crm_log(level, "%s < %s: failed", resource1->id, resource2->id); return -1; } do_crm_log(level, "%s == %s: default %d", resource1->id, resource2->id, node2->weight); return 0; } static node_t * can_run_instance(resource_t *rsc, node_t *node) { node_t *local_node = NULL; clone_variant_data_t *clone_data = NULL; if(can_run_resources(node) == FALSE) { goto bail; } local_node = parent_node_instance(rsc, node); get_clone_variant_data(clone_data, rsc->parent); if(local_node == NULL) { crm_warn("%s cannot run on %s: node not allowed", rsc->id, node->details->uname); goto bail; } else if(local_node->count < clone_data->clone_node_max) { return local_node; } else { crm_debug_2("%s cannot run on %s: node full", rsc->id, node->details->uname); } bail: if(node) { common_update_score(rsc, node->details->id, -INFINITY); } return NULL; } static node_t * color_instance(resource_t *rsc, pe_working_set_t *data_set) { node_t *chosen = NULL; node_t *local_node = NULL; crm_debug_2("Processing %s", rsc->id); if(is_not_set(rsc->flags, pe_rsc_provisional)) { return rsc->fns->location(rsc, NULL, FALSE); } else if(is_set(rsc->flags, pe_rsc_allocating)) { crm_debug("Dependancy loop detected involving %s", rsc->id); return NULL; } if(rsc->allowed_nodes) { slist_iter(try_node, node_t, rsc->allowed_nodes, lpc, can_run_instance(rsc, try_node); ); } chosen = rsc->cmds->color(rsc, data_set); if(chosen) { local_node = pe_find_node_id( rsc->parent->allowed_nodes, chosen->details->id); if(local_node) { local_node->count++; } else if(is_set(rsc->flags, pe_rsc_managed)) { /* what to do? we can't enforce per-node limits in this case */ crm_config_err("%s not found in %s (list=%d)", chosen->details->id, rsc->parent->id, g_list_length(rsc->parent->allowed_nodes)); } } return chosen; } static void append_parent_colocation(resource_t *rsc, resource_t *child, gboolean all) { slist_iter(cons, rsc_colocation_t, rsc->rsc_cons, lpc, if(all || cons->score < 0) { child->rsc_cons = g_list_append(child->rsc_cons, cons); } ); slist_iter(cons, rsc_colocation_t, rsc->rsc_cons_lhs, lpc, if(all || cons->score < 0) { child->rsc_cons_lhs = g_list_append(child->rsc_cons_lhs, cons); } ); } node_t * clone_color(resource_t *rsc, pe_working_set_t *data_set) { int allocated = 0; int available_nodes = 0; clone_variant_data_t *clone_data = NULL; get_clone_variant_data(clone_data, rsc); if(is_not_set(rsc->flags, pe_rsc_provisional)) { return NULL; } else if(is_set(rsc->flags, pe_rsc_allocating)) { crm_debug("Dependancy loop detected involving %s", rsc->id); return NULL; } set_bit(rsc->flags, pe_rsc_allocating); crm_debug_2("Processing %s", rsc->id); /* this information is used by sort_clone_instance() when deciding in which * order to allocate clone instances */ slist_iter( constraint, rsc_colocation_t, rsc->rsc_cons_lhs, lpc, rsc->allowed_nodes = constraint->rsc_lh->cmds->merge_weights( constraint->rsc_lh, rsc->id, rsc->allowed_nodes, constraint->score/INFINITY, TRUE); ); dump_node_scores(scores_log_level, rsc, __FUNCTION__, rsc->allowed_nodes); /* count now tracks the number of clones currently allocated */ slist_iter(node, node_t, rsc->allowed_nodes, lpc, node->count = 0; ); slist_iter(child, resource_t, rsc->children, lpc, if(g_list_length(child->running_on) > 0) { node_t *child_node = child->running_on->data; node_t *local_node = parent_node_instance( child, child->running_on->data); if(local_node) { local_node->count++; } else { crm_err("%s is running on %s which isn't allowed", child->id, child_node->details->uname); } } ); rsc->children = g_list_sort(rsc->children, sort_clone_instance); /* count now tracks the number of clones we have allocated */ slist_iter(node, node_t, rsc->allowed_nodes, lpc, node->count = 0; ); rsc->allowed_nodes = g_list_sort( rsc->allowed_nodes, sort_node_weight); slist_iter(node, node_t, rsc->allowed_nodes, lpc, if(can_run_resources(node) == FALSE) { available_nodes++; } ); slist_iter(child, resource_t, rsc->children, lpc, if(allocated >= clone_data->clone_max) { crm_debug("Child %s not allocated - limit reached", child->id); resource_location(child, NULL, -INFINITY, "clone_color:limit_reached", data_set); } else if (clone_data->clone_max < available_nodes) { /* Only include positive colocation preferences of dependant resources * if not every node will get a copy of the clone */ append_parent_colocation(rsc, child, TRUE); } else { append_parent_colocation(rsc, child, FALSE); } if(color_instance(child, data_set)) { allocated++; } ); crm_debug("Allocated %d %s instances of a possible %d", allocated, rsc->id, clone_data->clone_max); clear_bit(rsc->flags, pe_rsc_provisional); clear_bit(rsc->flags, pe_rsc_allocating); return NULL; } static void clone_update_pseudo_status( resource_t *rsc, gboolean *stopping, gboolean *starting) { if(rsc->children) { slist_iter(child, resource_t, rsc->children, lpc, clone_update_pseudo_status(child, stopping, starting) ); return; } CRM_ASSERT(stopping != NULL); CRM_ASSERT(starting != NULL); slist_iter( action, action_t, rsc->actions, lpc, if(*starting && *stopping) { return; } else if(action->optional) { crm_debug_3("Skipping optional: %s", action->uuid); continue; } else if(action->pseudo == FALSE && action->runnable == FALSE){ crm_debug_3("Skipping unrunnable: %s", action->uuid); continue; } else if(safe_str_eq(RSC_STOP, action->task)) { crm_debug_2("Stopping due to: %s", action->uuid); *stopping = TRUE; } else if(safe_str_eq(RSC_START, action->task)) { if(action->runnable == FALSE) { crm_debug_3("Skipping pseudo-op: %s run=%d, pseudo=%d", action->uuid, action->runnable, action->pseudo); } else { crm_debug_2("Starting due to: %s", action->uuid); crm_debug_3("%s run=%d, pseudo=%d", action->uuid, action->runnable, action->pseudo); *starting = TRUE; } } ); } void clone_create_actions(resource_t *rsc, pe_working_set_t *data_set) { gboolean child_starting = FALSE; gboolean child_stopping = FALSE; action_t *stop = NULL; action_t *start = NULL; action_t *action_complete = NULL; resource_t *last_start_rsc = NULL; resource_t *last_stop_rsc = NULL; clone_variant_data_t *clone_data = NULL; get_clone_variant_data(clone_data, rsc); crm_debug_2("Creating actions for %s", rsc->id); slist_iter( child_rsc, resource_t, rsc->children, lpc, child_rsc->cmds->create_actions(child_rsc, data_set); clone_update_pseudo_status( child_rsc, &child_stopping, &child_starting); if(is_set(child_rsc->flags, pe_rsc_starting)) { last_start_rsc = child_rsc; } if(is_set(child_rsc->flags, pe_rsc_stopping)) { last_stop_rsc = child_rsc; } ); /* start */ start = start_action(rsc, NULL, !child_starting); action_complete = custom_action( rsc, started_key(rsc), RSC_STARTED, NULL, !child_starting, TRUE, data_set); start->pseudo = TRUE; start->runnable = TRUE; action_complete->pseudo = TRUE; action_complete->runnable = TRUE; action_complete->priority = INFINITY; /* crm_err("Upgrading priority for %s to INFINITY", action_complete->uuid); */ child_starting_constraints(clone_data, rsc, NULL, last_start_rsc, data_set); clone_create_notifications( rsc, start, action_complete, data_set); /* stop */ stop = stop_action(rsc, NULL, !child_stopping); action_complete = custom_action( rsc, stopped_key(rsc), RSC_STOPPED, NULL, !child_stopping, TRUE, data_set); stop->pseudo = TRUE; stop->runnable = TRUE; action_complete->pseudo = TRUE; action_complete->runnable = TRUE; action_complete->priority = INFINITY; /* crm_err("Upgrading priority for %s to INFINITY", action_complete->uuid); */ child_stopping_constraints(clone_data, rsc, NULL, last_stop_rsc, data_set); clone_create_notifications(rsc, stop, action_complete, data_set); rsc->actions = rsc->actions; if(stop->post_notified != NULL && start->pre_notify != NULL) { order_actions(stop->post_notified, start->pre_notify, pe_order_optional); } } void clone_create_notifications( resource_t *rsc, action_t *action, action_t *action_complete, pe_working_set_t *data_set) { /* * pre_notify -> pre_notify_complete -> pseudo_action * -> (real actions) -> pseudo_action_complete * -> post_notify -> post_notify_complete * * if the pre_noitfy requires confirmation, * then a list of confirmations will be added as triggers * to pseudo_action in clone_expand() */ action_t *notify = NULL; action_t *notify_complete = NULL; enum action_tasks task; char *notify_key = NULL; clone_variant_data_t *clone_data = NULL; get_clone_variant_data(clone_data, rsc); if(is_not_set(rsc->flags, pe_rsc_notify)) { return; } task = text2task(action->task); /* create pre_notify */ notify_key = generate_notify_key( rsc->id, "pre", action->task); notify = custom_action(rsc, notify_key, RSC_NOTIFY, NULL, action->optional, TRUE, data_set); add_hash_param(notify->meta, "notify_type", "pre"); add_hash_param(notify->meta, "notify_operation", action->task); if(clone_data->notify_confirm) { add_hash_param(notify->meta, "notify_confirm", "yes"); } else { add_hash_param(notify->meta, "notify_confirm", "no"); } /* create pre_notify_complete */ notify_key = generate_notify_key( rsc->id, "confirmed-pre", action->task); notify_complete = custom_action(rsc, notify_key, RSC_NOTIFIED, NULL, action->optional, TRUE, data_set); add_hash_param(notify_complete->meta, "notify_type", "pre"); add_hash_param(notify_complete->meta, "notify_operation", action->task); if(clone_data->notify_confirm) { add_hash_param(notify->meta, "notify_confirm", "yes"); } else { add_hash_param(notify->meta, "notify_confirm", "no"); } notify->pseudo = TRUE; notify->runnable = TRUE; notify_complete->pseudo = TRUE; notify_complete->runnable = TRUE; /* pre_notify before pre_notify_complete */ order_actions(notify, notify_complete, pe_order_optional); /* pre_notify_complete before action */ order_actions(notify_complete, action, pe_order_optional); action->pre_notify = notify; action->pre_notified = notify_complete; /* create post_notify */ notify_key = generate_notify_key (rsc->id, "post", action->task); notify = custom_action(rsc, notify_key, RSC_NOTIFY, NULL, action_complete->optional, TRUE, data_set); add_hash_param(notify->meta, "notify_type", "post"); add_hash_param(notify->meta, "notify_operation", action->task); if(clone_data->notify_confirm) { add_hash_param(notify->meta, "notify_confirm", "yes"); } else { add_hash_param(notify->meta, "notify_confirm", "no"); } /* action_complete before post_notify */ order_actions(action_complete, notify, pe_order_optional); /* create post_notify_complete */ notify_key = generate_notify_key( rsc->id, "confirmed-post", action->task); notify_complete = custom_action(rsc, notify_key, RSC_NOTIFIED, NULL, action->optional, TRUE, data_set); add_hash_param(notify_complete->meta, "notify_type", "pre"); add_hash_param(notify_complete->meta, "notify_operation", action->task); if(clone_data->notify_confirm) { add_hash_param(notify->meta, "notify_confirm", "yes"); } else { add_hash_param(notify->meta, "notify_confirm", "no"); } notify->pseudo = TRUE; notify->runnable = TRUE; notify->priority = INFINITY; notify->runnable = action_complete->runnable; notify_complete->pseudo = TRUE; notify_complete->runnable = TRUE; notify_complete->priority = INFINITY; notify_complete->runnable = action_complete->runnable; /* post_notify before post_notify_complete */ order_actions(notify, notify_complete, pe_order_optional); action->post_notify = notify; action->post_notified = notify_complete; if(safe_str_eq(action->task, RSC_STOP)) { /* post_notify_complete before start */ custom_action_order( rsc, NULL, notify_complete, rsc, start_key(rsc), NULL, pe_order_optional, data_set); } else if(safe_str_eq(action->task, RSC_START)) { /* post_notify_complete before promote */ custom_action_order( rsc, NULL, notify_complete, rsc, promote_key(rsc), NULL, pe_order_optional, data_set); } else if(safe_str_eq(action->task, RSC_DEMOTE)) { /* post_notify_complete before promote */ custom_action_order( rsc, NULL, notify_complete, rsc, stop_key(rsc), NULL, pe_order_optional, data_set); } } void child_starting_constraints( clone_variant_data_t *clone_data, resource_t *rsc, resource_t *child, resource_t *last, pe_working_set_t *data_set) { if(child == NULL && last == NULL) { crm_debug("%s has no active children", rsc->id); return; } if(child != NULL) { order_start_start( rsc, child, pe_order_runnable_left|pe_order_implies_left_printed); new_rsc_order(child, RSC_START, rsc, RSC_STARTED, pe_order_implies_right_printed, data_set); } if(clone_data->ordered) { if(child == NULL) { /* last child start before global started */ new_rsc_order(last, RSC_START, rsc, RSC_STARTED, pe_order_runnable_left, data_set); } else if(last == NULL) { /* global start before first child start */ order_start_start( rsc, child, pe_order_implies_left); } else { /* child/child relative start */ order_start_start(last, child, pe_order_implies_left); } } } void child_stopping_constraints( clone_variant_data_t *clone_data, resource_t *rsc, resource_t *child, resource_t *last, pe_working_set_t *data_set) { if(child == NULL && last == NULL) { crm_debug("%s has no active children", rsc->id); return; } if(child != NULL) { order_stop_stop(rsc, child, pe_order_shutdown|pe_order_implies_left_printed); new_rsc_order(child, RSC_STOP, rsc, RSC_STOPPED, pe_order_implies_right_printed, data_set); } if(clone_data->ordered) { if(last == NULL) { /* first child stop before global stopped */ new_rsc_order(child, RSC_STOP, rsc, RSC_STOPPED, pe_order_runnable_left, data_set); } else if(child == NULL) { /* global stop before last child stop */ order_stop_stop( rsc, last, pe_order_implies_left); } else { /* child/child relative stop */ order_stop_stop(child, last, pe_order_implies_left); } } } void clone_internal_constraints(resource_t *rsc, pe_working_set_t *data_set) { resource_t *last_rsc = NULL; clone_variant_data_t *clone_data = NULL; get_clone_variant_data(clone_data, rsc); native_internal_constraints(rsc, data_set); /* global stop before stopped */ new_rsc_order(rsc, RSC_STOP, rsc, RSC_STOPPED, pe_order_runnable_left, data_set); /* global start before started */ new_rsc_order(rsc, RSC_START, rsc, RSC_STARTED, pe_order_runnable_left, data_set); /* global stopped before start */ new_rsc_order(rsc, RSC_STOPPED, rsc, RSC_START, pe_order_optional, data_set); slist_iter( child_rsc, resource_t, rsc->children, lpc, child_rsc->cmds->internal_constraints(child_rsc, data_set); child_starting_constraints( clone_data, rsc, child_rsc, last_rsc, data_set); child_stopping_constraints( clone_data, rsc, child_rsc, last_rsc, data_set); last_rsc = child_rsc; ); } resource_t* find_compatible_child( resource_t *local_child, resource_t *rsc, enum rsc_role_e filter, gboolean current) { node_t *local_node = NULL; node_t *node = NULL; clone_variant_data_t *clone_data = NULL; get_clone_variant_data(clone_data, rsc); local_node = local_child->fns->location(local_child, NULL, current); if(local_node == NULL) { crm_debug("Can't colocate unrunnable child %s with %s", local_child->id, rsc->id); return NULL; } slist_iter( child_rsc, resource_t, rsc->children, lpc, enum rsc_role_e next_role = child_rsc->fns->state(child_rsc, current); node = child_rsc->fns->location(child_rsc, NULL, current); if(filter != RSC_ROLE_UNKNOWN && next_role != filter) { crm_debug_2("Filtered %s", child_rsc->id); continue; } if(node && local_node && node->details == local_node->details) { crm_info("Colocating %s with %s on %s", local_child->id, child_rsc->id, node->details->uname); return child_rsc; } ); crm_debug("Can't colocate child %s with %s", local_child->id, rsc->id); return NULL; } void clone_rsc_colocation_lh( resource_t *rsc_lh, resource_t *rsc_rh, rsc_colocation_t *constraint) { gboolean do_interleave = FALSE; resource_t *rsc = constraint->rsc_lh; clone_variant_data_t *clone_data = NULL; clone_variant_data_t *clone_data_rh = NULL; if(rsc == NULL) { pe_err("rsc_lh was NULL for %s", constraint->id); return; } else if(constraint->rsc_rh == NULL) { pe_err("rsc_rh was NULL for %s", constraint->id); return; } else { crm_debug_4("Processing constraints from %s", rsc->id); } get_clone_variant_data(clone_data, rsc); if(constraint->rsc_rh->variant == pe_clone || constraint->rsc_rh->variant == pe_master) { get_clone_variant_data( clone_data_rh, constraint->rsc_rh); if(clone_data->clone_node_max != clone_data_rh->clone_node_max) { crm_config_err("Cannot interleave "XML_CIB_TAG_INCARNATION " %s and %s because" " they do not support the same number of" " resources per node", constraint->rsc_lh->id, constraint->rsc_rh->id); /* only the LHS side needs to be labeled as interleave */ } else if(clone_data->interleave) { do_interleave = TRUE; } else if(constraint->score >= INFINITY) { GListPtr lhs = NULL, rhs = NULL; lhs = rsc_lh->allowed_nodes; slist_iter( child_rsc, resource_t, rsc_rh->children, lpc, node_t *chosen = child_rsc->fns->location(child_rsc, NULL, FALSE); if(chosen != NULL) { rhs = g_list_append(rhs, chosen); } ); - rsc_lh->allowed_nodes = node_list_and(lhs, rhs, FALSE); + rsc_lh->allowed_nodes = node_list_exclude(lhs, rhs); pe_free_shallow_adv(rhs, FALSE); pe_free_shallow(lhs); return; } } else if(constraint->score >= INFINITY) { crm_config_err("Manditory co-location of clones (%s) with other" " non-clone (%s) resources is not supported", rsc_lh->id, rsc_rh->id); return; } if(do_interleave) { resource_t *rh_child = NULL; slist_iter(lh_child, resource_t, rsc->children, lpc, CRM_ASSERT(lh_child != NULL); rh_child = find_compatible_child( lh_child, rsc_rh, RSC_ROLE_UNKNOWN, FALSE); if(rh_child == NULL) { crm_debug_2("No match found for %s", lh_child->id); continue; } crm_debug("Interleaving %s with %s", lh_child->id, rh_child->id); lh_child->cmds->rsc_colocation_lh( lh_child, rh_child, constraint); ); return; } slist_iter( child_rsc, resource_t, rsc->children, lpc, child_rsc->cmds->rsc_colocation_lh(child_rsc, constraint->rsc_rh, constraint); ); } void clone_rsc_colocation_rh( resource_t *rsc_lh, resource_t *rsc_rh, rsc_colocation_t *constraint) { clone_variant_data_t *clone_data = NULL; CRM_CHECK(rsc_lh != NULL, return); CRM_CHECK(rsc_lh->variant == pe_native, return); get_clone_variant_data(clone_data, rsc_rh); crm_debug_3("Processing constraint %s: %d", constraint->id, constraint->score); if(rsc_rh == NULL) { pe_err("rsc_rh was NULL for %s", constraint->id); return; } else if(is_set(rsc_rh->flags, pe_rsc_provisional)) { crm_debug_3("%s is still provisional", rsc_rh->id); return; } else if(constraint->score >= INFINITY) { GListPtr lhs = NULL, rhs = NULL; lhs = rsc_lh->allowed_nodes; slist_iter( child_rsc, resource_t, rsc_rh->children, lpc, node_t *chosen = child_rsc->fns->location(child_rsc, NULL, FALSE); if(chosen != NULL) { rhs = g_list_append(rhs, chosen); } ); - rsc_lh->allowed_nodes = node_list_and(lhs, rhs, FALSE); + rsc_lh->allowed_nodes = node_list_exclude(lhs, rhs); pe_free_shallow_adv(rhs, FALSE); pe_free_shallow(lhs); return; } slist_iter( child_rsc, resource_t, rsc_rh->children, lpc, child_rsc->cmds->rsc_colocation_rh(rsc_lh, child_rsc, constraint); ); } void clone_rsc_order_lh(resource_t *rsc, order_constraint_t *order, pe_working_set_t *data_set) { resource_t *r1 = NULL; resource_t *r2 = NULL; gboolean do_interleave = FALSE; clone_variant_data_t *clone_data = NULL; get_clone_variant_data(clone_data, rsc); crm_debug_2("%s->%s", order->lh_action_task, order->rh_action_task); r1 = uber_parent(rsc); r2 = uber_parent(order->rh_rsc); if(r1 == r2) { native_rsc_order_lh(rsc, order, data_set); return; } if(order->rh_rsc->variant == pe_clone || order->rh_rsc->variant == pe_master) { clone_variant_data_t *clone_data_rh = NULL; get_clone_variant_data(clone_data_rh, order->rh_rsc); if(clone_data->clone_node_max != clone_data_rh->clone_node_max) { crm_config_err("Cannot interleave "XML_CIB_TAG_INCARNATION " %s and %s because they do not support the same" " number of resources per node", rsc->id, order->rh_rsc->id); /* only the LHS side needs to be labeled as interleave */ } else if(clone_data->interleave) { do_interleave = TRUE; } } if(do_interleave && order->rh_rsc) { resource_t *lh_child = NULL; resource_t *rh_saved = order->rh_rsc; gboolean current = FALSE; if(strstr(order->lh_action_task, "_stop_0") || strstr(order->lh_action_task, "_demote_0")) { current = TRUE; } slist_iter( rh_child, resource_t, rh_saved->children, lpc, CRM_ASSERT(rh_child != NULL); lh_child = find_compatible_child(rh_child, rsc, RSC_ROLE_UNKNOWN, current); if(lh_child == NULL) { crm_debug_2("No match found for %s", rh_child->id); continue; } crm_debug("Interleaving %s with %s", lh_child->id, rh_child->id); order->rh_rsc = rh_child; lh_child->cmds->rsc_order_lh(lh_child, order, data_set); order->rh_rsc = rh_saved; ); } else { #if 0 if(order->type != pe_order_optional) { crm_debug("Upgraded ordering constraint %d - 0x%.6x", order->id, order->type); native_rsc_order_lh(rsc, order, data_set); } #endif if(order->type & pe_order_implies_left) { if(rsc->variant == order->rh_rsc->variant) { crm_debug_2("Clone-to-clone ordering: %s -> %s 0x%.6x", order->lh_action_task, order->rh_action_task, order->type); /* stop instances on the same nodes as stopping RHS instances */ slist_iter( child_rsc, resource_t, rsc->children, lpc, native_rsc_order_lh(child_rsc, order, data_set); ); } else { /* stop everything */ crm_debug_2("Clone-to-* ordering: %s -> %s 0x%.6x", order->lh_action_task, order->rh_action_task, order->type); slist_iter( child_rsc, resource_t, rsc->children, lpc, native_rsc_order_lh(child_rsc, order, data_set); ); } } convert_non_atomic_task(rsc, order, FALSE); native_rsc_order_lh(rsc, order, data_set); } if(is_set(rsc->flags, pe_rsc_notify)) { order->type = pe_order_optional; convert_non_atomic_task(rsc, order, TRUE); native_rsc_order_lh(rsc, order, data_set); } } void clone_rsc_order_rh( action_t *lh_action, resource_t *rsc, order_constraint_t *order) { clone_variant_data_t *clone_data = NULL; get_clone_variant_data(clone_data, rsc); crm_debug_2("%s->%s", lh_action->uuid, order->rh_action_task); if(safe_str_eq(CRM_OP_PROBED, lh_action->uuid)) { slist_iter( child_rsc, resource_t, rsc->children, lpc, child_rsc->cmds->rsc_order_rh(lh_action, child_rsc, order); ); if(rsc->fns->state(rsc, TRUE) < RSC_ROLE_STARTED && rsc->fns->state(rsc, FALSE) > RSC_ROLE_STOPPED) { order->type |= pe_order_implies_right; } } native_rsc_order_rh(lh_action, rsc, order); } void clone_rsc_location(resource_t *rsc, rsc_to_node_t *constraint) { clone_variant_data_t *clone_data = NULL; get_clone_variant_data(clone_data, rsc); crm_debug_3("Processing location constraint %s for %s", constraint->id, rsc->id); native_rsc_location(rsc, constraint); slist_iter( child_rsc, resource_t, rsc->children, lpc, child_rsc->cmds->rsc_location(child_rsc, constraint); ); } static gint sort_notify_entries(gconstpointer a, gconstpointer b) { int tmp; const notify_entry_t *entry_a = a; const notify_entry_t *entry_b = b; if(entry_a == NULL && entry_b == NULL) { return 0; } if(entry_a == NULL) { return 1; } if(entry_b == NULL) { return -1; } if(entry_a->rsc == NULL && entry_b->rsc == NULL) { return 0; } if(entry_a->rsc == NULL) { return 1; } if(entry_b->rsc == NULL) { return -1; } tmp = strcmp(entry_a->rsc->id, entry_b->rsc->id); if(tmp != 0) { return tmp; } if(entry_a->node == NULL && entry_b->node == NULL) { return 0; } if(entry_a->node == NULL) { return 1; } if(entry_b->node == NULL) { return -1; } return strcmp(entry_a->node->details->id, entry_b->node->details->id); } static void expand_list(GListPtr list, int clones, char **rsc_list, char **node_list, char **uuid_list) { const char *uname = NULL; const char *rsc_id = NULL; const char *last_rsc_id = NULL; CRM_CHECK(list != NULL, return); if(rsc_list) { CRM_CHECK(*rsc_list == NULL, *rsc_list = NULL); } if(node_list) { CRM_CHECK(*node_list == NULL, *node_list = NULL); } slist_iter(entry, notify_entry_t, list, lpc, CRM_CHECK(entry != NULL, continue); CRM_CHECK(entry->rsc != NULL, continue); CRM_CHECK(entry->node != NULL, continue); rsc_id = entry->rsc->id; uname = entry->node->details->uname; CRM_ASSERT(uname != NULL); CRM_ASSERT(rsc_id != NULL); /* filter dups */ if(safe_str_eq(rsc_id, last_rsc_id)) { continue; } last_rsc_id = rsc_id; if(rsc_list != NULL) { int existing_len = 0; int len = 2 + strlen(rsc_id); /* +1 space, +1 EOS */ if(rsc_list && *rsc_list) { existing_len = strlen(*rsc_list); } crm_debug_5("Adding %s (%dc) at offset %d", rsc_id, len-2, existing_len); crm_realloc(*rsc_list, len + existing_len); sprintf(*rsc_list + existing_len, "%s ", rsc_id); } if(node_list != NULL) { int existing_len = 0; int len = 2 + strlen(uname); if(node_list && *node_list) { existing_len = strlen(*node_list); } crm_debug_5("Adding %s (%dc) at offset %d", uname, len-2, existing_len); crm_realloc(*node_list, len + existing_len); sprintf(*node_list + existing_len, "%s ", uname); } ); } static void mark_notifications_required(resource_t *rsc, enum action_tasks task, gboolean top) { char *key = NULL; char *key_complete = NULL; const char *task_s = task2text(task); if(top) { key = generate_op_key(rsc->id, task_s, 0); key_complete = generate_op_key(rsc->id, task2text(task+1), 0); } slist_iter(action, action_t, rsc->actions, lpc, if(action->optional == FALSE) { continue; } if(safe_str_eq(action->uuid, key) || safe_str_eq(action->uuid, key_complete)) { crm_debug_3("Marking top-level action %s as required", action->uuid); action->optional = FALSE; } if(strstr(action->uuid, task_s)) { if(safe_str_eq(RSC_NOTIFIED, action->task) || safe_str_eq(RSC_NOTIFY, action->task)) { crm_debug_3("Marking %s as required", action->uuid); action->optional = FALSE; } } ); slist_iter( child, resource_t, rsc->children, lpc, mark_notifications_required(child, task, FALSE); ); } void clone_expand(resource_t *rsc, pe_working_set_t *data_set) { char *rsc_list = NULL; char *node_list = NULL; char *uuid_list = NULL; notify_data_t *n_data = NULL; clone_variant_data_t *clone_data = NULL; get_clone_variant_data(clone_data, rsc); crm_malloc0(n_data, sizeof(notify_data_t)); n_data->keys = g_hash_table_new_full( g_str_hash, g_str_equal, g_hash_destroy_str, g_hash_destroy_str); crm_debug_2("Processing actions from %s", rsc->id); if(is_set(rsc->flags, pe_rsc_notify)) { slist_iter( child_rsc, resource_t, rsc->children, lpc, slist_iter( op, action_t, rsc->actions, lpc2, child_rsc->cmds->create_notify_element( child_rsc, op, n_data, data_set); ); ); } /* expand the notify data */ if(is_set(rsc->flags, pe_rsc_notify) && n_data->stop) { n_data->stop = g_list_sort( n_data->stop, sort_notify_entries); rsc_list = NULL; node_list = NULL; expand_list(n_data->stop, clone_data->clone_max, &rsc_list, &node_list, &uuid_list); g_hash_table_insert( n_data->keys, crm_strdup("notify_stop_resource"), rsc_list); g_hash_table_insert( n_data->keys, crm_strdup("notify_stop_uname"), node_list); if(rsc_list != NULL) { mark_notifications_required(rsc, stop_rsc, TRUE); } } if(is_set(rsc->flags, pe_rsc_notify) && n_data->start) { n_data->start = g_list_sort( n_data->start, sort_notify_entries); rsc_list = NULL; node_list = NULL; expand_list(n_data->start, clone_data->clone_max, &rsc_list, &node_list, &uuid_list); g_hash_table_insert( n_data->keys, crm_strdup("notify_start_resource"), rsc_list); g_hash_table_insert( n_data->keys, crm_strdup("notify_start_uname"), node_list); mark_notifications_required(rsc, start_rsc, TRUE); } if(is_set(rsc->flags, pe_rsc_notify) && n_data->demote) { n_data->demote = g_list_sort( n_data->demote, sort_notify_entries); rsc_list = NULL; node_list = NULL; expand_list(n_data->demote, clone_data->clone_max, &rsc_list, &node_list, &uuid_list); g_hash_table_insert( n_data->keys, crm_strdup("notify_demote_resource"), rsc_list); g_hash_table_insert( n_data->keys, crm_strdup("notify_demote_uname"), node_list); mark_notifications_required(rsc, action_demote, TRUE); } if(is_set(rsc->flags, pe_rsc_notify) && n_data->promote) { n_data->promote = g_list_sort( n_data->promote, sort_notify_entries); rsc_list = NULL; node_list = NULL; uuid_list = NULL; expand_list(n_data->promote, clone_data->clone_max, &rsc_list, &node_list, &uuid_list); g_hash_table_insert( n_data->keys, crm_strdup("notify_promote_resource"), rsc_list); g_hash_table_insert( n_data->keys, crm_strdup("notify_promote_uname"), node_list); mark_notifications_required(rsc, action_promote, TRUE); } if(is_set(rsc->flags, pe_rsc_notify) && n_data->active) { n_data->active = g_list_sort( n_data->active, sort_notify_entries); rsc_list = NULL; node_list = NULL; uuid_list = NULL; expand_list(n_data->active, clone_data->clone_max, &rsc_list, &node_list, &uuid_list); g_hash_table_insert( n_data->keys, crm_strdup("notify_active_resource"), rsc_list); g_hash_table_insert( n_data->keys, crm_strdup("notify_active_uname"), node_list); } if(is_set(rsc->flags, pe_rsc_notify) && n_data->slave) { n_data->slave = g_list_sort( n_data->slave, sort_notify_entries); rsc_list = NULL; node_list = NULL; uuid_list = NULL; expand_list(n_data->slave, clone_data->clone_max, &rsc_list, &node_list, &uuid_list); g_hash_table_insert( n_data->keys, crm_strdup("notify_slave_resource"), rsc_list); g_hash_table_insert( n_data->keys, crm_strdup("notify_slave_uname"), node_list); } if(is_set(rsc->flags, pe_rsc_notify) && n_data->master) { n_data->master = g_list_sort( n_data->master, sort_notify_entries); rsc_list = NULL; node_list = NULL; uuid_list = NULL; expand_list(n_data->master, clone_data->clone_max, &rsc_list, &node_list, &uuid_list); g_hash_table_insert( n_data->keys, crm_strdup("notify_master_resource"), rsc_list); g_hash_table_insert( n_data->keys, crm_strdup("notify_master_uname"), node_list); } if(is_set(rsc->flags, pe_rsc_notify) && n_data->inactive) { n_data->inactive = g_list_sort( n_data->inactive, sort_notify_entries); rsc_list = NULL; node_list = NULL; uuid_list = NULL; expand_list(n_data->inactive, clone_data->clone_max, &rsc_list, &node_list, &uuid_list); g_hash_table_insert( n_data->keys, crm_strdup("notify_inactive_resource"), rsc_list); g_hash_table_insert( n_data->keys, crm_strdup("notify_inactive_uname"), node_list); } /* yes, we DO need this second loop */ slist_iter( child_rsc, resource_t, rsc->children, lpc, child_rsc->cmds->expand(child_rsc, data_set); ); /* slist_iter( */ /* action, action_t, rsc->actions, lpc2, */ /* if(safe_str_eq(action->task, RSC_NOTIFY)) { */ /* action->meta_xml = notify_xml; */ /* } */ /* ); */ native_expand(rsc, data_set); /* destroy the notify_data */ pe_free_shallow(n_data->stop); pe_free_shallow(n_data->start); pe_free_shallow(n_data->demote); pe_free_shallow(n_data->promote); pe_free_shallow(n_data->master); pe_free_shallow(n_data->slave); pe_free_shallow(n_data->active); pe_free_shallow(n_data->inactive); g_hash_table_destroy(n_data->keys); crm_free(n_data); } static gint sort_rsc_id(gconstpointer a, gconstpointer b) { const resource_t *resource1 = (const resource_t*)a; const resource_t *resource2 = (const resource_t*)b; CRM_ASSERT(resource1 != NULL); CRM_ASSERT(resource2 != NULL); return strcmp(resource1->id, resource2->id); } static resource_t *find_instance_on(resource_t *rsc, node_t *node) { slist_iter(child, resource_t, rsc->children, lpc, slist_iter(known ,node_t, child->known_on, lpc2, if(node->details == known->details) { return child; } ); ); return NULL; } gboolean clone_create_probe(resource_t *rsc, node_t *node, action_t *complete, gboolean force, pe_working_set_t *data_set) { gboolean any_created = FALSE; clone_variant_data_t *clone_data = NULL; get_clone_variant_data(clone_data, rsc); rsc->children = g_list_sort(rsc->children, sort_rsc_id); if(is_not_set(rsc->flags, pe_rsc_unique) && clone_data->clone_node_max == 1) { /* only look for one copy */ resource_t *child = NULL; /* Try whoever we probed last time */ child = find_instance_on(rsc, node); if(child) { return child->cmds->create_probe( child, node, complete, force, data_set); } /* Try whoever we plan on starting there */ slist_iter( child_rsc, resource_t, rsc->children, lpc, node_t *local_node = child_rsc->fns->location(child_rsc, NULL, FALSE); if(local_node == NULL) { continue; } if(local_node->details == node->details) { return child_rsc->cmds->create_probe( child_rsc, node, complete, force, data_set); } ); /* Fall back to the first clone instance */ child = rsc->children->data; return child->cmds->create_probe(child, node, complete, force, data_set); } slist_iter( child_rsc, resource_t, rsc->children, lpc, if(child_rsc->cmds->create_probe( child_rsc, node, complete, force, data_set)) { any_created = TRUE; } if(any_created && is_not_set(rsc->flags, pe_rsc_unique) && clone_data->clone_node_max == 1) { /* only look for one copy (clone :0) */ break; } ); return any_created; } diff --git a/pengine/master.c b/pengine/master.c index 334ccdb2ee..1401b6a697 100644 --- a/pengine/master.c +++ b/pengine/master.c @@ -1,804 +1,807 @@ /* * Copyright (C) 2004 Andrew Beekhof * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #include #include #include #include #include #define VARIANT_CLONE 1 #include extern gint sort_clone_instance(gconstpointer a, gconstpointer b); +void node_list_update_one(GListPtr list1, node_t *other, int score); extern void clone_create_notifications( resource_t *rsc, action_t *action, action_t *action_complete, pe_working_set_t *data_set); extern int master_score(resource_t *rsc, node_t *node, int not_set_value); static void child_promoting_constraints( clone_variant_data_t *clone_data, enum pe_ordering type, resource_t *rsc, resource_t *child, resource_t *last, pe_working_set_t *data_set) { if(child == NULL) { if(clone_data->ordered && last != NULL) { crm_debug_4("Ordered version (last node)"); /* last child promote before promoted started */ new_rsc_order(last, RSC_PROMOTE, rsc, RSC_PROMOTED, type, data_set); } return; } /* child promote before global promoted */ new_rsc_order(child, RSC_PROMOTE, rsc, RSC_PROMOTED, type, data_set); /* global promote before child promote */ new_rsc_order(rsc, RSC_PROMOTE, child, RSC_PROMOTE, type, data_set); if(clone_data->ordered) { crm_debug_4("Ordered version"); if(last == NULL) { /* global promote before first child promote */ last = rsc; } /* else: child/child relative promote */ order_start_start(last, child, type); new_rsc_order(last, RSC_PROMOTE, child, RSC_PROMOTE, type, data_set); } else { crm_debug_4("Un-ordered version"); } } static void child_demoting_constraints( clone_variant_data_t *clone_data, enum pe_ordering type, resource_t *rsc, resource_t *child, resource_t *last, pe_working_set_t *data_set) { if(child == NULL) { if(clone_data->ordered && last != NULL) { crm_debug_4("Ordered version (last node)"); /* global demote before first child demote */ new_rsc_order(rsc, RSC_DEMOTE, last, RSC_DEMOTE, pe_order_implies_left, data_set); } return; } /* child demote before global demoted */ new_rsc_order(child, RSC_DEMOTE, rsc, RSC_DEMOTED, pe_order_implies_right_printed, data_set); /* global demote before child demote */ new_rsc_order(rsc, RSC_DEMOTE, child, RSC_DEMOTE, pe_order_implies_left_printed, data_set); if(clone_data->ordered && last != NULL) { crm_debug_4("Ordered version"); /* child/child relative demote */ new_rsc_order(child, RSC_DEMOTE, last, RSC_DEMOTE, type, data_set); } else if(clone_data->ordered) { crm_debug_4("Ordered version (1st node)"); /* first child stop before global stopped */ new_rsc_order(child, RSC_DEMOTE, rsc, RSC_DEMOTED, type, data_set); } else { crm_debug_4("Un-ordered version"); } } static void master_update_pseudo_status( resource_t *rsc, gboolean *demoting, gboolean *promoting) { if(rsc->children) { slist_iter(child, resource_t, rsc->children, lpc, master_update_pseudo_status(child, demoting, promoting) ); return; } CRM_ASSERT(demoting != NULL); CRM_ASSERT(promoting != NULL); slist_iter( action, action_t, rsc->actions, lpc, if(*promoting && *demoting) { return; } else if(action->optional) { continue; } else if(safe_str_eq(RSC_DEMOTE, action->task)) { *demoting = TRUE; } else if(safe_str_eq(RSC_PROMOTE, action->task)) { *promoting = TRUE; } ); } #define apply_master_location(list) \ slist_iter( \ cons, rsc_to_node_t, list, lpc2, \ cons_node = NULL; \ if(cons->role_filter == RSC_ROLE_MASTER) { \ crm_debug_2("Applying %s to %s", \ cons->id, child_rsc->id); \ cons_node = pe_find_node_id( \ cons->node_list_rh, chosen->details->id); \ } \ if(cons_node != NULL) { \ int new_priority = merge_weights( \ child_rsc->priority, cons_node->weight); \ crm_debug_2("\t%s: %d->%d (%d)", child_rsc->id, \ child_rsc->priority, new_priority, cons_node->weight); \ child_rsc->priority = new_priority; \ } \ ); static node_t * can_be_master(resource_t *rsc) { node_t *node = NULL; node_t *local_node = NULL; resource_t *parent = uber_parent(rsc); clone_variant_data_t *clone_data = NULL; int level = LOG_DEBUG_2; #if 0 enum rsc_role_e role = RSC_ROLE_UNKNOWN; role = rsc->fns->state(rsc, FALSE); crm_info("%s role: %s", rsc->id, role2text(role)); #endif if(rsc->children) { slist_iter( child, resource_t, rsc->children, lpc, if(can_be_master(child) == NULL) { do_crm_log(level, "Child %s of %s can't be promoted", child->id, rsc->id); return NULL; } ); } node = rsc->fns->location(rsc, NULL, FALSE); if(rsc->priority < 0) { do_crm_log(level, "%s cannot be master: preference: %d", rsc->id, rsc->priority); return NULL; } else if(node == NULL) { do_crm_log(level, "%s cannot be master: not allocated", rsc->id); return NULL; } else if(can_run_resources(node) == FALSE) { do_crm_log(level, "Node cant run any resources: %s", node->details->uname); return NULL; } get_clone_variant_data(clone_data, parent); local_node = pe_find_node_id( parent->allowed_nodes, node->details->id); if(local_node == NULL) { crm_err("%s cannot run on %s: node not allowed", rsc->id, node->details->uname); return NULL; } else if(local_node->count < clone_data->master_node_max) { return local_node; } else { do_crm_log(level, "%s cannot be master on %s: node full", rsc->id, node->details->uname); } return NULL; } static gint sort_master_instance(gconstpointer a, gconstpointer b) { int rc; enum rsc_role_e role1 = RSC_ROLE_UNKNOWN; enum rsc_role_e role2 = RSC_ROLE_UNKNOWN; const resource_t *resource1 = (const resource_t*)a; const resource_t *resource2 = (const resource_t*)b; CRM_ASSERT(resource1 != NULL); CRM_ASSERT(resource2 != NULL); role1 = resource1->fns->state(resource1, TRUE); role2 = resource2->fns->state(resource2, TRUE); rc = sort_rsc_index(a, b); if( rc != 0 ) { return rc; } if(role1 > role2) { return -1; } else if(role1 < role2) { return 1; } return sort_clone_instance(a, b); } static void master_promotion_order(resource_t *rsc) { node_t *node = NULL; node_t *chosen = NULL; clone_variant_data_t *clone_data = NULL; get_clone_variant_data(clone_data, rsc); if(clone_data->merged_master_weights) { return; } clone_data->merged_master_weights = TRUE; crm_debug_2("Merging weights for %s", rsc->id); slist_iter( child, resource_t, rsc->children, lpc, crm_debug_2("%s: %d", child->id, child->sort_index); ); dump_node_scores(LOG_DEBUG_3, rsc, "Before", rsc->allowed_nodes); slist_iter( child, resource_t, rsc->children, lpc, chosen = child->fns->location(child, NULL, FALSE); if(chosen == NULL || child->sort_index < 0) { crm_debug_3("Skipping %s", child->id); continue; } node = (node_t*)pe_find_node_id( rsc->allowed_nodes, chosen->details->id); CRM_ASSERT(node != NULL); /* adds in master preferences and rsc_location.role=Master */ node->weight = merge_weights(child->sort_index, node->weight); ); dump_node_scores(LOG_DEBUG_3, rsc, "Middle", rsc->allowed_nodes); slist_iter( constraint, rsc_colocation_t, rsc->rsc_cons_lhs, lpc, /* (re-)adds location preferences of resource that wish to be * colocated with the master instance */ if(constraint->role_rh == RSC_ROLE_MASTER) { rsc->allowed_nodes = constraint->rsc_lh->cmds->merge_weights( constraint->rsc_lh, rsc->id, rsc->allowed_nodes, constraint->score/INFINITY, TRUE); } ); dump_node_scores(LOG_DEBUG_3, rsc, "After", rsc->allowed_nodes); /* write them back and sort */ slist_iter( child, resource_t, rsc->children, lpc, chosen = child->fns->location(child, NULL, FALSE); if(chosen == NULL || child->sort_index < 0) { crm_debug_2("%s: %d", child->id, child->sort_index); continue; } node = (node_t*)pe_find_node_id( rsc->allowed_nodes, chosen->details->id); CRM_ASSERT(node != NULL); child->sort_index = node->weight; crm_debug_2("%s: %d", child->id, child->sort_index); ); rsc->children = g_list_sort(rsc->children, sort_master_instance); } int master_score(resource_t *rsc, node_t *node, int not_set_value) { char *attr_name; char *name = rsc->id; const char *attr_value; int score = not_set_value, len = 0; if(rsc->fns->state(rsc, TRUE) < RSC_ROLE_STARTED) { return score; } if(rsc->running_on) { node_t *match = pe_find_node_id(rsc->allowed_nodes, node->details->id); if(match->weight < 0) { crm_debug_2("%s on %s has score: %d - ignoring master pref", rsc->id, match->details->uname, match->weight); return score; } } #if 0 if(rsc->clone_name) { name = rsc->clone_name; crm_err("%s ::= %s", rsc->id, rsc->clone_name); } #endif len = 8 + strlen(name); crm_malloc0(attr_name, len); sprintf(attr_name, "master-%s", name); crm_debug_3("looking for %s on %s", attr_name, node->details->uname); attr_value = g_hash_table_lookup( node->details->attrs, attr_name); if(attr_value == NULL) { crm_free(attr_name); len = 8 + strlen(rsc->long_name); crm_malloc0(attr_name, len); sprintf(attr_name, "master-%s", rsc->long_name); crm_debug_3("looking for %s on %s", attr_name, node->details->uname); attr_value = g_hash_table_lookup( node->details->attrs, attr_name); } if(attr_value != NULL) { crm_debug_2("%s[%s] = %s", attr_name, node->details->uname, crm_str(attr_value)); score = char2score(attr_value); } crm_free(attr_name); return score; } #define max(a, b) aapplied_master_prefs) { /* Make sure we only do this once */ return; } clone_data->applied_master_prefs = TRUE; slist_iter( child_rsc, resource_t, rsc->children, lpc, slist_iter( node, node_t, child_rsc->allowed_nodes, lpc, if(can_run_resources(node) == FALSE) { /* This node will never be promoted to master, * so don't apply the master score as that may * lead to clone shuffling */ continue; } score = master_score(child_rsc, node, 0); new_score = merge_weights(node->weight, score); if(new_score != node->weight) { crm_debug_2("\t%s: Updating preference for %s (%d->%d)", child_rsc->id, node->details->uname, node->weight, new_score); node->weight = new_score; } new_score = max(child_rsc->priority, score); if(new_score != child_rsc->priority) { crm_debug_2("\t%s: Updating priority (%d->%d)", child_rsc->id, child_rsc->priority, new_score); child_rsc->priority = new_score; } ); ); } static void set_role(resource_t *rsc, enum rsc_role_e role, gboolean current) { if(current) { if(rsc->variant == pe_native && rsc->running_on != NULL && rsc->role > role) { crm_debug_6("Filtering change %s.role = %s (was %s)", rsc->id, role2text(role), role2text(rsc->role)); } else if(rsc->role < role) { crm_debug_5("Set %s.role = %s (was %s)", rsc->id, role2text(role), role2text(rsc->role)); rsc->role = role; } } else { if(rsc->next_role < role) { crm_debug_5("Set %s.next_role = %s (was %s)", rsc->id, role2text(role), role2text(rsc->next_role)); rsc->next_role = role; if(role == RSC_ROLE_MASTER) { add_hash_param(rsc->parameters, crm_meta_name("role"), role2text(role)); } } } slist_iter( child_rsc, resource_t, rsc->children, lpc, set_role(child_rsc, role, current); ); } node_t * master_color(resource_t *rsc, pe_working_set_t *data_set) { int promoted = 0; node_t *chosen = NULL; node_t *cons_node = NULL; enum rsc_role_e next_role = RSC_ROLE_UNKNOWN; clone_variant_data_t *clone_data = NULL; get_clone_variant_data(clone_data, rsc); apply_master_prefs(rsc); clone_color(rsc, data_set); /* count now tracks the number of masters allocated */ slist_iter(node, node_t, rsc->allowed_nodes, lpc, node->count = 0; ); /* * assign priority */ slist_iter( child_rsc, resource_t, rsc->children, lpc, GListPtr list = NULL; crm_debug_2("Assigning priority for %s", child_rsc->id); if(child_rsc->fns->state(child_rsc, TRUE) == RSC_ROLE_STARTED) { set_role(child_rsc, RSC_ROLE_SLAVE, TRUE); } chosen = child_rsc->fns->location(child_rsc, &list, FALSE); if(g_list_length(list) > 1) { crm_config_err("Cannot promote non-colocated child %s", child_rsc->id); } g_list_free(list); if(chosen == NULL) { continue; } next_role = child_rsc->fns->state(child_rsc, FALSE); switch(next_role) { case RSC_ROLE_STARTED: CRM_CHECK(chosen != NULL, break); /* * Default to -1 if no value is set * * This allows master locations to be specified * based solely on rsc_location constraints, * but prevents anyone from being promoted if * neither a constraint nor a master-score is present */ child_rsc->priority = master_score(child_rsc, chosen, -1); break; case RSC_ROLE_SLAVE: case RSC_ROLE_STOPPED: child_rsc->priority = -INFINITY; break; case RSC_ROLE_MASTER: /* the only reason we should be here is if * we're re-creating actions after a stonith */ promoted++; break; default: CRM_CHECK(FALSE/* unhandled */, crm_err("Unknown resource role: %d for %s", next_role, child_rsc->id)); } apply_master_location(child_rsc->rsc_location); apply_master_location(rsc->rsc_location); slist_iter( cons, rsc_colocation_t, child_rsc->rsc_cons, lpc2, child_rsc->cmds->rsc_colocation_lh(child_rsc, cons->rsc_rh, cons); ); child_rsc->sort_index = child_rsc->priority; crm_debug_2("Assigning priority for %s: %d", child_rsc->id, child_rsc->priority); if(next_role == RSC_ROLE_MASTER) { child_rsc->sort_index = INFINITY; } ); master_promotion_order(rsc); /* mark the first N as masters */ slist_iter( child_rsc, resource_t, rsc->children, lpc, chosen = NULL; crm_debug_2("Processing %s", child_rsc->id); chosen = child_rsc->fns->location(child_rsc, NULL, FALSE); do_crm_log(scores_log_level, "%s promotion score on %s: %d", child_rsc->id, chosen?chosen->details->uname:"none", child_rsc->sort_index); chosen = NULL; /* nuke 'chosen' so that we don't promote more than the * required number of instances */ if(promoted < clone_data->master_max) { chosen = can_be_master(child_rsc); } crm_debug("%s master score: %d", child_rsc->id, child_rsc->priority); if(chosen == NULL) { next_role = child_rsc->fns->state(child_rsc, FALSE); if(next_role == RSC_ROLE_STARTED) { set_role(child_rsc, RSC_ROLE_SLAVE, FALSE); } continue; } chosen->count++; crm_info("Promoting %s (%s %s)", child_rsc->id, role2text(child_rsc->role), chosen->details->uname); set_role(child_rsc, RSC_ROLE_MASTER, FALSE); clone_data->masters_allocated++; promoted++; ); crm_info("%s: Promoted %d instances of a possible %d to master", rsc->id, promoted, clone_data->master_max); return NULL; } void master_create_actions(resource_t *rsc, pe_working_set_t *data_set) { action_t *action = NULL; action_t *action_complete = NULL; gboolean any_promoting = FALSE; gboolean any_demoting = FALSE; resource_t *last_promote_rsc = NULL; resource_t *last_demote_rsc = NULL; clone_variant_data_t *clone_data = NULL; get_clone_variant_data(clone_data, rsc); crm_debug("Creating actions for %s", rsc->id); /* create actions as normal */ clone_create_actions(rsc, data_set); slist_iter( child_rsc, resource_t, rsc->children, lpc, gboolean child_promoting = FALSE; gboolean child_demoting = FALSE; crm_debug_2("Creating actions for %s", child_rsc->id); child_rsc->cmds->create_actions(child_rsc, data_set); master_update_pseudo_status( child_rsc, &child_demoting, &child_promoting); any_demoting = any_demoting || child_demoting; any_promoting = any_promoting || child_promoting; crm_debug_2("Created actions for %s: %d %d", child_rsc->id, child_promoting, child_demoting); ); /* promote */ action = promote_action(rsc, NULL, !any_promoting); action_complete = custom_action( rsc, promoted_key(rsc), RSC_PROMOTED, NULL, !any_promoting, TRUE, data_set); action->pseudo = TRUE; action->runnable = FALSE; action_complete->pseudo = TRUE; action_complete->runnable = FALSE; action_complete->priority = INFINITY; if(clone_data->masters_allocated > 0) { action->runnable = TRUE; action_complete->runnable = TRUE; } child_promoting_constraints(clone_data, pe_order_optional, rsc, NULL, last_promote_rsc, data_set); clone_create_notifications(rsc, action, action_complete, data_set); /* demote */ action = demote_action(rsc, NULL, !any_demoting); action_complete = custom_action( rsc, demoted_key(rsc), RSC_DEMOTED, NULL, !any_demoting, TRUE, data_set); action_complete->priority = INFINITY; action->pseudo = TRUE; action->runnable = TRUE; action_complete->pseudo = TRUE; action_complete->runnable = TRUE; child_demoting_constraints(clone_data, pe_order_optional, rsc, NULL, last_demote_rsc, data_set); clone_create_notifications(rsc, action, action_complete, data_set); /* restore the correct priority */ slist_iter( child_rsc, resource_t, rsc->children, lpc, child_rsc->priority = rsc->priority; ); } void master_internal_constraints(resource_t *rsc, pe_working_set_t *data_set) { resource_t *last_rsc = NULL; clone_variant_data_t *clone_data = NULL; get_clone_variant_data(clone_data, rsc); clone_internal_constraints(rsc, data_set); /* global stopped before start */ new_rsc_order(rsc, RSC_STOPPED, rsc, RSC_START, pe_order_optional, data_set); /* global stopped before promote */ new_rsc_order(rsc, RSC_STOPPED, rsc, RSC_PROMOTE, pe_order_optional, data_set); /* global demoted before start */ new_rsc_order(rsc, RSC_DEMOTED, rsc, RSC_START, pe_order_optional, data_set); /* global started before promote */ new_rsc_order(rsc, RSC_STARTED, rsc, RSC_PROMOTE, pe_order_optional, data_set); /* global demoted before stop */ new_rsc_order(rsc, RSC_DEMOTED, rsc, RSC_STOP, pe_order_optional, data_set); /* global demote before demoted */ new_rsc_order(rsc, RSC_DEMOTE, rsc, RSC_DEMOTED, pe_order_optional, data_set); /* global demoted before promote */ new_rsc_order(rsc, RSC_DEMOTED, rsc, RSC_PROMOTE, pe_order_optional, data_set); slist_iter( child_rsc, resource_t, rsc->children, lpc, /* child demote before promote */ new_rsc_order(child_rsc, RSC_DEMOTE, child_rsc, RSC_PROMOTE, pe_order_optional, data_set); child_promoting_constraints(clone_data, pe_order_optional, rsc, child_rsc, last_rsc, data_set); child_demoting_constraints(clone_data, pe_order_optional, rsc, child_rsc, last_rsc, data_set); last_rsc = child_rsc; ); } -static void node_list_update_one(GListPtr list1, node_t *other, int score) +void node_list_update_one(GListPtr list1, node_t *other, int score) { node_t *node = NULL; if(other == NULL) { return; } node = (node_t*)pe_find_node_id(list1, other->details->id); if(node != NULL) { crm_debug_2("%s: %d + %d", node->details->uname, node->weight, other->weight); node->weight = merge_weights(node->weight, score); } } void master_rsc_colocation_rh( resource_t *rsc_lh, resource_t *rsc_rh, rsc_colocation_t *constraint) { clone_variant_data_t *clone_data = NULL; get_clone_variant_data(clone_data, rsc_rh); CRM_CHECK(rsc_rh != NULL, return); if(is_set(rsc_rh->flags, pe_rsc_provisional)) { return; } else if(constraint->role_rh == RSC_ROLE_UNKNOWN) { crm_debug_3("Handling %s as a clone colocation", constraint->id); clone_rsc_colocation_rh(rsc_lh, rsc_rh, constraint); return; } CRM_CHECK(rsc_lh != NULL, return); CRM_CHECK(rsc_lh->variant == pe_native, return); crm_debug_2("Processing constraint %s: %d", constraint->id, constraint->score); if(constraint->role_rh == RSC_ROLE_UNKNOWN) { slist_iter( child_rsc, resource_t, rsc_rh->children, lpc, child_rsc->cmds->rsc_colocation_rh(rsc_lh, child_rsc, constraint); ); } else if(is_set(rsc_lh->flags, pe_rsc_provisional)) { GListPtr lhs = NULL, rhs = NULL; lhs = rsc_lh->allowed_nodes; - slist_iter( child_rsc, resource_t, rsc_rh->children, lpc, node_t *chosen = child_rsc->fns->location(child_rsc, NULL, FALSE); enum rsc_role_e next_role = child_rsc->fns->state(child_rsc, FALSE); crm_debug_3("Processing: %s", child_rsc->id); if(chosen != NULL && next_role == constraint->role_rh) { - crm_debug_3("Applying: %s %s", child_rsc->id, - role2text(next_role)); - node_list_update_one(rsc_lh->allowed_nodes, chosen, constraint->score); + crm_debug_3("Applying: %s %s %s %d", child_rsc->id, + role2text(next_role), chosen->details->uname, constraint->score); + if(constraint->score < INFINITY) { + node_list_update_one(rsc_lh->allowed_nodes, chosen, constraint->score); + } rhs = g_list_append(rhs, chosen); } ); /* Only do this if its not a master-master colocation * Doing this unconditionally would prevent the slaves from being started */ - if(constraint->score > 0 - && (constraint->role_lh != RSC_ROLE_MASTER - || constraint->role_rh != RSC_ROLE_MASTER)) { - rsc_lh->allowed_nodes = node_list_and(lhs, rhs, FALSE); - pe_free_shallow(lhs); + if(constraint->role_lh != RSC_ROLE_MASTER + || constraint->role_rh != RSC_ROLE_MASTER) { + if(constraint->score > 0) { + rsc_lh->allowed_nodes = node_list_exclude(lhs, rhs); + pe_free_shallow(lhs); + } } pe_free_shallow_adv(rhs, FALSE); } else if(constraint->role_lh == RSC_ROLE_MASTER) { resource_t *rh_child = find_compatible_child(rsc_lh, rsc_rh, constraint->role_rh, FALSE); if(rh_child == NULL && constraint->score >= INFINITY) { crm_debug_2("%s can't be promoted %s", rsc_lh->id, constraint->id); rsc_lh->priority = -INFINITY; } else if(rh_child != NULL) { int new_priority = merge_weights(rsc_lh->priority, constraint->score); crm_debug("Applying %s to %s", constraint->id, rsc_lh->id); crm_debug("\t%s: %d->%d", rsc_lh->id, rsc_lh->priority, new_priority); rsc_lh->priority = new_priority; } } return; }