diff --git a/crmd/control.c b/crmd/control.c index cee8a70588..242eaa2c57 100644 --- a/crmd/control.c +++ b/crmd/control.c @@ -1,916 +1,919 @@ /* * Copyright (C) 2004 Andrew Beekhof * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public * License as published by the Free Software Foundation; either * version 2 of the License, or (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ #include #include #include #include #include #include #include +#include "../lib/common/stack.h" #include #include #include #include #include #include #include #include #include char *ipc_server = NULL; extern gboolean crm_connect_corosync(void); extern void crmd_ha_connection_destroy(gpointer user_data); void crm_shutdown(int nsig); gboolean crm_read_options(gpointer user_data); gboolean fsa_has_quorum = FALSE; GHashTable *ipc_clients = NULL; crm_trigger_t *fsa_source = NULL; crm_trigger_t *config_read = NULL; /* A_HA_CONNECT */ void do_ha_control(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t *msg_data) { gboolean registered = FALSE; if(action & A_HA_DISCONNECT) { if(is_openais_cluster()) { crm_peer_destroy(); + terminate_ais_connection(); crm_info("Disconnected from OpenAIS"); + #if SUPPORT_HEARTBEAT } else if(fsa_cluster_conn != NULL) { set_bit_inplace(fsa_input_register, R_HA_DISCONNECTED); fsa_cluster_conn->llc_ops->signoff(fsa_cluster_conn, FALSE); crm_info("Disconnected from Heartbeat"); #endif } } if(action & A_HA_CONNECT) { crm_set_status_callback(&ais_status_callback); if(is_openais_cluster()) { #if SUPPORT_COROSYNC registered = crm_connect_corosync(); #endif } else if(is_heartbeat_cluster()) { #if SUPPORT_HEARTBEAT registered = crm_cluster_connect( &fsa_our_uname, &fsa_our_uuid, crmd_ha_msg_callback, crmd_ha_connection_destroy, &fsa_cluster_conn); #endif } #if SUPPORT_HEARTBEAT if(is_heartbeat_cluster()) { crm_debug_3("Be informed of Node Status changes"); if (registered && fsa_cluster_conn->llc_ops->set_nstatus_callback( fsa_cluster_conn, crmd_ha_status_callback, fsa_cluster_conn) != HA_OK){ crm_err("Cannot set nstatus callback: %s", fsa_cluster_conn->llc_ops->errmsg(fsa_cluster_conn)); registered = FALSE; } crm_debug_3("Be informed of CRM Client Status changes"); if (registered && fsa_cluster_conn->llc_ops->set_cstatus_callback( fsa_cluster_conn, crmd_client_status_callback, fsa_cluster_conn) != HA_OK) { crm_err("Cannot set cstatus callback: %s", fsa_cluster_conn->llc_ops->errmsg(fsa_cluster_conn)); registered = FALSE; } if(registered) { crm_debug_3("Requesting an initial dump of CRMD client_status"); fsa_cluster_conn->llc_ops->client_status( fsa_cluster_conn, NULL, CRM_SYSTEM_CRMD, -1); } } #endif if(registered == FALSE) { set_bit_inplace(fsa_input_register, R_HA_DISCONNECTED); register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL); return; } clear_bit_inplace(fsa_input_register, R_HA_DISCONNECTED); crm_info("Connected to the cluster"); } if(action & ~(A_HA_CONNECT|A_HA_DISCONNECT)) { crm_err("Unexpected action %s in %s", fsa_action2string(action), __FUNCTION__); } } /* A_SHUTDOWN */ void do_shutdown(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t *msg_data) { /* just in case */ set_bit_inplace(fsa_input_register, R_SHUTDOWN); if(is_heartbeat_cluster()) { if(is_set(fsa_input_register, pe_subsystem->flag_connected)) { crm_info("Terminating the %s", pe_subsystem->name); if(stop_subsystem(pe_subsystem, TRUE) == FALSE) { /* its gone... */ crm_err("Faking %s exit", pe_subsystem->name); clear_bit_inplace(fsa_input_register, pe_subsystem->flag_connected); } else { crm_info("Waiting for subsystems to exit"); crmd_fsa_stall(NULL); } } crm_info("All subsystems stopped, continuing"); } if(stonith_api) { /* Prevent it from comming up again */ clear_bit_inplace(fsa_input_register, R_ST_REQUIRED); crm_info("Disconnecting STONITH..."); stonith_api->cmds->disconnect(stonith_api); } } /* A_SHUTDOWN_REQ */ void do_shutdown_req(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t *msg_data) { xmlNode *msg = NULL; crm_info("Sending shutdown request to DC: %s", crm_str(fsa_our_dc)); msg = create_request( CRM_OP_SHUTDOWN_REQ, NULL, NULL, CRM_SYSTEM_DC, CRM_SYSTEM_CRMD, NULL); /* set_bit_inplace(fsa_input_register, R_STAYDOWN); */ if(send_cluster_message(NULL, crm_msg_crmd, msg, TRUE) == FALSE) { register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL); } free_xml(msg); } extern char *max_generation_from; extern xmlNode *max_generation_xml; extern GHashTable *resources; extern GHashTable *voted; extern GHashTable *reload_hash; void log_connected_client(gpointer key, gpointer value, gpointer user_data); void log_connected_client(gpointer key, gpointer value, gpointer user_data) { crmd_client_t *client = value; crm_err("%s is still connected at exit", client->table_key); } static void free_mem(fsa_data_t *msg_data) { g_main_loop_quit(crmd_mainloop); g_main_loop_unref(crmd_mainloop); #if SUPPORT_HEARTBEAT if(fsa_cluster_conn) { fsa_cluster_conn->llc_ops->delete(fsa_cluster_conn); fsa_cluster_conn = NULL; } #endif slist_destroy(fsa_data_t, fsa_data, fsa_message_queue, crm_info("Dropping %s: [ state=%s cause=%s origin=%s ]", fsa_input2string(fsa_data->fsa_input), fsa_state2string(fsa_state), fsa_cause2string(fsa_data->fsa_cause), fsa_data->origin); delete_fsa_input(fsa_data); ); delete_fsa_input(msg_data); if(ipc_clients) { crm_debug("Number of connected clients: %d", g_hash_table_size(ipc_clients)); /* g_hash_table_foreach(ipc_clients, log_connected_client, NULL); */ g_hash_table_destroy(ipc_clients); } empty_uuid_cache(); crm_peer_destroy(); clear_bit_inplace(fsa_input_register, R_CCM_DATA); if(te_subsystem->client && te_subsystem->client->client_source) { crm_debug("Full destroy: TE"); G_main_del_IPC_Channel(te_subsystem->client->client_source); } else { crm_debug("Partial destroy: TE"); crmd_ipc_connection_destroy(te_subsystem->client); } crm_free(te_subsystem); if(pe_subsystem->client && pe_subsystem->client->client_source) { crm_debug("Full destroy: PE"); G_main_del_IPC_Channel(pe_subsystem->client->client_source); } else { crm_debug("Partial destroy: PE"); crmd_ipc_connection_destroy(pe_subsystem->client); } crm_free(pe_subsystem); crm_free(cib_subsystem); if(integrated_nodes) { g_hash_table_destroy(integrated_nodes); } if(finalized_nodes) { g_hash_table_destroy(finalized_nodes); } if(confirmed_nodes) { g_hash_table_destroy(confirmed_nodes); } if(reload_hash) { g_hash_table_destroy(reload_hash); } if(resources) { g_hash_table_destroy(resources); } if(voted) { g_hash_table_destroy(voted); } cib_delete(fsa_cib_conn); fsa_cib_conn = NULL; if(fsa_lrm_conn) { fsa_lrm_conn->lrm_ops->delete(fsa_lrm_conn); } crm_free(integration_timer); crm_free(finalization_timer); crm_free(election_trigger); crm_free(election_timeout); crm_free(shutdown_escalation_timer); crm_free(wait_timer); crm_free(recheck_timer); crm_free(fsa_our_dc_version); crm_free(fsa_our_uname); crm_free(fsa_our_uuid); crm_free(fsa_our_dc); crm_free(ipc_server); crm_free(max_generation_from); free_xml(max_generation_xml); crm_xml_cleanup(); } /* A_EXIT_0, A_EXIT_1 */ void do_exit(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t *msg_data) { int exit_code = 0; int log_level = LOG_INFO; const char *exit_type = "gracefully"; if(action & A_EXIT_1) { exit_code = 1; log_level = LOG_ERR; exit_type = "forcefully"; } verify_stopped(cur_state, LOG_ERR); do_crm_log(log_level, "Performing %s - %s exiting the CRMd", fsa_action2string(action), exit_type); if(is_set(fsa_input_register, R_IN_RECOVERY)) { crm_err("Could not recover from internal error"); exit_code = 2; } if(is_set(fsa_input_register, R_STAYDOWN)) { crm_warn("Inhibiting respawn by Heartbeat"); exit_code = 100; } free_mem(msg_data); crm_info("[%s] stopped (%d)", crm_system_name, exit_code); cl_flush_logs(); exit(exit_code); } /* A_STARTUP */ void do_startup(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t *msg_data) { int was_error = 0; int interval = 1; /* seconds between DC heartbeats */ crm_debug("Registering Signal Handlers"); mainloop_add_signal(SIGTERM, crm_shutdown); fsa_source = mainloop_add_trigger(G_PRIORITY_HIGH, crm_fsa_trigger, NULL); config_read = mainloop_add_trigger(G_PRIORITY_HIGH, crm_read_options, NULL); ipc_clients = g_hash_table_new(g_str_hash, g_str_equal); crm_debug("Creating CIB and LRM objects"); fsa_cib_conn = cib_new(); fsa_lrm_conn = ll_lrm_new(XML_CIB_TAG_LRM); /* set up the timers */ crm_malloc0(integration_timer, sizeof(fsa_timer_t)); crm_malloc0(finalization_timer, sizeof(fsa_timer_t)); crm_malloc0(election_trigger, sizeof(fsa_timer_t)); crm_malloc0(election_timeout, sizeof(fsa_timer_t)); crm_malloc0(shutdown_escalation_timer, sizeof(fsa_timer_t)); crm_malloc0(wait_timer, sizeof(fsa_timer_t)); crm_malloc0(recheck_timer, sizeof(fsa_timer_t)); interval = interval * 1000; if(election_trigger != NULL) { election_trigger->source_id = 0; election_trigger->period_ms = -1; election_trigger->fsa_input = I_DC_TIMEOUT; election_trigger->callback = crm_timer_popped; election_trigger->repeat = FALSE; } else { was_error = TRUE; } if(election_timeout != NULL) { election_timeout->source_id = 0; election_timeout->period_ms = -1; election_timeout->fsa_input = I_ELECTION_DC; election_timeout->callback = crm_timer_popped; election_timeout->repeat = FALSE; } else { was_error = TRUE; } if(integration_timer != NULL) { integration_timer->source_id = 0; integration_timer->period_ms = -1; integration_timer->fsa_input = I_INTEGRATED; integration_timer->callback = crm_timer_popped; integration_timer->repeat = FALSE; } else { was_error = TRUE; } if(finalization_timer != NULL) { finalization_timer->source_id = 0; finalization_timer->period_ms = -1; finalization_timer->fsa_input = I_FINALIZED; finalization_timer->callback = crm_timer_popped; finalization_timer->repeat = FALSE; /* for possible enabling... a bug in the join protocol left * a slave in S_PENDING while we think its in S_NOT_DC * * raising I_FINALIZED put us into a transition loop which is * never resolved. * in this loop we continually send probes which the node * NACK's because its in S_PENDING * * if we have nodes where heartbeat is active but the * CRM is not... then this will be handled in the * integration phase */ finalization_timer->fsa_input = I_ELECTION; } else { was_error = TRUE; } if(shutdown_escalation_timer != NULL) { shutdown_escalation_timer->source_id = 0; shutdown_escalation_timer->period_ms = -1; shutdown_escalation_timer->fsa_input = I_STOP; shutdown_escalation_timer->callback = crm_timer_popped; shutdown_escalation_timer->repeat = FALSE; } else { was_error = TRUE; } if(wait_timer != NULL) { wait_timer->source_id = 0; wait_timer->period_ms = 2000; wait_timer->fsa_input = I_NULL; wait_timer->callback = crm_timer_popped; wait_timer->repeat = FALSE; } else { was_error = TRUE; } if(recheck_timer != NULL) { recheck_timer->source_id = 0; recheck_timer->period_ms = -1; recheck_timer->fsa_input = I_PE_CALC; recheck_timer->callback = crm_timer_popped; recheck_timer->repeat = FALSE; } else { was_error = TRUE; } /* set up the sub systems */ crm_malloc0(cib_subsystem, sizeof(struct crm_subsystem_s)); crm_malloc0(te_subsystem, sizeof(struct crm_subsystem_s)); crm_malloc0(pe_subsystem, sizeof(struct crm_subsystem_s)); if(cib_subsystem != NULL) { cib_subsystem->pid = -1; cib_subsystem->path = CRM_DAEMON_DIR; cib_subsystem->name = CRM_SYSTEM_CIB; cib_subsystem->command = CRM_DAEMON_DIR"/"CRM_SYSTEM_CIB; cib_subsystem->args = "-VVc"; cib_subsystem->flag_connected = R_CIB_CONNECTED; cib_subsystem->flag_required = R_CIB_REQUIRED; } else { was_error = TRUE; } if(te_subsystem != NULL) { te_subsystem->pid = -1; te_subsystem->path = CRM_DAEMON_DIR; te_subsystem->name = CRM_SYSTEM_TENGINE; te_subsystem->command = CRM_DAEMON_DIR"/"CRM_SYSTEM_TENGINE; te_subsystem->args = NULL; te_subsystem->flag_connected = R_TE_CONNECTED; te_subsystem->flag_required = R_TE_REQUIRED; } else { was_error = TRUE; } if(pe_subsystem != NULL) { pe_subsystem->pid = -1; pe_subsystem->path = CRM_DAEMON_DIR; pe_subsystem->name = CRM_SYSTEM_PENGINE; pe_subsystem->command = CRM_DAEMON_DIR"/"CRM_SYSTEM_PENGINE; pe_subsystem->args = NULL; pe_subsystem->flag_connected = R_PE_CONNECTED; pe_subsystem->flag_required = R_PE_REQUIRED; } else { was_error = TRUE; } if(was_error) { register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL); } welcomed_nodes = g_hash_table_new_full( g_str_hash, g_str_equal, g_hash_destroy_str, g_hash_destroy_str); integrated_nodes = g_hash_table_new_full( g_str_hash, g_str_equal, g_hash_destroy_str, g_hash_destroy_str); finalized_nodes = g_hash_table_new_full( g_str_hash, g_str_equal, g_hash_destroy_str, g_hash_destroy_str); confirmed_nodes = g_hash_table_new_full( g_str_hash, g_str_equal, g_hash_destroy_str, g_hash_destroy_str); set_sigchld_proctrack(G_PRIORITY_HIGH,DEFAULT_MAXDISPATCHTIME); } /* A_STOP */ void do_stop(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t *msg_data) { register_fsa_input(C_FSA_INTERNAL, I_TERMINATE, NULL); } /* A_STARTED */ void do_started(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t *msg_data) { if(cur_state != S_STARTING) { crm_err("Start cancelled... %s", fsa_state2string(cur_state)); return; } else if(is_set(fsa_input_register, R_CCM_DATA) == FALSE) { crm_info("Delaying start, no membership data (%.16llx)", R_CCM_DATA); crmd_fsa_stall(NULL); return; } else if(is_set(fsa_input_register, R_LRM_CONNECTED) == FALSE) { crm_info("Delaying start, LRM not connected (%.16llx)", R_LRM_CONNECTED); crmd_fsa_stall(NULL); return; } else if(is_set(fsa_input_register, R_CIB_CONNECTED) == FALSE) { crm_info("Delaying start, CIB not connected (%.16llx)", R_CIB_CONNECTED); crmd_fsa_stall(NULL); return; } else if(is_set(fsa_input_register, R_READ_CONFIG) == FALSE) { crm_info("Delaying start, Config not read (%.16llx)", R_READ_CONFIG); crmd_fsa_stall(NULL); return; } else if(is_set(fsa_input_register, R_PEER_DATA) == FALSE) { HA_Message *msg = NULL; /* try reading from HA */ crm_info("Delaying start, No peer data (%.16llx)", R_PEER_DATA); crm_debug_3("Looking for a HA message"); #if SUPPORT_HEARTBEAT if(is_heartbeat_cluster()) { msg = fsa_cluster_conn->llc_ops->readmsg(fsa_cluster_conn, 0); } #endif if(msg != NULL) { crm_debug_3("There was a HA message"); crm_msg_del(msg); } crmd_fsa_stall(NULL); return; } crm_debug("Init server comms"); if(ipc_server == NULL) { ipc_server = crm_strdup(CRM_SYSTEM_CRMD); } if(init_server_ipc_comms(ipc_server, crmd_client_connect, default_ipc_connection_destroy)) { crm_err("Couldn't start IPC server"); register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL); } if(stonith_reconnect == NULL) { int dummy; stonith_reconnect = mainloop_add_trigger( G_PRIORITY_LOW, te_connect_stonith, &dummy); } set_bit_inplace(fsa_input_register, R_ST_REQUIRED); mainloop_set_trigger(stonith_reconnect); crm_info("The local CRM is operational"); clear_bit_inplace(fsa_input_register, R_STARTING); register_fsa_input(msg_data->fsa_cause, I_PENDING, NULL); } /* A_RECOVER */ void do_recover(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t *msg_data) { set_bit_inplace(fsa_input_register, R_IN_RECOVERY); crm_err("Action %s (%.16llx) not supported", fsa_action2string(action), action); register_fsa_input(C_FSA_INTERNAL, I_TERMINATE, NULL); } pe_cluster_option crmd_opts[] = { /* name, old-name, validate, default, description */ { "dc-version", NULL, "string", NULL, "none", NULL, "Version of Pacemaker on the cluster's DC.", "Includes the hash which identifies the exact Mercurial changeset it was built from. Used for diagnostic purposes." }, { "cluster-infrastructure", NULL, "string", NULL, "heartbeat", NULL, "The messaging stack on which Pacemaker is currently running.", "Used for informational and diagnostic purposes." }, { XML_CONFIG_ATTR_DC_DEADTIME, "dc_deadtime", "time", NULL, "60s", &check_time, "How long to wait for a response from other nodes during startup.", "The \"correct\" value will depend on the speed/load of your network and the type of switches used." }, { XML_CONFIG_ATTR_RECHECK, "cluster_recheck_interval", "time", "Zero disables polling. Positive values are an interval in seconds (unless other SI units are specified. eg. 5min)", "15min", &check_timer, "Polling interval for time based changes to options, resource parameters and constraints.", "The Cluster is primarily event driven, however the configuration can have elements that change based on time." " To ensure these changes take effect, we can optionally poll the cluster's status for changes." }, { XML_CONFIG_ATTR_ELECTION_FAIL, "election_timeout", "time", NULL, "2min", &check_timer, "*** Advanced Use Only ***.", "If need to adjust this value, it probably indicates the presence of a bug." }, { XML_CONFIG_ATTR_FORCE_QUIT, "shutdown_escalation", "time", NULL, "20min", &check_timer, "*** Advanced Use Only ***.", "If need to adjust this value, it probably indicates the presence of a bug." }, { "crmd-integration-timeout", NULL, "time", NULL, "3min", &check_timer, "*** Advanced Use Only ***.", "If need to adjust this value, it probably indicates the presence of a bug." }, { "crmd-finalization-timeout", NULL, "time", NULL, "30min", &check_timer, "*** Advanced Use Only ***.", "If you need to adjust this value, it probably indicates the presence of a bug." }, { XML_ATTR_EXPECTED_VOTES, NULL, "integer", NULL, "2", &check_number, "The number of nodes expected to be in the cluster", "Used to calculate quorum in openais based clusters." }, }; void crmd_metadata(void) { config_metadata("CRM Daemon", "1.0", "CRM Daemon Options", "This is a fake resource that details the options that can be configured for the CRM Daemon.", crmd_opts, DIMOF(crmd_opts)); } static void verify_crmd_options(GHashTable *options) { verify_all_options(options, crmd_opts, DIMOF(crmd_opts)); } static const char * crmd_pref(GHashTable *options, const char *name) { return get_cluster_pref(options, crmd_opts, DIMOF(crmd_opts), name); } static void config_query_callback(xmlNode *msg, int call_id, int rc, xmlNode *output, void *user_data) { const char *value = NULL; GHashTable *config_hash = NULL; ha_time_t *now = new_ha_date(TRUE); if(rc != cib_ok) { fsa_data_t *msg_data = NULL; crm_err("Local CIB query resulted in an error: %s", cib_error2string(rc)); register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL); if(rc == cib_bad_permissions || rc == cib_dtd_validation || rc == cib_bad_digest || rc == cib_bad_config) { crm_err("The cluster is mis-configured - shutting down and staying down"); set_bit_inplace(fsa_input_register, R_STAYDOWN); } goto bail; } crm_debug("Call %d : Parsing CIB options", call_id); config_hash = g_hash_table_new_full( g_str_hash,g_str_equal, g_hash_destroy_str,g_hash_destroy_str); unpack_instance_attributes( output, output, XML_CIB_TAG_PROPSET, NULL, config_hash, CIB_OPTIONS_FIRST, FALSE, now); verify_crmd_options(config_hash); value = crmd_pref(config_hash, XML_CONFIG_ATTR_DC_DEADTIME); election_trigger->period_ms = crm_get_msec(value); value = crmd_pref(config_hash, XML_CONFIG_ATTR_FORCE_QUIT); shutdown_escalation_timer->period_ms = crm_get_msec(value); crm_info("Shutdown escalation occurs after: %dms", shutdown_escalation_timer->period_ms); value = crmd_pref(config_hash, XML_CONFIG_ATTR_ELECTION_FAIL); election_timeout->period_ms = crm_get_msec(value); value = crmd_pref(config_hash, XML_CONFIG_ATTR_RECHECK); recheck_timer->period_ms = crm_get_msec(value); crm_info("Checking for expired actions every %dms", recheck_timer->period_ms); value = crmd_pref(config_hash, "crmd-integration-timeout"); integration_timer->period_ms = crm_get_msec(value); value = crmd_pref(config_hash, "crmd-finalization-timeout"); finalization_timer->period_ms = crm_get_msec(value); #if SUPPORT_COROSYNC if(is_classic_ais_cluster()) { value = crmd_pref(config_hash, XML_ATTR_EXPECTED_VOTES); crm_info("Sending expected-votes=%s to corosync", value); send_ais_text(crm_class_quorum, value, TRUE, NULL, crm_msg_ais); } #endif set_bit_inplace(fsa_input_register, R_READ_CONFIG); crm_debug_3("Triggering FSA: %s", __FUNCTION__); mainloop_set_trigger(fsa_source); g_hash_table_destroy(config_hash); bail: free_ha_date(now); } gboolean crm_read_options(gpointer user_data) { int call_id = fsa_cib_conn->cmds->query( fsa_cib_conn, XML_CIB_TAG_CRMCONFIG, NULL, cib_scope_local); add_cib_op_callback(fsa_cib_conn, call_id, FALSE, NULL, config_query_callback); crm_debug_2("Querying the CIB... call %d", call_id); return TRUE; } /* A_READCONFIG */ void do_read_config(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t *msg_data) { mainloop_set_trigger(config_read); } void crm_shutdown(int nsig) { if (crmd_mainloop != NULL && g_main_is_running(crmd_mainloop)) { if(is_set(fsa_input_register, R_SHUTDOWN)) { crm_err("Escalating the shutdown"); register_fsa_input_before(C_SHUTDOWN, I_ERROR, NULL); } else { crm_info("Requesting shutdown"); set_bit_inplace(fsa_input_register, R_SHUTDOWN); register_fsa_input(C_SHUTDOWN,I_SHUTDOWN,NULL); if(shutdown_escalation_timer->period_ms < 1) { const char *value = crmd_pref(NULL, XML_CONFIG_ATTR_FORCE_QUIT); int msec = crm_get_msec(value); crm_info("Using default shutdown escalation: %dms", msec); shutdown_escalation_timer->period_ms = msec; } /* cant rely on this... */ crm_notice("Forcing shutdown in: %dms", shutdown_escalation_timer->period_ms); crm_timer_start(shutdown_escalation_timer); } } else { crm_info("exit from shutdown"); exit(LSB_EXIT_OK); } } static void default_cib_update_callback(xmlNode *msg, int call_id, int rc, xmlNode *output, void *user_data) { if(rc != cib_ok) { fsa_data_t *msg_data = NULL; crm_err("CIB Update failed: %s", cib_error2string(rc)); crm_log_xml_warn(output, "update:failed"); register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL); } } #if SUPPORT_HEARTBEAT static void populate_cib_nodes_ha(gboolean with_client_status) { int call_id = 0; const char *ha_node = NULL; xmlNode *cib_node_list = NULL; if(fsa_cluster_conn == NULL) { crm_debug("Not connected"); return; } /* Async get client status information in the cluster */ crm_info("Requesting the list of configured nodes"); fsa_cluster_conn->llc_ops->init_nodewalk(fsa_cluster_conn); cib_node_list = create_xml_node(NULL, XML_CIB_TAG_NODES); do { const char *ha_node_type = NULL; const char *ha_node_uuid = NULL; xmlNode *cib_new_node = NULL; ha_node = fsa_cluster_conn->llc_ops->nextnode(fsa_cluster_conn); if(ha_node == NULL) { continue; } ha_node_type = fsa_cluster_conn->llc_ops->node_type( fsa_cluster_conn, ha_node); if(safe_str_neq(NORMALNODE, ha_node_type)) { crm_debug("Node %s: skipping '%s'", ha_node, ha_node_type); continue; } ha_node_uuid = get_uuid(ha_node); if(ha_node_uuid == NULL) { crm_warn("Node %s: no uuid found", ha_node); continue; } crm_debug("Node: %s (uuid: %s)", ha_node, ha_node_uuid); cib_new_node = create_xml_node(cib_node_list, XML_CIB_TAG_NODE); crm_xml_add(cib_new_node, XML_ATTR_ID, ha_node_uuid); crm_xml_add(cib_new_node, XML_ATTR_UNAME, ha_node); crm_xml_add(cib_new_node, XML_ATTR_TYPE, ha_node_type); } while(ha_node != NULL); fsa_cluster_conn->llc_ops->end_nodewalk(fsa_cluster_conn); /* Now update the CIB with the list of nodes */ fsa_cib_update( XML_CIB_TAG_NODES, cib_node_list, cib_scope_local|cib_quorum_override, call_id, NULL); add_cib_op_callback(fsa_cib_conn, call_id, FALSE, NULL, default_cib_update_callback); free_xml(cib_node_list); crm_debug_2("Complete"); } #endif static void create_cib_node_definition( gpointer key, gpointer value, gpointer user_data) { crm_node_t *node = value; xmlNode *cib_nodes = user_data; xmlNode *cib_new_node = NULL; cib_new_node = create_xml_node(cib_nodes, XML_CIB_TAG_NODE); crm_xml_add(cib_new_node, XML_ATTR_ID, node->uuid); crm_xml_add(cib_new_node, XML_ATTR_UNAME, node->uname); crm_xml_add(cib_new_node, XML_ATTR_TYPE, NORMALNODE); } void populate_cib_nodes(gboolean with_client_status) { int call_id = 0; xmlNode *cib_node_list = NULL; #if SUPPORT_HEARTBEAT if(is_heartbeat_cluster()) { populate_cib_nodes_ha(with_client_status); return; } #endif cib_node_list = create_xml_node(NULL, XML_CIB_TAG_NODES); g_hash_table_foreach( crm_peer_cache, create_cib_node_definition, cib_node_list); fsa_cib_update( XML_CIB_TAG_NODES, cib_node_list, cib_scope_local|cib_quorum_override, call_id, NULL); add_cib_op_callback(fsa_cib_conn, call_id, FALSE, NULL, default_cib_update_callback); free_xml(cib_node_list); crm_debug_2("Complete"); } diff --git a/crmd/te_utils.c b/crmd/te_utils.c index 06cbc3f0d3..fb78a5376b 100644 --- a/crmd/te_utils.c +++ b/crmd/te_utils.c @@ -1,393 +1,390 @@ /* * Copyright (C) 2004 Andrew Beekhof * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public * License as published by the Free Software Foundation; either * version 2 of the License, or (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ #include #include #include #include #include #include #include #include #include #include GCHSource *stonith_src = NULL; crm_trigger_t *stonith_reconnect = NULL; static gboolean fail_incompletable_stonith(crm_graph_t *graph) { GListPtr lpc = NULL; const char *task = NULL; xmlNode *last_action = NULL; if(graph == NULL) { return FALSE; } for(lpc = graph->synapses; lpc != NULL; lpc = lpc->next) { GListPtr lpc2 = NULL; synapse_t *synapse = (synapse_t*)lpc->data; if (synapse->confirmed) { continue; } for(lpc2 = synapse->actions; lpc2 != NULL; lpc2 = lpc2->next) { crm_action_t *action = (crm_action_t*)lpc2->data; if(action->type != action_type_crm || action->confirmed) { continue; } task = crm_element_value(action->xml, XML_LRM_ATTR_TASK); if(task && safe_str_eq(task, CRM_OP_FENCE)) { action->failed = TRUE; last_action = action->xml; update_graph(graph, action); crm_notice("Failing action %d (%s): STONITHd terminated", action->id, ID(action->xml)); } } } if(last_action != NULL) { crm_warn("STONITHd failure resulted in un-runnable actions"); abort_transition(INFINITY, tg_restart, "Stonith failure", last_action); return TRUE; } return FALSE; } static void tengine_stonith_connection_destroy(stonith_t *st, const char *event, xmlNode *msg) { if(is_set(fsa_input_register, R_ST_REQUIRED)) { crm_crit("Fencing daemon connection failed"); mainloop_set_trigger(stonith_reconnect); } else { crm_info("Fencing daemon disconnected"); } /* cbchan will be garbage at this point, arrange for it to be reset */ stonith_api->state = stonith_disconnected; if(AM_I_DC) { fail_incompletable_stonith(transition_graph); trigger_graph(); } } /* */ #ifdef SUPPORT_CMAN # include # include "../lib/common/stack.h" #endif static void tengine_stonith_notify(stonith_t *st, const char *event, xmlNode *msg) { int rc = -99; const char *origin = NULL; const char *target = NULL; const char *executioner = NULL; xmlNode *action = get_xpath_object("//st-data", msg, LOG_ERR); if(action == NULL) { crm_log_xml(LOG_ERR, "Notify data not found", msg); return; } crm_log_xml(LOG_DEBUG, "stonith_notify", msg); crm_element_value_int(msg, F_STONITH_RC, &rc); origin = crm_element_value(action, F_STONITH_ORIGIN); target = crm_element_value(action, F_STONITH_TARGET); executioner = crm_element_value(action, F_STONITH_DELEGATE); - - if(rc == stonith_ok) { + + if(rc == stonith_ok && crm_str_eq(target, fsa_our_uname, TRUE)) { + crm_err("We were alegedly just fenced by %s for %s!", executioner, origin); + register_fsa_error_adv(C_FSA_INTERNAL, I_ERROR, NULL, NULL, __FUNCTION__); + + } else if(rc == stonith_ok) { crm_info("Peer %s was terminated (%s) by %s for %s (ref=%s): %s", target, crm_element_value(action, F_STONITH_OPERATION), executioner, origin, crm_element_value(action, F_STONITH_REMOTE), stonith_error2string(rc)); } else { crm_err("Peer %s could not be terminated (%s) by %s for %s (ref=%s): %s", target, crm_element_value(action, F_STONITH_OPERATION), executioner?executioner:"", origin, crm_element_value(action, F_STONITH_REMOTE), stonith_error2string(rc)); } #ifdef SUPPORT_CMAN if(rc == stonith_ok && is_cman_cluster()) { int rc = 0; char *target_copy = crm_strdup(target); crm_info("Notifing CMAN that '%s' is now fenced", target); - rc = fenced_join(); - if(rc != 0) { - crm_notice("Could not connect to fenced: rc=%d", rc); - - } else { - rc = fenced_external(target_copy); - if(rc != 0) { - crm_err("Could not notify fenced: rc=%d", rc); - } - fenced_leave(); - } - crm_free(target_copy); + rc = fenced_external(target_copy); + if(rc != 0) { + crm_err("Could not notify fenced that '%s' is down: rc=%d", target, rc); + } + crm_free(target_copy); } #endif if(rc == stonith_ok && safe_str_eq(target, origin)) { if(fsa_our_dc == NULL || safe_str_eq(fsa_our_dc, target)) { const char *uuid = get_uuid(target); crm_notice("Target was our leader %s/%s (recorded leader: %s)", target, uuid, fsa_our_dc?fsa_our_dc:""); /* There's no need for everyone to update the cib. * Have the node that performed the op do the update too. * In the unlikely event that both die, the DC would be * shot a second time which is not ideal but safe. */ if(safe_str_eq(executioner, fsa_our_uname)) { send_stonith_update(NULL, target, uuid); } } } } gboolean te_connect_stonith(gpointer user_data) { int lpc = 0; int rc = stonith_ok; if(stonith_api == NULL) { stonith_api = stonith_api_new(); } if(stonith_api->state != stonith_disconnected) { crm_debug_2("Still connected"); return TRUE; } for(lpc = 0; lpc < 30; lpc++) { crm_info("Attempting connection to fencing daemon..."); sleep(1); rc = stonith_api->cmds->connect(stonith_api, crm_system_name, NULL, NULL); if(rc == stonith_ok) { break; } if(user_data != NULL) { crm_err("Sign-in failed: triggered a retry"); mainloop_set_trigger(stonith_reconnect); return TRUE; } crm_err("Sign-in failed: pausing and trying again in 2s..."); sleep(1); } CRM_CHECK(rc == stonith_ok, return TRUE); /* If not, we failed 30 times... just get out */ stonith_api->cmds->register_notification( stonith_api, T_STONITH_NOTIFY_DISCONNECT, tengine_stonith_connection_destroy); stonith_api->cmds->register_notification( stonith_api, STONITH_OP_FENCE, tengine_stonith_notify); crm_info("Connected"); return TRUE; } gboolean stop_te_timer(crm_action_timer_t *timer) { const char *timer_desc = "action timer"; if(timer == NULL) { return FALSE; } if(timer->reason == timeout_abort) { timer_desc = "global timer"; crm_debug_2("Stopping %s", timer_desc); } if(timer->source_id != 0) { crm_debug_2("Stopping %s", timer_desc); g_source_remove(timer->source_id); timer->source_id = 0; } else { crm_debug_2("%s was already stopped", timer_desc); return FALSE; } return TRUE; } gboolean te_graph_trigger(gpointer user_data) { enum transition_status graph_rc = -1; if(transition_graph == NULL) { crm_debug("Nothing to do"); return TRUE; } crm_debug_2("Invoking graph %d in state %s", transition_graph->id, fsa_state2string(fsa_state)); switch(fsa_state) { case S_STARTING: case S_PENDING: case S_NOT_DC: case S_HALT: case S_ILLEGAL: case S_STOPPING: case S_TERMINATE: return TRUE; break; default: break; } if(transition_graph->complete == FALSE) { graph_rc = run_graph(transition_graph); print_graph(LOG_DEBUG_3, transition_graph); if(graph_rc == transition_active) { crm_debug_3("Transition not yet complete"); return TRUE; } else if(graph_rc == transition_pending) { crm_debug_3("Transition not yet complete - no actions fired"); return TRUE; } if(graph_rc != transition_complete) { crm_err("Transition failed: %s", transition_status(graph_rc)); print_graph(LOG_WARNING, transition_graph); } } crm_info("Transition %d is now complete", transition_graph->id); transition_graph->complete = TRUE; notify_crmd(transition_graph); return TRUE; } void trigger_graph_processing(const char *fn, int line) { mainloop_set_trigger(transition_trigger); crm_debug_2("%s:%d - Triggered graph processing", fn, line); } void abort_transition_graph( int abort_priority, enum transition_action abort_action, const char *abort_text, xmlNode *reason, const char *fn, int line) { int log_level = LOG_INFO; const char *magic = NULL; CRM_CHECK(transition_graph != NULL, return); if(reason) { int diff_add_updates = 0; int diff_add_epoch = 0; int diff_add_admin_epoch = 0; int diff_del_updates = 0; int diff_del_epoch = 0; int diff_del_admin_epoch = 0; xmlNode *diff = get_xpath_object("//"F_CIB_UPDATE_RESULT"//diff", reason, LOG_DEBUG_2); magic = crm_element_value(reason, XML_ATTR_TRANSITION_MAGIC); if(diff) { cib_diff_version_details( diff, &diff_add_admin_epoch, &diff_add_epoch, &diff_add_updates, &diff_del_admin_epoch, &diff_del_epoch, &diff_del_updates); do_crm_log(log_level, "%s:%d - Triggered transition abort (complete=%d, tag=%s, id=%s, magic=%s, cib=%d.%d.%d) : %s", fn, line, transition_graph->complete, TYPE(reason), ID(reason), magic?magic:"NA", diff_add_admin_epoch,diff_add_epoch,diff_add_updates, abort_text); } else { do_crm_log(log_level, "%s:%d - Triggered transition abort (complete=%d, tag=%s, id=%s, magic=%s) : %s", fn, line, transition_graph->complete, TYPE(reason), ID(reason), magic?magic:"NA", abort_text); } } else { do_crm_log(log_level, "%s:%d - Triggered transition abort (complete=%d) : %s", fn, line, transition_graph->complete, abort_text); } switch(fsa_state) { case S_STARTING: case S_PENDING: case S_NOT_DC: case S_HALT: case S_ILLEGAL: case S_STOPPING: case S_TERMINATE: do_crm_log(log_level, "Abort suppressed: state=%s (complete=%d)", fsa_state2string(fsa_state), transition_graph->complete); return; default: break; } if(magic == NULL && reason != NULL) { crm_log_xml(log_level+1, "Cause", reason); } /* Make sure any queued calculations are discarded ASAP */ crm_free(fsa_pe_ref); fsa_pe_ref = NULL; if(transition_graph->complete) { register_fsa_input(C_FSA_INTERNAL, I_PE_CALC, NULL); return; } update_abort_priority( transition_graph, abort_priority, abort_action, abort_text); mainloop_set_trigger(transition_trigger); } diff --git a/cts/CM_ais.py b/cts/CM_ais.py index ead4555753..b22c5b91a5 100644 --- a/cts/CM_ais.py +++ b/cts/CM_ais.py @@ -1,353 +1,351 @@ '''CTS: Cluster Testing System: AIS dependent modules... ''' __copyright__=''' Copyright (C) 2007 Andrew Beekhof ''' # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License # as published by the Free Software Foundation; either version 2 # of the License, or (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. import os, sys, warnings from cts.CTSvars import * from cts.CM_lha import crm_lha from cts.CTS import Process ####################################################################### # # LinuxHA v2 dependent modules # ####################################################################### class crm_ais(crm_lha): ''' The crm version 3 cluster manager class. It implements the things we need to talk to and manipulate crm clusters running on top of openais ''' def __init__(self, Environment, randseed=None): crm_lha.__init__(self, Environment, randseed=randseed) self.update({ "Name" : "crm-ais", - "UUIDQueryCmd" : "crmadmin -N --openais", "EpocheCmd" : "crm_node -e --openais", "QuorumCmd" : "crm_node -q --openais", "ParitionCmd" : "crm_node -p --openais", "Pat:They_stopped" : "%s crmd:.*Node %s: .* state=lost .new", "Pat:ChildExit" : "Child process .* exited", # Bad news Regexes. Should never occur. "BadRegexes" : ( r"ERROR:", r"CRIT:", r"TRACE:", r"Shutting down\.", r"Forcing shutdown\.", r"Timer I_TERMINATE just popped", r"input=I_ERROR", r"input=I_FAIL", r"input=I_INTEGRATED cause=C_TIMER_POPPED", r"input=I_FINALIZED cause=C_TIMER_POPPED", r"input=I_ERROR", r", exiting\.", r"WARN.*Ignoring HA message.*vote.*not in our membership list", r"pengine.*Attempting recovery of resource", r"is taking more than 2x its timeout", r"Confirm not received from", r"Welcome reply not received from", r"Attempting to schedule .* after a stop", r"Resource .* was active at shutdown", r"duplicate entries for call_id", r"Search terminated:", r":global_timer_callback", r"Faking parameter digest creation", r"Parameters to .* action changed:", r"Parameters to .* changed", r"Child process .* terminated with signal 11", r"Executing .* fencing operation", # Not inherently bad, but worth tracking r"No need to invoke the TE", r"ping.*: DEBUG: Updated connected = 0", r"Digest mis-match:", ), }) def errorstoignore(self): # At some point implement a more elegant solution that # also produces a report at the end '''Return list of errors which are known and very noisey should be ignored''' if 1: return [ "crm_mon:", "crmadmin:", "update_trace_data", "async_notify: strange, client not found", "ERROR: Message hist queue is filling up" ] return [] def NodeUUID(self, node): return node def ais_components(self): fullcomplist = {} self.complist = [] self.common_ignore = [ "Pending action:", "ERROR: crm_log_message_adv:", "ERROR: MSG: No message to dump", "pending LRM operations at shutdown", "Lost connection to the CIB service", "Connection to the CIB terminated...", "Sending message to CIB service FAILED", "apply_xml_diff: Diff application failed!", "crmd: .*Action A_RECOVER .* not supported", "pingd: .*ERROR: send_update: Could not send update", "send_ipc_message: IPC Channel to .* is not connected", "unconfirmed_actions: Waiting on .* unconfirmed actions", "cib_native_msgready: Message pending on command channel", "crmd:.*do_exit: Performing A_EXIT_1 - forcefully exiting the CRMd", "verify_stopped: Resource .* was active at shutdown. You may ignore this error if it is unmanaged.", "ERROR: attrd_connection_destroy: Lost connection to attrd", "nfo: te_fence_node: Executing .* fencing operation", ] fullcomplist["cib"] = Process(self, "cib", pats = [ "State transition .* S_RECOVERY", "Respawning .* crmd", "Respawning .* attrd", "Lost connection to the CIB service", "Connection to the CIB terminated...", "Child process crmd exited .* rc=2", "Child process attrd exited .* rc=1", "crmd: .*Input I_TERMINATE from do_recover", "crmd: .*I_ERROR.*crmd_cib_connection_destroy", "crmd:.*do_exit: Could not recover from internal error", ], badnews_ignore = self.common_ignore) fullcomplist["lrmd"] = Process(self, "lrmd", pats = [ "State transition .* S_RECOVERY", "LRM Connection failed", "Respawning .* crmd", "crmd: .*I_ERROR.*lrm_connection_destroy", "Child process crmd exited .* rc=2", "crmd: .*Input I_TERMINATE from do_recover", "crmd:.*do_exit: Could not recover from internal error", ], badnews_ignore = self.common_ignore) fullcomplist["crmd"] = Process(self, "crmd", pats = [ # "WARN: determine_online_status: Node .* is unclean", # "Scheduling Node .* for STONITH", # "Executing .* fencing operation", # Only if the node wasn't the DC: "State transition S_IDLE", "State transition .* -> S_IDLE", ], badnews_ignore = self.common_ignore) fullcomplist["attrd"] = Process(self, "attrd", pats = [ "crmd: .*ERROR: attrd_connection_destroy: Lost connection to attrd" ], badnews_ignore = self.common_ignore) fullcomplist["pengine"] = Process(self, "pengine", dc_pats = [ "State transition .* S_RECOVERY", "Respawning .* crmd", "Child process crmd exited .* rc=2", "crmd: .*pe_connection_destroy: Connection to the Policy Engine failed", "crmd: .*I_ERROR.*save_cib_contents", "crmd: .*Input I_TERMINATE from do_recover", "crmd:.*do_exit: Could not recover from internal error", ], badnews_ignore = self.common_ignore) stonith_ignore = [ "update_failcount: Updating failcount for child_DoFencing", "ERROR: te_connect_stonith: Sign-in failed: triggered a retry", ] stonith_ignore.extend(self.common_ignore) fullcomplist["stonith-ng"] = Process(self, "stonith-ng", process="stonithd", pats = [ "CRIT: stonith_dispatch: Lost connection to the STONITH service", "tengine_stonith_connection_destroy: Fencing daemon connection failed", "Attempting connection to fencing daemon", "te_connect_stonith: Connected", ], badnews_ignore = stonith_ignore) vgrind = self.Env["valgrind-procs"].split() for key in fullcomplist.keys(): if self.Env["valgrind-tests"]: if key in vgrind: # Processes running under valgrind can't be shot with "killall -9 processname" self.log("Filtering %s from the component list as it is being profiled by valgrind" % key) continue if key == "stonith-ng" and not self.Env["DoFencing"]: continue self.complist.append(fullcomplist[key]) #self.complist = [ fullcomplist["pengine"] ] return self.complist class crm_whitetank(crm_ais): ''' The crm version 3 cluster manager class. It implements the things we need to talk to and manipulate crm clusters running on top of openais ''' def __init__(self, Environment, randseed=None): crm_ais.__init__(self, Environment, randseed=randseed) self.update({ "Name" : "crm-whitetank", "StartCmd" : CTSvars.INITDIR+"/openais start", "StopCmd" : CTSvars.INITDIR+"/openais stop", "Pat:We_stopped" : "%s.*openais.*pcmk_shutdown: Shutdown complete", "Pat:They_stopped" : "%s crmd:.*Node %s: .* state=lost .new", "Pat:They_dead" : "openais:.*Node %s is now: lost", "Pat:ChildKilled" : "%s openais.*Child process %s terminated with signal 9", "Pat:ChildRespawn" : "%s openais.*Respawning failed child process: %s", "Pat:ChildExit" : "Child process .* exited", }) def Components(self): self.ais_components() aisexec_ignore = [ "ERROR: ais_dispatch: Receiving message .* failed", "crmd: .*I_ERROR.*crmd_cib_connection_destroy", "cib: .*ERROR: cib_ais_destroy: AIS connection terminated", #"crmd: .*ERROR: crm_ais_destroy: AIS connection terminated", "crmd:.*do_exit: Could not recover from internal error", "crmd: .*I_TERMINATE.*do_recover", "attrd: .*CRIT: attrd_ais_destroy: Lost connection to OpenAIS service!", "stonithd: .*ERROR: AIS connection terminated", ] aisexec_ignore.extend(self.common_ignore) self.complist.append(Process(self, "aisexec", pats = [ "ERROR: ais_dispatch: AIS connection failed", "crmd: .*ERROR: do_exit: Could not recover from internal error", "pengine: .*Scheduling Node .* for STONITH", "stonithd: .*requests a STONITH operation RESET on node", "stonithd: .*Succeeded to STONITH the node", ], badnews_ignore = aisexec_ignore)) class crm_flatiron(crm_ais): ''' The crm version 3 cluster manager class. It implements the things we need to talk to and manipulate crm clusters running on top of openais ''' def __init__(self, Environment, randseed=None): crm_ais.__init__(self, Environment, randseed=randseed) self.update({ "Name" : "crm-flatiron", "StartCmd" : "service corosync start", "StopCmd" : "service corosync stop", # The next pattern is too early # "Pat:We_stopped" : "%s.*Service engine unloaded: Pacemaker Cluster Manager", # The next pattern would be preferred, but it doesn't always come out # "Pat:We_stopped" : "%s.*Corosync Cluster Engine exiting with status", "Pat:We_stopped" : "%s.*Service engine unloaded: corosync cluster quorum service", "Pat:They_stopped" : "%s crmd:.*Node %s: .* state=lost .new", "Pat:They_dead" : "corosync:.*Node %s is now: lost", "Pat:ChildKilled" : "%s corosync.*Child process %s terminated with signal 9", "Pat:ChildRespawn" : "%s corosync.*Respawning failed child process: %s", }) def Components(self): self.ais_components() corosync_ignore = [ "ERROR: ais_dispatch: Receiving message .* failed", "crmd: .*I_ERROR.*crmd_cib_connection_destroy", "cib: .*ERROR: cib_ais_destroy: AIS connection terminated", #"crmd: .*ERROR: crm_ais_destroy: AIS connection terminated", "crmd:.*do_exit: Could not recover from internal error", "crmd: .*I_TERMINATE.*do_recover", "attrd: .*CRIT: attrd_ais_destroy: Lost connection to Corosync service!", "stonithd: .*ERROR: AIS connection terminated", ] # corosync_ignore.extend(self.common_ignore) # self.complist.append(Process(self, "corosync", pats = [ # "ERROR: ais_dispatch: AIS connection failed", # "crmd: .*ERROR: do_exit: Could not recover from internal error", # "pengine: .*Scheduling Node .* for STONITH", # "stonithd: .*requests a STONITH operation RESET on node", # "stonithd: .*Succeeded to STONITH the node", # ], badnews_ignore = corosync_ignore)) return self.complist class crm_mcp(crm_flatiron): ''' The crm version 3 cluster manager class. It implements the things we need to talk to and manipulate crm clusters running on top of openais ''' def __init__(self, Environment, randseed=None): crm_flatiron.__init__(self, Environment, randseed=randseed) self.update({ "Name" : "crm-mcp", "StartCmd" : "service corosync start; service pacemaker start", "StopCmd" : "service pacemaker stop; service corosync stop", "Pat:We_stopped" : "%s.*Service engine unloaded: corosync cluster quorum service", "Pat:They_stopped" : "%s crmd:.*Node %s: .* state=lost .new", "Pat:They_dead" : "crmd:.*Node %s: .* state=lost .new", "Pat:ChildKilled" : "%s pacemakerd.*Child process %s terminated with signal 9", "Pat:ChildRespawn" : "%s pacemakerd.*Respawning failed child process: %s", }) class crm_cman(crm_flatiron): ''' The crm version 3 cluster manager class. It implements the things we need to talk to and manipulate crm clusters running on top of openais ''' def __init__(self, Environment, randseed=None): crm_flatiron.__init__(self, Environment, randseed=randseed) self.update({ "Name" : "crm-cman", - "StartCmd" : "service corosync start; service pacemaker start", - "StopCmd" : "service pacemaker stop; cman_tool leave", + "StartCmd" : "service cman start; service pacemaker start", + "StopCmd" : "service pacemaker stop; service cman stop;", - "UUIDQueryCmd" : "crmadmin -N --cman", "EpocheCmd" : "crm_node -e --cman", "QuorumCmd" : "crm_node -q --cman", "ParitionCmd" : "crm_node -p --cman", "Pat:We_stopped" : "%s.*Service engine unloaded: corosync cluster quorum service", "Pat:They_stopped" : "%s crmd:.*Node %s: .* state=lost .new", "Pat:They_dead" : "crmd:.*Node %s: .* state=lost .new", "Pat:ChildKilled" : "%s pacemakerd.*Child process %s terminated with signal 9", "Pat:ChildRespawn" : "%s pacemakerd.*Respawning failed child process: %s", }) diff --git a/cts/CTSlab.py b/cts/CTSlab.py index b8f71ff972..5493bf9bc8 100755 --- a/cts/CTSlab.py +++ b/cts/CTSlab.py @@ -1,461 +1,468 @@ #!/usr/bin/python '''CTS: Cluster Testing System: Lab environment module ''' __copyright__=''' Copyright (C) 2001,2005 Alan Robertson Licensed under the GNU GPL. ''' # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License # as published by the Free Software Foundation; either version 2 # of the License, or (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. from UserDict import UserDict import sys, types, string, string, signal, os, socket pdir=os.path.dirname(sys.path[0]) sys.path.insert(0, pdir) # So that things work from the source directory try: from cts.CTSvars import * from cts.CM_ais import * from cts.CM_lha import crm_lha from cts.CTSaudits import AuditList from cts.CTStests import TestList from cts.CTSscenarios import * except ImportError: sys.stderr.write("abort: couldn't find cts libraries in [%s]\n" % ' '.join(sys.path)) sys.stderr.write("(check your install and PYTHONPATH)\n") sys.exit(-1) cm = None Tests = [] Chosen = [] scenario = None # Not really used, the handler in def sig_handler(signum, frame) : if cm: cm.log("Interrupted by signal %d"%signum) if scenario: scenario.summarize() if signum == 15 : if scenario: scenario.TearDown() sys.exit(1) class LabEnvironment(CtsLab): def __init__(self): CtsLab.__init__(self) # Get a random seed for the random number generator. - self["DoStonith"] = 1 self["DoStandby"] = 1 self["DoFencing"] = 1 self["XmitLoss"] = "0.0" self["RecvLoss"] = "0.0" self["IPBase"] = "127.0.0.10" self["ConnectivityHost"] = socket.gethostname() self["ClobberCIB"] = 0 self["CIBfilename"] = None self["CIBResource"] = 0 self["DoBSC"] = 0 self["use_logd"] = 0 self["oprofile"] = [] self["warn-inactive"] = 0 self["ListTests"] = 0 self["benchmark"] = 0 self["Schema"] = "pacemaker-1.0" self["Stack"] = "openais" self["stonith-type"] = "external/ssh" self["stonith-params"] = "hostlist=all,livedangerously=yes" self["at-boot"] = 1 # Does the cluster software start automatically when the node boots self["logger"] = ([StdErrLog(self)]) self["loop-minutes"] = 60 self["valgrind-prefix"] = None self["valgrind-procs"] = "cib crmd attrd pengine stonith-ng" self["valgrind-opts"] = """--leak-check=full --show-reachable=yes --trace-children=no --num-callers=25 --gen-suppressions=all --suppressions="""+CTSvars.CTS_home+"""/cts.supp""" #self["valgrind-opts"] = """--trace-children=no --num-callers=25 --gen-suppressions=all --suppressions="""+CTSvars.CTS_home+"""/cts.supp""" self["experimental-tests"] = 0 self["valgrind-tests"] = 0 self["unsafe-tests"] = 1 self["loop-tests"] = 1 self["scenario"] = "random" def usage(arg, status=1): print "Illegal argument " + arg print "usage: " + sys.argv[0] +" [options] number-of-iterations" print "\nCommon options: " print "\t [--at-boot (1|0)], does the cluster software start at boot time" print "\t [--nodes 'node list'], list of cluster nodes separated by whitespace" print "\t [--limit-nodes max], only use the first 'max' cluster nodes supplied with --nodes" print "\t [--stack (heartbeat|ais)], which cluster stack is installed" print "\t [--logfile path], where should the test software look for logs from cluster nodes" print "\t [--outputfile path], optional location for the test software to write logs to" print "\t [--syslog-facility name], which syslog facility should the test software log to" print "\t [--choose testcase-name], run only the named test" print "\t [--list-tests], list the valid tests" print "\t [--benchmark], add the timing information" print "\t " print "Options for release testing: " print "\t [--clobber-cib | -c ] Erase any existing configuration" print "\t [--populate-resources | -r] Generate a sample configuration" print "\t [--test-ip-base ip] Offset for generated IP address resources" print "\t " print "Additional (less common) options: " print "\t [--trunc (truncate logfile before starting)]" print "\t [--xmit-loss lost-rate(0.0-1.0)]" print "\t [--recv-loss lost-rate(0.0-1.0)]" print "\t [--standby (1 | 0 | yes | no)]" print "\t [--fencing (1 | 0 | yes | no)]" print "\t [--stonith (1 | 0 | yes | no)]" print "\t [--stonith-type type]" print "\t [--stonith-args name=value]" print "\t [--bsc]" print "\t [--once], run all valid tests once" print "\t [--no-loop-tests], dont run looping/time-based tests" print "\t [--no-unsafe-tests], dont run tests that are unsafe for use with ocfs2/drbd" print "\t [--valgrind-tests], include tests using valgrind" print "\t [--experimental-tests], include experimental tests" print "\t [--oprofile 'node list'], list of cluster nodes to run oprofile on]" print "\t [--qarsh] Use the QARSH backdoor to access nodes instead of SSH" print "\t [--seed random_seed]" print "\t [--set option=value]" sys.exit(status) # # A little test code... # if __name__ == '__main__': Environment = LabEnvironment() NumIter = 0 Version = 1 LimitNodes = 0 TruncateLog = 0 ListTests = 0 HaveSeed = 0 node_list = '' # Set the signal handler signal.signal(15, sig_handler) signal.signal(10, sig_handler) # Process arguments... skipthis=None args=sys.argv[1:] for i in range(0, len(args)): if skipthis: skipthis=None continue elif args[i] == "-l" or args[i] == "--limit-nodes": skipthis=1 LimitNodes = int(args[i+1]) elif args[i] == "-r" or args[i] == "--populate-resources": Environment["CIBResource"] = 1 elif args[i] == "-L" or args[i] == "--logfile": skipthis=1 Environment["LogFileName"] = args[i+1] elif args[i] == "--outputfile": skipthis=1 Environment["OutputFile"] = args[i+1] elif args[i] == "--test-ip-base": skipthis=1 Environment["IPBase"] = args[i+1] elif args[i] == "--oprofile": skipthis=1 Environment["oprofile"] = args[i+1].split(' ') elif args[i] == "--trunc": Environment["TruncateLog"]=1 elif args[i] == "--list-tests" or args[i] == "--list" : Environment["ListTests"]=1 elif args[i] == "--benchmark": Environment["benchmark"]=1 elif args[i] == "--bsc": Environment["DoBSC"] = 1 Environment["scenario"] = "basic-sanity" elif args[i] == "--qarsh": Environment.rsh.enable_qarsh() elif args[i] == "--fencing": skipthis=1 if args[i+1] == "1" or args[i+1] == "yes": Environment["DoFencing"] = 1 elif args[i+1] == "0" or args[i+1] == "no": Environment["DoFencing"] = 0 else: usage(args[i+1]) elif args[i] == "--stonith": skipthis=1 if args[i+1] == "1" or args[i+1] == "yes": - Environment["DoStonith"]=1 + Environment["DoFencing"]=1 elif args[i+1] == "0" or args[i+1] == "no": - Environment["DoStonith"]=0 + Environment["DoFencing"]=0 + elif args[i+1] == "rhcs": + Environment["DoStonith"]=1 + Environment["stonith-type"] = "fence_xvm" + Environment["stonith-params"] = "pcmk_arg_map=domain:uname" + elif args[i+1] == "lha": + Environment["DoStonith"]=1 + Environment["stonith-type"] = "external/ssh" + Environment["stonith-params"] = "hostlist=all,livedangerously=yes" else: usage(args[i+1]) elif args[i] == "--stonith-type": Environment["stonith-type"] = args[i+1] skipthis=1 elif args[i] == "--stonith-args": Environment["stonith-params"] = args[i+1] skipthis=1 elif args[i] == "--standby": skipthis=1 if args[i+1] == "1" or args[i+1] == "yes": Environment["DoStandby"] = 1 elif args[i+1] == "0" or args[i+1] == "no": Environment["DoStandby"] = 0 else: usage(args[i+1]) elif args[i] == "--clobber-cib" or args[i] == "-c": Environment["ClobberCIB"] = 1 elif args[i] == "--cib-filename": skipthis=1 Environment["CIBfilename"] = args[i+1] elif args[i] == "--xmit-loss": try: float(args[i+1]) except ValueError: print ("--xmit-loss parameter should be float") usage(args[i+1]) skipthis=1 Environment["XmitLoss"] = args[i+1] elif args[i] == "--recv-loss": try: float(args[i+1]) except ValueError: print ("--recv-loss parameter should be float") usage(args[i+1]) skipthis=1 Environment["RecvLoss"] = args[i+1] elif args[i] == "--choose": skipthis=1 Chosen.append(args[i+1]) Environment["scenario"] = "sequence" elif args[i] == "--nodes": skipthis=1 node_list = args[i+1].split(' ') elif args[i] == "--syslog-facility" or args[i] == "--facility": skipthis=1 Environment["SyslogFacility"] = args[i+1] elif args[i] == "--seed": skipthis=1 Environment.SeedRandom(args[i+1]) elif args[i] == "--warn-inactive": Environment["warn-inactive"] = 1 elif args[i] == "--schema": skipthis=1 Environment["Schema"] = args[i+1] elif args[i] == "--ais": Environment["Stack"] = "openais" elif args[i] == "--at-boot" or args[i] == "--cluster-starts-at-boot": skipthis=1 if args[i+1] == "1" or args[i+1] == "yes": Environment["at-boot"] = 1 elif args[i+1] == "0" or args[i+1] == "no": Environment["at-boot"] = 0 else: usage(args[i+1]) elif args[i] == "--heartbeat" or args[i] == "--lha": Environment["Stack"] = "heartbeat" elif args[i] == "--hae": Environment["Stack"] = "openais" Environment["Schema"] = "hae" elif args[i] == "--stack": Environment["Stack"] = args[i+1] skipthis=1 elif args[i] == "--once": Environment["scenario"] = "all-once" elif args[i] == "--valgrind-tests": Environment["valgrind-tests"] = 1 elif args[i] == "--no-loop-tests": Environment["loop-tests"] = 0 elif args[i] == "--loop-minutes": skipthis=1 try: Environment["loop-minutes"]=int(args[i+1]) except ValueError: usage(args[i]) elif args[i] == "--no-unsafe-tests": Environment["unsafe-tests"] = 0 elif args[i] == "--experimental-tests": Environment["experimental-tests"] = 1 elif args[i] == "--set": skipthis=1 (name, value) = args[i+1].split('=') Environment[name] = value else: try: NumIter=int(args[i]) except ValueError: usage(args[i]) if Environment["DoBSC"]: NumIter = 2 LimitNodes = 1 Chosen.append("AddResource") Environment["ClobberCIB"] = 1 Environment["CIBResource"] = 0 Environment["logger"].append(FileLog(Environment, Environment["LogFileName"])) elif Environment["OutputFile"]: Environment["logger"].append(FileLog(Environment, Environment["OutputFile"])) elif Environment["SyslogFacility"]: Environment["logger"].append(SysLog(Environment)) if Environment["Stack"] == "heartbeat" or Environment["Stack"] == "lha": Environment["Stack"] = "heartbeat" Environment['CMclass'] = crm_lha elif Environment["Stack"] == "openais" or Environment["Stack"] == "ais" or Environment["Stack"] == "whitetank": Environment["Stack"] = "openais (whitetank)" Environment['CMclass'] = crm_whitetank Environment["use_logd"] = 0 elif Environment["Stack"] == "corosync" or Environment["Stack"] == "cs" or Environment["Stack"] == "flatiron": Environment["Stack"] = "corosync (flatiron)" Environment['CMclass'] = crm_flatiron Environment["use_logd"] = 0 elif Environment["Stack"] == "cman": Environment["Stack"] = "corosync (cman)" Environment['CMclass'] = crm_cman Environment["use_logd"] = 0 elif Environment["Stack"] == "mcp": Environment["Stack"] = "corosync (mcp)" Environment['CMclass'] = crm_mcp Environment["use_logd"] = 0 else: print "Unknown stack: "+Environment["Stack"] sys.exit(1) if len(node_list) < 1: print "No nodes specified!" sys.exit(1) if LimitNodes > 0: if len(node_list) > LimitNodes: print("Limiting the number of nodes configured=%d (max=%d)" %(len(node_list), LimitNodes)) while len(node_list) > LimitNodes: node_list.pop(len(node_list)-1) Environment["nodes"] = node_list # Create the Cluster Manager object cm = Environment['CMclass'](Environment) if TruncateLog: Environment.log("Truncating %s" % LogFile) lf = open(LogFile, "w"); if lf != None: lf.truncate(0) lf.close() Audits = AuditList(cm) if Environment["ListTests"] == 1 : Tests = TestList(cm, Audits) Environment.log("Total %d tests"%len(Tests)) for test in Tests : Environment.log(str(test.name)); sys.exit(0) if len(Chosen) == 0: Tests = TestList(cm, Audits) else: for TestCase in Chosen: match = None for test in TestList(cm, Audits): if test.name == TestCase: match = test if not match: usage("--choose: No applicable/valid tests chosen") else: Tests.append(match) # Scenario selection if Environment["scenario"] == "basic-sanity": scenario = RandomTests(cm, [ BasicSanityCheck(Environment) ], Audits, Tests) elif Environment["scenario"] == "all-once": NumIter = len(Tests) scenario = AllOnce( cm, [ InitClusterManager(Environment), PacketLoss(Environment) ], Audits, Tests) elif Environment["scenario"] == "sequence": scenario = Sequence( cm, [ InitClusterManager(Environment), PacketLoss(Environment) ], Audits, Tests) else: scenario = RandomTests( cm, [ InitClusterManager(Environment), PacketLoss(Environment) ], Audits, Tests) Environment.log(">>>>>>>>>>>>>>>> BEGINNING " + repr(NumIter) + " TESTS ") Environment.log("Stack: %s" % Environment["Stack"]) Environment.log("Schema: %s" % Environment["Schema"]) Environment.log("Scenario: %s" % scenario.__doc__) Environment.log("Random Seed: %s" % Environment["RandSeed"]) Environment.log("System log files: %s" % Environment["LogFileName"]) Environment.dump() rc = Environment.run(scenario, NumIter) sys.exit(rc) diff --git a/cts/CTStests.py b/cts/CTStests.py index 2287b0efd9..6661e37fd7 100644 --- a/cts/CTStests.py +++ b/cts/CTStests.py @@ -1,2223 +1,2223 @@ '''CTS: Cluster Testing System: Tests module There are a few things we want to do here: ''' __copyright__=''' Copyright (C) 2000, 2001 Alan Robertson Licensed under the GNU GPL. Add RecourceRecover testcase Zhao Kai ''' # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License # as published by the Free Software Foundation; either version 2 # of the License, or (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # # SPECIAL NOTE: # # Tests may NOT implement any cluster-manager-specific code in them. # EXTEND the ClusterManager object to provide the base capabilities # the test needs if you need to do something that the current CM classes # do not. Otherwise you screw up the whole point of the object structure # in CTS. # # Thank you. # import time, os, re, types, string, tempfile, sys from stat import * from cts import CTS from cts.CTSaudits import * AllTestClasses = [ ] class CTSTest: ''' A Cluster test. We implement the basic set of properties and behaviors for a generic cluster test. Cluster tests track their own statistics. We keep each of the kinds of counts we track as separate {name,value} pairs. ''' def __init__(self, cm): #self.name="the unnamed test" self.Stats = {"calls":0 , "success":0 , "failure":0 , "skipped":0 , "auditfail":0} # if not issubclass(cm.__class__, ClusterManager): # raise ValueError("Must be a ClusterManager object") self.CM = cm self.Audits = [] self.timeout=120 self.passed = 1 self.is_loop = 0 self.is_unsafe = 0 self.is_experimental = 0 self.is_valgrind = 0 self.benchmark = 0 # which tests to benchmark self.timer = {} # timers def has_key(self, key): return self.Stats.has_key(key) def __setitem__(self, key, value): self.Stats[key] = value def __getitem__(self, key): return self.Stats[key] def log_mark(self, msg): self.CM.debug("MARK: test %s %s %d" % (self.name,msg,time.time())) return def get_timer(self,key = "test"): try: return self.timer[key] except: return 0 def set_timer(self,key = "test"): self.timer[key] = time.time() return self.timer[key] def log_timer(self,key = "test"): elapsed = 0 if key in self.timer: elapsed = time.time() - self.timer[key] s = key == "test" and self.name or "%s:%s" %(self.name,key) self.CM.debug("%s runtime: %.2f" % (s, elapsed)) del self.timer[key] return elapsed def incr(self, name): '''Increment (or initialize) the value associated with the given name''' if not self.Stats.has_key(name): self.Stats[name]=0 self.Stats[name] = self.Stats[name]+1 # Reset the test passed boolean if name == "calls": self.passed = 1 def failure(self, reason="none"): '''Increment the failure count''' self.passed = 0 self.incr("failure") self.CM.log(("Test %s" % self.name).ljust(35) +" FAILED: %s" % reason) return None def success(self): '''Increment the success count''' self.incr("success") return 1 def skipped(self): '''Increment the skipped count''' self.incr("skipped") return 1 def __call__(self, node): '''Perform the given test''' raise ValueError("Abstract Class member (__call__)") self.incr("calls") return self.failure() def audit(self): passed = 1 if len(self.Audits) > 0: for audit in self.Audits: if not audit(): self.CM.log("Internal %s Audit %s FAILED." % (self.name, audit.name())) self.incr("auditfail") passed = 0 return passed def setup(self, node): '''Setup the given test''' return self.success() def teardown(self, node): '''Tear down the given test''' return self.success() def create_watch(self, patterns, timeout, name=None): if not name: name = self.name return CTS.LogWatcher(self.CM.Env, self.CM["LogFileName"], patterns, name, timeout) def local_badnews(self, prefix, watch, local_ignore=[]): errcount = 0 if not prefix: prefix = "LocalBadNews:" ignorelist = [] ignorelist.append(" CTS: ") ignorelist.append(prefix) ignorelist.extend(local_ignore) while errcount < 100: match=watch.look(0) if match: add_err = 1 for ignore in ignorelist: if add_err == 1 and re.search(ignore, match): add_err = 0 if add_err == 1: self.CM.log(prefix + " " + match) errcount=errcount+1 else: break else: self.CM.log("Too many errors!") return errcount def is_applicable(self): return self.is_applicable_common() def is_applicable_common(self): '''Return TRUE if we are applicable in the current test configuration''' #raise ValueError("Abstract Class member (is_applicable)") if self.is_loop and not self.CM.Env["loop-tests"]: return 0 elif self.is_unsafe and not self.CM.Env["unsafe-tests"]: return 0 elif self.is_valgrind and not self.CM.Env["valgrind-tests"]: return 0 elif self.is_experimental and not self.CM.Env["experimental-tests"]: return 0 elif self.CM.Env["benchmark"] and self.benchmark == 0: return 0 return 1 def find_ocfs2_resources(self, node): self.r_o2cb = None self.r_ocfs2 = [] (rc, lines) = self.CM.rsh(node, "crm_resource -c", None) for line in lines: if re.search("^Resource", line): r = AuditResource(self.CM, line) if r.rtype == "o2cb" and r.parent != "NA": self.CM.debug("Found o2cb: %s" % self.r_o2cb) self.r_o2cb = r.parent if re.search("^Constraint", line): c = AuditConstraint(self.CM, line) if c.type == "rsc_colocation" and c.target == self.r_o2cb: self.r_ocfs2.append(c.rsc) self.CM.debug("Found ocfs2 filesystems: %s" % repr(self.r_ocfs2)) return len(self.r_ocfs2) def canrunnow(self, node): '''Return TRUE if we can meaningfully run right now''' return 1 def errorstoignore(self): '''Return list of errors which are 'normal' and should be ignored''' return [] ################################################################### class StopTest(CTSTest): ################################################################### '''Stop (deactivate) the cluster manager on a node''' def __init__(self, cm): CTSTest.__init__(self, cm) self.name="Stop" def __call__(self, node): '''Perform the 'stop' test. ''' self.incr("calls") if self.CM.ShouldBeStatus[node] != "up": return self.skipped() patterns = [] # Technically we should always be able to notice ourselves stopping patterns.append(self.CM["Pat:We_stopped"] % node) #if self.CM.Env["use_logd"]: # patterns.append(self.CM["Pat:Logd_stopped"] % node) # Any active node needs to notice this one left # NOTE: This wont work if we have multiple partitions for other in self.CM.Env["nodes"]: if self.CM.ShouldBeStatus[other] == "up" and other != node: patterns.append(self.CM["Pat:They_stopped"] %(other, self.CM.key_for_node(node))) #self.debug("Checking %s will notice %s left"%(other, node)) watch = self.create_watch(patterns, self.CM["DeadTime"]) watch.setwatch() if node == self.CM.OurNode: self.incr("us") else: if self.CM.upcount() <= 1: self.incr("all") else: self.incr("them") self.CM.StopaCM(node) watch_result = watch.lookforall() failreason=None UnmatchedList = "||" if watch.unmatched: (rc, output) = self.CM.rsh(node, "/bin/ps axf", None) for line in output: self.CM.debug(line) for regex in watch.unmatched: self.CM.log ("ERROR: Shutdown pattern not found: %s" % (regex)) UnmatchedList += regex + "||"; failreason="Missing shutdown pattern" self.CM.cluster_stable(self.CM["DeadTime"]) if not watch.unmatched or self.CM.upcount() == 0: return self.success() if len(watch.unmatched) >= self.CM.upcount(): return self.failure("no match against (%s)" % UnmatchedList) if failreason == None: return self.success() else: return self.failure(failreason) # # We don't register StopTest because it's better when called by # another test... # ################################################################### class StartTest(CTSTest): ################################################################### '''Start (activate) the cluster manager on a node''' def __init__(self, cm, debug=None): CTSTest.__init__(self,cm) self.name="start" self.debug = debug def __call__(self, node): '''Perform the 'start' test. ''' self.incr("calls") if self.CM.upcount() == 0: self.incr("us") else: self.incr("them") if self.CM.ShouldBeStatus[node] != "down": return self.skipped() elif self.CM.StartaCM(node): return self.success() else: return self.failure("Startup %s on node %s failed" %(self.CM["Name"], node)) # # We don't register StartTest because it's better when called by # another test... # ################################################################### class FlipTest(CTSTest): ################################################################### '''If it's running, stop it. If it's stopped start it. Overthrow the status quo... ''' def __init__(self, cm): CTSTest.__init__(self,cm) self.name="Flip" self.start = StartTest(cm) self.stop = StopTest(cm) def __call__(self, node): '''Perform the 'Flip' test. ''' self.incr("calls") if self.CM.ShouldBeStatus[node] == "up": self.incr("stopped") ret = self.stop(node) type="up->down" # Give the cluster time to recognize it's gone... time.sleep(self.CM["StableTime"]) elif self.CM.ShouldBeStatus[node] == "down": self.incr("started") ret = self.start(node) type="down->up" else: return self.skipped() self.incr(type) if ret: return self.success() else: return self.failure("%s failure" % type) # Register FlipTest as a good test to run AllTestClasses.append(FlipTest) ################################################################### class RestartTest(CTSTest): ################################################################### '''Stop and restart a node''' def __init__(self, cm): CTSTest.__init__(self,cm) self.name="Restart" self.start = StartTest(cm) self.stop = StopTest(cm) self.benchmark = 1 def __call__(self, node): '''Perform the 'restart' test. ''' self.incr("calls") self.incr("node:" + node) ret1 = 1 if self.CM.StataCM(node): self.incr("WasStopped") if not self.start(node): return self.failure("start (setup) failure: "+node) self.set_timer() if not self.stop(node): return self.failure("stop failure: "+node) if not self.start(node): return self.failure("start failure: "+node) return self.success() # Register RestartTest as a good test to run AllTestClasses.append(RestartTest) ################################################################### class StonithdTest(CTSTest): ################################################################### def __init__(self, cm): CTSTest.__init__(self, cm) self.name="Stonithd" self.startall = SimulStartLite(cm) self.benchmark = 1 def __call__(self, node): self.incr("calls") if len(self.CM.Env["nodes"]) < 2: return self.skipped() ret = self.startall(None) if not ret: return self.failure("Setup failed") is_dc = self.CM.is_node_dc(node) watchpats = [] watchpats.append("stonith-ng:.*Operation .* for host '%s' with device .* returned: 0" % node) watchpats.append("tengine_stonith_notify: Peer %s was terminated .*: OK" % node) if is_dc: watchpats.append("tengine_stonith_notify: Target was our leader .*%s" % node) else: watchpats.append("tengine_stonith_callback: .*: OK ") if self.CM.Env["LogWatcher"] != "remote" or not is_dc: # Often remote logs aren't flushed to disk by the time the node is shot, # so we wont be able to find them # Remote syslog doesn't suffer this problem because they're already on # the loghost when the node is shot watchpats.append("Node %s will be fenced because termination was requested" % node) watchpats.append("Scheduling Node %s for STONITH" % node) watchpats.append("Executing .* fencing operation") if self.CM.Env["at-boot"] == 0: self.CM.debug("Expecting %s to stay down" % node) self.CM.ShouldBeStatus[node]="down" else: self.CM.debug("Expecting %s to come up again %d" % (node, self.CM.Env["at-boot"])) watchpats.append("%s crmd: .* S_STARTING -> S_PENDING" % node) watchpats.append("%s crmd: .* S_PENDING -> S_NOT_DC" % node) watch = self.create_watch(watchpats, 30 + self.CM["DeadTime"] + self.CM["StableTime"] + self.CM["StartTime"]) watch.setwatch() self.CM.rsh(node, "crm_attribute --node %s --type status --attr-name terminate --attr-value true" % node) self.set_timer("fence") matched = watch.lookforall() self.log_timer("fence") self.set_timer("reform") if watch.unmatched: self.CM.log("Patterns not found: " + repr(watch.unmatched)) self.CM.debug("Waiting for the cluster to recover") self.CM.cluster_stable() self.CM.debug("Waiting STONITHd node to come back up") self.CM.ns.WaitForAllNodesToComeUp(self.CM.Env["nodes"], 600) self.CM.debug("Waiting for the cluster to re-stabilize with all nodes") is_stable = self.CM.cluster_stable(self.CM["StartTime"]) if not matched: return self.failure("Didn't find all expected patterns") elif not is_stable: return self.failure("Cluster did not become stable") self.log_timer("reform") return self.success() def errorstoignore(self): return [ "Executing .* fencing operation" ] def is_applicable(self): if not self.is_applicable_common(): return 0 - if self.CM.Env.has_key("DoStonith"): - return self.CM.Env["DoStonith"] + if self.CM.Env.has_key("DoFencing"): + return self.CM.Env["DoFencing"] return 1 AllTestClasses.append(StonithdTest) ################################################################### class StartOnebyOne(CTSTest): ################################################################### '''Start all the nodes ~ one by one''' def __init__(self, cm): CTSTest.__init__(self,cm) self.name="StartOnebyOne" self.stopall = SimulStopLite(cm) self.start = StartTest(cm) self.ns=CTS.NodeStatus(cm.Env) def __call__(self, dummy): '''Perform the 'StartOnebyOne' test. ''' self.incr("calls") # We ignore the "node" parameter... # Shut down all the nodes... ret = self.stopall(None) if not ret: return self.failure("Test setup failed") failed=[] self.set_timer() for node in self.CM.Env["nodes"]: if not self.start(node): failed.append(node) if len(failed) > 0: return self.failure("Some node failed to start: " + repr(failed)) return self.success() # Register StartOnebyOne as a good test to run AllTestClasses.append(StartOnebyOne) ################################################################### class SimulStart(CTSTest): ################################################################### '''Start all the nodes ~ simultaneously''' def __init__(self, cm): CTSTest.__init__(self,cm) self.name="SimulStart" self.stopall = SimulStopLite(cm) self.startall = SimulStartLite(cm) def __call__(self, dummy): '''Perform the 'SimulStart' test. ''' self.incr("calls") # We ignore the "node" parameter... # Shut down all the nodes... ret = self.stopall(None) if not ret: return self.failure("Setup failed") self.CM.clear_all_caches() if not self.startall(None): return self.failure("Startall failed") return self.success() # Register SimulStart as a good test to run AllTestClasses.append(SimulStart) ################################################################### class SimulStop(CTSTest): ################################################################### '''Stop all the nodes ~ simultaneously''' def __init__(self, cm): CTSTest.__init__(self,cm) self.name="SimulStop" self.startall = SimulStartLite(cm) self.stopall = SimulStopLite(cm) def __call__(self, dummy): '''Perform the 'SimulStop' test. ''' self.incr("calls") # We ignore the "node" parameter... # Start up all the nodes... ret = self.startall(None) if not ret: return self.failure("Setup failed") if not self.stopall(None): return self.failure("Stopall failed") return self.success() # Register SimulStop as a good test to run AllTestClasses.append(SimulStop) ################################################################### class StopOnebyOne(CTSTest): ################################################################### '''Stop all the nodes in order''' def __init__(self, cm): CTSTest.__init__(self,cm) self.name="StopOnebyOne" self.startall = SimulStartLite(cm) self.stop = StopTest(cm) def __call__(self, dummy): '''Perform the 'StopOnebyOne' test. ''' self.incr("calls") # We ignore the "node" parameter... # Start up all the nodes... ret = self.startall(None) if not ret: return self.failure("Setup failed") failed=[] self.set_timer() for node in self.CM.Env["nodes"]: if not self.stop(node): failed.append(node) if len(failed) > 0: return self.failure("Some node failed to stop: " + repr(failed)) self.CM.clear_all_caches() return self.success() # Register StopOnebyOne as a good test to run AllTestClasses.append(StopOnebyOne) ################################################################### class RestartOnebyOne(CTSTest): ################################################################### '''Restart all the nodes in order''' def __init__(self, cm): CTSTest.__init__(self,cm) self.name="RestartOnebyOne" self.startall = SimulStartLite(cm) def __call__(self, dummy): '''Perform the 'RestartOnebyOne' test. ''' self.incr("calls") # We ignore the "node" parameter... # Start up all the nodes... ret = self.startall(None) if not ret: return self.failure("Setup failed") did_fail=[] self.set_timer() self.restart = RestartTest(self.CM) for node in self.CM.Env["nodes"]: if not self.restart(node): did_fail.append(node) if did_fail: return self.failure("Could not restart %d nodes: %s" %(len(did_fail), repr(did_fail))) return self.success() # Register StopOnebyOne as a good test to run AllTestClasses.append(RestartOnebyOne) ################################################################### class PartialStart(CTSTest): ################################################################### '''Start a node - but tell it to stop before it finishes starting up''' def __init__(self, cm): CTSTest.__init__(self,cm) self.name="PartialStart" self.startall = SimulStartLite(cm) self.stopall = SimulStopLite(cm) #self.is_unsafe = 1 def __call__(self, node): '''Perform the 'PartialStart' test. ''' self.incr("calls") ret = self.stopall(None) if not ret: return self.failure("Setup failed") # FIXME! This should use the CM class to get the pattern # then it would be applicable in general watchpats = [] watchpats.append("Starting crmd") watch = self.create_watch(watchpats, self.CM["DeadTime"]+10) watch.setwatch() self.CM.StartaCMnoBlock(node) ret = watch.lookforall() if not ret: self.CM.log("Patterns not found: " + repr(watch.unmatched)) return self.failure("Setup of %s failed" % node) ret = self.stopall(None) if not ret: return self.failure("%s did not stop in time" % node) return self.success() # Register StopOnebyOne as a good test to run AllTestClasses.append(PartialStart) ####################################################################### class StandbyTest(CTSTest): ####################################################################### def __init__(self, cm): CTSTest.__init__(self,cm) self.name="Standby" self.benchmark = 1 self.start = StartTest(cm) self.startall = SimulStartLite(cm) # make sure the node is active # set the node to standby mode # check resources, none resource should be running on the node # set the node to active mode # check resouces, resources should have been migrated back (SHOULD THEY?) def __call__(self, node): self.incr("calls") ret=self.startall(None) if not ret: return self.failure("Start all nodes failed") self.CM.debug("Make sure node %s is active" % node) if self.CM.StandbyStatus(node) != "off": if not self.CM.SetStandbyMode(node, "off"): return self.failure("can't set node %s to active mode" % node) self.CM.cluster_stable() status = self.CM.StandbyStatus(node) if status != "off": return self.failure("standby status of %s is [%s] but we expect [off]" % (node, status)) self.CM.debug("Getting resources running on node %s" % node) rsc_on_node = self.CM.active_resources(node) self.CM.debug("Setting node %s to standby mode" % node) if not self.CM.SetStandbyMode(node, "on"): return self.failure("can't set node %s to standby mode" % node) self.set_timer("on") time.sleep(1) # Allow time for the update to be applied and cause something self.CM.cluster_stable() status = self.CM.StandbyStatus(node) if status != "on": return self.failure("standby status of %s is [%s] but we expect [on]" % (node, status)) self.log_timer("on") self.CM.debug("Checking resources") bad_run = self.CM.active_resources(node) if len(bad_run) > 0: rc = self.failure("%s set to standby, %s is still running on it" % (node, repr(bad_run))) self.CM.debug("Setting node %s to active mode" % node) self.CM.SetStandbyMode(node, "off") return rc self.CM.debug("Setting node %s to active mode" % node) if not self.CM.SetStandbyMode(node, "off"): return self.failure("can't set node %s to active mode" % node) self.set_timer("off") self.CM.cluster_stable() status = self.CM.StandbyStatus(node) if status != "off": return self.failure("standby status of %s is [%s] but we expect [off]" % (node, status)) self.log_timer("off") return self.success() AllTestClasses.append(StandbyTest) ####################################################################### class ValgrindTest(CTSTest): ####################################################################### '''Check for memory leaks''' def __init__(self, cm): CTSTest.__init__(self,cm) self.name="Valgrind" self.stopall = SimulStopLite(cm) self.startall = SimulStartLite(cm) self.is_valgrind = 1 self.is_loop = 1 def setup(self, node): self.incr("calls") ret=self.stopall(None) if not ret: return self.failure("Stop all nodes failed") # Enable valgrind self.logPat = "/tmp/%s-*.valgrind" % self.name self.CM.Env["valgrind-prefix"] = self.name self.CM.rsh(node, "rm -f %s" % self.logPat, None) ret=self.startall(None) if not ret: return self.failure("Start all nodes failed") for node in self.CM.Env["nodes"]: (rc, output) = self.CM.rsh(node, "ps u --ppid `pidofproc aisexec`", None) for line in output: self.CM.debug(line) return self.success() def teardown(self, node): # Disable valgrind self.CM.Env["valgrind-prefix"] = None # Return all nodes to normal ret=self.stopall(None) if not ret: return self.failure("Stop all nodes failed") return self.success() def find_leaks(self): # Check for leaks leaked = [] self.stop = StopTest(self.CM) for node in self.CM.Env["nodes"]: (rc, ps_out) = self.CM.rsh(node, "ps u --ppid `pidofproc aisexec`", None) rc = self.stop(node) if not rc: self.failure("Couldn't shut down %s" % node) rc = self.CM.rsh(node, "grep -e indirectly.*lost:.*[1-9] -e definitely.*lost:.*[1-9] -e ERROR.*SUMMARY:.*[1-9].*errors %s" % self.logPat, 0) if rc != 1: leaked.append(node) self.failure("Valgrind errors detected on %s" % node) for line in ps_out: self.CM.log(line) (rc, output) = self.CM.rsh(node, "grep -e lost: -e SUMMARY: %s" % self.logPat, None) for line in output: self.CM.log(line) (rc, output) = self.CM.rsh(node, "cat %s" % self.logPat, None) for line in output: self.CM.debug(line) self.CM.rsh(node, "rm -f %s" % self.logPat, None) return leaked def __call__(self, node): leaked = self.find_leaks() if len(leaked) > 0: return self.failure("Nodes %s leaked" % repr(leaked)) return self.success() def errorstoignore(self): '''Return list of errors which should be ignored''' return [ """cib:.*readCibXmlFile:""", """HA_VALGRIND_ENABLED""" ] ####################################################################### class StandbyLoopTest(ValgrindTest): ####################################################################### '''Check for memory leaks by putting a node in and out of standby for an hour''' def __init__(self, cm): ValgrindTest.__init__(self,cm) self.name="StandbyLoop" def __call__(self, node): lpc = 0 delay = 2 failed = 0 done=time.time() + self.CM.Env["loop-minutes"]*60 while time.time() <= done and not failed: lpc = lpc + 1 time.sleep(delay) if not self.CM.SetStandbyMode(node, "on"): self.failure("can't set node %s to standby mode" % node) failed = lpc time.sleep(delay) if not self.CM.SetStandbyMode(node, "off"): self.failure("can't set node %s to active mode" % node) failed = lpc leaked = self.find_leaks() if failed: return self.failure("Iteration %d failed" % failed) elif len(leaked) > 0: return self.failure("Nodes %s leaked" % repr(leaked)) return self.success() AllTestClasses.append(StandbyLoopTest) ############################################################################## class BandwidthTest(CTSTest): ############################################################################## # Tests should not be cluster-manager-specific # If you need to find out cluster manager configuration to do this, then # it should be added to the generic cluster manager API. '''Test the bandwidth which heartbeat uses''' def __init__(self, cm): CTSTest.__init__(self, cm) self.name = "Bandwidth" self.start = StartTest(cm) self.__setitem__("min",0) self.__setitem__("max",0) self.__setitem__("totalbandwidth",0) self.tempfile = tempfile.mktemp(".cts") self.startall = SimulStartLite(cm) def __call__(self, node): '''Perform the Bandwidth test''' self.incr("calls") if self.CM.upcount()<1: return self.skipped() Path = self.CM.InternalCommConfig() if "ip" not in Path["mediatype"]: return self.skipped() port = Path["port"][0] port = int(port) ret = self.startall(None) if not ret: return self.failure("Test setup failed") time.sleep(5) # We get extra messages right after startup. fstmpfile = "/var/run/band_estimate" dumpcmd = "tcpdump -p -n -c 102 -i any udp port %d > %s 2>&1" \ % (port, fstmpfile) rc = self.CM.rsh(node, dumpcmd) if rc == 0: farfile = "root@%s:%s" % (node, fstmpfile) self.CM.rsh.cp(farfile, self.tempfile) Bandwidth = self.countbandwidth(self.tempfile) if not Bandwidth: self.CM.log("Could not compute bandwidth.") return self.success() intband = int(Bandwidth + 0.5) self.CM.log("...bandwidth: %d bits/sec" % intband) self.Stats["totalbandwidth"] = self.Stats["totalbandwidth"] + Bandwidth if self.Stats["min"] == 0: self.Stats["min"] = Bandwidth if Bandwidth > self.Stats["max"]: self.Stats["max"] = Bandwidth if Bandwidth < self.Stats["min"]: self.Stats["min"] = Bandwidth self.CM.rsh(node, "rm -f %s" % fstmpfile) os.unlink(self.tempfile) return self.success() else: return self.failure("no response from tcpdump command [%d]!" % rc) def countbandwidth(self, file): fp = open(file, "r") fp.seek(0) count = 0 sum = 0 while 1: line = fp.readline() if not line: return None if re.search("udp",line) or re.search("UDP,", line): count=count+1 linesplit = string.split(line," ") for j in range(len(linesplit)-1): if linesplit[j]=="udp": break if linesplit[j]=="length:": break try: sum = sum + int(linesplit[j+1]) except ValueError: self.CM.log("Invalid tcpdump line: %s" % line) return None T1 = linesplit[0] timesplit = string.split(T1,":") time2split = string.split(timesplit[2],".") time1 = (long(timesplit[0])*60+long(timesplit[1]))*60+long(time2split[0])+long(time2split[1])*0.000001 break while count < 100: line = fp.readline() if not line: return None if re.search("udp",line) or re.search("UDP,", line): count = count+1 linessplit = string.split(line," ") for j in range(len(linessplit)-1): if linessplit[j] =="udp": break if linesplit[j]=="length:": break try: sum=int(linessplit[j+1])+sum except ValueError: self.CM.log("Invalid tcpdump line: %s" % line) return None T2 = linessplit[0] timesplit = string.split(T2,":") time2split = string.split(timesplit[2],".") time2 = (long(timesplit[0])*60+long(timesplit[1]))*60+long(time2split[0])+long(time2split[1])*0.000001 time = time2-time1 if (time <= 0): return 0 return (sum*8)/time def is_applicable(self): '''BandwidthTest never applicable''' return 0 AllTestClasses.append(BandwidthTest) ################################################################### class ResourceRecover(CTSTest): ################################################################### def __init__(self, cm): CTSTest.__init__(self,cm) self.name="ResourceRecover" self.start = StartTest(cm) self.startall = SimulStartLite(cm) self.max=30 self.rid=None #self.is_unsafe = 1 self.benchmark = 1 # these are the values used for the new LRM API call self.action = "asyncmon" self.interval = 0 def __call__(self, node): '''Perform the 'ResourceRecover' test. ''' self.incr("calls") ret = self.startall(None) if not ret: return self.failure("Setup failed") resourcelist = self.CM.active_resources(node) # if there are no resourcelist, return directly if len(resourcelist)==0: self.CM.log("No active resources on %s" % node) return self.skipped() self.rid = self.CM.Env.RandomGen.choice(resourcelist) rsc = None (rc, lines) = self.CM.rsh(node, "crm_resource -c", None) for line in lines: if re.search("^Resource", line): tmp = AuditResource(self.CM, line) if tmp.id == self.rid: rsc = tmp # Handle anonymous clones that get renamed self.rid = rsc.clone_id break if not rsc: return self.failure("Could not find %s in the resource list" % self.rid) self.CM.debug("Shooting %s aka. %s" % (rsc.clone_id, rsc.id)) pats = [] pats.append("Updating failcount for %s on .* after .* %s" % (self.rid, self.action)) if rsc.managed(): pats.append("crmd:.* Performing .* op=%s_stop_0" % self.rid) if rsc.unique(): pats.append("crmd:.* Performing .* op=%s_start_0" % self.rid) pats.append("crmd:.* LRM operation %s_start_0.*confirmed.*ok" % self.rid) else: # Anonymous clones may get restarted with a different clone number pats.append("crmd:.* Performing .* op=.*_start_0") pats.append("crmd:.* LRM operation .*_start_0.*confirmed.*ok") watch = self.create_watch(pats, 60) watch.setwatch() self.CM.rsh(node, "crm_resource -F -r %s -H %s &>/dev/null" % (self.rid, node)) self.set_timer("recover") watch.lookforall() self.log_timer("recover") self.CM.cluster_stable() recovered=self.CM.ResourceLocation(self.rid) if watch.unmatched: return self.failure("Patterns not found: %s" % repr(watch.unmatched)) elif rsc.unique() and len(recovered) > 1: return self.failure("%s is now active on more than one node: %s"%(self.rid, repr(recovered))) elif len(recovered) > 0: self.CM.debug("%s is running on: %s" %(self.rid, repr(recovered))) elif rsc.managed(): return self.failure("%s was not recovered and is inactive" % self.rid) return self.success() def errorstoignore(self): '''Return list of errors which should be ignored''' return [ """Updating failcount for %s""" % self.rid, """Unknown operation: fail""", """ERROR: sending stonithRA op to stonithd failed.""", """ERROR: process_lrm_event: LRM operation %s_%s_%d""" % (self.rid, self.action, self.interval), """ERROR: process_graph_event: Action %s_%s_%d .* initiated outside of a transition""" % (self.rid, self.action, self.interval), ] AllTestClasses.append(ResourceRecover) ################################################################### class ComponentFail(CTSTest): ################################################################### def __init__(self, cm): CTSTest.__init__(self,cm) self.name="ComponentFail" self.startall = SimulStartLite(cm) self.complist = cm.Components() self.patterns = [] self.okerrpatterns = [] self.is_unsafe = 1 def __call__(self, node): '''Perform the 'ComponentFail' test. ''' self.incr("calls") self.patterns = [] self.okerrpatterns = [] # start all nodes ret = self.startall(None) if not ret: return self.failure("Setup failed") if not self.CM.cluster_stable(self.CM["StableTime"]): return self.failure("Setup failed - unstable") node_is_dc = self.CM.is_node_dc(node, None) # select a component to kill chosen = self.CM.Env.RandomGen.choice(self.complist) while chosen.dc_only == 1 and node_is_dc == 0: chosen = self.CM.Env.RandomGen.choice(self.complist) self.CM.debug("...component %s (dc=%d,boot=%d)" % (chosen.name, node_is_dc,chosen.triggersreboot)) self.incr(chosen.name) if chosen.name != "aisexec": if self.CM["Name"] != "crm-lha" or chosen.name != "pengine": self.patterns.append(self.CM["Pat:ChildKilled"] %(node, chosen.name)) self.patterns.append(self.CM["Pat:ChildRespawn"] %(node, chosen.name)) self.patterns.extend(chosen.pats) if node_is_dc: self.patterns.extend(chosen.dc_pats) # In an ideal world, this next stuff should be in the "chosen" object as a member function if self.CM["Name"] == "crm-lha" and chosen.triggersreboot: # Make sure the node goes down and then comes back up if it should reboot... for other in self.CM.Env["nodes"]: if other != node: self.patterns.append(self.CM["Pat:They_stopped"] %(other, self.CM.key_for_node(node))) self.patterns.append(self.CM["Pat:Slave_started"] % node) self.patterns.append(self.CM["Pat:Local_started"] % node) if chosen.dc_only: # Sometimes these will be in the log, and sometimes they won't... self.okerrpatterns.append("%s crmd:.*Process %s:.* exited" %(node, chosen.name)) self.okerrpatterns.append("%s crmd:.*I_ERROR.*crmdManagedChildDied" %node) self.okerrpatterns.append("%s crmd:.*The %s subsystem terminated unexpectedly" %(node, chosen.name)) self.okerrpatterns.append("ERROR: Client .* exited with return code") else: # Sometimes this won't be in the log... self.okerrpatterns.append(self.CM["Pat:ChildKilled"] %(node, chosen.name)) self.okerrpatterns.append(self.CM["Pat:ChildRespawn"] %(node, chosen.name)) self.okerrpatterns.append(self.CM["Pat:ChildExit"]) # supply a copy so self.patterns doesnt end up empty tmpPats = [] tmpPats.extend(self.patterns) self.patterns.extend(chosen.badnews_ignore) # Look for STONITH ops, depending on Env["at-boot"] we might need to change the nodes status stonithPats = [] stonithPats.append("stonith-ng:.*Operation .* for host '%s' with device .* returned: 0" % node) stonith = self.create_watch(stonithPats, 0) stonith.setwatch() # set the watch for stable watch = self.create_watch( tmpPats, self.CM["DeadTime"] + self.CM["StableTime"] + self.CM["StartTime"]) watch.setwatch() # kill the component chosen.kill(node) # check to see Heartbeat noticed matched = watch.lookforall(allow_multiple_matches=1) if watch.unmatched: self.CM.log("Patterns not found: " + repr(watch.unmatched)) if self.CM.Env["at-boot"] == 0: self.CM.debug("Checking if %s was shot" % node) shot = stonith.look(60) if shot: self.CM.debug("Found: "+ repr(shot)) self.CM.ShouldBeStatus[node]="down" self.CM.debug("Waiting for the cluster to recover") self.CM.cluster_stable() self.CM.debug("Waiting for any STONITHd node to come back up") self.CM.ns.WaitForAllNodesToComeUp(self.CM.Env["nodes"], 600) self.CM.debug("Waiting for the cluster to re-stabilize with all nodes") is_stable = self.CM.cluster_stable(self.CM["StartTime"]) if not matched: return self.failure("Didn't find all expected patterns") elif not is_stable: return self.failure("Cluster did not become stable") return self.success() def errorstoignore(self): '''Return list of errors which should be ignored''' # Note that okerrpatterns refers to the last time we ran this test # The good news is that this works fine for us... self.okerrpatterns.extend(self.patterns) return self.okerrpatterns AllTestClasses.append(ComponentFail) #################################################################### class SplitBrainTest(CTSTest): #################################################################### '''It is used to test split-brain. when the path between the two nodes break check the two nodes both take over the resource''' def __init__(self,cm): CTSTest.__init__(self,cm) self.name = "SplitBrain" self.start = StartTest(cm) self.startall = SimulStartLite(cm) self.is_experimental = 1 def isolate_partition(self, partition): other_nodes = [] other_nodes.extend(self.CM.Env["nodes"]) for node in partition: try: other_nodes.remove(node) except ValueError: self.CM.log("Node "+node+" not in " + repr(self.CM.Env["nodes"]) + " from " +repr(partition)) if len(other_nodes) == 0: return 1 self.CM.debug("Creating partition: " + repr(partition)) self.CM.debug("Everyone else: " + repr(other_nodes)) for node in partition: if not self.CM.isolate_node(node, other_nodes): self.CM.log("Could not isolate %s" % node) return 0 return 1 def heal_partition(self, partition): other_nodes = [] other_nodes.extend(self.CM.Env["nodes"]) for node in partition: try: other_nodes.remove(node) except ValueError: self.CM.log("Node "+node+" not in " + repr(self.CM.Env["nodes"])) if len(other_nodes) == 0: return 1 self.CM.debug("Healing partition: " + repr(partition)) self.CM.debug("Everyone else: " + repr(other_nodes)) for node in partition: self.CM.unisolate_node(node, other_nodes) def __call__(self, node): '''Perform split-brain test''' self.incr("calls") self.passed = 1 partitions = {} ret = self.startall(None) if not ret: return self.failure("Setup failed") while 1: # Retry until we get multiple partitions partitions = {} p_max = len(self.CM.Env["nodes"]) for node in self.CM.Env["nodes"]: p = self.CM.Env.RandomGen.randint(1, p_max) if not partitions.has_key(p): partitions[p]= [] partitions[p].append(node) p_max = len(partitions.keys()) if p_max > 1: break # else, try again self.CM.debug("Created %d partitions" % p_max) for key in partitions.keys(): self.CM.debug("Partition["+str(key)+"]:\t"+repr(partitions[key])) # Disabling STONITH to reduce test complexity for now self.CM.rsh(node, "crm_attribute -n stonith-enabled -v false") for key in partitions.keys(): self.isolate_partition(partitions[key]) count = 30 while count > 0: if len(self.CM.find_partitions()) != p_max: time.sleep(10) else: break else: self.failure("Expected partitions were not created") # Target number of partitions formed - wait for stability if not self.CM.cluster_stable(): self.failure("Partitioned cluster not stable") # Now audit the cluster state self.CM.partitions_expected = p_max if not self.audit(): self.failure("Audits failed") self.CM.partitions_expected = 1 # And heal them again for key in partitions.keys(): self.heal_partition(partitions[key]) # Wait for a single partition to form count = 30 while count > 0: if len(self.CM.find_partitions()) != 1: time.sleep(10) count -= 1 else: break else: self.failure("Cluster did not reform") # Wait for it to have the right number of members count = 30 while count > 0: members = [] partitions = self.CM.find_partitions() if len(partitions) > 0: members = partitions[0].split() if len(members) != len(self.CM.Env["nodes"]): time.sleep(10) count -= 1 else: break else: self.failure("Cluster did not completely reform") # Wait up to 20 minutes - the delay is more preferable than # trying to continue with in a messed up state if not self.CM.cluster_stable(1200): self.failure("Reformed cluster not stable") answer = raw_input('Continue? [nY]') if answer and answer == "n": raise ValueError("Reformed cluster not stable") # Turn fencing back on if self.CM.Env["DoFencing"]: self.CM.rsh(node, "crm_attribute -D -n stonith-enabled") self.CM.cluster_stable() if self.passed: return self.success() return self.failure("See previous errors") def errorstoignore(self): '''Return list of errors which are 'normal' and should be ignored''' return [ "Another DC detected:", "ERROR: attrd_cib_callback: .*Application of an update diff failed", "crmd_ha_msg_callback:.*not in our membership list", "CRIT:.*node.*returning after partition", ] def is_applicable(self): if not self.is_applicable_common(): return 0 return len(self.CM.Env["nodes"]) > 2 AllTestClasses.append(SplitBrainTest) #################################################################### class Reattach(CTSTest): #################################################################### def __init__(self, cm): CTSTest.__init__(self,cm) self.name="Reattach" self.startall = SimulStartLite(cm) self.restart1 = RestartTest(cm) self.stopall = SimulStopLite(cm) self.is_unsafe = 0 # Handled by canrunnow() def setup(self, node): attempt=0 if not self.startall(None): return None # Make sure we are really _really_ stable and that all # resources, including those that depend on transient node # attributes, are started while not self.CM.cluster_stable(double_check=True): if attempt < 5: attempt += 1 self.CM.debug("Not stable yet, re-testing") else: self.CM.log("Cluster is not stable") return None return 1 def teardown(self, node): # Make sure 'node' is up start = StartTest(self.CM) start(node) is_managed = self.CM.rsh(node, "crm_attribute -GQ -t crm_config -n is-managed-default -d true", 1) is_managed = is_managed[:-1] # Strip off the newline if is_managed != "true": self.CM.log("Attempting to re-enable resource management on %s (%s)" % (node, is_managed)) managed = self.create_watch(["is-managed-default"], 60) managed.setwatch() self.CM.rsh(node, "crm_attribute -D -n is-managed-default") if not managed.lookforall(): self.CM.log("Patterns not found: " + repr(managed.unmatched)) self.CM.log("Could not re-enable resource management") return 0 return 1 def canrunnow(self, node): '''Return TRUE if we can meaningfully run right now''' if self.find_ocfs2_resources(node): self.CM.log("Detach/Reattach scenarios are not possible with OCFS2 services present") return 0 return 1 def __call__(self, node): self.incr("calls") pats = [] managed = self.create_watch(["is-managed-default"], 60) managed.setwatch() self.CM.debug("Disable resource management") self.CM.rsh(node, "crm_attribute -n is-managed-default -v false") if not managed.lookforall(): self.CM.log("Patterns not found: " + repr(managed.unmatched)) return self.failure("Resource management not disabled") pats = [] pats.append("crmd:.*Performing.*_stop_0") pats.append("crmd:.*Performing.*_start_0") pats.append("crmd:.*Performing.*_promote_0") pats.append("crmd:.*Performing.*_demote_0") pats.append("crmd:.*Performing.*_migrate_.*_0") watch = self.create_watch(pats, 60, "ShutdownActivity") watch.setwatch() self.CM.debug("Shutting down the cluster") ret = self.stopall(None) if not ret: self.CM.debug("Re-enable resource management") self.CM.rsh(node, "crm_attribute -D -n is-managed-default") return self.failure("Couldn't shut down the cluster") self.CM.debug("Bringing the cluster back up") ret = self.startall(None) time.sleep(5) # allow ping to update the CIB if not ret: self.CM.debug("Re-enable resource management") self.CM.rsh(node, "crm_attribute -D -n is-managed-default") return self.failure("Couldn't restart the cluster") if self.local_badnews("ResourceActivity:", watch): self.CM.debug("Re-enable resource management") self.CM.rsh(node, "crm_attribute -D -n is-managed-default") return self.failure("Resources stopped or started during cluster restart") watch = self.create_watch(pats, 60, "StartupActivity") watch.setwatch() managed = self.create_watch(["is-managed-default"], 60) managed.setwatch() self.CM.debug("Re-enable resource management") self.CM.rsh(node, "crm_attribute -D -n is-managed-default") if not managed.lookforall(): self.CM.log("Patterns not found: " + repr(managed.unmatched)) return self.failure("Resource management not enabled") self.CM.cluster_stable() # Ignore actions for STONITH resources ignore = [] (rc, lines) = self.CM.rsh(node, "crm_resource -c", None) for line in lines: if re.search("^Resource", line): r = AuditResource(self.CM, line) if r.rclass == "stonith": self.CM.debug("Ignoring: crmd:.*Performing.*op=%s_.*_0" % r.id) ignore.append("crmd:.*Performing.*op=%s_.*_0" % r.id) if self.local_badnews("ResourceActivity:", watch, ignore): return self.failure("Resources stopped or started after resource management was re-enabled") return ret def errorstoignore(self): '''Return list of errors which should be ignored''' return [ "You may ignore this error if it is unmanaged.", "pingd: .*ERROR: send_ipc_message:", "pingd: .*ERROR: send_update:", "lrmd: .*ERROR: notify_client:", ] def is_applicable(self): if self.CM["Name"] == "crm-lha": return None return 1 AllTestClasses.append(Reattach) #################################################################### class SpecialTest1(CTSTest): #################################################################### '''Set up a custom test to cause quorum failure issues for Andrew''' def __init__(self, cm): CTSTest.__init__(self,cm) self.name="SpecialTest1" self.startall = SimulStartLite(cm) self.restart1 = RestartTest(cm) self.stopall = SimulStopLite(cm) def __call__(self, node): '''Perform the 'SpecialTest1' test for Andrew. ''' self.incr("calls") # Shut down all the nodes... ret = self.stopall(None) if not ret: return self.failure("Could not stop all nodes") # Start the selected node ret = self.restart1(node) if not ret: return self.failure("Could not start "+node) # Start all remaining nodes ret = self.startall(None) if not ret: return self.failure("Could not start the remaining nodes") return self.success() AllTestClasses.append(SpecialTest1) #################################################################### class HAETest(CTSTest): #################################################################### '''Set up a custom test to cause quorum failure issues for Andrew''' def __init__(self, cm): CTSTest.__init__(self,cm) self.name="HAETest" self.stopall = SimulStopLite(cm) self.startall = SimulStartLite(cm) self.is_loop = 1 def setup(self, node): # Start all remaining nodes ret = self.startall(None) if not ret: return self.failure("Couldn't start all nodes") return self.success() def teardown(self, node): # Stop everything ret = self.stopall(None) if not ret: return self.failure("Couldn't stop all nodes") return self.success() def wait_on_state(self, node, resource, expected_clones, attempts=240): while attempts > 0: active=0 (rc, lines) = self.CM.rsh(node, "crm_resource -r %s -W -Q" % resource, stdout=None) # Hack until crm_resource does the right thing if rc == 0 and lines: active = len(lines) if len(lines) == expected_clones: return 1 elif rc == 1: self.CM.debug("Resource %s is still inactive" % resource) elif rc == 234: self.CM.log("Unknown resource %s" % resource) return 0 elif rc == 246: self.CM.log("Cluster is inactive") return 0 elif rc != 0: self.CM.log("Call to crm_resource failed, rc=%d" % rc) return 0 else: self.CM.debug("Resource %s is active on %d times instead of %d" % (resource, active, expected_clones)) attempts -= 1 time.sleep(1) return 0 def find_dlm(self, node): self.r_dlm = None (rc, lines) = self.CM.rsh(node, "crm_resource -c", None) for line in lines: if re.search("^Resource", line): r = AuditResource(self.CM, line) if r.rtype == "controld" and r.parent != "NA": self.CM.debug("Found dlm: %s" % self.r_dlm) self.r_dlm = r.parent return 1 return 0 def find_hae_resources(self, node): self.r_dlm = None self.r_o2cb = None self.r_ocfs2 = [] if self.find_dlm(node): self.find_ocfs2_resources(node) def is_applicable(self): if not self.is_applicable_common(): return 0 if self.CM.Env["Schema"] == "hae": return 1 return None #################################################################### class HAERoleTest(HAETest): #################################################################### def __init__(self, cm): '''Lars' mount/unmount test for the HA extension. ''' HAETest.__init__(self,cm) self.name="HAERoleTest" def change_state(self, node, resource, target): rc = self.CM.rsh(node, "crm_resource -r %s -p target-role -v %s --meta" % (resource, target)) return rc def __call__(self, node): self.incr("calls") lpc = 0 failed = 0 delay = 2 done=time.time() + self.CM.Env["loop-minutes"]*60 self.find_hae_resources(node) clone_max = len(self.CM.Env["nodes"]) while time.time() <= done and not failed: lpc = lpc + 1 self.change_state(node, self.r_dlm, "Stopped") if not self.wait_on_state(node, self.r_dlm, 0): self.failure("%s did not go down correctly" % self.r_dlm) failed = lpc self.change_state(node, self.r_dlm, "Started") if not self.wait_on_state(node, self.r_dlm, clone_max): self.failure("%s did not come up correctly" % self.r_dlm) failed = lpc if not self.wait_on_state(node, self.r_o2cb, clone_max): self.failure("%s did not come up correctly" % self.r_o2cb) failed = lpc for fs in self.r_ocfs2: if not self.wait_on_state(node, fs, clone_max): self.failure("%s did not come up correctly" % fs) failed = lpc if failed: return self.failure("iteration %d failed" % failed) return self.success() AllTestClasses.append(HAERoleTest) #################################################################### class HAEStandbyTest(HAETest): #################################################################### '''Set up a custom test to cause quorum failure issues for Andrew''' def __init__(self, cm): HAETest.__init__(self,cm) self.name="HAEStandbyTest" def change_state(self, node, resource, target): rc = self.CM.rsh(node, "crm_standby -l reboot -v %s" % (target)) return rc def __call__(self, node): self.incr("calls") lpc = 0 failed = 0 done=time.time() + self.CM.Env["loop-minutes"]*60 self.find_hae_resources(node) clone_max = len(self.CM.Env["nodes"]) while time.time() <= done and not failed: lpc = lpc + 1 self.change_state(node, self.r_dlm, "true") if not self.wait_on_state(node, self.r_dlm, clone_max-1): self.failure("%s did not go down correctly" % self.r_dlm) failed = lpc self.change_state(node, self.r_dlm, "false") if not self.wait_on_state(node, self.r_dlm, clone_max): self.failure("%s did not come up correctly" % self.r_dlm) failed = lpc if not self.wait_on_state(node, self.r_o2cb, clone_max): self.failure("%s did not come up correctly" % self.r_o2cb) failed = lpc for fs in self.r_ocfs2: if not self.wait_on_state(node, fs, clone_max): self.failure("%s did not come up correctly" % fs) failed = lpc if failed: return self.failure("iteration %d failed" % failed) return self.success() AllTestClasses.append(HAEStandbyTest) ################################################################### class NearQuorumPointTest(CTSTest): ################################################################### ''' This test brings larger clusters near the quorum point (50%). In addition, it will test doing starts and stops at the same time. Here is how I think it should work: - loop over the nodes and decide randomly which will be up and which will be down Use a 50% probability for each of up/down. - figure out what to do to get into that state from the current state - in parallel, bring up those going up and bring those going down. ''' def __init__(self, cm): CTSTest.__init__(self,cm) self.name="NearQuorumPoint" def __call__(self, dummy): '''Perform the 'NearQuorumPoint' test. ''' self.incr("calls") startset = [] stopset = [] #decide what to do with each node for node in self.CM.Env["nodes"]: action = self.CM.Env.RandomGen.choice(["start","stop"]) #action = self.CM.Env.RandomGen.choice(["start","stop","no change"]) if action == "start" : startset.append(node) elif action == "stop" : stopset.append(node) self.CM.debug("start nodes:" + repr(startset)) self.CM.debug("stop nodes:" + repr(stopset)) #add search patterns watchpats = [ ] for node in stopset: if self.CM.ShouldBeStatus[node] == "up": watchpats.append(self.CM["Pat:We_stopped"] % node) for node in startset: if self.CM.ShouldBeStatus[node] == "down": #watchpats.append(self.CM["Pat:Slave_started"] % node) watchpats.append(self.CM["Pat:Local_started"] % node) else: for stopping in stopset: if self.CM.ShouldBeStatus[stopping] == "up": watchpats.append(self.CM["Pat:They_stopped"] % (node, self.CM.key_for_node(stopping))) if len(watchpats) == 0: return self.skipped() if len(startset) != 0: watchpats.append(self.CM["Pat:DC_IDLE"]) watch = self.create_watch(watchpats, self.CM["DeadTime"]+10) watch.setwatch() #begin actions for node in stopset: if self.CM.ShouldBeStatus[node] == "up": self.CM.StopaCMnoBlock(node) for node in startset: if self.CM.ShouldBeStatus[node] == "down": self.CM.StartaCMnoBlock(node) #get the result if watch.lookforall(): self.CM.cluster_stable() return self.success() self.CM.log("Warn: Patterns not found: " + repr(watch.unmatched)) #get the "bad" nodes upnodes = [] for node in stopset: if self.CM.StataCM(node) == 1: upnodes.append(node) downnodes = [] for node in startset: if self.CM.StataCM(node) == 0: downnodes.append(node) if upnodes == [] and downnodes == []: self.CM.cluster_stable() # Make sure they're completely down with no residule for node in stopset: self.CM.rsh(node, self.CM["StopCmd"]) return self.success() if len(upnodes) > 0: self.CM.log("Warn: Unstoppable nodes: " + repr(upnodes)) if len(downnodes) > 0: self.CM.log("Warn: Unstartable nodes: " + repr(downnodes)) return self.failure() AllTestClasses.append(NearQuorumPointTest) ################################################################### class RollingUpgradeTest(CTSTest): ################################################################### '''Perform a rolling upgrade of the cluster''' def __init__(self, cm): CTSTest.__init__(self,cm) self.name="RollingUpgrade" self.start = StartTest(cm) self.stop = StopTest(cm) self.stopall = SimulStopLite(cm) self.startall = SimulStartLite(cm) def setup(self, node): # Start all remaining nodes ret = self.stopall(None) if not ret: return self.failure("Couldn't stop all nodes") for node in self.CM.Env["nodes"]: if not self.downgrade(node, None): return self.failure("Couldn't downgrade %s" % node) ret = self.startall(None) if not ret: return self.failure("Couldn't start all nodes") return self.success() def teardown(self, node): # Stop everything ret = self.stopall(None) if not ret: return self.failure("Couldn't stop all nodes") for node in self.CM.Env["nodes"]: if not self.upgrade(node, None): return self.failure("Couldn't upgrade %s" % node) return self.success() def install(self, node, version, start=1, flags="--force"): target_dir = "/tmp/rpm-%s" % version src_dir = "%s/%s" % (self.CM.Env["rpm-dir"], version) self.CM.log("Installing %s on %s with %s" % (version, node, flags)) if not self.stop(node): return self.failure("stop failure: "+node) rc = self.CM.rsh(node, "mkdir -p %s" % target_dir) rc = self.CM.rsh(node, "rm -f %s/*.rpm" % target_dir) (rc, lines) = self.CM.rsh(node, "ls -1 %s/*.rpm" % src_dir, None) for line in lines: line = line[:-1] rc = self.CM.rsh.cp("%s" % (line), "%s:%s/" % (node, target_dir)) rc = self.CM.rsh(node, "rpm -Uvh %s %s/*.rpm" % (flags, target_dir)) if start and not self.start(node): return self.failure("start failure: "+node) return self.success() def upgrade(self, node, start=1): return self.install(node, self.CM.Env["current-version"], start) def downgrade(self, node, start=1): return self.install(node, self.CM.Env["previous-version"], start, "--force --nodeps") def __call__(self, node): '''Perform the 'Rolling Upgrade' test. ''' self.incr("calls") for node in self.CM.Env["nodes"]: if self.upgrade(node): return self.failure("Couldn't upgrade %s" % node) self.CM.cluster_stable() return self.success() def is_applicable(self): if not self.is_applicable_common(): return None if not self.CM.Env.has_key("rpm-dir"): return None if not self.CM.Env.has_key("current-version"): return None if not self.CM.Env.has_key("previous-version"): return None return 1 # Register RestartTest as a good test to run AllTestClasses.append(RollingUpgradeTest) ################################################################### class BSC_AddResource(CTSTest): ################################################################### '''Add a resource to the cluster''' def __init__(self, cm): CTSTest.__init__(self, cm) self.name="AddResource" self.resource_offset = 0 self.cib_cmd="""cibadmin -C -o %s -X '%s' """ def __call__(self, node): self.incr("calls") self.resource_offset = self.resource_offset + 1 r_id = "bsc-rsc-%s-%d" % (node, self.resource_offset) start_pat = "crmd.*%s_start_0.*confirmed.*ok" patterns = [] patterns.append(start_pat % r_id) watch = self.create_watch(patterns, self.CM["DeadTime"]) watch.setwatch() fields = string.split(self.CM.Env["IPBase"], '.') fields[3] = str(int(fields[3])+1) ip = string.join(fields, '.') self.CM.Env["IPBase"] = ip if not self.make_ip_resource(node, r_id, "ocf", "IPaddr", ip): return self.failure("Make resource %s failed" % r_id) failed = 0 watch_result = watch.lookforall() if watch.unmatched: for regex in watch.unmatched: self.CM.log ("Warn: Pattern not found: %s" % (regex)) failed = 1 if failed: return self.failure("Resource pattern(s) not found") if not self.CM.cluster_stable(self.CM["DeadTime"]): return self.failure("Unstable cluster") return self.success() def make_ip_resource(self, node, id, rclass, type, ip): self.CM.log("Creating %s::%s:%s (%s) on %s" % (rclass,type,id,ip,node)) rsc_xml=""" """ % (id, rclass, type, id, id, ip) node_constraint=""" """ % (id, id, id, id, node) rc = 0 (rc, lines) = self.CM.rsh(node, self.cib_cmd % ("constraints", node_constraint), None) if rc != 0: self.CM.log("Constraint creation failed: %d" % rc) return None (rc, lines) = self.CM.rsh(node, self.cib_cmd % ("resources", rsc_xml), None) if rc != 0: self.CM.log("Resource creation failed: %d" % rc) return None return 1 def is_applicable(self): if self.CM.Env["DoBSC"]: return 1 return None AllTestClasses.append(BSC_AddResource) class SimulStopLite(CTSTest): ################################################################### '''Stop any active nodes ~ simultaneously''' def __init__(self, cm): CTSTest.__init__(self,cm) self.name="SimulStopLite" def __call__(self, dummy): '''Perform the 'SimulStopLite' setup work. ''' self.incr("calls") self.CM.debug("Setup: " + self.name) # We ignore the "node" parameter... watchpats = [ ] for node in self.CM.Env["nodes"]: if self.CM.ShouldBeStatus[node] == "up": self.incr("WasStarted") watchpats.append(self.CM["Pat:We_stopped"] % node) #if self.CM.Env["use_logd"]: # watchpats.append(self.CM["Pat:Logd_stopped"] % node) if len(watchpats) == 0: self.CM.clear_all_caches() return self.success() # Stop all the nodes - at about the same time... watch = self.create_watch(watchpats, self.CM["DeadTime"]+10) watch.setwatch() self.set_timer() for node in self.CM.Env["nodes"]: if self.CM.ShouldBeStatus[node] == "up": self.CM.StopaCMnoBlock(node) if watch.lookforall(): self.CM.clear_all_caches() # Make sure they're completely down with no residule for node in self.CM.Env["nodes"]: self.CM.rsh(node, self.CM["StopCmd"]) return self.success() did_fail=0 up_nodes = [] for node in self.CM.Env["nodes"]: if self.CM.StataCM(node) == 1: did_fail=1 up_nodes.append(node) if did_fail: return self.failure("Active nodes exist: " + repr(up_nodes)) self.CM.log("Warn: All nodes stopped but CTS didnt detect: " + repr(watch.unmatched)) self.CM.clear_all_caches() return self.failure("Missing log message: "+repr(watch.unmatched)) def is_applicable(self): '''SimulStopLite is a setup test and never applicable''' return 0 ################################################################### class SimulStartLite(CTSTest): ################################################################### '''Start any stopped nodes ~ simultaneously''' def __init__(self, cm): CTSTest.__init__(self,cm) self.name="SimulStartLite" def __call__(self, dummy): '''Perform the 'SimulStartList' setup work. ''' self.incr("calls") self.CM.debug("Setup: " + self.name) # We ignore the "node" parameter... watchpats = [ ] uppat = self.CM["Pat:Slave_started"] if self.CM.upcount() == 0: uppat = self.CM["Pat:Local_started"] for node in self.CM.Env["nodes"]: if self.CM.ShouldBeStatus[node] == "down": self.incr("WasStopped") watchpats.append(uppat % node) if len(watchpats) == 0: return self.success() watchpats.append(self.CM["Pat:DC_IDLE"]) # Start all the nodes - at about the same time... watch = self.create_watch(watchpats, self.CM["DeadTime"]+10) watch.setwatch() self.set_timer() for node in self.CM.Env["nodes"]: if self.CM.ShouldBeStatus[node] == "down": self.CM.StartaCMnoBlock(node) if watch.lookforall(): for attempt in (1, 2, 3, 4, 5): if self.CM.cluster_stable(): return self.success() return self.failure("Cluster did not stabilize") did_fail=0 unstable = [] for node in self.CM.Env["nodes"]: if self.CM.StataCM(node) == 0: did_fail=1 unstable.append(node) if did_fail: return self.failure("Unstarted nodes exist: " + repr(unstable)) unstable = [] for node in self.CM.Env["nodes"]: if not self.CM.node_stable(node): did_fail=1 unstable.append(node) if did_fail: return self.failure("Unstable cluster nodes exist: " + repr(unstable)) self.CM.log("ERROR: All nodes started but CTS didnt detect: " + repr(watch.unmatched)) return self.failure() def is_applicable(self): '''SimulStartLite is a setup test and never applicable''' return 0 def TestList(cm, audits): result = [] for testclass in AllTestClasses: bound_test = testclass(cm) if bound_test.is_applicable(): bound_test.Audits = audits result.append(bound_test) return result # vim:ts=4:sw=4:et: diff --git a/doc/Clusters_from_Scratch/en-US/Ap-Cman.xml b/doc/Clusters_from_Scratch/en-US/Ap-Cman.xml index e70080312e..af27515cab 100644 --- a/doc/Clusters_from_Scratch/en-US/Ap-Cman.xml +++ b/doc/Clusters_from_Scratch/en-US/Ap-Cman.xml @@ -1,131 +1,125 @@ %BOOK_ENTITIES; ]> Using CMAN for Cluster Membership and Quorum
Background CMAN v3 is a Corsync plugin that monitors the names and number of active cluster nodes in order to deliver membership and quorum information to clients (such as the Pacemaker daemons). In a traditional Corosync-Pacemaker cluster, a Pacemaker plugin is loaded to provide membership and quorum information. The motivation for wanting to use CMAN for this instead, is to ensure all elements of the cluster stack are making decisions based on the same membership and quorum data. A failure to do this can lead to what is called internal split-brain - a situation where different parts of the stack disagree about whether some nodes are alive or dead - which quickly leads to unnecssary down-time and/or data corruption. CMAN has been around longer than Pacemaker and is part of the Red Hat cluster stack, so it is available and supported by many distributions and other pieces of software (such as OCFS2 and GFS2). For this reason it makes sense to support it.
Adding CMAN Support Be sure to disable the Pacemaker plugin before continuing with this section. In most cases, this can be achieved by removing /etc/corosync/service.d/pcmk and stopping Corosync.
Adding CMAN Support - cluster.conf The preferred approach for enabling CMAN is to configure cluster.conf and use the /etc/init.d/cman script to start Corosync. Its far easier to maintain and start automatically starts the necessary pieces for using GFS2. You can find some documentation on Installing CMAN and Creating a Basic Cluster Configuration File at the Red Hat website. However please ignore the parts about Fencing, Failover Domains, or HA Services and anything to do with rgmanager and fenced. All these continue to be handled by Pacemaker in the normal manner. Sample cluster.conf for a two-node cluster - + - - - - - - + + - - - + ]]>
Adding CMAN Support - corosync.conf The alternative is to add the necessary cman configuration elements to corosync.conf. We recommend you place these directives in /etc/corosync/service.d/cman as they will differ between machines. If you choose this approach, you would continue to start and stop Corosync with it's init script as previously described in this document. Sample corosync.conf extensions for a two-node cluster [root@pcmk-1 ~]# cat <<-END >>/etc/corosync/service.d/cman cluster { name: beekhof clusternodes { clusternode { votes: 1 nodeid: 1 name: pcmk-1 } clusternode { votes: 1 nodeid: 2 name: pcmk-2 } } cman { expected_votes: 2 cluster_id: 123 nodename: `uname -n` two_node: 1 max_queued: 10 } } service { name: corosync_cman ver: 0 } quorum { provider: quorum_cman } END Verify that nodename was set appropriately on each host.
diff --git a/doc/Clusters_from_Scratch/en-US/Ch-Active-Active.xml b/doc/Clusters_from_Scratch/en-US/Ch-Active-Active.xml index 329cfd7178..8f709287dd 100644 --- a/doc/Clusters_from_Scratch/en-US/Ch-Active-Active.xml +++ b/doc/Clusters_from_Scratch/en-US/Ch-Active-Active.xml @@ -1,721 +1,721 @@ %BOOK_ENTITIES; ]> Conversion to Active/Active
Requirements The primary requirement for an Active/Active cluster is that the data required for your services are available, simultaneously, on both machines. Pacemaker makes no requirement on how this is achieved, you could use a SAN if you had one available, however since DRBD supports multiple Primaries, we can also use that. The only hitch is that we need to use a cluster-aware filesystem (and the one we used earlier with DRBD, ext4, is not one of those). Both OCFS2 and GFS2 are supported, however here we will use GFS2 which comes with &DISTRO; &DISTRO_VERSION; .
Install a Cluster Filesystem - GFS2 - The first thing to do is install gfs2-utils on each machine. + The first thing to do is install gfs2-utils and gfs2-cluster on each machine. -[root@pcmk-1 ~]# yum install -y gfs2-utils gfs-pcmk +[root@pcmk-1 ~]# yum install -y gfs2-utils gfs2-cluster gfs-pcmk Setting up Install Process Resolving Dependencies --> Running transaction check ---> Package gfs-pcmk.x86_64 0:3.0.5-2.fc12 set to be updated --> Processing Dependency: libSaCkpt.so.3(OPENAIS_CKPT_B.01.01)(64bit) for package: gfs-pcmk-3.0.5-2.fc12.x86_64 --> Processing Dependency: dlm-pcmk for package: gfs-pcmk-3.0.5-2.fc12.x86_64 --> Processing Dependency: libccs.so.3()(64bit) for package: gfs-pcmk-3.0.5-2.fc12.x86_64 --> Processing Dependency: libdlmcontrol.so.3()(64bit) for package: gfs-pcmk-3.0.5-2.fc12.x86_64 --> Processing Dependency: liblogthread.so.3()(64bit) for package: gfs-pcmk-3.0.5-2.fc12.x86_64 --> Processing Dependency: libSaCkpt.so.3()(64bit) for package: gfs-pcmk-3.0.5-2.fc12.x86_64 ---> Package gfs2-utils.x86_64 0:3.0.5-2.fc12 set to be updated --> Running transaction check ---> Package clusterlib.x86_64 0:3.0.5-2.fc12 set to be updated ---> Package dlm-pcmk.x86_64 0:3.0.5-2.fc12 set to be updated ---> Package openaislib.x86_64 0:1.1.0-1.fc12 set to be updated --> Finished Dependency Resolution Dependencies Resolved ===========================================================================================  Package                Arch               Version                   Repository        Size =========================================================================================== Installing:  gfs-pcmk               x86_64             3.0.5-2.fc12              custom           101 k  gfs2-utils             x86_64             3.0.5-2.fc12              custom           208 k Installing for dependencies:  clusterlib             x86_64             3.0.5-2.fc12              custom            65 k  dlm-pcmk               x86_64             3.0.5-2.fc12              custom            93 k  openaislib             x86_64             1.1.0-1.fc12              fedora            76 k Transaction Summary =========================================================================================== Install       5 Package(s) Upgrade       0 Package(s) Total download size: 541 k Downloading Packages: (1/5): clusterlib-3.0.5-2.fc12.x86_64.rpm                                |  65 kB     00:00 (2/5): dlm-pcmk-3.0.5-2.fc12.x86_64.rpm                                  |  93 kB     00:00 (3/5): gfs-pcmk-3.0.5-2.fc12.x86_64.rpm                                  | 101 kB     00:00 (4/5): gfs2-utils-3.0.5-2.fc12.x86_64.rpm                                | 208 kB     00:00 (5/5): openaislib-1.1.0-1.fc12.x86_64.rpm                                |  76 kB     00:00 ------------------------------------------------------------------------------------------- Total                                                           992 kB/s | 541 kB     00:00 Running rpm_check_debug Running Transaction Test Finished Transaction Test Transaction Test Succeeded Running Transaction   Installing     : clusterlib-3.0.5-2.fc12.x86_64                                       1/5   Installing     : openaislib-1.1.0-1.fc12.x86_64                                       2/5   Installing     : dlm-pcmk-3.0.5-2.fc12.x86_64                                         3/5   Installing     : gfs-pcmk-3.0.5-2.fc12.x86_64                                         4/5   Installing     : gfs2-utils-3.0.5-2.fc12.x86_64                                       5/5 Installed:   gfs-pcmk.x86_64 0:3.0.5-2.fc12                    gfs2-utils.x86_64 0:3.0.5-2.fc12 Dependency Installed:   clusterlib.x86_64 0:3.0.5-2.fc12   dlm-pcmk.x86_64 0:3.0.5-2.fc12   openaislib.x86_64 0:1.1.0-1.fc12   Complete! [root@pcmk-1 x86_64]# If this step fails, it is likely that your version/distribution does not ship the "Pacemaker" versions of dlm_controld and/or gfs_controld. Normally these files would be called dlm_controld.pcmk and gfs_controld.pcmk and live in the /usr/sbin directory. If you cannot locate an installation source for these files, you will need to install a package called cman and reconfigure Corosync to use it as outlined in . - When using CMAN, you can skip where dlm-clone and gfs-clone are created, and proceed directly to . + When using CMAN, you can skip where dlm-clone and gfs-clone are created, and proceed directly to after ensuring that gfs2-utils and gfs2-cluster were installed.
Setup Pacemaker-GFS2 Integration GFS2 needs two services to be running, the first is the user-space interface to the kernel’s distributed lock manager (DLM). The DLM is used to co-ordinate which node(s) can access a given file (and when) and integrates with Pacemaker to obtain node membership The list of nodes the cluster considers to be available information and fencing capabilities. The second service is GFS2’s own control daemon which also integrates with Pacemaker to obtain node membership data.
Add the DLM service The DLM control daemon needs to run on all active cluster nodes, so we will use the shells interactive mode to create a cloned resource. [root@pcmk-1 ~]# crm crm(live)# cib new stack-glue INFO: stack-glue shadow CIB created crm(stack-glue)# configure primitive dlm ocf:pacemaker:controld op monitor interval=120s crm(stack-glue)# configure clone dlm-clone dlm meta interleave=true crm(stack-glue)# configure show xml crm(stack-glue)# configure show node pcmk-1 node pcmk-2 primitive WebData ocf:linbit:drbd \         params drbd_resource="wwwdata" \         op monitor interval="60s" primitive WebFS ocf:heartbeat:Filesystem \         params device="/dev/drbd/by-res/wwwdata" directory="/var/www/html" fstype="ext4" primitive WebSite ocf:heartbeat:apache \         params configfile="/etc/httpd/conf/httpd.conf" \         op monitor interval="1min" primitive ClusterIP ocf:heartbeat:IPaddr2 \         params ip="192.168.122.101" cidr_netmask="32" \         op monitor interval="30s" primitive dlm ocf:pacemaker:controld \ op monitor interval="120s" ms WebDataClone WebData \         meta master-max="1" master-node-max="1" clone-max="2" clone-node-max="1" notify="true" clone dlm-clone dlm \ meta interleave="true" location prefer-pcmk-1 WebSite 50: pcmk-1 colocation WebSite-with-WebFS inf: WebSite WebFS colocation fs_on_drbd inf: WebFS WebDataClone:Master colocation website-with-ip inf: WebSite ClusterIP order WebFS-after-WebData inf: WebDataClone:promote WebFS:start order WebSite-after-WebFS inf: WebFS WebSite order apache-after-ip inf: ClusterIP WebSite property $id="cib-bootstrap-options" \         dc-version="1.1.5-bdd89e69ba545404d02445be1f3d72e6a203ba2f" \         cluster-infrastructure="openais" \         expected-quorum-votes=â€2†\         stonith-enabled="false" \         no-quorum-policy="ignore" rsc_defaults $id="rsc-options" \         resource-stickiness=â€100†TODO: Explain the meaning of the interleave option Review the configuration before uploading it to the cluster, quitting the shell and watching the cluster’s response crm(stack-glue)# cib commit stack-glue INFO: commited 'stack-glue' shadow CIB to the cluster crm(stack-glue)# quit bye [root@pcmk-1 ~]# crm_mon ============ Last updated: Thu Sep  3 20:49:54 2009 Stack: openais Current DC: pcmk-2 - partition with quorum Version: 1.1.5-bdd89e69ba545404d02445be1f3d72e6a203ba2f 2 Nodes configured, 2 expected votes 5 Resources configured. ============ Online: [ pcmk-1 pcmk-2 ] WebSite (ocf::heartbeat:apache):        Started pcmk-2 Master/Slave Set: WebDataClone         Masters: [ pcmk-1 ]         Slaves: [ pcmk-2 ] ClusterIP        (ocf::heartbeat:IPaddr):        Started pcmk-2 Clone Set: dlm-clone Started: [ pcmk-2 pcmk-1 ] WebFS   (ocf::heartbeat:Filesystem):    Started pcmk-2
Add the GFS2 service Once the DLM is active, we can add the GFS2 control daemon. Use the crm shell to create the gfs-control cluster resource: [root@pcmk-1 ~]# crm crm(live)# cib new gfs-glue --force INFO: gfs-glue shadow CIB created crm(gfs-glue)# configure primitive gfs-control ocf:pacemaker:controld params daemon=gfs_controld.pcmk args="-g 0" op monitor interval=120s crm(gfs-glue)# configure clone gfs-clone gfs-control meta interleave=true Now ensure Pacemaker only starts the gfs-control service on nodes that also have a copy of the dlm service (created above) already running crm(gfs-glue)# configure colocation gfs-with-dlm INFINITY: gfs-clone dlm-clone crm(gfs-glue)# configure order start-gfs-after-dlm mandatory: dlm-clone gfs-clone Review the configuration before uploading it to the cluster, quitting the shell and watching the cluster’s response crm(gfs-glue)# configure show node pcmk-1 node pcmk-2 primitive WebData ocf:linbit:drbd \         params drbd_resource="wwwdata" \         op monitor interval="60s" primitive WebFS ocf:heartbeat:Filesystem \         params device="/dev/drbd/by-res/wwwdata" directory="/var/www/html" fstype="ext4" primitive WebSite ocf:heartbeat:apache \         params configfile="/etc/httpd/conf/httpd.conf" \         op monitor interval="1min" primitive ClusterIP ocf:heartbeat:IPaddr2 \         params ip="192.168.122.101" cidr_netmask="32" \         op monitor interval="30s" primitive dlm ocf:pacemaker:controld \         op monitor interval="120s" primitive gfs-control ocf:pacemaker:controld \ params daemon=â€gfs_controld.pcmk†args=â€-g 0†\ op monitor interval="120s" ms WebDataClone WebData \         meta master-max="1" master-node-max="1" clone-max="2" clone-node-max="1" notify="true" clone dlm-clone dlm \         meta interleave="true" clone gfs-clone gfs-control \ meta interleave="true" location prefer-pcmk-1 WebSite 50: pcmk-1 colocation WebSite-with-WebFS inf: WebSite WebFS colocation fs_on_drbd inf: WebFS WebDataClone:Master colocation gfs-with-dlm inf: gfs-clone dlm-clone colocation website-with-ip inf: WebSite ClusterIP order WebFS-after-WebData inf: WebDataClone:promote WebFS:start order WebSite-after-WebFS inf: WebFS WebSite order apache-after-ip inf: ClusterIP WebSite order start-gfs-after-dlm inf: dlm-clone gfs-clone property $id="cib-bootstrap-options" \         dc-version="1.1.5-bdd89e69ba545404d02445be1f3d72e6a203ba2f" \         cluster-infrastructure="openais" \         expected-quorum-votes=â€2†\         stonith-enabled="false" \         no-quorum-policy="ignore" rsc_defaults $id="rsc-options" \         resource-stickiness=â€100†crm(gfs-glue)# cib commit gfs-glue INFO: commited 'gfs-glue' shadow CIB to the cluster crm(gfs-glue)# quit bye [root@pcmk-1 ~]# crm_mon ============ Last updated: Thu Sep  3 20:49:54 2009 Stack: openais Current DC: pcmk-2 - partition with quorum Version: 1.1.5-bdd89e69ba545404d02445be1f3d72e6a203ba2f 2 Nodes configured, 2 expected votes 6 Resources configured. ============ Online: [ pcmk-1 pcmk-2 ] WebSite (ocf::heartbeat:apache):        Started pcmk-2 Master/Slave Set: WebDataClone         Masters: [ pcmk-1 ]         Slaves: [ pcmk-2 ] ClusterIP        (ocf::heartbeat:IPaddr):        Started pcmk-2 Clone Set: dlm-clone         Started: [ pcmk-2 pcmk-1 ] Clone Set: gfs-clone Started: [ pcmk-2 pcmk-1 ] WebFS   (ocf::heartbeat:Filesystem):    Started pcmk-1
Create a GFS2 Filesystem
Preparation Before we do anything to the existing partition, we need to make sure it is unmounted. We do this by tell the cluster to stop the WebFS resource. This will ensure that other resources (in our case, Apache) using WebFS are not only stopped, but stopped in the correct order. [root@pcmk-1 ~]# crm_resource --resource WebFS --set-parameter target-role --meta --parameter-value Stopped [root@pcmk-1 ~]# crm_mon ============ Last updated: Thu Sep  3 15:18:06 2009 Stack: openais Current DC: pcmk-1 - partition with quorum Version: 1.1.5-bdd89e69ba545404d02445be1f3d72e6a203ba2f 2 Nodes configured, 2 expected votes 6 Resources configured. ============ Online: [ pcmk-1 pcmk-2 ] Master/Slave Set: WebDataClone         Masters: [ pcmk-1 ]         Slaves: [ pcmk-2 ] ClusterIP        (ocf::heartbeat:IPaddr):        Started pcmk-1 Clone Set: dlm-clone         Started: [ pcmk-2 pcmk-1 ] Clone Set: gfs-clone         Started: [ pcmk-2 pcmk-1 ] Note that both Apache and WebFS have been stopped.
Create and Populate an GFS2 Partition Now that the cluster stack and integration pieces are running smoothly, we can create an GFS2 partition. This will erase all previous content stored on the DRBD device. Ensure you have a copy of any important data. We need to specify a number of additional parameters when creating a GFS2 partition. First we must use the -p option to specify that we want to use the the Kernel’s DLM. Next we use -j to indicate that it should reserve enough space for two journals (one per node accessing the filesystem). Lastly, we use -t to specify the lock table name. The format for this field is clustername:fsname. For the fsname, we just need to pick something unique and descriptive and since we haven’t specified a clustername yet, we will use the default (pcmk). To specify an alternate name for the cluster, locate the service section containing “name: pacemaker†in corosync.conf and insert the following line anywhere inside the block: clustername: myname Do this on each node in the cluster and be sure to restart them before continuing. [root@pcmk-1 ~]# mkfs.gfs2 -p lock_dlm -j 2 -t pcmk:web /dev/drbd1 This will destroy any data on /dev/drbd1. It appears to contain: data Are you sure you want to proceed? [y/n] y Device:                    /dev/drbd1 Blocksize:                 4096 Device Size                1.00 GB (131072 blocks) Filesystem Size:           1.00 GB (131070 blocks) Journals:                  2 Resource Groups:           2 Locking Protocol:          "lock_dlm" Lock Table:                "pcmk:web" UUID:                      6B776F46-177B-BAF8-2C2B-292C0E078613 [root@pcmk-1 ~]# Then (re)populate the new filesystem with data (web pages). For now we’ll create another variation on our home page. [root@pcmk-1 ~]# mount /dev/drbd1 /mnt/ [root@pcmk-1 ~]# cat <<-END >/mnt/index.html <html> <body>My Test Site - GFS2</body> </html> END [root@pcmk-1 ~]# umount /dev/drbd1 [root@pcmk-1 ~]# drbdadm verify wwwdata [root@pcmk-1 ~]#
Reconfigure the Cluster for GFS2 [root@pcmk-1 ~]# crm crm(live)# cib new GFS2 INFO: GFS2 shadow CIB created crm(GFS2)# configure delete WebFS crm(GFS2)# configure primitive WebFS ocf:heartbeat:Filesystem params device="/dev/drbd/by-res/wwwdata" directory="/var/www/html" fstype=â€gfs2†Now that we’ve recreated the resource, we also need to recreate all the constraints that used it. This is because the shell will automatically remove any constraints that referenced WebFS. crm(GFS2)# configure colocation WebSite-with-WebFS inf: WebSite WebFS crm(GFS2)# configure colocation fs_on_drbd inf: WebFS WebDataClone:Master crm(GFS2)# configure order WebFS-after-WebData inf: WebDataClone:promote WebFS:start crm(GFS2)# configure order WebSite-after-WebFS inf: WebFS WebSite crm(GFS2)# configure colocation WebFS-with-gfs-control INFINITY: WebFS gfs-clone crm(GFS2)# configure order start-WebFS-after-gfs-control mandatory: gfs-clone WebFS crm(GFS2)# configure show node pcmk-1 node pcmk-2 primitive WebData ocf:linbit:drbd \         params drbd_resource="wwwdata" \         op monitor interval="60s" primitive WebFS ocf:heartbeat:Filesystem \ params device="/dev/drbd/by-res/wwwdata" directory="/var/www/html" fstype=â€gfs2†primitive WebSite ocf:heartbeat:apache \         params configfile="/etc/httpd/conf/httpd.conf" \         op monitor interval="1min" primitive ClusterIP ocf:heartbeat:IPaddr2 \         params ip="192.168.122.101" cidr_netmask="32" \         op monitor interval="30s" primitive dlm ocf:pacemaker:controld \         op monitor interval="120s" primitive gfs-control ocf:pacemaker:controld \    params daemon=â€gfs_controld.pcmk†args=â€-g 0†\         op monitor interval="120s" ms WebDataClone WebData \         meta master-max="1" master-node-max="1" clone-max="2" clone-node-max="1" notify="true" clone dlm-clone dlm \         meta interleave="true" clone gfs-clone gfs-control \         meta interleave="true" colocation WebFS-with-gfs-control inf: WebFS gfs-clone colocation WebSite-with-WebFS inf: WebSite WebFS colocation fs_on_drbd inf: WebFS WebDataClone:Master colocation gfs-with-dlm inf: gfs-clone dlm-clone colocation website-with-ip inf: WebSite ClusterIP order WebFS-after-WebData inf: WebDataClone:promote WebFS:start order WebSite-after-WebFS inf: WebFS WebSite order apache-after-ip inf: ClusterIP WebSite order start-WebFS-after-gfs-control inf: gfs-clone WebFS order start-gfs-after-dlm inf: dlm-clone gfs-clone property $id="cib-bootstrap-options" \         dc-version="1.1.5-bdd89e69ba545404d02445be1f3d72e6a203ba2f" \         cluster-infrastructure="openais" \         expected-quorum-votes=â€2†\         stonith-enabled="false" \         no-quorum-policy="ignore" rsc_defaults $id="rsc-options" \         resource-stickiness=â€100†Review the configuration before uploading it to the cluster, quitting the shell and watching the cluster’s response crm(GFS2)# cib commit GFS2 INFO: commited 'GFS2' shadow CIB to the cluster crm(GFS2)# quit bye [root@pcmk-1 ~]# crm_mon ============ Last updated: Thu Sep  3 20:49:54 2009 Stack: openais Current DC: pcmk-2 - partition with quorum Version: 1.1.5-bdd89e69ba545404d02445be1f3d72e6a203ba2f 2 Nodes configured, 2 expected votes 6 Resources configured. ============ Online: [ pcmk-1 pcmk-2 ] WebSite (ocf::heartbeat:apache):        Started pcmk-2 Master/Slave Set: WebDataClone         Masters: [ pcmk-1 ]         Slaves: [ pcmk-2 ] ClusterIP        (ocf::heartbeat:IPaddr):        Started pcmk-2 Clone Set: dlm-clone         Started: [ pcmk-2 pcmk-1 ] Clone Set: gfs-clone         Started: [ pcmk-2 pcmk-1 ] WebFS (ocf::heartbeat:Filesystem): Started pcmk-1
Reconfigure Pacemaker for Active/Active Almost everything is in place. Recent versions of DRBD are capable of operating in Primary/Primary mode and the filesystem we’re using is cluster aware. All we need to do now is reconfigure the cluster to take advantage of this. This will involve a number of changes, so we’ll again use interactive mode. [root@pcmk-1 ~]# crm [root@pcmk-1 ~]# cib new active There’s no point making the services active on both locations if we can’t reach them, so lets first clone the IP address. Cloned IPaddr2 resources use an iptables rule to ensure that each request only processed by one of the two clone instances. The additional meta options tell the cluster how many instances of the clone we want (one “request bucket†for each node) and that if all other nodes fail, then the remaining node should hold all of them. Otherwise the requests would be simply discarded. [root@pcmk-1 ~]# configure clone WebIP ClusterIP  \         meta globally-unique=â€true†clone-max=â€2†clone-node-max=â€2†Now we must tell the ClusterIP how to decide which requests are processed by which hosts. To do this we must specify the clusterip_hash parameter. Open the ClusterIP resource [root@pcmk-1 ~]# configure edit  ClusterIP And add the following to the params line clusterip_hash="sourceip" So that the complete definition looks like: primitive ClusterIP ocf:heartbeat:IPaddr2 \         params ip="192.168.122.101" cidr_netmask="32" clusterip_hash="sourceip" \         op monitor interval="30s" Here is the full transcript [root@pcmk-1 ~]# crm crm(live)# cib new active INFO: active shadow CIB created crm(active)# configure clone WebIP ClusterIP  \         meta globally-unique=â€true†clone-max=â€2†clone-node-max=â€2†crm(active)# configure show node pcmk-1 node pcmk-2 primitive WebData ocf:linbit:drbd \         params drbd_resource="wwwdata" \         op monitor interval="60s" primitive WebFS ocf:heartbeat:Filesystem \         params device="/dev/drbd/by-res/wwwdata" directory="/var/www/html" fstype=â€gfs2†primitive WebSite ocf:heartbeat:apache \         params configfile="/etc/httpd/conf/httpd.conf" \         op monitor interval="1min" primitive ClusterIP ocf:heartbeat:IPaddr2 \         params ip=â€192.168.122.101†cidr_netmask=â€32†clusterip_hash=â€sourceip†\         op monitor interval="30s" primitive dlm ocf:pacemaker:controld \         op monitor interval="120s" primitive gfs-control ocf:pacemaker:controld \    params daemon=â€gfs_controld.pcmk†args=â€-g 0†\         op monitor interval="120s" ms WebDataClone WebData \         meta master-max="1" master-node-max="1" clone-max="2" clone-node-max="1" notify="true" clone WebIP ClusterIP \ meta globally-unique=â€true†clone-max=â€2†clone-node-max=â€2†clone dlm-clone dlm \         meta interleave="true" clone gfs-clone gfs-control \         meta interleave="true" colocation WebFS-with-gfs-control inf: WebFS gfs-clone colocation WebSite-with-WebFS inf: WebSite WebFS colocation fs_on_drbd inf: WebFS WebDataClone:Master colocation gfs-with-dlm inf: gfs-clone dlm-clone colocation website-with-ip inf: WebSite WebIP order WebFS-after-WebData inf: WebDataClone:promote WebFS:start order WebSite-after-WebFS inf: WebFS WebSite order apache-after-ip inf: WebIP WebSite order start-WebFS-after-gfs-control inf: gfs-clone WebFS order start-gfs-after-dlm inf: dlm-clone gfs-clone property $id="cib-bootstrap-options" \         dc-version="1.1.5-bdd89e69ba545404d02445be1f3d72e6a203ba2f" \         cluster-infrastructure="openais" \         expected-quorum-votes=â€2†\         stonith-enabled="false" \         no-quorum-policy="ignore" rsc_defaults $id="rsc-options" \         resource-stickiness=â€100†Notice how any constraints that referenced ClusterIP have been updated to use WebIP instead. This is an additional benefit of using the crm shell. Next we need to convert the filesystem and Apache resources into clones. Again, the shell will automatically update any relevant constraints. crm(active)# configure clone WebFSClone WebFS crm(active)# configure clone WebSiteClone WebSite The last step is to tell the cluster that it is now allowed to promote both instances to be Primary (aka. Master). crm(active)# configure edit WebDataClone Change master-max to 2 crm(active)# configure show node pcmk-1 node pcmk-2 primitive WebData ocf:linbit:drbd \         params drbd_resource="wwwdata" \         op monitor interval="60s" primitive WebFS ocf:heartbeat:Filesystem \         params device="/dev/drbd/by-res/wwwdata" directory="/var/www/html" fstype=â€gfs2†primitive WebSite ocf:heartbeat:apache \         params configfile="/etc/httpd/conf/httpd.conf" \         op monitor interval="1min" primitive ClusterIP ocf:heartbeat:IPaddr2 \         params ip=â€192.168.122.101†cidr_netmask=â€32†clusterip_hash=â€sourceip†\         op monitor interval="30s" primitive dlm ocf:pacemaker:controld \         op monitor interval="120s" primitive gfs-control ocf:pacemaker:controld \    params daemon=â€gfs_controld.pcmk†args=â€-g 0†\         op monitor interval="120s" ms WebDataClone WebData \         meta master-max="2" master-node-max="1" clone-max="2" clone-node-max="1" notify="true" clone WebFSClone WebFS clone WebIP ClusterIP  \         meta globally-unique=â€true†clone-max=â€2†clone-node-max=â€2†clone WebSiteClone WebSite clone dlm-clone dlm \         meta interleave="true" clone gfs-clone gfs-control \         meta interleave="true" colocation WebFS-with-gfs-control inf: WebFSClone gfs-clone colocation WebSite-with-WebFS inf: WebSiteClone WebFSClone colocation fs_on_drbd inf: WebFSClone WebDataClone:Master colocation gfs-with-dlm inf: gfs-clone dlm-clone colocation website-with-ip inf: WebSiteClone WebIP order WebFS-after-WebData inf: WebDataClone:promote WebFSClone:start order WebSite-after-WebFS inf: WebFSClone WebSiteClone order apache-after-ip inf: WebIP WebSiteClone order start-WebFS-after-gfs-control inf: gfs-clone WebFSClone order start-gfs-after-dlm inf: dlm-clone gfs-clone property $id="cib-bootstrap-options" \         dc-version="1.1.5-bdd89e69ba545404d02445be1f3d72e6a203ba2f" \         cluster-infrastructure="openais" \         expected-quorum-votes=â€2†\         stonith-enabled="false" \         no-quorum-policy="ignore" rsc_defaults $id="rsc-options" \         resource-stickiness=â€100†Review the configuration before uploading it to the cluster, quitting the shell and watching the cluster’s response crm(active)# cib commit active INFO: commited 'active' shadow CIB to the cluster crm(active)# quit bye [root@pcmk-1 ~]# crm_mon ============ Last updated: Thu Sep  3 21:37:27 2009 Stack: openais Current DC: pcmk-2 - partition with quorum Version: 1.1.5-bdd89e69ba545404d02445be1f3d72e6a203ba2f 2 Nodes configured, 2 expected votes 6 Resources configured. ============ Online: [ pcmk-1 pcmk-2 ] Master/Slave Set: WebDataClone         Masters: [ pcmk-1 pcmk-2 ] Clone Set: dlm-clone         Started: [ pcmk-2 pcmk-1 ] Clone Set: gfs-clone         Started: [ pcmk-2 pcmk-1 ] Clone Set: WebIP Started: [ pcmk-1 pcmk-2 ] Clone Set: WebFSClone Started: [ pcmk-1 pcmk-2 ] Clone Set: WebSiteClone Started: [ pcmk-1 pcmk-2 ]
Testing Recovery TODO: Put one node into standby to demonstrate failover
diff --git a/doc/Clusters_from_Scratch/en-US/Ch-Apache.xml b/doc/Clusters_from_Scratch/en-US/Ch-Apache.xml index 667c2c391d..8377f05620 100644 --- a/doc/Clusters_from_Scratch/en-US/Ch-Apache.xml +++ b/doc/Clusters_from_Scratch/en-US/Ch-Apache.xml @@ -1,472 +1,490 @@ %BOOK_ENTITIES; ]> Apache - Adding More Services Now that we have a basic but functional active/passive two-node cluster, we’re ready to add some real services. We’re going to start with Apache because its a feature of many clusters and relatively simple to configure.
Installation Before continuing, we need to make sure Apache is installed on both hosts. [root@ppcmk-1 ~]# yum install -y httpd Setting up Install Process Resolving Dependencies --> Running transaction check ---> Package httpd.x86_64 0:2.2.13-2.fc12 set to be updated --> Processing Dependency: httpd-tools = 2.2.13-2.fc12 for package: httpd-2.2.13-2.fc12.x86_64 --> Processing Dependency: apr-util-ldap for package: httpd-2.2.13-2.fc12.x86_64 --> Processing Dependency: /etc/mime.types for package: httpd-2.2.13-2.fc12.x86_64 --> Processing Dependency: libaprutil-1.so.0()(64bit) for package: httpd-2.2.13-2.fc12.x86_64 --> Processing Dependency: libapr-1.so.0()(64bit) for package: httpd-2.2.13-2.fc12.x86_64 --> Running transaction check ---> Package apr.x86_64 0:1.3.9-2.fc12 set to be updated ---> Package apr-util.x86_64 0:1.3.9-2.fc12 set to be updated ---> Package apr-util-ldap.x86_64 0:1.3.9-2.fc12 set to be updated ---> Package httpd-tools.x86_64 0:2.2.13-2.fc12 set to be updated ---> Package mailcap.noarch 0:2.1.30-1.fc12 set to be updated --> Finished Dependency Resolution Dependencies Resolved =======================================================================================  Package               Arch             Version                Repository         Size ======================================================================================= Installing:  httpd               x86_64           2.2.13-2.fc12            rawhide           735 k Installing for dependencies:  apr                 x86_64           1.3.9-2.fc12             rawhide           117 k  apr-util            x86_64           1.3.9-2.fc12             rawhide            84 k  apr-util-ldap       x86_64           1.3.9-2.fc12             rawhide            15 k  httpd-tools         x86_64           2.2.13-2.fc12            rawhide            63 k  mailcap             noarch           2.1.30-1.fc12            rawhide            25 k Transaction Summary ======================================================================================= Install       6 Package(s) Upgrade       0 Package(s) Total download size: 1.0 M Downloading Packages: (1/6): apr-1.3.9-2.fc12.x86_64.rpm                                   | 117 kB     00:00     (2/6): apr-util-1.3.9-2.fc12.x86_64.rpm                             |  84 kB     00:00     (3/6): apr-util-ldap-1.3.9-2.fc12.x86_64.rpm                         |  15 kB     00:00     (4/6): httpd-2.2.13-2.fc12.x86_64.rpm                               | 735 kB     00:00     (5/6): httpd-tools-2.2.13-2.fc12.x86_64.rpm                         |  63 kB     00:00     (6/6): mailcap-2.1.30-1.fc12.noarch.rpm                             |  25 kB     00:00     ---------------------------------------------------------------------------------------- Total                                                       875 kB/s | 1.0 MB     00:01     Running rpm_check_debug Running Transaction Test Finished Transaction Test Transaction Test Succeeded Running Transaction   Installing     : apr-1.3.9-2.fc12.x86_64                                         1/6   Installing     : apr-util-1.3.9-2.fc12.x86_64                                     2/6   Installing     : apr-util-ldap-1.3.9-2.fc12.x86_64                               3/6   Installing     : httpd-tools-2.2.13-2.fc12.x86_64                                 4/6   Installing     : mailcap-2.1.30-1.fc12.noarch                                     5/6   Installing     : httpd-2.2.13-2.fc12.x86_64                                       6/6 Installed:   httpd.x86_64 0:2.2.13-2.fc12                                                         Dependency Installed:   apr.x86_64 0:1.3.9-2.fc12            apr-util.x86_64 0:1.3.9-2.fc12   apr-util-ldap.x86_64 0:1.3.9-2.fc12  httpd-tools.x86_64 0:2.2.13-2.fc12   mailcap.noarch 0:2.1.30-1.fc12   Complete! [root@pcmk-1 ~]# Also, we need the wget tool in order for the cluster to be able to check the status of the Apache server. [root@pcmk-1 ~]# yum install -y wget Setting up Install Process Resolving Dependencies --> Running transaction check ---> Package wget.x86_64 0:1.11.4-5.fc12 set to be updated --> Finished Dependency Resolution Dependencies Resolved ===========================================================================================  Package        Arch             Version                      Repository               Size =========================================================================================== Installing:  wget         x86_64          1.11.4-5.fc12                   rawhide                393 k Transaction Summary =========================================================================================== Install       1 Package(s) Upgrade       0 Package(s) Total download size: 393 k Downloading Packages: wget-1.11.4-5.fc12.x86_64.rpm                                            | 393 kB     00:00     Running rpm_check_debug Running Transaction Test Finished Transaction Test Transaction Test Succeeded Running Transaction   Installing     : wget-1.11.4-5.fc12.x86_64                                            1/1 Installed:   wget.x86_64 0:1.11.4-5.fc12 Complete! [root@pcmk-1 ~]#
Preparation First we need to create a page for Apache to serve up. On Fedora the default Apache docroot is /var/www/html, so we’ll create an index file there. [root@pcmk-1 ~]# cat <<-END >/var/www/html/index.html <html> <body>My Test Site - pcmk-1</body> </html> END [root@pcmk-1 ~]# For the moment, we will simplify things by serving up only a static site and manually sync the data between the two nodes. So run the command again on pcmk-2. [root@pcmk-2 ~]# cat <<-END >/var/www/html/index.html <html> <body>My Test Site - pcmk-2</body> </html> END [root@pcmk-2 ~]#
Enable the Apache status URL In order to monitor the health of your Apache instance, and recover it if it fails, the resource agent used by Pacemaker assumes the server-status URL is available. Look for the following in /etc/httpd/conf/httpd.conf and make sure it is not disabled or commented out: <Location /server-status> SetHandler server-status Order deny,allow Deny from all Allow from 127.0.0.1 </Location>
Update the Configuration At this point, Apache is ready to go, all that needs to be done is to add it to the cluster. Lets call the resource WebSite. We need to use an OCF script called apache in the heartbeat namespace Compare the key used here ocf:heartbeat:apache with the one we used earlier for the IP address: ocf:heartbeat:IPaddr2 , the only required parameter is the path to the main Apache configuration file and we’ll tell the cluster to check once a minute that apache is still running. [root@pcmk-1 ~]# crm configure primitive WebSite ocf:heartbeat:apache params configfile=/etc/httpd/conf/httpd.conf op monitor interval=1min [root@pcmk-1 ~]# crm configure show node pcmk-1 node pcmk-2 primitive WebSite ocf:heartbeat:apache \ params configfile="/etc/httpd/conf/httpd.conf" \ op monitor interval="1min" primitive ClusterIP ocf:heartbeat:IPaddr2 \         params ip="192.168.122.101" cidr_netmask="32" \         op monitor interval="30s" property $id="cib-bootstrap-options" \         dc-version="1.1.5-bdd89e69ba545404d02445be1f3d72e6a203ba2f" \         cluster-infrastructure="openais" \         expected-quorum-votes="2" \         stonith-enabled="false" \         no-quorum-policy="ignore" rsc_defaults $id="rsc-options" \         resource-stickiness="100" After a short delay, we should see the cluster start apache [root@pcmk-1 ~]# crm_mon ============ Last updated: Fri Aug 28 16:12:49 2009 Stack: openais Current DC: pcmk-2 - partition with quorum Version: 1.1.5-bdd89e69ba545404d02445be1f3d72e6a203ba2f 2 Nodes configured, 2 expected votes 2 Resources configured. ============ Online: [ pcmk-1 pcmk-2 ] ClusterIP        (ocf::heartbeat:IPaddr):        Started pcmk-2 WebSite        (ocf::heartbeat:apache):        Started pcmk-1 Wait a moment, the WebSite resource isn’t running on the same host as our IP address!
Ensuring Resources Run on the Same Host - To reduce the load on any one machine, Pacemaker will generally try to spread the configured resources across the cluster nodes. However we can tell the cluster that two resources are related and need to run on the same host (or not at all). Here we instruct the cluster that WebSite can only run on the host that ClusterIP is active on. If ClusterIP is not active anywhere, WebSite will not be permitted to run anywhere. + To reduce the load on any one machine, Pacemaker will generally try to spread the configured resources across the cluster nodes. + However we can tell the cluster that two resources are related and need to run on the same host (or not at all). + Here we instruct the cluster that WebSite can only run on the host that ClusterIP is active on. - + + For the constraint, we need a name (choose something descriptive like website-with-ip), indicate that its mandatory (so that if ClusterIP is not active anywhere, WebSite will not be permitted to run anywhere either) by specifying a score of INFINITY and finally list the two resources. + + + + If ClusterIP is not active anywhere, WebSite will not be permitted to run anywhere. + + + + + Colocation constraints are "directional", in that they imply certain things about the order in which the two resources will have a location chosen. + In this case we're saying WebSite needs to be placed on the same machine as ClusterIP, this implies that we must know the location of ClusterIP before choosing a location for WebSite. + + [root@pcmk-1 ~]# crm configure colocation website-with-ip INFINITY: WebSite ClusterIP [root@pcmk-1 ~]# crm configure show node pcmk-1 node pcmk-2 primitive WebSite ocf:heartbeat:apache \         params configfile="/etc/httpd/conf/httpd.conf" \         op monitor interval="1min" primitive ClusterIP ocf:heartbeat:IPaddr2 \         params ip="192.168.122.101" cidr_netmask="32" \         op monitor interval="30s" colocation website-with-ip inf: WebSite ClusterIP property $id="cib-bootstrap-options" \         dc-version="1.1.5-bdd89e69ba545404d02445be1f3d72e6a203ba2f" \         cluster-infrastructure="openais" \         expected-quorum-votes="2" \         stonith-enabled="false" \         no-quorum-policy="ignore" rsc_defaults $id="rsc-options" \         resource-stickiness="100" [root@pcmk-1 ~]# crm_mon ============ Last updated: Fri Aug 28 16:14:34 2009 Stack: openais Current DC: pcmk-2 - partition with quorum Version: 1.1.5-bdd89e69ba545404d02445be1f3d72e6a203ba2f 2 Nodes configured, 2 expected votes 2 Resources configured. ============ Online: [ pcmk-1 pcmk-2 ] ClusterIP        (ocf::heartbeat:IPaddr):        Started pcmk-2 WebSite        (ocf::heartbeat:apache):        Started pcmk-2
- Controlling Resource Start/Stop Ordering - - When Apache starts, it binds to the available IP addresses. It doesn’t know about any addresses we add afterwards, so not only do they need to run on the same node, but we need to make sure ClusterIP is already active before we start WebSite. We do this by adding an ordering constraint. We need to give it a name (chose something descriptive like apache-after-ip), indicate that its mandatory (so that any recovery for ClusterIP will also trigger recovery of WebSite) and list the two resources in the order we need them to start. - + Controlling Resource Start/Stop Ordering + + When Apache starts, it binds to the available IP addresses. + It doesn’t know about any addresses we add afterwards, so not only do they need to run on the same node, but we need to make sure ClusterIP is already active before we start WebSite. + We do this by adding an ordering constraint. + We need to give it a name (choose something descriptive like apache-after-ip), indicate that its mandatory (so that any recovery for ClusterIP will also trigger recovery of WebSite) and list the two resources in the order we need them to start. + [root@pcmk-1 ~]# crm configure order apache-after-ip mandatory: ClusterIP WebSite [root@pcmk-1 ~]# crm configure show node pcmk-1 node pcmk-2 primitive WebSite ocf:heartbeat:apache \         params configfile="/etc/httpd/conf/httpd.conf" \         op monitor interval="1min" primitive ClusterIP ocf:heartbeat:IPaddr2 \         params ip="192.168.122.101" cidr_netmask="32" \         op monitor interval="30s" colocation website-with-ip inf: WebSite ClusterIP order apache-after-ip inf: ClusterIP WebSite property $id="cib-bootstrap-options" \         dc-version="1.1.5-bdd89e69ba545404d02445be1f3d72e6a203ba2f" \         cluster-infrastructure="openais" \         expected-quorum-votes="2" \         stonith-enabled="false" \         no-quorum-policy="ignore" rsc_defaults $id="rsc-options" \         resource-stickiness="100"
Specifying a Preferred Location Pacemaker does not rely on any sort of hardware symmetry between nodes, so it may well be that one machine is more powerful than the other. In such cases it makes sense to host the resources there if it is available. To do this we create a location constraint. Again we give it a descriptive name (prefer-pcmk-1), specify the resource we want to run there (WebSite), how badly we’d like it to run there (we’ll use 50 for now, but in a two-node situation almost any value above 0 will do) and the host’s name. [root@pcmk-1 ~]# crm configure location prefer-pcmk-1 WebSite 50: pcmk-1 [root@pcmk-1 ~]# crm configure show node pcmk-1 node pcmk-2 primitive WebSite ocf:heartbeat:apache \         params configfile="/etc/httpd/conf/httpd.conf" \         op monitor interval="1min" primitive ClusterIP ocf:heartbeat:IPaddr2 \         params ip="192.168.122.101" cidr_netmask="32" \         op monitor interval="30s" location prefer-pcmk-1 WebSite 50: pcmk-1 colocation website-with-ip inf: WebSite ClusterIP property $id="cib-bootstrap-options" \         dc-version="1.1.5-bdd89e69ba545404d02445be1f3d72e6a203ba2f" \         cluster-infrastructure="openais" \         expected-quorum-votes="2" \         stonith-enabled="false" \         no-quorum-policy="ignore" rsc_defaults $id="rsc-options" \         resource-stickiness="100" [root@pcmk-1 ~]# crm_mon ============ Last updated: Fri Aug 28 16:17:35 2009 Stack: openais Current DC: pcmk-2 - partition with quorum Version: 1.1.5-bdd89e69ba545404d02445be1f3d72e6a203ba2f 2 Nodes configured, 2 expected votes 2 Resources configured. ============ Online: [ pcmk-1 pcmk-2 ] ClusterIP        (ocf::heartbeat:IPaddr):        Started pcmk-2 WebSite        (ocf::heartbeat:apache):        Started pcmk-2 Wait a minute, the resources are still on pcmk-2! Even though we now prefer pcmk-1 over pcmk-2, that preference is (intentionally) less than the resource stickiness (how much we preferred not to have unnecessary downtime). To see the current placement scores, you can use a tool called ptest ptest -sL Include output There is a way to force them to move though...
Manually Moving Resources Around the Cluster There are always times when an administrator needs to override the cluster and force resources to move to a specific location. Underneath we use location constraints like the one we created above, happily you don’t need to care. Just provide the name of the resource and the intended location, we’ll do the rest. [root@pcmk-1 ~]# crm resource move WebSite pcmk-1 [root@pcmk-1 ~]# crm_mon ============ Last updated: Fri Aug 28 16:19:24 2009 Stack: openais Current DC: pcmk-2 - partition with quorum Version: 1.1.5-bdd89e69ba545404d02445be1f3d72e6a203ba2f 2 Nodes configured, 2 expected votes 2 Resources configured. ============ Online: [ pcmk-1 pcmk-2 ] ClusterIP        (ocf::heartbeat:IPaddr):        Started pcmk-1 WebSite        (ocf::heartbeat:apache):        Started pcmk-1 Notice how the colocation rule we created has ensured that ClusterIP was also moved to pcmk-1. For the curious, we can see the effect of this command by examining the configuration crm configure show [root@pcmk-1 ~]# crm configure show node pcmk-1 node pcmk-2 primitive WebSite ocf:heartbeat:apache \         params configfile="/etc/httpd/conf/httpd.conf" \         op monitor interval="1min" primitive ClusterIP ocf:heartbeat:IPaddr2 \         params ip="192.168.122.101" cidr_netmask="32" \         op monitor interval="30s" location cli-prefer-WebSite WebSite \ rule $id="cli-prefer-rule-WebSite" inf: #uname eq pcmk-1 location prefer-pcmk-1 WebSite 50: pcmk-1 colocation website-with-ip inf: WebSite ClusterIP property $id="cib-bootstrap-options" \         dc-version="1.1.5-bdd89e69ba545404d02445be1f3d72e6a203ba2f" \         cluster-infrastructure="openais" \         expected-quorum-votes="2" \         stonith-enabled="false" \         no-quorum-policy="ignore" rsc_defaults $id="rsc-options" \         resource-stickiness="100" Highlighted is the automated constraint used to move the resources to pcmk-1
Giving Control Back to the Cluster Once we’ve finished whatever activity that required us to move the resources to pcmk-1, in our case nothing, we can then allow the cluster to resume normal operation with the unmove command. Since we previously configured a default stickiness, the resources will remain on pcmk-1. [root@pcmk-1 ~]# crm resource unmove WebSite [root@pcmk-1 ~]# crm configure show node pcmk-1 node pcmk-2 primitive WebSite ocf:heartbeat:apache \         params configfile="/etc/httpd/conf/httpd.conf" \         op monitor interval="1min" primitive ClusterIP ocf:heartbeat:IPaddr2 \         params ip="192.168.122.101" cidr_netmask="32" \         op monitor interval="30s" location prefer-pcmk-1 WebSite 50: pcmk-1 colocation website-with-ip inf: WebSite ClusterIP property $id="cib-bootstrap-options" \         dc-version="1.1.5-bdd89e69ba545404d02445be1f3d72e6a203ba2f" \         cluster-infrastructure="openais" \         expected-quorum-votes="2" \         stonith-enabled="false" \         no-quorum-policy="ignore" rsc_defaults $id="rsc-options" \         resource-stickiness="100" Note that the automated constraint is now gone. If we check the cluster status, we can also see that as expected the resources are still active on pcmk-1. [root@pcmk-1 ~]# crm_mon ============ Last updated: Fri Aug 28 16:20:53 2009 Stack: openais Current DC: pcmk-2 - partition with quorum Version: 1.1.5-bdd89e69ba545404d02445be1f3d72e6a203ba2f 2 Nodes configured, 2 expected votes 2 Resources configured. ============ Online: [ pcmk-1 pcmk-2 ] ClusterIP        (ocf::heartbeat:IPaddr):        Started pcmk-1 WebSite        (ocf::heartbeat:apache):        Started pcmk-1
diff --git a/doc/Clusters_from_Scratch/en-US/Ch-Shared-Storage.xml b/doc/Clusters_from_Scratch/en-US/Ch-Shared-Storage.xml index 03d974b410..5fc6805de8 100644 --- a/doc/Clusters_from_Scratch/en-US/Ch-Shared-Storage.xml +++ b/doc/Clusters_from_Scratch/en-US/Ch-Shared-Storage.xml @@ -1,528 +1,528 @@ %BOOK_ENTITIES; ]> Replicated Storage with DRBD Even if you’re serving up static websites, having to manually synchronize the contents of that website to all the machines in the cluster is not ideal. For dynamic websites, such as a wiki, its not even an option. Not everyone care afford network-attached storage but somehow the data needs to be kept in sync. Enter DRBD which can be thought of as network based RAID-1. See http://www.drbd.org/ for more details.
Install the DRBD Packages Since its inclusion in the upstream 2.6.33 kernel, everything needed to use DRBD ships with &DISTRO; &DISTRO_VERSION;. All you need to do is install it: -[root@pcmk-1 ~]# yum install -y drbd-pacemaker +[root@pcmk-1 ~]# yum install -y drbd-pacemaker drbd-udev Loaded plugins: presto, refresh-packagekit Setting up Install Process Resolving Dependencies --> Running transaction check ---> Package drbd-pacemaker.x86_64 0:8.3.7-2.fc13 set to be updated --> Processing Dependency: drbd-utils = 8.3.7-2.fc13 for package: drbd-pacemaker-8.3.7-2.fc13.x86_64 --> Running transaction check ---> Package drbd-utils.x86_64 0:8.3.7-2.fc13 set to be updated --> Finished Dependency Resolution Dependencies Resolved ================================================================================= Package Arch Version Repository Size ================================================================================= Installing: drbd-pacemaker x86_64 8.3.7-2.fc13 fedora 19 k Installing for dependencies: drbd-utils x86_64 8.3.7-2.fc13 fedora 165 k Transaction Summary ================================================================================= Install 2 Package(s) Upgrade 0 Package(s) Total download size: 184 k Installed size: 427 k Downloading Packages: Setting up and reading Presto delta metadata fedora/prestodelta | 1.7 kB 00:00 Processing delta metadata Package(s) data still to download: 184 k (1/2): drbd-pacemaker-8.3.7-2.fc13.x86_64.rpm | 19 kB 00:01 (2/2): drbd-utils-8.3.7-2.fc13.x86_64.rpm | 165 kB 00:02 --------------------------------------------------------------------------------- Total 45 kB/s | 184 kB 00:04 Running rpm_check_debug Running Transaction Test Transaction Test Succeeded Running Transaction Installing : drbd-utils-8.3.7-2.fc13.x86_64 1/2 Installing : drbd-pacemaker-8.3.7-2.fc13.x86_64 2/2 Installed: drbd-pacemaker.x86_64 0:8.3.7-2.fc13 Dependency Installed: drbd-utils.x86_64 0:8.3.7-2.fc13 Complete! [root@pcmk-1 ~]#
Configure DRBD Before we configure DRBD, we need to set aside some disk for it to use.
Create A Partition for DRBD If you have more than 1Gb free, feel free to use it. For this guide however, 1Gb is plenty of space for a single html file and sufficient for later holding the GFS2 metadata. [root@pcmk-1 ~]# lvcreate -n drbd-demo -L 1G VolGroup   Logical volume "drbd-demo" created [root@pcmk-1 ~]# lvs   LV        VG       Attr   LSize   Origin Snap%  Move Log Copy%  Convert   drbd-demo VolGroup -wi-a- 1.00G                                         lv_root   VolGroup -wi-ao   7.30G                                         lv_swap   VolGroup -wi-ao 500.00M Repeat this on the second node, be sure to use the same size partition. [root@pcmk-2 ~]# lvs   LV      VG       Attr   LSize   Origin Snap%  Move Log Copy%  Convert   lv_root VolGroup -wi-ao   7.30G                                         lv_swap VolGroup -wi-ao 500.00M                                       [root@pcmk-2 ~]# lvcreate -n drbd-demo -L 1G VolGroup   Logical volume "drbd-demo" created [root@pcmk-2 ~]# lvs   LV        VG       Attr   LSize   Origin Snap%  Move Log Copy%  Convert   drbd-demo VolGroup -wi-a- 1.00G                                         lv_root   VolGroup -wi-ao   7.30G                                         lv_swap   VolGroup -wi-ao 500.00M
Write the DRBD Config There is no series of commands for build a DRBD configuration, so simply copy the configuration below to /etc/drbd.conf Detailed information on the directives used in this configuration (and other alternatives) is available from http://www.drbd.org/users-guide/ch-configure.html Be sure to use the names and addresses of your nodes if they differ from the ones used in this guide. global {   usage-count yes; } common {   protocol C; } resource wwwdata {   meta-disk internal;   device    /dev/drbd1;   syncer {     verify-alg sha1;   }   net {     allow-two-primaries;   }   on pcmk-1 {     disk      /dev/mapper/VolGroup-drbd--demo;     address   192.168.122.101:7789;   }   on pcmk-2 {     disk      /dev/mapper/VolGroup-drbd--demo;     address   192.168.122.102:7789;   } } TODO: Explain the reason for the allow-two-primaries option
Initialize and Load DRBD With the configuration in place, we can now perform the DRBD initialization [root@pcmk-1 ~]# drbdadm create-md wwwdata md_offset 12578816 al_offset 12546048 bm_offset 12541952 Found some data  ==> This might destroy existing data! <== Do you want to proceed? [need to type 'yes' to confirm] yes Writing meta data... initializing activity log NOT initialized bitmap New drbd meta data block successfully created. success Now load the DRBD kernel module and confirm that everything is sane [root@pcmk-1 ~]# modprobe drbd [root@pcmk-1 ~]# drbdadm up wwwdata [root@pcmk-1 ~]# cat /proc/drbd version: 8.3.6 (api:88/proto:86-90) GIT-hash: f3606c47cc6fcf6b3f086e425cb34af8b7a81bbf build by root@pcmk-1, 2009-12-08 11:22:57 1: cs:WFConnection ro:Secondary/Unknown ds:Inconsistent/DUnknown C r----     ns:0 nr:0 dw:0 dr:0 al:0 bm:0 lo:0 pe:0 ua:0 ap:0 ep:1 wo:b oos:12248 [root@pcmk-1 ~]# Repeat on the second node drbdadm --force create-md wwwdata modprobe drbd drbdadm up wwwdata cat /proc/drbd [root@pcmk-2 ~]# drbdadm --force create-md wwwdata Writing meta data... initializing activity log NOT initialized bitmap New drbd meta data block successfully created. success [root@pcmk-2 ~]# modprobe drbd WARNING: Deprecated config file /etc/modprobe.conf, all config files belong into /etc/modprobe.d/. [root@pcmk-2 ~]# drbdadm up wwwdata [root@pcmk-2 ~]# cat /proc/drbd version: 8.3.6 (api:88/proto:86-90) GIT-hash: f3606c47cc6fcf6b3f086e425cb34af8b7a81bbf build by root@pcmk-1, 2009-12-08 11:22:57 1: cs:Connected ro:Secondary/Secondary ds:Inconsistent/Inconsistent C r----     ns:0 nr:0 dw:0 dr:0 al:0 bm:0 lo:0 pe:0 ua:0 ap:0 ep:1 wo:b oos:12248 Now we need to tell DRBD which set of data to use. Since both sides contain garbage, we can run the following on pcmk-1: [root@pcmk-1 ~]# drbdadm -- --overwrite-data-of-peer primary wwwdata [root@pcmk-1 ~]# cat /proc/drbd version: 8.3.6 (api:88/proto:86-90) GIT-hash: f3606c47cc6fcf6b3f086e425cb34af8b7a81bbf build by root@pcmk-1, 2009-12-08 11:22:57  1: cs:SyncSource ro:Primary/Secondary ds:UpToDate/Inconsistent C r----     ns:2184 nr:0 dw:0 dr:2472 al:0 bm:0 lo:0 pe:0 ua:0 ap:0 ep:1 wo:b oos:10064         [=====>..............] sync'ed: 33.4% (10064/12248)K         finish: 0:00:37 speed: 240 (240) K/sec [root@pcmk-1 ~]# cat /proc/drbd version: 8.3.6 (api:88/proto:86-90) GIT-hash: f3606c47cc6fcf6b3f086e425cb34af8b7a81bbf build by root@pcmk-1, 2009-12-08 11:22:57  1: cs:Connected ro:Primary/Secondary ds:UpToDate/UpToDate C r----     ns:12248 nr:0 dw:0 dr:12536 al:0 bm:1 lo:0 pe:0 ua:0 ap:0 ep:1 wo:b oos:0 pcmk-1 is now in the Primary state which allows it to be written to. Which means its a good point at which to create a filesystem and populate it with some data to serve up via our WebSite resource.
Populate DRBD with Data [root@pcmk-1 ~]# mkfs.ext4 /dev/drbd1 mke2fs 1.41.4 (27-Jan-2009) Filesystem label= OS type: Linux Block size=1024 (log=0) Fragment size=1024 (log=0) 3072 inodes, 12248 blocks 612 blocks (5.00%) reserved for the super user First data block=1 Maximum filesystem blocks=12582912 2 block groups 8192 blocks per group, 8192 fragments per group 1536 inodes per group Superblock backups stored on blocks:         8193 Writing inode tables: done                             Creating journal (1024 blocks): done Writing superblocks and filesystem accounting information: done This filesystem will be automatically checked every 26 mounts or 180 days, whichever comes first.  Use tune2fs -c or -i to override. Now mount the newly created filesystem so we can create our index file mount /dev/drbd1 /mnt/ cat <<-END >/mnt/index.html <html> <body>My Test Site - drbd</body> </html> END umount /dev/drbd1 [root@pcmk-1 ~]# mount /dev/drbd1 /mnt/ [root@pcmk-1 ~]# cat <<-END >/mnt/index.html > <html> > <body>My Test Site - drbd</body> > </html> > END [root@pcmk-1 ~]# umount /dev/drbd1
Configure the Cluster for DRBD One handy feature of the crm shell is that you can use it in interactive mode to make several changes atomically. First we launch the shell. The prompt will change to indicate you’re in interactive mode. [root@pcmk-1 ~]# crm cib crm(live)# Next we must create a working copy or the current configuration. This is where all our changes will go. The cluster will not see any of them until we say its ok. Notice again how the prompt changes, this time to indicate that we’re no longer looking at the live cluster. cib crm(live)# cib new drbd INFO: drbd shadow CIB created crm(drbd)# Now we can create our DRBD clone and display the revised configuration. crm(drbd)# configure primitive WebData ocf:linbit:drbd params drbd_resource=wwwdata \         op monitor interval=60s crm(drbd)# configure ms WebDataClone WebData meta master-max=1 master-node-max=1 \         clone-max=2 clone-node-max=1 notify=true crm(drbd)# configure show node pcmk-1 node pcmk-2 primitive WebData ocf:linbit:drbd \ params drbd_resource="wwwdata" \ op monitor interval="60s" primitive WebSite ocf:heartbeat:apache \         params configfile="/etc/httpd/conf/httpd.conf" \         op monitor interval="1min" primitive ClusterIP ocf:heartbeat:IPaddr2 \         params ip="192.168.122.101" cidr_netmask="32" \         op monitor interval="30s" ms WebDataClone WebData \ meta master-max="1" master-node-max="1" clone-max="2" clone-node-max="1" notify="true" location prefer-pcmk-1 WebSite 50: pcmk-1 colocation website-with-ip inf: WebSite ClusterIP order apache-after-ip inf: ClusterIP WebSite property $id="cib-bootstrap-options" \         dc-version="1.1.5-bdd89e69ba545404d02445be1f3d72e6a203ba2f" \         cluster-infrastructure="openais" \         expected-quorum-votes=â€2†\         stonith-enabled="false" \         no-quorum-policy="ignore" rsc_defaults $id="rsc-options" \         resource-stickiness=â€100†Once we’re happy with the changes, we can tell the cluster to start using them and use crm_mon to check everything is functioning. crm(drbd)# cib commit drbd INFO: commited 'drbd' shadow CIB to the cluster crm(drbd)# quit bye [root@pcmk-1 ~]# crm_mon ============ Last updated: Tue Sep  1 09:37:13 2009 Stack: openais Current DC: pcmk-1 - partition with quorum Version: 1.1.5-bdd89e69ba545404d02445be1f3d72e6a203ba2f 2 Nodes configured, 2 expected votes 3 Resources configured. ============ Online: [ pcmk-1 pcmk-2 ] ClusterIP        (ocf::heartbeat:IPaddr):        Started pcmk-1 WebSite (ocf::heartbeat:apache):        Started pcmk-1 Master/Slave Set: WebDataClone Masters: [ pcmk-2 ] Slaves: [ pcmk-1 ] Include details on adding a second DRBD resource Now that DRBD is functioning we can configure a Filesystem resource to use it. In addition to the filesystem’s definition, we also need to tell the cluster where it can be located (only on the DRBD Primary) and when it is allowed to start (after the Primary was promoted). Once again we’ll use the shell’s interactive mode [root@pcmk-1 ~]# crm crm(live)# cib new fs INFO: fs shadow CIB created crm(fs)# configure primitive WebFS ocf:heartbeat:Filesystem \         params device="/dev/drbd/by-res/wwwdata" directory="/var/www/html" fstype="ext4" crm(fs)# configure colocation fs_on_drbd inf: WebFS WebDataClone:Master crm(fs)# configure order WebFS-after-WebData inf: WebDataClone:promote WebFS:start We also need to tell the cluster that Apache needs to run on the same machine as the filesystem and that it must be active before Apache can start. crm(fs)# configure colocation WebSite-with-WebFS inf: WebSite WebFS crm(fs)# configure order WebSite-after-WebFS inf: WebFS WebSite Time to review the updated configuration: [root@pcmk-1 ~]# crm configure show node pcmk-1 node pcmk-2 primitive WebData ocf:linbit:drbd \         params drbd_resource="wwwdata" \         op monitor interval="60s" primitive WebFS ocf:heartbeat:Filesystem \         params device="/dev/drbd/by-res/wwwdata" directory="/var/www/html" fstype="ext4" primitive WebSite ocf:heartbeat:apache \         params configfile="/etc/httpd/conf/httpd.conf" \         op monitor interval="1min" primitive ClusterIP ocf:heartbeat:IPaddr2 \         params ip="192.168.122.101" cidr_netmask="32" \         op monitor interval="30s" ms WebDataClone WebData \         meta master-max="1" master-node-max="1" clone-max="2" clone-node-max="1" notify="true" location prefer-pcmk-1 WebSite 50: pcmk-1 colocation WebSite-with-WebFS inf: WebSite WebFS colocation fs_on_drbd inf: WebFS WebDataClone:Master colocation website-with-ip inf: WebSite ClusterIP order WebFS-after-WebData inf: WebDataClone:promote WebFS:start order WebSite-after-WebFS inf: WebFS WebSite order apache-after-ip inf: ClusterIP WebSite property $id="cib-bootstrap-options" \         dc-version="1.1.5-bdd89e69ba545404d02445be1f3d72e6a203ba2f" \         cluster-infrastructure="openais" \         expected-quorum-votes=â€2†\         stonith-enabled="false" \         no-quorum-policy="ignore" rsc_defaults $id="rsc-options" \         resource-stickiness=â€100†After reviewing the new configuration, we again upload it and watch the cluster put it into effect. crm(fs)# cib commit fs INFO: commited 'fs' shadow CIB to the cluster crm(fs)# quit bye [root@pcmk-1 ~]# crm_mon ============ Last updated: Tue Sep  1 10:08:44 2009 Stack: openais Current DC: pcmk-1 - partition with quorum Version: 1.1.5-bdd89e69ba545404d02445be1f3d72e6a203ba2f 2 Nodes configured, 2 expected votes 4 Resources configured. ============ Online: [ pcmk-1 pcmk-2 ] ClusterIP        (ocf::heartbeat:IPaddr):        Started pcmk-1 WebSite (ocf::heartbeat:apache): Started pcmk-1 Master/Slave Set: WebDataClone         Masters: [ pcmk-1 ]         Slaves: [ pcmk-2 ] WebFS (ocf::heartbeat:Filesystem): Started pcmk-1
Testing Migration We could shut down the active node again, but another way to safely simulate recovery is to put the node into what is called “standby modeâ€. Nodes in this state tell the cluster that they are not allowed to run resources. Any resources found active there will be moved elsewhere. This feature can be particularly useful when updating the resources’ packages. Put the local node into standby mode and observe the cluster move all the resources to the other node. Note also that the node’s status will change to indicate that it can no longer host resources. [root@pcmk-1 ~]# crm node standby [root@pcmk-1 ~]# crm_mon ============ Last updated: Tue Sep  1 10:09:57 2009 Stack: openais Current DC: pcmk-1 - partition with quorum Version: 1.1.5-bdd89e69ba545404d02445be1f3d72e6a203ba2f 2 Nodes configured, 2 expected votes 4 Resources configured. ============ Node pcmk-1: standby Online: [ pcmk-2 ] ClusterIP        (ocf::heartbeat:IPaddr):        Started pcmk-2 WebSite (ocf::heartbeat:apache):        Started pcmk-2 Master/Slave Set: WebDataClone         Masters: [ pcmk-2 ]         Stopped: [ WebData:1 ] WebFS   (ocf::heartbeat:Filesystem):    Started pcmk-2 Once we’ve done everything we needed to on pcmk-1 (in this case nothing, we just wanted to see the resources move), we can allow the node to be a full cluster member again. [root@pcmk-1 ~]# crm node online [root@pcmk-1 ~]# crm_mon ============ Last updated: Tue Sep  1 10:13:25 2009 Stack: openais Current DC: pcmk-1 - partition with quorum Version: 1.1.5-bdd89e69ba545404d02445be1f3d72e6a203ba2f 2 Nodes configured, 2 expected votes 4 Resources configured. ============ Online: [ pcmk-1 pcmk-2 ] ClusterIP        (ocf::heartbeat:IPaddr):        Started pcmk-2 WebSite (ocf::heartbeat:apache):        Started pcmk-2 Master/Slave Set: WebDataClone         Masters: [ pcmk-2 ]         Slaves: [ pcmk-1 ] WebFS   (ocf::heartbeat:Filesystem):    Started pcmk-2 Notice that our resource stickiness settings prevent the services from migrating back to pcmk-1.
diff --git a/extra/resources/SysInfo b/extra/resources/SysInfo index 8f9870bd1a..a2fd4ac126 100644 --- a/extra/resources/SysInfo +++ b/extra/resources/SysInfo @@ -1,388 +1,330 @@ #!/bin/sh # # # SysInfo OCF Resource Agent # It records (in the CIB) various attributes of a node # # Copyright (c) 2004 SUSE LINUX AG, Lars Marowsky-Brée # All Rights Reserved. # # This program is free software; you can redistribute it and/or modify # it under the terms of version 2 of the GNU General Public License as # published by the Free Software Foundation. # # This program is distributed in the hope that it would be useful, but # WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. # # Further, this software is distributed without any warranty that it is # free of the rightful claim of any third person regarding infringement # or the like. Any license provided herein, whether implied or # otherwise, applies only to this software file. Patent licenses, if # any, provided herein do not apply to combinations of this program with # other software, or any other product whatsoever. # # You should have received a copy of the GNU General Public License # along with this program; if not, write the Free Software Foundation, # Inc., 59 Temple Place - Suite 330, Boston MA 02111-1307, USA. # ####################################################################### # Initialization: . ${OCF_ROOT}/resource.d/heartbeat/.ocf-shellfuncs ####################################################################### meta_data() { cat < 1.0 This is a SysInfo Resource Agent. It records (in the CIB) various attributes of a node Sample Linux output: arch: i686 os: Linux-2.4.26-gentoo-r14 free_swap: 1999 cpu_info: Intel(R) Celeron(R) CPU 2.40GHz cpu_speed: 4771.02 cpu_cores: 1 cpu_load: 0.00 ram_total: 513 ram_free: 117 root_free: 2.4 Sample Darwin output: arch: i386 os: Darwin-8.6.2 cpu_info: Intel Core Duo cpu_speed: 2.16 cpu_cores: 2 cpu_load: 0.18 ram_total: 2016 ram_free: 787 root_free: 13 Units: free_swap: Mb ram_*: Mb cpu_speed (Linux): bogomips cpu_speed (Darwin): Ghz *_free: GB (or user-defined: disk_unit) SysInfo resource agent PID file PID file Interval to allow values to stabilize Dampening Delay Filesystems or Paths to be queried for free disk space as a SPACE separated list - e.g "/dev/sda1 /tmp". Results will be written to an attribute with leading slashes removed, and other slashes replaced with underscore, and the word 'free' appended - e.g /dev/sda1 -> dev_sda1_free Note: The root filesystem '/' is always queried to an attribute named 'root_free' List of Filesytems/Paths to query for free disk space content type="string" /> Unit to report disk free space in. Can be one of: B, K, M, G, T, P (case-insensitive) Unit to report disk free space in content type="string" default="G"/> END } ####################################################################### UpdateStat() { name=$1; shift value="$*" - echo -e "$name:\t$value" + printf "%s:\t%s\n" "$name" "$value" ${HA_SBIN_DIR}/attrd_updater ${OCF_RESKEY_delay} -S status -n $name -v "$value" } SysInfoStats() { UpdateStat arch "`uname -m`" UpdateStat os "`uname -s`-`uname -r`" case `uname -s` in "Darwin") mem=`top -l 1 | grep Mem: | awk '{print $10}'` mem_used=`top -l 1 | grep Mem: | awk '{print $8}'` mem=`SysInfo_mem_units $mem` mem_used=`SysInfo_mem_units $mem_used` mem_total=`expr $mem_used + $mem` - cpu_type=`system_profiler SPHardwareDataType | grep "CPU Type:"` - cpu_type=${cpu_type/*: /} - cpu_speed=`system_profiler SPHardwareDataType | grep "CPU Speed:" | awk '{print $3}'` - cpu_cores=`system_profiler SPHardwareDataType | grep "Number Of"` - cpu_cores=${cpu_cores/*: /} + cpu_type=`system_profiler SPHardwareDataType | awk -F': ' '/^CPU Type/ {print $2; exit}'` + cpu_speed=`system_profiler SPHardwareDataType | awk -F': ' '/^CPU Speed/ {print $2; exit}'` + cpu_cores=`system_profiler SPHardwareDataType | awk -F': ' '/^Number Of/ {print $2; exit}'` ;; "Linux") if [ -f /proc/cpuinfo ]; then - cpu_type=`grep "model name" /proc/cpuinfo | head -n 1` - cpu_type=${cpu_type/*: /} - cpu_speed=`grep "bogomips" /proc/cpuinfo | head -n 1` - cpu_speed=${cpu_speed/*: /} + cpu_type=`awk -F': ' '/model name/ {print $2; exit}' /proc/cpuinfo` + cpu_speed=`awk -F': ' '/bogomips/ {print $2; exit}' /proc/cpuinfo` cpu_cores=`grep "^processor" /proc/cpuinfo | wc -l` fi if [ -f /proc/meminfo ]; then # meminfo results are in kB mem=`grep "SwapFree" /proc/meminfo | awk '{print $2"k"}'` if [ ! -z $mem ]; then UpdateStat free_swap `SysInfo_mem_units $mem` fi mem=`grep "Inactive" /proc/meminfo | awk '{print $2"k"}'` mem_total=`grep "MemTotal" /proc/meminfo | awk '{print $2"k"}'` else mem=`top -n 1 | grep Mem: | awk '{print $7}'` fi ;; *) esac if [ x != x"$cpu_type" ]; then UpdateStat cpu_info "$cpu_type" fi if [ x != x"$cpu_speed" ]; then UpdateStat cpu_speed "$cpu_speed" fi if [ x != x"$cpu_cores" ]; then UpdateStat cpu_cores "$cpu_cores" fi loads=`uptime` load15=`echo ${loads} | awk '{print $10}'` UpdateStat cpu_load $load15 if [ ! -z "$mem" ]; then # Massage the memory values UpdateStat ram_total `SysInfo_mem_units $mem_total` UpdateStat ram_free `SysInfo_mem_units $mem` fi # Portability notes: # o tail: explicit "-n" not available in Solaris; instead simplify # 'tail -n ' to the equivalent 'tail -'. for disk in "/" ${OCF_RESKEY_disks}; do unset disk_free disk_label - disk_free=`df -k ${disk} | tail -1 | awk '{print $4}'` + disk_free=`df -h ${disk} | tail -1 | awk '{print $4}'` if [ x != x"$disk_free" ]; then disk_label=`echo $disk | sed -e 's#^/$#root#;s#^/*##;s#/#_#g'` UpdateStat ${disk_label}_free `SysInfo_hdd_units $disk_free` fi done } +SysInfo_megabytes() { + # Size in megabytes + echo $1 | awk '{ n = $0; + sub(/[0-9]+(.[0-9]+)?/, ""); + split(n, a, $0); + n=a[1]; + if ($0 == "G" || $0 == "") { n *= 1024 }; + if (/^kB?/) { n /= 1024 }; + printf "%d\n", n }' # Intentionaly round to an integer +} + SysInfo_mem_units() { mem=$1 if [ -z $1 ]; then return fi - memlen=`expr ${#mem} - 1` - memlen_alt=`expr ${#mem} - 2` - if [ ${mem:$memlen:1} = "G" ]; then - mem="${mem:0:$memlen}" - if [ $mem != ${mem/./} ]; then - mem_before=${mem/.*/} - mem_after=${mem/*./} - mem=$[mem_before*1024] - if [ ${#mem_after} = 0 ]; then - : - elif [ ${#mem_after} = 1 ]; then - mem=$[mem+100*$mem_after] - elif [ ${#mem_after} = 2 ]; then - mem=$[mem+10*$mem_after] - elif [ ${#mem_after} = 3 ]; then - mem=$[mem+$mem_after] - else - mem_after=${mem_after:0:3} - mem=$[mem+$mem_after] - fi - fi - elif [ ${mem:$memlen:1} = "M" ]; then - mem=${mem/.*/} - mem="${mem:0:$memlen}" - elif [ ${mem:$memlen:1} = "k" ]; then - mem="${mem:0:$memlen}" - mem=${mem/.*/} - mem=`expr $mem / 1024` - elif [ ${mem:$memlen_alt:2} = "kB" ]; then - mem="${mem:0:$memlen_alt}" - mem=${mem/.*/} - mem=`expr $mem / 1024` - elif [ ${mem:$memlen_alt:2} = "Mb" ]; then - mem="${mem:0:$memlen_alt}" - mem=${mem/.*/} - elif [ ${mem:$memlen_alt:2} = "MB" ]; then - mem="${mem:0:$memlen_alt}" - mem=${mem/.*/} - fi - + mem=$(SysInfo_megabytes "$1") # Round to the next multiple of 50 - memlen=`expr ${#mem} - 2` - mem_round="${mem:$memlen:2}" - if [ x$mem_round = x ]; then - : - elif [ $mem_round = "00" ]; then - : - elif [ $mem_round -lt "50" ]; then - mem=$[mem+50] - mem=$[mem-$mem_round] - - else - mem=$[mem+100] - mem=$[mem-$mem_round] + r=$(($mem % 50)) + if [ $r != 0 ]; then + mem=$(($mem + 50 - $r)) fi + echo $mem } SysInfo_hdd_units() { - disk_size=$1 - disk_unit=${OCF_RESKEY_disk_unit} - if [ -z $disk_unit ]; then - disk_unit="G" - fi - - case $disk_unit in - [Pp]) - disk_size=$((disk_size/1024/1024/1024/1024)) - ;; - [Tt]) - disk_size=$((disk_size/1024/1024/1024)) - ;; - [Gg]) - disk_size=$((disk_size/1024/1024)) - ;; - [Mm]) - disk_size=$((disk_size/1024)) - ;; - [Kk]) - #Already in KB - ;; - [Bb]) - disk_size=$((disk_size*1024)) - ;; - *) - ocf_log err "disk_unit set to invalid unit" - exit $OCF_ERR_ARGS + # Defauts to size in gigabytes + + case $OCF_RESKEY_disk_unit in + [Pp]) echo $(($(SysInfo_megabytes "$1") / 1024 / 1024 / 1024));; + [Tt]) echo $(($(SysInfo_megabytes "$1") / 1024 / 1024));; + [Gg]) echo $(($(SysInfo_megabytes "$1") / 1024));; + [Mm]) echo SysInfo_megabytes "$1";; + [Kk]) echo $(($(SysInfo_megabytes "$1") * 1024));; + [Bb]) echo $(($(SysInfo_megabytes "$1") * 1024 * 1024));; + *) + ocf_log err "Invalid value for disk_unit: $OCF_RESKEY_disk_unit" + echo $(($(SysInfo_megabytes "$1") / 1024));; esac - - echo $disk_size } SysInfo_usage() { cat < $OCF_RESKEY_pidfile SysInfoStats exit $OCF_SUCCESS } SysInfo_stop() { rm $OCF_RESKEY_pidfile exit $OCF_SUCCESS } SysInfo_monitor() { if [ -f $OCF_RESKEY_pidfile ]; then clone=`cat $OCF_RESKEY_pidfile` fi if [ x$clone = x ]; then rm $OCF_RESKEY_pidfile exit $OCF_NOT_RUNNING elif [ $clone = $OCF_RESKEY_clone ]; then SysInfoStats exit $OCF_SUCCESS elif [ x$OCF_RESKEY_CRM_meta_globally_unique = xtrue -o x$OCF_RESKEY_CRM_meta_globally_unique = xTrue -o x$OCF_RESKEY_CRM_meta_globally_unique = xyes -o x$OCF_RESKEY_CRM_meta_globally_unique = xYes ]; then SysInfoStats exit $OCF_SUCCESS fi exit $OCF_NOT_RUNNING } SysInfo_validate() { return $OCF_SUCCESS } if [ $# -ne 1 ]; then SysInfo_usage exit $OCF_ERR_ARGS fi : ${OCF_RESKEY_pidfile:="$HA_VARRUN/SysInfo-${OCF_RESOURCE_INSTANCE}"} +: ${OCF_RESKEY_disk_unit:="G"} : ${OCF_RESKEY_clone:="0"} if [ x != x${OCF_RESKEY_delay} ]; then OCF_RESKEY_delay="-d ${OCF_RESKEY_delay}" fi case $__OCF_ACTION in meta-data) meta_data exit $OCF_SUCCESS ;; start) SysInfo_start ;; stop) SysInfo_stop ;; monitor) SysInfo_monitor ;; validate-all) SysInfo_validate ;; usage|help) SysInfo_usage exit $OCF_SUCCESS ;; *) SysInfo_usage exit $OCF_ERR_UNIMPLEMENTED ;; esac exit $? diff --git a/lib/common/ais.c b/lib/common/ais.c index 15db59bc75..90559dc921 100644 --- a/lib/common/ais.c +++ b/lib/common/ais.c @@ -1,1225 +1,1229 @@ /* * Copyright (C) 2004 Andrew Beekhof * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ #include #include #include #include #include #include #include "stack.h" #ifdef SUPPORT_COROSYNC # include #endif #ifdef SUPPORT_CMAN # include cman_handle_t pcmk_cman_handle = NULL; #endif #ifdef SUPPORT_CS_QUORUM # include # include # include # include # include quorum_handle_t pcmk_quorum_handle = 0; cpg_handle_t pcmk_cpg_handle = 0; struct cpg_name pcmk_cpg_group = { .length = 0, .value[0] = 0, }; #endif static char *pcmk_uname = NULL; static int pcmk_uname_len = 0; static uint32_t pcmk_nodeid = 0; #define cs_repeat(counter, max, code) do { \ code; \ if(rc == CS_ERR_TRY_AGAIN) { \ counter++; \ crm_debug("Retrying operation after %ds", counter); \ sleep(counter); \ } \ } while(rc == CS_ERR_TRY_AGAIN && counter < max) enum crm_ais_msg_types text2msg_type(const char *text) { int type = crm_msg_none; CRM_CHECK(text != NULL, return type); if(safe_str_eq(text, "ais")) { type = crm_msg_ais; } else if(safe_str_eq(text, "crm_plugin")) { type = crm_msg_ais; } else if(safe_str_eq(text, CRM_SYSTEM_CIB)) { type = crm_msg_cib; } else if(safe_str_eq(text, CRM_SYSTEM_CRMD)) { type = crm_msg_crmd; } else if(safe_str_eq(text, CRM_SYSTEM_DC)) { type = crm_msg_crmd; } else if(safe_str_eq(text, CRM_SYSTEM_TENGINE)) { type = crm_msg_te; } else if(safe_str_eq(text, CRM_SYSTEM_PENGINE)) { type = crm_msg_pe; } else if(safe_str_eq(text, CRM_SYSTEM_LRMD)) { type = crm_msg_lrmd; } else if(safe_str_eq(text, CRM_SYSTEM_STONITHD)) { type = crm_msg_stonithd; } else if(safe_str_eq(text, "stonith-ng")) { type = crm_msg_stonith_ng; } else if(safe_str_eq(text, "attrd")) { type = crm_msg_attrd; } else { /* This will normally be a transient client rather than * a cluster daemon. Set the type to the pid of the client */ int scan_rc = sscanf(text, "%d", &type); if(scan_rc != 1) { /* Ensure its sane */ type = crm_msg_none; } } return type; } char *get_ais_data(const AIS_Message *msg) { int rc = BZ_OK; char *uncompressed = NULL; unsigned int new_size = msg->size + 1; if(msg->is_compressed == FALSE) { crm_debug_2("Returning uncompressed message data"); uncompressed = strdup(msg->data); } else { crm_debug_2("Decompressing message data"); crm_malloc0(uncompressed, new_size); rc = BZ2_bzBuffToBuffDecompress( uncompressed, &new_size, (char*)msg->data, msg->compressed_size, 1, 0); CRM_ASSERT(rc == BZ_OK); CRM_ASSERT(new_size == msg->size); } return uncompressed; } #if SUPPORT_COROSYNC int ais_fd_sync = -1; int ais_fd_async = -1; /* never send messages via this channel */ void *ais_ipc_ctx = NULL; hdb_handle_t ais_ipc_handle = 0; GFDSource *ais_source = NULL; GFDSource *ais_source_sync = NULL; GFDSource *cman_source = NULL; GFDSource *cpg_source = NULL; GFDSource *quorumd_source = NULL; static char *ais_cluster_name = NULL; gboolean get_ais_nodeid(uint32_t *id, char **uname) { struct iovec iov; int retries = 0; int rc = CS_OK; coroipc_response_header_t header; struct crm_ais_nodeid_resp_s answer; header.error = CS_OK; header.id = crm_class_nodeid; header.size = sizeof(coroipc_response_header_t); CRM_CHECK(id != NULL, return FALSE); CRM_CHECK(uname != NULL, return FALSE); iov.iov_base = &header; iov.iov_len = header.size; retry: errno = 0; rc = coroipcc_msg_send_reply_receive( ais_ipc_handle, &iov, 1, &answer, sizeof (answer)); if(rc == CS_OK) { CRM_CHECK(answer.header.size == sizeof (struct crm_ais_nodeid_resp_s), crm_err("Odd message: id=%d, size=%d, error=%d", answer.header.id, answer.header.size, answer.header.error)); CRM_CHECK(answer.header.id == crm_class_nodeid, crm_err("Bad response id: %d", answer.header.id)); } if(rc == CS_ERR_TRY_AGAIN && retries < 20) { retries++; crm_info("Peer overloaded: Re-sending message (Attempt %d of 20)", retries); sleep(retries); /* Proportional back off */ goto retry; } if(rc != CS_OK) { crm_err("Sending nodeid request: FAILED (rc=%d): %s", rc, ais_error2text(rc)); return FALSE; } else if(answer.header.error != CS_OK) { crm_err("Bad response from peer: (rc=%d): %s", rc, ais_error2text(rc)); return FALSE; } crm_info("Server details: id=%u uname=%s cname=%s", answer.id, answer.uname, answer.cname); *id = answer.id; *uname = crm_strdup(answer.uname); ais_cluster_name = crm_strdup(answer.cname); return TRUE; } gboolean crm_get_cluster_name(char **cname) { CRM_CHECK(cname != NULL, return FALSE); if(ais_cluster_name) { *cname = crm_strdup(ais_cluster_name); return TRUE; } return FALSE; } gboolean send_ais_text(int class, const char *data, gboolean local, const char *node, enum crm_ais_msg_types dest) { static int msg_id = 0; static int local_pid = 0; enum cluster_type_e cluster_type = get_cluster_type(); int retries = 0; int rc = CS_OK; int buf_len = sizeof(coroipc_response_header_t); char *buf = NULL; struct iovec iov; const char *transport = "pcmk"; coroipc_response_header_t *header = NULL; AIS_Message *ais_msg = NULL; enum crm_ais_msg_types sender = text2msg_type(crm_system_name); /* There are only 6 handlers registered to crm_lib_service in plugin.c */ CRM_CHECK(class < 6, crm_err("Invalid message class: %d", class); return FALSE); if(data == NULL) { data = ""; } if(local_pid == 0) { local_pid = getpid(); } if(sender == crm_msg_none) { sender = local_pid; } crm_malloc0(ais_msg, sizeof(AIS_Message)); ais_msg->id = msg_id++; ais_msg->header.id = class; ais_msg->header.error = CS_OK; ais_msg->host.type = dest; ais_msg->host.local = local; if(node) { ais_msg->host.size = strlen(node); memset(ais_msg->host.uname, 0, MAX_NAME); memcpy(ais_msg->host.uname, node, ais_msg->host.size); ais_msg->host.id = 0; } else { ais_msg->host.size = 0; memset(ais_msg->host.uname, 0, MAX_NAME); ais_msg->host.id = 0; } ais_msg->sender.id = 0; ais_msg->sender.type = sender; ais_msg->sender.pid = local_pid; ais_msg->sender.size = pcmk_uname_len; memset(ais_msg->sender.uname, 0, MAX_NAME); memcpy(ais_msg->sender.uname, pcmk_uname, ais_msg->sender.size); ais_msg->size = 1 + strlen(data); if(ais_msg->size < CRM_BZ2_THRESHOLD) { failback: crm_realloc(ais_msg, sizeof(AIS_Message) + ais_msg->size); memcpy(ais_msg->data, data, ais_msg->size); } else { char *compressed = NULL; char *uncompressed = crm_strdup(data); unsigned int len = (ais_msg->size * 1.1) + 600; /* recomended size */ crm_debug_5("Compressing message payload"); crm_malloc(compressed, len); rc = BZ2_bzBuffToBuffCompress( compressed, &len, uncompressed, ais_msg->size, CRM_BZ2_BLOCKS, 0, CRM_BZ2_WORK); crm_free(uncompressed); if(rc != BZ_OK) { crm_err("Compression failed: %d", rc); crm_free(compressed); goto failback; } crm_realloc(ais_msg, sizeof(AIS_Message) + len + 1); memcpy(ais_msg->data, compressed, len); ais_msg->data[len] = 0; crm_free(compressed); ais_msg->is_compressed = TRUE; ais_msg->compressed_size = len; crm_debug_2("Compression details: %d -> %d", ais_msg->size, ais_data_len(ais_msg)); } ais_msg->header.size = sizeof(AIS_Message) + ais_data_len(ais_msg); crm_debug_3("Sending%s message %d to %s.%s (data=%d, total=%d)", ais_msg->is_compressed?" compressed":"", ais_msg->id, ais_dest(&(ais_msg->host)), msg_type2text(dest), ais_data_len(ais_msg), ais_msg->header.size); iov.iov_base = ais_msg; iov.iov_len = ais_msg->header.size; crm_realloc(buf, buf_len); do { if(rc == CS_ERR_TRY_AGAIN) { retries++; crm_info("Peer overloaded or membership in flux:" " Re-sending message (Attempt %d of 20)", retries); sleep(retries); /* Proportional back off */ } errno = 0; switch(cluster_type) { case pcmk_cluster_classic_ais: rc = coroipcc_msg_send_reply_receive(ais_ipc_handle, &iov, 1, buf, buf_len); header = (coroipc_response_header_t *)buf; if(rc == CS_OK) { CRM_CHECK(header->size == sizeof (coroipc_response_header_t), crm_err("Odd message: id=%d, size=%d, class=%d, error=%d", header->id, header->size, class, header->error)); CRM_ASSERT(buf_len >= header->size); CRM_CHECK(header->id == CRM_MESSAGE_IPC_ACK, crm_err("Bad response id (%d) for request (%d)", header->id, ais_msg->header.id)); CRM_CHECK(header->error == CS_OK, rc = header->error); } break; case pcmk_cluster_corosync: case pcmk_cluster_cman: transport = "cpg"; CRM_CHECK(dest != crm_msg_ais, rc = CS_ERR_MESSAGE_ERROR; goto bail); rc = cpg_mcast_joined(pcmk_cpg_handle, CPG_TYPE_AGREED, &iov, 1); if(rc == CS_ERR_TRY_AGAIN) { cpg_flow_control_state_t fc_state = CPG_FLOW_CONTROL_DISABLED; int rc2 = cpg_flow_control_state_get (pcmk_cpg_handle, &fc_state); if (rc2 == CS_OK && fc_state == CPG_FLOW_CONTROL_ENABLED) { crm_warn("Connection overloaded, cannot send messages"); goto bail; } else if (rc2 != CS_OK) { crm_warn("Could not determin the connection state: %s (%d)", ais_error2text(rc2), rc2); goto bail; } } break; case pcmk_cluster_unknown: case pcmk_cluster_invalid: case pcmk_cluster_heartbeat: CRM_ASSERT(is_openais_cluster()); break; } } while (rc == CS_ERR_TRY_AGAIN && retries < 20); bail: if(rc != CS_OK) { crm_perror(LOG_ERR,"Sending message %d via %s: FAILED (rc=%d): %s", ais_msg->id, transport, rc, ais_error2text(rc)); } else { crm_debug_4("Message %d: sent", ais_msg->id); } crm_free(buf); crm_free(ais_msg); return (rc == CS_OK); } gboolean send_ais_message(xmlNode *msg, gboolean local, const char *node, enum crm_ais_msg_types dest) { gboolean rc = TRUE; char *data = NULL; if(is_classic_ais_cluster()) { if(ais_fd_async < 0 || ais_source == NULL) { crm_err("Not connected to AIS: %d %p", ais_fd_async, ais_source); return FALSE; } } data = dump_xml_unformatted(msg); rc = send_ais_text(0, data, local, node, dest); crm_free(data); return rc; } void terminate_ais_connection(void) { crm_notice("Disconnecting from AIS"); /* G_main_del_fd(ais_source); */ /* G_main_del_fd(ais_source_sync); */ -#ifdef SUPPORT_CMAN - if(is_cman_cluster()) { - cman_stop_notification(pcmk_cman_handle); - cman_finish(pcmk_cman_handle); + if(is_classic_ais_cluster() == FALSE) { + coroipcc_service_disconnect(ais_ipc_handle); + + } else { + cpg_leave(pcmk_cpg_handle, &pcmk_cpg_group); } -#endif if(is_corosync_cluster()) { quorum_finalize(pcmk_quorum_handle); } - if(is_classic_ais_cluster() == FALSE) { - coroipcc_service_disconnect(ais_ipc_handle); - - } else { - cpg_leave(pcmk_cpg_handle, &pcmk_cpg_group); +#ifdef SUPPORT_CMAN + if(is_cman_cluster()) { + cman_stop_notification(pcmk_cman_handle); + cman_finish(pcmk_cman_handle); } +#endif } int ais_membership_timer = 0; gboolean ais_membership_force = FALSE; static gboolean ais_dispatch_message( AIS_Message *msg, gboolean (*dispatch)(AIS_Message*,char*,int)) { char *data = NULL; char *uncompressed = NULL; xmlNode *xml = NULL; CRM_ASSERT(msg != NULL); crm_debug_3("Got new%s message (size=%d, %d, %d)", msg->is_compressed?" compressed":"", ais_data_len(msg), msg->size, msg->compressed_size); data = msg->data; if(msg->is_compressed && msg->size > 0) { int rc = BZ_OK; unsigned int new_size = msg->size + 1; if(check_message_sanity(msg, NULL) == FALSE) { goto badmsg; } crm_debug_5("Decompressing message data"); crm_malloc0(uncompressed, new_size); rc = BZ2_bzBuffToBuffDecompress( uncompressed, &new_size, data, msg->compressed_size, 1, 0); if(rc != BZ_OK) { crm_err("Decompression failed: %d", rc); goto badmsg; } CRM_ASSERT(rc == BZ_OK); CRM_ASSERT(new_size == msg->size); data = uncompressed; } else if(check_message_sanity(msg, data) == FALSE) { goto badmsg; } else if(safe_str_eq("identify", data)) { int pid = getpid(); char *pid_s = crm_itoa(pid); send_ais_text(0, pid_s, TRUE, NULL, crm_msg_ais); crm_free(pid_s); goto done; } if(msg->header.id != crm_class_members) { crm_update_peer(msg->sender.id, 0,0,0,0, msg->sender.uname, msg->sender.uname, NULL, NULL); } if(msg->header.id == crm_class_rmpeer) { uint32_t id = crm_int_helper(data, NULL); crm_info("Removing peer %s/%u", data, id); reap_crm_member(id); goto done; } else if(msg->header.id == crm_class_members || msg->header.id == crm_class_quorum) { xml = string2xml(data); if(xml == NULL) { crm_err("Invalid membership update: %s", data); goto badmsg; } if(is_classic_ais_cluster() == FALSE) { xmlNode *node = NULL; for(node = __xml_first_child(xml); node != NULL; node = __xml_next(node)) { crm_update_cman_node(node, crm_peer_seq); } } else { xmlNode *node = NULL; const char *value = NULL; gboolean quorate = FALSE; value = crm_element_value(xml, "quorate"); CRM_CHECK(value != NULL, crm_log_xml_err(xml, "No quorum value:"); goto badmsg); if(crm_is_true(value)) { quorate = TRUE; } value = crm_element_value(xml, "id"); CRM_CHECK(value != NULL, crm_log_xml_err(xml, "No membership id"); goto badmsg); crm_peer_seq = crm_int_helper(value, NULL); if(quorate != crm_have_quorum) { crm_notice("Membership %s: quorum %s", value, quorate?"acquired":"lost"); crm_have_quorum = quorate; } else { crm_info("Membership %s: quorum %s", value, quorate?"retained":"still lost"); } for(node = __xml_first_child(xml); node != NULL; node = __xml_next(node)) { crm_update_ais_node(node, crm_peer_seq); } } } if(dispatch != NULL) { dispatch(msg, data, 0); } done: crm_free(uncompressed); free_xml(xml); return TRUE; badmsg: crm_err("Invalid message (id=%d, dest=%s:%s, from=%s:%s.%d):" " min=%d, total=%d, size=%d, bz2_size=%d", msg->id, ais_dest(&(msg->host)), msg_type2text(msg->host.type), ais_dest(&(msg->sender)), msg_type2text(msg->sender.type), msg->sender.pid, (int)sizeof(AIS_Message), msg->header.size, msg->size, msg->compressed_size); goto done; } gboolean ais_dispatch(int sender, gpointer user_data) { int rc = CS_OK; char *buffer = NULL; gboolean good = TRUE; gboolean (*dispatch)(AIS_Message*,char*,int) = user_data; - rc = coroipcc_dispatch_get (ais_ipc_handle, (void**)&buffer, 0); + do { + rc = coroipcc_dispatch_get (ais_ipc_handle, (void**)&buffer, 0); - if (rc == 0 || buffer == NULL) { - /* Zero is a legal "no message afterall" value */ - return TRUE; + if (rc == 0 || buffer == NULL) { + /* Zero is a legal "no message afterall" value */ + return TRUE; + + } else if (rc != CS_OK) { + crm_perror(LOG_ERR,"Receiving message body failed: (%d) %s", rc, ais_error2text(rc)); + goto bail; + } - } else if (rc != CS_OK) { - crm_perror(LOG_ERR,"Receiving message body failed: (%d) %s", rc, ais_error2text(rc)); - goto bail; - } + good = ais_dispatch_message((AIS_Message*)buffer, dispatch); + coroipcc_dispatch_put (ais_ipc_handle); - good = ais_dispatch_message((AIS_Message*)buffer, dispatch); - coroipcc_dispatch_put (ais_ipc_handle); + } while(good); + return good; bail: crm_err("AIS connection failed"); return FALSE; } static void ais_destroy(gpointer user_data) { crm_err("AIS connection terminated"); ais_fd_sync = -1; exit(1); } static gboolean pcmk_proc_dispatch(IPC_Channel *ch, gpointer user_data) { xmlNode *msg = NULL; gboolean stay_connected = TRUE; while(IPC_ISRCONN(ch)) { if(ch->ops->is_message_pending(ch) == 0) { break; } msg = xmlfromIPC(ch, MAX_IPC_DELAY); if(msg) { xmlNode *node = NULL; for(node = __xml_first_child(msg); node != NULL; node = __xml_next(node)) { int id = 0; int children = 0; const char *uname = crm_element_value(node, "uname"); crm_element_value_int(node, "processes", &children); crm_update_peer(id, 0, 0, 0, children, NULL, uname, NULL, NULL); } free_xml(msg); } if(ch->ch_status != IPC_CONNECT) { break; } } if (ch->ch_status != IPC_CONNECT) { stay_connected = FALSE; } return stay_connected; } #ifdef SUPPORT_CMAN static gboolean pcmk_cman_dispatch(int sender, gpointer user_data) { - int rc = cman_dispatch(pcmk_cman_handle, CMAN_DISPATCH_ONE); + int rc = cman_dispatch(pcmk_cman_handle, CMAN_DISPATCH_ALL); if(rc < 0) { crm_err("Connection to cman failed: %d", rc); return FALSE; } return TRUE; } #define MAX_NODES 256 static void cman_event_callback(cman_handle_t handle, void *privdata, int reason, int arg) { int rc = 0, lpc = 0, node_count = 0; cman_cluster_t cluster; static cman_node_t cman_nodes[MAX_NODES]; gboolean (*dispatch)(unsigned long long, gboolean) = privdata; switch (reason) { case CMAN_REASON_STATECHANGE: memset(&cluster, 0, sizeof(cluster)); rc = cman_get_cluster(pcmk_cman_handle, &cluster); if (rc < 0) { crm_err("Couldn't query cman cluster details: %d %d", rc, errno); return; } crm_peer_seq = cluster.ci_generation; if(arg != crm_have_quorum) { crm_notice("Membership %llu: quorum %s", crm_peer_seq, arg?"acquired":"lost"); crm_have_quorum = arg; } else { crm_info("Membership %llu: quorum %s", crm_peer_seq, arg?"retained":"still lost"); } rc = cman_get_nodes(pcmk_cman_handle, MAX_NODES, &node_count, cman_nodes); if (rc < 0) { crm_err("Couldn't query cman node list: %d %d", rc, errno); return; } for (lpc = 0; lpc < node_count; lpc++) { if (cman_nodes[lpc].cn_nodeid == 0) { /* Never allow node ID 0 to be considered a member #315711 */ cman_nodes[lpc].cn_member = 0; } crm_update_peer(cman_nodes[lpc].cn_nodeid, cman_nodes[lpc].cn_incarnation, cman_nodes[lpc].cn_member?crm_peer_seq:0, 0, 0, cman_nodes[lpc].cn_name, cman_nodes[lpc].cn_name, NULL, cman_nodes[lpc].cn_member?CRM_NODE_MEMBER:CRM_NODE_LOST); } if(dispatch) { dispatch(crm_peer_seq, crm_have_quorum); } break; case CMAN_REASON_TRY_SHUTDOWN: /* Always reply with a negative - pacemaker needs to be stopped first */ crm_info("CMAN wants to shut down: %s", arg?"forced":"optional"); cman_replyto_shutdown(pcmk_cman_handle, 0); break; case CMAN_REASON_CONFIG_UPDATE: /* Ignore */ break; } } #endif gboolean init_cman_connection( gboolean (*dispatch)(unsigned long long, gboolean), void (*destroy)(gpointer)) { #ifdef SUPPORT_CMAN int rc = -1, fd = -1; cman_cluster_t cluster; crm_info("Configuring Pacemaker to obtain quorum from cman"); memset(&cluster, 0, sizeof(cluster)); pcmk_cman_handle = cman_init(dispatch); if(pcmk_cman_handle == NULL || cman_is_active(pcmk_cman_handle) == FALSE) { crm_err("Couldn't connect to cman"); goto cman_bail; } rc = cman_get_cluster(pcmk_cman_handle, &cluster); if (rc < 0) { crm_err("Couldn't query cman cluster details: %d %d", rc, errno); goto cman_bail; } ais_cluster_name = crm_strdup(cluster.ci_name); rc = cman_start_notification(pcmk_cman_handle, cman_event_callback); if (rc < 0) { crm_err("Couldn't register for cman notifications: %d %d", rc, errno); goto cman_bail; } /* Get the current membership state */ cman_event_callback(pcmk_cman_handle, dispatch, CMAN_REASON_STATECHANGE, cman_is_quorate(pcmk_cman_handle)); fd = cman_get_fd(pcmk_cman_handle); crm_debug("Adding fd=%d to mainloop", fd); cman_source = G_main_add_fd( G_PRIORITY_HIGH, fd, FALSE, pcmk_cman_dispatch, dispatch, destroy); cman_bail: if (rc < 0) { cman_finish(pcmk_cman_handle); return FALSE; } #else crm_err("cman qorum is not supported in this build"); exit(100); #endif return TRUE; } #ifdef SUPPORT_CS_QUORUM gboolean (*pcmk_cpg_dispatch_fn)(AIS_Message*,char*,int) = NULL; static gboolean pcmk_cpg_dispatch(int sender, gpointer user_data) { int rc = 0; pcmk_cpg_dispatch_fn = user_data; rc = cpg_dispatch(pcmk_cpg_handle, CS_DISPATCH_ALL); if(rc != CS_OK) { crm_err("Connection to the CPG API failed: %d", rc); return FALSE; } return TRUE; } static void pcmk_cpg_deliver ( cpg_handle_t handle, const struct cpg_name *groupName, uint32_t nodeid, uint32_t pid, void *msg, size_t msg_len) { AIS_Message *ais_msg = (AIS_Message*)msg; if(ais_msg->sender.id > 0 && ais_msg->sender.id != nodeid) { crm_err("Nodeid mismatch from %d.%d: claimed nodeid=%u", nodeid, pid, ais_msg->sender.id); return; } else if(ais_msg->host.size != 0 && safe_str_neq(ais_msg->host.uname, pcmk_uname)) { /* Not for us */ return; } ais_msg->sender.id = nodeid; if(ais_msg->sender.size == 0) { crm_node_t *peer = crm_get_peer(nodeid, NULL); if(peer == NULL) { crm_err("Peer with nodeid=%u is unknown", nodeid); } else if(peer->uname == NULL) { crm_err("No uname for peer with nodeid=%u", nodeid); } else { crm_notice("Fixing uname for peer with nodeid=%u", nodeid); ais_msg->sender.size = strlen(peer->uname); memset(ais_msg->sender.uname, 0, MAX_NAME); memcpy(ais_msg->sender.uname, peer->uname, ais_msg->sender.size); } } ais_dispatch_message(ais_msg, pcmk_cpg_dispatch_fn); } static void pcmk_cpg_membership( cpg_handle_t handle, const struct cpg_name *groupName, const struct cpg_address *member_list, size_t member_list_entries, const struct cpg_address *left_list, size_t left_list_entries, const struct cpg_address *joined_list, size_t joined_list_entries) { /* Don't care about CPG membership */ } static gboolean pcmk_quorum_dispatch(int sender, gpointer user_data) { int rc = 0; rc = quorum_dispatch(pcmk_quorum_handle, CS_DISPATCH_ALL); if(rc < 0) { crm_err("Connection to the Quorum API failed: %d", rc); return FALSE; } return TRUE; } static void pcmk_quorum_notification( quorum_handle_t handle, uint32_t quorate, uint64_t ring_id, uint32_t view_list_entries, uint32_t *view_list) { int i; if(quorate != crm_have_quorum) { crm_notice("Membership "U64T": quorum %s (%lu)", ring_id, quorate?"acquired":"lost", (long unsigned int)view_list_entries); crm_have_quorum = quorate; } else { crm_info("Membership "U64T": quorum %s (%lu)", ring_id, quorate?"retained":"still lost", (long unsigned int)view_list_entries); } for (i=0; ihost.type; int tmp_size = msg->header.size - sizeof(AIS_Message); if(sane && msg->header.size == 0) { crm_warn("Message with no size"); sane = FALSE; } if(sane && msg->header.error != CS_OK) { crm_warn("Message header contains an error: %d", msg->header.error); sane = FALSE; } if(sane && ais_data_len(msg) != tmp_size) { crm_warn("Message payload size is incorrect: expected %d, got %d", ais_data_len(msg), tmp_size); sane = TRUE; } if(sane && ais_data_len(msg) == 0) { crm_warn("Message with no payload"); sane = FALSE; } if(sane && data && msg->is_compressed == FALSE) { int str_size = strlen(data) + 1; if(ais_data_len(msg) != str_size) { int lpc = 0; crm_warn("Message payload is corrupted: expected %d bytes, got %d", ais_data_len(msg), str_size); sane = FALSE; for(lpc = (str_size - 10); lpc < msg->size; lpc++) { if(lpc < 0) { lpc = 0; } crm_debug("bad_data[%d]: %d / '%c'", lpc, data[lpc], data[lpc]); } } } if(sane == FALSE) { crm_err("Invalid message %d: (dest=%s:%s, from=%s:%s.%d, compressed=%d, size=%d, total=%d)", msg->id, ais_dest(&(msg->host)), msg_type2text(dest), ais_dest(&(msg->sender)), msg_type2text(msg->sender.type), msg->sender.pid, msg->is_compressed, ais_data_len(msg), msg->header.size); } else if(repaired) { crm_err("Repaired message %d: (dest=%s:%s, from=%s:%s.%d, compressed=%d, size=%d, total=%d)", msg->id, ais_dest(&(msg->host)), msg_type2text(dest), ais_dest(&(msg->sender)), msg_type2text(msg->sender.type), msg->sender.pid, msg->is_compressed, ais_data_len(msg), msg->header.size); } else { crm_debug_3("Verfied message %d: (dest=%s:%s, from=%s:%s.%d, compressed=%d, size=%d, total=%d)", msg->id, ais_dest(&(msg->host)), msg_type2text(dest), ais_dest(&(msg->sender)), msg_type2text(msg->sender.type), msg->sender.pid, msg->is_compressed, ais_data_len(msg), msg->header.size); } return sane; } #endif diff --git a/lib/common/utils.c b/lib/common/utils.c index 2f6332fdd7..cc8d3da248 100644 --- a/lib/common/utils.c +++ b/lib/common/utils.c @@ -1,2654 +1,2657 @@ /* * Copyright (C) 2004 Andrew Beekhof * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ #include #ifndef _GNU_SOURCE # define _GNU_SOURCE #endif #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #if HAVE_HB_CONFIG_H #include /* for HB_COREDIR */ #endif #if HAVE_GLUE_CONFIG_H #include /* for HB_COREDIR */ #endif #ifndef MAXLINE # define MAXLINE 512 #endif #ifdef HAVE_GETOPT_H # include #endif CRM_TRACE_INIT_DATA(common); static uint ref_counter = 0; unsigned int crm_log_level = LOG_INFO; gboolean crm_config_error = FALSE; gboolean crm_config_warning = FALSE; const char *crm_system_name = "unknown"; int node_score_red = 0; int node_score_green = 0; int node_score_yellow = 0; int node_score_infinity = INFINITY; void crm_set_env_options(void); gboolean check_time(const char *value) { if(crm_get_msec(value) < 5000) { return FALSE; } return TRUE; } gboolean check_timer(const char *value) { if(crm_get_msec(value) < 0) { return FALSE; } return TRUE; } gboolean check_boolean(const char *value) { int tmp = FALSE; if(crm_str_to_boolean(value, &tmp) != 1) { return FALSE; } return TRUE; } gboolean check_number(const char *value) { errno = 0; if(value == NULL) { return FALSE; } else if(safe_str_eq(value, MINUS_INFINITY_S)) { } else if(safe_str_eq(value, INFINITY_S)) { } else { crm_int_helper(value, NULL); } if(errno != 0) { return FALSE; } return TRUE; } int char2score(const char *score) { int score_f = 0; if(score == NULL) { } else if(safe_str_eq(score, MINUS_INFINITY_S)) { score_f = -node_score_infinity; } else if(safe_str_eq(score, INFINITY_S)) { score_f = node_score_infinity; } else if(safe_str_eq(score, "+"INFINITY_S)) { score_f = node_score_infinity; } else if(safe_str_eq(score, "red")) { score_f = node_score_red; } else if(safe_str_eq(score, "yellow")) { score_f = node_score_yellow; } else if(safe_str_eq(score, "green")) { score_f = node_score_green; } else { score_f = crm_parse_int(score, NULL); if(score_f > 0 && score_f > node_score_infinity) { score_f = node_score_infinity; } else if(score_f < 0 && score_f < -node_score_infinity) { score_f = -node_score_infinity; } } return score_f; } char * score2char(int score) { if(score >= node_score_infinity) { return crm_strdup(INFINITY_S); } else if(score <= -node_score_infinity) { return crm_strdup("-"INFINITY_S); } return crm_itoa(score); } const char * cluster_option(GHashTable* options, gboolean(*validate)(const char*), const char *name, const char *old_name, const char *def_value) { const char *value = NULL; CRM_ASSERT(name != NULL); if(options != NULL) { value = g_hash_table_lookup(options, name); } if(value == NULL && old_name && options != NULL) { value = g_hash_table_lookup(options, old_name); if(value != NULL) { crm_config_warn("Using deprecated name '%s' for" " cluster option '%s'", old_name, name); g_hash_table_insert( options, crm_strdup(name), crm_strdup(value)); value = g_hash_table_lookup(options, old_name); } } if(value == NULL) { crm_debug_2("Using default value '%s' for cluster option '%s'", def_value, name); if(options == NULL) { return def_value; } g_hash_table_insert( options, crm_strdup(name), crm_strdup(def_value)); value = g_hash_table_lookup(options, name); } if(validate && validate(value) == FALSE) { crm_config_err("Value '%s' for cluster option '%s' is invalid." " Defaulting to %s", value, name, def_value); g_hash_table_replace(options, crm_strdup(name), crm_strdup(def_value)); value = g_hash_table_lookup(options, name); } return value; } const char * get_cluster_pref(GHashTable *options, pe_cluster_option *option_list, int len, const char *name) { int lpc = 0; const char *value = NULL; gboolean found = FALSE; for(lpc = 0; lpc < len; lpc++) { if(safe_str_eq(name, option_list[lpc].name)) { found = TRUE; value = cluster_option(options, option_list[lpc].is_valid, option_list[lpc].name, option_list[lpc].alt_name, option_list[lpc].default_value); } } CRM_CHECK(found, crm_err("No option named: %s", name)); CRM_ASSERT(value != NULL); return value; } void config_metadata(const char *name, const char *version, const char *desc_short, const char *desc_long, pe_cluster_option *option_list, int len) { int lpc = 0; fprintf(stdout, "" "\n" "\n" " %s\n" " %s\n" " %s\n" " \n", name, version, desc_long, desc_short); for(lpc = 0; lpc < len; lpc++) { if(option_list[lpc].description_long == NULL && option_list[lpc].description_short == NULL) { continue; } fprintf(stdout, " \n" " %s\n" " \n" " %s%s%s\n" " \n", option_list[lpc].name, option_list[lpc].description_short, option_list[lpc].type, option_list[lpc].default_value, option_list[lpc].description_long?option_list[lpc].description_long:option_list[lpc].description_short, option_list[lpc].values?" Allowed values: ":"", option_list[lpc].values?option_list[lpc].values:""); } fprintf(stdout, " \n\n"); } void verify_all_options(GHashTable *options, pe_cluster_option *option_list, int len) { int lpc = 0; for(lpc = 0; lpc < len; lpc++) { cluster_option(options, option_list[lpc].is_valid, option_list[lpc].name, option_list[lpc].alt_name, option_list[lpc].default_value); } } char * generateReference(const char *custom1, const char *custom2) { const char *local_cust1 = custom1; const char *local_cust2 = custom2; int reference_len = 4; char *since_epoch = NULL; reference_len += 20; /* too big */ reference_len += 40; /* too big */ if(local_cust1 == NULL) { local_cust1 = "_empty_"; } reference_len += strlen(local_cust1); if(local_cust2 == NULL) { local_cust2 = "_empty_"; } reference_len += strlen(local_cust2); crm_malloc0(since_epoch, reference_len); if(since_epoch != NULL) { sprintf(since_epoch, "%s-%s-%ld-%u", local_cust1, local_cust2, (unsigned long)time(NULL), ref_counter++); } return since_epoch; } gboolean decodeNVpair(const char *srcstring, char separator, char **name, char **value) { int lpc = 0; int len = 0; const char *temp = NULL; CRM_ASSERT(name != NULL && value != NULL); *name = NULL; *value = NULL; crm_debug_4("Attempting to decode: [%s]", srcstring); if (srcstring != NULL) { len = strlen(srcstring); while(lpc <= len) { if (srcstring[lpc] == separator) { crm_malloc0(*name, lpc+1); if(*name == NULL) { break; /* and return FALSE */ } strncpy(*name, srcstring, lpc); (*name)[lpc] = '\0'; /* this sucks but as the strtok manpage says.. * it *is* a bug */ len = len-lpc; len--; if(len <= 0) { *value = NULL; } else { crm_malloc0(*value, len+1); if(*value == NULL) { crm_free(*name); break; /* and return FALSE */ } temp = srcstring+lpc+1; strncpy(*value, temp, len); (*value)[len] = '\0'; } return TRUE; } lpc++; } } if(*name != NULL) { crm_free(*name); } *name = NULL; *value = NULL; return FALSE; } char * crm_concat(const char *prefix, const char *suffix, char join) { int len = 0; char *new_str = NULL; CRM_ASSERT(prefix != NULL); CRM_ASSERT(suffix != NULL); len = strlen(prefix) + strlen(suffix) + 2; crm_malloc0(new_str, (len)); sprintf(new_str, "%s%c%s", prefix, join, suffix); new_str[len-1] = 0; return new_str; } char * generate_hash_key(const char *crm_msg_reference, const char *sys) { char *hash_key = crm_concat(sys?sys:"none", crm_msg_reference, '_'); crm_debug_3("created hash key: (%s)", hash_key); return hash_key; } char * generate_hash_value(const char *src_node, const char *src_subsys) { char *hash_value = NULL; if (src_node == NULL || src_subsys == NULL) { return NULL; } if (strcasecmp(CRM_SYSTEM_DC, src_subsys) == 0) { hash_value = crm_strdup(src_subsys); CRM_ASSERT(hash_value); return hash_value; } hash_value = crm_concat(src_node, src_subsys, '_'); crm_info("created hash value: (%s)", hash_value); return hash_value; } char * crm_itoa(int an_int) { int len = 32; char *buffer = NULL; crm_malloc0(buffer, (len+1)); if(buffer != NULL) { snprintf(buffer, len, "%d", an_int); } return buffer; } extern int LogToLoggingDaemon(int priority, const char * buf, int bstrlen, gboolean use_pri_str); #ifdef HAVE_G_LOG_SET_DEFAULT_HANDLER GLogFunc glib_log_default; static void crm_glib_handler(const gchar *log_domain, GLogLevelFlags flags, const gchar *message, gpointer user_data) { int log_level = LOG_WARNING; GLogLevelFlags msg_level = (flags & G_LOG_LEVEL_MASK); switch(msg_level) { case G_LOG_LEVEL_CRITICAL: /* log and record how we got here */ crm_abort(__FILE__,__PRETTY_FUNCTION__,__LINE__, message, TRUE, TRUE); return; case G_LOG_LEVEL_ERROR: log_level = LOG_ERR; break; case G_LOG_LEVEL_MESSAGE: log_level = LOG_NOTICE; break; case G_LOG_LEVEL_INFO: log_level = LOG_INFO; break; case G_LOG_LEVEL_DEBUG: log_level = LOG_DEBUG; break; case G_LOG_LEVEL_WARNING: case G_LOG_FLAG_RECURSION: case G_LOG_FLAG_FATAL: case G_LOG_LEVEL_MASK: log_level = LOG_WARNING; break; } do_crm_log(log_level, "%s: %s", log_domain, message); } #endif void crm_log_deinit(void) { #ifdef HAVE_G_LOG_SET_DEFAULT_HANDLER g_log_set_default_handler(glib_log_default, NULL); #endif } gboolean crm_log_init( const char *entity, int level, gboolean coredir, gboolean to_stderr, int argc, char **argv) { return crm_log_init_worker(entity, level, coredir, to_stderr, argc, argv, FALSE); } gboolean crm_log_init_quiet( const char *entity, int level, gboolean coredir, gboolean to_stderr, int argc, char **argv) { return crm_log_init_worker(entity, level, coredir, to_stderr, argc, argv, TRUE); } #if SUPPORT_TRACING static int update_trace_data(struct _pcmk_ddebug_query *query, struct _pcmk_ddebug *start, struct _pcmk_ddebug *stop) { int lpc = 0; unsigned nfound = 0; struct _pcmk_ddebug *dp; const char *match = "unknown"; CRM_ASSERT(stop != NULL); CRM_ASSERT(start != NULL); for (dp = start; dp != stop; dp++) { gboolean bump = FALSE; lpc++; /* fprintf(stderr, "checking: %-12s %20s:%u fmt:%s\n", */ /* dp->function, dp->filename, dp->lineno, dp->format); */ if (query->functions && strstr(query->functions, dp->function) != NULL) { match = "function"; bump = TRUE; } if(query->files) { char token[500]; const char *offset = NULL; const char *next = query->files; do { offset = next; next = strchrnul(offset, ','); snprintf(token, 499, "%.*s", (int)(next-offset), offset); if (query->files && strstr(dp->filename, token) != NULL) { match = "file"; bump = TRUE; } else if(next[0] != 0) { next++; } } while(bump == FALSE && next != NULL && next[0] != 0); } if (query->formats && strstr(query->formats, dp->format) != NULL) { match = "format"; bump = TRUE; } if(bump) { nfound++; dp->bump = LOG_NOTICE; do_crm_log_always(LOG_INFO, "Detected '%s' match: %-12s %20s:%u fmt:%s", match, dp->function, dp->filename, dp->lineno, dp->format); } } query->total += lpc; query->matches += nfound; return nfound; } #define _GNU_SOURCE #include #include #include static int ddebug_callback(struct dl_phdr_info *info, size_t size, void *data) { if(strlen(info->dlpi_name) > 0) { struct _pcmk_ddebug_query *query = data; void *handle; void *start; void *stop; char *error; handle = dlopen (info->dlpi_name, RTLD_LAZY); error = dlerror(); if (!handle || error) { crm_err("%s", error); if(handle) { dlclose(handle); } return 0; } start = dlsym(handle, "__start___verbose"); error = dlerror(); if (error) { goto done; } stop = dlsym(handle, "__stop___verbose"); error = dlerror(); if (error) { goto done; } else { unsigned long int len = (unsigned long int)stop - (unsigned long int)start; crm_info("Checking for query matches in %lu trace symbols from: %s (offset: %p)", len/sizeof(struct _pcmk_ddebug), info->dlpi_name, start); update_trace_data(query, start, stop); } done: dlclose(handle); } return 0; } #endif void update_all_trace_data(void) { #if SUPPORT_TRACING gboolean search = FALSE; const char *env_value = NULL; struct _pcmk_ddebug_query query; memset(&query, 0, sizeof(struct _pcmk_ddebug_query)); env_value = getenv("PCMK_trace_files"); if(env_value) { search = TRUE; query.files = env_value; } env_value = getenv("PCMK_trace_formats"); if(env_value) { search = TRUE; query.formats = env_value; } env_value = getenv("PCMK_trace_functions"); if(env_value) { search = TRUE; query.functions = env_value; } if(search) { update_trace_data(&query, __start___verbose, __stop___verbose); dl_iterate_phdr(ddebug_callback, &query); if(query.matches == 0) { do_crm_log_always(LOG_DEBUG, "no matches for query: {fn='%s', file='%s', fmt='%s'} in %llu entries", crm_str(query.functions), crm_str(query.files), crm_str(query.formats), query.total); } else { do_crm_log_always(LOG_INFO, "%llu matches for query: {fn='%s', file='%s', fmt='%s'} in %llu entries", query.matches, crm_str(query.functions), crm_str(query.files), crm_str(query.formats), query.total); } } /* return query.matches; */ #endif } gboolean crm_log_init_worker( const char *entity, int level, gboolean coredir, gboolean to_stderr, int argc, char **argv, gboolean quiet) { /* Redirect messages from glib functions to our handler */ /* cl_malloc_forced_for_glib(); */ #ifdef HAVE_G_LOG_SET_DEFAULT_HANDLER glib_log_default = g_log_set_default_handler(crm_glib_handler, NULL); #endif /* and for good measure... - this enum is a bit field (!) */ g_log_set_always_fatal((GLogLevelFlags)0); /*value out of range*/ if(entity) { crm_system_name = entity; } else if(argc > 0 && argv != NULL) { crm_system_name = basename(argv[0]); if(strstr(crm_system_name, "lt-") == crm_system_name) { crm_system_name += 3; } } else if(crm_system_name == NULL) { crm_system_name = "Unknown"; } setenv("PCMK_service", crm_system_name, 1); cl_log_set_entity(crm_system_name); set_crm_log_level(level); crm_set_env_options(); if(quiet) { /* Nuke any syslog activity */ unsetenv("HA_logfacility"); } else { cl_log_args(argc, argv); if(getenv("HA_logfacility") == NULL) { /* Set a default */ cl_log_set_facility(HA_LOG_FACILITY); } /* else: picked up by crm_set_env_options() */ } cl_log_enable_stderr(to_stderr); if(coredir) { const char *user = getenv("USER"); if(user != NULL && safe_str_neq(user, "root") && safe_str_neq(user, CRM_DAEMON_USER)) { crm_info("Not switching to corefile directory for %s", user); coredir = FALSE; } } if(coredir) { int user = getuid(); const char *base = HA_COREDIR; struct passwd *pwent = getpwuid(user); if (pwent == NULL) { crm_perror(LOG_ERR, "Cannot get name for uid: %d", user); } else if(safe_str_neq(pwent->pw_name, "root") && safe_str_neq(pwent->pw_name, "nobody") && safe_str_neq(pwent->pw_name, CRM_DAEMON_USER)) { crm_debug("Don't change active directory for regular user: %s", pwent->pw_name); } else if (chdir(base) < 0) { crm_perror(LOG_ERR, "Cannot change active directory to %s", base); } else if (chdir(pwent->pw_name) < 0) { crm_perror(LOG_ERR, "Cannot change active directory to %s/%s", base, pwent->pw_name); } else { crm_info("Changed active directory to %s/%s", base, pwent->pw_name); #if 0 { char path[512]; snprintf(path, 512, "%s-%d", crm_system_name, getpid()); mkdir(path, 0750); chdir(path); crm_info("Changed active directory to %s/%s/%s", base, pwent->pw_name, path); } #endif } } update_all_trace_data(); crm_signal(DEBUG_INC, alter_debug); crm_signal(DEBUG_DEC, alter_debug); return TRUE; } /* returns the old value */ unsigned int set_crm_log_level(unsigned int level) { unsigned int old = crm_log_level; crm_log_level = level; return old; } unsigned int get_crm_log_level(void) { return crm_log_level; } static int crm_version_helper(const char *text, char **end_text) { int atoi_result = -1; CRM_ASSERT(end_text != NULL); errno = 0; if(text != NULL && text[0] != 0) { atoi_result = (int)strtol(text, end_text, 10); if(errno == EINVAL) { crm_err("Conversion of '%s' %c failed", text, text[0]); atoi_result = -1; } } return atoi_result; } /* * version1 < version2 : -1 * version1 = version2 : 0 * version1 > version2 : 1 */ int compare_version(const char *version1, const char *version2) { int rc = 0; int lpc = 0; char *ver1_copy = NULL, *ver2_copy = NULL; char *rest1 = NULL, *rest2 = NULL; if(version1 == NULL && version2 == NULL) { return 0; } else if(version1 == NULL) { return -1; } else if(version2 == NULL) { return 1; } ver1_copy = crm_strdup(version1); ver2_copy = crm_strdup(version2); rest1 = ver1_copy; rest2 = ver2_copy; while(1) { int digit1 = 0; int digit2 = 0; lpc++; if(rest1 == rest2) { break; } if(rest1 != NULL) { digit1 = crm_version_helper(rest1, &rest1); } if(rest2 != NULL) { digit2 = crm_version_helper(rest2, &rest2); } if(digit1 < digit2){ rc = -1; crm_debug_5("%d < %d", digit1, digit2); break; } else if (digit1 > digit2){ rc = 1; crm_debug_5("%d > %d", digit1, digit2); break; } if(rest1 != NULL && rest1[0] == '.') { rest1++; } if(rest1 != NULL && rest1[0] == 0) { rest1 = NULL; } if(rest2 != NULL && rest2[0] == '.') { rest2++; } if(rest2 != NULL && rest2[0] == 0) { rest2 = NULL; } } crm_free(ver1_copy); crm_free(ver2_copy); if(rc == 0) { crm_debug_3("%s == %s (%d)", version1, version2, lpc); } else if(rc < 0) { crm_debug_3("%s < %s (%d)", version1, version2, lpc); } else if(rc > 0) { crm_debug_3("%s > %s (%d)", version1, version2, lpc); } return rc; } gboolean do_stderr = FALSE; void alter_debug(int nsig) { crm_signal(DEBUG_INC, alter_debug); crm_signal(DEBUG_DEC, alter_debug); switch(nsig) { case DEBUG_INC: if (crm_log_level < 100) { crm_log_level++; } break; case DEBUG_DEC: if (crm_log_level > 0) { crm_log_level--; } break; default: fprintf(stderr, "Unknown signal %d\n", nsig); cl_log(LOG_ERR, "Unknown signal %d", nsig); break; } } void g_hash_destroy_str(gpointer data) { crm_free(data); } #include /* #include */ /* #include */ long long crm_int_helper(const char *text, char **end_text) { long long result = -1; char *local_end_text = NULL; int saved_errno = 0; errno = 0; if(text != NULL) { #ifdef ANSI_ONLY if(end_text != NULL) { result = strtol(text, end_text, 10); } else { result = strtol(text, &local_end_text, 10); } #else if(end_text != NULL) { result = strtoll(text, end_text, 10); } else { result = strtoll(text, &local_end_text, 10); } #endif saved_errno = errno; /* CRM_CHECK(errno != EINVAL); */ if(errno == EINVAL) { crm_err("Conversion of %s failed", text); result = -1; } else if(errno == ERANGE) { crm_err("Conversion of %s was clipped: %lld", text, result); } else if(errno != 0) { crm_perror(LOG_ERR,"Conversion of %s failed:", text); } if(local_end_text != NULL && local_end_text[0] != '\0') { crm_err("Characters left over after parsing '%s': '%s'", text, local_end_text); } errno = saved_errno; } return result; } int crm_parse_int(const char *text, const char *default_text) { int atoi_result = -1; if(text != NULL) { atoi_result = crm_int_helper(text, NULL); if(errno == 0) { return atoi_result; } } if(default_text != NULL) { atoi_result = crm_int_helper(default_text, NULL); if(errno == 0) { return atoi_result; } } else { crm_err("No default conversion value supplied"); } return -1; } gboolean safe_str_neq(const char *a, const char *b) { if(a == b) { return FALSE; } else if(a==NULL || b==NULL) { return TRUE; } else if(strcasecmp(a, b) == 0) { return FALSE; } return TRUE; } char * crm_strdup_fn(const char *src, const char *file, const char *fn, int line) { char *dup = NULL; CRM_CHECK(src != NULL, crm_err("Could not perform copy at %s:%d (%s)", file, line, fn); return NULL); crm_malloc0(dup, strlen(src) + 1); return strcpy(dup, src); } #define ENV_PREFIX "HA_" void crm_set_env_options(void) { cl_inherit_logging_environment(500); cl_log_set_logd_channel_source(NULL, NULL); if(debug_level > 0 && (debug_level+LOG_INFO) > (int)crm_log_level) { set_crm_log_level(LOG_INFO + debug_level); } } gboolean crm_is_true(const char * s) { gboolean ret = FALSE; if(s != NULL) { crm_str_to_boolean(s, &ret); } return ret; } int crm_str_to_boolean(const char * s, int * ret) { if(s == NULL) { return -1; } else if (strcasecmp(s, "true") == 0 || strcasecmp(s, "on") == 0 || strcasecmp(s, "yes") == 0 || strcasecmp(s, "y") == 0 || strcasecmp(s, "1") == 0){ *ret = TRUE; return 1; } else if (strcasecmp(s, "false") == 0 || strcasecmp(s, "off") == 0 || strcasecmp(s, "no") == 0 || strcasecmp(s, "n") == 0 || strcasecmp(s, "0") == 0){ *ret = FALSE; return 1; } return -1; } #ifndef NUMCHARS # define NUMCHARS "0123456789." #endif #ifndef WHITESPACE # define WHITESPACE " \t\n\r\f" #endif unsigned long long crm_get_interval(const char * input) { ha_time_t *interval = NULL; char *input_copy = crm_strdup(input); char *input_copy_mutable = input_copy; unsigned long long msec = 0; if(input == NULL) { return 0; } else if(input[0] != 'P') { crm_free(input_copy); return crm_get_msec(input); } interval = parse_time_duration(&input_copy_mutable); msec = date_in_seconds(interval); free_ha_date(interval); crm_free(input_copy); return msec * 1000; } long long crm_get_msec(const char * input) { const char *cp = input; const char *units; long long multiplier = 1000; long long divisor = 1; long long msec = -1; char *end_text = NULL; /* double dret; */ if(input == NULL) { return msec; } cp += strspn(cp, WHITESPACE); units = cp + strspn(cp, NUMCHARS); units += strspn(units, WHITESPACE); if (strchr(NUMCHARS, *cp) == NULL) { return msec; } if (strncasecmp(units, "ms", 2) == 0 || strncasecmp(units, "msec", 4) == 0) { multiplier = 1; divisor = 1; } else if (strncasecmp(units, "us", 2) == 0 || strncasecmp(units, "usec", 4) == 0) { multiplier = 1; divisor = 1000; } else if (strncasecmp(units, "s", 1) == 0 || strncasecmp(units, "sec", 3) == 0) { multiplier = 1000; divisor = 1; } else if (strncasecmp(units, "m", 1) == 0 || strncasecmp(units, "min", 3) == 0) { multiplier = 60*1000; divisor = 1; } else if (strncasecmp(units, "h", 1) == 0 || strncasecmp(units, "hr", 2) == 0) { multiplier = 60*60*1000; divisor = 1; } else if (*units != EOS && *units != '\n' && *units != '\r') { return msec; } msec = crm_int_helper(cp, &end_text); msec *= multiplier; msec /= divisor; /* dret += 0.5; */ /* msec = (long long)dret; */ return msec; } const char * op_status2text(op_status_t status) { switch(status) { case LRM_OP_PENDING: return "pending"; break; case LRM_OP_DONE: return "complete"; break; case LRM_OP_ERROR: return "Error"; break; case LRM_OP_TIMEOUT: return "Timed Out"; break; case LRM_OP_NOTSUPPORTED: return "NOT SUPPORTED"; break; case LRM_OP_CANCELLED: return "Cancelled"; break; } crm_err("Unknown status: %d", status); return "UNKNOWN!"; } char * generate_op_key(const char *rsc_id, const char *op_type, int interval) { int len = 35; char *op_id = NULL; CRM_CHECK(rsc_id != NULL, return NULL); CRM_CHECK(op_type != NULL, return NULL); len += strlen(op_type); len += strlen(rsc_id); crm_malloc0(op_id, len); CRM_CHECK(op_id != NULL, return NULL); sprintf(op_id, "%s_%s_%d", rsc_id, op_type, interval); return op_id; } gboolean parse_op_key(const char *key, char **rsc_id, char **op_type, int *interval) { char *notify = NULL; char *mutable_key = NULL; char *mutable_key_ptr = NULL; int len = 0, offset = 0, ch = 0; CRM_CHECK(key != NULL, return FALSE); *interval = 0; len = strlen(key); offset = len-1; crm_debug_3("Source: %s", key); while(offset > 0 && isdigit(key[offset])) { int digits = len-offset; ch = key[offset] - '0'; CRM_CHECK(ch < 10, return FALSE); CRM_CHECK(ch >= 0, return FALSE); while(digits > 1) { digits--; ch = ch * 10; } *interval += ch; offset--; } crm_debug_3(" Interval: %d", *interval); CRM_CHECK(key[offset] == '_', return FALSE); mutable_key = crm_strdup(key); mutable_key_ptr = mutable_key_ptr; mutable_key[offset] = 0; offset--; while(offset > 0 && key[offset] != '_') { offset--; } CRM_CHECK(key[offset] == '_', crm_free(mutable_key); return FALSE); mutable_key_ptr = mutable_key+offset+1; crm_debug_3(" Action: %s", mutable_key_ptr); *op_type = crm_strdup(mutable_key_ptr); mutable_key[offset] = 0; offset--; CRM_CHECK(mutable_key != mutable_key_ptr, crm_free(mutable_key); return FALSE); notify = strstr(mutable_key, "_post_notify"); if(safe_str_eq(notify, "_post_notify")) { notify[0] = 0; } notify = strstr(mutable_key, "_pre_notify"); if(safe_str_eq(notify, "_pre_notify")) { notify[0] = 0; } crm_debug_3(" Resource: %s", mutable_key); *rsc_id = mutable_key; return TRUE; } char * generate_notify_key(const char *rsc_id, const char *notify_type, const char *op_type) { int len = 12; char *op_id = NULL; CRM_CHECK(rsc_id != NULL, return NULL); CRM_CHECK(op_type != NULL, return NULL); CRM_CHECK(notify_type != NULL, return NULL); len += strlen(op_type); len += strlen(rsc_id); len += strlen(notify_type); crm_malloc0(op_id, len); if(op_id != NULL) { sprintf(op_id, "%s_%s_notify_%s_0", rsc_id, notify_type, op_type); } return op_id; } char * generate_transition_magic_v202(const char *transition_key, int op_status) { int len = 80; char *fail_state = NULL; CRM_CHECK(transition_key != NULL, return NULL); len += strlen(transition_key); crm_malloc0(fail_state, len); if(fail_state != NULL) { snprintf(fail_state, len, "%d:%s", op_status,transition_key); } return fail_state; } char * generate_transition_magic(const char *transition_key, int op_status, int op_rc) { int len = 80; char *fail_state = NULL; CRM_CHECK(transition_key != NULL, return NULL); len += strlen(transition_key); crm_malloc0(fail_state, len); if(fail_state != NULL) { snprintf(fail_state, len, "%d:%d;%s", op_status, op_rc, transition_key); } return fail_state; } gboolean decode_transition_magic( const char *magic, char **uuid, int *transition_id, int *action_id, int *op_status, int *op_rc, int *target_rc) { int res = 0; char *key = NULL; gboolean result = TRUE; CRM_CHECK(magic != NULL, return FALSE); CRM_CHECK(op_rc != NULL, return FALSE); CRM_CHECK(op_status != NULL, return FALSE); crm_malloc0(key, strlen(magic)+1); res = sscanf(magic, "%d:%d;%s", op_status, op_rc, key); if(res != 3) { crm_crit("Only found %d items in: %s", res, magic); result = FALSE; goto bail; } CRM_CHECK(decode_transition_key(key, uuid, transition_id, action_id, target_rc), result = FALSE; goto bail; ); bail: crm_free(key); return result; } char * generate_transition_key(int transition_id, int action_id, int target_rc, const char *node) { int len = 40; char *fail_state = NULL; CRM_CHECK(node != NULL, return NULL); len += strlen(node); crm_malloc0(fail_state, len); if(fail_state != NULL) { snprintf(fail_state, len, "%d:%d:%d:%s", action_id, transition_id, target_rc, node); } return fail_state; } gboolean decode_transition_key( const char *key, char **uuid, int *transition_id, int *action_id, int *target_rc) { int res = 0; gboolean done = FALSE; CRM_CHECK(uuid != NULL, return FALSE); CRM_CHECK(target_rc != NULL, return FALSE); CRM_CHECK(action_id != NULL, return FALSE); CRM_CHECK(transition_id != NULL, return FALSE); crm_malloc0(*uuid, strlen(key)+1); res = sscanf(key, "%d:%d:%d:%s", action_id, transition_id, target_rc, *uuid); switch(res) { case 4: /* Post Pacemaker 0.6 */ done = TRUE; break; case 3: case 2: /* this can be tricky - the UUID might start with an integer */ /* Until Pacemaker 0.6 */ done = TRUE; *target_rc = -1; res = sscanf(key, "%d:%d:%s", action_id, transition_id, *uuid); if(res == 2) { *action_id = -1; res = sscanf(key, "%d:%s", transition_id, *uuid); CRM_CHECK(res == 2, done = FALSE); } else if(res != 3) { CRM_CHECK(res == 3, done = FALSE); } break; case 1: /* Prior to Heartbeat 2.0.8 */ done = TRUE; *action_id = -1; *target_rc = -1; res = sscanf(key, "%d:%s", transition_id, *uuid); CRM_CHECK(res == 2, done = FALSE); break; default: crm_crit("Unhandled sscanf result (%d) for %s", res, key); } if(strlen(*uuid) != 36) { crm_warn("Bad UUID (%s) in sscanf result (%d) for %s", *uuid, res, key); } if(done == FALSE) { crm_err("Cannot decode '%s' rc=%d", key, res); crm_free(*uuid); *uuid = NULL; *target_rc = -1; *action_id = -1; *transition_id = -1; } return done; } void filter_action_parameters(xmlNode *param_set, const char *version) { char *key = NULL; char *timeout = NULL; char *interval = NULL; const char *attr_filter[] = { XML_ATTR_ID, XML_ATTR_CRM_VERSION, XML_LRM_ATTR_OP_DIGEST, }; gboolean do_delete = FALSE; int lpc = 0; static int meta_len = 0; if(meta_len == 0) { meta_len = strlen(CRM_META); } if(param_set == NULL) { return; } for(lpc = 0; lpc < DIMOF(attr_filter); lpc++) { xml_remove_prop(param_set, attr_filter[lpc]); } key = crm_meta_name(XML_LRM_ATTR_INTERVAL); interval = crm_element_value_copy(param_set, key); crm_free(key); key = crm_meta_name(XML_ATTR_TIMEOUT); timeout = crm_element_value_copy(param_set, key); xml_prop_name_iter(param_set, prop_name, do_delete = FALSE; if(strncasecmp(prop_name, CRM_META, meta_len) == 0) { do_delete = TRUE; } if(do_delete) { xml_remove_prop(param_set, prop_name); } ); if(crm_get_msec(interval) > 0 && compare_version(version, "1.0.8") > 0) { /* Re-instate the operation's timeout value */ if(timeout != NULL) { crm_xml_add(param_set, key, timeout); } } crm_free(interval); crm_free(timeout); crm_free(key); } void filter_reload_parameters(xmlNode *param_set, const char *restart_string) { int len = 0; char *name = NULL; char *match = NULL; if(param_set == NULL) { return; } xml_prop_name_iter(param_set, prop_name, name = NULL; len = strlen(prop_name) + 3; crm_malloc0(name, len); sprintf(name, " %s ", prop_name); name[len-1] = 0; match = strstr(restart_string, name); if(match == NULL) { crm_debug_3("%s not found in %s", prop_name, restart_string); xml_remove_prop(param_set, prop_name); } crm_free(name); ); } void crm_abort(const char *file, const char *function, int line, const char *assert_condition, gboolean do_core, gboolean do_fork) { int rc = 0; int pid = 0; int status = 0; if(do_core == FALSE) { do_crm_log(LOG_ERR, "%s: Triggered assert at %s:%d : %s", function, file, line, assert_condition); return; } else if(do_fork) { pid=fork(); } else { do_crm_log(LOG_ERR, "%s: Triggered fatal assert at %s:%d : %s", function, file, line, assert_condition); } switch(pid) { case -1: do_crm_log(LOG_CRIT, "%s: Cannot create core for non-fatal assert at %s:%d : %s", function, file, line, assert_condition); return; default: /* Parent */ do_crm_log(LOG_ERR, "%s: Forked child %d to record non-fatal assert at %s:%d : %s", function, pid, file, line, assert_condition); do { rc = waitpid(pid, &status, 0); if(rc < 0 && errno != EINTR) { crm_perror(LOG_ERR,"%s: Cannot wait on forked child %d", function, pid); } } while(rc < 0 && errno == EINTR); return; case 0: /* Child */ abort(); break; } } char * generate_series_filename( const char *directory, const char *series, int sequence, gboolean bzip) { int len = 40; char *filename = NULL; const char *ext = "raw"; CRM_CHECK(directory != NULL, return NULL); CRM_CHECK(series != NULL, return NULL); len += strlen(directory); len += strlen(series); crm_malloc0(filename, len); CRM_CHECK(filename != NULL, return NULL); if(bzip) { ext = "bz2"; } sprintf(filename, "%s/%s-%d.%s", directory, series, sequence, ext); return filename; } int get_last_sequence(const char *directory, const char *series) { FILE *file_strm = NULL; int start = 0, length = 0, read_len = 0; char *series_file = NULL; char *buffer = NULL; int seq = 0; int len = 36; CRM_CHECK(directory != NULL, return 0); CRM_CHECK(series != NULL, return 0); len += strlen(directory); len += strlen(series); crm_malloc0(series_file, len); CRM_CHECK(series_file != NULL, return 0); sprintf(series_file, "%s/%s.last", directory, series); file_strm = fopen(series_file, "r"); if(file_strm == NULL) { crm_debug("Series file %s does not exist", series_file); crm_free(series_file); return 0; } /* see how big the file is */ start = ftell(file_strm); fseek(file_strm, 0L, SEEK_END); length = ftell(file_strm); fseek(file_strm, 0L, start); CRM_ASSERT(length >= 0); CRM_ASSERT(start == ftell(file_strm)); crm_debug_3("Reading %d bytes from file", length); crm_malloc0(buffer, (length+1)); read_len = fread(buffer, 1, length, file_strm); if(read_len != length) { crm_err("Calculated and read bytes differ: %d vs. %d", length, read_len); crm_free(buffer); buffer = NULL; } else if(length <= 0) { crm_info("%s was not valid", series_file); crm_free(buffer); buffer = NULL; } crm_free(series_file); seq = crm_parse_int(buffer, "0"); crm_free(buffer); fclose(file_strm); return seq; } void write_last_sequence( const char *directory, const char *series, int sequence, int max) { int rc = 0; int len = 36; FILE *file_strm = NULL; char *series_file = NULL; CRM_CHECK(directory != NULL, return); CRM_CHECK(series != NULL, return); if(max == 0) { return; } while(max > 0 && sequence > max) { sequence -= max; } len += strlen(directory); len += strlen(series); crm_malloc0(series_file, len); sprintf(series_file, "%s/%s.last", directory, series); file_strm = fopen(series_file, "w"); if(file_strm == NULL) { crm_err("Cannout open series file %s for writing", series_file); goto bail; } rc = fprintf(file_strm, "%d", sequence); if(rc < 0) { crm_perror(LOG_ERR,"Cannot write to series file %s", series_file); } bail: if(file_strm != NULL) { fflush(file_strm); fclose(file_strm); } crm_free(series_file); } #define LOCKSTRLEN 11 int crm_pid_active(long pid) { int rc = 0; int running = 0; char proc_path[PATH_MAX], exe_path[PATH_MAX], myexe_path[PATH_MAX]; if(pid <= 0) { return -1; } else if (kill(pid, 0) < 0 && errno == ESRCH) { return 0; } #ifndef HAVE_PROC_PID return 1; #endif /* check to make sure pid hasn't been reused by another process */ snprintf(proc_path, sizeof(proc_path), "/proc/%lu/exe", pid); rc = readlink(proc_path, exe_path, PATH_MAX-1); if(rc < 0) { crm_perror(LOG_ERR, "Could not read from %s", proc_path); goto bail; } exe_path[rc] = 0; snprintf(proc_path, sizeof(proc_path), "/proc/%lu/exe", (long unsigned int)getpid()); rc = readlink(proc_path, myexe_path, PATH_MAX-1); if(rc < 0) { crm_perror(LOG_ERR, "Could not read from %s", proc_path); goto bail; } myexe_path[rc] = 0; if(strcmp(exe_path, myexe_path) == 0) { running = 1; } bail: return running; } int crm_read_pidfile(const char *filename) { int fd; long pid = -1; char buf[LOCKSTRLEN+1]; if ((fd = open(filename, O_RDONLY)) < 0) { goto bail; } if (read(fd, buf, sizeof(buf)) < 1) { goto bail; } if (sscanf(buf, "%lu", &pid) > 0) { if (pid <= 0){ pid = -LSB_STATUS_STOPPED; } } bail: if(fd >= 0) { close(fd); } return pid; } int crm_lock_pidfile(const char *filename) { struct stat sbuf; int fd = 0, rc = 0; long pid = 0, mypid = 0; char lf_name[256], tf_name[256], buf[LOCKSTRLEN+1]; mypid = (unsigned long) getpid(); snprintf(lf_name, sizeof(lf_name), "%s",filename); snprintf(tf_name, sizeof(tf_name), "%s.%lu", filename, mypid); if ((fd = open(lf_name, O_RDONLY)) >= 0) { if (fstat(fd, &sbuf) >= 0 && sbuf.st_size < LOCKSTRLEN) { sleep(1); /* if someone was about to create one, * give'm a sec to do so * Though if they follow our protocol, * this won't happen. They should really * put the pid in, then link, not the * other way around. */ } if (read(fd, buf, sizeof(buf)) > 0) { if (sscanf(buf, "%lu", &pid) > 0) { if (pid > 1 && pid != getpid() && crm_pid_active(pid)) { /* locked by existing process - give up */ close(fd); return -1; } } } unlink(lf_name); close(fd); } if ((fd = open(tf_name, O_CREAT | O_WRONLY | O_EXCL, 0644)) < 0) { /* Hmmh, why did we fail? Anyway, nothing we can do about it */ return -3; } /* Slight overkill with the %*d format ;-) */ snprintf(buf, sizeof(buf), "%*lu\n", LOCKSTRLEN-1, mypid); if (write(fd, buf, LOCKSTRLEN) != LOCKSTRLEN) { /* Again, nothing we can do about this */ rc = -3; close(fd); goto out; } close(fd); switch (link(tf_name, lf_name)) { case 0: if (stat(tf_name, &sbuf) < 0) { /* something weird happened */ rc = -3; } else if (sbuf.st_nlink < 2) { /* somehow, it didn't get through - NFS trouble? */ rc = -2; } else { rc = 0; } break; case EEXIST: rc = -1; break; default: rc = -3; } out: unlink(tf_name); return rc; } void crm_make_daemon(const char *name, gboolean daemonize, const char *pidfile) { long pid; const char *devnull = "/dev/null"; if(daemonize == FALSE) { return; } pid = fork(); if (pid < 0) { fprintf(stderr, "%s: could not start daemon\n", name); crm_perror(LOG_ERR,"fork"); exit(LSB_EXIT_GENERIC); } else if (pid > 0) { exit(LSB_EXIT_OK); } if (crm_lock_pidfile(pidfile) < 0 ) { pid = crm_read_pidfile(pidfile); if(crm_pid_active(pid) > 0) { crm_warn("%s: already running [pid %ld] (%s).\n", name, pid, pidfile); exit(LSB_EXIT_OK); } } umask(022); close(STDIN_FILENO); (void)open(devnull, O_RDONLY); /* Stdin: fd 0 */ close(STDOUT_FILENO); (void)open(devnull, O_WRONLY); /* Stdout: fd 1 */ close(STDERR_FILENO); (void)open(devnull, O_WRONLY); /* Stderr: fd 2 */ } gboolean crm_is_writable(const char *dir, const char *file, const char *user, const char *group, gboolean need_both) { int s_res = -1; struct stat buf; char *full_file = NULL; const char *target = NULL; gboolean pass = TRUE; gboolean readwritable = FALSE; CRM_ASSERT(dir != NULL); if(file != NULL) { full_file = crm_concat(dir, file, '/'); target = full_file; s_res = stat(full_file, &buf); if( s_res == 0 && S_ISREG(buf.st_mode) == FALSE ) { crm_err("%s must be a regular file", target); pass = FALSE; goto out; } } if (s_res != 0) { target = dir; s_res = stat(dir, &buf); if(s_res != 0) { crm_err("%s must exist and be a directory", dir); pass = FALSE; goto out; } else if( S_ISDIR(buf.st_mode) == FALSE ) { crm_err("%s must be a directory", dir); pass = FALSE; } } if(user) { struct passwd *sys_user = NULL; sys_user = getpwnam(user); readwritable = (sys_user != NULL && buf.st_uid == sys_user->pw_uid && (buf.st_mode & (S_IRUSR|S_IWUSR))); if(readwritable == FALSE) { crm_err("%s must be owned and r/w by user %s", target, user); if(need_both) { pass = FALSE; } } } if(group) { struct group *sys_grp = getgrnam(group); readwritable = ( sys_grp != NULL && buf.st_gid == sys_grp->gr_gid && (buf.st_mode & (S_IRGRP|S_IWGRP))); if(readwritable == FALSE) { if(need_both || user == NULL) { pass = FALSE; crm_err("%s must be owned and r/w by group %s", target, group); } else { crm_warn("%s should be owned and r/w by group %s", target, group); } } } out: crm_free(full_file); return pass; } static unsigned long long crm_bit_filter = 0; /* 0x00000002ULL; */ static unsigned int bit_log_level = LOG_DEBUG_5; long long crm_clear_bit(const char *function, long long word, long long bit) { unsigned int level = bit_log_level; if(bit & crm_bit_filter) { level = LOG_ERR; } do_crm_log_unlikely(level, "Bit 0x%.16llx cleared by %s", bit, function); word &= ~bit; return word; } long long crm_set_bit(const char *function, long long word, long long bit) { unsigned int level = bit_log_level; if(bit & crm_bit_filter) { level = LOG_ERR; } do_crm_log_unlikely(level, "Bit 0x%.16llx set by %s", bit, function); word |= bit; return word; } const char * name_for_cluster_type(enum cluster_type_e type) { switch(type) { case pcmk_cluster_classic_ais: return "classic openais (with plugin)"; case pcmk_cluster_cman: return "cman"; case pcmk_cluster_corosync: return "corosync"; case pcmk_cluster_heartbeat: return "heartbeat"; case pcmk_cluster_unknown: return "unknown"; case pcmk_cluster_invalid: return "invalid"; } crm_err("Invalid cluster type: %d", type); return "invalid"; } /* Do not expose these two */ int set_cluster_type(enum cluster_type_e type); static enum cluster_type_e cluster_type = pcmk_cluster_unknown; int set_cluster_type(enum cluster_type_e type) { if(cluster_type == pcmk_cluster_unknown) { crm_info("Cluster type set to: %s", name_for_cluster_type(cluster_type)); cluster_type = type; return 0; } else if(cluster_type == type) { return 0; } else if(pcmk_cluster_unknown == type) { cluster_type = type; return 0; } crm_err("Cluster type already set to %s", name_for_cluster_type(cluster_type)); return -1; } enum cluster_type_e get_cluster_type(void) { if(cluster_type == pcmk_cluster_unknown) { const char *cluster = getenv("HA_cluster_type"); cluster_type = pcmk_cluster_invalid; if(cluster) { crm_info("Cluster type is: '%s'.", cluster); + } else { + cluster = "heartbeat"; } - if(cluster == NULL || safe_str_eq(cluster, "heartbeat")) { + + if(safe_str_eq(cluster, "heartbeat")) { #if SUPPORT_HEARTBEAT cluster_type = pcmk_cluster_heartbeat; #else crm_crit("This installation of Pacemaker does not support the '%s' cluster infrastructure. Terminating.", cluster); exit(100); #endif } else if(safe_str_eq(cluster, "openais")) { #if SUPPORT_COROSYNC cluster_type = pcmk_cluster_classic_ais; #else crm_crit("This installation of Pacemaker does not support the '%s' cluster infrastructure. Terminating.", cluster); exit(100); #endif } else if(safe_str_eq(cluster, "corosync")) { #if SUPPORT_COROSYNC cluster_type = pcmk_cluster_corosync; #else crm_crit("This installation of Pacemaker does not support the '%s' cluster infrastructure. Terminating.", cluster); exit(100); #endif } else if(safe_str_eq(cluster, "cman")) { #if SUPPORT_CMAN cluster_type = pcmk_cluster_cman; #else crm_crit("This installation of Pacemaker does not support the '%s' cluster infrastructure. Terminating.", cluster); exit(100); #endif } else { crm_crit("Unknown cluster type: '%s'. Terminating.", cluster); exit(100); } } return cluster_type; } gboolean is_cman_cluster(void) { return get_cluster_type() == pcmk_cluster_cman; } gboolean is_corosync_cluster(void) { return get_cluster_type() == pcmk_cluster_corosync; } gboolean is_classic_ais_cluster(void) { return get_cluster_type() == pcmk_cluster_classic_ais; } gboolean is_openais_cluster(void) { enum cluster_type_e type = get_cluster_type(); if(type == pcmk_cluster_classic_ais) { return TRUE; } else if(type == pcmk_cluster_corosync) { return TRUE; } else if(type == pcmk_cluster_cman) { return TRUE; } return FALSE; } gboolean is_heartbeat_cluster(void) { return get_cluster_type() == pcmk_cluster_heartbeat; } gboolean crm_str_eq(const char *a, const char *b, gboolean use_case) { if(a == b) { return TRUE; } else if(a == NULL || b == NULL) { /* shouldn't be comparing NULLs */ return FALSE; } else if(use_case && a[0] != b[0]) { return FALSE; } else if(strcasecmp(a, b) == 0) { return TRUE; } return FALSE; } char *crm_meta_name(const char *field) { int lpc = 0; int max = 0; char *crm_name = NULL; CRM_CHECK(field != NULL, return NULL); crm_name = crm_concat(CRM_META, field, '_'); /* Massage the names so they can be used as shell variables */ max = strlen(crm_name); for(; lpc < max; lpc++) { switch(crm_name[lpc]) { case '-': crm_name[lpc] = '_'; break; } } return crm_name; } const char *crm_meta_value(GHashTable *hash, const char *field) { char *key = NULL; const char *value = NULL; key = crm_meta_name(field); if(key) { value = g_hash_table_lookup(hash, key); crm_free(key); } return value; } static struct crm_option *crm_long_options = NULL; static const char *crm_app_description = NULL; static const char *crm_short_options = NULL; static const char *crm_app_usage = NULL; static struct option *crm_create_long_opts(struct crm_option *long_options) { struct option *long_opts = NULL; #ifdef HAVE_GETOPT_H int index = 0, lpc = 0; /* * A previous, possibly poor, choice of '?' as the short form of --help * means that getopt_long() returns '?' for both --help and for "unknown option" * * This dummy entry allows us to differentiate between the two in crm_get_option() * and exit with the correct error code */ crm_realloc(long_opts, (index+1) * sizeof(struct option)); long_opts[index].name = "__dummmy__"; long_opts[index].has_arg = 0; long_opts[index].flag = 0; long_opts[index].val = '_'; index++; for(lpc = 0; long_options[lpc].name != NULL; lpc++) { if(long_options[lpc].name[0] == '-') { continue; } crm_realloc(long_opts, (index+1) * sizeof(struct option)); /*fprintf(stderr, "Creating %d %s = %c\n", index, * long_options[lpc].name, long_options[lpc].val); */ long_opts[index].name = long_options[lpc].name; long_opts[index].has_arg = long_options[lpc].has_arg; long_opts[index].flag = long_options[lpc].flag; long_opts[index].val = long_options[lpc].val; index++; } /* Now create the list terminator */ crm_realloc(long_opts, (index+1) * sizeof(struct option)); long_opts[index].name = NULL; long_opts[index].has_arg = 0; long_opts[index].flag = 0; long_opts[index].val = 0; #endif return long_opts; } void crm_set_options(const char *short_options, const char *app_usage, struct crm_option *long_options, const char *app_desc) { if(short_options) { crm_short_options = short_options; } if(long_options) { crm_long_options = long_options; } if(app_desc) { crm_app_description = app_desc; } if(app_usage) { crm_app_usage = app_usage; } } int crm_get_option(int argc, char **argv, int *index) { #ifdef HAVE_GETOPT_H static struct option *long_opts = NULL; if(long_opts == NULL && crm_long_options) { long_opts = crm_create_long_opts(crm_long_options); } if(long_opts) { int flag = getopt_long(argc, argv, crm_short_options, long_opts, index); switch(flag) { case 0: return long_opts[*index].val; case -1: /* End of option processing */ break; case ':': crm_debug_2("Missing argument"); crm_help('?', 1); break; case '?': crm_help('?', *index?0:1); break; } return flag; } #endif if(crm_short_options) { return getopt(argc, argv, crm_short_options); } return -1; } void crm_help(char cmd, int exit_code) { int i = 0; FILE *stream = (exit_code ? stderr : stdout); if(cmd == 'v' || cmd == '$') { fprintf(stream, "Pacemaker %s\n", VERSION); fprintf(stream, "Written by Andrew Beekhof\n"); goto out; } if(cmd == '!') { fprintf(stream, "Pacemaker %s (Build: %s): %s\n", VERSION, BUILD_VERSION, CRM_FEATURES); goto out; } fprintf(stream, "%s - %s\n", crm_system_name, crm_app_description); if(crm_app_usage) { fprintf(stream, "Usage: %s %s\n", crm_system_name, crm_app_usage); } if(crm_long_options) { fprintf(stream, "Options:\n"); for(i = 0; crm_long_options[i].name != NULL; i++) { if(crm_long_options[i].flags & pcmk_option_hidden) { } else if(crm_long_options[i].flags & pcmk_option_paragraph) { fprintf(stream, "%s\n\n", crm_long_options[i].desc); } else if(crm_long_options[i].flags & pcmk_option_example) { fprintf(stream, "\t#%s\n\n", crm_long_options[i].desc); } else if(crm_long_options[i].val == '-' && crm_long_options[i].desc) { fprintf(stream, "%s\n", crm_long_options[i].desc); } else { fprintf(stream, " -%c, --%s%c%s\t%s\n", crm_long_options[i].val, crm_long_options[i].name, crm_long_options[i].has_arg?'=':' ',crm_long_options[i].has_arg?"value":"", crm_long_options[i].desc?crm_long_options[i].desc:""); } } } else if(crm_short_options) { fprintf(stream, "Usage: %s - %s\n", crm_system_name, crm_app_description); for(i = 0; crm_short_options[i] != 0; i++) { int has_arg = FALSE; if(crm_short_options[i+1] == ':') { has_arg = TRUE; } fprintf(stream, " -%c %s\n", crm_short_options[i], has_arg?"{value}":""); if(has_arg) { i++; } } } fprintf(stream, "\nReport bugs to %s\n", PACKAGE_BUGREPORT); out: if(exit_code >= 0) { exit(exit_code); } } #include <../../tools/attrd.h> gboolean attrd_update_delegate(IPC_Channel *cluster, char command, const char *host, const char *name, const char *value, const char *section, const char *set, const char *dampen, const char *user_name) { gboolean success = FALSE; const char *reason = "Cluster connection failed"; /* remap common aliases */ if(safe_str_eq(section, "reboot")) { section = XML_CIB_TAG_STATUS; } else if(safe_str_eq(section, "forever")) { section = XML_CIB_TAG_NODES; } if(cluster == NULL) { reason = "No connection to the cluster"; } else { xmlNode *update = create_xml_node(NULL, __FUNCTION__); crm_xml_add(update, F_TYPE, T_ATTRD); crm_xml_add(update, F_ORIG, crm_system_name); if(name == NULL && command == 'U') { command = 'R'; } switch(command) { case 'D': case 'U': case 'v': crm_xml_add(update, F_ATTRD_TASK, "update"); crm_xml_add(update, F_ATTRD_ATTRIBUTE, name); break; case 'R': crm_xml_add(update, F_ATTRD_TASK, "refresh"); break; case 'q': crm_xml_add(update, F_ATTRD_TASK, "query"); break; } crm_xml_add(update, F_ATTRD_VALUE, value); crm_xml_add(update, F_ATTRD_DAMPEN, dampen); crm_xml_add(update, F_ATTRD_SECTION, section); crm_xml_add(update, F_ATTRD_HOST, host); crm_xml_add(update, F_ATTRD_SET, set); #if ENABLE_ACL if (user_name) { crm_xml_add(update, F_ATTRD_USER, user_name); } #endif success = send_ipc_message(cluster, update); free_xml(update); } if(success) { crm_debug("Sent update: %s=%s for %s", name, value, host?host:"localhost"); return TRUE; } crm_info("Could not send update: %s=%s for %s", name, value, host?host:"localhost"); return FALSE; } gboolean attrd_lazy_update(char command, const char *host, const char *name, const char *value, const char *section, const char *set, const char *dampen) { int max = 5; gboolean updated = FALSE; static IPC_Channel *cluster = NULL; while(updated == 0 && max > 0) { if(cluster == NULL) { crm_info("Connecting to cluster... %d retries remaining", max); cluster = init_client_ipc_comms_nodispatch(T_ATTRD); } if(cluster != NULL) { updated = attrd_update(cluster, command, host, name, value, section, set, dampen); } if(updated == 0) { cluster = NULL; sleep(2); max--; } } return updated; } gboolean attrd_update_no_mainloop(int *connection, char command, const char *host, const char *name, const char *value, const char *section, const char *set, const char *dampen) { int max = 5; gboolean updated = FALSE; static IPC_Channel *cluster = NULL; if(connection && *connection == 0 && cluster) { crm_info("Forcing a new connection to the cluster"); cluster = NULL; } while(updated == 0 && max > 0) { if(cluster == NULL) { crm_info("Connecting to cluster... %d retries remaining", max); cluster = init_client_ipc_comms_nodispatch(T_ATTRD); } if(connection) { if(cluster != NULL) { *connection = cluster->ops->get_recv_select_fd(cluster); } else { *connection = 0; } } if(cluster != NULL) { updated = attrd_update(cluster, command, host, name, value, section, set, dampen); } if(updated == 0) { cluster = NULL; sleep(2); max--; } } return updated; } #define FAKE_TE_ID "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" static void append_digest(lrm_op_t *op, xmlNode *update, const char *version, const char *magic, int level) { /* this will enable us to later determine that the * resource's parameters have changed and we should force * a restart */ char *digest = NULL; xmlNode *args_xml = NULL; if(op->params == NULL) { return; } args_xml = create_xml_node(NULL, XML_TAG_PARAMS); g_hash_table_foreach(op->params, hash2field, args_xml); filter_action_parameters(args_xml, version); digest = calculate_operation_digest(args_xml, version); #if 0 if(level < crm_log_level && op->interval == 0 && crm_str_eq(op->op_type, CRMD_ACTION_START, TRUE)) { char *digest_source = dump_xml_unformatted(args_xml); do_crm_log(level, "Calculated digest %s for %s (%s). Source: %s\n", digest, ID(update), magic, digest_source); crm_free(digest_source); } #endif crm_xml_add(update, XML_LRM_ATTR_OP_DIGEST, digest); free_xml(args_xml); crm_free(digest); } xmlNode * create_operation_update( xmlNode *parent, lrm_op_t *op, const char *caller_version, int target_rc, const char *origin, int level) { char *magic = NULL; const char *task = NULL; xmlNode *xml_op = NULL; char *op_id = NULL; char *local_user_data = NULL; gboolean dc_munges_migrate_ops = (compare_version(caller_version, "3.0.3") < 0); CRM_CHECK(op != NULL, return NULL); do_crm_log(level, "%s: Updating resouce %s after %s %s op (interval=%d)", origin, op->rsc_id, op_status2text(op->op_status), op->op_type, op->interval); if(op->op_status == LRM_OP_CANCELLED) { crm_debug_3("Ignoring cancelled op"); return NULL; } crm_debug_3("DC version: %s", caller_version); task = op->op_type; /* remap the task name under various scenarios * this makes life easier for the PE when its trying determin the current state */ if(crm_str_eq(task, "reload", TRUE)) { if(op->op_status == LRM_OP_DONE) { task = CRMD_ACTION_START; } else { task = CRMD_ACTION_STATUS; } } else if(dc_munges_migrate_ops && crm_str_eq(task, CRMD_ACTION_MIGRATE, TRUE)) { /* if the migrate_from fails it will have enough info to do the right thing */ if(op->op_status == LRM_OP_DONE) { task = CRMD_ACTION_STOP; } else { task = CRMD_ACTION_STATUS; } } else if(dc_munges_migrate_ops && op->op_status == LRM_OP_DONE && crm_str_eq(task, CRMD_ACTION_MIGRATED, TRUE)) { task = CRMD_ACTION_START; } else if(crm_str_eq(task, CRMD_ACTION_NOTIFY, TRUE)) { const char *n_type = crm_meta_value(op->params, "notify_type"); const char *n_task = crm_meta_value(op->params, "notify_operation"); CRM_LOG_ASSERT(n_type != NULL); CRM_LOG_ASSERT(n_task != NULL); op_id = generate_notify_key(op->rsc_id, n_type, n_task); /* these are not yet allowed to fail */ op->op_status = LRM_OP_DONE; op->rc = 0; } if (op_id == NULL) { op_id = generate_op_key(op->rsc_id, task, op->interval); } xml_op = find_entity(parent, XML_LRM_TAG_RSC_OP, op_id); if(xml_op != NULL) { crm_log_xml(LOG_DEBUG, "Replacing existing entry", xml_op); } else { xml_op = create_xml_node(parent, XML_LRM_TAG_RSC_OP); } if(op->user_data == NULL) { crm_debug("Generating fake transition key for:" " %s_%s_%d %d from %s", op->rsc_id, op->op_type, op->interval, op->call_id, op->app_name); local_user_data = generate_transition_key(-1, op->call_id, target_rc, FAKE_TE_ID); op->user_data = local_user_data; } magic = generate_transition_magic(op->user_data, op->op_status, op->rc); crm_xml_add(xml_op, XML_ATTR_ID, op_id); crm_xml_add(xml_op, XML_LRM_ATTR_TASK, task); crm_xml_add(xml_op, XML_ATTR_ORIGIN, origin); crm_xml_add(xml_op, XML_ATTR_CRM_VERSION, caller_version); crm_xml_add(xml_op, XML_ATTR_TRANSITION_KEY, op->user_data); crm_xml_add(xml_op, XML_ATTR_TRANSITION_MAGIC, magic); crm_xml_add_int(xml_op, XML_LRM_ATTR_CALLID, op->call_id); crm_xml_add_int(xml_op, XML_LRM_ATTR_RC, op->rc); crm_xml_add_int(xml_op, XML_LRM_ATTR_OPSTATUS, op->op_status); crm_xml_add_int(xml_op, XML_LRM_ATTR_INTERVAL, op->interval); if(compare_version("2.1", caller_version) <= 0) { if(op->t_run || op->t_rcchange || op->exec_time || op->queue_time) { crm_debug_2("Timing data (%s_%s_%d): last=%lu change=%lu exec=%lu queue=%lu", op->rsc_id, op->op_type, op->interval, op->t_run, op->t_rcchange, op->exec_time, op->queue_time); if(op->interval == 0) { crm_xml_add_int(xml_op, "last-run", op->t_run); } crm_xml_add_int(xml_op, "last-rc-change", op->t_rcchange); crm_xml_add_int(xml_op, "exec-time", op->exec_time); crm_xml_add_int(xml_op, "queue-time", op->queue_time); } } if(crm_str_eq(op->op_type, CRMD_ACTION_MIGRATE, TRUE) || crm_str_eq(op->op_type, CRMD_ACTION_MIGRATED, TRUE)) { /* * Record migrate_source and migrate_target always for migrate ops. */ const char *name = XML_LRM_ATTR_MIGRATE_SOURCE; crm_xml_add(xml_op, name, crm_meta_value(op->params, name)); name = XML_LRM_ATTR_MIGRATE_TARGET; crm_xml_add(xml_op, name, crm_meta_value(op->params, name)); } append_digest(op, xml_op, caller_version, magic, LOG_DEBUG); if(local_user_data) { crm_free(local_user_data); op->user_data = NULL; } crm_free(magic); crm_free(op_id); return xml_op; } void free_lrm_op(lrm_op_t *op) { g_hash_table_destroy(op->params); crm_free(op->user_data); crm_free(op->output); crm_free(op->rsc_id); crm_free(op->op_type); crm_free(op->app_name); crm_free(op); } #if ENABLE_ACL void determine_request_user(char **user, IPC_Channel *channel, xmlNode *request, const char *field) { /* Get our internal validation out of the way first */ CRM_CHECK(user != NULL && channel != NULL && field != NULL, return); if(*user == NULL) { /* Figure out who our peer is and cache it... */ struct passwd *pwent = getpwuid(channel->farside_uid); if(pwent == NULL) { crm_perror(LOG_ERR, "Cannot get password entry of uid: %d", channel->farside_uid); } else { *user = crm_strdup(pwent->pw_name); } } /* If our peer is a privileged user, we might be doing something on behalf of someone else */ if(is_privileged(*user) == FALSE) { /* We're not a privileged user, set or overwrite any existing value for $field */ crm_xml_replace(request, field, *user); } else if(crm_element_value(request, field) == NULL) { /* Even if we're privileged, make sure there is always a value set */ crm_xml_replace(request, field, *user); /* } else { Legal delegation */ } crm_debug_2("Processing msg for user '%s'", crm_element_value(request, field)); } #endif diff --git a/pengine/pengine.c b/pengine/pengine.c index 0d9224b218..45c37b4b6c 100644 --- a/pengine/pengine.c +++ b/pengine/pengine.c @@ -1,297 +1,297 @@ /* * Copyright (C) 2004 Andrew Beekhof * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public * License as published by the Free Software Foundation; either * version 2 of the License, or (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ #include #include #include #include #include #include #include #include #include #include #include #include #include xmlNode * do_calculations( pe_working_set_t *data_set, xmlNode *xml_input, ha_time_t *now); gboolean show_scores = FALSE; int scores_log_level = LOG_DEBUG_2; gboolean show_utilization = FALSE; int utilization_log_level = LOG_DEBUG_2; extern int transition_id; #define get_series() was_processing_error?1:was_processing_warning?2:3 typedef struct series_s { int id; const char *name; const char *param; int wrap; } series_t; series_t series[] = { { 0, "pe-unknown", "_dont_match_anything_", -1 }, { 0, "pe-error", "pe-error-series-max", -1 }, { 0, "pe-warn", "pe-warn-series-max", 200 }, { 0, "pe-input", "pe-input-series-max", 400 }, }; gboolean process_pe_message(xmlNode *msg, xmlNode *xml_data, IPC_Channel *sender) { gboolean send_via_disk = FALSE; const char *sys_to = crm_element_value(msg, F_CRM_SYS_TO); const char *op = crm_element_value(msg, F_CRM_TASK); const char *ref = crm_element_value(msg, XML_ATTR_REFERENCE); crm_debug_3("Processing %s op (ref=%s)...", op, ref); if(op == NULL){ /* error */ } else if(strcasecmp(op, CRM_OP_HELLO) == 0) { /* ignore */ } else if(safe_str_eq(crm_element_value(msg, F_CRM_MSG_TYPE), XML_ATTR_RESPONSE)) { /* ignore */ } else if(sys_to == NULL || strcasecmp(sys_to, CRM_SYSTEM_PENGINE) != 0) { crm_debug_3("Bad sys-to %s", crm_str(sys_to)); return FALSE; } else if(strcasecmp(op, CRM_OP_PECALC) == 0) { int seq = -1; int series_id = 0; int series_wrap = 0; char *filename = NULL; char *graph_file = NULL; const char *value = NULL; pe_working_set_t data_set; xmlNode *converted = NULL; xmlNode *reply = NULL; gboolean process = TRUE; #if HAVE_BZLIB_H gboolean compress = TRUE; #else gboolean compress = FALSE; #endif crm_config_error = FALSE; crm_config_warning = FALSE; was_processing_error = FALSE; was_processing_warning = FALSE; graph_file = crm_strdup(CRM_STATE_DIR"/graph.XXXXXX"); graph_file = mktemp(graph_file); set_working_set_defaults(&data_set); converted = copy_xml(xml_data); if(cli_config_update(&converted, NULL, TRUE) == FALSE) { data_set.graph = create_xml_node(NULL, XML_TAG_GRAPH); crm_xml_add_int(data_set.graph, "transition_id", 0); crm_xml_add_int(data_set.graph, "cluster-delay", 0); process = FALSE; } if(process) { do_calculations(&data_set, converted, NULL); } series_id = get_series(); series_wrap = series[series_id].wrap; value = pe_pref(data_set.config_hash, series[series_id].param); if(value != NULL) { series_wrap = crm_int_helper(value, NULL); if(errno != 0) { series_wrap = series[series_id].wrap; } } else { crm_config_warn("No value specified for cluster" " preference: %s", series[series_id].param); } seq = get_last_sequence(PE_STATE_DIR, series[series_id].name); data_set.input = NULL; reply = create_reply(msg, data_set.graph); CRM_ASSERT(reply != NULL); filename = generate_series_filename( PE_STATE_DIR, series[series_id].name, seq, compress); crm_xml_add(reply, F_CRM_TGRAPH_INPUT, filename); crm_xml_add_int(reply, "graph-errors", was_processing_error); crm_xml_add_int(reply, "graph-warnings", was_processing_warning); crm_xml_add_int(reply, "config-errors", crm_config_error); crm_xml_add_int(reply, "config-warnings", crm_config_warning); if(send_ipc_message(sender, reply) == FALSE) { if(sender && sender->ops->get_chan_status(sender) == IPC_CONNECT) { send_via_disk = TRUE; crm_err("Answer could not be sent via IPC, send via the disk instead"); - crm_info("Writing the TE graph to %s", graph_file); + crm_notice("Writing the TE graph to %s", graph_file); if(write_xml_file(data_set.graph, graph_file, FALSE) < 0) { - crm_err("TE graph could not be written to disk"); + crm_err("TE graph could not be written to disk"); } } else { crm_info("Peer disconnected, discarding transition graph"); } } free_xml(reply); cleanup_alloc_calculations(&data_set); if(series_wrap != 0) { write_xml_file(xml_data, filename, compress); write_last_sequence(PE_STATE_DIR, series[series_id].name, seq+1, series_wrap); } if(was_processing_error) { crm_err("Transition %d:" " ERRORs found during PE processing." " PEngine Input stored in: %s", transition_id, filename); } else if(was_processing_warning) { crm_warn("Transition %d:" " WARNINGs found during PE processing." " PEngine Input stored in: %s", transition_id, filename); } else { - crm_info("Transition %d: PEngine Input stored in: %s", - transition_id, filename); + crm_notice("Transition %d: PEngine Input stored in: %s", + transition_id, filename); } if(crm_config_error) { - crm_info("Configuration ERRORs found during PE processing." - " Please run \"crm_verify -L\" to identify issues."); + crm_notice("Configuration ERRORs found during PE processing." + " Please run \"crm_verify -L\" to identify issues."); } else if(crm_config_warning) { - crm_info("Configuration WARNINGs found during PE processing." - " Please run \"crm_verify -L\" to identify issues."); + crm_notice("Configuration WARNINGs found during PE processing." + " Please run \"crm_verify -L\" to identify issues."); } if(send_via_disk) { reply = create_reply(msg, NULL); crm_xml_add(reply, F_CRM_TGRAPH, graph_file); crm_xml_add(reply, F_CRM_TGRAPH_INPUT, filename); CRM_ASSERT(reply != NULL); if(send_ipc_message(sender, reply) == FALSE) { crm_err("Answer could not be sent"); } free_xml(reply); } free_xml(converted); crm_free(graph_file); crm_free(filename); } else if(strcasecmp(op, CRM_OP_QUIT) == 0) { crm_warn("Received quit message, terminating"); exit(0); } return TRUE; } xmlNode * do_calculations(pe_working_set_t *data_set, xmlNode *xml_input, ha_time_t *now) { GListPtr gIter = NULL; - int rsc_log_level = LOG_NOTICE; + int rsc_log_level = LOG_INFO; /* pe_debug_on(); */ CRM_ASSERT(xml_input || is_set(data_set->flags, pe_flag_have_status)); if(is_set(data_set->flags, pe_flag_have_status) == FALSE) { set_working_set_defaults(data_set); data_set->input = xml_input; data_set->now = now; if(data_set->now == NULL) { data_set->now = new_ha_date(TRUE); } } else { crm_trace("Already have status - reusing"); } crm_debug_5("Calculate cluster status"); stage0(data_set); gIter = data_set->resources; for(; gIter != NULL; gIter = gIter->next) { resource_t *rsc = (resource_t*)gIter->data; if(is_set(rsc->flags, pe_rsc_orphan) && rsc->role == RSC_ROLE_STOPPED) { continue; } rsc->fns->print(rsc, NULL, pe_print_log, &rsc_log_level); } crm_trace("Applying placement constraints"); stage2(data_set); crm_trace("Create internal constraints"); stage3(data_set); crm_trace("Check actions"); stage4(data_set); crm_trace("Allocate resources"); stage5(data_set); crm_trace("Processing fencing and shutdown cases"); stage6(data_set); crm_trace("Applying ordering constraints"); stage7(data_set); crm_trace("Create transition graph"); stage8(data_set); crm_trace("=#=#=#=#= Summary =#=#=#=#="); crm_trace("\t========= Set %d (Un-runnable) =========", -1); if(crm_log_level > LOG_DEBUG) { gIter = data_set->actions; for(; gIter != NULL; gIter = gIter->next) { action_t *action = (action_t*)gIter->data; if(is_set(action->flags, pe_action_optional) == FALSE && is_set(action->flags, pe_action_runnable) == FALSE && is_set(action->flags, pe_action_pseudo) == FALSE) { log_action(LOG_DEBUG_2, "\t", action, TRUE); } } } return data_set->graph; } diff --git a/tools/crm_report.in b/tools/crm_report.in index cd6e63fd1d..15df3ec7ba 100755 --- a/tools/crm_report.in +++ b/tools/crm_report.in @@ -1,400 +1,429 @@ #!/bin/sh # Copyright (C) 2010 Andrew Beekhof # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public # License as published by the Free Software Foundation; either # version 2.1 of the License, or (at your option) any later version. # # This software is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # General Public License for more details. # # You should have received a copy of the GNU General Public # License along with this library; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA # # Note the quotes around `$TEMP': they are essential! TEMP=`getopt \ - -o hv?xl:f:t:n:T:Lpc:dSACHu:MV \ - --long help,cts:,node:,nodes:,from:,to:logfile:,as-directory,single-node,cluster:,user:,version,features \ - -n 'pcmk_report' -- "$@"` + -o hv?xl:f:t:n:T:Lpc:dSACHu:MVs \ + --long help,cts:,cts-log:,node:,nodes:,from:,to:logfile:,as-directory,single-node,cluster:,user:,version,features \ + -n 'crm_report' -- "$@"` eval set -- "$TEMP" times="" tests="" nodes="" compress=1 cluster="any" ssh_user="root" search_logs=1 report_data=`dirname $0` extra_logs="" -sanitize_patterns="passw.*" +sanitize_patterns="" log_patterns="CRIT: ERROR:" usage() { cat< "$l_base/$HALOG_F" fi cat<$l_base/.env LABEL="$label" REPORT_HOME="$r_base" REPORT_MASTER="$host" LOG_START=$start LOG_END=$end REMOVE=1 SANITIZE="$sanitize_patterns" CLUSTER=$cluster LOG_PATTERNS="$log_patterns" EXTRA_LOGS="$extra_logs" SEARCH_LOGS=$search_logs verbose=$verbose EOF for node in $nodes; do if [ `uname -n` = $node ]; then cat $l_base/.env $report_data/report.common $report_data/report.collector > $r_base/collector bash $r_base/collector else cat $l_base/.env $report_data/report.common $report_data/report.collector \ | ssh -l $ssh_user -T $node -- "mkdir -p $r_base; cat > $r_base/collector; bash $r_base/collector" | (cd $l_base && tar xf -) fi done analyze $l_base > $l_base/$ANALYSIS_F if [ -f $l_base/$HALOG_F ]; then node_events $l_base/$HALOG_F > $l_base/$EVENTS_F fi for node in $nodes; do cat $l_base/$node/$ANALYSIS_F >> $l_base/$ANALYSIS_F if [ -s $l_base/$node/$EVENTS_F ]; then cat $l_base/$node/$EVENTS_F >> $l_base/$EVENTS_F elif [ -s $l_base/$HALOG_F ]; then awk "\$4==\"$nodes\"" $l_base/$EVENTS_F >> $l_base/$n/$EVENTS_F fi done log " " if [ $compress = 1 ]; then fname=`shrink $l_base` rm -rf $l_base log "Collected results are available in $fname" log " " log "Please create a bug entry at" log " http://developerbugs.linux-foundation.org/enter_bug.cgi?product=Pacemaker" log "Include a description of your problem and attach this tarball" log " " log "Thank you for taking time to create this report." else log "Collected results are available in $l_base" fi log " " } # # check if files have same content in the cluster # cibdiff() { d1=`dirname $1` d2=`dirname $2` if [ -f $d1/RUNNING -a -f $d2/RUNNING ] || [ -f $d1/STOPPED -a -f $d2/STOPPED ]; then if which crm_diff > /dev/null 2>&1; then crm_diff -c -n $1 -o $2 else info "crm_diff(8) not found, cannot diff CIBs" fi else echo "can't compare cibs from running and stopped systems" fi } diffcheck() { [ -f "$1" ] || { echo "$1 does not exist" return 1 } [ -f "$2" ] || { echo "$2 does not exist" return 1 } case `basename $1` in $CIB_F) cibdiff $1 $2;; $B_CONF) diff -u $1 $2;; # confdiff? *) diff -u $1 $2;; esac } # # remove duplicates if files are same, make links instead # consolidate() { for n in $NODES; do if [ -f $1/$2 ]; then rm $1/$n/$2 else mv $1/$n/$2 $1 fi ln -s ../$2 $1/$n done } analyze_one() { rc=0 node0="" for n in $NODES; do if [ "$node0" ]; then diffcheck $1/$node0/$2 $1/$n/$2 rc=$(($rc+$?)) else node0=$n fi done return $rc } analyze() { flist="$HOSTCACHE $MEMBERSHIP_F $CIB_F $CRM_MON_F $B_CONF logd.cf $SYSINFO_F" for f in $flist; do printf "Diff $f... " ls $1/*/$f >/dev/null 2>&1 || { echo "no $1/*/$f :/" continue } if analyze_one $1 $f; then echo "OK" [ "$f" != $CIB_F ] && consolidate $1 $f else echo "" fi done } do_cts() { - if [ x$ctslog = x ]; then - ctslog=`findmsg 1 "CTS: Stack:"` - fi - if [ x$ctslog = x ]; then - fatal "No CTS control file detected" - fi - - if [ -z "$nodes" ]; then - debug "Using CTS control file: $ctslog" - nodes=`grep CTS: $ctslog | grep -v debug: | grep " \* " | sed s:.*\\\*::g | sort -u | tr '\\n' ' '` - fi - test_sets=`echo $tests | tr ',' ' '` for test_set in $test_sets; do + + start_time=0 start_test=`echo $test_set | tr '-' ' ' | awk '{print $1}'` + + end_time=0 end_test=`echo $test_set | tr '-' ' ' | awk '{print $2}'` if [ x$end_test = x ]; then msg="Extracting test $start_test" label="CTS-`date +"%a-%d-%b-%Y"`-$start_test" end_test=`expr $start_test + 1` else - msg="Extracting set $start_test to $end_test..." + msg="Extracting set $start_test to $end_test" label="CTS-`date +"%a-%d-%b-%Y"`-$start_test-$end_test" end_test=`expr $end_test + 1` fi if [ $start_test = 0 ]; then start_pat="BEGINNING [0-9].* TESTS" else start_pat="Running test.*\[ *$start_test\]" fi - ctslog=`findmsg 1 "$start_pat"` + + if [ x$ctslog = x ]; then + ctslog=`findmsg 1 "$start_pat"` + fi + if [ x$ctslog = x ]; then + fatal "No CTS control file detected" + fi + line=`grep -n "$start_pat" $ctslog | tail -1 | sed 's/:.*//'` - start_time=`linetime $ctslog $line` + if [ ! -z "$line" ]; then + start_time=`linetime $ctslog $line` + fi - ctslog=`findmsg 1 "Running test.*\[ *$end_test\]"` line=`grep -n "Running test.*\[ *$end_test\]" $ctslog | tail -1 | sed 's/:.*//'` - end_time=`linetime $ctslog $line` + if [ ! -z "$line" ]; then + end_time=`linetime $ctslog $line` + fi + if [ -z "$nodes" ]; then + debug "Using CTS control file: $ctslog" + nodes=`grep CTS: $ctslog | grep -v debug: | grep " \* " | sed s:.*\\\*::g | sort -u | tr '\\n' ' '` + fi + if [ $end_time -lt $start_time ]; then debug "Test didn't complete, grabbing everything up to now" end_time=`date +%s` fi - log "$msg (`time2str $start_time` to `time2str $end_time`)" - collect_data $label $start_time $end_time $ctslog + if [ $start_time != 0 ];then + log "$msg (`time2str $start_time` to `time2str $end_time`)" + collect_data $label $start_time $end_time $ctslog + else + fatal "$msg failed: not found" + fi done } getnodes() { if [ -z $1 ]; then cluster=`get_cluster_type` else cluster=$1 fi cluster_cf=`find_cluster_cf $cluster` # 1. Live if ps -ef | egrep -qs [c]rmd then debug "Querying CRM for nodes" cibadmin -Ql -o nodes | awk ' /type="normal"/ { for( i=1; i<=NF; i++ ) if( $i~/^uname=/ ) { sub("uname=.","",$i); sub("\".*","",$i); print $i; next; } } ' # 2. hostcache elif [ -f $HA_STATE_DIR/hostcache ]; then debug "Reading nodes from $HA_STATE_DIR/hostcache" awk '{print $1}' $HA_STATE_DIR/hostcache # 3. ha.cf elif [ "x$cluster" = "xheartbeat" ]; then debug "Reading nodes from $cluster_cf" getcfvar $cluster node $cluster_cf else # Look in the logs... logfile=`findmsg 1 "crm_update_peer"` debug "Reading nodes from $logfile" if [ ! -z "$logfile" ]; then grep crm_update_peer: $logfile | sed s/.*crm_update_peer// | sed s/://g | awk '{print $2}' | grep -v "(null)" | sort -u | tr '\n' ' ' fi fi } if [ "x$tests" != "x" ]; then do_cts elif [ "x$start_time" != "x" ]; then masterlog="" + + if [ -z "$sanitize_patterns" ]; then + log "WARNING: The tarball produced by this program may contain" + log " sensitive information such as passwords." + log "" + log "We will attempt to remove such information if you use the" + log "-p option. For example: -p \"pass.*\" -p \"user.*\"" + log "" + log "However, doing this may reduce the ability for the recipients" + log "to diagnose issues and generally provide assistance." + log "" + log "IT IS YOUR RESPONSIBILITY TO PROTECT SENSITIVE DATA FROM EXPOSURE" + log "" + fi + if [ -z "$nodes" ]; then nodes=`getnodes $cluster` log "Calculated node list: $nodes" fi if [ -z "$nodes" ]; then fatal "Cannot determine node list, please specify manually with --nodes" fi if echo $nodes | grep -qs $host then debug "We are a cluster node" else debug "We are a log master" masterlog=`findmsg 1 "crmd\\|CTS"` fi if [ -z $end_time ]; then end_time=`perl -e 'print time()'` fi label="pcmk-`date +"%a-%d-%b-%Y"`" log "Collecting data from $nodes (`time2str $start_time` to `time2str $end_time`)" collect_data $label $start_time $end_time $masterlog else - fatal "Not sure what to do, no tests or times to extract" + fatal "Not sure what to do, no tests or time ranges to extract" fi diff --git a/tools/hb2openais.sh.in b/tools/hb2openais.sh.in index 2acd9816bc..6723c3d789 100755 --- a/tools/hb2openais.sh.in +++ b/tools/hb2openais.sh.in @@ -1,804 +1,804 @@ #!/bin/bash # Copyright (C) 2008,2009 Dejan Muhamedagic # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public # License as published by the Free Software Foundation; either # version 2.1 of the License, or (at your option) any later version. # # This software is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # General Public License for more details. # # You should have received a copy of the GNU General Public # License along with this library; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA # . @sysconfdir@/ha.d/shellfuncs # utillib.sh moved (sigh!) # cluster-glue doesn't make its shared data dir available # we guess, and guess that that's safe, that the datadir is the same testdirs="@datadir@/cluster-glue $HA_NOARCHBIN/utillib.sh" for d in $testdirs; do if [ -f $d/utillib.sh ]; then NOARCH_DIR=$d break fi done test -f $NOARCH_DIR/utillib.sh || { echo "sorry, could not find utillib.sh in $testdirs" exit 1 } . $NOARCH_DIR/utillib.sh . $NOARCH_DIR/ha_cf_support.sh PROG=`basename $0` PROGDIR=`dirname $0` # the default syslog facility is not (yet) exported by heartbeat # to shell scripts # DEFAULT_HA_LOGFACILITY="daemon" export DEFAULT_HA_LOGFACILITY : ${SSH_OPTS="-T"} usage() { cat</dev/null else ssh -T -o Batchmode=yes $1 true 2>/dev/null fi } findsshuser() { for u in "" $TRY_SSH; do rc=0 for n in `getnodes`; do [ "$node" = "$WE" ] && continue testsshuser $n $u || { rc=1 break } done if [ $rc -eq 0 ]; then echo $u return 0 fi done return 1 } important() { echo "IMPORTANT: $*" >&2 } newportinfo() { important "the multicast port number on $1 is set to $2" important "please update your firewall rules (if any)" } changemediainfo() { important "$PRODUCT uses multicast for communication" important "please make sure that your network infrastructure supports it" } multicastinfo() { info "multicast for $PRODUCT ring $1 set to $2:$3" } netaddrinfo() { info "network address for $PRODUCT ring $1 set to $2" } backup_files() { [ "$TEST_DIR" ] && return info "backing up $BACKUP_FILES to $BACKUPDIR" $DRY mkdir $BACKUPDIR || { echo sorry, could not create $BACKUPDIR directory echo please cleanup exit 1 } if [ -z "$DRY" ]; then tar cf - $BACKUP_FILES | gzip > $BACKUPDIR/$WE.tar.gz || { echo sorry, could not create $BACKUPDIR/$WE.tar.gz exit 1 } else $DRY "tar cf - $BACKUP_FILES | gzip > $BACKUPDIR/$WE.tar.gz" fi } revert() { [ "$TEST_DIR" ] && return test -d $BACKUPDIR || { echo sorry, there is no $BACKUPDIR directory echo cannot revert exit 1 } info "restoring $BACKUP_FILES from $BACKUPDIR/$WE.tar.gz" gzip -dc $BACKUPDIR/$WE.tar.gz | (cd / && tar xf -) || { echo sorry, could not unpack $BACKUPDIR/$WE.tar.gz exit 1 } } pls_press_enter() { [ "$TEST_DIR" ] && return cat</dev/null | prochbmedia 2>/dev/null | sort -u | wc -l` if [ $mediacnt -ge 2 ]; then setvalue rrp_mode $RRP_MODE fi changemediainfo endstanza # the logging stanza getlogvars # enforce some syslog facility [ "$COROSYNC" ] && TO_FILE=to_logfile || TO_FILE=to_file debugsetting=`setdebug` newstanza logging setvalue debug $debugsetting setvalue fileline off setvalue to_stderr no setvalue timestamp off if [ "$HA_LOGFILE" ]; then setvalue $TO_FILE yes setvalue logfile $HA_LOGFILE else setvalue $TO_FILE no fi if [ "$HA_LOGFACILITY" ]; then setvalue to_syslog yes setvalue syslog_facility $HA_LOGFACILITY else setvalue to_syslog no fi newstanza logger_subsys setvalue subsys AMF setvalue debug $debugsetting endstanza endstanza newstanza amf setvalue mode disabled endstanza } if [ -z "$DRY" ]; then openaisconf > $AIS_CONF || fatal "cannot create $AIS_CONF" grep -wqs interface $AIS_CONF || fatal "no media found in $HA_CF" else openaisconf fi [ "$AIS_KEYF" ] && { info "Generating a key for OpenAIS authentication ..." if [ "$TEST_DIR" ]; then echo would run: $DRY $KEYGEN_PROG else $DRY $KEYGEN_PROG || fatal "cannot generate the key using $KEYGEN_PROG" fi } # remove various files which could get in a way if [ -z "$TEST_DIR" ]; then $DRY rm -f $RM_FILES fi fixcibperms() { [ "$TEST_DIR" ] && return uid=`ls -ldn $CRM_VARLIB | awk '{print $3}'` gid=`ls -ldn $CRM_VARLIB | awk '{print $4}'` $DRY $MYSUDO chown $uid:$gid $CIB } upgrade_cib() { $DRY $MYSUDO cibadmin --upgrade --force $DRY $MYSUDO crm_verify -V -x $CIB_file } py_proc_cib() { tmpfile=`maketempfile` $MYSUDO sh -c "python $PY_HELPER $* <$CIB >$tmpfile" || fatal "cannot process cib: $PY_HELPER $*" $DRY $MYSUDO mv $tmpfile $CIB } set_property() { py_proc_cib set_property $* } # remove the nodes section from the CIB py_proc_cib set_node_ids info "Edited the nodes' ids in the CIB" numnodes=`getnodes | wc -w` [ $numnodes -eq 2 ] && set_property no-quorum-policy ignore set_property expected-nodes $numnodes overwrite info "Done converting ha.cf to $AIS_CONF_BASE" important "Please check the resulting $AIS_CONF" important "and in particular interface stanzas and logging." important "If you find problems, please edit $AIS_CONF now!" # # first part done (openais), on to the CIB analyze_cib() { info "Analyzing the CIB..." $MYSUDO sh -c "python $PY_HELPER analyze_cib <$CIB" } check_respawns() { rc=1 for p in $SUPPORTED_RESPAWNS; do grep -qs "^respawn.*$p" $HA_CF && { info "a $p resource has to be created" rc=0 } done return $rc } part2() { intro_part2 || return 0 opts="-c $HA_CF" [ "$TEST_DIR" ] && opts="-T $opts" py_proc_cib $opts convert_cib info "Processed the CIB successfully" } # make the user believe that something's happening :) some_dots_idle() { [ "$TEST_DIR" ] && return cnt=0 printf "$2 ." while [ $cnt -lt $1 ]; do sleep 1 printf "." ctn=$((cnt+1)) done echo } print_dc() { crm_mon -1 | awk '/Current DC/{print $3}' } dcidle() { dc=`$MYSUDO print_dc` if [ "$dc" = "$WE" ]; then maxcnt=60 cnt=0 while [ $cnt -lt $maxcnt ]; do stat=`$MYSUDO crmadmin -S $dc` echo $stat | grep -qs S_IDLE && break [ "$1" = "-v" ] && echo $stat sleep 1 printf "." cnt=$((cnt+1)) done echo $stat | grep -qs S_IDLE else some_dots_idle 10 #just wait for 10 seconds fi } wait_crm() { [ "$TEST_DIR" ] && return cnt=10 dc="" while [ -z "$dc" -a $cnt -gt 0 ]; do dc=`$MYSUDO print_dc` cnt=$((cnt-1)) done if [ x = x"$dc" ]; then echo "sorry, no dc found/elected" exit 1 fi dcidle } manage_cluster() { if [ "$TEST_DIR" ]; then echo would run: /etc/init.d/openais $1 else $DRY /etc/init.d/openais $1 fi } tune_ocfs2() { cat< $MAN_TARF) fi diff --git a/tools/report.collector b/tools/report.collector index c708723372..930047e433 100644 --- a/tools/report.collector +++ b/tools/report.collector @@ -1,651 +1,654 @@ # Copyright (C) 2007 Dejan Muhamedagic # Almost everything as part of hb_report # Copyright (C) 2010 Andrew Beekhof # Cleanups, refactoring, extensions # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public # License as published by the Free Software Foundation; either # version 2.1 of the License, or (at your option) any later version. # # This software is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # General Public License for more details. # # You should have received a copy of the GNU General Public # License along with this library; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA # if echo $REPORT_HOME | grep -qs '^/' then - debug "Using full path to working directory" + debug "Using full path to working directory: $REPORT_HOME" else - debug "Canonicalizing working directory path" REPORT_HOME="$HOME/$REPORT_HOME" + debug "Canonicalizing working directory path: $REPORT_HOME" fi findlogdcf() { for f in \ `test -x $CRM_DAEMON_DIR/ha_logd && which strings > /dev/null 2>&1 && strings $CRM_DAEMON_DIR/ha_logd | grep 'logd\.cf'` \ `for d; do echo $d/logd.cf $d/ha_logd.cf; done` do if [ -f "$f" ]; then echo $f debug "Located logd.cf at: $f" return 0 fi done debug "Could not determine logd.cf location" return 1 } # # find files newer than a and older than b # isnumber() { echo "$*" | grep -qs '^[0-9][0-9]*$' } touchfile() { t=`mktemp` && perl -e "\$file=\"$t\"; \$tm=$1;" -e 'utime $tm, $tm, $file;' && echo $t } find_files_clean() { [ -z "$from_stamp" ] || rm -f "$from_stamp" [ -z "$to_stamp" ] || rm -f "$to_stamp" from_stamp="" to_stamp="" } find_files() { dirs=$1 from_time=$2 to_time=$3 isnumber "$from_time" && [ "$from_time" -gt 0 ] || { warning "sorry, can't find files in [ $1 ] based on time if you don't supply time" return } trap find_files_clean 0 if ! from_stamp=`touchfile $from_time`; then warning "sorry, can't create temporary file for find_files" return fi findexp="-newer $from_stamp" if isnumber "$to_time" && [ "$to_time" -gt 0 ]; then if ! to_stamp=`touchfile $to_time`; then warning "sorry, can't create temporary file for find_files" find_files_clean return fi findexp="$findexp ! -newer $to_stamp" fi find $dirs -type f $findexp find_files_clean trap "" 0 } # # check permissions of files/dirs # pl_checkperms() { perl -e ' # check permissions and ownership # uid and gid are numeric # everything must match exactly # no error checking! (file should exist, etc) ($filename, $perms, $in_uid, $in_gid) = @ARGV; ($mode,$uid,$gid) = (stat($filename))[2,4,5]; $p=sprintf("%04o", $mode & 07777); $p ne $perms and exit(1); $uid ne $in_uid and exit(1); $gid ne $in_gid and exit(1); ' $* } num_id() { getent $1 $2 | awk -F: '{print $3}' } chk_id() { [ "$2" ] && return 0 echo "$1: id not found" return 1 } check_perms() { while read type f p uid gid; do [ -$type $f ] || { echo "$f wrong type or doesn't exist" continue } n_uid=`num_id passwd $uid` chk_id "$uid" "$n_uid" || continue n_gid=`num_id group $gid` chk_id "$gid" "$n_gid" || continue pl_checkperms $f $p $n_uid $n_gid || { echo "wrong permissions or ownership for $f:" ls -ld $f } done } # # coredumps # findbinary() { random_binary=`which cat 2>/dev/null` # suppose we are lucky binary=`gdb $random_binary $1 < /dev/null 2>/dev/null | grep 'Core was generated' | awk '{print $5}' | sed "s/^.//;s/[.':]*$//"` if [ x = x"$binary" ]; then debug "Could not detect the program name for core $1 from the gdb output; will try with file(1)" binary=$(file $1 | awk '/from/{ for( i=1; i<=NF; i++ ) if( $i == "from" ) { print $(i+1) break } }') binary=`echo $binary | tr -d "'"` binary=$(echo $binary | tr -d '`') if [ "$binary" ]; then binary=`which $binary 2>/dev/null` fi fi if [ x = x"$binary" ]; then warning "Could not find the program path for core $1" return fi fullpath=`which $binary 2>/dev/null` if [ x = x"$fullpath" ]; then if [ -x $CRM_DAEMON_DIR/$binary ]; then echo $CRM_DAEMON_DIR/$binary debug "Found the program at $CRM_DAEMON_DIR/$binary for core $1" else warning "Could not find the program path for core $1" fi else echo $fullpath debug "Found the program at $fullpath for core $1" fi } getbt() { which gdb > /dev/null 2>&1 || { warning "Please install gdb to get backtraces" return } for corefile; do absbinpath=`findbinary $corefile` [ x = x"$absbinpath" ] && continue echo "====================== start backtrace ======================" ls -l $corefile gdb -batch -n -quiet -ex ${BT_OPTS:-"thread apply all bt full"} -ex quit \ $absbinpath $corefile 2>/dev/null echo "======================= end backtrace =======================" done } getconfig() { target=$1; shift; for cf in $*; do - if [ -f "$cf" ]; then - cp -p "$cf" $target/ + if [ -e "$cf" ]; then + cp -a "$cf" $target/ fi done crm_uuid -r > $target/$HB_UUID_F 2>&1 if ps -ef | egrep -qs [c]rmd then crm_mon -1 2>&1 | grep -v '^Last upd' > $target/$CRM_MON_F - cibadmin -Ql > $target/${CIB_F}.live + cibadmin -Ql 2>/dev/null > $target/${CIB_F}.live crm_node -p > $target/$MEMBERSHIP_F 2>&1 echo "$host" > $target/RUNNING else echo "$host" > $target/STOPPED fi if [ -f "$target/$CIB_F" ]; then crm_verify -V -x $target/$CIB_F >$target/$CRM_VERIFY_F 2>&1 CIB_file=$target/$CIB_F crm configure show >$target/$CIB_TXT_F 2>&1 fi } # # remove values of sensitive attributes # # this is not proper xml parsing, but it will work under the # circumstances sanitize_xml_attrs() { sed $( for patt in $SANITIZE; do echo "-e /name=\"$patt\"/s/value=\"[^\"]*\"/value=\"****\"/" done ) } sanitize_hacf() { awk ' $1=="stonith_host"{ for( i=5; i<=NF; i++ ) $i="****"; } {print} ' } sanitize_one_clean() { [ -z "$tmp" ] || rm -f "$tmp" tmp="" [ -z "$ref" ] || rm -f "$ref" ref="" } sanitize() { file=$1 compress="" if [ -z $SANITIZE ]; then return fi echo $file | grep -qs 'gz$' && compress=gzip echo $file | grep -qs 'bz2$' && compress=bzip2 if [ "$compress" ]; then decompress="$compress -dc" else compress=cat decompress=cat fi trap sanitize_one_clean 0 tmp=`mktemp` ref=`mktemp` if [ -z "$tmp" -o -z "$ref" ]; then sanitize_one_clean fatal "cannot create temporary files" fi touch -r $file $ref # save the mtime if [ "`basename $file`" = ha.cf ]; then sanitize_hacf else $decompress | sanitize_xml_attrs | $compress fi < $file > $tmp mv $tmp $file # note: cleaning $tmp up is still needed even after it's renamed # because its temp directory is still there. touch -r $ref $file sanitize_one_clean trap "" 0 } pickfirst() { for x; do which $x >/dev/null 2>&1 && { echo $x return 0 } done return 1 } # # get some system info # distro() { if which lsb_release >/dev/null 2>&1 then lsb_release -d debug "Using lsb_release for distribution info" return fi relf=`ls /etc/debian_version 2>/dev/null` || relf=`ls /etc/slackware-version 2>/dev/null` || relf=`ls -d /etc/*-release 2>/dev/null` && { for f in $relf; do test -f $f && { echo "`ls $f` `cat $f`" debug "Found `echo $relf | tr '\n' ' '` distribution release file(s)" return } done } warning "No lsb_release, no /etc/*-release, no /etc/debian_version: no distro information" } pkg_ver() { if which dpkg >/dev/null 2>&1 ; then pkg_mgr="deb" elif which rpm >/dev/null 2>&1 ; then pkg_mgr="rpm" elif which pkg_info >/dev/null 2>&1 ; then pkg_mgr="pkg_info" elif which pkginfo >/dev/null 2>&1 ; then pkg_mgr="pkginfo" else warning "Unknown package manager" return fi debug "The package manager is: $pkg_mgr" echo "The package manager is: $pkg_mgr" # for Linux .deb based systems case $pkg_mgr in deb) dpkg-query -f '${Package} ${Version} ${Architecture}\n' -W for pkg in $*; do if dpkg-query -W $pkg 2>/dev/null ; then debug "Verifying installation of: $pkg" echo "Verifying installation of: $pkg" debsums -s $pkg 2>/dev/null fi done ;; rpm) rpm -qa --qf '%{name} %{version}-%{release} - %{distribution} %{arch}\n' for pkg in $*; do if rpm -q $pkg >/dev/null 2>&1 ; then debug "Verifying installation of: $pkg" echo "Verifying installation of: $pkg" rpm --verify $pkg fi done ;; pkg_info) pkg_info ;; pkginfo) pkginfo | awk '{print $3}' # format? ;; esac } getbacktraces() { debug "Looking for backtraces: $*" flist=$( for f in `find_files "$CRM_CORE_DIRS" $1 $2`; do bf=`basename $f` test `expr match $bf core` -gt 0 && echo $f done) if [ "$flist" ]; then log "Found core files: `echo $flist | tr '\n' ' '`" getbt "$flist" fi } getpeinputs() { flist=$( find_files $PE_STATE_DIR $1 $2 | sed "s,`dirname $PE_STATE_DIR`/,,g" ) if [ "$flist" ]; then (cd `dirname $PE_STATE_DIR` && tar cf - $flist) | (cd $3 && tar xf -) debug "found `echo $flist | wc -w` pengine input files in $PE_STATE_DIR" fi } # # some basic system info and stats # sys_info() { cluster=$1; shift echo "Platform: `uname`" echo "Kernel release: `uname -r`" echo "Architecture: `uname -m`" if [ `uname` = Linux ]; then echo "Distribution: `distro`" fi cibadmin --version 2>&1 cibadmin -! 2>&1 case $1 in openais) : echo "openais version: how?" ;; corosync) /usr/sbin/corosync -v 2>&1 ;; heartbeat) heartbeat version: `$CRM_DAEMON_DIR/heartbeat -V` 2>&1 ;; esac - # TODO: Get cluster-glue build version + # Cluster glue version hash (if available) + stonith -V + + # Resource agents version hash echo "resource-agents: `grep 'Build version:' /usr/lib/ocf/resource.d/heartbeat/.ocf-shellfuncs`" pkg_ver $* } sys_stats() { set -x uname -n uptime ps axf ps auxw top -b -n 1 ifconfig -a ip addr list netstat -i arp -an test -d /proc && { cat /proc/cpuinfo } lsscsi lspci mount df set +x } dlm_dump() { if which dlm_tool >/dev/null 2>&1 ; then echo NOTICE - Lockspace overview: dlm_tool ls dlm_tool ls | grep name | while read X N ; do echo NOTICE - Lockspace $N: dlm_tool lockdump $N done echo NOTICE - Lockspace history: dlm_tool dump fi } iscfvarset() { test "`getcfvar $1 $2`" } iscfvartrue() { getcfvar $1 $2 $3 | egrep -qsi "^(true|y|yes|on|1)" } uselogd() { cf_file=$2 case $1 in heartbeat) iscfvartrue $1 use_logd $cf_file && return 0 # if use_logd true iscfvarset $1 logfacility $cf_file || iscfvarset $1 logfile $cf_file || iscfvarset $1 debugfile $cf_file || return 0 # or none of the log options set false ;; *) iscfvartrue $1 use_logd $cf_file ;; esac } get_logfile() { cf_type=$1 cf_file="$2" cf_logd="$3" facility_var="logfacility" if [ -f "$cf_logd" ]; then if uselogd; then cf_file="$cf_logd" cf_type="logd" fi fi debug "Reading $cf_type log settings" case $cf_type in openais|corosync) debug "Reading log settings from $cf_file" if iscfvartrue $cf_type to_syslog $cf_file; then facility_var=syslog_facility elif iscfvartrue $cf_type to_file $cf_file; then logfile=`getcfvar $cf_type logfile $cf_file syslog_facility` fi ;; heartbeat|logd) debug "Reading log settings from $cf_file" if iscfvartrue $cf_type debug $cf_file then logfile=`getcfvar $cf_type debugfile $cf_file` else logfile=`getcfvar $cf_type logfile $cf_file` fi ;; *) debug "Unknown cluster type: $cf_type" echo "/var/log/messages" ;; esac if [ "x$logfile" != "x" -a -f "$logfile" ]; then echo $logfile else facility=`getcfvar $cf_type $facility_var $cf_file` [ "" = "$facility" ] && facility="daemon" if [ "none" = "$facility" ]; then fatal "No logging is configured" fi msg="Mark:pcmk:`perl -e 'print time()'`" logger -p $facility.info $msg >/dev/null 2>&1 findmsg 1 "$msg" fi } essential_files() { cat< $SYSINFO_F essential_files $cluster | check_perms > $PERMISSIONS_F 2>&1 -getconfig "$REPORT_HOME/$host" "$cluster_cf" "$logd_cf" "$HA_STATE_DIR/crm/$CIB_F" "$HA_STATE_DIR/hostcache" - -dlm_dump > $DLM_DUMP_F 2>&1 -sys_stats > $SYSSTATS_F 2>&1 +getconfig "$REPORT_HOME/$host" "$cluster_cf" "$logd_cf" "$HA_STATE_DIR/crm/$CIB_F" "$HA_STATE_DIR/hostcache" "/etc/drbd.conf" "/etc/drbd.d" getpeinputs $LOG_START $LOG_END $REPORT_HOME/$host getbacktraces $LOG_START $LOG_END > $REPORT_HOME/$host/$BT_F dc=`crm_mon -1 2>/dev/null | awk '/Current DC/ {print $3}'` if [ "$host" = "$dc" ]; then echo "$host" > DC fi +dlm_dump > $DLM_DUMP_F 2>&1 +sys_stats > $SYSSTATS_F 2>&1 + debug "Sanitizing files" # # replace sensitive info with '****' # for f in `basename $cluster_cf` $CIB_F $CIB_TXT_F $CIB_F.live pengine/*; do if [ -f "$f" ]; then sanitize $f fi done # Grab logs #debug "Gathering logs: $logfile $EXTRA_LOGS" trap '[ -z "$pattfile" ] || rm -f "$pattfile"' 0 pattfile=`mktemp` || fatal "cannot create temporary files" for p in $LOG_PATTERNS; do echo "$p" done > $pattfile for l in $logfile $EXTRA_LOGS; do b=`basename $l` if [ ! -f "$l" ]; then # Not a file continue elif [ -f "$b" ]; then # We already have it continue fi dumplogset "$l" $LOG_START $LOG_END > "$b" echo "Log patterns $host:" > $ANALYSIS_F cat $b | grep -f $pattfile >> $ANALYSIS_F done rm -f $pattfile trap "" 0 # Purge files containing no information for f in `ls -1`; do if [ -d "$f" ]; then continue elif [ ! -s "$f" ]; then debug "Removing empty file: $f" rm -f $f fi done # Parse for events for l in $logfile $EXTRA_LOGS; do node_events `basename $logfile` > $EVENTS_F # Link the first logfile to a standard name if it doesn't yet exist if [ ! -e $HALOG_F ]; then ln -s `basename $l` $HALOG_F fi done if [ "$REPORT_MASTER" != "$host" ]; then debug "Streaming report back to $REPORT_MASTER" (cd $REPORT_HOME && tar cf - $host) if [ "$REMOVE" = "1" ]; then cd rm -rf $REPORT_HOME fi fi diff --git a/tools/report.common b/tools/report.common index b700a46df0..991c165110 100644 --- a/tools/report.common +++ b/tools/report.common @@ -1,629 +1,633 @@ # Copyright (C) 2007 Dejan Muhamedagic # Almost everything as part of hb_report # Copyright (C) 2010 Andrew Beekhof # Cleanups, refactoring, extensions # # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public # License as published by the Free Software Foundation; either # version 2.1 of the License, or (at your option) any later version. # # This software is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # General Public License for more details. # # You should have received a copy of the GNU General Public # License along with this library; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA # host=`uname -n` shorthost=`echo $host | sed s:\\\\..*::` if [ -z $verbose ]; then verbose=0 fi # Target Files EVENTS_F=events.txt ANALYSIS_F=analysis.txt DESCRIPTION_F=description.txt HALOG_F=cluster-log.txt BT_F=backtraces.txt SYSINFO_F=sysinfo.txt SYSSTATS_F=sysstats.txt DLM_DUMP_F=dlm_dump.txt CRM_MON_F=crm_mon.txt MEMBERSHIP_F=members.txt HB_UUID_F=hb_uuid.txt HOSTCACHE=hostcache CRM_VERIFY_F=crm_verify.txt PERMISSIONS_F=permissions.txt CIB_F=cib.xml CIB_TXT_F=cib.txt EVENT_PATTERNS=" state do_state_transition membership pcmk_peer_update.*(lost|memb): -quorum crmd.*ais.disp.*quorum.(lost|ac?quir) +quorum crmd.*crm_update_quorum|crmd.*ais.disp.*quorum.(lost|ac?quir) pause Process.pause.detected resources lrmd.*rsc:(start|stop) -stonith stonithd.*(requests|(Succeeded|Failed).to.STONITH|result=) -start_stop Executive.Service.RELEASE|crm_shutdown:.Requesting.shutdown|pcmk_shutdown:.Shutdown.complete +stonith te_fence_node|stonith-ng.*log_oper.*report|stonithd.*(requests|(Succeeded|Failed).to.STONITH|result=) +start_stop Starting.heartbeat|Corosync.Cluster.Engine|corosync.*Initializing.transport|Executive.Service.RELEASE|crm_shutdown:.Requesting.shutdown|pcmk_shutdown:.Shutdown.complete " PACKAGES="pacemaker pacemaker-libs libpacemaker3 pacemaker-pygui pacemaker-pymgmt pymgmt-client openais libopenais2 libopenais3 corosync libcorosync4 resource-agents cluster-glue-libs cluster-glue libglue2 ldirectord heartbeat heartbeat-common heartbeat-resources libheartbeat2 ocfs2-tools ocfs2-tools-o2cb ocfs2console ocfs2-kmp-default ocfs2-kmp-pae ocfs2-kmp-xen ocfs2-kmp-debug ocfs2-kmp-trace drbd drbd-kmp-xen drbd-kmp-pae drbd-kmp-default drbd-kmp-debug drbd-kmp-trace drbd-heartbeat drbd-pacemaker drbd-utils drbd-bash-completion drbd-xen lvm2 lvm2-clvm cmirrord libdlm libdlm2 libdlm3 hawk ruby lighttpd kernel-default kernel-pae kernel-xen glibc " # # keep the user posted # log() { printf "%-10s $*\n" "$shorthost:" 1>&2 } debug() { if [ $verbose -gt 0 ]; then log "Debug: $*" fi } info() { log "$*" } warning() { log "WARN: $*" } fatal() { log "ERROR: $*" exit 1 } detect_host() { local_state_dir=/var if [ -d $local_state_dir/run/crm ]; then CRM_STATE_DIR=$local_state_dir/run/crm else for d in `find / -type d -name run`; do if [ -d $d/crm ]; then CRM_STATE_DIR=$d/crm local_state_dir=`dirname $d` fi done fi if [ ! -d $CRM_STATE_DIR ]; then fatal "Non-standard Pacemaker installation: State directory not found" fi debug "Machine state directory: $local_state_dir" debug "State files located in: $CRM_STATE_DIR" if [ -d $local_state_dir/lib/pengine ]; then PE_STATE_DIR=$local_state_dir/lib/pengine else for d in `find / -type d -name pengine`; do PE_STATE_DIR=$d break done fi if [ -z $PE_STATE_DIR ]; then fatal "Non-standard Pacemaker installation: Policy Engine directory not found" fi debug "PE files located in: $PE_STATE_DIR" HA_STATE_DIR=$local_state_dir/lib/heartbeat if [ ! -d $HA_STATE_DIR ]; then # TODO: Go looking fatal "Non-standard Heartbeat installation: Heartbeat state directory not found" fi debug "Heartbeat state files located in: $HA_STATE_DIR" CRM_CORE_DIRS="" for d in $HA_STATE_DIR/cores $local_state_dir/lib/corosync $local_state_dir/lib/openais; do if [ -d $d ]; then CRM_CORE_DIRS="$CRM_CORE_DIRS $d" fi done debug "Core files located under: $CRM_CORE_DIRS" for d in /usr/lib/heartbeat /usr/lib64/heartbeat; do if [ -f $d/crmd ]; then CRM_DAEMON_DIR=$d break fi done if [ ! -d $CRM_DAEMON_DIR ]; then for d in `find / -type d -name heartbeat`; do if [ -f $d/crmd ]; then CRM_DAEMON_DIR=$d break fi done fi if [ ! -d $CRM_DAEMON_DIR ]; then for f in `find / -type f -name crmd`; do if [ -f $f ]; then CRM_DAEMON_DIR=`basename $f` fi done fi if [ ! -d $CRM_DAEMON_DIR ]; then fatal "Non-standard Pacemaker installation: daemons not found" fi debug "Pacemaker daemons located under: $CRM_DAEMON_DIR" } time2str() { perl -e "use POSIX; print strftime('%x %X',localtime($1));" } get_time() { perl -e "\$time='$*';" -e ' eval "use Date::Parse"; if (!$@) { print str2time($time); } else { eval "use Date::Manip"; if (!$@) { print UnixDate(ParseDateString($time), "%s"); } } ' } get_time_() { warning "No time format specified for: $*" } get_time_syslog() { awk '{print $1,$2,$3}' } get_time_legacy() { awk '{print $2}' | sed 's/_/ /' } get_time_format() { t=0 l="" func="" trycnt=10 while [ $trycnt -gt 0 ] && read l; do t=$(get_time `echo $l | get_time_syslog`) if [ "$t" ]; then func="syslog" break fi t=$(get_time `echo $l | get_time_legacy`) if [ "$t" ]; then func="legacy" break fi trycnt=$(($trycnt-1)) done #debug "Logfile uses the $func time format" echo $func } linetime() { format=`get_time_format < $1` l=`tail -n +$2 $1 | grep ":[0-5][0-9]:" | head -1 | get_time_$format` get_time "$l" } # Find pattern in a logfile somewhere # Return $max ordered results by age (newest first) findmsg() { max=$1 pattern=$2 logfiles="" syslogdirs="/var/log /var/logs /var/syslog /var/adm /var/log/ha /var/log/cluster" for d in $syslogdirs; do if [ -d $d ]; then logfiles=`grep -l -e "$pattern" $d/*` && break fi done 2>/dev/null if [ "x$logfiles" != "x" ]; then list=`ls -t $logfiles | head -n $max | tr '\n' ' '` echo $list debug "Pattern \'$pattern\' found in: [ $list ]" else debug "Pattern \'$pattern\' not found anywhere" fi } node_events() { Epatt=`echo "$EVENT_PATTERNS" | while read title p; do [ -n "$p" ] && echo -n "|$p"; done | sed 's/.//' ` grep -E "$Epatt" $1 } pickfirst() { for x; do which $x >/dev/null 2>&1 && { echo $x return 0 } done return 1 } shrink() { src=$* target=$1.tar tar_options=cf variant=`pickfirst bzip2 gzip false` case $variant in bz*) tar_options="jcf" target="$target.bz2" ;; gz*) tar_options="zcf" target="$target.gz" ;; *) warning "Could not find a compression program, the resulting tarball may be huge" ;; esac tar $tar_options $target $src >/dev/null 2>&1 echo $target } findln_by_time() { local logf=$1 local tm=$2 local first=1 local last=`wc -l < $logf` while [ $first -le $last ]; do mid=$((($last+$first)/2)) trycnt=10 while [ $trycnt -gt 0 ]; do tmid=`linetime $logf $mid` [ "$tmid" ] && break warning "cannot extract time: $logf:$mid; will try the next one" trycnt=$(($trycnt-1)) # shift the whole first-last segment first=$(($first-1)) last=$(($last-1)) mid=$((($last+$first)/2)) done if [ -z "$tmid" ]; then warning "giving up on log..." return fi if [ $tmid -gt $tm ]; then last=$(($mid-1)) elif [ $tmid -lt $tm ]; then first=$(($mid+1)) else break fi done echo $mid } dumplog() { local logf=$1 local from_line=$2 local to_line=$3 [ "$from_line" ] || return tail -n +$from_line $logf | if [ "$to_line" ]; then head -$(($to_line-$from_line+1)) else cat fi } # # find log/set of logs which are interesting for us # # # find log slices # find_decompressor() { if echo $1 | grep -qs 'bz2$'; then echo "bzip2 -dc" elif echo $1 | grep -qs 'gz$'; then echo "gzip -dc" else echo "cat" fi } # # check if the log contains a piece of our segment # is_our_log() { local logf=$1 local from_time=$2 local to_time=$3 local cat=`find_decompressor $logf` local format=`$cat $logf | get_time_format` local first_time=$(get_time "`$cat $logf | head -1 | get_time_$format`") local last_time=$(get_time "`$cat $logf | tail -1 | get_time_$format`") if [ x = "x$first_time" -o x = "x$last_time" ]; then return 0 # skip (empty log?) fi if [ $from_time -gt $last_time ]; then # we shouldn't get here anyway if the logs are in order return 2 # we're past good logs; exit fi if [ $from_time -ge $first_time ]; then return 3 # this is the last good log fi # have to go further back if [ x = "x$to_time" -o $to_time -ge $first_time ]; then return 1 # include this log else return 0 # don't include this log fi } # # go through archived logs (timewise backwards) and see if there # are lines belonging to us # (we rely on untouched log files, i.e. that modify time # hasn't been changed) # arch_logs() { local logf=$1 local from_time=$2 local to_time=$3 # look for files such as: ha-log-20090308 or # ha-log-20090308.gz (.bz2) or ha-log.0, etc ls -t $logf $logf*[0-9z] 2>/dev/null | while read next_log; do is_our_log $next_log $from_time $to_time case $? in 0) ;; # noop, continue 1) echo $next_log # include log and continue debug "Found log $next_log" ;; 2) break;; # don't go through older logs! 3) echo $next_log # include log and continue debug "Found log $next_log" break ;; # don't go through older logs! esac done } # # print part of the log # drop_tmp_file() { [ -z "$tmp" ] || rm -f "$tmp" } print_logseg() { local logf=$1 local from_time=$2 local to_time=$3 # uncompress to a temp file (if necessary) local cat=`find_decompressor $logf` if [ "$cat" != "cat" ]; then tmp=`mktemp` $cat $logf > $tmp trap drop_tmp_file 0 sourcef=$tmp else sourcef=$logf tmp="" fi if [ "$from_time" = 0 ]; then FROM_LINE=1 else FROM_LINE=`findln_by_time $sourcef $from_time` fi if [ -z "$FROM_LINE" ]; then warning "couldn't find line for time $from_time; corrupt log file?" return fi TO_LINE="" if [ "$to_time" != 0 ]; then TO_LINE=`findln_by_time $sourcef $to_time` if [ -z "$TO_LINE" ]; then warning "couldn't find line for time $to_time; corrupt log file?" return fi fi - dumplog $sourcef $FROM_LINE $TO_LINE - log "Including segment [$FROM_LINE-$TO_LINE] from $logf" + if [ $FROM_LINE -lt $TO_LINE ]; then + dumplog $sourcef $FROM_LINE $TO_LINE + log "Including segment [$FROM_LINE-$TO_LINE] from $logf" + else + log "Segment from $logf finished before it started, line: $FROM_LINE to $TO_LINE" + fi drop_tmp_file trap "" 0 } # # find log/set of logs which are interesting for us # dumplogset() { local logf=$1 local from_time=$2 local to_time=$3 local logf_set=`arch_logs $logf $from_time $to_time` if [ x = "x$logf_set" ]; then return fi local num_logs=`echo "$logf_set" | wc -l` local oldest=`echo $logf_set | awk '{print $NF}'` local newest=`echo $logf_set | awk '{print $1}'` local mid_logfiles=`echo $logf_set | awk '{for(i=NF-1; i>1; i--) print $i}'` # the first logfile: from $from_time to $to_time (or end) # logfiles in the middle: all # the last logfile: from beginning to $to_time (or end) case $num_logs in 1) print_logseg $newest $from_time $to_time;; *) print_logseg $oldest $from_time 0 for f in $mid_logfiles; do `find_decompressor $f` $f debug "including complete $f logfile" done print_logseg $newest 0 $to_time ;; esac } # cut out a stanza getstanza() { awk -v name="$1" ' !in_stanza && NF==2 && /^[a-z][a-z]*[[:space:]]*{/ { # stanza start if ($1 == name) in_stanza = 1 } in_stanza { print } in_stanza && NF==1 && $1 == "}" { exit } ' } # supply stanza in $1 and variable name in $2 # (stanza is optional) getcfvar() { cf_type=$1; shift; cf_var=$1; shift; cf_file=$* [ -f "$cf_file" ] || return case $cf_type in corosync|openais) sed 's/#.*//' < $cf_file | if [ $# -eq 2 ]; then getstanza "$cf_var" shift 1 else cat fi | awk -v varname="$cf_var" ' NF==2 && match($1,varname":$")==1 { print $2; exit; } ' ;; heartbeat) sed 's/#.*//' < $cf_file | grep -w "^$cf_var" | sed 's/^[^[:space:]]*[[:space:]]*//' ;; logd) sed 's/#.*//' < $cf_file | grep -w "^$cf_var" | sed 's/^[^[:space:]]*[[:space:]]*//' ;; esac } # # figure out the cluster type, depending on the process list # and existence of configuration files # get_cluster_type() { if ps -ef | egrep -qs '[c]orosync' then stack="corosync" elif ps -ef | egrep -qs '[a]isexec' then stack="openais" elif - ps -ef | egrep -qs '[h]eartbeat' + ps -ef | grep -v -e grep -e "eartbeat/[clasp]" | egrep -qs '[h]eartbeat' then stack="heartbeat" # Now we're guessing... # TODO: Technically these could be anywhere :-/ elif [ -f /etc/corosync/corosync.conf ]; then stack="corosync" elif [ -f /etc/ais/openais.conf ]; then stack="openais" else stack="heartbeat" fi - debug "Detected the $stack cluster stack" + debug "Detected the '$stack' cluster stack" echo $stack } find_cluster_cf() { case $1 in corosync) best_size=0 best_file="" # TODO: Technically these could be anywhere :-/ for cf in /etc/ais/openais.conf /etc/corosync/corosync.conf; do if [ -f $cf ]; then size=`wc -l $cf | awk '{print $1}'` if [ $size -gt $best_size ]; then best_size=$size best_file=$cf fi fi done echo "$best_file" ;; openais) # TODO: Technically it could be anywhere :-/ cf="/etc/ais/openais.conf" if [ -f $cf ]; then echo "$cf" fi ;; heartbeat) cf="/etc/ha.d/ha.cf" if [ -f $cf ]; then echo "$cf" fi ;; *) warning "Unknown cluster type: $1" ;; esac } # # check for the major prereq for a) parameter parsing and b) # parsing logs # t=`get_time "12:00"` if [ "$t" = "" ]; then fatal "please install the perl Date::Parse module" fi detect_host