diff --git a/crmd/callbacks.c b/crmd/callbacks.c
index 0bca6d9ddb..5b3c57e5a0 100644
--- a/crmd/callbacks.c
+++ b/crmd/callbacks.c
@@ -1,628 +1,654 @@
/*
* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* This software is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* General Public License for more details.
*
* You should have received a copy of the GNU General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*/
#include <crm_internal.h>
#include <sys/param.h>
#include <crm/crm.h>
#include <string.h>
#include <crmd_fsa.h>
#include <heartbeat.h>
#include <crm/msg_xml.h>
#include <crm/common/xml.h>
#include <crm/common/msg.h>
#include <crm/common/cluster.h>
#include <crm/cib.h>
#include <crmd.h>
#include <crmd_messages.h>
#include <crmd_callbacks.h>
#include <crmd_lrm.h>
void crmd_ha_connection_destroy(gpointer user_data);
void crmd_ha_msg_filter(xmlNode *msg);
/* From join_dc... */
extern gboolean check_join_state(
enum crmd_fsa_state cur_state, const char *source);
#define trigger_fsa(source) crm_debug_3("Triggering FSA: %s", __FUNCTION__); \
G_main_set_trigger(source);
#if SUPPORT_HEARTBEAT
gboolean
crmd_ha_msg_dispatch(ll_cluster_t *cluster_conn, gpointer user_data)
{
IPC_Channel *channel = NULL;
gboolean stay_connected = TRUE;
crm_debug_3("Invoked");
if(cluster_conn != NULL) {
channel = cluster_conn->llc_ops->ipcchan(cluster_conn);
}
CRM_CHECK(cluster_conn != NULL, ;);
CRM_CHECK(channel != NULL, ;);
if(channel != NULL && IPC_ISRCONN(channel)) {
if(cluster_conn->llc_ops->msgready(cluster_conn) == 0) {
crm_debug_2("no message ready yet");
}
/* invoke the callbacks but dont block */
cluster_conn->llc_ops->rcvmsg(cluster_conn, 0);
}
if (channel == NULL || channel->ch_status != IPC_CONNECT) {
if(is_set(fsa_input_register, R_HA_DISCONNECTED) == FALSE) {
crm_crit("Lost connection to heartbeat service.");
} else {
crm_info("Lost connection to heartbeat service.");
}
trigger_fsa(fsa_source);
stay_connected = FALSE;
}
return stay_connected;
}
#endif
void
crmd_ha_connection_destroy(gpointer user_data)
{
crm_debug_3("Invoked");
if(is_set(fsa_input_register, R_HA_DISCONNECTED)) {
/* we signed out, so this is expected */
crm_info("Heartbeat disconnection complete");
return;
}
crm_crit("Lost connection to heartbeat service!");
register_fsa_input(C_HA_DISCONNECT, I_ERROR, NULL);
trigger_fsa(fsa_source);
}
void
crmd_ha_msg_filter(xmlNode *msg)
{
if(AM_I_DC) {
const char *sys_from = crm_element_value(msg, F_CRM_SYS_FROM);
if(safe_str_eq(sys_from, CRM_SYSTEM_DC)) {
const char *from = crm_element_value(msg, F_ORIG);
if(safe_str_neq(from, fsa_our_uname)) {
const char *op = crm_element_value(msg, F_CRM_TASK);
crm_err("Another DC detected: %s (op=%s)", from, op);
/* make sure the election happens NOW */
if(fsa_state != S_ELECTION) {
ha_msg_input_t new_input;
new_input.msg = msg;
register_fsa_error_adv(
C_FSA_INTERNAL, I_ELECTION, NULL, &new_input, __FUNCTION__);
goto done;
}
}
}
} else {
const char *sys_to = crm_element_value(msg, F_CRM_SYS_TO);
if(safe_str_eq(sys_to, CRM_SYSTEM_DC)) {
return;
}
}
/* crm_log_xml(LOG_MSG, "HA[inbound]", msg); */
route_message(C_HA_MESSAGE, msg);
done:
trigger_fsa(fsa_source);
}
#if SUPPORT_HEARTBEAT
void
crmd_ha_msg_callback(HA_Message *hamsg, void* private_data)
{
int level = LOG_DEBUG;
crm_node_t *from_node = NULL;
xmlNode *msg = convert_ha_message(NULL, hamsg, __FUNCTION__);
const char *from = crm_element_value(msg, F_ORIG);
const char *op = crm_element_value(msg, F_CRM_TASK);
const char *sys_from = crm_element_value(msg, F_CRM_SYS_FROM);
CRM_CHECK(from != NULL, crm_log_xml_err(msg, "anon"); goto bail);
crm_debug_2("HA[inbound]: %s from %s", op, from);
if(crm_peer_cache == NULL || crm_active_members() == 0) {
crm_debug("Ignoring HA messages until we are"
" connected to the CCM (%s op from %s)", op, from);
crm_log_xml(LOG_MSG, "HA[inbound]: Ignore (No CCM)", msg);
goto bail;
}
from_node = crm_get_peer(0, from);
if(from_node == NULL || crm_is_member_active(from_node) == FALSE) {
if(safe_str_eq(op, CRM_OP_VOTE)) {
level = LOG_WARNING;
} else if(AM_I_DC && safe_str_eq(op, CRM_OP_JOIN_ANNOUNCE)) {
level = LOG_WARNING;
} else if(safe_str_eq(sys_from, CRM_SYSTEM_DC)) {
level = LOG_WARNING;
}
do_crm_log(level,
"Ignoring HA message (op=%s) from %s: not in our"
" membership list (size=%d)", op, from,
crm_active_members());
crm_log_xml(LOG_MSG, "HA[inbound]: CCM Discard", msg);
} else {
crmd_ha_msg_filter(msg);
return;
}
bail:
free_xml(msg);
return;
}
#endif
/*
* Apparently returning TRUE means "stay connected, keep doing stuff".
* Returning FALSE means "we're all done, close the connection"
*/
gboolean
crmd_ipc_msg_callback(IPC_Channel *client, gpointer user_data)
{
int lpc = 0;
xmlNode *msg = NULL;
crmd_client_t *curr_client = (crmd_client_t*)user_data;
gboolean stay_connected = TRUE;
crm_debug_2("Invoked: %s",
curr_client->table_key);
while(IPC_ISRCONN(client)) {
if(client->ops->is_message_pending(client) == 0) {
break;
}
msg = xmlfromIPC(client, 0);
if (msg == NULL) {
break;
}
lpc++;
crm_debug_2("Processing msg from %s", curr_client->table_key);
crm_log_xml(LOG_DEBUG_2, "CRMd[inbound]", msg);
if(crmd_authorize_message(msg, curr_client)) {
route_message(C_IPC_MESSAGE, msg);
}
free_xml(msg);
msg = NULL;
if(client->ch_status != IPC_CONNECT) {
break;
}
}
crm_debug_2("Processed %d messages", lpc);
if (client->ch_status != IPC_CONNECT) {
stay_connected = FALSE;
process_client_disconnect(curr_client);
}
trigger_fsa(fsa_source);
return stay_connected;
}
extern GCHSource *lrm_source;
gboolean
lrm_dispatch(IPC_Channel *src_not_used, gpointer user_data)
{
/* ?? src == lrm_channel ?? */
ll_lrm_t *lrm = (ll_lrm_t*)user_data;
IPC_Channel *lrm_channel = lrm->lrm_ops->ipcchan(lrm);
lrm->lrm_ops->rcvmsg(lrm, FALSE);
if(lrm_channel->ch_status != IPC_CONNECT) {
lrm_connection_destroy(NULL);
return FALSE;
}
return TRUE;
}
extern gboolean process_lrm_event(lrm_op_t *op);
void
lrm_op_callback(lrm_op_t* op)
{
CRM_CHECK(op != NULL, return);
process_lrm_event(op);
}
+void ais_status_callback(enum crm_status_type type, crm_node_t *node, const void *data)
+{
+ gboolean reset_status_entry = FALSE;
+ if(AM_I_DC == FALSE || node->uname == NULL) {
+ return;
+ }
+
+ switch(type) {
+ case crm_status_uname:
+ crm_info("status: %s is now %s", node->uname, node->state);
+ /* reset_status_entry = TRUE; */
+ /* If we've never seen the node, then it also wont be in the status section */
+ break;
+ case crm_status_nstate:
+ crm_info("status: %s is now %s (was %s)", node->uname, node->state, (const char *)data);
+ reset_status_entry = TRUE;
+ break;
+ case crm_status_processes:
+ break;
+ }
+
+ if(reset_status_entry && safe_str_eq(ONLINESTATUS, node->state)) {
+ erase_status_tag(fsa_our_uname, XML_CIB_TAG_LRM);
+ erase_status_tag(fsa_our_uname, XML_TAG_TRANSIENT_NODEATTRS);
+ }
+
+}
+
void
crmd_ha_status_callback(const char *node, const char *status, void *private)
{
xmlNode *update = NULL;
crm_node_t *member = NULL;
crm_notice("Status update: Node %s now has status [%s] (DC=%s)",
node, status, AM_I_DC?"true":"false");
member = crm_get_peer(0, node);
if(member == NULL || crm_is_member_active(member) == FALSE) {
/* Make sure it is created so crm_update_peer_proc() succeeds */
const char *uuid = get_uuid(node);
member = crm_update_peer(0, 0, 0, -1, 0, uuid, node, NULL, NULL);
}
if(safe_str_eq(status, PINGSTATUS)) {
return;
}
if(safe_str_eq(status, DEADSTATUS)) {
/* this node is toast */
crm_update_peer_proc(node, crm_proc_ais, OFFLINESTATUS);
if(AM_I_DC) {
update = create_node_state(
node, DEADSTATUS, XML_BOOLEAN_NO, OFFLINESTATUS,
CRMD_JOINSTATE_DOWN, NULL, TRUE, __FUNCTION__);
}
} else {
crm_update_peer_proc(node, crm_proc_ais, ONLINESTATUS);
if(AM_I_DC) {
update = create_node_state(
node, ACTIVESTATUS, NULL, NULL,
CRMD_JOINSTATE_PENDING, NULL, FALSE, __FUNCTION__);
}
}
trigger_fsa(fsa_source);
if(update != NULL) {
fsa_cib_anon_update(
XML_CIB_TAG_STATUS, update, cib_scope_local|cib_quorum_override|cib_can_create);
free_xml(update);
}
}
void
crmd_client_status_callback(const char * node, const char * client,
const char * status, void * private)
{
const char *join = NULL;
crm_node_t *member = NULL;
xmlNode *update = NULL;
gboolean clear_shutdown = FALSE;
crm_debug_3("Invoked");
if(safe_str_neq(client, CRM_SYSTEM_CRMD)) {
return;
}
if(safe_str_eq(status, JOINSTATUS)){
clear_shutdown = TRUE;
status = ONLINESTATUS;
join = CRMD_JOINSTATE_PENDING;
} else if(safe_str_eq(status, LEAVESTATUS)){
status = OFFLINESTATUS;
join = CRMD_JOINSTATE_DOWN;
/* clear_shutdown = TRUE; */
}
set_bit_inplace(fsa_input_register, R_PEER_DATA);
crm_notice("Status update: Client %s/%s now has status [%s] (DC=%s)",
node, client, status, AM_I_DC?"true":"false");
if(safe_str_eq(status, ONLINESTATUS)) {
/* remove the cached value in case it changed */
crm_debug_2("Uncaching UUID for %s", node);
unget_uuid(node);
}
member = crm_get_peer(0, node);
if(member == NULL || crm_is_member_active(member) == FALSE) {
/* Make sure it is created so crm_update_peer_proc() succeeds */
const char *uuid = get_uuid(node);
member = crm_update_peer(0, 0, 0, -1, 0, uuid, node, NULL, NULL);
}
crm_update_peer_proc(node, crm_proc_crmd, status);
if(is_set(fsa_input_register, R_CIB_CONNECTED) == FALSE) {
return;
} else if(fsa_state == S_STOPPING) {
return;
}
if(safe_str_eq(node, fsa_our_dc) && safe_str_eq(status, OFFLINESTATUS)){
/* did our DC leave us */
crm_info("Got client status callback - our DC is dead");
register_fsa_input(C_CRMD_STATUS_CALLBACK, I_ELECTION, NULL);
} else if(AM_I_DC == FALSE) {
crm_info("Not the DC");
} else {
crm_debug_3("Got client status callback");
-
- if(fsa_cib_conn != NULL && safe_str_eq(status, ONLINESTATUS)) {
+ if(safe_str_eq(status, ONLINESTATUS)) {
erase_status_tag(fsa_our_uname, XML_CIB_TAG_LRM);
erase_status_tag(fsa_our_uname, XML_TAG_TRANSIENT_NODEATTRS);
}
-
update = create_node_state(
node, NULL, NULL, status, join, NULL, clear_shutdown, __FUNCTION__);
fsa_cib_anon_update(
XML_CIB_TAG_STATUS, update, cib_scope_local|cib_quorum_override|cib_can_create);
free_xml(update);
if(safe_str_eq(status, OFFLINESTATUS)) {
erase_node_from_join(node);
check_join_state(fsa_state, __FUNCTION__);
}
}
trigger_fsa(fsa_source);
}
void
crmd_ipc_connection_destroy(gpointer user_data)
{
GCHSource *source = NULL;
crmd_client_t *client = user_data;
/* Calling this function on an _active_ connection results in:
* crmd_ipc_connection_destroy (callbacks.c:431)
* -> G_main_del_IPC_Channel (GSource.c:478)
* -> g_source_unref
* -> G_CH_destroy_int (GSource.c:647)
* -> crmd_ipc_connection_destroy (callbacks.c:437)\
*
* A better alternative is to call G_main_del_IPC_Channel() directly
*/
if(client == NULL) {
crm_debug_4("No client to delete");
return;
}
crm_debug_2("Disconnecting client %s (%p)", client->table_key, client);
source = client->client_source;
client->client_source = NULL;
if(source != NULL) {
crm_debug_3("Deleting %s (%p) from mainloop",
client->table_key, source);
G_main_del_IPC_Channel(source);
}
crm_free(client->table_key);
crm_free(client->sub_sys);
crm_free(client->uuid);
crm_free(client);
return;
}
gboolean
crmd_client_connect(IPC_Channel *client_channel, gpointer user_data)
{
crm_debug_3("Invoked");
if (client_channel == NULL) {
crm_err("Channel was NULL");
} else if (client_channel->ch_status == IPC_DISCONNECT) {
crm_err("Channel was disconnected");
} else {
crmd_client_t *blank_client = NULL;
crm_debug_3("Channel connected");
crm_malloc0(blank_client, sizeof(crmd_client_t));
CRM_ASSERT(blank_client != NULL);
crm_debug_2("Created client: %p", blank_client);
client_channel->ops->set_recv_qlen(client_channel, 1024);
client_channel->ops->set_send_qlen(client_channel, 1024);
blank_client->client_channel = client_channel;
blank_client->sub_sys = NULL;
blank_client->uuid = NULL;
blank_client->table_key = NULL;
blank_client->client_source =
G_main_add_IPC_Channel(
G_PRIORITY_LOW, client_channel,
FALSE, crmd_ipc_msg_callback,
blank_client, crmd_ipc_connection_destroy);
}
return TRUE;
}
#if SUPPORT_HEARTBEAT
static gboolean fsa_have_quorum = FALSE;
gboolean ccm_dispatch(int fd, gpointer user_data)
{
int rc = 0;
oc_ev_t *ccm_token = (oc_ev_t*)user_data;
gboolean was_error = FALSE;
crm_debug_3("Invoked");
rc = oc_ev_handle_event(ccm_token);
if(rc != 0) {
if(is_set(fsa_input_register, R_CCM_DISCONNECTED) == FALSE) {
/* we signed out, so this is expected */
register_fsa_input(C_CCM_CALLBACK, I_ERROR, NULL);
crm_err("CCM connection appears to have failed: rc=%d.",
rc);
}
was_error = TRUE;
}
trigger_fsa(fsa_source);
return !was_error;
}
void
crmd_ccm_msg_callback(
oc_ed_t event, void *cookie, size_t size, const void *data)
{
gboolean update_cache = FALSE;
const oc_ev_membership_t *membership = data;
gboolean update_quorum = FALSE;
crm_debug_3("Invoked");
CRM_ASSERT(data != NULL);
crm_info("Quorum %s after event=%s (id=%d)",
ccm_have_quorum(event)?"(re)attained":"lost",
ccm_event_name(event), membership->m_instance);
if(crm_peer_seq > membership->m_instance) {
crm_err("Membership instance ID went backwards! %llu->%d",
crm_peer_seq, membership->m_instance);
CRM_ASSERT(crm_peer_seq <= membership->m_instance);
return;
}
/*
* OC_EV_MS_NEW_MEMBERSHIP: membership with quorum
* OC_EV_MS_MS_INVALID: membership without quorum
* OC_EV_MS_NOT_PRIMARY: previous membership no longer valid
* OC_EV_MS_PRIMARY_RESTORED: previous membership restored
* OC_EV_MS_EVICTED: the client is evicted from ccm.
*/
switch(event) {
case OC_EV_MS_NEW_MEMBERSHIP:
case OC_EV_MS_INVALID:
update_cache = TRUE;
update_quorum = TRUE;
break;
case OC_EV_MS_NOT_PRIMARY:
break;
case OC_EV_MS_PRIMARY_RESTORED:
update_cache = TRUE;
crm_peer_seq = membership->m_instance;
break;
case OC_EV_MS_EVICTED:
update_quorum = TRUE;
register_fsa_input(C_FSA_INTERNAL, I_STOP, NULL);
crm_err("Shutting down after CCM event: %s",
ccm_event_name(event));
break;
default:
crm_err("Unknown CCM event: %d", event);
}
if(update_quorum) {
crm_have_quorum = ccm_have_quorum(event);
crm_update_quorum(crm_have_quorum);
if(crm_have_quorum == FALSE) {
/* did we just loose quorum? */
if(fsa_have_quorum) {
crm_info("Quorum lost: %s", ccm_event_name(event));
}
}
}
if(update_cache) {
crm_debug_2("Updating cache after event %s", ccm_event_name(event));
do_ccm_update_cache(C_CCM_CALLBACK, fsa_state, event, data, NULL);
} else if(event != OC_EV_MS_NOT_PRIMARY) {
crm_peer_seq = membership->m_instance;
register_fsa_action(A_TE_CANCEL);
}
oc_ev_callback_done(cookie);
return;
}
#endif
void
crmd_cib_connection_destroy(gpointer user_data)
{
CRM_CHECK(user_data == fsa_cib_conn, ;);
crm_debug_3("Invoked");
trigger_fsa(fsa_source);
fsa_cib_conn->state = cib_disconnected;
if(is_set(fsa_input_register, R_CIB_CONNECTED) == FALSE) {
crm_info("Connection to the CIB terminated...");
return;
}
/* eventually this will trigger a reconnect, not a shutdown */
crm_err("Connection to the CIB terminated...");
register_fsa_input(C_FSA_INTERNAL, I_ERROR, NULL);
clear_bit_inplace(fsa_input_register, R_CIB_CONNECTED);
return;
}
longclock_t fsa_start = 0;
longclock_t fsa_stop = 0;
longclock_t fsa_diff = 0;
gboolean
crm_fsa_trigger(gpointer user_data)
{
unsigned int fsa_diff_ms = 0;
if(fsa_diff_max_ms > 0) {
fsa_start = time_longclock();
}
crm_debug_2("Invoked (queue len: %d)", g_list_length(fsa_message_queue));
s_crmd_fsa(C_FSA_INTERNAL);
crm_debug_2("Exited (queue len: %d)", g_list_length(fsa_message_queue));
if(fsa_diff_max_ms > 0) {
fsa_stop = time_longclock();
fsa_diff = sub_longclock(fsa_stop, fsa_start);
fsa_diff_ms = longclockto_ms(fsa_diff);
if(fsa_diff_ms > fsa_diff_max_ms) {
crm_err("FSA took %dms to complete", fsa_diff_ms);
} else if(fsa_diff_ms > fsa_diff_warn_ms) {
crm_warn("FSA took %dms to complete", fsa_diff_ms);
}
}
return TRUE;
}
diff --git a/crmd/control.c b/crmd/control.c
index 2530c278cd..27b39d61da 100644
--- a/crmd/control.c
+++ b/crmd/control.c
@@ -1,961 +1,963 @@
/*
* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* This software is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* General Public License for more details.
*
* You should have received a copy of the GNU General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*/
#include <crm_internal.h>
#include <sys/param.h>
#include <heartbeat.h>
#include <crm/crm.h>
#include <crm/cib.h>
#include <crm/msg_xml.h>
#include <crm/common/ctrl.h>
#include <crm/pengine/rules.h>
#include <crm/common/cluster.h>
#include <crmd.h>
#include <crmd_fsa.h>
#include <fsa_proto.h>
#include <crmd_messages.h>
#include <crmd_callbacks.h>
#include <crmd_lrm.h>
#include <sys/types.h>
#include <sys/stat.h>
char *ipc_server = NULL;
extern void post_cache_update(int seq);
extern void crmd_ha_connection_destroy(gpointer user_data);
gboolean crm_shutdown(int nsig, gpointer unused);
gboolean fsa_has_quorum = FALSE;
GHashTable *ipc_clients = NULL;
GTRIGSource *fsa_source = NULL;
/* A_HA_CONNECT */
#if SUPPORT_AIS
extern void crmd_ha_msg_filter(xmlNode * msg);
static gboolean crm_ais_dispatch(AIS_Message *wrapper, char *data, int sender)
{
int seq = 0;
xmlNode *xml = NULL;
const char *seq_s = NULL;
if(wrapper->header.id == crm_class_notify) {
return TRUE;
}
xml = string2xml(data);
if(xml == NULL) {
crm_err("Message received: %d:'%.120s'", wrapper->id, data);
return TRUE;
}
crm_xml_add(xml, F_ORIG, wrapper->sender.uname);
crm_xml_add_int(xml, F_SEQ, wrapper->id);
switch(wrapper->header.id) {
case crm_class_members:
seq_s = crm_element_value(xml, "seq");
seq = crm_int_helper(seq_s, NULL);
set_bit_inplace(fsa_input_register, R_PEER_DATA);
post_cache_update(seq);
crm_update_quorum(crm_have_quorum);
break;
default:
crmd_ha_msg_filter(xml);
break;
}
free_xml(xml);
return TRUE;
}
static void
crm_ais_destroy(gpointer user_data)
{
crm_err("AIS connection terminated");
ais_fd_sync = -1;
exit(1);
}
#endif
void
do_ha_control(long long action,
enum crmd_fsa_cause cause,
enum crmd_fsa_state cur_state,
enum crmd_fsa_input current_input,
fsa_data_t *msg_data)
{
gboolean registered = FALSE;
if(action & A_HA_DISCONNECT) {
if(is_openais_cluster()) {
crm_peer_destroy();
crm_info("Disconnected from OpenAIS");
#if SUPPORT_HEARTBEAT
} else if(fsa_cluster_conn != NULL) {
set_bit_inplace(fsa_input_register, R_HA_DISCONNECTED);
fsa_cluster_conn->llc_ops->signoff(fsa_cluster_conn, FALSE);
crm_info("Disconnected from Heartbeat");
#endif
}
}
if(action & A_HA_CONNECT) {
void *dispatch = NULL;
void *destroy = NULL;
if(is_openais_cluster()) {
#if SUPPORT_AIS
destroy = crm_ais_destroy;
dispatch = crm_ais_dispatch;
+ crm_set_status_callback(&ais_status_callback);
+
#endif
} else if(is_heartbeat_cluster()) {
#if SUPPORT_HEARTBEAT
dispatch = crmd_ha_msg_callback;
destroy = crmd_ha_connection_destroy;
#endif
}
registered = crm_cluster_connect(
&fsa_our_uname, &fsa_our_uuid, dispatch, destroy,
#if SUPPORT_HEARTBEAT
&fsa_cluster_conn
#else
NULL
#endif
);
#if SUPPORT_HEARTBEAT
if(is_heartbeat_cluster()) {
crm_debug_3("Be informed of Node Status changes");
if (registered &&
fsa_cluster_conn->llc_ops->set_nstatus_callback(
fsa_cluster_conn, crmd_ha_status_callback,
fsa_cluster_conn) != HA_OK){
crm_err("Cannot set nstatus callback: %s",
fsa_cluster_conn->llc_ops->errmsg(fsa_cluster_conn));
registered = FALSE;
}
crm_debug_3("Be informed of CRM Client Status changes");
if (registered &&
fsa_cluster_conn->llc_ops->set_cstatus_callback(
fsa_cluster_conn, crmd_client_status_callback,
fsa_cluster_conn) != HA_OK) {
crm_err("Cannot set cstatus callback: %s",
fsa_cluster_conn->llc_ops->errmsg(fsa_cluster_conn));
registered = FALSE;
}
if(registered) {
crm_debug_3("Requesting an initial dump of CRMD client_status");
fsa_cluster_conn->llc_ops->client_status(
fsa_cluster_conn, NULL, CRM_SYSTEM_CRMD, -1);
}
}
#endif
if(registered == FALSE) {
set_bit_inplace(fsa_input_register, R_HA_DISCONNECTED);
register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
return;
}
clear_bit_inplace(fsa_input_register, R_HA_DISCONNECTED);
crm_info("Connected to Heartbeat");
}
if(action & ~(A_HA_CONNECT|A_HA_DISCONNECT)) {
crm_err("Unexpected action %s in %s",
fsa_action2string(action), __FUNCTION__);
}
}
/* A_SHUTDOWN */
void
do_shutdown(long long action,
enum crmd_fsa_cause cause,
enum crmd_fsa_state cur_state,
enum crmd_fsa_input current_input,
fsa_data_t *msg_data)
{
/* just in case */
set_bit_inplace(fsa_input_register, R_SHUTDOWN);
if(is_heartbeat_cluster()) {
if(is_set(fsa_input_register, pe_subsystem->flag_connected)) {
crm_info("Terminating the %s", pe_subsystem->name);
if(stop_subsystem(pe_subsystem, TRUE) == FALSE) {
/* its gone... */
crm_err("Faking %s exit", pe_subsystem->name);
clear_bit_inplace(fsa_input_register,
pe_subsystem->flag_connected);
} else {
crm_info("Waiting for subsystems to exit");
crmd_fsa_stall(NULL);
}
}
crm_info("All subsystems stopped, continuing");
}
}
/* A_SHUTDOWN_REQ */
void
do_shutdown_req(long long action,
enum crmd_fsa_cause cause,
enum crmd_fsa_state cur_state,
enum crmd_fsa_input current_input,
fsa_data_t *msg_data)
{
xmlNode *msg = NULL;
crm_info("Sending shutdown request to DC: %s", crm_str(fsa_our_dc));
msg = create_request(
CRM_OP_SHUTDOWN_REQ, NULL, NULL,
CRM_SYSTEM_DC, CRM_SYSTEM_CRMD, NULL);
/* set_bit_inplace(fsa_input_register, R_STAYDOWN); */
if(send_request(msg, NULL) == FALSE) {
if(AM_I_DC) {
crm_info("Processing shutdown locally");
} else {
register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
}
}
free_xml(msg);
}
extern char *max_generation_from;
extern xmlNode *max_generation_xml;
extern GHashTable *resources;
extern GHashTable *voted;
extern GHashTable *reload_hash;
void log_connected_client(gpointer key, gpointer value, gpointer user_data);
void
log_connected_client(gpointer key, gpointer value, gpointer user_data)
{
crmd_client_t *client = value;
crm_err("%s is still connected at exit", client->table_key);
}
static void free_mem(fsa_data_t *msg_data)
{
#if SUPPORT_HEARTBEAT
if(fsa_cluster_conn) {
fsa_cluster_conn->llc_ops->delete(fsa_cluster_conn);
fsa_cluster_conn = NULL;
}
#endif
slist_destroy(fsa_data_t, fsa_data, fsa_message_queue,
crm_info("Dropping %s: [ state=%s cause=%s origin=%s ]",
fsa_input2string(fsa_data->fsa_input),
fsa_state2string(fsa_state),
fsa_cause2string(fsa_data->fsa_cause),
fsa_data->origin);
delete_fsa_input(fsa_data);
);
delete_fsa_input(msg_data);
if(ipc_clients) {
crm_debug("Number of connected clients: %d",
g_hash_table_size(ipc_clients));
/* g_hash_table_foreach(ipc_clients, log_connected_client, NULL); */
g_hash_table_destroy(ipc_clients);
}
empty_uuid_cache();
crm_peer_destroy();
clear_bit_inplace(fsa_input_register, R_CCM_DATA);
if(te_subsystem->client && te_subsystem->client->client_source) {
crm_debug("Full destroy: TE");
G_main_del_IPC_Channel(te_subsystem->client->client_source);
} else {
crm_debug("Partial destroy: TE");
crmd_ipc_connection_destroy(te_subsystem->client);
}
crm_free(te_subsystem);
if(pe_subsystem->client && pe_subsystem->client->client_source) {
crm_debug("Full destroy: PE");
G_main_del_IPC_Channel(pe_subsystem->client->client_source);
} else {
crm_debug("Partial destroy: PE");
crmd_ipc_connection_destroy(pe_subsystem->client);
}
crm_free(pe_subsystem);
crm_free(cib_subsystem);
if(integrated_nodes) {
g_hash_table_destroy(integrated_nodes);
}
if(finalized_nodes) {
g_hash_table_destroy(finalized_nodes);
}
if(confirmed_nodes) {
g_hash_table_destroy(confirmed_nodes);
}
if(reload_hash) {
g_hash_table_destroy(reload_hash);
}
if(resources) {
g_hash_table_destroy(resources);
}
if(voted) {
g_hash_table_destroy(voted);
}
cib_delete(fsa_cib_conn);
fsa_cib_conn = NULL;
if(fsa_lrm_conn) {
fsa_lrm_conn->lrm_ops->delete(fsa_lrm_conn);
}
crm_free(integration_timer);
crm_free(finalization_timer);
crm_free(election_trigger);
crm_free(election_timeout);
crm_free(shutdown_escalation_timer);
crm_free(wait_timer);
crm_free(recheck_timer);
crm_free(fsa_our_dc_version);
crm_free(fsa_our_uname);
crm_free(fsa_our_uuid);
crm_free(fsa_our_dc);
crm_free(ipc_server);
crm_free(max_generation_from);
free_xml(max_generation_xml);
}
/* A_EXIT_0, A_EXIT_1 */
void
do_exit(long long action,
enum crmd_fsa_cause cause,
enum crmd_fsa_state cur_state,
enum crmd_fsa_input current_input,
fsa_data_t *msg_data)
{
int exit_code = 0;
int log_level = LOG_INFO;
const char *exit_type = "gracefully";
if(action & A_EXIT_1) {
exit_code = 1;
log_level = LOG_ERR;
exit_type = "forcefully";
}
verify_stopped(cur_state, LOG_ERR);
do_crm_log(log_level, "Performing %s - %s exiting the CRMd",
fsa_action2string(action), exit_type);
if(is_set(fsa_input_register, R_IN_RECOVERY)) {
crm_err("Could not recover from internal error");
exit_code = 2;
}
if(is_set(fsa_input_register, R_STAYDOWN)) {
crm_warn("Inhibiting respawn by Heartbeat");
exit_code = 100;
}
free_mem(msg_data);
crm_info("[%s] stopped (%d)", crm_system_name, exit_code);
cl_flush_logs();
exit(exit_code);
}
/* A_STARTUP */
void
do_startup(long long action,
enum crmd_fsa_cause cause,
enum crmd_fsa_state cur_state,
enum crmd_fsa_input current_input,
fsa_data_t *msg_data)
{
int was_error = 0;
int interval = 1; /* seconds between DC heartbeats */
crm_debug("Registering Signal Handlers");
G_main_add_SignalHandler(
G_PRIORITY_HIGH, SIGTERM, crm_shutdown, NULL, NULL);
fsa_source = G_main_add_TriggerHandler(
G_PRIORITY_HIGH, crm_fsa_trigger, NULL, NULL);
ipc_clients = g_hash_table_new(g_str_hash, g_str_equal);
crm_debug("Creating CIB and LRM objects");
fsa_cib_conn = cib_new();
fsa_lrm_conn = ll_lrm_new(XML_CIB_TAG_LRM);
/* set up the timers */
crm_malloc0(integration_timer, sizeof(fsa_timer_t));
crm_malloc0(finalization_timer, sizeof(fsa_timer_t));
crm_malloc0(election_trigger, sizeof(fsa_timer_t));
crm_malloc0(election_timeout, sizeof(fsa_timer_t));
crm_malloc0(shutdown_escalation_timer, sizeof(fsa_timer_t));
crm_malloc0(wait_timer, sizeof(fsa_timer_t));
crm_malloc0(recheck_timer, sizeof(fsa_timer_t));
interval = interval * 1000;
if(election_trigger != NULL) {
election_trigger->source_id = 0;
election_trigger->period_ms = -1;
election_trigger->fsa_input = I_DC_TIMEOUT;
election_trigger->callback = crm_timer_popped;
election_trigger->repeat = FALSE;
} else {
was_error = TRUE;
}
if(election_timeout != NULL) {
election_timeout->source_id = 0;
election_timeout->period_ms = -1;
election_timeout->fsa_input = I_ELECTION_DC;
election_timeout->callback = crm_timer_popped;
election_timeout->repeat = FALSE;
} else {
was_error = TRUE;
}
if(integration_timer != NULL) {
integration_timer->source_id = 0;
integration_timer->period_ms = -1;
integration_timer->fsa_input = I_INTEGRATED;
integration_timer->callback = crm_timer_popped;
integration_timer->repeat = FALSE;
} else {
was_error = TRUE;
}
if(finalization_timer != NULL) {
finalization_timer->source_id = 0;
finalization_timer->period_ms = -1;
finalization_timer->fsa_input = I_FINALIZED;
finalization_timer->callback = crm_timer_popped;
finalization_timer->repeat = FALSE;
/* for possible enabling... a bug in the join protocol left
* a slave in S_PENDING while we think its in S_NOT_DC
*
* raising I_FINALIZED put us into a transition loop which is
* never resolved.
* in this loop we continually send probes which the node
* NACK's because its in S_PENDING
*
* if we have nodes where heartbeat is active but the
* CRM is not... then this will be handled in the
* integration phase
*/
finalization_timer->fsa_input = I_ELECTION;
} else {
was_error = TRUE;
}
if(shutdown_escalation_timer != NULL) {
shutdown_escalation_timer->source_id = 0;
shutdown_escalation_timer->period_ms = -1;
shutdown_escalation_timer->fsa_input = I_STOP;
shutdown_escalation_timer->callback = crm_timer_popped;
shutdown_escalation_timer->repeat = FALSE;
} else {
was_error = TRUE;
}
if(wait_timer != NULL) {
wait_timer->source_id = 0;
wait_timer->period_ms = 2000;
wait_timer->fsa_input = I_NULL;
wait_timer->callback = crm_timer_popped;
wait_timer->repeat = FALSE;
} else {
was_error = TRUE;
}
if(recheck_timer != NULL) {
recheck_timer->source_id = 0;
recheck_timer->period_ms = -1;
recheck_timer->fsa_input = I_PE_CALC;
recheck_timer->callback = crm_timer_popped;
recheck_timer->repeat = FALSE;
} else {
was_error = TRUE;
}
/* set up the sub systems */
crm_malloc0(cib_subsystem, sizeof(struct crm_subsystem_s));
crm_malloc0(te_subsystem, sizeof(struct crm_subsystem_s));
crm_malloc0(pe_subsystem, sizeof(struct crm_subsystem_s));
if(cib_subsystem != NULL) {
cib_subsystem->pid = -1;
cib_subsystem->path = BIN_DIR;
cib_subsystem->name = CRM_SYSTEM_CIB;
cib_subsystem->command = BIN_DIR"/"CRM_SYSTEM_CIB;
cib_subsystem->args = "-VVc";
cib_subsystem->flag_connected = R_CIB_CONNECTED;
cib_subsystem->flag_required = R_CIB_REQUIRED;
} else {
was_error = TRUE;
}
if(te_subsystem != NULL) {
te_subsystem->pid = -1;
te_subsystem->path = BIN_DIR;
te_subsystem->name = CRM_SYSTEM_TENGINE;
te_subsystem->command = BIN_DIR"/"CRM_SYSTEM_TENGINE;
te_subsystem->args = NULL;
te_subsystem->flag_connected = R_TE_CONNECTED;
te_subsystem->flag_required = R_TE_REQUIRED;
} else {
was_error = TRUE;
}
if(pe_subsystem != NULL) {
pe_subsystem->pid = -1;
pe_subsystem->path = BIN_DIR;
pe_subsystem->name = CRM_SYSTEM_PENGINE;
pe_subsystem->command = BIN_DIR"/"CRM_SYSTEM_PENGINE;
pe_subsystem->args = NULL;
pe_subsystem->flag_connected = R_PE_CONNECTED;
pe_subsystem->flag_required = R_PE_REQUIRED;
} else {
was_error = TRUE;
}
if(was_error) {
register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
}
welcomed_nodes = g_hash_table_new_full(
g_str_hash, g_str_equal,
g_hash_destroy_str, g_hash_destroy_str);
integrated_nodes = g_hash_table_new_full(
g_str_hash, g_str_equal,
g_hash_destroy_str, g_hash_destroy_str);
finalized_nodes = g_hash_table_new_full(
g_str_hash, g_str_equal,
g_hash_destroy_str, g_hash_destroy_str);
confirmed_nodes = g_hash_table_new_full(
g_str_hash, g_str_equal,
g_hash_destroy_str, g_hash_destroy_str);
set_sigchld_proctrack(G_PRIORITY_HIGH,DEFAULT_MAXDISPATCHTIME);
}
/* A_STOP */
void
do_stop(long long action,
enum crmd_fsa_cause cause,
enum crmd_fsa_state cur_state,
enum crmd_fsa_input current_input,
fsa_data_t *msg_data)
{
register_fsa_input(C_FSA_INTERNAL, I_TERMINATE, NULL);
}
/* A_STARTED */
void
do_started(long long action,
enum crmd_fsa_cause cause,
enum crmd_fsa_state cur_state,
enum crmd_fsa_input current_input,
fsa_data_t *msg_data)
{
if(cur_state != S_STARTING) {
crm_err("Start cancelled... %s", fsa_state2string(cur_state));
return;
} else if(is_set(fsa_input_register, R_CCM_DATA) == FALSE) {
crm_info("Delaying start, CCM (%.16llx) not connected",
R_CCM_DATA);
crmd_fsa_stall(NULL);
return;
} else if(is_set(fsa_input_register, R_LRM_CONNECTED) == FALSE) {
crm_info("Delaying start, LRM (%.16llx) not connected",
R_LRM_CONNECTED);
crmd_fsa_stall(NULL);
return;
} else if(is_set(fsa_input_register, R_CIB_CONNECTED) == FALSE) {
crm_info("Delaying start, CIB (%.16llx) not connected",
R_CIB_CONNECTED);
crmd_fsa_stall(NULL);
return;
} else if(is_set(fsa_input_register, R_READ_CONFIG) == FALSE) {
crm_info("Delaying start, Config not read (%.16llx)",
R_READ_CONFIG);
crmd_fsa_stall(NULL);
return;
} else if(is_set(fsa_input_register, R_PEER_DATA) == FALSE) {
HA_Message *msg = NULL;
/* try reading from HA */
crm_info("Delaying start, Peer data (%.16llx) not recieved",
R_PEER_DATA);
crm_debug_3("Looking for a HA message");
#if SUPPORT_HEARTBEAT
if(is_heartbeat_cluster()) {
msg = fsa_cluster_conn->llc_ops->readmsg(fsa_cluster_conn, 0);
}
#endif
if(msg != NULL) {
crm_debug_3("There was a HA message");
crm_msg_del(msg);
}
/* this should no longer be required */
/* crm_timer_start(wait_timer); */
crmd_fsa_stall(NULL);
return;
}
crm_debug("Init server comms");
if(ipc_server == NULL) {
ipc_server = crm_strdup(CRM_SYSTEM_CRMD);
}
if(init_server_ipc_comms(ipc_server, crmd_client_connect,
default_ipc_connection_destroy)) {
crm_err("Couldn't start IPC server");
register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
}
crm_info("The local CRM is operational");
clear_bit_inplace(fsa_input_register, R_STARTING);
register_fsa_input(msg_data->fsa_cause, I_PENDING, NULL);
}
/* A_RECOVER */
void
do_recover(long long action,
enum crmd_fsa_cause cause,
enum crmd_fsa_state cur_state,
enum crmd_fsa_input current_input,
fsa_data_t *msg_data)
{
set_bit_inplace(fsa_input_register, R_IN_RECOVERY);
crm_err("Action %s (%.16llx) not supported",
fsa_action2string(action), action);
register_fsa_input(C_FSA_INTERNAL, I_TERMINATE, NULL);
}
pe_cluster_option crmd_opts[] = {
/* name, old-name, validate, default, description */
{ XML_CONFIG_ATTR_DC_DEADTIME, NULL, "time", NULL, "10s", &check_time, "How long to wait for a response from other nodes during startup.", "The \"correct\" value will depend on the speed and load of your network." },
{ XML_CONFIG_ATTR_RECHECK, NULL, "time", "Zero disables polling. Positive values are an interval in seconds (unless other SI units are specified. eg. 5min)", "0", &check_timer, "Polling interval for time based changes to options, resource parameters and constraints.", "The Cluster is primarily event driven, however the configuration can have elements that change based on time. To ensure these changes take effect, we can optionally poll the cluster's status for changes." },
{ XML_CONFIG_ATTR_ELECTION_FAIL, NULL, "time", NULL, "2min", &check_timer, "*** Advanced Use Only ***.", "If need to adjust this value, it probably indicates the presence of a bug." },
{ XML_CONFIG_ATTR_FORCE_QUIT, NULL, "time", NULL, "20min", &check_timer, "*** Advanced Use Only ***.", "If need to adjust this value, it probably indicates the presence of a bug." },
{ "crmd-integration-timeout", NULL, "time", NULL, "3min", &check_timer, "*** Advanced Use Only ***.", "If need to adjust this value, it probably indicates the presence of a bug." },
{ "crmd-finalization-timeout", NULL, "time", NULL, "30min", &check_timer, "*** Advanced Use Only ***.", "If you need to adjust this value, it probably indicates the presence of a bug." },
};
void
crmd_metadata(void)
{
config_metadata("CRM Daemon", "1.0",
"CRM Daemon Options",
"This is a fake resource that details the options that can be configured for the CRM Daemon.",
crmd_opts, DIMOF(crmd_opts));
}
static void
verify_crmd_options(GHashTable *options)
{
verify_all_options(options, crmd_opts, DIMOF(crmd_opts));
}
static const char *
crmd_pref(GHashTable *options, const char *name)
{
return get_cluster_pref(options, crmd_opts, DIMOF(crmd_opts), name);
}
static void
config_query_callback(xmlNode *msg, int call_id, int rc,
xmlNode *output, void *user_data)
{
const char *value = NULL;
GHashTable *config_hash = NULL;
ha_time_t *now = new_ha_date(TRUE);
if(rc != cib_ok) {
fsa_data_t *msg_data = NULL;
crm_err("Local CIB query resulted in an error: %s",
cib_error2string(rc));
register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
if(rc == cib_bad_permissions
|| rc == cib_bad_digest
|| rc == cib_bad_config) {
crm_err("The cluster is mis-configured - shutting down and staying down");
set_bit_inplace(fsa_input_register, R_STAYDOWN);
}
goto bail;
}
crm_debug("Call %d : Parsing CIB options", call_id);
config_hash = g_hash_table_new_full(
g_str_hash,g_str_equal, g_hash_destroy_str,g_hash_destroy_str);
unpack_instance_attributes(
output, XML_CIB_TAG_PROPSET, NULL, config_hash,
CIB_OPTIONS_FIRST, FALSE, now);
value = g_hash_table_lookup(config_hash, XML_CONFIG_ATTR_DC_DEADTIME);
if(value == NULL) {
/* apparently we're not allowed to free the result of getenv */
char *param_val = getenv(ENV_PREFIX "initdead");
value = crmd_pref(config_hash, XML_CONFIG_ATTR_DC_DEADTIME);
if(param_val != NULL) {
int from_env = crm_get_msec(param_val) / 2;
int from_defaults = crm_get_msec(value);
if(from_env > from_defaults) {
g_hash_table_replace(
config_hash, crm_strdup(XML_CONFIG_ATTR_DC_DEADTIME),
crm_strdup(param_val));
}
}
}
verify_crmd_options(config_hash);
value = crmd_pref(config_hash, XML_CONFIG_ATTR_DC_DEADTIME);
election_trigger->period_ms = crm_get_msec(value);
value = crmd_pref(config_hash, XML_CONFIG_ATTR_FORCE_QUIT);
shutdown_escalation_timer->period_ms = crm_get_msec(value);
value = crmd_pref(config_hash, XML_CONFIG_ATTR_ELECTION_FAIL);
election_timeout->period_ms = crm_get_msec(value);
value = crmd_pref(config_hash, XML_CONFIG_ATTR_RECHECK);
recheck_timer->period_ms = crm_get_msec(value);
value = crmd_pref(config_hash, "crmd-integration-timeout");
integration_timer->period_ms = crm_get_msec(value);
value = crmd_pref(config_hash, "crmd-finalization-timeout");
finalization_timer->period_ms = crm_get_msec(value);
set_bit_inplace(fsa_input_register, R_READ_CONFIG);
crm_debug_3("Triggering FSA: %s", __FUNCTION__);
G_main_set_trigger(fsa_source);
g_hash_table_destroy(config_hash);
bail:
free_ha_date(now);
}
/* A_READCONFIG */
void
do_read_config(long long action,
enum crmd_fsa_cause cause,
enum crmd_fsa_state cur_state,
enum crmd_fsa_input current_input,
fsa_data_t *msg_data)
{
int call_id = fsa_cib_conn->cmds->query(
fsa_cib_conn, XML_CIB_TAG_CRMCONFIG, NULL, cib_scope_local);
add_cib_op_callback(fsa_cib_conn, call_id, FALSE, NULL, config_query_callback);
crm_debug_2("Querying the CIB... call %d", call_id);
}
gboolean
crm_shutdown(int nsig, gpointer unused)
{
if (crmd_mainloop != NULL && g_main_is_running(crmd_mainloop)) {
if(is_set(fsa_input_register, R_SHUTDOWN)) {
crm_err("Escalating the shutdown");
register_fsa_input_before(C_SHUTDOWN, I_ERROR, NULL);
} else {
crm_info("Requesting shutdown");
set_bit_inplace(fsa_input_register, R_SHUTDOWN);
register_fsa_input(C_SHUTDOWN,I_SHUTDOWN,NULL);
if(shutdown_escalation_timer->period_ms < 1) {
GHashTable *config_hash = g_hash_table_new_full(
g_str_hash, g_str_equal,
g_hash_destroy_str, g_hash_destroy_str);
const char *value = crmd_pref(
config_hash, XML_CONFIG_ATTR_FORCE_QUIT);
int msec = crm_get_msec(value);
crm_info("Using default shutdown escalation: %dms", msec);
shutdown_escalation_timer->period_ms = msec;
g_hash_table_destroy(config_hash);
}
/* cant rely on this... */
crm_timer_start(shutdown_escalation_timer);
}
} else {
crm_info("exit from shutdown");
exit(LSB_EXIT_OK);
}
return TRUE;
}
static void
default_cib_update_callback(xmlNode *msg, int call_id, int rc,
xmlNode *output, void *user_data)
{
if(rc != cib_ok) {
fsa_data_t *msg_data = NULL;
crm_err("CIB Update failed: %s", cib_error2string(rc));
crm_log_xml_warn(output, "update:failed");
register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
}
}
#if SUPPORT_HEARTBEAT
static void
populate_cib_nodes_ha(gboolean with_client_status)
{
int call_id = 0;
const char *ha_node = NULL;
xmlNode *cib_node_list = NULL;
if(fsa_cluster_conn == NULL) {
crm_debug("Not connected");
return;
}
/* Async get client status information in the cluster */
crm_info("Requesting the list of configured nodes");
fsa_cluster_conn->llc_ops->init_nodewalk(fsa_cluster_conn);
cib_node_list = create_xml_node(NULL, XML_CIB_TAG_NODES);
do {
const char *ha_node_type = NULL;
const char *ha_node_uuid = NULL;
xmlNode *cib_new_node = NULL;
ha_node = fsa_cluster_conn->llc_ops->nextnode(fsa_cluster_conn);
if(ha_node == NULL) {
continue;
}
ha_node_type = fsa_cluster_conn->llc_ops->node_type(
fsa_cluster_conn, ha_node);
if(safe_str_neq(NORMALNODE, ha_node_type)) {
crm_debug("Node %s: skipping '%s'",
ha_node, ha_node_type);
continue;
}
ha_node_uuid = get_uuid(ha_node);
if(ha_node_uuid == NULL) {
crm_warn("Node %s: no uuid found", ha_node);
continue;
}
crm_debug("Node: %s (uuid: %s)", ha_node, ha_node_uuid);
cib_new_node = create_xml_node(cib_node_list, XML_CIB_TAG_NODE);
crm_xml_add(cib_new_node, XML_ATTR_ID, ha_node_uuid);
crm_xml_add(cib_new_node, XML_ATTR_UNAME, ha_node);
crm_xml_add(cib_new_node, XML_ATTR_TYPE, ha_node_type);
} while(ha_node != NULL);
fsa_cluster_conn->llc_ops->end_nodewalk(fsa_cluster_conn);
/* Now update the CIB with the list of nodes */
fsa_cib_update(
XML_CIB_TAG_NODES, cib_node_list,
cib_scope_local|cib_quorum_override, call_id);
add_cib_op_callback(fsa_cib_conn, call_id, FALSE, NULL, default_cib_update_callback);
free_xml(cib_node_list);
crm_debug_2("Complete");
}
#endif
static void create_cib_node_definition(
gpointer key, gpointer value, gpointer user_data)
{
crm_node_t *node = value;
xmlNode *cib_nodes = user_data;
xmlNode *cib_new_node = NULL;
cib_new_node = create_xml_node(cib_nodes, XML_CIB_TAG_NODE);
crm_xml_add(cib_new_node, XML_ATTR_ID, node->uuid);
crm_xml_add(cib_new_node, XML_ATTR_UNAME, node->uname);
crm_xml_add(cib_new_node, XML_ATTR_TYPE, NORMALNODE);
}
void
populate_cib_nodes(gboolean with_client_status)
{
int call_id = 0;
xmlNode *cib_node_list = NULL;
#if SUPPORT_HEARTBEAT
if(is_heartbeat_cluster()) {
populate_cib_nodes_ha(with_client_status);
return;
}
#endif
cib_node_list = create_xml_node(NULL, XML_CIB_TAG_NODES);
g_hash_table_foreach(
crm_peer_cache, create_cib_node_definition, cib_node_list);
fsa_cib_update(
XML_CIB_TAG_NODES, cib_node_list, cib_scope_local|cib_quorum_override, call_id);
add_cib_op_callback(fsa_cib_conn, call_id, FALSE, NULL, default_cib_update_callback);
free_xml(cib_node_list);
crm_debug_2("Complete");
}
diff --git a/crmd/crmd_callbacks.h b/crmd/crmd_callbacks.h
index 2d288640d0..725a6e6da6 100644
--- a/crmd/crmd_callbacks.h
+++ b/crmd/crmd_callbacks.h
@@ -1,63 +1,67 @@
/*
* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* This software is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* General Public License for more details.
*
* You should have received a copy of the GNU General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*/
#include <clplumbing/ipc.h>
+#include <crm/ais_common.h>
+#include <crm/common/cluster.h>
#if SUPPORT_HEARTBEAT
#include <hb_api.h>
extern void ccm_event_detail(const oc_ev_membership_t *oc, oc_ed_t event);
extern gboolean ccm_dispatch(int fd, gpointer user_data);
extern void crmd_ccm_msg_callback(
oc_ed_t event, void *cookie, size_t size, const void *data);
extern gboolean crmd_ha_msg_dispatch(
ll_cluster_t *cluster_conn, gpointer user_data);
#endif
/*
* Apparently returning TRUE means "stay connected, keep doing stuff".
* Returning FALSE means "we're all done, close the connection"
*/
extern void crmd_ipc_connection_destroy(gpointer user_data);
extern void crmd_ha_msg_callback(
HA_Message *hamsg, void* private_data);
extern gboolean crmd_ipc_msg_callback(
IPC_Channel *client, gpointer user_data);
extern gboolean crmd_ipc_msg_callback(
IPC_Channel *client, gpointer user_data);
extern gboolean lrm_dispatch(IPC_Channel*src, gpointer user_data);
extern void lrm_op_callback (lrm_op_t* op);
extern void crmd_ha_status_callback(
const char *node, const char * status, void* private_data);
extern void crmd_client_status_callback(
const char * node, const char * client, const char * status, void * private);
extern void msg_ccm_join(const xmlNode *msg, void *foo);
extern gboolean crmd_client_connect(
IPC_Channel *newclient, gpointer user_data);
extern void crmd_cib_connection_destroy(gpointer user_data);
extern gboolean crm_fsa_trigger(gpointer user_data);
+
+extern void ais_status_callback(enum crm_status_type type, crm_node_t *node, const void *data);
diff --git a/include/crm/common/cluster.h b/include/crm/common/cluster.h
index f07694972f..61dde710fd 100644
--- a/include/crm/common/cluster.h
+++ b/include/crm/common/cluster.h
@@ -1,93 +1,102 @@
/*
* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* This software is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* General Public License for more details.
*
* You should have received a copy of the GNU General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*/
#ifndef CRM_COMMON_CLUSTER__H
#define CRM_COMMON_CLUSTER__H
#include <clplumbing/ipc.h>
#include <clplumbing/GSource.h>
#include <crm/common/xml.h>
#include <crm/common/msg.h>
#include <crm/ais_common.h>
#if SUPPORT_HEARTBEAT
# include <heartbeat/hb_api.h>
#endif
extern gboolean crm_have_quorum;
extern GHashTable *crm_peer_cache;
extern unsigned long long crm_peer_seq;
extern void crm_peer_init(void);
extern void crm_peer_destroy(void);
extern gboolean crm_cluster_connect(
char **our_uname, char **our_uuid, void *dispatch, void *destroy,
#if SUPPORT_HEARTBEAT
ll_cluster_t **hb_conn
#else
void **unused
#endif
);
extern gboolean send_cluster_message(
const char *node, enum crm_ais_msg_types service, xmlNode *data, gboolean ordered);
extern void destroy_crm_node(gpointer data);
extern crm_node_t *crm_get_peer(unsigned int id, const char *uname);
extern crm_node_t *crm_update_ais_node(xmlNode *member, long long seq);
extern void crm_update_peer_proc(
const char *uname, uint32_t flag, const char *status);
extern crm_node_t *crm_update_peer(
unsigned int id, uint64_t born, uint64_t seen, int32_t votes, uint32_t children,
const char *uuid, const char *uname, const char *addr, const char *state);
extern gboolean crm_is_member_active(const crm_node_t *node);
extern guint crm_active_members(void);
extern guint reap_crm_membership(void);
extern guint crm_active_members(void);
extern guint crm_active_peers(uint32_t peer);
extern gboolean crm_calculate_quorum(void);
#if SUPPORT_HEARTBEAT
extern gboolean ccm_have_quorum(oc_ed_t event);
extern const char *ccm_event_name(oc_ed_t event);
extern crm_node_t *crm_update_ccm_node(
const oc_ev_membership_t *oc, int offset, const char *state, uint64_t seq);
#endif
#if SUPPORT_AIS
extern int ais_fd_sync;
extern GFDSource *ais_source;
extern gboolean send_ais_text(
int class, const char *data, gboolean local,
const char *node, enum crm_ais_msg_types dest);
extern int32_t get_ais_nodeid(void);
#endif
extern void empty_uuid_cache(void);
extern const char *get_uuid(const char *uname);
extern const char *get_uname(const char *uuid);
extern void set_uuid(xmlNode *node, const char *attr, const char *uname);
extern void unget_uuid(const char *uname);
+enum crm_status_type
+{
+ crm_status_uname,
+ crm_status_nstate,
+ crm_status_processes,
+};
+
enum crm_ais_msg_types text2msg_type(const char *text);
+extern void crm_set_status_callback(
+ void (*dispatch)(enum crm_status_type, crm_node_t*, const void*));
#endif
diff --git a/lib/common/membership.c b/lib/common/membership.c
index 11eb718825..b8263c60de 100644
--- a/lib/common/membership.c
+++ b/lib/common/membership.c
@@ -1,481 +1,503 @@
/*
* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* This software is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* General Public License for more details.
*
* You should have received a copy of the GNU General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*/
#include <crm_internal.h>
#ifndef _GNU_SOURCE
# define _GNU_SOURCE
#endif
#include <sys/param.h>
#include <sys/types.h>
#include <stdio.h>
#include <unistd.h>
#include <string.h>
#include <glib.h>
#include <crm/common/cluster.h>
#include <heartbeat.h>
#include <crm/ais.h>
struct quorum_count_s
{
guint votes_max;
guint votes_active;
guint votes_total;
guint nodes_max;
guint nodes_total;
};
GHashTable *crm_peer_id_cache = NULL;
GHashTable *crm_peer_cache = NULL;
unsigned long long crm_peer_seq = 0;
unsigned long long crm_max_peers = 0;
struct quorum_count_s quorum_stats;
gboolean crm_have_quorum = FALSE;
gboolean crm_is_member_active(const crm_node_t *node)
{
if(safe_str_eq(node->state, CRM_NODE_MEMBER)) {
return TRUE;
}
return FALSE;
}
static gboolean crm_reap_dead_member(
gpointer key, gpointer value, gpointer user_data)
{
if(crm_is_member_active(value) == FALSE) {
return TRUE;
}
return FALSE;
}
guint reap_crm_membership(void)
{
/* remove all dead members */
return g_hash_table_foreach_remove(
crm_peer_cache, crm_reap_dead_member, NULL);
}
static void crm_count_member(
gpointer key, gpointer value, gpointer user_data)
{
guint *count = user_data;
if(crm_is_member_active(value)) {
*count = *count + 1;
}
}
guint crm_active_members(void)
{
guint count = 0;
g_hash_table_foreach(crm_peer_cache, crm_count_member, &count);
return count;
}
struct peer_count_s
{
uint32_t peer;
guint count;
};
static void crm_count_peer(
gpointer key, gpointer value, gpointer user_data)
{
crm_node_t *node = value;
struct peer_count_s *search = user_data;
if(crm_is_member_active(node) && (node->processes & search->peer)) {
search->count = search->count + 1;
}
}
guint crm_active_peers(uint32_t peer)
{
struct peer_count_s search;
search.count = 0;
search.peer = peer;
g_hash_table_foreach(crm_peer_cache, crm_count_peer, &search);
return search.count;
}
void destroy_crm_node(gpointer data)
{
crm_node_t *node = data;
crm_debug_2("Destroying entry for node %u", node->id);
crm_free(node->addr);
crm_free(node->uname);
crm_free(node->state);
crm_free(node);
}
void crm_peer_init(void)
{
const char *expected = getenv("HA_expected_votes");
quorum_stats.votes_max = 2;
quorum_stats.votes_active = 0;
quorum_stats.votes_total = 0;
quorum_stats.nodes_max = 1;
quorum_stats.nodes_total = 0;
if(expected) {
crm_notice("%s expected quorum votes", expected);
quorum_stats.votes_max = crm_int_helper(expected, NULL);
}
if(quorum_stats.votes_max < 1) {
quorum_stats.votes_max = 1;
}
crm_peer_destroy();
if(crm_peer_cache == NULL) {
crm_peer_cache = g_hash_table_new_full(
g_str_hash, g_str_equal, NULL, destroy_crm_node);
}
if(crm_peer_id_cache == NULL) {
crm_peer_id_cache = g_hash_table_new_full(
g_direct_hash, g_direct_equal, NULL, NULL);
}
}
void crm_peer_destroy(void)
{
if(crm_peer_cache != NULL) {
g_hash_table_destroy(crm_peer_cache);
crm_peer_cache = NULL;
}
if(crm_peer_id_cache != NULL) {
g_hash_table_destroy(crm_peer_id_cache);
crm_peer_id_cache = NULL;
}
}
static crm_node_t *crm_new_peer(unsigned int id, const char *uname)
{
crm_node_t *node = NULL;
CRM_CHECK(uname != NULL || id > 0, return NULL);
crm_debug("Creating entry for node %s/%u", uname, id);
crm_malloc0(node, sizeof(crm_node_t));
node->state = crm_strdup("unknown");
return node;
}
crm_node_t *crm_get_peer(unsigned int id, const char *uname)
{
crm_node_t *node = NULL;
if(uname != NULL) {
node = g_hash_table_lookup(crm_peer_cache, uname);
}
if(node == NULL && id > 0) {
node = g_hash_table_lookup(crm_peer_id_cache, GUINT_TO_POINTER(id));
if(node && uname) {
CRM_ASSERT(node->uname == NULL);
crm_new_peer(id, uname);
}
}
return node;
}
+void (*crm_status_callback)(enum crm_status_type, crm_node_t*, const void*) = NULL;
+
+void crm_set_status_callback(
+ void (*dispatch)(enum crm_status_type,crm_node_t*, const void*))
+{
+ crm_status_callback = dispatch;
+}
+
+
crm_node_t *crm_update_peer(
unsigned int id, uint64_t born, uint64_t seen, int32_t votes, uint32_t children,
const char *uuid, const char *uname, const char *addr, const char *state)
{
gboolean id_changed = FALSE;
gboolean uname_changed = FALSE;
gboolean state_changed = FALSE;
gboolean addr_changed = FALSE;
gboolean procs_changed = FALSE;
gboolean votes_changed = FALSE;
crm_node_t *node = NULL;
CRM_CHECK(uname != NULL || id > 0, return NULL);
CRM_ASSERT(crm_peer_cache != NULL);
CRM_ASSERT(crm_peer_id_cache != NULL);
if(uname != NULL) {
node = g_hash_table_lookup(crm_peer_cache, uname);
}
if(node == NULL && id > 0) {
node = g_hash_table_lookup(crm_peer_id_cache, GUINT_TO_POINTER(id));
}
if(node == NULL) {
node = crm_new_peer(id, uname);
/* do it now so we don't get '(new)' everywhere */
node->votes = votes;
node->processes = children;
if(addr) {
node->addr = crm_strdup(addr);
}
}
if(votes > 0 && node->votes != votes) {
votes_changed = TRUE;
node->votes = votes;
}
if(uname != NULL && node->uname == NULL) {
uname_changed = TRUE;
node->uname = crm_strdup(uname);
crm_info("Node %u is now known as %s", id, uname);
g_hash_table_insert(crm_peer_cache, node->uname, node);
+ if(crm_status_callback) {
+ crm_status_callback(crm_status_uname, node, NULL);
+ }
}
if(node->uuid == NULL) {
if(uuid != NULL) {
node->uuid = crm_strdup(uuid);
} else if(node->uname != NULL && is_openais_cluster()) {
node->uuid = crm_strdup(node->uname);
}
}
if(id > 0 && id != node->id) {
id_changed = TRUE;
g_hash_table_remove(crm_peer_id_cache, GUINT_TO_POINTER(node->id));
g_hash_table_insert(crm_peer_id_cache, GUINT_TO_POINTER(id), node);
node->id = id;
crm_info("Node %s now has id: %u", crm_str(uname), id);
}
if(children > 0 && children != node->processes) {
- procs_changed = TRUE;
+ uint32_t last = node->processes;
node->processes = children;
+ procs_changed = TRUE;
+
+ if(crm_status_callback) {
+ crm_status_callback(crm_status_processes, node, &last);
+ }
}
if(born != 0) {
node->born = born;
}
if(state != NULL && safe_str_neq(node->state, state)) {
- state_changed = TRUE;
- crm_free(node->state);
+ char *last = node->state;
node->state = crm_strdup(state);
+ state_changed = TRUE;
+
+ if(crm_status_callback) {
+ crm_status_callback(crm_status_nstate, node, last);
+ }
+ crm_free(last);
}
if(seen != 0 && crm_is_member_active(node)) {
node->last_seen = seen;
}
if(addr != NULL) {
if(node->addr == NULL || crm_str_eq(node->addr, addr, FALSE) == FALSE) {
addr_changed = TRUE;
crm_free(node->addr);
node->addr = crm_strdup(addr);
}
}
if(id_changed || uname_changed || state_changed || addr_changed || votes_changed || procs_changed) {
crm_info("%sNode %s: id=%u%s state=%s%s addr=%s%s votes=%d%s born="U64T" seen="U64T" proc=%.32x%s",
uname_changed?"New ":"", node->uname,
node->id, id_changed?" (new)":"",
node->state, state_changed?" (new)":"",
node->addr, addr_changed?" (new)":"",
node->votes, votes_changed?" (new)":"",
node->born, node->last_seen,
node->processes, procs_changed?" (new)":""
);
}
return node;
}
crm_node_t *crm_update_ais_node(xmlNode *member, long long seq)
{
const char *id_s = crm_element_value(member, "id");
const char *addr = crm_element_value(member, "addr");
const char *uname = crm_element_value(member, "uname");
const char *state = crm_element_value(member, "state");
const char *born_s = crm_element_value(member, "born");
const char *seen_s = crm_element_value(member, "seen");
const char *votes_s = crm_element_value(member, "votes");
const char *procs_s = crm_element_value(member, "processes");
int votes = crm_int_helper(votes_s, NULL);
unsigned int id = crm_int_helper(id_s, NULL);
unsigned int procs = crm_int_helper(procs_s, NULL);
/* TODO: These values will contain garbage if version < 0.7.1 */
uint64_t born = crm_int_helper(born_s, NULL);
uint64_t seen = crm_int_helper(seen_s, NULL);
return crm_update_peer(id, born, seen, votes, procs, uname, uname, addr, state);
}
#if SUPPORT_HEARTBEAT
crm_node_t *crm_update_ccm_node(
const oc_ev_membership_t *oc, int offset, const char *state, uint64_t seq)
{
crm_node_t *node = NULL;
const char *uuid = NULL;
CRM_CHECK(oc->m_array[offset].node_uname != NULL, return NULL);
uuid = get_uuid(oc->m_array[offset].node_uname);
node = crm_update_peer(oc->m_array[offset].node_id,
oc->m_array[offset].node_born_on, seq, -1, 0,
uuid, oc->m_array[offset].node_uname, NULL, state);
if(safe_str_eq(CRM_NODE_ACTIVE, state)) {
crm_update_peer_proc(
oc->m_array[offset].node_uname, crm_proc_ais, ONLINESTATUS);
}
return node;
}
#endif
void crm_update_peer_proc(const char *uname, uint32_t flag, const char *status)
{
crm_node_t *node = NULL;
gboolean changed = FALSE;
CRM_ASSERT(crm_peer_cache != NULL);
CRM_CHECK(uname != NULL, return);
node = g_hash_table_lookup(crm_peer_cache, uname);
CRM_CHECK(node != NULL,
crm_err("Could not set %s.%s to %s", uname, peer2text(flag), status);
return);
if(safe_str_eq(status, ONLINESTATUS)) {
if((node->processes & flag) == 0) {
set_bit_inplace(node->processes, flag);
changed = TRUE;
}
} else if(node->processes & flag) {
clear_bit_inplace(node->processes, flag);
changed = TRUE;
}
if(changed) {
crm_info("%s.%s is now %s", uname, peer2text(flag), status);
}
}
static void crm_count_quorum(
gpointer key, gpointer value, gpointer user_data)
{
crm_node_t *node = value;
quorum_stats.nodes_total += 1;
quorum_stats.votes_total += node->votes;
if(crm_is_member_active(node)) {
quorum_stats.votes_active = quorum_stats.votes_active + node->votes;
}
}
gboolean crm_calculate_quorum(void)
{
unsigned int limit = 0;
gboolean quorate = TRUE;
quorum_stats.votes_total = 0;
quorum_stats.nodes_total = 0;
quorum_stats.votes_active = 0;
g_hash_table_foreach(crm_peer_cache, crm_count_quorum, NULL);
if(quorum_stats.votes_total > quorum_stats.votes_max) {
crm_info("Known quorum votes: %u -> %u",
quorum_stats.votes_max, quorum_stats.votes_total);
quorum_stats.votes_max = quorum_stats.votes_total;
}
if(quorum_stats.nodes_total > quorum_stats.nodes_max) {
crm_debug("Known quorum nodes: %u -> %u",
quorum_stats.nodes_max, quorum_stats.nodes_total);
quorum_stats.nodes_max = quorum_stats.nodes_total;
}
limit = (quorum_stats.votes_max + 2) / 2;
if(quorum_stats.votes_active < limit) {
quorate = FALSE;
}
crm_debug("known: %u, available: %u, limit: %u, active: %u: %s",
quorum_stats.votes_max, quorum_stats.votes_total,
limit, quorum_stats.votes_active, quorate?"true":"false");
if(quorate != crm_have_quorum) {
crm_notice("Membership %llu: quorum %s",
crm_peer_seq, quorate?"attained":"lost");
} else {
crm_debug("Membership %llu: quorum %s",
crm_peer_seq, quorate?"retained":"lost");
}
crm_have_quorum = quorate;
return quorate;
}
/* Code appropriated (with permission) from cman/daemon/commands.c under GPLv2 */
#if 0
static int calculate_quorum(int allow_decrease, int max_expected, unsigned int *ret_total_votes)
{
struct list *nodelist;
struct cluster_node *node;
unsigned int total_votes = 0;
unsigned int highest_expected = 0;
unsigned int newquorum, q1, q2;
unsigned int total_nodes = 0;
list_iterate(nodelist, &cluster_members_list) {
node = list_item(nodelist, struct cluster_node);
if (node->state == NODESTATE_MEMBER) {
highest_expected =
max(highest_expected, node->expected_votes);
total_votes += node->votes;
total_nodes++;
}
}
if (quorum_device && quorum_device->state == NODESTATE_MEMBER)
total_votes += quorum_device->votes;
if (max_expected > 0)
highest_expected = max_expected;
/* This quorum calculation is taken from the OpenVMS Cluster Systems
* manual, but, then, you guessed that didn't you */
q1 = (highest_expected + 2) / 2;
q2 = (total_votes + 2) / 2;
newquorum = max(q1, q2);
/* Normally quorum never decreases but the system administrator can
* force it down by setting expected votes to a maximum value */
if (!allow_decrease)
newquorum = max(quorum, newquorum);
/* The special two_node mode allows each of the two nodes to retain
* quorum if the other fails. Only one of the two should live past
* fencing (as both nodes try to fence each other in split-brain.)
* Also: if there are more than two nodes, force us inquorate to avoid
* any damage or confusion.
*/
if (two_node && total_nodes <= 2)
newquorum = 1;
if (ret_total_votes)
*ret_total_votes = total_votes;
return newquorum;
}
#endif